Make each receiver handle multiple sockets.

diff --git a/jam.c b/jam.c
index 1f2aedcf5d4d1855b3bade5776c997dbc2238a64..c1f9bb17adc729cca87bcbaef4cc82ff4d767432 100644 (file)
--- a/jam.c
+++ b/jam.c
@@ -25,7 +25,11 @@ unsigned room_sources = 0;
 
 unsigned num_senders = 128;
 unsigned num_sockets_per_sender = 16;
+unsigned num_receivers = 128;
 unsigned do_listen = 1;
+ 
+// quite arbitrary, should probably be a bit higher than num_sockets_per_sender
+unsigned epoll_room_in_receiver = 128;
 
 unsigned long long total_bytes_received = 0;
 pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -295,44 +299,114 @@ void *sender_worker(void *arg)
 
 void *receiver_worker(void *arg)
 {
-       int sock = (int)arg;
+       int server_sock = (int)arg;
        char buf[65536];
        unsigned long long bytes_received = 0;
+       int ep_fd, i;
+       struct epoll_event ev, *events;
+
+       ep_fd = epoll_create(num_sockets_per_sender);
+       if (ep_fd == -1) {
+               perror("epoll_create");
+               exit(1);
+       }
+
+       // malloc, since there might not be enough room on the stack
+       events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * epoll_room_in_receiver);
+       if (events == NULL) {
+               perror("malloc");
+               exit(1);
+       }
+       
+       // stick the receiver socket in the epoll set
+       ev.events = EPOLLIN;
+       ev.data.fd = server_sock;
+
+       if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, server_sock, &ev) == -1) {
+               perror("EPOLL_CTL_ADD");
+               exit(1);
+       }
 
        for ( ;; ) {
-               int ret = read(sock, buf, 65536);
-               if (ret == 0)
-                       break;
+               int num_active = epoll_wait(ep_fd, events, epoll_room_in_receiver, -1);
+               if (num_active == -1) {
+                       perror("epoll_wait");
+                       exit(1);
+               }
 
-               bytes_received += ret;
+               for (i = 0; i < num_active; ++i) {
+                       int sock = events[i].data.fd;
+                       int ret;
+                       if (sock == server_sock) {
+                               struct sockaddr_in addr;
+                               int sock;
+                               socklen_t addr_len = sizeof(addr);
+
+                               sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
+                               if (sock == -1) {
+                                       if (errno == EAGAIN) {
+                                               // another thread snatched it, ignore
+                                               continue;
+                                       } else {
+                                               perror("accept()");
+                                               exit(1);
+                                       }
+                               }
 
-               // update the central counter after every 1MB (8ms
-               // at gigabit speeds, should be enough) of received data
-               if (bytes_received > update_frequency) {
-                       pthread_mutex_lock(&receive_mutex);
-                       total_bytes_received += bytes_received;
-                       pthread_mutex_unlock(&receive_mutex);
+                               // add it to the epoll set
+                               ev.events = EPOLLIN | EPOLLHUP | EPOLLERR;
+                               ev.data.fd = sock;
 
-                       bytes_received = 0;
+                               if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
+                                       perror("EPOLL_CTL_ADD");
+                                       exit(1);
+                               }
+                               continue;
+                       }
+
+                       ret = read(sock, buf, 65536);
+                       if (ret == 0) {
+                               close(sock);
+                               
+                               if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, sock, NULL) == -1) {
+                                       perror("EPOLL_CTL_ADD");
+                                       exit(1);
+                               }
+
+                               continue;
+                       }
+
+                       bytes_received += ret;
+
+                       // update the central counter after every 1MB (8ms
+                       // at gigabit speeds, should be enough) of received data
+                       if (bytes_received > update_frequency) {
+                               pthread_mutex_lock(&receive_mutex);
+                               total_bytes_received += bytes_received;
+                               pthread_mutex_unlock(&receive_mutex);
+
+                               bytes_received = 0;
+                       }
                }
        }
        
-       if (close(sock) == -1) {
-               perror("close()");
-               exit(1);
-       }
-                       
        pthread_mutex_lock(&receive_mutex);
        total_bytes_received += bytes_received;
        pthread_mutex_unlock(&receive_mutex);
 
        pthread_exit(0);
 }
-       
+
+// We're keeping this separate receiver dispatcher around because we might eventually
+// want to move away from the model where each receiver worker does the accept(). Thus,
+// it makes sense to keep this dispatcher around, even though all it does at the moment
+// is spawn off a few new threads and then die itself.
 void *receiver_dispatcher(void *arg)
 {
        int server_sock = (int)arg;
+       int i;
        pthread_attr_t attr;
+       pthread_t thread;
        
        // FIXME: these do not really set errno
        if (pthread_attr_init(&attr) != 0) {
@@ -350,23 +424,8 @@ void *receiver_dispatcher(void *arg)
                exit(1);
        }
 
-       /*
-        * Listen for incoming connections, spawning off one receiver
-        * thread for each (which will just gobble up the data until
-        * we're done).
-        */
-       for ( ;; ) {
-               struct sockaddr_in addr;
-               socklen_t addr_len = sizeof(addr);
-               pthread_t thread;
-
-               int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
-               if (sock == -1) {
-                       perror("accept()");
-                       exit(1);
-               }
-
-               if (pthread_create(&thread, &attr, receiver_worker, (void *)sock) != 0) {
+       for (i = 0; i < num_receivers; ++i) {   
+               if (pthread_create(&thread, &attr, receiver_worker, (void *)server_sock) != 0) {
                        perror("pthread_create()");
                        exit(1);
                }