From: sgunderson@bigfoot.com <>
Date: Tue, 13 Feb 2007 22:13:52 +0000 (+0100)
Subject: Make each receiver handle multiple sockets.
X-Git-Url: https://git.sesse.net/?p=jam;a=commitdiff_plain;h=a5844b6a95d61ba003e9c0611d93031e4dbe467e

Make each receiver handle multiple sockets.
---

diff --git a/jam.c b/jam.c
index 1f2aedc..c1f9bb1 100644
--- a/jam.c
+++ b/jam.c
@@ -25,7 +25,11 @@ unsigned room_sources = 0;
 
 unsigned num_senders = 128;
 unsigned num_sockets_per_sender = 16;
+unsigned num_receivers = 128;
 unsigned do_listen = 1;
+ 
+// quite arbitrary, should probably be a bit higher than num_sockets_per_sender
+unsigned epoll_room_in_receiver = 128;
 
 unsigned long long total_bytes_received = 0;
 pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -295,44 +299,114 @@ void *sender_worker(void *arg)
 
 void *receiver_worker(void *arg)
 {
-	int sock = (int)arg;
+	int server_sock = (int)arg;
 	char buf[65536];
 	unsigned long long bytes_received = 0;
+	int ep_fd, i;
+	struct epoll_event ev, *events;
+
+	ep_fd = epoll_create(num_sockets_per_sender);
+	if (ep_fd == -1) {
+		perror("epoll_create");
+		exit(1);
+	}
+
+	// malloc, since there might not be enough room on the stack
+	events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * epoll_room_in_receiver);
+	if (events == NULL) {
+		perror("malloc");
+		exit(1);
+	}
+	
+	// stick the receiver socket in the epoll set
+	ev.events = EPOLLIN;
+	ev.data.fd = server_sock;
+
+	if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, server_sock, &ev) == -1) {
+		perror("EPOLL_CTL_ADD");
+		exit(1);
+	}
 
 	for ( ;; ) {
-		int ret = read(sock, buf, 65536);
-		if (ret == 0)
-			break;
+		int num_active = epoll_wait(ep_fd, events, epoll_room_in_receiver, -1);
+		if (num_active == -1) {
+			perror("epoll_wait");
+			exit(1);
+		}
 
-		bytes_received += ret;
+		for (i = 0; i < num_active; ++i) {
+			int sock = events[i].data.fd;
+			int ret;
+			if (sock == server_sock) {
+				struct sockaddr_in addr;
+				int sock;
+				socklen_t addr_len = sizeof(addr);
+
+				sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
+				if (sock == -1) {
+					if (errno == EAGAIN) {
+						// another thread snatched it, ignore
+						continue;
+					} else {
+						perror("accept()");
+						exit(1);
+					}
+				}
 
-		// update the central counter after every 1MB (8ms
-		// at gigabit speeds, should be enough) of received data
-		if (bytes_received > update_frequency) {
-			pthread_mutex_lock(&receive_mutex);
-			total_bytes_received += bytes_received;
-			pthread_mutex_unlock(&receive_mutex);
+				// add it to the epoll set
+				ev.events = EPOLLIN | EPOLLHUP | EPOLLERR;
+				ev.data.fd = sock;
 
-			bytes_received = 0;
+				if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
+					perror("EPOLL_CTL_ADD");
+					exit(1);
+				}
+				continue;
+			}
+
+			ret = read(sock, buf, 65536);
+			if (ret == 0) {
+				close(sock);
+				
+				if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, sock, NULL) == -1) {
+					perror("EPOLL_CTL_ADD");
+					exit(1);
+				}
+
+				continue;
+			}
+
+			bytes_received += ret;
+
+			// update the central counter after every 1MB (8ms
+			// at gigabit speeds, should be enough) of received data
+			if (bytes_received > update_frequency) {
+				pthread_mutex_lock(&receive_mutex);
+				total_bytes_received += bytes_received;
+				pthread_mutex_unlock(&receive_mutex);
+
+				bytes_received = 0;
+			}
 		}
 	}
 	
-	if (close(sock) == -1) {
-		perror("close()");
-		exit(1);
-	}
-			
 	pthread_mutex_lock(&receive_mutex);
 	total_bytes_received += bytes_received;
 	pthread_mutex_unlock(&receive_mutex);
 
 	pthread_exit(0);
 }
-	
+
+// We're keeping this separate receiver dispatcher around because we might eventually
+// want to move away from the model where each receiver worker does the accept(). Thus,
+// it makes sense to keep this dispatcher around, even though all it does at the moment
+// is spawn off a few new threads and then die itself.
 void *receiver_dispatcher(void *arg)
 {
 	int server_sock = (int)arg;
+	int i;
 	pthread_attr_t attr;
+	pthread_t thread;
 	
 	// FIXME: these do not really set errno
 	if (pthread_attr_init(&attr) != 0) {
@@ -350,23 +424,8 @@ void *receiver_dispatcher(void *arg)
 		exit(1);
 	}
 
-	/*
-	 * Listen for incoming connections, spawning off one receiver
-	 * thread for each (which will just gobble up the data until
-	 * we're done).
-	 */
-	for ( ;; ) {
-		struct sockaddr_in addr;
-		socklen_t addr_len = sizeof(addr);
-		pthread_t thread;
-
-		int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
-		if (sock == -1) {
-			perror("accept()");
-			exit(1);
-		}
-
-		if (pthread_create(&thread, &attr, receiver_worker, (void *)sock) != 0) {
+	for (i = 0; i < num_receivers; ++i) {	
+		if (pthread_create(&thread, &attr, receiver_worker, (void *)server_sock) != 0) {
 			perror("pthread_create()");
 			exit(1);
 		}