X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=jam.c;h=d0c7923f1688a3f4c328889c9c9d0aac55c53063;hb=8e6aa84fe8a01bcd49908d78e3310a6697b6294b;hp=d22ced453ff2c4769b9a558f895c54c59d799960;hpb=ca4fab9f783159edd91e2735125a2940c5723184;p=jam

diff --git a/jam.c b/jam.c
index d22ced4..d0c7923 100644
--- a/jam.c
+++ b/jam.c
@@ -1,20 +1,446 @@
 #include <stdio.h>
+#include <math.h>
+#include <string.h>
 #include <stdlib.h>
+#include <getopt.h>
+#include <netdb.h>
+#include <unistd.h>
 #include <pthread.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <sys/epoll.h>
+#include <sys/ioctl.h>
+#include <errno.h>
+
+unsigned short port = 2007;
+unsigned update_frequency = 1048576;
+
+struct in_addr *destinations = NULL;
+unsigned num_destinations = 0;
+unsigned room_destinations = 0;
+
+struct in_addr *sources = NULL;
+unsigned num_sources = 0;
+unsigned room_sources = 0;
+
+unsigned num_senders = 128;
+unsigned num_sockets_per_sender = 16;
+unsigned num_receivers = 128;
+unsigned do_listen = 1;
+ 
+// quite arbitrary, should probably be a bit higher than num_sockets_per_sender
+unsigned epoll_room_in_receiver = 128;
+
+unsigned long long total_bytes_received = 0;
+pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+unsigned long long total_bytes_sent = 0;
+pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+const static struct option longopts[] = {
+	{ "source-list", required_argument, NULL, 's' },
+	{ "destination-list", required_argument, NULL, 'd' },
+	{ "num-senders", required_argument, NULL, 'n' },
+	{ "num-sockets-per-sender", required_argument, NULL, 'N' },
+	{ "num-receivers", required_argument, NULL, 'r' },
+	{ "port", required_argument, NULL, 'p' },
+	{ "sender-only", no_argument, NULL, 'o' },
+	{ NULL, 0, NULL, 0 }
+};
+
+// generates from [0,1>
+double gen_uniform_random()
+{
+	return rand() / (RAND_MAX+1.0);
+}
+
+double gen_pareto_random(double min, double k)
+{
+	double u = gen_uniform_random();
+	return min * pow(u, -1.0 / k);
+}
+
+void read_ip_list(char *filename, struct in_addr **addr_list, unsigned *num, unsigned *room)
+{
+	char buf[256];
+	FILE *in = fopen(filename, "r");
+	if (in == NULL) {
+		perror(filename);
+		exit(1);
+	}
+
+	for ( ;; ) {
+		char *ptr;
+		struct in_addr addr;
+		struct hostent *he;
+
+		if (fgets(buf, 256, in) == NULL)
+			break;
+
+		ptr = strchr(buf, '\n');
+		if (ptr != NULL)
+			*ptr = 0;
+
+		ptr = strchr(buf, '\r');
+		if (ptr != NULL)
+			*ptr = 0;
+		
+		ptr = buf + strspn(buf, " \t");
+
+		if (ptr[0] == '#' || ptr[0] == 0)
+			continue;
+
+		he = gethostbyname(ptr);
+		if (he == NULL) {
+			perror(ptr);
+			exit(1);
+		}
+
+		// just pick the first for now
+		memcpy(&addr.s_addr, he->h_addr_list[0], sizeof(addr.s_addr));
+
+		if (*num >= *room) {
+			if (*room == 0) {
+				*room = 16;
+			} else {
+				*room <<= 1;
+			}
+			*addr_list = (struct in_addr *)realloc(*addr_list, *room * sizeof(struct in_addr));
+		}
+
+		(*addr_list)[*num] = addr;
+		++*num;
+	}
+
+	fclose(in);
+}
+
+void parse_options(int argc, char **argv)
+{
+	int option_index = 0;
+
+	for ( ;; ) {
+		int c = getopt_long(argc, argv, "s:d:n:N:r:p:o", longopts, &option_index); 
+		switch (c) {
+		case 's':
+			read_ip_list(optarg, &sources, &num_sources, &room_sources);
+			break;
+		case 'd':
+			read_ip_list(optarg, &destinations, &num_destinations, &room_destinations);
+			break;
+		case 'n':
+			num_senders = atoi(optarg);
+			break;
+		case 'r':
+			num_receivers = atoi(optarg);
+			break;
+		case 'N':
+			num_sockets_per_sender = atoi(optarg);
+			break;
+		case 'p':
+			port = atoi(optarg);
+			break;
+		case 'o':
+			do_listen = 0;
+			break;
+		case -1:
+			return;       // end of argument list
+		default:
+			fprintf(stderr, "Invalid option\n");
+			exit(1);
+		}
+	}
+}
+
+struct sender {
+	int fd;
+	unsigned long long bytes_left;
+};
+
+void generate_new_sender(int ep_fd, struct sender *s)
+{
+	int sock, one = 1;
+	unsigned src_num, dst_num;
+	unsigned long long num_bytes;
+	struct sockaddr_in sin;
+	struct epoll_event ev;
+	
+	sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+	if (sock == -1) {
+		perror("socket()");
+		exit(1);
+	}
+
+	// find the right parameters
+	src_num = (unsigned)(num_sources * gen_uniform_random());
+	dst_num = (unsigned)(num_destinations * gen_uniform_random());
+	num_bytes = (unsigned)gen_pareto_random(1048576.0, 1.0);
+
+	// FIXME: bind to the right source
+
+	if (ioctl(sock, FIONBIO, &one) == -1) {
+		perror("FIONBIO");
+		exit(1);
+	}
+
+	sin.sin_family = AF_INET;
+	sin.sin_port = htons(port);
+	sin.sin_addr = destinations[dst_num];
+
+	// non-blocking connect (will be detected by epoll later)
+	if (connect(sock, (struct sockaddr *)&sin, sizeof(sin)) == -1 && errno != EINPROGRESS) {
+		perror("connect()");
+		exit(1);
+	}
+	
+	// stick it in the epoll set (FIXME: make edge-triggered?)
+	ev.events = EPOLLOUT | EPOLLHUP | EPOLLERR;
+	ev.data.ptr = s;
+
+	s->fd = sock;
+	s->bytes_left = num_bytes;
+
+	if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
+		perror("EPOLL_CTL_ADD");
+		exit(1);
+	}
+}
+
+void *sender_worker(void *arg)
+{
+	unsigned i;
+	char buf[65536];
+	unsigned long long bytes_sent = 0;
+	int ep_fd;
+	struct epoll_event *events;
+
+	ep_fd = epoll_create(num_sockets_per_sender);
+	if (ep_fd == -1) {
+		perror("epoll_create");
+		exit(1);
+	}
+
+	// malloc, since there might not be enough room on the stack
+	events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * num_sockets_per_sender);
+	if (events == NULL) {
+		perror("malloc");
+		exit(1);
+	}
+
+	// fill the buffer with random junk
+	for (i = 0; i < 65536; ++i)
+		buf[i] = rand() & 0xff;
+
+	// allocate all the senders
+	for (i = 0; i < num_sockets_per_sender; ++i) {
+		struct sender *s = (struct sender *)malloc(sizeof(struct sender));
+		if (s == NULL) {
+			perror("malloc()");
+			exit(1);
+		}
+
+		generate_new_sender(ep_fd, s);
+	}
+
+	for ( ;; ) {
+		int num_active = epoll_wait(ep_fd, events, num_sockets_per_sender, -1);
+		if (num_active == -1) {
+			perror("epoll_wait");
+			exit(1);
+		}
+
+		for (i = 0; i < num_active; ++i) {
+			struct sender *s = (struct sender *)events[i].data.ptr;
+			unsigned long long bytes_to_send = s->bytes_left;
+			unsigned ret;
+
+			if (bytes_to_send > 65536) {
+				bytes_to_send = 65536;	
+			}
+
+			ret = send(s->fd, buf, bytes_to_send, MSG_NOSIGNAL);
+			if (ret == -1) {
+				if (errno == EAGAIN)
+					continue;
+
+				perror("send()");
+				exit(1);
+			}
+
+			s->bytes_left -= ret;
+			bytes_sent += ret;
+
+			// update the central counter after every 1MB (8ms
+			// at gigabit speeds, should be enough) of sent data
+			if (bytes_sent > update_frequency) {
+				pthread_mutex_lock(&send_mutex);
+				total_bytes_sent += bytes_sent;
+				pthread_mutex_unlock(&send_mutex);
+
+				bytes_sent = 0;
+			}
+			
+			if (s->bytes_left == 0) {
+				if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, s->fd, NULL) == -1) {
+					perror("EPOLL_CTL_DEL");
+					exit(1);
+				}
+				close(s->fd);
+				generate_new_sender(ep_fd, s);
+			}
+		}
+	}
+
+	pthread_mutex_lock(&send_mutex);
+	total_bytes_sent += bytes_sent;
+	pthread_mutex_unlock(&send_mutex);
+
+	free(events);
+	close(ep_fd);
+
+	pthread_exit(0);
+}
 
 void *receiver_worker(void *arg)
 {
-	int sock = (int)arg;
-	printf("Received worker for socket %u\n", sock);
+	int server_sock = (int)arg;
+	char buf[65536];
+	unsigned long long bytes_received = 0;
+	int ep_fd, i;
+	struct epoll_event ev, *events;
+
+	ep_fd = epoll_create(num_sockets_per_sender);
+	if (ep_fd == -1) {
+		perror("epoll_create");
+		exit(1);
+	}
+
+	// malloc, since there might not be enough room on the stack
+	events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * epoll_room_in_receiver);
+	if (events == NULL) {
+		perror("malloc");
+		exit(1);
+	}
+	
+	// stick the receiver socket in the epoll set
+	ev.events = EPOLLIN;
+	ev.data.fd = server_sock;
+
+	if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, server_sock, &ev) == -1) {
+		perror("EPOLL_CTL_ADD");
+		exit(1);
+	}
+
+	for ( ;; ) {
+		int num_active = epoll_wait(ep_fd, events, epoll_room_in_receiver, -1);
+		if (num_active == -1) {
+			perror("epoll_wait");
+			exit(1);
+		}
+
+		for (i = 0; i < num_active; ++i) {
+			int sock = events[i].data.fd;
+			int ret;
+			if (sock == server_sock) {
+				struct sockaddr_in addr;
+				int sock;
+				socklen_t addr_len = sizeof(addr);
+
+				sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
+				if (sock == -1) {
+					if (errno == EAGAIN) {
+						// another thread snatched it, ignore
+						continue;
+					} else {
+						perror("accept()");
+						exit(1);
+					}
+				}
+
+				// add it to the epoll set
+				ev.events = EPOLLIN | EPOLLHUP | EPOLLERR;
+				ev.data.fd = sock;
+
+				if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
+					perror("EPOLL_CTL_ADD");
+					exit(1);
+				}
+				continue;
+			}
+
+			ret = read(sock, buf, 65536);
+			if (ret == 0) {
+				if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, sock, NULL) == -1) {
+					perror("EPOLL_CTL_DEL");
+					exit(1);
+				}
+				close(sock);
+				continue;
+			}
+
+			bytes_received += ret;
+
+			// update the central counter after every 1MB (8ms
+			// at gigabit speeds, should be enough) of received data
+			if (bytes_received > update_frequency) {
+				pthread_mutex_lock(&receive_mutex);
+				total_bytes_received += bytes_received;
+				pthread_mutex_unlock(&receive_mutex);
+
+				bytes_received = 0;
+			}
+		}
+	}
+	
+	pthread_mutex_lock(&receive_mutex);
+	total_bytes_received += bytes_received;
+	pthread_mutex_unlock(&receive_mutex);
+
 	pthread_exit(0);
 }
 
+// We're keeping this separate receiver dispatcher around because we might eventually
+// want to move away from the model where each receiver worker does the accept(). Thus,
+// it makes sense to keep this dispatcher around, even though all it does at the moment
+// is spawn off a few new threads and then die itself.
+void *receiver_dispatcher(void *arg)
+{
+	int server_sock = (int)arg;
+	int i;
+	pthread_attr_t attr;
+	pthread_t thread;
+	
+	// FIXME: these do not really set errno
+	if (pthread_attr_init(&attr) != 0) {
+		perror("pthread_attr_init()");
+		exit(1);
+	}
+
+	if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) {
+		perror("pthread_attr_setstacksize");
+		exit(1);
+	}
+
+	if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
+		perror("pthread_attr_setdetachstate");
+		exit(1);
+	}
+
+	for (i = 0; i < num_receivers; ++i) {	
+		if (pthread_create(&thread, &attr, receiver_worker, (void *)server_sock) != 0) {
+			perror("pthread_create()");
+			exit(1);
+		}
+	}
+
+	return NULL;
+}
+
 int get_server_socket(unsigned short port)
 {
 	int server_sock;
 	struct sockaddr_in sin;
+	unsigned one = 1;
 
 	server_sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
 	if (server_sock == -1) {
@@ -22,6 +448,11 @@ int get_server_socket(unsigned short port)
 		exit(1);
 	}
 
+	if (setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) == -1) {
+		perror("setsocket(SO_REUSEADDR)");
+		exit(1);
+	}
+
 	sin.sin_family = AF_INET;
 	sin.sin_port = htons(port);
 	sin.sin_addr.s_addr = INADDR_ANY;
@@ -39,29 +470,88 @@ int get_server_socket(unsigned short port)
 	return server_sock;
 }
 
-int main()
+int main(int argc, char **argv)
 {
-	int server_sock = get_server_socket(2007);
+	int server_sock;
+	unsigned i;
+	pthread_attr_t attr;
 
-	// FIXME: fire off sender workers here
+	parse_options(argc, argv);
 
-	/*
-	 * Listen for incoming connections, spawning off one receiver
-	 * thread for each (which will just gobble up the data until
-	 * we're done).
-	 */
-	for ( ;; ) {
-		struct sockaddr_in addr;
-		socklen_t addr_len = sizeof(addr);
+	if (destinations == NULL || sources == NULL) {
+		fprintf(stderr, "Missing or empty source or destination host list. Aborting.\n");
+		exit(1);
+	}
+
+	if (do_listen) {
+		server_sock = get_server_socket(port);
+	}
+	
+	printf("Sending data on port %u from %u sources to %u destinations.\n\n",
+		port, num_sources, num_destinations);
+		
+	// FIXME: these do not really set errno
+	if (pthread_attr_init(&attr) != 0) {
+		perror("pthread_attr_init()");
+		exit(1);
+	}
+
+	if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) {
+		perror("pthread_attr_setstacksize");
+		exit(1);
+	}
+
+	if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
+		perror("pthread_attr_setdetachstate");
+		exit(1);
+	}
+		
+	// Fire off the master receiver.
+	if (do_listen) {
 		pthread_t thread;
+		if (pthread_create(&thread, &attr, receiver_dispatcher, (void *)server_sock) != 0) {
+			perror("pthread_create()");
+			exit(1);
+		}
+	}
+
+	printf("Waiting five seconds before starting senders...\n");
+	sleep(5);
 
-		int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
-		if (sock == -1) {
-			perror("accept()");
+	// Fire off sender workers.
+	for (i = 0; i < num_senders; ++i) {
+		pthread_t thread;
+		
+		if (pthread_create(&thread, &attr, sender_worker, NULL) != 0) {
+			perror("pthread_create()");
 			exit(1);
 		}
+	}
+
+	// Just stay around collecting statistics until we're done.
+	for ( ;; ) {
+		unsigned long long sent, received;
+		static unsigned long long last_sent = 0, last_received = 0;
+		double recv_rate, send_rate;
+
+		pthread_mutex_lock(&send_mutex);
+		sent = total_bytes_sent;
+		pthread_mutex_unlock(&send_mutex);
+		
+		pthread_mutex_lock(&receive_mutex);
+		received = total_bytes_received;
+		pthread_mutex_unlock(&receive_mutex);
+
+		send_rate = (sent - last_sent) * 8.0 / 1048576.0;	
+		recv_rate = (received - last_received) * 8.0 / 1048576.0;	
+		
+		printf("%12llu %12llu  %5.0f Mbit/sec  %5.0f Mbit/sec\n", sent, received,
+			send_rate, recv_rate);
+
+		last_sent = sent;
+		last_received = received;
 
-		pthread_create(&thread, NULL, receiver_worker, (void *)sock);
+		sleep(1);
 	}
 
 	exit(0);