9 #include <sys/socket.h>
10 #include <netinet/in.h>
11 #include <sys/epoll.h>
12 #include <sys/ioctl.h>
15 unsigned short port = 2007;
16 unsigned update_frequency = 1048576;
18 struct in_addr *destinations = NULL;
19 unsigned num_destinations = 0;
20 unsigned room_destinations = 0;
22 struct in_addr *sources = NULL;
23 unsigned num_sources = 0;
24 unsigned room_sources = 0;
26 unsigned num_senders = 128;
27 unsigned num_sockets_per_sender = 16;
28 unsigned num_receivers = 128;
29 unsigned do_listen = 1;
31 // quite arbitrary, should probably be a bit higher than num_sockets_per_sender
32 unsigned epoll_room_in_receiver = 128;
34 unsigned long long total_bytes_received = 0;
35 pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER;
37 unsigned long long total_bytes_sent = 0;
38 pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER;
40 const static struct option longopts[] = {
41 { "source-list", required_argument, NULL, 's' },
42 { "destination-list", required_argument, NULL, 'd' },
43 { "num-senders", required_argument, NULL, 'n' },
44 { "num-sockets-per-sender", required_argument, NULL, 'N' },
45 { "num-receivers", required_argument, NULL, 'r' },
46 { "port", required_argument, NULL, 'p' },
47 { "sender-only", no_argument, NULL, 'o' },
51 // generates from [0,1>
52 double gen_uniform_random()
54 return rand() / (RAND_MAX+1.0);
57 double gen_pareto_random(double min, double k)
59 double u = gen_uniform_random();
60 return min * pow(u, -1.0 / k);
63 void read_ip_list(char *filename, struct in_addr **addr_list, unsigned *num, unsigned *room)
66 FILE *in = fopen(filename, "r");
77 if (fgets(buf, 256, in) == NULL)
80 ptr = strchr(buf, '\n');
84 ptr = strchr(buf, '\r');
88 ptr = buf + strspn(buf, " \t");
90 if (ptr[0] == '#' || ptr[0] == 0)
93 he = gethostbyname(ptr);
99 // just pick the first for now
100 memcpy(&addr.s_addr, he->h_addr_list[0], sizeof(addr.s_addr));
108 *addr_list = (struct in_addr *)realloc(*addr_list, *room * sizeof(struct in_addr));
111 (*addr_list)[*num] = addr;
118 void parse_options(int argc, char **argv)
120 int option_index = 0;
123 int c = getopt_long(argc, argv, "s:d:n:N:r:p:o", longopts, &option_index);
126 read_ip_list(optarg, &sources, &num_sources, &room_sources);
129 read_ip_list(optarg, &destinations, &num_destinations, &room_destinations);
132 num_senders = atoi(optarg);
135 num_receivers = atoi(optarg);
138 num_sockets_per_sender = atoi(optarg);
147 return; // end of argument list
149 fprintf(stderr, "Invalid option\n");
157 unsigned long long bytes_left;
160 void generate_new_sender(int ep_fd, struct sender *s)
163 unsigned src_num, dst_num;
164 unsigned long long num_bytes;
165 struct sockaddr_in sin;
166 struct epoll_event ev;
168 sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
174 // find the right parameters
175 src_num = (unsigned)(num_sources * gen_uniform_random());
176 dst_num = (unsigned)(num_destinations * gen_uniform_random());
177 num_bytes = (unsigned)gen_pareto_random(1048576.0, 1.0);
179 // FIXME: bind to the right source
181 if (ioctl(sock, FIONBIO, &one) == -1) {
186 sin.sin_family = AF_INET;
187 sin.sin_port = htons(port);
188 sin.sin_addr = destinations[dst_num];
190 // non-blocking connect (will be detected by epoll later)
191 if (connect(sock, (struct sockaddr *)&sin, sizeof(sin)) == -1 && errno != EINPROGRESS) {
196 // stick it in the epoll set (FIXME: make edge-triggered?)
197 ev.events = EPOLLOUT | EPOLLHUP | EPOLLERR;
201 s->bytes_left = num_bytes;
203 if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
204 perror("EPOLL_CTL_ADD");
209 void *sender_worker(void *arg)
213 unsigned long long bytes_sent = 0;
215 struct epoll_event *events;
217 ep_fd = epoll_create(num_sockets_per_sender);
219 perror("epoll_create");
223 // malloc, since there might not be enough room on the stack
224 events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * num_sockets_per_sender);
225 if (events == NULL) {
230 // fill the buffer with random junk
231 for (i = 0; i < 65536; ++i)
232 buf[i] = rand() & 0xff;
234 // allocate all the senders
235 for (i = 0; i < num_sockets_per_sender; ++i) {
236 struct sender *s = (struct sender *)malloc(sizeof(struct sender));
242 generate_new_sender(ep_fd, s);
246 int num_active = epoll_wait(ep_fd, events, num_sockets_per_sender, -1);
247 if (num_active == -1) {
250 perror("epoll_wait");
254 for (i = 0; i < num_active; ++i) {
255 struct sender *s = (struct sender *)events[i].data.ptr;
256 unsigned long long bytes_to_send = s->bytes_left;
259 if (bytes_to_send > 65536) {
260 bytes_to_send = 65536;
263 ret = send(s->fd, buf, bytes_to_send, MSG_NOSIGNAL);
265 if (errno == EAGAIN || errno == EINTR)
272 s->bytes_left -= ret;
275 // update the central counter after every 1MB (8ms
276 // at gigabit speeds, should be enough) of sent data
277 if (bytes_sent > update_frequency) {
278 pthread_mutex_lock(&send_mutex);
279 total_bytes_sent += bytes_sent;
280 pthread_mutex_unlock(&send_mutex);
285 if (s->bytes_left == 0) {
286 if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, s->fd, NULL) == -1) {
287 perror("EPOLL_CTL_DEL");
291 generate_new_sender(ep_fd, s);
296 pthread_mutex_lock(&send_mutex);
297 total_bytes_sent += bytes_sent;
298 pthread_mutex_unlock(&send_mutex);
306 void *receiver_worker(void *arg)
308 int server_sock = (int)arg;
310 unsigned long long bytes_received = 0;
312 struct epoll_event ev, *events;
314 ep_fd = epoll_create(num_sockets_per_sender);
316 perror("epoll_create");
320 // malloc, since there might not be enough room on the stack
321 events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * epoll_room_in_receiver);
322 if (events == NULL) {
327 // stick the receiver socket in the epoll set
329 ev.data.fd = server_sock;
331 if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, server_sock, &ev) == -1) {
332 perror("EPOLL_CTL_ADD");
337 int num_active = epoll_wait(ep_fd, events, epoll_room_in_receiver, -1);
338 if (num_active == -1) {
341 perror("epoll_wait");
345 for (i = 0; i < num_active; ++i) {
346 int sock = events[i].data.fd;
348 if (sock == server_sock) {
349 struct sockaddr_in addr;
351 socklen_t addr_len = sizeof(addr);
353 sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
355 if (errno == EAGAIN || errno == EINTR) {
356 // another thread snatched it, ignore
364 // add it to the epoll set
365 ev.events = EPOLLIN | EPOLLHUP | EPOLLERR;
368 if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
369 perror("EPOLL_CTL_ADD");
375 ret = read(sock, buf, 65536);
377 if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, sock, NULL) == -1) {
378 perror("EPOLL_CTL_DEL");
385 bytes_received += ret;
387 // update the central counter after every 1MB (8ms
388 // at gigabit speeds, should be enough) of received data
389 if (bytes_received > update_frequency) {
390 pthread_mutex_lock(&receive_mutex);
391 total_bytes_received += bytes_received;
392 pthread_mutex_unlock(&receive_mutex);
399 pthread_mutex_lock(&receive_mutex);
400 total_bytes_received += bytes_received;
401 pthread_mutex_unlock(&receive_mutex);
406 // We're keeping this separate receiver dispatcher around because we might eventually
407 // want to move away from the model where each receiver worker does the accept(). Thus,
408 // it makes sense to keep this dispatcher around, even though all it does at the moment
409 // is spawn off a few new threads and then die itself.
410 void *receiver_dispatcher(void *arg)
412 int server_sock = (int)arg;
417 // FIXME: these do not really set errno
418 if (pthread_attr_init(&attr) != 0) {
419 perror("pthread_attr_init()");
423 if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) {
424 perror("pthread_attr_setstacksize");
428 if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
429 perror("pthread_attr_setdetachstate");
433 for (i = 0; i < num_receivers; ++i) {
434 if (pthread_create(&thread, &attr, receiver_worker, (void *)server_sock) != 0) {
435 perror("pthread_create()");
443 int get_server_socket(unsigned short port)
446 struct sockaddr_in sin;
449 server_sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
450 if (server_sock == -1) {
455 if (setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) == -1) {
456 perror("setsocket(SO_REUSEADDR)");
460 sin.sin_family = AF_INET;
461 sin.sin_port = htons(port);
462 sin.sin_addr.s_addr = INADDR_ANY;
464 if (bind(server_sock, (struct sockaddr *)&sin, sizeof(struct sockaddr)) == -1) {
469 if (listen(server_sock, 255) == -1) {
477 int main(int argc, char **argv)
483 parse_options(argc, argv);
485 if (destinations == NULL || sources == NULL) {
486 fprintf(stderr, "Missing or empty source or destination host list. Aborting.\n");
491 server_sock = get_server_socket(port);
494 printf("Sending data on port %u from %u sources to %u destinations.\n\n",
495 port, num_sources, num_destinations);
497 // FIXME: these do not really set errno
498 if (pthread_attr_init(&attr) != 0) {
499 perror("pthread_attr_init()");
503 if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) {
504 perror("pthread_attr_setstacksize");
508 if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
509 perror("pthread_attr_setdetachstate");
513 // Fire off the master receiver.
516 if (pthread_create(&thread, &attr, receiver_dispatcher, (void *)server_sock) != 0) {
517 perror("pthread_create()");
522 printf("Waiting five seconds before starting senders...\n");
525 // Fire off sender workers.
526 for (i = 0; i < num_senders; ++i) {
529 if (pthread_create(&thread, &attr, sender_worker, NULL) != 0) {
530 perror("pthread_create()");
535 // Just stay around collecting statistics until we're done.
537 unsigned long long sent, received;
538 static unsigned long long last_sent = 0, last_received = 0;
539 double recv_rate, send_rate;
541 pthread_mutex_lock(&send_mutex);
542 sent = total_bytes_sent;
543 pthread_mutex_unlock(&send_mutex);
545 pthread_mutex_lock(&receive_mutex);
546 received = total_bytes_received;
547 pthread_mutex_unlock(&receive_mutex);
549 send_rate = (sent - last_sent) * 8.0 / 1048576.0;
550 recv_rate = (received - last_received) * 8.0 / 1048576.0;
552 printf("%12llu %12llu %5.0f Mbit/sec %5.0f Mbit/sec\n", sent, received,
553 send_rate, recv_rate);
556 last_received = received;