X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=jam.c;h=0141e5ba45398df0f8cac42559eca84e85b166ae;hb=bc98f29eed4e03c5dc9b78bf4978ec7d490a5521;hp=1df2eb980ec7aad3f58f41fefd92c8fef4e8ae04;hpb=e824fa5d9096d07b4a6fdbba67280e655381863f;p=jam diff --git a/jam.c b/jam.c index 1df2eb9..0141e5b 100644 --- a/jam.c +++ b/jam.c @@ -8,8 +8,12 @@ #include #include #include +#include +#include +#include unsigned short port = 2007; +unsigned update_frequency = 1048576; struct in_addr *destinations = NULL; unsigned num_destinations = 0; @@ -20,12 +24,26 @@ unsigned num_sources = 0; unsigned room_sources = 0; unsigned num_senders = 128; +unsigned num_sockets_per_sender = 16; +unsigned num_receivers = 128; +unsigned do_listen = 1; + +// quite arbitrary, should probably be a bit higher than num_sockets_per_sender +unsigned epoll_room_in_receiver = 128; + +unsigned long long total_bytes_received = 0; +pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER; + +unsigned long long total_bytes_sent = 0; +pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER; const static struct option longopts[] = { { "source-list", required_argument, NULL, 's' }, { "destination-list", required_argument, NULL, 'd' }, { "num-senders", required_argument, NULL, 'n' }, + { "num-sockets-per-sender", required_argument, NULL, 'N' }, { "port", required_argument, NULL, 'p' }, + { "sender-only", no_argument, NULL, 'o' }, { NULL, 0, NULL, 0 } }; @@ -101,7 +119,7 @@ void parse_options(int argc, char **argv) int option_index = 0; for ( ;; ) { - int c = getopt_long(argc, argv, "s:d:n:p:", longopts, &option_index); + int c = getopt_long(argc, argv, "s:d:n:N:p:o", longopts, &option_index); switch (c) { case 's': read_ip_list(optarg, &sources, &num_sources, &room_sources); @@ -112,9 +130,15 @@ void parse_options(int argc, char **argv) case 'n': num_senders = atoi(optarg); break; + case 'N': + num_sockets_per_sender = atoi(optarg); + break; case 'p': port = atoi(optarg); break; + case 'o': + do_listen = 0; + break; case -1: return; // end of argument list default: @@ -124,65 +148,286 @@ void parse_options(int argc, char **argv) } } +struct sender { + int fd; + unsigned long long bytes_left; +}; + +void generate_new_sender(int ep_fd, struct sender *s) +{ + int sock, one = 1; + unsigned src_num, dst_num; + unsigned long long num_bytes; + struct sockaddr_in sin; + struct epoll_event ev; + + sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); + if (sock == -1) { + perror("socket()"); + exit(1); + } + + // find the right parameters + src_num = (unsigned)(num_sources * gen_uniform_random()); + dst_num = (unsigned)(num_destinations * gen_uniform_random()); + num_bytes = (unsigned)gen_pareto_random(1048576.0, 1.0); + + // FIXME: bind to the right source + + if (ioctl(sock, FIONBIO, &one) == -1) { + perror("FIONBIO"); + exit(1); + } + + sin.sin_family = AF_INET; + sin.sin_port = htons(port); + sin.sin_addr = destinations[dst_num]; + + // non-blocking connect (will be detected by epoll later) + if (connect(sock, (struct sockaddr *)&sin, sizeof(sin)) == -1 && errno != EINPROGRESS) { + perror("connect()"); + exit(1); + } + + // stick it in the epoll set (FIXME: make edge-triggered?) + ev.events = EPOLLOUT | EPOLLHUP | EPOLLERR; + ev.data.ptr = s; + + s->fd = sock; + s->bytes_left = num_bytes; + + if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) { + perror("EPOLL_CTL_ADD"); + exit(1); + } +} + void *sender_worker(void *arg) { unsigned i; - int sock; + char buf[65536]; + unsigned long long bytes_sent = 0; + int ep_fd; + struct epoll_event *events; + + ep_fd = epoll_create(num_sockets_per_sender); + if (ep_fd == -1) { + perror("epoll_create"); + exit(1); + } - for (i = 0; i < 1000; ++i) { - unsigned src_num = (unsigned)(num_sources * gen_uniform_random()); - unsigned dst_num = (unsigned)(num_destinations * gen_uniform_random()); - unsigned num_bytes = (unsigned)gen_pareto_random(1048576.0, 1.0); - struct sockaddr_in sin; + // malloc, since there might not be enough room on the stack + events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * num_sockets_per_sender); + if (events == NULL) { + perror("malloc"); + exit(1); + } - sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - if (sock == -1) { - perror("socket()"); + // fill the buffer with random junk + for (i = 0; i < 65536; ++i) + buf[i] = rand() & 0xff; + + // allocate all the senders + for (i = 0; i < num_sockets_per_sender; ++i) { + struct sender *s = (struct sender *)malloc(sizeof(struct sender)); + if (s == NULL) { + perror("malloc()"); exit(1); } - - // FIXME: bind to the right source - - sin.sin_family = AF_INET; - sin.sin_port = htons(port); - sin.sin_addr = destinations[dst_num]; - - if (connect(sock, (struct sockaddr *)&sin, sizeof(sin)) == -1) { - perror("connect()"); + + generate_new_sender(ep_fd, s); + } + + for ( ;; ) { + int num_active = epoll_wait(ep_fd, events, num_sockets_per_sender, -1); + if (num_active == -1) { + perror("epoll_wait"); exit(1); } - // FIXME: send data here + for (i = 0; i < num_active; ++i) { + struct sender *s = (struct sender *)events[i].data.ptr; + unsigned long long bytes_to_send = s->bytes_left; + unsigned ret; + + if (bytes_to_send > 65536) { + bytes_to_send = 65536; + } + + ret = send(s->fd, buf, bytes_to_send, MSG_NOSIGNAL); + if (ret == -1) { + if (errno == EAGAIN) + continue; + + perror("send()"); + exit(1); + } - close(sock); + s->bytes_left -= ret; + bytes_sent += ret; + + // update the central counter after every 1MB (8ms + // at gigabit speeds, should be enough) of sent data + if (bytes_sent > update_frequency) { + pthread_mutex_lock(&send_mutex); + total_bytes_sent += bytes_sent; + pthread_mutex_unlock(&send_mutex); + + bytes_sent = 0; + } + + if (s->bytes_left == 0) { + if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, s->fd, NULL) == -1) { + perror("EPOLL_CTL_DEL"); + exit(1); + } + close(s->fd); + generate_new_sender(ep_fd, s); + } + } } + pthread_mutex_lock(&send_mutex); + total_bytes_sent += bytes_sent; + pthread_mutex_unlock(&send_mutex); + + free(events); + close(ep_fd); + pthread_exit(0); } void *receiver_worker(void *arg) { - int sock = (int)arg; + int server_sock = (int)arg; char buf[65536]; + unsigned long long bytes_received = 0; + int ep_fd, i; + struct epoll_event ev, *events; + + ep_fd = epoll_create(num_sockets_per_sender); + if (ep_fd == -1) { + perror("epoll_create"); + exit(1); + } + + // malloc, since there might not be enough room on the stack + events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * epoll_room_in_receiver); + if (events == NULL) { + perror("malloc"); + exit(1); + } + + // stick the receiver socket in the epoll set + ev.events = EPOLLIN; + ev.data.fd = server_sock; - printf("Received worker for socket %u\n", sock); + if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, server_sock, &ev) == -1) { + perror("EPOLL_CTL_ADD"); + exit(1); + } for ( ;; ) { - int ret = read(sock, buf, 65536); - if (ret == 0) - break; + int num_active = epoll_wait(ep_fd, events, epoll_room_in_receiver, -1); + if (num_active == -1) { + perror("epoll_wait"); + exit(1); + } - // FIXME: update stats here + for (i = 0; i < num_active; ++i) { + int sock = events[i].data.fd; + int ret; + if (sock == server_sock) { + struct sockaddr_in addr; + int sock; + socklen_t addr_len = sizeof(addr); + + sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len); + if (sock == -1) { + if (errno == EAGAIN) { + // another thread snatched it, ignore + continue; + } else { + perror("accept()"); + exit(1); + } + } + + // add it to the epoll set + ev.events = EPOLLIN | EPOLLHUP | EPOLLERR; + ev.data.fd = sock; + + if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) { + perror("EPOLL_CTL_ADD"); + exit(1); + } + continue; + } + + ret = read(sock, buf, 65536); + if (ret == 0) { + if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, sock, NULL) == -1) { + perror("EPOLL_CTL_DEL"); + exit(1); + } + close(sock); + continue; + } + + bytes_received += ret; + + // update the central counter after every 1MB (8ms + // at gigabit speeds, should be enough) of received data + if (bytes_received > update_frequency) { + pthread_mutex_lock(&receive_mutex); + total_bytes_received += bytes_received; + pthread_mutex_unlock(&receive_mutex); + + bytes_received = 0; + } + } } + + pthread_mutex_lock(&receive_mutex); + total_bytes_received += bytes_received; + pthread_mutex_unlock(&receive_mutex); + + pthread_exit(0); +} - printf("Socket %u done\n", sock); +// We're keeping this separate receiver dispatcher around because we might eventually +// want to move away from the model where each receiver worker does the accept(). Thus, +// it makes sense to keep this dispatcher around, even though all it does at the moment +// is spawn off a few new threads and then die itself. +void *receiver_dispatcher(void *arg) +{ + int server_sock = (int)arg; + int i; + pthread_attr_t attr; + pthread_t thread; - if (close(sock) == -1) { - perror("close()"); + // FIXME: these do not really set errno + if (pthread_attr_init(&attr) != 0) { + perror("pthread_attr_init()"); exit(1); } - pthread_exit(0); + if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) { + perror("pthread_attr_setstacksize"); + exit(1); + } + + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) { + perror("pthread_attr_setdetachstate"); + exit(1); + } + + for (i = 0; i < num_receivers; ++i) { + if (pthread_create(&thread, &attr, receiver_worker, (void *)server_sock) != 0) { + perror("pthread_create()"); + exit(1); + } + } } int get_server_socket(unsigned short port) @@ -229,9 +474,12 @@ int main(int argc, char **argv) if (destinations == NULL || sources == NULL) { fprintf(stderr, "Missing or empty source or destination host list. Aborting.\n"); + exit(1); } - server_sock = get_server_socket(port); + if (do_listen) { + server_sock = get_server_socket(port); + } printf("Sending data on port %u from %u sources to %u destinations.\n\n", port, num_sources, num_destinations); @@ -247,6 +495,23 @@ int main(int argc, char **argv) exit(1); } + if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) { + perror("pthread_attr_setdetachstate"); + exit(1); + } + + // Fire off the master receiver. + if (do_listen) { + pthread_t thread; + if (pthread_create(&thread, &attr, receiver_dispatcher, (void *)server_sock) != 0) { + perror("pthread_create()"); + exit(1); + } + } + + printf("Waiting five seconds before starting senders...\n"); + sleep(5); + // Fire off sender workers. for (i = 0; i < num_senders; ++i) { pthread_t thread; @@ -257,26 +522,30 @@ int main(int argc, char **argv) } } - /* - * Listen for incoming connections, spawning off one receiver - * thread for each (which will just gobble up the data until - * we're done). - */ + // Just stay around collecting statistics until we're done. for ( ;; ) { - struct sockaddr_in addr; - socklen_t addr_len = sizeof(addr); - pthread_t thread; + unsigned long long sent, received; + static unsigned long long last_sent = 0, last_received = 0; + double recv_rate, send_rate; - int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len); - if (sock == -1) { - perror("accept()"); - exit(1); - } + pthread_mutex_lock(&send_mutex); + sent = total_bytes_sent; + pthread_mutex_unlock(&send_mutex); + + pthread_mutex_lock(&receive_mutex); + received = total_bytes_received; + pthread_mutex_unlock(&receive_mutex); - if (pthread_create(&thread, &attr, receiver_worker, (void *)sock) != 0) { - perror("pthread_create()"); - exit(1); - } + send_rate = (sent - last_sent) * 8.0 / 1048576.0; + recv_rate = (received - last_received) * 8.0 / 1048576.0; + + printf("%12llu %12llu %5.0f Mbit/sec %5.0f Mbit/sec\n", sent, received, + send_rate, recv_rate); + + last_sent = sent; + last_received = received; + + sleep(1); } exit(0);