#include <errno.h>
unsigned short port = 2007;
+unsigned update_frequency = 1048576;
struct in_addr *destinations = NULL;
unsigned num_destinations = 0;
unsigned num_senders = 128;
unsigned num_sockets_per_sender = 16;
+unsigned num_receivers = 128;
unsigned do_listen = 1;
+
+// quite arbitrary, should probably be a bit higher than num_sockets_per_sender
+unsigned epoll_room_in_receiver = 128;
unsigned long long total_bytes_received = 0;
pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER;
{ "destination-list", required_argument, NULL, 'd' },
{ "num-senders", required_argument, NULL, 'n' },
{ "num-sockets-per-sender", required_argument, NULL, 'N' },
+ { "num-receivers", required_argument, NULL, 'r' },
{ "port", required_argument, NULL, 'p' },
{ "sender-only", no_argument, NULL, 'o' },
{ NULL, 0, NULL, 0 }
int option_index = 0;
for ( ;; ) {
- int c = getopt_long(argc, argv, "s:d:n:p:o", longopts, &option_index);
+ int c = getopt_long(argc, argv, "s:d:n:N:r:p:o", longopts, &option_index);
switch (c) {
case 's':
read_ip_list(optarg, &sources, &num_sources, &room_sources);
case 'n':
num_senders = atoi(optarg);
break;
+ case 'r':
+ num_receivers = atoi(optarg);
+ break;
case 'N':
num_sockets_per_sender = atoi(optarg);
break;
}
}
-void generate_new_sender(int ep_fd)
+struct sender {
+ int fd;
+ unsigned long long bytes_left;
+};
+
+void generate_new_sender(int ep_fd, struct sender *s)
{
int sock, one = 1;
unsigned src_num, dst_num;
unsigned long long num_bytes;
struct sockaddr_in sin;
struct epoll_event ev;
-
+
sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (sock == -1) {
perror("socket()");
// stick it in the epoll set (FIXME: make edge-triggered?)
ev.events = EPOLLOUT | EPOLLHUP | EPOLLERR;
- ev.data.fd = sock;
- ev.data.u64 = num_bytes;
+ ev.data.ptr = s;
+
+ s->fd = sock;
+ s->bytes_left = num_bytes;
if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
perror("EPOLL_CTL_ADD");
buf[i] = rand() & 0xff;
// allocate all the senders
- for (i = 0; i < num_sockets_per_sender; ++i)
- generate_new_sender(ep_fd);
+ for (i = 0; i < num_sockets_per_sender; ++i) {
+ struct sender *s = (struct sender *)malloc(sizeof(struct sender));
+ if (s == NULL) {
+ perror("malloc()");
+ exit(1);
+ }
+
+ generate_new_sender(ep_fd, s);
+ }
for ( ;; ) {
- int ret = epoll_wait(ep_fd, events, num_sockets_per_sender, -1);
- for (i = 0; i < ret; ++i) {
- unsigned bytes_to_send = events[i].data.u64;
+ int num_active = epoll_wait(ep_fd, events, num_sockets_per_sender, -1);
+ if (num_active == -1) {
+ if (errno == EINTR)
+ continue;
+ perror("epoll_wait");
+ exit(1);
+ }
+
+ for (i = 0; i < num_active; ++i) {
+ struct sender *s = (struct sender *)events[i].data.ptr;
+ unsigned long long bytes_to_send = s->bytes_left;
unsigned ret;
if (bytes_to_send > 65536) {
bytes_to_send = 65536;
}
- ret = send(events[i].data.fd, buf, bytes_to_send, MSG_NOSIGNAL);
+ ret = send(s->fd, buf, bytes_to_send, MSG_NOSIGNAL);
if (ret == -1) {
+ if (errno == EAGAIN || errno == EINTR)
+ continue;
+
perror("send()");
exit(1);
}
- events[i].data.u64 -= ret;
+ s->bytes_left -= ret;
bytes_sent += ret;
// update the central counter after every 1MB (8ms
// at gigabit speeds, should be enough) of sent data
- if (bytes_sent > 1048576) {
+ if (bytes_sent > update_frequency) {
pthread_mutex_lock(&send_mutex);
total_bytes_sent += bytes_sent;
pthread_mutex_unlock(&send_mutex);
bytes_sent = 0;
}
- if (events[i].data.u64 == 0) {
- close(events[i].data.fd);
- if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, events[i].data.fd, NULL) == -1) {
- perror("EPOLL_CTL_ADD");
+ if (s->bytes_left == 0) {
+ if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, s->fd, NULL) == -1) {
+ perror("EPOLL_CTL_DEL");
exit(1);
}
-
- generate_new_sender(ep_fd);
+ close(s->fd);
+ generate_new_sender(ep_fd, s);
}
}
}
void *receiver_worker(void *arg)
{
- int sock = (int)arg;
+ int server_sock = (int)arg;
char buf[65536];
unsigned long long bytes_received = 0;
+ int ep_fd, i;
+ struct epoll_event ev, *events;
+
+ ep_fd = epoll_create(num_sockets_per_sender);
+ if (ep_fd == -1) {
+ perror("epoll_create");
+ exit(1);
+ }
+
+ // malloc, since there might not be enough room on the stack
+ events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * epoll_room_in_receiver);
+ if (events == NULL) {
+ perror("malloc");
+ exit(1);
+ }
+
+ // stick the receiver socket in the epoll set
+ ev.events = EPOLLIN;
+ ev.data.fd = server_sock;
+
+ if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, server_sock, &ev) == -1) {
+ perror("EPOLL_CTL_ADD");
+ exit(1);
+ }
for ( ;; ) {
- int ret = read(sock, buf, 65536);
- if (ret == 0)
- break;
+ int num_active = epoll_wait(ep_fd, events, epoll_room_in_receiver, -1);
+ if (num_active == -1) {
+ if (errno == EINTR)
+ continue;
+ perror("epoll_wait");
+ exit(1);
+ }
+
+ for (i = 0; i < num_active; ++i) {
+ int sock = events[i].data.fd;
+ int ret;
+ if (sock == server_sock) {
+ struct sockaddr_in addr;
+ int sock;
+ socklen_t addr_len = sizeof(addr);
+
+ sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
+ if (sock == -1) {
+ if (errno == EAGAIN || errno == EINTR) {
+ // another thread snatched it, ignore
+ continue;
+ } else {
+ perror("accept()");
+ exit(1);
+ }
+ }
+
+ // add it to the epoll set
+ ev.events = EPOLLIN | EPOLLHUP | EPOLLERR;
+ ev.data.fd = sock;
- bytes_received += ret;
+ if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
+ perror("EPOLL_CTL_ADD");
+ exit(1);
+ }
+ continue;
+ }
+
+ ret = read(sock, buf, 65536);
+ if (ret == 0) {
+ if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, sock, NULL) == -1) {
+ perror("EPOLL_CTL_DEL");
+ exit(1);
+ }
+ close(sock);
+ continue;
+ }
- // update the central counter after every 1MB (8ms
- // at gigabit speeds, should be enough) of received data
- if (bytes_received > 1048576) {
- pthread_mutex_lock(&receive_mutex);
- total_bytes_received += bytes_received;
- pthread_mutex_unlock(&receive_mutex);
+ bytes_received += ret;
- bytes_received = 0;
+ // update the central counter after every 1MB (8ms
+ // at gigabit speeds, should be enough) of received data
+ if (bytes_received > update_frequency) {
+ pthread_mutex_lock(&receive_mutex);
+ total_bytes_received += bytes_received;
+ pthread_mutex_unlock(&receive_mutex);
+
+ bytes_received = 0;
+ }
}
}
- if (close(sock) == -1) {
- perror("close()");
- exit(1);
- }
-
pthread_mutex_lock(&receive_mutex);
total_bytes_received += bytes_received;
pthread_mutex_unlock(&receive_mutex);
pthread_exit(0);
}
-
+
+// We're keeping this separate receiver dispatcher around because we might eventually
+// want to move away from the model where each receiver worker does the accept(). Thus,
+// it makes sense to keep this dispatcher around, even though all it does at the moment
+// is spawn off a few new threads and then die itself.
void *receiver_dispatcher(void *arg)
{
int server_sock = (int)arg;
+ int i;
pthread_attr_t attr;
+ pthread_t thread;
// FIXME: these do not really set errno
if (pthread_attr_init(&attr) != 0) {
exit(1);
}
- /*
- * Listen for incoming connections, spawning off one receiver
- * thread for each (which will just gobble up the data until
- * we're done).
- */
- for ( ;; ) {
- struct sockaddr_in addr;
- socklen_t addr_len = sizeof(addr);
- pthread_t thread;
-
- int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
- if (sock == -1) {
- perror("accept()");
- exit(1);
- }
-
- if (pthread_create(&thread, &attr, receiver_worker, (void *)sock) != 0) {
+ for (i = 0; i < num_receivers; ++i) {
+ if (pthread_create(&thread, &attr, receiver_worker, (void *)server_sock) != 0) {
perror("pthread_create()");
exit(1);
}
}
+
+ return NULL;
}
int get_server_socket(unsigned short port)
projects / jam / blobdiff