#include <stdio.h>
+#include <math.h>
+#include <string.h>
#include <stdlib.h>
+#include <getopt.h>
+#include <netdb.h>
#include <unistd.h>
#include <pthread.h>
#include <sys/socket.h>
#include <netinet/in.h>
+#include <sys/epoll.h>
+#include <sys/ioctl.h>
+#include <errno.h>
+
+unsigned short port = 2007;
+
+struct in_addr *destinations = NULL;
+unsigned num_destinations = 0;
+unsigned room_destinations = 0;
+
+struct in_addr *sources = NULL;
+unsigned num_sources = 0;
+unsigned room_sources = 0;
+
+unsigned num_senders = 128;
+unsigned num_sockets_per_sender = 16;
+unsigned do_listen = 1;
+
+unsigned long long total_bytes_received = 0;
+pthread_mutex_t receive_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+unsigned long long total_bytes_sent = 0;
+pthread_mutex_t send_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+const static struct option longopts[] = {
+ { "source-list", required_argument, NULL, 's' },
+ { "destination-list", required_argument, NULL, 'd' },
+ { "num-senders", required_argument, NULL, 'n' },
+ { "num-sockets-per-sender", required_argument, NULL, 'N' },
+ { "port", required_argument, NULL, 'p' },
+ { "sender-only", no_argument, NULL, 'o' },
+ { NULL, 0, NULL, 0 }
+};
+
+// generates from [0,1>
+double gen_uniform_random()
+{
+ return rand() / (RAND_MAX+1.0);
+}
+
+double gen_pareto_random(double min, double k)
+{
+ double u = gen_uniform_random();
+ return min * pow(u, -1.0 / k);
+}
+
+void read_ip_list(char *filename, struct in_addr **addr_list, unsigned *num, unsigned *room)
+{
+ char buf[256];
+ FILE *in = fopen(filename, "r");
+ if (in == NULL) {
+ perror(filename);
+ exit(1);
+ }
+
+ for ( ;; ) {
+ char *ptr;
+ struct in_addr addr;
+ struct hostent *he;
+
+ if (fgets(buf, 256, in) == NULL)
+ break;
+
+ ptr = strchr(buf, '\n');
+ if (ptr != NULL)
+ *ptr = 0;
+
+ ptr = strchr(buf, '\r');
+ if (ptr != NULL)
+ *ptr = 0;
+
+ ptr = buf + strspn(buf, " \t");
+
+ if (ptr[0] == '#' || ptr[0] == 0)
+ continue;
+
+ he = gethostbyname(ptr);
+ if (he == NULL) {
+ perror(ptr);
+ exit(1);
+ }
+
+ // just pick the first for now
+ memcpy(&addr.s_addr, he->h_addr_list[0], sizeof(addr.s_addr));
+
+ if (*num >= *room) {
+ if (*room == 0) {
+ *room = 16;
+ } else {
+ *room <<= 1;
+ }
+ *addr_list = (struct in_addr *)realloc(*addr_list, *room * sizeof(struct in_addr));
+ }
+
+ (*addr_list)[*num] = addr;
+ ++*num;
+ }
+
+ fclose(in);
+}
+
+void parse_options(int argc, char **argv)
+{
+ int option_index = 0;
+
+ for ( ;; ) {
+ int c = getopt_long(argc, argv, "s:d:n:N:p:o", longopts, &option_index);
+ switch (c) {
+ case 's':
+ read_ip_list(optarg, &sources, &num_sources, &room_sources);
+ break;
+ case 'd':
+ read_ip_list(optarg, &destinations, &num_destinations, &room_destinations);
+ break;
+ case 'n':
+ num_senders = atoi(optarg);
+ break;
+ case 'N':
+ num_sockets_per_sender = atoi(optarg);
+ break;
+ case 'p':
+ port = atoi(optarg);
+ break;
+ case 'o':
+ do_listen = 0;
+ break;
+ case -1:
+ return; // end of argument list
+ default:
+ fprintf(stderr, "Invalid option\n");
+ exit(1);
+ }
+ }
+}
+
+struct sender {
+ int fd;
+ unsigned long long bytes_left;
+};
+
+void generate_new_sender(int ep_fd)
+{
+ int sock, one = 1;
+ unsigned src_num, dst_num;
+ unsigned long long num_bytes;
+ struct sockaddr_in sin;
+ struct epoll_event ev;
+ struct sender *s;
+
+ s = (struct sender *)malloc(sizeof(struct sender));
+ if (s == NULL) {
+ perror("malloc()");
+ exit(1);
+ }
+
+ sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
+ if (sock == -1) {
+ perror("socket()");
+ exit(1);
+ }
+
+ // find the right parameters
+ src_num = (unsigned)(num_sources * gen_uniform_random());
+ dst_num = (unsigned)(num_destinations * gen_uniform_random());
+ num_bytes = (unsigned)gen_pareto_random(1048576.0, 1.0);
+
+ // FIXME: bind to the right source
+
+ if (ioctl(sock, FIONBIO, &one) == -1) {
+ perror("FIONBIO");
+ exit(1);
+ }
+
+ sin.sin_family = AF_INET;
+ sin.sin_port = htons(port);
+ sin.sin_addr = destinations[dst_num];
+
+ // non-blocking connect (will be detected by epoll later)
+ if (connect(sock, (struct sockaddr *)&sin, sizeof(sin)) == -1 && errno != EINPROGRESS) {
+ perror("connect()");
+ exit(1);
+ }
+
+ // stick it in the epoll set (FIXME: make edge-triggered?)
+ ev.events = EPOLLOUT | EPOLLHUP | EPOLLERR;
+ ev.data.ptr = s;
+
+ s->fd = sock;
+ s->bytes_left = num_bytes;
+
+ if (epoll_ctl(ep_fd, EPOLL_CTL_ADD, sock, &ev) == -1) {
+ perror("EPOLL_CTL_ADD");
+ exit(1);
+ }
+}
+
+void *sender_worker(void *arg)
+{
+ unsigned i;
+ char buf[65536];
+ unsigned long long bytes_sent = 0;
+ int ep_fd;
+ struct epoll_event *events;
+
+ ep_fd = epoll_create(num_sockets_per_sender);
+ if (ep_fd == -1) {
+ perror("epoll_create");
+ exit(1);
+ }
+
+ // malloc, since there might not be enough room on the stack
+ events = (struct epoll_event *)malloc(sizeof(struct epoll_event) * num_sockets_per_sender);
+ if (events == NULL) {
+ perror("malloc");
+ exit(1);
+ }
+
+ // fill the buffer with random junk
+ for (i = 0; i < 65536; ++i)
+ buf[i] = rand() & 0xff;
+
+ // allocate all the senders
+ for (i = 0; i < num_sockets_per_sender; ++i)
+ generate_new_sender(ep_fd);
+
+ for ( ;; ) {
+ int num_active = epoll_wait(ep_fd, events, num_sockets_per_sender, -1);
+ if (num_active == -1) {
+ perror("epoll_wait");
+ exit(1);
+ }
+
+ for (i = 0; i < num_active; ++i) {
+ struct sender *s = (struct sender *)events[i].data.ptr;
+ unsigned long long bytes_to_send = s->bytes_left;
+ unsigned ret;
+
+ if (bytes_to_send > 65536) {
+ bytes_to_send = 65536;
+ }
+
+ ret = send(s->fd, buf, bytes_to_send, MSG_NOSIGNAL);
+ if (ret == -1) {
+ if (errno == EAGAIN)
+ continue;
+
+ perror("send()");
+ exit(1);
+ }
+
+ s->bytes_left -= ret;
+ bytes_sent += ret;
+
+ // update the central counter after every 1MB (8ms
+ // at gigabit speeds, should be enough) of sent data
+ if (bytes_sent > 1048576) {
+ pthread_mutex_lock(&send_mutex);
+ total_bytes_sent += bytes_sent;
+ pthread_mutex_unlock(&send_mutex);
+
+ bytes_sent = 0;
+ }
+
+ if (s->bytes_left == 0) {
+ if (epoll_ctl(ep_fd, EPOLL_CTL_DEL, s->fd, NULL) == -1) {
+ perror("EPOLL_CTL_ADD");
+ exit(1);
+ }
+ close(s->fd);
+
+ free(s);
+ generate_new_sender(ep_fd);
+ }
+ }
+ }
+
+ pthread_mutex_lock(&send_mutex);
+ total_bytes_sent += bytes_sent;
+ pthread_mutex_unlock(&send_mutex);
+
+ free(events);
+ close(ep_fd);
+
+ pthread_exit(0);
+}
void *receiver_worker(void *arg)
{
int sock = (int)arg;
char buf[65536];
-
- printf("Received worker for socket %u\n", sock);
+ unsigned long long bytes_received = 0;
for ( ;; ) {
int ret = read(sock, buf, 65536);
if (ret == 0)
break;
- // FIXME: update stats here
- }
+ bytes_received += ret;
+
+ // update the central counter after every 1MB (8ms
+ // at gigabit speeds, should be enough) of received data
+ if (bytes_received > 1048576) {
+ pthread_mutex_lock(&receive_mutex);
+ total_bytes_received += bytes_received;
+ pthread_mutex_unlock(&receive_mutex);
- printf("Socket %u done\n", sock);
+ bytes_received = 0;
+ }
+ }
if (close(sock) == -1) {
perror("close()");
exit(1);
}
+
+ pthread_mutex_lock(&receive_mutex);
+ total_bytes_received += bytes_received;
+ pthread_mutex_unlock(&receive_mutex);
pthread_exit(0);
}
+
+void *receiver_dispatcher(void *arg)
+{
+ int server_sock = (int)arg;
+ pthread_attr_t attr;
+
+ // FIXME: these do not really set errno
+ if (pthread_attr_init(&attr) != 0) {
+ perror("pthread_attr_init()");
+ exit(1);
+ }
+
+ if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) {
+ perror("pthread_attr_setstacksize");
+ exit(1);
+ }
+
+ if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
+ perror("pthread_attr_setdetachstate");
+ exit(1);
+ }
+
+ /*
+ * Listen for incoming connections, spawning off one receiver
+ * thread for each (which will just gobble up the data until
+ * we're done).
+ */
+ for ( ;; ) {
+ struct sockaddr_in addr;
+ socklen_t addr_len = sizeof(addr);
+ pthread_t thread;
+
+ int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
+ if (sock == -1) {
+ perror("accept()");
+ exit(1);
+ }
+
+ if (pthread_create(&thread, &attr, receiver_worker, (void *)sock) != 0) {
+ perror("pthread_create()");
+ exit(1);
+ }
+ }
+}
int get_server_socket(unsigned short port)
{
int server_sock;
struct sockaddr_in sin;
+ unsigned one = 1;
server_sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
if (server_sock == -1) {
exit(1);
}
+ if (setsockopt(server_sock, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)) == -1) {
+ perror("setsocket(SO_REUSEADDR)");
+ exit(1);
+ }
+
sin.sin_family = AF_INET;
sin.sin_port = htons(port);
sin.sin_addr.s_addr = INADDR_ANY;
return server_sock;
}
-int main()
+int main(int argc, char **argv)
{
- int server_sock = get_server_socket(2007);
+ int server_sock;
+ unsigned i;
+ pthread_attr_t attr;
- // FIXME: fire off sender workers here
+ parse_options(argc, argv);
- /*
- * Listen for incoming connections, spawning off one receiver
- * thread for each (which will just gobble up the data until
- * we're done).
- */
- for ( ;; ) {
- struct sockaddr_in addr;
- socklen_t addr_len = sizeof(addr);
+ if (destinations == NULL || sources == NULL) {
+ fprintf(stderr, "Missing or empty source or destination host list. Aborting.\n");
+ exit(1);
+ }
+
+ if (do_listen) {
+ server_sock = get_server_socket(port);
+ }
+
+ printf("Sending data on port %u from %u sources to %u destinations.\n\n",
+ port, num_sources, num_destinations);
+
+ // FIXME: these do not really set errno
+ if (pthread_attr_init(&attr) != 0) {
+ perror("pthread_attr_init()");
+ exit(1);
+ }
+
+ if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN + 65536 + 0x4000) != 0) {
+ perror("pthread_attr_setstacksize");
+ exit(1);
+ }
+
+ if (pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0) {
+ perror("pthread_attr_setdetachstate");
+ exit(1);
+ }
+
+ // Fire off the master receiver.
+ if (do_listen) {
pthread_t thread;
+ if (pthread_create(&thread, &attr, receiver_dispatcher, (void *)server_sock) != 0) {
+ perror("pthread_create()");
+ exit(1);
+ }
+ }
- int sock = accept(server_sock, (struct sockaddr *)&addr, &addr_len);
- if (sock == -1) {
- perror("accept()");
+ printf("Waiting five seconds before starting senders...\n");
+ sleep(5);
+
+ // Fire off sender workers.
+ for (i = 0; i < num_senders; ++i) {
+ pthread_t thread;
+
+ if (pthread_create(&thread, &attr, sender_worker, NULL) != 0) {
+ perror("pthread_create()");
exit(1);
}
+ }
+
+ // Just stay around collecting statistics until we're done.
+ for ( ;; ) {
+ unsigned long long sent, received;
+ static unsigned long long last_sent = 0, last_received = 0;
+ double recv_rate, send_rate;
+
+ pthread_mutex_lock(&send_mutex);
+ sent = total_bytes_sent;
+ pthread_mutex_unlock(&send_mutex);
+
+ pthread_mutex_lock(&receive_mutex);
+ received = total_bytes_received;
+ pthread_mutex_unlock(&receive_mutex);
+
+ send_rate = (sent - last_sent) * 8.0 / 1048576.0;
+ recv_rate = (received - last_received) * 8.0 / 1048576.0;
+
+ printf("%12llu %12llu %5.0f Mbit/sec %5.0f Mbit/sec\n", sent, received,
+ send_rate, recv_rate);
+
+ last_sent = sent;
+ last_received = received;
- pthread_create(&thread, NULL, receiver_worker, (void *)sock);
+ sleep(1);
}
exit(0);