#include <stdint.h>
#include <pthread.h>
#include <sys/epoll.h>
+#include <time.h>
#include <string>
#include <map>
#include <vector>
-#define BACKLOG_SIZE 1048576
+#include "thread.h"
+
#define EPOLL_MAX_EVENTS 8192
#define EPOLL_TIMEOUT_MS 20
#define MAX_CLIENT_REQUEST 16384
class ClientProto;
class CubemapStateProto;
+class MarkPool;
+class Stream;
class StreamProto;
+// Digested statistics for writing to logs etc.
+struct ClientStats {
+ std::string stream_id;
+ std::string remote_addr;
+ time_t connect_time;
+ size_t bytes_sent;
+};
+
struct Client {
Client() {}
Client(int sock);
// Serialization/deserialization.
- Client(const ClientProto &serialized);
+ Client(const ClientProto &serialized, Stream *stream);
ClientProto serialize() const;
+ ClientStats get_stats() const;
+
// The file descriptor associated with this socket.
int sock;
+ // The fwmark associated with this socket (or 0).
+ int fwmark;
+
+ // Some information only used for logging.
+ std::string remote_addr;
+ time_t connect_time;
+
enum State { READING_REQUEST, SENDING_HEADER, SENDING_DATA, SENDING_ERROR };
State state;
// What stream we're connecting to; parsed from <request>.
// Not relevant for READING_REQUEST.
std::string stream_id;
+ Stream *stream;
// The header we want to send. This is nominally a copy of Stream::header,
// but since that might change on reconnects etc., we keep a local copy here.
};
struct Stream {
- Stream(const std::string &stream_id);
+ Stream(const std::string &stream_id, size_t backlog_size);
~Stream();
// Serialization/deserialization.
Stream(const StreamProto &serialized);
- StreamProto serialize() const;
+ StreamProto serialize();
std::string stream_id;
std::string header;
// The stream data itself, stored in a circular buffer.
- char *data;
+ //
+ // We store our data in a file, so that we can send the data to the
+ // kernel only once (with write()). We then use sendfile() for each
+ // client, which effectively zero-copies it out of the kernel's buffer
+ // cache. This is significantly more efficient than doing write() from
+ // a userspace memory buffer, since the latter makes the kernel copy
+ // the same data from userspace many times.
+ int data_fd;
+
+ // How many bytes <data_fd> can hold (the buffer size).
+ size_t backlog_size;
+
+ // How many bytes this stream have received. Can very well be larger
+ // than <backlog_size>, since the buffer wraps.
+ size_t bytes_received;
+
+ // Clients that are in SENDING_DATA, but that we don't listen on,
+ // because we currently don't have any data for them.
+ // See put_client_to_sleep() and wake_up_all_clients().
+ std::vector<Client *> sleeping_clients;
+
+ // Clients that we recently got data for (when they were in
+ // <sleeping_clients>).
+ std::vector<Client *> to_process;
+
+ // What pool to fetch marks from, or NULL.
+ MarkPool *mark_pool;
+
+ // Put client to sleep, since there is no more data for it; we will on
+ // longer listen on POLLOUT until we get more data. Also, it will be put
+ // in the list of clients to wake up when we do.
+ void put_client_to_sleep(Client *client);
- // How many bytes <data> contains. Can very well be larger than BACKLOG_SIZE,
- // since the buffer wraps.
- size_t data_size;
+ // We have more data, so mark all clients that are sleeping as ready to go.
+ void wake_up_all_clients();
private:
Stream(const Stream& other);
};
-class Server {
+class Server : public Thread {
public:
Server();
~Server();
- // Start a new thread that handles clients.
- void run();
+ // Get the list of all currently connected clients.
+ std::vector<ClientStats> get_client_stats() const;
- // Stop the thread.
- void stop();
+ // Set header (both HTTP header and any stream headers) for the given stream.
+ void set_header(const std::string &stream_id, const std::string &header);
- CubemapStateProto serialize() const;
+ // Set that the given stream should use the given mark pool from now on.
+ // NOTE: This should be set before any clients are connected!
+ void set_mark_pool(const std::string &stream_id, MarkPool *mark_pool);
- void add_client(int sock);
- void add_client_from_serialized(const ClientProto &client);
+ // These will be deferred until the next time an iteration in do_work() happens,
+ // and the order between them are undefined.
+ // XXX: header should ideally be ordered with respect to data.
+ void add_client_deferred(int sock);
+ void add_data_deferred(const std::string &stream_id, const char *data, size_t bytes);
- void add_stream(const std::string &stream_id);
+ // These should not be called while running, since that would violate
+ // threading assumptions (ie., that epoll is only called from one thread
+ // at the same time).
+ CubemapStateProto serialize();
+ void add_client_from_serialized(const ClientProto &client);
+ void add_stream(const std::string &stream_id, size_t bytes_received);
void add_stream_from_serialized(const StreamProto &stream);
- void set_header(const std::string &stream_id, const std::string &header);
- void add_data(const std::string &stream_id, const char *data, size_t bytes);
-
private:
- pthread_t worker_thread;
+ // Mutex protecting queued_data only. Note that if you want to hold both this
+ // and <mutex> below, you will need to take <mutex> before this one.
+ mutable pthread_mutex_t queued_data_mutex;
- // All variables below this line are protected by the mutex.
- pthread_mutex_t mutex;
+ // Deferred commands that should be run from the do_work() thread as soon as possible.
+ // We defer these for two reasons:
+ //
+ // - We only want to fiddle with epoll from one thread at any given time,
+ // and doing add_client() from the acceptor thread would violate that.
+ // - We don't want the input thread(s) hanging on <mutex> when doing
+ // add_data(), since they want to do add_data() rather often, and <mutex>
+ // can be taken a lot of the time.
+ //
+ // Protected by <queued_data_mutex>.
+ std::vector<int> queued_add_clients;
+ std::map<std::string, std::string> queued_data;
- // If the thread should stop or not.
- bool should_stop;
+ // All variables below this line are protected by the mutex.
+ mutable pthread_mutex_t mutex;
// Map from stream ID to stream.
std::map<std::string, Stream *> streams;
int epoll_fd;
epoll_event events[EPOLL_MAX_EVENTS];
- // Clients that are in SENDING_DATA, but that we don't listen on,
- // because we currently don't have any data for them.
- // See put_client_to_sleep() and wake_up_all_clients().
- std::vector<Client *> sleeping_clients;
-
- // Recover the this pointer, and call do_work().
- static void *do_work_thunk(void *arg);
-
// The actual worker thread.
- void do_work();
+ virtual void do_work();
// Process a client; read and write data as far as we can.
// After this call, one of these four is true:
// the SENDING_ERROR state.
void construct_error(Client *client, int error_code);
- // Put client to sleep, since there is no more data for it; we will on
- // longer listen on POLLOUT until we get more data. Also, it will be put
- // in the list of clients to wake up when we do.
- void put_client_to_sleep(Client *client);
-
- // We have more data, so mark all clients that are sleeping as ready to go.
- void wake_up_all_clients();
-
// TODO: This function should probably die.
Stream *find_stream(const std::string &stream_id);
+
+ void process_queued_data();
+
+ void add_client(int sock);
+ void add_data(const std::string &stream_id, const char *data, ssize_t bytes);
};
#endif // !defined(_SERVER_H)