Support configurable BACKLOG_SIZE (per-stream). No support for changing across restar...

[cubemap] / server.h

#ifndef _SERVER_H
#define _SERVER_H 1

#include <stdint.h>
#include <pthread.h>
#include <sys/epoll.h>
#include <time.h>
#include <string>
#include <map>
#include <vector>

#include "thread.h"

#define EPOLL_MAX_EVENTS 8192
#define EPOLL_TIMEOUT_MS 20
#define MAX_CLIENT_REQUEST 16384

class ClientProto;
class CubemapStateProto;
class MarkPool;
class Stream;
class StreamProto;

// Digested statistics for writing to logs etc.
struct ClientStats {
        std::string stream_id;
        std::string remote_addr;
        time_t connect_time;
        size_t bytes_sent;
};

struct Client {
        Client() {}
        Client(int sock);

        // Serialization/deserialization.
        Client(const ClientProto &serialized, Stream *stream);
        ClientProto serialize() const;

        ClientStats get_stats() const;

        // The file descriptor associated with this socket.
        int sock;

        // The fwmark associated with this socket (or 0).
        int fwmark;

        // Some information only used for logging.
        std::string remote_addr;
        time_t connect_time;

        enum State { READING_REQUEST, SENDING_HEADER, SENDING_DATA, SENDING_ERROR };
        State state;

        // The HTTP request, as sent by the client. If we are in READING_REQUEST,
        // this might not be finished.
        std::string request;

        // What stream we're connecting to; parsed from <request>.
        // Not relevant for READING_REQUEST.
        std::string stream_id;
        Stream *stream;

        // The header we want to send. This is nominally a copy of Stream::header,
        // but since that might change on reconnects etc., we keep a local copy here.
        // Only relevant for SENDING_HEADER or SENDING_ERROR; blank otherwise.
        std::string header_or_error;

        // Number of bytes we've sent of the header. Only relevant for SENDING_HEADER
        // or SENDING_ERROR.
        size_t header_or_error_bytes_sent;

        // Number of bytes we've sent of data. Only relevant for SENDING_DATA.
        size_t bytes_sent;
};

struct Stream {
        Stream(const std::string &stream_id, size_t backlog_size);
        ~Stream();

        // Serialization/deserialization.
        Stream(const StreamProto &serialized);
        StreamProto serialize();

        std::string stream_id;

        // The HTTP response header, plus the video stream header (if any).
        std::string header;

        // The stream data itself, stored in a circular buffer.
        //
        // We store our data in a file, so that we can send the data to the
        // kernel only once (with write()). We then use sendfile() for each
        // client, which effectively zero-copies it out of the kernel's buffer
        // cache. This is significantly more efficient than doing write() from
        // a userspace memory buffer, since the latter makes the kernel copy
        // the same data from userspace many times.
        int data_fd;

        // How many bytes <data_fd> can hold (the buffer size).
        size_t backlog_size;

        // How many bytes this stream have received. Can very well be larger
        // than <backlog_size>, since the buffer wraps.
        size_t bytes_received;
        
        // Clients that are in SENDING_DATA, but that we don't listen on,
        // because we currently don't have any data for them.
        // See put_client_to_sleep() and wake_up_all_clients().
        std::vector<Client *> sleeping_clients;

        // Clients that we recently got data for (when they were in
        // <sleeping_clients>).
        std::vector<Client *> to_process;

        // What pool to fetch marks from, or NULL.
        MarkPool *mark_pool;

        // Put client to sleep, since there is no more data for it; we will on
        // longer listen on POLLOUT until we get more data. Also, it will be put
        // in the list of clients to wake up when we do.
        void put_client_to_sleep(Client *client);

        // We have more data, so mark all clients that are sleeping as ready to go.
        void wake_up_all_clients();

private:
        Stream(const Stream& other);
};

class Server : public Thread {
public:
        Server();
        ~Server();

        // Get the list of all currently connected clients.     
        std::vector<ClientStats> get_client_stats() const;

        // Set header (both HTTP header and any stream headers) for the given stream.
        void set_header(const std::string &stream_id, const std::string &header);

        // Set that the given stream should use the given mark pool from now on.
        // NOTE: This should be set before any clients are connected!
        void set_mark_pool(const std::string &stream_id, MarkPool *mark_pool);

        // These will be deferred until the next time an iteration in do_work() happens,
        // and the order between them are undefined.
        // XXX: header should ideally be ordered with respect to data.
        void add_client_deferred(int sock);
        void add_data_deferred(const std::string &stream_id, const char *data, size_t bytes);

        // These should not be called while running, since that would violate
        // threading assumptions (ie., that epoll is only called from one thread
        // at the same time).
        CubemapStateProto serialize();
        void add_client_from_serialized(const ClientProto &client);
        void add_stream(const std::string &stream_id, size_t bytes_received);
        void add_stream_from_serialized(const StreamProto &stream);

private:
        // Mutex protecting queued_data only. Note that if you want to hold both this
        // and <mutex> below, you will need to take <mutex> before this one.
        mutable pthread_mutex_t queued_data_mutex;

        // Deferred commands that should be run from the do_work() thread as soon as possible.
        // We defer these for two reasons:
        //
        //  - We only want to fiddle with epoll from one thread at any given time,
        //    and doing add_client() from the acceptor thread would violate that.
        //  - We don't want the input thread(s) hanging on <mutex> when doing
        //    add_data(), since they want to do add_data() rather often, and <mutex>
        //    can be taken a lot of the time.
        //      
        // Protected by <queued_data_mutex>.
        std::vector<int> queued_add_clients;
        std::map<std::string, std::string> queued_data;

        // All variables below this line are protected by the mutex.
        mutable pthread_mutex_t mutex;

        // Map from stream ID to stream.
        std::map<std::string, Stream *> streams;

        // Map from file descriptor to client.
        std::map<int, Client> clients;

        // Used for epoll implementation (obviously).
        int epoll_fd;
        epoll_event events[EPOLL_MAX_EVENTS];

        // The actual worker thread.
        virtual void do_work();

        // Process a client; read and write data as far as we can.
        // After this call, one of these four is true:
        //
        //  1. The socket is closed, and the client deleted.
        //  2. We are still waiting for more data from the client.
        //  3. We've sent all the data we have to the client,
        //     and put it in <sleeping_clients>.
        //  4. The socket buffer is full (which means we still have
        //     data outstanding).
        //
        // For #2, we listen for EPOLLIN events. For #3 and #4, we listen
        // for EPOLLOUT in edge-triggered mode; it will never fire for #3,
        // but it's cheaper than taking it in and out all the time.
        void process_client(Client *client);

        // Close a given client socket, and clean up after it.
        void close_client(Client *client);

        // Parse the HTTP request. Returns a HTTP status code (200/400/404).
        int parse_request(Client *client);

        // Construct the HTTP header, and set the client into
        // the SENDING_HEADER state.
        void construct_header(Client *client);

        // Construct a generic error with the given line, and set the client into
        // the SENDING_ERROR state.
        void construct_error(Client *client, int error_code);

        // TODO: This function should probably die.
        Stream *find_stream(const std::string &stream_id);

        void process_queued_data();

        void add_client(int sock);
        void add_data(const std::string &stream_id, const char *data, ssize_t bytes);
};

#endif  // !defined(_SERVER_H)