X-Git-Url: https://git.sesse.net/?p=cubemap;a=blobdiff_plain;f=server.cpp;h=72664e33f665eda4bd09e49d22789e546b4ee81f;hp=7acf501236e1ee2db9cac22db0b72a5ddf8ecf69;hb=b757a4a2ce9d24835b52a185134835762af2f50c;hpb=cabe676b1637ad3d7193d59090f43d271253e81a diff --git a/server.cpp b/server.cpp index 7acf501..72664e3 100644 --- a/server.cpp +++ b/server.cpp @@ -19,8 +19,7 @@ #include "accesslog.h" #include "log.h" -#include "markpool.h" -#include "metacube.h" +#include "metacube2.h" #include "mutexlock.h" #include "parse.h" #include "server.h" @@ -28,14 +27,35 @@ #include "stream.h" #include "util.h" +#ifndef SO_MAX_PACING_RATE +#define SO_MAX_PACING_RATE 47 +#endif + using namespace std; extern AccessLogThread *access_log; +namespace { + +inline bool is_equal(timespec a, timespec b) +{ + return a.tv_sec == b.tv_sec && + a.tv_nsec == b.tv_nsec; +} + +inline bool is_earlier(timespec a, timespec b) +{ + if (a.tv_sec != b.tv_sec) + return a.tv_sec < b.tv_sec; + return a.tv_nsec < b.tv_nsec; +} + +} // namespace + Server::Server() { pthread_mutex_init(&mutex, NULL); - pthread_mutex_init(&queued_data_mutex, NULL); + pthread_mutex_init(&queued_clients_mutex, NULL); epoll_fd = epoll_create(1024); // Size argument is ignored. if (epoll_fd == -1) { @@ -87,6 +107,7 @@ void Server::do_work() process_queued_data(); + // Process each client where we have socket activity. for (int i = 0; i < nfds; ++i) { Client *client = reinterpret_cast(events[i].data.u64); @@ -98,6 +119,8 @@ void Server::do_work() process_client(client); } + // Process each client where its stream has new data, + // even if there was no socket activity. for (size_t i = 0; i < streams.size(); ++i) { vector to_process; swap(streams[i]->to_process, to_process); @@ -105,6 +128,49 @@ void Server::do_work() process_client(to_process[i]); } } + + // Finally, go through each client to see if it's timed out + // in the READING_REQUEST state. (Seemingly there are clients + // that can hold sockets up for days at a time without sending + // anything at all.) + timespec timeout_time; + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &timeout_time) == -1) { + log_perror("clock_gettime(CLOCK_MONOTONIC_COARSE)"); + continue; + } + timeout_time.tv_sec -= REQUEST_READ_TIMEOUT_SEC; + while (!clients_ordered_by_connect_time.empty()) { + const pair &connect_time_and_fd = clients_ordered_by_connect_time.front(); + + // See if we have reached the end of clients to process. + if (is_earlier(timeout_time, connect_time_and_fd.first)) { + break; + } + + // If this client doesn't exist anymore, just ignore it + // (it was deleted earlier). + map::iterator client_it = clients.find(connect_time_and_fd.second); + if (client_it == clients.end()) { + clients_ordered_by_connect_time.pop(); + continue; + } + Client *client = &client_it->second; + if (!is_equal(client->connect_time, connect_time_and_fd.first)) { + // Another client has taken this fd in the meantime. + clients_ordered_by_connect_time.pop(); + continue; + } + + if (client->state != Client::READING_REQUEST) { + // Only READING_REQUEST can time out. + clients_ordered_by_connect_time.pop(); + continue; + } + + // OK, it timed out. + close_client(client); + clients_ordered_by_connect_time.pop(); + } } } @@ -140,7 +206,7 @@ CubemapStateProto Server::serialize() void Server::add_client_deferred(int sock) { - MutexLock lock(&queued_data_mutex); + MutexLock lock(&queued_clients_mutex); queued_add_clients.push_back(sock); } @@ -151,6 +217,17 @@ void Server::add_client(int sock) assert(ret.second == true); // Should not already exist. Client *client_ptr = &ret.first->second; + // Connection timestamps must be nondecreasing. I can't find any guarantee + // that even the monotonic clock can't go backwards by a small amount + // (think switching between CPUs with non-synchronized TSCs), so if + // this actually should happen, we hack around it by fudging + // connect_time. + if (!clients_ordered_by_connect_time.empty() && + is_earlier(client_ptr->connect_time, clients_ordered_by_connect_time.back().first)) { + client_ptr->connect_time = clients_ordered_by_connect_time.back().first; + } + clients_ordered_by_connect_time.push(make_pair(client_ptr->connect_time, sock)); + // Start listening on data from this socket. epoll_event ev; ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP; @@ -179,6 +256,11 @@ void Server::add_client_from_serialized(const ClientProto &client) assert(ret.second == true); // Should not already exist. Client *client_ptr = &ret.first->second; + // Connection timestamps must be nondecreasing. + assert(clients_ordered_by_connect_time.empty() || + !is_earlier(client_ptr->connect_time, clients_ordered_by_connect_time.back().first)); + clients_ordered_by_connect_time.push(make_pair(client_ptr->connect_time, client.sock())); + // Start listening on data from this socket. epoll_event ev; if (client.state() == Client::READING_REQUEST) { @@ -194,15 +276,17 @@ void Server::add_client_from_serialized(const ClientProto &client) exit(1); } - if (client_ptr->state == Client::SENDING_DATA && - client_ptr->stream_pos == client_ptr->stream->bytes_received) { + if (client_ptr->state == Client::WAITING_FOR_KEYFRAME || + client_ptr->state == Client::PREBUFFERING || + (client_ptr->state == Client::SENDING_DATA && + client_ptr->stream_pos == client_ptr->stream->bytes_received)) { client_ptr->stream->put_client_to_sleep(client_ptr); } else { process_client(client_ptr); } } -int Server::lookup_stream_by_url(const std::string &url) const +int Server::lookup_stream_by_url(const string &url) const { map::const_iterator url_it = url_map.find(url); if (url_it == url_map.end()) { @@ -211,11 +295,11 @@ int Server::lookup_stream_by_url(const std::string &url) const return url_it->second; } -int Server::add_stream(const string &url, size_t backlog_size, Stream::Encoding encoding) +int Server::add_stream(const string &url, size_t backlog_size, size_t prebuffering_bytes, Stream::Encoding encoding) { MutexLock lock(&mutex); url_map.insert(make_pair(url, streams.size())); - streams.push_back(new Stream(url, backlog_size, encoding)); + streams.push_back(new Stream(url, backlog_size, prebuffering_bytes, encoding)); return streams.size() - 1; } @@ -233,6 +317,13 @@ void Server::set_backlog_size(int stream_index, size_t new_size) assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); streams[stream_index]->set_backlog_size(new_size); } + +void Server::set_prebuffering_bytes(int stream_index, size_t new_amount) +{ + MutexLock lock(&mutex); + assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); + streams[stream_index]->prebuffering_bytes = new_amount; +} void Server::set_encoding(int stream_index, Stream::Encoding encoding) { @@ -246,34 +337,32 @@ void Server::set_header(int stream_index, const string &http_header, const strin MutexLock lock(&mutex); assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); streams[stream_index]->http_header = http_header; - streams[stream_index]->stream_header = stream_header; - // If there are clients we haven't sent anything to yet, we should give - // them the header, so push back into the SENDING_HEADER state. - for (map::iterator client_it = clients.begin(); - client_it != clients.end(); - ++client_it) { - Client *client = &client_it->second; - if (client->state == Client::SENDING_DATA && - client->stream_pos == 0) { - construct_header(client); - } + if (stream_header != streams[stream_index]->stream_header) { + // We cannot start at any of the older starting points anymore, + // since they'd get the wrong header for the stream (not to mention + // that a changed header probably means the stream restarted, + // which means any client starting on the old one would probably + // stop playing properly at the change point). Next block + // should be a suitable starting point (if not, something is + // pretty strange), so it will fill up again soon enough. + streams[stream_index]->suitable_starting_points.clear(); } + streams[stream_index]->stream_header = stream_header; } -void Server::set_mark_pool(int stream_index, MarkPool *mark_pool) +void Server::set_pacing_rate(int stream_index, uint32_t pacing_rate) { MutexLock lock(&mutex); assert(clients.empty()); assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); - streams[stream_index]->mark_pool = mark_pool; + streams[stream_index]->pacing_rate = pacing_rate; } -void Server::add_data_deferred(int stream_index, const char *data, size_t bytes) +void Server::add_data_deferred(int stream_index, const char *data, size_t bytes, StreamStartSuitability suitable_for_stream_start) { - MutexLock lock(&queued_data_mutex); assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); - streams[stream_index]->add_data_deferred(data, bytes); + streams[stream_index]->add_data_deferred(data, bytes, suitable_for_stream_start); } // See the .h file for postconditions after this function. @@ -380,23 +469,76 @@ sending_header_or_error_again: return; } - // Start sending from the end. In other words, we won't send any of the backlog, - // but we'll start sending immediately as we get data. - // This is postcondition #3. - client->state = Client::SENDING_DATA; + Stream *stream = client->stream; if (client->stream_pos == size_t(-2)) { - client->stream_pos = std::min( - client->stream->bytes_received - client->stream->backlog_size, + // Start sending from the beginning of the backlog. + client->stream_pos = min( + stream->bytes_received - stream->backlog_size, 0); + client->state = Client::SENDING_DATA; + goto sending_data; + } else if (stream->prebuffering_bytes == 0) { + // Start sending from the first keyframe we get. In other + // words, we won't send any of the backlog, but we'll start + // sending immediately as we get the next keyframe block. + // Note that this is functionally identical to the next if branch, + // except that we save a binary search. + client->stream_pos = stream->bytes_received; + client->state = Client::WAITING_FOR_KEYFRAME; } else { - // client->stream_pos should be -1, but it might not be, - // if we have clients from an older version. - client->stream_pos = client->stream->bytes_received; + // We're not going to send anything to the client before we have + // N bytes. However, this wait might be boring; we can just as well + // use it to send older data if we have it. We use lower_bound() + // so that we are conservative and never add extra latency over just + // waiting (assuming CBR or nearly so); otherwise, we could want e.g. + // 100 kB prebuffer but end up sending a 10 MB GOP. + deque::const_iterator starting_point_it = + lower_bound(stream->suitable_starting_points.begin(), + stream->suitable_starting_points.end(), + stream->bytes_received - stream->prebuffering_bytes); + if (starting_point_it == stream->suitable_starting_points.end()) { + // None found. Just put us at the end, and then wait for the + // first keyframe to appear. + client->stream_pos = stream->bytes_received; + client->state = Client::WAITING_FOR_KEYFRAME; + } else { + client->stream_pos = *starting_point_it; + client->state = Client::PREBUFFERING; + goto prebuffering; + } } - client->stream->put_client_to_sleep(client); - return; + // Fall through. + } + case Client::WAITING_FOR_KEYFRAME: { + Stream *stream = client->stream; + if (stream->suitable_starting_points.empty() || + client->stream_pos > stream->suitable_starting_points.back()) { + // We haven't received a keyframe since this stream started waiting, + // so keep on waiting for one. + // This is postcondition #3. + stream->put_client_to_sleep(client); + return; + } + client->stream_pos = stream->suitable_starting_points.back(); + client->state = Client::PREBUFFERING; + // Fall through. + } + case Client::PREBUFFERING: { +prebuffering: + Stream *stream = client->stream; + size_t bytes_to_send = stream->bytes_received - client->stream_pos; + assert(bytes_to_send <= stream->backlog_size); + if (bytes_to_send < stream->prebuffering_bytes) { + // We don't have enough bytes buffered to start this client yet. + // This is postcondition #3. + stream->put_client_to_sleep(client); + return; + } + client->state = Client::SENDING_DATA; + // Fall through. } case Client::SENDING_DATA: { +sending_data: skip_lost_data(client); Stream *stream = client->stream; @@ -416,7 +558,7 @@ sending_data_again: ssize_t ret; do { - loff_t offset = client->stream_pos % stream->backlog_size; + off_t offset = client->stream_pos % stream->backlog_size; ret = sendfile(client->sock, stream->data_fd, &offset, bytes_to_send); } while (ret == -1 && errno == EINTR); @@ -454,18 +596,15 @@ sending_data_again: void Server::skip_lost_data(Client *client) { Stream *stream = client->stream; + if (stream == NULL) { + return; + } size_t bytes_to_send = stream->bytes_received - client->stream_pos; if (bytes_to_send > stream->backlog_size) { size_t bytes_lost = bytes_to_send - stream->backlog_size; client->stream_pos = stream->bytes_received - stream->backlog_size; client->bytes_lost += bytes_lost; ++client->num_loss_events; - - double loss_fraction = double(client->bytes_lost) / double(client->bytes_lost + client->bytes_sent); - log(WARNING, "[%s] Client lost %lld bytes (total loss: %.2f%%), maybe too slow connection", - client->remote_addr.c_str(), - (long long int)(bytes_lost), - 100.0 * loss_fraction); } } @@ -476,6 +615,18 @@ int Server::parse_request(Client *client) return 400; // Bad request (empty). } + // Parse the headers, for logging purposes. + // TODO: Case-insensitivity. + multimap headers = extract_headers(lines, client->remote_addr); + multimap::const_iterator referer_it = headers.find("Referer"); + if (referer_it != headers.end()) { + client->referer = referer_it->second; + } + multimap::const_iterator user_agent_it = headers.find("User-Agent"); + if (user_agent_it != headers.end()) { + client->user_agent = user_agent_it->second; + } + vector request_tokens = split_tokens(lines[0]); if (request_tokens.size() < 2) { return 400; // Bad request (empty). @@ -497,16 +648,17 @@ int Server::parse_request(Client *client) return 404; // Not found. } - client->url = request_tokens[1]; - client->stream = streams[url_map_it->second]; - if (client->stream->mark_pool != NULL) { - client->fwmark = client->stream->mark_pool->get_mark(); - } else { - client->fwmark = 0; // No mark. + Stream *stream = streams[url_map_it->second]; + if (stream->http_header.empty()) { + return 503; // Service unavailable. } - if (setsockopt(client->sock, SOL_SOCKET, SO_MARK, &client->fwmark, sizeof(client->fwmark)) == -1) { - if (client->fwmark != 0) { - log_perror("setsockopt(SO_MARK)"); + + client->url = request_tokens[1]; + + client->stream = stream; + if (setsockopt(client->sock, SOL_SOCKET, SO_MAX_PACING_RATE, &client->stream->pacing_rate, sizeof(client->stream->pacing_rate)) == -1) { + if (client->stream->pacing_rate != ~0U) { + log_perror("setsockopt(SO_MAX_PACING_RATE)"); } } client->request.clear(); @@ -526,10 +678,11 @@ void Server::construct_header(Client *client) "Content-encoding: metacube\r\n" + "\r\n"; if (!stream->stream_header.empty()) { - metacube_block_header hdr; - memcpy(hdr.sync, METACUBE_SYNC, sizeof(hdr.sync)); + metacube2_block_header hdr; + memcpy(hdr.sync, METACUBE2_SYNC, sizeof(hdr.sync)); hdr.size = htonl(stream->stream_header.size()); - hdr.flags = htonl(METACUBE_FLAGS_HEADER); + hdr.flags = htons(METACUBE_FLAGS_HEADER); + hdr.csum = htons(metacube2_compute_crc(&hdr)); client->header_or_error.append( string(reinterpret_cast(&hdr), sizeof(hdr))); } @@ -589,10 +742,6 @@ void Server::close_client(Client *client) if (client->stream != NULL) { delete_from(&client->stream->sleeping_clients, client); delete_from(&client->stream->to_process, client); - if (client->stream->mark_pool != NULL) { - int fwmark = client->fwmark; - client->stream->mark_pool->release_mark(fwmark); - } } // Log to access_log. @@ -606,12 +755,14 @@ void Server::close_client(Client *client) void Server::process_queued_data() { - MutexLock lock(&queued_data_mutex); + { + MutexLock lock(&queued_clients_mutex); - for (size_t i = 0; i < queued_add_clients.size(); ++i) { - add_client(queued_add_clients[i]); + for (size_t i = 0; i < queued_add_clients.size(); ++i) { + add_client(queued_add_clients[i]); + } + queued_add_clients.clear(); } - queued_add_clients.clear(); for (size_t i = 0; i < streams.size(); ++i) { streams[i]->process_queued_data();