X-Git-Url: https://git.sesse.net/?p=cubemap;a=blobdiff_plain;f=server.cpp;h=615e8fb31d48b137df3cdcb3acf8caa4cbf00c68;hp=093899939b4d832be6dd28a8a3e8966d46efa650;hb=38a2bb28fd8dcb5bb1e0cb56028936a35f20f503;hpb=c2080e598ec887ac7b087dbc287c42a14881d667 diff --git a/server.cpp b/server.cpp index 0938999..615e8fb 100644 --- a/server.cpp +++ b/server.cpp @@ -19,7 +19,6 @@ #include "accesslog.h" #include "log.h" -#include "markpool.h" #include "metacube2.h" #include "mutexlock.h" #include "parse.h" @@ -36,6 +35,23 @@ using namespace std; extern AccessLogThread *access_log; +namespace { + +inline bool is_equal(timespec a, timespec b) +{ + return a.tv_sec == b.tv_sec && + a.tv_nsec == b.tv_nsec; +} + +inline bool is_earlier(timespec a, timespec b) +{ + if (a.tv_sec != b.tv_sec) + return a.tv_sec < b.tv_sec; + return a.tv_nsec < b.tv_nsec; +} + +} // namespace + Server::Server() { pthread_mutex_init(&mutex, NULL); @@ -91,6 +107,7 @@ void Server::do_work() process_queued_data(); + // Process each client where we have socket activity. for (int i = 0; i < nfds; ++i) { Client *client = reinterpret_cast(events[i].data.u64); @@ -102,6 +119,8 @@ void Server::do_work() process_client(client); } + // Process each client where its stream has new data, + // even if there was no socket activity. for (size_t i = 0; i < streams.size(); ++i) { vector to_process; swap(streams[i]->to_process, to_process); @@ -109,6 +128,49 @@ void Server::do_work() process_client(to_process[i]); } } + + // Finally, go through each client to see if it's timed out + // in the READING_REQUEST state. (Seemingly there are clients + // that can hold sockets up for days at a time without sending + // anything at all.) + timespec timeout_time; + if (clock_gettime(CLOCK_MONOTONIC_COARSE, &timeout_time) == -1) { + log_perror("clock_gettime(CLOCK_MONOTONIC_COARSE)"); + continue; + } + timeout_time.tv_sec -= REQUEST_READ_TIMEOUT_SEC; + while (!clients_ordered_by_connect_time.empty()) { + const pair &connect_time_and_fd = clients_ordered_by_connect_time.front(); + + // See if we have reached the end of clients to process. + if (is_earlier(timeout_time, connect_time_and_fd.first)) { + break; + } + + // If this client doesn't exist anymore, just ignore it + // (it was deleted earlier). + map::iterator client_it = clients.find(connect_time_and_fd.second); + if (client_it == clients.end()) { + clients_ordered_by_connect_time.pop(); + continue; + } + Client *client = &client_it->second; + if (!is_equal(client->connect_time, connect_time_and_fd.first)) { + // Another client has taken this fd in the meantime. + clients_ordered_by_connect_time.pop(); + continue; + } + + if (client->state != Client::READING_REQUEST) { + // Only READING_REQUEST can time out. + clients_ordered_by_connect_time.pop(); + continue; + } + + // OK, it timed out. + close_client(client); + clients_ordered_by_connect_time.pop(); + } } } @@ -155,6 +217,17 @@ void Server::add_client(int sock) assert(ret.second == true); // Should not already exist. Client *client_ptr = &ret.first->second; + // Connection timestamps must be nondecreasing. I can't find any guarantee + // that even the monotonic clock can't go backwards by a small amount + // (think switching between CPUs with non-synchronized TSCs), so if + // this actually should happen, we hack around it by fudging + // connect_time. + if (!clients_ordered_by_connect_time.empty() && + is_earlier(client_ptr->connect_time, clients_ordered_by_connect_time.back().first)) { + client_ptr->connect_time = clients_ordered_by_connect_time.back().first; + } + clients_ordered_by_connect_time.push(make_pair(client_ptr->connect_time, sock)); + // Start listening on data from this socket. epoll_event ev; ev.events = EPOLLIN | EPOLLET | EPOLLRDHUP; @@ -183,6 +256,11 @@ void Server::add_client_from_serialized(const ClientProto &client) assert(ret.second == true); // Should not already exist. Client *client_ptr = &ret.first->second; + // Connection timestamps must be nondecreasing. + assert(clients_ordered_by_connect_time.empty() || + !is_earlier(client_ptr->connect_time, clients_ordered_by_connect_time.back().first)); + clients_ordered_by_connect_time.push(make_pair(client_ptr->connect_time, client.sock())); + // Start listening on data from this socket. epoll_event ev; if (client.state() == Client::READING_REQUEST) { @@ -199,6 +277,7 @@ void Server::add_client_from_serialized(const ClientProto &client) } if (client_ptr->state == Client::WAITING_FOR_KEYFRAME || + client_ptr->state == Client::PREBUFFERING || (client_ptr->state == Client::SENDING_DATA && client_ptr->stream_pos == client_ptr->stream->bytes_received)) { client_ptr->stream->put_client_to_sleep(client_ptr); @@ -207,7 +286,7 @@ void Server::add_client_from_serialized(const ClientProto &client) } } -int Server::lookup_stream_by_url(const std::string &url) const +int Server::lookup_stream_by_url(const string &url) const { map::const_iterator url_it = url_map.find(url); if (url_it == url_map.end()) { @@ -216,11 +295,11 @@ int Server::lookup_stream_by_url(const std::string &url) const return url_it->second; } -int Server::add_stream(const string &url, size_t backlog_size, Stream::Encoding encoding) +int Server::add_stream(const string &url, size_t backlog_size, size_t prebuffering_bytes, Stream::Encoding encoding) { MutexLock lock(&mutex); url_map.insert(make_pair(url, streams.size())); - streams.push_back(new Stream(url, backlog_size, encoding)); + streams.push_back(new Stream(url, backlog_size, prebuffering_bytes, encoding)); return streams.size() - 1; } @@ -238,6 +317,13 @@ void Server::set_backlog_size(int stream_index, size_t new_size) assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); streams[stream_index]->set_backlog_size(new_size); } + +void Server::set_prebuffering_bytes(int stream_index, size_t new_amount) +{ + MutexLock lock(&mutex); + assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); + streams[stream_index]->prebuffering_bytes = new_amount; +} void Server::set_encoding(int stream_index, Stream::Encoding encoding) { @@ -251,17 +337,20 @@ void Server::set_header(int stream_index, const string &http_header, const strin MutexLock lock(&mutex); assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); streams[stream_index]->http_header = http_header; + + if (stream_header != streams[stream_index]->stream_header) { + // We cannot start at any of the older starting points anymore, + // since they'd get the wrong header for the stream (not to mention + // that a changed header probably means the stream restarted, + // which means any client starting on the old one would probably + // stop playing properly at the change point). Next block + // should be a suitable starting point (if not, something is + // pretty strange), so it will fill up again soon enough. + streams[stream_index]->suitable_starting_points.clear(); + } streams[stream_index]->stream_header = stream_header; } -void Server::set_mark_pool(int stream_index, MarkPool *mark_pool) -{ - MutexLock lock(&mutex); - assert(clients.empty()); - assert(stream_index >= 0 && stream_index < ssize_t(streams.size())); - streams[stream_index]->mark_pool = mark_pool; -} - void Server::set_pacing_rate(int stream_index, uint32_t pacing_rate) { MutexLock lock(&mutex); @@ -334,17 +423,17 @@ read_request_again: } // We've changed states, so fall through. - assert(client->state == Client::SENDING_ERROR || + assert(client->state == Client::SENDING_SHORT_RESPONSE || client->state == Client::SENDING_HEADER); } - case Client::SENDING_ERROR: + case Client::SENDING_SHORT_RESPONSE: case Client::SENDING_HEADER: { -sending_header_or_error_again: +sending_header_or_short_response_again: int ret; do { ret = write(client->sock, - client->header_or_error.data() + client->header_or_error_bytes_sent, - client->header_or_error.size() - client->header_or_error_bytes_sent); + client->header_or_short_response.data() + client->header_or_short_response_bytes_sent, + client->header_or_short_response.size() - client->header_or_short_response_bytes_sent); } while (ret == -1 && errno == EINTR); if (ret == -1 && errno == EAGAIN) { @@ -362,56 +451,94 @@ sending_header_or_error_again: return; } - client->header_or_error_bytes_sent += ret; - assert(client->header_or_error_bytes_sent <= client->header_or_error.size()); + client->header_or_short_response_bytes_sent += ret; + assert(client->header_or_short_response_bytes_sent <= client->header_or_short_response.size()); - if (client->header_or_error_bytes_sent < client->header_or_error.size()) { + if (client->header_or_short_response_bytes_sent < client->header_or_short_response.size()) { // We haven't sent all yet. Fine; go another round. - goto sending_header_or_error_again; + goto sending_header_or_short_response_again; } // We're done sending the header or error! Clear it to release some memory. - client->header_or_error.clear(); + client->header_or_short_response.clear(); - if (client->state == Client::SENDING_ERROR) { + if (client->state == Client::SENDING_SHORT_RESPONSE) { // We're done sending the error, so now close. // This is postcondition #1. close_client(client); return; } - // Start sending from the first keyframe we get. In other - // words, we won't send any of the backlog, but we'll start - // sending immediately as we get the next keyframe block. - // This is postcondition #3. + Stream *stream = client->stream; if (client->stream_pos == size_t(-2)) { - client->stream_pos = std::min( - client->stream->bytes_received - client->stream->backlog_size, + // Start sending from the beginning of the backlog. + client->stream_pos = min( + stream->bytes_received - stream->backlog_size, 0); client->state = Client::SENDING_DATA; - } else { - // client->stream_pos should be -1, but it might not be, - // if we have clients from an older version. - client->stream_pos = client->stream->bytes_received; + goto sending_data; + } else if (stream->prebuffering_bytes == 0) { + // Start sending from the first keyframe we get. In other + // words, we won't send any of the backlog, but we'll start + // sending immediately as we get the next keyframe block. + // Note that this is functionally identical to the next if branch, + // except that we save a binary search. + client->stream_pos = stream->bytes_received; client->state = Client::WAITING_FOR_KEYFRAME; + } else { + // We're not going to send anything to the client before we have + // N bytes. However, this wait might be boring; we can just as well + // use it to send older data if we have it. We use lower_bound() + // so that we are conservative and never add extra latency over just + // waiting (assuming CBR or nearly so); otherwise, we could want e.g. + // 100 kB prebuffer but end up sending a 10 MB GOP. + deque::const_iterator starting_point_it = + lower_bound(stream->suitable_starting_points.begin(), + stream->suitable_starting_points.end(), + stream->bytes_received - stream->prebuffering_bytes); + if (starting_point_it == stream->suitable_starting_points.end()) { + // None found. Just put us at the end, and then wait for the + // first keyframe to appear. + client->stream_pos = stream->bytes_received; + client->state = Client::WAITING_FOR_KEYFRAME; + } else { + client->stream_pos = *starting_point_it; + client->state = Client::PREBUFFERING; + goto prebuffering; + } } - client->stream->put_client_to_sleep(client); - return; + // Fall through. } case Client::WAITING_FOR_KEYFRAME: { Stream *stream = client->stream; - if (ssize_t(client->stream_pos) > stream->last_suitable_starting_point) { + if (stream->suitable_starting_points.empty() || + client->stream_pos > stream->suitable_starting_points.back()) { // We haven't received a keyframe since this stream started waiting, // so keep on waiting for one. // This is postcondition #3. stream->put_client_to_sleep(client); return; } - client->stream_pos = stream->last_suitable_starting_point; + client->stream_pos = stream->suitable_starting_points.back(); + client->state = Client::PREBUFFERING; + // Fall through. + } + case Client::PREBUFFERING: { +prebuffering: + Stream *stream = client->stream; + size_t bytes_to_send = stream->bytes_received - client->stream_pos; + assert(bytes_to_send <= stream->backlog_size); + if (bytes_to_send < stream->prebuffering_bytes) { + // We don't have enough bytes buffered to start this client yet. + // This is postcondition #3. + stream->put_client_to_sleep(client); + return; + } client->state = Client::SENDING_DATA; // Fall through. } case Client::SENDING_DATA: { +sending_data: skip_lost_data(client); Stream *stream = client->stream; @@ -478,12 +605,6 @@ void Server::skip_lost_data(Client *client) client->stream_pos = stream->bytes_received - stream->backlog_size; client->bytes_lost += bytes_lost; ++client->num_loss_events; - - double loss_fraction = double(client->bytes_lost) / double(client->bytes_lost + client->bytes_sent); - log(WARNING, "[%s] Client lost %lld bytes (total loss: %.2f%%), maybe too slow connection", - client->remote_addr.c_str(), - (long long int)(bytes_lost), - 100.0 * loss_fraction); } } @@ -494,6 +615,18 @@ int Server::parse_request(Client *client) return 400; // Bad request (empty). } + // Parse the headers, for logging purposes. + // TODO: Case-insensitivity. + multimap headers = extract_headers(lines, client->remote_addr); + multimap::const_iterator referer_it = headers.find("Referer"); + if (referer_it != headers.end()) { + client->referer = referer_it->second; + } + multimap::const_iterator user_agent_it = headers.find("User-Agent"); + if (user_agent_it != headers.end()) { + client->user_agent = user_agent_it->second; + } + vector request_tokens = split_tokens(lines[0]); if (request_tokens.size() < 2) { return 400; // Bad request (empty). @@ -521,17 +654,8 @@ int Server::parse_request(Client *client) } client->url = request_tokens[1]; + client->stream = stream; - if (client->stream->mark_pool != NULL) { - client->fwmark = client->stream->mark_pool->get_mark(); - } else { - client->fwmark = 0; // No mark. - } - if (setsockopt(client->sock, SOL_SOCKET, SO_MARK, &client->fwmark, sizeof(client->fwmark)) == -1) { - if (client->fwmark != 0) { - log_perror("setsockopt(SO_MARK)"); - } - } if (setsockopt(client->sock, SOL_SOCKET, SO_MAX_PACING_RATE, &client->stream->pacing_rate, sizeof(client->stream->pacing_rate)) == -1) { if (client->stream->pacing_rate != ~0U) { log_perror("setsockopt(SO_MAX_PACING_RATE)"); @@ -546,11 +670,11 @@ void Server::construct_header(Client *client) { Stream *stream = client->stream; if (stream->encoding == Stream::STREAM_ENCODING_RAW) { - client->header_or_error = stream->http_header + + client->header_or_short_response = stream->http_header + "\r\n" + stream->stream_header; } else if (stream->encoding == Stream::STREAM_ENCODING_METACUBE) { - client->header_or_error = stream->http_header + + client->header_or_short_response = stream->http_header + "Content-encoding: metacube\r\n" + "\r\n"; if (!stream->stream_header.empty()) { @@ -559,10 +683,10 @@ void Server::construct_header(Client *client) hdr.size = htonl(stream->stream_header.size()); hdr.flags = htons(METACUBE_FLAGS_HEADER); hdr.csum = htons(metacube2_compute_crc(&hdr)); - client->header_or_error.append( + client->header_or_short_response.append( string(reinterpret_cast(&hdr), sizeof(hdr))); } - client->header_or_error.append(stream->stream_header); + client->header_or_short_response.append(stream->stream_header); } else { assert(false); } @@ -585,10 +709,10 @@ void Server::construct_error(Client *client, int error_code) char error[256]; snprintf(error, 256, "HTTP/1.0 %d Error\r\nContent-type: text/plain\r\n\r\nSomething went wrong. Sorry.\r\n", error_code); - client->header_or_error = error; + client->header_or_short_response = error; // Switch states. - client->state = Client::SENDING_ERROR; + client->state = Client::SENDING_SHORT_RESPONSE; epoll_event ev; ev.events = EPOLLOUT | EPOLLET | EPOLLRDHUP; @@ -618,10 +742,6 @@ void Server::close_client(Client *client) if (client->stream != NULL) { delete_from(&client->stream->sleeping_clients, client); delete_from(&client->stream->to_process, client); - if (client->stream->mark_pool != NULL) { - int fwmark = client->fwmark; - client->stream->mark_pool->release_mark(fwmark); - } } // Log to access_log.