Make the HTTP inputs time out after 30 seconds of no activity.

[cubemap] / httpinput.cpp

diff --git a/httpinput.cpp b/httpinput.cpp
index 1bd14dee87c7d022de1bea1d128059f43428aa8c..7f96970b678fb75c68a31746a39087a1a41e269d 100644 (file)
--- a/httpinput.cpp
+++ b/httpinput.cpp
@@ -28,7 +28,25 @@
 using namespace std;
 
 extern ServerPool *servers;
-         
+
+namespace {
+
+// Compute b-a.
+timespec clock_diff(const timespec &a, const timespec &b)
+{
+       timespec ret;
+       ret.tv_sec = b.tv_sec - a.tv_sec;
+       ret.tv_nsec = b.tv_nsec - a.tv_nsec;
+       if (ret.tv_nsec < 0) {
+               ret.tv_sec--;
+               ret.tv_nsec += 1000000000;
+       }
+       assert(ret.tv_nsec >= 0);
+       return ret;
+}
+
+}  // namespace
+
 HTTPInput::HTTPInput(const string &url)
        : state(NOT_CONNECTED),
          url(url),
@@ -39,6 +57,7 @@ HTTPInput::HTTPInput(const string &url)
        stats.url = url;
        stats.bytes_received = 0;
        stats.data_bytes_received = 0;
+       stats.connect_time = -1;
 }
 
 HTTPInput::HTTPInput(const InputProto &serialized)
@@ -48,6 +67,7 @@ HTTPInput::HTTPInput(const InputProto &serialized)
          request_bytes_sent(serialized.request_bytes_sent()),
          response(serialized.response()),
          http_header(serialized.http_header()),
+         stream_header(serialized.stream_header()),
          has_metacube_header(serialized.has_metacube_header()),
          sock(serialized.sock())
 {
@@ -68,6 +88,11 @@ HTTPInput::HTTPInput(const InputProto &serialized)
        stats.url = url;
        stats.bytes_received = serialized.bytes_received();
        stats.data_bytes_received = serialized.data_bytes_received();
+       if (serialized.has_connect_time()) {
+               stats.connect_time = serialized.connect_time();
+       } else {
+               stats.connect_time = time(NULL);
+       }
 }
 
 void HTTPInput::close_socket()
@@ -75,6 +100,9 @@ void HTTPInput::close_socket()
        if (sock != -1) {
                safe_close(sock);
        }
+
+       MutexLock lock(&stats_mutex);
+       stats.connect_time = -1;
 }
 
 InputProto HTTPInput::serialize() const
@@ -86,11 +114,13 @@ InputProto HTTPInput::serialize() const
        serialized.set_request_bytes_sent(request_bytes_sent);
        serialized.set_response(response);
        serialized.set_http_header(http_header);
+       serialized.set_stream_header(stream_header);
        serialized.set_pending_data(string(pending_data.begin(), pending_data.end()));
        serialized.set_has_metacube_header(has_metacube_header);
        serialized.set_sock(sock);
        serialized.set_bytes_received(stats.bytes_received);
        serialized.set_data_bytes_received(stats.data_bytes_received);
+       serialized.set_connect_time(stats.connect_time);
        return serialized;
 }
 
@@ -253,7 +283,7 @@ bool HTTPInput::parse_response(const std::string &request)
        }
 
        for (size_t i = 0; i < stream_indices.size(); ++i) {
-               servers->set_header(stream_indices[i], http_header, "");
+               servers->set_header(stream_indices[i], http_header, stream_header);
        }
 
        return true;
@@ -261,11 +291,43 @@ bool HTTPInput::parse_response(const std::string &request)
 
 void HTTPInput::do_work()
 {
+       timespec last_activity;
+
+       // TODO: Make the timeout persist across restarts.
+       if (state == SENDING_REQUEST || state == RECEIVING_HEADER || state == RECEIVING_DATA) {
+               int err = clock_gettime(CLOCK_MONOTONIC, &last_activity);
+               assert(err != -1);
+       }
+
        while (!should_stop()) {
                if (state == SENDING_REQUEST || state == RECEIVING_HEADER || state == RECEIVING_DATA) {
-                       bool activity = wait_for_activity(sock, (state == SENDING_REQUEST) ? POLLOUT : POLLIN, NULL);
-                       if (!activity) {
-                               // Most likely, should_stop was set.
+                       // Give the socket 30 seconds since last activity before we time out.
+                       static const int timeout_secs = 30;
+
+                       timespec now;
+                       int err = clock_gettime(CLOCK_MONOTONIC, &now);
+                       assert(err != -1);
+
+                       timespec elapsed = clock_diff(last_activity, now);
+                       if (elapsed.tv_sec >= timeout_secs) {
+                               // Timeout!
+                               log(ERROR, "[%s] Timeout after %d seconds, closing.", url.c_str(), elapsed.tv_sec);
+                               state = CLOSING_SOCKET;
+                               continue;
+                       }
+
+                       // Basically calculate (30 - (now - last_activity)) = (30 + (last_activity - now)).
+                       // Add a second of slack to account for differences between clocks.
+                       timespec timeout = clock_diff(now, last_activity);
+                       timeout.tv_sec += timeout_secs + 1;
+                       assert(timeout.tv_sec > 0 || (timeout.tv_sec >= 0 && timeout.tv_nsec > 0));
+
+                       bool activity = wait_for_activity(sock, (state == SENDING_REQUEST) ? POLLOUT : POLLIN, &timeout);
+                       if (activity) {
+                               err = clock_gettime(CLOCK_MONOTONIC, &last_activity);
+                               assert(err != -1);
+                       } else {
+                               // OK. Most likely, should_stop was set, or we have timed out.
                                continue;
                        }
                }
@@ -301,6 +363,10 @@ void HTTPInput::do_work()
                                        request = "GET " + path + " HTTP/1.0\r\nUser-Agent: cubemap\r\n\r\n";
                                        request_bytes_sent = 0;
                                }
+
+                               MutexLock lock(&stats_mutex);
+                               stats.connect_time = time(NULL);
+                               clock_gettime(CLOCK_MONOTONIC, &last_activity);
                        }
                        break;
                case SENDING_REQUEST: {
@@ -491,9 +557,9 @@ void HTTPInput::process_data(char *ptr, size_t bytes)
                }
                char *inner_data = pending_data.data() + sizeof(metacube_block_header);
                if (flags & METACUBE_FLAGS_HEADER) {
-                       string header(inner_data, inner_data + size);
+                       stream_header = string(inner_data, inner_data + size);
                        for (size_t i = 0; i < stream_indices.size(); ++i) {
-                               servers->set_header(stream_indices[i], http_header, header);
+                               servers->set_header(stream_indices[i], http_header, stream_header);
                        }
                } else {
                        StreamStartSuitability suitable_for_stream_start;
@@ -526,6 +592,12 @@ void HTTPInput::drop_pending_data(size_t num_bytes)
        pending_data.erase(pending_data.begin(), pending_data.begin() + num_bytes);
 }
 
+void HTTPInput::add_destination(int stream_index)
+{
+       stream_indices.push_back(stream_index);
+       servers->set_header(stream_index, http_header, stream_header);
+}
+
 InputStats HTTPInput::get_stats() const
 {
        MutexLock lock(&stats_mutex);