No longer serve Expires header; it is not needed.

[remoteglot] / www / serve-analysis.js

diff --git a/www/serve-analysis.js b/www/serve-analysis.js
index 868363d0926bc9d49fa8a4539fa0b4798d6765e6..edb4a18a9db7b6a933d77771f244806995525304 100644 (file)
--- a/www/serve-analysis.js
+++ b/www/serve-analysis.js
@@ -8,14 +8,22 @@ var url = require('url');
 var querystring = require('querystring');
 var path = require('path');
 var zlib = require('zlib');
 var querystring = require('querystring');
 var path = require('path');
 var zlib = require('zlib');

+var delta = require('./js/json_delta.js');

 
 // Constants.
 
 // Constants.

-var json_filename = '/srv/analysis.sesse.net/www/analysis.json';
+var JSON_FILENAME = '/srv/analysis.sesse.net/www/analysis.json';
+var HISTORY_TO_KEEP = 5;
+
+// If set to 1, we are already processing a JSON update and should not
+// start a new one. If set to 2, we are _also_ having one in the queue.
+var json_lock = 0;

 
 // The current contents of the file to hand out, and its last modified time.
 
 // The current contents of the file to hand out, and its last modified time.

-var json_contents = undefined;
-var json_contents_gz = undefined;
-var json_last_modified = undefined;
+var json = undefined;
+
+// The last five timestamps, and diffs from them to the latest version.
+var historic_json = [];
+var diff_json = {};

 
 // The list of clients that are waiting for new data to show up.
 // Uniquely keyed by request_id so that we can take them out of
 
 // The list of clients that are waiting for new data to show up.
 // Uniquely keyed by request_id so that we can take them out of


@@ -36,12 +44,81 @@ var touch_timer = undefined;
 // ourselves, so some external log-tailing daemon needs to tell us.
 var viewer_count_override = undefined;
 
 // ourselves, so some external log-tailing daemon needs to tell us.
 var viewer_count_override = undefined;
 

+var replace_json = function(new_json_contents, mtime) {
+       // Generate the list of diffs from the last five versions.
+       if (json !== undefined) {
+               // If two versions have the same mtime, clients could have either.
+               // Note the fact, so that we never insert it.
+               if (json.last_modified == mtime) {
+                       json.invalid_base = true;
+               }
+               if (!json.invalid_base) {
+                       historic_json.push(json);
+                       if (historic_json.length > HISTORY_TO_KEEP) {
+                               historic_json.shift();
+                       }
+               }
+       }
+
+       var new_json = {
+               parsed: JSON.parse(new_json_contents),
+               plain: new_json_contents,
+               last_modified: mtime
+       };
+       create_json_historic_diff(new_json, historic_json.slice(0), {}, function(new_diff_json) {
+               // gzip the new version (non-delta), and put it into place.
+               zlib.gzip(new_json_contents, function(err, buffer) {
+                       if (err) throw err;
+
+                       new_json.gzip = buffer;
+                       json = new_json;
+                       diff_json = new_diff_json;
+                       json_lock = 0;
+
+                       // Finally, wake up any sleeping clients.
+                       possibly_wakeup_clients();
+               });
+       });
+}
+
+var create_json_historic_diff = function(new_json, history_left, new_diff_json, cb) {
+       if (history_left.length == 0) {
+               cb(new_diff_json);
+               return;
+       }
+
+       var histobj = history_left.shift();
+       var diff = delta.JSON_delta.diff(histobj.parsed, new_json.parsed);
+       var diff_text = JSON.stringify(diff);
+       zlib.gzip(diff_text, function(err, buffer) {
+               if (err) throw err;
+               new_diff_json[histobj.last_modified] = {
+                       plain: diff,
+                       text: diff_text,
+                       gzip: buffer,
+                       last_modified: new_json.last_modified,
+               };
+               create_json_historic_diff(new_json, history_left, new_diff_json, cb);
+       });
+}
+

 var reread_file = function(event, filename) {
 var reread_file = function(event, filename) {

-       if (filename != path.basename(json_filename)) {
+       if (filename != path.basename(JSON_FILENAME)) {

                return;
        }
                return;
        }

-       console.log("Rereading " + json_filename);
-       fs.open(json_filename, 'r+', function(err, fd) {
+       if (json_lock >= 2) {
+               return;
+       }
+       if (json_lock == 1) {
+               // Already processing; wait a bit.
+               json_lock = 2;
+               setTimeout(function() { json_lock = 1; reread_file(event, filename); }, 100);
+               return;
+       }
+       json_lock = 1;
+
+       console.log("Rereading " + JSON_FILENAME);
+       fs.open(JSON_FILENAME, 'r+', function(err, fd) {

                if (err) throw err;
                fs.fstat(fd, function(err, st) {
                        if (err) throw err;
                if (err) throw err;
                fs.fstat(fd, function(err, st) {
                        if (err) throw err;


@@ -50,13 +127,7 @@ var reread_file = function(event, filename) {
                                if (err) throw err;
                                fs.close(fd, function() {
                                        var new_json_contents = buffer.toString('utf8', 0, bytesRead);
                                if (err) throw err;
                                fs.close(fd, function() {
                                        var new_json_contents = buffer.toString('utf8', 0, bytesRead);

-                                       zlib.gzip(new_json_contents, function(err, buffer) {
-                                               if (err) throw err;
-                                               json_contents = new_json_contents;
-                                               json_contents_gz = buffer;
-                                               json_last_modified = st.mtime.getTime();
-                                               possibly_wakeup_clients();
-                                       });
+                                       replace_json(new_json_contents, st.mtime.getTime());

                                });
                        });
                });
                                });
                        });
                });


@@ -68,7 +139,7 @@ var reread_file = function(event, filename) {
        touch_timer = setTimeout(function() {
                console.log("Touching analysis.json due to no other activity");
                var now = Date.now() / 1000;
        touch_timer = setTimeout(function() {
                console.log("Touching analysis.json due to no other activity");
                var now = Date.now() / 1000;

-               fs.utimes(json_filename, now, now);
+               fs.utimes(JSON_FILENAME, now, now);

        }, 30000);
 }
 var possibly_wakeup_clients = function() {
        }, 30000);
 }
 var possibly_wakeup_clients = function() {


@@ -76,6 +147,7 @@ var possibly_wakeup_clients = function() {
        for (var i in sleeping_clients) {
                mark_recently_seen(sleeping_clients[i].unique);
                send_json(sleeping_clients[i].response,
        for (var i in sleeping_clients) {
                mark_recently_seen(sleeping_clients[i].unique);
                send_json(sleeping_clients[i].response,

+                         sleeping_clients[i].ims,

                          sleeping_clients[i].accept_gzip,
                          num_viewers);
        }
                          sleeping_clients[i].accept_gzip,
                          num_viewers);
        }


@@ -103,24 +175,24 @@ var handle_viewer_override = function(request, u, response) {
                response.end();
        }
 }
                response.end();
        }
 }

-var send_json = function(response, accept_gzip, num_viewers) {
+var send_json = function(response, ims, accept_gzip, num_viewers) {
+       var this_json = diff_json[ims] || json;
+

        var headers = {
                'Content-Type': 'text/json',
        var headers = {
                'Content-Type': 'text/json',

-               'X-Remoteglot-Last-Modified': json_last_modified,
+               'X-Remoteglot-Last-Modified': this_json.last_modified,

                'X-Remoteglot-Num-Viewers': num_viewers,
                'X-Remoteglot-Num-Viewers': num_viewers,

-               'Access-Control-Allow-Origin': 'http://analysis.sesse.net',

                'Access-Control-Expose-Headers': 'X-Remoteglot-Last-Modified, X-Remoteglot-Num-Viewers',
                'Access-Control-Expose-Headers': 'X-Remoteglot-Last-Modified, X-Remoteglot-Num-Viewers',

-               'Expires': 'Mon, 01 Jan 1970 00:00:00 UTC',

                'Vary': 'Accept-Encoding',
        };
 
        if (accept_gzip) {
                headers['Content-Encoding'] = 'gzip';
                response.writeHead(200, headers);
                'Vary': 'Accept-Encoding',
        };
 
        if (accept_gzip) {
                headers['Content-Encoding'] = 'gzip';
                response.writeHead(200, headers);

-               response.write(json_contents_gz);
+               response.write(this_json.gzip);

        } else {
                response.writeHead(200, headers);
        } else {
                response.writeHead(200, headers);

-               response.write(json_contents);
+               response.write(this_json.text);

        }
        response.end();
 }
        }
        response.end();
 }


@@ -160,8 +232,8 @@ var count_viewers = function() {
 
 // Set up a watcher to catch changes to the file, then do an initial read
 // to make sure we have a copy.
 
 // Set up a watcher to catch changes to the file, then do an initial read
 // to make sure we have a copy.

-fs.watch(path.dirname(json_filename), reread_file);
-reread_file(null, path.basename(json_filename));
+fs.watch(path.dirname(JSON_FILENAME), reread_file);
+reread_file(null, path.basename(JSON_FILENAME));

 
 var server = http.createServer();
 server.on('request', function(request, response) {
 
 var server = http.createServer();
 server.on('request', function(request, response) {


@@ -192,8 +264,8 @@ server.on('request', function(request, response) {
 
        // If we already have something newer than what the user has,
        // just send it out and be done with it.
 
        // If we already have something newer than what the user has,
        // just send it out and be done with it.

-       if (json_last_modified !== undefined && (!ims || json_last_modified > ims)) {
-               send_json(response, accept_gzip, count_viewers());
+       if (json !== undefined && (!ims || json.last_modified > ims)) {
+               send_json(response, ims, accept_gzip, count_viewers());

                return;
        }
 
                return;
        }
 


@@ -204,6 +276,7 @@ server.on('request', function(request, response) {
        client.request_id = request_id;
        client.accept_gzip = accept_gzip;
        client.unique = unique;
        client.request_id = request_id;
        client.accept_gzip = accept_gzip;
        client.unique = unique;

+       client.ims = ims;

        sleeping_clients[request_id++] = client;
 
        request.socket.client = client;
        sleeping_clients[request_id++] = client;
 
        request.socket.client = client;