Make do with only one commit per sync.

[skvidarsync] / bin / sync.pl

diff --git a/bin/sync.pl b/bin/sync.pl
index bfac355fe6dd87b2dcc8d0a937cda65a2221d29f..8ca281d6c195591afc22e62b7c0043a3b7b675d0 100644 (file)
--- a/bin/sync.pl
+++ b/bin/sync.pl
@@ -10,12 +10,18 @@ use POSIX;
 use Time::HiRes;
 use IO::Select;
 use Unicode::Collate;
+use IO::Socket::SSL;
 binmode STDOUT, ':utf8';
 binmode STDERR, ':utf8';
 use utf8;
 
 require '../include/config.pm';
 
+my $global_ctx = IO::Socket::SSL::SSL_Context->new(
+       SSL_session_cache_size => 100,  # Probably overkill.
+);
+IO::Socket::SSL::set_default_context($global_ctx);
+
 my @log = ();
 my $uca = Unicode::Collate->new(level => 1);
 
@@ -113,19 +119,22 @@ sub get_spreadsheet_name {
 }
 
 sub matches_name {
-       my ($slack_name, $spreadsheet_name) = @_;
-       if (sort_key($slack_name) eq sort_key($spreadsheet_name)) {
-               return 1;
-       }
+       my ($slack_name, $spreadsheet_name, $ap) = @_;
 
-       my @ap = split /\s+/, $slack_name;
-       my @bp = split /\s+/, $spreadsheet_name;
-       if (scalar @ap >= 2 && scalar @bp >= 2 && sort_key($ap[0]) eq sort_key($bp[0])) {
+       # No need to check for an exact match; we already did that through $seen_names.
+       # if (sort_key($slack_name) eq sort_key($spreadsheet_name)) {
+       #       return 1;
+       # }
+
+       # @ap is precalculated by the caller.
+       # my @ap = map { sort_key($_) } split /\s+/, $slack_name;
+       my @bp = map { sort_key($_) } split /\s+/, $spreadsheet_name;
+       if (scalar @$ap >= 2 && scalar @bp >= 2 && $ap->[0] eq $bp[0]) {
                # First name matches, try to match some surname
                my $found = 0;
-               for my $ai (1..$#ap) {
+               for my $ai (1..(scalar @$ap)) {
                        for my $bi (1..$#bp) {
-                               $found = 1 if (sort_key($ap[$ai]) eq sort_key($bp[$bi]));
+                               $found = 1 if ($ap->[$ai] eq $bp[$bi]);
                        }
                }
                if ($found) {
@@ -226,7 +235,6 @@ sub get_group_assignments {
 sub update_assignment_db {
        my ($dbh, $channel, $ts, $assignments) = @_;
 
-       local $dbh->{AutoCommit} = 0;
        my %db_assignments = ();
        my $q = $dbh->prepare('SELECT name,group_name FROM current_group_membership_history WHERE channel=? AND ts=?');
        $q->execute($channel, $ts);
@@ -247,7 +255,6 @@ sub update_assignment_db {
                        $q->execute($channel, $ts, $name, undef);
                }
        }
-       $dbh->commit;
 }
 
 sub get_spreadsheet_with_title {
@@ -474,19 +481,21 @@ sub db_connect {
        if (!defined($dbh)) {
                return undef;
        }
+       $dbh->{AutoCommit} = 0;
        $dbh->do('LISTEN skvupdate') or return undef;
        return $dbh;
 }
 
 sub run {
-       my $dbh = shift;
+       my ($dbh, $ua) = @_;
        my $total_start = [Time::HiRes::gettimeofday];
 
        @log = ();
        skv_log("Siste sync startet: " . POSIX::ctime(time));
 
-       # Initialize the handles we need for communication.
-       my $ua = LWP::UserAgent->new('SKVidarLang/1.0');
+       # For the logic on the “applied” table below.
+       $dbh->do('SET TRANSACTION ISOLATION LEVEL SERIALIZABLE');
+
        my $token = get_oauth_bearer_token($dbh, $ua);
 
        # Find the newest message, what it is linked to, and what was the one before it (for group diffing).
@@ -557,7 +566,9 @@ sub run {
        my %assignments = get_group_assignments($main_sheet_json);
        update_assignment_db($dbh, $config::invitation_channel, $invitation_ts, \%assignments);
 
+       $start = [Time::HiRes::gettimeofday];
        my %seen_names = find_where_each_name_is($main_sheet_json);
+       log_timing($start, "Making sort key reverse mapping");
 
        # Find duplicates.
        for my $name (sort keys %seen_names) {
@@ -630,14 +641,17 @@ sub run {
                        # Do a search through all the available names in the sheet to find an obvious(ish) match.
                        my @candidates = ();
                        my $main_sheet_rows = $main_sheet_json->{'data'}[0]{'rowData'};
+                       $start = [Time::HiRes::gettimeofday];
+                       my @ap = map { sort_key($_) } split /\s+/, $slack_name;  # Precalc for matches_name().
                        for my $row (@$main_sheet_rows) {
                                for my $val (@{$row->{'values'}}) {
                                        my $name = get_spreadsheet_name($val);
-                                       if (defined($name) && matches_name($slack_name, $name)) {
+                                       if (defined($name) && matches_name($slack_name, $name, \@ap)) {
                                                push @candidates, $name;
                                        }
                                }
                        }
+                       log_timing($start, "Fuzzy-searching for Slack name “$slack_name”");
                        if ($#candidates == -1) {
                                skv_log("$slack_name ($userid) er påmeldt på Slack, men fant ikke et regneark-navn for dem.");
                                possibly_nag_user($dbh, $ua, $userid, $invitation_ts, undef, \%slack_userid_to_slack_name);
@@ -708,8 +722,6 @@ sub run {
 
        # Find the list of names we already marked yellow.
        my %have_colors = ();
-       $dbh->{AutoCommit} = 0;
-       $dbh->do('SET TRANSACTION ISOLATION LEVEL SERIALIZABLE');
        $q = $dbh->prepare('SELECT name,color FROM applied WHERE channel=? AND ts=?');
        $q->execute($config::invitation_channel, $invitation_ts);
        while (my $ref = $q->fetchrow_hashref) {
@@ -793,10 +805,12 @@ sub run {
        printf "Tok %.0f ms.\n", 1e3 * $elapsed;
 }
 
+# Initialize the handles we need for communication.
 my $dbh = db_connect() or die;
+my $ua = LWP::UserAgent->new(agent => 'SKVidarLang/1.0', keep_alive => 50);
 if ($#ARGV >= 0 && $ARGV[0] eq '--daemon') {
        # Start with a single, forced run.
-       run($dbh);
+       run($dbh, $ua);
 
        while (1) {
                while (!defined($dbh)) {
@@ -815,9 +829,7 @@ if ($#ARGV >= 0 && $ARGV[0] eq '--daemon') {
                }
                if (scalar @ready > 0) {  
                        eval {
-                               $dbh->{AutoCommit} = 1;
-                               run($dbh);
-                               $dbh->commit;
+                               run($dbh, $ua);
                        };
                        if ($@) {
                                warn "Died with: $@";
@@ -825,6 +837,10 @@ if ($#ARGV >= 0 && $ARGV[0] eq '--daemon') {
                        }
                }
        }
+} elsif ($#ARGV >= 0 && $ARGV[0] eq '--benchmark') {
+       for my $i (0..9) {
+               run($dbh, $ua);
+       }
 } else {
-       run($dbh);
+       run($dbh, $ua);
 }