From f95dfda2d14615a9816ba1abb32bd240a470e228 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sun, 14 Dec 2014 22:44:47 +0100 Subject: [PATCH] Build with the new transposition data from pgn-extract. A lot of UI is still missing, though. --- Makefile | 8 ++--- binloader.cpp | 2 +- binlookup.cpp | 31 ++++++++++++---- binmerger.cpp | 49 +++++-------------------- merge_count.cpp | 46 ++++++++++++++++++++++++ merge_count.h | 8 +++++ opening-stats.pl | 10 +++--- www/opening-stats.pl | 86 ++++++++++++++++++++++++++++---------------- 8 files changed, 152 insertions(+), 88 deletions(-) create mode 100644 merge_count.cpp create mode 100644 merge_count.h diff --git a/Makefile b/Makefile index 9d133ec..0e95cd4 100644 --- a/Makefile +++ b/Makefile @@ -5,12 +5,12 @@ PROTOC=protoc all: binloader binlookup binmerger binloader: binloader.o hash.o arena.o count.pb.o -binmerger: binmerger.o count.pb.o -binlookup: binlookup.o hash.o count.pb.o +binmerger: binmerger.o merge_count.o count.pb.o +binlookup: binlookup.o hash.o merge_count.o count.pb.o binloader.o: binloader.cpp arena.cpp count.pb.h -binmerger.o: binmerger.cpp count.pb.h -binlookup.o: binlookup.cpp count.pb.h +binmerger.o: binmerger.cpp merge_count.h count.pb.h +binlookup.o: binlookup.cpp merge_count.h count.pb.h %.pb.cc %.pb.h : %.proto $(PROTOC) --cpp_out=. $< diff --git a/binloader.cpp b/binloader.cpp index ab3e8ae..a030456 100644 --- a/binloader.cpp +++ b/binloader.cpp @@ -38,7 +38,7 @@ static int memcmp_different_len(const void *s1, size_t n1, const void *s2, size_ enum Result { WHITE = 0, DRAW, BLACK }; struct Element { - char *bpfen; + char *bpfen; // includes prev_board_hash int bpfen_len; char move[8]; // Na1xc3+ Result result; diff --git a/binlookup.cpp b/binlookup.cpp index 08455eb..894da9d 100644 --- a/binlookup.cpp +++ b/binlookup.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include "count.pb.h" +#include "merge_count.h" #include "hash.h" using namespace std; @@ -53,18 +55,33 @@ int main(int argc, char **argv) srcs[bucket] = mtbl_reader_source(mtbls[bucket]); } - mtbl_iter *it = mtbl_source_get(srcs[bucket], bpfen, bpfen_len); + // Give back the hash of the position to be kind to the user + uint16_t board_hash = util::Hash32((const char *)bpfen, bpfen_len); + printf("%d\n", board_hash); + + mtbl_iter *it = mtbl_source_get_prefix(srcs[bucket], bpfen, bpfen_len); const uint8_t *key, *val; size_t len_key, len_val; - if (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) { - Count c; - c.ParseFromArray(val, len_val); - printf("%d %d %d %u %f %f %d %ld %d %ld", + Count c; + bool has_c = false; + while (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) { + if (has_c) { + Count tmpc; + tmpc.ParseFromArray(val, len_val); + c = merge_count(c, tmpc); + } else { + c.ParseFromArray(val, len_val); + has_c = true; + } + } + + if (has_c) { + printf("%d %d %d %u %ld %ld %d %ld %d %ld", c.white(), c.draw(), c.black(), c.opening_num(), - double(c.sum_white_elo()) / c.num_elo(), - double(c.sum_black_elo()) / c.num_elo(), + c.sum_white_elo(), + c.sum_black_elo(), c.num_elo(), c.first_timestamp(), c.pgn_file_num(), c.pgn_start_position()); diff --git a/binmerger.cpp b/binmerger.cpp index 94e4780..7e25a8f 100644 --- a/binmerger.cpp +++ b/binmerger.cpp @@ -5,55 +5,22 @@ #include #include #include +#include "merge_count.h" #include "count.pb.h" using namespace std; -void merge_count(void* userdata, - const uint8_t *key, size_t len_key, - const uint8_t *val0, size_t len_val0, - const uint8_t *val1, size_t len_val1, - uint8_t **merged_val, size_t *len_merged_val) +void merge_serialized_count(void* userdata, + const uint8_t *key, size_t len_key, + const uint8_t *val0, size_t len_val0, + const uint8_t *val1, size_t len_val1, + uint8_t **merged_val, size_t *len_merged_val) { Count c0, c1; c0.ParseFromArray(val0, len_val0); c1.ParseFromArray(val1, len_val1); - Count c; - - c.set_white(c0.white() + c1.white()); - c.set_draw(c0.draw() + c1.draw()); - c.set_black(c0.black() + c1.black()); - c.set_sum_white_elo(c0.sum_white_elo() + c1.sum_white_elo()); - c.set_sum_black_elo(c0.sum_black_elo() + c1.sum_black_elo()); - c.set_num_elo(c0.num_elo() + c1.num_elo()); - if (c0.first_timestamp() <= c1.first_timestamp()) { - c.set_opening_num(c0.opening_num()); - if (c0.has_first_timestamp()) { - c.set_first_timestamp(c0.first_timestamp()); - } - c.set_pgn_file_num(c0.pgn_file_num()); - c.set_pgn_start_position(c0.pgn_start_position()); - } else { - c.set_opening_num(c1.opening_num()); - if (c1.has_first_timestamp()) { - c.set_first_timestamp(c1.first_timestamp()); - } - c.set_pgn_file_num(c1.pgn_file_num()); - c.set_pgn_start_position(c1.pgn_start_position()); - } - - // Merge the moves, with deduplication. - unordered_set moves; - for (int i = 0; i < c0.move_size(); ++i) { - moves.insert(c0.move(i)); - c.add_move(c0.move(i)); - } - for (int i = 0; i < c1.move_size(); ++i) { - if (!moves.count(c1.move(i))) { - c.add_move(c1.move(i)); - } - } + Count c = merge_count(c0, c1); static string buf; // Keep allocated. c.SerializeToString(&buf); @@ -66,7 +33,7 @@ void merge_count(void* userdata, int main(int argc, char **argv) { mtbl_merger_options* mopt = mtbl_merger_options_init(); - mtbl_merger_options_set_merge_func(mopt, merge_count, NULL); + mtbl_merger_options_set_merge_func(mopt, merge_serialized_count, NULL); mtbl_merger* merger = mtbl_merger_init(mopt); for (int i = 1; i < argc - 1; ++i) { diff --git a/merge_count.cpp b/merge_count.cpp new file mode 100644 index 0000000..dbf028d --- /dev/null +++ b/merge_count.cpp @@ -0,0 +1,46 @@ +#include "merge_count.h" + +#include + +using namespace std; + +Count merge_count(const Count& c0, const Count& c1) +{ + Count c; + + c.set_white(c0.white() + c1.white()); + c.set_draw(c0.draw() + c1.draw()); + c.set_black(c0.black() + c1.black()); + c.set_sum_white_elo(c0.sum_white_elo() + c1.sum_white_elo()); + c.set_sum_black_elo(c0.sum_black_elo() + c1.sum_black_elo()); + c.set_num_elo(c0.num_elo() + c1.num_elo()); + if (c0.first_timestamp() <= c1.first_timestamp()) { + c.set_opening_num(c0.opening_num()); + if (c0.has_first_timestamp()) { + c.set_first_timestamp(c0.first_timestamp()); + } + c.set_pgn_file_num(c0.pgn_file_num()); + c.set_pgn_start_position(c0.pgn_start_position()); + } else { + c.set_opening_num(c1.opening_num()); + if (c1.has_first_timestamp()) { + c.set_first_timestamp(c1.first_timestamp()); + } + c.set_pgn_file_num(c1.pgn_file_num()); + c.set_pgn_start_position(c1.pgn_start_position()); + } + + // Merge the moves, with deduplication. + unordered_set moves; + for (int i = 0; i < c0.move_size(); ++i) { + moves.insert(c0.move(i)); + c.add_move(c0.move(i)); + } + for (int i = 0; i < c1.move_size(); ++i) { + if (!moves.count(c1.move(i))) { + c.add_move(c1.move(i)); + } + } + + return c; +} diff --git a/merge_count.h b/merge_count.h new file mode 100644 index 0000000..247a720 --- /dev/null +++ b/merge_count.h @@ -0,0 +1,8 @@ +#ifndef _MERGE_COUNT_H +#define _MERGE_COUNT_H 1 + +#include "count.pb.h" + +Count merge_count(const Count& c0, const Count& c1); + +#endif // !defined(_MERGE_COUNT_H) diff --git a/opening-stats.pl b/opening-stats.pl index 27cdee0..24fdc99 100755 --- a/opening-stats.pl +++ b/opening-stats.pl @@ -16,11 +16,12 @@ my $pid = IPC::Open2::open2($chld_out, $chld_in, "./binlookup", "./open.mtbl", " my $pos = Position->from_fen($fen); my $hex = unpack('H*', $pos->bitpacked_fen); print $chld_in $hex, "\n"; -my $line = <$chld_out>; -print $line; -chomp $line; +chomp (my $line = <$chld_out>); # Root position hash. +print $line, "\n"; +chomp (my $line = <$chld_out>); # Actual stats. +print $line, "\n"; my ($white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo, $num_elo, $timestamp, $pgn_file_number, $pgn_start_position, @moves) = split / /, $line; # Explore one move out. @@ -28,6 +29,7 @@ for my $move (@moves) { my ($np, $uci_move) = $pos->make_pretty_move($move); my $hex = unpack('H*', $np->bitpacked_fen); print $chld_in $hex, "\n"; - my $line = <$chld_out>; + my $line = <$chld_out>; # Ignore position hash. + $line = <$chld_out>; print "$move $line"; } diff --git a/www/opening-stats.pl b/www/opening-stats.pl index 3affdd5..28dd443 100755 --- a/www/opening-stats.pl +++ b/www/opening-stats.pl @@ -17,28 +17,21 @@ my $pid = IPC::Open2::open2($chld_out, $chld_in, "../binlookup", "../open.mtbl", # Root position. Basically ignore everything except the opening (and later some root game stuff). my $fen = $cgi->param('fen') // 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1'; +my $includetransp = $cgi->param('includetransp') // 1; + my $pos = Position->from_fen($fen); -my $hex = unpack('H*', $pos->bitpacked_fen); -print $chld_in $hex, "\n"; -chomp (my $line = <$chld_out>); - -my ($white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo, $num_elo, $timestamp, $pgn_file_number, $pgn_start_position, @moves) = split / /, $line; -my @json_moves = (); -push @json_moves, { - white => $white * 1, - draw => $draw * 1, - black => $black * 1, - white_avg_elo => $num_elo == 0 ? undef : $white_avg_elo * 1, - black_avg_elo => $num_elo == 0 ? undef : $black_avg_elo * 1, - num_elo => $num_elo * 1 -}; +my ($json_root_pos, $root_aux_data) = get_json_move($pos, undef, $chld_in, $chld_out); # TODO: include previous hash if $includetransp == 0 +my $opening = $openings{$json_root_pos->{'opening_num'}} // 'A00: Start position'; -my $opening = $openings{$opening_num} // 'A00: Start position'; +my @json_moves = ($json_root_pos); my $root_game; eval { - die "Missing PGN position data." if (!defined($pgn_file_number) || !defined($pgn_start_position)); - my $pgntext = read_root_pgn($pgn_file_number, $pgn_start_position); + if (!exists($root_aux_data->{'pgn_file_number'}) || + !exists($root_aux_data->{'pgn_start_position'})) { + die "Missing PGN position data." + } + my $pgntext = read_root_pgn($root_aux_data->{'pgn_file_number'}, $root_aux_data->{'pgn_start_position'}); my $pgn = Chess::PGN::Parse->new(undef, $pgntext); $pgn->read_game() or die; $pgn->parse_game() or die; @@ -57,21 +50,16 @@ eval { }; # Explore one move out. -for my $move (@moves) { +for my $move (@{$root_aux_data->{'moves'}}) { my ($np, $uci_move) = $pos->make_pretty_move($move); - my $hex = unpack('H*', $np->bitpacked_fen); - print $chld_in $hex, "\n"; - my $line = <$chld_out>; - my ($white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo, $num_elo) = split / /, $line; - push @json_moves, { - move => $move, - white => $white * 1, - draw => $draw * 1, - black => $black * 1, - white_avg_elo => $num_elo == 0 ? undef : $white_avg_elo * 1, - black_avg_elo => $num_elo == 0 ? undef : $black_avg_elo * 1, - num_elo => $num_elo * 1 - }; + my $json_pos; + if ($includetransp) { + ($json_pos, undef) = get_json_move($np, undef, $chld_in, $chld_out); + } else { + ($json_pos, undef) = get_json_move($np, $root_aux_data->{'pos_hash'}, $chld_in, $chld_out); + } + $json_pos->{'move'} = $move; + push @json_moves, $json_pos; } print $cgi->header(-type=>'application/json'); @@ -93,6 +81,7 @@ sub read_openings { } sub read_root_pgn { + my ($pgn_file_number, $pgn_start_position) = @_; my @pgnnames; open my $pgnnamesfh, "<", "../pgnnames.txt" or die "../pgnnames.txt: $!"; @@ -119,3 +108,38 @@ sub read_root_pgn { return $root_pgn; } + +sub get_json_move { + my ($pos, $filter_prev_pos_hash, $chld_in, $chld_out) = @_; + my $hex = unpack('H*', $pos->bitpacked_fen); + if (defined($filter_prev_pos_hash)) { + $hex .= unpack('H*', pack('S', $filter_prev_pos_hash)); + } + print $chld_in $hex, "\n"; + + # Read the hash of this position. + chomp (my $pos_hash = <$chld_out>); + + chomp (my $line = <$chld_out>); + if ($line eq '-') { + return ({}, {}); + } + + my ($white, $draw, $black, $opening_num, $white_sum_elo, $black_sum_elo, $num_elo, $timestamp, $pgn_file_number, $pgn_start_position, @moves) = split / /, $line; + my $json_pos = { + white => $white, + draw => $draw, + black => $black, + white_avg_elo => $num_elo == 0 ? undef : $white_sum_elo / $num_elo, + black_avg_elo => $num_elo == 0 ? undef : $black_sum_elo / $num_elo, + num_elo => $num_elo, + opening_num => $opening_num, + }; + my $aux_data = { # Only relevant for the root. + pos_hash => $pos_hash * 1, + moves => \@moves, + pgn_file_number => $pgn_file_number, + pgn_start_position => $pgn_start_position, + }; + return ($json_pos, $aux_data); +} -- 2.39.2