all: binloader binlookup binmerger
binloader: binloader.o hash.o arena.o count.pb.o
-binmerger: binmerger.o count.pb.o
-binlookup: binlookup.o hash.o count.pb.o
+binmerger: binmerger.o merge_count.o count.pb.o
+binlookup: binlookup.o hash.o merge_count.o count.pb.o
binloader.o: binloader.cpp arena.cpp count.pb.h
-binmerger.o: binmerger.cpp count.pb.h
-binlookup.o: binlookup.cpp count.pb.h
+binmerger.o: binmerger.cpp merge_count.h count.pb.h
+binlookup.o: binlookup.cpp merge_count.h count.pb.h
%.pb.cc %.pb.h : %.proto
$(PROTOC) --cpp_out=. $<
enum Result { WHITE = 0, DRAW, BLACK };
struct Element {
- char *bpfen;
+ char *bpfen; // includes prev_board_hash
int bpfen_len;
char move[8]; // Na1xc3+
Result result;
#include <memory>
#include <string>
#include <string.h>
+#include <farmhash.h>
#include "count.pb.h"
+#include "merge_count.h"
#include "hash.h"
using namespace std;
srcs[bucket] = mtbl_reader_source(mtbls[bucket]);
}
- mtbl_iter *it = mtbl_source_get(srcs[bucket], bpfen, bpfen_len);
+ // Give back the hash of the position to be kind to the user
+ uint16_t board_hash = util::Hash32((const char *)bpfen, bpfen_len);
+ printf("%d\n", board_hash);
+
+ mtbl_iter *it = mtbl_source_get_prefix(srcs[bucket], bpfen, bpfen_len);
const uint8_t *key, *val;
size_t len_key, len_val;
- if (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) {
- Count c;
- c.ParseFromArray(val, len_val);
- printf("%d %d %d %u %f %f %d %ld %d %ld",
+ Count c;
+ bool has_c = false;
+ while (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) {
+ if (has_c) {
+ Count tmpc;
+ tmpc.ParseFromArray(val, len_val);
+ c = merge_count(c, tmpc);
+ } else {
+ c.ParseFromArray(val, len_val);
+ has_c = true;
+ }
+ }
+
+ if (has_c) {
+ printf("%d %d %d %u %ld %ld %d %ld %d %ld",
c.white(), c.draw(), c.black(), c.opening_num(),
- double(c.sum_white_elo()) / c.num_elo(),
- double(c.sum_black_elo()) / c.num_elo(),
+ c.sum_white_elo(),
+ c.sum_black_elo(),
c.num_elo(), c.first_timestamp(),
c.pgn_file_num(),
c.pgn_start_position());
#include <unordered_set>
#include <string.h>
#include <assert.h>
+#include "merge_count.h"
#include "count.pb.h"
using namespace std;
-void merge_count(void* userdata,
- const uint8_t *key, size_t len_key,
- const uint8_t *val0, size_t len_val0,
- const uint8_t *val1, size_t len_val1,
- uint8_t **merged_val, size_t *len_merged_val)
+void merge_serialized_count(void* userdata,
+ const uint8_t *key, size_t len_key,
+ const uint8_t *val0, size_t len_val0,
+ const uint8_t *val1, size_t len_val1,
+ uint8_t **merged_val, size_t *len_merged_val)
{
Count c0, c1;
c0.ParseFromArray(val0, len_val0);
c1.ParseFromArray(val1, len_val1);
- Count c;
-
- c.set_white(c0.white() + c1.white());
- c.set_draw(c0.draw() + c1.draw());
- c.set_black(c0.black() + c1.black());
- c.set_sum_white_elo(c0.sum_white_elo() + c1.sum_white_elo());
- c.set_sum_black_elo(c0.sum_black_elo() + c1.sum_black_elo());
- c.set_num_elo(c0.num_elo() + c1.num_elo());
- if (c0.first_timestamp() <= c1.first_timestamp()) {
- c.set_opening_num(c0.opening_num());
- if (c0.has_first_timestamp()) {
- c.set_first_timestamp(c0.first_timestamp());
- }
- c.set_pgn_file_num(c0.pgn_file_num());
- c.set_pgn_start_position(c0.pgn_start_position());
- } else {
- c.set_opening_num(c1.opening_num());
- if (c1.has_first_timestamp()) {
- c.set_first_timestamp(c1.first_timestamp());
- }
- c.set_pgn_file_num(c1.pgn_file_num());
- c.set_pgn_start_position(c1.pgn_start_position());
- }
-
- // Merge the moves, with deduplication.
- unordered_set<string> moves;
- for (int i = 0; i < c0.move_size(); ++i) {
- moves.insert(c0.move(i));
- c.add_move(c0.move(i));
- }
- for (int i = 0; i < c1.move_size(); ++i) {
- if (!moves.count(c1.move(i))) {
- c.add_move(c1.move(i));
- }
- }
+ Count c = merge_count(c0, c1);
static string buf; // Keep allocated.
c.SerializeToString(&buf);
int main(int argc, char **argv)
{
mtbl_merger_options* mopt = mtbl_merger_options_init();
- mtbl_merger_options_set_merge_func(mopt, merge_count, NULL);
+ mtbl_merger_options_set_merge_func(mopt, merge_serialized_count, NULL);
mtbl_merger* merger = mtbl_merger_init(mopt);
for (int i = 1; i < argc - 1; ++i) {
--- /dev/null
+#include "merge_count.h"
+
+#include <unordered_set>
+
+using namespace std;
+
+Count merge_count(const Count& c0, const Count& c1)
+{
+ Count c;
+
+ c.set_white(c0.white() + c1.white());
+ c.set_draw(c0.draw() + c1.draw());
+ c.set_black(c0.black() + c1.black());
+ c.set_sum_white_elo(c0.sum_white_elo() + c1.sum_white_elo());
+ c.set_sum_black_elo(c0.sum_black_elo() + c1.sum_black_elo());
+ c.set_num_elo(c0.num_elo() + c1.num_elo());
+ if (c0.first_timestamp() <= c1.first_timestamp()) {
+ c.set_opening_num(c0.opening_num());
+ if (c0.has_first_timestamp()) {
+ c.set_first_timestamp(c0.first_timestamp());
+ }
+ c.set_pgn_file_num(c0.pgn_file_num());
+ c.set_pgn_start_position(c0.pgn_start_position());
+ } else {
+ c.set_opening_num(c1.opening_num());
+ if (c1.has_first_timestamp()) {
+ c.set_first_timestamp(c1.first_timestamp());
+ }
+ c.set_pgn_file_num(c1.pgn_file_num());
+ c.set_pgn_start_position(c1.pgn_start_position());
+ }
+
+ // Merge the moves, with deduplication.
+ unordered_set<string> moves;
+ for (int i = 0; i < c0.move_size(); ++i) {
+ moves.insert(c0.move(i));
+ c.add_move(c0.move(i));
+ }
+ for (int i = 0; i < c1.move_size(); ++i) {
+ if (!moves.count(c1.move(i))) {
+ c.add_move(c1.move(i));
+ }
+ }
+
+ return c;
+}
--- /dev/null
+#ifndef _MERGE_COUNT_H
+#define _MERGE_COUNT_H 1
+
+#include "count.pb.h"
+
+Count merge_count(const Count& c0, const Count& c1);
+
+#endif // !defined(_MERGE_COUNT_H)
my $pos = Position->from_fen($fen);
my $hex = unpack('H*', $pos->bitpacked_fen);
print $chld_in $hex, "\n";
-my $line = <$chld_out>;
-print $line;
-chomp $line;
+chomp (my $line = <$chld_out>); # Root position hash.
+print $line, "\n";
+chomp (my $line = <$chld_out>); # Actual stats.
+print $line, "\n";
my ($white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo, $num_elo, $timestamp, $pgn_file_number, $pgn_start_position, @moves) = split / /, $line;
# Explore one move out.
my ($np, $uci_move) = $pos->make_pretty_move($move);
my $hex = unpack('H*', $np->bitpacked_fen);
print $chld_in $hex, "\n";
- my $line = <$chld_out>;
+ my $line = <$chld_out>; # Ignore position hash.
+ $line = <$chld_out>;
print "$move $line";
}
# Root position. Basically ignore everything except the opening (and later some root game stuff).
my $fen = $cgi->param('fen') // 'rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1';
+my $includetransp = $cgi->param('includetransp') // 1;
+
my $pos = Position->from_fen($fen);
-my $hex = unpack('H*', $pos->bitpacked_fen);
-print $chld_in $hex, "\n";
-chomp (my $line = <$chld_out>);
-
-my ($white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo, $num_elo, $timestamp, $pgn_file_number, $pgn_start_position, @moves) = split / /, $line;
-my @json_moves = ();
-push @json_moves, {
- white => $white * 1,
- draw => $draw * 1,
- black => $black * 1,
- white_avg_elo => $num_elo == 0 ? undef : $white_avg_elo * 1,
- black_avg_elo => $num_elo == 0 ? undef : $black_avg_elo * 1,
- num_elo => $num_elo * 1
-};
+my ($json_root_pos, $root_aux_data) = get_json_move($pos, undef, $chld_in, $chld_out); # TODO: include previous hash if $includetransp == 0
+my $opening = $openings{$json_root_pos->{'opening_num'}} // 'A00: Start position';
-my $opening = $openings{$opening_num} // 'A00: Start position';
+my @json_moves = ($json_root_pos);
my $root_game;
eval {
- die "Missing PGN position data." if (!defined($pgn_file_number) || !defined($pgn_start_position));
- my $pgntext = read_root_pgn($pgn_file_number, $pgn_start_position);
+ if (!exists($root_aux_data->{'pgn_file_number'}) ||
+ !exists($root_aux_data->{'pgn_start_position'})) {
+ die "Missing PGN position data."
+ }
+ my $pgntext = read_root_pgn($root_aux_data->{'pgn_file_number'}, $root_aux_data->{'pgn_start_position'});
my $pgn = Chess::PGN::Parse->new(undef, $pgntext);
$pgn->read_game() or die;
$pgn->parse_game() or die;
};
# Explore one move out.
-for my $move (@moves) {
+for my $move (@{$root_aux_data->{'moves'}}) {
my ($np, $uci_move) = $pos->make_pretty_move($move);
- my $hex = unpack('H*', $np->bitpacked_fen);
- print $chld_in $hex, "\n";
- my $line = <$chld_out>;
- my ($white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo, $num_elo) = split / /, $line;
- push @json_moves, {
- move => $move,
- white => $white * 1,
- draw => $draw * 1,
- black => $black * 1,
- white_avg_elo => $num_elo == 0 ? undef : $white_avg_elo * 1,
- black_avg_elo => $num_elo == 0 ? undef : $black_avg_elo * 1,
- num_elo => $num_elo * 1
- };
+ my $json_pos;
+ if ($includetransp) {
+ ($json_pos, undef) = get_json_move($np, undef, $chld_in, $chld_out);
+ } else {
+ ($json_pos, undef) = get_json_move($np, $root_aux_data->{'pos_hash'}, $chld_in, $chld_out);
+ }
+ $json_pos->{'move'} = $move;
+ push @json_moves, $json_pos;
}
print $cgi->header(-type=>'application/json');
}
sub read_root_pgn {
+ my ($pgn_file_number, $pgn_start_position) = @_;
my @pgnnames;
open my $pgnnamesfh, "<", "../pgnnames.txt"
or die "../pgnnames.txt: $!";
return $root_pgn;
}
+
+sub get_json_move {
+ my ($pos, $filter_prev_pos_hash, $chld_in, $chld_out) = @_;
+ my $hex = unpack('H*', $pos->bitpacked_fen);
+ if (defined($filter_prev_pos_hash)) {
+ $hex .= unpack('H*', pack('S', $filter_prev_pos_hash));
+ }
+ print $chld_in $hex, "\n";
+
+ # Read the hash of this position.
+ chomp (my $pos_hash = <$chld_out>);
+
+ chomp (my $line = <$chld_out>);
+ if ($line eq '-') {
+ return ({}, {});
+ }
+
+ my ($white, $draw, $black, $opening_num, $white_sum_elo, $black_sum_elo, $num_elo, $timestamp, $pgn_file_number, $pgn_start_position, @moves) = split / /, $line;
+ my $json_pos = {
+ white => $white,
+ draw => $draw,
+ black => $black,
+ white_avg_elo => $num_elo == 0 ? undef : $white_sum_elo / $num_elo,
+ black_avg_elo => $num_elo == 0 ? undef : $black_sum_elo / $num_elo,
+ num_elo => $num_elo,
+ opening_num => $opening_num,
+ };
+ my $aux_data = { # Only relevant for the root.
+ pos_hash => $pos_hash * 1,
+ moves => \@moves,
+ pgn_file_number => $pgn_file_number,
+ pgn_start_position => $pgn_start_position,
+ };
+ return ($json_pos, $aux_data);
+}
projects / remoteglot-book / commitdiff