openings.txt
www/history/
closure/
+book/part-*.bin
+book/ficsgame*.pgn
+book/binloader
+book/binlookup
+book/eco.pgn
+book/open.mtbl
--- /dev/null
+#! /usr/bin/perl
+#
+# Get eco.pgn from ftp://ftp.cs.kent.ac.uk/pub/djb/pgn-extract/eco.pgn,
+# or any other opening database you might want to use as a base.
+#
+use strict;
+use warnings;
+use Chess::PGN::Parse;
+
+require 'Position.pm';
+
+package ECO;
+
+our %fen_to_opening = ();
+our @openings = ();
+
+sub init {
+ {
+ my $pos = Position->start_pos("white", "black");
+ my $key = _key_for_pos($pos);
+ push @openings, { eco => 'A00', name => 'Start position' };
+ $fen_to_opening{$key} = $#openings;
+ }
+
+ my $pgn = Chess::PGN::Parse->new("eco.pgn")
+ or die "can't open eco.pgn\n";
+ while ($pgn->read_game()) {
+ my $tags = $pgn->tags();
+ $pgn->quick_parse_game;
+ my $pos = Position->start_pos("white", "black");
+ my $moves = $pgn->moves // [];
+ my $eco = $pgn->eco;
+ next if (!defined($eco));
+ my $name = $tags->{'Opening'};
+ if (exists($tags->{'Variation'}) && $tags->{'Variation'} ne '') {
+ $name .= ": " . $tags->{'Variation'};
+ }
+ for (my $i = 0; $i < scalar @$moves; ++$i) {
+ my ($from_row, $from_col, $to_row, $to_col, $promo) = $pos->parse_pretty_move($moves->[$i]);
+ $pos = $pos->make_move($from_row, $from_col, $to_row, $to_col, $promo, $moves->[$i]);
+ }
+ my $key = _key_for_pos($pos);
+ push @openings, { eco => $pgn->eco(), name => $name };
+ $fen_to_opening{$key} = $#openings;
+ }
+}
+
+sub persist {
+ my $filename = shift;
+ open my $fh, ">", $filename
+ or die "openings.txt: $!";
+ for my $opening (@openings) {
+ print $fh $opening->{'eco'}, " ", $opening->{'name'}, "\n";
+ }
+ close $fh;
+}
+
+sub unpersist {
+ my $filename = shift;
+ open my $fh, "<", $filename
+ or die "openings.txt: $!";
+ while (<$fh>) {
+ chomp;
+ push @openings, $_;
+ }
+ close $fh;
+}
+
+sub get_opening_num { # May return undef.
+ my $pos = shift;
+ return $fen_to_opening{_key_for_pos($pos)};
+}
+
+sub _key_for_pos {
+ my $pos = shift;
+ my $key = $pos->fen;
+ # Remove the move clocks.
+ $key =~ s/ \d+ \d+$//;
+ return $key;
+}
+
+1;
--- /dev/null
+//#define _GLIBCXX_PARALLEL
+#include <stdio.h>
+#include <vector>
+#include <mtbl.h>
+#include <algorithm>
+#include <utility>
+#include <memory>
+#include <string>
+#include <string.h>
+#include "count.h"
+
+using namespace std;
+
+enum Result { WHITE = 0, DRAW, BLACK };
+struct Element {
+ string bpfen_and_move;
+ Result result;
+ int opening_num, white_elo, black_elo;
+
+ bool operator< (const Element& other) const {
+ return bpfen_and_move < other.bpfen_and_move;
+ }
+};
+
+int main(int argc, char **argv)
+{
+ vector<Element> elems;
+
+ for (int i = 1; i < argc; ++i) {
+ FILE *fp = fopen(argv[i], "rb");
+ if (fp == NULL) {
+ perror(argv[i]);
+ exit(1);
+ }
+ for ( ;; ) {
+ int l = getc(fp);
+ if (l == -1) {
+ break;
+ }
+
+ string bpfen_and_move;
+ bpfen_and_move.resize(l);
+ if (fread(&bpfen_and_move[0], l, 1, fp) != 1) {
+ perror("fread()");
+ // exit(1);
+ break;
+ }
+
+ int r = getc(fp);
+ if (r == -1) {
+ perror("getc()");
+ //exit(1);
+ break;
+ }
+
+ int opening_num, white_elo, black_elo;
+ if (fread(&white_elo, sizeof(white_elo), 1, fp) != 1) {
+ perror("fread()");
+ //exit(1);
+ break;
+ }
+ if (fread(&black_elo, sizeof(black_elo), 1, fp) != 1) {
+ perror("fread()");
+ //exit(1);
+ break;
+ }
+ if (fread(&opening_num, sizeof(opening_num), 1, fp) != 1) {
+ perror("fread()");
+ //exit(1);
+ break;
+ }
+ elems.emplace_back(Element {move(bpfen_and_move), Result(r), opening_num, white_elo, black_elo});
+ }
+ fclose(fp);
+
+ printf("Read %ld elems\n", elems.size());
+ }
+
+ printf("Sorting...\n");
+ sort(elems.begin(), elems.end());
+
+ printf("Writing SSTable...\n");
+ mtbl_writer* mtbl = mtbl_writer_init("open.mtbl", NULL);
+ Count c;
+ int num_elo = 0;
+ double sum_white_elo = 0.0, sum_black_elo = 0.0;
+ for (int i = 0; i < elems.size(); ++i) {
+ if (elems[i].result == WHITE) {
+ ++c.white;
+ } else if (elems[i].result == DRAW) {
+ ++c.draw;
+ } else if (elems[i].result == BLACK) {
+ ++c.black;
+ }
+ c.opening_num = elems[i].opening_num;
+ if (elems[i].white_elo >= 100 && elems[i].black_elo >= 100) {
+ sum_white_elo += elems[i].white_elo;
+ sum_black_elo += elems[i].black_elo;
+ ++num_elo;
+ }
+ if (i == elems.size() - 1 || elems[i].bpfen_and_move != elems[i + 1].bpfen_and_move) {
+ c.avg_white_elo = sum_white_elo / num_elo;
+ c.avg_black_elo = sum_black_elo / num_elo;
+ mtbl_writer_add(mtbl,
+ (const uint8_t *)elems[i].bpfen_and_move.data(), elems[i].bpfen_and_move.size(),
+ (const uint8_t *)&c, sizeof(c));
+ c = Count();
+ num_elo = 0;
+ sum_white_elo = sum_black_elo = 0.0;
+ }
+ }
+ mtbl_writer_destroy(&mtbl);
+}
--- /dev/null
+#include <stdio.h>
+#include <vector>
+#include <mtbl.h>
+#include <algorithm>
+#include <utility>
+#include <memory>
+#include <string>
+#include <string.h>
+#include "count.h"
+
+using namespace std;
+
+int main(int argc, char **argv)
+{
+ const char *hex_prefix = argv[2];
+ const int prefix_len = strlen(hex_prefix) / 2;
+ uint8_t *prefix = new uint8_t[prefix_len];
+
+ for (int i = 0; i < prefix_len; ++i) {
+ char x[3];
+ x[0] = hex_prefix[i * 2 + 0];
+ x[1] = hex_prefix[i * 2 + 1];
+ x[2] = 0;
+ int k;
+ sscanf(x, "%02x", &k);
+ prefix[i] = k;
+ }
+
+ mtbl_reader* mtbl = mtbl_reader_init(argv[1], NULL);
+ const mtbl_source *src = mtbl_reader_source(mtbl);
+ mtbl_iter *it = mtbl_source_get_prefix(src, prefix, prefix_len);
+
+ const uint8_t *key, *val;
+ size_t len_key, len_val;
+
+ while (mtbl_iter_next(it, &key, &len_key, &val, &len_val)) {
+ string move((char *)(key + prefix_len), len_key - prefix_len);
+ const Count* c = (Count *)val;
+ printf("%s %d %d %d %d %f %f\n", move.c_str(), c->white, c->draw, c->black, c->opening_num, c->avg_white_elo, c->avg_black_elo);
+ }
+}
--- /dev/null
+
+struct Count {
+ int white = 0;
+ int draw = 0;
+ int black = 0;
+ int opening_num = -1;
+ float avg_white_elo = 0.0;
+ float avg_black_elo = 0.0;
+};
--- /dev/null
+#! /usr/bin/perl
+use strict;
+use warnings;
+require 'ECO.pm';
+
+ECO::init();
+ECO::persist();
+
--- /dev/null
+#! /usr/bin/perl
+use strict;
+use warnings;
+use CGI;
+use JSON::XS;
+use lib '.';
+use Position;
+require 'ECO.pm';
+
+#ECO::unpersist();
+
+my $cgi = CGI->new;
+my $fen = $ARGV[0];
+my $pos = Position->from_fen($fen);
+my $hex = unpack('H*', $pos->bitpacked_fen);
+system("./binlookup", "./open.mtbl", $hex);
+
--- /dev/null
+#! /bin/sh
+FILE=$1
+for X in $( seq 0 39 ); do
+ ( ./parse-pgn.pl $FILE $X 40 >> part-$X.bin ) &
+done
+wait
--- /dev/null
+#! /usr/bin/perl
+use Chess::PGN::Parse;
+use Data::Dumper;
+use strict;
+use warnings;
+use DBI;
+use DBD::Pg;
+require 'Position.pm';
+require 'Engine.pm';
+require 'ECO.pm';
+
+my $DRYRUN = 1;
+my $TEXTOUT = 0;
+my $BINOUT = 1;
+
+ECO::init();
+
+my $dbh = DBI->connect("dbi:Pg:dbname=ficsopening", "sesse", undef);
+$dbh->do("COPY opening FROM STDIN") unless $DRYRUN;
+
+my ($filename, $my_num, $tot_num) = @ARGV;
+
+my $pgn = Chess::PGN::Parse->new($filename)
+ or die "can't open $filename\n";
+my $game_num = 0;
+while ($pgn->read_game()) {
+ next unless ($game_num++ % $tot_num == $my_num);
+ my $tags = $pgn->tags();
+# next unless $tags->{'WhiteElo'} >= 2000;
+# next unless $tags->{'BlackElo'} >= 2000;
+ $pgn->quick_parse_game;
+ my $pos = Position->start_pos($pgn->white, $pgn->black);
+ my $result = $pgn->result;
+ my $binresult;
+ if ($result eq '1-0') {
+ $binresult = chr(0);
+ } elsif ($result eq '1/2-1/2') {
+ $binresult = chr(1);
+ } elsif ($result eq '0-1') {
+ $binresult = chr(2);
+ } else {
+ die "Unknown result $result";
+ }
+ my $binwhiteelo = pack('l', $tags->{'WhiteElo'});
+ my $binblackelo = pack('l', $tags->{'BlackElo'});
+ my $moves = $pgn->moves;
+ my $opening = ECO::get_opening_num($pos);
+# print STDERR $pgn->white, " ", $pgn->black, "\n";
+ for (my $i = 0; $i + 1 < scalar @$moves; ++$i) {
+ my ($from_row, $from_col, $to_row, $to_col, $promo) = $pos->parse_pretty_move($moves->[$i]);
+ my $next_move = $moves->[$i];
+ my $bpfen = $pos->bitpacked_fen;
+ my $bpfen_q = $dbh->quote($bpfen, { pg_type => DBD::Pg::PG_BYTEA });
+ my $fen = $pos->fen;
+ $opening = ECO::get_opening_num($pos) // $opening;
+ print "$fen $next_move $result $opening\n" if $TEXTOUT;
+ if ($BINOUT) {
+ print chr(length($bpfen) + length($next_move)) . $bpfen . $next_move;
+ print $binresult . $binwhiteelo . $binblackelo;
+ print pack('l', $opening);
+ }
+ $dbh->pg_putcopydata("$bpfen_q\t$next_move\t$result\n") unless $DRYRUN;
+ $pos = $pos->make_move($from_row, $from_col, $to_row, $to_col, $promo, $moves->[$i]);
+ }
+}
+$dbh->pg_putcopyend unless $DRYRUN;
--- /dev/null
+<!doctype html>
+<html>
+<head>
+ <meta charset="utf-8" />
+ <meta http-equiv="X-UA-Compatible" content="IE=edge" />
+ <title>analysis.sesse.net</title>
+
+ <link rel="stylesheet" href="css/chessboard-0.3.0.min.css" />
+ <link rel="stylesheet" href="css/remoteglot.css" />
+</head>
+<body>
+<h1 id="headline">Openings</h1>
+<div id="boardcontainer">
+ <div id="board"></div>
+ <div id="bottompanel">
+ <!-- CSS abuse... -->
+ <p id="whiteclock"><a href="javascript:prev_move()"><<<</a></p>
+ <p id="blackclock"><a href="javascript:next_move()">>>></a></p>
+ <p id="numviewers"></p>
+ </div>
+</div>
+<div id="analysis">
+ <table>
+ <thead>
+ <tr>
+ <th>Move</th>
+ <th>Games</th>
+ <th>%</th>
+ <th>Win%</th>
+ <th>WElo</th>
+ <th>BElo</th>
+ <th>AWin%</th>
+ <!--<th class="winbars">
+ <table><tr>
+ <td class="white" style="width: 35%;">White</td>
+ <td class="draw" style="width: 30%;">Draw</td>
+ <td class="black" style="width: 35%;">Black</td>
+ </tr></table>
+ </th> -->
+ </tr>
+ </thead>
+ <tbody id="lines">
+ </tbody>
+ </table>
+</div>
+<!-- For faster development -->
+<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
+<script type="text/javascript" src="js/chessboard-0.3.0.min.js"></script>
+<script type="text/javascript" src="js/chess.min.js"></script>
+<script type="text/javascript" src="js/book.js"></script>
+</body>
+</html>
#linenav {
display: none;
}
+
+/* Opening display */
+.num {
+ padding-left: 0.5em;
+ padding-right: 0.5em;
+ text-align: right;
+}
+.winbars {
+ width: 20em;
+ font-size: small;
+ font-weight: bold;
+ text-align: center;
+}
+.winbars table {
+ border: 1px solid black;
+ width: 100%;
+ border-collapse: collapse;
+}
+.winbars table td {
+ border: 1px solid black;
+ overflow: hidden;
+ max-width: 0px;
+}
+.winbars table td.white {
+ background-color: white;
+ color: black;
+}
+.winbars table td.draw {
+ background-color: gray;
+ color: white;
+}
+.winbars table td.black {
+ background-color: black;
+ color: white;
+}
--- /dev/null
+(function() {
+
+var board = null;
+var moves = [];
+var move_override = 0;
+
+var get_game = function() {
+ var game = new Chess();
+ for (var i = 0; i < move_override; ++i) {
+ game.move(moves[i]);
+ }
+ return game;
+}
+
+var update = function() {
+ var game = get_game();
+ board.position(game.fen());
+ fetch_analysis();
+}
+
+var fetch_analysis = function() {
+ var game = get_game();
+ $.ajax({
+ url: "/opening-stats.pl?fen=" + encodeURIComponent(game.fen())
+ }).done(function(data, textstatus, xhr) {
+ show_lines(data, game);
+ });
+}
+
+var add_td = function(tr, value) {
+ var td = document.createElement("td");
+ tr.appendChild(td);
+ $(td).addClass("num");
+ $(td).text(value);
+}
+
+var show_lines = function(data, game) {
+ var moves = data['moves'];
+ $('#numviewers').text(data['opening']);
+ var total_num = 0;
+ for (var i = 0; i < moves.length; ++i) {
+ var move = moves[i];
+ total_num += parseInt(move['white']);
+ total_num += parseInt(move['draw']);
+ total_num += parseInt(move['black']);
+ }
+
+ var tbl = $("#lines");
+ tbl.empty();
+
+ for (var i = 0; i < moves.length; ++i) {
+ var move = moves[i];
+ var tr = document.createElement("tr");
+
+ var white = parseInt(move['white']);
+ var draw = parseInt(move['draw']);
+ var black = parseInt(move['black']);
+
+ // Move.
+ var move_td = document.createElement("td");
+ tr.appendChild(move_td);
+ $(move_td).addClass("move");
+
+ var move_a = document.createElement("a");
+ move_a.href = "javascript:make_move('" + move['move'] + "')";
+ move_td.appendChild(move_a);
+ $(move_a).text(move['move']);
+
+ // #.
+ var num = white + draw + black;
+ add_td(tr, num);
+
+ // %.
+ add_td(tr, (100.0 * num / total_num).toFixed(1) + "%");
+
+ // Win%.
+ var white_win_ratio = (white + 0.5 * draw) / num;
+ var win_ratio = (game.turn() == 'w') ? white_win_ratio : 1.0 - white_win_ratio;
+ add_td(tr, ((100.0 * win_ratio).toFixed(1) + "%"));
+
+ // Elo.
+ add_td(tr, move['white_avg_elo'].toFixed(1));
+ add_td(tr, move['black_avg_elo'].toFixed(1));
+
+ // Win% corrected for Elo.
+ var win_elo = -400.0 * Math.log(1.0 / white_win_ratio - 1.0) / Math.LN10;
+ win_elo -= (move['white_avg_elo'] - move['black_avg_elo']);
+ white_win_ratio = 1.0 / (1.0 + Math.pow(10, win_elo / -400.0));
+ win_ratio = (game.turn() == 'w') ? white_win_ratio : 1.0 - white_win_ratio;
+ add_td(tr, ((100.0 * win_ratio).toFixed(1) + "%"));
+
+ if (false) {
+ // Win bars (W/D/B).
+ var winbar_td = document.createElement("td");
+ $(winbar_td).addClass("winbars");
+ tr.appendChild(winbar_td);
+ var winbar_table = document.createElement("table");
+ winbar_td.appendChild(winbar_table);
+ var winbar_tr = document.createElement("tr");
+ winbar_table.appendChild(winbar_tr);
+
+ if (white > 0) {
+ var white_percent = (100.0 * white / num).toFixed(0) + "%";
+ var white_td = document.createElement("td");
+ winbar_tr.appendChild(white_td);
+ $(white_td).addClass("white");
+ white_td.style.width = white_percent;
+ $(white_td).text(white_percent);
+ }
+ if (draw > 0) {
+ var draw_percent = (100.0 * draw / num).toFixed(0) + "%";
+ var draw_td = document.createElement("td");
+ winbar_tr.appendChild(draw_td);
+ $(draw_td).addClass("draw");
+ draw_td.style.width = draw_percent;
+ $(draw_td).text(draw_percent);
+ }
+ if (black > 0) {
+ var black_percent = (100.0 * black / num).toFixed(0) + "%";
+ var black_td = document.createElement("td");
+ winbar_tr.appendChild(black_td);
+ $(black_td).addClass("black");
+ black_td.style.width = black_percent;
+ $(black_td).text(black_percent);
+ }
+ }
+
+ tbl.append(tr);
+ }
+}
+
+var make_move = function(move) {
+ moves.length = move_override;
+ moves.push(move);
+ move_override = moves.length;
+ update();
+}
+window['make_move'] = make_move;
+
+var prev_move = function() {
+ if (move_override > 0) {
+ --move_override;
+ update();
+ }
+}
+window['prev_move'] = prev_move;
+
+var next_move = function() {
+ if (move_override < moves.length) {
+ ++move_override;
+ update();
+ }
+}
+window['next_move'] = next_move;
+
+// almost all of this stuff comes from the chessboard.js example page
+var onDragStart = function(source, piece, position, orientation) {
+ var game = get_game();
+ if (game.game_over() === true ||
+ (game.turn() === 'w' && piece.search(/^b/) !== -1) ||
+ (game.turn() === 'b' && piece.search(/^w/) !== -1)) {
+ return false;
+ }
+}
+
+var onDrop = function(source, target) {
+ // see if the move is legal
+ var game = get_game();
+ var move = game.move({
+ from: source,
+ to: target,
+ promotion: 'q' // NOTE: always promote to a queen for example simplicity
+ });
+
+ // illegal move
+ if (move === null) return 'snapback';
+
+ moves = game.history({ verbose: true });
+ move_override = moves.length;
+};
+
+// update the board position after the piece snap
+// for castling, en passant, pawn promotion
+var onSnapEnd = function() {
+ var game = get_game();
+ board.position(game.fen());
+ fetch_analysis();
+};
+
+var init = function() {
+ // Create board.
+ board = new window.ChessBoard('board', {
+ draggable: true,
+ position: 'start',
+ onDragStart: onDragStart,
+ onDrop: onDrop,
+ onSnapEnd: onSnapEnd
+ });
+ update();
+
+ $(window).keyup(function(event) {
+ if (event.which == 39) {
+ next_move();
+ } else if (event.which == 37) {
+ prev_move();
+ }
+ });
+}
+
+
+$(document).ready(init);
+
+})();
--- /dev/null
+#! /usr/bin/perl
+use strict;
+use warnings;
+use CGI;
+use JSON::XS;
+use lib '..';
+use Position;
+use ECO;
+
+ECO::unpersist("../book/openings.txt");
+
+my $cgi = CGI->new;
+my $fen = $cgi->param('fen');
+my $pos = Position->from_fen($fen);
+my $hex = unpack('H*', $pos->bitpacked_fen);
+open my $fh, "-|", "../book/binlookup", "../book/open.mtbl", $hex
+ or die "../book/binlookup: $!";
+
+my $opening;
+
+my @moves = ();
+while (<$fh>) {
+ chomp;
+ my ($move, $white, $draw, $black, $opening_num, $white_avg_elo, $black_avg_elo) = split;
+ push @moves, {
+ move => $move,
+ white => $white * 1,
+ draw => $draw * 1,
+ black => $black * 1,
+ white_avg_elo => $white_avg_elo * 1,
+ black_avg_elo => $black_avg_elo * 1
+ };
+ $opening = $ECO::openings[$opening_num];
+}
+close $fh;
+
+@moves = sort { num($b) <=> num($a) } @moves;
+
+print $cgi->header(-type=>'application/json');
+print JSON::XS::encode_json({ moves => \@moves, opening => $opening });
+
+sub num {
+ my $x = shift;
+ return $x->{'white'} + $x->{'draw'} + $x->{'black'};
+}
projects / remoteglot / commitdiff