#include <unordered_set>
#include <string.h>
#include "count.pb.h"
-#include "arena.h"
#include "hash.h"
#define DUMMY_TIMESTAMP 32503680000
using namespace std;
-static int memcmp_different_len(const void *s1, size_t n1, const void *s2, size_t n2)
+static inline int memcmp_different_len(const void *s1, size_t n1, const void *s2, size_t n2)
{
- int shared_len = min(n1, n2);
+ size_t shared_len = min(n1, n2);
+ if (shared_len >= 8) {
+ uint64_t a1 = *(const uint64_t *)s1;
+ uint64_t a2 = *(const uint64_t *)s2;
+ if (a1 != a2) {
+ a1 = __builtin_bswap64(a1);
+ a2 = __builtin_bswap64(a2);
+ return (a1 < a2) ? -1 : 1;
+ }
+ }
+
int s = memcmp(s1, s2, shared_len);
if (s != 0) {
return s;
}
- if (n1 < n2) {
- return -1;
- } else if (n1 > n2) {
- return 1;
- } else {
- return 0;
- }
+ return n2 - n1;
}
enum Result { WHITE = 0, DRAW, BLACK };
struct Element {
- char *bpfen; // includes prev_board_hash
+ char bpfen[32]; // includes prev_board_hash
int bpfen_len;
char move[8]; // Na1xc3+
Result result;
struct ShardData {
vector<Element> elems;
- unique_ptr<Arena> arena; // Used to allocate bpfen.
int num_written_subshards = 0;
};
c.set_pgn_file_num(e.file_num);
c.set_pgn_start_position(e.start_position);
}
- if (!moves.count(e.move)) {
+ if (strlen(e.move) > 0 && !moves.count(e.move)) {
moves.insert(e.move);
c.add_move(e.move);
}
e.bpfen_len != shard->elems[i + 1].bpfen_len ||
memcmp(e.bpfen, shard->elems[i + 1].bpfen, e.bpfen_len) != 0) {
c.SerializeToString(&buf);
- mtbl_writer_add(mtbl,
+ mtbl_res res = mtbl_writer_add(mtbl,
(const uint8_t *)e.bpfen, e.bpfen_len,
(const uint8_t *)buf.data(), buf.size());
+ assert(res == mtbl_res_success);
c = Count();
moves.clear();
}
mtbl_writer_destroy(&mtbl);
shard->elems.clear();
- shard->arena.reset(new Arena);
}
int main(int argc, char **argv)
for (int i = 0; i < num_buckets; ++i) {
shards[i].elems.reserve(num_pos_per_subshard);
- shards[i].arena.reset(new Arena);
}
size_t num_elems = 0;
if (bpfen_len == -1) {
break;
}
+ assert(bpfen_len <= 32);
if (bpfen_len >= int(sizeof(bpfen))) {
fprintf(stderr, "Overlong BPFEN (%d bytes)\n", bpfen_len);
// exit(1);
// exit(1);
break;
}
- if (fread(&move[0], l, 1, fp) != 1) {
+ if (l == 0) {
+ move[0] = 0;
+ } else if (fread(&move[0], l, 1, fp) != 1) {
perror("fread()");
// exit(1);
break;
int bucket = hash_key_to_bucket(bpfen, bpfen_len, num_buckets);
Element e;
- e.bpfen = shards[bucket].arena->alloc(bpfen_len);
memcpy(e.bpfen, bpfen, bpfen_len);
e.bpfen_len = bpfen_len;
strcpy(e.move, move);