X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=binloader.cpp;h=730f59279eba946af8073826b08cf775564cf994;hb=f3225963af3ef8bf33b49d5b51a37d36f47e1482;hp=2ed60cbe78250ff34eba9f7c358968f90c4ddb97;hpb=756a91677c1565b1bd555722ea795f911106104f;p=remoteglot-book

diff --git a/binloader.cpp b/binloader.cpp
index 2ed60cb..730f592 100644
--- a/binloader.cpp
+++ b/binloader.cpp
@@ -12,18 +12,36 @@
 #include <unordered_set>
 #include <string.h>
 #include "count.pb.h"
-#include "arena.h"
 #include "hash.h"
 
 #define DUMMY_TIMESTAMP 32503680000
 
 using namespace std;
 
-Arena arena;
+static inline int memcmp_different_len(const void *s1, size_t n1, const void *s2, size_t n2)
+{
+	size_t shared_len = min(n1, n2);
+	if (shared_len >= 8) {
+		uint64_t a1 = *(const uint64_t *)s1;
+		uint64_t a2 = *(const uint64_t *)s2;
+		if (a1 != a2) {
+			a1 = __builtin_bswap64(a1);
+			a2 = __builtin_bswap64(a2);
+			return (a1 < a2) ? -1 : 1;
+		}
+	}
+
+	int s = memcmp(s1, s2, shared_len);
+	if (s != 0) {
+		return s;
+	}
+
+	return n2 - n1;
+}
 
 enum Result { WHITE = 0, DRAW, BLACK };
 struct Element {
-	char *bpfen;
+	char bpfen[32];  // includes prev_board_hash
 	int bpfen_len;
 	char move[8];   // Na1xc3+
 	Result result;
@@ -33,21 +51,13 @@ struct Element {
 	long start_position;
 
 	bool operator< (const Element& other) const {
-		int shared_len = min(bpfen_len, other.bpfen_len);
-		int s = memcmp(bpfen, other.bpfen, shared_len);
-		if (s < 0) {
-			return true;
-		} else if (s > 0) {
-			return false;
-		} else {
-			return bpfen_len < other.bpfen_len;
-		}
+		int s = memcmp_different_len(bpfen, bpfen_len, other.bpfen, other.bpfen_len);
+		return s < 0;
 	}
 };
 
 struct ShardData {
 	vector<Element> elems;
-	unique_ptr<Arena> arena;  // Used to allocate bpfen.
 	int num_written_subshards = 0;
 };
 
@@ -88,7 +98,7 @@ void write_subshard(const char *basename, ShardData* shard, int bucket)
 			c.set_pgn_file_num(e.file_num);
 			c.set_pgn_start_position(e.start_position);
 		}
-		if (!moves.count(e.move)) {
+		if (strlen(e.move) > 0 && !moves.count(e.move)) {
 			moves.insert(e.move);
 			c.add_move(e.move);
 		}
@@ -96,9 +106,10 @@ void write_subshard(const char *basename, ShardData* shard, int bucket)
 		    e.bpfen_len != shard->elems[i + 1].bpfen_len ||
 		    memcmp(e.bpfen, shard->elems[i + 1].bpfen, e.bpfen_len) != 0) {
 			c.SerializeToString(&buf);
-			mtbl_writer_add(mtbl,
+			mtbl_res res = mtbl_writer_add(mtbl,
 				(const uint8_t *)e.bpfen, e.bpfen_len,
 				(const uint8_t *)buf.data(), buf.size());
+			assert(res == mtbl_res_success);
 			c = Count();
 			moves.clear();
 		}
@@ -106,7 +117,6 @@ void write_subshard(const char *basename, ShardData* shard, int bucket)
 	mtbl_writer_destroy(&mtbl);
 
 	shard->elems.clear();
-	shard->arena.reset(new Arena);
 }
 
 int main(int argc, char **argv)
@@ -119,7 +129,6 @@ int main(int argc, char **argv)
 
 	for (int i = 0; i < num_buckets; ++i) {
 		shards[i].elems.reserve(num_pos_per_subshard);
-		shards[i].arena.reset(new Arena);
 	}
 
 	size_t num_elems = 0;
@@ -141,6 +150,7 @@ int main(int argc, char **argv)
 			if (bpfen_len == -1) {
 				break;
 			}
+			assert(bpfen_len <= 32);
 			if (bpfen_len >= int(sizeof(bpfen))) {
 				fprintf(stderr, "Overlong BPFEN (%d bytes)\n", bpfen_len);
 		//		exit(1);
@@ -204,7 +214,9 @@ int main(int argc, char **argv)
 		//		exit(1);
 				break;
 			}
-			if (fread(&move[0], l, 1, fp) != 1) {
+			if (l == 0) {
+				move[0] = 0;
+			} else if (fread(&move[0], l, 1, fp) != 1) {
 				perror("fread()");
 		//		exit(1);
 				break;
@@ -213,7 +225,6 @@ int main(int argc, char **argv)
 
 			int bucket = hash_key_to_bucket(bpfen, bpfen_len, num_buckets);
 			Element e;
-			e.bpfen = shards[bucket].arena->alloc(bpfen_len);
 			memcpy(e.bpfen, bpfen, bpfen_len);
 			e.bpfen_len = bpfen_len;
 			strcpy(e.move, move);