Partition the SSTable; somewhat less efficient space-wise, it seems, but we avoid...

[remoteglot-book] / binloader.cpp

diff --git a/binloader.cpp b/binloader.cpp
index 9658c2ec992bfd6941a086628b6e7042911ca624..98b29457e8a807eccb7806d0ab5d230ac6953468 100644 (file)
--- a/binloader.cpp
+++ b/binloader.cpp
@@ -1,6 +1,6 @@
 //#define _GLIBCXX_PARALLEL
 
-// Usage: ./binloader IN1 IN2 IN3 ... OUT
+// Usage: ./binloader IN1 IN2 IN3 ... OUT NUM_BUCKETS
 
 #include <stdio.h>
 #include <vector>
@@ -11,6 +11,7 @@
 #include <string>
 #include <string.h>
 #include "count.h"
+#include "hash.h"
 
 using namespace std;
 
@@ -27,9 +28,13 @@ struct Element {
 
 int main(int argc, char **argv)
 {
-       vector<Element> elems;
+       int num_buckets = atoi(argv[argc - 1]);
 
-       for (int i = 1; i < argc - 1; ++i) {
+       vector<vector<Element>> elems;
+       elems.resize(num_buckets);
+
+       size_t num_elems = 0;
+       for (int i = 1; i < argc - 2; ++i) {
                FILE *fp = fopen(argv[i], "rb");
                if (fp == NULL) {
                        perror(argv[i]);
@@ -72,41 +77,52 @@ int main(int argc, char **argv)
                                //exit(1);
                                break;
                        }
-                       elems.emplace_back(Element {move(bpfen_and_move), Result(r), opening_num, white_elo, black_elo});
+
+                       int bucket = hash_key_to_bucket(bpfen_and_move.data(), bpfen_and_move.size(), num_buckets);
+                       elems[bucket].emplace_back(Element {move(bpfen_and_move), Result(r), opening_num, white_elo, black_elo});
+                       ++num_elems;
                }
                fclose(fp);
 
-               printf("Read %ld elems\n", elems.size());
+               printf("Read %ld elems\n", num_elems);
        }
 
        printf("Sorting...\n");
-       sort(elems.begin(), elems.end());
+       for (int i = 0; i < num_buckets; ++i) {
+               sort(elems[i].begin(), elems[i].end());
+       }
 
-       printf("Writing SSTable...\n");
-       mtbl_writer_options* wopt = mtbl_writer_options_init();
-       mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_SNAPPY);
-       mtbl_writer* mtbl = mtbl_writer_init(argv[argc - 1], wopt);
-       Count c;
-       for (int i = 0; i < elems.size(); ++i) {
-               if (elems[i].result == WHITE) {
-                       ++c.white;
-               } else if (elems[i].result == DRAW) {
-                       ++c.draw;
-               } else if (elems[i].result == BLACK) {
-                       ++c.black;
-               }
-               c.opening_num = elems[i].opening_num;
-               if (elems[i].white_elo >= 100 && elems[i].black_elo >= 100) {
-                       c.sum_white_elo += elems[i].white_elo;
-                       c.sum_black_elo += elems[i].black_elo;
-                       ++c.num_elo;
-               }
-               if (i == elems.size() - 1 || elems[i].bpfen_and_move != elems[i + 1].bpfen_and_move) {
-                       mtbl_writer_add(mtbl,
-                               (const uint8_t *)elems[i].bpfen_and_move.data(), elems[i].bpfen_and_move.size(),
-                               (const uint8_t *)&c, sizeof(c));
-                       c = Count();
+       printf("Writing SSTables...\n");
+       for (int i = 0; i < num_buckets; ++i) {
+               char filename[256];
+               snprintf(filename, sizeof(filename), "%s.part%04d", argv[argc - 2], i);
+
+               mtbl_writer_options* wopt = mtbl_writer_options_init();
+               mtbl_writer_options_set_compression(wopt, MTBL_COMPRESSION_SNAPPY);
+               mtbl_writer* mtbl = mtbl_writer_init(filename, wopt);
+               Count c;
+               for (size_t j = 0; j < elems[i].size(); ++j) {
+                       const Element &e = elems[i][j];
+                       if (e.result == WHITE) {
+                               ++c.white;
+                       } else if (e.result == DRAW) {
+                               ++c.draw;
+                       } else if (e.result == BLACK) {
+                               ++c.black;
+                       }
+                       c.opening_num = e.opening_num;
+                       if (e.white_elo >= 100 && e.black_elo >= 100) {
+                               c.sum_white_elo += e.white_elo;
+                               c.sum_black_elo += e.black_elo;
+                               ++c.num_elo;
+                       }
+                       if (j == elems[i].size() - 1 || e.bpfen_and_move != elems[i][j + 1].bpfen_and_move) {
+                               mtbl_writer_add(mtbl,
+                                       (const uint8_t *)e.bpfen_and_move.data(), e.bpfen_and_move.size(),
+                                       (const uint8_t *)&c, sizeof(c));
+                               c = Count();
+                       }
                }
+               mtbl_writer_destroy(&mtbl);
        }
-       mtbl_writer_destroy(&mtbl);
 }