return buf;
}
-Corpus::Corpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times)
+class EncodingCorpus : public DatabaseReceiver {
+public:
+ EncodingCorpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times);
+ ~EncodingCorpus();
+
+ void add_file(std::string filename, dir_time dt) override;
+ void flush_block() override;
+ void finish() override;
+
+ std::vector<uint64_t> filename_blocks;
+ size_t num_files = 0, num_files_in_block = 0, num_blocks = 0;
+ bool seen_trigram(uint32_t trgm)
+ {
+ return invindex[trgm] != nullptr;
+ }
+ size_t num_files_seen() const override { return num_files; }
+ PostingListBuilder &get_pl_builder(uint32_t trgm);
+ size_t num_trigrams() const;
+ std::string get_compressed_dir_times();
+
+private:
+ void compress_dir_times(size_t allowed_slop);
+
+ std::unique_ptr<PostingListBuilder *[]> invindex;
+ FILE *outfp;
+ std::string current_block;
+ std::string tempbuf;
+ const size_t block_size;
+ const bool store_dir_times;
+ ZSTD_CDict *cdict;
+
+ ZSTD_CStream *dir_time_ctx = nullptr;
+ std::string dir_times; // Buffer of still-uncompressed data.
+ std::string dir_times_compressed;
+};
+
+
+EncodingCorpus::EncodingCorpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times)
: invindex(new PostingListBuilder *[NUM_TRIGRAMS]), outfp(outfp), block_size(block_size), store_dir_times(store_dir_times), cdict(cdict)
{
fill(invindex.get(), invindex.get() + NUM_TRIGRAMS, nullptr);
}
}
-Corpus::~Corpus()
+EncodingCorpus::~EncodingCorpus()
{
for (unsigned i = 0; i < NUM_TRIGRAMS; ++i) {
delete invindex[i];
}
}
-PostingListBuilder &Corpus::get_pl_builder(uint32_t trgm)
+PostingListBuilder &EncodingCorpus::get_pl_builder(uint32_t trgm)
{
if (invindex[trgm] == nullptr) {
invindex[trgm] = new PostingListBuilder;
return *invindex[trgm];
}
-void Corpus::add_file(string filename, dir_time dt)
+void EncodingCorpus::add_file(string filename, dir_time dt)
{
++num_files;
if (!current_block.empty()) {
}
}
-void Corpus::compress_dir_times(size_t allowed_slop)
+void EncodingCorpus::compress_dir_times(size_t allowed_slop)
{
while (dir_times.size() >= allowed_slop) {
size_t old_size = dir_times_compressed.size();
}
}
-void Corpus::flush_block()
+void EncodingCorpus::flush_block()
{
if (current_block.empty()) {
return;
++num_blocks;
}
-void Corpus::finish()
+void EncodingCorpus::finish()
{
flush_block();
}
-size_t Corpus::num_trigrams() const
+size_t EncodingCorpus::num_trigrams() const
{
size_t num = 0;
for (unsigned trgm = 0; trgm < NUM_TRIGRAMS; ++trgm) {
return num;
}
-string Corpus::get_compressed_dir_times()
+string EncodingCorpus::get_compressed_dir_times()
{
if (!store_dir_times) {
return "";
return x;
}
-unique_ptr<Trigram[]> create_hashtable(Corpus &corpus, const vector<uint32_t> &all_trigrams, uint32_t ht_size, uint32_t num_overflow_slots)
+unique_ptr<Trigram[]> create_hashtable(EncodingCorpus &corpus, const vector<uint32_t> &all_trigrams, uint32_t ht_size, uint32_t num_overflow_slots)
{
unique_ptr<Trigram[]> ht(new Trigram[ht_size + num_overflow_slots + 1]); // 1 for the sentinel element at the end.
for (unsigned i = 0; i < ht_size + num_overflow_slots + 1; ++i) {
hdr.conf_block_offset_bytes = 0;
}
-Corpus *DatabaseBuilder::start_corpus(bool store_dir_times)
+DatabaseReceiver *DatabaseBuilder::start_corpus(bool store_dir_times)
{
corpus_start = steady_clock::now();
- corpus = new Corpus(outfp, block_size, cdict, store_dir_times);
+ corpus = new EncodingCorpus(outfp, block_size, cdict, store_dir_times);
return corpus;
}
virtual void add_file(std::string filename, dir_time dt) = 0;
virtual void flush_block() = 0;
virtual void finish() { flush_block(); }
+
+ // EncodingCorpus only.
+ virtual size_t num_files_seen() const { return -1; }
};
class DictionaryBuilder : public DatabaseReceiver {
std::vector<size_t> lengths;
};
-class Corpus : public DatabaseReceiver {
-public:
- Corpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times);
- ~Corpus();
-
- void add_file(std::string filename, dir_time dt) override;
- void flush_block() override;
- void finish() override;
-
- std::vector<uint64_t> filename_blocks;
- size_t num_files = 0, num_files_in_block = 0, num_blocks = 0;
- bool seen_trigram(uint32_t trgm)
- {
- return invindex[trgm] != nullptr;
- }
- PostingListBuilder &get_pl_builder(uint32_t trgm);
- size_t num_trigrams() const;
- std::string get_compressed_dir_times();
-
-private:
- void compress_dir_times(size_t allowed_slop);
-
- std::unique_ptr<PostingListBuilder *[]> invindex;
- FILE *outfp;
- std::string current_block;
- std::string tempbuf;
- const size_t block_size;
- const bool store_dir_times;
- ZSTD_CDict *cdict;
-
- ZSTD_CStream *dir_time_ctx = nullptr;
- std::string dir_times; // Buffer of still-uncompressed data.
- std::string dir_times_compressed;
-};
+class EncodingCorpus;
class DatabaseBuilder {
public:
DatabaseBuilder(const char *outfile, gid_t owner, int block_size, std::string dictionary, bool check_visibility);
- Corpus *start_corpus(bool store_dir_times);
+ DatabaseReceiver *start_corpus(bool store_dir_times);
void set_next_dictionary(std::string next_dictionary);
void set_conf_block(std::string conf_block);
void finish_corpus();
Header hdr;
const int block_size;
std::chrono::steady_clock::time_point corpus_start;
- Corpus *corpus = nullptr;
+ EncodingCorpus *corpus = nullptr;
ZSTD_CDict *cdict = nullptr;
std::string next_dictionary, conf_block;
};
string dictionary = builder.train(1024);
DatabaseBuilder db(outfile, /*owner=*/-1, block_size, dictionary, /*check_visibility=*/true);
- Corpus *corpus = db.start_corpus(/*store_dir_times=*/false);
+ DatabaseReceiver *corpus = db.start_corpus(/*store_dir_times=*/false);
if (plaintext) {
read_plaintext(infp, corpus);
} else {
}
fclose(infp);
- dprintf("Read %zu files from %s\n", corpus->num_files, infile);
+ dprintf("Read %zu files from %s\n", corpus->num_files_seen(), infile);
db.finish_corpus();
}
// “parent_dev” must be the device of the parent directory of “path”.
//
// Takes ownership of fd.
-int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, Corpus *corpus, DictionaryBuilder *dict_builder)
+int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, DatabaseReceiver *corpus, DictionaryBuilder *dict_builder)
{
if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, path)) {
if (conf_debug_pruning) {
DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary(), conf_check_visibility);
db.set_conf_block(conf_block);
- Corpus *corpus = db.start_corpus(/*store_dir_times=*/true);
+ DatabaseReceiver *corpus = db.start_corpus(/*store_dir_times=*/true);
int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
if (root_fd == -1) {