class PostingListBuilder {
public:
inline void add_docid(uint32_t docid);
+ inline void add_first_docid(uint32_t docid);
void finish();
- string encoded;
+ vector<unsigned char> encoded;
size_t num_docids = 0;
private:
vector<uint32_t> pending_deltas;
- uint32_t last_block_end, last_docid = -1;
+ uint32_t last_docid = -1;
};
void PostingListBuilder::add_docid(uint32_t docid)
return;
}
- if (num_docids == 0) {
- // Very first docid.
- write_header(docid);
- ++num_docids;
- last_block_end = last_docid = docid;
- return;
- }
+ assert(num_docids != 0);
pending_deltas.push_back(docid - last_docid - 1);
last_docid = docid;
if (pending_deltas.size() == 128) {
append_block();
pending_deltas.clear();
- last_block_end = docid;
}
++num_docids;
}
+void PostingListBuilder::add_first_docid(uint32_t docid)
+{
+ write_header(docid);
+ ++num_docids;
+ last_docid = docid;
+}
+
void PostingListBuilder::finish()
{
if (pending_deltas.empty()) {
// No interleaving for partial blocks.
unsigned char buf[P4NENC_BOUND(128)];
unsigned char *end = encode_pfor_single_block<128>(pending_deltas.data(), pending_deltas.size(), /*interleaved=*/false, buf);
- encoded.append(reinterpret_cast<char *>(buf), reinterpret_cast<char *>(end));
+ encoded.insert(encoded.end(), buf, end);
}
void PostingListBuilder::append_block()
unsigned char buf[P4NENC_BOUND(128)];
assert(pending_deltas.size() == 128);
unsigned char *end = encode_pfor_single_block<128>(pending_deltas.data(), 128, /*interleaved=*/true, buf);
- encoded.append(reinterpret_cast<char *>(buf), reinterpret_cast<char *>(end));
+ encoded.insert(encoded.end(), buf, end);
}
void PostingListBuilder::write_header(uint32_t docid)
{
unsigned char buf[P4NENC_BOUND(1)];
unsigned char *end = write_baseval(docid, buf);
- encoded.append(reinterpret_cast<char *>(buf), end - buf);
+ encoded.insert(encoded.end(), buf, end);
}
void DictionaryBuilder::add_file(string filename, dir_time)
string buf;
buf.resize(buf_size);
size_t ret = ZDICT_trainFromBuffer(&buf[0], buf_size, dictionary_buf.data(), lengths.data(), lengths.size());
- if (ret == size_t(-1)) {
+ if (ZDICT_isError(ret)) {
return "";
}
dprintf("Sampled %zu bytes in %zu blocks, built a dictionary of size %zu\n", dictionary_buf.size(), lengths.size(), ret);
return buf;
}
-Corpus::Corpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times)
+class EncodingCorpus : public DatabaseReceiver {
+public:
+ EncodingCorpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times);
+ ~EncodingCorpus();
+
+ void add_file(std::string filename, dir_time dt) override;
+ void flush_block() override;
+ void finish() override;
+
+ std::vector<uint64_t> filename_blocks;
+ size_t num_files = 0, num_files_in_block = 0, num_blocks = 0;
+ bool seen_trigram(uint32_t trgm)
+ {
+ return invindex[trgm] != nullptr;
+ }
+ size_t num_files_seen() const override { return num_files; }
+ PostingListBuilder &get_pl_builder(uint32_t trgm)
+ {
+ return *invindex[trgm];
+ }
+
+ void add_docid(uint32_t trgm, uint32_t docid)
+ {
+ if (invindex[trgm] == nullptr) {
+ invindex[trgm] = new PostingListBuilder;
+ invindex[trgm]->add_first_docid(docid);
+ } else {
+ invindex[trgm]->add_docid(docid);
+ }
+ }
+
+ size_t num_trigrams() const;
+ std::string get_compressed_dir_times();
+
+private:
+ void compress_dir_times(size_t allowed_slop);
+
+ std::unique_ptr<PostingListBuilder *[]> invindex;
+ FILE *outfp;
+ std::string current_block;
+ std::string tempbuf;
+ const size_t block_size;
+ const bool store_dir_times;
+ ZSTD_CDict *cdict;
+
+ ZSTD_CStream *dir_time_ctx = nullptr;
+ std::string dir_times; // Buffer of still-uncompressed data.
+ std::string dir_times_compressed;
+};
+
+
+EncodingCorpus::EncodingCorpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times)
: invindex(new PostingListBuilder *[NUM_TRIGRAMS]), outfp(outfp), block_size(block_size), store_dir_times(store_dir_times), cdict(cdict)
{
fill(invindex.get(), invindex.get() + NUM_TRIGRAMS, nullptr);
}
}
-Corpus::~Corpus()
+EncodingCorpus::~EncodingCorpus()
{
for (unsigned i = 0; i < NUM_TRIGRAMS; ++i) {
delete invindex[i];
}
}
-PostingListBuilder &Corpus::get_pl_builder(uint32_t trgm)
-{
- if (invindex[trgm] == nullptr) {
- invindex[trgm] = new PostingListBuilder;
- }
- return *invindex[trgm];
-}
-
-void Corpus::add_file(string filename, dir_time dt)
+void EncodingCorpus::add_file(string filename, dir_time dt)
{
++num_files;
if (!current_block.empty()) {
}
}
-void Corpus::compress_dir_times(size_t allowed_slop)
+void EncodingCorpus::compress_dir_times(size_t allowed_slop)
{
while (dir_times.size() >= allowed_slop) {
size_t old_size = dir_times_compressed.size();
}
}
-void Corpus::flush_block()
+void EncodingCorpus::flush_block()
{
if (current_block.empty()) {
return;
if (s.size() >= 3) {
for (size_t j = 0; j < s.size() - 2; ++j) {
uint32_t trgm = read_trigram(s, j);
- get_pl_builder(trgm).add_docid(docid);
+ add_docid(trgm, docid);
}
}
ptr += s.size() + 1;
++num_blocks;
}
-void Corpus::finish()
+void EncodingCorpus::finish()
{
flush_block();
}
-size_t Corpus::num_trigrams() const
+size_t EncodingCorpus::num_trigrams() const
{
size_t num = 0;
for (unsigned trgm = 0; trgm < NUM_TRIGRAMS; ++trgm) {
return num;
}
-string Corpus::get_compressed_dir_times()
+string EncodingCorpus::get_compressed_dir_times()
{
if (!store_dir_times) {
return "";
return x;
}
-unique_ptr<Trigram[]> create_hashtable(Corpus &corpus, const vector<uint32_t> &all_trigrams, uint32_t ht_size, uint32_t num_overflow_slots)
+unique_ptr<Trigram[]> create_hashtable(EncodingCorpus &corpus, const vector<uint32_t> &all_trigrams, uint32_t ht_size, uint32_t num_overflow_slots)
{
unique_ptr<Trigram[]> ht(new Trigram[ht_size + num_overflow_slots + 1]); // 1 for the sentinel element at the end.
for (unsigned i = 0; i < ht_size + num_overflow_slots + 1; ++i) {
if (path.empty()) {
path = ".";
}
+#ifdef O_TMPFILE
int fd = open(path.c_str(), O_WRONLY | O_TMPFILE, 0640);
if (fd == -1) {
perror(path.c_str());
exit(1);
}
+#else
+ temp_filename = string(outfile) + ".XXXXXX";
+ int fd = mkstemp(&temp_filename[0]);
+ if (fd == -1) {
+ perror(temp_filename.c_str());
+ exit(1);
+ }
+ if (fchmod(fd, 0640) == -1) {
+ perror("fchmod");
+ exit(1);
+ }
+#endif
if (owner != (gid_t)-1) {
if (fchown(fd, (uid_t)-1, owner) == -1) {
hdr.conf_block_offset_bytes = 0;
}
-Corpus *DatabaseBuilder::start_corpus(bool store_dir_times)
+DatabaseReceiver *DatabaseBuilder::start_corpus(bool store_dir_times)
{
corpus_start = steady_clock::now();
- corpus = new Corpus(outfp, block_size, cdict, store_dir_times);
+ corpus = new EncodingCorpus(outfp, block_size, cdict, store_dir_times);
return corpus;
}
continue;
}
- const string &encoded = corpus->get_pl_builder(hashtable[i].trgm).encoded;
+ const vector<unsigned char> &encoded = corpus->get_pl_builder(hashtable[i].trgm).encoded;
offset += encoded.size();
}
if (hashtable[i].num_docids == 0) {
continue;
}
- const string &encoded = corpus->get_pl_builder(hashtable[i].trgm).encoded;
+ const vector<unsigned char> &encoded = corpus->get_pl_builder(hashtable[i].trgm).encoded;
fwrite(encoded.data(), encoded.size(), 1, outfp);
}
fseek(outfp, 0, SEEK_SET);
fwrite(&hdr, sizeof(hdr), 1, outfp);
+#ifdef O_TMPFILE
// Give the file a proper name, making it visible in the file system.
// TODO: It would be nice to be able to do this atomically, like with rename.
unlink(outfile.c_str());
perror("linkat");
exit(1);
}
+#else
+ if (rename(temp_filename.c_str(), outfile.c_str()) == -1) {
+ perror("rename");
+ exit(1);
+ }
+#endif
fclose(outfp);
projects / plocate / blobdiff