#include <unistd.h>
#include <zdict.h>
#include <zstd.h>
+#include <unicode/coll.h>
+#include <unicode/locid.h>
#define P4NENC_BOUND(n) ((n + 127) / 128 + (n + 32) * sizeof(uint32_t))
void finish();
vector<unsigned char> encoded;
- size_t get_num_docids() const {
+ size_t get_num_docids() const
+ {
// Updated only when we flush, so check that we're finished.
assert(pending_deltas.empty());
return num_docids;
std::string dir_times_compressed;
};
-
EncodingCorpus::EncodingCorpus(FILE *outfp, size_t block_size, ZSTD_CDict *cdict, bool store_dir_times)
: invindex(new PostingListBuilder *[NUM_TRIGRAMS]), outfp(outfp), outfp_pos(ftell(outfp)), block_size(block_size), store_dir_times(store_dir_times), cdict(cdict)
{
uint32_t docid = num_blocks;
+ // Oh, ICU...
+ vector<uint8_t> sort_key;
+ sort_key.resize(32);
+ int32_t num_locales;
+ const icu::Locale* locales = icu::Collator::getAvailableLocales(num_locales);
+ for (int i = 0; i < num_locales; ++i) {
+ const icu::Locale &loc = locales[i];
+ if (strcmp(loc.getName(), "en_US_POSIX") == 0) {
+ continue; // Too weird.
+ }
+ UErrorCode status = U_ZERO_ERROR;
+ icu::Collator *coll = icu::Collator::createInstance(loc, status);
+ if (U_FAILURE(status)) {
+ fprintf(stderr, "ERROR: Failed to create collator\n");
+ exit(1);
+ }
+ coll->setStrength(icu::Collator::PRIMARY);
+ const char *ptr = current_block.c_str();
+ const char *end = ptr + current_block.size();
+ while (ptr < end) {
+ size_t len = strlen(ptr);
+ int32_t sortkey_len;
+ for ( ;; ) {
+ sortkey_len = coll->getSortKey(icu::UnicodeString::fromUTF8(icu::StringPiece(ptr, len)), sort_key.data(), sort_key.size());
+ if (sortkey_len < sort_key.size()) { // Note <, not <=; we need to keep a slop byte.
+ break;
+ }
+ sort_key.resize(sortkey_len * 3 / 2);
+ }
+
+ const uint8_t *keyptr = &sort_key[0];
+ const uint8_t *keyend = keyptr + sortkey_len;
+ while (keyptr < keyend - 3) {
+ // NOTE: Will read one byte past the end of the trigram, but it's OK,
+ // since we always call it from contexts where there's a terminating zero byte.
+ uint32_t trgm;
+ memcpy(&trgm, keyptr, sizeof(trgm));
+ ++keyptr;
+ trgm = le32toh(trgm);
+ add_docid(trgm & 0xffffff, docid);
+ }
+
+ ptr += len + 1;
+ }
+ }
+#if 0
// Create trigrams.
const char *ptr = current_block.c_str();
const char *end = ptr + current_block.size();
ptr += 3;
continue;
}
- for ( ;; ) {
+ for (;;) {
// NOTE: Will read one byte past the end of the trigram, but it's OK,
// since we always call it from contexts where there's a terminating zero byte.
uint32_t trgm;
}
}
}
+#endif
// Compress and add the filename block.
filename_blocks.push_back(outfp_pos);
int fd = -1;
#ifdef O_TMPFILE
fd = open(path.c_str(), O_WRONLY | O_TMPFILE, 0640);
- if (fd == -1 && errno != EOPNOTSUPP) {
+ if (fd == -1 && errno != EOPNOTSUPP && errno != EISDIR) {
perror(path.c_str());
exit(1);
}