3 Copyright (C) 2005, 2007, 2008 Red Hat, Inc. All rights reserved.
5 This copyrighted material is made available to anyone wishing to use, modify,
6 copy, or redistribute it subject to the terms and conditions of the GNU General
9 This program is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
11 PARTICULAR PURPOSE. See the GNU General Public License for more details.
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
15 Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 Author: Miloslav Trmac <mitr@redhat.com>
20 plocate modifications: Copyright (C) 2020 Steinar H. Gunderson.
21 plocate parts and modifications are licensed under the GPLv2 or, at your option,
25 #include "bind-mount.h"
26 #include "complete_pread.h"
28 #include "database-builder.h"
31 #include "io_uring_engine.h"
35 #include <arpa/inet.h>
52 #include <sys/resource.h>
55 #include <sys/types.h>
61 using namespace std::chrono;
63 /* Next conf_prunepaths entry */
64 static size_t conf_prunepaths_index; /* = 0; */
69 "Usage: updatedb PLOCATE_DB\n"
71 "Generate plocate index from mlocate.db, typically /var/lib/mlocate/mlocate.db.\n"
72 "Normally, the destination should be /var/lib/mlocate/plocate.db.\n"
74 " -b, --block-size SIZE number of filenames to store in each block (default 32)\n"
75 " -p, --plaintext input is a plaintext file, not an mlocate database\n"
76 " --help print this help\n"
77 " --version print version information\n");
82 printf("updatedb %s\n", PACKAGE_VERSION);
83 printf("Copyright (C) 2007 Red Hat, Inc. All rights reserved.\n");
84 printf("Copyright 2020 Steinar H. Gunderson\n");
85 printf("This software is distributed under the GPL v.2.\n");
87 printf("This program is provided with NO WARRANTY, to the extent permitted by law.\n");
90 int opendir_noatime(int dirfd, const char *path)
92 static bool noatime_failed = false;
94 if (!noatime_failed) {
95 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY | O_NOATIME);
98 } else if (errno == EPERM) {
99 /* EPERM is fairly O_NOATIME-specific; missing access rights cause
101 noatime_failed = true;
107 return openat(dirfd, path, O_RDONLY | O_DIRECTORY);
110 bool time_is_current(const dir_time &t)
112 static dir_time cache{ 0, 0 };
114 /* This is more difficult than it should be because Linux uses a cheaper time
115 source for filesystem timestamps than for gettimeofday() and they can get
116 slightly out of sync, see
117 https://bugzilla.redhat.com/show_bug.cgi?id=244697 . This affects even
118 nanosecond timestamps (and don't forget that tv_nsec existence doesn't
119 guarantee that the underlying filesystem has such resolution - it might be
120 microseconds or even coarser).
122 The worst case is probably FAT timestamps with 2-second resolution
123 (although using such a filesystem violates POSIX file times requirements).
125 So, to be on the safe side, require a >3.0 second difference (2 seconds to
126 make sure the FAT timestamp changed, 1 more to account for the Linux
127 timestamp races). This large margin might make updatedb marginally more
128 expensive, but it only makes a difference if the directory was very
129 recently updated _and_ is will not be updated again until the next
130 updatedb run; this is not likely to happen for most directories. */
132 /* Cache gettimeofday () results to rule out obviously old time stamps;
133 CACHE contains the earliest time we reject as too current. */
139 gettimeofday(&tv, nullptr);
140 cache.sec = tv.tv_sec - 3;
141 cache.nsec = tv.tv_usec * 1000;
150 // For directories only:
152 dir_time dt = unknown_dir_time;
153 dir_time db_modified = unknown_dir_time;
157 bool filesystem_is_excluded(const char *path)
159 if (conf_debug_pruning) {
160 /* This is debugging output, don't mark anything for translation */
161 fprintf(stderr, "Checking whether filesystem `%s' is excluded:\n", path);
163 FILE *f = setmntent("/proc/mounts", "r");
169 while ((me = getmntent(f)) != nullptr) {
170 if (conf_debug_pruning) {
171 /* This is debugging output, don't mark anything for translation */
172 fprintf(stderr, " `%s', type `%s'\n", me->mnt_dir, me->mnt_type);
174 string type(me->mnt_type);
175 for (char &p : type) {
178 if (find(conf_prunefs.begin(), conf_prunefs.end(), type) != conf_prunefs.end()) {
179 /* Paths in /proc/self/mounts contain no symbolic links. Besides
180 avoiding a few system calls, avoiding the realpath () avoids hangs
181 if the filesystem is unavailable hard-mounted NFS. */
182 char *dir = me->mnt_dir;
183 if (conf_debug_pruning) {
184 /* This is debugging output, don't mark anything for translation */
185 fprintf(stderr, " => type matches, dir `%s'\n", dir);
187 bool res = (strcmp(path, dir) == 0);
188 if (dir != me->mnt_dir)
196 if (conf_debug_pruning) {
197 /* This is debugging output, don't mark anything for translation */
198 fprintf(stderr, "...done\n");
204 dir_time get_dirtime_from_stat(const struct stat &buf)
206 dir_time ctime{ buf.st_ctim.tv_sec, int32_t(buf.st_ctim.tv_nsec) };
207 dir_time mtime{ buf.st_mtim.tv_sec, int32_t(buf.st_mtim.tv_nsec) };
208 dir_time dt = max(ctime, mtime);
210 if (time_is_current(dt)) {
211 /* The directory might be changing right now and we can't be sure the
212 timestamp will be changed again if more changes happen very soon, mark
213 the timestamp as invalid to force rescanning the directory next time
215 return unknown_dir_time;
221 // Represents the old database we are updating.
224 explicit ExistingDB(int fd);
227 pair<string, dir_time> read_next();
228 void unread(pair<string, dir_time> record)
230 unread_record = move(record);
232 string read_next_dictionary() const;
233 bool get_error() const { return error; }
239 uint32_t current_docid = 0;
241 string current_filename_block;
242 const char *current_filename_ptr = nullptr, *current_filename_end = nullptr;
244 off_t compressed_dir_time_pos;
245 string compressed_dir_time;
246 string current_dir_time_block;
247 const char *current_dir_time_ptr = nullptr, *current_dir_time_end = nullptr;
249 pair<string, dir_time> unread_record;
251 // Used in one-shot mode, repeatedly.
254 // Used in streaming mode.
255 ZSTD_DCtx *dir_time_ctx;
257 ZSTD_DDict *ddict = nullptr;
259 // If true, we've discovered an error or EOF, and will return only
260 // empty data from here.
261 bool eof = false, error = false;
264 ExistingDB::ExistingDB(int fd)
272 if (!try_complete_pread(fd, &hdr, sizeof(hdr), /*offset=*/0)) {
274 perror("pread(header)");
279 if (memcmp(hdr.magic, "\0plocate", 8) != 0) {
281 fprintf(stderr, "Old database had header mismatch, ignoring.\n");
286 if (hdr.version != 1 || hdr.max_version < 2) {
288 fprintf(stderr, "Old database had version mismatch (version=%d max_version=%d), ignoring.\n",
289 hdr.version, hdr.max_version);
295 // Compare the configuration block with our current one.
296 if (hdr.conf_block_length_bytes != conf_block.size()) {
298 fprintf(stderr, "Old database had different configuration block (size mismatch), ignoring.\n");
304 str.resize(hdr.conf_block_length_bytes);
305 if (!try_complete_pread(fd, str.data(), hdr.conf_block_length_bytes, hdr.conf_block_offset_bytes)) {
307 perror("pread(conf_block)");
312 if (str != conf_block) {
314 fprintf(stderr, "Old database had different configuration block (contents mismatch), ignoring.\n");
320 // Read dictionary, if it exists.
321 if (hdr.zstd_dictionary_length_bytes > 0) {
323 dictionary.resize(hdr.zstd_dictionary_length_bytes);
324 if (try_complete_pread(fd, &dictionary[0], hdr.zstd_dictionary_length_bytes, hdr.zstd_dictionary_offset_bytes)) {
325 ddict = ZSTD_createDDict(dictionary.data(), dictionary.size());
328 perror("pread(dictionary)");
334 compressed_dir_time_pos = hdr.directory_data_offset_bytes;
336 ctx = ZSTD_createDCtx();
337 dir_time_ctx = ZSTD_createDCtx();
340 ExistingDB::~ExistingDB()
347 pair<string, dir_time> ExistingDB::read_next()
349 if (!unread_record.first.empty()) {
350 auto ret = move(unread_record);
351 unread_record.first.clear();
356 return { "", not_a_dir };
359 // See if we need to read a new filename block.
360 if (current_filename_ptr == nullptr) {
361 if (current_docid >= hdr.num_docids) {
363 return { "", not_a_dir };
366 // Read the file offset from this docid and the next one.
367 // This is always allowed, since we have a sentinel block at the end.
368 off_t offset_for_block = hdr.filename_index_offset_bytes + current_docid * sizeof(uint64_t);
370 if (!try_complete_pread(fd, vals, sizeof(vals), offset_for_block)) {
372 perror("pread(offset)");
375 return { "", not_a_dir };
378 off_t offset = vals[0];
379 size_t compressed_len = vals[1] - vals[0];
380 unique_ptr<char[]> compressed(new char[compressed_len]);
381 if (!try_complete_pread(fd, compressed.get(), compressed_len, offset)) {
383 perror("pread(block)");
386 return { "", not_a_dir };
389 unsigned long long uncompressed_len = ZSTD_getFrameContentSize(compressed.get(), compressed_len);
390 if (uncompressed_len == ZSTD_CONTENTSIZE_UNKNOWN || uncompressed_len == ZSTD_CONTENTSIZE_ERROR) {
392 fprintf(stderr, "ZSTD_getFrameContentSize() failed\n");
395 return { "", not_a_dir };
399 block.resize(uncompressed_len + 1);
402 if (ddict != nullptr) {
403 err = ZSTD_decompress_usingDDict(ctx, &block[0], block.size(), compressed.get(),
404 compressed_len, ddict);
406 err = ZSTD_decompressDCtx(ctx, &block[0], block.size(), compressed.get(),
409 if (ZSTD_isError(err)) {
411 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
414 return { "", not_a_dir };
416 block[block.size() - 1] = '\0';
417 current_filename_block = move(block);
418 current_filename_ptr = current_filename_block.data();
419 current_filename_end = current_filename_block.data() + current_filename_block.size();
423 // See if we need to read more directory time data.
424 while (current_dir_time_ptr == current_dir_time_end ||
425 (*current_dir_time_ptr != 0 &&
426 size_t(current_dir_time_end - current_dir_time_ptr) < sizeof(dir_time) + 1)) {
427 if (current_dir_time_ptr != nullptr) {
428 const size_t bytes_consumed = current_dir_time_ptr - current_dir_time_block.data();
429 current_dir_time_block.erase(current_dir_time_block.begin(), current_dir_time_block.begin() + bytes_consumed);
432 // See if we can get more data out without reading more.
433 const size_t existing_data = current_dir_time_block.size();
434 current_dir_time_block.resize(existing_data + 4096);
436 ZSTD_outBuffer outbuf;
437 outbuf.dst = current_dir_time_block.data() + existing_data;
442 inbuf.src = compressed_dir_time.data();
443 inbuf.size = compressed_dir_time.size();
446 int err = ZSTD_decompressStream(dir_time_ctx, &outbuf, &inbuf);
449 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
452 return { "", not_a_dir };
454 compressed_dir_time.erase(compressed_dir_time.begin(), compressed_dir_time.begin() + inbuf.pos);
455 current_dir_time_block.resize(existing_data + outbuf.pos);
457 if (inbuf.pos == 0 && outbuf.pos == 0) {
458 // No movement, we'll need to try to read more data.
460 size_t bytes_to_read = min<size_t>(
461 hdr.directory_data_offset_bytes + hdr.directory_data_length_bytes - compressed_dir_time_pos,
463 if (bytes_to_read == 0) {
465 return { "", not_a_dir };
467 if (!try_complete_pread(fd, buf, bytes_to_read, compressed_dir_time_pos)) {
469 perror("pread(dirtime)");
472 return { "", not_a_dir };
474 compressed_dir_time_pos += bytes_to_read;
475 compressed_dir_time.insert(compressed_dir_time.end(), buf, buf + bytes_to_read);
477 // Next iteration will now try decompressing more.
480 current_dir_time_ptr = current_dir_time_block.data();
481 current_dir_time_end = current_dir_time_block.data() + current_dir_time_block.size();
484 string filename = current_filename_ptr;
485 current_filename_ptr += filename.size() + 1;
486 if (current_filename_ptr == current_filename_end) {
487 // End of this block.
488 current_filename_ptr = nullptr;
491 if (*current_dir_time_ptr == 0) {
492 ++current_dir_time_ptr;
493 return { move(filename), not_a_dir };
495 ++current_dir_time_ptr;
497 memcpy(&dt.sec, current_dir_time_ptr, sizeof(dt.sec));
498 current_dir_time_ptr += sizeof(dt.sec);
499 memcpy(&dt.nsec, current_dir_time_ptr, sizeof(dt.nsec));
500 current_dir_time_ptr += sizeof(dt.nsec);
501 return { move(filename), dt };
505 string ExistingDB::read_next_dictionary() const
507 if (hdr.next_zstd_dictionary_length_bytes == 0 || hdr.next_zstd_dictionary_length_bytes > 1048576) {
511 str.resize(hdr.next_zstd_dictionary_length_bytes);
512 if (!try_complete_pread(fd, str.data(), hdr.next_zstd_dictionary_length_bytes, hdr.next_zstd_dictionary_offset_bytes)) {
514 perror("pread(next_dictionary)");
521 // Scans the directory with absolute path “path”, which is opened as “fd”.
522 // Uses relative paths and openat() only, evading any issues with PATH_MAX
523 // and time-of-check-time-of-use race conditions. (mlocate's updatedb
524 // does a much more complicated dance with changing the current working
525 // directory, probably in the interest of portability to old platforms.)
526 // “parent_dev” must be the device of the parent directory of “path”.
528 // Takes ownership of fd.
529 int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, Corpus *corpus, DictionaryBuilder *dict_builder)
531 if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, path)) {
532 if (conf_debug_pruning) {
533 /* This is debugging output, don't mark anything for translation */
534 fprintf(stderr, "Skipping `%s': in prunepaths\n", path.c_str());
539 if (conf_prune_bind_mounts && is_bind_mount(path.c_str())) {
540 if (conf_debug_pruning) {
541 /* This is debugging output, don't mark anything for translation */
542 fprintf(stderr, "Skipping `%s': bind mount\n", path.c_str());
548 // We read in the old directory no matter whether it is current or not,
549 // because even if we're not going to use it, we'll need the modification directory
550 // of any subdirectories.
552 // Skip over anything before this directory; it is stuff that we would have
553 // consumed earlier if we wanted it.
555 pair<string, dir_time> record = existing_db->read_next();
556 if (record.first.empty()) {
559 if (dir_path_cmp(path, record.first) <= 0) {
560 existing_db->unread(move(record));
565 // Now read everything in this directory.
566 vector<entry> db_entries;
567 const string path_plus_slash = path.back() == '/' ? path : path + '/';
569 pair<string, dir_time> record = existing_db->read_next();
570 if (record.first.empty()) {
574 if (record.first.rfind(path_plus_slash, 0) != 0) {
575 // No longer starts with path, so we're in a different directory.
576 existing_db->unread(move(record));
579 if (record.first.find_first_of('/', path_plus_slash.size()) != string::npos) {
580 // Entered into a subdirectory of a subdirectory.
581 // Due to our ordering, this also means we're done.
582 existing_db->unread(move(record));
587 e.name = record.first.substr(path_plus_slash.size());
588 e.is_directory = (record.second.sec >= 0);
589 e.db_modified = record.second;
590 db_entries.push_back(e);
594 vector<entry> entries;
595 if (!existing_db->get_error() && db_modified.sec > 0 &&
596 modified.sec == db_modified.sec && modified.nsec == db_modified.nsec) {
597 // Not changed since the last database, so we can replace the readdir()
598 // by reading from the database. (We still need to open and stat everything,
599 // though, but that happens in a later step.)
600 entries = move(db_entries);
602 for (const entry &e : entries) {
603 printf("%s/%s\n", path.c_str(), e.name.c_str());
607 dir = fdopendir(fd); // Takes over ownership of fd.
608 if (dir == nullptr) {
614 while ((de = readdir(dir)) != nullptr) {
615 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
618 if (strlen(de->d_name) == 0) {
619 /* Unfortunately, this does happen, and mere assert() does not give
620 users enough information to complain to the right people. */
621 fprintf(stderr, "file system error: zero-length file name in directory %s", path.c_str());
627 e.is_directory = (de->d_type == DT_DIR);
630 printf("%s/%s\n", path.c_str(), de->d_name);
632 entries.push_back(move(e));
635 sort(entries.begin(), entries.end(), [](const entry &a, const entry &b) {
636 return a.name < b.name;
639 // Load directory modification times from the old database.
640 auto db_it = db_entries.begin();
641 for (entry &e : entries) {
642 for (; db_it != db_entries.end(); ++db_it) {
643 if (e.name < db_it->name) {
646 if (e.name == db_it->name) {
647 e.db_modified = db_it->db_modified;
654 // For each entry, we want to add it to the database. but this includes the modification time
655 // for directories, which means we need to open and stat it at this point.
657 // This means we may need to have many directories open at the same time, but it seems to be
658 // the simplest (only?) way of being compatible with mlocate's notion of listing all contents
659 // of a given directory before recursing, without buffering even more information. Hopefully,
660 // we won't go out of file descriptors here (it could happen if someone has tens of thousands
661 // of subdirectories in a single directory); if so, the admin will need to raise the limit.
662 for (entry &e : entries) {
663 if (!e.is_directory) {
668 if (find(conf_prunenames.begin(), conf_prunenames.end(), e.name) != conf_prunenames.end()) {
669 if (conf_debug_pruning) {
670 /* This is debugging output, don't mark anything for translation */
671 fprintf(stderr, "Skipping `%s': in prunenames\n", e.name.c_str());
676 e.fd = opendir_noatime(fd, e.name.c_str());
678 if (errno == EMFILE || errno == ENFILE) {
679 // The admin probably wants to know about this.
680 perror((path_plus_slash + e.name).c_str());
683 if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) {
684 fprintf(stderr, "Hint: Try `ulimit -n 131072' or similar.\n");
686 fprintf(stderr, "Hint: Try `ulimit -n %lu' or similar (current limit is %lu).\n",
687 rlim.rlim_cur * 2, rlim.rlim_cur);
695 if (fstat(e.fd, &buf) != 0) {
696 perror(path.c_str());
701 if (buf.st_dev != parent_dev) {
702 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
709 e.dt = get_dirtime_from_stat(buf);
712 // Actually add all the entries we figured out dates for above.
713 for (const entry &e : entries) {
714 corpus->add_file(path_plus_slash + e.name, e.dt);
715 dict_builder->add_file(path_plus_slash + e.name, e.dt);
718 // Now scan subdirectories.
719 for (const entry &e : entries) {
720 if (e.is_directory && e.fd != -1) {
721 int ret = scan(path_plus_slash + e.name, e.fd, e.dev, e.dt, e.db_modified, existing_db, corpus, dict_builder);
723 // TODO: The unscanned file descriptors will leak, but it doesn't really matter,
724 // as we're about to exit.
731 if (dir == nullptr) {
739 int main(int argc, char **argv)
741 // We want to bump the file limit; do it if we can (usually we are root
742 // and can set whatever we want). 128k should be ample for most setups.
744 if (getrlimit(RLIMIT_NOFILE, &rlim) != -1) {
745 rlim_t wanted = std::max<rlim_t>(rlim.rlim_cur, 131072);
746 rlim.rlim_cur = std::min<rlim_t>(wanted, rlim.rlim_max);
747 setrlimit(RLIMIT_NOFILE, &rlim); // Ignore errors.
750 conf_prepare(argc, argv);
751 if (conf_prune_bind_mounts) {
752 bind_mount_init(MOUNTINFO_PATH);
755 int fd = open(conf_output.c_str(), O_RDONLY);
756 ExistingDB existing_db(fd);
758 DictionaryBuilder dict_builder(/*blocks_to_keep=*/1000, conf_block_size);
761 if (conf_check_visibility) {
762 group *grp = getgrnam(GROUPNAME);
763 if (grp == nullptr) {
764 fprintf(stderr, "Unknown group %s\n", GROUPNAME);
770 DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary());
771 db.set_conf_block(conf_block);
772 Corpus *corpus = db.start_corpus(/*store_dir_times=*/true);
774 int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
781 if (fstat(root_fd, &buf) == -1) {
786 scan(conf_scan_root, root_fd, buf.st_dev, get_dirtime_from_stat(buf), /*db_modified=*/unknown_dir_time, &existing_db, corpus, &dict_builder);
788 // It's too late to use the dictionary for the data we already compressed,
789 // unless we wanted to either scan the entire file system again (acceptable
790 // for plocate-build where it's cheap, less so for us), or uncompressing
791 // and recompressing. Instead, we store it for next time, assuming that the
792 // data changes fairly little from time to time.
793 string next_dictionary = dict_builder.train(1024);
794 db.set_next_dictionary(next_dictionary);