3 Copyright (C) 2005, 2007, 2008 Red Hat, Inc. All rights reserved.
5 This copyrighted material is made available to anyone wishing to use, modify,
6 copy, or redistribute it subject to the terms and conditions of the GNU General
9 This program is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
11 PARTICULAR PURPOSE. See the GNU General Public License for more details.
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
15 Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 Author: Miloslav Trmac <mitr@redhat.com>
20 plocate modifications: Copyright (C) 2020 Steinar H. Gunderson.
21 plocate parts and modifications are licensed under the GPLv2 or, at your option,
25 #include "bind-mount.h"
26 #include "complete_pread.h"
28 #include "database-builder.h"
31 #include "io_uring_engine.h"
35 #include <arpa/inet.h>
52 #include <sys/resource.h>
55 #include <sys/types.h>
61 using namespace std::chrono;
63 /* Next conf_prunepaths entry */
64 static size_t conf_prunepaths_index; /* = 0; */
69 "Usage: updatedb PLOCATE_DB\n"
71 "Generate plocate index from mlocate.db, typically /var/lib/mlocate/mlocate.db.\n"
72 "Normally, the destination should be /var/lib/mlocate/plocate.db.\n"
74 " -b, --block-size SIZE number of filenames to store in each block (default 32)\n"
75 " -p, --plaintext input is a plaintext file, not an mlocate database\n"
76 " --help print this help\n"
77 " --version print version information\n");
82 printf("updatedb %s\n", PACKAGE_VERSION);
83 printf("Copyright (C) 2007 Red Hat, Inc. All rights reserved.\n");
84 printf("Copyright 2020 Steinar H. Gunderson\n");
85 printf("This software is distributed under the GPL v.2.\n");
87 printf("This program is provided with NO WARRANTY, to the extent permitted by law.\n");
90 int opendir_noatime(int dirfd, const char *path)
92 static bool noatime_failed = false;
94 if (!noatime_failed) {
96 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY | O_NOATIME);
98 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY);
102 } else if (errno == EPERM) {
103 /* EPERM is fairly O_NOATIME-specific; missing access rights cause
105 noatime_failed = true;
111 return openat(dirfd, path, O_RDONLY | O_DIRECTORY);
114 bool time_is_current(const dir_time &t)
116 static dir_time cache{ 0, 0 };
118 /* This is more difficult than it should be because Linux uses a cheaper time
119 source for filesystem timestamps than for gettimeofday() and they can get
120 slightly out of sync, see
121 https://bugzilla.redhat.com/show_bug.cgi?id=244697 . This affects even
122 nanosecond timestamps (and don't forget that tv_nsec existence doesn't
123 guarantee that the underlying filesystem has such resolution - it might be
124 microseconds or even coarser).
126 The worst case is probably FAT timestamps with 2-second resolution
127 (although using such a filesystem violates POSIX file times requirements).
129 So, to be on the safe side, require a >3.0 second difference (2 seconds to
130 make sure the FAT timestamp changed, 1 more to account for the Linux
131 timestamp races). This large margin might make updatedb marginally more
132 expensive, but it only makes a difference if the directory was very
133 recently updated _and_ is will not be updated again until the next
134 updatedb run; this is not likely to happen for most directories. */
136 /* Cache gettimeofday () results to rule out obviously old time stamps;
137 CACHE contains the earliest time we reject as too current. */
143 gettimeofday(&tv, nullptr);
144 cache.sec = tv.tv_sec - 3;
145 cache.nsec = tv.tv_usec * 1000;
154 // For directories only:
156 dir_time dt = unknown_dir_time;
157 dir_time db_modified = unknown_dir_time;
161 bool filesystem_is_excluded(const char *path)
163 if (conf_debug_pruning) {
164 /* This is debugging output, don't mark anything for translation */
165 fprintf(stderr, "Checking whether filesystem `%s' is excluded:\n", path);
167 FILE *f = setmntent("/proc/mounts", "r");
173 while ((me = getmntent(f)) != nullptr) {
174 if (conf_debug_pruning) {
175 /* This is debugging output, don't mark anything for translation */
176 fprintf(stderr, " `%s', type `%s'\n", me->mnt_dir, me->mnt_type);
178 string type(me->mnt_type);
179 for (char &p : type) {
182 if (find(conf_prunefs.begin(), conf_prunefs.end(), type) != conf_prunefs.end()) {
183 /* Paths in /proc/self/mounts contain no symbolic links. Besides
184 avoiding a few system calls, avoiding the realpath () avoids hangs
185 if the filesystem is unavailable hard-mounted NFS. */
186 char *dir = me->mnt_dir;
187 if (conf_debug_pruning) {
188 /* This is debugging output, don't mark anything for translation */
189 fprintf(stderr, " => type matches, dir `%s'\n", dir);
191 bool res = (strcmp(path, dir) == 0);
192 if (dir != me->mnt_dir)
200 if (conf_debug_pruning) {
201 /* This is debugging output, don't mark anything for translation */
202 fprintf(stderr, "...done\n");
208 dir_time get_dirtime_from_stat(const struct stat &buf)
210 dir_time ctime{ buf.st_ctim.tv_sec, int32_t(buf.st_ctim.tv_nsec) };
211 dir_time mtime{ buf.st_mtim.tv_sec, int32_t(buf.st_mtim.tv_nsec) };
212 dir_time dt = max(ctime, mtime);
214 if (time_is_current(dt)) {
215 /* The directory might be changing right now and we can't be sure the
216 timestamp will be changed again if more changes happen very soon, mark
217 the timestamp as invalid to force rescanning the directory next time
219 return unknown_dir_time;
225 // Represents the old database we are updating.
228 explicit ExistingDB(int fd);
231 pair<string, dir_time> read_next();
232 void unread(pair<string, dir_time> record)
234 unread_record = move(record);
236 string read_next_dictionary() const;
237 bool get_error() const { return error; }
243 uint32_t current_docid = 0;
245 string current_filename_block;
246 const char *current_filename_ptr = nullptr, *current_filename_end = nullptr;
248 off_t compressed_dir_time_pos;
249 string compressed_dir_time;
250 string current_dir_time_block;
251 const char *current_dir_time_ptr = nullptr, *current_dir_time_end = nullptr;
253 pair<string, dir_time> unread_record;
255 // Used in one-shot mode, repeatedly.
258 // Used in streaming mode.
259 ZSTD_DCtx *dir_time_ctx;
261 ZSTD_DDict *ddict = nullptr;
263 // If true, we've discovered an error or EOF, and will return only
264 // empty data from here.
265 bool eof = false, error = false;
268 ExistingDB::ExistingDB(int fd)
276 if (!try_complete_pread(fd, &hdr, sizeof(hdr), /*offset=*/0)) {
278 perror("pread(header)");
283 if (memcmp(hdr.magic, "\0plocate", 8) != 0) {
285 fprintf(stderr, "Old database had header mismatch, ignoring.\n");
290 if (hdr.version != 1 || hdr.max_version < 2) {
292 fprintf(stderr, "Old database had version mismatch (version=%d max_version=%d), ignoring.\n",
293 hdr.version, hdr.max_version);
299 // Compare the configuration block with our current one.
300 if (hdr.conf_block_length_bytes != conf_block.size()) {
302 fprintf(stderr, "Old database had different configuration block (size mismatch), ignoring.\n");
308 str.resize(hdr.conf_block_length_bytes);
309 if (!try_complete_pread(fd, str.data(), hdr.conf_block_length_bytes, hdr.conf_block_offset_bytes)) {
311 perror("pread(conf_block)");
316 if (str != conf_block) {
318 fprintf(stderr, "Old database had different configuration block (contents mismatch), ignoring.\n");
324 // Read dictionary, if it exists.
325 if (hdr.zstd_dictionary_length_bytes > 0) {
327 dictionary.resize(hdr.zstd_dictionary_length_bytes);
328 if (try_complete_pread(fd, &dictionary[0], hdr.zstd_dictionary_length_bytes, hdr.zstd_dictionary_offset_bytes)) {
329 ddict = ZSTD_createDDict(dictionary.data(), dictionary.size());
332 perror("pread(dictionary)");
338 compressed_dir_time_pos = hdr.directory_data_offset_bytes;
340 ctx = ZSTD_createDCtx();
341 dir_time_ctx = ZSTD_createDCtx();
344 ExistingDB::~ExistingDB()
351 pair<string, dir_time> ExistingDB::read_next()
353 if (!unread_record.first.empty()) {
354 auto ret = move(unread_record);
355 unread_record.first.clear();
360 return { "", not_a_dir };
363 // See if we need to read a new filename block.
364 if (current_filename_ptr == nullptr) {
365 if (current_docid >= hdr.num_docids) {
367 return { "", not_a_dir };
370 // Read the file offset from this docid and the next one.
371 // This is always allowed, since we have a sentinel block at the end.
372 off_t offset_for_block = hdr.filename_index_offset_bytes + current_docid * sizeof(uint64_t);
374 if (!try_complete_pread(fd, vals, sizeof(vals), offset_for_block)) {
376 perror("pread(offset)");
379 return { "", not_a_dir };
382 off_t offset = vals[0];
383 size_t compressed_len = vals[1] - vals[0];
384 unique_ptr<char[]> compressed(new char[compressed_len]);
385 if (!try_complete_pread(fd, compressed.get(), compressed_len, offset)) {
387 perror("pread(block)");
390 return { "", not_a_dir };
393 unsigned long long uncompressed_len = ZSTD_getFrameContentSize(compressed.get(), compressed_len);
394 if (uncompressed_len == ZSTD_CONTENTSIZE_UNKNOWN || uncompressed_len == ZSTD_CONTENTSIZE_ERROR) {
396 fprintf(stderr, "ZSTD_getFrameContentSize() failed\n");
399 return { "", not_a_dir };
403 block.resize(uncompressed_len + 1);
406 if (ddict != nullptr) {
407 err = ZSTD_decompress_usingDDict(ctx, &block[0], block.size(), compressed.get(),
408 compressed_len, ddict);
410 err = ZSTD_decompressDCtx(ctx, &block[0], block.size(), compressed.get(),
413 if (ZSTD_isError(err)) {
415 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
418 return { "", not_a_dir };
420 block[block.size() - 1] = '\0';
421 current_filename_block = move(block);
422 current_filename_ptr = current_filename_block.data();
423 current_filename_end = current_filename_block.data() + current_filename_block.size();
427 // See if we need to read more directory time data.
428 while (current_dir_time_ptr == current_dir_time_end ||
429 (*current_dir_time_ptr != 0 &&
430 size_t(current_dir_time_end - current_dir_time_ptr) < sizeof(dir_time) + 1)) {
431 if (current_dir_time_ptr != nullptr) {
432 const size_t bytes_consumed = current_dir_time_ptr - current_dir_time_block.data();
433 current_dir_time_block.erase(current_dir_time_block.begin(), current_dir_time_block.begin() + bytes_consumed);
436 // See if we can get more data out without reading more.
437 const size_t existing_data = current_dir_time_block.size();
438 current_dir_time_block.resize(existing_data + 4096);
440 ZSTD_outBuffer outbuf;
441 outbuf.dst = current_dir_time_block.data() + existing_data;
446 inbuf.src = compressed_dir_time.data();
447 inbuf.size = compressed_dir_time.size();
450 int err = ZSTD_decompressStream(dir_time_ctx, &outbuf, &inbuf);
453 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
456 return { "", not_a_dir };
458 compressed_dir_time.erase(compressed_dir_time.begin(), compressed_dir_time.begin() + inbuf.pos);
459 current_dir_time_block.resize(existing_data + outbuf.pos);
461 if (inbuf.pos == 0 && outbuf.pos == 0) {
462 // No movement, we'll need to try to read more data.
464 size_t bytes_to_read = min<size_t>(
465 hdr.directory_data_offset_bytes + hdr.directory_data_length_bytes - compressed_dir_time_pos,
467 if (bytes_to_read == 0) {
469 return { "", not_a_dir };
471 if (!try_complete_pread(fd, buf, bytes_to_read, compressed_dir_time_pos)) {
473 perror("pread(dirtime)");
476 return { "", not_a_dir };
478 compressed_dir_time_pos += bytes_to_read;
479 compressed_dir_time.insert(compressed_dir_time.end(), buf, buf + bytes_to_read);
481 // Next iteration will now try decompressing more.
484 current_dir_time_ptr = current_dir_time_block.data();
485 current_dir_time_end = current_dir_time_block.data() + current_dir_time_block.size();
488 string filename = current_filename_ptr;
489 current_filename_ptr += filename.size() + 1;
490 if (current_filename_ptr == current_filename_end) {
491 // End of this block.
492 current_filename_ptr = nullptr;
495 if (*current_dir_time_ptr == 0) {
496 ++current_dir_time_ptr;
497 return { move(filename), not_a_dir };
499 ++current_dir_time_ptr;
501 memcpy(&dt.sec, current_dir_time_ptr, sizeof(dt.sec));
502 current_dir_time_ptr += sizeof(dt.sec);
503 memcpy(&dt.nsec, current_dir_time_ptr, sizeof(dt.nsec));
504 current_dir_time_ptr += sizeof(dt.nsec);
505 return { move(filename), dt };
509 string ExistingDB::read_next_dictionary() const
511 if (hdr.next_zstd_dictionary_length_bytes == 0 || hdr.next_zstd_dictionary_length_bytes > 1048576) {
515 str.resize(hdr.next_zstd_dictionary_length_bytes);
516 if (!try_complete_pread(fd, str.data(), hdr.next_zstd_dictionary_length_bytes, hdr.next_zstd_dictionary_offset_bytes)) {
518 perror("pread(next_dictionary)");
525 // Scans the directory with absolute path “path”, which is opened as “fd”.
526 // Uses relative paths and openat() only, evading any issues with PATH_MAX
527 // and time-of-check-time-of-use race conditions. (mlocate's updatedb
528 // does a much more complicated dance with changing the current working
529 // directory, probably in the interest of portability to old platforms.)
530 // “parent_dev” must be the device of the parent directory of “path”.
532 // Takes ownership of fd.
533 int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, Corpus *corpus, DictionaryBuilder *dict_builder)
535 if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, path)) {
536 if (conf_debug_pruning) {
537 /* This is debugging output, don't mark anything for translation */
538 fprintf(stderr, "Skipping `%s': in prunepaths\n", path.c_str());
543 if (conf_prune_bind_mounts && is_bind_mount(path.c_str())) {
544 if (conf_debug_pruning) {
545 /* This is debugging output, don't mark anything for translation */
546 fprintf(stderr, "Skipping `%s': bind mount\n", path.c_str());
552 // We read in the old directory no matter whether it is current or not,
553 // because even if we're not going to use it, we'll need the modification directory
554 // of any subdirectories.
556 // Skip over anything before this directory; it is stuff that we would have
557 // consumed earlier if we wanted it.
559 pair<string, dir_time> record = existing_db->read_next();
560 if (record.first.empty()) {
563 if (dir_path_cmp(path, record.first) <= 0) {
564 existing_db->unread(move(record));
569 // Now read everything in this directory.
570 vector<entry> db_entries;
571 const string path_plus_slash = path.back() == '/' ? path : path + '/';
573 pair<string, dir_time> record = existing_db->read_next();
574 if (record.first.empty()) {
578 if (record.first.rfind(path_plus_slash, 0) != 0) {
579 // No longer starts with path, so we're in a different directory.
580 existing_db->unread(move(record));
583 if (record.first.find_first_of('/', path_plus_slash.size()) != string::npos) {
584 // Entered into a subdirectory of a subdirectory.
585 // Due to our ordering, this also means we're done.
586 existing_db->unread(move(record));
591 e.name = record.first.substr(path_plus_slash.size());
592 e.is_directory = (record.second.sec >= 0);
593 e.db_modified = record.second;
594 db_entries.push_back(e);
598 vector<entry> entries;
599 if (!existing_db->get_error() && db_modified.sec > 0 &&
600 modified.sec == db_modified.sec && modified.nsec == db_modified.nsec) {
601 // Not changed since the last database, so we can replace the readdir()
602 // by reading from the database. (We still need to open and stat everything,
603 // though, but that happens in a later step.)
604 entries = move(db_entries);
606 for (const entry &e : entries) {
607 printf("%s/%s\n", path.c_str(), e.name.c_str());
611 dir = fdopendir(fd); // Takes over ownership of fd.
612 if (dir == nullptr) {
618 while ((de = readdir(dir)) != nullptr) {
619 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
622 if (strlen(de->d_name) == 0) {
623 /* Unfortunately, this does happen, and mere assert() does not give
624 users enough information to complain to the right people. */
625 fprintf(stderr, "file system error: zero-length file name in directory %s", path.c_str());
631 e.is_directory = (de->d_type == DT_DIR);
634 printf("%s/%s\n", path.c_str(), de->d_name);
636 entries.push_back(move(e));
639 sort(entries.begin(), entries.end(), [](const entry &a, const entry &b) {
640 return a.name < b.name;
643 // Load directory modification times from the old database.
644 auto db_it = db_entries.begin();
645 for (entry &e : entries) {
646 for (; db_it != db_entries.end(); ++db_it) {
647 if (e.name < db_it->name) {
650 if (e.name == db_it->name) {
651 e.db_modified = db_it->db_modified;
658 // For each entry, we want to add it to the database. but this includes the modification time
659 // for directories, which means we need to open and stat it at this point.
661 // This means we may need to have many directories open at the same time, but it seems to be
662 // the simplest (only?) way of being compatible with mlocate's notion of listing all contents
663 // of a given directory before recursing, without buffering even more information. Hopefully,
664 // we won't go out of file descriptors here (it could happen if someone has tens of thousands
665 // of subdirectories in a single directory); if so, the admin will need to raise the limit.
666 for (entry &e : entries) {
667 if (!e.is_directory) {
672 if (find(conf_prunenames.begin(), conf_prunenames.end(), e.name) != conf_prunenames.end()) {
673 if (conf_debug_pruning) {
674 /* This is debugging output, don't mark anything for translation */
675 fprintf(stderr, "Skipping `%s': in prunenames\n", e.name.c_str());
680 e.fd = opendir_noatime(fd, e.name.c_str());
682 if (errno == EMFILE || errno == ENFILE) {
683 // The admin probably wants to know about this.
684 perror((path_plus_slash + e.name).c_str());
687 if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) {
688 fprintf(stderr, "Hint: Try `ulimit -n 131072' or similar.\n");
690 fprintf(stderr, "Hint: Try `ulimit -n %lu' or similar (current limit is %lu).\n",
691 rlim.rlim_cur * 2, rlim.rlim_cur);
699 if (fstat(e.fd, &buf) != 0) {
700 perror(path.c_str());
705 if (buf.st_dev != parent_dev) {
706 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
713 e.dt = get_dirtime_from_stat(buf);
716 // Actually add all the entries we figured out dates for above.
717 for (const entry &e : entries) {
718 corpus->add_file(path_plus_slash + e.name, e.dt);
719 dict_builder->add_file(path_plus_slash + e.name, e.dt);
722 // Now scan subdirectories.
723 for (const entry &e : entries) {
724 if (e.is_directory && e.fd != -1) {
725 int ret = scan(path_plus_slash + e.name, e.fd, e.dev, e.dt, e.db_modified, existing_db, corpus, dict_builder);
727 // TODO: The unscanned file descriptors will leak, but it doesn't really matter,
728 // as we're about to exit.
735 if (dir == nullptr) {
743 int main(int argc, char **argv)
745 // We want to bump the file limit; do it if we can (usually we are root
746 // and can set whatever we want). 128k should be ample for most setups.
748 if (getrlimit(RLIMIT_NOFILE, &rlim) != -1) {
749 // Even root cannot increase rlim_cur beyond rlim_max,
750 // so we need to try to increase rlim_max first.
751 // Ignore errors, though.
752 if (rlim.rlim_max < 131072) {
753 rlim.rlim_max = 131072;
754 setrlimit(RLIMIT_NOFILE, &rlim);
755 getrlimit(RLIMIT_NOFILE, &rlim);
758 rlim_t wanted = std::max<rlim_t>(rlim.rlim_cur, 131072);
759 rlim.rlim_cur = std::min<rlim_t>(wanted, rlim.rlim_max);
760 setrlimit(RLIMIT_NOFILE, &rlim); // Ignore errors.
763 conf_prepare(argc, argv);
764 if (conf_prune_bind_mounts) {
765 bind_mount_init(MOUNTINFO_PATH);
768 int fd = open(conf_output.c_str(), O_RDONLY);
769 ExistingDB existing_db(fd);
771 DictionaryBuilder dict_builder(/*blocks_to_keep=*/1000, conf_block_size);
774 if (conf_check_visibility) {
775 group *grp = getgrnam(GROUPNAME);
776 if (grp == nullptr) {
777 fprintf(stderr, "Unknown group %s\n", GROUPNAME);
783 DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary(), conf_check_visibility);
784 db.set_conf_block(conf_block);
785 Corpus *corpus = db.start_corpus(/*store_dir_times=*/true);
787 int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
794 if (fstat(root_fd, &buf) == -1) {
799 scan(conf_scan_root, root_fd, buf.st_dev, get_dirtime_from_stat(buf), /*db_modified=*/unknown_dir_time, &existing_db, corpus, &dict_builder);
801 // It's too late to use the dictionary for the data we already compressed,
802 // unless we wanted to either scan the entire file system again (acceptable
803 // for plocate-build where it's cheap, less so for us), or uncompressing
804 // and recompressing. Instead, we store it for next time, assuming that the
805 // data changes fairly little from time to time.
806 string next_dictionary = dict_builder.train(1024);
807 db.set_next_dictionary(next_dictionary);