3 Copyright (C) 2005, 2007, 2008 Red Hat, Inc. All rights reserved.
5 This copyrighted material is made available to anyone wishing to use, modify,
6 copy, or redistribute it subject to the terms and conditions of the GNU General
9 This program is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
11 PARTICULAR PURPOSE. See the GNU General Public License for more details.
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
15 Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 Author: Miloslav Trmac <mitr@redhat.com>
20 plocate modifications: Copyright (C) 2020 Steinar H. Gunderson.
21 plocate parts and modifications are licensed under the GPLv2 or, at your option,
25 #include "bind-mount.h"
26 #include "complete_pread.h"
28 #include "database-builder.h"
31 #include "io_uring_engine.h"
35 #include <arpa/inet.h>
53 #include <sys/resource.h>
56 #include <sys/types.h>
62 using namespace std::chrono;
64 /* Next conf_prunepaths entry */
65 static size_t conf_prunepaths_index; /* = 0; */
70 "Usage: updatedb PLOCATE_DB\n"
72 "Generate plocate index from mlocate.db, typically /var/lib/mlocate/mlocate.db.\n"
73 "Normally, the destination should be /var/lib/mlocate/plocate.db.\n"
75 " -b, --block-size SIZE number of filenames to store in each block (default 32)\n"
76 " -p, --plaintext input is a plaintext file, not an mlocate database\n"
77 " --help print this help\n"
78 " --version print version information\n");
83 printf("updatedb %s\n", PACKAGE_VERSION);
84 printf("Copyright (C) 2007 Red Hat, Inc. All rights reserved.\n");
85 printf("Copyright 2020 Steinar H. Gunderson\n");
86 printf("This software is distributed under the GPL v.2.\n");
88 printf("This program is provided with NO WARRANTY, to the extent permitted by law.\n");
91 int opendir_noatime(int dirfd, const char *path)
93 static bool noatime_failed = false;
95 if (!noatime_failed) {
97 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY | O_NOATIME);
99 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY);
103 } else if (errno == EPERM) {
104 /* EPERM is fairly O_NOATIME-specific; missing access rights cause
106 noatime_failed = true;
112 return openat(dirfd, path, O_RDONLY | O_DIRECTORY);
115 bool time_is_current(const dir_time &t)
117 static dir_time cache{ 0, 0 };
119 /* This is more difficult than it should be because Linux uses a cheaper time
120 source for filesystem timestamps than for gettimeofday() and they can get
121 slightly out of sync, see
122 https://bugzilla.redhat.com/show_bug.cgi?id=244697 . This affects even
123 nanosecond timestamps (and don't forget that tv_nsec existence doesn't
124 guarantee that the underlying filesystem has such resolution - it might be
125 microseconds or even coarser).
127 The worst case is probably FAT timestamps with 2-second resolution
128 (although using such a filesystem violates POSIX file times requirements).
130 So, to be on the safe side, require a >3.0 second difference (2 seconds to
131 make sure the FAT timestamp changed, 1 more to account for the Linux
132 timestamp races). This large margin might make updatedb marginally more
133 expensive, but it only makes a difference if the directory was very
134 recently updated _and_ is will not be updated again until the next
135 updatedb run; this is not likely to happen for most directories. */
137 /* Cache gettimeofday () results to rule out obviously old time stamps;
138 CACHE contains the earliest time we reject as too current. */
144 gettimeofday(&tv, nullptr);
145 cache.sec = tv.tv_sec - 3;
146 cache.nsec = tv.tv_usec * 1000;
155 // For directories only:
157 dir_time dt = unknown_dir_time;
158 dir_time db_modified = unknown_dir_time;
162 bool filesystem_is_excluded(const char *path)
164 if (conf_debug_pruning) {
165 /* This is debugging output, don't mark anything for translation */
166 fprintf(stderr, "Checking whether filesystem `%s' is excluded:\n", path);
168 FILE *f = setmntent("/proc/mounts", "r");
174 while ((me = getmntent(f)) != nullptr) {
175 if (conf_debug_pruning) {
176 /* This is debugging output, don't mark anything for translation */
177 fprintf(stderr, " `%s', type `%s'\n", me->mnt_dir, me->mnt_type);
179 string type(me->mnt_type);
180 for (char &p : type) {
183 if (find(conf_prunefs.begin(), conf_prunefs.end(), type) != conf_prunefs.end()) {
184 /* Paths in /proc/self/mounts contain no symbolic links. Besides
185 avoiding a few system calls, avoiding the realpath () avoids hangs
186 if the filesystem is unavailable hard-mounted NFS. */
187 char *dir = me->mnt_dir;
188 if (conf_debug_pruning) {
189 /* This is debugging output, don't mark anything for translation */
190 fprintf(stderr, " => type matches, dir `%s'\n", dir);
192 bool res = (strcmp(path, dir) == 0);
193 if (dir != me->mnt_dir)
201 if (conf_debug_pruning) {
202 /* This is debugging output, don't mark anything for translation */
203 fprintf(stderr, "...done\n");
209 dir_time get_dirtime_from_stat(const struct stat &buf)
211 dir_time ctime{ buf.st_ctim.tv_sec, int32_t(buf.st_ctim.tv_nsec) };
212 dir_time mtime{ buf.st_mtim.tv_sec, int32_t(buf.st_mtim.tv_nsec) };
213 dir_time dt = max(ctime, mtime);
215 if (time_is_current(dt)) {
216 /* The directory might be changing right now and we can't be sure the
217 timestamp will be changed again if more changes happen very soon, mark
218 the timestamp as invalid to force rescanning the directory next time
220 return unknown_dir_time;
226 // Represents the old database we are updating.
229 explicit ExistingDB(int fd);
232 pair<string, dir_time> read_next();
233 void unread(pair<string, dir_time> record)
235 unread_record = move(record);
237 string read_next_dictionary() const;
238 bool get_error() const { return error; }
244 uint32_t current_docid = 0;
246 string current_filename_block;
247 const char *current_filename_ptr = nullptr, *current_filename_end = nullptr;
249 off_t compressed_dir_time_pos;
250 string compressed_dir_time;
251 string current_dir_time_block;
252 const char *current_dir_time_ptr = nullptr, *current_dir_time_end = nullptr;
254 pair<string, dir_time> unread_record;
256 // Used in one-shot mode, repeatedly.
259 // Used in streaming mode.
260 ZSTD_DCtx *dir_time_ctx;
262 ZSTD_DDict *ddict = nullptr;
264 // If true, we've discovered an error or EOF, and will return only
265 // empty data from here.
266 bool eof = false, error = false;
269 ExistingDB::ExistingDB(int fd)
277 if (!try_complete_pread(fd, &hdr, sizeof(hdr), /*offset=*/0)) {
279 perror("pread(header)");
284 if (memcmp(hdr.magic, "\0plocate", 8) != 0) {
286 fprintf(stderr, "Old database had header mismatch, ignoring.\n");
291 if (hdr.version != 1 || hdr.max_version < 2) {
293 fprintf(stderr, "Old database had version mismatch (version=%d max_version=%d), ignoring.\n",
294 hdr.version, hdr.max_version);
300 // Compare the configuration block with our current one.
301 if (hdr.conf_block_length_bytes != conf_block.size()) {
303 fprintf(stderr, "Old database had different configuration block (size mismatch), ignoring.\n");
309 str.resize(hdr.conf_block_length_bytes);
310 if (!try_complete_pread(fd, str.data(), hdr.conf_block_length_bytes, hdr.conf_block_offset_bytes)) {
312 perror("pread(conf_block)");
317 if (str != conf_block) {
319 fprintf(stderr, "Old database had different configuration block (contents mismatch), ignoring.\n");
325 // Read dictionary, if it exists.
326 if (hdr.zstd_dictionary_length_bytes > 0) {
328 dictionary.resize(hdr.zstd_dictionary_length_bytes);
329 if (try_complete_pread(fd, &dictionary[0], hdr.zstd_dictionary_length_bytes, hdr.zstd_dictionary_offset_bytes)) {
330 ddict = ZSTD_createDDict(dictionary.data(), dictionary.size());
333 perror("pread(dictionary)");
339 compressed_dir_time_pos = hdr.directory_data_offset_bytes;
341 ctx = ZSTD_createDCtx();
342 dir_time_ctx = ZSTD_createDCtx();
345 ExistingDB::~ExistingDB()
352 pair<string, dir_time> ExistingDB::read_next()
354 if (!unread_record.first.empty()) {
355 auto ret = move(unread_record);
356 unread_record.first.clear();
361 return { "", not_a_dir };
364 // See if we need to read a new filename block.
365 if (current_filename_ptr == nullptr) {
366 if (current_docid >= hdr.num_docids) {
368 return { "", not_a_dir };
371 // Read the file offset from this docid and the next one.
372 // This is always allowed, since we have a sentinel block at the end.
373 off_t offset_for_block = hdr.filename_index_offset_bytes + current_docid * sizeof(uint64_t);
375 if (!try_complete_pread(fd, vals, sizeof(vals), offset_for_block)) {
377 perror("pread(offset)");
380 return { "", not_a_dir };
383 off_t offset = vals[0];
384 size_t compressed_len = vals[1] - vals[0];
385 unique_ptr<char[]> compressed(new char[compressed_len]);
386 if (!try_complete_pread(fd, compressed.get(), compressed_len, offset)) {
388 perror("pread(block)");
391 return { "", not_a_dir };
394 unsigned long long uncompressed_len = ZSTD_getFrameContentSize(compressed.get(), compressed_len);
395 if (uncompressed_len == ZSTD_CONTENTSIZE_UNKNOWN || uncompressed_len == ZSTD_CONTENTSIZE_ERROR) {
397 fprintf(stderr, "ZSTD_getFrameContentSize() failed\n");
400 return { "", not_a_dir };
404 block.resize(uncompressed_len + 1);
407 if (ddict != nullptr) {
408 err = ZSTD_decompress_usingDDict(ctx, &block[0], block.size(), compressed.get(),
409 compressed_len, ddict);
411 err = ZSTD_decompressDCtx(ctx, &block[0], block.size(), compressed.get(),
414 if (ZSTD_isError(err)) {
416 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
419 return { "", not_a_dir };
421 block[block.size() - 1] = '\0';
422 current_filename_block = move(block);
423 current_filename_ptr = current_filename_block.data();
424 current_filename_end = current_filename_block.data() + current_filename_block.size();
428 // See if we need to read more directory time data.
429 while (current_dir_time_ptr == current_dir_time_end ||
430 (*current_dir_time_ptr != 0 &&
431 size_t(current_dir_time_end - current_dir_time_ptr) < sizeof(dir_time) + 1)) {
432 if (current_dir_time_ptr != nullptr) {
433 const size_t bytes_consumed = current_dir_time_ptr - current_dir_time_block.data();
434 current_dir_time_block.erase(current_dir_time_block.begin(), current_dir_time_block.begin() + bytes_consumed);
437 // See if we can get more data out without reading more.
438 const size_t existing_data = current_dir_time_block.size();
439 current_dir_time_block.resize(existing_data + 4096);
441 ZSTD_outBuffer outbuf;
442 outbuf.dst = current_dir_time_block.data() + existing_data;
447 inbuf.src = compressed_dir_time.data();
448 inbuf.size = compressed_dir_time.size();
451 int err = ZSTD_decompressStream(dir_time_ctx, &outbuf, &inbuf);
454 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
457 return { "", not_a_dir };
459 compressed_dir_time.erase(compressed_dir_time.begin(), compressed_dir_time.begin() + inbuf.pos);
460 current_dir_time_block.resize(existing_data + outbuf.pos);
462 if (inbuf.pos == 0 && outbuf.pos == 0) {
463 // No movement, we'll need to try to read more data.
465 size_t bytes_to_read = min<size_t>(
466 hdr.directory_data_offset_bytes + hdr.directory_data_length_bytes - compressed_dir_time_pos,
468 if (bytes_to_read == 0) {
470 return { "", not_a_dir };
472 if (!try_complete_pread(fd, buf, bytes_to_read, compressed_dir_time_pos)) {
474 perror("pread(dirtime)");
477 return { "", not_a_dir };
479 compressed_dir_time_pos += bytes_to_read;
480 compressed_dir_time.insert(compressed_dir_time.end(), buf, buf + bytes_to_read);
482 // Next iteration will now try decompressing more.
485 current_dir_time_ptr = current_dir_time_block.data();
486 current_dir_time_end = current_dir_time_block.data() + current_dir_time_block.size();
489 string filename = current_filename_ptr;
490 current_filename_ptr += filename.size() + 1;
491 if (current_filename_ptr == current_filename_end) {
492 // End of this block.
493 current_filename_ptr = nullptr;
496 if (*current_dir_time_ptr == 0) {
497 ++current_dir_time_ptr;
498 return { move(filename), not_a_dir };
500 ++current_dir_time_ptr;
502 memcpy(&dt.sec, current_dir_time_ptr, sizeof(dt.sec));
503 current_dir_time_ptr += sizeof(dt.sec);
504 memcpy(&dt.nsec, current_dir_time_ptr, sizeof(dt.nsec));
505 current_dir_time_ptr += sizeof(dt.nsec);
506 return { move(filename), dt };
510 string ExistingDB::read_next_dictionary() const
512 if (hdr.next_zstd_dictionary_length_bytes == 0 || hdr.next_zstd_dictionary_length_bytes > 1048576) {
516 str.resize(hdr.next_zstd_dictionary_length_bytes);
517 if (!try_complete_pread(fd, str.data(), hdr.next_zstd_dictionary_length_bytes, hdr.next_zstd_dictionary_offset_bytes)) {
519 perror("pread(next_dictionary)");
526 // Scans the directory with absolute path “path”, which is opened as “fd”.
527 // Uses relative paths and openat() only, evading any issues with PATH_MAX
528 // and time-of-check-time-of-use race conditions. (mlocate's updatedb
529 // does a much more complicated dance with changing the current working
530 // directory, probably in the interest of portability to old platforms.)
531 // “parent_dev” must be the device of the parent directory of “path”.
533 // Takes ownership of fd.
534 int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, DatabaseReceiver *corpus, DictionaryBuilder *dict_builder)
536 if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, path)) {
537 if (conf_debug_pruning) {
538 /* This is debugging output, don't mark anything for translation */
539 fprintf(stderr, "Skipping `%s': in prunepaths\n", path.c_str());
544 if (conf_prune_bind_mounts && is_bind_mount(path.c_str())) {
545 if (conf_debug_pruning) {
546 /* This is debugging output, don't mark anything for translation */
547 fprintf(stderr, "Skipping `%s': bind mount\n", path.c_str());
553 // We read in the old directory no matter whether it is current or not,
554 // because even if we're not going to use it, we'll need the modification directory
555 // of any subdirectories.
557 // Skip over anything before this directory; it is stuff that we would have
558 // consumed earlier if we wanted it.
560 pair<string, dir_time> record = existing_db->read_next();
561 if (record.first.empty()) {
564 if (dir_path_cmp(path, record.first) <= 0) {
565 existing_db->unread(move(record));
570 // Now read everything in this directory.
571 vector<entry> db_entries;
572 const string path_plus_slash = path.back() == '/' ? path : path + '/';
574 pair<string, dir_time> record = existing_db->read_next();
575 if (record.first.empty()) {
579 if (record.first.rfind(path_plus_slash, 0) != 0) {
580 // No longer starts with path, so we're in a different directory.
581 existing_db->unread(move(record));
584 if (record.first.find_first_of('/', path_plus_slash.size()) != string::npos) {
585 // Entered into a subdirectory of a subdirectory.
586 // Due to our ordering, this also means we're done.
587 existing_db->unread(move(record));
592 e.name = record.first.substr(path_plus_slash.size());
593 e.is_directory = (record.second.sec >= 0);
594 e.db_modified = record.second;
595 db_entries.push_back(e);
599 vector<entry> entries;
600 if (!existing_db->get_error() && db_modified.sec > 0 &&
601 modified.sec == db_modified.sec && modified.nsec == db_modified.nsec) {
602 // Not changed since the last database, so we can replace the readdir()
603 // by reading from the database. (We still need to open and stat everything,
604 // though, but that happens in a later step.)
605 entries = move(db_entries);
607 for (const entry &e : entries) {
608 printf("%s/%s\n", path.c_str(), e.name.c_str());
612 dir = fdopendir(fd); // Takes over ownership of fd.
613 if (dir == nullptr) {
614 // fdopendir() wants to fstat() the fd to verify that it's indeed
615 // a directory, which can seemingly fail on at least CIFS filesystems
616 // if the server feels like it. We treat this as if we had an error
617 // on opening it, ie., ignore the directory.
623 while ((de = readdir(dir)) != nullptr) {
624 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
627 if (strlen(de->d_name) == 0) {
628 /* Unfortunately, this does happen, and mere assert() does not give
629 users enough information to complain to the right people. */
630 fprintf(stderr, "file system error: zero-length file name in directory %s", path.c_str());
636 if (de->d_type == DT_UNKNOWN) {
637 // Evidently some file systems, like older versions of XFS
638 // (mkfs.xfs -m crc=0 -n ftype=0), can return this,
639 // and we need a stat(). If we wanted to optimize for this,
640 // we could probably defer it to later (we're stat-ing directories
641 // when recursing), but this is rare, and not really worth it --
642 // the second stat() will be cached anyway.
644 if (fstatat(fd, de->d_name, &buf, AT_SYMLINK_NOFOLLOW) == 0 &&
645 S_ISDIR(buf.st_mode)) {
646 e.is_directory = true;
648 e.is_directory = false;
651 e.is_directory = (de->d_type == DT_DIR);
655 printf("%s/%s\n", path.c_str(), de->d_name);
657 entries.push_back(move(e));
660 sort(entries.begin(), entries.end(), [](const entry &a, const entry &b) {
661 return a.name < b.name;
664 // Load directory modification times from the old database.
665 auto db_it = db_entries.begin();
666 for (entry &e : entries) {
667 for (; db_it != db_entries.end(); ++db_it) {
668 if (e.name < db_it->name) {
671 if (e.name == db_it->name) {
672 e.db_modified = db_it->db_modified;
679 // For each entry, we want to add it to the database. but this includes the modification time
680 // for directories, which means we need to open and stat it at this point.
682 // This means we may need to have many directories open at the same time, but it seems to be
683 // the simplest (only?) way of being compatible with mlocate's notion of listing all contents
684 // of a given directory before recursing, without buffering even more information. Hopefully,
685 // we won't go out of file descriptors here (it could happen if someone has tens of thousands
686 // of subdirectories in a single directory); if so, the admin will need to raise the limit.
687 for (entry &e : entries) {
688 if (!e.is_directory) {
693 if (find(conf_prunenames.begin(), conf_prunenames.end(), e.name) != conf_prunenames.end()) {
694 if (conf_debug_pruning) {
695 /* This is debugging output, don't mark anything for translation */
696 fprintf(stderr, "Skipping `%s': in prunenames\n", e.name.c_str());
701 e.fd = opendir_noatime(fd, e.name.c_str());
703 if (errno == EMFILE || errno == ENFILE) {
704 // The admin probably wants to know about this.
705 perror((path_plus_slash + e.name).c_str());
708 if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) {
709 fprintf(stderr, "Hint: Try `ulimit -n 131072' or similar.\n");
711 fprintf(stderr, "Hint: Try `ulimit -n %" PRIu64 " or similar (current limit is %" PRIu64 ").\n",
712 static_cast<uint64_t>(rlim.rlim_cur * 2), static_cast<uint64_t>(rlim.rlim_cur));
720 if (fstat(e.fd, &buf) != 0) {
721 // It's possible that this is a filesystem that's excluded
722 // (and the failure is e.g. because the network is down).
723 // As a last-ditch effort, we try to check that before dying,
724 // i.e., duplicate the check from further down.
726 // It would be better to be able to run filesystem_is_excluded()
727 // for cheap on everything and just avoid the stat, but it seems
728 // hard to do that without any kind of raciness.
729 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
735 perror((path_plus_slash + e.name).c_str());
740 if (buf.st_dev != parent_dev) {
741 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
748 e.dt = get_dirtime_from_stat(buf);
751 // Actually add all the entries we figured out dates for above.
752 for (const entry &e : entries) {
753 corpus->add_file(path_plus_slash + e.name, e.dt);
754 dict_builder->add_file(path_plus_slash + e.name, e.dt);
757 // Now scan subdirectories.
758 for (const entry &e : entries) {
759 if (e.is_directory && e.fd != -1) {
760 int ret = scan(path_plus_slash + e.name, e.fd, e.dev, e.dt, e.db_modified, existing_db, corpus, dict_builder);
762 // TODO: The unscanned file descriptors will leak, but it doesn't really matter,
763 // as we're about to exit.
770 if (dir == nullptr) {
778 int main(int argc, char **argv)
780 // We want to bump the file limit; do it if we can (usually we are root
781 // and can set whatever we want). 128k should be ample for most setups.
783 if (getrlimit(RLIMIT_NOFILE, &rlim) != -1) {
784 // Even root cannot increase rlim_cur beyond rlim_max,
785 // so we need to try to increase rlim_max first.
786 // Ignore errors, though.
787 if (rlim.rlim_max < 131072) {
788 rlim.rlim_max = 131072;
789 setrlimit(RLIMIT_NOFILE, &rlim);
790 getrlimit(RLIMIT_NOFILE, &rlim);
793 rlim_t wanted = std::max<rlim_t>(rlim.rlim_cur, 131072);
794 rlim.rlim_cur = std::min<rlim_t>(wanted, rlim.rlim_max);
795 setrlimit(RLIMIT_NOFILE, &rlim); // Ignore errors.
798 conf_prepare(argc, argv);
799 if (conf_prune_bind_mounts) {
800 bind_mount_init(MOUNTINFO_PATH);
803 int fd = open(conf_output.c_str(), O_RDONLY);
804 ExistingDB existing_db(fd);
806 DictionaryBuilder dict_builder(/*blocks_to_keep=*/1000, conf_block_size);
809 if (conf_check_visibility) {
810 group *grp = getgrnam(GROUPNAME);
811 if (grp == nullptr) {
812 fprintf(stderr, "Unknown group %s\n", GROUPNAME);
818 DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary(), conf_check_visibility);
819 db.set_conf_block(conf_block);
820 DatabaseReceiver *corpus = db.start_corpus(/*store_dir_times=*/true);
822 int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
829 if (fstat(root_fd, &buf) == -1) {
834 scan(conf_scan_root, root_fd, buf.st_dev, get_dirtime_from_stat(buf), /*db_modified=*/unknown_dir_time, &existing_db, corpus, &dict_builder);
836 // It's too late to use the dictionary for the data we already compressed,
837 // unless we wanted to either scan the entire file system again (acceptable
838 // for plocate-build where it's cheap, less so for us), or uncompressing
839 // and recompressing. Instead, we store it for next time, assuming that the
840 // data changes fairly little from time to time.
841 string next_dictionary = dict_builder.train(1024);
842 db.set_next_dictionary(next_dictionary);