3 Copyright (C) 2005, 2007, 2008 Red Hat, Inc. All rights reserved.
5 This copyrighted material is made available to anyone wishing to use, modify,
6 copy, or redistribute it subject to the terms and conditions of the GNU General
9 This program is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
11 PARTICULAR PURPOSE. See the GNU General Public License for more details.
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
15 Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 Author: Miloslav Trmac <mitr@redhat.com>
20 plocate modifications: Copyright (C) 2020 Steinar H. Gunderson.
21 plocate parts and modifications are licensed under the GPLv2 or, at your option,
25 #include "bind-mount.h"
26 #include "complete_pread.h"
28 #include "database-builder.h"
31 #include "io_uring_engine.h"
35 #include <arpa/inet.h>
53 #include <sys/resource.h>
56 #include <sys/types.h>
62 using namespace std::chrono;
64 /* Next conf_prunepaths entry */
65 static size_t conf_prunepaths_index; /* = 0; */
70 "Usage: updatedb PLOCATE_DB\n"
72 "Generate plocate index from mlocate.db, typically /var/lib/mlocate/mlocate.db.\n"
73 "Normally, the destination should be /var/lib/mlocate/plocate.db.\n"
75 " -b, --block-size SIZE number of filenames to store in each block (default 32)\n"
76 " -p, --plaintext input is a plaintext file, not an mlocate database\n"
77 " --help print this help\n"
78 " --version print version information\n");
83 printf("updatedb %s\n", PACKAGE_VERSION);
84 printf("Copyright (C) 2007 Red Hat, Inc. All rights reserved.\n");
85 printf("Copyright 2020 Steinar H. Gunderson\n");
86 printf("This software is distributed under the GPL v.2.\n");
88 printf("This program is provided with NO WARRANTY, to the extent permitted by law.\n");
91 int opendir_noatime(int dirfd, const char *path)
93 static bool noatime_failed = false;
95 if (!noatime_failed) {
97 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY | O_NOATIME);
99 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY);
103 } else if (errno == EPERM) {
104 /* EPERM is fairly O_NOATIME-specific; missing access rights cause
106 noatime_failed = true;
112 return openat(dirfd, path, O_RDONLY | O_DIRECTORY);
115 bool time_is_current(const dir_time &t)
117 static dir_time cache{ 0, 0 };
119 /* This is more difficult than it should be because Linux uses a cheaper time
120 source for filesystem timestamps than for gettimeofday() and they can get
121 slightly out of sync, see
122 https://bugzilla.redhat.com/show_bug.cgi?id=244697 . This affects even
123 nanosecond timestamps (and don't forget that tv_nsec existence doesn't
124 guarantee that the underlying filesystem has such resolution - it might be
125 microseconds or even coarser).
127 The worst case is probably FAT timestamps with 2-second resolution
128 (although using such a filesystem violates POSIX file times requirements).
130 So, to be on the safe side, require a >3.0 second difference (2 seconds to
131 make sure the FAT timestamp changed, 1 more to account for the Linux
132 timestamp races). This large margin might make updatedb marginally more
133 expensive, but it only makes a difference if the directory was very
134 recently updated _and_ is will not be updated again until the next
135 updatedb run; this is not likely to happen for most directories. */
137 /* Cache gettimeofday () results to rule out obviously old time stamps;
138 CACHE contains the earliest time we reject as too current. */
144 gettimeofday(&tv, nullptr);
145 cache.sec = tv.tv_sec - 3;
146 cache.nsec = tv.tv_usec * 1000;
155 // For directories only:
157 dir_time dt = unknown_dir_time;
158 dir_time db_modified = unknown_dir_time;
162 bool filesystem_is_excluded(const char *path)
164 if (conf_debug_pruning) {
165 fprintf(stderr, "Checking whether filesystem `%s' is excluded:\n", path);
167 FILE *f = setmntent("/proc/mounts", "r");
173 while ((me = getmntent(f)) != nullptr) {
174 if (conf_debug_pruning) {
175 fprintf(stderr, " `%s', type `%s'\n", me->mnt_dir, me->mnt_type);
177 string type(me->mnt_type);
178 for (char &p : type) {
181 if (find(conf_prunefs.begin(), conf_prunefs.end(), type) != conf_prunefs.end()) {
182 /* Paths in /proc/self/mounts contain no symbolic links. Besides
183 avoiding a few system calls, avoiding the realpath () avoids hangs
184 if the filesystem is unavailable hard-mounted NFS. */
185 char *dir = me->mnt_dir;
186 if (conf_debug_pruning) {
187 fprintf(stderr, " => type matches, dir `%s'\n", dir);
189 bool res = (strcmp(path, dir) == 0);
190 if (dir != me->mnt_dir)
198 if (conf_debug_pruning) {
199 fprintf(stderr, "...done\n");
205 dir_time get_dirtime_from_stat(const struct stat &buf)
207 dir_time ctime{ buf.st_ctim.tv_sec, int32_t(buf.st_ctim.tv_nsec) };
208 dir_time mtime{ buf.st_mtim.tv_sec, int32_t(buf.st_mtim.tv_nsec) };
209 dir_time dt = max(ctime, mtime);
211 if (time_is_current(dt)) {
212 /* The directory might be changing right now and we can't be sure the
213 timestamp will be changed again if more changes happen very soon, mark
214 the timestamp as invalid to force rescanning the directory next time
216 return unknown_dir_time;
222 // Represents the old database we are updating.
225 explicit ExistingDB(int fd);
228 pair<string, dir_time> read_next();
229 void unread(pair<string, dir_time> record)
231 unread_record = move(record);
233 string read_next_dictionary() const;
234 bool get_error() const { return error; }
240 uint32_t current_docid = 0;
242 string current_filename_block;
243 const char *current_filename_ptr = nullptr, *current_filename_end = nullptr;
245 off_t compressed_dir_time_pos;
246 string compressed_dir_time;
247 string current_dir_time_block;
248 const char *current_dir_time_ptr = nullptr, *current_dir_time_end = nullptr;
250 pair<string, dir_time> unread_record;
252 // Used in one-shot mode, repeatedly.
255 // Used in streaming mode.
256 ZSTD_DCtx *dir_time_ctx;
258 ZSTD_DDict *ddict = nullptr;
260 // If true, we've discovered an error or EOF, and will return only
261 // empty data from here.
262 bool eof = false, error = false;
265 ExistingDB::ExistingDB(int fd)
273 if (!try_complete_pread(fd, &hdr, sizeof(hdr), /*offset=*/0)) {
275 perror("pread(header)");
280 if (memcmp(hdr.magic, "\0plocate", 8) != 0) {
282 fprintf(stderr, "Old database had header mismatch, ignoring.\n");
287 if (hdr.version != 1 || hdr.max_version < 2) {
289 fprintf(stderr, "Old database had version mismatch (version=%d max_version=%d), ignoring.\n",
290 hdr.version, hdr.max_version);
296 // Compare the configuration block with our current one.
297 if (hdr.conf_block_length_bytes != conf_block.size()) {
299 fprintf(stderr, "Old database had different configuration block (size mismatch), ignoring.\n");
305 str.resize(hdr.conf_block_length_bytes);
306 if (!try_complete_pread(fd, str.data(), hdr.conf_block_length_bytes, hdr.conf_block_offset_bytes)) {
308 perror("pread(conf_block)");
313 if (str != conf_block) {
315 fprintf(stderr, "Old database had different configuration block (contents mismatch), ignoring.\n");
321 // Read dictionary, if it exists.
322 if (hdr.zstd_dictionary_length_bytes > 0) {
324 dictionary.resize(hdr.zstd_dictionary_length_bytes);
325 if (try_complete_pread(fd, &dictionary[0], hdr.zstd_dictionary_length_bytes, hdr.zstd_dictionary_offset_bytes)) {
326 ddict = ZSTD_createDDict(dictionary.data(), dictionary.size());
329 perror("pread(dictionary)");
335 compressed_dir_time_pos = hdr.directory_data_offset_bytes;
337 ctx = ZSTD_createDCtx();
338 dir_time_ctx = ZSTD_createDCtx();
341 ExistingDB::~ExistingDB()
348 pair<string, dir_time> ExistingDB::read_next()
350 if (!unread_record.first.empty()) {
351 auto ret = move(unread_record);
352 unread_record.first.clear();
357 return { "", not_a_dir };
360 // See if we need to read a new filename block.
361 if (current_filename_ptr == nullptr) {
362 if (current_docid >= hdr.num_docids) {
364 return { "", not_a_dir };
367 // Read the file offset from this docid and the next one.
368 // This is always allowed, since we have a sentinel block at the end.
369 off_t offset_for_block = hdr.filename_index_offset_bytes + current_docid * sizeof(uint64_t);
371 if (!try_complete_pread(fd, vals, sizeof(vals), offset_for_block)) {
373 perror("pread(offset)");
376 return { "", not_a_dir };
379 off_t offset = vals[0];
380 size_t compressed_len = vals[1] - vals[0];
381 unique_ptr<char[]> compressed(new char[compressed_len]);
382 if (!try_complete_pread(fd, compressed.get(), compressed_len, offset)) {
384 perror("pread(block)");
387 return { "", not_a_dir };
390 unsigned long long uncompressed_len = ZSTD_getFrameContentSize(compressed.get(), compressed_len);
391 if (uncompressed_len == ZSTD_CONTENTSIZE_UNKNOWN || uncompressed_len == ZSTD_CONTENTSIZE_ERROR) {
393 fprintf(stderr, "ZSTD_getFrameContentSize() failed\n");
396 return { "", not_a_dir };
400 block.resize(uncompressed_len + 1);
403 if (ddict != nullptr) {
404 err = ZSTD_decompress_usingDDict(ctx, &block[0], block.size(), compressed.get(),
405 compressed_len, ddict);
407 err = ZSTD_decompressDCtx(ctx, &block[0], block.size(), compressed.get(),
410 if (ZSTD_isError(err)) {
412 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
415 return { "", not_a_dir };
417 block[block.size() - 1] = '\0';
418 current_filename_block = move(block);
419 current_filename_ptr = current_filename_block.data();
420 current_filename_end = current_filename_block.data() + current_filename_block.size();
424 // See if we need to read more directory time data.
425 while (current_dir_time_ptr == current_dir_time_end ||
426 (*current_dir_time_ptr != 0 &&
427 size_t(current_dir_time_end - current_dir_time_ptr) < sizeof(dir_time) + 1)) {
428 if (current_dir_time_ptr != nullptr) {
429 const size_t bytes_consumed = current_dir_time_ptr - current_dir_time_block.data();
430 current_dir_time_block.erase(current_dir_time_block.begin(), current_dir_time_block.begin() + bytes_consumed);
433 // See if we can get more data out without reading more.
434 const size_t existing_data = current_dir_time_block.size();
435 current_dir_time_block.resize(existing_data + 4096);
437 ZSTD_outBuffer outbuf;
438 outbuf.dst = current_dir_time_block.data() + existing_data;
443 inbuf.src = compressed_dir_time.data();
444 inbuf.size = compressed_dir_time.size();
447 int err = ZSTD_decompressStream(dir_time_ctx, &outbuf, &inbuf);
450 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
453 return { "", not_a_dir };
455 compressed_dir_time.erase(compressed_dir_time.begin(), compressed_dir_time.begin() + inbuf.pos);
456 current_dir_time_block.resize(existing_data + outbuf.pos);
458 if (inbuf.pos == 0 && outbuf.pos == 0) {
459 // No movement, we'll need to try to read more data.
461 size_t bytes_to_read = min<size_t>(
462 hdr.directory_data_offset_bytes + hdr.directory_data_length_bytes - compressed_dir_time_pos,
464 if (bytes_to_read == 0) {
466 return { "", not_a_dir };
468 if (!try_complete_pread(fd, buf, bytes_to_read, compressed_dir_time_pos)) {
470 perror("pread(dirtime)");
473 return { "", not_a_dir };
475 compressed_dir_time_pos += bytes_to_read;
476 compressed_dir_time.insert(compressed_dir_time.end(), buf, buf + bytes_to_read);
478 // Next iteration will now try decompressing more.
481 current_dir_time_ptr = current_dir_time_block.data();
482 current_dir_time_end = current_dir_time_block.data() + current_dir_time_block.size();
485 string filename = current_filename_ptr;
486 current_filename_ptr += filename.size() + 1;
487 if (current_filename_ptr == current_filename_end) {
488 // End of this block.
489 current_filename_ptr = nullptr;
492 if (*current_dir_time_ptr == 0) {
493 ++current_dir_time_ptr;
494 return { move(filename), not_a_dir };
496 ++current_dir_time_ptr;
498 memcpy(&dt.sec, current_dir_time_ptr, sizeof(dt.sec));
499 current_dir_time_ptr += sizeof(dt.sec);
500 memcpy(&dt.nsec, current_dir_time_ptr, sizeof(dt.nsec));
501 current_dir_time_ptr += sizeof(dt.nsec);
502 return { move(filename), dt };
506 string ExistingDB::read_next_dictionary() const
508 if (hdr.next_zstd_dictionary_length_bytes == 0 || hdr.next_zstd_dictionary_length_bytes > 1048576) {
512 str.resize(hdr.next_zstd_dictionary_length_bytes);
513 if (!try_complete_pread(fd, str.data(), hdr.next_zstd_dictionary_length_bytes, hdr.next_zstd_dictionary_offset_bytes)) {
515 perror("pread(next_dictionary)");
522 // Scans the directory with absolute path “path”, which is opened as “fd”.
523 // Uses relative paths and openat() only, evading any issues with PATH_MAX
524 // and time-of-check-time-of-use race conditions. (mlocate's updatedb
525 // does a much more complicated dance with changing the current working
526 // directory, probably in the interest of portability to old platforms.)
527 // “parent_dev” must be the device of the parent directory of “path”.
529 // Takes ownership of fd.
530 int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, DatabaseReceiver *corpus, DictionaryBuilder *dict_builder)
532 if (conf_prune_bind_mounts && is_bind_mount(path.c_str())) {
533 if (conf_debug_pruning) {
534 fprintf(stderr, "Skipping `%s': bind mount\n", path.c_str());
540 // We read in the old directory no matter whether it is current or not,
541 // because even if we're not going to use it, we'll need the modification directory
542 // of any subdirectories.
544 // Skip over anything before this directory; it is stuff that we would have
545 // consumed earlier if we wanted it.
547 pair<string, dir_time> record = existing_db->read_next();
548 if (record.first.empty()) {
551 if (dir_path_cmp(path, record.first) <= 0) {
552 existing_db->unread(move(record));
557 // Now read everything in this directory.
558 vector<entry> db_entries;
559 const string path_plus_slash = path.back() == '/' ? path : path + '/';
561 pair<string, dir_time> record = existing_db->read_next();
562 if (record.first.empty()) {
566 if (record.first.rfind(path_plus_slash, 0) != 0) {
567 // No longer starts with path, so we're in a different directory.
568 existing_db->unread(move(record));
571 if (record.first.find_first_of('/', path_plus_slash.size()) != string::npos) {
572 // Entered into a subdirectory of a subdirectory.
573 // Due to our ordering, this also means we're done.
574 existing_db->unread(move(record));
579 e.name = record.first.substr(path_plus_slash.size());
580 e.is_directory = (record.second.sec >= 0);
581 e.db_modified = record.second;
582 db_entries.push_back(e);
586 vector<entry> entries;
587 if (!existing_db->get_error() && db_modified.sec > 0 &&
588 modified.sec == db_modified.sec && modified.nsec == db_modified.nsec) {
589 // Not changed since the last database, so we can replace the readdir()
590 // by reading from the database. (We still need to open and stat everything,
591 // though, but that happens in a later step.)
592 entries = move(db_entries);
594 for (const entry &e : entries) {
595 printf("%s/%s\n", path.c_str(), e.name.c_str());
599 dir = fdopendir(fd); // Takes over ownership of fd.
600 if (dir == nullptr) {
601 // fdopendir() wants to fstat() the fd to verify that it's indeed
602 // a directory, which can seemingly fail on at least CIFS filesystems
603 // if the server feels like it. We treat this as if we had an error
604 // on opening it, ie., ignore the directory.
610 while ((de = readdir(dir)) != nullptr) {
611 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
614 if (strlen(de->d_name) == 0) {
615 /* Unfortunately, this does happen, and mere assert() does not give
616 users enough information to complain to the right people. */
617 fprintf(stderr, "file system error: zero-length file name in directory %s", path.c_str());
623 if (de->d_type == DT_UNKNOWN) {
624 // Evidently some file systems, like older versions of XFS
625 // (mkfs.xfs -m crc=0 -n ftype=0), can return this,
626 // and we need a stat(). If we wanted to optimize for this,
627 // we could probably defer it to later (we're stat-ing directories
628 // when recursing), but this is rare, and not really worth it --
629 // the second stat() will be cached anyway.
631 if (fstatat(fd, de->d_name, &buf, AT_SYMLINK_NOFOLLOW) == 0 &&
632 S_ISDIR(buf.st_mode)) {
633 e.is_directory = true;
635 e.is_directory = false;
638 e.is_directory = (de->d_type == DT_DIR);
642 printf("%s/%s\n", path.c_str(), de->d_name);
644 entries.push_back(move(e));
647 sort(entries.begin(), entries.end(), [](const entry &a, const entry &b) {
648 return a.name < b.name;
651 // Load directory modification times from the old database.
652 auto db_it = db_entries.begin();
653 for (entry &e : entries) {
654 for (; db_it != db_entries.end(); ++db_it) {
655 if (e.name < db_it->name) {
658 if (e.name == db_it->name) {
659 e.db_modified = db_it->db_modified;
666 // For each entry, we want to add it to the database. but this includes the modification time
667 // for directories, which means we need to open and stat it at this point.
669 // This means we may need to have many directories open at the same time, but it seems to be
670 // the simplest (only?) way of being compatible with mlocate's notion of listing all contents
671 // of a given directory before recursing, without buffering even more information. Hopefully,
672 // we won't go out of file descriptors here (it could happen if someone has tens of thousands
673 // of subdirectories in a single directory); if so, the admin will need to raise the limit.
674 for (entry &e : entries) {
675 if (!e.is_directory) {
680 if (find(conf_prunenames.begin(), conf_prunenames.end(), e.name) != conf_prunenames.end()) {
681 if (conf_debug_pruning) {
682 fprintf(stderr, "Skipping `%s': in prunenames\n", e.name.c_str());
686 if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, (path_plus_slash + e.name).c_str())) {
687 if (conf_debug_pruning) {
688 fprintf(stderr, "Skipping `%s/%s': in prunepaths\n", path.c_str(), e.name.c_str());
693 e.fd = opendir_noatime(fd, e.name.c_str());
695 if (errno == EMFILE || errno == ENFILE) {
696 // The admin probably wants to know about this.
697 perror((path_plus_slash + e.name).c_str());
700 if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) {
701 fprintf(stderr, "Hint: Try `ulimit -n 131072' or similar.\n");
703 fprintf(stderr, "Hint: Try `ulimit -n %" PRIu64 " or similar (current limit is %" PRIu64 ").\n",
704 static_cast<uint64_t>(rlim.rlim_cur * 2), static_cast<uint64_t>(rlim.rlim_cur));
712 if (fstat(e.fd, &buf) != 0) {
713 // It's possible that this is a filesystem that's excluded
714 // (and the failure is e.g. because the network is down).
715 // As a last-ditch effort, we try to check that before dying,
716 // i.e., duplicate the check from further down.
718 // It would be better to be able to run filesystem_is_excluded()
719 // for cheap on everything and just avoid the stat, but it seems
720 // hard to do that without any kind of raciness.
721 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
727 perror((path_plus_slash + e.name).c_str());
732 if (buf.st_dev != parent_dev) {
733 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
740 e.dt = get_dirtime_from_stat(buf);
743 // Actually add all the entries we figured out dates for above.
744 for (const entry &e : entries) {
745 corpus->add_file(path_plus_slash + e.name, e.dt);
746 dict_builder->add_file(path_plus_slash + e.name, e.dt);
749 // Now scan subdirectories.
750 for (const entry &e : entries) {
751 if (e.is_directory && e.fd != -1) {
752 int ret = scan(path_plus_slash + e.name, e.fd, e.dev, e.dt, e.db_modified, existing_db, corpus, dict_builder);
754 // TODO: The unscanned file descriptors will leak, but it doesn't really matter,
755 // as we're about to exit.
762 if (dir == nullptr) {
770 int main(int argc, char **argv)
772 // We want to bump the file limit; do it if we can (usually we are root
773 // and can set whatever we want). 128k should be ample for most setups.
775 if (getrlimit(RLIMIT_NOFILE, &rlim) != -1) {
776 // Even root cannot increase rlim_cur beyond rlim_max,
777 // so we need to try to increase rlim_max first.
778 // Ignore errors, though.
779 if (rlim.rlim_max < 131072) {
780 rlim.rlim_max = 131072;
781 setrlimit(RLIMIT_NOFILE, &rlim);
782 getrlimit(RLIMIT_NOFILE, &rlim);
785 rlim_t wanted = std::max<rlim_t>(rlim.rlim_cur, 131072);
786 rlim.rlim_cur = std::min<rlim_t>(wanted, rlim.rlim_max);
787 setrlimit(RLIMIT_NOFILE, &rlim); // Ignore errors.
790 conf_prepare(argc, argv);
791 if (conf_prune_bind_mounts) {
792 bind_mount_init(MOUNTINFO_PATH);
795 int fd = open(conf_output.c_str(), O_RDONLY);
796 ExistingDB existing_db(fd);
798 DictionaryBuilder dict_builder(/*blocks_to_keep=*/1000, conf_block_size);
801 if (conf_check_visibility) {
802 group *grp = getgrnam(GROUPNAME);
803 if (grp == nullptr) {
804 fprintf(stderr, "Unknown group %s\n", GROUPNAME);
810 DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary(), conf_check_visibility);
811 db.set_conf_block(conf_block);
812 DatabaseReceiver *corpus = db.start_corpus(/*store_dir_times=*/true);
814 int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
821 if (fstat(root_fd, &buf) == -1) {
826 scan(conf_scan_root, root_fd, buf.st_dev, get_dirtime_from_stat(buf), /*db_modified=*/unknown_dir_time, &existing_db, corpus, &dict_builder);
828 // It's too late to use the dictionary for the data we already compressed,
829 // unless we wanted to either scan the entire file system again (acceptable
830 // for plocate-build where it's cheap, less so for us), or uncompressing
831 // and recompressing. Instead, we store it for next time, assuming that the
832 // data changes fairly little from time to time.
833 string next_dictionary = dict_builder.train(1024);
834 db.set_next_dictionary(next_dictionary);