3 Copyright (C) 2005, 2007, 2008 Red Hat, Inc. All rights reserved.
5 This copyrighted material is made available to anyone wishing to use, modify,
6 copy, or redistribute it subject to the terms and conditions of the GNU General
9 This program is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
11 PARTICULAR PURPOSE. See the GNU General Public License for more details.
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
15 Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 Author: Miloslav Trmac <mitr@redhat.com>
20 plocate modifications: Copyright (C) 2020 Steinar H. Gunderson.
21 plocate parts and modifications are licensed under the GPLv2 or, at your option,
25 #include "bind-mount.h"
26 #include "complete_pread.h"
28 #include "database-builder.h"
31 #include "io_uring_engine.h"
35 #include <arpa/inet.h>
53 #include <sys/resource.h>
56 #include <sys/types.h>
62 using namespace std::chrono;
64 /* Next conf_prunepaths entry */
65 static size_t conf_prunepaths_index; /* = 0; */
70 "Usage: updatedb PLOCATE_DB\n"
72 "Generate plocate index from mlocate.db, typically /var/lib/mlocate/mlocate.db.\n"
73 "Normally, the destination should be /var/lib/mlocate/plocate.db.\n"
75 " -b, --block-size SIZE number of filenames to store in each block (default 32)\n"
76 " -p, --plaintext input is a plaintext file, not an mlocate database\n"
77 " --help print this help\n"
78 " --version print version information\n");
83 printf("updatedb %s\n", PACKAGE_VERSION);
84 printf("Copyright (C) 2007 Red Hat, Inc. All rights reserved.\n");
85 printf("Copyright 2020 Steinar H. Gunderson\n");
86 printf("This software is distributed under the GPL v.2.\n");
88 printf("This program is provided with NO WARRANTY, to the extent permitted by law.\n");
91 int opendir_noatime(int dirfd, const char *path)
93 static bool noatime_failed = false;
95 if (!noatime_failed) {
97 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY | O_NOATIME);
99 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY);
103 } else if (errno == EPERM) {
104 /* EPERM is fairly O_NOATIME-specific; missing access rights cause
106 noatime_failed = true;
112 return openat(dirfd, path, O_RDONLY | O_DIRECTORY);
115 bool time_is_current(const dir_time &t)
117 static dir_time cache{ 0, 0 };
119 /* This is more difficult than it should be because Linux uses a cheaper time
120 source for filesystem timestamps than for gettimeofday() and they can get
121 slightly out of sync, see
122 https://bugzilla.redhat.com/show_bug.cgi?id=244697 . This affects even
123 nanosecond timestamps (and don't forget that tv_nsec existence doesn't
124 guarantee that the underlying filesystem has such resolution - it might be
125 microseconds or even coarser).
127 The worst case is probably FAT timestamps with 2-second resolution
128 (although using such a filesystem violates POSIX file times requirements).
130 So, to be on the safe side, require a >3.0 second difference (2 seconds to
131 make sure the FAT timestamp changed, 1 more to account for the Linux
132 timestamp races). This large margin might make updatedb marginally more
133 expensive, but it only makes a difference if the directory was very
134 recently updated _and_ is will not be updated again until the next
135 updatedb run; this is not likely to happen for most directories. */
137 /* Cache gettimeofday () results to rule out obviously old time stamps;
138 CACHE contains the earliest time we reject as too current. */
144 gettimeofday(&tv, nullptr);
145 cache.sec = tv.tv_sec - 3;
146 cache.nsec = tv.tv_usec * 1000;
155 // For directories only:
157 dir_time dt = unknown_dir_time;
158 dir_time db_modified = unknown_dir_time;
162 bool filesystem_is_excluded(const char *path)
164 if (conf_debug_pruning) {
165 /* This is debugging output, don't mark anything for translation */
166 fprintf(stderr, "Checking whether filesystem `%s' is excluded:\n", path);
168 FILE *f = setmntent("/proc/mounts", "r");
174 while ((me = getmntent(f)) != nullptr) {
175 if (conf_debug_pruning) {
176 /* This is debugging output, don't mark anything for translation */
177 fprintf(stderr, " `%s', type `%s'\n", me->mnt_dir, me->mnt_type);
179 string type(me->mnt_type);
180 for (char &p : type) {
183 if (find(conf_prunefs.begin(), conf_prunefs.end(), type) != conf_prunefs.end()) {
184 /* Paths in /proc/self/mounts contain no symbolic links. Besides
185 avoiding a few system calls, avoiding the realpath () avoids hangs
186 if the filesystem is unavailable hard-mounted NFS. */
187 char *dir = me->mnt_dir;
188 if (conf_debug_pruning) {
189 /* This is debugging output, don't mark anything for translation */
190 fprintf(stderr, " => type matches, dir `%s'\n", dir);
192 bool res = (strcmp(path, dir) == 0);
193 if (dir != me->mnt_dir)
201 if (conf_debug_pruning) {
202 /* This is debugging output, don't mark anything for translation */
203 fprintf(stderr, "...done\n");
209 dir_time get_dirtime_from_stat(const struct stat &buf)
211 dir_time ctime{ buf.st_ctim.tv_sec, int32_t(buf.st_ctim.tv_nsec) };
212 dir_time mtime{ buf.st_mtim.tv_sec, int32_t(buf.st_mtim.tv_nsec) };
213 dir_time dt = max(ctime, mtime);
215 if (time_is_current(dt)) {
216 /* The directory might be changing right now and we can't be sure the
217 timestamp will be changed again if more changes happen very soon, mark
218 the timestamp as invalid to force rescanning the directory next time
220 return unknown_dir_time;
226 // Represents the old database we are updating.
229 explicit ExistingDB(int fd);
232 pair<string, dir_time> read_next();
233 void unread(pair<string, dir_time> record)
235 unread_record = move(record);
237 string read_next_dictionary() const;
238 bool get_error() const { return error; }
244 uint32_t current_docid = 0;
246 string current_filename_block;
247 const char *current_filename_ptr = nullptr, *current_filename_end = nullptr;
249 off_t compressed_dir_time_pos;
250 string compressed_dir_time;
251 string current_dir_time_block;
252 const char *current_dir_time_ptr = nullptr, *current_dir_time_end = nullptr;
254 pair<string, dir_time> unread_record;
256 // Used in one-shot mode, repeatedly.
259 // Used in streaming mode.
260 ZSTD_DCtx *dir_time_ctx;
262 ZSTD_DDict *ddict = nullptr;
264 // If true, we've discovered an error or EOF, and will return only
265 // empty data from here.
266 bool eof = false, error = false;
269 ExistingDB::ExistingDB(int fd)
277 if (!try_complete_pread(fd, &hdr, sizeof(hdr), /*offset=*/0)) {
279 perror("pread(header)");
284 if (memcmp(hdr.magic, "\0plocate", 8) != 0) {
286 fprintf(stderr, "Old database had header mismatch, ignoring.\n");
291 if (hdr.version != 1 || hdr.max_version < 2) {
293 fprintf(stderr, "Old database had version mismatch (version=%d max_version=%d), ignoring.\n",
294 hdr.version, hdr.max_version);
300 // Compare the configuration block with our current one.
301 if (hdr.conf_block_length_bytes != conf_block.size()) {
303 fprintf(stderr, "Old database had different configuration block (size mismatch), ignoring.\n");
309 str.resize(hdr.conf_block_length_bytes);
310 if (!try_complete_pread(fd, str.data(), hdr.conf_block_length_bytes, hdr.conf_block_offset_bytes)) {
312 perror("pread(conf_block)");
317 if (str != conf_block) {
319 fprintf(stderr, "Old database had different configuration block (contents mismatch), ignoring.\n");
325 // Read dictionary, if it exists.
326 if (hdr.zstd_dictionary_length_bytes > 0) {
328 dictionary.resize(hdr.zstd_dictionary_length_bytes);
329 if (try_complete_pread(fd, &dictionary[0], hdr.zstd_dictionary_length_bytes, hdr.zstd_dictionary_offset_bytes)) {
330 ddict = ZSTD_createDDict(dictionary.data(), dictionary.size());
333 perror("pread(dictionary)");
339 compressed_dir_time_pos = hdr.directory_data_offset_bytes;
341 ctx = ZSTD_createDCtx();
342 dir_time_ctx = ZSTD_createDCtx();
345 ExistingDB::~ExistingDB()
352 pair<string, dir_time> ExistingDB::read_next()
354 if (!unread_record.first.empty()) {
355 auto ret = move(unread_record);
356 unread_record.first.clear();
361 return { "", not_a_dir };
364 // See if we need to read a new filename block.
365 if (current_filename_ptr == nullptr) {
366 if (current_docid >= hdr.num_docids) {
368 return { "", not_a_dir };
371 // Read the file offset from this docid and the next one.
372 // This is always allowed, since we have a sentinel block at the end.
373 off_t offset_for_block = hdr.filename_index_offset_bytes + current_docid * sizeof(uint64_t);
375 if (!try_complete_pread(fd, vals, sizeof(vals), offset_for_block)) {
377 perror("pread(offset)");
380 return { "", not_a_dir };
383 off_t offset = vals[0];
384 size_t compressed_len = vals[1] - vals[0];
385 unique_ptr<char[]> compressed(new char[compressed_len]);
386 if (!try_complete_pread(fd, compressed.get(), compressed_len, offset)) {
388 perror("pread(block)");
391 return { "", not_a_dir };
394 unsigned long long uncompressed_len = ZSTD_getFrameContentSize(compressed.get(), compressed_len);
395 if (uncompressed_len == ZSTD_CONTENTSIZE_UNKNOWN || uncompressed_len == ZSTD_CONTENTSIZE_ERROR) {
397 fprintf(stderr, "ZSTD_getFrameContentSize() failed\n");
400 return { "", not_a_dir };
404 block.resize(uncompressed_len + 1);
407 if (ddict != nullptr) {
408 err = ZSTD_decompress_usingDDict(ctx, &block[0], block.size(), compressed.get(),
409 compressed_len, ddict);
411 err = ZSTD_decompressDCtx(ctx, &block[0], block.size(), compressed.get(),
414 if (ZSTD_isError(err)) {
416 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
419 return { "", not_a_dir };
421 block[block.size() - 1] = '\0';
422 current_filename_block = move(block);
423 current_filename_ptr = current_filename_block.data();
424 current_filename_end = current_filename_block.data() + current_filename_block.size();
428 // See if we need to read more directory time data.
429 while (current_dir_time_ptr == current_dir_time_end ||
430 (*current_dir_time_ptr != 0 &&
431 size_t(current_dir_time_end - current_dir_time_ptr) < sizeof(dir_time) + 1)) {
432 if (current_dir_time_ptr != nullptr) {
433 const size_t bytes_consumed = current_dir_time_ptr - current_dir_time_block.data();
434 current_dir_time_block.erase(current_dir_time_block.begin(), current_dir_time_block.begin() + bytes_consumed);
437 // See if we can get more data out without reading more.
438 const size_t existing_data = current_dir_time_block.size();
439 current_dir_time_block.resize(existing_data + 4096);
441 ZSTD_outBuffer outbuf;
442 outbuf.dst = current_dir_time_block.data() + existing_data;
447 inbuf.src = compressed_dir_time.data();
448 inbuf.size = compressed_dir_time.size();
451 int err = ZSTD_decompressStream(dir_time_ctx, &outbuf, &inbuf);
454 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
457 return { "", not_a_dir };
459 compressed_dir_time.erase(compressed_dir_time.begin(), compressed_dir_time.begin() + inbuf.pos);
460 current_dir_time_block.resize(existing_data + outbuf.pos);
462 if (inbuf.pos == 0 && outbuf.pos == 0) {
463 // No movement, we'll need to try to read more data.
465 size_t bytes_to_read = min<size_t>(
466 hdr.directory_data_offset_bytes + hdr.directory_data_length_bytes - compressed_dir_time_pos,
468 if (bytes_to_read == 0) {
470 return { "", not_a_dir };
472 if (!try_complete_pread(fd, buf, bytes_to_read, compressed_dir_time_pos)) {
474 perror("pread(dirtime)");
477 return { "", not_a_dir };
479 compressed_dir_time_pos += bytes_to_read;
480 compressed_dir_time.insert(compressed_dir_time.end(), buf, buf + bytes_to_read);
482 // Next iteration will now try decompressing more.
485 current_dir_time_ptr = current_dir_time_block.data();
486 current_dir_time_end = current_dir_time_block.data() + current_dir_time_block.size();
489 string filename = current_filename_ptr;
490 current_filename_ptr += filename.size() + 1;
491 if (current_filename_ptr == current_filename_end) {
492 // End of this block.
493 current_filename_ptr = nullptr;
496 if (*current_dir_time_ptr == 0) {
497 ++current_dir_time_ptr;
498 return { move(filename), not_a_dir };
500 ++current_dir_time_ptr;
502 memcpy(&dt.sec, current_dir_time_ptr, sizeof(dt.sec));
503 current_dir_time_ptr += sizeof(dt.sec);
504 memcpy(&dt.nsec, current_dir_time_ptr, sizeof(dt.nsec));
505 current_dir_time_ptr += sizeof(dt.nsec);
506 return { move(filename), dt };
510 string ExistingDB::read_next_dictionary() const
512 if (hdr.next_zstd_dictionary_length_bytes == 0 || hdr.next_zstd_dictionary_length_bytes > 1048576) {
516 str.resize(hdr.next_zstd_dictionary_length_bytes);
517 if (!try_complete_pread(fd, str.data(), hdr.next_zstd_dictionary_length_bytes, hdr.next_zstd_dictionary_offset_bytes)) {
519 perror("pread(next_dictionary)");
526 // Scans the directory with absolute path “path”, which is opened as “fd”.
527 // Uses relative paths and openat() only, evading any issues with PATH_MAX
528 // and time-of-check-time-of-use race conditions. (mlocate's updatedb
529 // does a much more complicated dance with changing the current working
530 // directory, probably in the interest of portability to old platforms.)
531 // “parent_dev” must be the device of the parent directory of “path”.
533 // Takes ownership of fd.
534 int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, DatabaseReceiver *corpus, DictionaryBuilder *dict_builder)
536 if (conf_prune_bind_mounts && is_bind_mount(path.c_str())) {
537 if (conf_debug_pruning) {
538 /* This is debugging output, don't mark anything for translation */
539 fprintf(stderr, "Skipping `%s': bind mount\n", path.c_str());
545 // We read in the old directory no matter whether it is current or not,
546 // because even if we're not going to use it, we'll need the modification directory
547 // of any subdirectories.
549 // Skip over anything before this directory; it is stuff that we would have
550 // consumed earlier if we wanted it.
552 pair<string, dir_time> record = existing_db->read_next();
553 if (record.first.empty()) {
556 if (dir_path_cmp(path, record.first) <= 0) {
557 existing_db->unread(move(record));
562 // Now read everything in this directory.
563 vector<entry> db_entries;
564 const string path_plus_slash = path.back() == '/' ? path : path + '/';
566 pair<string, dir_time> record = existing_db->read_next();
567 if (record.first.empty()) {
571 if (record.first.rfind(path_plus_slash, 0) != 0) {
572 // No longer starts with path, so we're in a different directory.
573 existing_db->unread(move(record));
576 if (record.first.find_first_of('/', path_plus_slash.size()) != string::npos) {
577 // Entered into a subdirectory of a subdirectory.
578 // Due to our ordering, this also means we're done.
579 existing_db->unread(move(record));
584 e.name = record.first.substr(path_plus_slash.size());
585 e.is_directory = (record.second.sec >= 0);
586 e.db_modified = record.second;
587 db_entries.push_back(e);
591 vector<entry> entries;
592 if (!existing_db->get_error() && db_modified.sec > 0 &&
593 modified.sec == db_modified.sec && modified.nsec == db_modified.nsec) {
594 // Not changed since the last database, so we can replace the readdir()
595 // by reading from the database. (We still need to open and stat everything,
596 // though, but that happens in a later step.)
597 entries = move(db_entries);
599 for (const entry &e : entries) {
600 printf("%s/%s\n", path.c_str(), e.name.c_str());
604 dir = fdopendir(fd); // Takes over ownership of fd.
605 if (dir == nullptr) {
606 // fdopendir() wants to fstat() the fd to verify that it's indeed
607 // a directory, which can seemingly fail on at least CIFS filesystems
608 // if the server feels like it. We treat this as if we had an error
609 // on opening it, ie., ignore the directory.
615 while ((de = readdir(dir)) != nullptr) {
616 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
619 if (strlen(de->d_name) == 0) {
620 /* Unfortunately, this does happen, and mere assert() does not give
621 users enough information to complain to the right people. */
622 fprintf(stderr, "file system error: zero-length file name in directory %s", path.c_str());
628 if (de->d_type == DT_UNKNOWN) {
629 // Evidently some file systems, like older versions of XFS
630 // (mkfs.xfs -m crc=0 -n ftype=0), can return this,
631 // and we need a stat(). If we wanted to optimize for this,
632 // we could probably defer it to later (we're stat-ing directories
633 // when recursing), but this is rare, and not really worth it --
634 // the second stat() will be cached anyway.
636 if (fstatat(fd, de->d_name, &buf, AT_SYMLINK_NOFOLLOW) == 0 &&
637 S_ISDIR(buf.st_mode)) {
638 e.is_directory = true;
640 e.is_directory = false;
643 e.is_directory = (de->d_type == DT_DIR);
647 printf("%s/%s\n", path.c_str(), de->d_name);
649 entries.push_back(move(e));
652 sort(entries.begin(), entries.end(), [](const entry &a, const entry &b) {
653 return a.name < b.name;
656 // Load directory modification times from the old database.
657 auto db_it = db_entries.begin();
658 for (entry &e : entries) {
659 for (; db_it != db_entries.end(); ++db_it) {
660 if (e.name < db_it->name) {
663 if (e.name == db_it->name) {
664 e.db_modified = db_it->db_modified;
671 // For each entry, we want to add it to the database. but this includes the modification time
672 // for directories, which means we need to open and stat it at this point.
674 // This means we may need to have many directories open at the same time, but it seems to be
675 // the simplest (only?) way of being compatible with mlocate's notion of listing all contents
676 // of a given directory before recursing, without buffering even more information. Hopefully,
677 // we won't go out of file descriptors here (it could happen if someone has tens of thousands
678 // of subdirectories in a single directory); if so, the admin will need to raise the limit.
679 for (entry &e : entries) {
680 if (!e.is_directory) {
685 if (find(conf_prunenames.begin(), conf_prunenames.end(), e.name) != conf_prunenames.end()) {
686 if (conf_debug_pruning) {
687 /* This is debugging output, don't mark anything for translation */
688 fprintf(stderr, "Skipping `%s': in prunenames\n", e.name.c_str());
692 if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, (path_plus_slash + e.name).c_str())) {
693 if (conf_debug_pruning) {
694 /* This is debugging output, don't mark anything for translation */
695 fprintf(stderr, "Skipping `%s/%s': in prunepaths\n", path.c_str(), e.name.c_str());
700 e.fd = opendir_noatime(fd, e.name.c_str());
702 if (errno == EMFILE || errno == ENFILE) {
703 // The admin probably wants to know about this.
704 perror((path_plus_slash + e.name).c_str());
707 if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) {
708 fprintf(stderr, "Hint: Try `ulimit -n 131072' or similar.\n");
710 fprintf(stderr, "Hint: Try `ulimit -n %" PRIu64 " or similar (current limit is %" PRIu64 ").\n",
711 static_cast<uint64_t>(rlim.rlim_cur * 2), static_cast<uint64_t>(rlim.rlim_cur));
719 if (fstat(e.fd, &buf) != 0) {
720 // It's possible that this is a filesystem that's excluded
721 // (and the failure is e.g. because the network is down).
722 // As a last-ditch effort, we try to check that before dying,
723 // i.e., duplicate the check from further down.
725 // It would be better to be able to run filesystem_is_excluded()
726 // for cheap on everything and just avoid the stat, but it seems
727 // hard to do that without any kind of raciness.
728 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
734 perror((path_plus_slash + e.name).c_str());
739 if (buf.st_dev != parent_dev) {
740 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
747 e.dt = get_dirtime_from_stat(buf);
750 // Actually add all the entries we figured out dates for above.
751 for (const entry &e : entries) {
752 corpus->add_file(path_plus_slash + e.name, e.dt);
753 dict_builder->add_file(path_plus_slash + e.name, e.dt);
756 // Now scan subdirectories.
757 for (const entry &e : entries) {
758 if (e.is_directory && e.fd != -1) {
759 int ret = scan(path_plus_slash + e.name, e.fd, e.dev, e.dt, e.db_modified, existing_db, corpus, dict_builder);
761 // TODO: The unscanned file descriptors will leak, but it doesn't really matter,
762 // as we're about to exit.
769 if (dir == nullptr) {
777 int main(int argc, char **argv)
779 // We want to bump the file limit; do it if we can (usually we are root
780 // and can set whatever we want). 128k should be ample for most setups.
782 if (getrlimit(RLIMIT_NOFILE, &rlim) != -1) {
783 // Even root cannot increase rlim_cur beyond rlim_max,
784 // so we need to try to increase rlim_max first.
785 // Ignore errors, though.
786 if (rlim.rlim_max < 131072) {
787 rlim.rlim_max = 131072;
788 setrlimit(RLIMIT_NOFILE, &rlim);
789 getrlimit(RLIMIT_NOFILE, &rlim);
792 rlim_t wanted = std::max<rlim_t>(rlim.rlim_cur, 131072);
793 rlim.rlim_cur = std::min<rlim_t>(wanted, rlim.rlim_max);
794 setrlimit(RLIMIT_NOFILE, &rlim); // Ignore errors.
797 conf_prepare(argc, argv);
798 if (conf_prune_bind_mounts) {
799 bind_mount_init(MOUNTINFO_PATH);
802 int fd = open(conf_output.c_str(), O_RDONLY);
803 ExistingDB existing_db(fd);
805 DictionaryBuilder dict_builder(/*blocks_to_keep=*/1000, conf_block_size);
808 if (conf_check_visibility) {
809 group *grp = getgrnam(GROUPNAME);
810 if (grp == nullptr) {
811 fprintf(stderr, "Unknown group %s\n", GROUPNAME);
817 DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary(), conf_check_visibility);
818 db.set_conf_block(conf_block);
819 DatabaseReceiver *corpus = db.start_corpus(/*store_dir_times=*/true);
821 int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
828 if (fstat(root_fd, &buf) == -1) {
833 scan(conf_scan_root, root_fd, buf.st_dev, get_dirtime_from_stat(buf), /*db_modified=*/unknown_dir_time, &existing_db, corpus, &dict_builder);
835 // It's too late to use the dictionary for the data we already compressed,
836 // unless we wanted to either scan the entire file system again (acceptable
837 // for plocate-build where it's cheap, less so for us), or uncompressing
838 // and recompressing. Instead, we store it for next time, assuming that the
839 // data changes fairly little from time to time.
840 string next_dictionary = dict_builder.train(1024);
841 db.set_next_dictionary(next_dictionary);