3 Copyright (C) 2005, 2007, 2008 Red Hat, Inc. All rights reserved.
5 This copyrighted material is made available to anyone wishing to use, modify,
6 copy, or redistribute it subject to the terms and conditions of the GNU General
9 This program is distributed in the hope that it will be useful, but WITHOUT ANY
10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
11 PARTICULAR PURPOSE. See the GNU General Public License for more details.
13 You should have received a copy of the GNU General Public License along with
14 this program; if not, write to the Free Software Foundation, Inc., 51 Franklin
15 Street, Fifth Floor, Boston, MA 02110-1301, USA.
17 Author: Miloslav Trmac <mitr@redhat.com>
20 plocate modifications: Copyright (C) 2020 Steinar H. Gunderson.
21 plocate parts and modifications are licensed under the GPLv2 or, at your option,
25 #include "bind-mount.h"
26 #include "complete_pread.h"
28 #include "database-builder.h"
31 #include "io_uring_engine.h"
35 #include <arpa/inet.h>
52 #include <sys/resource.h>
55 #include <sys/types.h>
61 using namespace std::chrono;
63 /* Next conf_prunepaths entry */
64 static size_t conf_prunepaths_index; /* = 0; */
69 "Usage: updatedb PLOCATE_DB\n"
71 "Generate plocate index from mlocate.db, typically /var/lib/mlocate/mlocate.db.\n"
72 "Normally, the destination should be /var/lib/mlocate/plocate.db.\n"
74 " -b, --block-size SIZE number of filenames to store in each block (default 32)\n"
75 " -p, --plaintext input is a plaintext file, not an mlocate database\n"
76 " --help print this help\n"
77 " --version print version information\n");
82 printf("updatedb %s\n", PACKAGE_VERSION);
83 printf("Copyright (C) 2007 Red Hat, Inc. All rights reserved.\n");
84 printf("Copyright 2020 Steinar H. Gunderson\n");
85 printf("This software is distributed under the GPL v.2.\n");
87 printf("This program is provided with NO WARRANTY, to the extent permitted by law.\n");
90 int opendir_noatime(int dirfd, const char *path)
92 static bool noatime_failed = false;
94 if (!noatime_failed) {
95 int fd = openat(dirfd, path, O_RDONLY | O_DIRECTORY | O_NOATIME);
98 } else if (errno == EPERM) {
99 /* EPERM is fairly O_NOATIME-specific; missing access rights cause
101 noatime_failed = true;
107 return openat(dirfd, path, O_RDONLY | O_DIRECTORY);
110 bool time_is_current(const dir_time &t)
112 static dir_time cache{ 0, 0 };
114 /* This is more difficult than it should be because Linux uses a cheaper time
115 source for filesystem timestamps than for gettimeofday() and they can get
116 slightly out of sync, see
117 https://bugzilla.redhat.com/show_bug.cgi?id=244697 . This affects even
118 nanosecond timestamps (and don't forget that tv_nsec existence doesn't
119 guarantee that the underlying filesystem has such resolution - it might be
120 microseconds or even coarser).
122 The worst case is probably FAT timestamps with 2-second resolution
123 (although using such a filesystem violates POSIX file times requirements).
125 So, to be on the safe side, require a >3.0 second difference (2 seconds to
126 make sure the FAT timestamp changed, 1 more to account for the Linux
127 timestamp races). This large margin might make updatedb marginally more
128 expensive, but it only makes a difference if the directory was very
129 recently updated _and_ is will not be updated again until the next
130 updatedb run; this is not likely to happen for most directories. */
132 /* Cache gettimeofday () results to rule out obviously old time stamps;
133 CACHE contains the earliest time we reject as too current. */
139 gettimeofday(&tv, nullptr);
140 cache.sec = tv.tv_sec - 3;
141 cache.nsec = tv.tv_usec * 1000;
150 // For directories only:
152 dir_time dt = unknown_dir_time;
153 dir_time db_modified = unknown_dir_time;
157 bool filesystem_is_excluded(const char *path)
159 if (conf_debug_pruning) {
160 /* This is debugging output, don't mark anything for translation */
161 fprintf(stderr, "Checking whether filesystem `%s' is excluded:\n", path);
163 FILE *f = setmntent("/proc/mounts", "r");
169 while ((me = getmntent(f)) != nullptr) {
170 if (conf_debug_pruning) {
171 /* This is debugging output, don't mark anything for translation */
172 fprintf(stderr, " `%s', type `%s'\n", me->mnt_dir, me->mnt_type);
174 string type(me->mnt_type);
175 for (char &p : type) {
178 if (find(conf_prunefs.begin(), conf_prunefs.end(), type) != conf_prunefs.end()) {
179 /* Paths in /proc/self/mounts contain no symbolic links. Besides
180 avoiding a few system calls, avoiding the realpath () avoids hangs
181 if the filesystem is unavailable hard-mounted NFS. */
182 char *dir = me->mnt_dir;
183 if (conf_debug_pruning) {
184 /* This is debugging output, don't mark anything for translation */
185 fprintf(stderr, " => type matches, dir `%s'\n", dir);
187 bool res = (strcmp(path, dir) == 0);
188 if (dir != me->mnt_dir)
196 if (conf_debug_pruning) {
197 /* This is debugging output, don't mark anything for translation */
198 fprintf(stderr, "...done\n");
204 dir_time get_dirtime_from_stat(const struct stat &buf)
206 dir_time ctime{ buf.st_ctim.tv_sec, int32_t(buf.st_ctim.tv_nsec) };
207 dir_time mtime{ buf.st_mtim.tv_sec, int32_t(buf.st_mtim.tv_nsec) };
208 dir_time dt = max(ctime, mtime);
210 if (time_is_current(dt)) {
211 /* The directory might be changing right now and we can't be sure the
212 timestamp will be changed again if more changes happen very soon, mark
213 the timestamp as invalid to force rescanning the directory next time
215 return unknown_dir_time;
221 // Represents the old database we are updating.
224 explicit ExistingDB(int fd);
227 pair<string, dir_time> read_next();
228 void unread(pair<string, dir_time> record)
230 unread_record = move(record);
232 string read_next_dictionary() const;
233 bool get_error() const { return error; }
239 uint32_t current_docid = 0;
241 string current_filename_block;
242 const char *current_filename_ptr = nullptr, *current_filename_end = nullptr;
244 off_t compressed_dir_time_pos;
245 string compressed_dir_time;
246 string current_dir_time_block;
247 const char *current_dir_time_ptr = nullptr, *current_dir_time_end = nullptr;
249 pair<string, dir_time> unread_record;
251 // Used in one-shot mode, repeatedly.
254 // Used in streaming mode.
255 ZSTD_DCtx *dir_time_ctx;
257 ZSTD_DDict *ddict = nullptr;
259 // If true, we've discovered an error or EOF, and will return only
260 // empty data from here.
261 bool eof = false, error = false;
264 ExistingDB::ExistingDB(int fd)
272 if (!try_complete_pread(fd, &hdr, sizeof(hdr), /*offset=*/0)) {
274 perror("pread(header)");
279 if (memcmp(hdr.magic, "\0plocate", 8) != 0) {
281 fprintf(stderr, "Old database had header mismatch, ignoring.\n");
286 if (hdr.version != 1 || hdr.max_version < 2) {
288 fprintf(stderr, "Old database had version mismatch (version=%d max_version=%d), ignoring.\n",
289 hdr.version, hdr.max_version);
295 // Compare the configuration block with our current one.
296 if (hdr.conf_block_length_bytes != conf_block.size()) {
298 fprintf(stderr, "Old database had different configuration block (size mismatch), ignoring.\n");
304 str.resize(hdr.conf_block_length_bytes);
305 if (!try_complete_pread(fd, str.data(), hdr.conf_block_length_bytes, hdr.conf_block_offset_bytes)) {
307 perror("pread(conf_block)");
312 if (str != conf_block) {
314 fprintf(stderr, "Old database had different configuration block (contents mismatch), ignoring.\n");
320 // Read dictionary, if it exists.
321 if (hdr.zstd_dictionary_length_bytes > 0) {
323 dictionary.resize(hdr.zstd_dictionary_length_bytes);
324 if (try_complete_pread(fd, &dictionary[0], hdr.zstd_dictionary_length_bytes, hdr.zstd_dictionary_offset_bytes)) {
325 ddict = ZSTD_createDDict(dictionary.data(), dictionary.size());
328 perror("pread(dictionary)");
334 compressed_dir_time_pos = hdr.directory_data_offset_bytes;
336 ctx = ZSTD_createDCtx();
337 dir_time_ctx = ZSTD_createDCtx();
340 ExistingDB::~ExistingDB()
347 pair<string, dir_time> ExistingDB::read_next()
349 if (!unread_record.first.empty()) {
350 auto ret = move(unread_record);
351 unread_record.first.clear();
356 return { "", not_a_dir };
359 // See if we need to read a new filename block.
360 if (current_filename_ptr == nullptr) {
361 if (current_docid >= hdr.num_docids) {
363 return { "", not_a_dir };
366 // Read the file offset from this docid and the next one.
367 // This is always allowed, since we have a sentinel block at the end.
368 off_t offset_for_block = hdr.filename_index_offset_bytes + current_docid * sizeof(uint64_t);
370 if (!try_complete_pread(fd, vals, sizeof(vals), offset_for_block)) {
372 perror("pread(offset)");
375 return { "", not_a_dir };
378 off_t offset = vals[0];
379 size_t compressed_len = vals[1] - vals[0];
380 unique_ptr<char[]> compressed(new char[compressed_len]);
381 if (!try_complete_pread(fd, compressed.get(), compressed_len, offset)) {
383 perror("pread(block)");
386 return { "", not_a_dir };
389 unsigned long long uncompressed_len = ZSTD_getFrameContentSize(compressed.get(), compressed_len);
390 if (uncompressed_len == ZSTD_CONTENTSIZE_UNKNOWN || uncompressed_len == ZSTD_CONTENTSIZE_ERROR) {
392 fprintf(stderr, "ZSTD_getFrameContentSize() failed\n");
395 return { "", not_a_dir };
399 block.resize(uncompressed_len + 1);
402 if (ddict != nullptr) {
403 err = ZSTD_decompress_usingDDict(ctx, &block[0], block.size(), compressed.get(),
404 compressed_len, ddict);
406 err = ZSTD_decompressDCtx(ctx, &block[0], block.size(), compressed.get(),
409 if (ZSTD_isError(err)) {
411 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
414 return { "", not_a_dir };
416 block[block.size() - 1] = '\0';
417 current_filename_block = move(block);
418 current_filename_ptr = current_filename_block.data();
419 current_filename_end = current_filename_block.data() + current_filename_block.size();
423 // See if we need to read more directory time data.
424 while (current_dir_time_ptr == current_dir_time_end ||
425 (*current_dir_time_ptr != 0 &&
426 size_t(current_dir_time_end - current_dir_time_ptr) < sizeof(dir_time) + 1)) {
427 if (current_dir_time_ptr != nullptr) {
428 const size_t bytes_consumed = current_dir_time_ptr - current_dir_time_block.data();
429 current_dir_time_block.erase(current_dir_time_block.begin(), current_dir_time_block.begin() + bytes_consumed);
432 // See if we can get more data out without reading more.
433 const size_t existing_data = current_dir_time_block.size();
434 current_dir_time_block.resize(existing_data + 4096);
436 ZSTD_outBuffer outbuf;
437 outbuf.dst = current_dir_time_block.data() + existing_data;
442 inbuf.src = compressed_dir_time.data();
443 inbuf.size = compressed_dir_time.size();
446 int err = ZSTD_decompressStream(dir_time_ctx, &outbuf, &inbuf);
449 fprintf(stderr, "ZSTD_decompress(): %s\n", ZSTD_getErrorName(err));
452 return { "", not_a_dir };
454 compressed_dir_time.erase(compressed_dir_time.begin(), compressed_dir_time.begin() + inbuf.pos);
455 current_dir_time_block.resize(existing_data + outbuf.pos);
457 if (inbuf.pos == 0 && outbuf.pos == 0) {
458 // No movement, we'll need to try to read more data.
460 size_t bytes_to_read = min<size_t>(
461 hdr.directory_data_offset_bytes + hdr.directory_data_length_bytes - compressed_dir_time_pos,
463 if (bytes_to_read == 0) {
465 return { "", not_a_dir };
467 if (!try_complete_pread(fd, buf, bytes_to_read, compressed_dir_time_pos)) {
469 perror("pread(dirtime)");
472 return { "", not_a_dir };
474 compressed_dir_time_pos += bytes_to_read;
475 compressed_dir_time.insert(compressed_dir_time.end(), buf, buf + bytes_to_read);
477 // Next iteration will now try decompressing more.
480 current_dir_time_ptr = current_dir_time_block.data();
481 current_dir_time_end = current_dir_time_block.data() + current_dir_time_block.size();
484 string filename = current_filename_ptr;
485 current_filename_ptr += filename.size() + 1;
486 if (current_filename_ptr == current_filename_end) {
487 // End of this block.
488 current_filename_ptr = nullptr;
491 if (*current_dir_time_ptr == 0) {
492 ++current_dir_time_ptr;
493 return { move(filename), not_a_dir };
495 ++current_dir_time_ptr;
497 memcpy(&dt.sec, current_dir_time_ptr, sizeof(dt.sec));
498 current_dir_time_ptr += sizeof(dt.sec);
499 memcpy(&dt.nsec, current_dir_time_ptr, sizeof(dt.nsec));
500 current_dir_time_ptr += sizeof(dt.nsec);
501 return { move(filename), dt };
505 string ExistingDB::read_next_dictionary() const
507 if (hdr.next_zstd_dictionary_length_bytes == 0 || hdr.next_zstd_dictionary_length_bytes > 1048576) {
511 str.resize(hdr.next_zstd_dictionary_length_bytes);
512 if (!try_complete_pread(fd, str.data(), hdr.next_zstd_dictionary_length_bytes, hdr.next_zstd_dictionary_offset_bytes)) {
514 perror("pread(next_dictionary)");
521 // Scans the directory with absolute path “path”, which is opened as “fd”.
522 // Uses relative paths and openat() only, evading any issues with PATH_MAX
523 // and time-of-check-time-of-use race conditions. (mlocate's updatedb
524 // does a much more complicated dance with changing the current working
525 // directory, probably in the interest of portability to old platforms.)
526 // “parent_dev” must be the device of the parent directory of “path”.
528 // Takes ownership of fd.
529 int scan(const string &path, int fd, dev_t parent_dev, dir_time modified, dir_time db_modified, ExistingDB *existing_db, Corpus *corpus, DictionaryBuilder *dict_builder)
531 if (string_list_contains_dir_path(&conf_prunepaths, &conf_prunepaths_index, path)) {
532 if (conf_debug_pruning) {
533 /* This is debugging output, don't mark anything for translation */
534 fprintf(stderr, "Skipping `%s': in prunepaths\n", path.c_str());
539 if (conf_prune_bind_mounts && is_bind_mount(path.c_str())) {
540 if (conf_debug_pruning) {
541 /* This is debugging output, don't mark anything for translation */
542 fprintf(stderr, "Skipping `%s': bind mount\n", path.c_str());
548 // We read in the old directory no matter whether it is current or not,
549 // because even if we're not going to use it, we'll need the modification directory
550 // of any subdirectories.
552 // Skip over anything before this directory; it is stuff that we would have
553 // consumed earlier if we wanted it.
555 pair<string, dir_time> record = existing_db->read_next();
556 if (record.first.empty()) {
559 if (dir_path_cmp(path, record.first) <= 0) {
560 existing_db->unread(move(record));
565 // Now read everything in this directory.
566 vector<entry> db_entries;
567 const string path_plus_slash = path.back() == '/' ? path : path + '/';
569 pair<string, dir_time> record = existing_db->read_next();
570 if (record.first.empty()) {
574 if (record.first.rfind(path_plus_slash, 0) != 0) {
575 // No longer starts with path, so we're in a different directory.
576 existing_db->unread(move(record));
579 if (record.first.find_first_of('/', path_plus_slash.size()) != string::npos) {
580 // Entered into a subdirectory of a subdirectory.
581 // Due to our ordering, this also means we're done.
582 existing_db->unread(move(record));
587 e.name = record.first.substr(path_plus_slash.size());
588 e.is_directory = (record.second.sec >= 0);
589 e.db_modified = record.second;
590 db_entries.push_back(e);
594 vector<entry> entries;
595 if (!existing_db->get_error() && db_modified.sec > 0 &&
596 modified.sec == db_modified.sec && modified.nsec == db_modified.nsec) {
597 // Not changed since the last database, so we can replace the readdir()
598 // by reading from the database. (We still need to open and stat everything,
599 // though, but that happens in a later step.)
600 entries = move(db_entries);
602 dir = fdopendir(fd); // Takes over ownership of fd.
603 if (dir == nullptr) {
609 while ((de = readdir(dir)) != nullptr) {
610 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
613 if (strlen(de->d_name) == 0) {
614 /* Unfortunately, this does happen, and mere assert() does not give
615 users enough information to complain to the right people. */
616 fprintf(stderr, "file system error: zero-length file name in directory %s", path.c_str());
622 e.is_directory = (de->d_type == DT_DIR);
625 printf("%s/%s\n", path.c_str(), de->d_name);
627 entries.push_back(move(e));
630 sort(entries.begin(), entries.end(), [](const entry &a, const entry &b) {
631 return a.name < b.name;
634 // Load directory modification times from the old database.
635 auto db_it = db_entries.begin();
636 for (entry &e : entries) {
637 for (; db_it != db_entries.end(); ++db_it) {
638 if (e.name < db_it->name) {
641 if (e.name == db_it->name) {
642 e.db_modified = db_it->db_modified;
649 // For each entry, we want to add it to the database. but this includes the modification time
650 // for directories, which means we need to open and stat it at this point.
652 // This means we may need to have many directories open at the same time, but it seems to be
653 // the simplest (only?) way of being compatible with mlocate's notion of listing all contents
654 // of a given directory before recursing, without buffering even more information. Hopefully,
655 // we won't go out of file descriptors here (it could happen if someone has tens of thousands
656 // of subdirectories in a single directory); if so, the admin will need to raise the limit.
657 for (entry &e : entries) {
658 if (!e.is_directory) {
663 if (find(conf_prunenames.begin(), conf_prunenames.end(), e.name) != conf_prunenames.end()) {
664 if (conf_debug_pruning) {
665 /* This is debugging output, don't mark anything for translation */
666 fprintf(stderr, "Skipping `%s': in prunenames\n", e.name.c_str());
671 e.fd = opendir_noatime(fd, e.name.c_str());
673 if (errno == EMFILE || errno == ENFILE) {
674 // The admin probably wants to know about this.
675 perror((path_plus_slash + e.name).c_str());
678 if (getrlimit(RLIMIT_NOFILE, &rlim) == -1) {
679 fprintf(stderr, "Hint: Try `ulimit -n 131072' or similar.\n");
681 fprintf(stderr, "Hint: Try `ulimit -n %lu' or similar (current limit is %lu).\n",
682 rlim.rlim_cur * 2, rlim.rlim_cur);
690 if (fstat(e.fd, &buf) != 0) {
691 perror(path.c_str());
696 if (buf.st_dev != parent_dev) {
697 if (filesystem_is_excluded((path_plus_slash + e.name).c_str())) {
704 e.dt = get_dirtime_from_stat(buf);
707 // Actually add all the entries we figured out dates for above.
708 for (const entry &e : entries) {
709 corpus->add_file(path_plus_slash + e.name, e.dt);
710 dict_builder->add_file(path_plus_slash + e.name, e.dt);
713 // Now scan subdirectories.
714 for (const entry &e : entries) {
715 if (e.is_directory && e.fd != -1) {
716 int ret = scan(path_plus_slash + e.name, e.fd, e.dev, e.dt, e.db_modified, existing_db, corpus, dict_builder);
718 // TODO: The unscanned file descriptors will leak, but it doesn't really matter,
719 // as we're about to exit.
726 if (dir == nullptr) {
734 int main(int argc, char **argv)
736 // We want to bump the file limit; do it if we can (usually we are root
737 // and can set whatever we want). 128k should be ample for most setups.
739 if (getrlimit(RLIMIT_NOFILE, &rlim) != -1) {
740 rlim_t wanted = std::max<rlim_t>(rlim.rlim_cur, 131072);
741 rlim.rlim_cur = std::min<rlim_t>(wanted, rlim.rlim_max);
742 setrlimit(RLIMIT_NOFILE, &rlim); // Ignore errors.
745 conf_prepare(argc, argv);
746 if (conf_prune_bind_mounts) {
747 bind_mount_init(MOUNTINFO_PATH);
750 int fd = open(conf_output.c_str(), O_RDONLY);
751 ExistingDB existing_db(fd);
753 DictionaryBuilder dict_builder(/*blocks_to_keep=*/1000, conf_block_size);
756 if (conf_check_visibility) {
757 group *grp = getgrnam(GROUPNAME);
758 if (grp == nullptr) {
759 fprintf(stderr, "Unknown group %s\n", GROUPNAME);
765 DatabaseBuilder db(conf_output.c_str(), owner, conf_block_size, existing_db.read_next_dictionary());
766 Corpus *corpus = db.start_corpus(/*store_dir_times=*/true);
768 int root_fd = opendir_noatime(AT_FDCWD, conf_scan_root);
775 if (fstat(root_fd, &buf) == -1) {
780 scan(conf_scan_root, root_fd, buf.st_dev, get_dirtime_from_stat(buf), /*db_modified=*/unknown_dir_time, &existing_db, corpus, &dict_builder);
782 // It's too late to use the dictionary for the data we already compressed,
783 // unless we wanted to either scan the entire file system again (acceptable
784 // for plocate-build where it's cheap, less so for us), or uncompressing
785 // and recompressing. Instead, we store it for next time, assuming that the
786 // data changes fairly little from time to time.
787 string next_dictionary = dict_builder.train(1024);
788 db.set_next_dictionary(next_dictionary);