1 // SPDX-License-Identifier: GPL-2.0
4 #include "alloc_background.h"
5 #include "backpointers.h"
6 #include "btree_cache.h"
7 #include "btree_update.h"
8 #include "btree_update_interior.h"
9 #include "btree_write_buffer.h"
14 static bool extent_matches_bp(struct bch_fs *c,
15 enum btree_id btree_id, unsigned level,
18 struct bch_backpointer bp)
20 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
21 const union bch_extent_entry *entry;
22 struct extent_ptr_decoded p;
24 bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
26 struct bch_backpointer bp2;
31 bch2_extent_ptr_to_bp(c, btree_id, level, k, p,
33 if (bpos_eq(bucket, bucket2) &&
34 !memcmp(&bp, &bp2, sizeof(bp)))
41 int bch2_backpointer_invalid(struct bch_fs *c, struct bkey_s_c k,
42 enum bkey_invalid_flags flags,
45 struct bkey_s_c_backpointer bp = bkey_s_c_to_backpointer(k);
46 struct bpos bucket = bp_pos_to_bucket(c, bp.k->p);
49 bkey_fsck_err_on(!bpos_eq(bp.k->p, bucket_pos_to_bp(c, bucket, bp.v->bucket_offset)),
51 backpointer_pos_wrong,
52 "backpointer at wrong pos");
57 void bch2_backpointer_to_text(struct printbuf *out, const struct bch_backpointer *bp)
59 prt_printf(out, "btree=%s l=%u offset=%llu:%u len=%u pos=",
60 bch2_btree_id_str(bp->btree_id),
62 (u64) (bp->bucket_offset >> MAX_EXTENT_COMPRESS_RATIO_SHIFT),
63 (u32) bp->bucket_offset & ~(~0U << MAX_EXTENT_COMPRESS_RATIO_SHIFT),
65 bch2_bpos_to_text(out, bp->pos);
68 void bch2_backpointer_k_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
70 prt_str(out, "bucket=");
71 bch2_bpos_to_text(out, bp_pos_to_bucket(c, k.k->p));
74 bch2_backpointer_to_text(out, bkey_s_c_to_backpointer(k).v);
77 void bch2_backpointer_swab(struct bkey_s k)
79 struct bkey_s_backpointer bp = bkey_s_to_backpointer(k);
81 bp.v->bucket_offset = swab40(bp.v->bucket_offset);
82 bp.v->bucket_len = swab32(bp.v->bucket_len);
83 bch2_bpos_swab(&bp.v->pos);
86 static noinline int backpointer_mod_err(struct btree_trans *trans,
87 struct bch_backpointer bp,
89 struct bkey_s_c orig_k,
92 struct bch_fs *c = trans->c;
93 struct printbuf buf = PRINTBUF;
96 prt_printf(&buf, "existing backpointer found when inserting ");
97 bch2_backpointer_to_text(&buf, &bp);
99 printbuf_indent_add(&buf, 2);
101 prt_printf(&buf, "found ");
102 bch2_bkey_val_to_text(&buf, c, bp_k);
105 prt_printf(&buf, "for ");
106 bch2_bkey_val_to_text(&buf, c, orig_k);
108 bch_err(c, "%s", buf.buf);
109 } else if (c->curr_recovery_pass > BCH_RECOVERY_PASS_check_extents_to_backpointers) {
110 prt_printf(&buf, "backpointer not found when deleting");
112 printbuf_indent_add(&buf, 2);
114 prt_printf(&buf, "searching for ");
115 bch2_backpointer_to_text(&buf, &bp);
118 prt_printf(&buf, "got ");
119 bch2_bkey_val_to_text(&buf, c, bp_k);
122 prt_printf(&buf, "for ");
123 bch2_bkey_val_to_text(&buf, c, orig_k);
125 bch_err(c, "%s", buf.buf);
130 if (c->curr_recovery_pass > BCH_RECOVERY_PASS_check_extents_to_backpointers) {
131 bch2_inconsistent_error(c);
138 int bch2_bucket_backpointer_mod_nowritebuffer(struct btree_trans *trans,
139 struct bkey_i_backpointer *bp_k,
140 struct bch_backpointer bp,
141 struct bkey_s_c orig_k,
144 struct btree_iter bp_iter;
148 k = bch2_bkey_get_iter(trans, &bp_iter, BTREE_ID_backpointers,
152 BTREE_ITER_WITH_UPDATES);
159 : (k.k->type != KEY_TYPE_backpointer ||
160 memcmp(bkey_s_c_to_backpointer(k).v, &bp, sizeof(bp)))) {
161 ret = backpointer_mod_err(trans, bp, k, orig_k, insert);
166 ret = bch2_trans_update(trans, &bp_iter, &bp_k->k_i, 0);
168 bch2_trans_iter_exit(trans, &bp_iter);
173 * Find the next backpointer >= *bp_offset:
175 int bch2_get_next_backpointer(struct btree_trans *trans,
176 struct bpos bucket, int gen,
178 struct bch_backpointer *bp,
181 struct bch_fs *c = trans->c;
182 struct bpos bp_end_pos = bucket_pos_to_bp(c, bpos_nosnap_successor(bucket), 0);
183 struct btree_iter alloc_iter = { NULL }, bp_iter = { NULL };
187 if (bpos_ge(*bp_pos, bp_end_pos))
191 k = bch2_bkey_get_iter(trans, &alloc_iter, BTREE_ID_alloc,
192 bucket, BTREE_ITER_CACHED|iter_flags);
197 if (k.k->type != KEY_TYPE_alloc_v4 ||
198 bkey_s_c_to_alloc_v4(k).v->gen != gen)
202 *bp_pos = bpos_max(*bp_pos, bucket_pos_to_bp(c, bucket, 0));
204 for_each_btree_key_norestart(trans, bp_iter, BTREE_ID_backpointers,
205 *bp_pos, iter_flags, k, ret) {
206 if (bpos_ge(k.k->p, bp_end_pos))
210 *bp = *bkey_s_c_to_backpointer(k).v;
216 bch2_trans_iter_exit(trans, &bp_iter);
217 bch2_trans_iter_exit(trans, &alloc_iter);
221 static void backpointer_not_found(struct btree_trans *trans,
223 struct bch_backpointer bp,
226 struct bch_fs *c = trans->c;
227 struct printbuf buf = PRINTBUF;
228 struct bpos bucket = bp_pos_to_bucket(c, bp_pos);
231 * If we're using the btree write buffer, the backpointer we were
232 * looking at may have already been deleted - failure to find what it
233 * pointed to is not an error:
235 if (likely(!bch2_backpointers_no_use_write_buffer))
238 prt_printf(&buf, "backpointer doesn't match %s it points to:\n ",
239 bp.level ? "btree node" : "extent");
240 prt_printf(&buf, "bucket: ");
241 bch2_bpos_to_text(&buf, bucket);
242 prt_printf(&buf, "\n ");
244 prt_printf(&buf, "backpointer pos: ");
245 bch2_bpos_to_text(&buf, bp_pos);
246 prt_printf(&buf, "\n ");
248 bch2_backpointer_to_text(&buf, &bp);
249 prt_printf(&buf, "\n ");
250 bch2_bkey_val_to_text(&buf, c, k);
251 if (c->curr_recovery_pass >= BCH_RECOVERY_PASS_check_extents_to_backpointers)
252 bch_err_ratelimited(c, "%s", buf.buf);
254 bch2_trans_inconsistent(trans, "%s", buf.buf);
259 struct bkey_s_c bch2_backpointer_get_key(struct btree_trans *trans,
260 struct btree_iter *iter,
262 struct bch_backpointer bp,
265 if (likely(!bp.level)) {
266 struct bch_fs *c = trans->c;
267 struct bpos bucket = bp_pos_to_bucket(c, bp_pos);
270 bch2_trans_node_iter_init(trans, iter,
275 k = bch2_btree_iter_peek_slot(iter);
277 bch2_trans_iter_exit(trans, iter);
281 if (k.k && extent_matches_bp(c, bp.btree_id, bp.level, k, bucket, bp))
284 bch2_trans_iter_exit(trans, iter);
285 backpointer_not_found(trans, bp_pos, bp, k);
286 return bkey_s_c_null;
288 struct btree *b = bch2_backpointer_get_node(trans, iter, bp_pos, bp);
290 if (IS_ERR_OR_NULL(b)) {
291 bch2_trans_iter_exit(trans, iter);
292 return IS_ERR(b) ? bkey_s_c_err(PTR_ERR(b)) : bkey_s_c_null;
294 return bkey_i_to_s_c(&b->key);
298 struct btree *bch2_backpointer_get_node(struct btree_trans *trans,
299 struct btree_iter *iter,
301 struct bch_backpointer bp)
303 struct bch_fs *c = trans->c;
304 struct bpos bucket = bp_pos_to_bucket(c, bp_pos);
309 bch2_trans_node_iter_init(trans, iter,
315 b = bch2_btree_iter_peek_node(iter);
316 if (IS_ERR_OR_NULL(b))
319 BUG_ON(b->c.level != bp.level - 1);
321 if (extent_matches_bp(c, bp.btree_id, bp.level,
322 bkey_i_to_s_c(&b->key),
326 if (btree_node_will_make_reachable(b)) {
327 b = ERR_PTR(-BCH_ERR_backpointer_to_overwritten_btree_node);
329 backpointer_not_found(trans, bp_pos, bp, bkey_i_to_s_c(&b->key));
333 bch2_trans_iter_exit(trans, iter);
337 static int bch2_check_btree_backpointer(struct btree_trans *trans, struct btree_iter *bp_iter,
340 struct bch_fs *c = trans->c;
341 struct btree_iter alloc_iter = { NULL };
342 struct bkey_s_c alloc_k;
343 struct printbuf buf = PRINTBUF;
346 if (fsck_err_on(!bch2_dev_exists2(c, k.k->p.inode), c,
347 backpointer_to_missing_device,
348 "backpointer for missing device:\n%s",
349 (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {
350 ret = bch2_btree_delete_at(trans, bp_iter, 0);
354 alloc_k = bch2_bkey_get_iter(trans, &alloc_iter, BTREE_ID_alloc,
355 bp_pos_to_bucket(c, k.k->p), 0);
356 ret = bkey_err(alloc_k);
360 if (fsck_err_on(alloc_k.k->type != KEY_TYPE_alloc_v4, c,
361 backpointer_to_missing_alloc,
362 "backpointer for nonexistent alloc key: %llu:%llu:0\n%s",
363 alloc_iter.pos.inode, alloc_iter.pos.offset,
364 (bch2_bkey_val_to_text(&buf, c, alloc_k), buf.buf))) {
365 ret = bch2_btree_delete_at(trans, bp_iter, 0);
370 bch2_trans_iter_exit(trans, &alloc_iter);
375 /* verify that every backpointer has a corresponding alloc key */
376 int bch2_check_btree_backpointers(struct bch_fs *c)
378 struct btree_iter iter;
382 ret = bch2_trans_run(c,
383 for_each_btree_key_commit(trans, iter,
384 BTREE_ID_backpointers, POS_MIN, 0, k,
385 NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
386 bch2_check_btree_backpointer(trans, &iter, k)));
397 static int check_bp_exists(struct btree_trans *trans,
399 struct bch_backpointer bp,
400 struct bkey_s_c orig_k,
401 struct bpos bucket_start,
402 struct bpos bucket_end,
403 struct bpos_level *last_flushed)
405 struct bch_fs *c = trans->c;
406 struct btree_iter bp_iter = { NULL };
407 struct printbuf buf = PRINTBUF;
408 struct bkey_s_c bp_k;
411 if (bpos_lt(bucket, bucket_start) ||
412 bpos_gt(bucket, bucket_end))
415 if (!bch2_dev_bucket_exists(c, bucket))
418 bp_k = bch2_bkey_get_iter(trans, &bp_iter, BTREE_ID_backpointers,
419 bucket_pos_to_bp(c, bucket, bp.bucket_offset),
421 ret = bkey_err(bp_k);
425 if (bp_k.k->type != KEY_TYPE_backpointer ||
426 memcmp(bkey_s_c_to_backpointer(bp_k).v, &bp, sizeof(bp))) {
427 if (last_flushed->level != bp.level ||
428 !bpos_eq(last_flushed->pos, orig_k.k->p)) {
429 last_flushed->level = bp.level;
430 last_flushed->pos = orig_k.k->p;
432 ret = bch2_btree_write_buffer_flush_sync(trans) ?:
433 -BCH_ERR_transaction_restart_write_buffer_flush;
441 bch2_trans_iter_exit(trans, &bp_iter);
445 prt_printf(&buf, "missing backpointer for btree=%s l=%u ",
446 bch2_btree_id_str(bp.btree_id), bp.level);
447 bch2_bkey_val_to_text(&buf, c, orig_k);
448 prt_printf(&buf, "\nbp pos ");
449 bch2_bpos_to_text(&buf, bp_iter.pos);
451 if (c->sb.version_upgrade_complete < bcachefs_metadata_version_backpointers ||
452 c->opts.reconstruct_alloc ||
453 fsck_err(c, ptr_to_missing_backpointer, "%s", buf.buf))
454 ret = bch2_bucket_backpointer_mod(trans, bucket, bp, orig_k, true);
459 static int check_extent_to_backpointers(struct btree_trans *trans,
460 enum btree_id btree, unsigned level,
461 struct bpos bucket_start,
462 struct bpos bucket_end,
463 struct bpos_level *last_flushed,
466 struct bch_fs *c = trans->c;
467 struct bkey_ptrs_c ptrs;
468 const union bch_extent_entry *entry;
469 struct extent_ptr_decoded p;
472 ptrs = bch2_bkey_ptrs_c(k);
473 bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
474 struct bpos bucket_pos;
475 struct bch_backpointer bp;
480 bch2_extent_ptr_to_bp(c, btree, level,
481 k, p, &bucket_pos, &bp);
483 ret = check_bp_exists(trans, bucket_pos, bp, k,
484 bucket_start, bucket_end,
493 static int check_btree_root_to_backpointers(struct btree_trans *trans,
494 enum btree_id btree_id,
495 struct bpos bucket_start,
496 struct bpos bucket_end,
497 struct bpos_level *last_flushed,
500 struct bch_fs *c = trans->c;
501 struct btree_iter iter;
506 bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN,
507 0, bch2_btree_id_root(c, btree_id)->b->c.level, 0);
508 b = bch2_btree_iter_peek_node(&iter);
509 ret = PTR_ERR_OR_ZERO(b);
513 if (b != btree_node_root(c, b)) {
514 bch2_trans_iter_exit(trans, &iter);
520 k = bkey_i_to_s_c(&b->key);
521 ret = check_extent_to_backpointers(trans, btree_id, b->c.level + 1,
522 bucket_start, bucket_end,
525 bch2_trans_iter_exit(trans, &iter);
529 static inline struct bbpos bp_to_bbpos(struct bch_backpointer bp)
531 return (struct bbpos) {
532 .btree = bp.btree_id,
537 static size_t btree_nodes_fit_in_ram(struct bch_fs *c)
543 mem_bytes = i.totalram * i.mem_unit;
544 return div_u64(mem_bytes >> 1, btree_bytes(c));
547 static int bch2_get_btree_in_memory_pos(struct btree_trans *trans,
548 unsigned btree_leaf_mask,
549 unsigned btree_interior_mask,
550 struct bbpos start, struct bbpos *end)
552 struct btree_iter iter;
554 size_t btree_nodes = btree_nodes_fit_in_ram(trans->c);
558 for (btree = start.btree; btree < BTREE_ID_NR && !ret; btree++) {
559 unsigned depth = ((1U << btree) & btree_leaf_mask) ? 1 : 2;
561 if (!((1U << btree) & btree_leaf_mask) &&
562 !((1U << btree) & btree_interior_mask))
565 bch2_trans_node_iter_init(trans, &iter, btree,
566 btree == start.btree ? start.pos : POS_MIN,
569 * for_each_btree_key_contineu() doesn't check the return value
570 * from bch2_btree_iter_advance(), which is needed when
571 * iterating over interior nodes where we'll see keys at
575 k = __bch2_btree_iter_peek_and_restart(trans, &iter, 0);
582 *end = BBPOS(btree, k.k->p);
583 bch2_trans_iter_exit(trans, &iter);
586 } while (bch2_btree_iter_advance(&iter));
587 bch2_trans_iter_exit(trans, &iter);
594 static int bch2_check_extents_to_backpointers_pass(struct btree_trans *trans,
595 struct bpos bucket_start,
596 struct bpos bucket_end)
598 struct bch_fs *c = trans->c;
599 struct btree_iter iter;
600 enum btree_id btree_id;
602 struct bpos_level last_flushed = { UINT_MAX, POS_MIN };
605 for (btree_id = 0; btree_id < btree_id_nr_alive(c); btree_id++) {
606 int level, depth = btree_type_has_ptrs(btree_id) ? 0 : 1;
608 ret = commit_do(trans, NULL, NULL,
609 BCH_TRANS_COMMIT_lazy_rw|
610 BCH_TRANS_COMMIT_no_enospc,
611 check_btree_root_to_backpointers(trans, btree_id,
612 bucket_start, bucket_end,
613 &last_flushed, &level));
617 while (level >= depth) {
618 bch2_trans_node_iter_init(trans, &iter, btree_id, POS_MIN, 0,
620 BTREE_ITER_PREFETCH);
621 for_each_btree_key_continue(trans, iter, BTREE_ITER_PREFETCH, k, ret) {
622 ret = commit_do(trans, NULL, NULL,
623 BCH_TRANS_COMMIT_lazy_rw|
624 BCH_TRANS_COMMIT_no_enospc,
625 check_extent_to_backpointers(trans, btree_id, level,
626 bucket_start, bucket_end,
631 if (bpos_eq(iter.pos, SPOS_MAX))
634 bch2_trans_iter_exit(trans, &iter);
646 static struct bpos bucket_pos_to_bp_safe(const struct bch_fs *c,
649 return bch2_dev_exists2(c, bucket.inode)
650 ? bucket_pos_to_bp(c, bucket, 0)
654 static int bch2_get_alloc_in_memory_pos(struct btree_trans *trans,
655 struct bpos start, struct bpos *end)
657 struct btree_iter alloc_iter;
658 struct btree_iter bp_iter;
659 struct bkey_s_c alloc_k, bp_k;
660 size_t btree_nodes = btree_nodes_fit_in_ram(trans->c);
661 bool alloc_end = false, bp_end = false;
664 bch2_trans_node_iter_init(trans, &alloc_iter, BTREE_ID_alloc,
666 bch2_trans_node_iter_init(trans, &bp_iter, BTREE_ID_backpointers,
667 bucket_pos_to_bp_safe(trans->c, start), 0, 1, 0);
670 ? __bch2_btree_iter_peek_and_restart(trans, &alloc_iter, 0)
673 ? __bch2_btree_iter_peek_and_restart(trans, &bp_iter, 0)
676 ret = bkey_err(alloc_k) ?: bkey_err(bp_k);
677 if ((!alloc_k.k && !bp_k.k) || ret) {
684 *end = alloc_k.k ? alloc_k.k->p : SPOS_MAX;
688 if (bpos_lt(alloc_iter.pos, SPOS_MAX) &&
689 bpos_lt(bucket_pos_to_bp_safe(trans->c, alloc_iter.pos), bp_iter.pos)) {
690 if (!bch2_btree_iter_advance(&alloc_iter))
693 if (!bch2_btree_iter_advance(&bp_iter))
697 bch2_trans_iter_exit(trans, &bp_iter);
698 bch2_trans_iter_exit(trans, &alloc_iter);
702 int bch2_check_extents_to_backpointers(struct bch_fs *c)
704 struct btree_trans *trans = bch2_trans_get(c);
705 struct bpos start = POS_MIN, end;
709 ret = bch2_get_alloc_in_memory_pos(trans, start, &end);
713 if (bpos_eq(start, POS_MIN) && !bpos_eq(end, SPOS_MAX))
714 bch_verbose(c, "%s(): alloc info does not fit in ram, running in multiple passes with %zu nodes per pass",
715 __func__, btree_nodes_fit_in_ram(c));
717 if (!bpos_eq(start, POS_MIN) || !bpos_eq(end, SPOS_MAX)) {
718 struct printbuf buf = PRINTBUF;
720 prt_str(&buf, "check_extents_to_backpointers(): ");
721 bch2_bpos_to_text(&buf, start);
723 bch2_bpos_to_text(&buf, end);
725 bch_verbose(c, "%s", buf.buf);
729 ret = bch2_check_extents_to_backpointers_pass(trans, start, end);
730 if (ret || bpos_eq(end, SPOS_MAX))
733 start = bpos_successor(end);
735 bch2_trans_put(trans);
742 static int check_one_backpointer(struct btree_trans *trans,
745 struct bkey_s_c_backpointer bp,
746 struct bpos *last_flushed_pos)
748 struct bch_fs *c = trans->c;
749 struct btree_iter iter;
750 struct bbpos pos = bp_to_bbpos(*bp.v);
752 struct printbuf buf = PRINTBUF;
755 if (bbpos_cmp(pos, start) < 0 ||
756 bbpos_cmp(pos, end) > 0)
759 k = bch2_backpointer_get_key(trans, &iter, bp.k->p, *bp.v, 0);
761 if (ret == -BCH_ERR_backpointer_to_overwritten_btree_node)
766 if (!k.k && !bpos_eq(*last_flushed_pos, bp.k->p)) {
767 *last_flushed_pos = bp.k->p;
768 ret = bch2_btree_write_buffer_flush_sync(trans) ?:
769 -BCH_ERR_transaction_restart_write_buffer_flush;
773 if (fsck_err_on(!k.k, c,
774 backpointer_to_missing_ptr,
775 "backpointer for missing %s\n %s",
776 bp.v->level ? "btree node" : "extent",
777 (bch2_bkey_val_to_text(&buf, c, bp.s_c), buf.buf))) {
778 ret = bch2_btree_delete_at_buffered(trans, BTREE_ID_backpointers, bp.k->p);
783 bch2_trans_iter_exit(trans, &iter);
788 static int bch2_check_backpointers_to_extents_pass(struct btree_trans *trans,
792 struct btree_iter iter;
794 struct bpos last_flushed_pos = SPOS_MAX;
796 return for_each_btree_key_commit(trans, iter, BTREE_ID_backpointers,
797 POS_MIN, BTREE_ITER_PREFETCH, k,
798 NULL, NULL, BCH_TRANS_COMMIT_lazy_rw|BCH_TRANS_COMMIT_no_enospc,
799 check_one_backpointer(trans, start, end,
800 bkey_s_c_to_backpointer(k),
804 int bch2_check_backpointers_to_extents(struct bch_fs *c)
806 struct btree_trans *trans = bch2_trans_get(c);
807 struct bbpos start = (struct bbpos) { .btree = 0, .pos = POS_MIN, }, end;
811 ret = bch2_get_btree_in_memory_pos(trans,
812 (1U << BTREE_ID_extents)|
813 (1U << BTREE_ID_reflink),
819 if (!bbpos_cmp(start, BBPOS_MIN) &&
820 bbpos_cmp(end, BBPOS_MAX))
821 bch_verbose(c, "%s(): extents do not fit in ram, running in multiple passes with %zu nodes per pass",
822 __func__, btree_nodes_fit_in_ram(c));
824 if (bbpos_cmp(start, BBPOS_MIN) ||
825 bbpos_cmp(end, BBPOS_MAX)) {
826 struct printbuf buf = PRINTBUF;
828 prt_str(&buf, "check_backpointers_to_extents(): ");
829 bch2_bbpos_to_text(&buf, start);
831 bch2_bbpos_to_text(&buf, end);
833 bch_verbose(c, "%s", buf.buf);
837 ret = bch2_check_backpointers_to_extents_pass(trans, start, end);
838 if (ret || !bbpos_cmp(end, BBPOS_MAX))
841 start = bbpos_successor(end);
843 bch2_trans_put(trans);