};
if (log_u64s[1] >= (log_u64s[0] + log_u64s[2]) / 2) {
- bch2_btree_node_write(c, b, SIX_LOCK_write);
+ bch2_btree_node_write(c, b, SIX_LOCK_write, 0);
reinit_iter = true;
}
}
#define btree_err(type, c, ca, b, i, msg, ...) \
({ \
__label__ out; \
- char _buf[300]; \
- char *_buf2 = _buf; \
- struct printbuf out = PBUF(_buf); \
- \
- _buf2 = kmalloc(4096, GFP_ATOMIC); \
- if (_buf2) \
- out = _PBUF(_buf2, 4986); \
+ struct printbuf out = PRINTBUF; \
\
btree_err_msg(&out, c, ca, b, i, b->written, write); \
pr_buf(&out, ": " msg, ##__VA_ARGS__); \
if (type == BTREE_ERR_FIXABLE && \
write == READ && \
!test_bit(BCH_FS_INITIAL_GC_DONE, &c->flags)) { \
- mustfix_fsck_err(c, "%s", _buf2); \
+ mustfix_fsck_err(c, "%s", out.buf); \
goto out; \
} \
\
switch (write) { \
case READ: \
- if (_buf2) \
- bch_err(c, "%s", _buf2); \
+ bch_err(c, "%s", out.buf); \
\
switch (type) { \
case BTREE_ERR_FIXABLE: \
} \
break; \
case WRITE: \
- bch_err(c, "corrupt metadata before write: %s", _buf2); \
+ bch_err(c, "corrupt metadata before write: %s", out.buf);\
\
if (bch2_fs_inconsistent(c)) { \
ret = BCH_FSCK_ERRORS_NOT_FIXED; \
break; \
} \
out: \
- if (_buf2 != _buf) \
- kfree(_buf2); \
+ printbuf_exit(&out); \
true; \
})
{
unsigned version = le16_to_cpu(i->version);
const char *err;
- char buf1[100];
- char buf2[100];
+ struct printbuf buf1 = PRINTBUF;
+ struct printbuf buf2 = PRINTBUF;
int ret = 0;
btree_err_on((version != BCH_BSET_VERSION_OLD &&
BTREE_ERR_FIXABLE, c, ca, b, i,
"bset past end of btree node")) {
i->u64s = 0;
- return 0;
+ ret = 0;
+ goto out;
}
btree_err_on(offset && !i->u64s,
btree_err_on(bpos_cmp(b->data->min_key, bp->min_key),
BTREE_ERR_MUST_RETRY, c, ca, b, NULL,
"incorrect min_key: got %s should be %s",
- (bch2_bpos_to_text(&PBUF(buf1), bn->min_key), buf1),
- (bch2_bpos_to_text(&PBUF(buf2), bp->min_key), buf2));
+ (printbuf_reset(&buf1),
+ bch2_bpos_to_text(&buf1, bn->min_key), buf1.buf),
+ (printbuf_reset(&buf2),
+ bch2_bpos_to_text(&buf2, bp->min_key), buf2.buf));
}
btree_err_on(bpos_cmp(bn->max_key, b->key.k.p),
BTREE_ERR_MUST_RETRY, c, ca, b, i,
"incorrect max key %s",
- (bch2_bpos_to_text(&PBUF(buf1), bn->max_key), buf1));
+ (printbuf_reset(&buf1),
+ bch2_bpos_to_text(&buf1, bn->max_key), buf1.buf));
if (write)
compat_btree_node(b->c.level, b->c.btree_id, version,
BSET_BIG_ENDIAN(i), write,
&bn->format);
}
+out:
fsck_err:
+ printbuf_exit(&buf2);
+ printbuf_exit(&buf1);
return ret;
}
+static int bset_key_invalid(struct bch_fs *c, struct btree *b,
+ struct bkey_s_c k,
+ bool updated_range, int rw,
+ struct printbuf *err)
+{
+ return __bch2_bkey_invalid(c, k, btree_node_type(b), READ, err) ?:
+ (!updated_range ? bch2_bkey_in_btree_node(b, k, err) : 0) ?:
+ (rw == WRITE ? bch2_bkey_val_invalid(c, k, READ, err) : 0);
+}
+
static int validate_bset_keys(struct bch_fs *c, struct btree *b,
struct bset *i, unsigned *whiteout_u64s,
int write, bool have_retry)
{
unsigned version = le16_to_cpu(i->version);
struct bkey_packed *k, *prev = NULL;
+ struct printbuf buf = PRINTBUF;
bool updated_range = b->key.k.type == KEY_TYPE_btree_ptr_v2 &&
BTREE_PTR_RANGE_UPDATED(&bkey_i_to_btree_ptr_v2(&b->key)->v);
int ret = 0;
k != vstruct_last(i);) {
struct bkey_s u;
struct bkey tmp;
- const char *invalid;
if (btree_err_on(bkey_next(k) > vstruct_last(i),
BTREE_ERR_FIXABLE, c, NULL, b, i,
u = __bkey_disassemble(b, k, &tmp);
- invalid = __bch2_bkey_invalid(c, u.s_c, btree_node_type(b)) ?:
- (!updated_range ? bch2_bkey_in_btree_node(b, u.s_c) : NULL) ?:
- (write ? bch2_bkey_val_invalid(c, u.s_c) : NULL);
- if (invalid) {
- char buf[160];
+ printbuf_reset(&buf);
+ if (bset_key_invalid(c, b, u.s_c, updated_range, write, &buf)) {
+ printbuf_reset(&buf);
+ pr_buf(&buf, "invalid bkey: ");
+ bset_key_invalid(c, b, u.s_c, updated_range, write, &buf);
+ pr_buf(&buf, "\n ");
+ bch2_bkey_val_to_text(&buf, c, u.s_c);
- bch2_bkey_val_to_text(&PBUF(buf), c, u.s_c);
- btree_err(BTREE_ERR_FIXABLE, c, NULL, b, i,
- "invalid bkey: %s\n%s", invalid, buf);
+ btree_err(BTREE_ERR_FIXABLE, c, NULL, b, i, "%s", buf.buf);
i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
memmove_u64s_down(k, bkey_next(k),
&b->format, k);
if (prev && bkey_iter_cmp(b, prev, k) > 0) {
- char buf1[80];
- char buf2[80];
struct bkey up = bkey_unpack_key(b, prev);
- bch2_bkey_to_text(&PBUF(buf1), &up);
- bch2_bkey_to_text(&PBUF(buf2), u.k);
+ printbuf_reset(&buf);
+ pr_buf(&buf, "keys out of order: ");
+ bch2_bkey_to_text(&buf, &up);
+ pr_buf(&buf, " > ");
+ bch2_bkey_to_text(&buf, u.k);
bch2_dump_bset(c, b, i, 0);
- if (btree_err(BTREE_ERR_FIXABLE, c, NULL, b, i,
- "keys out of order: %s > %s",
- buf1, buf2)) {
+ if (btree_err(BTREE_ERR_FIXABLE, c, NULL, b, i, "%s", buf.buf)) {
i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
memmove_u64s_down(k, bkey_next(k),
(u64 *) vstruct_end(i) - (u64 *) k);
k = bkey_next(k);
}
fsck_err:
+ printbuf_exit(&buf);
return ret;
}
unsigned u64s;
unsigned blacklisted_written, nonblacklisted_written = 0;
unsigned ptr_written = btree_ptr_sectors_written(&b->key);
+ struct printbuf buf = PRINTBUF;
int ret, retry_read = 0, write = READ;
b->version_ondisk = U16_MAX;
btree_err_on(le64_to_cpu(b->data->magic) != bset_magic(c),
BTREE_ERR_MUST_RETRY, c, ca, b, NULL,
- "bad magic");
+ "bad magic: want %llx, got %llx",
+ bset_magic(c), le64_to_cpu(b->data->magic));
btree_err_on(!b->data->keys.seq,
BTREE_ERR_MUST_RETRY, c, ca, b, NULL,
- "bad btree header");
+ "bad btree header: seq 0");
if (b->key.k.type == KEY_TYPE_btree_ptr_v2) {
struct bch_btree_ptr_v2 *bp =
"error decrypting btree node: %i", ret))
goto fsck_err;
- btree_err_on(btree_node_is_extents(b) &&
+ btree_err_on(btree_node_type_is_extents(btree_node_type(b)) &&
!BTREE_NODE_NEW_EXTENT_OVERWRITE(b->data),
BTREE_ERR_FATAL, c, NULL, b, NULL,
"btree node does not have NEW_EXTENT_OVERWRITE set");
for (k = i->start; k != vstruct_last(i);) {
struct bkey tmp;
struct bkey_s u = __bkey_disassemble(b, k, &tmp);
- const char *invalid = bch2_bkey_val_invalid(c, u.s_c);
- if (invalid ||
+ printbuf_reset(&buf);
+
+ if (bch2_bkey_val_invalid(c, u.s_c, READ, &buf) ||
(bch2_inject_invalid_keys &&
!bversion_cmp(u.k->version, MAX_VERSION))) {
- char buf[160];
+ printbuf_reset(&buf);
- bch2_bkey_val_to_text(&PBUF(buf), c, u.s_c);
- btree_err(BTREE_ERR_FIXABLE, c, NULL, b, i,
- "invalid bkey %s: %s", buf, invalid);
+ pr_buf(&buf, "invalid bkey: ");
+ bch2_bkey_val_invalid(c, u.s_c, READ, &buf);
+ pr_buf(&buf, "\n ");
+ bch2_bkey_val_to_text(&buf, c, u.s_c);
+
+ btree_err(BTREE_ERR_FIXABLE, c, NULL, b, i, "%s", buf.buf);
btree_keys_account_key_drop(&b->nr, 0, k);
set_btree_node_need_rewrite(b);
out:
mempool_free(iter, &c->fill_iter);
+ printbuf_exit(&buf);
return retry_read;
fsck_err:
if (ret == BTREE_RETRY_READ) {
struct bch_dev *ca = bch_dev_bkey_exists(c, rb->pick.ptr.dev);
struct bio *bio = &rb->bio;
struct bch_io_failures failed = { .nr = 0 };
- char buf[200];
- struct printbuf out;
+ struct printbuf buf = PRINTBUF;
bool saw_error = false;
bool can_retry;
bio->bi_status = BLK_STS_REMOVED;
}
start:
- out = PBUF(buf);
- btree_pos_to_text(&out, c, b);
+ printbuf_reset(&buf);
+ btree_pos_to_text(&buf, c, b);
bch2_dev_io_err_on(bio->bi_status, ca, "btree read error %s for %s",
- bch2_blk_status_to_str(bio->bi_status), buf);
+ bch2_blk_status_to_str(bio->bi_status), buf.buf);
if (rb->have_ioref)
percpu_ref_put(&ca->io_ref);
rb->have_ioref = false;
bch2_time_stats_update(&c->times[BCH_TIME_btree_node_read],
rb->start_time);
bio_put(&rb->bio);
+ printbuf_exit(&buf);
if (saw_error && !btree_node_read_error(b))
bch2_btree_node_rewrite_async(c, b);
container_of(cl, struct btree_node_read_all, cl);
struct bch_fs *c = ra->c;
struct btree *b = ra->b;
+ struct printbuf buf = PRINTBUF;
bool dump_bset_maps = false;
bool have_retry = false;
int ret = 0, best = -1, write = READ;
fsck_err:
if (dump_bset_maps) {
for (i = 0; i < ra->nr; i++) {
- char buf[200];
- struct printbuf out = PBUF(buf);
struct btree_node *bn = ra->buf[i];
struct btree_node_entry *bne = NULL;
unsigned offset = 0, sectors;
if (ra->err[i])
continue;
+ printbuf_reset(&buf);
+
while (offset < btree_sectors(c)) {
if (!offset) {
sectors = vstruct_sectors(bn, c->block_bits);
sectors = vstruct_sectors(bne, c->block_bits);
}
- pr_buf(&out, " %u-%u", offset, offset + sectors);
+ pr_buf(&buf, " %u-%u", offset, offset + sectors);
if (bne && bch2_journal_seq_is_blacklisted(c,
le64_to_cpu(bne->keys.journal_seq), false))
- pr_buf(&out, "*");
+ pr_buf(&buf, "*");
offset += sectors;
}
bne = ra->buf[i] + (offset << 9);
if (bne->keys.seq == bn->keys.seq) {
if (!gap)
- pr_buf(&out, " GAP");
+ pr_buf(&buf, " GAP");
gap = true;
sectors = vstruct_sectors(bne, c->block_bits);
- pr_buf(&out, " %u-%u", offset, offset + sectors);
+ pr_buf(&buf, " %u-%u", offset, offset + sectors);
if (bch2_journal_seq_is_blacklisted(c,
le64_to_cpu(bne->keys.journal_seq), false))
- pr_buf(&out, "*");
+ pr_buf(&buf, "*");
}
offset++;
}
- bch_err(c, "replica %u:%s", i, buf);
+ bch_err(c, "replica %u:%s", i, buf.buf);
}
}
closure_debug_destroy(&ra->cl);
kfree(ra);
+ printbuf_exit(&buf);
clear_btree_node_read_in_flight(b);
wake_up_bit(&b->flags, BTREE_NODE_read_in_flight);
struct btree_read_bio *rb;
struct bch_dev *ca;
struct bio *bio;
- char buf[200];
+ struct printbuf buf = PRINTBUF;
int ret;
- btree_pos_to_text(&PBUF(buf), c, b);
+ btree_pos_to_text(&buf, c, b);
trace_btree_read(c, b);
if (bch2_verify_all_btree_replicas &&
!btree_node_read_all_replicas(c, b, sync))
- return;
+ goto out;
ret = bch2_bkey_pick_read_device(c, bkey_i_to_s_c(&b->key),
NULL, &pick);
if (bch2_fs_fatal_err_on(ret <= 0, c,
"btree node read error: no device to read from\n"
- " at %s", buf)) {
+ " at %s", buf.buf)) {
set_btree_node_read_error(b);
- return;
+ goto out;
}
ca = bch_dev_bkey_exists(c, pick.ptr.dev);
else
queue_work(c->io_complete_wq, &rb->work);
}
+out:
+ printbuf_exit(&buf);
}
int bch2_btree_root_read(struct bch_fs *c, enum btree_id id,
closure_sync(&cl);
} while (ret);
- b = bch2_btree_node_mem_alloc(c);
+ b = bch2_btree_node_mem_alloc(c, level != 0);
bch2_btree_cache_cannibalize_unlock(c);
BUG_ON(IS_ERR(b));
bch2_journal_pin_drop(&c->journal, &w->journal);
}
-static void btree_node_write_done(struct bch_fs *c, struct btree *b)
+static void __btree_node_write_done(struct bch_fs *c, struct btree *b)
{
struct btree_write *w = btree_prev_write(b);
unsigned long old, new, v;
bch2_btree_complete_write(c, b, w);
- v = READ_ONCE(b->flags);
- do {
- old = new = v;
-
- if (old & (1U << BTREE_NODE_need_write))
- goto do_write;
-
- new &= ~(1U << BTREE_NODE_write_in_flight);
- new &= ~(1U << BTREE_NODE_write_in_flight_inner);
- } while ((v = cmpxchg(&b->flags, old, new)) != old);
-
- wake_up_bit(&b->flags, BTREE_NODE_write_in_flight);
- return;
-
-do_write:
- six_lock_read(&b->c.lock, NULL, NULL);
v = READ_ONCE(b->flags);
do {
old = new = v;
if ((old & (1U << BTREE_NODE_dirty)) &&
(old & (1U << BTREE_NODE_need_write)) &&
!(old & (1U << BTREE_NODE_never_write)) &&
- btree_node_may_write(b)) {
+ !(old & (1U << BTREE_NODE_write_blocked)) &&
+ !(old & (1U << BTREE_NODE_will_make_reachable))) {
new &= ~(1U << BTREE_NODE_dirty);
new &= ~(1U << BTREE_NODE_need_write);
new |= (1U << BTREE_NODE_write_in_flight);
} while ((v = cmpxchg(&b->flags, old, new)) != old);
if (new & (1U << BTREE_NODE_write_in_flight))
- __bch2_btree_node_write(c, b, true);
+ __bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED);
+ else
+ wake_up_bit(&b->flags, BTREE_NODE_write_in_flight);
+}
+static void btree_node_write_done(struct bch_fs *c, struct btree *b)
+{
+ six_lock_read(&b->c.lock, NULL, NULL);
+ __btree_node_write_done(c, b);
six_unlock_read(&b->c.lock);
}
struct bset *i, unsigned sectors)
{
unsigned whiteout_u64s = 0;
+ struct printbuf buf = PRINTBUF;
int ret;
- if (bch2_bkey_invalid(c, bkey_i_to_s_c(&b->key), BKEY_TYPE_btree))
- return -1;
+ ret = bch2_bkey_invalid(c, bkey_i_to_s_c(&b->key),
+ BKEY_TYPE_btree, WRITE, &buf);
+
+ if (ret)
+ bch2_fs_inconsistent(c, "invalid btree node key before write: %s", buf.buf);
+ printbuf_exit(&buf);
+ if (ret)
+ return ret;
ret = validate_bset_keys(c, b, i, &whiteout_u64s, WRITE, false) ?:
validate_bset(c, NULL, b, i, b->written, sectors, WRITE, false);
bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree, &tmp.k);
}
-void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, bool already_started)
+void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags)
{
struct btree_write_bio *wbio;
struct bset_tree *t;
void *data;
int ret;
- if (already_started)
+ if (flags & BTREE_WRITE_ALREADY_STARTED)
goto do_write;
- if (test_bit(BCH_FS_HOLD_BTREE_WRITES, &c->flags))
- return;
-
/*
* We may only have a read lock on the btree node - the dirty bit is our
* "lock" against racing with other threads that may be trying to start
if (!(old & (1 << BTREE_NODE_dirty)))
return;
- if (!btree_node_may_write(b))
+ if ((flags & BTREE_WRITE_ONLY_IF_NEED) &&
+ !(old & (1 << BTREE_NODE_need_write)))
return;
- if (old & (1 << BTREE_NODE_never_write))
+ if (old &
+ ((1 << BTREE_NODE_never_write)|
+ (1 << BTREE_NODE_write_blocked)))
return;
- BUG_ON(old & (1 << BTREE_NODE_write_in_flight));
+ if (b->written &&
+ (old & (1 << BTREE_NODE_will_make_reachable)))
+ return;
+
+ if (old & (1 << BTREE_NODE_write_in_flight))
+ return;
new &= ~(1 << BTREE_NODE_dirty);
new &= ~(1 << BTREE_NODE_need_write);
BUG_ON(BSET_BIG_ENDIAN(i) != CPU_BIG_ENDIAN);
BUG_ON(i->seq != b->data->keys.seq);
- i->version = c->sb.version < bcachefs_metadata_version_new_versioning
+ i->version = c->sb.version < bcachefs_metadata_version_bkey_renumber
? cpu_to_le16(BCH_BSET_VERSION_OLD)
: cpu_to_le16(c->sb.version);
SET_BSET_OFFSET(i, b->written);
b->written += sectors_to_write;
nowrite:
btree_bounce_free(c, bytes, used_mempool, data);
- btree_node_write_done(c, b);
+ __btree_node_write_done(c, b);
}
/*
* Use this one if the node is intent locked:
*/
void bch2_btree_node_write(struct bch_fs *c, struct btree *b,
- enum six_lock_type lock_type_held)
+ enum six_lock_type lock_type_held,
+ unsigned flags)
{
if (lock_type_held == SIX_LOCK_intent ||
(lock_type_held == SIX_LOCK_read &&
six_lock_tryupgrade(&b->c.lock))) {
- __bch2_btree_node_write(c, b, false);
+ __bch2_btree_node_write(c, b, flags);
/* don't cycle lock unnecessarily: */
if (btree_node_just_written(b) &&
if (lock_type_held == SIX_LOCK_read)
six_lock_downgrade(&b->c.lock);
} else {
- __bch2_btree_node_write(c, b, false);
+ __bch2_btree_node_write(c, b, flags);
if (lock_type_held == SIX_LOCK_write &&
btree_node_just_written(b))
bch2_btree_post_write_cleanup(c, b);
}
}
-static void __bch2_btree_flush_all(struct bch_fs *c, unsigned flag)
+static bool __bch2_btree_flush_all(struct bch_fs *c, unsigned flag)
{
struct bucket_table *tbl;
struct rhash_head *pos;
struct btree *b;
unsigned i;
+ bool ret = false;
restart:
rcu_read_lock();
for_each_cached_btree(b, c, tbl, i, pos)
if (test_bit(flag, &b->flags)) {
rcu_read_unlock();
wait_on_bit_io(&b->flags, flag, TASK_UNINTERRUPTIBLE);
+ ret = true;
goto restart;
-
}
rcu_read_unlock();
-}
-void bch2_btree_flush_all_reads(struct bch_fs *c)
-{
- __bch2_btree_flush_all(c, BTREE_NODE_read_in_flight);
+ return ret;
}
-void bch2_btree_flush_all_writes(struct bch_fs *c)
+bool bch2_btree_flush_all_reads(struct bch_fs *c)
{
- __bch2_btree_flush_all(c, BTREE_NODE_write_in_flight);
+ return __bch2_btree_flush_all(c, BTREE_NODE_read_in_flight);
}
-void bch2_dirty_btree_nodes_to_text(struct printbuf *out, struct bch_fs *c)
+bool bch2_btree_flush_all_writes(struct bch_fs *c)
{
- struct bucket_table *tbl;
- struct rhash_head *pos;
- struct btree *b;
- unsigned i;
-
- rcu_read_lock();
- for_each_cached_btree(b, c, tbl, i, pos) {
- unsigned long flags = READ_ONCE(b->flags);
-
- if (!(flags & (1 << BTREE_NODE_dirty)))
- continue;
-
- pr_buf(out, "%p d %u n %u l %u w %u b %u r %u:%lu\n",
- b,
- (flags & (1 << BTREE_NODE_dirty)) != 0,
- (flags & (1 << BTREE_NODE_need_write)) != 0,
- b->c.level,
- b->written,
- !list_empty_careful(&b->write_blocked),
- b->will_make_reachable != 0,
- b->will_make_reachable & 1);
- }
- rcu_read_unlock();
+ return __bch2_btree_flush_all(c, BTREE_NODE_write_in_flight);
}
projects / bcachefs-tools-debian / blobdiff