From: Kent Overstreet Date: Fri, 18 Nov 2022 23:21:11 +0000 (-0500) Subject: Update bcachefs sources to 6406e05835 bcachefs: Nocow support X-Git-Url: https://git.sesse.net/?a=commitdiff_plain;h=c1f55a60c41ca5ab8ed7a0893c3d29f8006da82a;hp=e4716b10ed0210a13efdd3252c12199da3d52aad;p=bcachefs-tools-debian Update bcachefs sources to 6406e05835 bcachefs: Nocow support --- diff --git a/.bcachefs_revision b/.bcachefs_revision index 1b5f928..7a950b3 100644 --- a/.bcachefs_revision +++ b/.bcachefs_revision @@ -1 +1 @@ -8d3fc97ca3f24d8f7ab1e9ed04d8ca354c44dd8c +6406e0583565af5f7c51f9c43a34e5f7a92dadcd diff --git a/libbcachefs/alloc_foreground.c b/libbcachefs/alloc_foreground.c index 949c068..c4f971c 100644 --- a/libbcachefs/alloc_foreground.c +++ b/libbcachefs/alloc_foreground.c @@ -216,6 +216,11 @@ static struct open_bucket *__try_alloc_bucket(struct bch_fs *c, struct bch_dev * return NULL; } + if (bch2_bucket_nocow_is_locked(&c->nocow_locks, POS(ca->dev_idx, bucket))) { + s->skipped_nocow++; + return NULL; + } + spin_lock(&c->freelist_lock); if (unlikely(c->open_buckets_nr_free <= open_buckets_reserved(reserve))) { diff --git a/libbcachefs/alloc_types.h b/libbcachefs/alloc_types.h index e66a85f..271b4bf 100644 --- a/libbcachefs/alloc_types.h +++ b/libbcachefs/alloc_types.h @@ -13,6 +13,7 @@ struct bucket_alloc_state { u64 buckets_seen; u64 skipped_open; u64 skipped_need_journal_commit; + u64 skipped_nocow; u64 skipped_nouse; }; diff --git a/libbcachefs/bcachefs.h b/libbcachefs/bcachefs.h index 8a43fcf..3334100 100644 --- a/libbcachefs/bcachefs.h +++ b/libbcachefs/bcachefs.h @@ -206,6 +206,7 @@ #include "bcachefs_format.h" #include "errcode.h" #include "fifo.h" +#include "nocow_locking.h" #include "opts.h" #include "util.h" @@ -230,15 +231,27 @@ do { \ #endif #ifdef BCACHEFS_LOG_PREFIX -#define bch2_log_msg(_c, fmt) "bcachefs (%s): " fmt, ((_c)->name) -#define bch2_fmt(_c, fmt) bch2_log_msg(_c, fmt "\n") -#define bch2_fmt_inum(_c, _inum, fmt) "bcachefs (%s inum %llu): " fmt "\n", ((_c)->name), (_inum) + +#define bch2_log_msg(_c, fmt) "bcachefs (%s): " fmt, ((_c)->name) +#define bch2_fmt_dev(_ca, fmt) "bcachefs (%s): " fmt "\n", ((_ca)->name) +#define bch2_fmt_dev_offset(_ca, _offset, fmt) "bcachefs (%s sector %llu): " fmt "\n", ((_ca)->name), (_offset) +#define bch2_fmt_inum(_c, _inum, fmt) "bcachefs (%s inum %llu): " fmt "\n", ((_c)->name), (_inum) +#define bch2_fmt_inum_offset(_c, _inum, _offset, fmt) \ + "bcachefs (%s inum %llu offset %llu): " fmt "\n", ((_c)->name), (_inum), (_offset) + #else -#define bch2_log_msg(_c, fmt) fmt -#define bch2_fmt(_c, fmt) fmt "\n" -#define bch2_fmt_inum(_c, _inum, fmt) "inum %llu: " fmt "\n", (_inum) + +#define bch2_log_msg(_c, fmt) fmt +#define bch2_fmt_dev(_ca, fmt) "%s: " fmt "\n", ((_ca)->name) +#define bch2_fmt_dev_offset(_ca, _offset, fmt) "%s sector %llu: " fmt "\n", ((_ca)->name), (_offset) +#define bch2_fmt_inum(_c, _inum, fmt) "inum %llu: " fmt "\n", (_inum) +#define bch2_fmt_inum_offset(_c, _inum, _offset, fmt) \ + "inum %llu offset %llu: " fmt "\n", (_inum), (_offset) + #endif +#define bch2_fmt(_c, fmt) bch2_log_msg(_c, fmt "\n") + #define bch_info(c, fmt, ...) \ printk(KERN_INFO bch2_fmt(c, fmt), ##__VA_ARGS__) #define bch_notice(c, fmt, ...) \ @@ -247,13 +260,28 @@ do { \ printk(KERN_WARNING bch2_fmt(c, fmt), ##__VA_ARGS__) #define bch_warn_ratelimited(c, fmt, ...) \ printk_ratelimited(KERN_WARNING bch2_fmt(c, fmt), ##__VA_ARGS__) + #define bch_err(c, fmt, ...) \ printk(KERN_ERR bch2_fmt(c, fmt), ##__VA_ARGS__) +#define bch_err_dev(ca, fmt, ...) \ + printk(KERN_ERR bch2_fmt_dev(ca, fmt), ##__VA_ARGS__) +#define bch_err_dev_offset(ca, _offset, fmt, ...) \ + printk(KERN_ERR bch2_fmt_dev_offset(ca, _offset, fmt), ##__VA_ARGS__) +#define bch_err_inum(c, _inum, fmt, ...) \ + printk(KERN_ERR bch2_fmt_inum(c, _inum, fmt), ##__VA_ARGS__) +#define bch_err_inum_offset(c, _inum, _offset, fmt, ...) \ + printk(KERN_ERR bch2_fmt_inum_offset(c, _inum, _offset, fmt), ##__VA_ARGS__) #define bch_err_ratelimited(c, fmt, ...) \ printk_ratelimited(KERN_ERR bch2_fmt(c, fmt), ##__VA_ARGS__) +#define bch_err_dev_ratelimited(ca, fmt, ...) \ + printk_ratelimited(KERN_ERR bch2_fmt_dev(ca, fmt), ##__VA_ARGS__) +#define bch_err_dev_offset_ratelimited(ca, _offset, fmt, ...) \ + printk_ratelimited(KERN_ERR bch2_fmt_dev_offset(ca, _offset, fmt), ##__VA_ARGS__) #define bch_err_inum_ratelimited(c, _inum, fmt, ...) \ printk_ratelimited(KERN_ERR bch2_fmt_inum(c, _inum, fmt), ##__VA_ARGS__) +#define bch_err_inum_offset_ratelimited(c, _inum, _offset, fmt, ...) \ + printk_ratelimited(KERN_ERR bch2_fmt_inum_offset(c, _inum, _offset, fmt), ##__VA_ARGS__) #define bch_verbose(c, fmt, ...) \ do { \ @@ -349,7 +377,8 @@ BCH_DEBUG_PARAMS_DEBUG() x(journal_flush_seq) \ x(blocked_journal) \ x(blocked_allocate) \ - x(blocked_allocate_open_bucket) + x(blocked_allocate_open_bucket) \ + x(nocow_lock_contended) enum bch_time_stats { #define x(name) BCH_TIME_##name, @@ -448,6 +477,7 @@ struct bch_dev { struct bch_sb *sb_read_scratch; int sb_write_error; dev_t dev; + atomic_t flush_seq; struct bch_devs_mask self; @@ -602,23 +632,6 @@ typedef struct { #define BCACHEFS_ROOT_SUBVOL_INUM \ ((subvol_inum) { BCACHEFS_ROOT_SUBVOL, BCACHEFS_ROOT_INO }) -#define BCH_BTREE_WRITE_TYPES() \ - x(initial, 0) \ - x(init_next_bset, 1) \ - x(cache_reclaim, 2) \ - x(journal_reclaim, 3) \ - x(interior, 4) - -enum btree_write_type { -#define x(t, n) BTREE_WRITE_##t, - BCH_BTREE_WRITE_TYPES() -#undef x - BTREE_WRITE_TYPE_NR, -}; - -#define BTREE_WRITE_TYPE_MASK (roundup_pow_of_two(BTREE_WRITE_TYPE_NR) - 1) -#define BTREE_WRITE_TYPE_BITS ilog2(BTREE_WRITE_TYPE_MASK) - struct bch_fs { struct closure cl; @@ -847,6 +860,8 @@ struct bch_fs { struct bio_set bio_write; struct mutex bio_bounce_pages_lock; mempool_t bio_bounce_pages; + struct bucket_nocow_lock_table + nocow_locks; struct rhashtable promote_table; mempool_t compression_bounce[2]; @@ -908,6 +923,7 @@ struct bch_fs { struct bio_set writepage_bioset; struct bio_set dio_write_bioset; struct bio_set dio_read_bioset; + struct bio_set nocow_flush_bioset; /* ERRORS */ struct list_head fsck_errors; diff --git a/libbcachefs/bcachefs_format.h b/libbcachefs/bcachefs_format.h index 5da9f3a..0aa522b 100644 --- a/libbcachefs/bcachefs_format.h +++ b/libbcachefs/bcachefs_format.h @@ -578,7 +578,7 @@ struct bch_extent_ptr { __u64 type:1, cached:1, unused:1, - reservation:1, + unwritten:1, offset:44, /* 8 petabytes */ dev:8, gen:8; @@ -586,7 +586,7 @@ struct bch_extent_ptr { __u64 gen:8, dev:8, offset:44, - reservation:1, + unwritten:1, unused:1, cached:1, type:1; @@ -794,7 +794,8 @@ struct bch_inode_generation { x(bi_dir, 64) \ x(bi_dir_offset, 64) \ x(bi_subvol, 32) \ - x(bi_parent_subvol, 32) + x(bi_parent_subvol, 32) \ + x(bi_nocow, 8) /* subset of BCH_INODE_FIELDS */ #define BCH_INODE_OPTS() \ @@ -806,7 +807,8 @@ struct bch_inode_generation { x(promote_target, 16) \ x(foreground_target, 16) \ x(background_target, 16) \ - x(erasure_code, 16) + x(erasure_code, 16) \ + x(nocow, 8) enum inode_opt_id { #define x(name, ...) \ @@ -1543,7 +1545,8 @@ struct bch_sb_field_journal_seq_blacklist { x(alloc_v4, 20) \ x(new_data_types, 21) \ x(backpointers, 22) \ - x(inode_v3, 23) + x(inode_v3, 23) \ + x(unwritten_extents, 24) enum bcachefs_metadata_version { bcachefs_metadata_version_min = 9, @@ -1692,6 +1695,7 @@ LE64_BITMASK(BCH_SB_JOURNAL_FLUSH_DISABLED,struct bch_sb, flags[3], 62, 63); LE64_BITMASK(BCH_SB_JOURNAL_RECLAIM_DELAY,struct bch_sb, flags[4], 0, 32); /* Obsolete, always enabled: */ LE64_BITMASK(BCH_SB_JOURNAL_TRANSACTION_NAMES,struct bch_sb, flags[4], 32, 33); +LE64_BITMASK(BCH_SB_NOCOW, struct bch_sb, flags[4], 33, 34); /* * Features: diff --git a/libbcachefs/bkey_sort.c b/libbcachefs/bkey_sort.c index be0d4bc..557a79c 100644 --- a/libbcachefs/bkey_sort.c +++ b/libbcachefs/bkey_sort.c @@ -144,6 +144,8 @@ bch2_sort_repack(struct bset *dst, struct btree *src, else bch2_bkey_unpack(src, (void *) out, in); + out->needs_whiteout = false; + btree_keys_account_key_add(&nr, 0, out); out = bkey_next(out); } diff --git a/libbcachefs/bset.h b/libbcachefs/bset.h index 72e6376..acef143 100644 --- a/libbcachefs/bset.h +++ b/libbcachefs/bset.h @@ -432,6 +432,11 @@ struct bkey_s_c bch2_btree_node_iter_peek_unpack(struct btree_node_iter *, struct btree *, struct bkey *); +#define for_each_btree_node_key(b, k, iter) \ + for (bch2_btree_node_iter_init_from_start((iter), (b)); \ + (k = bch2_btree_node_iter_peek((iter), (b))); \ + bch2_btree_node_iter_advance(iter, b)) + #define for_each_btree_node_key_unpack(b, k, iter, unpacked) \ for (bch2_btree_node_iter_init_from_start((iter), (b)); \ (k = bch2_btree_node_iter_peek_unpack((iter), (b), (unpacked))).k;\ diff --git a/libbcachefs/btree_cache.c b/libbcachefs/btree_cache.c index 75e7447..2ca0a9d 100644 --- a/libbcachefs/btree_cache.c +++ b/libbcachefs/btree_cache.c @@ -677,7 +677,6 @@ out: b->flags = 0; b->written = 0; b->nsets = 0; - b->write_type = 0; b->sib_u64s[0] = 0; b->sib_u64s[1] = 0; b->whiteout_u64s = 0; diff --git a/libbcachefs/btree_io.c b/libbcachefs/btree_io.c index cee3b50..48f213f 100644 --- a/libbcachefs/btree_io.c +++ b/libbcachefs/btree_io.c @@ -1641,6 +1641,7 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b) { struct btree_write *w = btree_prev_write(b); unsigned long old, new, v; + unsigned type = 0; bch2_btree_complete_write(c, b, w); @@ -1659,6 +1660,9 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b) new |= (1U << BTREE_NODE_write_in_flight_inner); new |= (1U << BTREE_NODE_just_written); new ^= (1U << BTREE_NODE_write_idx); + + type = new & BTREE_WRITE_TYPE_MASK; + new &= ~BTREE_WRITE_TYPE_MASK; } else { new &= ~(1U << BTREE_NODE_write_in_flight); new &= ~(1U << BTREE_NODE_write_in_flight_inner); @@ -1666,7 +1670,7 @@ static void __btree_node_write_done(struct bch_fs *c, struct btree *b) } while ((v = cmpxchg(&b->flags, old, new)) != old); if (new & (1U << BTREE_NODE_write_in_flight)) - __bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|b->write_type); + __bch2_btree_node_write(c, b, BTREE_WRITE_ALREADY_STARTED|type); else wake_up_bit(&b->flags, BTREE_NODE_write_in_flight); } @@ -1798,7 +1802,8 @@ static void btree_write_submit(struct work_struct *work) bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&tmp.k)), ptr) ptr->offset += wbio->sector_offset; - bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree, &tmp.k); + bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree, + &tmp.k, false); } void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) @@ -1851,6 +1856,10 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) if (old & (1 << BTREE_NODE_write_in_flight)) return; + if (flags & BTREE_WRITE_ONLY_IF_NEED) + type = new & BTREE_WRITE_TYPE_MASK; + new &= ~BTREE_WRITE_TYPE_MASK; + new &= ~(1 << BTREE_NODE_dirty); new &= ~(1 << BTREE_NODE_need_write); new |= (1 << BTREE_NODE_write_in_flight); @@ -1862,10 +1871,6 @@ void __bch2_btree_node_write(struct bch_fs *c, struct btree *b, unsigned flags) if (new & (1U << BTREE_NODE_need_write)) return; do_write: - if ((flags & BTREE_WRITE_ONLY_IF_NEED)) - type = b->write_type; - b->write_type = 0; - BUG_ON((type == BTREE_WRITE_initial) != (b->written == 0)); atomic_dec(&c->btree_cache.dirty); diff --git a/libbcachefs/btree_iter.c b/libbcachefs/btree_iter.c index 5080f56..b6a761d 100644 --- a/libbcachefs/btree_iter.c +++ b/libbcachefs/btree_iter.c @@ -1091,7 +1091,7 @@ static int btree_path_traverse_one(struct btree_trans *trans, unsigned long trace_ip) { unsigned depth_want = path->level; - int ret = trans->restarted; + int ret = -((int) trans->restarted); if (unlikely(ret)) goto out; diff --git a/libbcachefs/btree_iter.h b/libbcachefs/btree_iter.h index bad51ce..8ed5aee 100644 --- a/libbcachefs/btree_iter.h +++ b/libbcachefs/btree_iter.h @@ -347,6 +347,11 @@ __btree_iter_peek_node_and_restart(struct btree_trans *trans, struct btree_iter return b; } +/* + * XXX + * this does not handle transaction restarts from bch2_btree_iter_next_node() + * correctly + */ #define __for_each_btree_node(_trans, _iter, _btree_id, _start, \ _locks_want, _depth, _flags, _b, _ret) \ for (bch2_trans_node_iter_init((_trans), &(_iter), (_btree_id), \ @@ -473,11 +478,11 @@ __bch2_btree_iter_peek_and_restart(struct btree_trans *trans, \ while (1) { \ u32 _restart_count = bch2_trans_begin(_trans); \ + \ + _ret = 0; \ (_k) = bch2_btree_iter_peek_type(&(_iter), (_flags)); \ - if (!(_k).k) { \ - _ret = 0; \ + if (!(_k).k) \ break; \ - } \ \ _ret = bkey_err(_k) ?: (_do); \ if (bch2_err_matches(_ret, BCH_ERR_transaction_restart))\ diff --git a/libbcachefs/btree_key_cache.c b/libbcachefs/btree_key_cache.c index 66fb698..634c673 100644 --- a/libbcachefs/btree_key_cache.c +++ b/libbcachefs/btree_key_cache.c @@ -190,7 +190,8 @@ static void bkey_cached_free_fast(struct btree_key_cache *bc, } static struct bkey_cached * -bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path) +bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path, + bool *was_new) { struct bch_fs *c = trans->c; struct btree_key_cache *bc = &c->btree_key_cache; @@ -274,6 +275,7 @@ bkey_cached_alloc(struct btree_trans *trans, struct btree_path *path) ck->c.cached = true; BUG_ON(!six_trylock_intent(&ck->c.lock)); BUG_ON(!six_trylock_write(&ck->c.lock)); + *was_new = true; return ck; } @@ -312,9 +314,9 @@ btree_key_cache_create(struct btree_trans *trans, struct btree_path *path) struct bch_fs *c = trans->c; struct btree_key_cache *bc = &c->btree_key_cache; struct bkey_cached *ck; - bool was_new = true; + bool was_new = false; - ck = bkey_cached_alloc(trans, path); + ck = bkey_cached_alloc(trans, path, &was_new); if (IS_ERR(ck)) return ck; @@ -327,7 +329,6 @@ btree_key_cache_create(struct btree_trans *trans, struct btree_path *path) } mark_btree_node_locked(trans, path, 0, SIX_LOCK_intent); - was_new = false; } else { if (path->btree_id == BTREE_ID_subvolumes) six_lock_pcpu_alloc(&ck->c.lock); @@ -348,12 +349,12 @@ btree_key_cache_create(struct btree_trans *trans, struct btree_path *path) if (likely(was_new)) { six_unlock_write(&ck->c.lock); six_unlock_intent(&ck->c.lock); - mark_btree_node_locked(trans, path, 0, BTREE_NODE_UNLOCKED); kfree(ck); } else { bkey_cached_free_fast(bc, ck); } + mark_btree_node_locked(trans, path, 0, BTREE_NODE_UNLOCKED); return NULL; } diff --git a/libbcachefs/btree_types.h b/libbcachefs/btree_types.h index d89489e..6d44b2e 100644 --- a/libbcachefs/btree_types.h +++ b/libbcachefs/btree_types.h @@ -77,7 +77,6 @@ struct btree { u8 nsets; u8 nr_key_bits; u16 version_ondisk; - u8 write_type; struct bkey_format format; @@ -457,6 +456,23 @@ struct btree_trans { struct replicas_delta_list *fs_usage_deltas; }; +#define BCH_BTREE_WRITE_TYPES() \ + x(initial, 0) \ + x(init_next_bset, 1) \ + x(cache_reclaim, 2) \ + x(journal_reclaim, 3) \ + x(interior, 4) + +enum btree_write_type { +#define x(t, n) BTREE_WRITE_##t, + BCH_BTREE_WRITE_TYPES() +#undef x + BTREE_WRITE_TYPE_NR, +}; + +#define BTREE_WRITE_TYPE_MASK (roundup_pow_of_two(BTREE_WRITE_TYPE_NR) - 1) +#define BTREE_WRITE_TYPE_BITS ilog2(roundup_pow_of_two(BTREE_WRITE_TYPE_NR)) + #define BTREE_FLAGS() \ x(read_in_flight) \ x(read_error) \ @@ -476,6 +492,8 @@ struct btree_trans { x(never_write) enum btree_flags { + /* First bits for btree node write type */ + BTREE_NODE_FLAGS_START = BTREE_WRITE_TYPE_BITS - 1, #define x(flag) BTREE_NODE_##flag, BTREE_FLAGS() #undef x diff --git a/libbcachefs/btree_update_interior.c b/libbcachefs/btree_update_interior.c index dac2fa6..d4f1920 100644 --- a/libbcachefs/btree_update_interior.c +++ b/libbcachefs/btree_update_interior.c @@ -378,14 +378,19 @@ static void btree_set_max(struct btree *b, struct bpos pos) b->data->max_key = pos; } -struct btree *__bch2_btree_node_alloc_replacement(struct btree_update *as, - struct btree_trans *trans, - struct btree *b, - struct bkey_format format) +static struct btree *bch2_btree_node_alloc_replacement(struct btree_update *as, + struct btree_trans *trans, + struct btree *b) { - struct btree *n; + struct btree *n = bch2_btree_node_alloc(as, trans, b->c.level); + struct bkey_format format = bch2_btree_calc_format(b); - n = bch2_btree_node_alloc(as, trans, b->c.level); + /* + * The keys might expand with the new format - if they wouldn't fit in + * the btree node anymore, use the old format for now: + */ + if (!bch2_btree_node_format_fits(as->c, b, &format)) + format = b->format; SET_BTREE_NODE_SEQ(n->data, BTREE_NODE_SEQ(b->data) + 1); @@ -398,27 +403,9 @@ struct btree *__bch2_btree_node_alloc_replacement(struct btree_update *as, bch2_btree_sort_into(as->c, n, b); btree_node_reset_sib_u64s(n); - - n->key.k.p = b->key.k.p; return n; } -static struct btree *bch2_btree_node_alloc_replacement(struct btree_update *as, - struct btree_trans *trans, - struct btree *b) -{ - struct bkey_format new_f = bch2_btree_calc_format(b); - - /* - * The keys might expand with the new format - if they wouldn't fit in - * the btree node anymore, use the old format for now: - */ - if (!bch2_btree_node_format_fits(as->c, b, &new_f)) - new_f = b->format; - - return __bch2_btree_node_alloc_replacement(as, trans, b, new_f); -} - static struct btree *__btree_root_alloc(struct btree_update *as, struct btree_trans *trans, unsigned level) { @@ -1271,6 +1258,7 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, struct bch_fs *c = as->c; struct bkey_packed *k; struct printbuf buf = PRINTBUF; + unsigned long old, new, v; BUG_ON(insert->k.type == KEY_TYPE_btree_ptr_v2 && !btree_ptr_sectors_written(insert)); @@ -1308,8 +1296,15 @@ static void bch2_insert_fixup_btree_ptr(struct btree_update *as, bch2_btree_bset_insert_key(trans, path, b, node_iter, insert); set_btree_node_dirty_acct(c, b); - set_btree_node_need_write(b); - b->write_type = BTREE_WRITE_interior; + + v = READ_ONCE(b->flags); + do { + old = new = v; + + new &= ~BTREE_WRITE_TYPE_MASK; + new |= BTREE_WRITE_interior; + new |= 1 << BTREE_NODE_need_write; + } while ((v = cmpxchg(&b->flags, old, new)) != old); printbuf_exit(&buf); } @@ -1332,8 +1327,12 @@ __bch2_btree_insert_keys_interior(struct btree_update *as, ; while (!bch2_keylist_empty(keys)) { - bch2_insert_fixup_btree_ptr(as, trans, path, b, - &node_iter, bch2_keylist_front(keys)); + struct bkey_i *k = bch2_keylist_front(keys); + + if (bpos_cmp(k->k.p, b->key.k.p) > 0) + break; + + bch2_insert_fixup_btree_ptr(as, trans, path, b, &node_iter, k); bch2_keylist_pop_front(keys); } } @@ -1342,109 +1341,91 @@ __bch2_btree_insert_keys_interior(struct btree_update *as, * Move keys from n1 (original replacement node, now lower node) to n2 (higher * node) */ -static struct btree *__btree_split_node(struct btree_update *as, - struct btree_trans *trans, - struct btree *n1) +static void __btree_split_node(struct btree_update *as, + struct btree_trans *trans, + struct btree *b, + struct btree *n[2]) { - struct bkey_format_state s; - size_t nr_packed = 0, nr_unpacked = 0; - struct btree *n2; - struct bset *set1, *set2; - struct bkey_packed *k, *set2_start, *set2_end, *out, *prev = NULL; + struct bkey_packed *k; struct bpos n1_pos; + struct btree_node_iter iter; + struct bset *bsets[2]; + struct bkey_format_state format[2]; + struct bkey_packed *out[2]; + struct bkey uk; + unsigned u64s, n1_u64s = (b->nr.live_u64s * 3) / 5; + int i; - n2 = bch2_btree_node_alloc(as, trans, n1->c.level); + for (i = 0; i < 2; i++) { + BUG_ON(n[i]->nsets != 1); - n2->data->max_key = n1->data->max_key; - n2->data->format = n1->format; - SET_BTREE_NODE_SEQ(n2->data, BTREE_NODE_SEQ(n1->data)); - n2->key.k.p = n1->key.k.p; + bsets[i] = btree_bset_first(n[i]); + out[i] = bsets[i]->start; - set1 = btree_bset_first(n1); - set2 = btree_bset_first(n2); + SET_BTREE_NODE_SEQ(n[i]->data, BTREE_NODE_SEQ(b->data) + 1); + bch2_bkey_format_init(&format[i]); + } - /* - * Has to be a linear search because we don't have an auxiliary - * search tree yet - */ - k = set1->start; - while (1) { - struct bkey_packed *n = bkey_next(k); + u64s = 0; + for_each_btree_node_key(b, k, &iter) { + if (bkey_deleted(k)) + continue; + + i = u64s >= n1_u64s; + u64s += k->u64s; + uk = bkey_unpack_key(b, k); + if (!i) + n1_pos = uk.p; + bch2_bkey_format_add_key(&format[i], &uk); + } - if (n == vstruct_last(set1)) - break; - if (k->_data - set1->_data >= (le16_to_cpu(set1->u64s) * 3) / 5) - break; + btree_set_min(n[0], b->data->min_key); + btree_set_max(n[0], n1_pos); + btree_set_min(n[1], bpos_successor(n1_pos)); + btree_set_max(n[1], b->data->max_key); - if (bkey_packed(k)) - nr_packed++; - else - nr_unpacked++; + for (i = 0; i < 2; i++) { + bch2_bkey_format_add_pos(&format[i], n[i]->data->min_key); + bch2_bkey_format_add_pos(&format[i], n[i]->data->max_key); - prev = k; - k = n; + n[i]->data->format = bch2_bkey_format_done(&format[i]); + btree_node_set_format(n[i], n[i]->data->format); } - BUG_ON(!prev); - set2_start = k; - set2_end = vstruct_last(set1); - - set1->u64s = cpu_to_le16((u64 *) set2_start - set1->_data); - set_btree_bset_end(n1, n1->set); + u64s = 0; + for_each_btree_node_key(b, k, &iter) { + if (bkey_deleted(k)) + continue; - n1->nr.live_u64s = le16_to_cpu(set1->u64s); - n1->nr.bset_u64s[0] = le16_to_cpu(set1->u64s); - n1->nr.packed_keys = nr_packed; - n1->nr.unpacked_keys = nr_unpacked; + i = u64s >= n1_u64s; + u64s += k->u64s; - n1_pos = bkey_unpack_pos(n1, prev); - if (as->c->sb.version < bcachefs_metadata_version_snapshot) - n1_pos.snapshot = U32_MAX; - - btree_set_max(n1, n1_pos); - btree_set_min(n2, bpos_successor(n1->key.k.p)); + if (bch2_bkey_transform(&n[i]->format, out[i], bkey_packed(k) + ? &b->format: &bch2_bkey_format_current, k)) + out[i]->format = KEY_FORMAT_LOCAL_BTREE; + else + bch2_bkey_unpack(b, (void *) out[i], k); - bch2_bkey_format_init(&s); - bch2_bkey_format_add_pos(&s, n2->data->min_key); - bch2_bkey_format_add_pos(&s, n2->data->max_key); + out[i]->needs_whiteout = false; - for (k = set2_start; k != set2_end; k = bkey_next(k)) { - struct bkey uk = bkey_unpack_key(n1, k); - bch2_bkey_format_add_key(&s, &uk); + btree_keys_account_key_add(&n[i]->nr, 0, out[i]); + out[i] = bkey_next(out[i]); } - n2->data->format = bch2_bkey_format_done(&s); - btree_node_set_format(n2, n2->data->format); - - out = set2->start; - memset(&n2->nr, 0, sizeof(n2->nr)); - - for (k = set2_start; k != set2_end; k = bkey_next(k)) { - BUG_ON(!bch2_bkey_transform(&n2->format, out, bkey_packed(k) - ? &n1->format : &bch2_bkey_format_current, k)); - out->format = KEY_FORMAT_LOCAL_BTREE; - btree_keys_account_key_add(&n2->nr, 0, out); - out = bkey_next(out); - } + for (i = 0; i < 2; i++) { + bsets[i]->u64s = cpu_to_le16((u64 *) out[i] - bsets[i]->_data); - set2->u64s = cpu_to_le16((u64 *) out - set2->_data); - set_btree_bset_end(n2, n2->set); + BUG_ON(!bsets[i]->u64s); - BUG_ON(!set1->u64s); - BUG_ON(!set2->u64s); + set_btree_bset_end(n[i], n[i]->set); - btree_node_reset_sib_u64s(n1); - btree_node_reset_sib_u64s(n2); + btree_node_reset_sib_u64s(n[i]); - bch2_verify_btree_nr_keys(n1); - bch2_verify_btree_nr_keys(n2); + bch2_verify_btree_nr_keys(n[i]); - if (n1->c.level) { - btree_node_interior_verify(as->c, n1); - btree_node_interior_verify(as->c, n2); + if (b->c.level) + btree_node_interior_verify(as->c, n[i]); } - - return n2; } /* @@ -1464,41 +1445,17 @@ static void btree_split_insert_keys(struct btree_update *as, struct btree *b, struct keylist *keys) { - struct btree_node_iter node_iter; - struct bkey_i *k = bch2_keylist_front(keys); - struct bkey_packed *src, *dst, *n; - struct bset *i; + if (!bch2_keylist_empty(keys) && + bpos_cmp(bch2_keylist_front(keys)->k.p, + b->data->max_key) <= 0) { + struct btree_node_iter node_iter; - bch2_btree_node_iter_init(&node_iter, b, &k->k.p); + bch2_btree_node_iter_init(&node_iter, b, &bch2_keylist_front(keys)->k.p); - __bch2_btree_insert_keys_interior(as, trans, path, b, node_iter, keys); + __bch2_btree_insert_keys_interior(as, trans, path, b, node_iter, keys); - /* - * We can't tolerate whiteouts here - with whiteouts there can be - * duplicate keys, and it would be rather bad if we picked a duplicate - * for the pivot: - */ - i = btree_bset_first(b); - src = dst = i->start; - while (src != vstruct_last(i)) { - n = bkey_next(src); - if (!bkey_deleted(src)) { - memmove_u64s_down(dst, src, src->u64s); - dst = bkey_next(dst); - } - src = n; + btree_node_interior_verify(as->c, b); } - - /* Also clear out the unwritten whiteouts area: */ - b->whiteout_u64s = 0; - - i->u64s = cpu_to_le16((u64 *) dst - i->_data); - set_btree_bset_end(b, b->set); - - BUG_ON(b->nsets != 1 || - b->nr.live_u64s != le16_to_cpu(btree_bset_first(b)->u64s)); - - btree_node_interior_verify(as->c, b); } static int btree_split(struct btree_update *as, struct btree_trans *trans, @@ -1517,15 +1474,21 @@ static int btree_split(struct btree_update *as, struct btree_trans *trans, bch2_btree_interior_update_will_free_node(as, b); - n1 = bch2_btree_node_alloc_replacement(as, trans, b); + if (b->nr.live_u64s > BTREE_SPLIT_THRESHOLD(c)) { + struct btree *n[2]; - if (keys) - btree_split_insert_keys(as, trans, path, n1, keys); - - if (bset_u64s(&n1->set[0]) > BTREE_SPLIT_THRESHOLD(c)) { trace_and_count(c, btree_node_split, c, b); - n2 = __btree_split_node(as, trans, n1); + n[0] = n1 = bch2_btree_node_alloc(as, trans, b->c.level); + n[1] = n2 = bch2_btree_node_alloc(as, trans, b->c.level); + + __btree_split_node(as, trans, b, n); + + if (keys) { + btree_split_insert_keys(as, trans, path, n1, keys); + btree_split_insert_keys(as, trans, path, n2, keys); + BUG_ON(!bch2_keylist_empty(keys)); + } bch2_btree_build_aux_trees(n2); bch2_btree_build_aux_trees(n1); @@ -1574,6 +1537,13 @@ static int btree_split(struct btree_update *as, struct btree_trans *trans, } else { trace_and_count(c, btree_node_compact, c, b); + n1 = bch2_btree_node_alloc_replacement(as, trans, b); + + if (keys) { + btree_split_insert_keys(as, trans, path, n1, keys); + BUG_ON(!bch2_keylist_empty(keys)); + } + bch2_btree_build_aux_trees(n1); bch2_btree_update_add_new_node(as, n1); six_unlock_write(&n1->c.lock); diff --git a/libbcachefs/btree_update_leaf.c b/libbcachefs/btree_update_leaf.c index b930b78..05c1b28 100644 --- a/libbcachefs/btree_update_leaf.c +++ b/libbcachefs/btree_update_leaf.c @@ -178,11 +178,11 @@ static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, w->journal.seq != seq) break; + new &= ~BTREE_WRITE_TYPE_MASK; + new |= BTREE_WRITE_journal_reclaim; new |= 1 << BTREE_NODE_need_write; } while ((v = cmpxchg(&b->flags, old, new)) != old); - b->write_type = BTREE_WRITE_journal_reclaim; - btree_node_write_if_need(c, b, SIX_LOCK_read); six_unlock_read(&b->c.lock); diff --git a/libbcachefs/buckets_waiting_for_journal.c b/libbcachefs/buckets_waiting_for_journal.c index 2e5b955..0f4ef9e 100644 --- a/libbcachefs/buckets_waiting_for_journal.c +++ b/libbcachefs/buckets_waiting_for_journal.c @@ -3,6 +3,7 @@ #include "bcachefs.h" #include "buckets_waiting_for_journal.h" #include +#include static inline struct bucket_hashed * bucket_hash(struct buckets_waiting_for_journal_table *t, @@ -10,7 +11,7 @@ bucket_hash(struct buckets_waiting_for_journal_table *t, { unsigned h = siphash_1u64(dev_bucket, &t->hash_seeds[hash_seed_idx]); - BUG_ON(!is_power_of_2(t->size)); + EBUG_ON(!is_power_of_2(t->size)); return t->d + (h & (t->size - 1)); } diff --git a/libbcachefs/checksum.h b/libbcachefs/checksum.h index f7ccef7..409ad53 100644 --- a/libbcachefs/checksum.h +++ b/libbcachefs/checksum.h @@ -99,14 +99,17 @@ static inline enum bch_csum_type bch2_csum_opt_to_type(enum bch_csum_opts type, } static inline enum bch_csum_type bch2_data_checksum_type(struct bch_fs *c, - unsigned opt) + struct bch_io_opts opts) { + if (opts.nocow) + return 0; + if (c->sb.encryption_type) return c->opts.wide_macs ? BCH_CSUM_chacha20_poly1305_128 : BCH_CSUM_chacha20_poly1305_80; - return bch2_csum_opt_to_type(opt, true); + return bch2_csum_opt_to_type(opts.data_checksum, true); } static inline enum bch_csum_type bch2_meta_checksum_type(struct bch_fs *c) diff --git a/libbcachefs/data_update.c b/libbcachefs/data_update.c index be45cf5..3015528 100644 --- a/libbcachefs/data_update.c +++ b/libbcachefs/data_update.c @@ -97,10 +97,10 @@ static void bch2_bkey_mark_dev_cached(struct bkey_s k, unsigned dev) ptr->cached = true; } -int bch2_data_update_index_update(struct bch_write_op *op) +static int __bch2_data_update_index_update(struct btree_trans *trans, + struct bch_write_op *op) { struct bch_fs *c = op->c; - struct btree_trans trans; struct btree_iter iter; struct data_update *m = container_of(op, struct data_update, op); @@ -112,9 +112,7 @@ int bch2_data_update_index_update(struct bch_write_op *op) bch2_bkey_buf_init(&_insert); bch2_bkey_buf_realloc(&_insert, c, U8_MAX); - bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024); - - bch2_trans_iter_init(&trans, &iter, m->btree_id, + bch2_trans_iter_init(trans, &iter, m->btree_id, bkey_start_pos(&bch2_keylist_front(keys)->k), BTREE_ITER_SLOTS|BTREE_ITER_INTENT); @@ -131,7 +129,7 @@ int bch2_data_update_index_update(struct bch_write_op *op) s64 i_sectors_delta = 0, disk_sectors_delta = 0; unsigned i; - bch2_trans_begin(&trans); + bch2_trans_begin(trans); k = bch2_btree_iter_peek_slot(&iter); ret = bkey_err(k); @@ -202,7 +200,7 @@ int bch2_data_update_index_update(struct bch_write_op *op) bch2_bkey_narrow_crcs(insert, (struct bch_extent_crc_unpacked) { 0 }); bch2_extent_normalize(c, bkey_i_to_s(insert)); - ret = bch2_sum_sector_overwrites(&trans, &iter, insert, + ret = bch2_sum_sector_overwrites(trans, &iter, insert, &should_check_enospc, &i_sectors_delta, &disk_sectors_delta); @@ -220,11 +218,11 @@ int bch2_data_update_index_update(struct bch_write_op *op) next_pos = insert->k.p; - ret = insert_snapshot_whiteouts(&trans, m->btree_id, + ret = insert_snapshot_whiteouts(trans, m->btree_id, k.k->p, insert->k.p) ?: - bch2_trans_update(&trans, &iter, insert, + bch2_trans_update(trans, &iter, insert, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?: - bch2_trans_commit(&trans, &op->res, + bch2_trans_commit(trans, &op->res, NULL, BTREE_INSERT_NOFAIL| m->data_opts.btree_insert_flags); @@ -261,14 +259,26 @@ nomatch: goto next; } out: - bch2_trans_iter_exit(&trans, &iter); - bch2_trans_exit(&trans); + bch2_trans_iter_exit(trans, &iter); bch2_bkey_buf_exit(&_insert, c); bch2_bkey_buf_exit(&_new, c); BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart)); return ret; } +int bch2_data_update_index_update(struct bch_write_op *op) +{ + struct bch_fs *c = op->c; + struct btree_trans trans; + int ret; + + bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024); + ret = __bch2_data_update_index_update(&trans, op); + bch2_trans_exit(&trans); + + return ret; +} + void bch2_data_update_read_done(struct data_update *m, struct bch_extent_crc_unpacked crc) { @@ -284,12 +294,99 @@ void bch2_data_update_read_done(struct data_update *m, void bch2_data_update_exit(struct data_update *update) { struct bch_fs *c = update->op.c; + struct bkey_ptrs_c ptrs = + bch2_bkey_ptrs_c(bkey_i_to_s_c(update->k.k)); + const struct bch_extent_ptr *ptr; + + bkey_for_each_ptr(ptrs, ptr) + bch2_bucket_nocow_unlock(&c->nocow_locks, + PTR_BUCKET_POS(c, ptr), 0); bch2_bkey_buf_exit(&update->k, c); bch2_disk_reservation_put(c, &update->op.res); bch2_bio_free_pages_pool(c, &update->op.wbio.bio); } +void bch2_update_unwritten_extent(struct btree_trans *trans, + struct data_update *update) +{ + struct bch_fs *c = update->op.c; + struct bio *bio = &update->op.wbio.bio; + struct bkey_i_extent *e; + struct write_point *wp; + struct bch_extent_ptr *ptr; + struct closure cl; + struct btree_iter iter; + struct bkey_s_c k; + int ret; + + closure_init_stack(&cl); + bch2_keylist_init(&update->op.insert_keys, update->op.inline_keys); + + while (bio_sectors(bio)) { + unsigned sectors = bio_sectors(bio); + + bch2_trans_iter_init(trans, &iter, update->btree_id, update->op.pos, + BTREE_ITER_SLOTS); + ret = lockrestart_do(trans, ({ + k = bch2_btree_iter_peek_slot(&iter); + bkey_err(k); + })); + bch2_trans_iter_exit(trans, &iter); + + if (ret || !bch2_extents_match(k, bkey_i_to_s_c(update->k.k))) + break; + + e = bkey_extent_init(update->op.insert_keys.top); + e->k.p = update->op.pos; + + ret = bch2_alloc_sectors_start_trans(trans, + update->op.target, + false, + update->op.write_point, + &update->op.devs_have, + update->op.nr_replicas, + update->op.nr_replicas, + update->op.alloc_reserve, + 0, &cl, &wp); + if (ret == -EAGAIN) { + bch2_trans_unlock(trans); + closure_sync(&cl); + continue; + } + + if (ret) + return; + + sectors = min(sectors, wp->sectors_free); + + bch2_key_resize(&e->k, sectors); + + bch2_open_bucket_get(c, wp, &update->op.open_buckets); + bch2_alloc_sectors_append_ptrs(c, wp, &e->k_i, sectors, false); + bch2_alloc_sectors_done(c, wp); + + bio_advance(bio, sectors << 9); + update->op.pos.offset += sectors; + + extent_for_each_ptr(extent_i_to_s(e), ptr) + ptr->unwritten = true; + bch2_keylist_push(&update->op.insert_keys); + + ret = __bch2_data_update_index_update(trans, &update->op); + + bch2_open_buckets_put(c, &update->op.open_buckets); + + if (ret) + break; + } + + if ((atomic_read(&cl.remaining) & CLOSURE_REMAINING_MASK) != 1) { + bch2_trans_unlock(trans); + closure_sync(&cl); + } +} + int bch2_data_update_init(struct bch_fs *c, struct data_update *m, struct write_point_specifier wp, struct bch_io_opts io_opts, @@ -351,6 +448,9 @@ int bch2_data_update_init(struct bch_fs *c, struct data_update *m, m->op.incompressible = true; i++; + + bch2_bucket_nocow_lock(&c->nocow_locks, + PTR_BUCKET_POS(c, &p.ptr), 0); } if (reserve_sectors) { @@ -366,6 +466,10 @@ int bch2_data_update_init(struct bch_fs *c, struct data_update *m, hweight32(m->data_opts.rewrite_ptrs) + m->data_opts.extra_replicas; BUG_ON(!m->op.nr_replicas); + + /* Special handling required: */ + if (bkey_extent_is_unwritten(k)) + return -BCH_ERR_unwritten_extent_update; return 0; } diff --git a/libbcachefs/data_update.h b/libbcachefs/data_update.h index 5d86907..f304c33 100644 --- a/libbcachefs/data_update.h +++ b/libbcachefs/data_update.h @@ -32,6 +32,7 @@ void bch2_data_update_read_done(struct data_update *, struct bch_extent_crc_unpacked); void bch2_data_update_exit(struct data_update *); +void bch2_update_unwritten_extent(struct btree_trans *, struct data_update *); int bch2_data_update_init(struct bch_fs *, struct data_update *, struct write_point_specifier, struct bch_io_opts, struct data_update_opts, diff --git a/libbcachefs/errcode.h b/libbcachefs/errcode.h index 9f29304..93b515e 100644 --- a/libbcachefs/errcode.h +++ b/libbcachefs/errcode.h @@ -63,7 +63,8 @@ x(BCH_ERR_fsck, fsck_repair_unimplemented) \ x(BCH_ERR_fsck, fsck_repair_impossible) \ x(0, need_snapshot_cleanup) \ - x(0, need_topology_repair) + x(0, need_topology_repair) \ + x(0, unwritten_extent_update) enum bch_errcode { BCH_ERR_START = 2048, diff --git a/libbcachefs/error.h b/libbcachefs/error.h index bbf9b6d..dae7262 100644 --- a/libbcachefs/error.h +++ b/libbcachefs/error.h @@ -186,36 +186,25 @@ void bch2_io_error_work(struct work_struct *); /* Does the error handling without logging a message */ void bch2_io_error(struct bch_dev *); -/* Logs message and handles the error: */ -#define bch2_dev_io_error(ca, fmt, ...) \ -do { \ - printk_ratelimited(KERN_ERR "bcachefs (%s): " fmt, \ - (ca)->name, ##__VA_ARGS__); \ - bch2_io_error(ca); \ -} while (0) - -#define bch2_dev_inum_io_error(ca, _inum, _offset, fmt, ...) \ -do { \ - printk_ratelimited(KERN_ERR "bcachefs (%s inum %llu offset %llu): " fmt,\ - (ca)->name, (_inum), (_offset), ##__VA_ARGS__); \ - bch2_io_error(ca); \ -} while (0) - #define bch2_dev_io_err_on(cond, ca, ...) \ ({ \ bool _ret = (cond); \ \ - if (_ret) \ - bch2_dev_io_error(ca, __VA_ARGS__); \ + if (_ret) { \ + bch_err_dev_ratelimited(ca, __VA_ARGS__); \ + bch2_io_error(ca); \ + } \ _ret; \ }) -#define bch2_dev_inum_io_err_on(cond, ca, _inum, _offset, ...) \ +#define bch2_dev_inum_io_err_on(cond, ca, ...) \ ({ \ bool _ret = (cond); \ \ - if (_ret) \ - bch2_dev_inum_io_error(ca, _inum, _offset, __VA_ARGS__);\ + if (_ret) { \ + bch_err_inum_offset_ratelimited(ca, __VA_ARGS__); \ + bch2_io_error(ca); \ + } \ _ret; \ }) diff --git a/libbcachefs/extents.c b/libbcachefs/extents.c index 9e2a4ed..1274d50 100644 --- a/libbcachefs/extents.c +++ b/libbcachefs/extents.c @@ -117,6 +117,13 @@ int bch2_bkey_pick_read_device(struct bch_fs *c, struct bkey_s_c k, return -EIO; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) { + /* + * Unwritten extent: no need to actually read, treat it as a + * hole and return 0s: + */ + if (p.ptr.unwritten) + return 0; + ca = bch_dev_bkey_exists(c, p.ptr.dev); /* @@ -270,6 +277,7 @@ bool bch2_extent_merge(struct bch_fs *c, struct bkey_s l, struct bkey_s_c r) rp.ptr.offset + rp.crc.offset || lp.ptr.dev != rp.ptr.dev || lp.ptr.gen != rp.ptr.gen || + lp.ptr.unwritten != rp.ptr.unwritten || lp.has_ec != rp.has_ec) return false; @@ -657,22 +665,21 @@ unsigned bch2_bkey_replicas(struct bch_fs *c, struct bkey_s_c k) return replicas; } -static unsigned bch2_extent_ptr_durability(struct bch_fs *c, - struct extent_ptr_decoded p) +unsigned bch2_extent_ptr_durability(struct bch_fs *c, struct extent_ptr_decoded *p) { unsigned durability = 0; struct bch_dev *ca; - if (p.ptr.cached) + if (p->ptr.cached) return 0; - ca = bch_dev_bkey_exists(c, p.ptr.dev); + ca = bch_dev_bkey_exists(c, p->ptr.dev); if (ca->mi.state != BCH_MEMBER_STATE_failed) durability = max_t(unsigned, durability, ca->mi.durability); - if (p.has_ec) - durability += p.ec.redundancy; + if (p->has_ec) + durability += p->ec.redundancy; return durability; } @@ -685,7 +692,7 @@ unsigned bch2_bkey_durability(struct bch_fs *c, struct bkey_s_c k) unsigned durability = 0; bkey_for_each_ptr_decode(k.k, ptrs, p, entry) - durability += bch2_extent_ptr_durability(c, p); + durability += bch2_extent_ptr_durability(c,& p); return durability; } @@ -923,20 +930,31 @@ bool bch2_bkey_matches_ptr(struct bch_fs *c, struct bkey_s_c k, */ bool bch2_extents_match(struct bkey_s_c k1, struct bkey_s_c k2) { - struct bkey_ptrs_c ptrs1 = bch2_bkey_ptrs_c(k1); - struct bkey_ptrs_c ptrs2 = bch2_bkey_ptrs_c(k2); - const union bch_extent_entry *entry1, *entry2; - struct extent_ptr_decoded p1, p2; + if (k1.k->type != k2.k->type) + return false; + + if (bkey_extent_is_direct_data(k1.k)) { + struct bkey_ptrs_c ptrs1 = bch2_bkey_ptrs_c(k1); + struct bkey_ptrs_c ptrs2 = bch2_bkey_ptrs_c(k2); + const union bch_extent_entry *entry1, *entry2; + struct extent_ptr_decoded p1, p2; - bkey_for_each_ptr_decode(k1.k, ptrs1, p1, entry1) - bkey_for_each_ptr_decode(k2.k, ptrs2, p2, entry2) + if (bkey_extent_is_unwritten(k1) != bkey_extent_is_unwritten(k2)) + return false; + + bkey_for_each_ptr_decode(k1.k, ptrs1, p1, entry1) + bkey_for_each_ptr_decode(k2.k, ptrs2, p2, entry2) if (p1.ptr.dev == p2.ptr.dev && p1.ptr.gen == p2.ptr.gen && (s64) p1.ptr.offset + p1.crc.offset - bkey_start_offset(k1.k) == (s64) p2.ptr.offset + p2.crc.offset - bkey_start_offset(k2.k)) return true; - return false; + return false; + } else { + /* KEY_TYPE_deleted, etc. */ + return true; + } } bool bch2_extent_has_ptr(struct bkey_s_c k1, struct extent_ptr_decoded p1, @@ -1005,10 +1023,12 @@ void bch2_bkey_ptrs_to_text(struct printbuf *out, struct bch_fs *c, u32 offset; u64 b = sector_to_bucket_and_offset(ca, ptr->offset, &offset); - prt_printf(out, "ptr: %u:%llu:%u gen %u%s", ptr->dev, - b, offset, ptr->gen, - ptr->cached ? " cached" : ""); - + prt_printf(out, "ptr: %u:%llu:%u gen %u", + ptr->dev, b, offset, ptr->gen); + if (ptr->cached) + prt_str(out, " cached"); + if (ptr->unwritten) + prt_str(out, " unwritten"); if (ca && ptr_stale(ca, ptr)) prt_printf(out, " stale"); } @@ -1097,6 +1117,7 @@ int bch2_bkey_ptrs_invalid(const struct bch_fs *c, struct bkey_s_c k, unsigned size_ondisk = k.k->size; unsigned nonce = UINT_MAX; unsigned nr_ptrs = 0; + bool unwritten = false; int ret; if (bkey_is_btree_ptr(k.k)) @@ -1121,6 +1142,18 @@ int bch2_bkey_ptrs_invalid(const struct bch_fs *c, struct bkey_s_c k, false, err); if (ret) return ret; + + if (nr_ptrs && unwritten != entry->ptr.unwritten) { + prt_printf(err, "extent with unwritten and written ptrs"); + return -EINVAL; + } + + if (k.k->type != KEY_TYPE_extent && entry->ptr.unwritten) { + prt_printf(err, "has unwritten ptrs"); + return -EINVAL; + } + + unwritten = entry->ptr.unwritten; nr_ptrs++; break; case BCH_EXTENT_ENTRY_crc32: diff --git a/libbcachefs/extents.h b/libbcachefs/extents.h index 224df17..84737af 100644 --- a/libbcachefs/extents.h +++ b/libbcachefs/extents.h @@ -510,6 +510,23 @@ static inline bool bkey_extent_is_allocation(const struct bkey *k) } } +static inline bool bkey_extent_is_unwritten(struct bkey_s_c k) +{ + struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); + const struct bch_extent_ptr *ptr; + + bkey_for_each_ptr(ptrs, ptr) + if (ptr->unwritten) + return true; + return false; +} + +static inline bool bkey_extent_is_reservation(struct bkey_s_c k) +{ + return k.k->type == KEY_TYPE_reservation || + bkey_extent_is_unwritten(k); +} + static inline struct bch_devs_list bch2_bkey_devs(struct bkey_s_c k) { struct bch_devs_list ret = (struct bch_devs_list) { 0 }; @@ -579,6 +596,7 @@ bool bch2_bkey_is_incompressible(struct bkey_s_c); unsigned bch2_bkey_sectors_compressed(struct bkey_s_c); unsigned bch2_bkey_replicas(struct bch_fs *, struct bkey_s_c); +unsigned bch2_extent_ptr_durability(struct bch_fs *, struct extent_ptr_decoded *); unsigned bch2_bkey_durability(struct bch_fs *, struct bkey_s_c); void bch2_bkey_extent_entry_drop(struct bkey_i *, union bch_extent_entry *); diff --git a/libbcachefs/fs-io.c b/libbcachefs/fs-io.c index 6d0a6de..dc2352d 100644 --- a/libbcachefs/fs-io.c +++ b/libbcachefs/fs-io.c @@ -35,6 +35,72 @@ #include #include +struct nocow_flush { + struct closure *cl; + struct bch_dev *ca; + struct bio bio; +}; + +static void nocow_flush_endio(struct bio *_bio) +{ + + struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio); + + closure_put(bio->cl); + percpu_ref_put(&bio->ca->io_ref); + bio_put(&bio->bio); +} + +static void bch2_inode_flush_nocow_writes_async(struct bch_fs *c, + struct bch_inode_info *inode, + struct closure *cl) +{ + struct nocow_flush *bio; + struct bch_dev *ca; + struct bch_devs_mask devs; + unsigned dev; + + dev = find_first_bit(inode->ei_devs_need_flush.d, BCH_SB_MEMBERS_MAX); + if (dev == BCH_SB_MEMBERS_MAX) + return; + + devs = inode->ei_devs_need_flush; + memset(&inode->ei_devs_need_flush, 0, sizeof(inode->ei_devs_need_flush)); + + for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) { + rcu_read_lock(); + ca = rcu_dereference(c->devs[dev]); + if (ca && !percpu_ref_tryget(&ca->io_ref)) + ca = NULL; + rcu_read_unlock(); + + if (!ca) + continue; + + bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, 0, + REQ_OP_FLUSH, + GFP_KERNEL, + &c->nocow_flush_bioset), + struct nocow_flush, bio); + bio->cl = cl; + bio->ca = ca; + bio->bio.bi_end_io = nocow_flush_endio; + closure_bio_submit(&bio->bio, cl); + } +} + +static int bch2_inode_flush_nocow_writes(struct bch_fs *c, + struct bch_inode_info *inode) +{ + struct closure cl; + + closure_init_stack(&cl); + bch2_inode_flush_nocow_writes_async(c, inode, &cl); + closure_sync(&cl); + + return 0; +} + static inline bool bio_full(struct bio *bio, unsigned len) { if (bio->bi_vcnt >= bio->bi_max_vecs) @@ -77,6 +143,7 @@ struct dio_write { struct bch_inode_info *inode; struct mm_struct *mm; unsigned loop:1, + extending:1, sync:1, flush:1, free_iov:1; @@ -131,24 +198,29 @@ static noinline int write_invalidate_inode_pages_range(struct address_space *map #ifdef CONFIG_BCACHEFS_QUOTA -static void bch2_quota_reservation_put(struct bch_fs *c, - struct bch_inode_info *inode, - struct quota_res *res) +static void __bch2_quota_reservation_put(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res) { - if (!res->sectors) - return; - - mutex_lock(&inode->ei_quota_lock); BUG_ON(res->sectors > inode->ei_quota_reserved); bch2_quota_acct(c, inode->ei_qid, Q_SPC, -((s64) res->sectors), KEY_TYPE_QUOTA_PREALLOC); inode->ei_quota_reserved -= res->sectors; - mutex_unlock(&inode->ei_quota_lock); - res->sectors = 0; } +static void bch2_quota_reservation_put(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res) +{ + if (res->sectors) { + mutex_lock(&inode->ei_quota_lock); + __bch2_quota_reservation_put(c, inode, res); + mutex_unlock(&inode->ei_quota_lock); + } +} + static int bch2_quota_reservation_add(struct bch_fs *c, struct bch_inode_info *inode, struct quota_res *res, @@ -171,11 +243,13 @@ static int bch2_quota_reservation_add(struct bch_fs *c, #else +static void __bch2_quota_reservation_put(struct bch_fs *c, + struct bch_inode_info *inode, + struct quota_res *res) {} + static void bch2_quota_reservation_put(struct bch_fs *c, struct bch_inode_info *inode, - struct quota_res *res) -{ -} + struct quota_res *res) {} static int bch2_quota_reservation_add(struct bch_fs *c, struct bch_inode_info *inode, @@ -226,13 +300,9 @@ int __must_check bch2_write_inode_size(struct bch_fs *c, return bch2_write_inode(c, inode, inode_set_size, &s, fields); } -static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, +static void __i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, struct quota_res *quota_res, s64 sectors) { - if (!sectors) - return; - - mutex_lock(&inode->ei_quota_lock); bch2_fs_inconsistent_on((s64) inode->v.i_blocks + sectors < 0, c, "inode %lu i_blocks underflow: %llu + %lli < 0 (ondisk %lli)", inode->v.i_ino, (u64) inode->v.i_blocks, sectors, @@ -250,7 +320,16 @@ static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, KEY_TYPE_QUOTA_WARN); } #endif - mutex_unlock(&inode->ei_quota_lock); +} + +static void i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, + struct quota_res *quota_res, s64 sectors) +{ + if (sectors) { + mutex_lock(&inode->ei_quota_lock); + __i_sectors_acct(c, inode, quota_res, sectors); + mutex_unlock(&inode->ei_quota_lock); + } } /* page state: */ @@ -328,11 +407,11 @@ static struct bch_page_state *bch2_page_state_create(struct page *page, return bch2_page_state(page) ?: __bch2_page_state_create(page, gfp); } -static unsigned bkey_to_sector_state(const struct bkey *k) +static unsigned bkey_to_sector_state(struct bkey_s_c k) { - if (k->type == KEY_TYPE_reservation) + if (bkey_extent_is_reservation(k)) return SECTOR_RESERVED; - if (bkey_extent_is_allocation(k)) + if (bkey_extent_is_allocation(k.k)) return SECTOR_ALLOCATED; return SECTOR_UNALLOCATED; } @@ -383,7 +462,7 @@ retry: SPOS(inum.inum, offset, snapshot), BTREE_ITER_SLOTS, k, ret) { unsigned nr_ptrs = bch2_bkey_nr_ptrs_fully_allocated(k); - unsigned state = bkey_to_sector_state(k.k); + unsigned state = bkey_to_sector_state(k); while (pg_idx < nr_pages) { struct page *page = pages[pg_idx]; @@ -423,7 +502,7 @@ static void bch2_bio_page_state_set(struct bio *bio, struct bkey_s_c k) struct bio_vec bv; unsigned nr_ptrs = k.k->type == KEY_TYPE_reflink_v ? 0 : bch2_bkey_nr_ptrs_fully_allocated(k); - unsigned state = bkey_to_sector_state(k.k); + unsigned state = bkey_to_sector_state(k); bio_for_each_segment(bv, bio, iter) __bch2_page_state_set(bv.bv_page, bv.bv_offset >> 9, @@ -1074,7 +1153,9 @@ err: goto retry; if (ret) { - bch_err_inum_ratelimited(c, inum.inum, + bch_err_inum_offset_ratelimited(c, + iter.pos.inode, + iter.pos.offset << 9, "read error %i from btree lookup", ret); rbio->bio.bi_status = BLK_STS_IOERR; bio_endio(&rbio->bio); @@ -1306,6 +1387,7 @@ static void bch2_writepage_io_alloc(struct bch_fs *c, op->subvol = inode->ei_subvol; op->pos = POS(inode->v.i_ino, sector); op->end_io = bch2_writepage_io_done; + op->devs_need_flush = &inode->ei_devs_need_flush; op->wbio.bio.bi_iter.bi_sector = sector; op->wbio.bio.bi_opf = wbc_to_write_flags(wbc); } @@ -1428,9 +1510,13 @@ do_io: sectors << 9, offset << 9)); /* Check for writing past i_size: */ - WARN_ON_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) > - round_up(i_size, block_bytes(c)) && - !test_bit(BCH_FS_EMERGENCY_RO, &c->flags)); + WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) > + round_up(i_size, block_bytes(c)) && + !test_bit(BCH_FS_EMERGENCY_RO, &c->flags), + "writing past i_size: %llu > %llu (unrounded %llu)\n", + bio_end_sector(&w->io->op.wbio.bio) << 9, + round_up(i_size, block_bytes(c)), + i_size); w->io->op.res.sectors += reserved_sectors; w->io->op.i_sectors_delta -= dirty_sectors; @@ -2101,10 +2187,12 @@ static noinline void bch2_dio_write_flush(struct dio_write *dio) if (!dio->op.error) { ret = bch2_inode_find_by_inum(c, inode_inum(dio->inode), &inode); - if (ret) + if (ret) { dio->op.error = ret; - else + } else { bch2_journal_flush_seq_async(&c->journal, inode.bi_journal_seq, &dio->op.cl); + bch2_inode_flush_nocow_writes_async(c, dio->inode, &dio->op.cl); + } } if (dio->sync) { @@ -2117,7 +2205,6 @@ static noinline void bch2_dio_write_flush(struct dio_write *dio) static __always_inline long bch2_dio_write_done(struct dio_write *dio) { - struct bch_fs *c = dio->op.c; struct kiocb *req = dio->req; struct bch_inode_info *inode = dio->inode; bool sync = dio->sync; @@ -2130,7 +2217,6 @@ static __always_inline long bch2_dio_write_done(struct dio_write *dio) } bch2_pagecache_block_put(inode); - bch2_quota_reservation_put(c, inode, &dio->quota_res); if (dio->free_iov) kfree(dio->iter.iov); @@ -2160,14 +2246,22 @@ static __always_inline void bch2_dio_write_end(struct dio_write *dio) struct bvec_iter_all iter; struct bio_vec *bv; - i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta); - req->ki_pos += (u64) dio->op.written << 9; - dio->written += dio->op.written; + req->ki_pos += (u64) dio->op.written << 9; + dio->written += dio->op.written; - spin_lock(&inode->v.i_lock); - if (req->ki_pos > inode->v.i_size) - i_size_write(&inode->v, req->ki_pos); - spin_unlock(&inode->v.i_lock); + if (dio->extending) { + spin_lock(&inode->v.i_lock); + if (req->ki_pos > inode->v.i_size) + i_size_write(&inode->v, req->ki_pos); + spin_unlock(&inode->v.i_lock); + } + + if (dio->op.i_sectors_delta || dio->quota_res.sectors) { + mutex_lock(&inode->ei_quota_lock); + __i_sectors_acct(c, inode, &dio->quota_res, dio->op.i_sectors_delta); + __bch2_quota_reservation_put(c, inode, &dio->quota_res); + mutex_unlock(&inode->ei_quota_lock); + } if (likely(!bio_flagged(bio, BIO_NO_PAGE_REF))) bio_for_each_segment_all(bv, bio, iter) @@ -2244,11 +2338,17 @@ static long bch2_dio_write_loop(struct dio_write *dio) dio->op.nr_replicas = dio->op.opts.data_replicas; dio->op.subvol = inode->ei_subvol; dio->op.pos = POS(inode->v.i_ino, (u64) req->ki_pos >> 9); + dio->op.devs_need_flush = &inode->ei_devs_need_flush; if (sync) dio->op.flags |= BCH_WRITE_SYNC; dio->op.flags |= BCH_WRITE_CHECK_ENOSPC; + ret = bch2_quota_reservation_add(c, inode, &dio->quota_res, + bio_sectors(bio), true); + if (unlikely(ret)) + goto err; + ret = bch2_disk_reservation_get(c, &dio->op.res, bio_sectors(bio), dio->op.opts.data_replicas, 0); if (unlikely(ret) && @@ -2288,6 +2388,8 @@ err: bio_for_each_segment_all(bv, bio, iter) put_page(bv->bv_page); } + + bch2_quota_reservation_put(c, inode, &dio->quota_res); goto out; } @@ -2366,6 +2468,7 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) dio->inode = inode; dio->mm = current->mm; dio->loop = false; + dio->extending = extending; dio->sync = is_sync_kiocb(req) || extending; dio->flush = iocb_is_dsync(req) && !c->opts.journal_flush_disabled; dio->free_iov = false; @@ -2374,11 +2477,6 @@ ssize_t bch2_direct_write(struct kiocb *req, struct iov_iter *iter) dio->iter = *iter; dio->op.c = c; - ret = bch2_quota_reservation_add(c, inode, &dio->quota_res, - iter->count >> 9, true); - if (unlikely(ret)) - goto err_put_bio; - if (unlikely(mapping->nrpages)) { ret = write_invalidate_inode_pages_range(mapping, req->ki_pos, @@ -2394,7 +2492,6 @@ err: return ret; err_put_bio: bch2_pagecache_block_put(inode); - bch2_quota_reservation_put(c, inode, &dio->quota_res); bio_put(bio); inode_dio_end(&inode->v); goto err; @@ -2446,19 +2543,21 @@ out: * inode->ei_inode.bi_journal_seq won't be up to date since it's set in an * insert trigger: look up the btree inode instead */ -static int bch2_flush_inode(struct bch_fs *c, subvol_inum inum) +static int bch2_flush_inode(struct bch_fs *c, + struct bch_inode_info *inode) { - struct bch_inode_unpacked inode; + struct bch_inode_unpacked u; int ret; if (c->opts.journal_flush_disabled) return 0; - ret = bch2_inode_find_by_inum(c, inum, &inode); + ret = bch2_inode_find_by_inum(c, inode_inum(inode), &u); if (ret) return ret; - return bch2_journal_flush_seq(&c->journal, inode.bi_journal_seq); + return bch2_journal_flush_seq(&c->journal, u.bi_journal_seq) ?: + bch2_inode_flush_nocow_writes(c, inode); } int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) @@ -2469,7 +2568,7 @@ int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) ret = file_write_and_wait_range(file, start, end); ret2 = sync_inode_metadata(&inode->v, 1); - ret3 = bch2_flush_inode(c, inode_inum(inode)); + ret3 = bch2_flush_inode(c, inode); return bch2_err_class(ret ?: ret2 ?: ret3); } @@ -2712,8 +2811,10 @@ int bch2_truncate(struct user_namespace *mnt_userns, if (ret) goto err; - WARN_ON(!test_bit(EI_INODE_ERROR, &inode->ei_flags) && - inode->v.i_size < inode_u.bi_size); + WARN_ONCE(!test_bit(EI_INODE_ERROR, &inode->ei_flags) && + inode->v.i_size < inode_u.bi_size, + "truncate spotted in mem i_size < btree i_size: %llu < %llu\n", + (u64) inode->v.i_size, inode_u.bi_size); if (iattr->ia_size > inode->v.i_size) { ret = bch2_extend(mnt_userns, inode, &inode_u, iattr); @@ -3015,7 +3116,7 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode, struct btree_trans trans; struct btree_iter iter; struct bpos end_pos = POS(inode->v.i_ino, end_sector); - unsigned replicas = io_opts(c, &inode->ei_inode).data_replicas; + struct bch_io_opts opts = io_opts(c, &inode->ei_inode); int ret = 0; bch2_trans_init(&trans, c, BTREE_ITER_MAX, 512); @@ -3026,9 +3127,7 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode, while (!ret && bkey_cmp(iter.pos, end_pos) < 0) { s64 i_sectors_delta = 0; - struct disk_reservation disk_res = { 0 }; struct quota_res quota_res = { 0 }; - struct bkey_i_reservation reservation; struct bkey_s_c k; unsigned sectors; u32 snapshot; @@ -3047,8 +3146,8 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode, goto bkey_err; /* already reserved */ - if (k.k->type == KEY_TYPE_reservation && - bkey_s_c_to_reservation(k).v->nr_replicas >= replicas) { + if (bkey_extent_is_reservation(k) && + bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) { bch2_btree_iter_advance(&iter); continue; } @@ -3059,16 +3158,12 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode, continue; } - bkey_reservation_init(&reservation.k_i); - reservation.k.type = KEY_TYPE_reservation; - reservation.k.p = k.k->p; - reservation.k.size = k.k->size; - - bch2_cut_front(iter.pos, &reservation.k_i); - bch2_cut_back(end_pos, &reservation.k_i); + /* + * XXX: for nocow mode, we should promote shared extents to + * unshared here + */ - sectors = reservation.k.size; - reservation.v.nr_replicas = bch2_bkey_nr_ptrs_allocated(k); + sectors = bpos_min(k.k->p, end_pos).offset - iter.pos.offset; if (!bkey_extent_is_allocation(k.k)) { ret = bch2_quota_reservation_add(c, inode, @@ -3078,25 +3173,15 @@ static int __bchfs_fallocate(struct bch_inode_info *inode, int mode, goto bkey_err; } - if (reservation.v.nr_replicas < replicas || - bch2_bkey_sectors_compressed(k)) { - ret = bch2_disk_reservation_get(c, &disk_res, sectors, - replicas, 0); - if (unlikely(ret)) - goto bkey_err; - - reservation.v.nr_replicas = disk_res.nr_replicas; - } - - ret = bch2_extent_update(&trans, inode_inum(inode), &iter, - &reservation.k_i, &disk_res, - 0, &i_sectors_delta, true); + ret = bch2_extent_fallocate(&trans, inode_inum(inode), &iter, + sectors, opts, &i_sectors_delta, + writepoint_hashed((unsigned long) current)); if (ret) goto bkey_err; + i_sectors_acct(c, inode, "a_res, i_sectors_delta); bkey_err: bch2_quota_reservation_put(c, inode, "a_res); - bch2_disk_reservation_put(c, &disk_res); if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) ret = 0; } @@ -3337,7 +3422,7 @@ loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src, if ((file_dst->f_flags & (__O_SYNC | O_DSYNC)) || IS_SYNC(file_inode(file_dst))) - ret = bch2_flush_inode(c, inode_inum(dst)); + ret = bch2_flush_inode(c, dst); err: bch2_quota_reservation_put(c, dst, "a_res); bch2_unlock_inodes(INODE_LOCK|INODE_PAGECACHE_BLOCK, src, dst); @@ -3593,6 +3678,7 @@ loff_t bch2_llseek(struct file *file, loff_t offset, int whence) void bch2_fs_fsio_exit(struct bch_fs *c) { + bioset_exit(&c->nocow_flush_bioset); bioset_exit(&c->dio_write_bioset); bioset_exit(&c->dio_read_bioset); bioset_exit(&c->writepage_bioset); @@ -3612,7 +3698,9 @@ int bch2_fs_fsio_init(struct bch_fs *c) BIOSET_NEED_BVECS) || bioset_init(&c->dio_write_bioset, 4, offsetof(struct dio_write, op.wbio.bio), - BIOSET_NEED_BVECS)) + BIOSET_NEED_BVECS) || + bioset_init(&c->nocow_flush_bioset, + 1, offsetof(struct nocow_flush, bio), 0)) ret = -ENOMEM; pr_verbose_init(c->opts, "ret %i", ret); diff --git a/libbcachefs/fs.c b/libbcachefs/fs.c index 4591b75..8621738 100644 --- a/libbcachefs/fs.c +++ b/libbcachefs/fs.c @@ -810,6 +810,9 @@ static int bch2_fill_extent(struct bch_fs *c, int flags2 = 0; u64 offset = p.ptr.offset; + if (p.ptr.unwritten) + flags2 |= FIEMAP_EXTENT_UNWRITTEN; + if (p.crc.compression_type) flags2 |= FIEMAP_EXTENT_ENCODED; else diff --git a/libbcachefs/fs.h b/libbcachefs/fs.h index b11a150..6b91bbe 100644 --- a/libbcachefs/fs.h +++ b/libbcachefs/fs.h @@ -26,6 +26,17 @@ struct bch_inode_info { u32 ei_subvol; + /* + * When we've been doing nocow writes we'll need to issue flushes to the + * underlying block devices + * + * XXX: a device may have had a flush issued by some other codepath. It + * would be better to keep for each device a sequence number that's + * incremented when we isusue a cache flush, and track here the sequence + * number that needs flushing. + */ + struct bch_devs_mask ei_devs_need_flush; + /* copy of inode in btree: */ struct bch_inode_unpacked ei_inode; }; diff --git a/libbcachefs/fsck.c b/libbcachefs/fsck.c index f4f0e0c..ffc2671 100644 --- a/libbcachefs/fsck.c +++ b/libbcachefs/fsck.c @@ -1257,8 +1257,8 @@ static int check_extent(struct btree_trans *trans, struct btree_iter *iter, continue; if (fsck_err_on(!(i->inode.bi_flags & BCH_INODE_I_SIZE_DIRTY) && - k.k->type != KEY_TYPE_reservation && - k.k->p.offset > round_up(i->inode.bi_size, block_bytes(c)) >> 9, c, + k.k->p.offset > round_up(i->inode.bi_size, block_bytes(c)) >> 9 && + !bkey_extent_is_reservation(k), c, "extent type past end of inode %llu:%u, i_size %llu\n %s", i->inode.bi_inum, i->snapshot, i->inode.bi_size, (bch2_bkey_val_to_text(&buf, c, k), buf.buf))) { diff --git a/libbcachefs/inode.c b/libbcachefs/inode.c index 1a0d260..a91465e 100644 --- a/libbcachefs/inode.c +++ b/libbcachefs/inode.c @@ -733,7 +733,6 @@ static int bch2_inode_delete_keys(struct btree_trans *trans, * iterator: */ bch2_trans_iter_init(trans, &iter, id, POS(inum.inum, 0), - BTREE_ITER_NOT_EXTENTS| BTREE_ITER_INTENT); while (1) { @@ -756,6 +755,14 @@ static int bch2_inode_delete_keys(struct btree_trans *trans, bkey_init(&delete.k); delete.k.p = iter.pos; + if (iter.flags & BTREE_ITER_IS_EXTENTS) { + bch2_key_resize(&delete.k, k.k->p.offset - iter.pos.offset); + + ret = bch2_extent_trim_atomic(trans, &iter, &delete); + if (ret) + goto err; + } + ret = bch2_trans_update(trans, &iter, &delete, 0) ?: bch2_trans_commit(trans, NULL, NULL, BTREE_INSERT_NOFAIL); diff --git a/libbcachefs/inode.h b/libbcachefs/inode.h index a9742bb..be01604 100644 --- a/libbcachefs/inode.h +++ b/libbcachefs/inode.h @@ -157,6 +157,8 @@ io_opts(struct bch_fs *c, struct bch_inode_unpacked *inode) struct bch_io_opts opts = bch2_opts_to_inode_opts(c->opts); bch2_io_opts_apply(&opts, bch2_inode_opts_get(inode)); + if (opts.nocow) + opts.compression = opts.background_compression = opts.data_checksum = opts.erasure_code; return opts; } diff --git a/libbcachefs/io.c b/libbcachefs/io.c index 82caaf5..0ff835e 100644 --- a/libbcachefs/io.c +++ b/libbcachefs/io.c @@ -233,6 +233,62 @@ int bch2_sum_sector_overwrites(struct btree_trans *trans, return ret; } +static int bch2_extent_update_i_size_sectors(struct btree_trans *trans, + struct btree_iter *extent_iter, + u64 new_i_size, + s64 i_sectors_delta) +{ + struct btree_iter iter; + struct bkey_s_c inode_k; + struct bkey_s_c_inode_v3 inode; + struct bkey_i_inode_v3 *new_inode; + int ret; + + bch2_trans_iter_init(trans, &iter, BTREE_ID_inodes, + SPOS(0, + extent_iter->pos.inode, + extent_iter->snapshot), + BTREE_ITER_INTENT|BTREE_ITER_CACHED); + inode_k = bch2_btree_iter_peek_slot(&iter); + ret = bkey_err(inode_k); + if (unlikely(ret)) + goto err; + + ret = bkey_is_inode(inode_k.k) ? 0 : -ENOENT; + if (unlikely(ret)) + goto err; + + if (unlikely(inode_k.k->type != KEY_TYPE_inode_v3)) { + inode_k = bch2_inode_to_v3(trans, inode_k); + ret = bkey_err(inode_k); + if (unlikely(ret)) + goto err; + } + + inode = bkey_s_c_to_inode_v3(inode_k); + + new_inode = bch2_trans_kmalloc(trans, bkey_bytes(inode_k.k)); + ret = PTR_ERR_OR_ZERO(new_inode); + if (unlikely(ret)) + goto err; + + bkey_reassemble(&new_inode->k_i, inode.s_c); + + if (!(le64_to_cpu(inode.v->bi_flags) & BCH_INODE_I_SIZE_DIRTY) && + new_i_size > le64_to_cpu(inode.v->bi_size)) + new_inode->v.bi_size = cpu_to_le64(new_i_size); + + le64_add_cpu(&new_inode->v.bi_sectors, i_sectors_delta); + + new_inode->k.p.snapshot = iter.snapshot; + + ret = bch2_trans_update(trans, &iter, &new_inode->k_i, + BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); +err: + bch2_trans_iter_exit(trans, &iter); + return ret; +} + int bch2_extent_update(struct btree_trans *trans, subvol_inum inum, struct btree_iter *iter, @@ -242,10 +298,6 @@ int bch2_extent_update(struct btree_trans *trans, s64 *i_sectors_delta_total, bool check_enospc) { - struct btree_iter inode_iter = { NULL }; - struct bkey_s_c inode_k; - struct bkey_s_c_inode_v3 inode; - struct bkey_i_inode_v3 *new_inode; struct bpos next_pos; bool usage_increasing; s64 i_sectors_delta = 0, disk_sectors_delta = 0; @@ -265,7 +317,6 @@ int bch2_extent_update(struct btree_trans *trans, if (ret) return ret; - new_i_size = min(k->k.p.offset << 9, new_i_size); next_pos = k->k.p; ret = bch2_sum_sector_overwrites(trans, iter, k, @@ -285,61 +336,149 @@ int bch2_extent_update(struct btree_trans *trans, return ret; } - bch2_trans_iter_init(trans, &inode_iter, BTREE_ID_inodes, - SPOS(0, inum.inum, iter->snapshot), - BTREE_ITER_INTENT|BTREE_ITER_CACHED); - inode_k = bch2_btree_iter_peek_slot(&inode_iter); - ret = bkey_err(inode_k); - if (unlikely(ret)) - goto err; - - ret = bkey_is_inode(inode_k.k) ? 0 : -ENOENT; - if (unlikely(ret)) - goto err; - - if (unlikely(inode_k.k->type != KEY_TYPE_inode_v3)) { - inode_k = bch2_inode_to_v3(trans, inode_k); - ret = bkey_err(inode_k); - if (unlikely(ret)) - goto err; - } - - inode = bkey_s_c_to_inode_v3(inode_k); - - new_inode = bch2_trans_kmalloc(trans, bkey_bytes(inode_k.k)); - ret = PTR_ERR_OR_ZERO(new_inode); - if (unlikely(ret)) - goto err; - - bkey_reassemble(&new_inode->k_i, inode.s_c); - - if (!(le64_to_cpu(inode.v->bi_flags) & BCH_INODE_I_SIZE_DIRTY) && - new_i_size > le64_to_cpu(inode.v->bi_size)) - new_inode->v.bi_size = cpu_to_le64(new_i_size); - - le64_add_cpu(&new_inode->v.bi_sectors, i_sectors_delta); - - new_inode->k.p.snapshot = iter->snapshot; - /* * Note: - * We always have to do an inode updated - even when i_size/i_sectors + * We always have to do an inode update - even when i_size/i_sectors * aren't changing - for fsync to work properly; fsync relies on * inode->bi_journal_seq which is updated by the trigger code: */ - ret = bch2_trans_update(trans, &inode_iter, &new_inode->k_i, 0) ?: + ret = bch2_extent_update_i_size_sectors(trans, iter, + min(k->k.p.offset << 9, new_i_size), + i_sectors_delta) ?: bch2_trans_update(trans, iter, k, 0) ?: bch2_trans_commit(trans, disk_res, NULL, BTREE_INSERT_NOCHECK_RW| BTREE_INSERT_NOFAIL); if (unlikely(ret)) - goto err; + return ret; if (i_sectors_delta_total) *i_sectors_delta_total += i_sectors_delta; bch2_btree_iter_set_pos(iter, next_pos); -err: - bch2_trans_iter_exit(trans, &inode_iter); + return 0; +} + +/* Overwrites whatever was present with zeroes: */ +int bch2_extent_fallocate(struct btree_trans *trans, + subvol_inum inum, + struct btree_iter *iter, + unsigned sectors, + struct bch_io_opts opts, + s64 *i_sectors_delta, + struct write_point_specifier write_point) +{ + struct bch_fs *c = trans->c; + struct disk_reservation disk_res = { 0 }; + struct closure cl; + struct open_buckets open_buckets; + struct bkey_s_c k; + struct bkey_buf old, new; + bool have_reservation = false; + bool unwritten = opts.nocow && + c->sb.version >= bcachefs_metadata_version_unwritten_extents; + int ret; + + bch2_bkey_buf_init(&old); + bch2_bkey_buf_init(&new); + closure_init_stack(&cl); + open_buckets.nr = 0; +retry: + k = bch2_btree_iter_peek_slot(iter); + ret = bkey_err(k); + if (ret) + return ret; + + sectors = min_t(u64, sectors, k.k->p.offset - iter->pos.offset); + + if (!have_reservation) { + unsigned new_replicas = + max(0, (int) opts.data_replicas - + (int) bch2_bkey_nr_ptrs_fully_allocated(k)); + /* + * Get a disk reservation before (in the nocow case) calling + * into the allocator: + */ + ret = bch2_disk_reservation_get(c, &disk_res, sectors, new_replicas, 0); + if (unlikely(ret)) + goto out; + + bch2_bkey_buf_reassemble(&old, c, k); + } + + if (have_reservation) { + if (!bch2_extents_match(k, bkey_i_to_s_c(old.k))) + goto out; + + bch2_key_resize(&new.k->k, sectors); + } else if (!unwritten) { + struct bkey_i_reservation *reservation; + + bch2_bkey_buf_realloc(&new, c, sizeof(*reservation) / sizeof(u64)); + reservation = bkey_reservation_init(new.k); + reservation->k.p = iter->pos; + bch2_key_resize(&reservation->k, sectors); + reservation->v.nr_replicas = opts.data_replicas; + } else { + struct bkey_i_extent *e; + struct bch_devs_list devs_have; + struct write_point *wp; + struct bch_extent_ptr *ptr; + + devs_have.nr = 0; + + bch2_bkey_buf_realloc(&new, c, BKEY_EXTENT_U64s_MAX); + + e = bkey_extent_init(new.k); + e->k.p = iter->pos; + + ret = bch2_alloc_sectors_start_trans(trans, + opts.foreground_target, + false, + write_point, + &devs_have, + opts.data_replicas, + opts.data_replicas, + RESERVE_none, 0, &cl, &wp); + if (ret == -EAGAIN) { + bch2_trans_unlock(trans); + closure_sync(&cl); + goto retry; + } + if (ret) + return ret; + + sectors = min(sectors, wp->sectors_free); + + bch2_key_resize(&e->k, sectors); + + bch2_open_bucket_get(c, wp, &open_buckets); + bch2_alloc_sectors_append_ptrs(c, wp, &e->k_i, sectors, false); + bch2_alloc_sectors_done(c, wp); + + extent_for_each_ptr(extent_i_to_s(e), ptr) + ptr->unwritten = true; + } + + have_reservation = true; + + ret = bch2_extent_update(trans, inum, iter, new.k, &disk_res, + 0, i_sectors_delta, true); +out: + if ((atomic_read(&cl.remaining) & CLOSURE_REMAINING_MASK) != 1) { + bch2_trans_unlock(trans); + closure_sync(&cl); + } + + if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) { + bch2_trans_begin(trans); + goto retry; + } + + bch2_open_buckets_put(c, &open_buckets); + bch2_disk_reservation_put(c, &disk_res); + bch2_bkey_buf_exit(&new, c); + bch2_bkey_buf_exit(&old, c); + return ret; } @@ -485,7 +624,8 @@ static int bch2_write_index_default(struct bch_write_op *op) void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, enum bch_data_type type, - const struct bkey_i *k) + const struct bkey_i *k, + bool nocow) { struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(k)); const struct bch_extent_ptr *ptr; @@ -519,9 +659,11 @@ void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c, n->c = c; n->dev = ptr->dev; - n->have_ioref = bch2_dev_get_ioref(ca, + n->have_ioref = nocow || bch2_dev_get_ioref(ca, type == BCH_DATA_btree ? READ : WRITE); + n->nocow = nocow; n->submit_time = local_clock(); + n->inode_offset = bkey_start_offset(&k->k); n->bio.bi_iter.bi_sector = ptr->offset; if (likely(n->have_ioref)) { @@ -624,8 +766,12 @@ static void __bch2_write_index(struct bch_write_op *op) op->written += sectors_start - keylist_sectors(keys); if (ret) { - bch_err_inum_ratelimited(c, op->pos.inode, - "write error while doing btree update: %s", bch2_err_str(ret)); + struct bkey_i *k = bch2_keylist_front(&op->insert_keys); + + bch_err_inum_offset_ratelimited(c, + k->k.p.inode, k->k.p.offset << 9, + "write error while doing btree update: %s", + bch2_err_str(ret)); goto err; } } @@ -692,13 +838,16 @@ static void bch2_write_endio(struct bio *bio) if (bch2_dev_inum_io_err_on(bio->bi_status, ca, op->pos.inode, - op->pos.offset - bio_sectors(bio), /* XXX definitely wrong */ + wbio->inode_offset << 9, "data write error: %s", bch2_blk_status_to_str(bio->bi_status))) { set_bit(wbio->dev, op->failed.d); op->flags |= BCH_WRITE_IO_ERROR; } + if (wbio->nocow) + set_bit(wbio->dev, op->devs_need_flush->d); + if (wbio->have_ioref) { bch2_latency_acct(ca, wbio->submit_time, WRITE); percpu_ref_put(&ca->io_ref); @@ -1122,6 +1271,301 @@ err: return ret; } +static bool bch2_extent_is_writeable(struct bch_write_op *op, + struct bkey_s_c k) +{ + struct bch_fs *c = op->c; + struct bkey_s_c_extent e; + struct extent_ptr_decoded p; + const union bch_extent_entry *entry; + unsigned replicas = 0; + + if (k.k->type != KEY_TYPE_extent) + return false; + + e = bkey_s_c_to_extent(k); + extent_for_each_ptr_decode(e, p, entry) { + if (p.crc.csum_type || + crc_is_compressed(p.crc) || + p.has_ec) + return false; + + replicas += bch2_extent_ptr_durability(c, &p); + } + + return replicas >= op->opts.data_replicas; +} + +static inline void bch2_nocow_write_unlock(struct bch_write_op *op) +{ + struct bch_fs *c = op->c; + const struct bch_extent_ptr *ptr; + struct bkey_i *k; + + for_each_keylist_key(&op->insert_keys, k) { + struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(k)); + + bkey_for_each_ptr(ptrs, ptr) + bch2_bucket_nocow_unlock(&c->nocow_locks, + PTR_BUCKET_POS(c, ptr), + BUCKET_NOCOW_LOCK_UPDATE); + } +} + +static int bch2_nocow_write_convert_one_unwritten(struct btree_trans *trans, + struct btree_iter *iter, + struct bkey_i *orig, + struct bkey_s_c k, + u64 new_i_size) +{ + struct bkey_i *new; + struct bkey_ptrs ptrs; + struct bch_extent_ptr *ptr; + int ret; + + if (!bch2_extents_match(bkey_i_to_s_c(orig), k)) { + /* trace this */ + return 0; + } + + new = bch2_trans_kmalloc(trans, bkey_bytes(k.k)); + ret = PTR_ERR_OR_ZERO(new); + if (ret) + return ret; + + bkey_reassemble(new, k); + + bch2_cut_front(bkey_start_pos(&orig->k), new); + bch2_cut_back(orig->k.p, new); + + ptrs = bch2_bkey_ptrs(bkey_i_to_s(new)); + bkey_for_each_ptr(ptrs, ptr) + ptr->unwritten = 0; + + /* + * Note that we're not calling bch2_subvol_get_snapshot() in this path - + * that was done when we kicked off the write, and here it's important + * that we update the extent that we wrote to - even if a snapshot has + * since been created. The write is still outstanding, so we're ok + * w.r.t. snapshot atomicity: + */ + return bch2_extent_update_i_size_sectors(trans, iter, + min(new->k.p.offset << 9, new_i_size), 0) ?: + bch2_trans_update(trans, iter, new, + BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE); +} + +static void bch2_nocow_write_convert_unwritten(struct bch_write_op *op) +{ + struct bch_fs *c = op->c; + struct btree_trans trans; + struct btree_iter iter; + struct bkey_i *orig; + struct bkey_s_c k; + int ret; + + bch2_trans_init(&trans, c, 0, 0); + + for_each_keylist_key(&op->insert_keys, orig) { + ret = for_each_btree_key_commit(&trans, iter, BTREE_ID_extents, + bkey_start_pos(&orig->k), + BTREE_ITER_INTENT, k, + NULL, NULL, BTREE_INSERT_NOFAIL, ({ + if (bkey_cmp(bkey_start_pos(k.k), orig->k.p) >= 0) + break; + + bch2_nocow_write_convert_one_unwritten(&trans, &iter, orig, k, op->new_i_size); + })); + + if (ret) { + struct bkey_i *k = bch2_keylist_front(&op->insert_keys); + + bch_err_inum_offset_ratelimited(c, + k->k.p.inode, k->k.p.offset << 9, + "write error while doing btree update: %s", + bch2_err_str(ret)); + op->error = ret; + break; + } + } + + bch2_trans_exit(&trans); +} + +static void __bch2_nocow_write_done(struct bch_write_op *op) +{ + bch2_nocow_write_unlock(op); + + if (unlikely(op->flags & BCH_WRITE_IO_ERROR)) { + op->error = -EIO; + } else if (unlikely(op->flags & BCH_WRITE_CONVERT_UNWRITTEN)) + bch2_nocow_write_convert_unwritten(op); +} + +static void bch2_nocow_write_done(struct closure *cl) +{ + struct bch_write_op *op = container_of(cl, struct bch_write_op, cl); + + __bch2_nocow_write_done(op); + bch2_write_done(cl); +} + +static void bch2_nocow_write(struct bch_write_op *op) +{ + struct bch_fs *c = op->c; + struct btree_trans trans; + struct btree_iter iter; + struct bkey_s_c k; + struct bkey_ptrs_c ptrs; + const struct bch_extent_ptr *ptr, *ptr2; + u32 snapshot; + int ret; + + if (op->flags & BCH_WRITE_MOVE) + return; + + bch2_trans_init(&trans, c, 0, 0); +retry: + bch2_trans_begin(&trans); + + ret = bch2_subvolume_get_snapshot(&trans, op->subvol, &snapshot); + if (unlikely(ret)) + goto err; + + bch2_trans_iter_init(&trans, &iter, BTREE_ID_extents, + SPOS(op->pos.inode, op->pos.offset, snapshot), + BTREE_ITER_SLOTS); + while (1) { + struct bio *bio = &op->wbio.bio; + + k = bch2_btree_iter_peek_slot(&iter); + ret = bkey_err(k); + if (ret) + break; + + /* fall back to normal cow write path? */ + if (unlikely(k.k->p.snapshot != snapshot || + !bch2_extent_is_writeable(op, k))) + break; + + if (bch2_keylist_realloc(&op->insert_keys, + op->inline_keys, + ARRAY_SIZE(op->inline_keys), + k.k->u64s)) + break; + + /* Get iorefs before dropping btree locks: */ + ptrs = bch2_bkey_ptrs_c(k); + bkey_for_each_ptr(ptrs, ptr) + if (unlikely(!bch2_dev_get_ioref(bch_dev_bkey_exists(c, ptr->dev), WRITE))) + goto err_get_ioref; + + /* Unlock before taking nocow locks, doing IO: */ + bkey_reassemble(op->insert_keys.top, k); + bch2_trans_unlock(&trans); + + bch2_cut_front(op->pos, op->insert_keys.top); + bch2_cut_back(POS(op->pos.inode, op->pos.offset + bio_sectors(bio)), op->insert_keys.top); + + ptrs = bch2_bkey_ptrs_c(bkey_i_to_s_c(op->insert_keys.top)); + bkey_for_each_ptr(ptrs, ptr) { + bch2_bucket_nocow_lock(&c->nocow_locks, + PTR_BUCKET_POS(c, ptr), + BUCKET_NOCOW_LOCK_UPDATE); + if (unlikely(ptr_stale(bch_dev_bkey_exists(c, ptr->dev), ptr))) + goto err_bucket_stale; + + if (ptr->unwritten) + op->flags |= BCH_WRITE_CONVERT_UNWRITTEN; + } + + bio = &op->wbio.bio; + if (k.k->p.offset < op->pos.offset + bio_sectors(bio)) { + bio = bio_split(bio, k.k->p.offset - op->pos.offset, + GFP_KERNEL, &c->bio_write); + wbio_init(bio)->put_bio = true; + bio->bi_opf = op->wbio.bio.bi_opf; + } else { + op->flags |= BCH_WRITE_DONE; + } + + op->pos.offset += bio_sectors(bio); + op->written += bio_sectors(bio); + + bio->bi_end_io = bch2_write_endio; + bio->bi_private = &op->cl; + bio->bi_opf |= REQ_OP_WRITE; + closure_get(&op->cl); + bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_user, + op->insert_keys.top, true); + + bch2_keylist_push(&op->insert_keys); + if (op->flags & BCH_WRITE_DONE) + break; + bch2_btree_iter_advance(&iter); + } +out: + bch2_trans_iter_exit(&trans, &iter); +err: + if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) + goto retry; + + if (ret) { + bch_err_inum_offset_ratelimited(c, + op->pos.inode, + op->pos.offset << 9, + "%s: btree lookup error %s", + __func__, bch2_err_str(ret)); + op->error = ret; + op->flags |= BCH_WRITE_DONE; + } + + bch2_trans_exit(&trans); + + /* fallback to cow write path? */ + if (!(op->flags & BCH_WRITE_DONE)) { + closure_sync(&op->cl); + __bch2_nocow_write_done(op); + op->insert_keys.top = op->insert_keys.keys; + } else if (op->flags & BCH_WRITE_SYNC) { + closure_sync(&op->cl); + bch2_nocow_write_done(&op->cl); + } else { + /* + * XXX + * needs to run out of process context because ei_quota_lock is + * a mutex + */ + continue_at(&op->cl, bch2_nocow_write_done, index_update_wq(op)); + } + return; +err_get_ioref: + bkey_for_each_ptr(ptrs, ptr2) { + if (ptr2 == ptr) + break; + + percpu_ref_put(&bch_dev_bkey_exists(c, ptr2->dev)->io_ref); + } + + /* Fall back to COW path: */ + goto out; +err_bucket_stale: + bkey_for_each_ptr(ptrs, ptr2) { + bch2_bucket_nocow_unlock(&c->nocow_locks, + PTR_BUCKET_POS(c, ptr2), + BUCKET_NOCOW_LOCK_UPDATE); + if (ptr2 == ptr) + break; + } + + bkey_for_each_ptr(ptrs, ptr2) + percpu_ref_put(&bch_dev_bkey_exists(c, ptr2->dev)->io_ref); + + /* We can retry this: */ + ret = BCH_ERR_transaction_restart; + goto out; +} + static void __bch2_write(struct bch_write_op *op) { struct bch_fs *c = op->c; @@ -1131,6 +1575,12 @@ static void __bch2_write(struct bch_write_op *op) int ret; nofs_flags = memalloc_nofs_save(); + + if (unlikely(op->opts.nocow)) { + bch2_nocow_write(op); + if (op->flags & BCH_WRITE_DONE) + goto out_nofs_restore; + } again: memset(&op->failed, 0, sizeof(op->failed)); op->btree_update_ready = false; @@ -1202,7 +1652,7 @@ again: key_to_write_offset); bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_user, - key_to_write); + key_to_write, false); } while (ret); /* @@ -1227,7 +1677,7 @@ again: continue_at(&op->cl, bch2_write_index, NULL); } - +out_nofs_restore: memalloc_nofs_restore(nofs_flags); } @@ -1307,8 +1757,10 @@ void bch2_write(struct closure *cl) wbio_init(bio)->put_bio = false; if (bio->bi_iter.bi_size & (c->opts.block_size - 1)) { - bch_err_inum_ratelimited(c, op->pos.inode, - "misaligned write"); + bch_err_inum_offset_ratelimited(c, + op->pos.inode, + op->pos.offset << 9, + "misaligned write"); op->error = -EIO; goto err; } @@ -1374,6 +1826,9 @@ static inline bool should_promote(struct bch_fs *c, struct bkey_s_c k, if (bch2_bkey_has_target(c, k, opts.promote_target)) return false; + if (bkey_extent_is_unwritten(k)) + return false; + if (bch2_target_congested(c, opts.promote_target)) { /* XXX trace this */ return false; @@ -1889,20 +2344,25 @@ csum_err: goto out; } - bch2_dev_inum_io_error(ca, rbio->read_pos.inode, (u64) rbio->bvec_iter.bi_sector, + bch_err_inum_offset_ratelimited(ca, + rbio->read_pos.inode, + rbio->read_pos.offset << 9, "data checksum error: expected %0llx:%0llx got %0llx:%0llx (type %s)", rbio->pick.crc.csum.hi, rbio->pick.crc.csum.lo, csum.hi, csum.lo, bch2_csum_types[crc.csum_type]); + bch2_io_error(ca); bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR); goto out; decompression_err: - bch_err_inum_ratelimited(c, rbio->read_pos.inode, - "decompression error"); + bch_err_inum_offset_ratelimited(c, rbio->read_pos.inode, + rbio->read_pos.offset << 9, + "decompression error"); bch2_rbio_error(rbio, READ_ERR, BLK_STS_IOERR); goto out; decrypt_err: - bch_err_inum_ratelimited(c, rbio->read_pos.inode, - "decrypt error"); + bch_err_inum_offset_ratelimited(c, rbio->read_pos.inode, + rbio->read_pos.offset << 9, + "decrypt error"); bch2_rbio_error(rbio, READ_ERR, BLK_STS_IOERR); goto out; } @@ -1977,7 +2437,9 @@ int __bch2_read_indirect_extent(struct btree_trans *trans, if (k.k->type != KEY_TYPE_reflink_v && k.k->type != KEY_TYPE_indirect_inline_data) { - bch_err_inum_ratelimited(trans->c, orig_k->k->k.p.inode, + bch_err_inum_offset_ratelimited(trans->c, + orig_k->k->k.p.inode, + orig_k->k->k.p.offset << 9, "%llu len %u points to nonexistent indirect extent %llu", orig_k->k->k.p.offset, orig_k->k->k.size, @@ -2063,8 +2525,9 @@ retry_pick: goto hole; if (pick_ret < 0) { - bch_err_inum_ratelimited(c, k.k->p.inode, - "no device to read from"); + bch_err_inum_offset_ratelimited(c, + read_pos.inode, read_pos.offset << 9, + "no device to read from"); goto err; } @@ -2243,8 +2706,10 @@ get_bio: if (!rbio->pick.idx) { if (!rbio->have_ioref) { - bch_err_inum_ratelimited(c, k.k->p.inode, - "no device to read from"); + bch_err_inum_offset_ratelimited(c, + read_pos.inode, + read_pos.offset << 9, + "no device to read from"); bch2_rbio_error(rbio, READ_RETRY_AVOID, BLK_STS_IOERR); goto out; } @@ -2412,8 +2877,9 @@ err: bch2_bkey_buf_exit(&sk, c); if (ret) { - bch_err_inum_ratelimited(c, inum.inum, - "read error %i from btree lookup", ret); + bch_err_inum_offset_ratelimited(c, inum.inum, + bvec_iter.bi_sector << 9, + "read error %i from btree lookup", ret); rbio->bio.bi_status = BLK_STS_IOERR; bch2_rbio_done(rbio); } @@ -2431,6 +2897,11 @@ void bch2_fs_io_exit(struct bch_fs *c) int bch2_fs_io_init(struct bch_fs *c) { + unsigned i; + + for (i = 0; i < ARRAY_SIZE(c->nocow_locks.l); i++) + two_state_lock_init(&c->nocow_locks.l[i]); + if (bioset_init(&c->bio_read, 1, offsetof(struct bch_read_bio, bio), BIOSET_NEED_BVECS) || bioset_init(&c->bio_read_split, 1, offsetof(struct bch_read_bio, bio), diff --git a/libbcachefs/io.h b/libbcachefs/io.h index e23ff0e..68e4d76 100644 --- a/libbcachefs/io.h +++ b/libbcachefs/io.h @@ -18,7 +18,7 @@ void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, size_t); void bch2_latency_acct(struct bch_dev *, u64, int); void bch2_submit_wbio_replicas(struct bch_write_bio *, struct bch_fs *, - enum bch_data_type, const struct bkey_i *); + enum bch_data_type, const struct bkey_i *, bool); #define BLK_STS_REMOVED ((__force blk_status_t)128) @@ -39,6 +39,7 @@ enum bch_write_flags { /* Internal: */ BCH_WRITE_DONE = (1 << 10), BCH_WRITE_IO_ERROR = (1 << 11), + BCH_WRITE_CONVERT_UNWRITTEN = (1 << 12), }; static inline struct workqueue_struct *index_update_wq(struct bch_write_op *op) @@ -53,6 +54,9 @@ int bch2_sum_sector_overwrites(struct btree_trans *, struct btree_iter *, int bch2_extent_update(struct btree_trans *, subvol_inum, struct btree_iter *, struct bkey_i *, struct disk_reservation *, u64, s64 *, bool); +int bch2_extent_fallocate(struct btree_trans *, subvol_inum, struct btree_iter *, + unsigned, struct bch_io_opts, s64 *, + struct write_point_specifier); int bch2_fpunch_at(struct btree_trans *, struct btree_iter *, subvol_inum, u64, s64 *); @@ -66,7 +70,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c, op->flags = 0; op->written = 0; op->error = 0; - op->csum_type = bch2_data_checksum_type(c, opts.data_checksum); + op->csum_type = bch2_data_checksum_type(c, opts); op->compression_type = bch2_compression_opt_to_type[opts.compression]; op->nr_replicas = 0; op->nr_replicas_required = c->opts.data_replicas_required; @@ -83,6 +87,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c, op->res = (struct disk_reservation) { 0 }; op->new_i_size = U64_MAX; op->i_sectors_delta = 0; + op->devs_need_flush = NULL; } void bch2_write(struct closure *); diff --git a/libbcachefs/io_types.h b/libbcachefs/io_types.h index a91635d..4e5d310 100644 --- a/libbcachefs/io_types.h +++ b/libbcachefs/io_types.h @@ -87,6 +87,7 @@ struct bch_write_bio { struct bch_write_bio *parent; u64 submit_time; + u64 inode_offset; struct bch_devs_list failed; u8 dev; @@ -95,6 +96,7 @@ struct bch_write_bio { bounce:1, put_bio:1, have_ioref:1, + nocow:1, used_mempool:1, first_btree_write:1; @@ -148,6 +150,12 @@ struct bch_write_op { struct keylist insert_keys; u64 inline_keys[BKEY_EXTENT_U64s_MAX * 2]; + /* + * Bitmask of devices that have had nocow writes issued to them since + * last flush: + */ + struct bch_devs_mask *devs_need_flush; + /* Must be last: */ struct bch_write_bio wbio; }; diff --git a/libbcachefs/journal_io.c b/libbcachefs/journal_io.c index c4922c6..fb7d0bf 100644 --- a/libbcachefs/journal_io.c +++ b/libbcachefs/journal_io.c @@ -1269,10 +1269,10 @@ int bch2_journal_read(struct bch_fs *c, u64 *blacklist_seq, u64 *start_seq) struct bch_dev *ca = bch_dev_bkey_exists(c, i->ptrs[ptr].dev); if (!i->ptrs[ptr].csum_good) - printk(KERN_ERR "bcachefs (%s) sector %llu: invalid journal checksum, seq %llu%s\n", - ca->name, i->ptrs[ptr].sector, - le64_to_cpu(i->j.seq), - i->csum_good ? " (had good copy on another device)" : ""); + bch_err_dev_offset(ca, i->ptrs[ptr].sector, + "invalid journal checksum, seq %llu%s", + le64_to_cpu(i->j.seq), + i->csum_good ? " (had good copy on another device)" : ""); } ret = jset_validate(c, diff --git a/libbcachefs/move.c b/libbcachefs/move.c index 1d11cf0..700f847 100644 --- a/libbcachefs/move.c +++ b/libbcachefs/move.c @@ -261,6 +261,12 @@ static int bch2_move_extent(struct btree_trans *trans, if (!percpu_ref_tryget_live(&c->writes)) return -EROFS; + /* + * Before memory allocations & taking nocow locks in + * bch2_data_update_init(): + */ + bch2_trans_unlock(trans); + /* write path might have to decompress data: */ bkey_for_each_ptr_decode(k.k, ptrs, p, entry) sectors = max_t(unsigned, sectors, p.crc.uncompressed_size); @@ -296,7 +302,7 @@ static int bch2_move_extent(struct btree_trans *trans, ret = bch2_data_update_init(c, &io->write, ctxt->wp, io_opts, data_opts, btree_id, k); - if (ret) + if (ret && ret != -BCH_ERR_unwritten_extent_update) goto err_free_pages; io->write.ctxt = ctxt; @@ -304,6 +310,15 @@ static int bch2_move_extent(struct btree_trans *trans, atomic64_inc(&ctxt->stats->keys_moved); atomic64_add(k.k->size, &ctxt->stats->sectors_moved); + + if (ret == -BCH_ERR_unwritten_extent_update) { + bch2_update_unwritten_extent(trans, &io->write); + move_free(io); + return 0; + } + + BUG_ON(ret); + this_cpu_add(c->counters[BCH_COUNTER_io_move], k.k->size); this_cpu_add(c->counters[BCH_COUNTER_move_extent_read], k.k->size); trace_move_extent_read(k.k); @@ -499,6 +514,7 @@ static int __bch2_move_data(struct moving_context *ctxt, */ bch2_bkey_buf_reassemble(&sk, c, k); k = bkey_i_to_s_c(sk.k); + bch2_trans_unlock(&trans); ret2 = bch2_move_extent(&trans, &iter, ctxt, io_opts, btree_id, k, data_opts); @@ -573,6 +589,9 @@ static int verify_bucket_evacuated(struct btree_trans *trans, struct bpos bucket struct bch_fs *c = trans->c; struct btree_iter iter; struct bkey_s_c k; + struct printbuf buf = PRINTBUF; + struct bch_backpointer bp; + u64 bp_offset = 0; int ret; bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc, @@ -586,24 +605,53 @@ again: if (a.v->gen == gen && a.v->dirty_sectors) { - struct printbuf buf = PRINTBUF; - if (a.v->data_type == BCH_DATA_btree) { bch2_trans_unlock(trans); if (bch2_btree_interior_updates_flush(c)) goto again; + goto failed_to_evacuate; } - - prt_str(&buf, "failed to evacuate bucket "); - bch2_bkey_val_to_text(&buf, c, k); - - bch_err(c, "%s", buf.buf); - printbuf_exit(&buf); } } bch2_trans_iter_exit(trans, &iter); return ret; +failed_to_evacuate: + bch2_trans_iter_exit(trans, &iter); + + prt_printf(&buf, bch2_log_msg(c, "failed to evacuate bucket ")); + bch2_bkey_val_to_text(&buf, c, k); + + while (1) { + bch2_trans_begin(trans); + + ret = bch2_get_next_backpointer(trans, bucket, gen, + &bp_offset, &bp, + BTREE_ITER_CACHED); + if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) + continue; + if (ret) + break; + if (bp_offset == U64_MAX) + break; + + k = bch2_backpointer_get_key(trans, &iter, + bucket, bp_offset, bp); + ret = bkey_err(k); + if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) + continue; + if (ret) + break; + if (!k.k) + continue; + prt_newline(&buf); + bch2_bkey_val_to_text(&buf, c, k); + bch2_trans_iter_exit(trans, &iter); + } + + bch2_print_string_as_lines(KERN_ERR, buf.buf); + printbuf_exit(&buf); + return 0; } int __bch2_evacuate_bucket(struct moving_context *ctxt, diff --git a/libbcachefs/nocow_locking.c b/libbcachefs/nocow_locking.c new file mode 100644 index 0000000..54e8669 --- /dev/null +++ b/libbcachefs/nocow_locking.c @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "bcachefs.h" +#include "nocow_locking.h" +#include "util.h" + +void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t, + struct bpos bucket, int flags) +{ + struct bch_fs *c = container_of(t, struct bch_fs, nocow_locks); + two_state_lock_t *l = bucket_nocow_lock(t, bucket); + u64 start_time = local_clock(); + + bch2_two_state_lock(l, flags & BUCKET_NOCOW_LOCK_UPDATE); + bch2_time_stats_update(&c->times[BCH_TIME_nocow_lock_contended], start_time); +} diff --git a/libbcachefs/nocow_locking.h b/libbcachefs/nocow_locking.h new file mode 100644 index 0000000..09ab85a --- /dev/null +++ b/libbcachefs/nocow_locking.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BCACHEFS_NOCOW_LOCKING_H +#define _BCACHEFS_NOCOW_LOCKING_H + +#include "bcachefs_format.h" +#include "two_state_shared_lock.h" + +#include + +#define BUCKET_NOCOW_LOCKS (1U << 10) + +struct bucket_nocow_lock_table { + siphash_key_t key; + two_state_lock_t l[BUCKET_NOCOW_LOCKS]; +}; + +#define BUCKET_NOCOW_LOCK_UPDATE (1 << 0) + +static inline two_state_lock_t *bucket_nocow_lock(struct bucket_nocow_lock_table *t, + struct bpos bucket) +{ + u64 dev_bucket = bucket.inode << 56 | bucket.offset; + unsigned h = siphash_1u64(dev_bucket, &t->key); + + return t->l + (h & (BUCKET_NOCOW_LOCKS - 1)); +} + +static inline bool bch2_bucket_nocow_is_locked(struct bucket_nocow_lock_table *t, + struct bpos bucket) +{ + two_state_lock_t *l = bucket_nocow_lock(t, bucket); + + return atomic_long_read(&l->v) != 0; +} + +static inline void bch2_bucket_nocow_unlock(struct bucket_nocow_lock_table *t, + struct bpos bucket, int flags) +{ + two_state_lock_t *l = bucket_nocow_lock(t, bucket); + + bch2_two_state_unlock(l, flags & BUCKET_NOCOW_LOCK_UPDATE); +} + +void __bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *, struct bpos, int); + +static inline void bch2_bucket_nocow_lock(struct bucket_nocow_lock_table *t, + struct bpos bucket, int flags) +{ + two_state_lock_t *l = bucket_nocow_lock(t, bucket); + + if (!bch2_two_state_trylock(l, flags & BUCKET_NOCOW_LOCK_UPDATE)) + __bch2_bucket_nocow_lock(t, bucket, flags); +} + +#endif /* _BCACHEFS_NOCOW_LOCKING_H */ diff --git a/libbcachefs/opts.h b/libbcachefs/opts.h index 5b8586e..ad0b13e 100644 --- a/libbcachefs/opts.h +++ b/libbcachefs/opts.h @@ -382,6 +382,13 @@ enum opt_type { OPT_BOOL(), \ BCH2_NO_SB_OPT, false, \ NULL, NULL) \ + x(nocow, u8, \ + OPT_FS|OPT_FORMAT|OPT_MOUNT|OPT_RUNTIME|OPT_INODE, \ + OPT_BOOL(), \ + BCH_SB_NOCOW, false, \ + NULL, "Nocow mode: Writes will be done in place when possible.\n"\ + "Snapshots and reflink will still caused writes to be COW\n"\ + "Implicitly disables data checksumming, compression and encryption")\ x(fs_size, u64, \ OPT_DEVICE, \ OPT_UINT(0, S64_MAX), \ diff --git a/libbcachefs/quota.c b/libbcachefs/quota.c index db81727..7f74c02 100644 --- a/libbcachefs/quota.c +++ b/libbcachefs/quota.c @@ -364,16 +364,16 @@ int bch2_quota_acct(struct bch_fs *c, struct bch_qid qid, memset(&msgs, 0, sizeof(msgs)); + for_each_set_qtype(c, i, q, qtypes) { + mq[i] = genradix_ptr_alloc(&q->table, qid.q[i], GFP_KERNEL); + if (!mq[i]) + return -ENOMEM; + } + for_each_set_qtype(c, i, q, qtypes) mutex_lock_nested(&q->lock, i); for_each_set_qtype(c, i, q, qtypes) { - mq[i] = genradix_ptr_alloc(&q->table, qid.q[i], GFP_NOFS); - if (!mq[i]) { - ret = -ENOMEM; - goto err; - } - ret = bch2_quota_check_limit(c, i, mq[i], &msgs, counter, v, mode); if (ret) goto err; @@ -416,18 +416,17 @@ int bch2_quota_transfer(struct bch_fs *c, unsigned qtypes, memset(&msgs, 0, sizeof(msgs)); + for_each_set_qtype(c, i, q, qtypes) { + src_q[i] = genradix_ptr_alloc(&q->table, src.q[i], GFP_KERNEL); + dst_q[i] = genradix_ptr_alloc(&q->table, dst.q[i], GFP_KERNEL); + if (!src_q[i] || !dst_q[i]) + return -ENOMEM; + } + for_each_set_qtype(c, i, q, qtypes) mutex_lock_nested(&q->lock, i); for_each_set_qtype(c, i, q, qtypes) { - src_q[i] = genradix_ptr_alloc(&q->table, src.q[i], GFP_NOFS); - dst_q[i] = genradix_ptr_alloc(&q->table, dst.q[i], GFP_NOFS); - - if (!src_q[i] || !dst_q[i]) { - ret = -ENOMEM; - goto err; - } - ret = bch2_quota_check_limit(c, i, dst_q[i], &msgs, Q_SPC, dst_q[i]->c[Q_SPC].v + space, mode); diff --git a/libbcachefs/reflink.c b/libbcachefs/reflink.c index 0d4c004..94f2b30 100644 --- a/libbcachefs/reflink.c +++ b/libbcachefs/reflink.c @@ -255,6 +255,9 @@ static struct bkey_s_c get_next_src(struct btree_iter *iter, struct bpos end) if (bkey_cmp(iter->pos, end) >= 0) break; + if (bkey_extent_is_unwritten(k)) + continue; + if (bkey_extent_is_data(k.k)) return k; } diff --git a/libbcachefs/super.h b/libbcachefs/super.h index 8501ada..3c83e9b 100644 --- a/libbcachefs/super.h +++ b/libbcachefs/super.h @@ -88,9 +88,10 @@ static inline void bch2_dev_list_drop_dev(struct bch_devs_list *devs, static inline void bch2_dev_list_add_dev(struct bch_devs_list *devs, unsigned dev) { - BUG_ON(bch2_dev_list_has_dev(*devs, dev)); - BUG_ON(devs->nr >= ARRAY_SIZE(devs->devs)); - devs->devs[devs->nr++] = dev; + if (!bch2_dev_list_has_dev(*devs, dev)) { + BUG_ON(devs->nr >= ARRAY_SIZE(devs->devs)); + devs->devs[devs->nr++] = dev; + } } static inline struct bch_devs_list bch2_dev_list_single(unsigned dev) diff --git a/libbcachefs/util.h b/libbcachefs/util.h index 846e602..473c969 100644 --- a/libbcachefs/util.h +++ b/libbcachefs/util.h @@ -23,33 +23,9 @@ struct closure; #ifdef CONFIG_BCACHEFS_DEBUG - #define EBUG_ON(cond) BUG_ON(cond) -#define atomic_dec_bug(v) BUG_ON(atomic_dec_return(v) < 0) -#define atomic_inc_bug(v, i) BUG_ON(atomic_inc_return(v) <= i) -#define atomic_sub_bug(i, v) BUG_ON(atomic_sub_return(i, v) < 0) -#define atomic_add_bug(i, v) BUG_ON(atomic_add_return(i, v) < 0) -#define atomic_long_dec_bug(v) BUG_ON(atomic_long_dec_return(v) < 0) -#define atomic_long_sub_bug(i, v) BUG_ON(atomic_long_sub_return(i, v) < 0) -#define atomic64_dec_bug(v) BUG_ON(atomic64_dec_return(v) < 0) -#define atomic64_inc_bug(v, i) BUG_ON(atomic64_inc_return(v) <= i) -#define atomic64_sub_bug(i, v) BUG_ON(atomic64_sub_return(i, v) < 0) -#define atomic64_add_bug(i, v) BUG_ON(atomic64_add_return(i, v) < 0) - -#else /* DEBUG */ - +#else #define EBUG_ON(cond) -#define atomic_dec_bug(v) atomic_dec(v) -#define atomic_inc_bug(v, i) atomic_inc(v) -#define atomic_sub_bug(i, v) atomic_sub(i, v) -#define atomic_add_bug(i, v) atomic_add(i, v) -#define atomic_long_dec_bug(v) atomic_long_dec(v) -#define atomic_long_sub_bug(i, v) atomic_long_sub(i, v) -#define atomic64_dec_bug(v) atomic64_dec(v) -#define atomic64_inc_bug(v, i) atomic64_inc(v) -#define atomic64_sub_bug(i, v) atomic64_sub(i, v) -#define atomic64_add_bug(i, v) atomic64_add(i, v) - #endif #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__