X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libbcachefs%2Fbtree_update_leaf.c;h=496855233c4c82a962c5ee6b91a8e0419d51e9ae;hb=dd707c9bfdc13fd90856576ddbb383efccef751c;hp=d05ed96c42804145292f08e2c2ad6f74aade6c49;hpb=304691592738dc272f4150107b54a53ab43fc8be;p=bcachefs-tools-debian diff --git a/libbcachefs/btree_update_leaf.c b/libbcachefs/btree_update_leaf.c index d05ed96..4968552 100644 --- a/libbcachefs/btree_update_leaf.c +++ b/libbcachefs/btree_update_leaf.c @@ -6,6 +6,7 @@ #include "btree_gc.h" #include "btree_io.h" #include "btree_iter.h" +#include "btree_key_cache.h" #include "btree_locking.h" #include "buckets.h" #include "debug.h" @@ -23,16 +24,18 @@ static inline bool same_leaf_as_prev(struct btree_trans *trans, struct btree_insert_entry *i) { - return i != trans->updates && - i[0].iter->l[0].b == i[-1].iter->l[0].b; + return i != trans->updates2 && + iter_l(i[0].iter)->b == iter_l(i[-1].iter)->b; } - inline void bch2_btree_node_lock_for_insert(struct bch_fs *c, struct btree *b, struct btree_iter *iter) { bch2_btree_node_lock_write(b, iter); + if (btree_iter_type(iter) == BTREE_ITER_CACHED) + return; + if (unlikely(btree_node_just_written(b)) && bch2_btree_post_write_cleanup(c, b)) bch2_btree_iter_reinit_node(iter, b); @@ -53,82 +56,76 @@ bool bch2_btree_bset_insert_key(struct btree_iter *iter, struct btree_node_iter *node_iter, struct bkey_i *insert) { - const struct bkey_format *f = &b->format; struct bkey_packed *k; - unsigned clobber_u64s; + unsigned clobber_u64s = 0, new_u64s = 0; EBUG_ON(btree_node_just_written(b)); EBUG_ON(bset_written(b, btree_bset_last(b))); EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k)); - EBUG_ON(bkey_cmp(bkey_start_pos(&insert->k), b->data->min_key) < 0 || - bkey_cmp(insert->k.p, b->data->max_key) > 0); + EBUG_ON(bkey_cmp(b->data->min_key, POS_MIN) && + bkey_cmp(bkey_start_pos(&insert->k), + bkey_predecessor(b->data->min_key)) < 0); + EBUG_ON(bkey_cmp(insert->k.p, b->data->min_key) < 0); + EBUG_ON(bkey_cmp(insert->k.p, b->data->max_key) > 0); + EBUG_ON(insert->k.u64s > + bch_btree_keys_u64s_remaining(iter->trans->c, b)); + EBUG_ON(iter->flags & BTREE_ITER_IS_EXTENTS); k = bch2_btree_node_iter_peek_all(node_iter, b); - if (k && !bkey_cmp_packed(b, k, &insert->k)) { - BUG_ON(bkey_whiteout(k)); - - if (!bkey_written(b, k) && - bkey_val_u64s(&insert->k) == bkeyp_val_u64s(f, k) && - !bkey_whiteout(&insert->k)) { - k->type = insert->k.type; - memcpy_u64s(bkeyp_val(f, k), &insert->v, - bkey_val_u64s(&insert->k)); - return true; - } - - btree_account_key_drop(b, k); - - if (bkey_whiteout(&insert->k)) { - unsigned clobber_u64s = k->u64s, new_u64s = k->u64s; - - k->type = KEY_TYPE_deleted; + if (k && bkey_cmp_left_packed(b, k, &insert->k.p)) + k = NULL; - if (k->needs_whiteout) { - push_whiteout(iter->trans->c, b, k); - k->needs_whiteout = false; - } + /* @k is the key being overwritten/deleted, if any: */ + EBUG_ON(k && bkey_whiteout(k)); - if (k >= btree_bset_last(b)->start) { - bch2_bset_delete(b, k, clobber_u64s); - new_u64s = 0; - } + /* Deleting, but not found? nothing to do: */ + if (bkey_whiteout(&insert->k) && !k) + return false; - bch2_btree_node_iter_fix(iter, b, node_iter, k, - clobber_u64s, new_u64s); - return true; + if (bkey_whiteout(&insert->k)) { + /* Deleting: */ + btree_account_key_drop(b, k); + k->type = KEY_TYPE_deleted; - } + if (k->needs_whiteout) + push_whiteout(iter->trans->c, b, insert->k.p); + k->needs_whiteout = false; if (k >= btree_bset_last(b)->start) { clobber_u64s = k->u64s; - goto overwrite; + bch2_bset_delete(b, k, clobber_u64s); + goto fix_iter; + } else { + bch2_btree_iter_fix_key_modified(iter, b, k); } + return true; + } + + if (k) { + /* Overwriting: */ + btree_account_key_drop(b, k); + k->type = KEY_TYPE_deleted; + insert->k.needs_whiteout = k->needs_whiteout; k->needs_whiteout = false; - k->type = KEY_TYPE_deleted; - /* - * XXX: we should be able to do this without two calls to - * bch2_btree_node_iter_fix: - */ - bch2_btree_node_iter_fix(iter, b, node_iter, k, - k->u64s, k->u64s); - } else { - /* - * Deleting, but the key to delete wasn't found - nothing to do: - */ - if (bkey_whiteout(&insert->k)) - return false; - insert->k.needs_whiteout = false; + if (k >= btree_bset_last(b)->start) { + clobber_u64s = k->u64s; + goto overwrite; + } else { + bch2_btree_iter_fix_key_modified(iter, b, k); + } } k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b)); - clobber_u64s = 0; overwrite: bch2_bset_insert(b, node_iter, k, insert, clobber_u64s); - bch2_btree_node_iter_fix(iter, b, node_iter, k, - clobber_u64s, k->u64s); + new_u64s = k->u64s; +fix_iter: + if (clobber_u64s != new_u64s) + bch2_btree_node_iter_fix(iter, b, node_iter, k, + clobber_u64s, new_u64s); return true; } @@ -142,7 +139,7 @@ static void __btree_node_flush(struct journal *j, struct journal_entry_pin *pin, btree_node_lock_type(c, b, SIX_LOCK_read); bch2_btree_node_write_cond(c, b, (btree_current_write(b) == w && w->journal.seq == seq)); - six_unlock_read(&b->lock); + six_unlock_read(&b->c.lock); } static void btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq) @@ -155,95 +152,46 @@ static void btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, return __btree_node_flush(j, pin, 1, seq); } -static inline void __btree_journal_key(struct btree_trans *trans, - enum btree_id btree_id, - struct bkey_i *insert) -{ - struct journal *j = &trans->c->journal; - u64 seq = trans->journal_res.seq; - bool needs_whiteout = insert->k.needs_whiteout; - - /* ick */ - insert->k.needs_whiteout = false; - bch2_journal_add_keys(j, &trans->journal_res, - btree_id, insert); - insert->k.needs_whiteout = needs_whiteout; - - bch2_journal_set_has_inode(j, &trans->journal_res, - insert->k.p.inode); - - if (trans->journal_seq) - *trans->journal_seq = seq; -} - -void bch2_btree_journal_key(struct btree_trans *trans, - struct btree_iter *iter, - struct bkey_i *insert) +inline void bch2_btree_add_journal_pin(struct bch_fs *c, + struct btree *b, u64 seq) { - struct bch_fs *c = trans->c; - struct journal *j = &c->journal; - struct btree *b = iter->l[0].b; struct btree_write *w = btree_current_write(b); - EBUG_ON(iter->level || b->level); - EBUG_ON(trans->journal_res.ref != - !(trans->flags & BTREE_INSERT_JOURNAL_REPLAY)); - - if (likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))) { - __btree_journal_key(trans, iter->btree_id, insert); - btree_bset_last(b)->journal_seq = - cpu_to_le64(trans->journal_res.seq); - } - - if (unlikely(!journal_pin_active(&w->journal))) { - u64 seq = likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY)) - ? trans->journal_res.seq - : j->replay_journal_seq; - - bch2_journal_pin_add(j, seq, &w->journal, - btree_node_write_idx(b) == 0 - ? btree_node_flush0 - : btree_node_flush1); - } - - if (unlikely(!btree_node_dirty(b))) - set_btree_node_dirty(b); -} - -static void bch2_insert_fixup_key(struct btree_trans *trans, - struct btree_iter *iter, - struct bkey_i *insert) -{ - struct btree_iter_level *l = &iter->l[0]; - - EBUG_ON(iter->level); - EBUG_ON(insert->k.u64s > - bch_btree_keys_u64s_remaining(trans->c, l->b)); - - if (likely(bch2_btree_bset_insert_key(iter, l->b, &l->iter, insert))) - bch2_btree_journal_key(trans, iter, insert); + bch2_journal_pin_add(&c->journal, seq, &w->journal, + btree_node_write_idx(b) == 0 + ? btree_node_flush0 + : btree_node_flush1); } /** * btree_insert_key - insert a key one key into a leaf node */ -static void btree_insert_key_leaf(struct btree_trans *trans, +static bool btree_insert_key_leaf(struct btree_trans *trans, struct btree_iter *iter, struct bkey_i *insert) { struct bch_fs *c = trans->c; - struct btree *b = iter->l[0].b; + struct btree *b = iter_l(iter)->b; struct bset_tree *t = bset_tree_last(b); + struct bset *i = bset(b, t); int old_u64s = bset_u64s(t); int old_live_u64s = b->nr.live_u64s; int live_u64s_added, u64s_added; - insert->k.needs_whiteout = false; + EBUG_ON(!iter->level && + !test_bit(BCH_FS_BTREE_INTERIOR_REPLAY_DONE, &c->flags)); + + if (unlikely(!bch2_btree_bset_insert_key(iter, b, + &iter_l(iter)->iter, insert))) + return false; - if (!btree_node_is_extents(b)) - bch2_insert_fixup_key(trans, iter, insert); - else - bch2_insert_fixup_extent(trans, iter, insert); + i->journal_seq = cpu_to_le64(max(trans->journal_res.seq, + le64_to_cpu(i->journal_seq))); + + bch2_btree_add_journal_pin(c, b, trans->journal_res.seq); + + if (unlikely(!btree_node_dirty(b))) + set_btree_node_dirty(c, b); live_u64s_added = (int) b->nr.live_u64s - old_live_u64s; u64s_added = (int) bset_u64s(t) - old_u64s; @@ -258,8 +206,11 @@ static void btree_insert_key_leaf(struct btree_trans *trans, bch2_btree_iter_reinit_node(iter, b); trace_btree_insert_key(c, b, insert); + return true; } +/* Cached btree updates: */ + /* Normal update interface: */ static inline void btree_insert_entry_checks(struct btree_trans *trans, @@ -268,14 +219,10 @@ static inline void btree_insert_entry_checks(struct btree_trans *trans, { struct bch_fs *c = trans->c; - BUG_ON(iter->level); - BUG_ON(bkey_cmp(bkey_start_pos(&insert->k), iter->pos)); - EBUG_ON((iter->flags & BTREE_ITER_IS_EXTENTS) && - bkey_cmp(insert->k.p, iter->l[0].b->key.k.p) > 0); - - BUG_ON(debug_check_bkeys(c) && - !bkey_deleted(&insert->k) && - bch2_bkey_invalid(c, bkey_i_to_s_c(insert), iter->btree_id)); + BUG_ON(bkey_cmp(insert->k.p, iter->pos)); + BUG_ON(bch2_debug_check_bkeys && + bch2_bkey_invalid(c, bkey_i_to_s_c(insert), + __btree_node_type(iter->level, iter->btree_id))); } static noinline int @@ -317,25 +264,42 @@ static inline int bch2_trans_journal_res_get(struct btree_trans *trans, static enum btree_insert_ret btree_key_can_insert(struct btree_trans *trans, struct btree_iter *iter, - struct bkey_i *insert, - unsigned *u64s) + unsigned u64s) { struct bch_fs *c = trans->c; - struct btree *b = iter->l[0].b; - static enum btree_insert_ret ret; + struct btree *b = iter_l(iter)->b; - if (unlikely(btree_node_fake(b))) + if (!bch2_btree_node_insert_fits(c, b, u64s)) return BTREE_INSERT_BTREE_NODE_FULL; - ret = !btree_node_is_extents(b) - ? BTREE_INSERT_OK - : bch2_extent_can_insert(trans, iter, insert, u64s); - if (ret) - return ret; + return BTREE_INSERT_OK; +} - if (*u64s > bch_btree_keys_u64s_remaining(c, b)) - return BTREE_INSERT_BTREE_NODE_FULL; +static enum btree_insert_ret +btree_key_can_insert_cached(struct btree_trans *trans, + struct btree_iter *iter, + unsigned u64s) +{ + struct bkey_cached *ck = (void *) iter->l[0].b; + unsigned new_u64s; + struct bkey_i *new_k; + + BUG_ON(iter->level); + + if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) && + bch2_btree_key_cache_must_wait(trans->c)) + return BTREE_INSERT_NEED_JOURNAL_RECLAIM; + + if (u64s <= ck->u64s) + return BTREE_INSERT_OK; + new_u64s = roundup_pow_of_two(u64s); + new_k = krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOFS); + if (!new_k) + return -ENOMEM; + + ck->u64s = new_u64s; + ck->k = new_k; return BTREE_INSERT_OK; } @@ -343,7 +307,31 @@ static inline void do_btree_insert_one(struct btree_trans *trans, struct btree_iter *iter, struct bkey_i *insert) { - btree_insert_key_leaf(trans, iter, insert); + struct bch_fs *c = trans->c; + struct journal *j = &c->journal; + bool did_work; + + EBUG_ON(trans->journal_res.ref != + !(trans->flags & BTREE_INSERT_JOURNAL_REPLAY)); + + insert->k.needs_whiteout = false; + + did_work = (btree_iter_type(iter) != BTREE_ITER_CACHED) + ? btree_insert_key_leaf(trans, iter, insert) + : bch2_btree_insert_key_cached(trans, iter, insert); + if (!did_work) + return; + + if (likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))) { + bch2_journal_add_keys(j, &trans->journal_res, + iter->btree_id, insert); + + bch2_journal_set_has_inode(j, &trans->journal_res, + insert->k.p.inode); + + if (trans->journal_seq) + *trans->journal_seq = trans->journal_res.seq; + } } static inline bool iter_has_trans_triggers(struct btree_iter *iter) @@ -353,8 +341,9 @@ static inline bool iter_has_trans_triggers(struct btree_iter *iter) static inline bool iter_has_nontrans_triggers(struct btree_iter *iter) { - return (BTREE_NODE_TYPE_HAS_TRIGGERS & - ~BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS) & + return (((BTREE_NODE_TYPE_HAS_TRIGGERS & + ~BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS)) | + (1U << BTREE_ID_EC)) & (1U << iter->btree_id); } @@ -368,10 +357,16 @@ static noinline void bch2_trans_mark_gc(struct btree_trans *trans) struct bch_fs *c = trans->c; struct btree_insert_entry *i; - trans_for_each_update(trans, i) + trans_for_each_update(trans, i) { + /* + * XXX: synchronization of cached update triggers with gc + */ + BUG_ON(btree_iter_type(i->iter) == BTREE_ITER_CACHED); + if (gc_visited(c, gc_pos_btree_node(i->iter->l[0].b))) bch2_mark_update(trans, i->iter, i->k, NULL, i->trigger_flags|BTREE_TRIGGER_GC); + } } static inline int @@ -398,13 +393,15 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, prefetch(&trans->c->journal.flags); - trans_for_each_update(trans, i) { + trans_for_each_update2(trans, i) { /* Multiple inserts might go to same leaf: */ if (!same_leaf_as_prev(trans, i)) u64s = 0; u64s += i->k->k.u64s; - ret = btree_key_can_insert(trans, i->iter, i->k, &u64s); + ret = btree_iter_type(i->iter) != BTREE_ITER_CACHED + ? btree_key_can_insert(trans, i->iter, u64s) + : btree_key_can_insert_cached(trans, i->iter, u64s); if (ret) { *stopped_at = i; return ret; @@ -428,6 +425,17 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, JOURNAL_RES_GET_NONBLOCK); if (ret) goto err; + } else { + trans->journal_res.seq = c->journal.replay_journal_seq; + } + + if (unlikely(trans->extra_journal_entry_u64s)) { + memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res), + trans->extra_journal_entries, + trans->extra_journal_entry_u64s); + + trans->journal_res.offset += trans->extra_journal_entry_u64s; + trans->journal_res.u64s -= trans->extra_journal_entry_u64s; } /* @@ -436,11 +444,11 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, */ if (!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY)) { - if (journal_seq_verify(c)) - trans_for_each_update(trans, i) + if (bch2_journal_seq_verify) + trans_for_each_update2(trans, i) i->k->k.version.lo = trans->journal_res.seq; - else if (inject_invalid_keys(c)) - trans_for_each_update(trans, i) + else if (bch2_inject_invalid_keys) + trans_for_each_update2(trans, i) i->k->k.version = MAX_VERSION; } @@ -463,7 +471,7 @@ bch2_trans_commit_write_locked(struct btree_trans *trans, if (unlikely(c->gc_pos.phase)) bch2_trans_mark_gc(trans); - trans_for_each_update(trans, i) + trans_for_each_update2(trans, i) do_btree_insert_one(trans, i->iter, i->k); err: if (marking) { @@ -484,12 +492,14 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans, struct btree_iter *iter; int ret; - trans_for_each_update(trans, i) - BUG_ON(!btree_node_intent_locked(i->iter, 0)); + trans_for_each_update2(trans, i) + BUG_ON(!btree_node_intent_locked(i->iter, i->iter->level)); ret = bch2_journal_preres_get(&trans->c->journal, &trans->journal_preres, trans->journal_preres_u64s, - JOURNAL_RES_GET_NONBLOCK); + JOURNAL_RES_GET_NONBLOCK| + ((trans->flags & BTREE_INSERT_JOURNAL_RECLAIM) + ? JOURNAL_RES_GET_RECLAIM : 0)); if (unlikely(ret == -EAGAIN)) ret = bch2_trans_journal_preres_get_cold(trans, trans->journal_preres_u64s); @@ -503,7 +513,7 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans, * or anything else that might call bch2_trans_relock(), since that * would just retake the read locks: */ - trans_for_each_iter_all(trans, iter) { + trans_for_each_iter(trans, iter) { if (iter->nodes_locked != iter->nodes_intent_locked) { EBUG_ON(iter->flags & BTREE_ITER_KEEP_UNTIL_COMMIT); EBUG_ON(trans->iters_live & (1ULL << iter->idx)); @@ -512,22 +522,26 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans, } if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG)) - trans_for_each_update(trans, i) + trans_for_each_update2(trans, i) btree_insert_entry_checks(trans, i->iter, i->k); bch2_btree_trans_verify_locks(trans); - trans_for_each_update(trans, i) + trans_for_each_update2(trans, i) if (!same_leaf_as_prev(trans, i)) bch2_btree_node_lock_for_insert(trans->c, - i->iter->l[0].b, i->iter); + iter_l(i->iter)->b, i->iter); ret = bch2_trans_commit_write_locked(trans, stopped_at); - trans_for_each_update(trans, i) + trans_for_each_update2(trans, i) if (!same_leaf_as_prev(trans, i)) - bch2_btree_node_unlock_write_inlined(i->iter->l[0].b, + bch2_btree_node_unlock_write_inlined(iter_l(i->iter)->b, i->iter); + if (!ret && trans->journal_pin) + bch2_journal_pin_add(&trans->c->journal, trans->journal_res.seq, + trans->journal_pin, NULL); + /* * Drop journal reservation after dropping write locks, since dropping * the journal reservation may kick off a journal write: @@ -540,15 +554,15 @@ static inline int do_bch2_trans_commit(struct btree_trans *trans, if (trans->flags & BTREE_INSERT_NOUNLOCK) trans->nounlock = true; - trans_for_each_update(trans, i) - if (!same_leaf_as_prev(trans, i)) + trans_for_each_update2(trans, i) + if (btree_iter_type(i->iter) != BTREE_ITER_CACHED && + !same_leaf_as_prev(trans, i)) bch2_foreground_maybe_merge(trans->c, i->iter, 0, trans->flags); trans->nounlock = false; - trans_for_each_update(trans, i) - bch2_btree_iter_downgrade(i->iter); + bch2_trans_downgrade(trans); return 0; } @@ -632,20 +646,24 @@ int bch2_trans_commit_error(struct btree_trans *trans, trace_trans_restart_journal_res_get(trans->ip); ret = -EINTR; break; - default: - BUG_ON(ret >= 0); - break; - } - - if (ret == -EINTR) { - int ret2 = bch2_btree_iter_traverse_all(trans); + case BTREE_INSERT_NEED_JOURNAL_RECLAIM: + bch2_trans_unlock(trans); - if (ret2) { - trace_trans_restart_traverse(trans->ip); - return ret2; + while (bch2_btree_key_cache_must_wait(c)) { + mutex_lock(&c->journal.reclaim_lock); + bch2_journal_reclaim(&c->journal); + mutex_unlock(&c->journal.reclaim_lock); } - trace_trans_restart_atomic(trans->ip); + if (bch2_trans_relock(trans)) + return 0; + + trace_trans_restart_journal_reclaim(trans->ip); + ret = -EINTR; + break; + default: + BUG_ON(ret >= 0); + break; } return ret; @@ -670,6 +688,135 @@ bch2_trans_commit_get_rw_cold(struct btree_trans *trans) return 0; } +static inline int btree_iter_pos_cmp(const struct btree_iter *l, + const struct btree_iter *r) +{ + return cmp_int(l->btree_id, r->btree_id) ?: + bkey_cmp(l->pos, r->pos); +} + +static void bch2_trans_update2(struct btree_trans *trans, + struct btree_iter *iter, + struct bkey_i *insert) +{ + struct btree_insert_entry *i, n = (struct btree_insert_entry) { + .iter = iter, .k = insert + }; + + btree_insert_entry_checks(trans, n.iter, n.k); + + BUG_ON(iter->uptodate > BTREE_ITER_NEED_PEEK); + + EBUG_ON(trans->nr_updates2 >= BTREE_ITER_MAX); + + iter->flags |= BTREE_ITER_KEEP_UNTIL_COMMIT; + + trans_for_each_update2(trans, i) { + if (btree_iter_pos_cmp(n.iter, i->iter) == 0) { + *i = n; + return; + } + + if (btree_iter_pos_cmp(n.iter, i->iter) <= 0) + break; + } + + array_insert_item(trans->updates2, trans->nr_updates2, + i - trans->updates2, n); +} + +static int extent_update_to_keys(struct btree_trans *trans, + struct btree_iter *orig_iter, + struct bkey_i *insert) +{ + struct btree_iter *iter; + int ret; + + ret = bch2_extent_can_insert(trans, orig_iter, insert); + if (ret) + return ret; + + if (bkey_deleted(&insert->k)) + return 0; + + iter = bch2_trans_copy_iter(trans, orig_iter); + + iter->flags |= BTREE_ITER_INTENT; + __bch2_btree_iter_set_pos(iter, insert->k.p, false); + bch2_trans_update2(trans, iter, insert); + bch2_trans_iter_put(trans, iter); + return 0; +} + +static int extent_handle_overwrites(struct btree_trans *trans, + enum btree_id btree_id, + struct bpos start, struct bpos end) +{ + struct btree_iter *iter = NULL, *update_iter; + struct bkey_i *update; + struct bkey_s_c k; + int ret = 0; + + iter = bch2_trans_get_iter(trans, btree_id, start, BTREE_ITER_INTENT); + k = bch2_btree_iter_peek_with_updates(iter); + + while (k.k && !(ret = bkey_err(k))) { + if (bkey_cmp(end, bkey_start_pos(k.k)) <= 0) + break; + + if (bkey_cmp(bkey_start_pos(k.k), start) < 0) { + update_iter = bch2_trans_copy_iter(trans, iter); + + update = bch2_trans_kmalloc(trans, bkey_bytes(k.k)); + if ((ret = PTR_ERR_OR_ZERO(update))) + goto err; + + bkey_reassemble(update, k); + bch2_cut_back(start, update); + + __bch2_btree_iter_set_pos(update_iter, update->k.p, false); + bch2_trans_update2(trans, update_iter, update); + bch2_trans_iter_put(trans, update_iter); + } + + if (bkey_cmp(k.k->p, end) > 0) { + update_iter = bch2_trans_copy_iter(trans, iter); + + update = bch2_trans_kmalloc(trans, bkey_bytes(k.k)); + if ((ret = PTR_ERR_OR_ZERO(update))) + goto err; + + bkey_reassemble(update, k); + bch2_cut_front(end, update); + + __bch2_btree_iter_set_pos(update_iter, update->k.p, false); + bch2_trans_update2(trans, update_iter, update); + bch2_trans_iter_put(trans, update_iter); + } else { + update_iter = bch2_trans_copy_iter(trans, iter); + + update = bch2_trans_kmalloc(trans, sizeof(struct bkey)); + if ((ret = PTR_ERR_OR_ZERO(update))) + goto err; + + update->k = *k.k; + set_bkey_val_u64s(&update->k, 0); + update->k.type = KEY_TYPE_deleted; + update->k.size = 0; + + __bch2_btree_iter_set_pos(update_iter, update->k.p, false); + bch2_trans_update2(trans, update_iter, update); + bch2_trans_iter_put(trans, update_iter); + } + + k = bch2_btree_iter_next_with_updates(iter); + } +err: + if (!IS_ERR_OR_NULL(iter)) + bch2_trans_iter_put(trans, iter); + return ret; +} + int __bch2_trans_commit(struct btree_trans *trans) { struct btree_insert_entry *i = NULL; @@ -678,8 +825,6 @@ int __bch2_trans_commit(struct btree_trans *trans) unsigned u64s; int ret = 0; - BUG_ON(trans->need_reset); - if (!trans->nr_updates) goto out_noupdates; @@ -688,7 +833,7 @@ int __bch2_trans_commit(struct btree_trans *trans) memset(&trans->journal_preres, 0, sizeof(trans->journal_preres)); - trans->journal_u64s = 0; + trans->journal_u64s = trans->extra_journal_entry_u64s; trans->journal_preres_u64s = 0; if (!(trans->flags & BTREE_INSERT_NOCHECK_RW) && @@ -698,6 +843,14 @@ int __bch2_trans_commit(struct btree_trans *trans) return ret; } +#ifdef CONFIG_BCACHEFS_DEBUG + trans_for_each_update(trans, i) + if (btree_iter_type(i->iter) != BTREE_ITER_CACHED && + !(i->trigger_flags & BTREE_TRIGGER_NORUN)) + bch2_btree_key_cache_verify_clean(trans, + i->iter->btree_id, i->iter->pos); +#endif + /* * Running triggers will append more updates to the list of updates as * we're walking it: @@ -706,10 +859,18 @@ int __bch2_trans_commit(struct btree_trans *trans) trans_trigger_run = false; trans_for_each_update(trans, i) { - /* we know trans->nounlock won't be set here: */ - if (unlikely(!(i->iter->locks_want < 1 - ? __bch2_btree_iter_upgrade(i->iter, 1) - : i->iter->uptodate <= BTREE_ITER_NEED_PEEK))) { + if (unlikely(i->iter->uptodate > BTREE_ITER_NEED_PEEK && + (ret = bch2_btree_iter_traverse(i->iter)))) { + trace_trans_restart_traverse(trans->ip); + goto out; + } + + /* + * We're not using bch2_btree_iter_upgrade here because + * we know trans->nounlock can't be set: + */ + if (unlikely(i->iter->locks_want < 1 && + !__bch2_btree_iter_upgrade(i->iter, 1))) { trace_trans_restart_upgrade(trans->ip); ret = -EINTR; goto out; @@ -731,9 +892,39 @@ int __bch2_trans_commit(struct btree_trans *trans) } } while (trans_trigger_run); + /* Turn extents updates into keys: */ + trans_for_each_update(trans, i) + if (i->iter->flags & BTREE_ITER_IS_EXTENTS) { + struct bpos start = bkey_start_pos(&i->k->k); + + while (i + 1 < trans->updates + trans->nr_updates && + i[0].iter->btree_id == i[1].iter->btree_id && + !bkey_cmp(i[0].k->k.p, bkey_start_pos(&i[1].k->k))) + i++; + + ret = extent_handle_overwrites(trans, i->iter->btree_id, + start, i->k->k.p); + if (ret) + goto out; + } + trans_for_each_update(trans, i) { + if (i->iter->flags & BTREE_ITER_IS_EXTENTS) { + ret = extent_update_to_keys(trans, i->iter, i->k); + if (ret) + goto out; + } else { + bch2_trans_update2(trans, i->iter, i->k); + } + } + + trans_for_each_update2(trans, i) { + BUG_ON(i->iter->uptodate > BTREE_ITER_NEED_PEEK); + BUG_ON(i->iter->locks_want < 1); + u64s = jset_u64s(i->k->k.u64s); - if (0) + if (btree_iter_type(i->iter) == BTREE_ITER_CACHED && + likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))) trans->journal_preres_u64s += u64s; trans->journal_u64s += u64s; } @@ -762,7 +953,7 @@ out: if (likely(!(trans->flags & BTREE_INSERT_NOCHECK_RW))) percpu_ref_put(&trans->c->writes); out_noupdates: - bch2_trans_reset(trans, TRANS_RESET_MEM|TRANS_RESET_NOTRAVERSE); + bch2_trans_reset(trans, !ret ? TRANS_RESET_NOTRAVERSE : 0); return ret; err: @@ -780,11 +971,14 @@ int bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter, .trigger_flags = flags, .iter = iter, .k = k }; - EBUG_ON(bkey_cmp(iter->pos, bkey_start_pos(&k->k))); + EBUG_ON(bkey_cmp(iter->pos, + (iter->flags & BTREE_ITER_IS_EXTENTS) + ? bkey_start_pos(&k->k) + : k->k.p)); iter->flags |= BTREE_ITER_KEEP_UNTIL_COMMIT; - if (iter->flags & BTREE_ITER_IS_EXTENTS) { + if (btree_node_type_is_extents(iter->btree_id)) { iter->pos_after_commit = k->k.p; iter->flags |= BTREE_ITER_SET_POS_AFTER_COMMIT; } @@ -793,7 +987,7 @@ int bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter, * Pending updates are kept sorted: first, find position of new update: */ trans_for_each_update(trans, i) - if (btree_iter_cmp(iter, i->iter) <= 0) + if (btree_iter_pos_cmp(iter, i->iter) <= 0) break; /* @@ -823,10 +1017,6 @@ int bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter, */ if (trans->iters_live & (1ULL << i->iter->idx)) { i->iter = bch2_trans_copy_iter(trans, i->iter); - if (IS_ERR(i->iter)) { - trans->need_reset = true; - return PTR_ERR(i->iter); - } i->iter->flags |= BTREE_ITER_KEEP_UNTIL_COMMIT; bch2_trans_iter_put(trans, i->iter); @@ -836,25 +1026,26 @@ int bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter, bch2_btree_iter_set_pos(i->iter, n.k->k.p); } - EBUG_ON(trans->nr_updates >= trans->nr_iters); + EBUG_ON(trans->nr_updates >= BTREE_ITER_MAX); array_insert_item(trans->updates, trans->nr_updates, i - trans->updates, n); return 0; } -static int __bch2_btree_insert(struct btree_trans *trans, - enum btree_id id, struct bkey_i *k) +int __bch2_btree_insert(struct btree_trans *trans, + enum btree_id id, struct bkey_i *k) { struct btree_iter *iter; + int ret; iter = bch2_trans_get_iter(trans, id, bkey_start_pos(&k->k), BTREE_ITER_INTENT); - if (IS_ERR(iter)) - return PTR_ERR(iter); - bch2_trans_update(trans, iter, k, 0); - return 0; + ret = bch2_btree_iter_traverse(iter) ?: + bch2_trans_update(trans, iter, k, 0); + bch2_trans_iter_put(trans, iter); + return ret; } /** @@ -873,23 +1064,43 @@ int bch2_btree_insert(struct bch_fs *c, enum btree_id id, __bch2_btree_insert(&trans, id, k)); } -int bch2_btree_delete_at_range(struct btree_trans *trans, - struct btree_iter *iter, - struct bpos end, - u64 *journal_seq) +int bch2_btree_delete_at(struct btree_trans *trans, + struct btree_iter *iter, unsigned flags) { + struct bkey_i k; + + bkey_init(&k.k); + k.k.p = iter->pos; + + bch2_trans_update(trans, iter, &k, 0); + return bch2_trans_commit(trans, NULL, NULL, + BTREE_INSERT_NOFAIL| + BTREE_INSERT_USE_RESERVE|flags); +} + +int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id, + struct bpos start, struct bpos end, + u64 *journal_seq) +{ + struct btree_iter *iter; struct bkey_s_c k; int ret = 0; + + iter = bch2_trans_get_iter(trans, id, start, BTREE_ITER_INTENT); retry: while ((k = bch2_btree_iter_peek(iter)).k && !(ret = bkey_err(k)) && bkey_cmp(iter->pos, end) < 0) { struct bkey_i delete; - bch2_trans_reset(trans, TRANS_RESET_MEM); + bch2_trans_begin(trans); bkey_init(&delete.k); + /* + * This could probably be more efficient for extents: + */ + /* * For extents, iter.pos won't necessarily be the same as * bkey_start_pos(k.k) (for non extents they always will be the @@ -902,7 +1113,7 @@ retry: */ delete.k.p = iter->pos; - if (iter->flags & BTREE_ITER_IS_EXTENTS) { + if (btree_node_type_is_extents(iter->btree_id)) { unsigned max_sectors = KEY_SIZE_MAX & (~0 << trans->c->block_bits); @@ -929,22 +1140,8 @@ retry: goto retry; } + bch2_trans_iter_put(trans, iter); return ret; - -} - -int bch2_btree_delete_at(struct btree_trans *trans, - struct btree_iter *iter, unsigned flags) -{ - struct bkey_i k; - - bkey_init(&k.k); - k.k.p = iter->pos; - - bch2_trans_update(trans, iter, &k, 0); - return bch2_trans_commit(trans, NULL, NULL, - BTREE_INSERT_NOFAIL| - BTREE_INSERT_USE_RESERVE|flags); } /* @@ -956,21 +1153,6 @@ int bch2_btree_delete_range(struct bch_fs *c, enum btree_id id, struct bpos start, struct bpos end, u64 *journal_seq) { - struct btree_trans trans; - struct btree_iter *iter; - int ret = 0; - - /* - * XXX: whether we need mem/more iters depends on whether this btree id - * has triggers - */ - bch2_trans_init(&trans, c, BTREE_ITER_MAX, 512); - - iter = bch2_trans_get_iter(&trans, id, start, BTREE_ITER_INTENT); - - ret = bch2_btree_delete_at_range(&trans, iter, end, journal_seq); - ret = bch2_trans_exit(&trans) ?: ret; - - BUG_ON(ret == -EINTR); - return ret; + return bch2_trans_do(c, NULL, journal_seq, 0, + bch2_btree_delete_range_trans(&trans, id, start, end, journal_seq)); }