- if (*prev &&
- bch2_extent_merge(c, b, (void *) *prev, &tmp.k))
- return;
-
- if (*prev) {
- bch2_bkey_pack(*prev, (void *) *prev, f);
-
- btree_keys_account_key_add(nr, 0, *prev);
- *prev = bkey_next(*prev);
- } else {
- *prev = start;
- }
-
- bkey_copy(*prev, &tmp.k);
-}
-
-struct btree_nr_keys bch2_extent_sort_fix_overlapping(struct bch_fs *c,
- struct bset *dst,
- struct btree *b,
- struct btree_node_iter_large *iter)
-{
- struct bkey_format *f = &b->format;
- struct btree_node_iter_set *_l = iter->data, *_r;
- struct bkey_packed *prev = NULL, *out, *lk, *rk;
- struct bkey l_unpacked, r_unpacked;
- struct bkey_s l, r;
- struct btree_nr_keys nr;
-
- memset(&nr, 0, sizeof(nr));
-
- heap_resort(iter, extent_sort_cmp, NULL);
-
- while (!bch2_btree_node_iter_large_end(iter)) {
- lk = __btree_node_offset_to_key(b, _l->k);
-
- if (iter->used == 1) {
- extent_sort_append(c, b, &nr, dst->start, &prev, lk);
- extent_sort_next(iter, b, _l);
- continue;
- }
-
- _r = iter->data + 1;
- if (iter->used > 2 &&
- extent_sort_cmp(iter, _r[0], _r[1]) >= 0)
- _r++;
-
- rk = __btree_node_offset_to_key(b, _r->k);
-
- l = __bkey_disassemble(b, lk, &l_unpacked);
- r = __bkey_disassemble(b, rk, &r_unpacked);
-
- /* If current key and next key don't overlap, just append */
- if (bkey_cmp(l.k->p, bkey_start_pos(r.k)) <= 0) {
- extent_sort_append(c, b, &nr, dst->start, &prev, lk);
- extent_sort_next(iter, b, _l);
- continue;
- }
-
- /* Skip 0 size keys */
- if (!r.k->size) {
- extent_sort_next(iter, b, _r);
- continue;
- }
-
- /*
- * overlap: keep the newer key and trim the older key so they
- * don't overlap. comparing pointers tells us which one is
- * newer, since the bsets are appended one after the other.
- */
-
- /* can't happen because of comparison func */
- BUG_ON(_l->k < _r->k &&
- !bkey_cmp(bkey_start_pos(l.k), bkey_start_pos(r.k)));
-
- if (_l->k > _r->k) {
- /* l wins, trim r */
- if (bkey_cmp(l.k->p, r.k->p) >= 0) {
- sort_key_next(iter, b, _r);
- } else {
- __bch2_cut_front(l.k->p, r);
- extent_save(b, rk, r.k);
- }
-
- extent_sort_sift(iter, b, _r - iter->data);
- } else if (bkey_cmp(l.k->p, r.k->p) > 0) {
- BKEY_PADDED(k) tmp;
-
- /*
- * r wins, but it overlaps in the middle of l - split l:
- */
- bkey_reassemble(&tmp.k, l.s_c);
- bch2_cut_back(bkey_start_pos(r.k), &tmp.k.k);
-
- __bch2_cut_front(r.k->p, l);
- extent_save(b, lk, l.k);
-
- extent_sort_sift(iter, b, 0);
-
- extent_sort_append(c, b, &nr, dst->start, &prev,
- bkey_to_packed(&tmp.k));
- } else {
- bch2_cut_back(bkey_start_pos(r.k), l.k);
- extent_save(b, lk, l.k);
- }
- }
-
- if (prev) {
- bch2_bkey_pack(prev, (void *) prev, f);
- btree_keys_account_key_add(&nr, 0, prev);
- out = bkey_next(prev);
- } else {
- out = dst->start;
- }
-
- dst->u64s = cpu_to_le16((u64 *) out - dst->_data);
- return nr;
-}
-
-struct extent_insert_state {
- struct btree_insert *trans;
- struct btree_insert_entry *insert;
- struct bpos committed;
- struct bch_fs_usage stats;
-
- /* for deleting: */
- struct bkey_i whiteout;
- bool update_journal;
- bool update_btree;
- bool deleting;
-};
-
-static void bch2_add_sectors(struct extent_insert_state *s,
- struct bkey_s_c k, u64 offset, s64 sectors)
-{
- struct bch_fs *c = s->trans->c;
- struct btree *b = s->insert->iter->l[0].b;
-
- EBUG_ON(bkey_cmp(bkey_start_pos(k.k), b->data->min_key) < 0);
-
- if (!sectors)
- return;
-
- bch2_mark_key(c, BKEY_TYPE_EXTENTS, k, sectors > 0, sectors,
- gc_pos_btree_node(b), &s->stats,
- s->trans->journal_res.seq, 0);
-}
-
-static void bch2_subtract_sectors(struct extent_insert_state *s,
- struct bkey_s_c k, u64 offset, s64 sectors)
-{
- bch2_add_sectors(s, k, offset, -sectors);
-}
-
-/* These wrappers subtract exactly the sectors that we're removing from @k */
-static void bch2_cut_subtract_back(struct extent_insert_state *s,
- struct bpos where, struct bkey_s k)
-{
- bch2_subtract_sectors(s, k.s_c, where.offset,
- k.k->p.offset - where.offset);
- bch2_cut_back(where, k.k);
-}
-
-static void bch2_cut_subtract_front(struct extent_insert_state *s,
- struct bpos where, struct bkey_s k)
-{
- bch2_subtract_sectors(s, k.s_c, bkey_start_offset(k.k),
- where.offset - bkey_start_offset(k.k));
- __bch2_cut_front(where, k);
-}
-
-static void bch2_drop_subtract(struct extent_insert_state *s, struct bkey_s k)
-{
- if (k.k->size)
- bch2_subtract_sectors(s, k.s_c,
- bkey_start_offset(k.k), k.k->size);
- k.k->size = 0;
- k.k->type = KEY_TYPE_DELETED;
-}
-
-static bool bch2_extent_merge_inline(struct bch_fs *,
- struct btree_iter *,
- struct bkey_packed *,
- struct bkey_packed *,
- bool);
-
-static void verify_extent_nonoverlapping(struct btree *b,
- struct btree_node_iter *_iter,
- struct bkey_i *insert)
-{
-#ifdef CONFIG_BCACHEFS_DEBUG
- struct btree_node_iter iter;
- struct bkey_packed *k;
- struct bkey uk;
-
- iter = *_iter;
- k = bch2_btree_node_iter_prev_filter(&iter, b, KEY_TYPE_DISCARD);
- BUG_ON(k &&
- (uk = bkey_unpack_key(b, k),
- bkey_cmp(uk.p, bkey_start_pos(&insert->k)) > 0));
-
- iter = *_iter;
- k = bch2_btree_node_iter_peek_filter(&iter, b, KEY_TYPE_DISCARD);
-#if 0
- BUG_ON(k &&
- (uk = bkey_unpack_key(b, k),
- bkey_cmp(insert->k.p, bkey_start_pos(&uk))) > 0);
-#else
- if (k &&
- (uk = bkey_unpack_key(b, k),
- bkey_cmp(insert->k.p, bkey_start_pos(&uk))) > 0) {
- char buf1[100];
- char buf2[100];
-
- bch2_bkey_to_text(&PBUF(buf1), &insert->k);
- bch2_bkey_to_text(&PBUF(buf2), &uk);
-
- bch2_dump_btree_node(b);
- panic("insert > next :\n"
- "insert %s\n"
- "next %s\n",
- buf1, buf2);
- }
-#endif
-
-#endif
-}
-
-static void verify_modified_extent(struct btree_iter *iter,
- struct bkey_packed *k)
-{
- bch2_btree_iter_verify(iter, iter->l[0].b);
- bch2_verify_insert_pos(iter->l[0].b, k, k, k->u64s);
-}
-
-static void extent_bset_insert(struct bch_fs *c, struct btree_iter *iter,
- struct bkey_i *insert)
-{
- struct btree_iter_level *l = &iter->l[0];
- struct btree_node_iter node_iter;
- struct bkey_packed *k;
-
- BUG_ON(insert->k.u64s > bch_btree_keys_u64s_remaining(c, l->b));
-
- EBUG_ON(bkey_deleted(&insert->k) || !insert->k.size);
- verify_extent_nonoverlapping(l->b, &l->iter, insert);
-
- node_iter = l->iter;
- k = bch2_btree_node_iter_prev_filter(&node_iter, l->b, KEY_TYPE_DISCARD);
- if (k && !bkey_written(l->b, k) &&
- bch2_extent_merge_inline(c, iter, k, bkey_to_packed(insert), true))
- return;
-
- node_iter = l->iter;
- k = bch2_btree_node_iter_peek_filter(&node_iter, l->b, KEY_TYPE_DISCARD);
- if (k && !bkey_written(l->b, k) &&
- bch2_extent_merge_inline(c, iter, bkey_to_packed(insert), k, false))
- return;
-
- k = bch2_btree_node_iter_bset_pos(&l->iter, l->b, bset_tree_last(l->b));
-
- bch2_bset_insert(l->b, &l->iter, k, insert, 0);
- bch2_btree_node_iter_fix(iter, l->b, &l->iter, k, 0, k->u64s);
- bch2_btree_iter_verify(iter, l->b);
-}
-
-static void extent_insert_committed(struct extent_insert_state *s)
-{
- struct bch_fs *c = s->trans->c;
- struct btree_iter *iter = s->insert->iter;
- struct bkey_i *insert = s->insert->k;
- BKEY_PADDED(k) split;
-
- EBUG_ON(bkey_cmp(insert->k.p, s->committed) < 0);
- EBUG_ON(bkey_cmp(s->committed, bkey_start_pos(&insert->k)) < 0);
-
- bkey_copy(&split.k, insert);
- if (s->deleting)
- split.k.k.type = KEY_TYPE_DISCARD;
-
- if (!(s->trans->flags & BTREE_INSERT_JOURNAL_REPLAY))
- bch2_cut_subtract_back(s, s->committed,
- bkey_i_to_s(&split.k));
- else
- bch2_cut_back(s->committed, &split.k.k);
-
- if (!bkey_cmp(s->committed, iter->pos))
- return;
-
- bch2_btree_iter_set_pos_same_leaf(iter, s->committed);
-
- if (s->update_btree) {
- if (debug_check_bkeys(c))
- bch2_bkey_debugcheck(c, iter->l[0].b,
- bkey_i_to_s_c(&split.k));
-
- EBUG_ON(bkey_deleted(&split.k.k) || !split.k.k.size);
-
- extent_bset_insert(c, iter, &split.k);
- }
-
- if (s->update_journal) {
- bkey_copy(&split.k, !s->deleting ? insert : &s->whiteout);
- if (s->deleting)
- split.k.k.type = KEY_TYPE_DISCARD;
-
- bch2_cut_back(s->committed, &split.k.k);
-
- EBUG_ON(bkey_deleted(&split.k.k) || !split.k.k.size);
-
- bch2_btree_journal_key(s->trans, iter, &split.k);
- }
-
- bch2_cut_front(s->committed, insert);
-
- insert->k.needs_whiteout = false;
- s->trans->did_work = true;
-}
-
-void bch2_extent_trim_atomic(struct bkey_i *k, struct btree_iter *iter)
-{
- struct btree *b = iter->l[0].b;
-
- BUG_ON(iter->uptodate > BTREE_ITER_NEED_PEEK);
-
- bch2_cut_back(b->key.k.p, &k->k);
-
- BUG_ON(bkey_cmp(bkey_start_pos(&k->k), b->data->min_key) < 0);
-}
-
-enum btree_insert_ret
-bch2_extent_can_insert(struct btree_insert *trans,
- struct btree_insert_entry *insert,
- unsigned *u64s)
-{
- struct btree_iter_level *l = &insert->iter->l[0];
- struct btree_node_iter node_iter = l->iter;
- enum bch_extent_overlap overlap;
- struct bkey_packed *_k;
- struct bkey unpacked;
- struct bkey_s_c k;
- int sectors;
-
- BUG_ON(trans->flags & BTREE_INSERT_ATOMIC &&
- !bch2_extent_is_atomic(&insert->k->k, insert->iter));
-
- /*
- * We avoid creating whiteouts whenever possible when deleting, but
- * those optimizations mean we may potentially insert two whiteouts
- * instead of one (when we overlap with the front of one extent and the
- * back of another):
- */
- if (bkey_whiteout(&insert->k->k))
- *u64s += BKEY_U64s;
-
- _k = bch2_btree_node_iter_peek_filter(&node_iter, l->b,
- KEY_TYPE_DISCARD);
- if (!_k)
- return BTREE_INSERT_OK;
-
- k = bkey_disassemble(l->b, _k, &unpacked);
-
- overlap = bch2_extent_overlap(&insert->k->k, k.k);
-
- /* account for having to split existing extent: */
- if (overlap == BCH_EXTENT_OVERLAP_MIDDLE)
- *u64s += _k->u64s;
-
- if (overlap == BCH_EXTENT_OVERLAP_MIDDLE &&
- (sectors = bch2_extent_is_compressed(k))) {
- int flags = BCH_DISK_RESERVATION_BTREE_LOCKS_HELD;
-
- if (trans->flags & BTREE_INSERT_NOFAIL)
- flags |= BCH_DISK_RESERVATION_NOFAIL;
-
- switch (bch2_disk_reservation_add(trans->c,
- trans->disk_res,
- sectors, flags)) {
- case 0:
- break;
- case -ENOSPC:
- return BTREE_INSERT_ENOSPC;
- case -EINTR:
- return BTREE_INSERT_NEED_GC_LOCK;
- default:
- BUG();
- }
- }
-
- return BTREE_INSERT_OK;
-}
-
-static void
-extent_squash(struct extent_insert_state *s, struct bkey_i *insert,
- struct bkey_packed *_k, struct bkey_s k,
- enum bch_extent_overlap overlap)
-{
- struct bch_fs *c = s->trans->c;
- struct btree_iter *iter = s->insert->iter;
- struct btree_iter_level *l = &iter->l[0];
-
- switch (overlap) {
- case BCH_EXTENT_OVERLAP_FRONT:
- /* insert overlaps with start of k: */
- bch2_cut_subtract_front(s, insert->k.p, k);
- BUG_ON(bkey_deleted(k.k));
- extent_save(l->b, _k, k.k);
- verify_modified_extent(iter, _k);
- break;
-
- case BCH_EXTENT_OVERLAP_BACK:
- /* insert overlaps with end of k: */
- bch2_cut_subtract_back(s, bkey_start_pos(&insert->k), k);
- BUG_ON(bkey_deleted(k.k));
- extent_save(l->b, _k, k.k);
-
- /*
- * As the auxiliary tree is indexed by the end of the
- * key and we've just changed the end, update the
- * auxiliary tree.
- */
- bch2_bset_fix_invalidated_key(l->b, _k);
- bch2_btree_node_iter_fix(iter, l->b, &l->iter,
- _k, _k->u64s, _k->u64s);
- verify_modified_extent(iter, _k);
- break;
-
- case BCH_EXTENT_OVERLAP_ALL: {
- /* The insert key completely covers k, invalidate k */
- if (!bkey_whiteout(k.k))
- btree_account_key_drop(l->b, _k);
-
- bch2_drop_subtract(s, k);
-
- if (_k >= btree_bset_last(l->b)->start) {
- unsigned u64s = _k->u64s;
-
- bch2_bset_delete(l->b, _k, _k->u64s);
- bch2_btree_node_iter_fix(iter, l->b, &l->iter,
- _k, u64s, 0);
- bch2_btree_iter_verify(iter, l->b);
- } else {
- extent_save(l->b, _k, k.k);
- bch2_btree_node_iter_fix(iter, l->b, &l->iter,
- _k, _k->u64s, _k->u64s);
- verify_modified_extent(iter, _k);
- }
-
- break;
- }
- case BCH_EXTENT_OVERLAP_MIDDLE: {
- BKEY_PADDED(k) split;
- /*
- * The insert key falls 'in the middle' of k
- * The insert key splits k in 3:
- * - start only in k, preserve
- * - middle common section, invalidate in k
- * - end only in k, preserve
- *
- * We update the old key to preserve the start,
- * insert will be the new common section,
- * we manually insert the end that we are preserving.
- *
- * modify k _before_ doing the insert (which will move
- * what k points to)
- */
- bkey_reassemble(&split.k, k.s_c);
- split.k.k.needs_whiteout |= bkey_written(l->b, _k);
-
- bch2_cut_back(bkey_start_pos(&insert->k), &split.k.k);
- BUG_ON(bkey_deleted(&split.k.k));
-
- bch2_cut_subtract_front(s, insert->k.p, k);
- BUG_ON(bkey_deleted(k.k));
- extent_save(l->b, _k, k.k);
- verify_modified_extent(iter, _k);
-
- bch2_add_sectors(s, bkey_i_to_s_c(&split.k),
- bkey_start_offset(&split.k.k),
- split.k.k.size);
- extent_bset_insert(c, iter, &split.k);
- break;
- }
- }
-}
-
-static void __bch2_insert_fixup_extent(struct extent_insert_state *s)
-{
- struct btree_iter *iter = s->insert->iter;
- struct btree_iter_level *l = &iter->l[0];
- struct bkey_packed *_k;
- struct bkey unpacked;
- struct bkey_i *insert = s->insert->k;
-
- while (bkey_cmp(s->committed, insert->k.p) < 0 &&
- (_k = bch2_btree_node_iter_peek_filter(&l->iter, l->b,
- KEY_TYPE_DISCARD))) {
- struct bkey_s k = __bkey_disassemble(l->b, _k, &unpacked);
- enum bch_extent_overlap overlap = bch2_extent_overlap(&insert->k, k.k);
-
- EBUG_ON(bkey_cmp(iter->pos, k.k->p) >= 0);
-
- if (bkey_cmp(bkey_start_pos(k.k), insert->k.p) >= 0)
- break;
-
- s->committed = bpos_min(s->insert->k->k.p, k.k->p);
-
- if (!bkey_whiteout(k.k))
- s->update_journal = true;
-
- if (!s->update_journal) {
- bch2_cut_front(s->committed, insert);
- bch2_cut_front(s->committed, &s->whiteout);
- bch2_btree_iter_set_pos_same_leaf(iter, s->committed);
- goto next;
- }
-
- /*
- * When deleting, if possible just do it by switching the type
- * of the key we're deleting, instead of creating and inserting
- * a new whiteout:
- */
- if (s->deleting &&
- !s->update_btree &&
- !bkey_cmp(insert->k.p, k.k->p) &&
- !bkey_cmp(bkey_start_pos(&insert->k), bkey_start_pos(k.k))) {
- if (!bkey_whiteout(k.k)) {
- btree_account_key_drop(l->b, _k);
- bch2_subtract_sectors(s, k.s_c,
- bkey_start_offset(k.k), k.k->size);
- _k->type = KEY_TYPE_DISCARD;
- reserve_whiteout(l->b, _k);
- }
- break;
- }
-
- if (k.k->needs_whiteout || bkey_written(l->b, _k)) {
- insert->k.needs_whiteout = true;
- s->update_btree = true;
- }
-
- if (s->update_btree &&
- overlap == BCH_EXTENT_OVERLAP_ALL &&
- bkey_whiteout(k.k) &&
- k.k->needs_whiteout) {
- unreserve_whiteout(l->b, _k);
- _k->needs_whiteout = false;
- }
-
- extent_squash(s, insert, _k, k, overlap);