git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_update_leaf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "btree_update.h"
   5 #include "btree_update_interior.h"
   6 #include "btree_gc.h"
   7 #include "btree_io.h"
   8 #include "btree_iter.h"
   9 #include "btree_key_cache.h"
  10 #include "btree_locking.h"
  11 #include "btree_write_buffer.h"
  12 #include "buckets.h"
  13 #include "debug.h"
  14 #include "errcode.h"
  15 #include "error.h"
  16 #include "extent_update.h"
  17 #include "journal.h"
  18 #include "journal_reclaim.h"
  19 #include "keylist.h"
  20 #include "recovery.h"
  21 #include "subvolume.h"
  22 #include "replicas.h"
  23
  24 #include <linux/prefetch.h>
  25 #include <linux/sort.h>
  26 #include <trace/events/bcachefs.h>
  27
  28 /*
  29  * bch2_btree_path_peek_slot() for a cached iterator might return a key in a
  30  * different snapshot:
  31  */
  32 struct bkey_s_c bch2_btree_path_peek_slot_exact(struct btree_path *path, struct bkey *u)
  33 {
  34         struct bkey_s_c k = bch2_btree_path_peek_slot(path, u);
  35
  36         if (k.k && bpos_eq(path->pos, k.k->p))
  37                 return k;
  38
  39         bkey_init(u);
  40         u->p = path->pos;
  41         return (struct bkey_s_c) { u, NULL };
  42 }
  43
  44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i)
  45 {
  46 #ifdef CONFIG_BCACHEFS_DEBUG
  47         struct bch_fs *c = trans->c;
  48         struct bkey u;
  49         struct bkey_s_c k = bch2_btree_path_peek_slot_exact(i->path, &u);
  50
  51         if (unlikely(trans->journal_replay_not_finished)) {
  52                 struct bkey_i *j_k =
  53                         bch2_journal_keys_peek_slot(c, i->btree_id, i->level, i->k->k.p);
  54
  55                 if (j_k)
  56                         k = bkey_i_to_s_c(j_k);
  57         }
  58
  59         i->old_k.needs_whiteout = k.k->needs_whiteout;
  60
  61         BUG_ON(memcmp(&i->old_k, k.k, sizeof(struct bkey)));
  62         BUG_ON(i->old_v != k.v);
  63 #endif
  64 }
  65
  66 static int __must_check
  67 bch2_trans_update_by_path(struct btree_trans *, struct btree_path *,
  68                           struct bkey_i *, enum btree_update_flags);
  69
  70 static inline int btree_insert_entry_cmp(const struct btree_insert_entry *l,
  71                                          const struct btree_insert_entry *r)
  72 {
  73         return   cmp_int(l->btree_id,   r->btree_id) ?:
  74                  cmp_int(l->cached,     r->cached) ?:
  75                  -cmp_int(l->level,     r->level) ?:
  76                  bpos_cmp(l->k->k.p,    r->k->k.p);
  77 }
  78
  79 static inline struct btree_path_level *insert_l(struct btree_insert_entry *i)
  80 {
  81         return i->path->l + i->level;
  82 }
  83
  84 static inline bool same_leaf_as_prev(struct btree_trans *trans,
  85                                      struct btree_insert_entry *i)
  86 {
  87         return i != trans->updates &&
  88                 insert_l(&i[0])->b == insert_l(&i[-1])->b;
  89 }
  90
  91 static inline bool same_leaf_as_next(struct btree_trans *trans,
  92                                      struct btree_insert_entry *i)
  93 {
  94         return i + 1 < trans->updates + trans->nr_updates &&
  95                 insert_l(&i[0])->b == insert_l(&i[1])->b;
  96 }
  97
  98 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans,
  99                                            struct btree_path *path,
 100                                            struct btree *b)
 101 {
 102         struct bch_fs *c = trans->c;
 103
 104         if (unlikely(btree_node_just_written(b)) &&
 105             bch2_btree_post_write_cleanup(c, b))
 106                 bch2_trans_node_reinit_iter(trans, b);
 107
 108         /*
 109          * If the last bset has been written, or if it's gotten too big - start
 110          * a new bset to insert into:
 111          */
 112         if (want_new_bset(c, b))
 113                 bch2_btree_init_next(trans, b);
 114 }
 115
 116 /* Inserting into a given leaf node (last stage of insert): */
 117
 118 /* Handle overwrites and do insert, for non extents: */
 119 bool bch2_btree_bset_insert_key(struct btree_trans *trans,
 120                                 struct btree_path *path,
 121                                 struct btree *b,
 122                                 struct btree_node_iter *node_iter,
 123                                 struct bkey_i *insert)
 124 {
 125         struct bkey_packed *k;
 126         unsigned clobber_u64s = 0, new_u64s = 0;
 127
 128         EBUG_ON(btree_node_just_written(b));
 129         EBUG_ON(bset_written(b, btree_bset_last(b)));
 130         EBUG_ON(bkey_deleted(&insert->k) && bkey_val_u64s(&insert->k));
 131         EBUG_ON(bpos_lt(insert->k.p, b->data->min_key));
 132         EBUG_ON(bpos_gt(insert->k.p, b->data->max_key));
 133         EBUG_ON(insert->k.u64s >
 134                 bch_btree_keys_u64s_remaining(trans->c, b));
 135
 136         k = bch2_btree_node_iter_peek_all(node_iter, b);
 137         if (k && bkey_cmp_left_packed(b, k, &insert->k.p))
 138                 k = NULL;
 139
 140         /* @k is the key being overwritten/deleted, if any: */
 141         EBUG_ON(k && bkey_deleted(k));
 142
 143         /* Deleting, but not found? nothing to do: */
 144         if (bkey_deleted(&insert->k) && !k)
 145                 return false;
 146
 147         if (bkey_deleted(&insert->k)) {
 148                 /* Deleting: */
 149                 btree_account_key_drop(b, k);
 150                 k->type = KEY_TYPE_deleted;
 151
 152                 if (k->needs_whiteout)
 153                         push_whiteout(trans->c, b, insert->k.p);
 154                 k->needs_whiteout = false;
 155
 156                 if (k >= btree_bset_last(b)->start) {
 157                         clobber_u64s = k->u64s;
 158                         bch2_bset_delete(b, k, clobber_u64s);
 159                         goto fix_iter;
 160                 } else {
 161                         bch2_btree_path_fix_key_modified(trans, b, k);
 162                 }
 163
 164                 return true;
 165         }
 166
 167         if (k) {
 168                 /* Overwriting: */
 169                 btree_account_key_drop(b, k);
 170                 k->type = KEY_TYPE_deleted;
 171
 172                 insert->k.needs_whiteout = k->needs_whiteout;
 173                 k->needs_whiteout = false;
 174
 175                 if (k >= btree_bset_last(b)->start) {
 176                         clobber_u64s = k->u64s;
 177                         goto overwrite;
 178                 } else {
 179                         bch2_btree_path_fix_key_modified(trans, b, k);
 180                 }
 181         }
 182
 183         k = bch2_btree_node_iter_bset_pos(node_iter, b, bset_tree_last(b));
 184 overwrite:
 185         bch2_bset_insert(b, node_iter, k, insert, clobber_u64s);
 186         new_u64s = k->u64s;
 187 fix_iter:
 188         if (clobber_u64s != new_u64s)
 189                 bch2_btree_node_iter_fix(trans, path, b, node_iter, k,
 190                                          clobber_u64s, new_u64s);
 191         return true;
 192 }
 193
 194 static int __btree_node_flush(struct journal *j, struct journal_entry_pin *pin,
 195                                unsigned i, u64 seq)
 196 {
 197         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 198         struct btree_write *w = container_of(pin, struct btree_write, journal);
 199         struct btree *b = container_of(w, struct btree, writes[i]);
 200         struct btree_trans trans;
 201         unsigned long old, new, v;
 202         unsigned idx = w - b->writes;
 203
 204         bch2_trans_init(&trans, c, 0, 0);
 205
 206         btree_node_lock_nopath_nofail(&trans, &b->c, SIX_LOCK_read);
 207         v = READ_ONCE(b->flags);
 208
 209         do {
 210                 old = new = v;
 211
 212                 if (!(old & (1 << BTREE_NODE_dirty)) ||
 213                     !!(old & (1 << BTREE_NODE_write_idx)) != idx ||
 214                     w->journal.seq != seq)
 215                         break;
 216
 217                 new &= ~BTREE_WRITE_TYPE_MASK;
 218                 new |= BTREE_WRITE_journal_reclaim;
 219                 new |= 1 << BTREE_NODE_need_write;
 220         } while ((v = cmpxchg(&b->flags, old, new)) != old);
 221
 222         btree_node_write_if_need(c, b, SIX_LOCK_read);
 223         six_unlock_read(&b->c.lock);
 224
 225         bch2_trans_exit(&trans);
 226         return 0;
 227 }
 228
 229 static int btree_node_flush0(struct journal *j, struct journal_entry_pin *pin, u64 seq)
 230 {
 231         return __btree_node_flush(j, pin, 0, seq);
 232 }
 233
 234 static int btree_node_flush1(struct journal *j, struct journal_entry_pin *pin, u64 seq)
 235 {
 236         return __btree_node_flush(j, pin, 1, seq);
 237 }
 238
 239 inline void bch2_btree_add_journal_pin(struct bch_fs *c,
 240                                        struct btree *b, u64 seq)
 241 {
 242         struct btree_write *w = btree_current_write(b);
 243
 244         bch2_journal_pin_add(&c->journal, seq, &w->journal,
 245                              btree_node_write_idx(b) == 0
 246                              ? btree_node_flush0
 247                              : btree_node_flush1);
 248 }
 249
 250 /**
 251  * btree_insert_key - insert a key one key into a leaf node
 252  */
 253 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans,
 254                                        struct btree_path *path,
 255                                        struct bkey_i *insert,
 256                                        u64 journal_seq)
 257 {
 258         struct bch_fs *c = trans->c;
 259         struct btree *b = path_l(path)->b;
 260         struct bset_tree *t = bset_tree_last(b);
 261         struct bset *i = bset(b, t);
 262         int old_u64s = bset_u64s(t);
 263         int old_live_u64s = b->nr.live_u64s;
 264         int live_u64s_added, u64s_added;
 265
 266         if (unlikely(!bch2_btree_bset_insert_key(trans, path, b,
 267                                         &path_l(path)->iter, insert)))
 268                 return;
 269
 270         i->journal_seq = cpu_to_le64(max(journal_seq, le64_to_cpu(i->journal_seq)));
 271
 272         bch2_btree_add_journal_pin(c, b, journal_seq);
 273
 274         if (unlikely(!btree_node_dirty(b)))
 275                 set_btree_node_dirty_acct(c, b);
 276
 277         live_u64s_added = (int) b->nr.live_u64s - old_live_u64s;
 278         u64s_added = (int) bset_u64s(t) - old_u64s;
 279
 280         if (b->sib_u64s[0] != U16_MAX && live_u64s_added < 0)
 281                 b->sib_u64s[0] = max(0, (int) b->sib_u64s[0] + live_u64s_added);
 282         if (b->sib_u64s[1] != U16_MAX && live_u64s_added < 0)
 283                 b->sib_u64s[1] = max(0, (int) b->sib_u64s[1] + live_u64s_added);
 284
 285         if (u64s_added > live_u64s_added &&
 286             bch2_maybe_compact_whiteouts(c, b))
 287                 bch2_trans_node_reinit_iter(trans, b);
 288 }
 289
 290 static void btree_insert_key_leaf(struct btree_trans *trans,
 291                                   struct btree_insert_entry *insert)
 292 {
 293         bch2_btree_insert_key_leaf(trans, insert->path, insert->k, trans->journal_res.seq);
 294 }
 295
 296 /* Cached btree updates: */
 297
 298 /* Normal update interface: */
 299
 300 static inline void btree_insert_entry_checks(struct btree_trans *trans,
 301                                              struct btree_insert_entry *i)
 302 {
 303         BUG_ON(!bpos_eq(i->k->k.p, i->path->pos));
 304         BUG_ON(i->cached        != i->path->cached);
 305         BUG_ON(i->level         != i->path->level);
 306         BUG_ON(i->btree_id      != i->path->btree_id);
 307         EBUG_ON(!i->level &&
 308                 !(i->flags & BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) &&
 309                 test_bit(JOURNAL_REPLAY_DONE, &trans->c->journal.flags) &&
 310                 i->k->k.p.snapshot &&
 311                 bch2_snapshot_internal_node(trans->c, i->k->k.p.snapshot));
 312 }
 313
 314 static noinline int
 315 bch2_trans_journal_preres_get_cold(struct btree_trans *trans, unsigned flags,
 316                                    unsigned long trace_ip)
 317 {
 318         struct bch_fs *c = trans->c;
 319         int ret;
 320
 321         bch2_trans_unlock(trans);
 322
 323         ret = bch2_journal_preres_get(&c->journal,
 324                         &trans->journal_preres,
 325                         trans->journal_preres_u64s,
 326                         (flags & JOURNAL_WATERMARK_MASK));
 327         if (ret)
 328                 return ret;
 329
 330         ret = bch2_trans_relock(trans);
 331         if (ret) {
 332                 trace_and_count(c, trans_restart_journal_preres_get, trans, trace_ip, 0);
 333                 return ret;
 334         }
 335
 336         return 0;
 337 }
 338
 339 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans,
 340                                                       unsigned flags)
 341 {
 342         return bch2_journal_res_get(&trans->c->journal, &trans->journal_res,
 343                                     trans->journal_u64s, flags);
 344 }
 345
 346 #define JSET_ENTRY_LOG_U64s             4
 347
 348 static noinline void journal_transaction_name(struct btree_trans *trans)
 349 {
 350         struct bch_fs *c = trans->c;
 351         struct journal *j = &c->journal;
 352         struct jset_entry *entry =
 353                 bch2_journal_add_entry(j, &trans->journal_res,
 354                                        BCH_JSET_ENTRY_log, 0, 0,
 355                                        JSET_ENTRY_LOG_U64s);
 356         struct jset_entry_log *l =
 357                 container_of(entry, struct jset_entry_log, entry);
 358
 359         strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64));
 360 }
 361
 362 static inline int btree_key_can_insert(struct btree_trans *trans,
 363                                        struct btree *b, unsigned u64s)
 364 {
 365         struct bch_fs *c = trans->c;
 366
 367         if (!bch2_btree_node_insert_fits(c, b, u64s))
 368                 return -BCH_ERR_btree_insert_btree_node_full;
 369
 370         return 0;
 371 }
 372
 373 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags,
 374                                        struct btree_path *path, unsigned u64s)
 375 {
 376         struct bch_fs *c = trans->c;
 377         struct bkey_cached *ck = (void *) path->l[0].b;
 378         struct btree_insert_entry *i;
 379         unsigned new_u64s;
 380         struct bkey_i *new_k;
 381
 382         EBUG_ON(path->level);
 383
 384         if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
 385             bch2_btree_key_cache_must_wait(c) &&
 386             !(flags & BTREE_INSERT_JOURNAL_RECLAIM))
 387                 return -BCH_ERR_btree_insert_need_journal_reclaim;
 388
 389         /*
 390          * bch2_varint_decode can read past the end of the buffer by at most 7
 391          * bytes (it won't be used):
 392          */
 393         u64s += 1;
 394
 395         if (u64s <= ck->u64s)
 396                 return 0;
 397
 398         new_u64s        = roundup_pow_of_two(u64s);
 399         new_k           = krealloc(ck->k, new_u64s * sizeof(u64), GFP_NOFS);
 400         if (!new_k) {
 401                 bch_err(c, "error allocating memory for key cache key, btree %s u64s %u",
 402                         bch2_btree_ids[path->btree_id], new_u64s);
 403                 return -ENOMEM;
 404         }
 405
 406         trans_for_each_update(trans, i)
 407                 if (i->old_v == &ck->k->v)
 408                         i->old_v = &new_k->v;
 409
 410         ck->u64s        = new_u64s;
 411         ck->k           = new_k;
 412         return 0;
 413 }
 414
 415 /* Triggers: */
 416
 417 static int run_one_mem_trigger(struct btree_trans *trans,
 418                                struct btree_insert_entry *i,
 419                                unsigned flags)
 420 {
 421         struct bkey_s_c old = { &i->old_k, i->old_v };
 422         struct bkey_i *new = i->k;
 423         int ret;
 424
 425         verify_update_old_key(trans, i);
 426
 427         if (unlikely(flags & BTREE_TRIGGER_NORUN))
 428                 return 0;
 429
 430         if (!btree_node_type_needs_gc(i->btree_id))
 431                 return 0;
 432
 433         if (bch2_bkey_ops[old.k->type].atomic_trigger ==
 434             bch2_bkey_ops[i->k->k.type].atomic_trigger &&
 435             ((1U << old.k->type) & BTREE_TRIGGER_WANTS_OLD_AND_NEW)) {
 436                 ret   = bch2_mark_key(trans, old, bkey_i_to_s_c(new),
 437                                 BTREE_TRIGGER_INSERT|BTREE_TRIGGER_OVERWRITE|flags);
 438         } else {
 439                 struct bkey             _deleted = KEY(0, 0, 0);
 440                 struct bkey_s_c         deleted = (struct bkey_s_c) { &_deleted, NULL };
 441
 442                 _deleted.p = i->path->pos;
 443
 444                 ret   = bch2_mark_key(trans, deleted, bkey_i_to_s_c(new),
 445                                 BTREE_TRIGGER_INSERT|flags) ?:
 446                         bch2_mark_key(trans, old, deleted,
 447                                 BTREE_TRIGGER_OVERWRITE|flags);
 448         }
 449
 450         return ret;
 451 }
 452
 453 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i,
 454                                  bool overwrite)
 455 {
 456         /*
 457          * Transactional triggers create new btree_insert_entries, so we can't
 458          * pass them a pointer to a btree_insert_entry, that memory is going to
 459          * move:
 460          */
 461         struct bkey old_k = i->old_k;
 462         struct bkey_s_c old = { &old_k, i->old_v };
 463
 464         verify_update_old_key(trans, i);
 465
 466         if ((i->flags & BTREE_TRIGGER_NORUN) ||
 467             !(BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type)))
 468                 return 0;
 469
 470         if (!i->insert_trigger_run &&
 471             !i->overwrite_trigger_run &&
 472             bch2_bkey_ops[old.k->type].trans_trigger ==
 473             bch2_bkey_ops[i->k->k.type].trans_trigger &&
 474             ((1U << old.k->type) & BTREE_TRIGGER_WANTS_OLD_AND_NEW)) {
 475                 i->overwrite_trigger_run = true;
 476                 i->insert_trigger_run = true;
 477                 return bch2_trans_mark_key(trans, i->btree_id, i->level, old, i->k,
 478                                            BTREE_TRIGGER_INSERT|
 479                                            BTREE_TRIGGER_OVERWRITE|
 480                                            i->flags) ?: 1;
 481         } else if (overwrite && !i->overwrite_trigger_run) {
 482                 i->overwrite_trigger_run = true;
 483                 return bch2_trans_mark_old(trans, i->btree_id, i->level, old, i->flags) ?: 1;
 484         } else if (!overwrite && !i->insert_trigger_run) {
 485                 i->insert_trigger_run = true;
 486                 return bch2_trans_mark_new(trans, i->btree_id, i->level, i->k, i->flags) ?: 1;
 487         } else {
 488                 return 0;
 489         }
 490 }
 491
 492 static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id,
 493                               struct btree_insert_entry *btree_id_start)
 494 {
 495         struct btree_insert_entry *i;
 496         bool trans_trigger_run;
 497         int ret, overwrite;
 498
 499         for (overwrite = 1; overwrite >= 0; --overwrite) {
 500
 501                 /*
 502                  * Running triggers will append more updates to the list of updates as
 503                  * we're walking it:
 504                  */
 505                 do {
 506                         trans_trigger_run = false;
 507
 508                         for (i = btree_id_start;
 509                              i < trans->updates + trans->nr_updates && i->btree_id <= btree_id;
 510                              i++) {
 511                                 if (i->btree_id != btree_id)
 512                                         continue;
 513
 514                                 ret = run_one_trans_trigger(trans, i, overwrite);
 515                                 if (ret < 0)
 516                                         return ret;
 517                                 if (ret)
 518                                         trans_trigger_run = true;
 519                         }
 520                 } while (trans_trigger_run);
 521         }
 522
 523         return 0;
 524 }
 525
 526 static int bch2_trans_commit_run_triggers(struct btree_trans *trans)
 527 {
 528         struct btree_insert_entry *i = NULL, *btree_id_start = trans->updates;
 529         unsigned btree_id = 0;
 530         int ret = 0;
 531
 532         /*
 533          *
 534          * For a given btree, this algorithm runs insert triggers before
 535          * overwrite triggers: this is so that when extents are being moved
 536          * (e.g. by FALLOCATE_FL_INSERT_RANGE), we don't drop references before
 537          * they are re-added.
 538          */
 539         for (btree_id = 0; btree_id < BTREE_ID_NR; btree_id++) {
 540                 if (btree_id == BTREE_ID_alloc)
 541                         continue;
 542
 543                 while (btree_id_start < trans->updates + trans->nr_updates &&
 544                        btree_id_start->btree_id < btree_id)
 545                         btree_id_start++;
 546
 547                 ret = run_btree_triggers(trans, btree_id, btree_id_start);
 548                 if (ret)
 549                         return ret;
 550         }
 551
 552         trans_for_each_update(trans, i) {
 553                 if (i->btree_id > BTREE_ID_alloc)
 554                         break;
 555                 if (i->btree_id == BTREE_ID_alloc) {
 556                         ret = run_btree_triggers(trans, BTREE_ID_alloc, i);
 557                         if (ret)
 558                                 return ret;
 559                         break;
 560                 }
 561         }
 562
 563 #ifdef CONFIG_BCACHEFS_DEBUG
 564         trans_for_each_update(trans, i)
 565                 BUG_ON(!(i->flags & BTREE_TRIGGER_NORUN) &&
 566                        (BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS & (1U << i->bkey_type)) &&
 567                        (!i->insert_trigger_run || !i->overwrite_trigger_run));
 568 #endif
 569         return 0;
 570 }
 571
 572 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans)
 573 {
 574         struct bch_fs *c = trans->c;
 575         struct btree_insert_entry *i;
 576         int ret = 0;
 577
 578         trans_for_each_update(trans, i) {
 579                 /*
 580                  * XXX: synchronization of cached update triggers with gc
 581                  * XXX: synchronization of interior node updates with gc
 582                  */
 583                 BUG_ON(i->cached || i->level);
 584
 585                 if (gc_visited(c, gc_pos_btree_node(insert_l(i)->b))) {
 586                         ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_GC);
 587                         if (ret)
 588                                 break;
 589                 }
 590         }
 591
 592         return ret;
 593 }
 594
 595 static inline int
 596 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags,
 597                                struct btree_insert_entry **stopped_at,
 598                                unsigned long trace_ip)
 599 {
 600         struct bch_fs *c = trans->c;
 601         struct btree_insert_entry *i;
 602         struct btree_write_buffered_key *wb;
 603         struct btree_trans_commit_hook *h;
 604         unsigned u64s = 0;
 605         bool marking = false;
 606         int ret;
 607
 608         if (race_fault()) {
 609                 trace_and_count(c, trans_restart_fault_inject, trans, trace_ip);
 610                 return btree_trans_restart_nounlock(trans, BCH_ERR_transaction_restart_fault_inject);
 611         }
 612
 613         /*
 614          * Check if the insert will fit in the leaf node with the write lock
 615          * held, otherwise another thread could write the node changing the
 616          * amount of space available:
 617          */
 618
 619         prefetch(&trans->c->journal.flags);
 620
 621         h = trans->hooks;
 622         while (h) {
 623                 ret = h->fn(trans, h);
 624                 if (ret)
 625                         return ret;
 626                 h = h->next;
 627         }
 628
 629         trans_for_each_update(trans, i) {
 630                 /* Multiple inserts might go to same leaf: */
 631                 if (!same_leaf_as_prev(trans, i))
 632                         u64s = 0;
 633
 634                 u64s += i->k->k.u64s;
 635                 ret = !i->cached
 636                         ? btree_key_can_insert(trans, insert_l(i)->b, u64s)
 637                         : btree_key_can_insert_cached(trans, flags, i->path, u64s);
 638                 if (ret) {
 639                         *stopped_at = i;
 640                         return ret;
 641                 }
 642
 643                 if (btree_node_type_needs_gc(i->bkey_type))
 644                         marking = true;
 645         }
 646
 647         if (trans->nr_wb_updates &&
 648             trans->nr_wb_updates + c->btree_write_buffer.state.nr > c->btree_write_buffer.size)
 649                 return -BCH_ERR_btree_insert_need_flush_buffer;
 650
 651         /*
 652          * Don't get journal reservation until after we know insert will
 653          * succeed:
 654          */
 655         if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) {
 656                 ret = bch2_trans_journal_res_get(trans,
 657                                 (flags & JOURNAL_WATERMARK_MASK)|
 658                                 JOURNAL_RES_GET_NONBLOCK);
 659                 if (ret)
 660                         return ret;
 661
 662                 if (unlikely(trans->journal_transaction_names))
 663                         journal_transaction_name(trans);
 664         } else {
 665                 trans->journal_res.seq = c->journal.replay_journal_seq;
 666         }
 667
 668         /*
 669          * Not allowed to fail after we've gotten our journal reservation - we
 670          * have to use it:
 671          */
 672
 673         if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) &&
 674             !(flags & BTREE_INSERT_JOURNAL_REPLAY)) {
 675                 if (bch2_journal_seq_verify)
 676                         trans_for_each_update(trans, i)
 677                                 i->k->k.version.lo = trans->journal_res.seq;
 678                 else if (bch2_inject_invalid_keys)
 679                         trans_for_each_update(trans, i)
 680                                 i->k->k.version = MAX_VERSION;
 681         }
 682
 683         if (trans->fs_usage_deltas &&
 684             bch2_trans_fs_usage_apply(trans, trans->fs_usage_deltas))
 685                 return -BCH_ERR_btree_insert_need_mark_replicas;
 686
 687         if (trans->nr_wb_updates) {
 688                 EBUG_ON(flags & BTREE_INSERT_JOURNAL_REPLAY);
 689
 690                 ret = bch2_btree_insert_keys_write_buffer(trans);
 691                 if (ret)
 692                         goto revert_fs_usage;
 693         }
 694
 695         trans_for_each_update(trans, i)
 696                 if (BTREE_NODE_TYPE_HAS_MEM_TRIGGERS & (1U << i->bkey_type)) {
 697                         ret = run_one_mem_trigger(trans, i, i->flags);
 698                         if (ret)
 699                                 goto fatal_err;
 700                 }
 701
 702         if (unlikely(c->gc_pos.phase)) {
 703                 ret = bch2_trans_commit_run_gc_triggers(trans);
 704                 if  (ret)
 705                         goto fatal_err;
 706         }
 707
 708         if (unlikely(trans->extra_journal_entries.nr)) {
 709                 memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res),
 710                                   trans->extra_journal_entries.data,
 711                                   trans->extra_journal_entries.nr);
 712
 713                 trans->journal_res.offset       += trans->extra_journal_entries.nr;
 714                 trans->journal_res.u64s         -= trans->extra_journal_entries.nr;
 715         }
 716
 717         if (likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY))) {
 718                 struct journal *j = &c->journal;
 719                 struct jset_entry *entry;
 720
 721                 trans_for_each_update(trans, i) {
 722                         if (i->key_cache_already_flushed)
 723                                 continue;
 724
 725                         if (i->flags & BTREE_UPDATE_NOJOURNAL)
 726                                 continue;
 727
 728                         verify_update_old_key(trans, i);
 729
 730                         if (trans->journal_transaction_names) {
 731                                 entry = bch2_journal_add_entry(j, &trans->journal_res,
 732                                                        BCH_JSET_ENTRY_overwrite,
 733                                                        i->btree_id, i->level,
 734                                                        i->old_k.u64s);
 735                                 bkey_reassemble(&entry->start[0],
 736                                                 (struct bkey_s_c) { &i->old_k, i->old_v });
 737                         }
 738
 739                         entry = bch2_journal_add_entry(j, &trans->journal_res,
 740                                                BCH_JSET_ENTRY_btree_keys,
 741                                                i->btree_id, i->level,
 742                                                i->k->k.u64s);
 743                         bkey_copy(&entry->start[0], i->k);
 744                 }
 745
 746                 trans_for_each_wb_update(trans, wb) {
 747                         entry = bch2_journal_add_entry(j, &trans->journal_res,
 748                                                BCH_JSET_ENTRY_btree_keys,
 749                                                wb->btree, 0,
 750                                                wb->k.k.u64s);
 751                         bkey_copy(&entry->start[0], &wb->k);
 752                 }
 753
 754                 if (trans->journal_seq)
 755                         *trans->journal_seq = trans->journal_res.seq;
 756         }
 757
 758         trans_for_each_update(trans, i) {
 759                 i->k->k.needs_whiteout = false;
 760
 761                 if (!i->cached)
 762                         btree_insert_key_leaf(trans, i);
 763                 else if (!i->key_cache_already_flushed)
 764                         bch2_btree_insert_key_cached(trans, flags, i->path, i->k);
 765                 else {
 766                         bch2_btree_key_cache_drop(trans, i->path);
 767                         btree_path_set_dirty(i->path, BTREE_ITER_NEED_TRAVERSE);
 768                 }
 769         }
 770
 771         return 0;
 772 fatal_err:
 773         bch2_fatal_error(c);
 774 revert_fs_usage:
 775         if (trans->fs_usage_deltas)
 776                 bch2_trans_fs_usage_revert(trans, trans->fs_usage_deltas);
 777         return ret;
 778 }
 779
 780 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i)
 781 {
 782         while (--i >= trans->updates) {
 783                 if (same_leaf_as_prev(trans, i))
 784                         continue;
 785
 786                 bch2_btree_node_unlock_write(trans, i->path, insert_l(i)->b);
 787         }
 788
 789         trace_and_count(trans->c, trans_restart_would_deadlock_write, trans);
 790         return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write);
 791 }
 792
 793 static inline int trans_lock_write(struct btree_trans *trans)
 794 {
 795         struct btree_insert_entry *i;
 796
 797         trans_for_each_update(trans, i) {
 798                 if (same_leaf_as_prev(trans, i))
 799                         continue;
 800
 801                 if (bch2_btree_node_lock_write(trans, i->path, &insert_l(i)->b->c))
 802                         return trans_lock_write_fail(trans, i);
 803
 804                 if (!i->cached)
 805                         bch2_btree_node_prep_for_write(trans, i->path, insert_l(i)->b);
 806         }
 807
 808         return 0;
 809 }
 810
 811 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans)
 812 {
 813         struct btree_insert_entry *i;
 814         struct btree_write_buffered_key *wb;
 815
 816         trans_for_each_update(trans, i)
 817                 bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p);
 818
 819         trans_for_each_wb_update(trans, wb)
 820                 bch2_journal_key_overwritten(trans->c, wb->btree, 0, wb->k.k.p);
 821 }
 822
 823 #ifdef CONFIG_BCACHEFS_DEBUG
 824 static noinline int bch2_trans_commit_bkey_invalid(struct btree_trans *trans, unsigned flags,
 825                                                    struct btree_insert_entry *i,
 826                                                    struct printbuf *err)
 827 {
 828         struct bch_fs *c = trans->c;
 829         int rw = (flags & BTREE_INSERT_JOURNAL_REPLAY) ? READ : WRITE;
 830
 831         printbuf_reset(err);
 832         prt_printf(err, "invalid bkey on insert from %s -> %ps",
 833                    trans->fn, (void *) i->ip_allocated);
 834         prt_newline(err);
 835         printbuf_indent_add(err, 2);
 836
 837         bch2_bkey_val_to_text(err, c, bkey_i_to_s_c(i->k));
 838         prt_newline(err);
 839
 840         bch2_bkey_invalid(c, bkey_i_to_s_c(i->k),
 841                           i->bkey_type, rw, err);
 842         bch2_print_string_as_lines(KERN_ERR, err->buf);
 843
 844         bch2_inconsistent_error(c);
 845         bch2_dump_trans_updates(trans);
 846         printbuf_exit(err);
 847
 848         return -EINVAL;
 849 }
 850 #endif
 851
 852 /*
 853  * Get journal reservation, take write locks, and attempt to do btree update(s):
 854  */
 855 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags,
 856                                        struct btree_insert_entry **stopped_at,
 857                                        unsigned long trace_ip)
 858 {
 859         struct bch_fs *c = trans->c;
 860         struct btree_insert_entry *i;
 861         int ret, u64s_delta = 0;
 862
 863 #ifdef CONFIG_BCACHEFS_DEBUG
 864         struct printbuf buf = PRINTBUF;
 865
 866         trans_for_each_update(trans, i) {
 867                 int rw = (flags & BTREE_INSERT_JOURNAL_REPLAY) ? READ : WRITE;
 868
 869                 if (unlikely(bch2_bkey_invalid(c, bkey_i_to_s_c(i->k),
 870                                                i->bkey_type, rw, &buf)))
 871                         return bch2_trans_commit_bkey_invalid(trans, flags, i, &buf);
 872                 btree_insert_entry_checks(trans, i);
 873         }
 874         printbuf_exit(&buf);
 875 #endif
 876
 877         trans_for_each_update(trans, i) {
 878                 if (i->cached)
 879                         continue;
 880
 881                 u64s_delta += !bkey_deleted(&i->k->k) ? i->k->k.u64s : 0;
 882                 u64s_delta -= i->old_btree_u64s;
 883
 884                 if (!same_leaf_as_next(trans, i)) {
 885                         if (u64s_delta <= 0) {
 886                                 ret = bch2_foreground_maybe_merge(trans, i->path,
 887                                                         i->level, flags);
 888                                 if (unlikely(ret))
 889                                         return ret;
 890                         }
 891
 892                         u64s_delta = 0;
 893                 }
 894         }
 895
 896         ret = bch2_journal_preres_get(&c->journal,
 897                         &trans->journal_preres, trans->journal_preres_u64s,
 898                         (flags & JOURNAL_WATERMARK_MASK)|JOURNAL_RES_GET_NONBLOCK);
 899         if (unlikely(ret == -BCH_ERR_journal_preres_get_blocked))
 900                 ret = bch2_trans_journal_preres_get_cold(trans, flags, trace_ip);
 901         if (unlikely(ret))
 902                 return ret;
 903
 904         ret = trans_lock_write(trans);
 905         if (unlikely(ret))
 906                 return ret;
 907
 908         ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip);
 909
 910         if (!ret && unlikely(trans->journal_replay_not_finished))
 911                 bch2_drop_overwrites_from_journal(trans);
 912
 913         trans_for_each_update(trans, i)
 914                 if (!same_leaf_as_prev(trans, i))
 915                         bch2_btree_node_unlock_write_inlined(trans, i->path,
 916                                                         insert_l(i)->b);
 917
 918         if (!ret && trans->journal_pin)
 919                 bch2_journal_pin_add(&c->journal, trans->journal_res.seq,
 920                                      trans->journal_pin, NULL);
 921
 922         /*
 923          * Drop journal reservation after dropping write locks, since dropping
 924          * the journal reservation may kick off a journal write:
 925          */
 926         bch2_journal_res_put(&c->journal, &trans->journal_res);
 927
 928         if (unlikely(ret))
 929                 return ret;
 930
 931         bch2_trans_downgrade(trans);
 932
 933         return 0;
 934 }
 935
 936 static int journal_reclaim_wait_done(struct bch_fs *c)
 937 {
 938         int ret = bch2_journal_error(&c->journal) ?:
 939                 !bch2_btree_key_cache_must_wait(c);
 940
 941         if (!ret)
 942                 journal_reclaim_kick(&c->journal);
 943         return ret;
 944 }
 945
 946 static noinline
 947 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags,
 948                             struct btree_insert_entry *i,
 949                             int ret, unsigned long trace_ip)
 950 {
 951         struct bch_fs *c = trans->c;
 952
 953         switch (ret) {
 954         case -BCH_ERR_btree_insert_btree_node_full:
 955                 ret = bch2_btree_split_leaf(trans, i->path, flags);
 956                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 957                         trace_and_count(c, trans_restart_btree_node_split, trans, trace_ip, i->path);
 958                 break;
 959         case -BCH_ERR_btree_insert_need_mark_replicas:
 960                 bch2_trans_unlock(trans);
 961
 962                 ret = bch2_replicas_delta_list_mark(c, trans->fs_usage_deltas);
 963                 if (ret)
 964                         break;
 965
 966                 ret = bch2_trans_relock(trans);
 967                 if (ret)
 968                         trace_and_count(c, trans_restart_mark_replicas, trans, trace_ip);
 969                 break;
 970         case -BCH_ERR_journal_res_get_blocked:
 971                 bch2_trans_unlock(trans);
 972
 973                 if ((flags & BTREE_INSERT_JOURNAL_RECLAIM) &&
 974                     !(flags & JOURNAL_WATERMARK_reserved)) {
 975                         ret = -BCH_ERR_journal_reclaim_would_deadlock;
 976                         break;
 977                 }
 978
 979                 ret = bch2_trans_journal_res_get(trans,
 980                                         (flags & JOURNAL_WATERMARK_MASK)|
 981                                         JOURNAL_RES_GET_CHECK);
 982                 if (ret)
 983                         break;
 984
 985                 ret = bch2_trans_relock(trans);
 986                 if (ret)
 987                         trace_and_count(c, trans_restart_journal_res_get, trans, trace_ip);
 988                 break;
 989         case -BCH_ERR_btree_insert_need_journal_reclaim:
 990                 bch2_trans_unlock(trans);
 991
 992                 trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip);
 993
 994                 wait_event_freezable(c->journal.reclaim_wait,
 995                                      (ret = journal_reclaim_wait_done(c)));
 996                 if (ret < 0)
 997                         break;
 998
 999                 ret = bch2_trans_relock(trans);
1000                 if (ret)
1001                         trace_and_count(c, trans_restart_journal_reclaim, trans, trace_ip);
1002                 break;
1003         case -BCH_ERR_btree_insert_need_flush_buffer: {
1004                 struct btree_write_buffer *wb = &c->btree_write_buffer;
1005
1006                 ret = 0;
1007
1008                 if (wb->state.nr > wb->size * 3 / 4) {
1009                         bch2_trans_reset_updates(trans);
1010                         bch2_trans_unlock(trans);
1011
1012                         mutex_lock(&wb->flush_lock);
1013
1014                         if (wb->state.nr > wb->size * 3 / 4)
1015                                 ret = __bch2_btree_write_buffer_flush(trans,
1016                                                 flags|BTREE_INSERT_NOCHECK_RW, true);
1017                         else
1018                                 mutex_unlock(&wb->flush_lock);
1019
1020                         if (!ret) {
1021                                 trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_);
1022                                 ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush);
1023                         }
1024                 }
1025                 break;
1026         }
1027         default:
1028                 BUG_ON(ret >= 0);
1029                 break;
1030         }
1031
1032         BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted);
1033
1034         bch2_fs_inconsistent_on(bch2_err_matches(ret, ENOSPC) &&
1035                                 !(flags & BTREE_INSERT_NOWAIT) &&
1036                                 (flags & BTREE_INSERT_NOFAIL), c,
1037                 "%s: incorrectly got %s\n", __func__, bch2_err_str(ret));
1038
1039         return ret;
1040 }
1041
1042 static noinline int
1043 bch2_trans_commit_get_rw_cold(struct btree_trans *trans, unsigned flags)
1044 {
1045         struct bch_fs *c = trans->c;
1046         int ret;
1047
1048         if (likely(!(flags & BTREE_INSERT_LAZY_RW)) ||
1049             test_bit(BCH_FS_STARTED, &c->flags))
1050                 return -BCH_ERR_erofs_trans_commit;
1051
1052         bch2_trans_unlock(trans);
1053
1054         ret =   bch2_fs_read_write_early(c) ?:
1055                 bch2_trans_relock(trans);
1056         if (ret)
1057                 return ret;
1058
1059         bch2_write_ref_get(c, BCH_WRITE_REF_trans);
1060         return 0;
1061 }
1062
1063 /*
1064  * This is for updates done in the early part of fsck - btree_gc - before we've
1065  * gone RW. we only add the new key to the list of keys for journal replay to
1066  * do.
1067  */
1068 static noinline int
1069 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans)
1070 {
1071         struct bch_fs *c = trans->c;
1072         struct btree_insert_entry *i;
1073         int ret = 0;
1074
1075         trans_for_each_update(trans, i) {
1076                 ret = bch2_journal_key_insert(c, i->btree_id, i->level, i->k);
1077                 if (ret)
1078                         break;
1079         }
1080
1081         return ret;
1082 }
1083
1084 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags)
1085 {
1086         struct bch_fs *c = trans->c;
1087         struct btree_insert_entry *i = NULL;
1088         struct btree_write_buffered_key *wb;
1089         unsigned u64s;
1090         int ret = 0;
1091
1092         if (!trans->nr_updates &&
1093             !trans->nr_wb_updates &&
1094             !trans->extra_journal_entries.nr)
1095                 goto out_reset;
1096
1097         if (flags & BTREE_INSERT_GC_LOCK_HELD)
1098                 lockdep_assert_held(&c->gc_lock);
1099
1100         ret = bch2_trans_commit_run_triggers(trans);
1101         if (ret)
1102                 goto out_reset;
1103
1104         if (unlikely(!test_bit(BCH_FS_MAY_GO_RW, &c->flags))) {
1105                 ret = do_bch2_trans_commit_to_journal_replay(trans);
1106                 goto out_reset;
1107         }
1108
1109         if (!(flags & BTREE_INSERT_NOCHECK_RW) &&
1110             unlikely(!bch2_write_ref_tryget(c, BCH_WRITE_REF_trans))) {
1111                 ret = bch2_trans_commit_get_rw_cold(trans, flags);
1112                 if (ret)
1113                         goto out_reset;
1114         }
1115
1116         if (c->btree_write_buffer.state.nr > c->btree_write_buffer.size / 2 &&
1117             mutex_trylock(&c->btree_write_buffer.flush_lock)) {
1118                 bch2_trans_begin(trans);
1119                 bch2_trans_unlock(trans);
1120
1121                 ret = __bch2_btree_write_buffer_flush(trans,
1122                                         flags|BTREE_INSERT_NOCHECK_RW, true);
1123                 if (!ret) {
1124                         trace_and_count(c, trans_restart_write_buffer_flush, trans, _THIS_IP_);
1125                         ret = btree_trans_restart(trans, BCH_ERR_transaction_restart_write_buffer_flush);
1126                 }
1127                 goto out;
1128         }
1129
1130         EBUG_ON(test_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags));
1131
1132         memset(&trans->journal_preres, 0, sizeof(trans->journal_preres));
1133
1134         trans->journal_u64s             = trans->extra_journal_entries.nr;
1135         trans->journal_preres_u64s      = 0;
1136
1137         trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names);
1138
1139         if (trans->journal_transaction_names)
1140                 trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s);
1141
1142         trans_for_each_update(trans, i) {
1143                 EBUG_ON(!i->path->should_be_locked);
1144
1145                 ret = bch2_btree_path_upgrade(trans, i->path, i->level + 1);
1146                 if (unlikely(ret))
1147                         goto out;
1148
1149                 EBUG_ON(!btree_node_intent_locked(i->path, i->level));
1150
1151                 if (i->key_cache_already_flushed)
1152                         continue;
1153
1154                 /* we're going to journal the key being updated: */
1155                 u64s = jset_u64s(i->k->k.u64s);
1156                 if (i->cached &&
1157                     likely(!(flags & BTREE_INSERT_JOURNAL_REPLAY)))
1158                         trans->journal_preres_u64s += u64s;
1159
1160                 if (i->flags & BTREE_UPDATE_NOJOURNAL)
1161                         continue;
1162
1163                 trans->journal_u64s += u64s;
1164
1165                 /* and we're also going to log the overwrite: */
1166                 if (trans->journal_transaction_names)
1167                         trans->journal_u64s += jset_u64s(i->old_k.u64s);
1168         }
1169
1170         trans_for_each_wb_update(trans, wb)
1171                 trans->journal_u64s += jset_u64s(wb->k.k.u64s);
1172
1173         if (trans->extra_journal_res) {
1174                 ret = bch2_disk_reservation_add(c, trans->disk_res,
1175                                 trans->extra_journal_res,
1176                                 (flags & BTREE_INSERT_NOFAIL)
1177                                 ? BCH_DISK_RESERVATION_NOFAIL : 0);
1178                 if (ret)
1179                         goto err;
1180         }
1181 retry:
1182         bch2_trans_verify_not_in_restart(trans);
1183         memset(&trans->journal_res, 0, sizeof(trans->journal_res));
1184
1185         ret = do_bch2_trans_commit(trans, flags, &i, _RET_IP_);
1186
1187         /* make sure we didn't drop or screw up locks: */
1188         bch2_trans_verify_locks(trans);
1189
1190         if (ret)
1191                 goto err;
1192
1193         trace_and_count(c, transaction_commit, trans, _RET_IP_);
1194 out:
1195         bch2_journal_preres_put(&c->journal, &trans->journal_preres);
1196
1197         if (likely(!(flags & BTREE_INSERT_NOCHECK_RW)))
1198                 bch2_write_ref_put(c, BCH_WRITE_REF_trans);
1199 out_reset:
1200         bch2_trans_reset_updates(trans);
1201
1202         return ret;
1203 err:
1204         ret = bch2_trans_commit_error(trans, flags, i, ret, _RET_IP_);
1205         if (ret)
1206                 goto out;
1207
1208         goto retry;
1209 }
1210
1211 static noinline int __check_pos_snapshot_overwritten(struct btree_trans *trans,
1212                                           enum btree_id id,
1213                                           struct bpos pos)
1214 {
1215         struct bch_fs *c = trans->c;
1216         struct btree_iter iter;
1217         struct bkey_s_c k;
1218         int ret;
1219
1220         bch2_trans_iter_init(trans, &iter, id, pos,
1221                              BTREE_ITER_NOT_EXTENTS|
1222                              BTREE_ITER_ALL_SNAPSHOTS);
1223         while (1) {
1224                 k = bch2_btree_iter_prev(&iter);
1225                 ret = bkey_err(k);
1226                 if (ret)
1227                         break;
1228
1229                 if (!k.k)
1230                         break;
1231
1232                 if (!bkey_eq(pos, k.k->p))
1233                         break;
1234
1235                 if (bch2_snapshot_is_ancestor(c, k.k->p.snapshot, pos.snapshot)) {
1236                         ret = 1;
1237                         break;
1238                 }
1239         }
1240         bch2_trans_iter_exit(trans, &iter);
1241
1242         return ret;
1243 }
1244
1245 static inline int check_pos_snapshot_overwritten(struct btree_trans *trans,
1246                                           enum btree_id id,
1247                                           struct bpos pos)
1248 {
1249         if (!btree_type_has_snapshots(id) ||
1250             pos.snapshot == U32_MAX ||
1251             !snapshot_t(trans->c, pos.snapshot)->children[0])
1252                 return 0;
1253
1254         return __check_pos_snapshot_overwritten(trans, id, pos);
1255 }
1256
1257 static noinline int extent_front_merge(struct btree_trans *trans,
1258                                        struct btree_iter *iter,
1259                                        struct bkey_s_c k,
1260                                        struct bkey_i **insert,
1261                                        enum btree_update_flags flags)
1262 {
1263         struct bch_fs *c = trans->c;
1264         struct bkey_i *update;
1265         int ret;
1266
1267         update = bch2_bkey_make_mut(trans, k);
1268         ret = PTR_ERR_OR_ZERO(update);
1269         if (ret)
1270                 return ret;
1271
1272         if (!bch2_bkey_merge(c, bkey_i_to_s(update), bkey_i_to_s_c(*insert)))
1273                 return 0;
1274
1275         ret =   check_pos_snapshot_overwritten(trans, iter->btree_id, k.k->p) ?:
1276                 check_pos_snapshot_overwritten(trans, iter->btree_id, (*insert)->k.p);
1277         if (ret < 0)
1278                 return ret;
1279         if (ret)
1280                 return 0;
1281
1282         ret = bch2_btree_delete_at(trans, iter, flags);
1283         if (ret)
1284                 return ret;
1285
1286         *insert = update;
1287         return 0;
1288 }
1289
1290 static noinline int extent_back_merge(struct btree_trans *trans,
1291                                       struct btree_iter *iter,
1292                                       struct bkey_i *insert,
1293                                       struct bkey_s_c k)
1294 {
1295         struct bch_fs *c = trans->c;
1296         int ret;
1297
1298         ret =   check_pos_snapshot_overwritten(trans, iter->btree_id, insert->k.p) ?:
1299                 check_pos_snapshot_overwritten(trans, iter->btree_id, k.k->p);
1300         if (ret < 0)
1301                 return ret;
1302         if (ret)
1303                 return 0;
1304
1305         bch2_bkey_merge(c, bkey_i_to_s(insert), k);
1306         return 0;
1307 }
1308
1309 int bch2_trans_update_extent(struct btree_trans *trans,
1310                              struct btree_iter *orig_iter,
1311                              struct bkey_i *insert,
1312                              enum btree_update_flags flags)
1313 {
1314         struct btree_iter iter, update_iter;
1315         struct bpos start = bkey_start_pos(&insert->k);
1316         struct bkey_i *update;
1317         struct bkey_s_c k;
1318         enum btree_id btree_id = orig_iter->btree_id;
1319         int ret = 0, compressed_sectors;
1320
1321         bch2_trans_iter_init(trans, &iter, btree_id, start,
1322                              BTREE_ITER_INTENT|
1323                              BTREE_ITER_WITH_UPDATES|
1324                              BTREE_ITER_NOT_EXTENTS);
1325         k = bch2_btree_iter_peek_upto(&iter, POS(insert->k.p.inode, U64_MAX));
1326         if ((ret = bkey_err(k)))
1327                 goto err;
1328         if (!k.k)
1329                 goto out;
1330
1331         if (bkey_eq(k.k->p, bkey_start_pos(&insert->k))) {
1332                 if (bch2_bkey_maybe_mergable(k.k, &insert->k)) {
1333                         ret = extent_front_merge(trans, &iter, k, &insert, flags);
1334                         if (ret)
1335                                 goto err;
1336                 }
1337
1338                 goto next;
1339         }
1340
1341         while (bkey_gt(insert->k.p, bkey_start_pos(k.k))) {
1342                 bool front_split = bkey_lt(bkey_start_pos(k.k), start);
1343                 bool back_split  = bkey_gt(k.k->p, insert->k.p);
1344
1345                 /*
1346                  * If we're going to be splitting a compressed extent, note it
1347                  * so that __bch2_trans_commit() can increase our disk
1348                  * reservation:
1349                  */
1350                 if (((front_split && back_split) ||
1351                      ((front_split || back_split) && k.k->p.snapshot != insert->k.p.snapshot)) &&
1352                     (compressed_sectors = bch2_bkey_sectors_compressed(k)))
1353                         trans->extra_journal_res += compressed_sectors;
1354
1355                 if (front_split) {
1356                         update = bch2_bkey_make_mut(trans, k);
1357                         if ((ret = PTR_ERR_OR_ZERO(update)))
1358                                 goto err;
1359
1360                         bch2_cut_back(start, update);
1361
1362                         bch2_trans_iter_init(trans, &update_iter, btree_id, update->k.p,
1363                                              BTREE_ITER_NOT_EXTENTS|
1364                                              BTREE_ITER_ALL_SNAPSHOTS|
1365                                              BTREE_ITER_INTENT);
1366                         ret   = bch2_btree_iter_traverse(&update_iter) ?:
1367                                 bch2_trans_update(trans, &update_iter, update,
1368                                                   BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|
1369                                                   flags);
1370                         bch2_trans_iter_exit(trans, &update_iter);
1371
1372                         if (ret)
1373                                 goto err;
1374                 }
1375
1376                 if (k.k->p.snapshot != insert->k.p.snapshot &&
1377                     (front_split || back_split)) {
1378                         update = bch2_bkey_make_mut(trans, k);
1379                         if ((ret = PTR_ERR_OR_ZERO(update)))
1380                                 goto err;
1381
1382                         bch2_cut_front(start, update);
1383                         bch2_cut_back(insert->k.p, update);
1384
1385                         bch2_trans_iter_init(trans, &update_iter, btree_id, update->k.p,
1386                                              BTREE_ITER_NOT_EXTENTS|
1387                                              BTREE_ITER_ALL_SNAPSHOTS|
1388                                              BTREE_ITER_INTENT);
1389                         ret   = bch2_btree_iter_traverse(&update_iter) ?:
1390                                 bch2_trans_update(trans, &update_iter, update,
1391                                                   BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|
1392                                                   flags);
1393                         bch2_trans_iter_exit(trans, &update_iter);
1394                         if (ret)
1395                                 goto err;
1396                 }
1397
1398                 if (bkey_le(k.k->p, insert->k.p)) {
1399                         update = bch2_trans_kmalloc(trans, sizeof(*update));
1400                         if ((ret = PTR_ERR_OR_ZERO(update)))
1401                                 goto err;
1402
1403                         bkey_init(&update->k);
1404                         update->k.p = k.k->p;
1405
1406                         if (insert->k.p.snapshot != k.k->p.snapshot) {
1407                                 update->k.p.snapshot = insert->k.p.snapshot;
1408                                 update->k.type = KEY_TYPE_whiteout;
1409                         }
1410
1411                         bch2_trans_iter_init(trans, &update_iter, btree_id, update->k.p,
1412                                              BTREE_ITER_NOT_EXTENTS|
1413                                              BTREE_ITER_INTENT);
1414                         ret   = bch2_btree_iter_traverse(&update_iter) ?:
1415                                 bch2_trans_update(trans, &update_iter, update,
1416                                                   BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|
1417                                                   flags);
1418                         bch2_trans_iter_exit(trans, &update_iter);
1419
1420                         if (ret)
1421                                 goto err;
1422                 }
1423
1424                 if (back_split) {
1425                         update = bch2_bkey_make_mut(trans, k);
1426                         if ((ret = PTR_ERR_OR_ZERO(update)))
1427                                 goto err;
1428
1429                         bch2_cut_front(insert->k.p, update);
1430
1431                         ret = bch2_trans_update_by_path(trans, iter.path, update,
1432                                                   BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE|
1433                                                   flags);
1434                         if (ret)
1435                                 goto err;
1436                         goto out;
1437                 }
1438 next:
1439                 bch2_btree_iter_advance(&iter);
1440                 k = bch2_btree_iter_peek_upto(&iter, POS(insert->k.p.inode, U64_MAX));
1441                 if ((ret = bkey_err(k)))
1442                         goto err;
1443                 if (!k.k)
1444                         goto out;
1445         }
1446
1447         if (bch2_bkey_maybe_mergable(&insert->k, k.k)) {
1448                 ret = extent_back_merge(trans, &iter, insert, k);
1449                 if (ret)
1450                         goto err;
1451         }
1452 out:
1453         if (!bkey_deleted(&insert->k)) {
1454                 /*
1455                  * Rewinding iterators is expensive: get a new one and the one
1456                  * that points to the start of insert will be cloned from:
1457                  */
1458                 bch2_trans_iter_exit(trans, &iter);
1459                 bch2_trans_iter_init(trans, &iter, btree_id, insert->k.p,
1460                                      BTREE_ITER_NOT_EXTENTS|
1461                                      BTREE_ITER_INTENT);
1462                 ret   = bch2_btree_iter_traverse(&iter) ?:
1463                         bch2_trans_update(trans, &iter, insert, flags);
1464         }
1465 err:
1466         bch2_trans_iter_exit(trans, &iter);
1467
1468         return ret;
1469 }
1470
1471 /*
1472  * When deleting, check if we need to emit a whiteout (because we're overwriting
1473  * something in an ancestor snapshot)
1474  */
1475 static int need_whiteout_for_snapshot(struct btree_trans *trans,
1476                                       enum btree_id btree_id, struct bpos pos)
1477 {
1478         struct btree_iter iter;
1479         struct bkey_s_c k;
1480         u32 snapshot = pos.snapshot;
1481         int ret;
1482
1483         if (!bch2_snapshot_parent(trans->c, pos.snapshot))
1484                 return 0;
1485
1486         pos.snapshot++;
1487
1488         for_each_btree_key_norestart(trans, iter, btree_id, pos,
1489                            BTREE_ITER_ALL_SNAPSHOTS|
1490                            BTREE_ITER_NOPRESERVE, k, ret) {
1491                 if (!bkey_eq(k.k->p, pos))
1492                         break;
1493
1494                 if (bch2_snapshot_is_ancestor(trans->c, snapshot,
1495                                               k.k->p.snapshot)) {
1496                         ret = !bkey_whiteout(k.k);
1497                         break;
1498                 }
1499         }
1500         bch2_trans_iter_exit(trans, &iter);
1501
1502         return ret;
1503 }
1504
1505 static int __must_check
1506 bch2_trans_update_by_path_trace(struct btree_trans *trans, struct btree_path *path,
1507                                 struct bkey_i *k, enum btree_update_flags flags,
1508                                 unsigned long ip);
1509
1510 static noinline int flush_new_cached_update(struct btree_trans *trans,
1511                                             struct btree_path *path,
1512                                             struct btree_insert_entry *i,
1513                                             enum btree_update_flags flags,
1514                                             unsigned long ip)
1515 {
1516         struct btree_path *btree_path;
1517         int ret;
1518
1519         i->key_cache_already_flushed = true;
1520         i->flags |= BTREE_TRIGGER_NORUN;
1521
1522         btree_path = bch2_path_get(trans, path->btree_id, path->pos, 1, 0,
1523                                    BTREE_ITER_INTENT, _THIS_IP_);
1524
1525         ret = bch2_btree_path_traverse(trans, btree_path, 0);
1526         if (ret)
1527                 goto err;
1528
1529         btree_path_set_should_be_locked(btree_path);
1530         ret = bch2_trans_update_by_path_trace(trans, btree_path, i->k, flags, ip);
1531 err:
1532         bch2_path_put(trans, btree_path, true);
1533         return ret;
1534 }
1535
1536 static int __must_check
1537 bch2_trans_update_by_path_trace(struct btree_trans *trans, struct btree_path *path,
1538                                 struct bkey_i *k, enum btree_update_flags flags,
1539                                 unsigned long ip)
1540 {
1541         struct bch_fs *c = trans->c;
1542         struct btree_insert_entry *i, n;
1543         int cmp;
1544
1545         EBUG_ON(!path->should_be_locked);
1546         EBUG_ON(trans->nr_updates >= BTREE_ITER_MAX);
1547         EBUG_ON(!bpos_eq(k->k.p, path->pos));
1548
1549         n = (struct btree_insert_entry) {
1550                 .flags          = flags,
1551                 .bkey_type      = __btree_node_type(path->level, path->btree_id),
1552                 .btree_id       = path->btree_id,
1553                 .level          = path->level,
1554                 .cached         = path->cached,
1555                 .path           = path,
1556                 .k              = k,
1557                 .ip_allocated   = ip,
1558         };
1559
1560 #ifdef CONFIG_BCACHEFS_DEBUG
1561         trans_for_each_update(trans, i)
1562                 BUG_ON(i != trans->updates &&
1563                        btree_insert_entry_cmp(i - 1, i) >= 0);
1564 #endif
1565
1566         /*
1567          * Pending updates are kept sorted: first, find position of new update,
1568          * then delete/trim any updates the new update overwrites:
1569          */
1570         trans_for_each_update(trans, i) {
1571                 cmp = btree_insert_entry_cmp(&n, i);
1572                 if (cmp <= 0)
1573                         break;
1574         }
1575
1576         if (!cmp && i < trans->updates + trans->nr_updates) {
1577                 EBUG_ON(i->insert_trigger_run || i->overwrite_trigger_run);
1578
1579                 bch2_path_put(trans, i->path, true);
1580                 i->flags        = n.flags;
1581                 i->cached       = n.cached;
1582                 i->k            = n.k;
1583                 i->path         = n.path;
1584                 i->ip_allocated = n.ip_allocated;
1585         } else {
1586                 array_insert_item(trans->updates, trans->nr_updates,
1587                                   i - trans->updates, n);
1588
1589                 i->old_v = bch2_btree_path_peek_slot_exact(path, &i->old_k).v;
1590                 i->old_btree_u64s = !bkey_deleted(&i->old_k) ? i->old_k.u64s : 0;
1591
1592                 if (unlikely(trans->journal_replay_not_finished)) {
1593                         struct bkey_i *j_k =
1594                                 bch2_journal_keys_peek_slot(c, n.btree_id, n.level, k->k.p);
1595
1596                         if (j_k) {
1597                                 i->old_k = j_k->k;
1598                                 i->old_v = &j_k->v;
1599                         }
1600                 }
1601         }
1602
1603         __btree_path_get(i->path, true);
1604
1605         /*
1606          * If a key is present in the key cache, it must also exist in the
1607          * btree - this is necessary for cache coherency. When iterating over
1608          * a btree that's cached in the key cache, the btree iter code checks
1609          * the key cache - but the key has to exist in the btree for that to
1610          * work:
1611          */
1612         if (path->cached &&
1613             bkey_deleted(&i->old_k) &&
1614             !(flags & BTREE_UPDATE_NO_KEY_CACHE_COHERENCY))
1615                 return flush_new_cached_update(trans, path, i, flags, ip);
1616
1617         return 0;
1618 }
1619
1620 static inline int __must_check
1621 bch2_trans_update_by_path(struct btree_trans *trans, struct btree_path *path,
1622                           struct bkey_i *k, enum btree_update_flags flags)
1623 {
1624         return bch2_trans_update_by_path_trace(trans, path, k, flags, _RET_IP_);
1625 }
1626
1627 int __must_check bch2_trans_update(struct btree_trans *trans, struct btree_iter *iter,
1628                                    struct bkey_i *k, enum btree_update_flags flags)
1629 {
1630         struct btree_path *path = iter->update_path ?: iter->path;
1631         struct bkey_cached *ck;
1632         int ret;
1633
1634         if (iter->flags & BTREE_ITER_IS_EXTENTS)
1635                 return bch2_trans_update_extent(trans, iter, k, flags);
1636
1637         if (bkey_deleted(&k->k) &&
1638             !(flags & BTREE_UPDATE_KEY_CACHE_RECLAIM) &&
1639             (iter->flags & BTREE_ITER_FILTER_SNAPSHOTS)) {
1640                 ret = need_whiteout_for_snapshot(trans, iter->btree_id, k->k.p);
1641                 if (unlikely(ret < 0))
1642                         return ret;
1643
1644                 if (ret)
1645                         k->k.type = KEY_TYPE_whiteout;
1646         }
1647
1648         /*
1649          * Ensure that updates to cached btrees go to the key cache:
1650          */
1651         if (!(flags & BTREE_UPDATE_KEY_CACHE_RECLAIM) &&
1652             !path->cached &&
1653             !path->level &&
1654             btree_id_cached(trans->c, path->btree_id)) {
1655                 if (!iter->key_cache_path ||
1656                     !iter->key_cache_path->should_be_locked ||
1657                     !bpos_eq(iter->key_cache_path->pos, k->k.p)) {
1658                         if (!iter->key_cache_path)
1659                                 iter->key_cache_path =
1660                                         bch2_path_get(trans, path->btree_id, path->pos, 1, 0,
1661                                                       BTREE_ITER_INTENT|
1662                                                       BTREE_ITER_CACHED, _THIS_IP_);
1663
1664                         iter->key_cache_path =
1665                                 bch2_btree_path_set_pos(trans, iter->key_cache_path, path->pos,
1666                                                         iter->flags & BTREE_ITER_INTENT,
1667                                                         _THIS_IP_);
1668
1669                         ret = bch2_btree_path_traverse(trans, iter->key_cache_path,
1670                                                        BTREE_ITER_CACHED);
1671                         if (unlikely(ret))
1672                                 return ret;
1673
1674                         ck = (void *) iter->key_cache_path->l[0].b;
1675
1676                         if (test_bit(BKEY_CACHED_DIRTY, &ck->flags)) {
1677                                 trace_and_count(trans->c, trans_restart_key_cache_raced, trans, _RET_IP_);
1678                                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_key_cache_raced);
1679                         }
1680
1681                         btree_path_set_should_be_locked(iter->key_cache_path);
1682                 }
1683
1684                 path = iter->key_cache_path;
1685         }
1686
1687         return bch2_trans_update_by_path(trans, path, k, flags);
1688 }
1689
1690 int __must_check bch2_trans_update_buffered(struct btree_trans *trans,
1691                                             enum btree_id btree,
1692                                             struct bkey_i *k)
1693 {
1694         struct btree_write_buffered_key *i;
1695         int ret;
1696
1697         EBUG_ON(trans->nr_wb_updates > trans->wb_updates_size);
1698         EBUG_ON(k->k.u64s > BTREE_WRITE_BUFERED_U64s_MAX);
1699
1700         trans_for_each_wb_update(trans, i) {
1701                 if (i->btree == btree && bpos_eq(i->k.k.p, k->k.p)) {
1702                         bkey_copy(&i->k, k);
1703                         return 0;
1704                 }
1705         }
1706
1707         if (!trans->wb_updates ||
1708             trans->nr_wb_updates == trans->wb_updates_size) {
1709                 struct btree_write_buffered_key *u;
1710
1711                 if (trans->nr_wb_updates == trans->wb_updates_size) {
1712                         struct btree_transaction_stats *s = btree_trans_stats(trans);
1713
1714                         BUG_ON(trans->wb_updates_size > U8_MAX / 2);
1715                         trans->wb_updates_size = max(1, trans->wb_updates_size * 2);
1716                         if (s)
1717                                 s->wb_updates_size = trans->wb_updates_size;
1718                 }
1719
1720                 u = bch2_trans_kmalloc_nomemzero(trans,
1721                                         trans->wb_updates_size *
1722                                         sizeof(struct btree_write_buffered_key));
1723                 ret = PTR_ERR_OR_ZERO(u);
1724                 if (ret)
1725                         return ret;
1726
1727                 if (trans->nr_wb_updates)
1728                         memcpy(u, trans->wb_updates, trans->nr_wb_updates *
1729                                sizeof(struct btree_write_buffered_key));
1730                 trans->wb_updates = u;
1731         }
1732
1733         trans->wb_updates[trans->nr_wb_updates] = (struct btree_write_buffered_key) {
1734                 .btree  = btree,
1735         };
1736
1737         bkey_copy(&trans->wb_updates[trans->nr_wb_updates].k, k);
1738         trans->nr_wb_updates++;
1739
1740         return 0;
1741 }
1742
1743 void bch2_trans_commit_hook(struct btree_trans *trans,
1744                             struct btree_trans_commit_hook *h)
1745 {
1746         h->next = trans->hooks;
1747         trans->hooks = h;
1748 }
1749
1750 int __bch2_btree_insert(struct btree_trans *trans,
1751                         enum btree_id id,
1752                         struct bkey_i *k, enum btree_update_flags flags)
1753 {
1754         struct btree_iter iter;
1755         int ret;
1756
1757         bch2_trans_iter_init(trans, &iter, id, bkey_start_pos(&k->k),
1758                              BTREE_ITER_INTENT);
1759         ret   = bch2_btree_iter_traverse(&iter) ?:
1760                 bch2_trans_update(trans, &iter, k, flags);
1761         bch2_trans_iter_exit(trans, &iter);
1762         return ret;
1763 }
1764
1765 /**
1766  * bch2_btree_insert - insert keys into the extent btree
1767  * @c:                  pointer to struct bch_fs
1768  * @id:                 btree to insert into
1769  * @insert_keys:        list of keys to insert
1770  * @hook:               insert callback
1771  */
1772 int bch2_btree_insert(struct bch_fs *c, enum btree_id id,
1773                       struct bkey_i *k,
1774                       struct disk_reservation *disk_res,
1775                       u64 *journal_seq, int flags)
1776 {
1777         return bch2_trans_do(c, disk_res, journal_seq, flags,
1778                              __bch2_btree_insert(&trans, id, k, 0));
1779 }
1780
1781 int bch2_btree_delete_extent_at(struct btree_trans *trans, struct btree_iter *iter,
1782                                 unsigned len, unsigned update_flags)
1783 {
1784         struct bkey_i *k;
1785
1786         k = bch2_trans_kmalloc(trans, sizeof(*k));
1787         if (IS_ERR(k))
1788                 return PTR_ERR(k);
1789
1790         bkey_init(&k->k);
1791         k->k.p = iter->pos;
1792         bch2_key_resize(&k->k, len);
1793         return bch2_trans_update(trans, iter, k, update_flags);
1794 }
1795
1796 int bch2_btree_delete_at(struct btree_trans *trans,
1797                          struct btree_iter *iter, unsigned update_flags)
1798 {
1799         return bch2_btree_delete_extent_at(trans, iter, 0, update_flags);
1800 }
1801
1802 int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id,
1803                                   struct bpos start, struct bpos end,
1804                                   unsigned update_flags,
1805                                   u64 *journal_seq)
1806 {
1807         u32 restart_count = trans->restart_count;
1808         struct btree_iter iter;
1809         struct bkey_s_c k;
1810         int ret = 0;
1811
1812         bch2_trans_iter_init(trans, &iter, id, start, BTREE_ITER_INTENT);
1813         while ((k = bch2_btree_iter_peek_upto(&iter, end)).k) {
1814                 struct disk_reservation disk_res =
1815                         bch2_disk_reservation_init(trans->c, 0);
1816                 struct bkey_i delete;
1817
1818                 ret = bkey_err(k);
1819                 if (ret)
1820                         goto err;
1821
1822                 bkey_init(&delete.k);
1823
1824                 /*
1825                  * This could probably be more efficient for extents:
1826                  */
1827
1828                 /*
1829                  * For extents, iter.pos won't necessarily be the same as
1830                  * bkey_start_pos(k.k) (for non extents they always will be the
1831                  * same). It's important that we delete starting from iter.pos
1832                  * because the range we want to delete could start in the middle
1833                  * of k.
1834                  *
1835                  * (bch2_btree_iter_peek() does guarantee that iter.pos >=
1836                  * bkey_start_pos(k.k)).
1837                  */
1838                 delete.k.p = iter.pos;
1839
1840                 if (iter.flags & BTREE_ITER_IS_EXTENTS)
1841                         bch2_key_resize(&delete.k,
1842                                         bpos_min(end, k.k->p).offset -
1843                                         iter.pos.offset);
1844
1845                 ret   = bch2_trans_update(trans, &iter, &delete, update_flags) ?:
1846                         bch2_trans_commit(trans, &disk_res, journal_seq,
1847                                           BTREE_INSERT_NOFAIL);
1848                 bch2_disk_reservation_put(trans->c, &disk_res);
1849 err:
1850                 /*
1851                  * the bch2_trans_begin() call is in a weird place because we
1852                  * need to call it after every transaction commit, to avoid path
1853                  * overflow, but don't want to call it if the delete operation
1854                  * is a no-op and we have no work to do:
1855                  */
1856                 bch2_trans_begin(trans);
1857
1858                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
1859                         ret = 0;
1860                 if (ret)
1861                         break;
1862         }
1863         bch2_trans_iter_exit(trans, &iter);
1864
1865         if (!ret && trans_was_restarted(trans, restart_count))
1866                 ret = -BCH_ERR_transaction_restart_nested;
1867         return ret;
1868 }
1869
1870 /*
1871  * bch_btree_delete_range - delete everything within a given range
1872  *
1873  * Range is a half open interval - [start, end)
1874  */
1875 int bch2_btree_delete_range(struct bch_fs *c, enum btree_id id,
1876                             struct bpos start, struct bpos end,
1877                             unsigned update_flags,
1878                             u64 *journal_seq)
1879 {
1880         int ret = bch2_trans_run(c,
1881                         bch2_btree_delete_range_trans(&trans, id, start, end,
1882                                                       update_flags, journal_seq));
1883         if (ret == -BCH_ERR_transaction_restart_nested)
1884                 ret = 0;
1885         return ret;
1886 }
1887
1888 static int __bch2_trans_log_msg(darray_u64 *entries, const char *fmt, va_list args)
1889 {
1890         struct printbuf buf = PRINTBUF;
1891         struct jset_entry_log *l;
1892         unsigned u64s;
1893         int ret;
1894
1895         prt_vprintf(&buf, fmt, args);
1896         ret = buf.allocation_failure ? -ENOMEM : 0;
1897         if (ret)
1898                 goto err;
1899
1900         u64s = DIV_ROUND_UP(buf.pos, sizeof(u64));
1901
1902         ret = darray_make_room(entries, jset_u64s(u64s));
1903         if (ret)
1904                 goto err;
1905
1906         l = (void *) &darray_top(*entries);
1907         l->entry.u64s           = cpu_to_le16(u64s);
1908         l->entry.btree_id       = 0;
1909         l->entry.level          = 1;
1910         l->entry.type           = BCH_JSET_ENTRY_log;
1911         l->entry.pad[0]         = 0;
1912         l->entry.pad[1]         = 0;
1913         l->entry.pad[2]         = 0;
1914         memcpy(l->d, buf.buf, buf.pos);
1915         while (buf.pos & 7)
1916                 l->d[buf.pos++] = '\0';
1917
1918         entries->nr += jset_u64s(u64s);
1919 err:
1920         printbuf_exit(&buf);
1921         return ret;
1922 }
1923
1924 int bch2_trans_log_msg(struct btree_trans *trans, const char *fmt, ...)
1925 {
1926         va_list args;
1927         int ret;
1928
1929         va_start(args, fmt);
1930         ret = __bch2_trans_log_msg(&trans->extra_journal_entries, fmt, args);
1931         va_end(args);
1932
1933         return ret;
1934 }
1935
1936 int bch2_fs_log_msg(struct bch_fs *c, const char *fmt, ...)
1937 {
1938         va_list args;
1939         int ret;
1940
1941         va_start(args, fmt);
1942
1943         if (!test_bit(JOURNAL_STARTED, &c->journal.flags)) {
1944                 ret = __bch2_trans_log_msg(&c->journal.early_journal_entries, fmt, args);
1945         } else {
1946                 ret = bch2_trans_do(c, NULL, NULL, BTREE_INSERT_LAZY_RW,
1947                         __bch2_trans_log_msg(&trans.extra_journal_entries, fmt, args));
1948         }
1949
1950         va_end(args);
1951
1952         return ret;
1953
1954 }