git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_locking.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "btree_locking.h"
   5 #include "btree_types.h"
   6
   7 static struct lock_class_key bch2_btree_node_lock_key;
   8
   9 void bch2_btree_lock_init(struct btree_bkey_cached_common *b,
  10                           enum six_lock_init_flags flags)
  11 {
  12         __six_lock_init(&b->lock, "b->c.lock", &bch2_btree_node_lock_key, flags);
  13 #ifdef CONFIG_DEBUG_LOCK_ALLOC
  14         lockdep_set_no_check_recursion(&b->lock.dep_map);
  15 #endif
  16 }
  17
  18 #ifdef CONFIG_LOCKDEP
  19 void bch2_assert_btree_nodes_not_locked(void)
  20 {
  21         BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key));
  22 }
  23 #endif
  24
  25 /* Btree node locking: */
  26
  27 struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *trans,
  28                                                   struct btree_path *skip,
  29                                                   struct btree_bkey_cached_common *b,
  30                                                   unsigned level)
  31 {
  32         struct btree_path *path;
  33         struct six_lock_count ret;
  34
  35         memset(&ret, 0, sizeof(ret));
  36
  37         if (IS_ERR_OR_NULL(b))
  38                 return ret;
  39
  40         trans_for_each_path(trans, path)
  41                 if (path != skip && &path->l[level].b->c == b) {
  42                         int t = btree_node_locked_type(path, level);
  43
  44                         if (t != BTREE_NODE_UNLOCKED)
  45                                 ret.n[t]++;
  46                 }
  47
  48         return ret;
  49 }
  50
  51 /* unlock */
  52
  53 void bch2_btree_node_unlock_write(struct btree_trans *trans,
  54                         struct btree_path *path, struct btree *b)
  55 {
  56         bch2_btree_node_unlock_write_inlined(trans, path, b);
  57 }
  58
  59 /* lock */
  60
  61 /*
  62  * @trans wants to lock @b with type @type
  63  */
  64 struct trans_waiting_for_lock {
  65         struct btree_trans              *trans;
  66         struct btree_bkey_cached_common *node_want;
  67         enum six_lock_type              lock_want;
  68
  69         /* for iterating over held locks :*/
  70         u8                              path_idx;
  71         u8                              level;
  72         u64                             lock_start_time;
  73 };
  74
  75 struct lock_graph {
  76         struct trans_waiting_for_lock   g[8];
  77         unsigned                        nr;
  78 };
  79
  80 static noinline void print_cycle(struct printbuf *out, struct lock_graph *g)
  81 {
  82         struct trans_waiting_for_lock *i;
  83
  84         prt_printf(out, "Found lock cycle (%u entries):", g->nr);
  85         prt_newline(out);
  86
  87         for (i = g->g; i < g->g + g->nr; i++)
  88                 bch2_btree_trans_to_text(out, i->trans);
  89 }
  90
  91 static noinline void print_chain(struct printbuf *out, struct lock_graph *g)
  92 {
  93         struct trans_waiting_for_lock *i;
  94
  95         for (i = g->g; i != g->g + g->nr; i++) {
  96                 if (i != g->g)
  97                         prt_str(out, "<- ");
  98                 prt_printf(out, "%u ", i->trans->locking_wait.task->pid);
  99         }
 100         prt_newline(out);
 101 }
 102
 103 static void lock_graph_up(struct lock_graph *g)
 104 {
 105         closure_put(&g->g[--g->nr].trans->ref);
 106 }
 107
 108 static noinline void lock_graph_pop_all(struct lock_graph *g)
 109 {
 110         while (g->nr)
 111                 lock_graph_up(g);
 112 }
 113
 114 static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
 115 {
 116         closure_get(&trans->ref);
 117
 118         g->g[g->nr++] = (struct trans_waiting_for_lock) {
 119                 .trans          = trans,
 120                 .node_want      = trans->locking,
 121                 .lock_want      = trans->locking_wait.lock_want,
 122         };
 123 }
 124
 125 static bool lock_graph_remove_non_waiters(struct lock_graph *g)
 126 {
 127         struct trans_waiting_for_lock *i;
 128
 129         for (i = g->g + 1; i < g->g + g->nr; i++)
 130                 if (i->trans->locking != i->node_want ||
 131                     i->trans->locking_wait.start_time != i[-1].lock_start_time) {
 132                         while (g->g + g->nr > i)
 133                                 lock_graph_up(g);
 134                         return true;
 135                 }
 136
 137         return false;
 138 }
 139
 140 static int abort_lock(struct lock_graph *g, struct trans_waiting_for_lock *i)
 141 {
 142         if (i == g->g) {
 143                 trace_and_count(i->trans->c, trans_restart_would_deadlock, i->trans, _RET_IP_);
 144                 return btree_trans_restart(i->trans, BCH_ERR_transaction_restart_would_deadlock);
 145         } else {
 146                 i->trans->lock_must_abort = true;
 147                 wake_up_process(i->trans->locking_wait.task);
 148                 return 0;
 149         }
 150 }
 151
 152 static int btree_trans_abort_preference(struct btree_trans *trans)
 153 {
 154         if (trans->lock_may_not_fail)
 155                 return 0;
 156         if (trans->locking_wait.lock_want == SIX_LOCK_write)
 157                 return 1;
 158         if (!trans->in_traverse_all)
 159                 return 2;
 160         return 3;
 161 }
 162
 163 static noinline int break_cycle(struct lock_graph *g, struct printbuf *cycle)
 164 {
 165         struct trans_waiting_for_lock *i, *abort = NULL;
 166         unsigned best = 0, pref;
 167         int ret;
 168
 169         if (lock_graph_remove_non_waiters(g))
 170                 return 0;
 171
 172         /* Only checking, for debugfs: */
 173         if (cycle) {
 174                 print_cycle(cycle, g);
 175                 ret = -1;
 176                 goto out;
 177         }
 178
 179         for (i = g->g; i < g->g + g->nr; i++) {
 180                 pref = btree_trans_abort_preference(i->trans);
 181                 if (pref > best) {
 182                         abort = i;
 183                         best = pref;
 184                 }
 185         }
 186
 187         if (unlikely(!best)) {
 188                 struct printbuf buf = PRINTBUF;
 189
 190                 prt_printf(&buf, bch2_fmt(g->g->trans->c, "cycle of nofail locks"));
 191
 192                 for (i = g->g; i < g->g + g->nr; i++) {
 193                         struct btree_trans *trans = i->trans;
 194
 195                         bch2_btree_trans_to_text(&buf, trans);
 196
 197                         prt_printf(&buf, "backtrace:");
 198                         prt_newline(&buf);
 199                         printbuf_indent_add(&buf, 2);
 200                         bch2_prt_task_backtrace(&buf, trans->locking_wait.task);
 201                         printbuf_indent_sub(&buf, 2);
 202                         prt_newline(&buf);
 203                 }
 204
 205                 bch2_print_string_as_lines(KERN_ERR, buf.buf);
 206                 printbuf_exit(&buf);
 207                 BUG();
 208         }
 209
 210         ret = abort_lock(g, abort);
 211 out:
 212         if (ret)
 213                 while (g->nr)
 214                         lock_graph_up(g);
 215         return ret;
 216 }
 217
 218 static int lock_graph_descend(struct lock_graph *g, struct btree_trans *trans,
 219                               struct printbuf *cycle)
 220 {
 221         struct btree_trans *orig_trans = g->g->trans;
 222         struct trans_waiting_for_lock *i;
 223
 224         for (i = g->g; i < g->g + g->nr; i++)
 225                 if (i->trans == trans)
 226                         return break_cycle(g, cycle);
 227
 228         if (g->nr == ARRAY_SIZE(g->g)) {
 229                 if (orig_trans->lock_may_not_fail)
 230                         return 0;
 231
 232                 while (g->nr)
 233                         lock_graph_up(g);
 234
 235                 if (cycle)
 236                         return 0;
 237
 238                 trace_and_count(trans->c, trans_restart_would_deadlock_recursion_limit, trans, _RET_IP_);
 239                 return btree_trans_restart(orig_trans, BCH_ERR_transaction_restart_deadlock_recursion_limit);
 240         }
 241
 242         lock_graph_down(g, trans);
 243         return 0;
 244 }
 245
 246 static bool lock_type_conflicts(enum six_lock_type t1, enum six_lock_type t2)
 247 {
 248         return t1 + t2 > 1;
 249 }
 250
 251 int bch2_check_for_deadlock(struct btree_trans *trans, struct printbuf *cycle)
 252 {
 253         struct lock_graph g;
 254         struct trans_waiting_for_lock *top;
 255         struct btree_bkey_cached_common *b;
 256         struct btree_path *path;
 257         int ret;
 258
 259         if (trans->lock_must_abort) {
 260                 if (cycle)
 261                         return -1;
 262
 263                 trace_and_count(trans->c, trans_restart_would_deadlock, trans, _RET_IP_);
 264                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock);
 265         }
 266
 267         g.nr = 0;
 268         lock_graph_down(&g, trans);
 269 next:
 270         if (!g.nr)
 271                 return 0;
 272
 273         top = &g.g[g.nr - 1];
 274
 275         trans_for_each_path_from(top->trans, path, top->path_idx) {
 276                 if (!path->nodes_locked)
 277                         continue;
 278
 279                 if (top->path_idx != path->idx) {
 280                         top->path_idx           = path->idx;
 281                         top->level              = 0;
 282                         top->lock_start_time    = 0;
 283                 }
 284
 285                 for (;
 286                      top->level < BTREE_MAX_DEPTH;
 287                      top->level++, top->lock_start_time = 0) {
 288                         int lock_held = btree_node_locked_type(path, top->level);
 289
 290                         if (lock_held == BTREE_NODE_UNLOCKED)
 291                                 continue;
 292
 293                         b = &READ_ONCE(path->l[top->level].b)->c;
 294
 295                         if (IS_ERR_OR_NULL(b)) {
 296                                 /*
 297                                  * If we get here, it means we raced with the
 298                                  * other thread updating its btree_path
 299                                  * structures - which means it can't be blocked
 300                                  * waiting on a lock:
 301                                  */
 302                                 if (!lock_graph_remove_non_waiters(&g)) {
 303                                         /*
 304                                          * If lock_graph_remove_non_waiters()
 305                                          * didn't do anything, it must be
 306                                          * because we're being called by debugfs
 307                                          * checking for lock cycles, which
 308                                          * invokes us on btree_transactions that
 309                                          * aren't actually waiting on anything.
 310                                          * Just bail out:
 311                                          */
 312                                         lock_graph_pop_all(&g);
 313                                 }
 314
 315                                 goto next;
 316                         }
 317
 318                         if (list_empty_careful(&b->lock.wait_list))
 319                                 continue;
 320
 321                         raw_spin_lock(&b->lock.wait_lock);
 322                         list_for_each_entry(trans, &b->lock.wait_list, locking_wait.list) {
 323                                 BUG_ON(b != trans->locking);
 324
 325                                 if (top->lock_start_time &&
 326                                     time_after_eq64(top->lock_start_time, trans->locking_wait.start_time))
 327                                         continue;
 328
 329                                 top->lock_start_time = trans->locking_wait.start_time;
 330
 331                                 /* Don't check for self deadlock: */
 332                                 if (trans == top->trans ||
 333                                     !lock_type_conflicts(lock_held, trans->locking_wait.lock_want))
 334                                         continue;
 335
 336                                 ret = lock_graph_descend(&g, trans, cycle);
 337                                 raw_spin_unlock(&b->lock.wait_lock);
 338
 339                                 if (ret)
 340                                         return ret;
 341                                 goto next;
 342
 343                         }
 344                         raw_spin_unlock(&b->lock.wait_lock);
 345                 }
 346         }
 347
 348         if (g.nr > 1 && cycle)
 349                 print_chain(cycle, &g);
 350         lock_graph_up(&g);
 351         goto next;
 352 }
 353
 354 int bch2_six_check_for_deadlock(struct six_lock *lock, void *p)
 355 {
 356         struct btree_trans *trans = p;
 357
 358         return bch2_check_for_deadlock(trans, NULL);
 359 }
 360
 361 int __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree_path *path,
 362                                  struct btree_bkey_cached_common *b,
 363                                  bool lock_may_not_fail)
 364 {
 365         int readers = bch2_btree_node_lock_counts(trans, NULL, b, b->level).n[SIX_LOCK_read];
 366         int ret;
 367
 368         /*
 369          * Must drop our read locks before calling six_lock_write() -
 370          * six_unlock() won't do wakeups until the reader count
 371          * goes to 0, and it's safe because we have the node intent
 372          * locked:
 373          */
 374         six_lock_readers_add(&b->lock, -readers);
 375         ret = __btree_node_lock_nopath(trans, b, SIX_LOCK_write,
 376                                        lock_may_not_fail, _RET_IP_);
 377         six_lock_readers_add(&b->lock, readers);
 378
 379         if (ret)
 380                 mark_btree_node_locked_noreset(path, b->level, SIX_LOCK_intent);
 381
 382         return ret;
 383 }
 384
 385 void bch2_btree_node_lock_write_nofail(struct btree_trans *trans,
 386                                        struct btree_path *path,
 387                                        struct btree_bkey_cached_common *b)
 388 {
 389         struct btree_path *linked;
 390         unsigned i;
 391         int ret;
 392
 393         /*
 394          * XXX BIG FAT NOTICE
 395          *
 396          * Drop all read locks before taking a write lock:
 397          *
 398          * This is a hack, because bch2_btree_node_lock_write_nofail() is a
 399          * hack - but by dropping read locks first, this should never fail, and
 400          * we only use this in code paths where whatever read locks we've
 401          * already taken are no longer needed:
 402          */
 403
 404         trans_for_each_path(trans, linked) {
 405                 if (!linked->nodes_locked)
 406                         continue;
 407
 408                 for (i = 0; i < BTREE_MAX_DEPTH; i++)
 409                         if (btree_node_read_locked(linked, i)) {
 410                                 btree_node_unlock(trans, linked, i);
 411                                 btree_path_set_dirty(linked, BTREE_ITER_NEED_RELOCK);
 412                         }
 413         }
 414
 415         ret = __btree_node_lock_write(trans, path, b, true);
 416         BUG_ON(ret);
 417 }
 418
 419 /* relock */
 420
 421 static inline bool btree_path_get_locks(struct btree_trans *trans,
 422                                         struct btree_path *path,
 423                                         bool upgrade)
 424 {
 425         unsigned l = path->level;
 426         int fail_idx = -1;
 427
 428         do {
 429                 if (!btree_path_node(path, l))
 430                         break;
 431
 432                 if (!(upgrade
 433                       ? bch2_btree_node_upgrade(trans, path, l)
 434                       : bch2_btree_node_relock(trans, path, l)))
 435                         fail_idx = l;
 436
 437                 l++;
 438         } while (l < path->locks_want);
 439
 440         /*
 441          * When we fail to get a lock, we have to ensure that any child nodes
 442          * can't be relocked so bch2_btree_path_traverse has to walk back up to
 443          * the node that we failed to relock:
 444          */
 445         if (fail_idx >= 0) {
 446                 __bch2_btree_path_unlock(trans, path);
 447                 btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
 448
 449                 do {
 450                         path->l[fail_idx].b = upgrade
 451                                 ? ERR_PTR(-BCH_ERR_no_btree_node_upgrade)
 452                                 : ERR_PTR(-BCH_ERR_no_btree_node_relock);
 453                         --fail_idx;
 454                 } while (fail_idx >= 0);
 455         }
 456
 457         if (path->uptodate == BTREE_ITER_NEED_RELOCK)
 458                 path->uptodate = BTREE_ITER_UPTODATE;
 459
 460         bch2_trans_verify_locks(trans);
 461
 462         return path->uptodate < BTREE_ITER_NEED_RELOCK;
 463 }
 464
 465 bool __bch2_btree_node_relock(struct btree_trans *trans,
 466                               struct btree_path *path, unsigned level,
 467                               bool trace)
 468 {
 469         struct btree *b = btree_path_node(path, level);
 470         int want = __btree_lock_want(path, level);
 471
 472         if (race_fault())
 473                 goto fail;
 474
 475         if (six_relock_type(&b->c.lock, want, path->l[level].lock_seq) ||
 476             (btree_node_lock_seq_matches(path, b, level) &&
 477              btree_node_lock_increment(trans, &b->c, level, want))) {
 478                 mark_btree_node_locked(trans, path, level, want);
 479                 return true;
 480         }
 481 fail:
 482         if (trace && !trans->notrace_relock_fail)
 483                 trace_and_count(trans->c, btree_path_relock_fail, trans, _RET_IP_, path, level);
 484         return false;
 485 }
 486
 487 /* upgrade */
 488
 489 bool bch2_btree_node_upgrade(struct btree_trans *trans,
 490                              struct btree_path *path, unsigned level)
 491 {
 492         struct btree *b = path->l[level].b;
 493         struct six_lock_count count = bch2_btree_node_lock_counts(trans, path, &b->c, level);
 494
 495         if (!is_btree_node(path, level))
 496                 return false;
 497
 498         switch (btree_lock_want(path, level)) {
 499         case BTREE_NODE_UNLOCKED:
 500                 BUG_ON(btree_node_locked(path, level));
 501                 return true;
 502         case BTREE_NODE_READ_LOCKED:
 503                 BUG_ON(btree_node_intent_locked(path, level));
 504                 return bch2_btree_node_relock(trans, path, level);
 505         case BTREE_NODE_INTENT_LOCKED:
 506                 break;
 507         case BTREE_NODE_WRITE_LOCKED:
 508                 BUG();
 509         }
 510
 511         if (btree_node_intent_locked(path, level))
 512                 return true;
 513
 514         if (race_fault())
 515                 return false;
 516
 517         if (btree_node_locked(path, level)) {
 518                 bool ret;
 519
 520                 six_lock_readers_add(&b->c.lock, -count.n[SIX_LOCK_read]);
 521                 ret = six_lock_tryupgrade(&b->c.lock);
 522                 six_lock_readers_add(&b->c.lock, count.n[SIX_LOCK_read]);
 523
 524                 if (ret)
 525                         goto success;
 526         } else {
 527                 if (six_relock_type(&b->c.lock, SIX_LOCK_intent, path->l[level].lock_seq))
 528                         goto success;
 529         }
 530
 531         /*
 532          * Do we already have an intent lock via another path? If so, just bump
 533          * lock count:
 534          */
 535         if (btree_node_lock_seq_matches(path, b, level) &&
 536             btree_node_lock_increment(trans, &b->c, level, BTREE_NODE_INTENT_LOCKED)) {
 537                 btree_node_unlock(trans, path, level);
 538                 goto success;
 539         }
 540
 541         trace_and_count(trans->c, btree_path_upgrade_fail, trans, _RET_IP_, path, level);
 542         return false;
 543 success:
 544         mark_btree_node_locked_noreset(path, level, SIX_LOCK_intent);
 545         return true;
 546 }
 547
 548 /* Btree path locking: */
 549
 550 /*
 551  * Only for btree_cache.c - only relocks intent locks
 552  */
 553 int bch2_btree_path_relock_intent(struct btree_trans *trans,
 554                                   struct btree_path *path)
 555 {
 556         unsigned l;
 557
 558         for (l = path->level;
 559              l < path->locks_want && btree_path_node(path, l);
 560              l++) {
 561                 if (!bch2_btree_node_relock(trans, path, l)) {
 562                         __bch2_btree_path_unlock(trans, path);
 563                         btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
 564                         trace_and_count(trans->c, trans_restart_relock_path_intent, trans, _RET_IP_, path);
 565                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path_intent);
 566                 }
 567         }
 568
 569         return 0;
 570 }
 571
 572 __flatten
 573 bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
 574                         struct btree_path *path, unsigned long trace_ip)
 575 {
 576         return btree_path_get_locks(trans, path, false);
 577 }
 578
 579 int __bch2_btree_path_relock(struct btree_trans *trans,
 580                         struct btree_path *path, unsigned long trace_ip)
 581 {
 582         if (!bch2_btree_path_relock_norestart(trans, path, trace_ip)) {
 583                 trace_and_count(trans->c, trans_restart_relock_path, trans, trace_ip, path);
 584                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path);
 585         }
 586
 587         return 0;
 588 }
 589
 590 __flatten
 591 bool bch2_btree_path_upgrade_norestart(struct btree_trans *trans,
 592                         struct btree_path *path, unsigned long trace_ip)
 593 {
 594         return btree_path_get_locks(trans, path, true);
 595 }
 596
 597 bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans,
 598                                struct btree_path *path,
 599                                unsigned new_locks_want)
 600 {
 601         EBUG_ON(path->locks_want >= new_locks_want);
 602
 603         path->locks_want = new_locks_want;
 604
 605         return btree_path_get_locks(trans, path, true);
 606 }
 607
 608 bool __bch2_btree_path_upgrade(struct btree_trans *trans,
 609                                struct btree_path *path,
 610                                unsigned new_locks_want)
 611 {
 612         struct btree_path *linked;
 613
 614         if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want))
 615                 return true;
 616
 617         /*
 618          * XXX: this is ugly - we'd prefer to not be mucking with other
 619          * iterators in the btree_trans here.
 620          *
 621          * On failure to upgrade the iterator, setting iter->locks_want and
 622          * calling get_locks() is sufficient to make bch2_btree_path_traverse()
 623          * get the locks we want on transaction restart.
 624          *
 625          * But if this iterator was a clone, on transaction restart what we did
 626          * to this iterator isn't going to be preserved.
 627          *
 628          * Possibly we could add an iterator field for the parent iterator when
 629          * an iterator is a copy - for now, we'll just upgrade any other
 630          * iterators with the same btree id.
 631          *
 632          * The code below used to be needed to ensure ancestor nodes get locked
 633          * before interior nodes - now that's handled by
 634          * bch2_btree_path_traverse_all().
 635          */
 636         if (!path->cached && !trans->in_traverse_all)
 637                 trans_for_each_path(trans, linked)
 638                         if (linked != path &&
 639                             linked->cached == path->cached &&
 640                             linked->btree_id == path->btree_id &&
 641                             linked->locks_want < new_locks_want) {
 642                                 linked->locks_want = new_locks_want;
 643                                 btree_path_get_locks(trans, linked, true);
 644                         }
 645
 646         return false;
 647 }
 648
 649 void __bch2_btree_path_downgrade(struct btree_trans *trans,
 650                                  struct btree_path *path,
 651                                  unsigned new_locks_want)
 652 {
 653         unsigned l;
 654
 655         EBUG_ON(path->locks_want < new_locks_want);
 656
 657         path->locks_want = new_locks_want;
 658
 659         while (path->nodes_locked &&
 660                (l = btree_path_highest_level_locked(path)) >= path->locks_want) {
 661                 if (l > path->level) {
 662                         btree_node_unlock(trans, path, l);
 663                 } else {
 664                         if (btree_node_intent_locked(path, l)) {
 665                                 six_lock_downgrade(&path->l[l].b->c.lock);
 666                                 mark_btree_node_locked_noreset(path, l, SIX_LOCK_read);
 667                         }
 668                         break;
 669                 }
 670         }
 671
 672         bch2_btree_path_verify_locks(path);
 673 }
 674
 675 /* Btree transaction locking: */
 676
 677 void bch2_trans_downgrade(struct btree_trans *trans)
 678 {
 679         struct btree_path *path;
 680
 681         trans_for_each_path(trans, path)
 682                 bch2_btree_path_downgrade(trans, path);
 683 }
 684
 685 int bch2_trans_relock(struct btree_trans *trans)
 686 {
 687         struct btree_path *path;
 688
 689         if (unlikely(trans->restarted))
 690                 return -((int) trans->restarted);
 691
 692         trans_for_each_path(trans, path)
 693                 if (path->should_be_locked &&
 694                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
 695                         trace_and_count(trans->c, trans_restart_relock, trans, _RET_IP_, path);
 696                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
 697                 }
 698         return 0;
 699 }
 700
 701 int bch2_trans_relock_notrace(struct btree_trans *trans)
 702 {
 703         struct btree_path *path;
 704
 705         if (unlikely(trans->restarted))
 706                 return -((int) trans->restarted);
 707
 708         trans_for_each_path(trans, path)
 709                 if (path->should_be_locked &&
 710                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
 711                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
 712                 }
 713         return 0;
 714 }
 715
 716 void bch2_trans_unlock(struct btree_trans *trans)
 717 {
 718         struct btree_path *path;
 719
 720         trans_for_each_path(trans, path)
 721                 __bch2_btree_path_unlock(trans, path);
 722
 723         /*
 724          * bch2_gc_btree_init_recurse() doesn't use btree iterators for walking
 725          * btree nodes, it implements its own walking:
 726          */
 727         if (!trans->is_initial_gc)
 728                 bch2_assert_btree_nodes_not_locked();
 729 }
 730
 731 bool bch2_trans_locked(struct btree_trans *trans)
 732 {
 733         struct btree_path *path;
 734
 735         trans_for_each_path(trans, path)
 736                 if (path->nodes_locked)
 737                         return true;
 738         return false;
 739 }
 740
 741 int __bch2_trans_mutex_lock(struct btree_trans *trans,
 742                             struct mutex *lock)
 743 {
 744         int ret = drop_locks_do(trans, (mutex_lock(lock), 0));
 745
 746         if (ret)
 747                 mutex_unlock(lock);
 748         return ret;
 749 }
 750
 751 /* Debug */
 752
 753 #ifdef CONFIG_BCACHEFS_DEBUG
 754
 755 void bch2_btree_path_verify_locks(struct btree_path *path)
 756 {
 757         unsigned l;
 758
 759         if (!path->nodes_locked) {
 760                 BUG_ON(path->uptodate == BTREE_ITER_UPTODATE &&
 761                        btree_path_node(path, path->level));
 762                 return;
 763         }
 764
 765         for (l = 0; l < BTREE_MAX_DEPTH; l++) {
 766                 int want = btree_lock_want(path, l);
 767                 int have = btree_node_locked_type(path, l);
 768
 769                 BUG_ON(!is_btree_node(path, l) && have != BTREE_NODE_UNLOCKED);
 770
 771                 BUG_ON(is_btree_node(path, l) &&
 772                        (want == BTREE_NODE_UNLOCKED ||
 773                         have != BTREE_NODE_WRITE_LOCKED) &&
 774                        want != have);
 775         }
 776 }
 777
 778 void bch2_trans_verify_locks(struct btree_trans *trans)
 779 {
 780         struct btree_path *path;
 781
 782         trans_for_each_path(trans, path)
 783                 bch2_btree_path_verify_locks(path);
 784 }
 785
 786 #endif