git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_locking.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "btree_locking.h"
   5 #include "btree_types.h"
   6
   7 static struct lock_class_key bch2_btree_node_lock_key;
   8
   9 void bch2_btree_lock_init(struct btree_bkey_cached_common *b,
  10                           enum six_lock_init_flags flags)
  11 {
  12         __six_lock_init(&b->lock, "b->c.lock", &bch2_btree_node_lock_key, flags);
  13         lockdep_set_novalidate_class(&b->lock);
  14 }
  15
  16 #ifdef CONFIG_LOCKDEP
  17 void bch2_assert_btree_nodes_not_locked(void)
  18 {
  19 #if 0
  20         //Re-enable when lock_class_is_held() is merged:
  21         BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key));
  22 #endif
  23 }
  24 #endif
  25
  26 /* Btree node locking: */
  27
  28 struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *trans,
  29                                                   struct btree_path *skip,
  30                                                   struct btree_bkey_cached_common *b,
  31                                                   unsigned level)
  32 {
  33         struct btree_path *path;
  34         struct six_lock_count ret;
  35
  36         memset(&ret, 0, sizeof(ret));
  37
  38         if (IS_ERR_OR_NULL(b))
  39                 return ret;
  40
  41         trans_for_each_path(trans, path)
  42                 if (path != skip && &path->l[level].b->c == b) {
  43                         int t = btree_node_locked_type(path, level);
  44
  45                         if (t != BTREE_NODE_UNLOCKED)
  46                                 ret.n[t]++;
  47                 }
  48
  49         return ret;
  50 }
  51
  52 /* unlock */
  53
  54 void bch2_btree_node_unlock_write(struct btree_trans *trans,
  55                         struct btree_path *path, struct btree *b)
  56 {
  57         bch2_btree_node_unlock_write_inlined(trans, path, b);
  58 }
  59
  60 /* lock */
  61
  62 /*
  63  * @trans wants to lock @b with type @type
  64  */
  65 struct trans_waiting_for_lock {
  66         struct btree_trans              *trans;
  67         struct btree_bkey_cached_common *node_want;
  68         enum six_lock_type              lock_want;
  69
  70         /* for iterating over held locks :*/
  71         u8                              path_idx;
  72         u8                              level;
  73         u64                             lock_start_time;
  74 };
  75
  76 struct lock_graph {
  77         struct trans_waiting_for_lock   g[8];
  78         unsigned                        nr;
  79 };
  80
  81 static noinline void print_cycle(struct printbuf *out, struct lock_graph *g)
  82 {
  83         struct trans_waiting_for_lock *i;
  84
  85         prt_printf(out, "Found lock cycle (%u entries):", g->nr);
  86         prt_newline(out);
  87
  88         for (i = g->g; i < g->g + g->nr; i++)
  89                 bch2_btree_trans_to_text(out, i->trans);
  90 }
  91
  92 static noinline void print_chain(struct printbuf *out, struct lock_graph *g)
  93 {
  94         struct trans_waiting_for_lock *i;
  95
  96         for (i = g->g; i != g->g + g->nr; i++) {
  97                 if (i != g->g)
  98                         prt_str(out, "<- ");
  99                 prt_printf(out, "%u ", i->trans->locking_wait.task->pid);
 100         }
 101         prt_newline(out);
 102 }
 103
 104 static void lock_graph_up(struct lock_graph *g)
 105 {
 106         closure_put(&g->g[--g->nr].trans->ref);
 107 }
 108
 109 static noinline void lock_graph_pop_all(struct lock_graph *g)
 110 {
 111         while (g->nr)
 112                 lock_graph_up(g);
 113 }
 114
 115 static void __lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
 116 {
 117         g->g[g->nr++] = (struct trans_waiting_for_lock) {
 118                 .trans          = trans,
 119                 .node_want      = trans->locking,
 120                 .lock_want      = trans->locking_wait.lock_want,
 121         };
 122 }
 123
 124 static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
 125 {
 126         closure_get(&trans->ref);
 127         __lock_graph_down(g, trans);
 128 }
 129
 130 static bool lock_graph_remove_non_waiters(struct lock_graph *g)
 131 {
 132         struct trans_waiting_for_lock *i;
 133
 134         for (i = g->g + 1; i < g->g + g->nr; i++)
 135                 if (i->trans->locking != i->node_want ||
 136                     i->trans->locking_wait.start_time != i[-1].lock_start_time) {
 137                         while (g->g + g->nr > i)
 138                                 lock_graph_up(g);
 139                         return true;
 140                 }
 141
 142         return false;
 143 }
 144
 145 static void trace_would_deadlock(struct lock_graph *g, struct btree_trans *trans,
 146                                  unsigned long ip)
 147 {
 148         struct bch_fs *c = trans->c;
 149
 150         count_event(c, trans_restart_would_deadlock);
 151
 152         if (trace_trans_restart_would_deadlock_enabled()) {
 153                 struct printbuf buf = PRINTBUF;
 154
 155                 buf.atomic++;
 156                 print_cycle(&buf, g);
 157
 158                 trace_trans_restart_would_deadlock(trans, ip, buf.buf);
 159                 printbuf_exit(&buf);
 160         }
 161 }
 162
 163 static int abort_lock(struct lock_graph *g, struct trans_waiting_for_lock *i)
 164 {
 165         if (i == g->g) {
 166                 trace_would_deadlock(g, i->trans, _RET_IP_);
 167                 return btree_trans_restart(i->trans, BCH_ERR_transaction_restart_would_deadlock);
 168         } else {
 169                 i->trans->lock_must_abort = true;
 170                 wake_up_process(i->trans->locking_wait.task);
 171                 return 0;
 172         }
 173 }
 174
 175 static int btree_trans_abort_preference(struct btree_trans *trans)
 176 {
 177         if (trans->lock_may_not_fail)
 178                 return 0;
 179         if (trans->locking_wait.lock_want == SIX_LOCK_write)
 180                 return 1;
 181         if (!trans->in_traverse_all)
 182                 return 2;
 183         return 3;
 184 }
 185
 186 static noinline int break_cycle(struct lock_graph *g, struct printbuf *cycle)
 187 {
 188         struct trans_waiting_for_lock *i, *abort = NULL;
 189         unsigned best = 0, pref;
 190         int ret;
 191
 192         if (lock_graph_remove_non_waiters(g))
 193                 return 0;
 194
 195         /* Only checking, for debugfs: */
 196         if (cycle) {
 197                 print_cycle(cycle, g);
 198                 ret = -1;
 199                 goto out;
 200         }
 201
 202         for (i = g->g; i < g->g + g->nr; i++) {
 203                 pref = btree_trans_abort_preference(i->trans);
 204                 if (pref > best) {
 205                         abort = i;
 206                         best = pref;
 207                 }
 208         }
 209
 210         if (unlikely(!best)) {
 211                 struct printbuf buf = PRINTBUF;
 212
 213                 prt_printf(&buf, bch2_fmt(g->g->trans->c, "cycle of nofail locks"));
 214
 215                 for (i = g->g; i < g->g + g->nr; i++) {
 216                         struct btree_trans *trans = i->trans;
 217
 218                         bch2_btree_trans_to_text(&buf, trans);
 219
 220                         prt_printf(&buf, "backtrace:");
 221                         prt_newline(&buf);
 222                         printbuf_indent_add(&buf, 2);
 223                         bch2_prt_task_backtrace(&buf, trans->locking_wait.task);
 224                         printbuf_indent_sub(&buf, 2);
 225                         prt_newline(&buf);
 226                 }
 227
 228                 bch2_print_string_as_lines(KERN_ERR, buf.buf);
 229                 printbuf_exit(&buf);
 230                 BUG();
 231         }
 232
 233         ret = abort_lock(g, abort);
 234 out:
 235         if (ret)
 236                 while (g->nr)
 237                         lock_graph_up(g);
 238         return ret;
 239 }
 240
 241 static int lock_graph_descend(struct lock_graph *g, struct btree_trans *trans,
 242                               struct printbuf *cycle)
 243 {
 244         struct btree_trans *orig_trans = g->g->trans;
 245         struct trans_waiting_for_lock *i;
 246
 247         for (i = g->g; i < g->g + g->nr; i++)
 248                 if (i->trans == trans) {
 249                         closure_put(&trans->ref);
 250                         return break_cycle(g, cycle);
 251                 }
 252
 253         if (g->nr == ARRAY_SIZE(g->g)) {
 254                 closure_put(&trans->ref);
 255
 256                 if (orig_trans->lock_may_not_fail)
 257                         return 0;
 258
 259                 while (g->nr)
 260                         lock_graph_up(g);
 261
 262                 if (cycle)
 263                         return 0;
 264
 265                 trace_and_count(trans->c, trans_restart_would_deadlock_recursion_limit, trans, _RET_IP_);
 266                 return btree_trans_restart(orig_trans, BCH_ERR_transaction_restart_deadlock_recursion_limit);
 267         }
 268
 269         __lock_graph_down(g, trans);
 270         return 0;
 271 }
 272
 273 static bool lock_type_conflicts(enum six_lock_type t1, enum six_lock_type t2)
 274 {
 275         return t1 + t2 > 1;
 276 }
 277
 278 int bch2_check_for_deadlock(struct btree_trans *trans, struct printbuf *cycle)
 279 {
 280         struct lock_graph g;
 281         struct trans_waiting_for_lock *top;
 282         struct btree_bkey_cached_common *b;
 283         struct btree_path *path;
 284         unsigned path_idx;
 285         int ret;
 286
 287         g.nr = 0;
 288
 289         if (trans->lock_must_abort) {
 290                 if (cycle)
 291                         return -1;
 292
 293                 trace_would_deadlock(&g, trans, _RET_IP_);
 294                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock);
 295         }
 296
 297         lock_graph_down(&g, trans);
 298 next:
 299         if (!g.nr)
 300                 return 0;
 301
 302         top = &g.g[g.nr - 1];
 303
 304         trans_for_each_path_safe_from(top->trans, path, path_idx, top->path_idx) {
 305                 if (!path->nodes_locked)
 306                         continue;
 307
 308                 if (path_idx != top->path_idx) {
 309                         top->path_idx           = path_idx;
 310                         top->level              = 0;
 311                         top->lock_start_time    = 0;
 312                 }
 313
 314                 for (;
 315                      top->level < BTREE_MAX_DEPTH;
 316                      top->level++, top->lock_start_time = 0) {
 317                         int lock_held = btree_node_locked_type(path, top->level);
 318
 319                         if (lock_held == BTREE_NODE_UNLOCKED)
 320                                 continue;
 321
 322                         b = &READ_ONCE(path->l[top->level].b)->c;
 323
 324                         if (IS_ERR_OR_NULL(b)) {
 325                                 /*
 326                                  * If we get here, it means we raced with the
 327                                  * other thread updating its btree_path
 328                                  * structures - which means it can't be blocked
 329                                  * waiting on a lock:
 330                                  */
 331                                 if (!lock_graph_remove_non_waiters(&g)) {
 332                                         /*
 333                                          * If lock_graph_remove_non_waiters()
 334                                          * didn't do anything, it must be
 335                                          * because we're being called by debugfs
 336                                          * checking for lock cycles, which
 337                                          * invokes us on btree_transactions that
 338                                          * aren't actually waiting on anything.
 339                                          * Just bail out:
 340                                          */
 341                                         lock_graph_pop_all(&g);
 342                                 }
 343
 344                                 goto next;
 345                         }
 346
 347                         if (list_empty_careful(&b->lock.wait_list))
 348                                 continue;
 349
 350                         raw_spin_lock(&b->lock.wait_lock);
 351                         list_for_each_entry(trans, &b->lock.wait_list, locking_wait.list) {
 352                                 BUG_ON(b != trans->locking);
 353
 354                                 if (top->lock_start_time &&
 355                                     time_after_eq64(top->lock_start_time, trans->locking_wait.start_time))
 356                                         continue;
 357
 358                                 top->lock_start_time = trans->locking_wait.start_time;
 359
 360                                 /* Don't check for self deadlock: */
 361                                 if (trans == top->trans ||
 362                                     !lock_type_conflicts(lock_held, trans->locking_wait.lock_want))
 363                                         continue;
 364
 365                                 closure_get(&trans->ref);
 366                                 raw_spin_unlock(&b->lock.wait_lock);
 367
 368                                 ret = lock_graph_descend(&g, trans, cycle);
 369                                 if (ret)
 370                                         return ret;
 371                                 goto next;
 372
 373                         }
 374                         raw_spin_unlock(&b->lock.wait_lock);
 375                 }
 376         }
 377
 378         if (g.nr > 1 && cycle)
 379                 print_chain(cycle, &g);
 380         lock_graph_up(&g);
 381         goto next;
 382 }
 383
 384 int bch2_six_check_for_deadlock(struct six_lock *lock, void *p)
 385 {
 386         struct btree_trans *trans = p;
 387
 388         return bch2_check_for_deadlock(trans, NULL);
 389 }
 390
 391 int __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree_path *path,
 392                                  struct btree_bkey_cached_common *b,
 393                                  bool lock_may_not_fail)
 394 {
 395         int readers = bch2_btree_node_lock_counts(trans, NULL, b, b->level).n[SIX_LOCK_read];
 396         int ret;
 397
 398         /*
 399          * Must drop our read locks before calling six_lock_write() -
 400          * six_unlock() won't do wakeups until the reader count
 401          * goes to 0, and it's safe because we have the node intent
 402          * locked:
 403          */
 404         six_lock_readers_add(&b->lock, -readers);
 405         ret = __btree_node_lock_nopath(trans, b, SIX_LOCK_write,
 406                                        lock_may_not_fail, _RET_IP_);
 407         six_lock_readers_add(&b->lock, readers);
 408
 409         if (ret)
 410                 mark_btree_node_locked_noreset(path, b->level, BTREE_NODE_INTENT_LOCKED);
 411
 412         return ret;
 413 }
 414
 415 void bch2_btree_node_lock_write_nofail(struct btree_trans *trans,
 416                                        struct btree_path *path,
 417                                        struct btree_bkey_cached_common *b)
 418 {
 419         struct btree_path *linked;
 420         unsigned i;
 421         int ret;
 422
 423         /*
 424          * XXX BIG FAT NOTICE
 425          *
 426          * Drop all read locks before taking a write lock:
 427          *
 428          * This is a hack, because bch2_btree_node_lock_write_nofail() is a
 429          * hack - but by dropping read locks first, this should never fail, and
 430          * we only use this in code paths where whatever read locks we've
 431          * already taken are no longer needed:
 432          */
 433
 434         trans_for_each_path(trans, linked) {
 435                 if (!linked->nodes_locked)
 436                         continue;
 437
 438                 for (i = 0; i < BTREE_MAX_DEPTH; i++)
 439                         if (btree_node_read_locked(linked, i)) {
 440                                 btree_node_unlock(trans, linked, i);
 441                                 btree_path_set_dirty(linked, BTREE_ITER_NEED_RELOCK);
 442                         }
 443         }
 444
 445         ret = __btree_node_lock_write(trans, path, b, true);
 446         BUG_ON(ret);
 447 }
 448
 449 /* relock */
 450
 451 static inline bool btree_path_get_locks(struct btree_trans *trans,
 452                                         struct btree_path *path,
 453                                         bool upgrade,
 454                                         struct get_locks_fail *f)
 455 {
 456         unsigned l = path->level;
 457         int fail_idx = -1;
 458
 459         do {
 460                 if (!btree_path_node(path, l))
 461                         break;
 462
 463                 if (!(upgrade
 464                       ? bch2_btree_node_upgrade(trans, path, l)
 465                       : bch2_btree_node_relock(trans, path, l))) {
 466                         fail_idx        = l;
 467
 468                         if (f) {
 469                                 f->l    = l;
 470                                 f->b    = path->l[l].b;
 471                         }
 472                 }
 473
 474                 l++;
 475         } while (l < path->locks_want);
 476
 477         /*
 478          * When we fail to get a lock, we have to ensure that any child nodes
 479          * can't be relocked so bch2_btree_path_traverse has to walk back up to
 480          * the node that we failed to relock:
 481          */
 482         if (fail_idx >= 0) {
 483                 __bch2_btree_path_unlock(trans, path);
 484                 btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
 485
 486                 do {
 487                         path->l[fail_idx].b = upgrade
 488                                 ? ERR_PTR(-BCH_ERR_no_btree_node_upgrade)
 489                                 : ERR_PTR(-BCH_ERR_no_btree_node_relock);
 490                         --fail_idx;
 491                 } while (fail_idx >= 0);
 492         }
 493
 494         if (path->uptodate == BTREE_ITER_NEED_RELOCK)
 495                 path->uptodate = BTREE_ITER_UPTODATE;
 496
 497         bch2_trans_verify_locks(trans);
 498
 499         return path->uptodate < BTREE_ITER_NEED_RELOCK;
 500 }
 501
 502 bool __bch2_btree_node_relock(struct btree_trans *trans,
 503                               struct btree_path *path, unsigned level,
 504                               bool trace)
 505 {
 506         struct btree *b = btree_path_node(path, level);
 507         int want = __btree_lock_want(path, level);
 508
 509         if (race_fault())
 510                 goto fail;
 511
 512         if (six_relock_type(&b->c.lock, want, path->l[level].lock_seq) ||
 513             (btree_node_lock_seq_matches(path, b, level) &&
 514              btree_node_lock_increment(trans, &b->c, level, want))) {
 515                 mark_btree_node_locked(trans, path, level, want);
 516                 return true;
 517         }
 518 fail:
 519         if (trace && !trans->notrace_relock_fail)
 520                 trace_and_count(trans->c, btree_path_relock_fail, trans, _RET_IP_, path, level);
 521         return false;
 522 }
 523
 524 /* upgrade */
 525
 526 bool bch2_btree_node_upgrade(struct btree_trans *trans,
 527                              struct btree_path *path, unsigned level)
 528 {
 529         struct btree *b = path->l[level].b;
 530         struct six_lock_count count = bch2_btree_node_lock_counts(trans, path, &b->c, level);
 531
 532         if (!is_btree_node(path, level))
 533                 return false;
 534
 535         switch (btree_lock_want(path, level)) {
 536         case BTREE_NODE_UNLOCKED:
 537                 BUG_ON(btree_node_locked(path, level));
 538                 return true;
 539         case BTREE_NODE_READ_LOCKED:
 540                 BUG_ON(btree_node_intent_locked(path, level));
 541                 return bch2_btree_node_relock(trans, path, level);
 542         case BTREE_NODE_INTENT_LOCKED:
 543                 break;
 544         case BTREE_NODE_WRITE_LOCKED:
 545                 BUG();
 546         }
 547
 548         if (btree_node_intent_locked(path, level))
 549                 return true;
 550
 551         if (race_fault())
 552                 return false;
 553
 554         if (btree_node_locked(path, level)) {
 555                 bool ret;
 556
 557                 six_lock_readers_add(&b->c.lock, -count.n[SIX_LOCK_read]);
 558                 ret = six_lock_tryupgrade(&b->c.lock);
 559                 six_lock_readers_add(&b->c.lock, count.n[SIX_LOCK_read]);
 560
 561                 if (ret)
 562                         goto success;
 563         } else {
 564                 if (six_relock_type(&b->c.lock, SIX_LOCK_intent, path->l[level].lock_seq))
 565                         goto success;
 566         }
 567
 568         /*
 569          * Do we already have an intent lock via another path? If so, just bump
 570          * lock count:
 571          */
 572         if (btree_node_lock_seq_matches(path, b, level) &&
 573             btree_node_lock_increment(trans, &b->c, level, BTREE_NODE_INTENT_LOCKED)) {
 574                 btree_node_unlock(trans, path, level);
 575                 goto success;
 576         }
 577
 578         trace_and_count(trans->c, btree_path_upgrade_fail, trans, _RET_IP_, path, level);
 579         return false;
 580 success:
 581         mark_btree_node_locked_noreset(path, level, BTREE_NODE_INTENT_LOCKED);
 582         return true;
 583 }
 584
 585 /* Btree path locking: */
 586
 587 /*
 588  * Only for btree_cache.c - only relocks intent locks
 589  */
 590 int bch2_btree_path_relock_intent(struct btree_trans *trans,
 591                                   struct btree_path *path)
 592 {
 593         unsigned l;
 594
 595         for (l = path->level;
 596              l < path->locks_want && btree_path_node(path, l);
 597              l++) {
 598                 if (!bch2_btree_node_relock(trans, path, l)) {
 599                         __bch2_btree_path_unlock(trans, path);
 600                         btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
 601                         trace_and_count(trans->c, trans_restart_relock_path_intent, trans, _RET_IP_, path);
 602                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path_intent);
 603                 }
 604         }
 605
 606         return 0;
 607 }
 608
 609 __flatten
 610 bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
 611                         struct btree_path *path, unsigned long trace_ip)
 612 {
 613         struct get_locks_fail f;
 614
 615         return btree_path_get_locks(trans, path, false, &f);
 616 }
 617
 618 int __bch2_btree_path_relock(struct btree_trans *trans,
 619                         struct btree_path *path, unsigned long trace_ip)
 620 {
 621         if (!bch2_btree_path_relock_norestart(trans, path, trace_ip)) {
 622                 trace_and_count(trans->c, trans_restart_relock_path, trans, trace_ip, path);
 623                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path);
 624         }
 625
 626         return 0;
 627 }
 628
 629 bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans,
 630                                struct btree_path *path,
 631                                unsigned new_locks_want,
 632                                struct get_locks_fail *f)
 633 {
 634         EBUG_ON(path->locks_want >= new_locks_want);
 635
 636         path->locks_want = new_locks_want;
 637
 638         return btree_path_get_locks(trans, path, true, f);
 639 }
 640
 641 bool __bch2_btree_path_upgrade(struct btree_trans *trans,
 642                                struct btree_path *path,
 643                                unsigned new_locks_want,
 644                                struct get_locks_fail *f)
 645 {
 646         struct btree_path *linked;
 647
 648         if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want, f))
 649                 return true;
 650
 651         /*
 652          * XXX: this is ugly - we'd prefer to not be mucking with other
 653          * iterators in the btree_trans here.
 654          *
 655          * On failure to upgrade the iterator, setting iter->locks_want and
 656          * calling get_locks() is sufficient to make bch2_btree_path_traverse()
 657          * get the locks we want on transaction restart.
 658          *
 659          * But if this iterator was a clone, on transaction restart what we did
 660          * to this iterator isn't going to be preserved.
 661          *
 662          * Possibly we could add an iterator field for the parent iterator when
 663          * an iterator is a copy - for now, we'll just upgrade any other
 664          * iterators with the same btree id.
 665          *
 666          * The code below used to be needed to ensure ancestor nodes get locked
 667          * before interior nodes - now that's handled by
 668          * bch2_btree_path_traverse_all().
 669          */
 670         if (!path->cached && !trans->in_traverse_all)
 671                 trans_for_each_path(trans, linked)
 672                         if (linked != path &&
 673                             linked->cached == path->cached &&
 674                             linked->btree_id == path->btree_id &&
 675                             linked->locks_want < new_locks_want) {
 676                                 linked->locks_want = new_locks_want;
 677                                 btree_path_get_locks(trans, linked, true, NULL);
 678                         }
 679
 680         return false;
 681 }
 682
 683 void __bch2_btree_path_downgrade(struct btree_trans *trans,
 684                                  struct btree_path *path,
 685                                  unsigned new_locks_want)
 686 {
 687         unsigned l, old_locks_want = path->locks_want;
 688
 689         if (trans->restarted)
 690                 return;
 691
 692         EBUG_ON(path->locks_want < new_locks_want);
 693
 694         path->locks_want = new_locks_want;
 695
 696         while (path->nodes_locked &&
 697                (l = btree_path_highest_level_locked(path)) >= path->locks_want) {
 698                 if (l > path->level) {
 699                         btree_node_unlock(trans, path, l);
 700                 } else {
 701                         if (btree_node_intent_locked(path, l)) {
 702                                 six_lock_downgrade(&path->l[l].b->c.lock);
 703                                 mark_btree_node_locked_noreset(path, l, BTREE_NODE_READ_LOCKED);
 704                         }
 705                         break;
 706                 }
 707         }
 708
 709         bch2_btree_path_verify_locks(path);
 710
 711         path->downgrade_seq++;
 712         trace_path_downgrade(trans, _RET_IP_, path, old_locks_want);
 713 }
 714
 715 /* Btree transaction locking: */
 716
 717 void bch2_trans_downgrade(struct btree_trans *trans)
 718 {
 719         struct btree_path *path;
 720
 721         if (trans->restarted)
 722                 return;
 723
 724         trans_for_each_path(trans, path)
 725                 bch2_btree_path_downgrade(trans, path);
 726 }
 727
 728 int bch2_trans_relock(struct btree_trans *trans)
 729 {
 730         struct btree_path *path;
 731
 732         if (unlikely(trans->restarted))
 733                 return -((int) trans->restarted);
 734
 735         trans_for_each_path(trans, path)
 736                 if (path->should_be_locked &&
 737                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
 738                         trace_and_count(trans->c, trans_restart_relock, trans, _RET_IP_, path);
 739                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
 740                 }
 741         return 0;
 742 }
 743
 744 int bch2_trans_relock_notrace(struct btree_trans *trans)
 745 {
 746         struct btree_path *path;
 747
 748         if (unlikely(trans->restarted))
 749                 return -((int) trans->restarted);
 750
 751         trans_for_each_path(trans, path)
 752                 if (path->should_be_locked &&
 753                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
 754                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
 755                 }
 756         return 0;
 757 }
 758
 759 void bch2_trans_unlock_noassert(struct btree_trans *trans)
 760 {
 761         struct btree_path *path;
 762
 763         trans_for_each_path(trans, path)
 764                 __bch2_btree_path_unlock(trans, path);
 765 }
 766
 767 void bch2_trans_unlock(struct btree_trans *trans)
 768 {
 769         struct btree_path *path;
 770
 771         trans_for_each_path(trans, path)
 772                 __bch2_btree_path_unlock(trans, path);
 773 }
 774
 775 void bch2_trans_unlock_long(struct btree_trans *trans)
 776 {
 777         bch2_trans_unlock(trans);
 778         bch2_trans_srcu_unlock(trans);
 779 }
 780
 781 bool bch2_trans_locked(struct btree_trans *trans)
 782 {
 783         struct btree_path *path;
 784
 785         trans_for_each_path(trans, path)
 786                 if (path->nodes_locked)
 787                         return true;
 788         return false;
 789 }
 790
 791 int __bch2_trans_mutex_lock(struct btree_trans *trans,
 792                             struct mutex *lock)
 793 {
 794         int ret = drop_locks_do(trans, (mutex_lock(lock), 0));
 795
 796         if (ret)
 797                 mutex_unlock(lock);
 798         return ret;
 799 }
 800
 801 /* Debug */
 802
 803 #ifdef CONFIG_BCACHEFS_DEBUG
 804
 805 void bch2_btree_path_verify_locks(struct btree_path *path)
 806 {
 807         unsigned l;
 808
 809         if (!path->nodes_locked) {
 810                 BUG_ON(path->uptodate == BTREE_ITER_UPTODATE &&
 811                        btree_path_node(path, path->level));
 812                 return;
 813         }
 814
 815         for (l = 0; l < BTREE_MAX_DEPTH; l++) {
 816                 int want = btree_lock_want(path, l);
 817                 int have = btree_node_locked_type(path, l);
 818
 819                 BUG_ON(!is_btree_node(path, l) && have != BTREE_NODE_UNLOCKED);
 820
 821                 BUG_ON(is_btree_node(path, l) &&
 822                        (want == BTREE_NODE_UNLOCKED ||
 823                         have != BTREE_NODE_WRITE_LOCKED) &&
 824                        want != have);
 825         }
 826 }
 827
 828 void bch2_trans_verify_locks(struct btree_trans *trans)
 829 {
 830         struct btree_path *path;
 831
 832         trans_for_each_path(trans, path)
 833                 bch2_btree_path_verify_locks(path);
 834 }
 835
 836 #endif