git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_locking.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "btree_locking.h"
   5 #include "btree_types.h"
   6
   7 static struct lock_class_key bch2_btree_node_lock_key;
   8
   9 void bch2_btree_lock_init(struct btree_bkey_cached_common *b,
  10                           enum six_lock_init_flags flags)
  11 {
  12         __six_lock_init(&b->lock, "b->c.lock", &bch2_btree_node_lock_key, flags);
  13         lockdep_set_novalidate_class(&b->lock);
  14 }
  15
  16 #ifdef CONFIG_LOCKDEP
  17 void bch2_assert_btree_nodes_not_locked(void)
  18 {
  19 #if 0
  20         //Re-enable when lock_class_is_held() is merged:
  21         BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key));
  22 #endif
  23 }
  24 #endif
  25
  26 /* Btree node locking: */
  27
  28 struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *trans,
  29                                                   struct btree_path *skip,
  30                                                   struct btree_bkey_cached_common *b,
  31                                                   unsigned level)
  32 {
  33         struct btree_path *path;
  34         struct six_lock_count ret;
  35         unsigned i;
  36
  37         memset(&ret, 0, sizeof(ret));
  38
  39         if (IS_ERR_OR_NULL(b))
  40                 return ret;
  41
  42         trans_for_each_path(trans, path, i)
  43                 if (path != skip && &path->l[level].b->c == b) {
  44                         int t = btree_node_locked_type(path, level);
  45
  46                         if (t != BTREE_NODE_UNLOCKED)
  47                                 ret.n[t]++;
  48                 }
  49
  50         return ret;
  51 }
  52
  53 /* unlock */
  54
  55 void bch2_btree_node_unlock_write(struct btree_trans *trans,
  56                         struct btree_path *path, struct btree *b)
  57 {
  58         bch2_btree_node_unlock_write_inlined(trans, path, b);
  59 }
  60
  61 /* lock */
  62
  63 /*
  64  * @trans wants to lock @b with type @type
  65  */
  66 struct trans_waiting_for_lock {
  67         struct btree_trans              *trans;
  68         struct btree_bkey_cached_common *node_want;
  69         enum six_lock_type              lock_want;
  70
  71         /* for iterating over held locks :*/
  72         u8                              path_idx;
  73         u8                              level;
  74         u64                             lock_start_time;
  75 };
  76
  77 struct lock_graph {
  78         struct trans_waiting_for_lock   g[8];
  79         unsigned                        nr;
  80 };
  81
  82 static noinline void print_cycle(struct printbuf *out, struct lock_graph *g)
  83 {
  84         struct trans_waiting_for_lock *i;
  85
  86         prt_printf(out, "Found lock cycle (%u entries):", g->nr);
  87         prt_newline(out);
  88
  89         for (i = g->g; i < g->g + g->nr; i++)
  90                 bch2_btree_trans_to_text(out, i->trans);
  91 }
  92
  93 static noinline void print_chain(struct printbuf *out, struct lock_graph *g)
  94 {
  95         struct trans_waiting_for_lock *i;
  96
  97         for (i = g->g; i != g->g + g->nr; i++) {
  98                 struct task_struct *task = i->trans->locking_wait.task;
  99                 if (i != g->g)
 100                         prt_str(out, "<- ");
 101                 prt_printf(out, "%u ", task ?task->pid : 0);
 102         }
 103         prt_newline(out);
 104 }
 105
 106 static void lock_graph_up(struct lock_graph *g)
 107 {
 108         closure_put(&g->g[--g->nr].trans->ref);
 109 }
 110
 111 static noinline void lock_graph_pop_all(struct lock_graph *g)
 112 {
 113         while (g->nr)
 114                 lock_graph_up(g);
 115 }
 116
 117 static void __lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
 118 {
 119         g->g[g->nr++] = (struct trans_waiting_for_lock) {
 120                 .trans          = trans,
 121                 .node_want      = trans->locking,
 122                 .lock_want      = trans->locking_wait.lock_want,
 123         };
 124 }
 125
 126 static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
 127 {
 128         closure_get(&trans->ref);
 129         __lock_graph_down(g, trans);
 130 }
 131
 132 static bool lock_graph_remove_non_waiters(struct lock_graph *g)
 133 {
 134         struct trans_waiting_for_lock *i;
 135
 136         for (i = g->g + 1; i < g->g + g->nr; i++)
 137                 if (i->trans->locking != i->node_want ||
 138                     i->trans->locking_wait.start_time != i[-1].lock_start_time) {
 139                         while (g->g + g->nr > i)
 140                                 lock_graph_up(g);
 141                         return true;
 142                 }
 143
 144         return false;
 145 }
 146
 147 static void trace_would_deadlock(struct lock_graph *g, struct btree_trans *trans,
 148                                  unsigned long ip)
 149 {
 150         struct bch_fs *c = trans->c;
 151
 152         count_event(c, trans_restart_would_deadlock);
 153
 154         if (trace_trans_restart_would_deadlock_enabled()) {
 155                 struct printbuf buf = PRINTBUF;
 156
 157                 buf.atomic++;
 158                 print_cycle(&buf, g);
 159
 160                 trace_trans_restart_would_deadlock(trans, ip, buf.buf);
 161                 printbuf_exit(&buf);
 162         }
 163 }
 164
 165 static int abort_lock(struct lock_graph *g, struct trans_waiting_for_lock *i)
 166 {
 167         if (i == g->g) {
 168                 trace_would_deadlock(g, i->trans, _RET_IP_);
 169                 return btree_trans_restart(i->trans, BCH_ERR_transaction_restart_would_deadlock);
 170         } else {
 171                 i->trans->lock_must_abort = true;
 172                 wake_up_process(i->trans->locking_wait.task);
 173                 return 0;
 174         }
 175 }
 176
 177 static int btree_trans_abort_preference(struct btree_trans *trans)
 178 {
 179         if (trans->lock_may_not_fail)
 180                 return 0;
 181         if (trans->locking_wait.lock_want == SIX_LOCK_write)
 182                 return 1;
 183         if (!trans->in_traverse_all)
 184                 return 2;
 185         return 3;
 186 }
 187
 188 static noinline int break_cycle(struct lock_graph *g, struct printbuf *cycle)
 189 {
 190         struct trans_waiting_for_lock *i, *abort = NULL;
 191         unsigned best = 0, pref;
 192         int ret;
 193
 194         if (lock_graph_remove_non_waiters(g))
 195                 return 0;
 196
 197         /* Only checking, for debugfs: */
 198         if (cycle) {
 199                 print_cycle(cycle, g);
 200                 ret = -1;
 201                 goto out;
 202         }
 203
 204         for (i = g->g; i < g->g + g->nr; i++) {
 205                 pref = btree_trans_abort_preference(i->trans);
 206                 if (pref > best) {
 207                         abort = i;
 208                         best = pref;
 209                 }
 210         }
 211
 212         if (unlikely(!best)) {
 213                 struct printbuf buf = PRINTBUF;
 214
 215                 prt_printf(&buf, bch2_fmt(g->g->trans->c, "cycle of nofail locks"));
 216
 217                 for (i = g->g; i < g->g + g->nr; i++) {
 218                         struct btree_trans *trans = i->trans;
 219
 220                         bch2_btree_trans_to_text(&buf, trans);
 221
 222                         prt_printf(&buf, "backtrace:");
 223                         prt_newline(&buf);
 224                         printbuf_indent_add(&buf, 2);
 225                         bch2_prt_task_backtrace(&buf, trans->locking_wait.task);
 226                         printbuf_indent_sub(&buf, 2);
 227                         prt_newline(&buf);
 228                 }
 229
 230                 bch2_print_string_as_lines(KERN_ERR, buf.buf);
 231                 printbuf_exit(&buf);
 232                 BUG();
 233         }
 234
 235         ret = abort_lock(g, abort);
 236 out:
 237         if (ret)
 238                 while (g->nr)
 239                         lock_graph_up(g);
 240         return ret;
 241 }
 242
 243 static int lock_graph_descend(struct lock_graph *g, struct btree_trans *trans,
 244                               struct printbuf *cycle)
 245 {
 246         struct btree_trans *orig_trans = g->g->trans;
 247         struct trans_waiting_for_lock *i;
 248
 249         for (i = g->g; i < g->g + g->nr; i++)
 250                 if (i->trans == trans) {
 251                         closure_put(&trans->ref);
 252                         return break_cycle(g, cycle);
 253                 }
 254
 255         if (g->nr == ARRAY_SIZE(g->g)) {
 256                 closure_put(&trans->ref);
 257
 258                 if (orig_trans->lock_may_not_fail)
 259                         return 0;
 260
 261                 while (g->nr)
 262                         lock_graph_up(g);
 263
 264                 if (cycle)
 265                         return 0;
 266
 267                 trace_and_count(trans->c, trans_restart_would_deadlock_recursion_limit, trans, _RET_IP_);
 268                 return btree_trans_restart(orig_trans, BCH_ERR_transaction_restart_deadlock_recursion_limit);
 269         }
 270
 271         __lock_graph_down(g, trans);
 272         return 0;
 273 }
 274
 275 static bool lock_type_conflicts(enum six_lock_type t1, enum six_lock_type t2)
 276 {
 277         return t1 + t2 > 1;
 278 }
 279
 280 int bch2_check_for_deadlock(struct btree_trans *trans, struct printbuf *cycle)
 281 {
 282         struct lock_graph g;
 283         struct trans_waiting_for_lock *top;
 284         struct btree_bkey_cached_common *b;
 285         btree_path_idx_t path_idx;
 286         int ret = 0;
 287
 288         g.nr = 0;
 289
 290         if (trans->lock_must_abort) {
 291                 if (cycle)
 292                         return -1;
 293
 294                 trace_would_deadlock(&g, trans, _RET_IP_);
 295                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock);
 296         }
 297
 298         lock_graph_down(&g, trans);
 299
 300         /* trans->paths is rcu protected vs. freeing */
 301         rcu_read_lock();
 302         if (cycle)
 303                 cycle->atomic++;
 304 next:
 305         if (!g.nr)
 306                 goto out;
 307
 308         top = &g.g[g.nr - 1];
 309
 310         struct btree_path *paths = rcu_dereference(top->trans->paths);
 311         if (!paths)
 312                 goto up;
 313
 314         unsigned long *paths_allocated = trans_paths_allocated(paths);
 315
 316         trans_for_each_path_idx_from(paths_allocated, *trans_paths_nr(paths),
 317                                      path_idx, top->path_idx) {
 318                 struct btree_path *path = paths + path_idx;
 319                 if (!path->nodes_locked)
 320                         continue;
 321
 322                 if (path_idx != top->path_idx) {
 323                         top->path_idx           = path_idx;
 324                         top->level              = 0;
 325                         top->lock_start_time    = 0;
 326                 }
 327
 328                 for (;
 329                      top->level < BTREE_MAX_DEPTH;
 330                      top->level++, top->lock_start_time = 0) {
 331                         int lock_held = btree_node_locked_type(path, top->level);
 332
 333                         if (lock_held == BTREE_NODE_UNLOCKED)
 334                                 continue;
 335
 336                         b = &READ_ONCE(path->l[top->level].b)->c;
 337
 338                         if (IS_ERR_OR_NULL(b)) {
 339                                 /*
 340                                  * If we get here, it means we raced with the
 341                                  * other thread updating its btree_path
 342                                  * structures - which means it can't be blocked
 343                                  * waiting on a lock:
 344                                  */
 345                                 if (!lock_graph_remove_non_waiters(&g)) {
 346                                         /*
 347                                          * If lock_graph_remove_non_waiters()
 348                                          * didn't do anything, it must be
 349                                          * because we're being called by debugfs
 350                                          * checking for lock cycles, which
 351                                          * invokes us on btree_transactions that
 352                                          * aren't actually waiting on anything.
 353                                          * Just bail out:
 354                                          */
 355                                         lock_graph_pop_all(&g);
 356                                 }
 357
 358                                 goto next;
 359                         }
 360
 361                         if (list_empty_careful(&b->lock.wait_list))
 362                                 continue;
 363
 364                         raw_spin_lock(&b->lock.wait_lock);
 365                         list_for_each_entry(trans, &b->lock.wait_list, locking_wait.list) {
 366                                 BUG_ON(b != trans->locking);
 367
 368                                 if (top->lock_start_time &&
 369                                     time_after_eq64(top->lock_start_time, trans->locking_wait.start_time))
 370                                         continue;
 371
 372                                 top->lock_start_time = trans->locking_wait.start_time;
 373
 374                                 /* Don't check for self deadlock: */
 375                                 if (trans == top->trans ||
 376                                     !lock_type_conflicts(lock_held, trans->locking_wait.lock_want))
 377                                         continue;
 378
 379                                 closure_get(&trans->ref);
 380                                 raw_spin_unlock(&b->lock.wait_lock);
 381
 382                                 ret = lock_graph_descend(&g, trans, cycle);
 383                                 if (ret)
 384                                         goto out;
 385                                 goto next;
 386
 387                         }
 388                         raw_spin_unlock(&b->lock.wait_lock);
 389                 }
 390         }
 391 up:
 392         if (g.nr > 1 && cycle)
 393                 print_chain(cycle, &g);
 394         lock_graph_up(&g);
 395         goto next;
 396 out:
 397         if (cycle)
 398                 --cycle->atomic;
 399         rcu_read_unlock();
 400         return ret;
 401 }
 402
 403 int bch2_six_check_for_deadlock(struct six_lock *lock, void *p)
 404 {
 405         struct btree_trans *trans = p;
 406
 407         return bch2_check_for_deadlock(trans, NULL);
 408 }
 409
 410 int __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree_path *path,
 411                                  struct btree_bkey_cached_common *b,
 412                                  bool lock_may_not_fail)
 413 {
 414         int readers = bch2_btree_node_lock_counts(trans, NULL, b, b->level).n[SIX_LOCK_read];
 415         int ret;
 416
 417         /*
 418          * Must drop our read locks before calling six_lock_write() -
 419          * six_unlock() won't do wakeups until the reader count
 420          * goes to 0, and it's safe because we have the node intent
 421          * locked:
 422          */
 423         six_lock_readers_add(&b->lock, -readers);
 424         ret = __btree_node_lock_nopath(trans, b, SIX_LOCK_write,
 425                                        lock_may_not_fail, _RET_IP_);
 426         six_lock_readers_add(&b->lock, readers);
 427
 428         if (ret)
 429                 mark_btree_node_locked_noreset(path, b->level, BTREE_NODE_INTENT_LOCKED);
 430
 431         return ret;
 432 }
 433
 434 void bch2_btree_node_lock_write_nofail(struct btree_trans *trans,
 435                                        struct btree_path *path,
 436                                        struct btree_bkey_cached_common *b)
 437 {
 438         struct btree_path *linked;
 439         unsigned i, iter;
 440         int ret;
 441
 442         /*
 443          * XXX BIG FAT NOTICE
 444          *
 445          * Drop all read locks before taking a write lock:
 446          *
 447          * This is a hack, because bch2_btree_node_lock_write_nofail() is a
 448          * hack - but by dropping read locks first, this should never fail, and
 449          * we only use this in code paths where whatever read locks we've
 450          * already taken are no longer needed:
 451          */
 452
 453         trans_for_each_path(trans, linked, iter) {
 454                 if (!linked->nodes_locked)
 455                         continue;
 456
 457                 for (i = 0; i < BTREE_MAX_DEPTH; i++)
 458                         if (btree_node_read_locked(linked, i)) {
 459                                 btree_node_unlock(trans, linked, i);
 460                                 btree_path_set_dirty(linked, BTREE_ITER_NEED_RELOCK);
 461                         }
 462         }
 463
 464         ret = __btree_node_lock_write(trans, path, b, true);
 465         BUG_ON(ret);
 466 }
 467
 468 /* relock */
 469
 470 static inline bool btree_path_get_locks(struct btree_trans *trans,
 471                                         struct btree_path *path,
 472                                         bool upgrade,
 473                                         struct get_locks_fail *f)
 474 {
 475         unsigned l = path->level;
 476         int fail_idx = -1;
 477
 478         do {
 479                 if (!btree_path_node(path, l))
 480                         break;
 481
 482                 if (!(upgrade
 483                       ? bch2_btree_node_upgrade(trans, path, l)
 484                       : bch2_btree_node_relock(trans, path, l))) {
 485                         fail_idx        = l;
 486
 487                         if (f) {
 488                                 f->l    = l;
 489                                 f->b    = path->l[l].b;
 490                         }
 491                 }
 492
 493                 l++;
 494         } while (l < path->locks_want);
 495
 496         /*
 497          * When we fail to get a lock, we have to ensure that any child nodes
 498          * can't be relocked so bch2_btree_path_traverse has to walk back up to
 499          * the node that we failed to relock:
 500          */
 501         if (fail_idx >= 0) {
 502                 __bch2_btree_path_unlock(trans, path);
 503                 btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
 504
 505                 do {
 506                         path->l[fail_idx].b = upgrade
 507                                 ? ERR_PTR(-BCH_ERR_no_btree_node_upgrade)
 508                                 : ERR_PTR(-BCH_ERR_no_btree_node_relock);
 509                         --fail_idx;
 510                 } while (fail_idx >= 0);
 511         }
 512
 513         if (path->uptodate == BTREE_ITER_NEED_RELOCK)
 514                 path->uptodate = BTREE_ITER_UPTODATE;
 515
 516         bch2_trans_verify_locks(trans);
 517
 518         return path->uptodate < BTREE_ITER_NEED_RELOCK;
 519 }
 520
 521 bool __bch2_btree_node_relock(struct btree_trans *trans,
 522                               struct btree_path *path, unsigned level,
 523                               bool trace)
 524 {
 525         struct btree *b = btree_path_node(path, level);
 526         int want = __btree_lock_want(path, level);
 527
 528         if (race_fault())
 529                 goto fail;
 530
 531         if (six_relock_type(&b->c.lock, want, path->l[level].lock_seq) ||
 532             (btree_node_lock_seq_matches(path, b, level) &&
 533              btree_node_lock_increment(trans, &b->c, level, want))) {
 534                 mark_btree_node_locked(trans, path, level, want);
 535                 return true;
 536         }
 537 fail:
 538         if (trace && !trans->notrace_relock_fail)
 539                 trace_and_count(trans->c, btree_path_relock_fail, trans, _RET_IP_, path, level);
 540         return false;
 541 }
 542
 543 /* upgrade */
 544
 545 bool bch2_btree_node_upgrade(struct btree_trans *trans,
 546                              struct btree_path *path, unsigned level)
 547 {
 548         struct btree *b = path->l[level].b;
 549         struct six_lock_count count = bch2_btree_node_lock_counts(trans, path, &b->c, level);
 550
 551         if (!is_btree_node(path, level))
 552                 return false;
 553
 554         switch (btree_lock_want(path, level)) {
 555         case BTREE_NODE_UNLOCKED:
 556                 BUG_ON(btree_node_locked(path, level));
 557                 return true;
 558         case BTREE_NODE_READ_LOCKED:
 559                 BUG_ON(btree_node_intent_locked(path, level));
 560                 return bch2_btree_node_relock(trans, path, level);
 561         case BTREE_NODE_INTENT_LOCKED:
 562                 break;
 563         case BTREE_NODE_WRITE_LOCKED:
 564                 BUG();
 565         }
 566
 567         if (btree_node_intent_locked(path, level))
 568                 return true;
 569
 570         if (race_fault())
 571                 return false;
 572
 573         if (btree_node_locked(path, level)) {
 574                 bool ret;
 575
 576                 six_lock_readers_add(&b->c.lock, -count.n[SIX_LOCK_read]);
 577                 ret = six_lock_tryupgrade(&b->c.lock);
 578                 six_lock_readers_add(&b->c.lock, count.n[SIX_LOCK_read]);
 579
 580                 if (ret)
 581                         goto success;
 582         } else {
 583                 if (six_relock_type(&b->c.lock, SIX_LOCK_intent, path->l[level].lock_seq))
 584                         goto success;
 585         }
 586
 587         /*
 588          * Do we already have an intent lock via another path? If so, just bump
 589          * lock count:
 590          */
 591         if (btree_node_lock_seq_matches(path, b, level) &&
 592             btree_node_lock_increment(trans, &b->c, level, BTREE_NODE_INTENT_LOCKED)) {
 593                 btree_node_unlock(trans, path, level);
 594                 goto success;
 595         }
 596
 597         trace_and_count(trans->c, btree_path_upgrade_fail, trans, _RET_IP_, path, level);
 598         return false;
 599 success:
 600         mark_btree_node_locked_noreset(path, level, BTREE_NODE_INTENT_LOCKED);
 601         return true;
 602 }
 603
 604 /* Btree path locking: */
 605
 606 /*
 607  * Only for btree_cache.c - only relocks intent locks
 608  */
 609 int bch2_btree_path_relock_intent(struct btree_trans *trans,
 610                                   struct btree_path *path)
 611 {
 612         unsigned l;
 613
 614         for (l = path->level;
 615              l < path->locks_want && btree_path_node(path, l);
 616              l++) {
 617                 if (!bch2_btree_node_relock(trans, path, l)) {
 618                         __bch2_btree_path_unlock(trans, path);
 619                         btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
 620                         trace_and_count(trans->c, trans_restart_relock_path_intent, trans, _RET_IP_, path);
 621                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path_intent);
 622                 }
 623         }
 624
 625         return 0;
 626 }
 627
 628 __flatten
 629 bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
 630                         struct btree_path *path, unsigned long trace_ip)
 631 {
 632         struct get_locks_fail f;
 633
 634         return btree_path_get_locks(trans, path, false, &f);
 635 }
 636
 637 int __bch2_btree_path_relock(struct btree_trans *trans,
 638                         struct btree_path *path, unsigned long trace_ip)
 639 {
 640         if (!bch2_btree_path_relock_norestart(trans, path, trace_ip)) {
 641                 trace_and_count(trans->c, trans_restart_relock_path, trans, trace_ip, path);
 642                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path);
 643         }
 644
 645         return 0;
 646 }
 647
 648 bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans,
 649                                struct btree_path *path,
 650                                unsigned new_locks_want,
 651                                struct get_locks_fail *f)
 652 {
 653         EBUG_ON(path->locks_want >= new_locks_want);
 654
 655         path->locks_want = new_locks_want;
 656
 657         return btree_path_get_locks(trans, path, true, f);
 658 }
 659
 660 bool __bch2_btree_path_upgrade(struct btree_trans *trans,
 661                                struct btree_path *path,
 662                                unsigned new_locks_want,
 663                                struct get_locks_fail *f)
 664 {
 665         if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want, f))
 666                 return true;
 667
 668         /*
 669          * XXX: this is ugly - we'd prefer to not be mucking with other
 670          * iterators in the btree_trans here.
 671          *
 672          * On failure to upgrade the iterator, setting iter->locks_want and
 673          * calling get_locks() is sufficient to make bch2_btree_path_traverse()
 674          * get the locks we want on transaction restart.
 675          *
 676          * But if this iterator was a clone, on transaction restart what we did
 677          * to this iterator isn't going to be preserved.
 678          *
 679          * Possibly we could add an iterator field for the parent iterator when
 680          * an iterator is a copy - for now, we'll just upgrade any other
 681          * iterators with the same btree id.
 682          *
 683          * The code below used to be needed to ensure ancestor nodes get locked
 684          * before interior nodes - now that's handled by
 685          * bch2_btree_path_traverse_all().
 686          */
 687         if (!path->cached && !trans->in_traverse_all) {
 688                 struct btree_path *linked;
 689                 unsigned i;
 690
 691                 trans_for_each_path(trans, linked, i)
 692                         if (linked != path &&
 693                             linked->cached == path->cached &&
 694                             linked->btree_id == path->btree_id &&
 695                             linked->locks_want < new_locks_want) {
 696                                 linked->locks_want = new_locks_want;
 697                                 btree_path_get_locks(trans, linked, true, NULL);
 698                         }
 699         }
 700
 701         return false;
 702 }
 703
 704 void __bch2_btree_path_downgrade(struct btree_trans *trans,
 705                                  struct btree_path *path,
 706                                  unsigned new_locks_want)
 707 {
 708         unsigned l, old_locks_want = path->locks_want;
 709
 710         if (trans->restarted)
 711                 return;
 712
 713         EBUG_ON(path->locks_want < new_locks_want);
 714
 715         path->locks_want = new_locks_want;
 716
 717         while (path->nodes_locked &&
 718                (l = btree_path_highest_level_locked(path)) >= path->locks_want) {
 719                 if (l > path->level) {
 720                         btree_node_unlock(trans, path, l);
 721                 } else {
 722                         if (btree_node_intent_locked(path, l)) {
 723                                 six_lock_downgrade(&path->l[l].b->c.lock);
 724                                 mark_btree_node_locked_noreset(path, l, BTREE_NODE_READ_LOCKED);
 725                         }
 726                         break;
 727                 }
 728         }
 729
 730         bch2_btree_path_verify_locks(path);
 731
 732         trace_path_downgrade(trans, _RET_IP_, path, old_locks_want);
 733 }
 734
 735 /* Btree transaction locking: */
 736
 737 void bch2_trans_downgrade(struct btree_trans *trans)
 738 {
 739         struct btree_path *path;
 740         unsigned i;
 741
 742         if (trans->restarted)
 743                 return;
 744
 745         trans_for_each_path(trans, path, i)
 746                 bch2_btree_path_downgrade(trans, path);
 747 }
 748
 749 int bch2_trans_relock(struct btree_trans *trans)
 750 {
 751         struct btree_path *path;
 752         unsigned i;
 753
 754         if (unlikely(trans->restarted))
 755                 return -((int) trans->restarted);
 756
 757         trans_for_each_path(trans, path, i)
 758                 if (path->should_be_locked &&
 759                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
 760                         trace_and_count(trans->c, trans_restart_relock, trans, _RET_IP_, path);
 761                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
 762                 }
 763         return 0;
 764 }
 765
 766 int bch2_trans_relock_notrace(struct btree_trans *trans)
 767 {
 768         struct btree_path *path;
 769         unsigned i;
 770
 771         if (unlikely(trans->restarted))
 772                 return -((int) trans->restarted);
 773
 774         trans_for_each_path(trans, path, i)
 775                 if (path->should_be_locked &&
 776                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
 777                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
 778                 }
 779         return 0;
 780 }
 781
 782 void bch2_trans_unlock_noassert(struct btree_trans *trans)
 783 {
 784         struct btree_path *path;
 785         unsigned i;
 786
 787         trans_for_each_path(trans, path, i)
 788                 __bch2_btree_path_unlock(trans, path);
 789 }
 790
 791 void bch2_trans_unlock(struct btree_trans *trans)
 792 {
 793         struct btree_path *path;
 794         unsigned i;
 795
 796         trans_for_each_path(trans, path, i)
 797                 __bch2_btree_path_unlock(trans, path);
 798 }
 799
 800 void bch2_trans_unlock_long(struct btree_trans *trans)
 801 {
 802         bch2_trans_unlock(trans);
 803         bch2_trans_srcu_unlock(trans);
 804 }
 805
 806 bool bch2_trans_locked(struct btree_trans *trans)
 807 {
 808         struct btree_path *path;
 809         unsigned i;
 810
 811         trans_for_each_path(trans, path, i)
 812                 if (path->nodes_locked)
 813                         return true;
 814         return false;
 815 }
 816
 817 int __bch2_trans_mutex_lock(struct btree_trans *trans,
 818                             struct mutex *lock)
 819 {
 820         int ret = drop_locks_do(trans, (mutex_lock(lock), 0));
 821
 822         if (ret)
 823                 mutex_unlock(lock);
 824         return ret;
 825 }
 826
 827 /* Debug */
 828
 829 #ifdef CONFIG_BCACHEFS_DEBUG
 830
 831 void bch2_btree_path_verify_locks(struct btree_path *path)
 832 {
 833         unsigned l;
 834
 835         if (!path->nodes_locked) {
 836                 BUG_ON(path->uptodate == BTREE_ITER_UPTODATE &&
 837                        btree_path_node(path, path->level));
 838                 return;
 839         }
 840
 841         for (l = 0; l < BTREE_MAX_DEPTH; l++) {
 842                 int want = btree_lock_want(path, l);
 843                 int have = btree_node_locked_type(path, l);
 844
 845                 BUG_ON(!is_btree_node(path, l) && have != BTREE_NODE_UNLOCKED);
 846
 847                 BUG_ON(is_btree_node(path, l) &&
 848                        (want == BTREE_NODE_UNLOCKED ||
 849                         have != BTREE_NODE_WRITE_LOCKED) &&
 850                        want != have);
 851         }
 852 }
 853
 854 void bch2_trans_verify_locks(struct btree_trans *trans)
 855 {
 856         struct btree_path *path;
 857         unsigned i;
 858
 859         trans_for_each_path(trans, path, i)
 860                 bch2_btree_path_verify_locks(path);
 861 }
 862
 863 #endif