]> git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_locking.c
Update bcachefs sources to eb83f1f842bb mean and variance: Promote to lib/math
[bcachefs-tools-debian] / libbcachefs / btree_locking.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "bcachefs.h"
4 #include "btree_locking.h"
5 #include "btree_types.h"
6
7 static struct lock_class_key bch2_btree_node_lock_key;
8
9 void bch2_btree_lock_init(struct btree_bkey_cached_common *b,
10                           enum six_lock_init_flags flags)
11 {
12         __six_lock_init(&b->lock, "b->c.lock", &bch2_btree_node_lock_key, flags);
13 #ifdef CONFIG_DEBUG_LOCK_ALLOC
14         lockdep_set_no_check_recursion(&b->lock.dep_map);
15 #endif
16 }
17
18 #ifdef CONFIG_LOCKDEP
19 void bch2_assert_btree_nodes_not_locked(void)
20 {
21         BUG_ON(lock_class_is_held(&bch2_btree_node_lock_key));
22 }
23 #endif
24
25 /* Btree node locking: */
26
27 struct six_lock_count bch2_btree_node_lock_counts(struct btree_trans *trans,
28                                                   struct btree_path *skip,
29                                                   struct btree_bkey_cached_common *b,
30                                                   unsigned level)
31 {
32         struct btree_path *path;
33         struct six_lock_count ret;
34
35         memset(&ret, 0, sizeof(ret));
36
37         if (IS_ERR_OR_NULL(b))
38                 return ret;
39
40         trans_for_each_path(trans, path)
41                 if (path != skip && &path->l[level].b->c == b) {
42                         int t = btree_node_locked_type(path, level);
43
44                         if (t != BTREE_NODE_UNLOCKED)
45                                 ret.n[t]++;
46                 }
47
48         return ret;
49 }
50
51 /* unlock */
52
53 void bch2_btree_node_unlock_write(struct btree_trans *trans,
54                         struct btree_path *path, struct btree *b)
55 {
56         bch2_btree_node_unlock_write_inlined(trans, path, b);
57 }
58
59 /* lock */
60
61 /*
62  * @trans wants to lock @b with type @type
63  */
64 struct trans_waiting_for_lock {
65         struct btree_trans              *trans;
66         struct btree_bkey_cached_common *node_want;
67         enum six_lock_type              lock_want;
68
69         /* for iterating over held locks :*/
70         u8                              path_idx;
71         u8                              level;
72         u64                             lock_start_time;
73 };
74
75 struct lock_graph {
76         struct trans_waiting_for_lock   g[8];
77         unsigned                        nr;
78 };
79
80 static noinline void print_cycle(struct printbuf *out, struct lock_graph *g)
81 {
82         struct trans_waiting_for_lock *i;
83
84         prt_printf(out, "Found lock cycle (%u entries):", g->nr);
85         prt_newline(out);
86
87         for (i = g->g; i < g->g + g->nr; i++)
88                 bch2_btree_trans_to_text(out, i->trans);
89 }
90
91 static noinline void print_chain(struct printbuf *out, struct lock_graph *g)
92 {
93         struct trans_waiting_for_lock *i;
94
95         for (i = g->g; i != g->g + g->nr; i++) {
96                 if (i != g->g)
97                         prt_str(out, "<- ");
98                 prt_printf(out, "%u ", i->trans->locking_wait.task->pid);
99         }
100         prt_newline(out);
101 }
102
103 static void lock_graph_up(struct lock_graph *g)
104 {
105         closure_put(&g->g[--g->nr].trans->ref);
106 }
107
108 static noinline void lock_graph_pop_all(struct lock_graph *g)
109 {
110         while (g->nr)
111                 lock_graph_up(g);
112 }
113
114 static void __lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
115 {
116         g->g[g->nr++] = (struct trans_waiting_for_lock) {
117                 .trans          = trans,
118                 .node_want      = trans->locking,
119                 .lock_want      = trans->locking_wait.lock_want,
120         };
121 }
122
123 static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
124 {
125         closure_get(&trans->ref);
126         __lock_graph_down(g, trans);
127 }
128
129 static bool lock_graph_remove_non_waiters(struct lock_graph *g)
130 {
131         struct trans_waiting_for_lock *i;
132
133         for (i = g->g + 1; i < g->g + g->nr; i++)
134                 if (i->trans->locking != i->node_want ||
135                     i->trans->locking_wait.start_time != i[-1].lock_start_time) {
136                         while (g->g + g->nr > i)
137                                 lock_graph_up(g);
138                         return true;
139                 }
140
141         return false;
142 }
143
144 static void trace_would_deadlock(struct lock_graph *g, struct btree_trans *trans,
145                                  unsigned long ip)
146 {
147         struct bch_fs *c = trans->c;
148
149         count_event(c, trans_restart_would_deadlock);
150
151         if (trace_trans_restart_would_deadlock_enabled()) {
152                 struct printbuf buf = PRINTBUF;
153
154                 buf.atomic++;
155                 print_cycle(&buf, g);
156
157                 trace_trans_restart_would_deadlock(trans, ip, buf.buf);
158                 printbuf_exit(&buf);
159         }
160 }
161
162 static int abort_lock(struct lock_graph *g, struct trans_waiting_for_lock *i)
163 {
164         if (i == g->g) {
165                 trace_would_deadlock(g, i->trans, _RET_IP_);
166                 return btree_trans_restart(i->trans, BCH_ERR_transaction_restart_would_deadlock);
167         } else {
168                 i->trans->lock_must_abort = true;
169                 wake_up_process(i->trans->locking_wait.task);
170                 return 0;
171         }
172 }
173
174 static int btree_trans_abort_preference(struct btree_trans *trans)
175 {
176         if (trans->lock_may_not_fail)
177                 return 0;
178         if (trans->locking_wait.lock_want == SIX_LOCK_write)
179                 return 1;
180         if (!trans->in_traverse_all)
181                 return 2;
182         return 3;
183 }
184
185 static noinline int break_cycle(struct lock_graph *g, struct printbuf *cycle)
186 {
187         struct trans_waiting_for_lock *i, *abort = NULL;
188         unsigned best = 0, pref;
189         int ret;
190
191         if (lock_graph_remove_non_waiters(g))
192                 return 0;
193
194         /* Only checking, for debugfs: */
195         if (cycle) {
196                 print_cycle(cycle, g);
197                 ret = -1;
198                 goto out;
199         }
200
201         for (i = g->g; i < g->g + g->nr; i++) {
202                 pref = btree_trans_abort_preference(i->trans);
203                 if (pref > best) {
204                         abort = i;
205                         best = pref;
206                 }
207         }
208
209         if (unlikely(!best)) {
210                 struct printbuf buf = PRINTBUF;
211
212                 prt_printf(&buf, bch2_fmt(g->g->trans->c, "cycle of nofail locks"));
213
214                 for (i = g->g; i < g->g + g->nr; i++) {
215                         struct btree_trans *trans = i->trans;
216
217                         bch2_btree_trans_to_text(&buf, trans);
218
219                         prt_printf(&buf, "backtrace:");
220                         prt_newline(&buf);
221                         printbuf_indent_add(&buf, 2);
222                         bch2_prt_task_backtrace(&buf, trans->locking_wait.task);
223                         printbuf_indent_sub(&buf, 2);
224                         prt_newline(&buf);
225                 }
226
227                 bch2_print_string_as_lines(KERN_ERR, buf.buf);
228                 printbuf_exit(&buf);
229                 BUG();
230         }
231
232         ret = abort_lock(g, abort);
233 out:
234         if (ret)
235                 while (g->nr)
236                         lock_graph_up(g);
237         return ret;
238 }
239
240 static int lock_graph_descend(struct lock_graph *g, struct btree_trans *trans,
241                               struct printbuf *cycle)
242 {
243         struct btree_trans *orig_trans = g->g->trans;
244         struct trans_waiting_for_lock *i;
245
246         for (i = g->g; i < g->g + g->nr; i++)
247                 if (i->trans == trans) {
248                         closure_put(&trans->ref);
249                         return break_cycle(g, cycle);
250                 }
251
252         if (g->nr == ARRAY_SIZE(g->g)) {
253                 closure_put(&trans->ref);
254
255                 if (orig_trans->lock_may_not_fail)
256                         return 0;
257
258                 while (g->nr)
259                         lock_graph_up(g);
260
261                 if (cycle)
262                         return 0;
263
264                 trace_and_count(trans->c, trans_restart_would_deadlock_recursion_limit, trans, _RET_IP_);
265                 return btree_trans_restart(orig_trans, BCH_ERR_transaction_restart_deadlock_recursion_limit);
266         }
267
268         __lock_graph_down(g, trans);
269         return 0;
270 }
271
272 static bool lock_type_conflicts(enum six_lock_type t1, enum six_lock_type t2)
273 {
274         return t1 + t2 > 1;
275 }
276
277 int bch2_check_for_deadlock(struct btree_trans *trans, struct printbuf *cycle)
278 {
279         struct lock_graph g;
280         struct trans_waiting_for_lock *top;
281         struct btree_bkey_cached_common *b;
282         struct btree_path *path;
283         unsigned path_idx;
284         int ret;
285
286         g.nr = 0;
287
288         if (trans->lock_must_abort) {
289                 if (cycle)
290                         return -1;
291
292                 trace_would_deadlock(&g, trans, _RET_IP_);
293                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock);
294         }
295
296         lock_graph_down(&g, trans);
297 next:
298         if (!g.nr)
299                 return 0;
300
301         top = &g.g[g.nr - 1];
302
303         trans_for_each_path_safe_from(top->trans, path, path_idx, top->path_idx) {
304                 if (!path->nodes_locked)
305                         continue;
306
307                 if (path_idx != top->path_idx) {
308                         top->path_idx           = path_idx;
309                         top->level              = 0;
310                         top->lock_start_time    = 0;
311                 }
312
313                 for (;
314                      top->level < BTREE_MAX_DEPTH;
315                      top->level++, top->lock_start_time = 0) {
316                         int lock_held = btree_node_locked_type(path, top->level);
317
318                         if (lock_held == BTREE_NODE_UNLOCKED)
319                                 continue;
320
321                         b = &READ_ONCE(path->l[top->level].b)->c;
322
323                         if (IS_ERR_OR_NULL(b)) {
324                                 /*
325                                  * If we get here, it means we raced with the
326                                  * other thread updating its btree_path
327                                  * structures - which means it can't be blocked
328                                  * waiting on a lock:
329                                  */
330                                 if (!lock_graph_remove_non_waiters(&g)) {
331                                         /*
332                                          * If lock_graph_remove_non_waiters()
333                                          * didn't do anything, it must be
334                                          * because we're being called by debugfs
335                                          * checking for lock cycles, which
336                                          * invokes us on btree_transactions that
337                                          * aren't actually waiting on anything.
338                                          * Just bail out:
339                                          */
340                                         lock_graph_pop_all(&g);
341                                 }
342
343                                 goto next;
344                         }
345
346                         if (list_empty_careful(&b->lock.wait_list))
347                                 continue;
348
349                         raw_spin_lock(&b->lock.wait_lock);
350                         list_for_each_entry(trans, &b->lock.wait_list, locking_wait.list) {
351                                 BUG_ON(b != trans->locking);
352
353                                 if (top->lock_start_time &&
354                                     time_after_eq64(top->lock_start_time, trans->locking_wait.start_time))
355                                         continue;
356
357                                 top->lock_start_time = trans->locking_wait.start_time;
358
359                                 /* Don't check for self deadlock: */
360                                 if (trans == top->trans ||
361                                     !lock_type_conflicts(lock_held, trans->locking_wait.lock_want))
362                                         continue;
363
364                                 closure_get(&trans->ref);
365                                 raw_spin_unlock(&b->lock.wait_lock);
366
367                                 ret = lock_graph_descend(&g, trans, cycle);
368                                 if (ret)
369                                         return ret;
370                                 goto next;
371
372                         }
373                         raw_spin_unlock(&b->lock.wait_lock);
374                 }
375         }
376
377         if (g.nr > 1 && cycle)
378                 print_chain(cycle, &g);
379         lock_graph_up(&g);
380         goto next;
381 }
382
383 int bch2_six_check_for_deadlock(struct six_lock *lock, void *p)
384 {
385         struct btree_trans *trans = p;
386
387         return bch2_check_for_deadlock(trans, NULL);
388 }
389
390 int __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree_path *path,
391                                  struct btree_bkey_cached_common *b,
392                                  bool lock_may_not_fail)
393 {
394         int readers = bch2_btree_node_lock_counts(trans, NULL, b, b->level).n[SIX_LOCK_read];
395         int ret;
396
397         /*
398          * Must drop our read locks before calling six_lock_write() -
399          * six_unlock() won't do wakeups until the reader count
400          * goes to 0, and it's safe because we have the node intent
401          * locked:
402          */
403         six_lock_readers_add(&b->lock, -readers);
404         ret = __btree_node_lock_nopath(trans, b, SIX_LOCK_write,
405                                        lock_may_not_fail, _RET_IP_);
406         six_lock_readers_add(&b->lock, readers);
407
408         if (ret)
409                 mark_btree_node_locked_noreset(path, b->level, BTREE_NODE_INTENT_LOCKED);
410
411         return ret;
412 }
413
414 void bch2_btree_node_lock_write_nofail(struct btree_trans *trans,
415                                        struct btree_path *path,
416                                        struct btree_bkey_cached_common *b)
417 {
418         struct btree_path *linked;
419         unsigned i;
420         int ret;
421
422         /*
423          * XXX BIG FAT NOTICE
424          *
425          * Drop all read locks before taking a write lock:
426          *
427          * This is a hack, because bch2_btree_node_lock_write_nofail() is a
428          * hack - but by dropping read locks first, this should never fail, and
429          * we only use this in code paths where whatever read locks we've
430          * already taken are no longer needed:
431          */
432
433         trans_for_each_path(trans, linked) {
434                 if (!linked->nodes_locked)
435                         continue;
436
437                 for (i = 0; i < BTREE_MAX_DEPTH; i++)
438                         if (btree_node_read_locked(linked, i)) {
439                                 btree_node_unlock(trans, linked, i);
440                                 btree_path_set_dirty(linked, BTREE_ITER_NEED_RELOCK);
441                         }
442         }
443
444         ret = __btree_node_lock_write(trans, path, b, true);
445         BUG_ON(ret);
446 }
447
448 /* relock */
449
450 static inline bool btree_path_get_locks(struct btree_trans *trans,
451                                         struct btree_path *path,
452                                         bool upgrade,
453                                         struct get_locks_fail *f)
454 {
455         unsigned l = path->level;
456         int fail_idx = -1;
457
458         do {
459                 if (!btree_path_node(path, l))
460                         break;
461
462                 if (!(upgrade
463                       ? bch2_btree_node_upgrade(trans, path, l)
464                       : bch2_btree_node_relock(trans, path, l))) {
465                         fail_idx        = l;
466
467                         if (f) {
468                                 f->l    = l;
469                                 f->b    = path->l[l].b;
470                         }
471                 }
472
473                 l++;
474         } while (l < path->locks_want);
475
476         /*
477          * When we fail to get a lock, we have to ensure that any child nodes
478          * can't be relocked so bch2_btree_path_traverse has to walk back up to
479          * the node that we failed to relock:
480          */
481         if (fail_idx >= 0) {
482                 __bch2_btree_path_unlock(trans, path);
483                 btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
484
485                 do {
486                         path->l[fail_idx].b = upgrade
487                                 ? ERR_PTR(-BCH_ERR_no_btree_node_upgrade)
488                                 : ERR_PTR(-BCH_ERR_no_btree_node_relock);
489                         --fail_idx;
490                 } while (fail_idx >= 0);
491         }
492
493         if (path->uptodate == BTREE_ITER_NEED_RELOCK)
494                 path->uptodate = BTREE_ITER_UPTODATE;
495
496         bch2_trans_verify_locks(trans);
497
498         return path->uptodate < BTREE_ITER_NEED_RELOCK;
499 }
500
501 bool __bch2_btree_node_relock(struct btree_trans *trans,
502                               struct btree_path *path, unsigned level,
503                               bool trace)
504 {
505         struct btree *b = btree_path_node(path, level);
506         int want = __btree_lock_want(path, level);
507
508         if (race_fault())
509                 goto fail;
510
511         if (six_relock_type(&b->c.lock, want, path->l[level].lock_seq) ||
512             (btree_node_lock_seq_matches(path, b, level) &&
513              btree_node_lock_increment(trans, &b->c, level, want))) {
514                 mark_btree_node_locked(trans, path, level, want);
515                 return true;
516         }
517 fail:
518         if (trace && !trans->notrace_relock_fail)
519                 trace_and_count(trans->c, btree_path_relock_fail, trans, _RET_IP_, path, level);
520         return false;
521 }
522
523 /* upgrade */
524
525 bool bch2_btree_node_upgrade(struct btree_trans *trans,
526                              struct btree_path *path, unsigned level)
527 {
528         struct btree *b = path->l[level].b;
529         struct six_lock_count count = bch2_btree_node_lock_counts(trans, path, &b->c, level);
530
531         if (!is_btree_node(path, level))
532                 return false;
533
534         switch (btree_lock_want(path, level)) {
535         case BTREE_NODE_UNLOCKED:
536                 BUG_ON(btree_node_locked(path, level));
537                 return true;
538         case BTREE_NODE_READ_LOCKED:
539                 BUG_ON(btree_node_intent_locked(path, level));
540                 return bch2_btree_node_relock(trans, path, level);
541         case BTREE_NODE_INTENT_LOCKED:
542                 break;
543         case BTREE_NODE_WRITE_LOCKED:
544                 BUG();
545         }
546
547         if (btree_node_intent_locked(path, level))
548                 return true;
549
550         if (race_fault())
551                 return false;
552
553         if (btree_node_locked(path, level)) {
554                 bool ret;
555
556                 six_lock_readers_add(&b->c.lock, -count.n[SIX_LOCK_read]);
557                 ret = six_lock_tryupgrade(&b->c.lock);
558                 six_lock_readers_add(&b->c.lock, count.n[SIX_LOCK_read]);
559
560                 if (ret)
561                         goto success;
562         } else {
563                 if (six_relock_type(&b->c.lock, SIX_LOCK_intent, path->l[level].lock_seq))
564                         goto success;
565         }
566
567         /*
568          * Do we already have an intent lock via another path? If so, just bump
569          * lock count:
570          */
571         if (btree_node_lock_seq_matches(path, b, level) &&
572             btree_node_lock_increment(trans, &b->c, level, BTREE_NODE_INTENT_LOCKED)) {
573                 btree_node_unlock(trans, path, level);
574                 goto success;
575         }
576
577         trace_and_count(trans->c, btree_path_upgrade_fail, trans, _RET_IP_, path, level);
578         return false;
579 success:
580         mark_btree_node_locked_noreset(path, level, BTREE_NODE_INTENT_LOCKED);
581         return true;
582 }
583
584 /* Btree path locking: */
585
586 /*
587  * Only for btree_cache.c - only relocks intent locks
588  */
589 int bch2_btree_path_relock_intent(struct btree_trans *trans,
590                                   struct btree_path *path)
591 {
592         unsigned l;
593
594         for (l = path->level;
595              l < path->locks_want && btree_path_node(path, l);
596              l++) {
597                 if (!bch2_btree_node_relock(trans, path, l)) {
598                         __bch2_btree_path_unlock(trans, path);
599                         btree_path_set_dirty(path, BTREE_ITER_NEED_TRAVERSE);
600                         trace_and_count(trans->c, trans_restart_relock_path_intent, trans, _RET_IP_, path);
601                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path_intent);
602                 }
603         }
604
605         return 0;
606 }
607
608 __flatten
609 bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
610                         struct btree_path *path, unsigned long trace_ip)
611 {
612         struct get_locks_fail f;
613
614         return btree_path_get_locks(trans, path, false, &f);
615 }
616
617 int __bch2_btree_path_relock(struct btree_trans *trans,
618                         struct btree_path *path, unsigned long trace_ip)
619 {
620         if (!bch2_btree_path_relock_norestart(trans, path, trace_ip)) {
621                 trace_and_count(trans->c, trans_restart_relock_path, trans, trace_ip, path);
622                 return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock_path);
623         }
624
625         return 0;
626 }
627
628 bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans,
629                                struct btree_path *path,
630                                unsigned new_locks_want,
631                                struct get_locks_fail *f)
632 {
633         EBUG_ON(path->locks_want >= new_locks_want);
634
635         path->locks_want = new_locks_want;
636
637         return btree_path_get_locks(trans, path, true, f);
638 }
639
640 bool __bch2_btree_path_upgrade(struct btree_trans *trans,
641                                struct btree_path *path,
642                                unsigned new_locks_want,
643                                struct get_locks_fail *f)
644 {
645         struct btree_path *linked;
646
647         if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want, f))
648                 return true;
649
650         /*
651          * XXX: this is ugly - we'd prefer to not be mucking with other
652          * iterators in the btree_trans here.
653          *
654          * On failure to upgrade the iterator, setting iter->locks_want and
655          * calling get_locks() is sufficient to make bch2_btree_path_traverse()
656          * get the locks we want on transaction restart.
657          *
658          * But if this iterator was a clone, on transaction restart what we did
659          * to this iterator isn't going to be preserved.
660          *
661          * Possibly we could add an iterator field for the parent iterator when
662          * an iterator is a copy - for now, we'll just upgrade any other
663          * iterators with the same btree id.
664          *
665          * The code below used to be needed to ensure ancestor nodes get locked
666          * before interior nodes - now that's handled by
667          * bch2_btree_path_traverse_all().
668          */
669         if (!path->cached && !trans->in_traverse_all)
670                 trans_for_each_path(trans, linked)
671                         if (linked != path &&
672                             linked->cached == path->cached &&
673                             linked->btree_id == path->btree_id &&
674                             linked->locks_want < new_locks_want) {
675                                 linked->locks_want = new_locks_want;
676                                 btree_path_get_locks(trans, linked, true, NULL);
677                         }
678
679         return false;
680 }
681
682 void __bch2_btree_path_downgrade(struct btree_trans *trans,
683                                  struct btree_path *path,
684                                  unsigned new_locks_want)
685 {
686         unsigned l, old_locks_want = path->locks_want;
687
688         if (trans->restarted)
689                 return;
690
691         EBUG_ON(path->locks_want < new_locks_want);
692
693         path->locks_want = new_locks_want;
694
695         while (path->nodes_locked &&
696                (l = btree_path_highest_level_locked(path)) >= path->locks_want) {
697                 if (l > path->level) {
698                         btree_node_unlock(trans, path, l);
699                 } else {
700                         if (btree_node_intent_locked(path, l)) {
701                                 six_lock_downgrade(&path->l[l].b->c.lock);
702                                 mark_btree_node_locked_noreset(path, l, BTREE_NODE_READ_LOCKED);
703                         }
704                         break;
705                 }
706         }
707
708         bch2_btree_path_verify_locks(path);
709
710         path->downgrade_seq++;
711         trace_path_downgrade(trans, _RET_IP_, path, old_locks_want);
712 }
713
714 /* Btree transaction locking: */
715
716 void bch2_trans_downgrade(struct btree_trans *trans)
717 {
718         struct btree_path *path;
719
720         if (trans->restarted)
721                 return;
722
723         trans_for_each_path(trans, path)
724                 bch2_btree_path_downgrade(trans, path);
725 }
726
727 int bch2_trans_relock(struct btree_trans *trans)
728 {
729         struct btree_path *path;
730
731         if (unlikely(trans->restarted))
732                 return -((int) trans->restarted);
733
734         trans_for_each_path(trans, path)
735                 if (path->should_be_locked &&
736                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
737                         trace_and_count(trans->c, trans_restart_relock, trans, _RET_IP_, path);
738                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
739                 }
740         return 0;
741 }
742
743 int bch2_trans_relock_notrace(struct btree_trans *trans)
744 {
745         struct btree_path *path;
746
747         if (unlikely(trans->restarted))
748                 return -((int) trans->restarted);
749
750         trans_for_each_path(trans, path)
751                 if (path->should_be_locked &&
752                     !bch2_btree_path_relock_norestart(trans, path, _RET_IP_)) {
753                         return btree_trans_restart(trans, BCH_ERR_transaction_restart_relock);
754                 }
755         return 0;
756 }
757
758 void bch2_trans_unlock_noassert(struct btree_trans *trans)
759 {
760         struct btree_path *path;
761
762         trans_for_each_path(trans, path)
763                 __bch2_btree_path_unlock(trans, path);
764 }
765
766 void bch2_trans_unlock(struct btree_trans *trans)
767 {
768         struct btree_path *path;
769
770         trans_for_each_path(trans, path)
771                 __bch2_btree_path_unlock(trans, path);
772
773         /*
774          * bch2_gc_btree_init_recurse() doesn't use btree iterators for walking
775          * btree nodes, it implements its own walking:
776          */
777         if (!trans->is_initial_gc)
778                 bch2_assert_btree_nodes_not_locked();
779 }
780
781 void bch2_trans_unlock_long(struct btree_trans *trans)
782 {
783         bch2_trans_unlock(trans);
784         bch2_trans_srcu_unlock(trans);
785 }
786
787 bool bch2_trans_locked(struct btree_trans *trans)
788 {
789         struct btree_path *path;
790
791         trans_for_each_path(trans, path)
792                 if (path->nodes_locked)
793                         return true;
794         return false;
795 }
796
797 int __bch2_trans_mutex_lock(struct btree_trans *trans,
798                             struct mutex *lock)
799 {
800         int ret = drop_locks_do(trans, (mutex_lock(lock), 0));
801
802         if (ret)
803                 mutex_unlock(lock);
804         return ret;
805 }
806
807 /* Debug */
808
809 #ifdef CONFIG_BCACHEFS_DEBUG
810
811 void bch2_btree_path_verify_locks(struct btree_path *path)
812 {
813         unsigned l;
814
815         if (!path->nodes_locked) {
816                 BUG_ON(path->uptodate == BTREE_ITER_UPTODATE &&
817                        btree_path_node(path, path->level));
818                 return;
819         }
820
821         for (l = 0; l < BTREE_MAX_DEPTH; l++) {
822                 int want = btree_lock_want(path, l);
823                 int have = btree_node_locked_type(path, l);
824
825                 BUG_ON(!is_btree_node(path, l) && have != BTREE_NODE_UNLOCKED);
826
827                 BUG_ON(is_btree_node(path, l) &&
828                        (want == BTREE_NODE_UNLOCKED ||
829                         have != BTREE_NODE_WRITE_LOCKED) &&
830                        want != have);
831         }
832 }
833
834 void bch2_trans_verify_locks(struct btree_trans *trans)
835 {
836         struct btree_path *path;
837
838         trans_for_each_path(trans, path)
839                 bch2_btree_path_verify_locks(path);
840 }
841
842 #endif