btree_write_buffer: ensure atomic64_sub_return_release availability

[bcachefs-tools-debian] / libbcachefs / btree_locking.c

diff --git a/libbcachefs/btree_locking.c b/libbcachefs/btree_locking.c
index 70639a1572ff9ef1516287b46e6da56f3b3f236e..c4266835b8646f45adf9eb1a5f116bac12470fe1 100644 (file)
--- a/libbcachefs/btree_locking.c
+++ b/libbcachefs/btree_locking.c
@@ -10,6 +10,9 @@ void bch2_btree_lock_init(struct btree_bkey_cached_common *b,
                          enum six_lock_init_flags flags)
 {
        __six_lock_init(&b->lock, "b->c.lock", &bch2_btree_node_lock_key, flags);
+#ifdef CONFIG_DEBUG_LOCK_ALLOC
+       lockdep_set_no_check_recursion(&b->lock.dep_map);
+#endif
 }
 
 #ifdef CONFIG_LOCKDEP
@@ -108,10 +111,8 @@ static noinline void lock_graph_pop_all(struct lock_graph *g)
                lock_graph_up(g);
 }
 
-static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
+static void __lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
 {
-       closure_get(&trans->ref);
-
        g->g[g->nr++] = (struct trans_waiting_for_lock) {
                .trans          = trans,
                .node_want      = trans->locking,
@@ -119,6 +120,12 @@ static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
        };
 }
 
+static void lock_graph_down(struct lock_graph *g, struct btree_trans *trans)
+{
+       closure_get(&trans->ref);
+       __lock_graph_down(g, trans);
+}
+
 static bool lock_graph_remove_non_waiters(struct lock_graph *g)
 {
        struct trans_waiting_for_lock *i;
@@ -219,10 +226,14 @@ static int lock_graph_descend(struct lock_graph *g, struct btree_trans *trans,
        struct trans_waiting_for_lock *i;
 
        for (i = g->g; i < g->g + g->nr; i++)
-               if (i->trans == trans)
+               if (i->trans == trans) {
+                       closure_put(&trans->ref);
                        return break_cycle(g, cycle);
+               }
 
        if (g->nr == ARRAY_SIZE(g->g)) {
+               closure_put(&trans->ref);
+
                if (orig_trans->lock_may_not_fail)
                        return 0;
 
@@ -236,7 +247,7 @@ static int lock_graph_descend(struct lock_graph *g, struct btree_trans *trans,
                return btree_trans_restart(orig_trans, BCH_ERR_transaction_restart_deadlock_recursion_limit);
        }
 
-       lock_graph_down(g, trans);
+       __lock_graph_down(g, trans);
        return 0;
 }
 
@@ -251,6 +262,7 @@ int bch2_check_for_deadlock(struct btree_trans *trans, struct printbuf *cycle)
        struct trans_waiting_for_lock *top;
        struct btree_bkey_cached_common *b;
        struct btree_path *path;
+       unsigned path_idx;
        int ret;
 
        if (trans->lock_must_abort) {
@@ -269,12 +281,12 @@ next:
 
        top = &g.g[g.nr - 1];
 
-       trans_for_each_path_from(top->trans, path, top->path_idx) {
+       trans_for_each_path_safe_from(top->trans, path, path_idx, top->path_idx) {
                if (!path->nodes_locked)
                        continue;
 
-               if (top->path_idx != path->idx) {
-                       top->path_idx           = path->idx;
+               if (path_idx != top->path_idx) {
+                       top->path_idx           = path_idx;
                        top->level              = 0;
                        top->lock_start_time    = 0;
                }
@@ -330,9 +342,10 @@ next:
                                    !lock_type_conflicts(lock_held, trans->locking_wait.lock_want))
                                        continue;
 
-                               ret = lock_graph_descend(&g, trans, cycle);
+                               closure_get(&trans->ref);
                                raw_spin_unlock(&b->lock.wait_lock);
 
+                               ret = lock_graph_descend(&g, trans, cycle);
                                if (ret)
                                        return ret;
                                goto next;
@@ -374,7 +387,7 @@ int __bch2_btree_node_lock_write(struct btree_trans *trans, struct btree_path *p
        six_lock_readers_add(&b->lock, readers);
 
        if (ret)
-               mark_btree_node_locked_noreset(path, b->level, SIX_LOCK_intent);
+               mark_btree_node_locked_noreset(path, b->level, BTREE_NODE_INTENT_LOCKED);
 
        return ret;
 }
@@ -417,7 +430,8 @@ void bch2_btree_node_lock_write_nofail(struct btree_trans *trans,
 
 static inline bool btree_path_get_locks(struct btree_trans *trans,
                                        struct btree_path *path,
-                                       bool upgrade)
+                                       bool upgrade,
+                                       struct get_locks_fail *f)
 {
        unsigned l = path->level;
        int fail_idx = -1;
@@ -428,8 +442,14 @@ static inline bool btree_path_get_locks(struct btree_trans *trans,
 
                if (!(upgrade
                      ? bch2_btree_node_upgrade(trans, path, l)
-                     : bch2_btree_node_relock(trans, path, l)))
-                       fail_idx = l;
+                     : bch2_btree_node_relock(trans, path, l))) {
+                       fail_idx        = l;
+
+                       if (f) {
+                               f->l    = l;
+                               f->b    = path->l[l].b;
+                       }
+               }
 
                l++;
        } while (l < path->locks_want);
@@ -538,7 +558,7 @@ bool bch2_btree_node_upgrade(struct btree_trans *trans,
        trace_and_count(trans->c, btree_path_upgrade_fail, trans, _RET_IP_, path, level);
        return false;
 success:
-       mark_btree_node_locked_noreset(path, level, SIX_LOCK_intent);
+       mark_btree_node_locked_noreset(path, level, BTREE_NODE_INTENT_LOCKED);
        return true;
 }
 
@@ -570,7 +590,9 @@ __flatten
 bool bch2_btree_path_relock_norestart(struct btree_trans *trans,
                        struct btree_path *path, unsigned long trace_ip)
 {
-       return btree_path_get_locks(trans, path, false);
+       struct get_locks_fail f;
+
+       return btree_path_get_locks(trans, path, false, &f);
 }
 
 int __bch2_btree_path_relock(struct btree_trans *trans,
@@ -584,31 +606,26 @@ int __bch2_btree_path_relock(struct btree_trans *trans,
        return 0;
 }
 
-__flatten
-bool bch2_btree_path_upgrade_norestart(struct btree_trans *trans,
-                       struct btree_path *path, unsigned long trace_ip)
-{
-       return btree_path_get_locks(trans, path, true);
-}
-
 bool bch2_btree_path_upgrade_noupgrade_sibs(struct btree_trans *trans,
                               struct btree_path *path,
-                              unsigned new_locks_want)
+                              unsigned new_locks_want,
+                              struct get_locks_fail *f)
 {
        EBUG_ON(path->locks_want >= new_locks_want);
 
        path->locks_want = new_locks_want;
 
-       return btree_path_get_locks(trans, path, true);
+       return btree_path_get_locks(trans, path, true, f);
 }
 
 bool __bch2_btree_path_upgrade(struct btree_trans *trans,
                               struct btree_path *path,
-                              unsigned new_locks_want)
+                              unsigned new_locks_want,
+                              struct get_locks_fail *f)
 {
        struct btree_path *linked;
 
-       if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want))
+       if (bch2_btree_path_upgrade_noupgrade_sibs(trans, path, new_locks_want, f))
                return true;
 
        /*
@@ -637,7 +654,7 @@ bool __bch2_btree_path_upgrade(struct btree_trans *trans,
                            linked->btree_id == path->btree_id &&
                            linked->locks_want < new_locks_want) {
                                linked->locks_want = new_locks_want;
-                               btree_path_get_locks(trans, linked, true);
+                               btree_path_get_locks(trans, linked, true, NULL);
                        }
 
        return false;
@@ -649,6 +666,9 @@ void __bch2_btree_path_downgrade(struct btree_trans *trans,
 {
        unsigned l;
 
+       if (trans->restarted)
+               return;
+
        EBUG_ON(path->locks_want < new_locks_want);
 
        path->locks_want = new_locks_want;
@@ -660,13 +680,16 @@ void __bch2_btree_path_downgrade(struct btree_trans *trans,
                } else {
                        if (btree_node_intent_locked(path, l)) {
                                six_lock_downgrade(&path->l[l].b->c.lock);
-                               mark_btree_node_locked_noreset(path, l, SIX_LOCK_read);
+                               mark_btree_node_locked_noreset(path, l, BTREE_NODE_READ_LOCKED);
                        }
                        break;
                }
        }
 
        bch2_btree_path_verify_locks(path);
+
+       path->downgrade_seq++;
+       trace_path_downgrade(trans, _RET_IP_, path);
 }
 
 /* Btree transaction locking: */
@@ -675,6 +698,9 @@ void bch2_trans_downgrade(struct btree_trans *trans)
 {
        struct btree_path *path;
 
+       if (trans->restarted)
+               return;
+
        trans_for_each_path(trans, path)
                bch2_btree_path_downgrade(trans, path);
 }
@@ -710,6 +736,14 @@ int bch2_trans_relock_notrace(struct btree_trans *trans)
        return 0;
 }
 
+void bch2_trans_unlock_noassert(struct btree_trans *trans)
+{
+       struct btree_path *path;
+
+       trans_for_each_path(trans, path)
+               __bch2_btree_path_unlock(trans, path);
+}
+
 void bch2_trans_unlock(struct btree_trans *trans)
 {
        struct btree_path *path;
@@ -725,6 +759,12 @@ void bch2_trans_unlock(struct btree_trans *trans)
                bch2_assert_btree_nodes_not_locked();
 }
 
+void bch2_trans_unlock_long(struct btree_trans *trans)
+{
+       bch2_trans_unlock(trans);
+       bch2_trans_srcu_unlock(trans);
+}
+
 bool bch2_trans_locked(struct btree_trans *trans)
 {
        struct btree_path *path;
@@ -738,11 +778,8 @@ bool bch2_trans_locked(struct btree_trans *trans)
 int __bch2_trans_mutex_lock(struct btree_trans *trans,
                            struct mutex *lock)
 {
-       int ret;
+       int ret = drop_locks_do(trans, (mutex_lock(lock), 0));
 
-       bch2_trans_unlock(trans);
-       mutex_lock(lock);
-       ret = bch2_trans_relock(trans);
        if (ret)
                mutex_unlock(lock);
        return ret;