]> git.sesse.net Git - bcachefs-tools-debian/commitdiff
Update bcachefs sources to ff83eed5f5 bcachefs: Fix bch2_mark_stripe()
authorKent Overstreet <kent.overstreet@gmail.com>
Fri, 23 Oct 2020 22:43:51 +0000 (18:43 -0400)
committerKent Overstreet <kent.overstreet@gmail.com>
Fri, 23 Oct 2020 22:43:51 +0000 (18:43 -0400)
17 files changed:
.bcachefs_revision
libbcachefs/alloc_background.c
libbcachefs/alloc_background.h
libbcachefs/bcachefs_format.h
libbcachefs/btree_types.h
libbcachefs/btree_update_leaf.c
libbcachefs/buckets.c
libbcachefs/buckets_types.h
libbcachefs/ec.c
libbcachefs/ec.h
libbcachefs/io.c
libbcachefs/move.c
libbcachefs/move.h
libbcachefs/movinggc.c
libbcachefs/rebalance.c
libbcachefs/replicas.c
libbcachefs/super.c

index 59bf491a458edd91a279004ed6aa7bce7a2dd35b..4d0b59defe2ce6d4ac5d4969fd8405d748a15f24 100644 (file)
@@ -1 +1 @@
-26c226917f0455877387c1a325282e67e3283f54
+ff83eed5f5f7083d7d111979f68239fe8efd100e
index 54096e8311d35b8dff346fd8256d20e152d78bc0..97508de9f7214204f8f6297b0d0d6d6be901732e 100644 (file)
@@ -544,8 +544,6 @@ out:
  * commands to the newly free buckets, then puts them on the various freelists.
  */
 
-#define BUCKET_GC_GEN_MAX      96U
-
 /**
  * wait_buckets_available - wait on reclaimable buckets
  *
index 870714ff16a7f496972bc1fec2c2fa564cd4c9f4..cbaff56f7473f4f77edb5da718312ee7f29c598b 100644 (file)
@@ -13,6 +13,9 @@ struct bkey_alloc_unpacked {
 #undef  x
 };
 
+/* How out of date a pointer gen is allowed to be: */
+#define BUCKET_GC_GEN_MAX      96U
+
 /* returns true if not equal */
 static inline bool bkey_alloc_unpacked_cmp(struct bkey_alloc_unpacked l,
                                           struct bkey_alloc_unpacked r)
index d5a2230e403c4750ae77ec8debde4f1f36af477a..45dc42865811bb65cb4528277e3f2189dd7d9ae8 100644 (file)
@@ -1032,7 +1032,8 @@ LE64_BITMASK(BCH_KDF_SCRYPT_P,    struct bch_sb_field_crypt, kdf_flags, 32, 48);
        x(journal,      2)              \
        x(btree,        3)              \
        x(user,         4)              \
-       x(cached,       5)
+       x(cached,       5)              \
+       x(parity,       6)
 
 enum bch_data_type {
 #define x(t, n) BCH_DATA_##t,
index c1717b7c8c38712a4278cfd32abb787d5e9951f4..cc01baeec138daaf5e06c3e030e095e4e20b7443 100644 (file)
@@ -591,6 +591,7 @@ static inline bool btree_iter_is_extents(struct btree_iter *iter)
 #define BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS             \
        ((1U << BKEY_TYPE_EXTENTS)|                     \
         (1U << BKEY_TYPE_INODES)|                      \
+        (1U << BKEY_TYPE_EC)|                          \
         (1U << BKEY_TYPE_REFLINK))
 
 enum btree_trigger_flags {
index cd699c257244ddc246e389784d9f2f398d9ad494..49995cd00c16c26c1f4f77c9d9b5eb4da83d3634 100644 (file)
@@ -337,8 +337,9 @@ static inline bool iter_has_trans_triggers(struct btree_iter *iter)
 
 static inline bool iter_has_nontrans_triggers(struct btree_iter *iter)
 {
-       return (BTREE_NODE_TYPE_HAS_TRIGGERS &
-               ~BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS) &
+       return (((BTREE_NODE_TYPE_HAS_TRIGGERS &
+                 ~BTREE_NODE_TYPE_HAS_TRANS_TRIGGERS)) |
+               (1U << BTREE_ID_EC)) &
                (1U << iter->btree_id);
 }
 
index c3fc3abbc0dc1ef368efccf73bd5661f9b3f7963..c8e8e978d143cd1d7345cba29f029b8a1232a777 100644 (file)
 #include <linux/preempt.h>
 #include <trace/events/bcachefs.h>
 
+static inline void fs_usage_data_type_to_base(struct bch_fs_usage *fs_usage,
+                                             enum bch_data_type data_type,
+                                             s64 sectors)
+{
+       switch (data_type) {
+       case BCH_DATA_btree:
+               fs_usage->btree         += sectors;
+               break;
+       case BCH_DATA_user:
+       case BCH_DATA_parity:
+               fs_usage->data          += sectors;
+               break;
+       case BCH_DATA_cached:
+               fs_usage->cached        += sectors;
+               break;
+       default:
+               break;
+       }
+}
+
 /*
  * Clear journal_seq_valid for buckets for which it's not needed, to prevent
  * wraparound:
@@ -132,17 +152,7 @@ void bch2_fs_usage_initialize(struct bch_fs *c)
                struct bch_replicas_entry *e =
                        cpu_replicas_entry(&c->replicas, i);
 
-               switch (e->data_type) {
-               case BCH_DATA_btree:
-                       usage->btree    += usage->replicas[i];
-                       break;
-               case BCH_DATA_user:
-                       usage->data     += usage->replicas[i];
-                       break;
-               case BCH_DATA_cached:
-                       usage->cached   += usage->replicas[i];
-                       break;
-               }
+               fs_usage_data_type_to_base(usage, e->data_type, usage->replicas[i]);
        }
 
        percpu_up_write(&c->mark_lock);
@@ -376,9 +386,14 @@ static inline int is_fragmented_bucket(struct bucket_mark m,
        return 0;
 }
 
+static inline int is_stripe_data_bucket(struct bucket_mark m)
+{
+       return m.stripe && m.data_type != BCH_DATA_parity;
+}
+
 static inline int bucket_stripe_sectors(struct bucket_mark m)
 {
-       return m.stripe ? m.dirty_sectors : 0;
+       return is_stripe_data_bucket(m) ? m.dirty_sectors : 0;
 }
 
 static inline enum bch_data_type bucket_type(struct bucket_mark m)
@@ -412,8 +427,8 @@ int bch2_fs_usage_apply(struct bch_fs *c,
         */
        should_not_have_added = added - (s64) (disk_res ? disk_res->sectors : 0);
        if (WARN_ONCE(should_not_have_added > 0,
-                     "disk usage increased by %lli without a reservation",
-                     should_not_have_added)) {
+                     "disk usage increased by %lli more than reservation of %llu",
+                     added, disk_res ? disk_res->sectors : 0)) {
                atomic64_sub(should_not_have_added, &c->sectors_available);
                added -= should_not_have_added;
                ret = -1;
@@ -522,17 +537,7 @@ static inline int update_replicas(struct bch_fs *c,
        if (!fs_usage)
                return 0;
 
-       switch (r->data_type) {
-       case BCH_DATA_btree:
-               fs_usage->btree         += sectors;
-               break;
-       case BCH_DATA_user:
-               fs_usage->data          += sectors;
-               break;
-       case BCH_DATA_cached:
-               fs_usage->cached        += sectors;
-               break;
-       }
+       fs_usage_data_type_to_base(fs_usage, r->data_type, sectors);
        fs_usage->replicas[idx]         += sectors;
        return 0;
 }
@@ -884,124 +889,155 @@ static s64 ptr_disk_sectors_delta(struct extent_ptr_decoded p,
                                        p.crc.uncompressed_size);
 }
 
-static void bucket_set_stripe(struct bch_fs *c,
-                             const struct bch_extent_ptr *ptr,
-                             struct bch_fs_usage *fs_usage,
-                             u64 journal_seq,
-                             unsigned flags,
-                             bool enabled)
+static int check_bucket_ref(struct bch_fs *c, struct bkey_s_c k,
+                           const struct bch_extent_ptr *ptr,
+                           s64 sectors, enum bch_data_type ptr_data_type,
+                           u8 bucket_gen, u8 bucket_data_type,
+                           u16 dirty_sectors, u16 cached_sectors)
 {
-       bool gc = flags & BTREE_TRIGGER_GC;
-       struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
-       struct bucket *g = PTR_BUCKET(ca, ptr, gc);
-       struct bucket_mark new, old;
-
-       old = bucket_cmpxchg(g, new, ({
-               new.stripe                      = enabled;
-               if (journal_seq) {
-                       new.journal_seq_valid   = 1;
-                       new.journal_seq         = journal_seq;
-               }
-       }));
-
-       bch2_dev_usage_update(c, ca, fs_usage, old, new, gc);
-
-       /*
-        * XXX write repair code for these, flag stripe as possibly bad
-        */
-       if (old.gen != ptr->gen)
-               bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
-                             "stripe with stale pointer");
-#if 0
-       /*
-        * We'd like to check for these, but these checks don't work
-        * yet:
-        */
-       if (old.stripe && enabled)
-               bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
-                             "multiple stripes using same bucket");
-
-       if (!old.stripe && !enabled)
-               bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
-                             "deleting stripe but bucket not marked as stripe bucket");
-#endif
-}
-
-static int __mark_pointer(struct bch_fs *c, struct bkey_s_c k,
-                         struct extent_ptr_decoded p,
-                         s64 sectors, enum bch_data_type ptr_data_type,
-                         u8 bucket_gen, u8 *bucket_data_type,
-                         u16 *dirty_sectors, u16 *cached_sectors)
-{
-       u16 *dst_sectors = !p.ptr.cached
+       size_t bucket_nr = PTR_BUCKET_NR(bch_dev_bkey_exists(c, ptr->dev), ptr);
+       u16 bucket_sectors = !ptr->cached
                ? dirty_sectors
                : cached_sectors;
-       u16 orig_sectors = *dst_sectors;
        char buf[200];
 
-       if (gen_after(p.ptr.gen, bucket_gen)) {
+       if (gen_after(ptr->gen, bucket_gen)) {
                bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
                        "bucket %u:%zu gen %u data type %s: ptr gen %u newer than bucket gen\n"
                        "while marking %s",
-                       p.ptr.dev, PTR_BUCKET_NR(bch_dev_bkey_exists(c, p.ptr.dev), &p.ptr),
-                       bucket_gen,
-                       bch2_data_types[*bucket_data_type ?: ptr_data_type],
-                       p.ptr.gen,
+                       ptr->dev, bucket_nr, bucket_gen,
+                       bch2_data_types[bucket_data_type ?: ptr_data_type],
+                       ptr->gen,
                        (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
                return -EIO;
        }
 
-       if (gen_cmp(bucket_gen, p.ptr.gen) > 96U) {
+       if (gen_cmp(bucket_gen, ptr->gen) > BUCKET_GC_GEN_MAX) {
                bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
                        "bucket %u:%zu gen %u data type %s: ptr gen %u too stale\n"
                        "while marking %s",
-                       p.ptr.dev, PTR_BUCKET_NR(bch_dev_bkey_exists(c, p.ptr.dev), &p.ptr),
-                       bucket_gen,
-                       bch2_data_types[*bucket_data_type ?: ptr_data_type],
-                       p.ptr.gen,
+                       ptr->dev, bucket_nr, bucket_gen,
+                       bch2_data_types[bucket_data_type ?: ptr_data_type],
+                       ptr->gen,
                        (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
                return -EIO;
        }
 
-       if (bucket_gen != p.ptr.gen && !p.ptr.cached) {
+       if (bucket_gen != ptr->gen && !ptr->cached) {
                bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
                        "bucket %u:%zu gen %u data type %s: stale dirty ptr (gen %u)\n"
                        "while marking %s",
-                       p.ptr.dev, PTR_BUCKET_NR(bch_dev_bkey_exists(c, p.ptr.dev), &p.ptr),
-                       bucket_gen,
-                       bch2_data_types[*bucket_data_type ?: ptr_data_type],
-                       p.ptr.gen,
+                       ptr->dev, bucket_nr, bucket_gen,
+                       bch2_data_types[bucket_data_type ?: ptr_data_type],
+                       ptr->gen,
                        (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
                return -EIO;
        }
 
-       if (bucket_gen != p.ptr.gen)
+       if (bucket_gen != ptr->gen)
                return 1;
 
-       if (*bucket_data_type && *bucket_data_type != ptr_data_type) {
+       if (bucket_data_type && ptr_data_type &&
+           bucket_data_type != ptr_data_type) {
                bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
                        "bucket %u:%zu gen %u different types of data in same bucket: %s, %s\n"
                        "while marking %s",
-                       p.ptr.dev, PTR_BUCKET_NR(bch_dev_bkey_exists(c, p.ptr.dev), &p.ptr),
-                       bucket_gen,
-                       bch2_data_types[*bucket_data_type],
+                       ptr->dev, bucket_nr, bucket_gen,
+                       bch2_data_types[bucket_data_type],
                        bch2_data_types[ptr_data_type],
                        (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
                return -EIO;
        }
 
-       if (checked_add(*dst_sectors, sectors)) {
+       if ((unsigned) (bucket_sectors + sectors) > U16_MAX) {
                bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
                        "bucket %u:%zu gen %u data type %s sector count overflow: %u + %lli > U16_MAX\n"
                        "while marking %s",
-                       p.ptr.dev, PTR_BUCKET_NR(bch_dev_bkey_exists(c, p.ptr.dev), &p.ptr),
-                       bucket_gen,
-                       bch2_data_types[*bucket_data_type ?: ptr_data_type],
-                       orig_sectors, sectors,
+                       ptr->dev, bucket_nr, bucket_gen,
+                       bch2_data_types[bucket_data_type ?: ptr_data_type],
+                       bucket_sectors, sectors,
                        (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
                return -EIO;
        }
 
+       return 0;
+}
+
+static int bucket_set_stripe(struct bch_fs *c, struct bkey_s_c k,
+                            unsigned ptr_idx,
+                            struct bch_fs_usage *fs_usage,
+                            u64 journal_seq, unsigned flags,
+                            bool enabled)
+{
+       const struct bch_stripe *s = bkey_s_c_to_stripe(k).v;
+       unsigned nr_data = s->nr_blocks - s->nr_redundant;
+       bool parity = ptr_idx >= nr_data;
+       const struct bch_extent_ptr *ptr = s->ptrs + ptr_idx;
+       bool gc = flags & BTREE_TRIGGER_GC;
+       struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+       struct bucket *g = PTR_BUCKET(ca, ptr, gc);
+       struct bucket_mark new, old;
+       char buf[200];
+       int ret;
+
+       if (enabled)
+               g->ec_redundancy = s->nr_redundant;
+
+       old = bucket_cmpxchg(g, new, ({
+               ret = check_bucket_ref(c, k, ptr, 0, 0, new.gen, new.data_type,
+                                      new.dirty_sectors, new.cached_sectors);
+               if (ret)
+                       return ret;
+
+               if (new.stripe && enabled)
+                       bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
+                                     "bucket %u:%zu gen %u: multiple stripes using same bucket\n%s",
+                                     ptr->dev, PTR_BUCKET_NR(ca, ptr), new.gen,
+                                     (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
+
+               if (!new.stripe && !enabled)
+                       bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
+                                     "bucket %u:%zu gen %u: deleting stripe but not marked\n%s",
+                                     ptr->dev, PTR_BUCKET_NR(ca, ptr), new.gen,
+                                     (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf));
+
+               new.stripe                      = enabled;
+
+               if ((flags & BTREE_TRIGGER_GC) && parity) {
+                       new.data_type = enabled ? BCH_DATA_parity : 0;
+                       new.dirty_sectors = enabled ? le16_to_cpu(s->sectors): 0;
+               }
+
+               if (journal_seq) {
+                       new.journal_seq_valid   = 1;
+                       new.journal_seq         = journal_seq;
+               }
+       }));
+
+       if (!enabled)
+               g->ec_redundancy = 0;
+
+       bch2_dev_usage_update(c, ca, fs_usage, old, new, gc);
+       return 0;
+}
+
+static int __mark_pointer(struct bch_fs *c, struct bkey_s_c k,
+                         const struct bch_extent_ptr *ptr,
+                         s64 sectors, enum bch_data_type ptr_data_type,
+                         u8 bucket_gen, u8 *bucket_data_type,
+                         u16 *dirty_sectors, u16 *cached_sectors)
+{
+       u16 *dst_sectors = !ptr->cached
+               ? dirty_sectors
+               : cached_sectors;
+       int ret = check_bucket_ref(c, k, ptr, sectors, ptr_data_type,
+                                  bucket_gen, *bucket_data_type,
+                                  *dirty_sectors, *cached_sectors);
+
+       if (ret)
+               return ret;
+
+       *dst_sectors += sectors;
        *bucket_data_type = *dirty_sectors || *cached_sectors
                ? ptr_data_type : 0;
        return 0;
@@ -1026,7 +1062,7 @@ static int bch2_mark_pointer(struct bch_fs *c, struct bkey_s_c k,
                new.v.counter = old.v.counter = v;
                bucket_data_type = new.data_type;
 
-               ret = __mark_pointer(c, k, p, sectors, data_type, new.gen,
+               ret = __mark_pointer(c, k, &p.ptr, sectors, data_type, new.gen,
                                     &bucket_data_type,
                                     &new.dirty_sectors,
                                     &new.cached_sectors);
@@ -1059,12 +1095,10 @@ static int bch2_mark_stripe_ptr(struct bch_fs *c,
                                struct bch_extent_stripe_ptr p,
                                enum bch_data_type data_type,
                                struct bch_fs_usage *fs_usage,
-                               s64 sectors, unsigned flags,
-                               struct bch_replicas_padded *r,
-                               unsigned *nr_data,
-                               unsigned *nr_parity)
+                               s64 sectors, unsigned flags)
 {
        bool gc = flags & BTREE_TRIGGER_GC;
+       struct bch_replicas_padded r;
        struct stripe *m;
        unsigned i, blocks_nonempty = 0;
 
@@ -1079,14 +1113,10 @@ static int bch2_mark_stripe_ptr(struct bch_fs *c,
                return -EIO;
        }
 
-       BUG_ON(m->r.e.data_type != data_type);
-
-       *nr_data        = m->nr_blocks - m->nr_redundant;
-       *nr_parity      = m->nr_redundant;
-       *r = m->r;
-
        m->block_sectors[p.block] += sectors;
 
+       r = m->r;
+
        for (i = 0; i < m->nr_blocks; i++)
                blocks_nonempty += m->block_sectors[i] != 0;
 
@@ -1098,6 +1128,9 @@ static int bch2_mark_stripe_ptr(struct bch_fs *c,
 
        spin_unlock(&c->ec_stripes_heap_lock);
 
+       r.e.data_type = data_type;
+       update_replicas(c, fs_usage, &r.e, sectors);
+
        return 0;
 }
 
@@ -1143,25 +1176,11 @@ static int bch2_mark_extent(struct bch_fs *c,
                        dirty_sectors          += disk_sectors;
                        r.e.devs[r.e.nr_devs++] = p.ptr.dev;
                } else {
-                       struct bch_replicas_padded ec_r;
-                       unsigned nr_data, nr_parity;
-                       s64 parity_sectors;
-
                        ret = bch2_mark_stripe_ptr(c, p.ec, data_type,
-                                       fs_usage, disk_sectors, flags,
-                                       &ec_r, &nr_data, &nr_parity);
+                                       fs_usage, disk_sectors, flags);
                        if (ret)
                                return ret;
 
-                       parity_sectors =
-                               __ptr_disk_sectors_delta(p.crc.live_size,
-                                       offset, sectors, flags,
-                                       p.crc.compressed_size * nr_parity,
-                                       p.crc.uncompressed_size * nr_data);
-
-                       update_replicas(c, fs_usage, &ec_r.e,
-                                       disk_sectors + parity_sectors);
-
                        /*
                         * There may be other dirty pointers in this extent, but
                         * if so they're not required for mounting if we have an
@@ -1190,6 +1209,7 @@ static int bch2_mark_stripe(struct bch_fs *c,
                ? bkey_s_c_to_stripe(new).v : NULL;
        struct stripe *m = genradix_ptr(&c->stripes[gc], idx);
        unsigned i;
+       int ret;
 
        if (!m || (old_s && !m->alive)) {
                bch_err_ratelimited(c, "error marking nonexistent stripe %zu",
@@ -1199,9 +1219,12 @@ static int bch2_mark_stripe(struct bch_fs *c,
 
        if (!new_s) {
                /* Deleting: */
-               for (i = 0; i < old_s->nr_blocks; i++)
-                       bucket_set_stripe(c, old_s->ptrs + i, fs_usage,
-                                         journal_seq, flags, false);
+               for (i = 0; i < old_s->nr_blocks; i++) {
+                       ret = bucket_set_stripe(c, old, i, fs_usage,
+                                               journal_seq, flags, false);
+                       if (ret)
+                               return ret;
+               }
 
                if (!gc && m->on_heap) {
                        spin_lock(&c->ec_stripes_heap_lock);
@@ -1209,6 +1232,10 @@ static int bch2_mark_stripe(struct bch_fs *c,
                        spin_unlock(&c->ec_stripes_heap_lock);
                }
 
+               if (gc)
+                       update_replicas(c, fs_usage, &m->r.e,
+                                       -((s64) m->sectors * m->nr_redundant));
+
                memset(m, 0, sizeof(*m));
        } else {
                BUG_ON(old_s && new_s->nr_blocks != old_s->nr_blocks);
@@ -1220,11 +1247,16 @@ static int bch2_mark_stripe(struct bch_fs *c,
                                   old_s->ptrs + i,
                                   sizeof(struct bch_extent_ptr))) {
 
-                               if (old_s)
-                                       bucket_set_stripe(c, old_s->ptrs + i, fs_usage,
+                               if (old_s) {
+                                       bucket_set_stripe(c, old, i, fs_usage,
                                                          journal_seq, flags, false);
-                               bucket_set_stripe(c, new_s->ptrs + i, fs_usage,
-                                                 journal_seq, flags, true);
+                                       if (ret)
+                                               return ret;
+                               }
+                               ret = bucket_set_stripe(c, new, i, fs_usage,
+                                                       journal_seq, flags, true);
+                               if (ret)
+                                       return ret;
                        }
                }
 
@@ -1233,19 +1265,23 @@ static int bch2_mark_stripe(struct bch_fs *c,
                m->algorithm    = new_s->algorithm;
                m->nr_blocks    = new_s->nr_blocks;
                m->nr_redundant = new_s->nr_redundant;
+               m->blocks_nonempty = 0;
 
-               bch2_bkey_to_replicas(&m->r.e, new);
+               for (i = 0; i < new_s->nr_blocks; i++) {
+                       m->block_sectors[i] =
+                               stripe_blockcount_get(new_s, i);
+                       m->blocks_nonempty += !!m->block_sectors[i];
+               }
 
-               /* gc recalculates these fields: */
-               if (!(flags & BTREE_TRIGGER_GC)) {
-                       m->blocks_nonempty = 0;
+               if (gc && old_s)
+                       update_replicas(c, fs_usage, &m->r.e,
+                                       -((s64) m->sectors * m->nr_redundant));
 
-                       for (i = 0; i < new_s->nr_blocks; i++) {
-                               m->block_sectors[i] =
-                                       stripe_blockcount_get(new_s, i);
-                               m->blocks_nonempty += !!m->block_sectors[i];
-                       }
-               }
+               bch2_bkey_to_replicas(&m->r.e, new);
+
+               if (gc)
+                       update_replicas(c, fs_usage, &m->r.e,
+                                       ((s64) m->sectors * m->nr_redundant));
 
                if (!gc) {
                        spin_lock(&c->ec_stripes_heap_lock);
@@ -1550,23 +1586,21 @@ static int trans_get_key(struct btree_trans *trans,
        return ret;
 }
 
-static int bch2_trans_mark_pointer(struct btree_trans *trans,
-                       struct bkey_s_c k, struct extent_ptr_decoded p,
-                       s64 sectors, enum bch_data_type data_type)
+static int bch2_trans_start_alloc_update(struct btree_trans *trans, struct btree_iter **_iter,
+                                        const struct bch_extent_ptr *ptr,
+                                        struct bkey_alloc_unpacked *u)
 {
        struct bch_fs *c = trans->c;
-       struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev);
-       struct bpos pos = POS(p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr));
-       struct btree_iter *iter;
-       struct bkey_s_c k_a;
-       struct bkey_alloc_unpacked u;
-       struct bkey_i_alloc *a;
+       struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+       struct bpos pos = POS(ptr->dev, PTR_BUCKET_NR(ca, ptr));
        struct bucket *g;
+       struct btree_iter *iter;
+       struct bkey_s_c k;
        int ret;
 
-       iter = trans_get_update(trans, BTREE_ID_ALLOC, pos, &k_a);
+       iter = trans_get_update(trans, BTREE_ID_ALLOC, pos, &k);
        if (iter) {
-               u = bch2_alloc_unpack(k_a);
+               *u = bch2_alloc_unpack(k);
        } else {
                iter = bch2_trans_get_iter(trans, BTREE_ID_ALLOC, pos,
                                           BTREE_ITER_CACHED|
@@ -1576,16 +1610,36 @@ static int bch2_trans_mark_pointer(struct btree_trans *trans,
                        return PTR_ERR(iter);
 
                ret = bch2_btree_iter_traverse(iter);
-               if (ret)
-                       goto out;
+               if (ret) {
+                       bch2_trans_iter_put(trans, iter);
+                       return ret;
+               }
 
                percpu_down_read(&c->mark_lock);
                g = bucket(ca, pos.offset);
-               u = alloc_mem_to_key(g, READ_ONCE(g->mark));
+               *u = alloc_mem_to_key(g, READ_ONCE(g->mark));
                percpu_up_read(&c->mark_lock);
        }
 
-       ret = __mark_pointer(c, k, p, sectors, data_type, u.gen, &u.data_type,
+       *_iter = iter;
+       return 0;
+}
+
+static int bch2_trans_mark_pointer(struct btree_trans *trans,
+                       struct bkey_s_c k, struct extent_ptr_decoded p,
+                       s64 sectors, enum bch_data_type data_type)
+{
+       struct bch_fs *c = trans->c;
+       struct btree_iter *iter;
+       struct bkey_alloc_unpacked u;
+       struct bkey_i_alloc *a;
+       int ret;
+
+       ret = bch2_trans_start_alloc_update(trans, &iter, &p.ptr, &u);
+       if (ret)
+               return ret;
+
+       ret = __mark_pointer(c, k, &p.ptr, sectors, data_type, u.gen, &u.data_type,
                             &u.dirty_sectors, &u.cached_sectors);
        if (ret)
                goto out;
@@ -1596,7 +1650,7 @@ static int bch2_trans_mark_pointer(struct btree_trans *trans,
                goto out;
 
        bkey_alloc_init(&a->k_i);
-       a->k.p = pos;
+       a->k.p = iter->pos;
        bch2_alloc_pack(a, u);
        bch2_trans_update(trans, iter, &a->k_i, 0);
 out:
@@ -1606,15 +1660,13 @@ out:
 
 static int bch2_trans_mark_stripe_ptr(struct btree_trans *trans,
                        struct bch_extent_stripe_ptr p,
-                       s64 sectors, enum bch_data_type data_type,
-                       struct bch_replicas_padded *r,
-                       unsigned *nr_data,
-                       unsigned *nr_parity)
+                       s64 sectors, enum bch_data_type data_type)
 {
        struct bch_fs *c = trans->c;
        struct btree_iter *iter;
        struct bkey_s_c k;
        struct bkey_i_stripe *s;
+       struct bch_replicas_padded r;
        int ret = 0;
 
        ret = trans_get_key(trans, BTREE_ID_EC, POS(0, p.idx), &iter, &k);
@@ -1635,15 +1687,14 @@ static int bch2_trans_mark_stripe_ptr(struct btree_trans *trans,
                goto out;
 
        bkey_reassemble(&s->k_i, k);
-
        stripe_blockcount_set(&s->v, p.block,
                stripe_blockcount_get(&s->v, p.block) +
                sectors);
-
-       *nr_data        = s->v.nr_blocks - s->v.nr_redundant;
-       *nr_parity      = s->v.nr_redundant;
-       bch2_bkey_to_replicas(&r->e, bkey_i_to_s_c(&s->k_i));
        bch2_trans_update(trans, iter, &s->k_i, 0);
+
+       bch2_bkey_to_replicas(&r.e, bkey_i_to_s_c(&s->k_i));
+       r.e.data_type = data_type;
+       update_replicas_list(trans, &r.e, sectors);
 out:
        bch2_trans_iter_put(trans, iter);
        return ret;
@@ -1688,25 +1739,11 @@ static int bch2_trans_mark_extent(struct btree_trans *trans,
                        dirty_sectors          += disk_sectors;
                        r.e.devs[r.e.nr_devs++] = p.ptr.dev;
                } else {
-                       struct bch_replicas_padded ec_r;
-                       unsigned nr_data, nr_parity;
-                       s64 parity_sectors;
-
                        ret = bch2_trans_mark_stripe_ptr(trans, p.ec,
-                                       disk_sectors, data_type,
-                                       &ec_r, &nr_data, &nr_parity);
+                                       disk_sectors, data_type);
                        if (ret)
                                return ret;
 
-                       parity_sectors =
-                               __ptr_disk_sectors_delta(p.crc.live_size,
-                                       offset, sectors, flags,
-                                       p.crc.compressed_size * nr_parity,
-                                       p.crc.uncompressed_size * nr_data);
-
-                       update_replicas_list(trans, &ec_r.e,
-                                            disk_sectors + parity_sectors);
-
                        r.e.nr_required = 0;
                }
        }
@@ -1717,6 +1754,64 @@ static int bch2_trans_mark_extent(struct btree_trans *trans,
        return 0;
 }
 
+static int bch2_trans_mark_stripe(struct btree_trans *trans,
+                                 struct bkey_s_c k,
+                                 unsigned flags)
+{
+       const struct bch_stripe *s = bkey_s_c_to_stripe(k).v;
+       unsigned nr_data = s->nr_blocks - s->nr_redundant;
+       struct bch_replicas_padded r;
+       struct bkey_alloc_unpacked u;
+       struct bkey_i_alloc *a;
+       struct btree_iter *iter;
+       bool deleting = flags & BTREE_TRIGGER_OVERWRITE;
+       s64 sectors = le16_to_cpu(s->sectors);
+       unsigned i;
+       int ret = 0;
+
+       if (deleting)
+               sectors = -sectors;
+
+       bch2_bkey_to_replicas(&r.e, k);
+       update_replicas_list(trans, &r.e, sectors * s->nr_redundant);
+
+       /*
+        * The allocator code doesn't necessarily update bucket gens in the
+        * btree when incrementing them, right before handing out new buckets -
+        * we just need to persist those updates here along with the new stripe:
+        */
+
+       for (i = 0; i < s->nr_blocks && !ret; i++) {
+               bool parity = i >= nr_data;
+
+               ret = bch2_trans_start_alloc_update(trans, &iter,
+                                                   &s->ptrs[i], &u);
+               if (ret)
+                       break;
+
+               if (parity) {
+                       u.dirty_sectors += sectors;
+                       u.data_type = u.dirty_sectors
+                               ? BCH_DATA_parity
+                               : 0;
+               }
+
+               a = bch2_trans_kmalloc(trans, BKEY_ALLOC_U64s_MAX * 8);
+               ret = PTR_ERR_OR_ZERO(a);
+               if (ret)
+                       goto put_iter;
+
+               bkey_alloc_init(&a->k_i);
+               a->k.p = iter->pos;
+               bch2_alloc_pack(a, u);
+               bch2_trans_update(trans, iter, &a->k_i, 0);
+put_iter:
+               bch2_trans_iter_put(trans, iter);
+       }
+
+       return ret;
+}
+
 static int __bch2_trans_mark_reflink_p(struct btree_trans *trans,
                        struct bkey_s_c_reflink_p p,
                        u64 idx, unsigned sectors,
@@ -1816,6 +1911,8 @@ int bch2_trans_mark_key(struct btree_trans *trans, struct bkey_s_c k,
        case KEY_TYPE_reflink_v:
                return bch2_trans_mark_extent(trans, k, offset, sectors,
                                              flags, BCH_DATA_user);
+       case KEY_TYPE_stripe:
+               return bch2_trans_mark_stripe(trans, k, flags);
        case KEY_TYPE_inode:
                d = replicas_deltas_realloc(trans, 0);
 
index d5215b14d7d9b5353ea4df535658b478c5fa203b..d6057d22b18e5b14e4bf56a194bb48d7d4953b37 100644 (file)
@@ -41,6 +41,7 @@ struct bucket {
        u8                              oldest_gen;
        u8                              gc_gen;
        unsigned                        gen_valid:1;
+       u8                              ec_redundancy;
 };
 
 struct bucket_array {
@@ -125,6 +126,7 @@ struct disk_reservation {
 struct copygc_heap_entry {
        u8                      dev;
        u8                      gen;
+       u8                      replicas;
        u16                     fragmentation;
        u32                     sectors;
        u64                     offset;
index eac750ad2240f254b3690f386021d5140ae21f24..e4a4805ef218d4973d15933c34a30c8a1e3c45af 100644 (file)
@@ -343,12 +343,17 @@ static void ec_block_io(struct bch_fs *c, struct ec_stripe_buf *buf,
        unsigned offset = 0, bytes = buf->size << 9;
        struct bch_extent_ptr *ptr = &v->ptrs[idx];
        struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+       enum bch_data_type data_type = idx < buf->key.v.nr_blocks - buf->key.v.nr_redundant
+               ? BCH_DATA_user
+               : BCH_DATA_parity;
 
        if (!bch2_dev_get_ioref(ca, rw)) {
                clear_bit(idx, buf->valid);
                return;
        }
 
+       this_cpu_add(ca->io_done->sectors[rw][data_type], buf->size);
+
        while (offset < bytes) {
                unsigned nr_iovecs = min_t(size_t, BIO_MAX_PAGES,
                                           DIV_ROUND_UP(bytes, PAGE_SIZE));
@@ -670,6 +675,7 @@ static void ec_stripe_delete_work(struct work_struct *work)
 /* stripe creation: */
 
 static int ec_stripe_bkey_insert(struct bch_fs *c,
+                                struct ec_stripe_new *s,
                                 struct bkey_i_stripe *stripe)
 {
        struct btree_trans trans;
@@ -711,7 +717,7 @@ found_slot:
 
        bch2_trans_update(&trans, iter, &stripe->k_i, 0);
 
-       ret = bch2_trans_commit(&trans, NULL, NULL,
+       ret = bch2_trans_commit(&trans, &s->res, NULL,
                                BTREE_INSERT_NOFAIL);
 err:
        bch2_trans_iter_put(&trans, iter);
@@ -858,8 +864,8 @@ static void ec_stripe_create(struct ec_stripe_new *s)
 
        ret = s->existing_stripe
                ? bch2_btree_insert(c, BTREE_ID_EC, &s->stripe.key.k_i,
-                                   NULL, NULL, BTREE_INSERT_NOFAIL)
-               : ec_stripe_bkey_insert(c, &s->stripe.key);
+                                   &s->res, NULL, BTREE_INSERT_NOFAIL)
+               : ec_stripe_bkey_insert(c, s, &s->stripe.key);
        if (ret) {
                bch_err(c, "error creating stripe: error creating stripe key");
                goto err_put_writes;
@@ -886,6 +892,8 @@ static void ec_stripe_create(struct ec_stripe_new *s)
 err_put_writes:
        percpu_ref_put(&c->writes);
 err:
+       bch2_disk_reservation_put(c, &s->res);
+
        open_bucket_for_each(c, &s->blocks, ob, i) {
                ob->ec = NULL;
                __bch2_open_bucket_put(c, ob);
@@ -1325,6 +1333,7 @@ struct ec_stripe_head *bch2_ec_stripe_head_get(struct bch_fs *c,
        struct open_bucket *ob;
        unsigned i, data_idx = 0;
        s64 idx;
+       int ret;
 
        closure_init_stack(&cl);
 
@@ -1356,6 +1365,22 @@ struct ec_stripe_head *bch2_ec_stripe_head_get(struct bch_fs *c,
                                }
                }
 
+               if (!h->s->existing_stripe &&
+                   !h->s->res.sectors) {
+                       ret = bch2_disk_reservation_get(c, &h->s->res,
+                                                       h->blocksize,
+                                                       h->s->nr_parity, 0);
+                       if (ret) {
+                               /* What should we do here? */
+                               bch_err(c, "unable to create new stripe: %i", ret);
+                               bch2_ec_stripe_head_put(c, h);
+                               h = NULL;
+                               goto out;
+
+                       }
+
+               }
+
                if (new_stripe_alloc_buckets(c, h)) {
                        bch2_ec_stripe_head_put(c, h);
                        h = NULL;
index 6db16cf768daa40c8c91b8e2523208c146bfeac7..15f751fc2a35d32bea03efde0cf5cbbd332bb41a 100644 (file)
@@ -3,6 +3,7 @@
 #define _BCACHEFS_EC_H
 
 #include "ec_types.h"
+#include "buckets_types.h"
 #include "keylist_types.h"
 
 const char *bch2_stripe_invalid(const struct bch_fs *, struct bkey_s_c);
@@ -105,6 +106,7 @@ struct ec_stripe_new {
        struct open_buckets     blocks;
        u8                      data_block_idx[EC_STRIPE_MAX];
        struct open_buckets     parity;
+       struct disk_reservation res;
 
        struct keylist          keys;
        u64                     inline_keys[BKEY_U64s * 8];
index 0a4b4eed465ced685aee2393c2af1c97efaec0dd..58cc90b198f593e947f02b025f6b7e105f4d27c2 100644 (file)
@@ -1475,7 +1475,8 @@ static struct promote_op *__promote_alloc(struct bch_fs *c,
                        opts,
                        DATA_PROMOTE,
                        (struct data_opts) {
-                               .target = opts.promote_target
+                               .target         = opts.promote_target,
+                               .nr_replicas    = 1,
                        },
                        btree_id, k);
        BUG_ON(ret);
index 62dcac79ed068ab3ef7aeca4755bcef3af5f5a2f..6633d21f604ab00fc476b8530be18ebc5ba13606 100644 (file)
@@ -266,8 +266,8 @@ int bch2_migrate_write_init(struct bch_fs *c, struct migrate_write *m,
                BCH_WRITE_DATA_ENCODED|
                BCH_WRITE_FROM_INTERNAL;
 
-       m->op.nr_replicas       = 1;
-       m->op.nr_replicas_required = 1;
+       m->op.nr_replicas       = data_opts.nr_replicas;
+       m->op.nr_replicas_required = data_opts.nr_replicas;
        m->op.index_update_fn   = bch2_migrate_index_update;
 
        switch (data_cmd) {
@@ -756,6 +756,7 @@ static enum data_cmd rereplicate_pred(struct bch_fs *c, void *arg,
                return DATA_SKIP;
 
        data_opts->target               = 0;
+       data_opts->nr_replicas          = 1;
        data_opts->btree_insert_flags   = 0;
        return DATA_ADD_REPLICAS;
 }
@@ -771,6 +772,7 @@ static enum data_cmd migrate_pred(struct bch_fs *c, void *arg,
                return DATA_SKIP;
 
        data_opts->target               = 0;
+       data_opts->nr_replicas          = 1;
        data_opts->btree_insert_flags   = 0;
        data_opts->rewrite_dev          = op->migrate.dev;
        return DATA_REWRITE;
index 0acd1720d4f8571b3f7cf6f3e883be575c952d2b..b04bc669226de68582c59bdd8cb13ac035b219b2 100644 (file)
@@ -20,7 +20,8 @@ enum data_cmd {
 
 struct data_opts {
        u16             target;
-       unsigned        rewrite_dev;
+       u8              rewrite_dev;
+       u8              nr_replicas;
        int             btree_insert_flags;
 };
 
index de0a7974ec9f57074bac23cefc50ff3ea43055ce..ddfda1ef8a799a369006fb53f3836aabbaa059e3 100644 (file)
@@ -53,17 +53,21 @@ static int bucket_offset_cmp(const void *_l, const void *_r, size_t size)
                cmp_int(l->offset, r->offset);
 }
 
-static int __copygc_pred(struct bch_fs *c, struct bkey_s_c k)
+static enum data_cmd copygc_pred(struct bch_fs *c, void *arg,
+                                struct bkey_s_c k,
+                                struct bch_io_opts *io_opts,
+                                struct data_opts *data_opts)
 {
        copygc_heap *h = &c->copygc_heap;
        struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
-       const struct bch_extent_ptr *ptr;
+       const union bch_extent_entry *entry;
+       struct extent_ptr_decoded p;
 
-       bkey_for_each_ptr(ptrs, ptr) {
-               struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+       bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
+               struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev);
                struct copygc_heap_entry search = {
-                       .dev = ptr->dev,
-                       .offset = ptr->offset
+                       .dev    = p.ptr.dev,
+                       .offset = p.ptr.offset,
                };
 
                ssize_t i = eytzinger0_find_le(h->data, h->used,
@@ -81,27 +85,24 @@ static int __copygc_pred(struct bch_fs *c, struct bkey_s_c k)
                BUG_ON(i != j);
 #endif
                if (i >= 0 &&
-                   ptr->offset < h->data[i].offset + ca->mi.bucket_size &&
-                   ptr->gen == h->data[i].gen)
-                       return ptr->dev;
-       }
+                   p.ptr.offset < h->data[i].offset + ca->mi.bucket_size &&
+                   p.ptr.gen == h->data[i].gen) {
+                       data_opts->target               = io_opts->background_target;
+                       data_opts->nr_replicas          = 1;
+                       data_opts->btree_insert_flags   = BTREE_INSERT_USE_RESERVE;
+                       data_opts->rewrite_dev          = p.ptr.dev;
 
-       return -1;
-}
+                       if (p.has_ec) {
+                               struct stripe *m = genradix_ptr(&c->stripes[0], p.ec.idx);
 
-static enum data_cmd copygc_pred(struct bch_fs *c, void *arg,
-                                struct bkey_s_c k,
-                                struct bch_io_opts *io_opts,
-                                struct data_opts *data_opts)
-{
-       int dev_idx = __copygc_pred(c, k);
-       if (dev_idx < 0)
-               return DATA_SKIP;
-
-       data_opts->target               = io_opts->background_target;
-       data_opts->btree_insert_flags   = BTREE_INSERT_USE_RESERVE;
-       data_opts->rewrite_dev          = dev_idx;
-       return DATA_REWRITE;
+                               data_opts->nr_replicas += m->nr_redundant;
+                       }
+
+                       return DATA_REWRITE;
+               }
+       }
+
+       return DATA_SKIP;
 }
 
 static bool have_copygc_reserve(struct bch_dev *ca)
@@ -168,7 +169,8 @@ static int bch2_copygc(struct bch_fs *c)
                buckets = bucket_array(ca);
 
                for (b = buckets->first_bucket; b < buckets->nbuckets; b++) {
-                       struct bucket_mark m = READ_ONCE(buckets->b[b].mark);
+                       struct bucket *g = buckets->b + b;
+                       struct bucket_mark m = READ_ONCE(g->mark);
                        struct copygc_heap_entry e;
 
                        if (m.owned_by_allocator ||
@@ -177,9 +179,12 @@ static int bch2_copygc(struct bch_fs *c)
                            bucket_sectors_used(m) >= ca->mi.bucket_size)
                                continue;
 
+                       WARN_ON(m.stripe && !g->ec_redundancy);
+
                        e = (struct copygc_heap_entry) {
                                .dev            = dev_idx,
                                .gen            = m.gen,
+                               .replicas       = 1 + g->ec_redundancy,
                                .fragmentation  = bucket_sectors_used(m) * (1U << 15)
                                        / ca->mi.bucket_size,
                                .sectors        = bucket_sectors_used(m),
@@ -196,11 +201,11 @@ static int bch2_copygc(struct bch_fs *c)
        }
 
        for (i = h->data; i < h->data + h->used; i++)
-               sectors_to_move += i->sectors;
+               sectors_to_move += i->sectors * i->replicas;
 
        while (sectors_to_move > sectors_reserved) {
                BUG_ON(!heap_pop(h, e, -fragmentation_cmp, NULL));
-               sectors_to_move -= e.sectors;
+               sectors_to_move -= e.sectors * e.replicas;
        }
 
        buckets_to_move = h->used;
index 56a1f761271f304bd1c5aea3f71690d9d2d804c4..44d2651be9700590177844f29e14b0938efc8235 100644 (file)
@@ -73,6 +73,7 @@ static enum data_cmd rebalance_pred(struct bch_fs *c, void *arg,
 {
        if (__bch2_rebalance_pred(c, k, io_opts) >= 0) {
                data_opts->target               = io_opts->background_target;
+               data_opts->nr_replicas          = 1;
                data_opts->btree_insert_flags   = 0;
                return DATA_ADD_REPLICAS;
        } else {
index 6b6506c68609b1ec35e16ad11745b2eb96cddc67..91518c0d67948b5a8d3832dc2168f90763614953 100644 (file)
@@ -122,7 +122,7 @@ void bch2_bkey_to_replicas(struct bch_replicas_entry *e,
                extent_to_replicas(k, e);
                break;
        case KEY_TYPE_stripe:
-               e->data_type = BCH_DATA_user;
+               e->data_type = BCH_DATA_parity;
                stripe_to_replicas(k, e);
                break;
        }
@@ -446,7 +446,23 @@ static int __bch2_mark_bkey_replicas(struct bch_fs *c, struct bkey_s_c k,
 
        bch2_bkey_to_replicas(&search.e, k);
 
-       return __bch2_mark_replicas(c, &search.e, check);
+       ret = __bch2_mark_replicas(c, &search.e, check);
+       if (ret)
+               return ret;
+
+       if (search.e.data_type == BCH_DATA_parity) {
+               search.e.data_type = BCH_DATA_cached;
+               ret = __bch2_mark_replicas(c, &search.e, check);
+               if (ret)
+                       return ret;
+
+               search.e.data_type = BCH_DATA_user;
+               ret = __bch2_mark_replicas(c, &search.e, check);
+               if (ret)
+                       return ret;
+       }
+
+       return 0;
 }
 
 bool bch2_bkey_replicas_marked(struct bch_fs *c,
index 7f301fa6e5a1dc4023708b2e4953337519da6460..015bbd9f21fd933774c0982b285a42014da3ba35 100644 (file)
@@ -175,6 +175,9 @@ static void __bch2_fs_read_only(struct bch_fs *c)
        bch2_copygc_stop(c);
        bch2_gc_thread_stop(c);
 
+       bch2_io_timer_del(&c->io_clock[READ], &c->bucket_clock[READ].rescale);
+       bch2_io_timer_del(&c->io_clock[WRITE], &c->bucket_clock[WRITE].rescale);
+
        /*
         * Flush journal before stopping allocators, because flushing journal
         * blacklist entries involves allocating new btree nodes:
@@ -224,9 +227,6 @@ nowrote_alloc:
        for_each_member_device(ca, c, i)
                bch2_dev_allocator_stop(ca);
 
-       bch2_io_timer_del(&c->io_clock[READ], &c->bucket_clock[READ].rescale);
-       bch2_io_timer_del(&c->io_clock[WRITE], &c->bucket_clock[WRITE].rescale);
-
        clear_bit(BCH_FS_ALLOCATOR_RUNNING, &c->flags);
        clear_bit(BCH_FS_ALLOCATOR_STOPPING, &c->flags);