]> git.sesse.net Git - bcachefs-tools-debian/blobdiff - libbcachefs/super.c
Update bcachefs sources to 9a555a741e80 bcachefs: omit alignment attribute on big...
[bcachefs-tools-debian] / libbcachefs / super.c
index e7f186b45df103ad51662a1497efb92184a3ba61..a7f9de220d903f45899c9c515ad0bee26ffd2b60 100644 (file)
@@ -23,7 +23,6 @@
 #include "checksum.h"
 #include "clock.h"
 #include "compress.h"
-#include "counters.h"
 #include "debug.h"
 #include "disk_groups.h"
 #include "ec.h"
@@ -49,6 +48,7 @@
 #include "recovery.h"
 #include "replicas.h"
 #include "sb-clean.h"
+#include "sb-counters.h"
 #include "sb-errors.h"
 #include "sb-members.h"
 #include "snapshot.h"
 #include <linux/percpu.h>
 #include <linux/random.h>
 #include <linux/sysfs.h>
+#include <linux/thread_with_file.h>
 #include <crypto/hash.h>
 
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
 MODULE_DESCRIPTION("bcachefs filesystem");
+MODULE_SOFTDEP("pre: crc32c");
+MODULE_SOFTDEP("pre: crc64");
+MODULE_SOFTDEP("pre: sha256");
+MODULE_SOFTDEP("pre: chacha20");
+MODULE_SOFTDEP("pre: poly1305");
+MODULE_SOFTDEP("pre: xxhash");
 
 const char * const bch2_fs_flag_strs[] = {
 #define x(n)           #n,
@@ -80,6 +87,40 @@ const char * const bch2_fs_flag_strs[] = {
        NULL
 };
 
+void bch2_print_opts(struct bch_opts *opts, const char *fmt, ...)
+{
+       struct stdio_redirect *stdio = (void *)(unsigned long)opts->stdio;
+
+       va_list args;
+       va_start(args, fmt);
+       if (likely(!stdio)) {
+               vprintk(fmt, args);
+       } else {
+               if (fmt[0] == KERN_SOH[0])
+                       fmt += 2;
+
+               stdio_redirect_vprintf(stdio, true, fmt, args);
+       }
+       va_end(args);
+}
+
+void __bch2_print(struct bch_fs *c, const char *fmt, ...)
+{
+       struct stdio_redirect *stdio = bch2_fs_stdio_redirect(c);
+
+       va_list args;
+       va_start(args, fmt);
+       if (likely(!stdio)) {
+               vprintk(fmt, args);
+       } else {
+               if (fmt[0] == KERN_SOH[0])
+                       fmt += 2;
+
+               stdio_redirect_vprintf(stdio, true, fmt, args);
+       }
+       va_end(args);
+}
+
 #define KTYPE(type)                                                    \
 static const struct attribute_group type ## _group = {                 \
        .attrs = type ## _files                                         \
@@ -135,14 +176,12 @@ static void __bch2_dev_read_only(struct bch_fs *, struct bch_dev *);
 struct bch_fs *bch2_dev_to_fs(dev_t dev)
 {
        struct bch_fs *c;
-       struct bch_dev *ca;
-       unsigned i;
 
        mutex_lock(&bch_fs_list_lock);
        rcu_read_lock();
 
        list_for_each_entry(c, &bch_fs_list, list)
-               for_each_member_device_rcu(ca, c, i, NULL)
+               for_each_member_device_rcu(c, ca, NULL)
                        if (ca->disk_sb.bdev && ca->disk_sb.bdev->bd_dev == dev) {
                                closure_get(&c->cl);
                                goto found;
@@ -183,14 +222,13 @@ struct bch_fs *bch2_uuid_to_fs(__uuid_t uuid)
 
 static void bch2_dev_usage_journal_reserve(struct bch_fs *c)
 {
-       struct bch_dev *ca;
-       unsigned i, nr = 0, u64s =
+       unsigned nr = 0, u64s =
                ((sizeof(struct jset_entry_dev_usage) +
                  sizeof(struct jset_entry_dev_usage_type) * BCH_DATA_NR)) /
                sizeof(u64);
 
        rcu_read_lock();
-       for_each_member_device_rcu(ca, c, i, NULL)
+       for_each_member_device_rcu(c, ca, NULL)
                nr++;
        rcu_read_unlock();
 
@@ -217,8 +255,7 @@ static void bch2_dev_usage_journal_reserve(struct bch_fs *c)
 
 static void __bch2_fs_read_only(struct bch_fs *c)
 {
-       struct bch_dev *ca;
-       unsigned i, clean_passes = 0;
+       unsigned clean_passes = 0;
        u64 seq = 0;
 
        bch2_fs_ec_stop(c);
@@ -254,7 +291,7 @@ static void __bch2_fs_read_only(struct bch_fs *c)
        /*
         * After stopping journal:
         */
-       for_each_member_device(ca, c, i)
+       for_each_member_device(c, ca)
                bch2_dev_allocator_remove(c, ca);
 }
 
@@ -331,7 +368,8 @@ void bch2_fs_read_only(struct bch_fs *c)
                BUG_ON(c->journal.last_empty_seq != journal_cur_seq(&c->journal));
                BUG_ON(atomic_read(&c->btree_cache.dirty));
                BUG_ON(atomic_long_read(&c->btree_key_cache.nr_dirty));
-               BUG_ON(c->btree_write_buffer.state.nr);
+               BUG_ON(c->btree_write_buffer.inc.keys.nr);
+               BUG_ON(c->btree_write_buffer.flushing.keys.nr);
 
                bch_verbose(c, "marking filesystem clean");
                bch2_fs_mark_clean(c);
@@ -394,8 +432,6 @@ static int bch2_fs_read_write_late(struct bch_fs *c)
 
 static int __bch2_fs_read_write(struct bch_fs *c, bool early)
 {
-       struct bch_dev *ca;
-       unsigned i;
        int ret;
 
        if (test_bit(BCH_FS_initial_gc_unfixed, &c->flags)) {
@@ -406,16 +442,6 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early)
        if (test_bit(BCH_FS_rw, &c->flags))
                return 0;
 
-       if (c->opts.norecovery)
-               return -BCH_ERR_erofs_norecovery;
-
-       /*
-        * nochanges is used for fsck -n mode - we have to allow going rw
-        * during recovery for that to work:
-        */
-       if (c->opts.nochanges && (!early || c->opts.read_only))
-               return -BCH_ERR_erofs_nochanges;
-
        bch_info(c, "going read-write");
 
        ret = bch2_sb_members_v2_init(c);
@@ -436,7 +462,7 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early)
         */
        set_bit(JOURNAL_NEED_FLUSH_WRITE, &c->journal.flags);
 
-       for_each_rw_member(ca, c, i)
+       for_each_rw_member(c, ca)
                bch2_dev_allocator_add(c, ca);
        bch2_recalc_capacity(c);
 
@@ -446,7 +472,7 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early)
 #ifndef BCH_WRITE_REF_DEBUG
        percpu_ref_reinit(&c->writes);
 #else
-       for (i = 0; i < BCH_WRITE_REF_NR; i++) {
+       for (unsigned i = 0; i < BCH_WRITE_REF_NR; i++) {
                BUG_ON(atomic_long_read(&c->writes[i]));
                atomic_long_inc(&c->writes[i]);
        }
@@ -483,6 +509,12 @@ err:
 
 int bch2_fs_read_write(struct bch_fs *c)
 {
+       if (c->opts.norecovery)
+               return -BCH_ERR_erofs_norecovery;
+
+       if (c->opts.nochanges)
+               return -BCH_ERR_erofs_nochanges;
+
        return __bch2_fs_read_write(c, false);
 }
 
@@ -500,7 +532,7 @@ static void __bch2_fs_free(struct bch_fs *c)
        unsigned i;
 
        for (i = 0; i < BCH_TIME_STAT_NR; i++)
-               bch2_time_stats_exit(&c->times[i]);
+               time_stats_exit(&c->times[i]);
 
        bch2_free_pending_node_rewrites(c);
        bch2_fs_sb_errors_exit(c);
@@ -556,7 +588,7 @@ static void __bch2_fs_free(struct bch_fs *c)
                destroy_workqueue(c->btree_update_wq);
 
        bch2_free_super(&c->disk_sb);
-       kvpfree(c, sizeof(*c));
+       kvfree(c);
        module_put(THIS_MODULE);
 }
 
@@ -569,9 +601,6 @@ static void bch2_fs_release(struct kobject *kobj)
 
 void __bch2_fs_stop(struct bch_fs *c)
 {
-       struct bch_dev *ca;
-       unsigned i;
-
        bch_verbose(c, "shutting down");
 
        set_bit(BCH_FS_stopping, &c->flags);
@@ -582,7 +611,7 @@ void __bch2_fs_stop(struct bch_fs *c)
        bch2_fs_read_only(c);
        up_write(&c->state_lock);
 
-       for_each_member_device(ca, c, i)
+       for_each_member_device(c, ca)
                if (ca->kobj.state_in_sysfs &&
                    ca->disk_sb.bdev)
                        sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs");
@@ -593,6 +622,9 @@ void __bch2_fs_stop(struct bch_fs *c)
        bch2_fs_debug_exit(c);
        bch2_fs_chardev_exit(c);
 
+       bch2_ro_ref_put(c);
+       wait_event(c->ro_ref_wait, !refcount_read(&c->ro_ref));
+
        kobject_put(&c->counters_kobj);
        kobject_put(&c->time_stats);
        kobject_put(&c->opts_dir);
@@ -601,7 +633,7 @@ void __bch2_fs_stop(struct bch_fs *c)
        /* btree prefetch might have kicked off reads in the background: */
        bch2_btree_flush_all_reads(c);
 
-       for_each_member_device(ca, c, i)
+       for_each_member_device(c, ca)
                cancel_work_sync(&ca->io_error_work);
 
        cancel_work_sync(&c->read_only_work);
@@ -640,8 +672,6 @@ void bch2_fs_stop(struct bch_fs *c)
 
 static int bch2_fs_online(struct bch_fs *c)
 {
-       struct bch_dev *ca;
-       unsigned i;
        int ret = 0;
 
        lockdep_assert_held(&bch_fs_list_lock);
@@ -674,7 +704,7 @@ static int bch2_fs_online(struct bch_fs *c)
 
        down_write(&c->state_lock);
 
-       for_each_member_device(ca, c, i) {
+       for_each_member_device(c, ca) {
                ret = bch2_dev_sysfs_online(c, ca);
                if (ret) {
                        bch_err(c, "error creating sysfs objects");
@@ -697,12 +727,14 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
        unsigned i, iter_size;
        int ret = 0;
 
-       c = kvpmalloc(sizeof(struct bch_fs), GFP_KERNEL|__GFP_ZERO);
+       c = kvmalloc(sizeof(struct bch_fs), GFP_KERNEL|__GFP_ZERO);
        if (!c) {
                c = ERR_PTR(-BCH_ERR_ENOMEM_fs_alloc);
                goto out;
        }
 
+       c->stdio = (void *)(unsigned long) opts.stdio;
+
        __module_get(THIS_MODULE);
 
        closure_init(&c->cl, NULL);
@@ -723,16 +755,21 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
        mutex_init(&c->btree_root_lock);
        INIT_WORK(&c->read_only_work, bch2_fs_read_only_work);
 
+       refcount_set(&c->ro_ref, 1);
+       init_waitqueue_head(&c->ro_ref_wait);
+       sema_init(&c->online_fsck_mutex, 1);
+
        init_rwsem(&c->gc_lock);
        mutex_init(&c->gc_gens_lock);
        atomic_set(&c->journal_keys.ref, 1);
        c->journal_keys.initial_ref_held = true;
 
        for (i = 0; i < BCH_TIME_STAT_NR; i++)
-               bch2_time_stats_init(&c->times[i]);
+               time_stats_init(&c->times[i]);
 
        bch2_fs_copygc_init(c);
        bch2_fs_btree_key_cache_init_early(&c->btree_key_cache);
+       bch2_fs_btree_iter_init_early(c);
        bch2_fs_btree_interior_update_init_early(c);
        bch2_fs_allocator_background_init(c);
        bch2_fs_allocator_foreground_init(c);
@@ -837,13 +874,13 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
        c->inode_shard_bits = ilog2(roundup_pow_of_two(num_possible_cpus()));
 
        if (!(c->btree_update_wq = alloc_workqueue("bcachefs",
-                               WQ_FREEZABLE|WQ_UNBOUND|WQ_MEM_RECLAIM, 512)) ||
+                               WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_UNBOUND, 512)) ||
            !(c->btree_io_complete_wq = alloc_workqueue("bcachefs_btree_io",
-                               WQ_FREEZABLE|WQ_MEM_RECLAIM, 1)) ||
+                               WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM, 1)) ||
            !(c->copygc_wq = alloc_workqueue("bcachefs_copygc",
-                               WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) ||
+                               WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) ||
            !(c->io_complete_wq = alloc_workqueue("bcachefs_io",
-                               WQ_FREEZABLE|WQ_HIGHPRI|WQ_MEM_RECLAIM, 1)) ||
+                               WQ_HIGHPRI|WQ_FREEZABLE|WQ_MEM_RECLAIM, 512)) ||
            !(c->write_ref_wq = alloc_workqueue("bcachefs_write_ref",
                                WQ_FREEZABLE, 0)) ||
 #ifndef BCH_WRITE_REF_DEBUG
@@ -857,8 +894,8 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
                        BIOSET_NEED_BVECS) ||
            !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) ||
            !(c->online_reserved = alloc_percpu(u64)) ||
-           mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1,
-                                       btree_bytes(c)) ||
+           mempool_init_kvmalloc_pool(&c->btree_bounce_pool, 1,
+                                      c->opts.btree_node_size) ||
            mempool_init_kmalloc_pool(&c->large_bkey_pool, 1, 2048) ||
            !(c->unused_inode_hints = kcalloc(1U << c->inode_shard_bits,
                                              sizeof(u64), GFP_KERNEL))) {
@@ -957,9 +994,7 @@ static void print_mount_opts(struct bch_fs *c)
 
 int bch2_fs_start(struct bch_fs *c)
 {
-       struct bch_dev *ca;
        time64_t now = ktime_get_real_seconds();
-       unsigned i;
        int ret;
 
        print_mount_opts(c);
@@ -976,12 +1011,12 @@ int bch2_fs_start(struct bch_fs *c)
                goto err;
        }
 
-       for_each_online_member(ca, c, i)
-               bch2_members_v2_get_mut(c->disk_sb.sb, i)->last_mount = cpu_to_le64(now);
+       for_each_online_member(c, ca)
+               bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx)->last_mount = cpu_to_le64(now);
 
        mutex_unlock(&c->sb_lock);
 
-       for_each_rw_member(ca, c, i)
+       for_each_rw_member(c, ca)
                bch2_dev_allocator_add(c, ca);
        bch2_recalc_capacity(c);
 
@@ -1003,7 +1038,7 @@ int bch2_fs_start(struct bch_fs *c)
 
        set_bit(BCH_FS_started, &c->flags);
 
-       if (c->opts.read_only || c->opts.nochanges) {
+       if (c->opts.read_only) {
                bch2_fs_read_only(c);
        } else {
                ret = !test_bit(BCH_FS_rw, &c->flags)
@@ -1014,12 +1049,13 @@ int bch2_fs_start(struct bch_fs *c)
        }
 
        ret = 0;
-out:
+err:
+       if (ret)
+               bch_err_msg(c, ret, "starting filesystem");
+       else
+               bch_verbose(c, "done starting filesystem");
        up_write(&c->state_lock);
        return ret;
-err:
-       bch_err_msg(c, ret, "starting filesystem");
-       goto out;
 }
 
 static int bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c)
@@ -1036,20 +1072,83 @@ static int bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c)
        return 0;
 }
 
-static int bch2_dev_in_fs(struct bch_sb *fs, struct bch_sb *sb)
+static int bch2_dev_in_fs(struct bch_sb_handle *fs,
+                         struct bch_sb_handle *sb)
 {
-       struct bch_sb *newest =
-               le64_to_cpu(fs->seq) > le64_to_cpu(sb->seq) ? fs : sb;
+       if (fs == sb)
+               return 0;
 
-       if (!uuid_equal(&fs->uuid, &sb->uuid))
+       if (!uuid_equal(&fs->sb->uuid, &sb->sb->uuid))
                return -BCH_ERR_device_not_a_member_of_filesystem;
 
-       if (!bch2_dev_exists(newest, sb->dev_idx))
+       if (!bch2_dev_exists(fs->sb, sb->sb->dev_idx))
                return -BCH_ERR_device_has_been_removed;
 
-       if (fs->block_size != sb->block_size)
+       if (fs->sb->block_size != sb->sb->block_size)
                return -BCH_ERR_mismatched_block_size;
 
+       if (le16_to_cpu(fs->sb->version) < bcachefs_metadata_version_member_seq ||
+           le16_to_cpu(sb->sb->version) < bcachefs_metadata_version_member_seq)
+               return 0;
+
+       if (fs->sb->seq == sb->sb->seq &&
+           fs->sb->write_time != sb->sb->write_time) {
+               struct printbuf buf = PRINTBUF;
+
+               prt_str(&buf, "Split brain detected between ");
+               prt_bdevname(&buf, sb->bdev);
+               prt_str(&buf, " and ");
+               prt_bdevname(&buf, fs->bdev);
+               prt_char(&buf, ':');
+               prt_newline(&buf);
+               prt_printf(&buf, "seq=%llu but write_time different, got", le64_to_cpu(sb->sb->seq));
+               prt_newline(&buf);
+
+               prt_bdevname(&buf, fs->bdev);
+               prt_char(&buf, ' ');
+               bch2_prt_datetime(&buf, le64_to_cpu(fs->sb->write_time));;
+               prt_newline(&buf);
+
+               prt_bdevname(&buf, sb->bdev);
+               prt_char(&buf, ' ');
+               bch2_prt_datetime(&buf, le64_to_cpu(sb->sb->write_time));;
+               prt_newline(&buf);
+
+               prt_printf(&buf, "Not using older sb");
+
+               pr_err("%s", buf.buf);
+               printbuf_exit(&buf);
+               return -BCH_ERR_device_splitbrain;
+       }
+
+       struct bch_member m = bch2_sb_member_get(fs->sb, sb->sb->dev_idx);
+       u64 seq_from_fs         = le64_to_cpu(m.seq);
+       u64 seq_from_member     = le64_to_cpu(sb->sb->seq);
+
+       if (seq_from_fs && seq_from_fs < seq_from_member) {
+               struct printbuf buf = PRINTBUF;
+
+               prt_str(&buf, "Split brain detected between ");
+               prt_bdevname(&buf, sb->bdev);
+               prt_str(&buf, " and ");
+               prt_bdevname(&buf, fs->bdev);
+               prt_char(&buf, ':');
+               prt_newline(&buf);
+
+               prt_bdevname(&buf, fs->bdev);
+               prt_str(&buf, " believes seq of ");
+               prt_bdevname(&buf, sb->bdev);
+               prt_printf(&buf, " to be %llu, but ", seq_from_fs);
+               prt_bdevname(&buf, sb->bdev);
+               prt_printf(&buf, " has %llu\n", seq_from_member);
+               prt_str(&buf, "Not using ");
+               prt_bdevname(&buf, sb->bdev);
+
+               pr_err("%s", buf.buf);
+               printbuf_exit(&buf);
+               return -BCH_ERR_device_splitbrain;
+       }
+
        return 0;
 }
 
@@ -1081,8 +1180,8 @@ static void bch2_dev_free(struct bch_dev *ca)
        bch2_dev_buckets_free(ca);
        free_page((unsigned long) ca->sb_read_scratch);
 
-       bch2_time_stats_exit(&ca->io_latency[WRITE]);
-       bch2_time_stats_exit(&ca->io_latency[READ]);
+       time_stats_quantiles_exit(&ca->io_latency[WRITE]);
+       time_stats_quantiles_exit(&ca->io_latency[READ]);
 
        percpu_ref_exit(&ca->io_ref);
        percpu_ref_exit(&ca->ref);
@@ -1173,8 +1272,8 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
 
        INIT_WORK(&ca->io_error_work, bch2_io_error_work);
 
-       bch2_time_stats_init(&ca->io_latency[READ]);
-       bch2_time_stats_init(&ca->io_latency[WRITE]);
+       time_stats_quantiles_init(&ca->io_latency[READ]);
+       time_stats_quantiles_init(&ca->io_latency[WRITE]);
 
        ca->mi = bch2_mi_to_cpu(member);
 
@@ -1295,9 +1394,14 @@ static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb)
 
        bch2_dev_sysfs_online(c, ca);
 
+       struct printbuf name = PRINTBUF;
+       prt_bdevname(&name, ca->disk_sb.bdev);
+
        if (c->sb.nr_devices == 1)
-               snprintf(c->name, sizeof(c->name), "%pg", ca->disk_sb.bdev);
-       snprintf(ca->name, sizeof(ca->name), "%pg", ca->disk_sb.bdev);
+               strscpy(c->name, name.buf, sizeof(c->name));
+       strscpy(ca->name, name.buf, sizeof(ca->name));
+
+       printbuf_exit(&name);
 
        rebalance_wakeup(c);
        return 0;
@@ -1318,8 +1422,7 @@ bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca,
                            enum bch_member_state new_state, int flags)
 {
        struct bch_devs_mask new_online_devs;
-       struct bch_dev *ca2;
-       int i, nr_rw = 0, required;
+       int nr_rw = 0, required;
 
        lockdep_assert_held(&c->state_lock);
 
@@ -1331,16 +1434,16 @@ bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca,
                        return true;
 
                /* do we have enough devices to write to?  */
-               for_each_member_device(ca2, c, i)
+               for_each_member_device(c, ca2)
                        if (ca2 != ca)
                                nr_rw += ca2->mi.state == BCH_MEMBER_STATE_rw;
 
                required = max(!(flags & BCH_FORCE_IF_METADATA_DEGRADED)
                               ? c->opts.metadata_replicas
-                              : c->opts.metadata_replicas_required,
+                              : metadata_replicas_required(c),
                               !(flags & BCH_FORCE_IF_DATA_DEGRADED)
                               ? c->opts.data_replicas
-                              : c->opts.data_replicas_required);
+                              : data_replicas_required(c));
 
                return nr_rw >= required;
        case BCH_MEMBER_STATE_failed:
@@ -1479,9 +1582,7 @@ static int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca)
                                        BTREE_TRIGGER_NORUN, NULL) ?:
                bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end,
                                        BTREE_TRIGGER_NORUN, NULL);
-       if (ret)
-               bch_err_msg(c, ret, "removing dev alloc info");
-
+       bch_err_msg(c, ret, "removing dev alloc info");
        return ret;
 }
 
@@ -1508,40 +1609,35 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
        __bch2_dev_read_only(c, ca);
 
        ret = bch2_dev_data_drop(c, ca->dev_idx, flags);
-       if (ret) {
-               bch_err_msg(ca, ret, "dropping data");
+       bch_err_msg(ca, ret, "bch2_dev_data_drop()");
+       if (ret)
                goto err;
-       }
 
        ret = bch2_dev_remove_alloc(c, ca);
-       if (ret) {
-               bch_err_msg(ca, ret, "deleting alloc info");
+       bch_err_msg(ca, ret, "bch2_dev_remove_alloc()");
+       if (ret)
                goto err;
-       }
 
        ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx);
-       if (ret) {
-               bch_err_msg(ca, ret, "flushing journal");
+       bch_err_msg(ca, ret, "bch2_journal_flush_device_pins()");
+       if (ret)
                goto err;
-       }
 
        ret = bch2_journal_flush(&c->journal);
-       if (ret) {
-               bch_err(ca, "journal error");
+       bch_err_msg(ca, ret, "bch2_journal_flush()");
+       if (ret)
                goto err;
-       }
 
        ret = bch2_replicas_gc2(c);
-       if (ret) {
-               bch_err_msg(ca, ret, "in replicas_gc2()");
+       bch_err_msg(ca, ret, "bch2_replicas_gc2()");
+       if (ret)
                goto err;
-       }
 
        data = bch2_dev_has_data(c, ca);
        if (data) {
                struct printbuf data_has = PRINTBUF;
 
-               prt_bitflags(&data_has, bch2_data_types, data);
+               prt_bitflags(&data_has, __bch2_data_types, data);
                bch_err(ca, "Remove failed, still has data (%s)", data_has.buf);
                printbuf_exit(&data_has);
                ret = -EBUSY;
@@ -1607,10 +1703,9 @@ int bch2_dev_add(struct bch_fs *c, const char *path)
        int ret;
 
        ret = bch2_read_super(path, &opts, &sb);
-       if (ret) {
-               bch_err_msg(c, ret, "reading super");
+       bch_err_msg(c, ret, "reading super");
+       if (ret)
                goto err;
-       }
 
        dev_mi = bch2_sb_member_get(sb.sb, sb.sb->dev_idx);
 
@@ -1623,10 +1718,8 @@ int bch2_dev_add(struct bch_fs *c, const char *path)
        }
 
        ret = bch2_dev_may_add(sb.sb, c);
-       if (ret) {
-               bch_err_fn(c, ret);
+       if (ret)
                goto err;
-       }
 
        ca = __bch2_dev_alloc(c, &dev_mi);
        if (!ca) {
@@ -1641,19 +1734,17 @@ int bch2_dev_add(struct bch_fs *c, const char *path)
                goto err;
 
        ret = bch2_dev_journal_alloc(ca);
-       if (ret) {
-               bch_err_msg(c, ret, "allocating journal");
+       bch_err_msg(c, ret, "allocating journal");
+       if (ret)
                goto err;
-       }
 
        down_write(&c->state_lock);
        mutex_lock(&c->sb_lock);
 
        ret = bch2_sb_from_fs(c, ca);
-       if (ret) {
-               bch_err_msg(c, ret, "setting up new superblock");
+       bch_err_msg(c, ret, "setting up new superblock");
+       if (ret)
                goto err_unlock;
-       }
 
        if (dynamic_fault("bcachefs:add:no_slot"))
                goto no_slot;
@@ -1692,10 +1783,9 @@ have_slot:
 
        if (BCH_MEMBER_GROUP(&dev_mi)) {
                ret = __bch2_dev_group_set(c, ca, label.buf);
-               if (ret) {
-                       bch_err_msg(c, ret, "creating new label");
+               bch_err_msg(c, ret, "creating new label");
+               if (ret)
                        goto err_unlock;
-               }
        }
 
        bch2_write_super(c);
@@ -1704,16 +1794,14 @@ have_slot:
        bch2_dev_usage_journal_reserve(c);
 
        ret = bch2_trans_mark_dev_sb(c, ca);
-       if (ret) {
-               bch_err_msg(ca, ret, "marking new superblock");
+       bch_err_msg(ca, ret, "marking new superblock");
+       if (ret)
                goto err_late;
-       }
 
        ret = bch2_fs_freespace_init(c);
-       if (ret) {
-               bch_err_msg(ca, ret, "initializing free space");
+       bch_err_msg(ca, ret, "initializing free space");
+       if (ret)
                goto err_late;
-       }
 
        ca->new_fs_bucket_idx = 0;
 
@@ -1732,6 +1820,7 @@ err:
        bch2_free_super(&sb);
        printbuf_exit(&label);
        printbuf_exit(&errbuf);
+       bch_err_fn(c, ret);
        return ret;
 err_late:
        up_write(&c->state_lock);
@@ -1758,11 +1847,10 @@ int bch2_dev_online(struct bch_fs *c, const char *path)
 
        dev_idx = sb.sb->dev_idx;
 
-       ret = bch2_dev_in_fs(c->disk_sb.sb, sb.sb);
-       if (ret) {
-               bch_err_msg(c, ret, "bringing %s online", path);
+       ret = bch2_dev_in_fs(&c->disk_sb, &sb);
+       bch_err_msg(c, ret, "bringing %s online", path);
+       if (ret)
                goto err;
-       }
 
        ret = bch2_dev_attach_bdev(c, &sb);
        if (ret)
@@ -1771,10 +1859,9 @@ int bch2_dev_online(struct bch_fs *c, const char *path)
        ca = bch_dev_locked(c, dev_idx);
 
        ret = bch2_trans_mark_dev_sb(c, ca);
-       if (ret) {
-               bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path);
+       bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path);
+       if (ret)
                goto err;
-       }
 
        if (ca->mi.state == BCH_MEMBER_STATE_rw)
                __bch2_dev_read_write(c, ca);
@@ -1853,10 +1940,9 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
        }
 
        ret = bch2_dev_buckets_resize(c, ca, nbuckets);
-       if (ret) {
-               bch_err_msg(ca, ret, "resizing buckets");
+       bch_err_msg(ca, ret, "resizing buckets");
+       if (ret)
                goto err;
-       }
 
        ret = bch2_trans_mark_dev_sb(c, ca);
        if (ret)
@@ -1890,28 +1976,30 @@ err:
 /* return with ref on ca->ref: */
 struct bch_dev *bch2_dev_lookup(struct bch_fs *c, const char *name)
 {
-       struct bch_dev *ca;
-       unsigned i;
-
        rcu_read_lock();
-       for_each_member_device_rcu(ca, c, i, NULL)
-               if (!strcmp(name, ca->name))
-                       goto found;
-       ca = ERR_PTR(-BCH_ERR_ENOENT_dev_not_found);
-found:
+       for_each_member_device_rcu(c, ca, NULL)
+               if (!strcmp(name, ca->name)) {
+                       rcu_read_unlock();
+                       return ca;
+               }
        rcu_read_unlock();
-
-       return ca;
+       return ERR_PTR(-BCH_ERR_ENOENT_dev_not_found);
 }
 
 /* Filesystem open: */
 
+static inline int sb_cmp(struct bch_sb *l, struct bch_sb *r)
+{
+       return  cmp_int(le64_to_cpu(l->seq), le64_to_cpu(r->seq)) ?:
+               cmp_int(le64_to_cpu(l->write_time), le64_to_cpu(r->write_time));
+}
+
 struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices,
                            struct bch_opts opts)
 {
        DARRAY(struct bch_sb_handle) sbs = { 0 };
        struct bch_fs *c = NULL;
-       struct bch_sb_handle *sb, *best = NULL;
+       struct bch_sb_handle *best = NULL;
        struct printbuf errbuf = PRINTBUF;
        int ret = 0;
 
@@ -1937,20 +2025,27 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices,
                BUG_ON(darray_push(&sbs, sb));
        }
 
+       if (opts.nochanges && !opts.read_only) {
+               ret = -BCH_ERR_erofs_nochanges;
+               goto err_print;
+       }
+
        darray_for_each(sbs, sb)
-               if (!best || le64_to_cpu(sb->sb->seq) > le64_to_cpu(best->sb->seq))
+               if (!best || sb_cmp(sb->sb, best->sb) > 0)
                        best = sb;
 
        darray_for_each_reverse(sbs, sb) {
-               if (sb != best && !bch2_dev_exists(best->sb, sb->sb->dev_idx)) {
-                       pr_info("%pg has been removed, skipping", sb->bdev);
+               ret = bch2_dev_in_fs(best, sb);
+
+               if (ret == -BCH_ERR_device_has_been_removed ||
+                   ret == -BCH_ERR_device_splitbrain) {
                        bch2_free_super(sb);
                        darray_remove_item(&sbs, sb);
                        best -= best > sb;
+                       ret = 0;
                        continue;
                }
 
-               ret = bch2_dev_in_fs(best->sb, sb->sb);
                if (ret)
                        goto err_print;
        }