X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libbcachefs%2Fsuper.c;h=4eaa4cee82c5601329230764d950855d7443f576;hb=9f69a652dc33611db3c4a3d4ebda00a3ff8b3ca6;hp=d347389771e0ea73b236190018145e219bf395fc;hpb=34c9be19b376a007041555a7c9a47dfef3d0b1e6;p=bcachefs-tools-debian diff --git a/libbcachefs/super.c b/libbcachefs/super.c index d347389..4eaa4ce 100644 --- a/libbcachefs/super.c +++ b/libbcachefs/super.c @@ -13,6 +13,7 @@ #include "bkey_sort.h" #include "btree_cache.h" #include "btree_gc.h" +#include "btree_key_cache.h" #include "btree_update_interior.h" #include "btree_io.h" #include "chardev.h" @@ -48,7 +49,6 @@ #include #include #include -#include #include #include #include @@ -148,43 +148,21 @@ struct bch_fs *bch2_uuid_to_fs(uuid_le uuid) return c; } -int bch2_congested(void *data, int bdi_bits) +static void bch2_dev_usage_journal_reserve(struct bch_fs *c) { - struct bch_fs *c = data; - struct backing_dev_info *bdi; struct bch_dev *ca; - unsigned i; - int ret = 0; + unsigned i, nr = 0, u64s = + ((sizeof(struct jset_entry_dev_usage) + + sizeof(struct jset_entry_dev_usage_type) * BCH_DATA_NR)) / + sizeof(u64); rcu_read_lock(); - if (bdi_bits & (1 << WB_sync_congested)) { - /* Reads - check all devices: */ - for_each_readable_member(ca, c, i) { - bdi = ca->disk_sb.bdev->bd_bdi; - - if (bdi_congested(bdi, bdi_bits)) { - ret = 1; - break; - } - } - } else { - unsigned target = READ_ONCE(c->opts.foreground_target); - const struct bch_devs_mask *devs = target - ? bch2_target_to_mask(c, target) - : &c->rw_devs[BCH_DATA_USER]; - - for_each_member_device_rcu(ca, c, i, devs) { - bdi = ca->disk_sb.bdev->bd_bdi; - - if (bdi_congested(bdi, bdi_bits)) { - ret = 1; - break; - } - } - } + for_each_member_device_rcu(ca, c, i, NULL) + nr++; rcu_read_unlock(); - return ret; + bch2_journal_entry_res_resize(&c->journal, + &c->dev_usage_journal_res, u64s * nr); } /* Filesystem RO/RW: */ @@ -207,15 +185,10 @@ int bch2_congested(void *data, int bdi_bits) static void __bch2_fs_read_only(struct bch_fs *c) { struct bch_dev *ca; - bool wrote; unsigned i, clean_passes = 0; - int ret; bch2_rebalance_stop(c); - - for_each_member_device(ca, c, i) - bch2_copygc_stop(ca); - + bch2_copygc_stop(c); bch2_gc_thread_stop(c); /* @@ -224,56 +197,59 @@ static void __bch2_fs_read_only(struct bch_fs *c) */ bch2_journal_flush_all_pins(&c->journal); + /* + * If the allocator threads didn't all start up, the btree updates to + * write out alloc info aren't going to work: + */ if (!test_bit(BCH_FS_ALLOCATOR_RUNNING, &c->flags)) - goto allocator_not_running; + goto nowrote_alloc; - do { - wrote = false; - - ret = bch2_stripes_write(c, BTREE_INSERT_NOCHECK_RW, &wrote) ?: - bch2_alloc_write(c, BTREE_INSERT_NOCHECK_RW, &wrote); - - if (ret && !test_bit(BCH_FS_EMERGENCY_RO, &c->flags)) - bch2_fs_inconsistent(c, "error writing out alloc info %i", ret); + bch_verbose(c, "flushing journal and stopping allocators"); - if (ret) - break; + bch2_journal_flush_all_pins(&c->journal); + set_bit(BCH_FS_ALLOCATOR_STOPPING, &c->flags); - for_each_member_device(ca, c, i) - bch2_dev_allocator_quiesce(c, ca); + do { + clean_passes++; - bch2_journal_flush_all_pins(&c->journal); + if (bch2_journal_flush_all_pins(&c->journal)) + clean_passes = 0; /* - * We need to explicitly wait on btree interior updates to complete - * before stopping the journal, flushing all journal pins isn't - * sufficient, because in the BTREE_INTERIOR_UPDATING_ROOT case btree - * interior updates have to drop their journal pin before they're - * fully complete: + * In flight interior btree updates will generate more journal + * updates and btree updates (alloc btree): */ - closure_wait_event(&c->btree_interior_update_wait, - !bch2_btree_interior_updates_nr_pending(c)); + if (bch2_btree_interior_updates_nr_pending(c)) { + closure_wait_event(&c->btree_interior_update_wait, + !bch2_btree_interior_updates_nr_pending(c)); + clean_passes = 0; + } + flush_work(&c->btree_interior_update_work); - clean_passes = wrote ? 0 : clean_passes + 1; + if (bch2_journal_flush_all_pins(&c->journal)) + clean_passes = 0; } while (clean_passes < 2); -allocator_not_running: + bch_verbose(c, "flushing journal and stopping allocators complete"); + + set_bit(BCH_FS_ALLOC_CLEAN, &c->flags); +nowrote_alloc: + closure_wait_event(&c->btree_interior_update_wait, + !bch2_btree_interior_updates_nr_pending(c)); + flush_work(&c->btree_interior_update_work); + for_each_member_device(ca, c, i) bch2_dev_allocator_stop(ca); clear_bit(BCH_FS_ALLOCATOR_RUNNING, &c->flags); + clear_bit(BCH_FS_ALLOCATOR_STOPPING, &c->flags); bch2_fs_journal_stop(&c->journal); - /* XXX: mark super that alloc info is persistent */ - /* * the journal kicks off btree writes via reclaim - wait for in flight * writes after stopping journal: */ - if (test_bit(BCH_FS_EMERGENCY_RO, &c->flags)) - bch2_btree_flush_all_writes(c); - else - bch2_btree_verify_flushed(c); + bch2_btree_flush_all_writes(c); /* * After stopping journal: @@ -293,7 +269,7 @@ static void bch2_writes_disabled(struct percpu_ref *writes) void bch2_fs_read_only(struct bch_fs *c) { if (!test_bit(BCH_FS_RW, &c->flags)) { - cancel_delayed_work_sync(&c->journal.reclaim_work); + BUG_ON(c->journal.reclaim_thread); return; } @@ -338,8 +314,11 @@ void bch2_fs_read_only(struct bch_fs *c) !test_bit(BCH_FS_ERROR, &c->flags) && !test_bit(BCH_FS_EMERGENCY_RO, &c->flags) && test_bit(BCH_FS_STARTED, &c->flags) && - !c->opts.norecovery) + test_bit(BCH_FS_ALLOC_CLEAN, &c->flags) && + !c->opts.norecovery) { + bch_verbose(c, "marking filesystem clean"); bch2_fs_mark_clean(c); + } clear_bit(BCH_FS_RW, &c->flags); } @@ -349,9 +328,9 @@ static void bch2_fs_read_only_work(struct work_struct *work) struct bch_fs *c = container_of(work, struct bch_fs, read_only_work); - mutex_lock(&c->state_lock); + down_write(&c->state_lock); bch2_fs_read_only(c); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); } static void bch2_fs_read_only_async(struct bch_fs *c) @@ -363,8 +342,8 @@ bool bch2_fs_emergency_read_only(struct bch_fs *c) { bool ret = !test_and_set_bit(BCH_FS_EMERGENCY_RO, &c->flags); - bch2_fs_read_only_async(c); bch2_journal_halt(&c->journal); + bch2_fs_read_only_async(c); wake_up(&bch_read_only_wait); return ret; @@ -372,8 +351,6 @@ bool bch2_fs_emergency_read_only(struct bch_fs *c) static int bch2_fs_read_write_late(struct bch_fs *c) { - struct bch_dev *ca; - unsigned i; int ret; ret = bch2_gc_thread_start(c); @@ -382,13 +359,10 @@ static int bch2_fs_read_write_late(struct bch_fs *c) return ret; } - for_each_rw_member(ca, c, i) { - ret = bch2_copygc_start(c, ca); - if (ret) { - bch_err(c, "error starting copygc threads"); - percpu_ref_put(&ca->io_ref); - return ret; - } + ret = bch2_copygc_start(c); + if (ret) { + bch_err(c, "error starting copygc thread"); + return ret; } ret = bch2_rebalance_start(c); @@ -422,24 +396,25 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) (!early || c->opts.read_only))) return -EROFS; + bch_info(c, "going read-write"); + ret = bch2_fs_mark_dirty(c); if (ret) goto err; + /* + * We need to write out a journal entry before we start doing btree + * updates, to ensure that on unclean shutdown new journal blacklist + * entries are created: + */ + bch2_journal_meta(&c->journal); + + clear_bit(BCH_FS_ALLOC_CLEAN, &c->flags); + for_each_rw_member(ca, c, i) bch2_dev_allocator_add(c, ca); bch2_recalc_capacity(c); - if (!test_bit(BCH_FS_ALLOCATOR_STARTED, &c->flags)) { - ret = bch2_fs_allocator_start(c); - if (ret) { - bch_err(c, "error initializing allocator"); - goto err; - } - - set_bit(BCH_FS_ALLOCATOR_STARTED, &c->flags); - } - for_each_rw_member(ca, c, i) { ret = bch2_dev_allocator_start(ca); if (ret) { @@ -451,6 +426,15 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) set_bit(BCH_FS_ALLOCATOR_RUNNING, &c->flags); + for_each_rw_member(ca, c, i) + bch2_wake_allocator(ca); + + ret = bch2_journal_reclaim_start(&c->journal); + if (ret) { + bch_err(c, "error starting journal reclaim: %i", ret); + return ret; + } + if (!early) { ret = bch2_fs_read_write_late(c); if (ret) @@ -459,9 +443,6 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) percpu_ref_reinit(&c->writes); set_bit(BCH_FS_RW, &c->flags); - - queue_delayed_work(c->journal_reclaim_wq, - &c->journal.reclaim_work, 0); return 0; err: __bch2_fs_read_only(c); @@ -482,9 +463,10 @@ int bch2_fs_read_write_early(struct bch_fs *c) /* Filesystem startup/shutdown: */ -static void bch2_fs_free(struct bch_fs *c) +static void __bch2_fs_free(struct bch_fs *c) { unsigned i; + int cpu; for (i = 0; i < BCH_TIME_STAT_NR; i++) bch2_time_stats_exit(&c->times[i]); @@ -494,7 +476,9 @@ static void bch2_fs_free(struct bch_fs *c) bch2_fs_ec_exit(c); bch2_fs_encryption_exit(c); bch2_fs_io_exit(c); + bch2_fs_btree_interior_update_exit(c); bch2_fs_btree_iter_exit(c); + bch2_fs_btree_key_cache_exit(&c->btree_key_cache); bch2_fs_btree_cache_exit(c); bch2_fs_journal_exit(&c->journal); bch2_io_clock_exit(&c->io_clock[WRITE]); @@ -504,24 +488,28 @@ static void bch2_fs_free(struct bch_fs *c) bch2_journal_entries_free(&c->journal_entries); percpu_free_rwsem(&c->mark_lock); kfree(c->usage_scratch); - free_percpu(c->usage[1]); - free_percpu(c->usage[0]); + for (i = 0; i < ARRAY_SIZE(c->usage); i++) + free_percpu(c->usage[i]); kfree(c->usage_base); + + if (c->btree_iters_bufs) + for_each_possible_cpu(cpu) + kfree(per_cpu_ptr(c->btree_iters_bufs, cpu)->iter); + + free_percpu(c->btree_iters_bufs); free_percpu(c->pcpu); mempool_exit(&c->large_bkey_pool); mempool_exit(&c->btree_bounce_pool); bioset_exit(&c->btree_bio); - mempool_exit(&c->btree_interior_update_pool); - mempool_exit(&c->btree_reserve_pool); mempool_exit(&c->fill_iter); percpu_ref_exit(&c->writes); kfree(c->replicas.entries); kfree(c->replicas_gc.entries); kfree(rcu_dereference_protected(c->disk_groups, 1)); kfree(c->journal_seq_blacklist_table); + kfree(c->unused_inode_hints); + free_heap(&c->copygc_heap); - if (c->journal_reclaim_wq) - destroy_workqueue(c->journal_reclaim_wq); if (c->copygc_wq) destroy_workqueue(c->copygc_wq); if (c->wq) @@ -537,10 +525,10 @@ static void bch2_fs_release(struct kobject *kobj) { struct bch_fs *c = container_of(kobj, struct bch_fs, kobj); - bch2_fs_free(c); + __bch2_fs_free(c); } -void bch2_fs_stop(struct bch_fs *c) +void __bch2_fs_stop(struct bch_fs *c) { struct bch_dev *ca; unsigned i; @@ -551,9 +539,9 @@ void bch2_fs_stop(struct bch_fs *c) cancel_work_sync(&c->journal_seq_blacklist_gc_work); - mutex_lock(&c->state_lock); + down_write(&c->state_lock); bch2_fs_read_only(c); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); for_each_member_device(ca, c, i) if (ca->kobj.state_in_sysfs && @@ -571,13 +559,6 @@ void bch2_fs_stop(struct bch_fs *c) kobject_put(&c->opts_dir); kobject_put(&c->internal); - mutex_lock(&bch_fs_list_lock); - list_del(&c->list); - mutex_unlock(&bch_fs_list_lock); - - closure_sync(&c->cl); - closure_debug_destroy(&c->cl); - /* btree prefetch might have kicked off reads in the background: */ bch2_btree_flush_all_reads(c); @@ -588,6 +569,22 @@ void bch2_fs_stop(struct bch_fs *c) cancel_delayed_work_sync(&c->pd_controllers_update); cancel_work_sync(&c->read_only_work); + for (i = 0; i < c->sb.nr_devices; i++) + if (c->devs[i]) + bch2_free_super(&c->devs[i]->disk_sb); +} + +void bch2_fs_free(struct bch_fs *c) +{ + unsigned i; + + mutex_lock(&bch_fs_list_lock); + list_del(&c->list); + mutex_unlock(&bch_fs_list_lock); + + closure_sync(&c->cl); + closure_debug_destroy(&c->cl); + for (i = 0; i < c->sb.nr_devices; i++) if (c->devs[i]) bch2_dev_free(rcu_dereference_protected(c->devs[i], 1)); @@ -597,6 +594,12 @@ void bch2_fs_stop(struct bch_fs *c) kobject_put(&c->kobj); } +void bch2_fs_stop(struct bch_fs *c) +{ + __bch2_fs_stop(c); + bch2_fs_free(c); +} + static const char *bch2_fs_online(struct bch_fs *c) { struct bch_dev *ca; @@ -625,7 +628,7 @@ static const char *bch2_fs_online(struct bch_fs *c) bch2_opts_create_sysfs_files(&c->opts_dir)) return "error creating sysfs objects"; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); err = "error creating sysfs objects"; __for_each_member_device(ca, c, i, NULL) @@ -635,7 +638,7 @@ static const char *bch2_fs_online(struct bch_fs *c) list_add(&c->list, &bch_fs_list); err = NULL; err: - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return err; } @@ -654,10 +657,18 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) __module_get(THIS_MODULE); + closure_init(&c->cl, NULL); + + c->kobj.kset = bcachefs_kset; + kobject_init(&c->kobj, &bch2_fs_ktype); + kobject_init(&c->internal, &bch2_fs_internal_ktype); + kobject_init(&c->opts_dir, &bch2_fs_opts_dir_ktype); + kobject_init(&c->time_stats, &bch2_fs_time_stats_ktype); + c->minor = -1; c->disk_sb.fs_sb = true; - mutex_init(&c->state_lock); + init_rwsem(&c->state_lock); mutex_init(&c->sb_lock); mutex_init(&c->replicas_gc_lock); mutex_init(&c->btree_root_lock); @@ -668,6 +679,8 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) for (i = 0; i < BCH_TIME_STAT_NR; i++) bch2_time_stats_init(&c->times[i]); + bch2_fs_copygc_init(c); + bch2_fs_btree_key_cache_init_early(&c->btree_key_cache); bch2_fs_allocator_background_init(c); bch2_fs_allocator_foreground_init(c); bch2_fs_rebalance_init(c); @@ -675,11 +688,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) INIT_LIST_HEAD(&c->list); - INIT_LIST_HEAD(&c->btree_interior_update_list); - INIT_LIST_HEAD(&c->btree_interior_updates_unwritten); - mutex_init(&c->btree_reserve_cache_lock); - mutex_init(&c->btree_interior_update_lock); - mutex_init(&c->usage_scratch_lock); mutex_init(&c->bio_bounce_pages_lock); @@ -692,19 +700,25 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_blacklist_entries_gc); INIT_LIST_HEAD(&c->journal_entries); + INIT_LIST_HEAD(&c->journal_iters); INIT_LIST_HEAD(&c->fsck_errors); mutex_init(&c->fsck_error_lock); - INIT_LIST_HEAD(&c->ec_new_stripe_list); - mutex_init(&c->ec_new_stripe_lock); - mutex_init(&c->ec_stripe_create_lock); + INIT_LIST_HEAD(&c->ec_stripe_head_list); + mutex_init(&c->ec_stripe_head_lock); + + INIT_LIST_HEAD(&c->ec_stripe_new_list); + mutex_init(&c->ec_stripe_new_lock); + spin_lock_init(&c->ec_stripes_heap_lock); seqcount_init(&c->gc_pos_lock); seqcount_init(&c->usage_lock); + sema_init(&c->io_in_flight, 64); + c->copy_gc_enabled = 1; c->rebalance.enabled = 1; c->promote_whole_extents = true; @@ -716,6 +730,8 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_fs_btree_cache_init_early(&c->btree_cache); + mutex_init(&c->sectors_available_lock); + if (percpu_init_rwsem(&c->mark_lock)) goto err; @@ -744,33 +760,34 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) (btree_blocks(c) + 1) * 2 * sizeof(struct sort_iter_set); + c->inode_shard_bits = ilog2(roundup_pow_of_two(num_possible_cpus())); + if (!(c->wq = alloc_workqueue("bcachefs", WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) || - !(c->copygc_wq = alloc_workqueue("bcache_copygc", + !(c->copygc_wq = alloc_workqueue("bcachefs_copygc", WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) || - !(c->journal_reclaim_wq = alloc_workqueue("bcache_journal", - WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_HIGHPRI, 1)) || percpu_ref_init(&c->writes, bch2_writes_disabled, PERCPU_REF_INIT_DEAD, GFP_KERNEL) || - mempool_init_kmalloc_pool(&c->btree_reserve_pool, 1, - sizeof(struct btree_reserve)) || - mempool_init_kmalloc_pool(&c->btree_interior_update_pool, 1, - sizeof(struct btree_update)) || mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) || bioset_init(&c->btree_bio, 1, max(offsetof(struct btree_read_bio, bio), offsetof(struct btree_write_bio, wbio.bio)), BIOSET_NEED_BVECS) || !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) || + !(c->btree_iters_bufs = alloc_percpu(struct btree_iter_buf)) || mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1, btree_bytes(c)) || mempool_init_kmalloc_pool(&c->large_bkey_pool, 1, 2048) || + !(c->unused_inode_hints = kcalloc(1U << c->inode_shard_bits, + sizeof(u64), GFP_KERNEL)) || bch2_io_clock_init(&c->io_clock[READ]) || bch2_io_clock_init(&c->io_clock[WRITE]) || bch2_fs_journal_init(&c->journal) || bch2_fs_replicas_init(c) || bch2_fs_btree_cache_init(c) || + bch2_fs_btree_key_cache_init(&c->btree_key_cache) || bch2_fs_btree_iter_init(c) || + bch2_fs_btree_interior_update_init(c) || bch2_fs_io_init(c) || bch2_fs_encryption_init(c) || bch2_fs_compress_init(c) || @@ -784,17 +801,13 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_dev_alloc(c, i)) goto err; - /* - * Now that all allocations have succeeded, init various refcounty - * things that let us shutdown: - */ - closure_init(&c->cl, NULL); - - c->kobj.kset = bcachefs_kset; - kobject_init(&c->kobj, &bch2_fs_ktype); - kobject_init(&c->internal, &bch2_fs_internal_ktype); - kobject_init(&c->opts_dir, &bch2_fs_opts_dir_ktype); - kobject_init(&c->time_stats, &bch2_fs_time_stats_ktype); + bch2_journal_entry_res_resize(&c->journal, + &c->btree_root_journal_res, + BTREE_ID_NR * (JSET_KEYS_U64s + BKEY_BTREE_PTR_U64s_MAX)); + bch2_dev_usage_journal_reserve(c); + bch2_journal_entry_res_resize(&c->journal, + &c->clock_journal_res, + (sizeof(struct jset_entry_clock) / sizeof(u64)) * 2); mutex_lock(&bch_fs_list_lock); err = bch2_fs_online(c); @@ -855,7 +868,7 @@ int bch2_fs_start(struct bch_fs *c) unsigned i; int ret = -EINVAL; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); BUG_ON(test_bit(BCH_FS_STARTED, &c->flags)); @@ -891,6 +904,13 @@ int bch2_fs_start(struct bch_fs *c) set_bit(BCH_FS_STARTED, &c->flags); + /* + * Allocator threads don't start filling copygc reserve until after we + * set BCH_FS_STARTED - wake them now: + */ + for_each_online_member(ca, c, i) + bch2_wake_allocator(ca); + if (c->opts.read_only || c->opts.nochanges) { bch2_fs_read_only(c); } else { @@ -905,7 +925,7 @@ int bch2_fs_start(struct bch_fs *c) print_mount_opts(c); ret = 0; out: - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return ret; err: switch (ret) { @@ -986,6 +1006,8 @@ static void bch2_dev_release(struct kobject *kobj) static void bch2_dev_free(struct bch_dev *ca) { + bch2_dev_allocator_stop(ca); + cancel_work_sync(&ca->io_error_work); if (ca->kobj.state_in_sysfs && @@ -1096,10 +1118,6 @@ static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c, init_rwsem(&ca->bucket_lock); - writepoint_init(&ca->copygc_write_point, BCH_DATA_USER); - - bch2_dev_copygc_init(ca); - INIT_WORK(&ca->io_error_work, bch2_io_error_work); bch2_time_stats_init(&ca->io_latency[READ]); @@ -1158,6 +1176,14 @@ static int bch2_dev_alloc(struct bch_fs *c, unsigned dev_idx) if (!ca) goto err; + ca->fs = c; + + if (ca->mi.state == BCH_MEMBER_STATE_rw && + bch2_dev_allocator_start(ca)) { + bch2_dev_free(ca); + goto err; + } + bch2_dev_attach(c, ca, dev_idx); out: pr_verbose_init(c->opts, "ret %i", ret); @@ -1228,13 +1254,6 @@ static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb) if (ret) return ret; - if (test_bit(BCH_FS_ALLOC_READ_DONE, &c->flags) && - !percpu_u64_get(&ca->usage[0]->buckets[BCH_DATA_SB])) { - mutex_lock(&c->sb_lock); - bch2_mark_dev_superblock(ca->fs, ca, 0); - mutex_unlock(&c->sb_lock); - } - bch2_dev_sysfs_online(c, ca); if (c->sb.nr_devices == 1) @@ -1260,23 +1279,22 @@ bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca, enum bch_member_state new_state, int flags) { struct bch_devs_mask new_online_devs; - struct replicas_status s; struct bch_dev *ca2; int i, nr_rw = 0, required; lockdep_assert_held(&c->state_lock); switch (new_state) { - case BCH_MEMBER_STATE_RW: + case BCH_MEMBER_STATE_rw: return true; - case BCH_MEMBER_STATE_RO: - if (ca->mi.state != BCH_MEMBER_STATE_RW) + case BCH_MEMBER_STATE_ro: + if (ca->mi.state != BCH_MEMBER_STATE_rw) return true; /* do we have enough devices to write to? */ for_each_member_device(ca2, c, i) if (ca2 != ca) - nr_rw += ca2->mi.state == BCH_MEMBER_STATE_RW; + nr_rw += ca2->mi.state == BCH_MEMBER_STATE_rw; required = max(!(flags & BCH_FORCE_IF_METADATA_DEGRADED) ? c->opts.metadata_replicas @@ -1286,19 +1304,17 @@ bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca, : c->opts.data_replicas_required); return nr_rw >= required; - case BCH_MEMBER_STATE_FAILED: - case BCH_MEMBER_STATE_SPARE: - if (ca->mi.state != BCH_MEMBER_STATE_RW && - ca->mi.state != BCH_MEMBER_STATE_RO) + case BCH_MEMBER_STATE_failed: + case BCH_MEMBER_STATE_spare: + if (ca->mi.state != BCH_MEMBER_STATE_rw && + ca->mi.state != BCH_MEMBER_STATE_ro) return true; /* do we have enough devices to read from? */ new_online_devs = bch2_online_devs(c); __clear_bit(ca->dev_idx, new_online_devs.d); - s = __bch2_replicas_status(c, new_online_devs); - - return bch2_have_enough_devs(s, flags); + return bch2_have_enough_devs(c, new_online_devs, flags, false); default: BUG(); } @@ -1306,14 +1322,18 @@ bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca, static bool bch2_fs_may_start(struct bch_fs *c) { - struct replicas_status s; struct bch_sb_field_members *mi; struct bch_dev *ca; - unsigned i, flags = c->opts.degraded - ? BCH_FORCE_IF_DEGRADED - : 0; + unsigned i, flags = 0; + + if (c->opts.very_degraded) + flags |= BCH_FORCE_IF_DEGRADED|BCH_FORCE_IF_LOST; + + if (c->opts.degraded) + flags |= BCH_FORCE_IF_DEGRADED; - if (!c->opts.degraded) { + if (!c->opts.degraded && + !c->opts.very_degraded) { mutex_lock(&c->sb_lock); mi = bch2_sb_get_members(c->disk_sb.sb); @@ -1324,8 +1344,8 @@ static bool bch2_fs_may_start(struct bch_fs *c) ca = bch_dev_locked(c, i); if (!bch2_dev_is_online(ca) && - (ca->mi.state == BCH_MEMBER_STATE_RW || - ca->mi.state == BCH_MEMBER_STATE_RO)) { + (ca->mi.state == BCH_MEMBER_STATE_rw || + ca->mi.state == BCH_MEMBER_STATE_ro)) { mutex_unlock(&c->sb_lock); return false; } @@ -1333,14 +1353,16 @@ static bool bch2_fs_may_start(struct bch_fs *c) mutex_unlock(&c->sb_lock); } - s = bch2_replicas_status(c); - - return bch2_have_enough_devs(s, flags); + return bch2_have_enough_devs(c, bch2_online_devs(c), flags, true); } static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca) { - bch2_copygc_stop(ca); + /* + * Device going read only means the copygc reserve get smaller, so we + * don't want that happening while copygc is in progress: + */ + bch2_copygc_stop(c); /* * The allocator thread itself allocates btree nodes, so stop it first: @@ -1348,13 +1370,15 @@ static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca) bch2_dev_allocator_stop(ca); bch2_dev_allocator_remove(c, ca); bch2_dev_journal_stop(&c->journal, ca); + + bch2_copygc_start(c); } static const char *__bch2_dev_read_write(struct bch_fs *c, struct bch_dev *ca) { lockdep_assert_held(&c->state_lock); - BUG_ON(ca->mi.state != BCH_MEMBER_STATE_RW); + BUG_ON(ca->mi.state != BCH_MEMBER_STATE_rw); bch2_dev_allocator_add(c, ca); bch2_recalc_capacity(c); @@ -1362,9 +1386,6 @@ static const char *__bch2_dev_read_write(struct bch_fs *c, struct bch_dev *ca) if (bch2_dev_allocator_start(ca)) return "error starting allocator thread"; - if (bch2_copygc_start(c, ca)) - return "error starting copygc thread"; - return NULL; } @@ -1380,10 +1401,10 @@ int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, if (!bch2_dev_state_allowed(c, ca, new_state, flags)) return -EINVAL; - if (new_state != BCH_MEMBER_STATE_RW) + if (new_state != BCH_MEMBER_STATE_rw) __bch2_dev_read_only(c, ca); - bch_notice(ca, "%s", bch2_dev_state[new_state]); + bch_notice(ca, "%s", bch2_member_states[new_state]); mutex_lock(&c->sb_lock); mi = bch2_sb_get_members(c->disk_sb.sb); @@ -1391,7 +1412,7 @@ int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, bch2_write_super(c); mutex_unlock(&c->sb_lock); - if (new_state == BCH_MEMBER_STATE_RW && + if (new_state == BCH_MEMBER_STATE_rw && __bch2_dev_read_write(c, ca)) ret = -ENOMEM; @@ -1405,22 +1426,47 @@ int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, { int ret; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); ret = __bch2_dev_set_state(c, ca, new_state, flags); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return ret; } /* Device add/removal: */ +int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca) +{ + struct btree_trans trans; + size_t i; + int ret; + + bch2_trans_init(&trans, c, 0, 0); + + for (i = 0; i < ca->mi.nbuckets; i++) { + ret = bch2_btree_key_cache_flush(&trans, + BTREE_ID_alloc, POS(ca->dev_idx, i)); + if (ret) + break; + } + bch2_trans_exit(&trans); + + if (ret) + return ret; + + return bch2_btree_delete_range(c, BTREE_ID_alloc, + POS(ca->dev_idx, 0), + POS(ca->dev_idx + 1, 0), + NULL); +} + int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) { struct bch_sb_field_members *mi; unsigned dev_idx = ca->dev_idx, data; int ret = -EINVAL; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); /* * We consume a reference to ca->ref, regardless of whether we succeed @@ -1428,7 +1474,7 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) */ percpu_ref_put(&ca->ref); - if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_FAILED, flags)) { + if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) { bch_err(ca, "Cannot remove without losing data"); goto err; } @@ -1447,10 +1493,7 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) goto err; } - ret = bch2_btree_delete_range(c, BTREE_ID_ALLOC, - POS(ca->dev_idx, 0), - POS(ca->dev_idx + 1, 0), - NULL); + ret = bch2_dev_remove_alloc(c, ca); if (ret) { bch_err(ca, "Remove failed, error deleting alloc info"); goto err; @@ -1510,29 +1553,18 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) bch2_write_super(c); mutex_unlock(&c->sb_lock); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); + + bch2_dev_usage_journal_reserve(c); return 0; err: - if (ca->mi.state == BCH_MEMBER_STATE_RW && + if (ca->mi.state == BCH_MEMBER_STATE_rw && !percpu_ref_is_zero(&ca->io_ref)) __bch2_dev_read_write(c, ca); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return ret; } -static void dev_usage_clear(struct bch_dev *ca) -{ - struct bucket_array *buckets; - - percpu_memset(ca->usage[0], 0, sizeof(*ca->usage[0])); - - down_read(&ca->bucket_lock); - buckets = bucket_array(ca); - - memset(buckets->b, 0, sizeof(buckets->b[0]) * buckets->nbuckets); - up_read(&ca->bucket_lock); -} - /* Add new device to running filesystem: */ int bch2_dev_add(struct bch_fs *c, const char *path) { @@ -1583,16 +1615,14 @@ int bch2_dev_add(struct bch_fs *c, const char *path) * allocate the journal, reset all the marks, then remark after we * attach... */ - bch2_mark_dev_superblock(ca->fs, ca, 0); + bch2_mark_dev_superblock(NULL, ca, 0); err = "journal alloc failed"; ret = bch2_dev_journal_alloc(ca); if (ret) goto err; - dev_usage_clear(ca); - - mutex_lock(&c->state_lock); + down_write(&c->state_lock); mutex_lock(&c->sb_lock); err = "insufficient space in new superblock"; @@ -1642,23 +1672,28 @@ have_slot: ca->disk_sb.sb->dev_idx = dev_idx; bch2_dev_attach(c, ca, dev_idx); - bch2_mark_dev_superblock(c, ca, 0); - bch2_write_super(c); mutex_unlock(&c->sb_lock); - if (ca->mi.state == BCH_MEMBER_STATE_RW) { + bch2_dev_usage_journal_reserve(c); + + err = "error marking superblock"; + ret = bch2_trans_mark_dev_sb(c, NULL, ca); + if (ret) + goto err_late; + + if (ca->mi.state == BCH_MEMBER_STATE_rw) { err = __bch2_dev_read_write(c, ca); if (err) goto err_late; } - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return 0; err_unlock: mutex_unlock(&c->sb_lock); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); err: if (ca) bch2_dev_free(ca); @@ -1666,6 +1701,7 @@ err: bch_err(c, "Unable to add device: %s", err); return ret; err_late: + up_write(&c->state_lock); bch_err(c, "Error going rw after adding device: %s", err); return -EINVAL; } @@ -1681,11 +1717,11 @@ int bch2_dev_online(struct bch_fs *c, const char *path) const char *err; int ret; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); ret = bch2_read_super(path, &opts, &sb); if (ret) { - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return ret; } @@ -1701,7 +1737,13 @@ int bch2_dev_online(struct bch_fs *c, const char *path) } ca = bch_dev_locked(c, dev_idx); - if (ca->mi.state == BCH_MEMBER_STATE_RW) { + + if (bch2_trans_mark_dev_sb(c, NULL, ca)) { + err = "bch2_trans_mark_dev_sb() error"; + goto err; + } + + if (ca->mi.state == BCH_MEMBER_STATE_rw) { err = __bch2_dev_read_write(c, ca); if (err) goto err; @@ -1716,10 +1758,10 @@ int bch2_dev_online(struct bch_fs *c, const char *path) bch2_write_super(c); mutex_unlock(&c->sb_lock); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return 0; err: - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); bch2_free_super(&sb); bch_err(c, "error bringing %s online: %s", path, err); return -EINVAL; @@ -1727,23 +1769,23 @@ err: int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags) { - mutex_lock(&c->state_lock); + down_write(&c->state_lock); if (!bch2_dev_is_online(ca)) { bch_err(ca, "Already offline"); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return 0; } - if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_FAILED, flags)) { + if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) { bch_err(ca, "Cannot offline required disk"); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return -EINVAL; } __bch2_dev_offline(c, ca); - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return 0; } @@ -1752,7 +1794,7 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) struct bch_member *mi; int ret = 0; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); if (nbuckets < ca->mi.nbuckets) { bch_err(ca, "Cannot shrink yet"); @@ -1783,14 +1825,13 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) bch2_recalc_capacity(c); err: - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); return ret; } /* return with ref on ca->ref: */ struct bch_dev *bch2_dev_lookup(struct bch_fs *c, const char *path) { - struct block_device *bdev = lookup_bdev(path); struct bch_dev *ca; unsigned i; @@ -1815,6 +1856,7 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices, { struct bch_sb_handle *sb = NULL; struct bch_fs *c = NULL; + struct bch_sb_field_members *mi; unsigned i, best_sb = 0; const char *err; int ret = -ENOMEM; @@ -1850,10 +1892,24 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices, le64_to_cpu(sb[best_sb].sb->seq)) best_sb = i; - for (i = 0; i < nr_devices; i++) { + mi = bch2_sb_get_members(sb[best_sb].sb); + + i = 0; + while (i < nr_devices) { + if (i != best_sb && + !bch2_dev_exists(sb[best_sb].sb, mi, sb[i].sb->dev_idx)) { + char buf[BDEVNAME_SIZE]; + pr_info("%s has been removed, skipping", + bdevname(sb[i].bdev, buf)); + bch2_free_super(&sb[i]); + array_remove_item(sb, nr_devices, i); + continue; + } + err = bch2_dev_in_fs(sb[best_sb].sb, sb[i].sb); if (err) goto err_print; + i++; } ret = -ENOMEM; @@ -1862,13 +1918,13 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices, goto err; err = "bch2_dev_online() error"; - mutex_lock(&c->state_lock); + down_write(&c->state_lock); for (i = 0; i < nr_devices; i++) if (bch2_dev_attach_bdev(c, &sb[i])) { - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); goto err_print; } - mutex_unlock(&c->state_lock); + up_write(&c->state_lock); err = "insufficient devices"; if (!bch2_fs_may_start(c)) @@ -1980,6 +2036,7 @@ static void bcachefs_exit(void) bch2_debug_exit(); bch2_vfs_exit(); bch2_chardev_exit(); + bch2_btree_key_cache_exit(); if (bcachefs_kset) kset_unregister(bcachefs_kset); } @@ -1987,9 +2044,9 @@ static void bcachefs_exit(void) static int __init bcachefs_init(void) { bch2_bkey_pack_test(); - bch2_inode_pack_test(); if (!(bcachefs_kset = kset_create_and_add("bcachefs", NULL, fs_kobj)) || + bch2_btree_key_cache_init() || bch2_chardev_init() || bch2_vfs_init() || bch2_debug_init())