X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libbcachefs%2Fsuper.c;h=359ca164e0889764d03cd930449debc89327b59a;hb=b0c9ad15f4e5cee60973a8f5f6dc49acfeec9755;hp=5be4c40afa47500725e6ea95bb8ab0b0d3acebed;hpb=188b6d0c8ef1c02462a744b176557c27220112c9;p=bcachefs-tools-debian diff --git a/libbcachefs/super.c b/libbcachefs/super.c index 5be4c40..359ca16 100644 --- a/libbcachefs/super.c +++ b/libbcachefs/super.c @@ -16,6 +16,7 @@ #include "btree_key_cache.h" #include "btree_update_interior.h" #include "btree_io.h" +#include "btree_write_buffer.h" #include "buckets_waiting_for_journal.h" #include "chardev.h" #include "checksum.h" @@ -37,6 +38,7 @@ #include "move.h" #include "migrate.h" #include "movinggc.h" +#include "nocow_locking.h" #include "quota.h" #include "rebalance.h" #include "recovery.h" @@ -54,7 +56,6 @@ #include #include #include -#include #include #include #include @@ -109,7 +110,7 @@ static struct kset *bcachefs_kset; static LIST_HEAD(bch_fs_list); static DEFINE_MUTEX(bch_fs_list_lock); -static DECLARE_WAIT_QUEUE_HEAD(bch_read_only_wait); +DECLARE_WAIT_QUEUE_HEAD(bch2_read_only_wait); static void bch2_dev_free(struct bch_dev *); static int bch2_dev_alloc(struct bch_fs *, unsigned); @@ -237,13 +238,15 @@ static void __bch2_fs_read_only(struct bch_fs *c) bch2_dev_allocator_remove(c, ca); } +#ifndef BCH_WRITE_REF_DEBUG static void bch2_writes_disabled(struct percpu_ref *writes) { struct bch_fs *c = container_of(writes, struct bch_fs, writes); set_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags); - wake_up(&bch_read_only_wait); + wake_up(&bch2_read_only_wait); } +#endif void bch2_fs_read_only(struct bch_fs *c) { @@ -258,9 +261,13 @@ void bch2_fs_read_only(struct bch_fs *c) * Block new foreground-end write operations from starting - any new * writes will return -EROFS: */ + set_bit(BCH_FS_GOING_RO, &c->flags); +#ifndef BCH_WRITE_REF_DEBUG percpu_ref_kill(&c->writes); - - cancel_work_sync(&c->ec_stripe_delete_work); +#else + for (unsigned i = 0; i < BCH_WRITE_REF_NR; i++) + bch2_write_ref_put(c, i); +#endif /* * If we're not doing an emergency shutdown, we want to wait on @@ -273,16 +280,17 @@ void bch2_fs_read_only(struct bch_fs *c) * we do need to wait on them before returning and signalling * that going RO is complete: */ - wait_event(bch_read_only_wait, + wait_event(bch2_read_only_wait, test_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags) || test_bit(BCH_FS_EMERGENCY_RO, &c->flags)); __bch2_fs_read_only(c); - wait_event(bch_read_only_wait, + wait_event(bch2_read_only_wait, test_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags)); clear_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags); + clear_bit(BCH_FS_GOING_RO, &c->flags); if (!bch2_journal_error(&c->journal) && !test_bit(BCH_FS_ERROR, &c->flags) && @@ -319,7 +327,7 @@ bool bch2_fs_emergency_read_only(struct bch_fs *c) bch2_journal_halt(&c->journal); bch2_fs_read_only_async(c); - wake_up(&bch_read_only_wait); + wake_up(&bch2_read_only_wait); return ret; } @@ -367,6 +375,14 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) clear_bit(BCH_FS_CLEAN_SHUTDOWN, &c->flags); + /* + * First journal write must be a flush write: after a clean shutdown we + * don't read the journal, so the first journal write may end up + * overwriting whatever was there previously, and there must always be + * at least one non-flush write in the journal or recovery will fail: + */ + set_bit(JOURNAL_NEED_FLUSH_WRITE, &c->journal.flags); + for_each_rw_member(ca, c, i) bch2_dev_allocator_add(c, ca); bch2_recalc_capacity(c); @@ -383,20 +399,27 @@ static int __bch2_fs_read_write(struct bch_fs *c, bool early) return ret; } - schedule_work(&c->ec_stripe_delete_work); - - bch2_do_discards(c); - bch2_do_invalidates(c); - if (!early) { ret = bch2_fs_read_write_late(c); if (ret) goto err; } +#ifndef BCH_WRITE_REF_DEBUG percpu_ref_reinit(&c->writes); +#else + for (unsigned i = 0; i < BCH_WRITE_REF_NR; i++) { + BUG_ON(atomic_long_read(&c->writes[i])); + atomic_long_inc(&c->writes[i]); + } +#endif set_bit(BCH_FS_RW, &c->flags); set_bit(BCH_FS_WAS_RW, &c->flags); + + bch2_do_discards(c); + bch2_do_invalidates(c); + bch2_do_stripe_deletes(c); + bch2_do_pending_node_rewrites(c); return 0; err: __bch2_fs_read_only(c); @@ -425,6 +448,7 @@ static void __bch2_fs_free(struct bch_fs *c) for (i = 0; i < BCH_TIME_STAT_NR; i++) bch2_time_stats_exit(&c->times[i]); + bch2_free_pending_node_rewrites(c); bch2_fs_counters_exit(c); bch2_fs_snapshots_exit(c); bch2_fs_quota_exit(c); @@ -444,24 +468,26 @@ static void __bch2_fs_free(struct bch_fs *c) bch2_fs_compress_exit(c); bch2_journal_keys_free(&c->journal_keys); bch2_journal_entries_free(c); + bch2_fs_btree_write_buffer_exit(c); percpu_free_rwsem(&c->mark_lock); + free_percpu(c->online_reserved); if (c->btree_paths_bufs) for_each_possible_cpu(cpu) kfree(per_cpu_ptr(c->btree_paths_bufs, cpu)->path); - free_percpu(c->online_reserved); free_percpu(c->btree_paths_bufs); free_percpu(c->pcpu); mempool_exit(&c->large_bkey_pool); mempool_exit(&c->btree_bounce_pool); bioset_exit(&c->btree_bio); mempool_exit(&c->fill_iter); +#ifndef BCH_WRITE_REF_DEBUG percpu_ref_exit(&c->writes); +#endif kfree(rcu_dereference_protected(c->disk_groups, 1)); kfree(c->journal_seq_blacklist_table); kfree(c->unused_inode_hints); - free_heap(&c->copygc_heap); if (c->io_complete_wq) destroy_workqueue(c->io_complete_wq); @@ -680,12 +706,14 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) INIT_LIST_HEAD(&c->data_progress_list); mutex_init(&c->data_progress_lock); - spin_lock_init(&c->ec_stripes_heap_lock); + mutex_init(&c->ec_stripes_heap_lock); seqcount_init(&c->gc_pos_lock); seqcount_init(&c->usage_lock); + sema_init(&c->io_in_flight, 128); + c->copy_gc_enabled = 1; c->rebalance.enabled = 1; c->promote_whole_extents = true; @@ -734,9 +762,6 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_opts_apply(&c->opts, opts); - /* key cache currently disabled for inodes, because of snapshots: */ - c->opts.inodes_use_key_cache = 0; - c->btree_key_cache_btrees |= 1U << BTREE_ID_alloc; if (c->opts.inodes_use_key_cache) c->btree_key_cache_btrees |= 1U << BTREE_ID_inodes; @@ -757,23 +782,25 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) c->inode_shard_bits = ilog2(roundup_pow_of_two(num_possible_cpus())); if (!(c->btree_update_wq = alloc_workqueue("bcachefs", - WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) || + WQ_FREEZABLE|WQ_UNBOUND|WQ_MEM_RECLAIM, 512)) || !(c->btree_io_complete_wq = alloc_workqueue("bcachefs_btree_io", - WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) || + WQ_FREEZABLE|WQ_MEM_RECLAIM, 1)) || !(c->copygc_wq = alloc_workqueue("bcachefs_copygc", WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) || !(c->io_complete_wq = alloc_workqueue("bcachefs_io", WQ_FREEZABLE|WQ_HIGHPRI|WQ_MEM_RECLAIM, 1)) || +#ifndef BCH_WRITE_REF_DEBUG percpu_ref_init(&c->writes, bch2_writes_disabled, PERCPU_REF_INIT_DEAD, GFP_KERNEL) || +#endif mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) || bioset_init(&c->btree_bio, 1, max(offsetof(struct btree_read_bio, bio), offsetof(struct btree_write_bio, wbio.bio)), BIOSET_NEED_BVECS) || !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) || - !(c->btree_paths_bufs = alloc_percpu(struct btree_path_buf)) || !(c->online_reserved = alloc_percpu(u64)) || + !(c->btree_paths_bufs = alloc_percpu(struct btree_path_buf)) || mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1, btree_bytes(c)) || mempool_init_kmalloc_pool(&c->large_bkey_pool, 1, 2048) || @@ -793,8 +820,10 @@ static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts) bch2_fs_btree_iter_init(c) ?: bch2_fs_btree_interior_update_init(c) ?: bch2_fs_buckets_waiting_for_journal_init(c) ?: + bch2_fs_btree_write_buffer_init(c) ?: bch2_fs_subvolumes_init(c) ?: bch2_fs_io_init(c) ?: + bch2_fs_nocow_locking_init(c) ?: bch2_fs_encryption_init(c) ?: bch2_fs_compress_init(c) ?: bch2_fs_ec_init(c) ?: @@ -840,9 +869,12 @@ static void print_mount_opts(struct bch_fs *c) struct printbuf p = PRINTBUF; bool first = true; + prt_printf(&p, "mounted version=%s", bch2_metadata_versions[c->sb.version]); + if (c->opts.read_only) { - prt_printf(&p, "ro"); + prt_str(&p, " opts="); first = false; + prt_printf(&p, "ro"); } for (i = 0; i < bch2_opts_nr; i++) { @@ -855,16 +887,12 @@ static void print_mount_opts(struct bch_fs *c) if (v == bch2_opt_get_by_id(&bch2_opts_default, i)) continue; - if (!first) - prt_printf(&p, ","); + prt_str(&p, first ? " opts=" : ","); first = false; bch2_opt_to_text(&p, c, c->disk_sb.sb, opt, v, OPT_SHOW_MOUNT_STYLE); } - if (!p.pos) - prt_printf(&p, "(null)"); - - bch_info(c, "mounted version=%s opts=%s", bch2_metadata_versions[c->sb.version], p.buf); + bch_info(c, "%s", p.buf); printbuf_exit(&p); } @@ -874,7 +902,7 @@ int bch2_fs_start(struct bch_fs *c) struct bch_dev *ca; time64_t now = ktime_get_real_seconds(); unsigned i; - int ret = -EINVAL; + int ret; down_write(&c->state_lock); @@ -911,9 +939,9 @@ int bch2_fs_start(struct bch_fs *c) if (ret) goto err; - ret = -EINVAL; if (bch2_fs_init_fault("fs_start")) { bch_err(c, "fs_start fault injected"); + ret = -EINVAL; goto err; } @@ -936,46 +964,43 @@ out: return ret; err: bch_err(c, "error starting filesystem: %s", bch2_err_str(ret)); - - if (ret < -BCH_ERR_START) - ret = -EINVAL; goto out; } -static const char *bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c) +static int bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c) { struct bch_sb_field_members *sb_mi; sb_mi = bch2_sb_get_members(sb); if (!sb_mi) - return "Invalid superblock: member info area missing"; + return -BCH_ERR_member_info_missing; if (le16_to_cpu(sb->block_size) != block_sectors(c)) - return "mismatched block size"; + return -BCH_ERR_mismatched_block_size; if (le16_to_cpu(sb_mi->members[sb->dev_idx].bucket_size) < BCH_SB_BTREE_NODE_SIZE(c->disk_sb.sb)) - return "new cache bucket size is too small"; + return -BCH_ERR_bucket_size_too_small; - return NULL; + return 0; } -static const char *bch2_dev_in_fs(struct bch_sb *fs, struct bch_sb *sb) +static int bch2_dev_in_fs(struct bch_sb *fs, struct bch_sb *sb) { struct bch_sb *newest = le64_to_cpu(fs->seq) > le64_to_cpu(sb->seq) ? fs : sb; struct bch_sb_field_members *mi = bch2_sb_get_members(newest); if (uuid_le_cmp(fs->uuid, sb->uuid)) - return "device not a member of filesystem"; + return -BCH_ERR_device_not_a_member_of_filesystem; if (!bch2_dev_exists(newest, mi, sb->dev_idx)) - return "device has been removed"; + return -BCH_ERR_device_has_been_removed; if (fs->block_size != sb->block_size) - return "mismatched block size"; + return -BCH_ERR_mismatched_block_size; - return NULL; + return 0; } /* Device startup/shutdown: */ @@ -1173,23 +1198,17 @@ static int __bch2_dev_attach_bdev(struct bch_dev *ca, struct bch_sb_handle *sb) if (bch2_dev_is_online(ca)) { bch_err(ca, "already have device online in slot %u", sb->sb->dev_idx); - return -EINVAL; + return -BCH_ERR_device_already_online; } if (get_capacity(sb->bdev->bd_disk) < ca->mi.bucket_size * ca->mi.nbuckets) { bch_err(ca, "cannot online: device too small"); - return -EINVAL; + return -BCH_ERR_device_size_too_small; } BUG_ON(!percpu_ref_is_zero(&ca->io_ref)); - if (get_capacity(sb->bdev->bd_disk) < - ca->mi.bucket_size * ca->mi.nbuckets) { - bch_err(ca, "device too small"); - return -EINVAL; - } - ret = bch2_dev_journal_init(ca, sb->sb); if (ret) return ret; @@ -1358,7 +1377,7 @@ int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca, return 0; if (!bch2_dev_state_allowed(c, ca, new_state, flags)) - return -EINVAL; + return -BCH_ERR_device_state_not_allowed; if (new_state != BCH_MEMBER_STATE_rw) __bch2_dev_read_only(c, ca); @@ -1423,7 +1442,7 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) { struct bch_sb_field_members *mi; unsigned dev_idx = ca->dev_idx, data; - int ret = -EINVAL; + int ret; down_write(&c->state_lock); @@ -1435,6 +1454,7 @@ int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags) if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) { bch_err(ca, "Cannot remove without losing data"); + ret = -BCH_ERR_device_state_not_allowed; goto err; } @@ -1520,7 +1540,6 @@ int bch2_dev_add(struct bch_fs *c, const char *path) { struct bch_opts opts = bch2_opts_empty(); struct bch_sb_handle sb; - const char *err; struct bch_dev *ca = NULL; struct bch_sb_field_members *mi; struct bch_member dev_mi; @@ -1545,10 +1564,9 @@ int bch2_dev_add(struct bch_fs *c, const char *path) } } - err = bch2_dev_may_add(sb.sb, c); - if (err) { - bch_err(c, "device add error: %s", err); - ret = -EINVAL; + ret = bch2_dev_may_add(sb.sb, c); + if (ret) { + bch_err(c, "device add error: %s", bch2_err_str(ret)); goto err; } @@ -1682,7 +1700,6 @@ int bch2_dev_online(struct bch_fs *c, const char *path) struct bch_sb_field_members *mi; struct bch_dev *ca; unsigned dev_idx; - const char *err; int ret; down_write(&c->state_lock); @@ -1695,9 +1712,9 @@ int bch2_dev_online(struct bch_fs *c, const char *path) dev_idx = sb.sb->dev_idx; - err = bch2_dev_in_fs(c->disk_sb.sb, sb.sb); - if (err) { - bch_err(c, "error bringing %s online: %s", path, err); + ret = bch2_dev_in_fs(c->disk_sb.sb, sb.sb); + if (ret) { + bch_err(c, "error bringing %s online: %s", path, bch2_err_str(ret)); goto err; } @@ -1731,7 +1748,7 @@ int bch2_dev_online(struct bch_fs *c, const char *path) err: up_write(&c->state_lock); bch2_free_super(&sb); - return -EINVAL; + return ret; } int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags) @@ -1747,7 +1764,7 @@ int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags) if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) { bch_err(ca, "Cannot offline required disk"); up_write(&c->state_lock); - return -EINVAL; + return -BCH_ERR_device_state_not_allowed; } __bch2_dev_offline(c, ca); @@ -1773,7 +1790,7 @@ int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets) get_capacity(ca->disk_sb.bdev->bd_disk) < ca->mi.bucket_size * nbuckets) { bch_err(ca, "New size larger than device"); - ret = -EINVAL; + ret = -BCH_ERR_device_size_too_small; goto err; } @@ -1826,7 +1843,6 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices, struct bch_fs *c = NULL; struct bch_sb_field_members *mi; unsigned i, best_sb = 0; - const char *err; struct printbuf errbuf = PRINTBUF; int ret = 0; @@ -1870,8 +1886,8 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices, continue; } - err = bch2_dev_in_fs(sb[best_sb].sb, sb[i].sb); - if (err) + ret = bch2_dev_in_fs(sb[best_sb].sb, sb[i].sb); + if (ret) goto err_print; i++; } @@ -1892,9 +1908,10 @@ struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices, } up_write(&c->state_lock); - err = "insufficient devices"; - if (!bch2_fs_may_start(c)) + if (!bch2_fs_may_start(c)) { + ret = -BCH_ERR_insufficient_devices_to_start; goto err_print; + } if (!c->opts.nostart) { ret = bch2_fs_start(c); @@ -1905,12 +1922,12 @@ out: kfree(sb); printbuf_exit(&errbuf); module_put(THIS_MODULE); - pr_verbose_init(opts, "ret %i", PTR_ERR_OR_ZERO(c)); + pr_verbose_init(opts, "ret %s (%i)", bch2_err_str(PTR_ERR_OR_ZERO(c)), + PTR_ERR_OR_ZERO(c)); return c; err_print: pr_err("bch_fs_open err opening %s: %s", - devices[0], err); - ret = -EINVAL; + devices[0], bch2_err_str(ret)); err: if (!IS_ERR_OR_NULL(c)) bch2_fs_stop(c); @@ -1957,5 +1974,8 @@ err: BCH_DEBUG_PARAMS() #undef BCH_DEBUG_PARAM +unsigned bch2_metadata_version = bcachefs_metadata_version_current; +module_param_named(version, bch2_metadata_version, uint, 0400); + module_exit(bcachefs_exit); module_init(bcachefs_init);