git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/super.c

   1 /*
   2  * bcachefs setup/teardown code, and some metadata io - read a superblock and
   3  * figure out what to do with it.
   4  *
   5  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
   6  * Copyright 2012 Google, Inc.
   7  */
   8
   9 #include "bcachefs.h"
  10 #include "alloc_background.h"
  11 #include "alloc_foreground.h"
  12 #include "btree_cache.h"
  13 #include "btree_gc.h"
  14 #include "btree_update_interior.h"
  15 #include "btree_io.h"
  16 #include "chardev.h"
  17 #include "checksum.h"
  18 #include "clock.h"
  19 #include "compress.h"
  20 #include "debug.h"
  21 #include "disk_groups.h"
  22 #include "error.h"
  23 #include "fs.h"
  24 #include "fs-io.h"
  25 #include "fsck.h"
  26 #include "inode.h"
  27 #include "io.h"
  28 #include "journal.h"
  29 #include "journal_reclaim.h"
  30 #include "move.h"
  31 #include "migrate.h"
  32 #include "movinggc.h"
  33 #include "quota.h"
  34 #include "rebalance.h"
  35 #include "recovery.h"
  36 #include "replicas.h"
  37 #include "super.h"
  38 #include "super-io.h"
  39 #include "sysfs.h"
  40
  41 #include <linux/backing-dev.h>
  42 #include <linux/blkdev.h>
  43 #include <linux/debugfs.h>
  44 #include <linux/device.h>
  45 #include <linux/genhd.h>
  46 #include <linux/idr.h>
  47 #include <linux/kthread.h>
  48 #include <linux/module.h>
  49 #include <linux/percpu.h>
  50 #include <linux/random.h>
  51 #include <linux/sysfs.h>
  52 #include <crypto/hash.h>
  53
  54 #include <trace/events/bcachefs.h>
  55
  56 MODULE_LICENSE("GPL");
  57 MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
  58
  59 #define KTYPE(type)                                                     \
  60 struct kobj_type type ## _ktype = {                                     \
  61         .release        = type ## _release,                             \
  62         .sysfs_ops      = &type ## _sysfs_ops,                          \
  63         .default_attrs  = type ## _files                                \
  64 }
  65
  66 static void bch2_fs_release(struct kobject *);
  67 static void bch2_dev_release(struct kobject *);
  68
  69 static void bch2_fs_internal_release(struct kobject *k)
  70 {
  71 }
  72
  73 static void bch2_fs_opts_dir_release(struct kobject *k)
  74 {
  75 }
  76
  77 static void bch2_fs_time_stats_release(struct kobject *k)
  78 {
  79 }
  80
  81 static KTYPE(bch2_fs);
  82 static KTYPE(bch2_fs_internal);
  83 static KTYPE(bch2_fs_opts_dir);
  84 static KTYPE(bch2_fs_time_stats);
  85 static KTYPE(bch2_dev);
  86
  87 static struct kset *bcachefs_kset;
  88 static LIST_HEAD(bch_fs_list);
  89 static DEFINE_MUTEX(bch_fs_list_lock);
  90
  91 static DECLARE_WAIT_QUEUE_HEAD(bch_read_only_wait);
  92
  93 static void bch2_dev_free(struct bch_dev *);
  94 static int bch2_dev_alloc(struct bch_fs *, unsigned);
  95 static int bch2_dev_sysfs_online(struct bch_fs *, struct bch_dev *);
  96 static void __bch2_dev_read_only(struct bch_fs *, struct bch_dev *);
  97
  98 struct bch_fs *bch2_bdev_to_fs(struct block_device *bdev)
  99 {
 100         struct bch_fs *c;
 101         struct bch_dev *ca;
 102         unsigned i;
 103
 104         mutex_lock(&bch_fs_list_lock);
 105         rcu_read_lock();
 106
 107         list_for_each_entry(c, &bch_fs_list, list)
 108                 for_each_member_device_rcu(ca, c, i, NULL)
 109                         if (ca->disk_sb.bdev == bdev) {
 110                                 closure_get(&c->cl);
 111                                 goto found;
 112                         }
 113         c = NULL;
 114 found:
 115         rcu_read_unlock();
 116         mutex_unlock(&bch_fs_list_lock);
 117
 118         return c;
 119 }
 120
 121 static struct bch_fs *__bch2_uuid_to_fs(uuid_le uuid)
 122 {
 123         struct bch_fs *c;
 124
 125         lockdep_assert_held(&bch_fs_list_lock);
 126
 127         list_for_each_entry(c, &bch_fs_list, list)
 128                 if (!memcmp(&c->disk_sb.sb->uuid, &uuid, sizeof(uuid_le)))
 129                         return c;
 130
 131         return NULL;
 132 }
 133
 134 struct bch_fs *bch2_uuid_to_fs(uuid_le uuid)
 135 {
 136         struct bch_fs *c;
 137
 138         mutex_lock(&bch_fs_list_lock);
 139         c = __bch2_uuid_to_fs(uuid);
 140         if (c)
 141                 closure_get(&c->cl);
 142         mutex_unlock(&bch_fs_list_lock);
 143
 144         return c;
 145 }
 146
 147 int bch2_congested(void *data, int bdi_bits)
 148 {
 149         struct bch_fs *c = data;
 150         struct backing_dev_info *bdi;
 151         struct bch_dev *ca;
 152         unsigned i;
 153         int ret = 0;
 154
 155         rcu_read_lock();
 156         if (bdi_bits & (1 << WB_sync_congested)) {
 157                 /* Reads - check all devices: */
 158                 for_each_readable_member(ca, c, i) {
 159                         bdi = ca->disk_sb.bdev->bd_bdi;
 160
 161                         if (bdi_congested(bdi, bdi_bits)) {
 162                                 ret = 1;
 163                                 break;
 164                         }
 165                 }
 166         } else {
 167                 unsigned target = READ_ONCE(c->opts.foreground_target);
 168                 const struct bch_devs_mask *devs = target
 169                         ? bch2_target_to_mask(c, target)
 170                         : &c->rw_devs[BCH_DATA_USER];
 171
 172                 for_each_member_device_rcu(ca, c, i, devs) {
 173                         bdi = ca->disk_sb.bdev->bd_bdi;
 174
 175                         if (bdi_congested(bdi, bdi_bits)) {
 176                                 ret = 1;
 177                                 break;
 178                         }
 179                 }
 180         }
 181         rcu_read_unlock();
 182
 183         return ret;
 184 }
 185
 186 /* Filesystem RO/RW: */
 187
 188 /*
 189  * For startup/shutdown of RW stuff, the dependencies are:
 190  *
 191  * - foreground writes depend on copygc and rebalance (to free up space)
 192  *
 193  * - copygc and rebalance depend on mark and sweep gc (they actually probably
 194  *   don't because they either reserve ahead of time or don't block if
 195  *   allocations fail, but allocations can require mark and sweep gc to run
 196  *   because of generation number wraparound)
 197  *
 198  * - all of the above depends on the allocator threads
 199  *
 200  * - allocator depends on the journal (when it rewrites prios and gens)
 201  */
 202
 203 static void __bch2_fs_read_only(struct bch_fs *c)
 204 {
 205         struct bch_dev *ca;
 206         unsigned i;
 207
 208         bch2_rebalance_stop(c);
 209
 210         for_each_member_device(ca, c, i)
 211                 bch2_copygc_stop(ca);
 212
 213         bch2_gc_thread_stop(c);
 214
 215         /*
 216          * Flush journal before stopping allocators, because flushing journal
 217          * blacklist entries involves allocating new btree nodes:
 218          */
 219         bch2_journal_flush_all_pins(&c->journal);
 220
 221         for_each_member_device(ca, c, i)
 222                 bch2_dev_allocator_stop(ca);
 223
 224         bch2_journal_flush_all_pins(&c->journal);
 225
 226         /*
 227          * We need to explicitly wait on btree interior updates to complete
 228          * before stopping the journal, flushing all journal pins isn't
 229          * sufficient, because in the BTREE_INTERIOR_UPDATING_ROOT case btree
 230          * interior updates have to drop their journal pin before they're
 231          * fully complete:
 232          */
 233         closure_wait_event(&c->btree_interior_update_wait,
 234                            !bch2_btree_interior_updates_nr_pending(c));
 235
 236         bch2_fs_journal_stop(&c->journal);
 237
 238         /*
 239          * the journal kicks off btree writes via reclaim - wait for in flight
 240          * writes after stopping journal:
 241          */
 242         if (test_bit(BCH_FS_EMERGENCY_RO, &c->flags))
 243                 bch2_btree_flush_all_writes(c);
 244         else
 245                 bch2_btree_verify_flushed(c);
 246
 247         /*
 248          * After stopping journal:
 249          */
 250         for_each_member_device(ca, c, i)
 251                 bch2_dev_allocator_remove(c, ca);
 252 }
 253
 254 static void bch2_writes_disabled(struct percpu_ref *writes)
 255 {
 256         struct bch_fs *c = container_of(writes, struct bch_fs, writes);
 257
 258         set_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags);
 259         wake_up(&bch_read_only_wait);
 260 }
 261
 262 void bch2_fs_read_only(struct bch_fs *c)
 263 {
 264         if (c->state == BCH_FS_RO)
 265                 return;
 266
 267         BUG_ON(test_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags));
 268
 269         /*
 270          * Block new foreground-end write operations from starting - any new
 271          * writes will return -EROFS:
 272          *
 273          * (This is really blocking new _allocations_, writes to previously
 274          * allocated space can still happen until stopping the allocator in
 275          * bch2_dev_allocator_stop()).
 276          */
 277         percpu_ref_kill(&c->writes);
 278
 279         cancel_delayed_work(&c->pd_controllers_update);
 280
 281         /*
 282          * If we're not doing an emergency shutdown, we want to wait on
 283          * outstanding writes to complete so they don't see spurious errors due
 284          * to shutting down the allocator:
 285          *
 286          * If we are doing an emergency shutdown outstanding writes may
 287          * hang until we shutdown the allocator so we don't want to wait
 288          * on outstanding writes before shutting everything down - but
 289          * we do need to wait on them before returning and signalling
 290          * that going RO is complete:
 291          */
 292         wait_event(bch_read_only_wait,
 293                    test_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags) ||
 294                    test_bit(BCH_FS_EMERGENCY_RO, &c->flags));
 295
 296         __bch2_fs_read_only(c);
 297
 298         wait_event(bch_read_only_wait,
 299                    test_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags));
 300
 301         clear_bit(BCH_FS_WRITE_DISABLE_COMPLETE, &c->flags);
 302
 303         if (!bch2_journal_error(&c->journal) &&
 304             !test_bit(BCH_FS_ERROR, &c->flags) &&
 305             !test_bit(BCH_FS_EMERGENCY_RO, &c->flags))
 306                 bch2_fs_mark_clean(c, true);
 307
 308         if (c->state != BCH_FS_STOPPING)
 309                 c->state = BCH_FS_RO;
 310 }
 311
 312 static void bch2_fs_read_only_work(struct work_struct *work)
 313 {
 314         struct bch_fs *c =
 315                 container_of(work, struct bch_fs, read_only_work);
 316
 317         mutex_lock(&c->state_lock);
 318         bch2_fs_read_only(c);
 319         mutex_unlock(&c->state_lock);
 320 }
 321
 322 static void bch2_fs_read_only_async(struct bch_fs *c)
 323 {
 324         queue_work(system_long_wq, &c->read_only_work);
 325 }
 326
 327 bool bch2_fs_emergency_read_only(struct bch_fs *c)
 328 {
 329         bool ret = !test_and_set_bit(BCH_FS_EMERGENCY_RO, &c->flags);
 330
 331         bch2_fs_read_only_async(c);
 332         bch2_journal_halt(&c->journal);
 333
 334         wake_up(&bch_read_only_wait);
 335         return ret;
 336 }
 337
 338 const char *bch2_fs_read_write(struct bch_fs *c)
 339 {
 340         struct bch_dev *ca;
 341         const char *err = NULL;
 342         unsigned i;
 343
 344         if (c->state == BCH_FS_RW)
 345                 return NULL;
 346
 347         bch2_fs_mark_clean(c, false);
 348
 349         for_each_rw_member(ca, c, i)
 350                 bch2_dev_allocator_add(c, ca);
 351         bch2_recalc_capacity(c);
 352
 353         err = "error starting allocator thread";
 354         for_each_rw_member(ca, c, i)
 355                 if (bch2_dev_allocator_start(ca)) {
 356                         percpu_ref_put(&ca->io_ref);
 357                         goto err;
 358                 }
 359
 360         err = "error starting btree GC thread";
 361         if (bch2_gc_thread_start(c))
 362                 goto err;
 363
 364         err = "error starting copygc thread";
 365         for_each_rw_member(ca, c, i)
 366                 if (bch2_copygc_start(c, ca)) {
 367                         percpu_ref_put(&ca->io_ref);
 368                         goto err;
 369                 }
 370
 371         err = "error starting rebalance thread";
 372         if (bch2_rebalance_start(c))
 373                 goto err;
 374
 375         schedule_delayed_work(&c->pd_controllers_update, 5 * HZ);
 376
 377         if (c->state != BCH_FS_STARTING)
 378                 percpu_ref_reinit(&c->writes);
 379
 380         c->state = BCH_FS_RW;
 381         return NULL;
 382 err:
 383         __bch2_fs_read_only(c);
 384         return err;
 385 }
 386
 387 /* Filesystem startup/shutdown: */
 388
 389 static void bch2_fs_free(struct bch_fs *c)
 390 {
 391         unsigned i;
 392
 393         for (i = 0; i < BCH_TIME_STAT_NR; i++)
 394                 bch2_time_stats_exit(&c->times[i]);
 395
 396         bch2_fs_quota_exit(c);
 397         bch2_fs_fsio_exit(c);
 398         bch2_fs_encryption_exit(c);
 399         bch2_fs_io_exit(c);
 400         bch2_fs_btree_cache_exit(c);
 401         bch2_fs_journal_exit(&c->journal);
 402         bch2_io_clock_exit(&c->io_clock[WRITE]);
 403         bch2_io_clock_exit(&c->io_clock[READ]);
 404         bch2_fs_compress_exit(c);
 405         percpu_free_rwsem(&c->usage_lock);
 406         free_percpu(c->usage_percpu);
 407         mempool_exit(&c->btree_iters_pool);
 408         mempool_exit(&c->btree_bounce_pool);
 409         bioset_exit(&c->btree_bio);
 410         mempool_exit(&c->btree_interior_update_pool);
 411         mempool_exit(&c->btree_reserve_pool);
 412         mempool_exit(&c->fill_iter);
 413         percpu_ref_exit(&c->writes);
 414         kfree(rcu_dereference_protected(c->replicas, 1));
 415         kfree(rcu_dereference_protected(c->disk_groups, 1));
 416
 417         if (c->copygc_wq)
 418                 destroy_workqueue(c->copygc_wq);
 419         if (c->wq)
 420                 destroy_workqueue(c->wq);
 421
 422         free_pages((unsigned long) c->disk_sb.sb,
 423                    c->disk_sb.page_order);
 424         kvpfree(c, sizeof(*c));
 425         module_put(THIS_MODULE);
 426 }
 427
 428 static void bch2_fs_release(struct kobject *kobj)
 429 {
 430         struct bch_fs *c = container_of(kobj, struct bch_fs, kobj);
 431
 432         bch2_fs_free(c);
 433 }
 434
 435 void bch2_fs_stop(struct bch_fs *c)
 436 {
 437         struct bch_dev *ca;
 438         unsigned i;
 439
 440         bch_verbose(c, "shutting down");
 441
 442         for_each_member_device(ca, c, i)
 443                 if (ca->kobj.state_in_sysfs &&
 444                     ca->disk_sb.bdev)
 445                         sysfs_remove_link(&part_to_dev(ca->disk_sb.bdev->bd_part)->kobj,
 446                                           "bcachefs");
 447
 448         if (c->kobj.state_in_sysfs)
 449                 kobject_del(&c->kobj);
 450
 451         bch2_fs_debug_exit(c);
 452         bch2_fs_chardev_exit(c);
 453
 454         kobject_put(&c->time_stats);
 455         kobject_put(&c->opts_dir);
 456         kobject_put(&c->internal);
 457
 458         mutex_lock(&bch_fs_list_lock);
 459         list_del(&c->list);
 460         mutex_unlock(&bch_fs_list_lock);
 461
 462         closure_sync(&c->cl);
 463         closure_debug_destroy(&c->cl);
 464
 465         mutex_lock(&c->state_lock);
 466         bch2_fs_read_only(c);
 467         mutex_unlock(&c->state_lock);
 468
 469         /* btree prefetch might have kicked off reads in the background: */
 470         bch2_btree_flush_all_reads(c);
 471
 472         for_each_member_device(ca, c, i)
 473                 cancel_work_sync(&ca->io_error_work);
 474
 475         cancel_work_sync(&c->btree_write_error_work);
 476         cancel_delayed_work_sync(&c->pd_controllers_update);
 477         cancel_work_sync(&c->read_only_work);
 478
 479         for (i = 0; i < c->sb.nr_devices; i++)
 480                 if (c->devs[i])
 481                         bch2_dev_free(rcu_dereference_protected(c->devs[i], 1));
 482
 483         bch_verbose(c, "shutdown complete");
 484
 485         kobject_put(&c->kobj);
 486 }
 487
 488 static const char *bch2_fs_online(struct bch_fs *c)
 489 {
 490         struct bch_dev *ca;
 491         const char *err = NULL;
 492         unsigned i;
 493         int ret;
 494
 495         lockdep_assert_held(&bch_fs_list_lock);
 496
 497         if (!list_empty(&c->list))
 498                 return NULL;
 499
 500         if (__bch2_uuid_to_fs(c->sb.uuid))
 501                 return "filesystem UUID already open";
 502
 503         ret = bch2_fs_chardev_init(c);
 504         if (ret)
 505                 return "error creating character device";
 506
 507         bch2_fs_debug_init(c);
 508
 509         if (kobject_add(&c->kobj, NULL, "%pU", c->sb.user_uuid.b) ||
 510             kobject_add(&c->internal, &c->kobj, "internal") ||
 511             kobject_add(&c->opts_dir, &c->kobj, "options") ||
 512             kobject_add(&c->time_stats, &c->kobj, "time_stats") ||
 513             bch2_opts_create_sysfs_files(&c->opts_dir))
 514                 return "error creating sysfs objects";
 515
 516         mutex_lock(&c->state_lock);
 517
 518         err = "error creating sysfs objects";
 519         __for_each_member_device(ca, c, i, NULL)
 520                 if (bch2_dev_sysfs_online(c, ca))
 521                         goto err;
 522
 523         list_add(&c->list, &bch_fs_list);
 524         err = NULL;
 525 err:
 526         mutex_unlock(&c->state_lock);
 527         return err;
 528 }
 529
 530 static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
 531 {
 532         struct bch_sb_field_members *mi;
 533         struct bch_fs *c;
 534         unsigned i, iter_size;
 535         const char *err;
 536
 537         pr_verbose_init(opts, "");
 538
 539         c = kvpmalloc(sizeof(struct bch_fs), GFP_KERNEL|__GFP_ZERO);
 540         if (!c)
 541                 goto out;
 542
 543         __module_get(THIS_MODULE);
 544
 545         c->minor                = -1;
 546         c->disk_sb.fs_sb        = true;
 547
 548         mutex_init(&c->state_lock);
 549         mutex_init(&c->sb_lock);
 550         mutex_init(&c->replicas_gc_lock);
 551         mutex_init(&c->btree_root_lock);
 552         INIT_WORK(&c->read_only_work, bch2_fs_read_only_work);
 553
 554         init_rwsem(&c->gc_lock);
 555
 556         for (i = 0; i < BCH_TIME_STAT_NR; i++)
 557                 bch2_time_stats_init(&c->times[i]);
 558
 559         bch2_fs_allocator_background_init(c);
 560         bch2_fs_allocator_foreground_init(c);
 561         bch2_fs_rebalance_init(c);
 562         bch2_fs_quota_init(c);
 563
 564         INIT_LIST_HEAD(&c->list);
 565
 566         INIT_LIST_HEAD(&c->btree_interior_update_list);
 567         mutex_init(&c->btree_reserve_cache_lock);
 568         mutex_init(&c->btree_interior_update_lock);
 569
 570         mutex_init(&c->bio_bounce_pages_lock);
 571
 572         bio_list_init(&c->btree_write_error_list);
 573         spin_lock_init(&c->btree_write_error_lock);
 574         INIT_WORK(&c->btree_write_error_work, bch2_btree_write_error_work);
 575
 576         INIT_LIST_HEAD(&c->fsck_errors);
 577         mutex_init(&c->fsck_error_lock);
 578
 579         seqcount_init(&c->gc_pos_lock);
 580
 581         c->copy_gc_enabled              = 1;
 582         c->rebalance.enabled            = 1;
 583         c->promote_whole_extents        = true;
 584
 585         c->journal.write_time   = &c->times[BCH_TIME_journal_write];
 586         c->journal.delay_time   = &c->times[BCH_TIME_journal_delay];
 587         c->journal.blocked_time = &c->times[BCH_TIME_journal_blocked];
 588         c->journal.flush_seq_time = &c->times[BCH_TIME_journal_flush_seq];
 589
 590         bch2_fs_btree_cache_init_early(&c->btree_cache);
 591
 592         mutex_lock(&c->sb_lock);
 593
 594         if (bch2_sb_to_fs(c, sb)) {
 595                 mutex_unlock(&c->sb_lock);
 596                 goto err;
 597         }
 598
 599         mutex_unlock(&c->sb_lock);
 600
 601         scnprintf(c->name, sizeof(c->name), "%pU", &c->sb.user_uuid);
 602
 603         c->opts = bch2_opts_default;
 604         bch2_opts_apply(&c->opts, bch2_opts_from_sb(sb));
 605         bch2_opts_apply(&c->opts, opts);
 606
 607         c->block_bits           = ilog2(c->opts.block_size);
 608         c->btree_foreground_merge_threshold = BTREE_FOREGROUND_MERGE_THRESHOLD(c);
 609
 610         c->opts.nochanges       |= c->opts.noreplay;
 611         c->opts.read_only       |= c->opts.nochanges;
 612
 613         if (bch2_fs_init_fault("fs_alloc"))
 614                 goto err;
 615
 616         iter_size = sizeof(struct btree_node_iter_large) +
 617                 (btree_blocks(c) + 1) * 2 *
 618                 sizeof(struct btree_node_iter_set);
 619
 620         if (!(c->wq = alloc_workqueue("bcachefs",
 621                                 WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_HIGHPRI, 1)) ||
 622             !(c->copygc_wq = alloc_workqueue("bcache_copygc",
 623                                 WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_HIGHPRI, 1)) ||
 624             percpu_ref_init(&c->writes, bch2_writes_disabled, 0, GFP_KERNEL) ||
 625             mempool_init_kmalloc_pool(&c->btree_reserve_pool, 1,
 626                                       sizeof(struct btree_reserve)) ||
 627             mempool_init_kmalloc_pool(&c->btree_interior_update_pool, 1,
 628                                       sizeof(struct btree_update)) ||
 629             mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) ||
 630             bioset_init(&c->btree_bio, 1,
 631                         max(offsetof(struct btree_read_bio, bio),
 632                             offsetof(struct btree_write_bio, wbio.bio)),
 633                         BIOSET_NEED_BVECS) ||
 634             !(c->usage_percpu = alloc_percpu(struct bch_fs_usage)) ||
 635             percpu_init_rwsem(&c->usage_lock) ||
 636             mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1,
 637                                         btree_bytes(c)) ||
 638             mempool_init_kmalloc_pool(&c->btree_iters_pool, 1,
 639                         sizeof(struct btree_iter) * BTREE_ITER_MAX) ||
 640             bch2_io_clock_init(&c->io_clock[READ]) ||
 641             bch2_io_clock_init(&c->io_clock[WRITE]) ||
 642             bch2_fs_journal_init(&c->journal) ||
 643             bch2_fs_btree_cache_init(c) ||
 644             bch2_fs_io_init(c) ||
 645             bch2_fs_encryption_init(c) ||
 646             bch2_fs_compress_init(c) ||
 647             bch2_fs_fsio_init(c))
 648                 goto err;
 649
 650         mi = bch2_sb_get_members(c->disk_sb.sb);
 651         for (i = 0; i < c->sb.nr_devices; i++)
 652                 if (bch2_dev_exists(c->disk_sb.sb, mi, i) &&
 653                     bch2_dev_alloc(c, i))
 654                         goto err;
 655
 656         /*
 657          * Now that all allocations have succeeded, init various refcounty
 658          * things that let us shutdown:
 659          */
 660         closure_init(&c->cl, NULL);
 661
 662         c->kobj.kset = bcachefs_kset;
 663         kobject_init(&c->kobj, &bch2_fs_ktype);
 664         kobject_init(&c->internal, &bch2_fs_internal_ktype);
 665         kobject_init(&c->opts_dir, &bch2_fs_opts_dir_ktype);
 666         kobject_init(&c->time_stats, &bch2_fs_time_stats_ktype);
 667
 668         mutex_lock(&bch_fs_list_lock);
 669         err = bch2_fs_online(c);
 670         mutex_unlock(&bch_fs_list_lock);
 671         if (err) {
 672                 bch_err(c, "bch2_fs_online() error: %s", err);
 673                 goto err;
 674         }
 675 out:
 676         pr_verbose_init(opts, "ret %i", c ? 0 : -ENOMEM);
 677         return c;
 678 err:
 679         bch2_fs_free(c);
 680         c = NULL;
 681         goto out;
 682 }
 683
 684 const char *bch2_fs_start(struct bch_fs *c)
 685 {
 686         const char *err = "cannot allocate memory";
 687         struct bch_sb_field_members *mi;
 688         struct bch_dev *ca;
 689         time64_t now = ktime_get_real_seconds();
 690         unsigned i;
 691         int ret = -EINVAL;
 692
 693         mutex_lock(&c->state_lock);
 694
 695         BUG_ON(c->state != BCH_FS_STARTING);
 696
 697         mutex_lock(&c->sb_lock);
 698
 699         for_each_online_member(ca, c, i)
 700                 bch2_sb_from_fs(c, ca);
 701
 702         mi = bch2_sb_get_members(c->disk_sb.sb);
 703         for_each_online_member(ca, c, i)
 704                 mi->members[ca->dev_idx].last_mount = cpu_to_le64(now);
 705
 706         mutex_unlock(&c->sb_lock);
 707
 708         for_each_rw_member(ca, c, i)
 709                 bch2_dev_allocator_add(c, ca);
 710         bch2_recalc_capacity(c);
 711
 712         ret = BCH_SB_INITIALIZED(c->disk_sb.sb)
 713                 ? bch2_fs_recovery(c)
 714                 : bch2_fs_initialize(c);
 715         if (ret)
 716                 goto err;
 717
 718         err = "dynamic fault";
 719         if (bch2_fs_init_fault("fs_start"))
 720                 goto err;
 721
 722         if (c->opts.read_only) {
 723                 bch2_fs_read_only(c);
 724         } else {
 725                 err = bch2_fs_read_write(c);
 726                 if (err)
 727                         goto err;
 728         }
 729
 730         set_bit(BCH_FS_STARTED, &c->flags);
 731
 732         err = NULL;
 733 out:
 734         mutex_unlock(&c->state_lock);
 735         return err;
 736 err:
 737         switch (ret) {
 738         case BCH_FSCK_ERRORS_NOT_FIXED:
 739                 bch_err(c, "filesystem contains errors: please report this to the developers");
 740                 pr_cont("mount with -o fix_errors to repair\n");
 741                 err = "fsck error";
 742                 break;
 743         case BCH_FSCK_REPAIR_UNIMPLEMENTED:
 744                 bch_err(c, "filesystem contains errors: please report this to the developers");
 745                 pr_cont("repair unimplemented: inform the developers so that it can be added\n");
 746                 err = "fsck error";
 747                 break;
 748         case BCH_FSCK_REPAIR_IMPOSSIBLE:
 749                 bch_err(c, "filesystem contains errors, but repair impossible");
 750                 err = "fsck error";
 751                 break;
 752         case BCH_FSCK_UNKNOWN_VERSION:
 753                 err = "unknown metadata version";;
 754                 break;
 755         case -ENOMEM:
 756                 err = "cannot allocate memory";
 757                 break;
 758         case -EIO:
 759                 err = "IO error";
 760                 break;
 761         }
 762
 763         BUG_ON(!err);
 764         set_bit(BCH_FS_ERROR, &c->flags);
 765         goto out;
 766 }
 767
 768 static const char *bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c)
 769 {
 770         struct bch_sb_field_members *sb_mi;
 771
 772         sb_mi = bch2_sb_get_members(sb);
 773         if (!sb_mi)
 774                 return "Invalid superblock: member info area missing";
 775
 776         if (le16_to_cpu(sb->block_size) != c->opts.block_size)
 777                 return "mismatched block size";
 778
 779         if (le16_to_cpu(sb_mi->members[sb->dev_idx].bucket_size) <
 780             BCH_SB_BTREE_NODE_SIZE(c->disk_sb.sb))
 781                 return "new cache bucket size is too small";
 782
 783         return NULL;
 784 }
 785
 786 static const char *bch2_dev_in_fs(struct bch_sb *fs, struct bch_sb *sb)
 787 {
 788         struct bch_sb *newest =
 789                 le64_to_cpu(fs->seq) > le64_to_cpu(sb->seq) ? fs : sb;
 790         struct bch_sb_field_members *mi = bch2_sb_get_members(newest);
 791
 792         if (uuid_le_cmp(fs->uuid, sb->uuid))
 793                 return "device not a member of filesystem";
 794
 795         if (!bch2_dev_exists(newest, mi, sb->dev_idx))
 796                 return "device has been removed";
 797
 798         if (fs->block_size != sb->block_size)
 799                 return "mismatched block size";
 800
 801         return NULL;
 802 }
 803
 804 /* Device startup/shutdown: */
 805
 806 static void bch2_dev_release(struct kobject *kobj)
 807 {
 808         struct bch_dev *ca = container_of(kobj, struct bch_dev, kobj);
 809
 810         kfree(ca);
 811 }
 812
 813 static void bch2_dev_free(struct bch_dev *ca)
 814 {
 815         cancel_work_sync(&ca->io_error_work);
 816
 817         if (ca->kobj.state_in_sysfs &&
 818             ca->disk_sb.bdev)
 819                 sysfs_remove_link(&part_to_dev(ca->disk_sb.bdev->bd_part)->kobj,
 820                                   "bcachefs");
 821
 822         if (ca->kobj.state_in_sysfs)
 823                 kobject_del(&ca->kobj);
 824
 825         bch2_free_super(&ca->disk_sb);
 826         bch2_dev_journal_exit(ca);
 827
 828         free_percpu(ca->io_done);
 829         bioset_exit(&ca->replica_set);
 830         bch2_dev_buckets_free(ca);
 831
 832         bch2_time_stats_exit(&ca->io_latency[WRITE]);
 833         bch2_time_stats_exit(&ca->io_latency[READ]);
 834
 835         percpu_ref_exit(&ca->io_ref);
 836         percpu_ref_exit(&ca->ref);
 837         kobject_put(&ca->kobj);
 838 }
 839
 840 static void __bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca)
 841 {
 842
 843         lockdep_assert_held(&c->state_lock);
 844
 845         if (percpu_ref_is_zero(&ca->io_ref))
 846                 return;
 847
 848         __bch2_dev_read_only(c, ca);
 849
 850         reinit_completion(&ca->io_ref_completion);
 851         percpu_ref_kill(&ca->io_ref);
 852         wait_for_completion(&ca->io_ref_completion);
 853
 854         if (ca->kobj.state_in_sysfs) {
 855                 struct kobject *block =
 856                         &part_to_dev(ca->disk_sb.bdev->bd_part)->kobj;
 857
 858                 sysfs_remove_link(block, "bcachefs");
 859                 sysfs_remove_link(&ca->kobj, "block");
 860         }
 861
 862         bch2_free_super(&ca->disk_sb);
 863         bch2_dev_journal_exit(ca);
 864 }
 865
 866 static void bch2_dev_ref_complete(struct percpu_ref *ref)
 867 {
 868         struct bch_dev *ca = container_of(ref, struct bch_dev, ref);
 869
 870         complete(&ca->ref_completion);
 871 }
 872
 873 static void bch2_dev_io_ref_complete(struct percpu_ref *ref)
 874 {
 875         struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref);
 876
 877         complete(&ca->io_ref_completion);
 878 }
 879
 880 static int bch2_dev_sysfs_online(struct bch_fs *c, struct bch_dev *ca)
 881 {
 882         int ret;
 883
 884         if (!c->kobj.state_in_sysfs)
 885                 return 0;
 886
 887         if (!ca->kobj.state_in_sysfs) {
 888                 ret = kobject_add(&ca->kobj, &c->kobj,
 889                                   "dev-%u", ca->dev_idx);
 890                 if (ret)
 891                         return ret;
 892         }
 893
 894         if (ca->disk_sb.bdev) {
 895                 struct kobject *block =
 896                         &part_to_dev(ca->disk_sb.bdev->bd_part)->kobj;
 897
 898                 ret = sysfs_create_link(block, &ca->kobj, "bcachefs");
 899                 if (ret)
 900                         return ret;
 901                 ret = sysfs_create_link(&ca->kobj, block, "block");
 902                 if (ret)
 903                         return ret;
 904         }
 905
 906         return 0;
 907 }
 908
 909 static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
 910                                         struct bch_member *member)
 911 {
 912         struct bch_dev *ca;
 913
 914         ca = kzalloc(sizeof(*ca), GFP_KERNEL);
 915         if (!ca)
 916                 return NULL;
 917
 918         kobject_init(&ca->kobj, &bch2_dev_ktype);
 919         init_completion(&ca->ref_completion);
 920         init_completion(&ca->io_ref_completion);
 921
 922         init_rwsem(&ca->bucket_lock);
 923
 924         writepoint_init(&ca->copygc_write_point, BCH_DATA_USER);
 925
 926         spin_lock_init(&ca->freelist_lock);
 927         bch2_dev_copygc_init(ca);
 928
 929         INIT_WORK(&ca->io_error_work, bch2_io_error_work);
 930
 931         bch2_time_stats_init(&ca->io_latency[READ]);
 932         bch2_time_stats_init(&ca->io_latency[WRITE]);
 933
 934         ca->mi = bch2_mi_to_cpu(member);
 935         ca->uuid = member->uuid;
 936
 937         if (opt_defined(c->opts, discard))
 938                 ca->mi.discard = opt_get(c->opts, discard);
 939
 940         if (percpu_ref_init(&ca->ref, bch2_dev_ref_complete,
 941                             0, GFP_KERNEL) ||
 942             percpu_ref_init(&ca->io_ref, bch2_dev_io_ref_complete,
 943                             PERCPU_REF_INIT_DEAD, GFP_KERNEL) ||
 944             bch2_dev_buckets_alloc(c, ca) ||
 945             bioset_init(&ca->replica_set, 4,
 946                         offsetof(struct bch_write_bio, bio), 0) ||
 947             !(ca->io_done       = alloc_percpu(*ca->io_done)))
 948                 goto err;
 949
 950         return ca;
 951 err:
 952         bch2_dev_free(ca);
 953         return NULL;
 954 }
 955
 956 static void bch2_dev_attach(struct bch_fs *c, struct bch_dev *ca,
 957                             unsigned dev_idx)
 958 {
 959         ca->dev_idx = dev_idx;
 960         __set_bit(ca->dev_idx, ca->self.d);
 961         scnprintf(ca->name, sizeof(ca->name), "dev-%u", dev_idx);
 962
 963         ca->fs = c;
 964         rcu_assign_pointer(c->devs[ca->dev_idx], ca);
 965
 966         if (bch2_dev_sysfs_online(c, ca))
 967                 pr_warn("error creating sysfs objects");
 968 }
 969
 970 static int bch2_dev_alloc(struct bch_fs *c, unsigned dev_idx)
 971 {
 972         struct bch_member *member =
 973                 bch2_sb_get_members(c->disk_sb.sb)->members + dev_idx;
 974         struct bch_dev *ca = NULL;
 975         int ret = 0;
 976
 977         pr_verbose_init(c->opts, "");
 978
 979         if (bch2_fs_init_fault("dev_alloc"))
 980                 goto err;
 981
 982         ca = __bch2_dev_alloc(c, member);
 983         if (!ca)
 984                 goto err;
 985
 986         bch2_dev_attach(c, ca, dev_idx);
 987 out:
 988         pr_verbose_init(c->opts, "ret %i", ret);
 989         return ret;
 990 err:
 991         if (ca)
 992                 bch2_dev_free(ca);
 993         ret = -ENOMEM;
 994         goto out;
 995 }
 996
 997 static int __bch2_dev_attach_bdev(struct bch_dev *ca, struct bch_sb_handle *sb)
 998 {
 999         unsigned ret;
1000
1001         if (bch2_dev_is_online(ca)) {
1002                 bch_err(ca, "already have device online in slot %u",
1003                         sb->sb->dev_idx);
1004                 return -EINVAL;
1005         }
1006
1007         if (get_capacity(sb->bdev->bd_disk) <
1008             ca->mi.bucket_size * ca->mi.nbuckets) {
1009                 bch_err(ca, "cannot online: device too small");
1010                 return -EINVAL;
1011         }
1012
1013         BUG_ON(!percpu_ref_is_zero(&ca->io_ref));
1014
1015         if (get_capacity(sb->bdev->bd_disk) <
1016             ca->mi.bucket_size * ca->mi.nbuckets) {
1017                 bch_err(ca, "device too small");
1018                 return -EINVAL;
1019         }
1020
1021         ret = bch2_dev_journal_init(ca, sb->sb);
1022         if (ret)
1023                 return ret;
1024
1025         /* Commit: */
1026         ca->disk_sb = *sb;
1027         if (sb->mode & FMODE_EXCL)
1028                 ca->disk_sb.bdev->bd_holder = ca;
1029         memset(sb, 0, sizeof(*sb));
1030
1031         percpu_ref_reinit(&ca->io_ref);
1032
1033         return 0;
1034 }
1035
1036 static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb)
1037 {
1038         struct bch_dev *ca;
1039         int ret;
1040
1041         lockdep_assert_held(&c->state_lock);
1042
1043         if (le64_to_cpu(sb->sb->seq) >
1044             le64_to_cpu(c->disk_sb.sb->seq))
1045                 bch2_sb_to_fs(c, sb->sb);
1046
1047         BUG_ON(sb->sb->dev_idx >= c->sb.nr_devices ||
1048                !c->devs[sb->sb->dev_idx]);
1049
1050         ca = bch_dev_locked(c, sb->sb->dev_idx);
1051
1052         ret = __bch2_dev_attach_bdev(ca, sb);
1053         if (ret)
1054                 return ret;
1055
1056         mutex_lock(&c->sb_lock);
1057         bch2_mark_dev_superblock(ca->fs, ca,
1058                         BCH_BUCKET_MARK_MAY_MAKE_UNAVAILABLE);
1059         mutex_unlock(&c->sb_lock);
1060
1061         bch2_dev_sysfs_online(c, ca);
1062
1063         if (c->sb.nr_devices == 1)
1064                 bdevname(ca->disk_sb.bdev, c->name);
1065         bdevname(ca->disk_sb.bdev, ca->name);
1066
1067         rebalance_wakeup(c);
1068         return 0;
1069 }
1070
1071 /* Device management: */
1072
1073 /*
1074  * Note: this function is also used by the error paths - when a particular
1075  * device sees an error, we call it to determine whether we can just set the
1076  * device RO, or - if this function returns false - we'll set the whole
1077  * filesystem RO:
1078  *
1079  * XXX: maybe we should be more explicit about whether we're changing state
1080  * because we got an error or what have you?
1081  */
1082 bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca,
1083                             enum bch_member_state new_state, int flags)
1084 {
1085         struct bch_devs_mask new_online_devs;
1086         struct replicas_status s;
1087         struct bch_dev *ca2;
1088         int i, nr_rw = 0, required;
1089
1090         lockdep_assert_held(&c->state_lock);
1091
1092         switch (new_state) {
1093         case BCH_MEMBER_STATE_RW:
1094                 return true;
1095         case BCH_MEMBER_STATE_RO:
1096                 if (ca->mi.state != BCH_MEMBER_STATE_RW)
1097                         return true;
1098
1099                 /* do we have enough devices to write to?  */
1100                 for_each_member_device(ca2, c, i)
1101                         if (ca2 != ca)
1102                                 nr_rw += ca2->mi.state == BCH_MEMBER_STATE_RW;
1103
1104                 required = max(!(flags & BCH_FORCE_IF_METADATA_DEGRADED)
1105                                ? c->opts.metadata_replicas
1106                                : c->opts.metadata_replicas_required,
1107                                !(flags & BCH_FORCE_IF_DATA_DEGRADED)
1108                                ? c->opts.data_replicas
1109                                : c->opts.data_replicas_required);
1110
1111                 return nr_rw >= required;
1112         case BCH_MEMBER_STATE_FAILED:
1113         case BCH_MEMBER_STATE_SPARE:
1114                 if (ca->mi.state != BCH_MEMBER_STATE_RW &&
1115                     ca->mi.state != BCH_MEMBER_STATE_RO)
1116                         return true;
1117
1118                 /* do we have enough devices to read from?  */
1119                 new_online_devs = bch2_online_devs(c);
1120                 __clear_bit(ca->dev_idx, new_online_devs.d);
1121
1122                 s = __bch2_replicas_status(c, new_online_devs);
1123
1124                 return bch2_have_enough_devs(s, flags);
1125         default:
1126                 BUG();
1127         }
1128 }
1129
1130 static bool bch2_fs_may_start(struct bch_fs *c)
1131 {
1132         struct replicas_status s;
1133         struct bch_sb_field_members *mi;
1134         struct bch_dev *ca;
1135         unsigned i, flags = c->opts.degraded
1136                 ? BCH_FORCE_IF_DEGRADED
1137                 : 0;
1138
1139         if (!c->opts.degraded) {
1140                 mutex_lock(&c->sb_lock);
1141                 mi = bch2_sb_get_members(c->disk_sb.sb);
1142
1143                 for (i = 0; i < c->disk_sb.sb->nr_devices; i++) {
1144                         if (!bch2_dev_exists(c->disk_sb.sb, mi, i))
1145                                 continue;
1146
1147                         ca = bch_dev_locked(c, i);
1148
1149                         if (!bch2_dev_is_online(ca) &&
1150                             (ca->mi.state == BCH_MEMBER_STATE_RW ||
1151                              ca->mi.state == BCH_MEMBER_STATE_RO)) {
1152                                 mutex_unlock(&c->sb_lock);
1153                                 return false;
1154                         }
1155                 }
1156                 mutex_unlock(&c->sb_lock);
1157         }
1158
1159         s = bch2_replicas_status(c);
1160
1161         return bch2_have_enough_devs(s, flags);
1162 }
1163
1164 static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca)
1165 {
1166         bch2_copygc_stop(ca);
1167
1168         /*
1169          * The allocator thread itself allocates btree nodes, so stop it first:
1170          */
1171         bch2_dev_allocator_stop(ca);
1172         bch2_dev_allocator_remove(c, ca);
1173         bch2_dev_journal_stop(&c->journal, ca);
1174 }
1175
1176 static const char *__bch2_dev_read_write(struct bch_fs *c, struct bch_dev *ca)
1177 {
1178         lockdep_assert_held(&c->state_lock);
1179
1180         BUG_ON(ca->mi.state != BCH_MEMBER_STATE_RW);
1181
1182         bch2_dev_allocator_add(c, ca);
1183         bch2_recalc_capacity(c);
1184
1185         if (bch2_dev_allocator_start(ca))
1186                 return "error starting allocator thread";
1187
1188         if (bch2_copygc_start(c, ca))
1189                 return "error starting copygc thread";
1190
1191         return NULL;
1192 }
1193
1194 int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,
1195                          enum bch_member_state new_state, int flags)
1196 {
1197         struct bch_sb_field_members *mi;
1198         int ret = 0;
1199
1200         if (ca->mi.state == new_state)
1201                 return 0;
1202
1203         if (!bch2_dev_state_allowed(c, ca, new_state, flags))
1204                 return -EINVAL;
1205
1206         if (new_state != BCH_MEMBER_STATE_RW)
1207                 __bch2_dev_read_only(c, ca);
1208
1209         bch_notice(ca, "%s", bch2_dev_state[new_state]);
1210
1211         mutex_lock(&c->sb_lock);
1212         mi = bch2_sb_get_members(c->disk_sb.sb);
1213         SET_BCH_MEMBER_STATE(&mi->members[ca->dev_idx], new_state);
1214         bch2_write_super(c);
1215         mutex_unlock(&c->sb_lock);
1216
1217         if (new_state == BCH_MEMBER_STATE_RW &&
1218             __bch2_dev_read_write(c, ca))
1219                 ret = -ENOMEM;
1220
1221         rebalance_wakeup(c);
1222
1223         return ret;
1224 }
1225
1226 int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,
1227                        enum bch_member_state new_state, int flags)
1228 {
1229         int ret;
1230
1231         mutex_lock(&c->state_lock);
1232         ret = __bch2_dev_set_state(c, ca, new_state, flags);
1233         mutex_unlock(&c->state_lock);
1234
1235         return ret;
1236 }
1237
1238 /* Device add/removal: */
1239
1240 int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
1241 {
1242         struct bch_sb_field_members *mi;
1243         unsigned dev_idx = ca->dev_idx, data;
1244         int ret = -EINVAL;
1245
1246         mutex_lock(&c->state_lock);
1247
1248         percpu_ref_put(&ca->ref); /* XXX */
1249
1250         if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_FAILED, flags)) {
1251                 bch_err(ca, "Cannot remove without losing data");
1252                 goto err;
1253         }
1254
1255         __bch2_dev_read_only(c, ca);
1256
1257         /*
1258          * XXX: verify that dev_idx is really not in use anymore, anywhere
1259          *
1260          * flag_data_bad() does not check btree pointers
1261          */
1262         ret = bch2_dev_data_drop(c, ca->dev_idx, flags);
1263         if (ret) {
1264                 bch_err(ca, "Remove failed: error %i dropping data", ret);
1265                 goto err;
1266         }
1267
1268         ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx);
1269         if (ret) {
1270                 bch_err(ca, "Remove failed: error %i flushing journal", ret);
1271                 goto err;
1272         }
1273
1274         data = bch2_dev_has_data(c, ca);
1275         if (data) {
1276                 char data_has_str[100];
1277                 bch2_scnprint_flag_list(data_has_str,
1278                                         sizeof(data_has_str),
1279                                         bch2_data_types,
1280                                         data);
1281                 bch_err(ca, "Remove failed, still has data (%s)", data_has_str);
1282                 ret = -EBUSY;
1283                 goto err;
1284         }
1285
1286         ret = bch2_btree_delete_range(c, BTREE_ID_ALLOC,
1287                                       POS(ca->dev_idx, 0),
1288                                       POS(ca->dev_idx + 1, 0),
1289                                       NULL);
1290         if (ret) {
1291                 bch_err(ca, "Remove failed, error deleting alloc info");
1292                 goto err;
1293         }
1294
1295         /*
1296          * must flush all existing journal entries, they might have
1297          * (overwritten) keys that point to the device we're removing:
1298          */
1299         bch2_journal_flush_all_pins(&c->journal);
1300         ret = bch2_journal_error(&c->journal);
1301         if (ret) {
1302                 bch_err(ca, "Remove failed, journal error");
1303                 goto err;
1304         }
1305
1306         __bch2_dev_offline(c, ca);
1307
1308         mutex_lock(&c->sb_lock);
1309         rcu_assign_pointer(c->devs[ca->dev_idx], NULL);
1310         mutex_unlock(&c->sb_lock);
1311
1312         percpu_ref_kill(&ca->ref);
1313         wait_for_completion(&ca->ref_completion);
1314
1315         bch2_dev_free(ca);
1316
1317         /*
1318          * Free this device's slot in the bch_member array - all pointers to
1319          * this device must be gone:
1320          */
1321         mutex_lock(&c->sb_lock);
1322         mi = bch2_sb_get_members(c->disk_sb.sb);
1323         memset(&mi->members[dev_idx].uuid, 0, sizeof(mi->members[dev_idx].uuid));
1324
1325         bch2_write_super(c);
1326
1327         mutex_unlock(&c->sb_lock);
1328         mutex_unlock(&c->state_lock);
1329         return 0;
1330 err:
1331         if (ca->mi.state == BCH_MEMBER_STATE_RW)
1332                 __bch2_dev_read_write(c, ca);
1333         mutex_unlock(&c->state_lock);
1334         return ret;
1335 }
1336
1337 static void dev_usage_clear(struct bch_dev *ca)
1338 {
1339         struct bucket_array *buckets;
1340         int cpu;
1341
1342         for_each_possible_cpu(cpu) {
1343                 struct bch_dev_usage *p =
1344                         per_cpu_ptr(ca->usage_percpu, cpu);
1345                 memset(p, 0, sizeof(*p));
1346         }
1347
1348         down_read(&ca->bucket_lock);
1349         buckets = bucket_array(ca);
1350
1351         memset(buckets->b, 0, sizeof(buckets->b[0]) * buckets->nbuckets);
1352         up_read(&ca->bucket_lock);
1353 }
1354
1355 /* Add new device to running filesystem: */
1356 int bch2_dev_add(struct bch_fs *c, const char *path)
1357 {
1358         struct bch_opts opts = bch2_opts_empty();
1359         struct bch_sb_handle sb;
1360         const char *err;
1361         struct bch_dev *ca = NULL;
1362         struct bch_sb_field_members *mi;
1363         struct bch_member dev_mi;
1364         unsigned dev_idx, nr_devices, u64s;
1365         int ret;
1366
1367         ret = bch2_read_super(path, &opts, &sb);
1368         if (ret)
1369                 return ret;
1370
1371         err = bch2_sb_validate(&sb);
1372         if (err)
1373                 return -EINVAL;
1374
1375         dev_mi = bch2_sb_get_members(sb.sb)->members[sb.sb->dev_idx];
1376
1377         err = bch2_dev_may_add(sb.sb, c);
1378         if (err)
1379                 return -EINVAL;
1380
1381         ca = __bch2_dev_alloc(c, &dev_mi);
1382         if (!ca) {
1383                 bch2_free_super(&sb);
1384                 return -ENOMEM;
1385         }
1386
1387         ret = __bch2_dev_attach_bdev(ca, &sb);
1388         if (ret) {
1389                 bch2_dev_free(ca);
1390                 return ret;
1391         }
1392
1393         /*
1394          * We want to allocate journal on the new device before adding the new
1395          * device to the filesystem because allocating after we attach requires
1396          * spinning up the allocator thread, and the allocator thread requires
1397          * doing btree writes, which if the existing devices are RO isn't going
1398          * to work
1399          *
1400          * So we have to mark where the superblocks are, but marking allocated
1401          * data normally updates the filesystem usage too, so we have to mark,
1402          * allocate the journal, reset all the marks, then remark after we
1403          * attach...
1404          */
1405         bch2_mark_dev_superblock(ca->fs, ca,
1406                         BCH_BUCKET_MARK_MAY_MAKE_UNAVAILABLE);
1407
1408         err = "journal alloc failed";
1409         ret = bch2_dev_journal_alloc(ca);
1410         if (ret)
1411                 goto err;
1412
1413         dev_usage_clear(ca);
1414
1415         mutex_lock(&c->state_lock);
1416         mutex_lock(&c->sb_lock);
1417
1418         err = "insufficient space in new superblock";
1419         ret = bch2_sb_from_fs(c, ca);
1420         if (ret)
1421                 goto err_unlock;
1422
1423         mi = bch2_sb_get_members(ca->disk_sb.sb);
1424
1425         if (!bch2_sb_resize_members(&ca->disk_sb,
1426                                 le32_to_cpu(mi->field.u64s) +
1427                                 sizeof(dev_mi) / sizeof(u64))) {
1428                 ret = -ENOSPC;
1429                 goto err_unlock;
1430         }
1431
1432         if (dynamic_fault("bcachefs:add:no_slot"))
1433                 goto no_slot;
1434
1435         mi = bch2_sb_get_members(c->disk_sb.sb);
1436         for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++)
1437                 if (!bch2_dev_exists(c->disk_sb.sb, mi, dev_idx))
1438                         goto have_slot;
1439 no_slot:
1440         err = "no slots available in superblock";
1441         ret = -ENOSPC;
1442         goto err_unlock;
1443
1444 have_slot:
1445         nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices);
1446         u64s = (sizeof(struct bch_sb_field_members) +
1447                 sizeof(struct bch_member) * nr_devices) / sizeof(u64);
1448
1449         err = "no space in superblock for member info";
1450         ret = -ENOSPC;
1451
1452         mi = bch2_sb_resize_members(&c->disk_sb, u64s);
1453         if (!mi)
1454                 goto err_unlock;
1455
1456         /* success: */
1457
1458         mi->members[dev_idx] = dev_mi;
1459         mi->members[dev_idx].last_mount = cpu_to_le64(ktime_get_real_seconds());
1460         c->disk_sb.sb->nr_devices       = nr_devices;
1461
1462         ca->disk_sb.sb->dev_idx = dev_idx;
1463         bch2_dev_attach(c, ca, dev_idx);
1464
1465         bch2_mark_dev_superblock(c, ca,
1466                         BCH_BUCKET_MARK_MAY_MAKE_UNAVAILABLE);
1467
1468         bch2_write_super(c);
1469         mutex_unlock(&c->sb_lock);
1470
1471         if (ca->mi.state == BCH_MEMBER_STATE_RW) {
1472                 err = __bch2_dev_read_write(c, ca);
1473                 if (err)
1474                         goto err_late;
1475         }
1476
1477         mutex_unlock(&c->state_lock);
1478         return 0;
1479
1480 err_unlock:
1481         mutex_unlock(&c->sb_lock);
1482         mutex_unlock(&c->state_lock);
1483 err:
1484         if (ca)
1485                 bch2_dev_free(ca);
1486         bch2_free_super(&sb);
1487         bch_err(c, "Unable to add device: %s", err);
1488         return ret;
1489 err_late:
1490         bch_err(c, "Error going rw after adding device: %s", err);
1491         return -EINVAL;
1492 }
1493
1494 /* Hot add existing device to running filesystem: */
1495 int bch2_dev_online(struct bch_fs *c, const char *path)
1496 {
1497         struct bch_opts opts = bch2_opts_empty();
1498         struct bch_sb_handle sb = { NULL };
1499         struct bch_sb_field_members *mi;
1500         struct bch_dev *ca;
1501         unsigned dev_idx;
1502         const char *err;
1503         int ret;
1504
1505         mutex_lock(&c->state_lock);
1506
1507         ret = bch2_read_super(path, &opts, &sb);
1508         if (ret) {
1509                 mutex_unlock(&c->state_lock);
1510                 return ret;
1511         }
1512
1513         dev_idx = sb.sb->dev_idx;
1514
1515         err = bch2_dev_in_fs(c->disk_sb.sb, sb.sb);
1516         if (err)
1517                 goto err;
1518
1519         if (bch2_dev_attach_bdev(c, &sb)) {
1520                 err = "bch2_dev_attach_bdev() error";
1521                 goto err;
1522         }
1523
1524         ca = bch_dev_locked(c, dev_idx);
1525         if (ca->mi.state == BCH_MEMBER_STATE_RW) {
1526                 err = __bch2_dev_read_write(c, ca);
1527                 if (err)
1528                         goto err;
1529         }
1530
1531         mutex_lock(&c->sb_lock);
1532         mi = bch2_sb_get_members(c->disk_sb.sb);
1533
1534         mi->members[ca->dev_idx].last_mount =
1535                 cpu_to_le64(ktime_get_real_seconds());
1536
1537         bch2_write_super(c);
1538         mutex_unlock(&c->sb_lock);
1539
1540         mutex_unlock(&c->state_lock);
1541         return 0;
1542 err:
1543         mutex_unlock(&c->state_lock);
1544         bch2_free_super(&sb);
1545         bch_err(c, "error bringing %s online: %s", path, err);
1546         return -EINVAL;
1547 }
1548
1549 int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags)
1550 {
1551         mutex_lock(&c->state_lock);
1552
1553         if (!bch2_dev_is_online(ca)) {
1554                 bch_err(ca, "Already offline");
1555                 mutex_unlock(&c->state_lock);
1556                 return 0;
1557         }
1558
1559         if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_FAILED, flags)) {
1560                 bch_err(ca, "Cannot offline required disk");
1561                 mutex_unlock(&c->state_lock);
1562                 return -EINVAL;
1563         }
1564
1565         __bch2_dev_offline(c, ca);
1566
1567         mutex_unlock(&c->state_lock);
1568         return 0;
1569 }
1570
1571 int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
1572 {
1573         struct bch_member *mi;
1574         int ret = 0;
1575
1576         mutex_lock(&c->state_lock);
1577
1578         if (nbuckets < ca->mi.nbuckets) {
1579                 bch_err(ca, "Cannot shrink yet");
1580                 ret = -EINVAL;
1581                 goto err;
1582         }
1583
1584         if (bch2_dev_is_online(ca) &&
1585             get_capacity(ca->disk_sb.bdev->bd_disk) <
1586             ca->mi.bucket_size * nbuckets) {
1587                 bch_err(ca, "New size larger than device");
1588                 ret = -EINVAL;
1589                 goto err;
1590         }
1591
1592         ret = bch2_dev_buckets_resize(c, ca, nbuckets);
1593         if (ret) {
1594                 bch_err(ca, "Resize error: %i", ret);
1595                 goto err;
1596         }
1597
1598         mutex_lock(&c->sb_lock);
1599         mi = &bch2_sb_get_members(c->disk_sb.sb)->members[ca->dev_idx];
1600         mi->nbuckets = cpu_to_le64(nbuckets);
1601
1602         bch2_write_super(c);
1603         mutex_unlock(&c->sb_lock);
1604
1605         bch2_recalc_capacity(c);
1606 err:
1607         mutex_unlock(&c->state_lock);
1608         return ret;
1609 }
1610
1611 /* return with ref on ca->ref: */
1612 struct bch_dev *bch2_dev_lookup(struct bch_fs *c, const char *path)
1613 {
1614
1615         struct block_device *bdev = lookup_bdev(path);
1616         struct bch_dev *ca;
1617         unsigned i;
1618
1619         if (IS_ERR(bdev))
1620                 return ERR_CAST(bdev);
1621
1622         for_each_member_device(ca, c, i)
1623                 if (ca->disk_sb.bdev == bdev)
1624                         goto found;
1625
1626         ca = ERR_PTR(-ENOENT);
1627 found:
1628         bdput(bdev);
1629         return ca;
1630 }
1631
1632 /* Filesystem open: */
1633
1634 struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices,
1635                             struct bch_opts opts)
1636 {
1637         struct bch_sb_handle *sb = NULL;
1638         struct bch_fs *c = NULL;
1639         unsigned i, best_sb = 0;
1640         const char *err;
1641         int ret = -ENOMEM;
1642
1643         pr_verbose_init(opts, "");
1644
1645         if (!nr_devices) {
1646                 c = ERR_PTR(-EINVAL);
1647                 goto out2;
1648         }
1649
1650         if (!try_module_get(THIS_MODULE)) {
1651                 c = ERR_PTR(-ENODEV);
1652                 goto out2;
1653         }
1654
1655         sb = kcalloc(nr_devices, sizeof(*sb), GFP_KERNEL);
1656         if (!sb)
1657                 goto err;
1658
1659         for (i = 0; i < nr_devices; i++) {
1660                 ret = bch2_read_super(devices[i], &opts, &sb[i]);
1661                 if (ret)
1662                         goto err;
1663
1664                 err = bch2_sb_validate(&sb[i]);
1665                 if (err)
1666                         goto err_print;
1667         }
1668
1669         for (i = 1; i < nr_devices; i++)
1670                 if (le64_to_cpu(sb[i].sb->seq) >
1671                     le64_to_cpu(sb[best_sb].sb->seq))
1672                         best_sb = i;
1673
1674         for (i = 0; i < nr_devices; i++) {
1675                 err = bch2_dev_in_fs(sb[best_sb].sb, sb[i].sb);
1676                 if (err)
1677                         goto err_print;
1678         }
1679
1680         ret = -ENOMEM;
1681         c = bch2_fs_alloc(sb[best_sb].sb, opts);
1682         if (!c)
1683                 goto err;
1684
1685         err = "bch2_dev_online() error";
1686         mutex_lock(&c->state_lock);
1687         for (i = 0; i < nr_devices; i++)
1688                 if (bch2_dev_attach_bdev(c, &sb[i])) {
1689                         mutex_unlock(&c->state_lock);
1690                         goto err_print;
1691                 }
1692         mutex_unlock(&c->state_lock);
1693
1694         err = "insufficient devices";
1695         if (!bch2_fs_may_start(c))
1696                 goto err_print;
1697
1698         if (!c->opts.nostart) {
1699                 err = bch2_fs_start(c);
1700                 if (err)
1701                         goto err_print;
1702         }
1703 out:
1704         kfree(sb);
1705         module_put(THIS_MODULE);
1706 out2:
1707         pr_verbose_init(opts, "ret %i", PTR_ERR_OR_ZERO(c));
1708         return c;
1709 err_print:
1710         pr_err("bch_fs_open err opening %s: %s",
1711                devices[0], err);
1712         ret = -EINVAL;
1713 err:
1714         if (c)
1715                 bch2_fs_stop(c);
1716         for (i = 0; i < nr_devices; i++)
1717                 bch2_free_super(&sb[i]);
1718         c = ERR_PTR(ret);
1719         goto out;
1720 }
1721
1722 static const char *__bch2_fs_open_incremental(struct bch_sb_handle *sb,
1723                                               struct bch_opts opts)
1724 {
1725         const char *err;
1726         struct bch_fs *c;
1727         bool allocated_fs = false;
1728
1729         err = bch2_sb_validate(sb);
1730         if (err)
1731                 return err;
1732
1733         mutex_lock(&bch_fs_list_lock);
1734         c = __bch2_uuid_to_fs(sb->sb->uuid);
1735         if (c) {
1736                 closure_get(&c->cl);
1737
1738                 err = bch2_dev_in_fs(c->disk_sb.sb, sb->sb);
1739                 if (err)
1740                         goto err;
1741         } else {
1742                 c = bch2_fs_alloc(sb->sb, opts);
1743                 err = "cannot allocate memory";
1744                 if (!c)
1745                         goto err;
1746
1747                 allocated_fs = true;
1748         }
1749
1750         err = "bch2_dev_online() error";
1751
1752         mutex_lock(&c->sb_lock);
1753         if (bch2_dev_attach_bdev(c, sb)) {
1754                 mutex_unlock(&c->sb_lock);
1755                 goto err;
1756         }
1757         mutex_unlock(&c->sb_lock);
1758
1759         if (!c->opts.nostart && bch2_fs_may_start(c)) {
1760                 err = bch2_fs_start(c);
1761                 if (err)
1762                         goto err;
1763         }
1764
1765         closure_put(&c->cl);
1766         mutex_unlock(&bch_fs_list_lock);
1767
1768         return NULL;
1769 err:
1770         mutex_unlock(&bch_fs_list_lock);
1771
1772         if (allocated_fs)
1773                 bch2_fs_stop(c);
1774         else if (c)
1775                 closure_put(&c->cl);
1776
1777         return err;
1778 }
1779
1780 const char *bch2_fs_open_incremental(const char *path)
1781 {
1782         struct bch_sb_handle sb;
1783         struct bch_opts opts = bch2_opts_empty();
1784         const char *err;
1785
1786         if (bch2_read_super(path, &opts, &sb))
1787                 return "error reading superblock";
1788
1789         err = __bch2_fs_open_incremental(&sb, opts);
1790         bch2_free_super(&sb);
1791
1792         return err;
1793 }
1794
1795 /* Global interfaces/init */
1796
1797 static void bcachefs_exit(void)
1798 {
1799         bch2_debug_exit();
1800         bch2_vfs_exit();
1801         bch2_chardev_exit();
1802         if (bcachefs_kset)
1803                 kset_unregister(bcachefs_kset);
1804 }
1805
1806 static int __init bcachefs_init(void)
1807 {
1808         bch2_bkey_pack_test();
1809         bch2_inode_pack_test();
1810
1811         if (!(bcachefs_kset = kset_create_and_add("bcachefs", NULL, fs_kobj)) ||
1812             bch2_chardev_init() ||
1813             bch2_vfs_init() ||
1814             bch2_debug_init())
1815                 goto err;
1816
1817         return 0;
1818 err:
1819         bcachefs_exit();
1820         return -ENOMEM;
1821 }
1822
1823 #define BCH_DEBUG_PARAM(name, description)                      \
1824         bool bch2_##name;                                       \
1825         module_param_named(name, bch2_##name, bool, 0644);      \
1826         MODULE_PARM_DESC(name, description);
1827 BCH_DEBUG_PARAMS()
1828 #undef BCH_DEBUG_PARAM
1829
1830 module_exit(bcachefs_exit);
1831 module_init(bcachefs_init);