]> git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/super.c
e7f186b45df103ad51662a1497efb92184a3ba61
[bcachefs-tools-debian] / libbcachefs / super.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * bcachefs setup/teardown code, and some metadata io - read a superblock and
4  * figure out what to do with it.
5  *
6  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
7  * Copyright 2012 Google, Inc.
8  */
9
10 #include "bcachefs.h"
11 #include "alloc_background.h"
12 #include "alloc_foreground.h"
13 #include "bkey_sort.h"
14 #include "btree_cache.h"
15 #include "btree_gc.h"
16 #include "btree_journal_iter.h"
17 #include "btree_key_cache.h"
18 #include "btree_update_interior.h"
19 #include "btree_io.h"
20 #include "btree_write_buffer.h"
21 #include "buckets_waiting_for_journal.h"
22 #include "chardev.h"
23 #include "checksum.h"
24 #include "clock.h"
25 #include "compress.h"
26 #include "counters.h"
27 #include "debug.h"
28 #include "disk_groups.h"
29 #include "ec.h"
30 #include "errcode.h"
31 #include "error.h"
32 #include "fs.h"
33 #include "fs-io.h"
34 #include "fs-io-buffered.h"
35 #include "fs-io-direct.h"
36 #include "fsck.h"
37 #include "inode.h"
38 #include "io_read.h"
39 #include "io_write.h"
40 #include "journal.h"
41 #include "journal_reclaim.h"
42 #include "journal_seq_blacklist.h"
43 #include "move.h"
44 #include "migrate.h"
45 #include "movinggc.h"
46 #include "nocow_locking.h"
47 #include "quota.h"
48 #include "rebalance.h"
49 #include "recovery.h"
50 #include "replicas.h"
51 #include "sb-clean.h"
52 #include "sb-errors.h"
53 #include "sb-members.h"
54 #include "snapshot.h"
55 #include "subvolume.h"
56 #include "super.h"
57 #include "super-io.h"
58 #include "sysfs.h"
59 #include "trace.h"
60
61 #include <linux/backing-dev.h>
62 #include <linux/blkdev.h>
63 #include <linux/debugfs.h>
64 #include <linux/device.h>
65 #include <linux/idr.h>
66 #include <linux/module.h>
67 #include <linux/percpu.h>
68 #include <linux/random.h>
69 #include <linux/sysfs.h>
70 #include <crypto/hash.h>
71
72 MODULE_LICENSE("GPL");
73 MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");
74 MODULE_DESCRIPTION("bcachefs filesystem");
75
76 const char * const bch2_fs_flag_strs[] = {
77 #define x(n)            #n,
78         BCH_FS_FLAGS()
79 #undef x
80         NULL
81 };
82
83 #define KTYPE(type)                                                     \
84 static const struct attribute_group type ## _group = {                  \
85         .attrs = type ## _files                                         \
86 };                                                                      \
87                                                                         \
88 static const struct attribute_group *type ## _groups[] = {              \
89         &type ## _group,                                                \
90         NULL                                                            \
91 };                                                                      \
92                                                                         \
93 static const struct kobj_type type ## _ktype = {                        \
94         .release        = type ## _release,                             \
95         .sysfs_ops      = &type ## _sysfs_ops,                          \
96         .default_groups = type ## _groups                               \
97 }
98
99 static void bch2_fs_release(struct kobject *);
100 static void bch2_dev_release(struct kobject *);
101 static void bch2_fs_counters_release(struct kobject *k)
102 {
103 }
104
105 static void bch2_fs_internal_release(struct kobject *k)
106 {
107 }
108
109 static void bch2_fs_opts_dir_release(struct kobject *k)
110 {
111 }
112
113 static void bch2_fs_time_stats_release(struct kobject *k)
114 {
115 }
116
117 KTYPE(bch2_fs);
118 KTYPE(bch2_fs_counters);
119 KTYPE(bch2_fs_internal);
120 KTYPE(bch2_fs_opts_dir);
121 KTYPE(bch2_fs_time_stats);
122 KTYPE(bch2_dev);
123
124 static struct kset *bcachefs_kset;
125 static LIST_HEAD(bch_fs_list);
126 static DEFINE_MUTEX(bch_fs_list_lock);
127
128 DECLARE_WAIT_QUEUE_HEAD(bch2_read_only_wait);
129
130 static void bch2_dev_free(struct bch_dev *);
131 static int bch2_dev_alloc(struct bch_fs *, unsigned);
132 static int bch2_dev_sysfs_online(struct bch_fs *, struct bch_dev *);
133 static void __bch2_dev_read_only(struct bch_fs *, struct bch_dev *);
134
135 struct bch_fs *bch2_dev_to_fs(dev_t dev)
136 {
137         struct bch_fs *c;
138         struct bch_dev *ca;
139         unsigned i;
140
141         mutex_lock(&bch_fs_list_lock);
142         rcu_read_lock();
143
144         list_for_each_entry(c, &bch_fs_list, list)
145                 for_each_member_device_rcu(ca, c, i, NULL)
146                         if (ca->disk_sb.bdev && ca->disk_sb.bdev->bd_dev == dev) {
147                                 closure_get(&c->cl);
148                                 goto found;
149                         }
150         c = NULL;
151 found:
152         rcu_read_unlock();
153         mutex_unlock(&bch_fs_list_lock);
154
155         return c;
156 }
157
158 static struct bch_fs *__bch2_uuid_to_fs(__uuid_t uuid)
159 {
160         struct bch_fs *c;
161
162         lockdep_assert_held(&bch_fs_list_lock);
163
164         list_for_each_entry(c, &bch_fs_list, list)
165                 if (!memcmp(&c->disk_sb.sb->uuid, &uuid, sizeof(uuid)))
166                         return c;
167
168         return NULL;
169 }
170
171 struct bch_fs *bch2_uuid_to_fs(__uuid_t uuid)
172 {
173         struct bch_fs *c;
174
175         mutex_lock(&bch_fs_list_lock);
176         c = __bch2_uuid_to_fs(uuid);
177         if (c)
178                 closure_get(&c->cl);
179         mutex_unlock(&bch_fs_list_lock);
180
181         return c;
182 }
183
184 static void bch2_dev_usage_journal_reserve(struct bch_fs *c)
185 {
186         struct bch_dev *ca;
187         unsigned i, nr = 0, u64s =
188                 ((sizeof(struct jset_entry_dev_usage) +
189                   sizeof(struct jset_entry_dev_usage_type) * BCH_DATA_NR)) /
190                 sizeof(u64);
191
192         rcu_read_lock();
193         for_each_member_device_rcu(ca, c, i, NULL)
194                 nr++;
195         rcu_read_unlock();
196
197         bch2_journal_entry_res_resize(&c->journal,
198                         &c->dev_usage_journal_res, u64s * nr);
199 }
200
201 /* Filesystem RO/RW: */
202
203 /*
204  * For startup/shutdown of RW stuff, the dependencies are:
205  *
206  * - foreground writes depend on copygc and rebalance (to free up space)
207  *
208  * - copygc and rebalance depend on mark and sweep gc (they actually probably
209  *   don't because they either reserve ahead of time or don't block if
210  *   allocations fail, but allocations can require mark and sweep gc to run
211  *   because of generation number wraparound)
212  *
213  * - all of the above depends on the allocator threads
214  *
215  * - allocator depends on the journal (when it rewrites prios and gens)
216  */
217
218 static void __bch2_fs_read_only(struct bch_fs *c)
219 {
220         struct bch_dev *ca;
221         unsigned i, clean_passes = 0;
222         u64 seq = 0;
223
224         bch2_fs_ec_stop(c);
225         bch2_open_buckets_stop(c, NULL, true);
226         bch2_rebalance_stop(c);
227         bch2_copygc_stop(c);
228         bch2_gc_thread_stop(c);
229         bch2_fs_ec_flush(c);
230
231         bch_verbose(c, "flushing journal and stopping allocators, journal seq %llu",
232                     journal_cur_seq(&c->journal));
233
234         do {
235                 clean_passes++;
236
237                 if (bch2_btree_interior_updates_flush(c) ||
238                     bch2_journal_flush_all_pins(&c->journal) ||
239                     bch2_btree_flush_all_writes(c) ||
240                     seq != atomic64_read(&c->journal.seq)) {
241                         seq = atomic64_read(&c->journal.seq);
242                         clean_passes = 0;
243                 }
244         } while (clean_passes < 2);
245
246         bch_verbose(c, "flushing journal and stopping allocators complete, journal seq %llu",
247                     journal_cur_seq(&c->journal));
248
249         if (test_bit(JOURNAL_REPLAY_DONE, &c->journal.flags) &&
250             !test_bit(BCH_FS_emergency_ro, &c->flags))
251                 set_bit(BCH_FS_clean_shutdown, &c->flags);
252         bch2_fs_journal_stop(&c->journal);
253
254         /*
255          * After stopping journal:
256          */
257         for_each_member_device(ca, c, i)
258                 bch2_dev_allocator_remove(c, ca);
259 }
260
261 #ifndef BCH_WRITE_REF_DEBUG
262 static void bch2_writes_disabled(struct percpu_ref *writes)
263 {
264         struct bch_fs *c = container_of(writes, struct bch_fs, writes);
265
266         set_bit(BCH_FS_write_disable_complete, &c->flags);
267         wake_up(&bch2_read_only_wait);
268 }
269 #endif
270
271 void bch2_fs_read_only(struct bch_fs *c)
272 {
273         if (!test_bit(BCH_FS_rw, &c->flags)) {
274                 bch2_journal_reclaim_stop(&c->journal);
275                 return;
276         }
277
278         BUG_ON(test_bit(BCH_FS_write_disable_complete, &c->flags));
279
280         bch_verbose(c, "going read-only");
281
282         /*
283          * Block new foreground-end write operations from starting - any new
284          * writes will return -EROFS:
285          */
286         set_bit(BCH_FS_going_ro, &c->flags);
287 #ifndef BCH_WRITE_REF_DEBUG
288         percpu_ref_kill(&c->writes);
289 #else
290         for (unsigned i = 0; i < BCH_WRITE_REF_NR; i++)
291                 bch2_write_ref_put(c, i);
292 #endif
293
294         /*
295          * If we're not doing an emergency shutdown, we want to wait on
296          * outstanding writes to complete so they don't see spurious errors due
297          * to shutting down the allocator:
298          *
299          * If we are doing an emergency shutdown outstanding writes may
300          * hang until we shutdown the allocator so we don't want to wait
301          * on outstanding writes before shutting everything down - but
302          * we do need to wait on them before returning and signalling
303          * that going RO is complete:
304          */
305         wait_event(bch2_read_only_wait,
306                    test_bit(BCH_FS_write_disable_complete, &c->flags) ||
307                    test_bit(BCH_FS_emergency_ro, &c->flags));
308
309         bool writes_disabled = test_bit(BCH_FS_write_disable_complete, &c->flags);
310         if (writes_disabled)
311                 bch_verbose(c, "finished waiting for writes to stop");
312
313         __bch2_fs_read_only(c);
314
315         wait_event(bch2_read_only_wait,
316                    test_bit(BCH_FS_write_disable_complete, &c->flags));
317
318         if (!writes_disabled)
319                 bch_verbose(c, "finished waiting for writes to stop");
320
321         clear_bit(BCH_FS_write_disable_complete, &c->flags);
322         clear_bit(BCH_FS_going_ro, &c->flags);
323         clear_bit(BCH_FS_rw, &c->flags);
324
325         if (!bch2_journal_error(&c->journal) &&
326             !test_bit(BCH_FS_error, &c->flags) &&
327             !test_bit(BCH_FS_emergency_ro, &c->flags) &&
328             test_bit(BCH_FS_started, &c->flags) &&
329             test_bit(BCH_FS_clean_shutdown, &c->flags) &&
330             !c->opts.norecovery) {
331                 BUG_ON(c->journal.last_empty_seq != journal_cur_seq(&c->journal));
332                 BUG_ON(atomic_read(&c->btree_cache.dirty));
333                 BUG_ON(atomic_long_read(&c->btree_key_cache.nr_dirty));
334                 BUG_ON(c->btree_write_buffer.state.nr);
335
336                 bch_verbose(c, "marking filesystem clean");
337                 bch2_fs_mark_clean(c);
338         } else {
339                 bch_verbose(c, "done going read-only, filesystem not clean");
340         }
341 }
342
343 static void bch2_fs_read_only_work(struct work_struct *work)
344 {
345         struct bch_fs *c =
346                 container_of(work, struct bch_fs, read_only_work);
347
348         down_write(&c->state_lock);
349         bch2_fs_read_only(c);
350         up_write(&c->state_lock);
351 }
352
353 static void bch2_fs_read_only_async(struct bch_fs *c)
354 {
355         queue_work(system_long_wq, &c->read_only_work);
356 }
357
358 bool bch2_fs_emergency_read_only(struct bch_fs *c)
359 {
360         bool ret = !test_and_set_bit(BCH_FS_emergency_ro, &c->flags);
361
362         bch2_journal_halt(&c->journal);
363         bch2_fs_read_only_async(c);
364
365         wake_up(&bch2_read_only_wait);
366         return ret;
367 }
368
369 static int bch2_fs_read_write_late(struct bch_fs *c)
370 {
371         int ret;
372
373         /*
374          * Data move operations can't run until after check_snapshots has
375          * completed, and bch2_snapshot_is_ancestor() is available.
376          *
377          * Ideally we'd start copygc/rebalance earlier instead of waiting for
378          * all of recovery/fsck to complete:
379          */
380         ret = bch2_copygc_start(c);
381         if (ret) {
382                 bch_err(c, "error starting copygc thread");
383                 return ret;
384         }
385
386         ret = bch2_rebalance_start(c);
387         if (ret) {
388                 bch_err(c, "error starting rebalance thread");
389                 return ret;
390         }
391
392         return 0;
393 }
394
395 static int __bch2_fs_read_write(struct bch_fs *c, bool early)
396 {
397         struct bch_dev *ca;
398         unsigned i;
399         int ret;
400
401         if (test_bit(BCH_FS_initial_gc_unfixed, &c->flags)) {
402                 bch_err(c, "cannot go rw, unfixed btree errors");
403                 return -BCH_ERR_erofs_unfixed_errors;
404         }
405
406         if (test_bit(BCH_FS_rw, &c->flags))
407                 return 0;
408
409         if (c->opts.norecovery)
410                 return -BCH_ERR_erofs_norecovery;
411
412         /*
413          * nochanges is used for fsck -n mode - we have to allow going rw
414          * during recovery for that to work:
415          */
416         if (c->opts.nochanges && (!early || c->opts.read_only))
417                 return -BCH_ERR_erofs_nochanges;
418
419         bch_info(c, "going read-write");
420
421         ret = bch2_sb_members_v2_init(c);
422         if (ret)
423                 goto err;
424
425         ret = bch2_fs_mark_dirty(c);
426         if (ret)
427                 goto err;
428
429         clear_bit(BCH_FS_clean_shutdown, &c->flags);
430
431         /*
432          * First journal write must be a flush write: after a clean shutdown we
433          * don't read the journal, so the first journal write may end up
434          * overwriting whatever was there previously, and there must always be
435          * at least one non-flush write in the journal or recovery will fail:
436          */
437         set_bit(JOURNAL_NEED_FLUSH_WRITE, &c->journal.flags);
438
439         for_each_rw_member(ca, c, i)
440                 bch2_dev_allocator_add(c, ca);
441         bch2_recalc_capacity(c);
442
443         set_bit(BCH_FS_rw, &c->flags);
444         set_bit(BCH_FS_was_rw, &c->flags);
445
446 #ifndef BCH_WRITE_REF_DEBUG
447         percpu_ref_reinit(&c->writes);
448 #else
449         for (i = 0; i < BCH_WRITE_REF_NR; i++) {
450                 BUG_ON(atomic_long_read(&c->writes[i]));
451                 atomic_long_inc(&c->writes[i]);
452         }
453 #endif
454
455         ret = bch2_gc_thread_start(c);
456         if (ret) {
457                 bch_err(c, "error starting gc thread");
458                 return ret;
459         }
460
461         ret = bch2_journal_reclaim_start(&c->journal);
462         if (ret)
463                 goto err;
464
465         if (!early) {
466                 ret = bch2_fs_read_write_late(c);
467                 if (ret)
468                         goto err;
469         }
470
471         bch2_do_discards(c);
472         bch2_do_invalidates(c);
473         bch2_do_stripe_deletes(c);
474         bch2_do_pending_node_rewrites(c);
475         return 0;
476 err:
477         if (test_bit(BCH_FS_rw, &c->flags))
478                 bch2_fs_read_only(c);
479         else
480                 __bch2_fs_read_only(c);
481         return ret;
482 }
483
484 int bch2_fs_read_write(struct bch_fs *c)
485 {
486         return __bch2_fs_read_write(c, false);
487 }
488
489 int bch2_fs_read_write_early(struct bch_fs *c)
490 {
491         lockdep_assert_held(&c->state_lock);
492
493         return __bch2_fs_read_write(c, true);
494 }
495
496 /* Filesystem startup/shutdown: */
497
498 static void __bch2_fs_free(struct bch_fs *c)
499 {
500         unsigned i;
501
502         for (i = 0; i < BCH_TIME_STAT_NR; i++)
503                 bch2_time_stats_exit(&c->times[i]);
504
505         bch2_free_pending_node_rewrites(c);
506         bch2_fs_sb_errors_exit(c);
507         bch2_fs_counters_exit(c);
508         bch2_fs_snapshots_exit(c);
509         bch2_fs_quota_exit(c);
510         bch2_fs_fs_io_direct_exit(c);
511         bch2_fs_fs_io_buffered_exit(c);
512         bch2_fs_fsio_exit(c);
513         bch2_fs_ec_exit(c);
514         bch2_fs_encryption_exit(c);
515         bch2_fs_nocow_locking_exit(c);
516         bch2_fs_io_write_exit(c);
517         bch2_fs_io_read_exit(c);
518         bch2_fs_buckets_waiting_for_journal_exit(c);
519         bch2_fs_btree_interior_update_exit(c);
520         bch2_fs_btree_iter_exit(c);
521         bch2_fs_btree_key_cache_exit(&c->btree_key_cache);
522         bch2_fs_btree_cache_exit(c);
523         bch2_fs_replicas_exit(c);
524         bch2_fs_journal_exit(&c->journal);
525         bch2_io_clock_exit(&c->io_clock[WRITE]);
526         bch2_io_clock_exit(&c->io_clock[READ]);
527         bch2_fs_compress_exit(c);
528         bch2_journal_keys_put_initial(c);
529         BUG_ON(atomic_read(&c->journal_keys.ref));
530         bch2_fs_btree_write_buffer_exit(c);
531         percpu_free_rwsem(&c->mark_lock);
532         free_percpu(c->online_reserved);
533
534         darray_exit(&c->btree_roots_extra);
535         free_percpu(c->pcpu);
536         mempool_exit(&c->large_bkey_pool);
537         mempool_exit(&c->btree_bounce_pool);
538         bioset_exit(&c->btree_bio);
539         mempool_exit(&c->fill_iter);
540 #ifndef BCH_WRITE_REF_DEBUG
541         percpu_ref_exit(&c->writes);
542 #endif
543         kfree(rcu_dereference_protected(c->disk_groups, 1));
544         kfree(c->journal_seq_blacklist_table);
545         kfree(c->unused_inode_hints);
546
547         if (c->write_ref_wq)
548                 destroy_workqueue(c->write_ref_wq);
549         if (c->io_complete_wq)
550                 destroy_workqueue(c->io_complete_wq);
551         if (c->copygc_wq)
552                 destroy_workqueue(c->copygc_wq);
553         if (c->btree_io_complete_wq)
554                 destroy_workqueue(c->btree_io_complete_wq);
555         if (c->btree_update_wq)
556                 destroy_workqueue(c->btree_update_wq);
557
558         bch2_free_super(&c->disk_sb);
559         kvpfree(c, sizeof(*c));
560         module_put(THIS_MODULE);
561 }
562
563 static void bch2_fs_release(struct kobject *kobj)
564 {
565         struct bch_fs *c = container_of(kobj, struct bch_fs, kobj);
566
567         __bch2_fs_free(c);
568 }
569
570 void __bch2_fs_stop(struct bch_fs *c)
571 {
572         struct bch_dev *ca;
573         unsigned i;
574
575         bch_verbose(c, "shutting down");
576
577         set_bit(BCH_FS_stopping, &c->flags);
578
579         cancel_work_sync(&c->journal_seq_blacklist_gc_work);
580
581         down_write(&c->state_lock);
582         bch2_fs_read_only(c);
583         up_write(&c->state_lock);
584
585         for_each_member_device(ca, c, i)
586                 if (ca->kobj.state_in_sysfs &&
587                     ca->disk_sb.bdev)
588                         sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs");
589
590         if (c->kobj.state_in_sysfs)
591                 kobject_del(&c->kobj);
592
593         bch2_fs_debug_exit(c);
594         bch2_fs_chardev_exit(c);
595
596         kobject_put(&c->counters_kobj);
597         kobject_put(&c->time_stats);
598         kobject_put(&c->opts_dir);
599         kobject_put(&c->internal);
600
601         /* btree prefetch might have kicked off reads in the background: */
602         bch2_btree_flush_all_reads(c);
603
604         for_each_member_device(ca, c, i)
605                 cancel_work_sync(&ca->io_error_work);
606
607         cancel_work_sync(&c->read_only_work);
608 }
609
610 void bch2_fs_free(struct bch_fs *c)
611 {
612         unsigned i;
613
614         mutex_lock(&bch_fs_list_lock);
615         list_del(&c->list);
616         mutex_unlock(&bch_fs_list_lock);
617
618         closure_sync(&c->cl);
619         closure_debug_destroy(&c->cl);
620
621         for (i = 0; i < c->sb.nr_devices; i++) {
622                 struct bch_dev *ca = rcu_dereference_protected(c->devs[i], true);
623
624                 if (ca) {
625                         bch2_free_super(&ca->disk_sb);
626                         bch2_dev_free(ca);
627                 }
628         }
629
630         bch_verbose(c, "shutdown complete");
631
632         kobject_put(&c->kobj);
633 }
634
635 void bch2_fs_stop(struct bch_fs *c)
636 {
637         __bch2_fs_stop(c);
638         bch2_fs_free(c);
639 }
640
641 static int bch2_fs_online(struct bch_fs *c)
642 {
643         struct bch_dev *ca;
644         unsigned i;
645         int ret = 0;
646
647         lockdep_assert_held(&bch_fs_list_lock);
648
649         if (__bch2_uuid_to_fs(c->sb.uuid)) {
650                 bch_err(c, "filesystem UUID already open");
651                 return -EINVAL;
652         }
653
654         ret = bch2_fs_chardev_init(c);
655         if (ret) {
656                 bch_err(c, "error creating character device");
657                 return ret;
658         }
659
660         bch2_fs_debug_init(c);
661
662         ret = kobject_add(&c->kobj, NULL, "%pU", c->sb.user_uuid.b) ?:
663             kobject_add(&c->internal, &c->kobj, "internal") ?:
664             kobject_add(&c->opts_dir, &c->kobj, "options") ?:
665 #ifndef CONFIG_BCACHEFS_NO_LATENCY_ACCT
666             kobject_add(&c->time_stats, &c->kobj, "time_stats") ?:
667 #endif
668             kobject_add(&c->counters_kobj, &c->kobj, "counters") ?:
669             bch2_opts_create_sysfs_files(&c->opts_dir);
670         if (ret) {
671                 bch_err(c, "error creating sysfs objects");
672                 return ret;
673         }
674
675         down_write(&c->state_lock);
676
677         for_each_member_device(ca, c, i) {
678                 ret = bch2_dev_sysfs_online(c, ca);
679                 if (ret) {
680                         bch_err(c, "error creating sysfs objects");
681                         percpu_ref_put(&ca->ref);
682                         goto err;
683                 }
684         }
685
686         BUG_ON(!list_empty(&c->list));
687         list_add(&c->list, &bch_fs_list);
688 err:
689         up_write(&c->state_lock);
690         return ret;
691 }
692
693 static struct bch_fs *bch2_fs_alloc(struct bch_sb *sb, struct bch_opts opts)
694 {
695         struct bch_fs *c;
696         struct printbuf name = PRINTBUF;
697         unsigned i, iter_size;
698         int ret = 0;
699
700         c = kvpmalloc(sizeof(struct bch_fs), GFP_KERNEL|__GFP_ZERO);
701         if (!c) {
702                 c = ERR_PTR(-BCH_ERR_ENOMEM_fs_alloc);
703                 goto out;
704         }
705
706         __module_get(THIS_MODULE);
707
708         closure_init(&c->cl, NULL);
709
710         c->kobj.kset = bcachefs_kset;
711         kobject_init(&c->kobj, &bch2_fs_ktype);
712         kobject_init(&c->internal, &bch2_fs_internal_ktype);
713         kobject_init(&c->opts_dir, &bch2_fs_opts_dir_ktype);
714         kobject_init(&c->time_stats, &bch2_fs_time_stats_ktype);
715         kobject_init(&c->counters_kobj, &bch2_fs_counters_ktype);
716
717         c->minor                = -1;
718         c->disk_sb.fs_sb        = true;
719
720         init_rwsem(&c->state_lock);
721         mutex_init(&c->sb_lock);
722         mutex_init(&c->replicas_gc_lock);
723         mutex_init(&c->btree_root_lock);
724         INIT_WORK(&c->read_only_work, bch2_fs_read_only_work);
725
726         init_rwsem(&c->gc_lock);
727         mutex_init(&c->gc_gens_lock);
728         atomic_set(&c->journal_keys.ref, 1);
729         c->journal_keys.initial_ref_held = true;
730
731         for (i = 0; i < BCH_TIME_STAT_NR; i++)
732                 bch2_time_stats_init(&c->times[i]);
733
734         bch2_fs_copygc_init(c);
735         bch2_fs_btree_key_cache_init_early(&c->btree_key_cache);
736         bch2_fs_btree_interior_update_init_early(c);
737         bch2_fs_allocator_background_init(c);
738         bch2_fs_allocator_foreground_init(c);
739         bch2_fs_rebalance_init(c);
740         bch2_fs_quota_init(c);
741         bch2_fs_ec_init_early(c);
742         bch2_fs_move_init(c);
743         bch2_fs_sb_errors_init_early(c);
744
745         INIT_LIST_HEAD(&c->list);
746
747         mutex_init(&c->usage_scratch_lock);
748
749         mutex_init(&c->bio_bounce_pages_lock);
750         mutex_init(&c->snapshot_table_lock);
751         init_rwsem(&c->snapshot_create_lock);
752
753         spin_lock_init(&c->btree_write_error_lock);
754
755         INIT_WORK(&c->journal_seq_blacklist_gc_work,
756                   bch2_blacklist_entries_gc);
757
758         INIT_LIST_HEAD(&c->journal_iters);
759
760         INIT_LIST_HEAD(&c->fsck_error_msgs);
761         mutex_init(&c->fsck_error_msgs_lock);
762
763         seqcount_init(&c->gc_pos_lock);
764
765         seqcount_init(&c->usage_lock);
766
767         sema_init(&c->io_in_flight, 128);
768
769         INIT_LIST_HEAD(&c->vfs_inodes_list);
770         mutex_init(&c->vfs_inodes_lock);
771
772         c->copy_gc_enabled              = 1;
773         c->rebalance.enabled            = 1;
774         c->promote_whole_extents        = true;
775
776         c->journal.flush_write_time     = &c->times[BCH_TIME_journal_flush_write];
777         c->journal.noflush_write_time   = &c->times[BCH_TIME_journal_noflush_write];
778         c->journal.flush_seq_time       = &c->times[BCH_TIME_journal_flush_seq];
779
780         bch2_fs_btree_cache_init_early(&c->btree_cache);
781
782         mutex_init(&c->sectors_available_lock);
783
784         ret = percpu_init_rwsem(&c->mark_lock);
785         if (ret)
786                 goto err;
787
788         mutex_lock(&c->sb_lock);
789         ret = bch2_sb_to_fs(c, sb);
790         mutex_unlock(&c->sb_lock);
791
792         if (ret)
793                 goto err;
794
795         pr_uuid(&name, c->sb.user_uuid.b);
796         strscpy(c->name, name.buf, sizeof(c->name));
797         printbuf_exit(&name);
798
799         ret = name.allocation_failure ? -BCH_ERR_ENOMEM_fs_name_alloc : 0;
800         if (ret)
801                 goto err;
802
803         /* Compat: */
804         if (le16_to_cpu(sb->version) <= bcachefs_metadata_version_inode_v2 &&
805             !BCH_SB_JOURNAL_FLUSH_DELAY(sb))
806                 SET_BCH_SB_JOURNAL_FLUSH_DELAY(sb, 1000);
807
808         if (le16_to_cpu(sb->version) <= bcachefs_metadata_version_inode_v2 &&
809             !BCH_SB_JOURNAL_RECLAIM_DELAY(sb))
810                 SET_BCH_SB_JOURNAL_RECLAIM_DELAY(sb, 100);
811
812         c->opts = bch2_opts_default;
813         ret = bch2_opts_from_sb(&c->opts, sb);
814         if (ret)
815                 goto err;
816
817         bch2_opts_apply(&c->opts, opts);
818
819         c->btree_key_cache_btrees |= 1U << BTREE_ID_alloc;
820         if (c->opts.inodes_use_key_cache)
821                 c->btree_key_cache_btrees |= 1U << BTREE_ID_inodes;
822         c->btree_key_cache_btrees |= 1U << BTREE_ID_logged_ops;
823
824         c->block_bits           = ilog2(block_sectors(c));
825         c->btree_foreground_merge_threshold = BTREE_FOREGROUND_MERGE_THRESHOLD(c);
826
827         if (bch2_fs_init_fault("fs_alloc")) {
828                 bch_err(c, "fs_alloc fault injected");
829                 ret = -EFAULT;
830                 goto err;
831         }
832
833         iter_size = sizeof(struct sort_iter) +
834                 (btree_blocks(c) + 1) * 2 *
835                 sizeof(struct sort_iter_set);
836
837         c->inode_shard_bits = ilog2(roundup_pow_of_two(num_possible_cpus()));
838
839         if (!(c->btree_update_wq = alloc_workqueue("bcachefs",
840                                 WQ_FREEZABLE|WQ_UNBOUND|WQ_MEM_RECLAIM, 512)) ||
841             !(c->btree_io_complete_wq = alloc_workqueue("bcachefs_btree_io",
842                                 WQ_FREEZABLE|WQ_MEM_RECLAIM, 1)) ||
843             !(c->copygc_wq = alloc_workqueue("bcachefs_copygc",
844                                 WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) ||
845             !(c->io_complete_wq = alloc_workqueue("bcachefs_io",
846                                 WQ_FREEZABLE|WQ_HIGHPRI|WQ_MEM_RECLAIM, 1)) ||
847             !(c->write_ref_wq = alloc_workqueue("bcachefs_write_ref",
848                                 WQ_FREEZABLE, 0)) ||
849 #ifndef BCH_WRITE_REF_DEBUG
850             percpu_ref_init(&c->writes, bch2_writes_disabled,
851                             PERCPU_REF_INIT_DEAD, GFP_KERNEL) ||
852 #endif
853             mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) ||
854             bioset_init(&c->btree_bio, 1,
855                         max(offsetof(struct btree_read_bio, bio),
856                             offsetof(struct btree_write_bio, wbio.bio)),
857                         BIOSET_NEED_BVECS) ||
858             !(c->pcpu = alloc_percpu(struct bch_fs_pcpu)) ||
859             !(c->online_reserved = alloc_percpu(u64)) ||
860             mempool_init_kvpmalloc_pool(&c->btree_bounce_pool, 1,
861                                         btree_bytes(c)) ||
862             mempool_init_kmalloc_pool(&c->large_bkey_pool, 1, 2048) ||
863             !(c->unused_inode_hints = kcalloc(1U << c->inode_shard_bits,
864                                               sizeof(u64), GFP_KERNEL))) {
865                 ret = -BCH_ERR_ENOMEM_fs_other_alloc;
866                 goto err;
867         }
868
869         ret = bch2_fs_counters_init(c) ?:
870             bch2_fs_sb_errors_init(c) ?:
871             bch2_io_clock_init(&c->io_clock[READ]) ?:
872             bch2_io_clock_init(&c->io_clock[WRITE]) ?:
873             bch2_fs_journal_init(&c->journal) ?:
874             bch2_fs_replicas_init(c) ?:
875             bch2_fs_btree_cache_init(c) ?:
876             bch2_fs_btree_key_cache_init(&c->btree_key_cache) ?:
877             bch2_fs_btree_iter_init(c) ?:
878             bch2_fs_btree_interior_update_init(c) ?:
879             bch2_fs_buckets_waiting_for_journal_init(c) ?:
880             bch2_fs_btree_write_buffer_init(c) ?:
881             bch2_fs_subvolumes_init(c) ?:
882             bch2_fs_io_read_init(c) ?:
883             bch2_fs_io_write_init(c) ?:
884             bch2_fs_nocow_locking_init(c) ?:
885             bch2_fs_encryption_init(c) ?:
886             bch2_fs_compress_init(c) ?:
887             bch2_fs_ec_init(c) ?:
888             bch2_fs_fsio_init(c) ?:
889             bch2_fs_fs_io_buffered_init(c) ?:
890             bch2_fs_fs_io_direct_init(c);
891         if (ret)
892                 goto err;
893
894         for (i = 0; i < c->sb.nr_devices; i++)
895                 if (bch2_dev_exists(c->disk_sb.sb, i) &&
896                     bch2_dev_alloc(c, i)) {
897                         ret = -EEXIST;
898                         goto err;
899                 }
900
901         bch2_journal_entry_res_resize(&c->journal,
902                         &c->btree_root_journal_res,
903                         BTREE_ID_NR * (JSET_KEYS_U64s + BKEY_BTREE_PTR_U64s_MAX));
904         bch2_dev_usage_journal_reserve(c);
905         bch2_journal_entry_res_resize(&c->journal,
906                         &c->clock_journal_res,
907                         (sizeof(struct jset_entry_clock) / sizeof(u64)) * 2);
908
909         mutex_lock(&bch_fs_list_lock);
910         ret = bch2_fs_online(c);
911         mutex_unlock(&bch_fs_list_lock);
912
913         if (ret)
914                 goto err;
915 out:
916         return c;
917 err:
918         bch2_fs_free(c);
919         c = ERR_PTR(ret);
920         goto out;
921 }
922
923 noinline_for_stack
924 static void print_mount_opts(struct bch_fs *c)
925 {
926         enum bch_opt_id i;
927         struct printbuf p = PRINTBUF;
928         bool first = true;
929
930         prt_str(&p, "mounting version ");
931         bch2_version_to_text(&p, c->sb.version);
932
933         if (c->opts.read_only) {
934                 prt_str(&p, " opts=");
935                 first = false;
936                 prt_printf(&p, "ro");
937         }
938
939         for (i = 0; i < bch2_opts_nr; i++) {
940                 const struct bch_option *opt = &bch2_opt_table[i];
941                 u64 v = bch2_opt_get_by_id(&c->opts, i);
942
943                 if (!(opt->flags & OPT_MOUNT))
944                         continue;
945
946                 if (v == bch2_opt_get_by_id(&bch2_opts_default, i))
947                         continue;
948
949                 prt_str(&p, first ? " opts=" : ",");
950                 first = false;
951                 bch2_opt_to_text(&p, c, c->disk_sb.sb, opt, v, OPT_SHOW_MOUNT_STYLE);
952         }
953
954         bch_info(c, "%s", p.buf);
955         printbuf_exit(&p);
956 }
957
958 int bch2_fs_start(struct bch_fs *c)
959 {
960         struct bch_dev *ca;
961         time64_t now = ktime_get_real_seconds();
962         unsigned i;
963         int ret;
964
965         print_mount_opts(c);
966
967         down_write(&c->state_lock);
968
969         BUG_ON(test_bit(BCH_FS_started, &c->flags));
970
971         mutex_lock(&c->sb_lock);
972
973         ret = bch2_sb_members_v2_init(c);
974         if (ret) {
975                 mutex_unlock(&c->sb_lock);
976                 goto err;
977         }
978
979         for_each_online_member(ca, c, i)
980                 bch2_members_v2_get_mut(c->disk_sb.sb, i)->last_mount = cpu_to_le64(now);
981
982         mutex_unlock(&c->sb_lock);
983
984         for_each_rw_member(ca, c, i)
985                 bch2_dev_allocator_add(c, ca);
986         bch2_recalc_capacity(c);
987
988         ret = BCH_SB_INITIALIZED(c->disk_sb.sb)
989                 ? bch2_fs_recovery(c)
990                 : bch2_fs_initialize(c);
991         if (ret)
992                 goto err;
993
994         ret = bch2_opts_check_may_set(c);
995         if (ret)
996                 goto err;
997
998         if (bch2_fs_init_fault("fs_start")) {
999                 bch_err(c, "fs_start fault injected");
1000                 ret = -EINVAL;
1001                 goto err;
1002         }
1003
1004         set_bit(BCH_FS_started, &c->flags);
1005
1006         if (c->opts.read_only || c->opts.nochanges) {
1007                 bch2_fs_read_only(c);
1008         } else {
1009                 ret = !test_bit(BCH_FS_rw, &c->flags)
1010                         ? bch2_fs_read_write(c)
1011                         : bch2_fs_read_write_late(c);
1012                 if (ret)
1013                         goto err;
1014         }
1015
1016         ret = 0;
1017 out:
1018         up_write(&c->state_lock);
1019         return ret;
1020 err:
1021         bch_err_msg(c, ret, "starting filesystem");
1022         goto out;
1023 }
1024
1025 static int bch2_dev_may_add(struct bch_sb *sb, struct bch_fs *c)
1026 {
1027         struct bch_member m = bch2_sb_member_get(sb, sb->dev_idx);
1028
1029         if (le16_to_cpu(sb->block_size) != block_sectors(c))
1030                 return -BCH_ERR_mismatched_block_size;
1031
1032         if (le16_to_cpu(m.bucket_size) <
1033             BCH_SB_BTREE_NODE_SIZE(c->disk_sb.sb))
1034                 return -BCH_ERR_bucket_size_too_small;
1035
1036         return 0;
1037 }
1038
1039 static int bch2_dev_in_fs(struct bch_sb *fs, struct bch_sb *sb)
1040 {
1041         struct bch_sb *newest =
1042                 le64_to_cpu(fs->seq) > le64_to_cpu(sb->seq) ? fs : sb;
1043
1044         if (!uuid_equal(&fs->uuid, &sb->uuid))
1045                 return -BCH_ERR_device_not_a_member_of_filesystem;
1046
1047         if (!bch2_dev_exists(newest, sb->dev_idx))
1048                 return -BCH_ERR_device_has_been_removed;
1049
1050         if (fs->block_size != sb->block_size)
1051                 return -BCH_ERR_mismatched_block_size;
1052
1053         return 0;
1054 }
1055
1056 /* Device startup/shutdown: */
1057
1058 static void bch2_dev_release(struct kobject *kobj)
1059 {
1060         struct bch_dev *ca = container_of(kobj, struct bch_dev, kobj);
1061
1062         kfree(ca);
1063 }
1064
1065 static void bch2_dev_free(struct bch_dev *ca)
1066 {
1067         cancel_work_sync(&ca->io_error_work);
1068
1069         if (ca->kobj.state_in_sysfs &&
1070             ca->disk_sb.bdev)
1071                 sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs");
1072
1073         if (ca->kobj.state_in_sysfs)
1074                 kobject_del(&ca->kobj);
1075
1076         bch2_free_super(&ca->disk_sb);
1077         bch2_dev_journal_exit(ca);
1078
1079         free_percpu(ca->io_done);
1080         bioset_exit(&ca->replica_set);
1081         bch2_dev_buckets_free(ca);
1082         free_page((unsigned long) ca->sb_read_scratch);
1083
1084         bch2_time_stats_exit(&ca->io_latency[WRITE]);
1085         bch2_time_stats_exit(&ca->io_latency[READ]);
1086
1087         percpu_ref_exit(&ca->io_ref);
1088         percpu_ref_exit(&ca->ref);
1089         kobject_put(&ca->kobj);
1090 }
1091
1092 static void __bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca)
1093 {
1094
1095         lockdep_assert_held(&c->state_lock);
1096
1097         if (percpu_ref_is_zero(&ca->io_ref))
1098                 return;
1099
1100         __bch2_dev_read_only(c, ca);
1101
1102         reinit_completion(&ca->io_ref_completion);
1103         percpu_ref_kill(&ca->io_ref);
1104         wait_for_completion(&ca->io_ref_completion);
1105
1106         if (ca->kobj.state_in_sysfs) {
1107                 sysfs_remove_link(bdev_kobj(ca->disk_sb.bdev), "bcachefs");
1108                 sysfs_remove_link(&ca->kobj, "block");
1109         }
1110
1111         bch2_free_super(&ca->disk_sb);
1112         bch2_dev_journal_exit(ca);
1113 }
1114
1115 static void bch2_dev_ref_complete(struct percpu_ref *ref)
1116 {
1117         struct bch_dev *ca = container_of(ref, struct bch_dev, ref);
1118
1119         complete(&ca->ref_completion);
1120 }
1121
1122 static void bch2_dev_io_ref_complete(struct percpu_ref *ref)
1123 {
1124         struct bch_dev *ca = container_of(ref, struct bch_dev, io_ref);
1125
1126         complete(&ca->io_ref_completion);
1127 }
1128
1129 static int bch2_dev_sysfs_online(struct bch_fs *c, struct bch_dev *ca)
1130 {
1131         int ret;
1132
1133         if (!c->kobj.state_in_sysfs)
1134                 return 0;
1135
1136         if (!ca->kobj.state_in_sysfs) {
1137                 ret = kobject_add(&ca->kobj, &c->kobj,
1138                                   "dev-%u", ca->dev_idx);
1139                 if (ret)
1140                         return ret;
1141         }
1142
1143         if (ca->disk_sb.bdev) {
1144                 struct kobject *block = bdev_kobj(ca->disk_sb.bdev);
1145
1146                 ret = sysfs_create_link(block, &ca->kobj, "bcachefs");
1147                 if (ret)
1148                         return ret;
1149
1150                 ret = sysfs_create_link(&ca->kobj, block, "block");
1151                 if (ret)
1152                         return ret;
1153         }
1154
1155         return 0;
1156 }
1157
1158 static struct bch_dev *__bch2_dev_alloc(struct bch_fs *c,
1159                                         struct bch_member *member)
1160 {
1161         struct bch_dev *ca;
1162         unsigned i;
1163
1164         ca = kzalloc(sizeof(*ca), GFP_KERNEL);
1165         if (!ca)
1166                 return NULL;
1167
1168         kobject_init(&ca->kobj, &bch2_dev_ktype);
1169         init_completion(&ca->ref_completion);
1170         init_completion(&ca->io_ref_completion);
1171
1172         init_rwsem(&ca->bucket_lock);
1173
1174         INIT_WORK(&ca->io_error_work, bch2_io_error_work);
1175
1176         bch2_time_stats_init(&ca->io_latency[READ]);
1177         bch2_time_stats_init(&ca->io_latency[WRITE]);
1178
1179         ca->mi = bch2_mi_to_cpu(member);
1180
1181         for (i = 0; i < ARRAY_SIZE(member->errors); i++)
1182                 atomic64_set(&ca->errors[i], le64_to_cpu(member->errors[i]));
1183
1184         ca->uuid = member->uuid;
1185
1186         ca->nr_btree_reserve = DIV_ROUND_UP(BTREE_NODE_RESERVE,
1187                              ca->mi.bucket_size / btree_sectors(c));
1188
1189         if (percpu_ref_init(&ca->ref, bch2_dev_ref_complete,
1190                             0, GFP_KERNEL) ||
1191             percpu_ref_init(&ca->io_ref, bch2_dev_io_ref_complete,
1192                             PERCPU_REF_INIT_DEAD, GFP_KERNEL) ||
1193             !(ca->sb_read_scratch = (void *) __get_free_page(GFP_KERNEL)) ||
1194             bch2_dev_buckets_alloc(c, ca) ||
1195             bioset_init(&ca->replica_set, 4,
1196                         offsetof(struct bch_write_bio, bio), 0) ||
1197             !(ca->io_done       = alloc_percpu(*ca->io_done)))
1198                 goto err;
1199
1200         return ca;
1201 err:
1202         bch2_dev_free(ca);
1203         return NULL;
1204 }
1205
1206 static void bch2_dev_attach(struct bch_fs *c, struct bch_dev *ca,
1207                             unsigned dev_idx)
1208 {
1209         ca->dev_idx = dev_idx;
1210         __set_bit(ca->dev_idx, ca->self.d);
1211         scnprintf(ca->name, sizeof(ca->name), "dev-%u", dev_idx);
1212
1213         ca->fs = c;
1214         rcu_assign_pointer(c->devs[ca->dev_idx], ca);
1215
1216         if (bch2_dev_sysfs_online(c, ca))
1217                 pr_warn("error creating sysfs objects");
1218 }
1219
1220 static int bch2_dev_alloc(struct bch_fs *c, unsigned dev_idx)
1221 {
1222         struct bch_member member = bch2_sb_member_get(c->disk_sb.sb, dev_idx);
1223         struct bch_dev *ca = NULL;
1224         int ret = 0;
1225
1226         if (bch2_fs_init_fault("dev_alloc"))
1227                 goto err;
1228
1229         ca = __bch2_dev_alloc(c, &member);
1230         if (!ca)
1231                 goto err;
1232
1233         ca->fs = c;
1234
1235         bch2_dev_attach(c, ca, dev_idx);
1236         return ret;
1237 err:
1238         if (ca)
1239                 bch2_dev_free(ca);
1240         return -BCH_ERR_ENOMEM_dev_alloc;
1241 }
1242
1243 static int __bch2_dev_attach_bdev(struct bch_dev *ca, struct bch_sb_handle *sb)
1244 {
1245         unsigned ret;
1246
1247         if (bch2_dev_is_online(ca)) {
1248                 bch_err(ca, "already have device online in slot %u",
1249                         sb->sb->dev_idx);
1250                 return -BCH_ERR_device_already_online;
1251         }
1252
1253         if (get_capacity(sb->bdev->bd_disk) <
1254             ca->mi.bucket_size * ca->mi.nbuckets) {
1255                 bch_err(ca, "cannot online: device too small");
1256                 return -BCH_ERR_device_size_too_small;
1257         }
1258
1259         BUG_ON(!percpu_ref_is_zero(&ca->io_ref));
1260
1261         ret = bch2_dev_journal_init(ca, sb->sb);
1262         if (ret)
1263                 return ret;
1264
1265         /* Commit: */
1266         ca->disk_sb = *sb;
1267         memset(sb, 0, sizeof(*sb));
1268
1269         ca->dev = ca->disk_sb.bdev->bd_dev;
1270
1271         percpu_ref_reinit(&ca->io_ref);
1272
1273         return 0;
1274 }
1275
1276 static int bch2_dev_attach_bdev(struct bch_fs *c, struct bch_sb_handle *sb)
1277 {
1278         struct bch_dev *ca;
1279         int ret;
1280
1281         lockdep_assert_held(&c->state_lock);
1282
1283         if (le64_to_cpu(sb->sb->seq) >
1284             le64_to_cpu(c->disk_sb.sb->seq))
1285                 bch2_sb_to_fs(c, sb->sb);
1286
1287         BUG_ON(sb->sb->dev_idx >= c->sb.nr_devices ||
1288                !c->devs[sb->sb->dev_idx]);
1289
1290         ca = bch_dev_locked(c, sb->sb->dev_idx);
1291
1292         ret = __bch2_dev_attach_bdev(ca, sb);
1293         if (ret)
1294                 return ret;
1295
1296         bch2_dev_sysfs_online(c, ca);
1297
1298         if (c->sb.nr_devices == 1)
1299                 snprintf(c->name, sizeof(c->name), "%pg", ca->disk_sb.bdev);
1300         snprintf(ca->name, sizeof(ca->name), "%pg", ca->disk_sb.bdev);
1301
1302         rebalance_wakeup(c);
1303         return 0;
1304 }
1305
1306 /* Device management: */
1307
1308 /*
1309  * Note: this function is also used by the error paths - when a particular
1310  * device sees an error, we call it to determine whether we can just set the
1311  * device RO, or - if this function returns false - we'll set the whole
1312  * filesystem RO:
1313  *
1314  * XXX: maybe we should be more explicit about whether we're changing state
1315  * because we got an error or what have you?
1316  */
1317 bool bch2_dev_state_allowed(struct bch_fs *c, struct bch_dev *ca,
1318                             enum bch_member_state new_state, int flags)
1319 {
1320         struct bch_devs_mask new_online_devs;
1321         struct bch_dev *ca2;
1322         int i, nr_rw = 0, required;
1323
1324         lockdep_assert_held(&c->state_lock);
1325
1326         switch (new_state) {
1327         case BCH_MEMBER_STATE_rw:
1328                 return true;
1329         case BCH_MEMBER_STATE_ro:
1330                 if (ca->mi.state != BCH_MEMBER_STATE_rw)
1331                         return true;
1332
1333                 /* do we have enough devices to write to?  */
1334                 for_each_member_device(ca2, c, i)
1335                         if (ca2 != ca)
1336                                 nr_rw += ca2->mi.state == BCH_MEMBER_STATE_rw;
1337
1338                 required = max(!(flags & BCH_FORCE_IF_METADATA_DEGRADED)
1339                                ? c->opts.metadata_replicas
1340                                : c->opts.metadata_replicas_required,
1341                                !(flags & BCH_FORCE_IF_DATA_DEGRADED)
1342                                ? c->opts.data_replicas
1343                                : c->opts.data_replicas_required);
1344
1345                 return nr_rw >= required;
1346         case BCH_MEMBER_STATE_failed:
1347         case BCH_MEMBER_STATE_spare:
1348                 if (ca->mi.state != BCH_MEMBER_STATE_rw &&
1349                     ca->mi.state != BCH_MEMBER_STATE_ro)
1350                         return true;
1351
1352                 /* do we have enough devices to read from?  */
1353                 new_online_devs = bch2_online_devs(c);
1354                 __clear_bit(ca->dev_idx, new_online_devs.d);
1355
1356                 return bch2_have_enough_devs(c, new_online_devs, flags, false);
1357         default:
1358                 BUG();
1359         }
1360 }
1361
1362 static bool bch2_fs_may_start(struct bch_fs *c)
1363 {
1364         struct bch_dev *ca;
1365         unsigned i, flags = 0;
1366
1367         if (c->opts.very_degraded)
1368                 flags |= BCH_FORCE_IF_DEGRADED|BCH_FORCE_IF_LOST;
1369
1370         if (c->opts.degraded)
1371                 flags |= BCH_FORCE_IF_DEGRADED;
1372
1373         if (!c->opts.degraded &&
1374             !c->opts.very_degraded) {
1375                 mutex_lock(&c->sb_lock);
1376
1377                 for (i = 0; i < c->disk_sb.sb->nr_devices; i++) {
1378                         if (!bch2_dev_exists(c->disk_sb.sb, i))
1379                                 continue;
1380
1381                         ca = bch_dev_locked(c, i);
1382
1383                         if (!bch2_dev_is_online(ca) &&
1384                             (ca->mi.state == BCH_MEMBER_STATE_rw ||
1385                              ca->mi.state == BCH_MEMBER_STATE_ro)) {
1386                                 mutex_unlock(&c->sb_lock);
1387                                 return false;
1388                         }
1389                 }
1390                 mutex_unlock(&c->sb_lock);
1391         }
1392
1393         return bch2_have_enough_devs(c, bch2_online_devs(c), flags, true);
1394 }
1395
1396 static void __bch2_dev_read_only(struct bch_fs *c, struct bch_dev *ca)
1397 {
1398         /*
1399          * The allocator thread itself allocates btree nodes, so stop it first:
1400          */
1401         bch2_dev_allocator_remove(c, ca);
1402         bch2_dev_journal_stop(&c->journal, ca);
1403 }
1404
1405 static void __bch2_dev_read_write(struct bch_fs *c, struct bch_dev *ca)
1406 {
1407         lockdep_assert_held(&c->state_lock);
1408
1409         BUG_ON(ca->mi.state != BCH_MEMBER_STATE_rw);
1410
1411         bch2_dev_allocator_add(c, ca);
1412         bch2_recalc_capacity(c);
1413 }
1414
1415 int __bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,
1416                          enum bch_member_state new_state, int flags)
1417 {
1418         struct bch_member *m;
1419         int ret = 0;
1420
1421         if (ca->mi.state == new_state)
1422                 return 0;
1423
1424         if (!bch2_dev_state_allowed(c, ca, new_state, flags))
1425                 return -BCH_ERR_device_state_not_allowed;
1426
1427         if (new_state != BCH_MEMBER_STATE_rw)
1428                 __bch2_dev_read_only(c, ca);
1429
1430         bch_notice(ca, "%s", bch2_member_states[new_state]);
1431
1432         mutex_lock(&c->sb_lock);
1433         m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
1434         SET_BCH_MEMBER_STATE(m, new_state);
1435         bch2_write_super(c);
1436         mutex_unlock(&c->sb_lock);
1437
1438         if (new_state == BCH_MEMBER_STATE_rw)
1439                 __bch2_dev_read_write(c, ca);
1440
1441         rebalance_wakeup(c);
1442
1443         return ret;
1444 }
1445
1446 int bch2_dev_set_state(struct bch_fs *c, struct bch_dev *ca,
1447                        enum bch_member_state new_state, int flags)
1448 {
1449         int ret;
1450
1451         down_write(&c->state_lock);
1452         ret = __bch2_dev_set_state(c, ca, new_state, flags);
1453         up_write(&c->state_lock);
1454
1455         return ret;
1456 }
1457
1458 /* Device add/removal: */
1459
1460 static int bch2_dev_remove_alloc(struct bch_fs *c, struct bch_dev *ca)
1461 {
1462         struct bpos start       = POS(ca->dev_idx, 0);
1463         struct bpos end         = POS(ca->dev_idx, U64_MAX);
1464         int ret;
1465
1466         /*
1467          * We clear the LRU and need_discard btrees first so that we don't race
1468          * with bch2_do_invalidates() and bch2_do_discards()
1469          */
1470         ret =   bch2_btree_delete_range(c, BTREE_ID_lru, start, end,
1471                                         BTREE_TRIGGER_NORUN, NULL) ?:
1472                 bch2_btree_delete_range(c, BTREE_ID_need_discard, start, end,
1473                                         BTREE_TRIGGER_NORUN, NULL) ?:
1474                 bch2_btree_delete_range(c, BTREE_ID_freespace, start, end,
1475                                         BTREE_TRIGGER_NORUN, NULL) ?:
1476                 bch2_btree_delete_range(c, BTREE_ID_backpointers, start, end,
1477                                         BTREE_TRIGGER_NORUN, NULL) ?:
1478                 bch2_btree_delete_range(c, BTREE_ID_alloc, start, end,
1479                                         BTREE_TRIGGER_NORUN, NULL) ?:
1480                 bch2_btree_delete_range(c, BTREE_ID_bucket_gens, start, end,
1481                                         BTREE_TRIGGER_NORUN, NULL);
1482         if (ret)
1483                 bch_err_msg(c, ret, "removing dev alloc info");
1484
1485         return ret;
1486 }
1487
1488 int bch2_dev_remove(struct bch_fs *c, struct bch_dev *ca, int flags)
1489 {
1490         struct bch_member *m;
1491         unsigned dev_idx = ca->dev_idx, data;
1492         int ret;
1493
1494         down_write(&c->state_lock);
1495
1496         /*
1497          * We consume a reference to ca->ref, regardless of whether we succeed
1498          * or fail:
1499          */
1500         percpu_ref_put(&ca->ref);
1501
1502         if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) {
1503                 bch_err(ca, "Cannot remove without losing data");
1504                 ret = -BCH_ERR_device_state_not_allowed;
1505                 goto err;
1506         }
1507
1508         __bch2_dev_read_only(c, ca);
1509
1510         ret = bch2_dev_data_drop(c, ca->dev_idx, flags);
1511         if (ret) {
1512                 bch_err_msg(ca, ret, "dropping data");
1513                 goto err;
1514         }
1515
1516         ret = bch2_dev_remove_alloc(c, ca);
1517         if (ret) {
1518                 bch_err_msg(ca, ret, "deleting alloc info");
1519                 goto err;
1520         }
1521
1522         ret = bch2_journal_flush_device_pins(&c->journal, ca->dev_idx);
1523         if (ret) {
1524                 bch_err_msg(ca, ret, "flushing journal");
1525                 goto err;
1526         }
1527
1528         ret = bch2_journal_flush(&c->journal);
1529         if (ret) {
1530                 bch_err(ca, "journal error");
1531                 goto err;
1532         }
1533
1534         ret = bch2_replicas_gc2(c);
1535         if (ret) {
1536                 bch_err_msg(ca, ret, "in replicas_gc2()");
1537                 goto err;
1538         }
1539
1540         data = bch2_dev_has_data(c, ca);
1541         if (data) {
1542                 struct printbuf data_has = PRINTBUF;
1543
1544                 prt_bitflags(&data_has, bch2_data_types, data);
1545                 bch_err(ca, "Remove failed, still has data (%s)", data_has.buf);
1546                 printbuf_exit(&data_has);
1547                 ret = -EBUSY;
1548                 goto err;
1549         }
1550
1551         __bch2_dev_offline(c, ca);
1552
1553         mutex_lock(&c->sb_lock);
1554         rcu_assign_pointer(c->devs[ca->dev_idx], NULL);
1555         mutex_unlock(&c->sb_lock);
1556
1557         percpu_ref_kill(&ca->ref);
1558         wait_for_completion(&ca->ref_completion);
1559
1560         bch2_dev_free(ca);
1561
1562         /*
1563          * At this point the device object has been removed in-core, but the
1564          * on-disk journal might still refer to the device index via sb device
1565          * usage entries. Recovery fails if it sees usage information for an
1566          * invalid device. Flush journal pins to push the back of the journal
1567          * past now invalid device index references before we update the
1568          * superblock, but after the device object has been removed so any
1569          * further journal writes elide usage info for the device.
1570          */
1571         bch2_journal_flush_all_pins(&c->journal);
1572
1573         /*
1574          * Free this device's slot in the bch_member array - all pointers to
1575          * this device must be gone:
1576          */
1577         mutex_lock(&c->sb_lock);
1578         m = bch2_members_v2_get_mut(c->disk_sb.sb, dev_idx);
1579         memset(&m->uuid, 0, sizeof(m->uuid));
1580
1581         bch2_write_super(c);
1582
1583         mutex_unlock(&c->sb_lock);
1584         up_write(&c->state_lock);
1585
1586         bch2_dev_usage_journal_reserve(c);
1587         return 0;
1588 err:
1589         if (ca->mi.state == BCH_MEMBER_STATE_rw &&
1590             !percpu_ref_is_zero(&ca->io_ref))
1591                 __bch2_dev_read_write(c, ca);
1592         up_write(&c->state_lock);
1593         return ret;
1594 }
1595
1596 /* Add new device to running filesystem: */
1597 int bch2_dev_add(struct bch_fs *c, const char *path)
1598 {
1599         struct bch_opts opts = bch2_opts_empty();
1600         struct bch_sb_handle sb;
1601         struct bch_dev *ca = NULL;
1602         struct bch_sb_field_members_v2 *mi;
1603         struct bch_member dev_mi;
1604         unsigned dev_idx, nr_devices, u64s;
1605         struct printbuf errbuf = PRINTBUF;
1606         struct printbuf label = PRINTBUF;
1607         int ret;
1608
1609         ret = bch2_read_super(path, &opts, &sb);
1610         if (ret) {
1611                 bch_err_msg(c, ret, "reading super");
1612                 goto err;
1613         }
1614
1615         dev_mi = bch2_sb_member_get(sb.sb, sb.sb->dev_idx);
1616
1617         if (BCH_MEMBER_GROUP(&dev_mi)) {
1618                 bch2_disk_path_to_text_sb(&label, sb.sb, BCH_MEMBER_GROUP(&dev_mi) - 1);
1619                 if (label.allocation_failure) {
1620                         ret = -ENOMEM;
1621                         goto err;
1622                 }
1623         }
1624
1625         ret = bch2_dev_may_add(sb.sb, c);
1626         if (ret) {
1627                 bch_err_fn(c, ret);
1628                 goto err;
1629         }
1630
1631         ca = __bch2_dev_alloc(c, &dev_mi);
1632         if (!ca) {
1633                 ret = -ENOMEM;
1634                 goto err;
1635         }
1636
1637         bch2_dev_usage_init(ca);
1638
1639         ret = __bch2_dev_attach_bdev(ca, &sb);
1640         if (ret)
1641                 goto err;
1642
1643         ret = bch2_dev_journal_alloc(ca);
1644         if (ret) {
1645                 bch_err_msg(c, ret, "allocating journal");
1646                 goto err;
1647         }
1648
1649         down_write(&c->state_lock);
1650         mutex_lock(&c->sb_lock);
1651
1652         ret = bch2_sb_from_fs(c, ca);
1653         if (ret) {
1654                 bch_err_msg(c, ret, "setting up new superblock");
1655                 goto err_unlock;
1656         }
1657
1658         if (dynamic_fault("bcachefs:add:no_slot"))
1659                 goto no_slot;
1660
1661         for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++)
1662                 if (!bch2_dev_exists(c->disk_sb.sb, dev_idx))
1663                         goto have_slot;
1664 no_slot:
1665         ret = -BCH_ERR_ENOSPC_sb_members;
1666         bch_err_msg(c, ret, "setting up new superblock");
1667         goto err_unlock;
1668
1669 have_slot:
1670         nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices);
1671
1672         mi = bch2_sb_field_get(c->disk_sb.sb, members_v2);
1673         u64s = DIV_ROUND_UP(sizeof(struct bch_sb_field_members_v2) +
1674                             le16_to_cpu(mi->member_bytes) * nr_devices, sizeof(u64));
1675
1676         mi = bch2_sb_field_resize(&c->disk_sb, members_v2, u64s);
1677         if (!mi) {
1678                 ret = -BCH_ERR_ENOSPC_sb_members;
1679                 bch_err_msg(c, ret, "setting up new superblock");
1680                 goto err_unlock;
1681         }
1682         struct bch_member *m = bch2_members_v2_get_mut(c->disk_sb.sb, dev_idx);
1683
1684         /* success: */
1685
1686         *m = dev_mi;
1687         m->last_mount = cpu_to_le64(ktime_get_real_seconds());
1688         c->disk_sb.sb->nr_devices       = nr_devices;
1689
1690         ca->disk_sb.sb->dev_idx = dev_idx;
1691         bch2_dev_attach(c, ca, dev_idx);
1692
1693         if (BCH_MEMBER_GROUP(&dev_mi)) {
1694                 ret = __bch2_dev_group_set(c, ca, label.buf);
1695                 if (ret) {
1696                         bch_err_msg(c, ret, "creating new label");
1697                         goto err_unlock;
1698                 }
1699         }
1700
1701         bch2_write_super(c);
1702         mutex_unlock(&c->sb_lock);
1703
1704         bch2_dev_usage_journal_reserve(c);
1705
1706         ret = bch2_trans_mark_dev_sb(c, ca);
1707         if (ret) {
1708                 bch_err_msg(ca, ret, "marking new superblock");
1709                 goto err_late;
1710         }
1711
1712         ret = bch2_fs_freespace_init(c);
1713         if (ret) {
1714                 bch_err_msg(ca, ret, "initializing free space");
1715                 goto err_late;
1716         }
1717
1718         ca->new_fs_bucket_idx = 0;
1719
1720         if (ca->mi.state == BCH_MEMBER_STATE_rw)
1721                 __bch2_dev_read_write(c, ca);
1722
1723         up_write(&c->state_lock);
1724         return 0;
1725
1726 err_unlock:
1727         mutex_unlock(&c->sb_lock);
1728         up_write(&c->state_lock);
1729 err:
1730         if (ca)
1731                 bch2_dev_free(ca);
1732         bch2_free_super(&sb);
1733         printbuf_exit(&label);
1734         printbuf_exit(&errbuf);
1735         return ret;
1736 err_late:
1737         up_write(&c->state_lock);
1738         ca = NULL;
1739         goto err;
1740 }
1741
1742 /* Hot add existing device to running filesystem: */
1743 int bch2_dev_online(struct bch_fs *c, const char *path)
1744 {
1745         struct bch_opts opts = bch2_opts_empty();
1746         struct bch_sb_handle sb = { NULL };
1747         struct bch_dev *ca;
1748         unsigned dev_idx;
1749         int ret;
1750
1751         down_write(&c->state_lock);
1752
1753         ret = bch2_read_super(path, &opts, &sb);
1754         if (ret) {
1755                 up_write(&c->state_lock);
1756                 return ret;
1757         }
1758
1759         dev_idx = sb.sb->dev_idx;
1760
1761         ret = bch2_dev_in_fs(c->disk_sb.sb, sb.sb);
1762         if (ret) {
1763                 bch_err_msg(c, ret, "bringing %s online", path);
1764                 goto err;
1765         }
1766
1767         ret = bch2_dev_attach_bdev(c, &sb);
1768         if (ret)
1769                 goto err;
1770
1771         ca = bch_dev_locked(c, dev_idx);
1772
1773         ret = bch2_trans_mark_dev_sb(c, ca);
1774         if (ret) {
1775                 bch_err_msg(c, ret, "bringing %s online: error from bch2_trans_mark_dev_sb", path);
1776                 goto err;
1777         }
1778
1779         if (ca->mi.state == BCH_MEMBER_STATE_rw)
1780                 __bch2_dev_read_write(c, ca);
1781
1782         if (!ca->mi.freespace_initialized) {
1783                 ret = bch2_dev_freespace_init(c, ca, 0, ca->mi.nbuckets);
1784                 bch_err_msg(ca, ret, "initializing free space");
1785                 if (ret)
1786                         goto err;
1787         }
1788
1789         if (!ca->journal.nr) {
1790                 ret = bch2_dev_journal_alloc(ca);
1791                 bch_err_msg(ca, ret, "allocating journal");
1792                 if (ret)
1793                         goto err;
1794         }
1795
1796         mutex_lock(&c->sb_lock);
1797         bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx)->last_mount =
1798                 cpu_to_le64(ktime_get_real_seconds());
1799         bch2_write_super(c);
1800         mutex_unlock(&c->sb_lock);
1801
1802         up_write(&c->state_lock);
1803         return 0;
1804 err:
1805         up_write(&c->state_lock);
1806         bch2_free_super(&sb);
1807         return ret;
1808 }
1809
1810 int bch2_dev_offline(struct bch_fs *c, struct bch_dev *ca, int flags)
1811 {
1812         down_write(&c->state_lock);
1813
1814         if (!bch2_dev_is_online(ca)) {
1815                 bch_err(ca, "Already offline");
1816                 up_write(&c->state_lock);
1817                 return 0;
1818         }
1819
1820         if (!bch2_dev_state_allowed(c, ca, BCH_MEMBER_STATE_failed, flags)) {
1821                 bch_err(ca, "Cannot offline required disk");
1822                 up_write(&c->state_lock);
1823                 return -BCH_ERR_device_state_not_allowed;
1824         }
1825
1826         __bch2_dev_offline(c, ca);
1827
1828         up_write(&c->state_lock);
1829         return 0;
1830 }
1831
1832 int bch2_dev_resize(struct bch_fs *c, struct bch_dev *ca, u64 nbuckets)
1833 {
1834         struct bch_member *m;
1835         u64 old_nbuckets;
1836         int ret = 0;
1837
1838         down_write(&c->state_lock);
1839         old_nbuckets = ca->mi.nbuckets;
1840
1841         if (nbuckets < ca->mi.nbuckets) {
1842                 bch_err(ca, "Cannot shrink yet");
1843                 ret = -EINVAL;
1844                 goto err;
1845         }
1846
1847         if (bch2_dev_is_online(ca) &&
1848             get_capacity(ca->disk_sb.bdev->bd_disk) <
1849             ca->mi.bucket_size * nbuckets) {
1850                 bch_err(ca, "New size larger than device");
1851                 ret = -BCH_ERR_device_size_too_small;
1852                 goto err;
1853         }
1854
1855         ret = bch2_dev_buckets_resize(c, ca, nbuckets);
1856         if (ret) {
1857                 bch_err_msg(ca, ret, "resizing buckets");
1858                 goto err;
1859         }
1860
1861         ret = bch2_trans_mark_dev_sb(c, ca);
1862         if (ret)
1863                 goto err;
1864
1865         mutex_lock(&c->sb_lock);
1866         m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
1867         m->nbuckets = cpu_to_le64(nbuckets);
1868
1869         bch2_write_super(c);
1870         mutex_unlock(&c->sb_lock);
1871
1872         if (ca->mi.freespace_initialized) {
1873                 ret = bch2_dev_freespace_init(c, ca, old_nbuckets, nbuckets);
1874                 if (ret)
1875                         goto err;
1876
1877                 /*
1878                  * XXX: this is all wrong transactionally - we'll be able to do
1879                  * this correctly after the disk space accounting rewrite
1880                  */
1881                 ca->usage_base->d[BCH_DATA_free].buckets += nbuckets - old_nbuckets;
1882         }
1883
1884         bch2_recalc_capacity(c);
1885 err:
1886         up_write(&c->state_lock);
1887         return ret;
1888 }
1889
1890 /* return with ref on ca->ref: */
1891 struct bch_dev *bch2_dev_lookup(struct bch_fs *c, const char *name)
1892 {
1893         struct bch_dev *ca;
1894         unsigned i;
1895
1896         rcu_read_lock();
1897         for_each_member_device_rcu(ca, c, i, NULL)
1898                 if (!strcmp(name, ca->name))
1899                         goto found;
1900         ca = ERR_PTR(-BCH_ERR_ENOENT_dev_not_found);
1901 found:
1902         rcu_read_unlock();
1903
1904         return ca;
1905 }
1906
1907 /* Filesystem open: */
1908
1909 struct bch_fs *bch2_fs_open(char * const *devices, unsigned nr_devices,
1910                             struct bch_opts opts)
1911 {
1912         DARRAY(struct bch_sb_handle) sbs = { 0 };
1913         struct bch_fs *c = NULL;
1914         struct bch_sb_handle *sb, *best = NULL;
1915         struct printbuf errbuf = PRINTBUF;
1916         int ret = 0;
1917
1918         if (!try_module_get(THIS_MODULE))
1919                 return ERR_PTR(-ENODEV);
1920
1921         if (!nr_devices) {
1922                 ret = -EINVAL;
1923                 goto err;
1924         }
1925
1926         ret = darray_make_room(&sbs, nr_devices);
1927         if (ret)
1928                 goto err;
1929
1930         for (unsigned i = 0; i < nr_devices; i++) {
1931                 struct bch_sb_handle sb = { NULL };
1932
1933                 ret = bch2_read_super(devices[i], &opts, &sb);
1934                 if (ret)
1935                         goto err;
1936
1937                 BUG_ON(darray_push(&sbs, sb));
1938         }
1939
1940         darray_for_each(sbs, sb)
1941                 if (!best || le64_to_cpu(sb->sb->seq) > le64_to_cpu(best->sb->seq))
1942                         best = sb;
1943
1944         darray_for_each_reverse(sbs, sb) {
1945                 if (sb != best && !bch2_dev_exists(best->sb, sb->sb->dev_idx)) {
1946                         pr_info("%pg has been removed, skipping", sb->bdev);
1947                         bch2_free_super(sb);
1948                         darray_remove_item(&sbs, sb);
1949                         best -= best > sb;
1950                         continue;
1951                 }
1952
1953                 ret = bch2_dev_in_fs(best->sb, sb->sb);
1954                 if (ret)
1955                         goto err_print;
1956         }
1957
1958         c = bch2_fs_alloc(best->sb, opts);
1959         ret = PTR_ERR_OR_ZERO(c);
1960         if (ret)
1961                 goto err;
1962
1963         down_write(&c->state_lock);
1964         darray_for_each(sbs, sb) {
1965                 ret = bch2_dev_attach_bdev(c, sb);
1966                 if (ret) {
1967                         up_write(&c->state_lock);
1968                         goto err;
1969                 }
1970         }
1971         up_write(&c->state_lock);
1972
1973         if (!bch2_fs_may_start(c)) {
1974                 ret = -BCH_ERR_insufficient_devices_to_start;
1975                 goto err_print;
1976         }
1977
1978         if (!c->opts.nostart) {
1979                 ret = bch2_fs_start(c);
1980                 if (ret)
1981                         goto err;
1982         }
1983 out:
1984         darray_for_each(sbs, sb)
1985                 bch2_free_super(sb);
1986         darray_exit(&sbs);
1987         printbuf_exit(&errbuf);
1988         module_put(THIS_MODULE);
1989         return c;
1990 err_print:
1991         pr_err("bch_fs_open err opening %s: %s",
1992                devices[0], bch2_err_str(ret));
1993 err:
1994         if (!IS_ERR_OR_NULL(c))
1995                 bch2_fs_stop(c);
1996         c = ERR_PTR(ret);
1997         goto out;
1998 }
1999
2000 /* Global interfaces/init */
2001
2002 static void bcachefs_exit(void)
2003 {
2004         bch2_debug_exit();
2005         bch2_vfs_exit();
2006         bch2_chardev_exit();
2007         bch2_btree_key_cache_exit();
2008         if (bcachefs_kset)
2009                 kset_unregister(bcachefs_kset);
2010 }
2011
2012 static int __init bcachefs_init(void)
2013 {
2014         bch2_bkey_pack_test();
2015
2016         if (!(bcachefs_kset = kset_create_and_add("bcachefs", NULL, fs_kobj)) ||
2017             bch2_btree_key_cache_init() ||
2018             bch2_chardev_init() ||
2019             bch2_vfs_init() ||
2020             bch2_debug_init())
2021                 goto err;
2022
2023         return 0;
2024 err:
2025         bcachefs_exit();
2026         return -ENOMEM;
2027 }
2028
2029 #define BCH_DEBUG_PARAM(name, description)                      \
2030         bool bch2_##name;                                       \
2031         module_param_named(name, bch2_##name, bool, 0644);      \
2032         MODULE_PARM_DESC(name, description);
2033 BCH_DEBUG_PARAMS()
2034 #undef BCH_DEBUG_PARAM
2035
2036 __maybe_unused
2037 static unsigned bch2_metadata_version = bcachefs_metadata_version_current;
2038 module_param_named(version, bch2_metadata_version, uint, 0400);
2039
2040 module_exit(bcachefs_exit);
2041 module_init(bcachefs_init);