git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/super-io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "checksum.h"
   5 #include "counters.h"
   6 #include "disk_groups.h"
   7 #include "ec.h"
   8 #include "error.h"
   9 #include "journal.h"
  10 #include "journal_sb.h"
  11 #include "journal_seq_blacklist.h"
  12 #include "recovery.h"
  13 #include "replicas.h"
  14 #include "quota.h"
  15 #include "sb-clean.h"
  16 #include "sb-errors.h"
  17 #include "sb-members.h"
  18 #include "super-io.h"
  19 #include "super.h"
  20 #include "trace.h"
  21 #include "vstructs.h"
  22
  23 #include <linux/backing-dev.h>
  24 #include <linux/sort.h>
  25
  26 static const struct blk_holder_ops bch2_sb_handle_bdev_ops = {
  27 };
  28
  29 struct bch2_metadata_version {
  30         u16             version;
  31         const char      *name;
  32         u64             recovery_passes;
  33 };
  34
  35 static const struct bch2_metadata_version bch2_metadata_versions[] = {
  36 #define x(n, v, _recovery_passes) {             \
  37         .version = v,                           \
  38         .name = #n,                             \
  39         .recovery_passes = _recovery_passes,    \
  40 },
  41         BCH_METADATA_VERSIONS()
  42 #undef x
  43 };
  44
  45 void bch2_version_to_text(struct printbuf *out, unsigned v)
  46 {
  47         const char *str = "(unknown version)";
  48
  49         for (unsigned i = 0; i < ARRAY_SIZE(bch2_metadata_versions); i++)
  50                 if (bch2_metadata_versions[i].version == v) {
  51                         str = bch2_metadata_versions[i].name;
  52                         break;
  53                 }
  54
  55         prt_printf(out, "%u.%u: %s", BCH_VERSION_MAJOR(v), BCH_VERSION_MINOR(v), str);
  56 }
  57
  58 unsigned bch2_latest_compatible_version(unsigned v)
  59 {
  60         if (!BCH_VERSION_MAJOR(v))
  61                 return v;
  62
  63         for (unsigned i = 0; i < ARRAY_SIZE(bch2_metadata_versions); i++)
  64                 if (bch2_metadata_versions[i].version > v &&
  65                     BCH_VERSION_MAJOR(bch2_metadata_versions[i].version) ==
  66                     BCH_VERSION_MAJOR(v))
  67                         v = bch2_metadata_versions[i].version;
  68
  69         return v;
  70 }
  71
  72 u64 bch2_upgrade_recovery_passes(struct bch_fs *c,
  73                                  unsigned old_version,
  74                                  unsigned new_version)
  75 {
  76         u64 ret = 0;
  77
  78         for (const struct bch2_metadata_version *i = bch2_metadata_versions;
  79              i < bch2_metadata_versions + ARRAY_SIZE(bch2_metadata_versions);
  80              i++)
  81                 if (i->version > old_version && i->version <= new_version) {
  82                         if (i->recovery_passes & RECOVERY_PASS_ALL_FSCK)
  83                                 ret |= bch2_fsck_recovery_passes();
  84                         ret |= i->recovery_passes;
  85                 }
  86
  87         return ret &= ~RECOVERY_PASS_ALL_FSCK;
  88 }
  89
  90 const char * const bch2_sb_fields[] = {
  91 #define x(name, nr)     #name,
  92         BCH_SB_FIELDS()
  93 #undef x
  94         NULL
  95 };
  96
  97 static int bch2_sb_field_validate(struct bch_sb *, struct bch_sb_field *,
  98                                   struct printbuf *);
  99
 100 struct bch_sb_field *bch2_sb_field_get_id(struct bch_sb *sb,
 101                                       enum bch_sb_field_type type)
 102 {
 103         struct bch_sb_field *f;
 104
 105         /* XXX: need locking around superblock to access optional fields */
 106
 107         vstruct_for_each(sb, f)
 108                 if (le32_to_cpu(f->type) == type)
 109                         return f;
 110         return NULL;
 111 }
 112
 113 static struct bch_sb_field *__bch2_sb_field_resize(struct bch_sb_handle *sb,
 114                                                    struct bch_sb_field *f,
 115                                                    unsigned u64s)
 116 {
 117         unsigned old_u64s = f ? le32_to_cpu(f->u64s) : 0;
 118         unsigned sb_u64s = le32_to_cpu(sb->sb->u64s) + u64s - old_u64s;
 119
 120         BUG_ON(__vstruct_bytes(struct bch_sb, sb_u64s) > sb->buffer_size);
 121
 122         if (!f && !u64s) {
 123                 /* nothing to do: */
 124         } else if (!f) {
 125                 f = vstruct_last(sb->sb);
 126                 memset(f, 0, sizeof(u64) * u64s);
 127                 f->u64s = cpu_to_le32(u64s);
 128                 f->type = 0;
 129         } else {
 130                 void *src, *dst;
 131
 132                 src = vstruct_end(f);
 133
 134                 if (u64s) {
 135                         f->u64s = cpu_to_le32(u64s);
 136                         dst = vstruct_end(f);
 137                 } else {
 138                         dst = f;
 139                 }
 140
 141                 memmove(dst, src, vstruct_end(sb->sb) - src);
 142
 143                 if (dst > src)
 144                         memset(src, 0, dst - src);
 145         }
 146
 147         sb->sb->u64s = cpu_to_le32(sb_u64s);
 148
 149         return u64s ? f : NULL;
 150 }
 151
 152 void bch2_sb_field_delete(struct bch_sb_handle *sb,
 153                           enum bch_sb_field_type type)
 154 {
 155         struct bch_sb_field *f = bch2_sb_field_get_id(sb->sb, type);
 156
 157         if (f)
 158                 __bch2_sb_field_resize(sb, f, 0);
 159 }
 160
 161 /* Superblock realloc/free: */
 162
 163 void bch2_free_super(struct bch_sb_handle *sb)
 164 {
 165         kfree(sb->bio);
 166         if (!IS_ERR_OR_NULL(sb->bdev))
 167                 blkdev_put(sb->bdev, sb->holder);
 168         kfree(sb->holder);
 169         kfree(sb->sb_name);
 170
 171         kfree(sb->sb);
 172         memset(sb, 0, sizeof(*sb));
 173 }
 174
 175 int bch2_sb_realloc(struct bch_sb_handle *sb, unsigned u64s)
 176 {
 177         size_t new_bytes = __vstruct_bytes(struct bch_sb, u64s);
 178         size_t new_buffer_size;
 179         struct bch_sb *new_sb;
 180         struct bio *bio;
 181
 182         if (sb->bdev)
 183                 new_bytes = max_t(size_t, new_bytes, bdev_logical_block_size(sb->bdev));
 184
 185         new_buffer_size = roundup_pow_of_two(new_bytes);
 186
 187         if (sb->sb && sb->buffer_size >= new_buffer_size)
 188                 return 0;
 189
 190         if (sb->sb && sb->have_layout) {
 191                 u64 max_bytes = 512 << sb->sb->layout.sb_max_size_bits;
 192
 193                 if (new_bytes > max_bytes) {
 194                         pr_err("%pg: superblock too big: want %zu but have %llu",
 195                                sb->bdev, new_bytes, max_bytes);
 196                         return -BCH_ERR_ENOSPC_sb;
 197                 }
 198         }
 199
 200         if (sb->buffer_size >= new_buffer_size && sb->sb)
 201                 return 0;
 202
 203         if (dynamic_fault("bcachefs:add:super_realloc"))
 204                 return -BCH_ERR_ENOMEM_sb_realloc_injected;
 205
 206         new_sb = krealloc(sb->sb, new_buffer_size, GFP_NOFS|__GFP_ZERO);
 207         if (!new_sb)
 208                 return -BCH_ERR_ENOMEM_sb_buf_realloc;
 209
 210         sb->sb = new_sb;
 211
 212         if (sb->have_bio) {
 213                 unsigned nr_bvecs = buf_pages(sb->sb, new_buffer_size);
 214
 215                 bio = bio_kmalloc(nr_bvecs, GFP_KERNEL);
 216                 if (!bio)
 217                         return -BCH_ERR_ENOMEM_sb_bio_realloc;
 218
 219                 bio_init(bio, NULL, bio->bi_inline_vecs, nr_bvecs, 0);
 220
 221                 kfree(sb->bio);
 222                 sb->bio = bio;
 223         }
 224
 225         sb->buffer_size = new_buffer_size;
 226
 227         return 0;
 228 }
 229
 230 struct bch_sb_field *bch2_sb_field_resize_id(struct bch_sb_handle *sb,
 231                                           enum bch_sb_field_type type,
 232                                           unsigned u64s)
 233 {
 234         struct bch_sb_field *f = bch2_sb_field_get_id(sb->sb, type);
 235         ssize_t old_u64s = f ? le32_to_cpu(f->u64s) : 0;
 236         ssize_t d = -old_u64s + u64s;
 237
 238         if (bch2_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d))
 239                 return NULL;
 240
 241         if (sb->fs_sb) {
 242                 struct bch_fs *c = container_of(sb, struct bch_fs, disk_sb);
 243                 struct bch_dev *ca;
 244                 unsigned i;
 245
 246                 lockdep_assert_held(&c->sb_lock);
 247
 248                 /* XXX: we're not checking that offline device have enough space */
 249
 250                 for_each_online_member(ca, c, i) {
 251                         struct bch_sb_handle *dev_sb = &ca->disk_sb;
 252
 253                         if (bch2_sb_realloc(dev_sb, le32_to_cpu(dev_sb->sb->u64s) + d)) {
 254                                 percpu_ref_put(&ca->ref);
 255                                 return NULL;
 256                         }
 257                 }
 258         }
 259
 260         f = bch2_sb_field_get_id(sb->sb, type);
 261         f = __bch2_sb_field_resize(sb, f, u64s);
 262         if (f)
 263                 f->type = cpu_to_le32(type);
 264         return f;
 265 }
 266
 267 /* Superblock validate: */
 268
 269 static int validate_sb_layout(struct bch_sb_layout *layout, struct printbuf *out)
 270 {
 271         u64 offset, prev_offset, max_sectors;
 272         unsigned i;
 273
 274         BUILD_BUG_ON(sizeof(struct bch_sb_layout) != 512);
 275
 276         if (!uuid_equal(&layout->magic, &BCACHE_MAGIC) &&
 277             !uuid_equal(&layout->magic, &BCHFS_MAGIC)) {
 278                 prt_printf(out, "Not a bcachefs superblock layout");
 279                 return -BCH_ERR_invalid_sb_layout;
 280         }
 281
 282         if (layout->layout_type != 0) {
 283                 prt_printf(out, "Invalid superblock layout type %u",
 284                        layout->layout_type);
 285                 return -BCH_ERR_invalid_sb_layout_type;
 286         }
 287
 288         if (!layout->nr_superblocks) {
 289                 prt_printf(out, "Invalid superblock layout: no superblocks");
 290                 return -BCH_ERR_invalid_sb_layout_nr_superblocks;
 291         }
 292
 293         if (layout->nr_superblocks > ARRAY_SIZE(layout->sb_offset)) {
 294                 prt_printf(out, "Invalid superblock layout: too many superblocks");
 295                 return -BCH_ERR_invalid_sb_layout_nr_superblocks;
 296         }
 297
 298         max_sectors = 1 << layout->sb_max_size_bits;
 299
 300         prev_offset = le64_to_cpu(layout->sb_offset[0]);
 301
 302         for (i = 1; i < layout->nr_superblocks; i++) {
 303                 offset = le64_to_cpu(layout->sb_offset[i]);
 304
 305                 if (offset < prev_offset + max_sectors) {
 306                         prt_printf(out, "Invalid superblock layout: superblocks overlap\n"
 307                                "  (sb %u ends at %llu next starts at %llu",
 308                                i - 1, prev_offset + max_sectors, offset);
 309                         return -BCH_ERR_invalid_sb_layout_superblocks_overlap;
 310                 }
 311                 prev_offset = offset;
 312         }
 313
 314         return 0;
 315 }
 316
 317 static int bch2_sb_compatible(struct bch_sb *sb, struct printbuf *out)
 318 {
 319         u16 version             = le16_to_cpu(sb->version);
 320         u16 version_min         = le16_to_cpu(sb->version_min);
 321
 322         if (!bch2_version_compatible(version)) {
 323                 prt_str(out, "Unsupported superblock version ");
 324                 bch2_version_to_text(out, version);
 325                 prt_str(out, " (min ");
 326                 bch2_version_to_text(out, bcachefs_metadata_version_min);
 327                 prt_str(out, ", max ");
 328                 bch2_version_to_text(out, bcachefs_metadata_version_current);
 329                 prt_str(out, ")");
 330                 return -BCH_ERR_invalid_sb_version;
 331         }
 332
 333         if (!bch2_version_compatible(version_min)) {
 334                 prt_str(out, "Unsupported superblock version_min ");
 335                 bch2_version_to_text(out, version_min);
 336                 prt_str(out, " (min ");
 337                 bch2_version_to_text(out, bcachefs_metadata_version_min);
 338                 prt_str(out, ", max ");
 339                 bch2_version_to_text(out, bcachefs_metadata_version_current);
 340                 prt_str(out, ")");
 341                 return -BCH_ERR_invalid_sb_version;
 342         }
 343
 344         if (version_min > version) {
 345                 prt_str(out, "Bad minimum version ");
 346                 bch2_version_to_text(out, version_min);
 347                 prt_str(out, ", greater than version field ");
 348                 bch2_version_to_text(out, version);
 349                 return -BCH_ERR_invalid_sb_version;
 350         }
 351
 352         return 0;
 353 }
 354
 355 static int bch2_sb_validate(struct bch_sb_handle *disk_sb, struct printbuf *out,
 356                             int rw)
 357 {
 358         struct bch_sb *sb = disk_sb->sb;
 359         struct bch_sb_field *f;
 360         struct bch_sb_field_members_v1 *mi;
 361         enum bch_opt_id opt_id;
 362         u16 block_size;
 363         int ret;
 364
 365         ret = bch2_sb_compatible(sb, out);
 366         if (ret)
 367                 return ret;
 368
 369         if (sb->features[1] ||
 370             (le64_to_cpu(sb->features[0]) & (~0ULL << BCH_FEATURE_NR))) {
 371                 prt_printf(out, "Filesystem has incompatible features");
 372                 return -BCH_ERR_invalid_sb_features;
 373         }
 374
 375         block_size = le16_to_cpu(sb->block_size);
 376
 377         if (block_size > PAGE_SECTORS) {
 378                 prt_printf(out, "Block size too big (got %u, max %u)",
 379                        block_size, PAGE_SECTORS);
 380                 return -BCH_ERR_invalid_sb_block_size;
 381         }
 382
 383         if (bch2_is_zero(sb->user_uuid.b, sizeof(sb->user_uuid))) {
 384                 prt_printf(out, "Bad user UUID (got zeroes)");
 385                 return -BCH_ERR_invalid_sb_uuid;
 386         }
 387
 388         if (bch2_is_zero(sb->uuid.b, sizeof(sb->uuid))) {
 389                 prt_printf(out, "Bad internal UUID (got zeroes)");
 390                 return -BCH_ERR_invalid_sb_uuid;
 391         }
 392
 393         if (!sb->nr_devices ||
 394             sb->nr_devices > BCH_SB_MEMBERS_MAX) {
 395                 prt_printf(out, "Bad number of member devices %u (max %u)",
 396                        sb->nr_devices, BCH_SB_MEMBERS_MAX);
 397                 return -BCH_ERR_invalid_sb_too_many_members;
 398         }
 399
 400         if (sb->dev_idx >= sb->nr_devices) {
 401                 prt_printf(out, "Bad dev_idx (got %u, nr_devices %u)",
 402                        sb->dev_idx, sb->nr_devices);
 403                 return -BCH_ERR_invalid_sb_dev_idx;
 404         }
 405
 406         if (!sb->time_precision ||
 407             le32_to_cpu(sb->time_precision) > NSEC_PER_SEC) {
 408                 prt_printf(out, "Invalid time precision: %u (min 1, max %lu)",
 409                        le32_to_cpu(sb->time_precision), NSEC_PER_SEC);
 410                 return -BCH_ERR_invalid_sb_time_precision;
 411         }
 412
 413         if (rw == READ) {
 414                 /*
 415                  * Been seeing a bug where these are getting inexplicably
 416                  * zeroed, so we're now validating them, but we have to be
 417                  * careful not to preven people's filesystems from mounting:
 418                  */
 419                 if (!BCH_SB_JOURNAL_FLUSH_DELAY(sb))
 420                         SET_BCH_SB_JOURNAL_FLUSH_DELAY(sb, 1000);
 421                 if (!BCH_SB_JOURNAL_RECLAIM_DELAY(sb))
 422                         SET_BCH_SB_JOURNAL_RECLAIM_DELAY(sb, 1000);
 423
 424                 if (!BCH_SB_VERSION_UPGRADE_COMPLETE(sb))
 425                         SET_BCH_SB_VERSION_UPGRADE_COMPLETE(sb, le16_to_cpu(sb->version));
 426         }
 427
 428         for (opt_id = 0; opt_id < bch2_opts_nr; opt_id++) {
 429                 const struct bch_option *opt = bch2_opt_table + opt_id;
 430
 431                 if (opt->get_sb != BCH2_NO_SB_OPT) {
 432                         u64 v = bch2_opt_from_sb(sb, opt_id);
 433
 434                         prt_printf(out, "Invalid option ");
 435                         ret = bch2_opt_validate(opt, v, out);
 436                         if (ret)
 437                                 return ret;
 438
 439                         printbuf_reset(out);
 440                 }
 441         }
 442
 443         /* validate layout */
 444         ret = validate_sb_layout(&sb->layout, out);
 445         if (ret)
 446                 return ret;
 447
 448         vstruct_for_each(sb, f) {
 449                 if (!f->u64s) {
 450                         prt_printf(out, "Invalid superblock: optional field with size 0 (type %u)",
 451                                le32_to_cpu(f->type));
 452                         return -BCH_ERR_invalid_sb_field_size;
 453                 }
 454
 455                 if (vstruct_next(f) > vstruct_last(sb)) {
 456                         prt_printf(out, "Invalid superblock: optional field extends past end of superblock (type %u)",
 457                                le32_to_cpu(f->type));
 458                         return -BCH_ERR_invalid_sb_field_size;
 459                 }
 460         }
 461
 462         /* members must be validated first: */
 463         mi = bch2_sb_field_get(sb, members_v1);
 464         if (!mi) {
 465                 prt_printf(out, "Invalid superblock: member info area missing");
 466                 return -BCH_ERR_invalid_sb_members_missing;
 467         }
 468
 469         ret = bch2_sb_field_validate(sb, &mi->field, out);
 470         if (ret)
 471                 return ret;
 472
 473         vstruct_for_each(sb, f) {
 474                 if (le32_to_cpu(f->type) == BCH_SB_FIELD_members_v1)
 475                         continue;
 476
 477                 ret = bch2_sb_field_validate(sb, f, out);
 478                 if (ret)
 479                         return ret;
 480         }
 481
 482         return 0;
 483 }
 484
 485 /* device open: */
 486
 487 static void bch2_sb_update(struct bch_fs *c)
 488 {
 489         struct bch_sb *src = c->disk_sb.sb;
 490         struct bch_dev *ca;
 491         unsigned i;
 492
 493         lockdep_assert_held(&c->sb_lock);
 494
 495         c->sb.uuid              = src->uuid;
 496         c->sb.user_uuid         = src->user_uuid;
 497         c->sb.version           = le16_to_cpu(src->version);
 498         c->sb.version_min       = le16_to_cpu(src->version_min);
 499         c->sb.version_upgrade_complete = BCH_SB_VERSION_UPGRADE_COMPLETE(src);
 500         c->sb.nr_devices        = src->nr_devices;
 501         c->sb.clean             = BCH_SB_CLEAN(src);
 502         c->sb.encryption_type   = BCH_SB_ENCRYPTION_TYPE(src);
 503
 504         c->sb.nsec_per_time_unit = le32_to_cpu(src->time_precision);
 505         c->sb.time_units_per_sec = NSEC_PER_SEC / c->sb.nsec_per_time_unit;
 506
 507         /* XXX this is wrong, we need a 96 or 128 bit integer type */
 508         c->sb.time_base_lo      = div_u64(le64_to_cpu(src->time_base_lo),
 509                                           c->sb.nsec_per_time_unit);
 510         c->sb.time_base_hi      = le32_to_cpu(src->time_base_hi);
 511
 512         c->sb.features          = le64_to_cpu(src->features[0]);
 513         c->sb.compat            = le64_to_cpu(src->compat[0]);
 514
 515         for_each_member_device(ca, c, i) {
 516                 struct bch_member m = bch2_sb_member_get(src, i);
 517                 ca->mi = bch2_mi_to_cpu(&m);
 518         }
 519 }
 520
 521 static int __copy_super(struct bch_sb_handle *dst_handle, struct bch_sb *src)
 522 {
 523         struct bch_sb_field *src_f, *dst_f;
 524         struct bch_sb *dst = dst_handle->sb;
 525         unsigned i;
 526
 527         dst->version            = src->version;
 528         dst->version_min        = src->version_min;
 529         dst->seq                = src->seq;
 530         dst->uuid               = src->uuid;
 531         dst->user_uuid          = src->user_uuid;
 532         memcpy(dst->label,      src->label, sizeof(dst->label));
 533
 534         dst->block_size         = src->block_size;
 535         dst->nr_devices         = src->nr_devices;
 536
 537         dst->time_base_lo       = src->time_base_lo;
 538         dst->time_base_hi       = src->time_base_hi;
 539         dst->time_precision     = src->time_precision;
 540
 541         memcpy(dst->flags,      src->flags,     sizeof(dst->flags));
 542         memcpy(dst->features,   src->features,  sizeof(dst->features));
 543         memcpy(dst->compat,     src->compat,    sizeof(dst->compat));
 544
 545         for (i = 0; i < BCH_SB_FIELD_NR; i++) {
 546                 int d;
 547
 548                 if ((1U << i) & BCH_SINGLE_DEVICE_SB_FIELDS)
 549                         continue;
 550
 551                 src_f = bch2_sb_field_get_id(src, i);
 552                 dst_f = bch2_sb_field_get_id(dst, i);
 553
 554                 d = (src_f ? le32_to_cpu(src_f->u64s) : 0) -
 555                     (dst_f ? le32_to_cpu(dst_f->u64s) : 0);
 556                 if (d > 0) {
 557                         int ret = bch2_sb_realloc(dst_handle,
 558                                         le32_to_cpu(dst_handle->sb->u64s) + d);
 559
 560                         if (ret)
 561                                 return ret;
 562
 563                         dst = dst_handle->sb;
 564                         dst_f = bch2_sb_field_get_id(dst, i);
 565                 }
 566
 567                 dst_f = __bch2_sb_field_resize(dst_handle, dst_f,
 568                                 src_f ? le32_to_cpu(src_f->u64s) : 0);
 569
 570                 if (src_f)
 571                         memcpy(dst_f, src_f, vstruct_bytes(src_f));
 572         }
 573
 574         return 0;
 575 }
 576
 577 int bch2_sb_to_fs(struct bch_fs *c, struct bch_sb *src)
 578 {
 579         int ret;
 580
 581         lockdep_assert_held(&c->sb_lock);
 582
 583         ret =   bch2_sb_realloc(&c->disk_sb, 0) ?:
 584                 __copy_super(&c->disk_sb, src) ?:
 585                 bch2_sb_replicas_to_cpu_replicas(c) ?:
 586                 bch2_sb_disk_groups_to_cpu(c);
 587         if (ret)
 588                 return ret;
 589
 590         bch2_sb_update(c);
 591         return 0;
 592 }
 593
 594 int bch2_sb_from_fs(struct bch_fs *c, struct bch_dev *ca)
 595 {
 596         return __copy_super(&ca->disk_sb, c->disk_sb.sb);
 597 }
 598
 599 /* read superblock: */
 600
 601 static int read_one_super(struct bch_sb_handle *sb, u64 offset, struct printbuf *err)
 602 {
 603         struct bch_csum csum;
 604         size_t bytes;
 605         int ret;
 606 reread:
 607         bio_reset(sb->bio, sb->bdev, REQ_OP_READ|REQ_SYNC|REQ_META);
 608         sb->bio->bi_iter.bi_sector = offset;
 609         bch2_bio_map(sb->bio, sb->sb, sb->buffer_size);
 610
 611         ret = submit_bio_wait(sb->bio);
 612         if (ret) {
 613                 prt_printf(err, "IO error: %i", ret);
 614                 return ret;
 615         }
 616
 617         if (!uuid_equal(&sb->sb->magic, &BCACHE_MAGIC) &&
 618             !uuid_equal(&sb->sb->magic, &BCHFS_MAGIC)) {
 619                 prt_printf(err, "Not a bcachefs superblock");
 620                 return -BCH_ERR_invalid_sb_magic;
 621         }
 622
 623         ret = bch2_sb_compatible(sb->sb, err);
 624         if (ret)
 625                 return ret;
 626
 627         bytes = vstruct_bytes(sb->sb);
 628
 629         if (bytes > 512 << sb->sb->layout.sb_max_size_bits) {
 630                 prt_printf(err, "Invalid superblock: too big (got %zu bytes, layout max %lu)",
 631                        bytes, 512UL << sb->sb->layout.sb_max_size_bits);
 632                 return -BCH_ERR_invalid_sb_too_big;
 633         }
 634
 635         if (bytes > sb->buffer_size) {
 636                 ret = bch2_sb_realloc(sb, le32_to_cpu(sb->sb->u64s));
 637                 if (ret)
 638                         return ret;
 639                 goto reread;
 640         }
 641
 642         if (BCH_SB_CSUM_TYPE(sb->sb) >= BCH_CSUM_NR) {
 643                 prt_printf(err, "unknown checksum type %llu", BCH_SB_CSUM_TYPE(sb->sb));
 644                 return -BCH_ERR_invalid_sb_csum_type;
 645         }
 646
 647         /* XXX: verify MACs */
 648         csum = csum_vstruct(NULL, BCH_SB_CSUM_TYPE(sb->sb),
 649                             null_nonce(), sb->sb);
 650
 651         if (bch2_crc_cmp(csum, sb->sb->csum)) {
 652                 prt_printf(err, "bad checksum");
 653                 return -BCH_ERR_invalid_sb_csum;
 654         }
 655
 656         sb->seq = le64_to_cpu(sb->sb->seq);
 657
 658         return 0;
 659 }
 660
 661 static int __bch2_read_super(const char *path, struct bch_opts *opts,
 662                     struct bch_sb_handle *sb, bool ignore_notbchfs_msg)
 663 {
 664         u64 offset = opt_get(*opts, sb);
 665         struct bch_sb_layout layout;
 666         struct printbuf err = PRINTBUF;
 667         struct printbuf err2 = PRINTBUF;
 668         __le64 *i;
 669         int ret;
 670 #ifndef __KERNEL__
 671 retry:
 672 #endif
 673         memset(sb, 0, sizeof(*sb));
 674         sb->mode        = BLK_OPEN_READ;
 675         sb->have_bio    = true;
 676         sb->holder      = kmalloc(1, GFP_KERNEL);
 677         if (!sb->holder)
 678                 return -ENOMEM;
 679
 680         sb->sb_name = kstrdup(path, GFP_KERNEL);
 681         if (!sb->sb_name)
 682                 return -ENOMEM;
 683
 684 #ifndef __KERNEL__
 685         if (opt_get(*opts, direct_io) == false)
 686                 sb->mode |= BLK_OPEN_BUFFERED;
 687 #endif
 688
 689         if (!opt_get(*opts, noexcl))
 690                 sb->mode |= BLK_OPEN_EXCL;
 691
 692         if (!opt_get(*opts, nochanges))
 693                 sb->mode |= BLK_OPEN_WRITE;
 694
 695         sb->bdev = blkdev_get_by_path(path, sb->mode, sb->holder, &bch2_sb_handle_bdev_ops);
 696         if (IS_ERR(sb->bdev) &&
 697             PTR_ERR(sb->bdev) == -EACCES &&
 698             opt_get(*opts, read_only)) {
 699                 sb->mode &= ~BLK_OPEN_WRITE;
 700
 701                 sb->bdev = blkdev_get_by_path(path, sb->mode, sb->holder, &bch2_sb_handle_bdev_ops);
 702                 if (!IS_ERR(sb->bdev))
 703                         opt_set(*opts, nochanges, true);
 704         }
 705
 706         if (IS_ERR(sb->bdev)) {
 707                 ret = PTR_ERR(sb->bdev);
 708                 goto out;
 709         }
 710
 711         ret = bch2_sb_realloc(sb, 0);
 712         if (ret) {
 713                 prt_printf(&err, "error allocating memory for superblock");
 714                 goto err;
 715         }
 716
 717         if (bch2_fs_init_fault("read_super")) {
 718                 prt_printf(&err, "dynamic fault");
 719                 ret = -EFAULT;
 720                 goto err;
 721         }
 722
 723         ret = read_one_super(sb, offset, &err);
 724         if (!ret)
 725                 goto got_super;
 726
 727         if (opt_defined(*opts, sb))
 728                 goto err;
 729
 730         prt_printf(&err2, "bcachefs (%s): error reading default superblock: %s\n",
 731                path, err.buf);
 732         if (ret == -BCH_ERR_invalid_sb_magic && ignore_notbchfs_msg)
 733                 printk(KERN_INFO "%s", err2.buf);
 734         else
 735                 printk(KERN_ERR "%s", err2.buf);
 736
 737         printbuf_exit(&err2);
 738         printbuf_reset(&err);
 739
 740         /*
 741          * Error reading primary superblock - read location of backup
 742          * superblocks:
 743          */
 744         bio_reset(sb->bio, sb->bdev, REQ_OP_READ|REQ_SYNC|REQ_META);
 745         sb->bio->bi_iter.bi_sector = BCH_SB_LAYOUT_SECTOR;
 746         /*
 747          * use sb buffer to read layout, since sb buffer is page aligned but
 748          * layout won't be:
 749          */
 750         bch2_bio_map(sb->bio, sb->sb, sizeof(struct bch_sb_layout));
 751
 752         ret = submit_bio_wait(sb->bio);
 753         if (ret) {
 754                 prt_printf(&err, "IO error: %i", ret);
 755                 goto err;
 756         }
 757
 758         memcpy(&layout, sb->sb, sizeof(layout));
 759         ret = validate_sb_layout(&layout, &err);
 760         if (ret)
 761                 goto err;
 762
 763         for (i = layout.sb_offset;
 764              i < layout.sb_offset + layout.nr_superblocks; i++) {
 765                 offset = le64_to_cpu(*i);
 766
 767                 if (offset == opt_get(*opts, sb))
 768                         continue;
 769
 770                 ret = read_one_super(sb, offset, &err);
 771                 if (!ret)
 772                         goto got_super;
 773         }
 774
 775         goto err;
 776
 777 got_super:
 778         if (le16_to_cpu(sb->sb->block_size) << 9 <
 779             bdev_logical_block_size(sb->bdev) &&
 780             opt_get(*opts, direct_io)) {
 781 #ifndef __KERNEL__
 782                 opt_set(*opts, direct_io, false);
 783                 bch2_free_super(sb);
 784                 goto retry;
 785 #endif
 786                 prt_printf(&err, "block size (%u) smaller than device block size (%u)",
 787                        le16_to_cpu(sb->sb->block_size) << 9,
 788                        bdev_logical_block_size(sb->bdev));
 789                 ret = -BCH_ERR_block_size_too_small;
 790                 goto err;
 791         }
 792
 793         ret = 0;
 794         sb->have_layout = true;
 795
 796         ret = bch2_sb_validate(sb, &err, READ);
 797         if (ret) {
 798                 printk(KERN_ERR "bcachefs (%s): error validating superblock: %s\n",
 799                        path, err.buf);
 800                 goto err_no_print;
 801         }
 802 out:
 803         printbuf_exit(&err);
 804         return ret;
 805 err:
 806         printk(KERN_ERR "bcachefs (%s): error reading superblock: %s\n",
 807                path, err.buf);
 808 err_no_print:
 809         bch2_free_super(sb);
 810         goto out;
 811 }
 812
 813 int bch2_read_super(const char *path, struct bch_opts *opts,
 814                     struct bch_sb_handle *sb)
 815 {
 816         return __bch2_read_super(path, opts, sb, false);
 817 }
 818
 819 /* provide a silenced version for mount.bcachefs */
 820
 821 int bch2_read_super_silent(const char *path, struct bch_opts *opts,
 822                     struct bch_sb_handle *sb)
 823 {
 824         return __bch2_read_super(path, opts, sb, true);
 825 }
 826
 827 /* write superblock: */
 828
 829 static void write_super_endio(struct bio *bio)
 830 {
 831         struct bch_dev *ca = bio->bi_private;
 832
 833         /* XXX: return errors directly */
 834
 835         if (bch2_dev_io_err_on(bio->bi_status, ca,
 836                                bio_data_dir(bio)
 837                                ? BCH_MEMBER_ERROR_write
 838                                : BCH_MEMBER_ERROR_read,
 839                                "superblock %s error: %s",
 840                                bio_data_dir(bio) ? "write" : "read",
 841                                bch2_blk_status_to_str(bio->bi_status)))
 842                 ca->sb_write_error = 1;
 843
 844         closure_put(&ca->fs->sb_write);
 845         percpu_ref_put(&ca->io_ref);
 846 }
 847
 848 static void read_back_super(struct bch_fs *c, struct bch_dev *ca)
 849 {
 850         struct bch_sb *sb = ca->disk_sb.sb;
 851         struct bio *bio = ca->disk_sb.bio;
 852
 853         bio_reset(bio, ca->disk_sb.bdev, REQ_OP_READ|REQ_SYNC|REQ_META);
 854         bio->bi_iter.bi_sector  = le64_to_cpu(sb->layout.sb_offset[0]);
 855         bio->bi_end_io          = write_super_endio;
 856         bio->bi_private         = ca;
 857         bch2_bio_map(bio, ca->sb_read_scratch, PAGE_SIZE);
 858
 859         this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_sb],
 860                      bio_sectors(bio));
 861
 862         percpu_ref_get(&ca->io_ref);
 863         closure_bio_submit(bio, &c->sb_write);
 864 }
 865
 866 static void write_one_super(struct bch_fs *c, struct bch_dev *ca, unsigned idx)
 867 {
 868         struct bch_sb *sb = ca->disk_sb.sb;
 869         struct bio *bio = ca->disk_sb.bio;
 870
 871         sb->offset = sb->layout.sb_offset[idx];
 872
 873         SET_BCH_SB_CSUM_TYPE(sb, bch2_csum_opt_to_type(c->opts.metadata_checksum, false));
 874         sb->csum = csum_vstruct(c, BCH_SB_CSUM_TYPE(sb),
 875                                 null_nonce(), sb);
 876
 877         bio_reset(bio, ca->disk_sb.bdev, REQ_OP_WRITE|REQ_SYNC|REQ_META);
 878         bio->bi_iter.bi_sector  = le64_to_cpu(sb->offset);
 879         bio->bi_end_io          = write_super_endio;
 880         bio->bi_private         = ca;
 881         bch2_bio_map(bio, sb,
 882                      roundup((size_t) vstruct_bytes(sb),
 883                              bdev_logical_block_size(ca->disk_sb.bdev)));
 884
 885         this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_sb],
 886                      bio_sectors(bio));
 887
 888         percpu_ref_get(&ca->io_ref);
 889         closure_bio_submit(bio, &c->sb_write);
 890 }
 891
 892 int bch2_write_super(struct bch_fs *c)
 893 {
 894         struct closure *cl = &c->sb_write;
 895         struct bch_dev *ca;
 896         struct printbuf err = PRINTBUF;
 897         unsigned i, sb = 0, nr_wrote;
 898         struct bch_devs_mask sb_written;
 899         bool wrote, can_mount_without_written, can_mount_with_written;
 900         unsigned degraded_flags = BCH_FORCE_IF_DEGRADED;
 901         int ret = 0;
 902
 903         trace_and_count(c, write_super, c, _RET_IP_);
 904
 905         if (c->opts.very_degraded)
 906                 degraded_flags |= BCH_FORCE_IF_LOST;
 907
 908         lockdep_assert_held(&c->sb_lock);
 909
 910         closure_init_stack(cl);
 911         memset(&sb_written, 0, sizeof(sb_written));
 912
 913         /* Make sure we're using the new magic numbers: */
 914         c->disk_sb.sb->magic = BCHFS_MAGIC;
 915         c->disk_sb.sb->layout.magic = BCHFS_MAGIC;
 916
 917         le64_add_cpu(&c->disk_sb.sb->seq, 1);
 918
 919         if (test_bit(BCH_FS_ERROR, &c->flags))
 920                 SET_BCH_SB_HAS_ERRORS(c->disk_sb.sb, 1);
 921         if (test_bit(BCH_FS_TOPOLOGY_ERROR, &c->flags))
 922                 SET_BCH_SB_HAS_TOPOLOGY_ERRORS(c->disk_sb.sb, 1);
 923
 924         SET_BCH_SB_BIG_ENDIAN(c->disk_sb.sb, CPU_BIG_ENDIAN);
 925
 926         bch2_sb_counters_from_cpu(c);
 927         bch2_sb_members_from_cpu(c);
 928         bch2_sb_members_cpy_v2_v1(&c->disk_sb);
 929         bch2_sb_errors_from_cpu(c);
 930
 931         for_each_online_member(ca, c, i)
 932                 bch2_sb_from_fs(c, ca);
 933
 934         for_each_online_member(ca, c, i) {
 935                 printbuf_reset(&err);
 936
 937                 ret = bch2_sb_validate(&ca->disk_sb, &err, WRITE);
 938                 if (ret) {
 939                         bch2_fs_inconsistent(c, "sb invalid before write: %s", err.buf);
 940                         percpu_ref_put(&ca->io_ref);
 941                         goto out;
 942                 }
 943         }
 944
 945         if (c->opts.nochanges)
 946                 goto out;
 947
 948         /*
 949          * Defer writing the superblock until filesystem initialization is
 950          * complete - don't write out a partly initialized superblock:
 951          */
 952         if (!BCH_SB_INITIALIZED(c->disk_sb.sb))
 953                 goto out;
 954
 955         for_each_online_member(ca, c, i) {
 956                 __set_bit(ca->dev_idx, sb_written.d);
 957                 ca->sb_write_error = 0;
 958         }
 959
 960         for_each_online_member(ca, c, i)
 961                 read_back_super(c, ca);
 962         closure_sync(cl);
 963
 964         for_each_online_member(ca, c, i) {
 965                 if (ca->sb_write_error)
 966                         continue;
 967
 968                 if (le64_to_cpu(ca->sb_read_scratch->seq) < ca->disk_sb.seq) {
 969                         bch2_fs_fatal_error(c,
 970                                 "Superblock write was silently dropped! (seq %llu expected %llu)",
 971                                 le64_to_cpu(ca->sb_read_scratch->seq),
 972                                 ca->disk_sb.seq);
 973                         percpu_ref_put(&ca->io_ref);
 974                         ret = -BCH_ERR_erofs_sb_err;
 975                         goto out;
 976                 }
 977
 978                 if (le64_to_cpu(ca->sb_read_scratch->seq) > ca->disk_sb.seq) {
 979                         bch2_fs_fatal_error(c,
 980                                 "Superblock modified by another process (seq %llu expected %llu)",
 981                                 le64_to_cpu(ca->sb_read_scratch->seq),
 982                                 ca->disk_sb.seq);
 983                         percpu_ref_put(&ca->io_ref);
 984                         ret = -BCH_ERR_erofs_sb_err;
 985                         goto out;
 986                 }
 987         }
 988
 989         do {
 990                 wrote = false;
 991                 for_each_online_member(ca, c, i)
 992                         if (!ca->sb_write_error &&
 993                             sb < ca->disk_sb.sb->layout.nr_superblocks) {
 994                                 write_one_super(c, ca, sb);
 995                                 wrote = true;
 996                         }
 997                 closure_sync(cl);
 998                 sb++;
 999         } while (wrote);
1000
1001         for_each_online_member(ca, c, i) {
1002                 if (ca->sb_write_error)
1003                         __clear_bit(ca->dev_idx, sb_written.d);
1004                 else
1005                         ca->disk_sb.seq = le64_to_cpu(ca->disk_sb.sb->seq);
1006         }
1007
1008         nr_wrote = dev_mask_nr(&sb_written);
1009
1010         can_mount_with_written =
1011                 bch2_have_enough_devs(c, sb_written, degraded_flags, false);
1012
1013         for (i = 0; i < ARRAY_SIZE(sb_written.d); i++)
1014                 sb_written.d[i] = ~sb_written.d[i];
1015
1016         can_mount_without_written =
1017                 bch2_have_enough_devs(c, sb_written, degraded_flags, false);
1018
1019         /*
1020          * If we would be able to mount _without_ the devices we successfully
1021          * wrote superblocks to, we weren't able to write to enough devices:
1022          *
1023          * Exception: if we can mount without the successes because we haven't
1024          * written anything (new filesystem), we continue if we'd be able to
1025          * mount with the devices we did successfully write to:
1026          */
1027         if (bch2_fs_fatal_err_on(!nr_wrote ||
1028                                  !can_mount_with_written ||
1029                                  (can_mount_without_written &&
1030                                   !can_mount_with_written), c,
1031                 "Unable to write superblock to sufficient devices (from %ps)",
1032                 (void *) _RET_IP_))
1033                 ret = -1;
1034 out:
1035         /* Make new options visible after they're persistent: */
1036         bch2_sb_update(c);
1037         printbuf_exit(&err);
1038         return ret;
1039 }
1040
1041 void __bch2_check_set_feature(struct bch_fs *c, unsigned feat)
1042 {
1043         mutex_lock(&c->sb_lock);
1044         if (!(c->sb.features & (1ULL << feat))) {
1045                 c->disk_sb.sb->features[0] |= cpu_to_le64(1ULL << feat);
1046
1047                 bch2_write_super(c);
1048         }
1049         mutex_unlock(&c->sb_lock);
1050 }
1051
1052 /* Downgrade if superblock is at a higher version than currently supported: */
1053 void bch2_sb_maybe_downgrade(struct bch_fs *c)
1054 {
1055         lockdep_assert_held(&c->sb_lock);
1056
1057         /*
1058          * Downgrade, if superblock is at a higher version than currently
1059          * supported:
1060          */
1061         if (BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb) > bcachefs_metadata_version_current)
1062                 SET_BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb, bcachefs_metadata_version_current);
1063         if (c->sb.version > bcachefs_metadata_version_current)
1064                 c->disk_sb.sb->version = cpu_to_le16(bcachefs_metadata_version_current);
1065         if (c->sb.version_min > bcachefs_metadata_version_current)
1066                 c->disk_sb.sb->version_min = cpu_to_le16(bcachefs_metadata_version_current);
1067         c->disk_sb.sb->compat[0] &= cpu_to_le64((1ULL << BCH_COMPAT_NR) - 1);
1068 }
1069
1070 void bch2_sb_upgrade(struct bch_fs *c, unsigned new_version)
1071 {
1072         lockdep_assert_held(&c->sb_lock);
1073
1074         c->disk_sb.sb->version = cpu_to_le16(new_version);
1075         c->disk_sb.sb->features[0] |= cpu_to_le64(BCH_SB_FEATURES_ALL);
1076 }
1077
1078 static const struct bch_sb_field_ops *bch2_sb_field_ops[] = {
1079 #define x(f, nr)                                        \
1080         [BCH_SB_FIELD_##f] = &bch_sb_field_ops_##f,
1081         BCH_SB_FIELDS()
1082 #undef x
1083 };
1084
1085 static const struct bch_sb_field_ops bch2_sb_field_null_ops;
1086
1087 static const struct bch_sb_field_ops *bch2_sb_field_type_ops(unsigned type)
1088 {
1089         return likely(type < ARRAY_SIZE(bch2_sb_field_ops))
1090                 ? bch2_sb_field_ops[type]
1091                 : &bch2_sb_field_null_ops;
1092 }
1093
1094 static int bch2_sb_field_validate(struct bch_sb *sb, struct bch_sb_field *f,
1095                                   struct printbuf *err)
1096 {
1097         unsigned type = le32_to_cpu(f->type);
1098         struct printbuf field_err = PRINTBUF;
1099         const struct bch_sb_field_ops *ops = bch2_sb_field_type_ops(type);
1100         int ret;
1101
1102         ret = ops->validate ? ops->validate(sb, f, &field_err) : 0;
1103         if (ret) {
1104                 prt_printf(err, "Invalid superblock section %s: %s",
1105                            bch2_sb_fields[type], field_err.buf);
1106                 prt_newline(err);
1107                 bch2_sb_field_to_text(err, sb, f);
1108         }
1109
1110         printbuf_exit(&field_err);
1111         return ret;
1112 }
1113
1114 void bch2_sb_field_to_text(struct printbuf *out, struct bch_sb *sb,
1115                            struct bch_sb_field *f)
1116 {
1117         unsigned type = le32_to_cpu(f->type);
1118         const struct bch_sb_field_ops *ops = bch2_sb_field_type_ops(type);
1119
1120         if (!out->nr_tabstops)
1121                 printbuf_tabstop_push(out, 32);
1122
1123         if (type < BCH_SB_FIELD_NR)
1124                 prt_printf(out, "%s", bch2_sb_fields[type]);
1125         else
1126                 prt_printf(out, "(unknown field %u)", type);
1127
1128         prt_printf(out, " (size %zu):", vstruct_bytes(f));
1129         prt_newline(out);
1130
1131         if (ops->to_text) {
1132                 printbuf_indent_add(out, 2);
1133                 ops->to_text(out, sb, f);
1134                 printbuf_indent_sub(out, 2);
1135         }
1136 }
1137
1138 void bch2_sb_layout_to_text(struct printbuf *out, struct bch_sb_layout *l)
1139 {
1140         unsigned i;
1141
1142         prt_printf(out, "Type:                    %u", l->layout_type);
1143         prt_newline(out);
1144
1145         prt_str(out, "Superblock max size:     ");
1146         prt_units_u64(out, 512 << l->sb_max_size_bits);
1147         prt_newline(out);
1148
1149         prt_printf(out, "Nr superblocks:          %u", l->nr_superblocks);
1150         prt_newline(out);
1151
1152         prt_str(out, "Offsets:                 ");
1153         for (i = 0; i < l->nr_superblocks; i++) {
1154                 if (i)
1155                         prt_str(out, ", ");
1156                 prt_printf(out, "%llu", le64_to_cpu(l->sb_offset[i]));
1157         }
1158         prt_newline(out);
1159 }
1160
1161 void bch2_sb_to_text(struct printbuf *out, struct bch_sb *sb,
1162                      bool print_layout, unsigned fields)
1163 {
1164         struct bch_sb_field *f;
1165         u64 fields_have = 0;
1166         unsigned nr_devices = 0;
1167
1168         if (!out->nr_tabstops)
1169                 printbuf_tabstop_push(out, 44);
1170
1171         for (int i = 0; i < sb->nr_devices; i++)
1172                 nr_devices += bch2_dev_exists(sb, i);
1173
1174         prt_printf(out, "External UUID:");
1175         prt_tab(out);
1176         pr_uuid(out, sb->user_uuid.b);
1177         prt_newline(out);
1178
1179         prt_printf(out, "Internal UUID:");
1180         prt_tab(out);
1181         pr_uuid(out, sb->uuid.b);
1182         prt_newline(out);
1183
1184         prt_str(out, "Device index:");
1185         prt_tab(out);
1186         prt_printf(out, "%u", sb->dev_idx);
1187         prt_newline(out);
1188
1189         prt_str(out, "Label:");
1190         prt_tab(out);
1191         prt_printf(out, "%.*s", (int) sizeof(sb->label), sb->label);
1192         prt_newline(out);
1193
1194         prt_str(out, "Version:");
1195         prt_tab(out);
1196         bch2_version_to_text(out, le16_to_cpu(sb->version));
1197         prt_newline(out);
1198
1199         prt_str(out, "Version upgrade complete:");
1200         prt_tab(out);
1201         bch2_version_to_text(out, BCH_SB_VERSION_UPGRADE_COMPLETE(sb));
1202         prt_newline(out);
1203
1204         prt_printf(out, "Oldest version on disk:");
1205         prt_tab(out);
1206         bch2_version_to_text(out, le16_to_cpu(sb->version_min));
1207         prt_newline(out);
1208
1209         prt_printf(out, "Created:");
1210         prt_tab(out);
1211         if (sb->time_base_lo)
1212                 bch2_prt_datetime(out, div_u64(le64_to_cpu(sb->time_base_lo), NSEC_PER_SEC));
1213         else
1214                 prt_printf(out, "(not set)");
1215         prt_newline(out);
1216
1217         prt_printf(out, "Sequence number:");
1218         prt_tab(out);
1219         prt_printf(out, "%llu", le64_to_cpu(sb->seq));
1220         prt_newline(out);
1221
1222         prt_printf(out, "Superblock size:");
1223         prt_tab(out);
1224         prt_printf(out, "%zu", vstruct_bytes(sb));
1225         prt_newline(out);
1226
1227         prt_printf(out, "Clean:");
1228         prt_tab(out);
1229         prt_printf(out, "%llu", BCH_SB_CLEAN(sb));
1230         prt_newline(out);
1231
1232         prt_printf(out, "Devices:");
1233         prt_tab(out);
1234         prt_printf(out, "%u", nr_devices);
1235         prt_newline(out);
1236
1237         prt_printf(out, "Sections:");
1238         vstruct_for_each(sb, f)
1239                 fields_have |= 1 << le32_to_cpu(f->type);
1240         prt_tab(out);
1241         prt_bitflags(out, bch2_sb_fields, fields_have);
1242         prt_newline(out);
1243
1244         prt_printf(out, "Features:");
1245         prt_tab(out);
1246         prt_bitflags(out, bch2_sb_features, le64_to_cpu(sb->features[0]));
1247         prt_newline(out);
1248
1249         prt_printf(out, "Compat features:");
1250         prt_tab(out);
1251         prt_bitflags(out, bch2_sb_compat, le64_to_cpu(sb->compat[0]));
1252         prt_newline(out);
1253
1254         prt_newline(out);
1255         prt_printf(out, "Options:");
1256         prt_newline(out);
1257         printbuf_indent_add(out, 2);
1258         {
1259                 enum bch_opt_id id;
1260
1261                 for (id = 0; id < bch2_opts_nr; id++) {
1262                         const struct bch_option *opt = bch2_opt_table + id;
1263
1264                         if (opt->get_sb != BCH2_NO_SB_OPT) {
1265                                 u64 v = bch2_opt_from_sb(sb, id);
1266
1267                                 prt_printf(out, "%s:", opt->attr.name);
1268                                 prt_tab(out);
1269                                 bch2_opt_to_text(out, NULL, sb, opt, v,
1270                                                  OPT_HUMAN_READABLE|OPT_SHOW_FULL_LIST);
1271                                 prt_newline(out);
1272                         }
1273                 }
1274         }
1275
1276         printbuf_indent_sub(out, 2);
1277
1278         if (print_layout) {
1279                 prt_newline(out);
1280                 prt_printf(out, "layout:");
1281                 prt_newline(out);
1282                 printbuf_indent_add(out, 2);
1283                 bch2_sb_layout_to_text(out, &sb->layout);
1284                 printbuf_indent_sub(out, 2);
1285         }
1286
1287         vstruct_for_each(sb, f)
1288                 if (fields & (1 << le32_to_cpu(f->type))) {
1289                         prt_newline(out);
1290                         bch2_sb_field_to_text(out, sb, f);
1291                 }
1292 }