12 #include <linux/backing-dev.h>
14 static inline void __bch_sb_layout_size_assert(void)
16 BUILD_BUG_ON(sizeof(struct bch_sb_layout) != 512);
19 struct bch_sb_field *bch_sb_field_get(struct bch_sb *sb,
20 enum bch_sb_field_types type)
22 struct bch_sb_field *f;
24 /* XXX: need locking around superblock to access optional fields */
26 vstruct_for_each(sb, f)
27 if (le32_to_cpu(f->type) == type)
32 void bch_free_super(struct bcache_superblock *sb)
36 if (!IS_ERR_OR_NULL(sb->bdev))
37 blkdev_put(sb->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
39 free_pages((unsigned long) sb->sb, sb->page_order);
40 memset(sb, 0, sizeof(*sb));
43 static int __bch_super_realloc(struct bcache_superblock *sb, unsigned order)
45 struct bch_sb *new_sb;
48 if (sb->page_order >= order && sb->sb)
51 if (dynamic_fault("bcache:add:super_realloc"))
54 bio = bio_kmalloc(GFP_KERNEL, 1 << order);
62 new_sb = (void *) __get_free_pages(GFP_KERNEL, order);
67 memcpy(new_sb, sb->sb, PAGE_SIZE << sb->page_order);
69 free_pages((unsigned long) sb->sb, sb->page_order);
72 sb->page_order = order;
77 int bch_dev_sb_realloc(struct bcache_superblock *sb, unsigned u64s)
79 u64 new_bytes = __vstruct_bytes(struct bch_sb, u64s);
80 u64 max_bytes = 512 << sb->sb->layout.sb_max_size_bits;
82 if (new_bytes > max_bytes) {
83 char buf[BDEVNAME_SIZE];
85 pr_err("%s: superblock too big: want %llu but have %llu",
86 bdevname(sb->bdev, buf), new_bytes, max_bytes);
90 return __bch_super_realloc(sb, get_order(new_bytes));
93 static int bch_fs_sb_realloc(struct cache_set *c, unsigned u64s)
95 u64 bytes = __vstruct_bytes(struct bch_sb, u64s);
97 unsigned order = get_order(bytes);
99 if (c->disk_sb && order <= c->disk_sb_order)
102 sb = (void *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, order);
107 memcpy(sb, c->disk_sb, PAGE_SIZE << c->disk_sb_order);
109 free_pages((unsigned long) c->disk_sb, c->disk_sb_order);
112 c->disk_sb_order = order;
116 static struct bch_sb_field *__bch_sb_field_resize(struct bch_sb *sb,
117 struct bch_sb_field *f,
120 unsigned old_u64s = f ? le32_to_cpu(f->u64s) : 0;
123 f = vstruct_last(sb);
124 memset(f, 0, sizeof(u64) * u64s);
125 f->u64s = cpu_to_le32(u64s);
130 src = vstruct_end(f);
131 f->u64s = cpu_to_le32(u64s);
132 dst = vstruct_end(f);
134 memmove(dst, src, vstruct_end(sb) - src);
137 memset(src, 0, dst - src);
140 le32_add_cpu(&sb->u64s, u64s - old_u64s);
146 struct bch_sb_field *bch_fs_sb_field_resize(struct cache_set *c,
147 struct bch_sb_field *f,
150 ssize_t old_u64s = f ? le32_to_cpu(f->u64s) : 0;
151 ssize_t d = -old_u64s + u64s;
155 lockdep_assert_held(&c->sb_lock);
157 if (bch_fs_sb_realloc(c, le32_to_cpu(c->disk_sb->u64s) + d))
160 for_each_cache(ca, c, i) {
161 struct bcache_superblock *sb = &ca->disk_sb;
163 if (bch_dev_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d)) {
164 percpu_ref_put(&ca->ref);
169 return __bch_sb_field_resize(c->disk_sb, f, u64s);
172 struct bch_sb_field *bch_dev_sb_field_resize(struct bcache_superblock *sb,
173 struct bch_sb_field *f,
176 ssize_t old_u64s = f ? le32_to_cpu(f->u64s) : 0;
177 ssize_t d = -old_u64s + u64s;
179 if (bch_dev_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d))
182 return __bch_sb_field_resize(sb->sb, f, u64s);
185 static const char *validate_sb_layout(struct bch_sb_layout *layout)
187 u64 offset, prev_offset, max_sectors;
190 if (uuid_le_cmp(layout->magic, BCACHE_MAGIC))
191 return "Not a bcache superblock layout";
193 if (layout->layout_type != 0)
194 return "Invalid superblock layout type";
196 if (!layout->nr_superblocks)
197 return "Invalid superblock layout: no superblocks";
199 if (layout->nr_superblocks > ARRAY_SIZE(layout->sb_offset))
200 return "Invalid superblock layout: too many superblocks";
202 max_sectors = 1 << layout->sb_max_size_bits;
204 prev_offset = le64_to_cpu(layout->sb_offset[0]);
206 if (prev_offset != BCH_SB_SECTOR)
207 return "Invalid superblock layout: doesn't have default superblock location";
209 for (i = 1; i < layout->nr_superblocks; i++) {
210 offset = le64_to_cpu(layout->sb_offset[i]);
212 if (offset < prev_offset + max_sectors)
213 return "Invalid superblock layout: superblocks overlap";
214 prev_offset = offset;
220 const char *bch_validate_cache_super(struct bcache_superblock *disk_sb)
222 struct bch_sb *sb = disk_sb->sb;
223 struct bch_sb_field *f;
224 struct bch_sb_field_members *sb_mi;
225 struct bch_sb_field_journal *journal;
226 struct cache_member_cpu mi;
231 switch (le64_to_cpu(sb->version)) {
232 case BCACHE_SB_VERSION_CDEV_V4:
235 return"Unsupported superblock version";
238 if (BCH_SB_INITIALIZED(sb) &&
239 le64_to_cpu(sb->version) != BCACHE_SB_VERSION_CDEV_V4)
240 return "Unsupported superblock version";
242 block_size = le16_to_cpu(sb->block_size);
244 if (!is_power_of_2(block_size) ||
245 block_size > PAGE_SECTORS)
246 return "Bad block size";
248 if (bch_is_zero(sb->user_uuid.b, sizeof(uuid_le)))
249 return "Bad user UUID";
251 if (bch_is_zero(sb->uuid.b, sizeof(uuid_le)))
252 return "Bad internal UUID";
254 if (!sb->nr_devices ||
255 sb->nr_devices <= sb->dev_idx ||
256 sb->nr_devices > BCH_SB_MEMBERS_MAX)
257 return "Bad cache device number in set";
259 if (!BCH_SB_META_REPLICAS_WANT(sb) ||
260 BCH_SB_META_REPLICAS_WANT(sb) >= BCH_REPLICAS_MAX)
261 return "Invalid number of metadata replicas";
263 if (!BCH_SB_META_REPLICAS_HAVE(sb) ||
264 BCH_SB_META_REPLICAS_HAVE(sb) >
265 BCH_SB_META_REPLICAS_WANT(sb))
266 return "Invalid number of metadata replicas";
268 if (!BCH_SB_DATA_REPLICAS_WANT(sb) ||
269 BCH_SB_DATA_REPLICAS_WANT(sb) >= BCH_REPLICAS_MAX)
270 return "Invalid number of data replicas";
272 if (!BCH_SB_DATA_REPLICAS_HAVE(sb) ||
273 BCH_SB_DATA_REPLICAS_HAVE(sb) >
274 BCH_SB_DATA_REPLICAS_WANT(sb))
275 return "Invalid number of data replicas";
277 if (!BCH_SB_BTREE_NODE_SIZE(sb))
278 return "Btree node size not set";
280 if (!is_power_of_2(BCH_SB_BTREE_NODE_SIZE(sb)))
281 return "Btree node size not a power of two";
283 if (BCH_SB_BTREE_NODE_SIZE(sb) > BTREE_NODE_SIZE_MAX)
284 return "Btree node size too large";
286 if (BCH_SB_GC_RESERVE(sb) < 5)
287 return "gc reserve percentage too small";
289 if (1U << BCH_SB_JOURNAL_ENTRY_SIZE(sb) < block_size)
290 return "max journal entry size too small";
293 if (512U << BCH_SB_JOURNAL_ENTRY_SIZE(sb) > JOURNAL_ENTRY_SIZE_MAX)
294 return "max journal entry size too big";
296 if (!sb->time_precision ||
297 le32_to_cpu(sb->time_precision) > NSEC_PER_SEC)
298 return "invalid time precision";
300 /* validate layout */
301 err = validate_sb_layout(&sb->layout);
305 vstruct_for_each(sb, f) {
307 return "Invalid superblock: invalid optional field";
309 if (vstruct_next(f) > vstruct_last(sb))
310 return "Invalid superblock: invalid optional field";
312 if (le32_to_cpu(f->type) >= BCH_SB_FIELD_NR)
313 return "Invalid superblock: unknown optional field type";
316 /* Validate member info: */
317 sb_mi = bch_sb_get_members(sb);
319 return "Invalid superblock: member info area missing";
321 if ((void *) (sb_mi->members + sb->nr_devices) >
322 vstruct_end(&sb_mi->field))
323 return "Invalid superblock: bad member info";
325 mi = cache_mi_to_cpu_mi(sb_mi->members + sb->dev_idx);
327 for (i = 0; i < sb->layout.nr_superblocks; i++) {
328 u64 offset = le64_to_cpu(sb->layout.sb_offset[i]);
329 u64 max_size = 1 << sb->layout.sb_max_size_bits;
331 if (offset + max_size > mi.first_bucket * mi.bucket_size)
332 return "Invalid superblock: first bucket comes before end of super";
335 if (mi.nbuckets > LONG_MAX)
336 return "Too many buckets";
338 if (mi.nbuckets - mi.first_bucket < 1 << 10)
339 return "Not enough buckets";
341 if (!is_power_of_2(mi.bucket_size) ||
342 mi.bucket_size < PAGE_SECTORS ||
343 mi.bucket_size < block_size)
344 return "Bad bucket size";
346 if (get_capacity(disk_sb->bdev->bd_disk) <
347 mi.bucket_size * mi.nbuckets)
348 return "Invalid superblock: device too small";
350 /* Validate journal buckets: */
351 journal = bch_sb_get_journal(sb);
353 for (i = 0; i < bch_nr_journal_buckets(journal); i++) {
354 u64 b = le64_to_cpu(journal->buckets[i]);
356 if (b < mi.first_bucket || b >= mi.nbuckets)
357 return "bad journal bucket";
366 static bool bch_is_open_cache(struct block_device *bdev)
373 list_for_each_entry(c, &bch_cache_sets, list)
374 for_each_cache_rcu(ca, c, i)
375 if (ca->disk_sb.bdev == bdev) {
383 static bool bch_is_open(struct block_device *bdev)
385 lockdep_assert_held(&bch_register_lock);
387 return bch_is_open_cache(bdev) || bch_is_open_backing_dev(bdev);
390 static const char *bch_blkdev_open(const char *path, void *holder,
391 struct cache_set_opts opts,
392 struct block_device **ret)
394 struct block_device *bdev;
395 fmode_t mode = opts.nochanges > 0
397 : FMODE_READ|FMODE_WRITE|FMODE_EXCL;
401 bdev = blkdev_get_by_path(path, mode, holder);
403 if (bdev == ERR_PTR(-EBUSY)) {
404 bdev = lookup_bdev(path);
406 return "device busy";
408 err = bch_is_open(bdev)
409 ? "device already registered"
417 return "failed to open device";
419 bdev_get_queue(bdev)->backing_dev_info.capabilities |= BDI_CAP_STABLE_WRITES;
425 /* Update cached mi: */
426 int bch_cache_set_mi_update(struct cache_set *c,
427 struct bch_member *mi,
430 struct cache_member_rcu *new, *old;
434 lockdep_assert_held(&c->sb_lock);
436 new = kzalloc(sizeof(struct cache_member_rcu) +
437 sizeof(struct cache_member_cpu) * nr_devices,
442 new->nr_devices = nr_devices;
444 for (i = 0; i < nr_devices; i++)
445 new->m[i] = cache_mi_to_cpu_mi(&mi[i]);
448 for_each_cache(ca, c, i)
452 old = rcu_dereference_protected(c->members,
453 lockdep_is_held(&c->sb_lock));
455 rcu_assign_pointer(c->members, new);
462 static void bch_sb_update(struct cache_set *c)
464 struct bch_sb *src = c->disk_sb;
466 lockdep_assert_held(&c->sb_lock);
468 c->sb.uuid = src->uuid;
469 c->sb.user_uuid = src->user_uuid;
470 c->sb.block_size = le16_to_cpu(src->block_size);
471 c->sb.btree_node_size = BCH_SB_BTREE_NODE_SIZE(src);
472 c->sb.nr_devices = src->nr_devices;
473 c->sb.clean = BCH_SB_CLEAN(src);
474 c->sb.meta_replicas_have= BCH_SB_META_REPLICAS_HAVE(src);
475 c->sb.data_replicas_have= BCH_SB_DATA_REPLICAS_HAVE(src);
476 c->sb.str_hash_type = BCH_SB_STR_HASH_TYPE(src);
477 c->sb.encryption_type = BCH_SB_ENCRYPTION_TYPE(src);
478 c->sb.time_base_lo = le64_to_cpu(src->time_base_lo);
479 c->sb.time_base_hi = le32_to_cpu(src->time_base_hi);
480 c->sb.time_precision = le32_to_cpu(src->time_precision);
483 /* doesn't copy member info */
484 static void __copy_super(struct bch_sb *dst, struct bch_sb *src)
486 struct bch_sb_field *src_f, *dst_f;
488 dst->version = src->version;
490 dst->uuid = src->uuid;
491 dst->user_uuid = src->user_uuid;
492 memcpy(dst->label, src->label, sizeof(dst->label));
494 dst->block_size = src->block_size;
495 dst->nr_devices = src->nr_devices;
497 dst->time_base_lo = src->time_base_lo;
498 dst->time_base_hi = src->time_base_hi;
499 dst->time_precision = src->time_precision;
501 memcpy(dst->flags, src->flags, sizeof(dst->flags));
502 memcpy(dst->features, src->features, sizeof(dst->features));
503 memcpy(dst->compat, src->compat, sizeof(dst->compat));
505 vstruct_for_each(src, src_f) {
506 if (src_f->type == BCH_SB_FIELD_journal)
509 dst_f = bch_sb_field_get(dst, src_f->type);
510 dst_f = __bch_sb_field_resize(dst, dst_f,
511 le32_to_cpu(src_f->u64s));
513 memcpy(dst_f, src_f, vstruct_bytes(src_f));
517 int bch_sb_to_cache_set(struct cache_set *c, struct bch_sb *src)
519 struct bch_sb_field_members *members =
520 bch_sb_get_members(src);
521 struct bch_sb_field_journal *journal_buckets =
522 bch_sb_get_journal(src);
523 unsigned journal_u64s = journal_buckets
524 ? le32_to_cpu(journal_buckets->field.u64s)
527 lockdep_assert_held(&c->sb_lock);
529 if (bch_fs_sb_realloc(c, le32_to_cpu(src->u64s) - journal_u64s))
532 if (bch_cache_set_mi_update(c, members->members, src->nr_devices))
535 __copy_super(c->disk_sb, src);
541 int bch_sb_from_cache_set(struct cache_set *c, struct cache *ca)
543 struct bch_sb *src = c->disk_sb, *dst = ca->disk_sb.sb;
544 struct bch_sb_field_journal *journal_buckets =
545 bch_sb_get_journal(dst);
546 unsigned journal_u64s = journal_buckets
547 ? le32_to_cpu(journal_buckets->field.u64s)
549 unsigned u64s = le32_to_cpu(src->u64s) + journal_u64s;
552 ret = bch_dev_sb_realloc(&ca->disk_sb, u64s);
556 __copy_super(dst, src);
561 /* read superblock: */
563 static const char *read_one_super(struct bcache_superblock *sb, u64 offset)
565 struct bch_csum csum;
570 sb->bio->bi_bdev = sb->bdev;
571 sb->bio->bi_iter.bi_sector = BCH_SB_SECTOR;
572 sb->bio->bi_iter.bi_size = PAGE_SIZE << sb->page_order;
573 bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
574 bch_bio_map(sb->bio, sb->sb);
576 if (submit_bio_wait(sb->bio))
579 if (uuid_le_cmp(sb->sb->magic, BCACHE_MAGIC))
580 return "Not a bcache superblock";
582 if (le64_to_cpu(sb->sb->version) != BCACHE_SB_VERSION_CDEV_V4)
583 return "Unsupported superblock version";
585 bytes = vstruct_bytes(sb->sb);
587 if (bytes > 512 << sb->sb->layout.sb_max_size_bits)
588 return "Bad superblock: too big";
590 order = get_order(bytes);
591 if (order > sb->page_order) {
592 if (__bch_super_realloc(sb, order))
593 return "cannot allocate memory";
597 if (BCH_SB_CSUM_TYPE(sb->sb) >= BCH_CSUM_NR)
598 return "unknown csum type";
600 /* XXX: verify MACs */
601 csum = csum_vstruct(NULL, BCH_SB_CSUM_TYPE(sb->sb),
602 (struct nonce) { 0 }, sb->sb);
604 if (bch_crc_cmp(csum, sb->sb->csum))
605 return "bad checksum reading superblock";
610 const char *bch_read_super(struct bcache_superblock *sb,
611 struct cache_set_opts opts,
614 struct bch_sb_layout layout;
618 lockdep_assert_held(&bch_register_lock);
620 memset(sb, 0, sizeof(*sb));
622 err = bch_blkdev_open(path, &sb, opts, &sb->bdev);
626 err = "cannot allocate memory";
627 if (__bch_super_realloc(sb, 0))
630 err = "dynamic fault";
631 if (cache_set_init_fault("read_super"))
634 err = read_one_super(sb, BCH_SB_SECTOR);
638 pr_err("error reading default super: %s", err);
641 * Error reading primary superblock - read location of backup
645 sb->bio->bi_bdev = sb->bdev;
646 sb->bio->bi_iter.bi_sector = BCH_SB_LAYOUT_SECTOR;
647 sb->bio->bi_iter.bi_size = sizeof(struct bch_sb_layout);
648 bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
650 * use sb buffer to read layout, since sb buffer is page aligned but
653 bch_bio_map(sb->bio, sb->sb);
656 if (submit_bio_wait(sb->bio))
659 memcpy(&layout, sb->sb, sizeof(layout));
660 err = validate_sb_layout(&layout);
664 for (i = 0; i < layout.nr_superblocks; i++) {
665 u64 offset = le64_to_cpu(layout.sb_offset[i]);
667 if (offset == BCH_SB_SECTOR)
670 err = read_one_super(sb, offset);
676 pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
677 le64_to_cpu(sb->sb->version),
678 le64_to_cpu(sb->sb->flags),
679 le64_to_cpu(sb->sb->seq),
680 le16_to_cpu(sb->sb->u64s));
682 err = "Superblock block size smaller than device block size";
683 if (le16_to_cpu(sb->sb->block_size) << 9 <
684 bdev_logical_block_size(sb->bdev))
693 /* write superblock: */
695 static void write_super_endio(struct bio *bio)
697 struct cache *ca = bio->bi_private;
699 /* XXX: return errors directly */
701 cache_fatal_io_err_on(bio->bi_error, ca, "superblock write");
703 bch_account_io_completion(ca);
705 closure_put(&ca->set->sb_write);
706 percpu_ref_put(&ca->ref);
709 static bool write_one_super(struct cache_set *c, struct cache *ca, unsigned idx)
711 struct bch_sb *sb = ca->disk_sb.sb;
712 struct bio *bio = ca->disk_sb.bio;
714 if (idx >= sb->layout.nr_superblocks)
717 sb->offset = sb->layout.sb_offset[idx];
719 SET_BCH_SB_CSUM_TYPE(sb, c->opts.metadata_checksum);
720 sb->csum = csum_vstruct(c, BCH_SB_CSUM_TYPE(sb),
721 (struct nonce) { 0 }, sb);
724 bio->bi_bdev = ca->disk_sb.bdev;
725 bio->bi_iter.bi_sector = le64_to_cpu(sb->offset);
726 bio->bi_iter.bi_size =
727 roundup(vstruct_bytes(sb),
728 bdev_logical_block_size(ca->disk_sb.bdev));
729 bio->bi_end_io = write_super_endio;
730 bio->bi_private = ca;
731 bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META);
732 bch_bio_map(bio, sb);
734 percpu_ref_get(&ca->ref);
735 closure_bio_submit_punt(bio, &c->sb_write, c);
740 void bch_write_super(struct cache_set *c)
742 struct bch_sb_field_members *members =
743 bch_sb_get_members(c->disk_sb);
744 struct closure *cl = &c->sb_write;
746 unsigned i, super_idx = 0;
749 lockdep_assert_held(&c->sb_lock);
751 closure_init_stack(cl);
753 le64_add_cpu(&c->disk_sb->seq, 1);
755 for_each_cache(ca, c, i)
756 bch_sb_from_cache_set(c, ca);
760 for_each_cache(ca, c, i)
761 if (write_one_super(c, ca, super_idx))
768 /* Make new options visible after they're persistent: */
769 bch_cache_set_mi_update(c, members->members, c->sb.nr_devices);
773 void bch_check_mark_super_slowpath(struct cache_set *c, const struct bkey_i *k,
776 struct bch_member *mi;
777 struct bkey_s_c_extent e = bkey_i_to_s_c_extent(k);
778 const struct bch_extent_ptr *ptr;
780 mutex_lock(&c->sb_lock);
782 /* recheck, might have raced */
783 if (bch_check_super_marked(c, k, meta)) {
784 mutex_unlock(&c->sb_lock);
788 mi = bch_sb_get_members(c->disk_sb)->members;
790 extent_for_each_ptr(e, ptr)
793 ? SET_BCH_MEMBER_HAS_METADATA
794 : SET_BCH_MEMBER_HAS_DATA)(mi + ptr->dev, true);
797 mutex_unlock(&c->sb_lock);