git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/replicas.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "buckets.h"
   5 #include "journal.h"
   6 #include "replicas.h"
   7 #include "super-io.h"
   8
   9 static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *,
  10                                             struct bch_replicas_cpu *);
  11
  12 /* Replicas tracking - in memory: */
  13
  14 static void verify_replicas_entry(struct bch_replicas_entry *e)
  15 {
  16 #ifdef CONFIG_BCACHEFS_DEBUG
  17         unsigned i;
  18
  19         BUG_ON(e->data_type >= BCH_DATA_NR);
  20         BUG_ON(!e->nr_devs);
  21         BUG_ON(e->nr_required > 1 &&
  22                e->nr_required >= e->nr_devs);
  23
  24         for (i = 0; i + 1 < e->nr_devs; i++)
  25                 BUG_ON(e->devs[i] >= e->devs[i + 1]);
  26 #endif
  27 }
  28
  29 void bch2_replicas_entry_sort(struct bch_replicas_entry *e)
  30 {
  31         bubble_sort(e->devs, e->nr_devs, u8_cmp);
  32 }
  33
  34 static void bch2_cpu_replicas_sort(struct bch_replicas_cpu *r)
  35 {
  36         eytzinger0_sort(r->entries, r->nr, r->entry_size, memcmp, NULL);
  37 }
  38
  39 void bch2_replicas_entry_to_text(struct printbuf *out,
  40                                  struct bch_replicas_entry *e)
  41 {
  42         unsigned i;
  43
  44         if (e->data_type < BCH_DATA_NR)
  45                 pr_buf(out, "%s", bch2_data_types[e->data_type]);
  46         else
  47                 pr_buf(out, "(invalid data type %u)", e->data_type);
  48
  49         pr_buf(out, ": %u/%u [", e->nr_required, e->nr_devs);
  50         for (i = 0; i < e->nr_devs; i++)
  51                 pr_buf(out, i ? " %u" : "%u", e->devs[i]);
  52         pr_buf(out, "]");
  53 }
  54
  55 void bch2_cpu_replicas_to_text(struct printbuf *out,
  56                                struct bch_replicas_cpu *r)
  57 {
  58         struct bch_replicas_entry *e;
  59         bool first = true;
  60
  61         for_each_cpu_replicas_entry(r, e) {
  62                 if (!first)
  63                         pr_buf(out, " ");
  64                 first = false;
  65
  66                 bch2_replicas_entry_to_text(out, e);
  67         }
  68 }
  69
  70 static void extent_to_replicas(struct bkey_s_c k,
  71                                struct bch_replicas_entry *r)
  72 {
  73         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
  74         const union bch_extent_entry *entry;
  75         struct extent_ptr_decoded p;
  76
  77         r->nr_required  = 1;
  78
  79         bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
  80                 if (p.ptr.cached)
  81                         continue;
  82
  83                 if (!p.has_ec)
  84                         r->devs[r->nr_devs++] = p.ptr.dev;
  85                 else
  86                         r->nr_required = 0;
  87         }
  88 }
  89
  90 static void stripe_to_replicas(struct bkey_s_c k,
  91                                struct bch_replicas_entry *r)
  92 {
  93         struct bkey_s_c_stripe s = bkey_s_c_to_stripe(k);
  94         const struct bch_extent_ptr *ptr;
  95
  96         r->nr_required  = s.v->nr_blocks - s.v->nr_redundant;
  97
  98         for (ptr = s.v->ptrs;
  99              ptr < s.v->ptrs + s.v->nr_blocks;
 100              ptr++)
 101                 r->devs[r->nr_devs++] = ptr->dev;
 102 }
 103
 104 void bch2_bkey_to_replicas(struct bch_replicas_entry *e,
 105                            struct bkey_s_c k)
 106 {
 107         e->nr_devs = 0;
 108
 109         switch (k.k->type) {
 110         case KEY_TYPE_btree_ptr:
 111         case KEY_TYPE_btree_ptr_v2:
 112                 e->data_type = BCH_DATA_btree;
 113                 extent_to_replicas(k, e);
 114                 break;
 115         case KEY_TYPE_extent:
 116         case KEY_TYPE_reflink_v:
 117                 e->data_type = BCH_DATA_user;
 118                 extent_to_replicas(k, e);
 119                 break;
 120         case KEY_TYPE_stripe:
 121                 e->data_type = BCH_DATA_parity;
 122                 stripe_to_replicas(k, e);
 123                 break;
 124         }
 125
 126         bch2_replicas_entry_sort(e);
 127 }
 128
 129 void bch2_devlist_to_replicas(struct bch_replicas_entry *e,
 130                               enum bch_data_type data_type,
 131                               struct bch_devs_list devs)
 132 {
 133         unsigned i;
 134
 135         BUG_ON(!data_type ||
 136                data_type == BCH_DATA_sb ||
 137                data_type >= BCH_DATA_NR);
 138
 139         e->data_type    = data_type;
 140         e->nr_devs      = 0;
 141         e->nr_required  = 1;
 142
 143         for (i = 0; i < devs.nr; i++)
 144                 e->devs[e->nr_devs++] = devs.devs[i];
 145
 146         bch2_replicas_entry_sort(e);
 147 }
 148
 149 static struct bch_replicas_cpu
 150 cpu_replicas_add_entry(struct bch_replicas_cpu *old,
 151                        struct bch_replicas_entry *new_entry)
 152 {
 153         unsigned i;
 154         struct bch_replicas_cpu new = {
 155                 .nr             = old->nr + 1,
 156                 .entry_size     = max_t(unsigned, old->entry_size,
 157                                         replicas_entry_bytes(new_entry)),
 158         };
 159
 160         BUG_ON(!new_entry->data_type);
 161         verify_replicas_entry(new_entry);
 162
 163         new.entries = kcalloc(new.nr, new.entry_size, GFP_KERNEL);
 164         if (!new.entries)
 165                 return new;
 166
 167         for (i = 0; i < old->nr; i++)
 168                 memcpy(cpu_replicas_entry(&new, i),
 169                        cpu_replicas_entry(old, i),
 170                        old->entry_size);
 171
 172         memcpy(cpu_replicas_entry(&new, old->nr),
 173                new_entry,
 174                replicas_entry_bytes(new_entry));
 175
 176         bch2_cpu_replicas_sort(&new);
 177         return new;
 178 }
 179
 180 static inline int __replicas_entry_idx(struct bch_replicas_cpu *r,
 181                                        struct bch_replicas_entry *search)
 182 {
 183         int idx, entry_size = replicas_entry_bytes(search);
 184
 185         if (unlikely(entry_size > r->entry_size))
 186                 return -1;
 187
 188         verify_replicas_entry(search);
 189
 190 #define entry_cmp(_l, _r, size) memcmp(_l, _r, entry_size)
 191         idx = eytzinger0_find(r->entries, r->nr, r->entry_size,
 192                               entry_cmp, search);
 193 #undef entry_cmp
 194
 195         return idx < r->nr ? idx : -1;
 196 }
 197
 198 int bch2_replicas_entry_idx(struct bch_fs *c,
 199                             struct bch_replicas_entry *search)
 200 {
 201         bch2_replicas_entry_sort(search);
 202
 203         return __replicas_entry_idx(&c->replicas, search);
 204 }
 205
 206 static bool __replicas_has_entry(struct bch_replicas_cpu *r,
 207                                  struct bch_replicas_entry *search)
 208 {
 209         return __replicas_entry_idx(r, search) >= 0;
 210 }
 211
 212 bool bch2_replicas_marked(struct bch_fs *c,
 213                           struct bch_replicas_entry *search)
 214 {
 215         bool marked;
 216
 217         if (!search->nr_devs)
 218                 return true;
 219
 220         verify_replicas_entry(search);
 221
 222         percpu_down_read(&c->mark_lock);
 223         marked = __replicas_has_entry(&c->replicas, search) &&
 224                 (likely((!c->replicas_gc.entries)) ||
 225                  __replicas_has_entry(&c->replicas_gc, search));
 226         percpu_up_read(&c->mark_lock);
 227
 228         return marked;
 229 }
 230
 231 static void __replicas_table_update(struct bch_fs_usage *dst,
 232                                     struct bch_replicas_cpu *dst_r,
 233                                     struct bch_fs_usage *src,
 234                                     struct bch_replicas_cpu *src_r)
 235 {
 236         int src_idx, dst_idx;
 237
 238         *dst = *src;
 239
 240         for (src_idx = 0; src_idx < src_r->nr; src_idx++) {
 241                 if (!src->replicas[src_idx])
 242                         continue;
 243
 244                 dst_idx = __replicas_entry_idx(dst_r,
 245                                 cpu_replicas_entry(src_r, src_idx));
 246                 BUG_ON(dst_idx < 0);
 247
 248                 dst->replicas[dst_idx] = src->replicas[src_idx];
 249         }
 250 }
 251
 252 static void __replicas_table_update_pcpu(struct bch_fs_usage __percpu *dst_p,
 253                                     struct bch_replicas_cpu *dst_r,
 254                                     struct bch_fs_usage __percpu *src_p,
 255                                     struct bch_replicas_cpu *src_r)
 256 {
 257         unsigned src_nr = sizeof(struct bch_fs_usage) / sizeof(u64) + src_r->nr;
 258         struct bch_fs_usage *dst, *src = (void *)
 259                 bch2_acc_percpu_u64s((void *) src_p, src_nr);
 260
 261         preempt_disable();
 262         dst = this_cpu_ptr(dst_p);
 263         preempt_enable();
 264
 265         __replicas_table_update(dst, dst_r, src, src_r);
 266 }
 267
 268 /*
 269  * Resize filesystem accounting:
 270  */
 271 static int replicas_table_update(struct bch_fs *c,
 272                                  struct bch_replicas_cpu *new_r)
 273 {
 274         struct bch_fs_usage __percpu *new_usage[JOURNAL_BUF_NR];
 275         struct bch_fs_usage_online *new_scratch = NULL;
 276         struct bch_fs_usage __percpu *new_gc = NULL;
 277         struct bch_fs_usage *new_base = NULL;
 278         unsigned i, bytes = sizeof(struct bch_fs_usage) +
 279                 sizeof(u64) * new_r->nr;
 280         unsigned scratch_bytes = sizeof(struct bch_fs_usage_online) +
 281                 sizeof(u64) * new_r->nr;
 282         int ret = 0;
 283
 284         memset(new_usage, 0, sizeof(new_usage));
 285
 286         for (i = 0; i < ARRAY_SIZE(new_usage); i++)
 287                 if (!(new_usage[i] = __alloc_percpu_gfp(bytes,
 288                                         sizeof(u64), GFP_KERNEL)))
 289                         goto err;
 290
 291         if (!(new_base = kzalloc(bytes, GFP_KERNEL)) ||
 292             !(new_scratch  = kmalloc(scratch_bytes, GFP_KERNEL)) ||
 293             (c->usage_gc &&
 294              !(new_gc = __alloc_percpu_gfp(bytes, sizeof(u64), GFP_KERNEL))))
 295                 goto err;
 296
 297         for (i = 0; i < ARRAY_SIZE(new_usage); i++)
 298                 if (c->usage[i])
 299                         __replicas_table_update_pcpu(new_usage[i], new_r,
 300                                                      c->usage[i], &c->replicas);
 301         if (c->usage_base)
 302                 __replicas_table_update(new_base,               new_r,
 303                                         c->usage_base,          &c->replicas);
 304         if (c->usage_gc)
 305                 __replicas_table_update_pcpu(new_gc,            new_r,
 306                                              c->usage_gc,       &c->replicas);
 307
 308         for (i = 0; i < ARRAY_SIZE(new_usage); i++)
 309                 swap(c->usage[i],       new_usage[i]);
 310         swap(c->usage_base,     new_base);
 311         swap(c->usage_scratch,  new_scratch);
 312         swap(c->usage_gc,       new_gc);
 313         swap(c->replicas,       *new_r);
 314 out:
 315         free_percpu(new_gc);
 316         kfree(new_scratch);
 317         for (i = 0; i < ARRAY_SIZE(new_usage); i++)
 318                 free_percpu(new_usage[i]);
 319         kfree(new_base);
 320         return ret;
 321 err:
 322         bch_err(c, "error updating replicas table: memory allocation failure");
 323         ret = -ENOMEM;
 324         goto out;
 325 }
 326
 327 static unsigned reserve_journal_replicas(struct bch_fs *c,
 328                                      struct bch_replicas_cpu *r)
 329 {
 330         struct bch_replicas_entry *e;
 331         unsigned journal_res_u64s = 0;
 332
 333         /* nr_inodes: */
 334         journal_res_u64s +=
 335                 DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64));
 336
 337         /* key_version: */
 338         journal_res_u64s +=
 339                 DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64));
 340
 341         /* persistent_reserved: */
 342         journal_res_u64s +=
 343                 DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64)) *
 344                 BCH_REPLICAS_MAX;
 345
 346         for_each_cpu_replicas_entry(r, e)
 347                 journal_res_u64s +=
 348                         DIV_ROUND_UP(sizeof(struct jset_entry_data_usage) +
 349                                      e->nr_devs, sizeof(u64));
 350         return journal_res_u64s;
 351 }
 352
 353 noinline
 354 static int bch2_mark_replicas_slowpath(struct bch_fs *c,
 355                                 struct bch_replicas_entry *new_entry)
 356 {
 357         struct bch_replicas_cpu new_r, new_gc;
 358         int ret = 0;
 359
 360         verify_replicas_entry(new_entry);
 361
 362         memset(&new_r, 0, sizeof(new_r));
 363         memset(&new_gc, 0, sizeof(new_gc));
 364
 365         mutex_lock(&c->sb_lock);
 366
 367         if (c->replicas_gc.entries &&
 368             !__replicas_has_entry(&c->replicas_gc, new_entry)) {
 369                 new_gc = cpu_replicas_add_entry(&c->replicas_gc, new_entry);
 370                 if (!new_gc.entries)
 371                         goto err;
 372         }
 373
 374         if (!__replicas_has_entry(&c->replicas, new_entry)) {
 375                 new_r = cpu_replicas_add_entry(&c->replicas, new_entry);
 376                 if (!new_r.entries)
 377                         goto err;
 378
 379                 ret = bch2_cpu_replicas_to_sb_replicas(c, &new_r);
 380                 if (ret)
 381                         goto err;
 382
 383                 bch2_journal_entry_res_resize(&c->journal,
 384                                 &c->replicas_journal_res,
 385                                 reserve_journal_replicas(c, &new_r));
 386         }
 387
 388         if (!new_r.entries &&
 389             !new_gc.entries)
 390                 goto out;
 391
 392         /* allocations done, now commit: */
 393
 394         if (new_r.entries)
 395                 bch2_write_super(c);
 396
 397         /* don't update in memory replicas until changes are persistent */
 398         percpu_down_write(&c->mark_lock);
 399         if (new_r.entries)
 400                 ret = replicas_table_update(c, &new_r);
 401         if (new_gc.entries)
 402                 swap(new_gc, c->replicas_gc);
 403         percpu_up_write(&c->mark_lock);
 404 out:
 405         mutex_unlock(&c->sb_lock);
 406
 407         kfree(new_r.entries);
 408         kfree(new_gc.entries);
 409
 410         return ret;
 411 err:
 412         bch_err(c, "error adding replicas entry: memory allocation failure");
 413         ret = -ENOMEM;
 414         goto out;
 415 }
 416
 417 int bch2_mark_replicas(struct bch_fs *c, struct bch_replicas_entry *r)
 418 {
 419         return likely(bch2_replicas_marked(c, r))
 420                 ? 0 : bch2_mark_replicas_slowpath(c, r);
 421 }
 422
 423 /* replicas delta list: */
 424
 425 int bch2_replicas_delta_list_mark(struct bch_fs *c,
 426                                   struct replicas_delta_list *r)
 427 {
 428         struct replicas_delta *d = r->d;
 429         struct replicas_delta *top = (void *) r->d + r->used;
 430         int ret = 0;
 431
 432         for (d = r->d; !ret && d != top; d = replicas_delta_next(d))
 433                 ret = bch2_mark_replicas(c, &d->r);
 434         return ret;
 435 }
 436
 437 /*
 438  * Old replicas_gc mechanism: only used for journal replicas entries now, should
 439  * die at some point:
 440  */
 441
 442 int bch2_replicas_gc_end(struct bch_fs *c, int ret)
 443 {
 444         unsigned i;
 445
 446         lockdep_assert_held(&c->replicas_gc_lock);
 447
 448         mutex_lock(&c->sb_lock);
 449         percpu_down_write(&c->mark_lock);
 450
 451         /*
 452          * this is kind of crappy; the replicas gc mechanism needs to be ripped
 453          * out
 454          */
 455
 456         for (i = 0; i < c->replicas.nr; i++) {
 457                 struct bch_replicas_entry *e =
 458                         cpu_replicas_entry(&c->replicas, i);
 459                 struct bch_replicas_cpu n;
 460
 461                 if (!__replicas_has_entry(&c->replicas_gc, e) &&
 462                     bch2_fs_usage_read_one(c, &c->usage_base->replicas[i])) {
 463                         n = cpu_replicas_add_entry(&c->replicas_gc, e);
 464                         if (!n.entries) {
 465                                 ret = -ENOSPC;
 466                                 goto err;
 467                         }
 468
 469                         swap(n, c->replicas_gc);
 470                         kfree(n.entries);
 471                 }
 472         }
 473
 474         if (bch2_cpu_replicas_to_sb_replicas(c, &c->replicas_gc)) {
 475                 ret = -ENOSPC;
 476                 goto err;
 477         }
 478
 479         ret = replicas_table_update(c, &c->replicas_gc);
 480 err:
 481         kfree(c->replicas_gc.entries);
 482         c->replicas_gc.entries = NULL;
 483
 484         percpu_up_write(&c->mark_lock);
 485
 486         if (!ret)
 487                 bch2_write_super(c);
 488
 489         mutex_unlock(&c->sb_lock);
 490
 491         return ret;
 492 }
 493
 494 int bch2_replicas_gc_start(struct bch_fs *c, unsigned typemask)
 495 {
 496         struct bch_replicas_entry *e;
 497         unsigned i = 0;
 498
 499         lockdep_assert_held(&c->replicas_gc_lock);
 500
 501         mutex_lock(&c->sb_lock);
 502         BUG_ON(c->replicas_gc.entries);
 503
 504         c->replicas_gc.nr               = 0;
 505         c->replicas_gc.entry_size       = 0;
 506
 507         for_each_cpu_replicas_entry(&c->replicas, e)
 508                 if (!((1 << e->data_type) & typemask)) {
 509                         c->replicas_gc.nr++;
 510                         c->replicas_gc.entry_size =
 511                                 max_t(unsigned, c->replicas_gc.entry_size,
 512                                       replicas_entry_bytes(e));
 513                 }
 514
 515         c->replicas_gc.entries = kcalloc(c->replicas_gc.nr,
 516                                          c->replicas_gc.entry_size,
 517                                          GFP_KERNEL);
 518         if (!c->replicas_gc.entries) {
 519                 mutex_unlock(&c->sb_lock);
 520                 bch_err(c, "error allocating c->replicas_gc");
 521                 return -ENOMEM;
 522         }
 523
 524         for_each_cpu_replicas_entry(&c->replicas, e)
 525                 if (!((1 << e->data_type) & typemask))
 526                         memcpy(cpu_replicas_entry(&c->replicas_gc, i++),
 527                                e, c->replicas_gc.entry_size);
 528
 529         bch2_cpu_replicas_sort(&c->replicas_gc);
 530         mutex_unlock(&c->sb_lock);
 531
 532         return 0;
 533 }
 534
 535 /* New much simpler mechanism for clearing out unneeded replicas entries: */
 536
 537 int bch2_replicas_gc2(struct bch_fs *c)
 538 {
 539         struct bch_replicas_cpu new = { 0 };
 540         unsigned i, nr;
 541         int ret = 0;
 542
 543         bch2_journal_meta(&c->journal);
 544 retry:
 545         nr              = READ_ONCE(c->replicas.nr);
 546         new.entry_size  = READ_ONCE(c->replicas.entry_size);
 547         new.entries     = kcalloc(nr, new.entry_size, GFP_KERNEL);
 548         if (!new.entries) {
 549                 bch_err(c, "error allocating c->replicas_gc");
 550                 return -ENOMEM;
 551         }
 552
 553         mutex_lock(&c->sb_lock);
 554         percpu_down_write(&c->mark_lock);
 555
 556         if (nr                  != c->replicas.nr ||
 557             new.entry_size      != c->replicas.entry_size) {
 558                 percpu_up_write(&c->mark_lock);
 559                 mutex_unlock(&c->sb_lock);
 560                 kfree(new.entries);
 561                 goto retry;
 562         }
 563
 564         for (i = 0; i < c->replicas.nr; i++) {
 565                 struct bch_replicas_entry *e =
 566                         cpu_replicas_entry(&c->replicas, i);
 567
 568                 if (e->data_type == BCH_DATA_journal ||
 569                     c->usage_base->replicas[i] ||
 570                     percpu_u64_get(&c->usage[0]->replicas[i]) ||
 571                     percpu_u64_get(&c->usage[1]->replicas[i]) ||
 572                     percpu_u64_get(&c->usage[2]->replicas[i]) ||
 573                     percpu_u64_get(&c->usage[3]->replicas[i]))
 574                         memcpy(cpu_replicas_entry(&new, new.nr++),
 575                                e, new.entry_size);
 576         }
 577
 578         bch2_cpu_replicas_sort(&new);
 579
 580         if (bch2_cpu_replicas_to_sb_replicas(c, &new)) {
 581                 ret = -ENOSPC;
 582                 goto err;
 583         }
 584
 585         ret = replicas_table_update(c, &new);
 586 err:
 587         kfree(new.entries);
 588
 589         percpu_up_write(&c->mark_lock);
 590
 591         if (!ret)
 592                 bch2_write_super(c);
 593
 594         mutex_unlock(&c->sb_lock);
 595
 596         return ret;
 597 }
 598
 599 int bch2_replicas_set_usage(struct bch_fs *c,
 600                             struct bch_replicas_entry *r,
 601                             u64 sectors)
 602 {
 603         int ret, idx = bch2_replicas_entry_idx(c, r);
 604
 605         if (idx < 0) {
 606                 struct bch_replicas_cpu n;
 607
 608                 n = cpu_replicas_add_entry(&c->replicas, r);
 609                 if (!n.entries)
 610                         return -ENOMEM;
 611
 612                 ret = replicas_table_update(c, &n);
 613                 if (ret)
 614                         return ret;
 615
 616                 kfree(n.entries);
 617
 618                 idx = bch2_replicas_entry_idx(c, r);
 619                 BUG_ON(ret < 0);
 620         }
 621
 622         c->usage_base->replicas[idx] = sectors;
 623
 624         return 0;
 625 }
 626
 627 /* Replicas tracking - superblock: */
 628
 629 static int
 630 __bch2_sb_replicas_to_cpu_replicas(struct bch_sb_field_replicas *sb_r,
 631                                    struct bch_replicas_cpu *cpu_r)
 632 {
 633         struct bch_replicas_entry *e, *dst;
 634         unsigned nr = 0, entry_size = 0, idx = 0;
 635
 636         for_each_replicas_entry(sb_r, e) {
 637                 entry_size = max_t(unsigned, entry_size,
 638                                    replicas_entry_bytes(e));
 639                 nr++;
 640         }
 641
 642         cpu_r->entries = kcalloc(nr, entry_size, GFP_KERNEL);
 643         if (!cpu_r->entries)
 644                 return -ENOMEM;
 645
 646         cpu_r->nr               = nr;
 647         cpu_r->entry_size       = entry_size;
 648
 649         for_each_replicas_entry(sb_r, e) {
 650                 dst = cpu_replicas_entry(cpu_r, idx++);
 651                 memcpy(dst, e, replicas_entry_bytes(e));
 652                 bch2_replicas_entry_sort(dst);
 653         }
 654
 655         return 0;
 656 }
 657
 658 static int
 659 __bch2_sb_replicas_v0_to_cpu_replicas(struct bch_sb_field_replicas_v0 *sb_r,
 660                                       struct bch_replicas_cpu *cpu_r)
 661 {
 662         struct bch_replicas_entry_v0 *e;
 663         unsigned nr = 0, entry_size = 0, idx = 0;
 664
 665         for_each_replicas_entry(sb_r, e) {
 666                 entry_size = max_t(unsigned, entry_size,
 667                                    replicas_entry_bytes(e));
 668                 nr++;
 669         }
 670
 671         entry_size += sizeof(struct bch_replicas_entry) -
 672                 sizeof(struct bch_replicas_entry_v0);
 673
 674         cpu_r->entries = kcalloc(nr, entry_size, GFP_KERNEL);
 675         if (!cpu_r->entries)
 676                 return -ENOMEM;
 677
 678         cpu_r->nr               = nr;
 679         cpu_r->entry_size       = entry_size;
 680
 681         for_each_replicas_entry(sb_r, e) {
 682                 struct bch_replicas_entry *dst =
 683                         cpu_replicas_entry(cpu_r, idx++);
 684
 685                 dst->data_type  = e->data_type;
 686                 dst->nr_devs    = e->nr_devs;
 687                 dst->nr_required = 1;
 688                 memcpy(dst->devs, e->devs, e->nr_devs);
 689                 bch2_replicas_entry_sort(dst);
 690         }
 691
 692         return 0;
 693 }
 694
 695 int bch2_sb_replicas_to_cpu_replicas(struct bch_fs *c)
 696 {
 697         struct bch_sb_field_replicas *sb_v1;
 698         struct bch_sb_field_replicas_v0 *sb_v0;
 699         struct bch_replicas_cpu new_r = { 0, 0, NULL };
 700         int ret = 0;
 701
 702         if ((sb_v1 = bch2_sb_get_replicas(c->disk_sb.sb)))
 703                 ret = __bch2_sb_replicas_to_cpu_replicas(sb_v1, &new_r);
 704         else if ((sb_v0 = bch2_sb_get_replicas_v0(c->disk_sb.sb)))
 705                 ret = __bch2_sb_replicas_v0_to_cpu_replicas(sb_v0, &new_r);
 706
 707         if (ret)
 708                 return -ENOMEM;
 709
 710         bch2_cpu_replicas_sort(&new_r);
 711
 712         percpu_down_write(&c->mark_lock);
 713
 714         ret = replicas_table_update(c, &new_r);
 715         percpu_up_write(&c->mark_lock);
 716
 717         kfree(new_r.entries);
 718
 719         return 0;
 720 }
 721
 722 static int bch2_cpu_replicas_to_sb_replicas_v0(struct bch_fs *c,
 723                                                struct bch_replicas_cpu *r)
 724 {
 725         struct bch_sb_field_replicas_v0 *sb_r;
 726         struct bch_replicas_entry_v0 *dst;
 727         struct bch_replicas_entry *src;
 728         size_t bytes;
 729
 730         bytes = sizeof(struct bch_sb_field_replicas);
 731
 732         for_each_cpu_replicas_entry(r, src)
 733                 bytes += replicas_entry_bytes(src) - 1;
 734
 735         sb_r = bch2_sb_resize_replicas_v0(&c->disk_sb,
 736                         DIV_ROUND_UP(bytes, sizeof(u64)));
 737         if (!sb_r)
 738                 return -ENOSPC;
 739
 740         bch2_sb_field_delete(&c->disk_sb, BCH_SB_FIELD_replicas);
 741         sb_r = bch2_sb_get_replicas_v0(c->disk_sb.sb);
 742
 743         memset(&sb_r->entries, 0,
 744                vstruct_end(&sb_r->field) -
 745                (void *) &sb_r->entries);
 746
 747         dst = sb_r->entries;
 748         for_each_cpu_replicas_entry(r, src) {
 749                 dst->data_type  = src->data_type;
 750                 dst->nr_devs    = src->nr_devs;
 751                 memcpy(dst->devs, src->devs, src->nr_devs);
 752
 753                 dst = replicas_entry_next(dst);
 754
 755                 BUG_ON((void *) dst > vstruct_end(&sb_r->field));
 756         }
 757
 758         return 0;
 759 }
 760
 761 static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *c,
 762                                             struct bch_replicas_cpu *r)
 763 {
 764         struct bch_sb_field_replicas *sb_r;
 765         struct bch_replicas_entry *dst, *src;
 766         bool need_v1 = false;
 767         size_t bytes;
 768
 769         bytes = sizeof(struct bch_sb_field_replicas);
 770
 771         for_each_cpu_replicas_entry(r, src) {
 772                 bytes += replicas_entry_bytes(src);
 773                 if (src->nr_required != 1)
 774                         need_v1 = true;
 775         }
 776
 777         if (!need_v1)
 778                 return bch2_cpu_replicas_to_sb_replicas_v0(c, r);
 779
 780         sb_r = bch2_sb_resize_replicas(&c->disk_sb,
 781                         DIV_ROUND_UP(bytes, sizeof(u64)));
 782         if (!sb_r)
 783                 return -ENOSPC;
 784
 785         bch2_sb_field_delete(&c->disk_sb, BCH_SB_FIELD_replicas_v0);
 786         sb_r = bch2_sb_get_replicas(c->disk_sb.sb);
 787
 788         memset(&sb_r->entries, 0,
 789                vstruct_end(&sb_r->field) -
 790                (void *) &sb_r->entries);
 791
 792         dst = sb_r->entries;
 793         for_each_cpu_replicas_entry(r, src) {
 794                 memcpy(dst, src, replicas_entry_bytes(src));
 795
 796                 dst = replicas_entry_next(dst);
 797
 798                 BUG_ON((void *) dst > vstruct_end(&sb_r->field));
 799         }
 800
 801         return 0;
 802 }
 803
 804 static int bch2_cpu_replicas_validate(struct bch_replicas_cpu *cpu_r,
 805                                       struct bch_sb *sb,
 806                                       struct printbuf *err)
 807 {
 808         struct bch_sb_field_members *mi = bch2_sb_get_members(sb);
 809         unsigned i, j;
 810
 811         sort_cmp_size(cpu_r->entries,
 812                       cpu_r->nr,
 813                       cpu_r->entry_size,
 814                       memcmp, NULL);
 815
 816         for (i = 0; i < cpu_r->nr; i++) {
 817                 struct bch_replicas_entry *e =
 818                         cpu_replicas_entry(cpu_r, i);
 819
 820                 if (e->data_type >= BCH_DATA_NR) {
 821                         pr_buf(err, "invalid data type in entry ");
 822                         bch2_replicas_entry_to_text(err, e);
 823                         return -EINVAL;
 824                 }
 825
 826                 if (!e->nr_devs) {
 827                         pr_buf(err, "no devices in entry ");
 828                         bch2_replicas_entry_to_text(err, e);
 829                         return -EINVAL;
 830                 }
 831
 832                 if (e->nr_required > 1 &&
 833                     e->nr_required >= e->nr_devs) {
 834                         pr_buf(err, "bad nr_required in entry ");
 835                         bch2_replicas_entry_to_text(err, e);
 836                         return -EINVAL;
 837                 }
 838
 839                 for (j = 0; j < e->nr_devs; j++)
 840                         if (!bch2_dev_exists(sb, mi, e->devs[j])) {
 841                                 pr_buf(err, "invalid device %u in entry ", e->devs[j]);
 842                                 bch2_replicas_entry_to_text(err, e);
 843                                 return -EINVAL;
 844                         }
 845
 846                 if (i + 1 < cpu_r->nr) {
 847                         struct bch_replicas_entry *n =
 848                                 cpu_replicas_entry(cpu_r, i + 1);
 849
 850                         BUG_ON(memcmp(e, n, cpu_r->entry_size) > 0);
 851
 852                         if (!memcmp(e, n, cpu_r->entry_size)) {
 853                                 pr_buf(err, "duplicate replicas entry ");
 854                                 bch2_replicas_entry_to_text(err, e);
 855                                 return -EINVAL;
 856                         }
 857                 }
 858         }
 859
 860         return 0;
 861 }
 862
 863 static int bch2_sb_validate_replicas(struct bch_sb *sb, struct bch_sb_field *f,
 864                                      struct printbuf *err)
 865 {
 866         struct bch_sb_field_replicas *sb_r = field_to_type(f, replicas);
 867         struct bch_replicas_cpu cpu_r;
 868         int ret;
 869
 870         if (__bch2_sb_replicas_to_cpu_replicas(sb_r, &cpu_r))
 871                 return -ENOMEM;
 872
 873         ret = bch2_cpu_replicas_validate(&cpu_r, sb, err);
 874         kfree(cpu_r.entries);
 875         return ret;
 876 }
 877
 878 static void bch2_sb_replicas_to_text(struct printbuf *out,
 879                                      struct bch_sb *sb,
 880                                      struct bch_sb_field *f)
 881 {
 882         struct bch_sb_field_replicas *r = field_to_type(f, replicas);
 883         struct bch_replicas_entry *e;
 884         bool first = true;
 885
 886         for_each_replicas_entry(r, e) {
 887                 if (!first)
 888                         pr_buf(out, " ");
 889                 first = false;
 890
 891                 bch2_replicas_entry_to_text(out, e);
 892         }
 893 }
 894
 895 const struct bch_sb_field_ops bch_sb_field_ops_replicas = {
 896         .validate       = bch2_sb_validate_replicas,
 897         .to_text        = bch2_sb_replicas_to_text,
 898 };
 899
 900 static int bch2_sb_validate_replicas_v0(struct bch_sb *sb, struct bch_sb_field *f,
 901                                         struct printbuf *err)
 902 {
 903         struct bch_sb_field_replicas_v0 *sb_r = field_to_type(f, replicas_v0);
 904         struct bch_replicas_cpu cpu_r;
 905         int ret;
 906
 907         if (__bch2_sb_replicas_v0_to_cpu_replicas(sb_r, &cpu_r))
 908                 return -ENOMEM;
 909
 910         ret = bch2_cpu_replicas_validate(&cpu_r, sb, err);
 911         kfree(cpu_r.entries);
 912         return ret;
 913 }
 914
 915 const struct bch_sb_field_ops bch_sb_field_ops_replicas_v0 = {
 916         .validate       = bch2_sb_validate_replicas_v0,
 917 };
 918
 919 /* Query replicas: */
 920
 921 bool bch2_have_enough_devs(struct bch_fs *c, struct bch_devs_mask devs,
 922                            unsigned flags, bool print)
 923 {
 924         struct bch_replicas_entry *e;
 925         bool ret = true;
 926
 927         percpu_down_read(&c->mark_lock);
 928         for_each_cpu_replicas_entry(&c->replicas, e) {
 929                 unsigned i, nr_online = 0, nr_failed = 0, dflags = 0;
 930                 bool metadata = e->data_type < BCH_DATA_user;
 931
 932                 if (e->data_type == BCH_DATA_cached)
 933                         continue;
 934
 935                 for (i = 0; i < e->nr_devs; i++) {
 936                         struct bch_dev *ca = bch_dev_bkey_exists(c, e->devs[i]);
 937
 938                         nr_online += test_bit(e->devs[i], devs.d);
 939                         nr_failed += ca->mi.state == BCH_MEMBER_STATE_failed;
 940                 }
 941
 942                 if (nr_failed == e->nr_devs)
 943                         continue;
 944
 945                 if (nr_online < e->nr_required)
 946                         dflags |= metadata
 947                                 ? BCH_FORCE_IF_METADATA_LOST
 948                                 : BCH_FORCE_IF_DATA_LOST;
 949
 950                 if (nr_online < e->nr_devs)
 951                         dflags |= metadata
 952                                 ? BCH_FORCE_IF_METADATA_DEGRADED
 953                                 : BCH_FORCE_IF_DATA_DEGRADED;
 954
 955                 if (dflags & ~flags) {
 956                         if (print) {
 957                                 char buf[100];
 958
 959                                 bch2_replicas_entry_to_text(&PBUF(buf), e);
 960                                 bch_err(c, "insufficient devices online (%u) for replicas entry %s",
 961                                         nr_online, buf);
 962                         }
 963                         ret = false;
 964                         break;
 965                 }
 966
 967         }
 968         percpu_up_read(&c->mark_lock);
 969
 970         return ret;
 971 }
 972
 973 unsigned bch2_dev_has_data(struct bch_fs *c, struct bch_dev *ca)
 974 {
 975         struct bch_replicas_entry *e;
 976         unsigned i, ret = 0;
 977
 978         percpu_down_read(&c->mark_lock);
 979
 980         for_each_cpu_replicas_entry(&c->replicas, e)
 981                 for (i = 0; i < e->nr_devs; i++)
 982                         if (e->devs[i] == ca->dev_idx)
 983                                 ret |= 1 << e->data_type;
 984
 985         percpu_up_read(&c->mark_lock);
 986
 987         return ret;
 988 }
 989
 990 void bch2_fs_replicas_exit(struct bch_fs *c)
 991 {
 992         unsigned i;
 993
 994         kfree(c->usage_scratch);
 995         for (i = 0; i < ARRAY_SIZE(c->usage); i++)
 996                 free_percpu(c->usage[i]);
 997         kfree(c->usage_base);
 998         kfree(c->replicas.entries);
 999         kfree(c->replicas_gc.entries);
1000
1001         mempool_exit(&c->replicas_delta_pool);
1002 }
1003
1004 int bch2_fs_replicas_init(struct bch_fs *c)
1005 {
1006         bch2_journal_entry_res_resize(&c->journal,
1007                         &c->replicas_journal_res,
1008                         reserve_journal_replicas(c, &c->replicas));
1009
1010         return mempool_init_kmalloc_pool(&c->replicas_delta_pool, 1,
1011                                          REPLICAS_DELTA_LIST_MAX) ?:
1012                 replicas_table_update(c, &c->replicas);
1013 }