]> git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/replicas.c
Update bcachefs sources to 10ab39f2fa bcachefs: Improvements to the journal read...
[bcachefs-tools-debian] / libbcachefs / replicas.c
1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "bcachefs.h"
4 #include "buckets.h"
5 #include "journal.h"
6 #include "replicas.h"
7 #include "super-io.h"
8
9 static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *,
10                                             struct bch_replicas_cpu *);
11
12 /* Replicas tracking - in memory: */
13
14 static inline int u8_cmp(u8 l, u8 r)
15 {
16         return cmp_int(l, r);
17 }
18
19 static void verify_replicas_entry(struct bch_replicas_entry *e)
20 {
21 #ifdef CONFIG_BCACHEFS_DEBUG
22         unsigned i;
23
24         BUG_ON(e->data_type >= BCH_DATA_NR);
25         BUG_ON(!e->nr_devs);
26         BUG_ON(e->nr_required > 1 &&
27                e->nr_required >= e->nr_devs);
28
29         for (i = 0; i + 1 < e->nr_devs; i++)
30                 BUG_ON(e->devs[i] >= e->devs[i + 1]);
31 #endif
32 }
33
34 static void replicas_entry_sort(struct bch_replicas_entry *e)
35 {
36         bubble_sort(e->devs, e->nr_devs, u8_cmp);
37 }
38
39 static void bch2_cpu_replicas_sort(struct bch_replicas_cpu *r)
40 {
41         eytzinger0_sort(r->entries, r->nr, r->entry_size, memcmp, NULL);
42 }
43
44 void bch2_replicas_entry_to_text(struct printbuf *out,
45                                  struct bch_replicas_entry *e)
46 {
47         unsigned i;
48
49         pr_buf(out, "%s: %u/%u [",
50                bch2_data_types[e->data_type],
51                e->nr_required,
52                e->nr_devs);
53
54         for (i = 0; i < e->nr_devs; i++)
55                 pr_buf(out, i ? " %u" : "%u", e->devs[i]);
56         pr_buf(out, "]");
57 }
58
59 void bch2_cpu_replicas_to_text(struct printbuf *out,
60                               struct bch_replicas_cpu *r)
61 {
62         struct bch_replicas_entry *e;
63         bool first = true;
64
65         for_each_cpu_replicas_entry(r, e) {
66                 if (!first)
67                         pr_buf(out, " ");
68                 first = false;
69
70                 bch2_replicas_entry_to_text(out, e);
71         }
72 }
73
74 static void extent_to_replicas(struct bkey_s_c k,
75                                struct bch_replicas_entry *r)
76 {
77         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
78         const union bch_extent_entry *entry;
79         struct extent_ptr_decoded p;
80
81         r->nr_required  = 1;
82
83         bkey_for_each_ptr_decode(k.k, ptrs, p, entry) {
84                 if (p.ptr.cached)
85                         continue;
86
87                 if (!p.has_ec)
88                         r->devs[r->nr_devs++] = p.ptr.dev;
89                 else
90                         r->nr_required = 0;
91         }
92 }
93
94 static void stripe_to_replicas(struct bkey_s_c k,
95                                struct bch_replicas_entry *r)
96 {
97         struct bkey_s_c_stripe s = bkey_s_c_to_stripe(k);
98         const struct bch_extent_ptr *ptr;
99
100         r->nr_required  = s.v->nr_blocks - s.v->nr_redundant;
101
102         for (ptr = s.v->ptrs;
103              ptr < s.v->ptrs + s.v->nr_blocks;
104              ptr++)
105                 r->devs[r->nr_devs++] = ptr->dev;
106 }
107
108 void bch2_bkey_to_replicas(struct bch_replicas_entry *e,
109                            struct bkey_s_c k)
110 {
111         e->nr_devs = 0;
112
113         switch (k.k->type) {
114         case KEY_TYPE_btree_ptr:
115         case KEY_TYPE_btree_ptr_v2:
116                 e->data_type = BCH_DATA_btree;
117                 extent_to_replicas(k, e);
118                 break;
119         case KEY_TYPE_extent:
120         case KEY_TYPE_reflink_v:
121                 e->data_type = BCH_DATA_user;
122                 extent_to_replicas(k, e);
123                 break;
124         case KEY_TYPE_stripe:
125                 e->data_type = BCH_DATA_user;
126                 stripe_to_replicas(k, e);
127                 break;
128         }
129
130         replicas_entry_sort(e);
131 }
132
133 void bch2_devlist_to_replicas(struct bch_replicas_entry *e,
134                               enum bch_data_type data_type,
135                               struct bch_devs_list devs)
136 {
137         unsigned i;
138
139         BUG_ON(!data_type ||
140                data_type == BCH_DATA_sb ||
141                data_type >= BCH_DATA_NR);
142
143         e->data_type    = data_type;
144         e->nr_devs      = 0;
145         e->nr_required  = 1;
146
147         for (i = 0; i < devs.nr; i++)
148                 e->devs[e->nr_devs++] = devs.devs[i];
149
150         replicas_entry_sort(e);
151 }
152
153 static struct bch_replicas_cpu
154 cpu_replicas_add_entry(struct bch_replicas_cpu *old,
155                        struct bch_replicas_entry *new_entry)
156 {
157         unsigned i;
158         struct bch_replicas_cpu new = {
159                 .nr             = old->nr + 1,
160                 .entry_size     = max_t(unsigned, old->entry_size,
161                                         replicas_entry_bytes(new_entry)),
162         };
163
164         BUG_ON(!new_entry->data_type);
165         verify_replicas_entry(new_entry);
166
167         new.entries = kcalloc(new.nr, new.entry_size, GFP_NOIO);
168         if (!new.entries)
169                 return new;
170
171         for (i = 0; i < old->nr; i++)
172                 memcpy(cpu_replicas_entry(&new, i),
173                        cpu_replicas_entry(old, i),
174                        old->entry_size);
175
176         memcpy(cpu_replicas_entry(&new, old->nr),
177                new_entry,
178                replicas_entry_bytes(new_entry));
179
180         bch2_cpu_replicas_sort(&new);
181         return new;
182 }
183
184 static inline int __replicas_entry_idx(struct bch_replicas_cpu *r,
185                                        struct bch_replicas_entry *search)
186 {
187         int idx, entry_size = replicas_entry_bytes(search);
188
189         if (unlikely(entry_size > r->entry_size))
190                 return -1;
191
192         verify_replicas_entry(search);
193
194 #define entry_cmp(_l, _r, size) memcmp(_l, _r, entry_size)
195         idx = eytzinger0_find(r->entries, r->nr, r->entry_size,
196                               entry_cmp, search);
197 #undef entry_cmp
198
199         return idx < r->nr ? idx : -1;
200 }
201
202 int bch2_replicas_entry_idx(struct bch_fs *c,
203                             struct bch_replicas_entry *search)
204 {
205         replicas_entry_sort(search);
206
207         return __replicas_entry_idx(&c->replicas, search);
208 }
209
210 static bool __replicas_has_entry(struct bch_replicas_cpu *r,
211                                  struct bch_replicas_entry *search)
212 {
213         return __replicas_entry_idx(r, search) >= 0;
214 }
215
216 bool bch2_replicas_marked(struct bch_fs *c,
217                           struct bch_replicas_entry *search)
218 {
219         bool marked;
220
221         if (!search->nr_devs)
222                 return true;
223
224         verify_replicas_entry(search);
225
226         percpu_down_read(&c->mark_lock);
227         marked = __replicas_has_entry(&c->replicas, search) &&
228                 (likely((!c->replicas_gc.entries)) ||
229                  __replicas_has_entry(&c->replicas_gc, search));
230         percpu_up_read(&c->mark_lock);
231
232         return marked;
233 }
234
235 static void __replicas_table_update(struct bch_fs_usage *dst,
236                                     struct bch_replicas_cpu *dst_r,
237                                     struct bch_fs_usage *src,
238                                     struct bch_replicas_cpu *src_r)
239 {
240         int src_idx, dst_idx;
241
242         *dst = *src;
243
244         for (src_idx = 0; src_idx < src_r->nr; src_idx++) {
245                 if (!src->replicas[src_idx])
246                         continue;
247
248                 dst_idx = __replicas_entry_idx(dst_r,
249                                 cpu_replicas_entry(src_r, src_idx));
250                 BUG_ON(dst_idx < 0);
251
252                 dst->replicas[dst_idx] = src->replicas[src_idx];
253         }
254 }
255
256 static void __replicas_table_update_pcpu(struct bch_fs_usage __percpu *dst_p,
257                                     struct bch_replicas_cpu *dst_r,
258                                     struct bch_fs_usage __percpu *src_p,
259                                     struct bch_replicas_cpu *src_r)
260 {
261         unsigned src_nr = sizeof(struct bch_fs_usage) / sizeof(u64) + src_r->nr;
262         struct bch_fs_usage *dst, *src = (void *)
263                 bch2_acc_percpu_u64s((void *) src_p, src_nr);
264
265         preempt_disable();
266         dst = this_cpu_ptr(dst_p);
267         preempt_enable();
268
269         __replicas_table_update(dst, dst_r, src, src_r);
270 }
271
272 /*
273  * Resize filesystem accounting:
274  */
275 static int replicas_table_update(struct bch_fs *c,
276                                  struct bch_replicas_cpu *new_r)
277 {
278         struct bch_fs_usage __percpu *new_usage[2] = { NULL, NULL };
279         struct bch_fs_usage *new_scratch = NULL;
280         struct bch_fs_usage __percpu *new_gc = NULL;
281         struct bch_fs_usage *new_base = NULL;
282         unsigned bytes = sizeof(struct bch_fs_usage) +
283                 sizeof(u64) * new_r->nr;
284         int ret = -ENOMEM;
285
286         if (!(new_base = kzalloc(bytes, GFP_NOIO)) ||
287             !(new_usage[0] = __alloc_percpu_gfp(bytes, sizeof(u64),
288                                                 GFP_NOIO)) ||
289             !(new_usage[1] = __alloc_percpu_gfp(bytes, sizeof(u64),
290                                                 GFP_NOIO)) ||
291             !(new_scratch  = kmalloc(bytes, GFP_NOIO)) ||
292             (c->usage_gc &&
293              !(new_gc = __alloc_percpu_gfp(bytes, sizeof(u64), GFP_NOIO)))) {
294                 bch_err(c, "error updating replicas table: memory allocation failure");
295                 goto err;
296         }
297
298         if (c->usage_base)
299                 __replicas_table_update(new_base,               new_r,
300                                         c->usage_base,          &c->replicas);
301         if (c->usage[0])
302                 __replicas_table_update_pcpu(new_usage[0],      new_r,
303                                              c->usage[0],       &c->replicas);
304         if (c->usage[1])
305                 __replicas_table_update_pcpu(new_usage[1],      new_r,
306                                              c->usage[1],       &c->replicas);
307         if (c->usage_gc)
308                 __replicas_table_update_pcpu(new_gc,            new_r,
309                                              c->usage_gc,       &c->replicas);
310
311         swap(c->usage_base,     new_base);
312         swap(c->usage[0],       new_usage[0]);
313         swap(c->usage[1],       new_usage[1]);
314         swap(c->usage_scratch,  new_scratch);
315         swap(c->usage_gc,       new_gc);
316         swap(c->replicas,       *new_r);
317         ret = 0;
318 err:
319         free_percpu(new_gc);
320         kfree(new_scratch);
321         free_percpu(new_usage[1]);
322         free_percpu(new_usage[0]);
323         kfree(new_base);
324         return ret;
325 }
326
327 static unsigned reserve_journal_replicas(struct bch_fs *c,
328                                      struct bch_replicas_cpu *r)
329 {
330         struct bch_replicas_entry *e;
331         unsigned journal_res_u64s = 0;
332
333         /* nr_inodes: */
334         journal_res_u64s +=
335                 DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64));
336
337         /* key_version: */
338         journal_res_u64s +=
339                 DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64));
340
341         /* persistent_reserved: */
342         journal_res_u64s +=
343                 DIV_ROUND_UP(sizeof(struct jset_entry_usage), sizeof(u64)) *
344                 BCH_REPLICAS_MAX;
345
346         for_each_cpu_replicas_entry(r, e)
347                 journal_res_u64s +=
348                         DIV_ROUND_UP(sizeof(struct jset_entry_data_usage) +
349                                      e->nr_devs, sizeof(u64));
350         return journal_res_u64s;
351 }
352
353 noinline
354 static int bch2_mark_replicas_slowpath(struct bch_fs *c,
355                                 struct bch_replicas_entry *new_entry)
356 {
357         struct bch_replicas_cpu new_r, new_gc;
358         int ret = 0;
359
360         verify_replicas_entry(new_entry);
361
362         memset(&new_r, 0, sizeof(new_r));
363         memset(&new_gc, 0, sizeof(new_gc));
364
365         mutex_lock(&c->sb_lock);
366
367         if (c->replicas_gc.entries &&
368             !__replicas_has_entry(&c->replicas_gc, new_entry)) {
369                 new_gc = cpu_replicas_add_entry(&c->replicas_gc, new_entry);
370                 if (!new_gc.entries)
371                         goto err;
372         }
373
374         if (!__replicas_has_entry(&c->replicas, new_entry)) {
375                 new_r = cpu_replicas_add_entry(&c->replicas, new_entry);
376                 if (!new_r.entries)
377                         goto err;
378
379                 ret = bch2_cpu_replicas_to_sb_replicas(c, &new_r);
380                 if (ret)
381                         goto err;
382
383                 bch2_journal_entry_res_resize(&c->journal,
384                                 &c->replicas_journal_res,
385                                 reserve_journal_replicas(c, &new_r));
386         }
387
388         if (!new_r.entries &&
389             !new_gc.entries)
390                 goto out;
391
392         /* allocations done, now commit: */
393
394         if (new_r.entries)
395                 bch2_write_super(c);
396
397         /* don't update in memory replicas until changes are persistent */
398         percpu_down_write(&c->mark_lock);
399         if (new_r.entries)
400                 ret = replicas_table_update(c, &new_r);
401         if (new_gc.entries)
402                 swap(new_gc, c->replicas_gc);
403         percpu_up_write(&c->mark_lock);
404 out:
405         mutex_unlock(&c->sb_lock);
406
407         kfree(new_r.entries);
408         kfree(new_gc.entries);
409
410         return ret;
411 err:
412         bch_err(c, "error adding replicas entry: memory allocation failure");
413         ret = -ENOMEM;
414         goto out;
415 }
416
417 static int __bch2_mark_replicas(struct bch_fs *c,
418                                 struct bch_replicas_entry *r,
419                                 bool check)
420 {
421         return likely(bch2_replicas_marked(c, r))       ? 0
422                 : check                                 ? -1
423                 : bch2_mark_replicas_slowpath(c, r);
424 }
425
426 int bch2_mark_replicas(struct bch_fs *c, struct bch_replicas_entry *r)
427 {
428         return __bch2_mark_replicas(c, r, false);
429 }
430
431 static int __bch2_mark_bkey_replicas(struct bch_fs *c, struct bkey_s_c k,
432                                      bool check)
433 {
434         struct bch_replicas_padded search;
435         struct bch_devs_list cached = bch2_bkey_cached_devs(k);
436         unsigned i;
437         int ret;
438
439         for (i = 0; i < cached.nr; i++) {
440                 bch2_replicas_entry_cached(&search.e, cached.devs[i]);
441
442                 ret = __bch2_mark_replicas(c, &search.e, check);
443                 if (ret)
444                         return ret;
445         }
446
447         bch2_bkey_to_replicas(&search.e, k);
448
449         return __bch2_mark_replicas(c, &search.e, check);
450 }
451
452 bool bch2_bkey_replicas_marked(struct bch_fs *c,
453                                struct bkey_s_c k)
454 {
455         return __bch2_mark_bkey_replicas(c, k, true) == 0;
456 }
457
458 int bch2_mark_bkey_replicas(struct bch_fs *c, struct bkey_s_c k)
459 {
460         return __bch2_mark_bkey_replicas(c, k, false);
461 }
462
463 int bch2_replicas_gc_end(struct bch_fs *c, int ret)
464 {
465         unsigned i;
466
467         lockdep_assert_held(&c->replicas_gc_lock);
468
469         mutex_lock(&c->sb_lock);
470         percpu_down_write(&c->mark_lock);
471
472         /*
473          * this is kind of crappy; the replicas gc mechanism needs to be ripped
474          * out
475          */
476
477         for (i = 0; i < c->replicas.nr; i++) {
478                 struct bch_replicas_entry *e =
479                         cpu_replicas_entry(&c->replicas, i);
480                 struct bch_replicas_cpu n;
481
482                 if (!__replicas_has_entry(&c->replicas_gc, e) &&
483                     (c->usage_base->replicas[i] ||
484                      percpu_u64_get(&c->usage[0]->replicas[i]) ||
485                      percpu_u64_get(&c->usage[1]->replicas[i]))) {
486                         n = cpu_replicas_add_entry(&c->replicas_gc, e);
487                         if (!n.entries) {
488                                 ret = -ENOSPC;
489                                 goto err;
490                         }
491
492                         swap(n, c->replicas_gc);
493                         kfree(n.entries);
494                 }
495         }
496
497         if (bch2_cpu_replicas_to_sb_replicas(c, &c->replicas_gc)) {
498                 ret = -ENOSPC;
499                 goto err;
500         }
501
502         ret = replicas_table_update(c, &c->replicas_gc);
503 err:
504         kfree(c->replicas_gc.entries);
505         c->replicas_gc.entries = NULL;
506
507         percpu_up_write(&c->mark_lock);
508
509         if (!ret)
510                 bch2_write_super(c);
511
512         mutex_unlock(&c->sb_lock);
513
514         return ret;
515 }
516
517 int bch2_replicas_gc_start(struct bch_fs *c, unsigned typemask)
518 {
519         struct bch_replicas_entry *e;
520         unsigned i = 0;
521
522         lockdep_assert_held(&c->replicas_gc_lock);
523
524         mutex_lock(&c->sb_lock);
525         BUG_ON(c->replicas_gc.entries);
526
527         c->replicas_gc.nr               = 0;
528         c->replicas_gc.entry_size       = 0;
529
530         for_each_cpu_replicas_entry(&c->replicas, e)
531                 if (!((1 << e->data_type) & typemask)) {
532                         c->replicas_gc.nr++;
533                         c->replicas_gc.entry_size =
534                                 max_t(unsigned, c->replicas_gc.entry_size,
535                                       replicas_entry_bytes(e));
536                 }
537
538         c->replicas_gc.entries = kcalloc(c->replicas_gc.nr,
539                                          c->replicas_gc.entry_size,
540                                          GFP_NOIO);
541         if (!c->replicas_gc.entries) {
542                 mutex_unlock(&c->sb_lock);
543                 bch_err(c, "error allocating c->replicas_gc");
544                 return -ENOMEM;
545         }
546
547         for_each_cpu_replicas_entry(&c->replicas, e)
548                 if (!((1 << e->data_type) & typemask))
549                         memcpy(cpu_replicas_entry(&c->replicas_gc, i++),
550                                e, c->replicas_gc.entry_size);
551
552         bch2_cpu_replicas_sort(&c->replicas_gc);
553         mutex_unlock(&c->sb_lock);
554
555         return 0;
556 }
557
558 int bch2_replicas_gc2(struct bch_fs *c)
559 {
560         struct bch_replicas_cpu new = { 0 };
561         unsigned i, nr;
562         int ret = 0;
563
564         bch2_journal_meta(&c->journal);
565 retry:
566         nr              = READ_ONCE(c->replicas.nr);
567         new.entry_size  = READ_ONCE(c->replicas.entry_size);
568         new.entries     = kcalloc(nr, new.entry_size, GFP_KERNEL);
569         if (!new.entries) {
570                 bch_err(c, "error allocating c->replicas_gc");
571                 return -ENOMEM;
572         }
573
574         mutex_lock(&c->sb_lock);
575         percpu_down_write(&c->mark_lock);
576
577         if (nr                  != c->replicas.nr ||
578             new.entry_size      != c->replicas.entry_size) {
579                 percpu_up_write(&c->mark_lock);
580                 mutex_unlock(&c->sb_lock);
581                 kfree(new.entries);
582                 goto retry;
583         }
584
585         for (i = 0; i < c->replicas.nr; i++) {
586                 struct bch_replicas_entry *e =
587                         cpu_replicas_entry(&c->replicas, i);
588
589                 if (e->data_type == BCH_DATA_journal ||
590                     c->usage_base->replicas[i] ||
591                     percpu_u64_get(&c->usage[0]->replicas[i]) ||
592                     percpu_u64_get(&c->usage[1]->replicas[i]))
593                         memcpy(cpu_replicas_entry(&new, new.nr++),
594                                e, new.entry_size);
595         }
596
597         bch2_cpu_replicas_sort(&new);
598
599         if (bch2_cpu_replicas_to_sb_replicas(c, &new)) {
600                 ret = -ENOSPC;
601                 goto err;
602         }
603
604         ret = replicas_table_update(c, &new);
605 err:
606         kfree(new.entries);
607
608         percpu_up_write(&c->mark_lock);
609
610         if (!ret)
611                 bch2_write_super(c);
612
613         mutex_unlock(&c->sb_lock);
614
615         return ret;
616 }
617
618 int bch2_replicas_set_usage(struct bch_fs *c,
619                             struct bch_replicas_entry *r,
620                             u64 sectors)
621 {
622         int ret, idx = bch2_replicas_entry_idx(c, r);
623
624         if (idx < 0) {
625                 struct bch_replicas_cpu n;
626
627                 n = cpu_replicas_add_entry(&c->replicas, r);
628                 if (!n.entries)
629                         return -ENOMEM;
630
631                 ret = replicas_table_update(c, &n);
632                 if (ret)
633                         return ret;
634
635                 kfree(n.entries);
636
637                 idx = bch2_replicas_entry_idx(c, r);
638                 BUG_ON(ret < 0);
639         }
640
641         c->usage_base->replicas[idx] = sectors;
642
643         return 0;
644 }
645
646 /* Replicas tracking - superblock: */
647
648 static int
649 __bch2_sb_replicas_to_cpu_replicas(struct bch_sb_field_replicas *sb_r,
650                                    struct bch_replicas_cpu *cpu_r)
651 {
652         struct bch_replicas_entry *e, *dst;
653         unsigned nr = 0, entry_size = 0, idx = 0;
654
655         for_each_replicas_entry(sb_r, e) {
656                 entry_size = max_t(unsigned, entry_size,
657                                    replicas_entry_bytes(e));
658                 nr++;
659         }
660
661         cpu_r->entries = kcalloc(nr, entry_size, GFP_NOIO);
662         if (!cpu_r->entries)
663                 return -ENOMEM;
664
665         cpu_r->nr               = nr;
666         cpu_r->entry_size       = entry_size;
667
668         for_each_replicas_entry(sb_r, e) {
669                 dst = cpu_replicas_entry(cpu_r, idx++);
670                 memcpy(dst, e, replicas_entry_bytes(e));
671                 replicas_entry_sort(dst);
672         }
673
674         return 0;
675 }
676
677 static int
678 __bch2_sb_replicas_v0_to_cpu_replicas(struct bch_sb_field_replicas_v0 *sb_r,
679                                       struct bch_replicas_cpu *cpu_r)
680 {
681         struct bch_replicas_entry_v0 *e;
682         unsigned nr = 0, entry_size = 0, idx = 0;
683
684         for_each_replicas_entry(sb_r, e) {
685                 entry_size = max_t(unsigned, entry_size,
686                                    replicas_entry_bytes(e));
687                 nr++;
688         }
689
690         entry_size += sizeof(struct bch_replicas_entry) -
691                 sizeof(struct bch_replicas_entry_v0);
692
693         cpu_r->entries = kcalloc(nr, entry_size, GFP_NOIO);
694         if (!cpu_r->entries)
695                 return -ENOMEM;
696
697         cpu_r->nr               = nr;
698         cpu_r->entry_size       = entry_size;
699
700         for_each_replicas_entry(sb_r, e) {
701                 struct bch_replicas_entry *dst =
702                         cpu_replicas_entry(cpu_r, idx++);
703
704                 dst->data_type  = e->data_type;
705                 dst->nr_devs    = e->nr_devs;
706                 dst->nr_required = 1;
707                 memcpy(dst->devs, e->devs, e->nr_devs);
708                 replicas_entry_sort(dst);
709         }
710
711         return 0;
712 }
713
714 int bch2_sb_replicas_to_cpu_replicas(struct bch_fs *c)
715 {
716         struct bch_sb_field_replicas *sb_v1;
717         struct bch_sb_field_replicas_v0 *sb_v0;
718         struct bch_replicas_cpu new_r = { 0, 0, NULL };
719         int ret = 0;
720
721         if ((sb_v1 = bch2_sb_get_replicas(c->disk_sb.sb)))
722                 ret = __bch2_sb_replicas_to_cpu_replicas(sb_v1, &new_r);
723         else if ((sb_v0 = bch2_sb_get_replicas_v0(c->disk_sb.sb)))
724                 ret = __bch2_sb_replicas_v0_to_cpu_replicas(sb_v0, &new_r);
725
726         if (ret)
727                 return -ENOMEM;
728
729         bch2_cpu_replicas_sort(&new_r);
730
731         percpu_down_write(&c->mark_lock);
732
733         ret = replicas_table_update(c, &new_r);
734         percpu_up_write(&c->mark_lock);
735
736         kfree(new_r.entries);
737
738         return 0;
739 }
740
741 static int bch2_cpu_replicas_to_sb_replicas_v0(struct bch_fs *c,
742                                                struct bch_replicas_cpu *r)
743 {
744         struct bch_sb_field_replicas_v0 *sb_r;
745         struct bch_replicas_entry_v0 *dst;
746         struct bch_replicas_entry *src;
747         size_t bytes;
748
749         bytes = sizeof(struct bch_sb_field_replicas);
750
751         for_each_cpu_replicas_entry(r, src)
752                 bytes += replicas_entry_bytes(src) - 1;
753
754         sb_r = bch2_sb_resize_replicas_v0(&c->disk_sb,
755                         DIV_ROUND_UP(bytes, sizeof(u64)));
756         if (!sb_r)
757                 return -ENOSPC;
758
759         bch2_sb_field_delete(&c->disk_sb, BCH_SB_FIELD_replicas);
760         sb_r = bch2_sb_get_replicas_v0(c->disk_sb.sb);
761
762         memset(&sb_r->entries, 0,
763                vstruct_end(&sb_r->field) -
764                (void *) &sb_r->entries);
765
766         dst = sb_r->entries;
767         for_each_cpu_replicas_entry(r, src) {
768                 dst->data_type  = src->data_type;
769                 dst->nr_devs    = src->nr_devs;
770                 memcpy(dst->devs, src->devs, src->nr_devs);
771
772                 dst = replicas_entry_next(dst);
773
774                 BUG_ON((void *) dst > vstruct_end(&sb_r->field));
775         }
776
777         return 0;
778 }
779
780 static int bch2_cpu_replicas_to_sb_replicas(struct bch_fs *c,
781                                             struct bch_replicas_cpu *r)
782 {
783         struct bch_sb_field_replicas *sb_r;
784         struct bch_replicas_entry *dst, *src;
785         bool need_v1 = false;
786         size_t bytes;
787
788         bytes = sizeof(struct bch_sb_field_replicas);
789
790         for_each_cpu_replicas_entry(r, src) {
791                 bytes += replicas_entry_bytes(src);
792                 if (src->nr_required != 1)
793                         need_v1 = true;
794         }
795
796         if (!need_v1)
797                 return bch2_cpu_replicas_to_sb_replicas_v0(c, r);
798
799         sb_r = bch2_sb_resize_replicas(&c->disk_sb,
800                         DIV_ROUND_UP(bytes, sizeof(u64)));
801         if (!sb_r)
802                 return -ENOSPC;
803
804         bch2_sb_field_delete(&c->disk_sb, BCH_SB_FIELD_replicas_v0);
805         sb_r = bch2_sb_get_replicas(c->disk_sb.sb);
806
807         memset(&sb_r->entries, 0,
808                vstruct_end(&sb_r->field) -
809                (void *) &sb_r->entries);
810
811         dst = sb_r->entries;
812         for_each_cpu_replicas_entry(r, src) {
813                 memcpy(dst, src, replicas_entry_bytes(src));
814
815                 dst = replicas_entry_next(dst);
816
817                 BUG_ON((void *) dst > vstruct_end(&sb_r->field));
818         }
819
820         return 0;
821 }
822
823 static const char *check_dup_replicas_entries(struct bch_replicas_cpu *cpu_r)
824 {
825         unsigned i;
826
827         sort_cmp_size(cpu_r->entries,
828                       cpu_r->nr,
829                       cpu_r->entry_size,
830                       memcmp, NULL);
831
832         for (i = 0; i + 1 < cpu_r->nr; i++) {
833                 struct bch_replicas_entry *l =
834                         cpu_replicas_entry(cpu_r, i);
835                 struct bch_replicas_entry *r =
836                         cpu_replicas_entry(cpu_r, i + 1);
837
838                 BUG_ON(memcmp(l, r, cpu_r->entry_size) > 0);
839
840                 if (!memcmp(l, r, cpu_r->entry_size))
841                         return "duplicate replicas entry";
842         }
843
844         return NULL;
845 }
846
847 static const char *bch2_sb_validate_replicas(struct bch_sb *sb, struct bch_sb_field *f)
848 {
849         struct bch_sb_field_replicas *sb_r = field_to_type(f, replicas);
850         struct bch_sb_field_members *mi = bch2_sb_get_members(sb);
851         struct bch_replicas_cpu cpu_r = { .entries = NULL };
852         struct bch_replicas_entry *e;
853         const char *err;
854         unsigned i;
855
856         for_each_replicas_entry(sb_r, e) {
857                 err = "invalid replicas entry: invalid data type";
858                 if (e->data_type >= BCH_DATA_NR)
859                         goto err;
860
861                 err = "invalid replicas entry: no devices";
862                 if (!e->nr_devs)
863                         goto err;
864
865                 err = "invalid replicas entry: bad nr_required";
866                 if (e->nr_required > 1 &&
867                     e->nr_required >= e->nr_devs)
868                         goto err;
869
870                 err = "invalid replicas entry: invalid device";
871                 for (i = 0; i < e->nr_devs; i++)
872                         if (!bch2_dev_exists(sb, mi, e->devs[i]))
873                                 goto err;
874         }
875
876         err = "cannot allocate memory";
877         if (__bch2_sb_replicas_to_cpu_replicas(sb_r, &cpu_r))
878                 goto err;
879
880         err = check_dup_replicas_entries(&cpu_r);
881 err:
882         kfree(cpu_r.entries);
883         return err;
884 }
885
886 static void bch2_sb_replicas_to_text(struct printbuf *out,
887                                      struct bch_sb *sb,
888                                      struct bch_sb_field *f)
889 {
890         struct bch_sb_field_replicas *r = field_to_type(f, replicas);
891         struct bch_replicas_entry *e;
892         bool first = true;
893
894         for_each_replicas_entry(r, e) {
895                 if (!first)
896                         pr_buf(out, " ");
897                 first = false;
898
899                 bch2_replicas_entry_to_text(out, e);
900         }
901 }
902
903 const struct bch_sb_field_ops bch_sb_field_ops_replicas = {
904         .validate       = bch2_sb_validate_replicas,
905         .to_text        = bch2_sb_replicas_to_text,
906 };
907
908 static const char *bch2_sb_validate_replicas_v0(struct bch_sb *sb, struct bch_sb_field *f)
909 {
910         struct bch_sb_field_replicas_v0 *sb_r = field_to_type(f, replicas_v0);
911         struct bch_sb_field_members *mi = bch2_sb_get_members(sb);
912         struct bch_replicas_cpu cpu_r = { .entries = NULL };
913         struct bch_replicas_entry_v0 *e;
914         const char *err;
915         unsigned i;
916
917         for_each_replicas_entry_v0(sb_r, e) {
918                 err = "invalid replicas entry: invalid data type";
919                 if (e->data_type >= BCH_DATA_NR)
920                         goto err;
921
922                 err = "invalid replicas entry: no devices";
923                 if (!e->nr_devs)
924                         goto err;
925
926                 err = "invalid replicas entry: invalid device";
927                 for (i = 0; i < e->nr_devs; i++)
928                         if (!bch2_dev_exists(sb, mi, e->devs[i]))
929                                 goto err;
930         }
931
932         err = "cannot allocate memory";
933         if (__bch2_sb_replicas_v0_to_cpu_replicas(sb_r, &cpu_r))
934                 goto err;
935
936         err = check_dup_replicas_entries(&cpu_r);
937 err:
938         kfree(cpu_r.entries);
939         return err;
940 }
941
942 const struct bch_sb_field_ops bch_sb_field_ops_replicas_v0 = {
943         .validate       = bch2_sb_validate_replicas_v0,
944 };
945
946 /* Query replicas: */
947
948 struct replicas_status __bch2_replicas_status(struct bch_fs *c,
949                                               struct bch_devs_mask online_devs)
950 {
951         struct bch_sb_field_members *mi;
952         struct bch_replicas_entry *e;
953         unsigned i, nr_online, nr_offline;
954         struct replicas_status ret;
955
956         memset(&ret, 0, sizeof(ret));
957
958         for (i = 0; i < ARRAY_SIZE(ret.replicas); i++)
959                 ret.replicas[i].redundancy = INT_MAX;
960
961         mi = bch2_sb_get_members(c->disk_sb.sb);
962
963         percpu_down_read(&c->mark_lock);
964
965         for_each_cpu_replicas_entry(&c->replicas, e) {
966                 if (e->data_type >= ARRAY_SIZE(ret.replicas))
967                         panic("e %p data_type %u\n", e, e->data_type);
968
969                 nr_online = nr_offline = 0;
970
971                 for (i = 0; i < e->nr_devs; i++) {
972                         BUG_ON(!bch2_dev_exists(c->disk_sb.sb, mi,
973                                                 e->devs[i]));
974
975                         if (test_bit(e->devs[i], online_devs.d))
976                                 nr_online++;
977                         else
978                                 nr_offline++;
979                 }
980
981                 ret.replicas[e->data_type].redundancy =
982                         min(ret.replicas[e->data_type].redundancy,
983                             (int) nr_online - (int) e->nr_required);
984
985                 ret.replicas[e->data_type].nr_offline =
986                         max(ret.replicas[e->data_type].nr_offline,
987                             nr_offline);
988         }
989
990         percpu_up_read(&c->mark_lock);
991
992         for (i = 0; i < ARRAY_SIZE(ret.replicas); i++)
993                 if (ret.replicas[i].redundancy == INT_MAX)
994                         ret.replicas[i].redundancy = 0;
995
996         return ret;
997 }
998
999 struct replicas_status bch2_replicas_status(struct bch_fs *c)
1000 {
1001         return __bch2_replicas_status(c, bch2_online_devs(c));
1002 }
1003
1004 static bool have_enough_devs(struct replicas_status s,
1005                              enum bch_data_type type,
1006                              bool force_if_degraded,
1007                              bool force_if_lost)
1008 {
1009         return (!s.replicas[type].nr_offline || force_if_degraded) &&
1010                 (s.replicas[type].redundancy >= 0 || force_if_lost);
1011 }
1012
1013 bool bch2_have_enough_devs(struct replicas_status s, unsigned flags)
1014 {
1015         return (have_enough_devs(s, BCH_DATA_journal,
1016                                  flags & BCH_FORCE_IF_METADATA_DEGRADED,
1017                                  flags & BCH_FORCE_IF_METADATA_LOST) &&
1018                 have_enough_devs(s, BCH_DATA_btree,
1019                                  flags & BCH_FORCE_IF_METADATA_DEGRADED,
1020                                  flags & BCH_FORCE_IF_METADATA_LOST) &&
1021                 have_enough_devs(s, BCH_DATA_user,
1022                                  flags & BCH_FORCE_IF_DATA_DEGRADED,
1023                                  flags & BCH_FORCE_IF_DATA_LOST));
1024 }
1025
1026 int bch2_replicas_online(struct bch_fs *c, bool meta)
1027 {
1028         struct replicas_status s = bch2_replicas_status(c);
1029
1030         return (meta
1031                 ? min(s.replicas[BCH_DATA_journal].redundancy,
1032                       s.replicas[BCH_DATA_btree].redundancy)
1033                 : s.replicas[BCH_DATA_user].redundancy) + 1;
1034 }
1035
1036 unsigned bch2_dev_has_data(struct bch_fs *c, struct bch_dev *ca)
1037 {
1038         struct bch_replicas_entry *e;
1039         unsigned i, ret = 0;
1040
1041         percpu_down_read(&c->mark_lock);
1042
1043         for_each_cpu_replicas_entry(&c->replicas, e)
1044                 for (i = 0; i < e->nr_devs; i++)
1045                         if (e->devs[i] == ca->dev_idx)
1046                                 ret |= 1 << e->data_type;
1047
1048         percpu_up_read(&c->mark_lock);
1049
1050         return ret;
1051 }
1052
1053 int bch2_fs_replicas_init(struct bch_fs *c)
1054 {
1055         c->journal.entry_u64s_reserved +=
1056                 reserve_journal_replicas(c, &c->replicas);
1057
1058         return replicas_table_update(c, &c->replicas);
1059 }