git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/recovery.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "bkey_buf.h"
   5 #include "alloc_background.h"
   6 #include "btree_gc.h"
   7 #include "btree_update.h"
   8 #include "btree_update_interior.h"
   9 #include "btree_io.h"
  10 #include "buckets.h"
  11 #include "dirent.h"
  12 #include "ec.h"
  13 #include "error.h"
  14 #include "fs-common.h"
  15 #include "fsck.h"
  16 #include "journal_io.h"
  17 #include "journal_reclaim.h"
  18 #include "journal_seq_blacklist.h"
  19 #include "move.h"
  20 #include "quota.h"
  21 #include "recovery.h"
  22 #include "replicas.h"
  23 #include "super-io.h"
  24
  25 #include <linux/sort.h>
  26 #include <linux/stat.h>
  27
  28 #define QSTR(n) { { { .len = strlen(n) } }, .name = n }
  29
  30 /* for -o reconstruct_alloc: */
  31 static void drop_alloc_keys(struct journal_keys *keys)
  32 {
  33         size_t src, dst;
  34
  35         for (src = 0, dst = 0; src < keys->nr; src++)
  36                 if (keys->d[src].btree_id != BTREE_ID_alloc)
  37                         keys->d[dst++] = keys->d[src];
  38
  39         keys->nr = dst;
  40 }
  41
  42 /*
  43  * Btree node pointers have a field to stack a pointer to the in memory btree
  44  * node; we need to zero out this field when reading in btree nodes, or when
  45  * reading in keys from the journal:
  46  */
  47 static void zero_out_btree_mem_ptr(struct journal_keys *keys)
  48 {
  49         struct journal_key *i;
  50
  51         for (i = keys->d; i < keys->d + keys->nr; i++)
  52                 if (i->k->k.type == KEY_TYPE_btree_ptr_v2)
  53                         bkey_i_to_btree_ptr_v2(i->k)->v.mem_ptr = 0;
  54 }
  55
  56 /* iterate over keys read from the journal: */
  57
  58 static int __journal_key_cmp(enum btree_id      l_btree_id,
  59                              unsigned           l_level,
  60                              struct bpos        l_pos,
  61                              struct journal_key *r)
  62 {
  63         return (cmp_int(l_btree_id,     r->btree_id) ?:
  64                 cmp_int(l_level,        r->level) ?:
  65                 bpos_cmp(l_pos, r->k->k.p));
  66 }
  67
  68 static int journal_key_cmp(struct journal_key *l, struct journal_key *r)
  69 {
  70         return (cmp_int(l->btree_id,    r->btree_id) ?:
  71                 cmp_int(l->level,       r->level) ?:
  72                 bpos_cmp(l->k->k.p,     r->k->k.p));
  73 }
  74
  75 static size_t journal_key_search(struct journal_keys *journal_keys,
  76                                  enum btree_id id, unsigned level,
  77                                  struct bpos pos)
  78 {
  79         size_t l = 0, r = journal_keys->nr, m;
  80
  81         while (l < r) {
  82                 m = l + ((r - l) >> 1);
  83                 if (__journal_key_cmp(id, level, pos, &journal_keys->d[m]) > 0)
  84                         l = m + 1;
  85                 else
  86                         r = m;
  87         }
  88
  89         BUG_ON(l < journal_keys->nr &&
  90                __journal_key_cmp(id, level, pos, &journal_keys->d[l]) > 0);
  91
  92         BUG_ON(l &&
  93                __journal_key_cmp(id, level, pos, &journal_keys->d[l - 1]) <= 0);
  94
  95         return l;
  96 }
  97
  98 static void journal_iter_fix(struct bch_fs *c, struct journal_iter *iter, unsigned idx)
  99 {
 100         struct bkey_i *n = iter->keys->d[idx].k;
 101         struct btree_and_journal_iter *biter =
 102                 container_of(iter, struct btree_and_journal_iter, journal);
 103
 104         if (iter->idx > idx ||
 105             (iter->idx == idx &&
 106              biter->last &&
 107              bpos_cmp(n->k.p, biter->unpacked.p) <= 0))
 108                 iter->idx++;
 109 }
 110
 111 int bch2_journal_key_insert(struct bch_fs *c, enum btree_id id,
 112                             unsigned level, struct bkey_i *k)
 113 {
 114         struct journal_key n = {
 115                 .btree_id       = id,
 116                 .level          = level,
 117                 .k              = k,
 118                 .allocated      = true
 119         };
 120         struct journal_keys *keys = &c->journal_keys;
 121         struct journal_iter *iter;
 122         unsigned idx = journal_key_search(keys, id, level, k->k.p);
 123
 124         if (idx < keys->nr &&
 125             journal_key_cmp(&n, &keys->d[idx]) == 0) {
 126                 if (keys->d[idx].allocated)
 127                         kfree(keys->d[idx].k);
 128                 keys->d[idx] = n;
 129                 return 0;
 130         }
 131
 132         if (keys->nr == keys->size) {
 133                 struct journal_keys new_keys = {
 134                         .nr                     = keys->nr,
 135                         .size                   = keys->size * 2,
 136                         .journal_seq_base       = keys->journal_seq_base,
 137                 };
 138
 139                 new_keys.d = kvmalloc(sizeof(new_keys.d[0]) * new_keys.size, GFP_KERNEL);
 140                 if (!new_keys.d) {
 141                         bch_err(c, "%s: error allocating new key array (size %zu)",
 142                                 __func__, new_keys.size);
 143                         return -ENOMEM;
 144                 }
 145
 146                 memcpy(new_keys.d, keys->d, sizeof(keys->d[0]) * keys->nr);
 147                 kvfree(keys->d);
 148                 *keys = new_keys;
 149         }
 150
 151         array_insert_item(keys->d, keys->nr, idx, n);
 152
 153         list_for_each_entry(iter, &c->journal_iters, list)
 154                 journal_iter_fix(c, iter, idx);
 155
 156         return 0;
 157 }
 158
 159 int bch2_journal_key_delete(struct bch_fs *c, enum btree_id id,
 160                             unsigned level, struct bpos pos)
 161 {
 162         struct bkey_i *whiteout =
 163                 kmalloc(sizeof(struct bkey), GFP_KERNEL);
 164         int ret;
 165
 166         if (!whiteout) {
 167                 bch_err(c, "%s: error allocating new key", __func__);
 168                 return -ENOMEM;
 169         }
 170
 171         bkey_init(&whiteout->k);
 172         whiteout->k.p = pos;
 173
 174         ret = bch2_journal_key_insert(c, id, level, whiteout);
 175         if (ret)
 176                 kfree(whiteout);
 177         return ret;
 178 }
 179
 180 static struct bkey_i *bch2_journal_iter_peek(struct journal_iter *iter)
 181 {
 182         struct journal_key *k = iter->idx - iter->keys->nr
 183                 ? iter->keys->d + iter->idx : NULL;
 184
 185         if (k &&
 186             k->btree_id == iter->btree_id &&
 187             k->level    == iter->level)
 188                 return k->k;
 189
 190         iter->idx = iter->keys->nr;
 191         return NULL;
 192 }
 193
 194 static void bch2_journal_iter_advance(struct journal_iter *iter)
 195 {
 196         if (iter->idx < iter->keys->nr)
 197                 iter->idx++;
 198 }
 199
 200 static void bch2_journal_iter_exit(struct journal_iter *iter)
 201 {
 202         list_del(&iter->list);
 203 }
 204
 205 static void bch2_journal_iter_init(struct bch_fs *c,
 206                                    struct journal_iter *iter,
 207                                    enum btree_id id, unsigned level,
 208                                    struct bpos pos)
 209 {
 210         iter->btree_id  = id;
 211         iter->level     = level;
 212         iter->keys      = &c->journal_keys;
 213         iter->idx       = journal_key_search(&c->journal_keys, id, level, pos);
 214         list_add(&iter->list, &c->journal_iters);
 215 }
 216
 217 static struct bkey_s_c bch2_journal_iter_peek_btree(struct btree_and_journal_iter *iter)
 218 {
 219         return bch2_btree_node_iter_peek_unpack(&iter->node_iter,
 220                                                 iter->b, &iter->unpacked);
 221 }
 222
 223 static void bch2_journal_iter_advance_btree(struct btree_and_journal_iter *iter)
 224 {
 225         bch2_btree_node_iter_advance(&iter->node_iter, iter->b);
 226 }
 227
 228 void bch2_btree_and_journal_iter_advance(struct btree_and_journal_iter *iter)
 229 {
 230         switch (iter->last) {
 231         case none:
 232                 break;
 233         case btree:
 234                 bch2_journal_iter_advance_btree(iter);
 235                 break;
 236         case journal:
 237                 bch2_journal_iter_advance(&iter->journal);
 238                 break;
 239         }
 240
 241         iter->last = none;
 242 }
 243
 244 struct bkey_s_c bch2_btree_and_journal_iter_peek(struct btree_and_journal_iter *iter)
 245 {
 246         struct bkey_s_c ret;
 247
 248         while (1) {
 249                 struct bkey_s_c btree_k         =
 250                         bch2_journal_iter_peek_btree(iter);
 251                 struct bkey_s_c journal_k       =
 252                         bkey_i_to_s_c(bch2_journal_iter_peek(&iter->journal));
 253
 254                 if (btree_k.k && journal_k.k) {
 255                         int cmp = bpos_cmp(btree_k.k->p, journal_k.k->p);
 256
 257                         if (!cmp)
 258                                 bch2_journal_iter_advance_btree(iter);
 259
 260                         iter->last = cmp < 0 ? btree : journal;
 261                 } else if (btree_k.k) {
 262                         iter->last = btree;
 263                 } else if (journal_k.k) {
 264                         iter->last = journal;
 265                 } else {
 266                         iter->last = none;
 267                         return bkey_s_c_null;
 268                 }
 269
 270                 ret = iter->last == journal ? journal_k : btree_k;
 271
 272                 if (iter->b &&
 273                     bpos_cmp(ret.k->p, iter->b->data->max_key) > 0) {
 274                         iter->journal.idx = iter->journal.keys->nr;
 275                         iter->last = none;
 276                         return bkey_s_c_null;
 277                 }
 278
 279                 if (!bkey_deleted(ret.k))
 280                         break;
 281
 282                 bch2_btree_and_journal_iter_advance(iter);
 283         }
 284
 285         return ret;
 286 }
 287
 288 struct bkey_s_c bch2_btree_and_journal_iter_next(struct btree_and_journal_iter *iter)
 289 {
 290         bch2_btree_and_journal_iter_advance(iter);
 291
 292         return bch2_btree_and_journal_iter_peek(iter);
 293 }
 294
 295 void bch2_btree_and_journal_iter_exit(struct btree_and_journal_iter *iter)
 296 {
 297         bch2_journal_iter_exit(&iter->journal);
 298 }
 299
 300 void bch2_btree_and_journal_iter_init_node_iter(struct btree_and_journal_iter *iter,
 301                                                 struct bch_fs *c,
 302                                                 struct btree *b)
 303 {
 304         memset(iter, 0, sizeof(*iter));
 305
 306         iter->b = b;
 307         bch2_btree_node_iter_init_from_start(&iter->node_iter, iter->b);
 308         bch2_journal_iter_init(c, &iter->journal,
 309                                b->c.btree_id, b->c.level, b->data->min_key);
 310 }
 311
 312 /* Walk btree, overlaying keys from the journal: */
 313
 314 static void btree_and_journal_iter_prefetch(struct bch_fs *c, struct btree *b,
 315                                            struct btree_and_journal_iter iter)
 316 {
 317         unsigned i = 0, nr = b->c.level > 1 ? 2 : 16;
 318         struct bkey_s_c k;
 319         struct bkey_buf tmp;
 320
 321         BUG_ON(!b->c.level);
 322
 323         bch2_bkey_buf_init(&tmp);
 324
 325         while (i < nr &&
 326                (k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 327                 bch2_bkey_buf_reassemble(&tmp, c, k);
 328
 329                 bch2_btree_node_prefetch(c, NULL, tmp.k,
 330                                         b->c.btree_id, b->c.level - 1);
 331
 332                 bch2_btree_and_journal_iter_advance(&iter);
 333                 i++;
 334         }
 335
 336         bch2_bkey_buf_exit(&tmp, c);
 337 }
 338
 339 static int bch2_btree_and_journal_walk_recurse(struct bch_fs *c, struct btree *b,
 340                                 enum btree_id btree_id,
 341                                 btree_walk_key_fn key_fn)
 342 {
 343         struct btree_and_journal_iter iter;
 344         struct bkey_s_c k;
 345         struct bkey_buf tmp;
 346         struct btree *child;
 347         int ret = 0;
 348
 349         bch2_bkey_buf_init(&tmp);
 350         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 351
 352         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 353                 if (b->c.level) {
 354                         bch2_bkey_buf_reassemble(&tmp, c, k);
 355
 356                         child = bch2_btree_node_get_noiter(c, tmp.k,
 357                                                 b->c.btree_id, b->c.level - 1,
 358                                                 false);
 359
 360                         ret = PTR_ERR_OR_ZERO(child);
 361                         if (ret)
 362                                 break;
 363
 364                         btree_and_journal_iter_prefetch(c, b, iter);
 365
 366                         ret = bch2_btree_and_journal_walk_recurse(c, child,
 367                                         btree_id, key_fn);
 368                         six_unlock_read(&child->c.lock);
 369                 } else {
 370                         ret = key_fn(c, k);
 371                 }
 372
 373                 if (ret)
 374                         break;
 375
 376                 bch2_btree_and_journal_iter_advance(&iter);
 377         }
 378
 379         bch2_btree_and_journal_iter_exit(&iter);
 380         bch2_bkey_buf_exit(&tmp, c);
 381         return ret;
 382 }
 383
 384 int bch2_btree_and_journal_walk(struct bch_fs *c, enum btree_id btree_id,
 385                                 btree_walk_key_fn key_fn)
 386 {
 387         struct btree *b = c->btree_roots[btree_id].b;
 388         int ret = 0;
 389
 390         if (btree_node_fake(b))
 391                 return 0;
 392
 393         six_lock_read(&b->c.lock, NULL, NULL);
 394         ret = bch2_btree_and_journal_walk_recurse(c, b, btree_id, key_fn);
 395         six_unlock_read(&b->c.lock);
 396
 397         return ret;
 398 }
 399
 400 /* sort and dedup all keys in the journal: */
 401
 402 void bch2_journal_entries_free(struct list_head *list)
 403 {
 404
 405         while (!list_empty(list)) {
 406                 struct journal_replay *i =
 407                         list_first_entry(list, struct journal_replay, list);
 408                 list_del(&i->list);
 409                 kvpfree(i, offsetof(struct journal_replay, j) +
 410                         vstruct_bytes(&i->j));
 411         }
 412 }
 413
 414 /*
 415  * When keys compare equal, oldest compares first:
 416  */
 417 static int journal_sort_key_cmp(const void *_l, const void *_r)
 418 {
 419         const struct journal_key *l = _l;
 420         const struct journal_key *r = _r;
 421
 422         return  cmp_int(l->btree_id,    r->btree_id) ?:
 423                 cmp_int(l->level,       r->level) ?:
 424                 bpos_cmp(l->k->k.p, r->k->k.p) ?:
 425                 cmp_int(l->journal_seq, r->journal_seq) ?:
 426                 cmp_int(l->journal_offset, r->journal_offset);
 427 }
 428
 429 void bch2_journal_keys_free(struct journal_keys *keys)
 430 {
 431         struct journal_key *i;
 432
 433         for (i = keys->d; i < keys->d + keys->nr; i++)
 434                 if (i->allocated)
 435                         kfree(i->k);
 436
 437         kvfree(keys->d);
 438         keys->d = NULL;
 439         keys->nr = 0;
 440 }
 441
 442 static struct journal_keys journal_keys_sort(struct list_head *journal_entries)
 443 {
 444         struct journal_replay *i;
 445         struct jset_entry *entry;
 446         struct bkey_i *k, *_n;
 447         struct journal_keys keys = { NULL };
 448         struct journal_key *src, *dst;
 449         size_t nr_keys = 0;
 450
 451         if (list_empty(journal_entries))
 452                 return keys;
 453
 454         list_for_each_entry(i, journal_entries, list) {
 455                 if (i->ignore)
 456                         continue;
 457
 458                 if (!keys.journal_seq_base)
 459                         keys.journal_seq_base = le64_to_cpu(i->j.seq);
 460
 461                 for_each_jset_key(k, _n, entry, &i->j)
 462                         nr_keys++;
 463         }
 464
 465         keys.size = roundup_pow_of_two(nr_keys);
 466
 467         keys.d = kvmalloc(sizeof(keys.d[0]) * keys.size, GFP_KERNEL);
 468         if (!keys.d)
 469                 goto err;
 470
 471         list_for_each_entry(i, journal_entries, list) {
 472                 if (i->ignore)
 473                         continue;
 474
 475                 BUG_ON(le64_to_cpu(i->j.seq) - keys.journal_seq_base > U32_MAX);
 476
 477                 for_each_jset_key(k, _n, entry, &i->j)
 478                         keys.d[keys.nr++] = (struct journal_key) {
 479                                 .btree_id       = entry->btree_id,
 480                                 .level          = entry->level,
 481                                 .k              = k,
 482                                 .journal_seq    = le64_to_cpu(i->j.seq) -
 483                                         keys.journal_seq_base,
 484                                 .journal_offset = k->_data - i->j._data,
 485                         };
 486         }
 487
 488         sort(keys.d, keys.nr, sizeof(keys.d[0]), journal_sort_key_cmp, NULL);
 489
 490         src = dst = keys.d;
 491         while (src < keys.d + keys.nr) {
 492                 while (src + 1 < keys.d + keys.nr &&
 493                        src[0].btree_id  == src[1].btree_id &&
 494                        src[0].level     == src[1].level &&
 495                        !bpos_cmp(src[0].k->k.p, src[1].k->k.p))
 496                         src++;
 497
 498                 *dst++ = *src++;
 499         }
 500
 501         keys.nr = dst - keys.d;
 502 err:
 503         return keys;
 504 }
 505
 506 /* journal replay: */
 507
 508 static void replay_now_at(struct journal *j, u64 seq)
 509 {
 510         BUG_ON(seq < j->replay_journal_seq);
 511         BUG_ON(seq > j->replay_journal_seq_end);
 512
 513         while (j->replay_journal_seq < seq)
 514                 bch2_journal_pin_put(j, j->replay_journal_seq++);
 515 }
 516
 517 static int __bch2_journal_replay_key(struct btree_trans *trans,
 518                                      enum btree_id id, unsigned level,
 519                                      struct bkey_i *k)
 520 {
 521         struct btree_iter *iter;
 522         int ret;
 523
 524         iter = bch2_trans_get_node_iter(trans, id, k->k.p,
 525                                         BTREE_MAX_DEPTH, level,
 526                                         BTREE_ITER_INTENT|
 527                                         BTREE_ITER_NOT_EXTENTS);
 528         ret   = bch2_btree_iter_traverse(iter) ?:
 529                 bch2_trans_update(trans, iter, k, BTREE_TRIGGER_NORUN);
 530         bch2_trans_iter_put(trans, iter);
 531         return ret;
 532 }
 533
 534 static int bch2_journal_replay_key(struct bch_fs *c, struct journal_key *k)
 535 {
 536         unsigned commit_flags = BTREE_INSERT_NOFAIL|
 537                 BTREE_INSERT_LAZY_RW;
 538
 539         if (!k->allocated)
 540                 commit_flags |= BTREE_INSERT_JOURNAL_REPLAY;
 541
 542         return bch2_trans_do(c, NULL, NULL, commit_flags,
 543                              __bch2_journal_replay_key(&trans, k->btree_id, k->level, k->k));
 544 }
 545
 546 static int __bch2_alloc_replay_key(struct btree_trans *trans, struct bkey_i *k)
 547 {
 548         struct btree_iter *iter;
 549         int ret;
 550
 551         iter = bch2_trans_get_iter(trans, BTREE_ID_alloc, k->k.p,
 552                                    BTREE_ITER_CACHED|
 553                                    BTREE_ITER_CACHED_NOFILL|
 554                                    BTREE_ITER_INTENT);
 555         ret   = bch2_btree_iter_traverse(iter) ?:
 556                 bch2_trans_update(trans, iter, k, BTREE_TRIGGER_NORUN);
 557         bch2_trans_iter_put(trans, iter);
 558         return ret;
 559 }
 560
 561 static int bch2_alloc_replay_key(struct bch_fs *c, struct bkey_i *k)
 562 {
 563         return bch2_trans_do(c, NULL, NULL,
 564                              BTREE_INSERT_NOFAIL|
 565                              BTREE_INSERT_USE_RESERVE|
 566                              BTREE_INSERT_LAZY_RW|
 567                              BTREE_INSERT_JOURNAL_REPLAY,
 568                         __bch2_alloc_replay_key(&trans, k));
 569 }
 570
 571 static int journal_sort_seq_cmp(const void *_l, const void *_r)
 572 {
 573         const struct journal_key *l = _l;
 574         const struct journal_key *r = _r;
 575
 576         return  cmp_int(r->level,       l->level) ?:
 577                 cmp_int(l->journal_seq, r->journal_seq) ?:
 578                 cmp_int(l->btree_id,    r->btree_id) ?:
 579                 bpos_cmp(l->k->k.p,     r->k->k.p);
 580 }
 581
 582 static int bch2_journal_replay(struct bch_fs *c,
 583                                struct journal_keys keys)
 584 {
 585         struct journal *j = &c->journal;
 586         struct journal_key *i;
 587         u64 seq;
 588         int ret;
 589
 590         sort(keys.d, keys.nr, sizeof(keys.d[0]), journal_sort_seq_cmp, NULL);
 591
 592         if (keys.nr)
 593                 replay_now_at(j, keys.journal_seq_base);
 594
 595         seq = j->replay_journal_seq;
 596
 597         /*
 598          * First replay updates to the alloc btree - these will only update the
 599          * btree key cache:
 600          */
 601         for_each_journal_key(keys, i) {
 602                 cond_resched();
 603
 604                 if (!i->level && i->btree_id == BTREE_ID_alloc) {
 605                         j->replay_journal_seq = keys.journal_seq_base + i->journal_seq;
 606                         ret = bch2_alloc_replay_key(c, i->k);
 607                         if (ret)
 608                                 goto err;
 609                 }
 610         }
 611
 612         /*
 613          * Next replay updates to interior btree nodes:
 614          */
 615         for_each_journal_key(keys, i) {
 616                 cond_resched();
 617
 618                 if (i->level) {
 619                         j->replay_journal_seq = keys.journal_seq_base + i->journal_seq;
 620                         ret = bch2_journal_replay_key(c, i);
 621                         if (ret)
 622                                 goto err;
 623                 }
 624         }
 625
 626         /*
 627          * Now that the btree is in a consistent state, we can start journal
 628          * reclaim (which will be flushing entries from the btree key cache back
 629          * to the btree:
 630          */
 631         set_bit(BCH_FS_BTREE_INTERIOR_REPLAY_DONE, &c->flags);
 632         set_bit(JOURNAL_RECLAIM_STARTED, &j->flags);
 633         journal_reclaim_kick(j);
 634
 635         j->replay_journal_seq = seq;
 636
 637         /*
 638          * Now replay leaf node updates:
 639          */
 640         for_each_journal_key(keys, i) {
 641                 cond_resched();
 642
 643                 if (i->level || i->btree_id == BTREE_ID_alloc)
 644                         continue;
 645
 646                 replay_now_at(j, keys.journal_seq_base + i->journal_seq);
 647
 648                 ret = bch2_journal_replay_key(c, i);
 649                 if (ret)
 650                         goto err;
 651         }
 652
 653         replay_now_at(j, j->replay_journal_seq_end);
 654         j->replay_journal_seq = 0;
 655
 656         bch2_journal_set_replay_done(j);
 657         bch2_journal_flush_all_pins(j);
 658         return bch2_journal_error(j);
 659 err:
 660         bch_err(c, "journal replay: error %d while replaying key at btree %s level %u",
 661                 ret, bch2_btree_ids[i->btree_id], i->level);
 662         return ret;
 663 }
 664
 665 /* journal replay early: */
 666
 667 static int journal_replay_entry_early(struct bch_fs *c,
 668                                       struct jset_entry *entry)
 669 {
 670         int ret = 0;
 671
 672         switch (entry->type) {
 673         case BCH_JSET_ENTRY_btree_root: {
 674                 struct btree_root *r;
 675
 676                 if (entry->btree_id >= BTREE_ID_NR) {
 677                         bch_err(c, "filesystem has unknown btree type %u",
 678                                 entry->btree_id);
 679                         return -EINVAL;
 680                 }
 681
 682                 r = &c->btree_roots[entry->btree_id];
 683
 684                 if (entry->u64s) {
 685                         r->level = entry->level;
 686                         bkey_copy(&r->key, &entry->start[0]);
 687                         r->error = 0;
 688                 } else {
 689                         r->error = -EIO;
 690                 }
 691                 r->alive = true;
 692                 break;
 693         }
 694         case BCH_JSET_ENTRY_usage: {
 695                 struct jset_entry_usage *u =
 696                         container_of(entry, struct jset_entry_usage, entry);
 697
 698                 switch (entry->btree_id) {
 699                 case FS_USAGE_RESERVED:
 700                         if (entry->level < BCH_REPLICAS_MAX)
 701                                 c->usage_base->persistent_reserved[entry->level] =
 702                                         le64_to_cpu(u->v);
 703                         break;
 704                 case FS_USAGE_INODES:
 705                         c->usage_base->nr_inodes = le64_to_cpu(u->v);
 706                         break;
 707                 case FS_USAGE_KEY_VERSION:
 708                         atomic64_set(&c->key_version,
 709                                      le64_to_cpu(u->v));
 710                         break;
 711                 }
 712
 713                 break;
 714         }
 715         case BCH_JSET_ENTRY_data_usage: {
 716                 struct jset_entry_data_usage *u =
 717                         container_of(entry, struct jset_entry_data_usage, entry);
 718
 719                 ret = bch2_replicas_set_usage(c, &u->r,
 720                                               le64_to_cpu(u->v));
 721                 break;
 722         }
 723         case BCH_JSET_ENTRY_dev_usage: {
 724                 struct jset_entry_dev_usage *u =
 725                         container_of(entry, struct jset_entry_dev_usage, entry);
 726                 struct bch_dev *ca = bch_dev_bkey_exists(c, le32_to_cpu(u->dev));
 727                 unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 728                 unsigned nr_types = (bytes - sizeof(struct jset_entry_dev_usage)) /
 729                         sizeof(struct jset_entry_dev_usage_type);
 730                 unsigned i;
 731
 732                 ca->usage_base->buckets_ec              = le64_to_cpu(u->buckets_ec);
 733                 ca->usage_base->buckets_unavailable     = le64_to_cpu(u->buckets_unavailable);
 734
 735                 for (i = 0; i < min_t(unsigned, nr_types, BCH_DATA_NR); i++) {
 736                         ca->usage_base->d[i].buckets    = le64_to_cpu(u->d[i].buckets);
 737                         ca->usage_base->d[i].sectors    = le64_to_cpu(u->d[i].sectors);
 738                         ca->usage_base->d[i].fragmented = le64_to_cpu(u->d[i].fragmented);
 739                 }
 740
 741                 break;
 742         }
 743         case BCH_JSET_ENTRY_blacklist: {
 744                 struct jset_entry_blacklist *bl_entry =
 745                         container_of(entry, struct jset_entry_blacklist, entry);
 746
 747                 ret = bch2_journal_seq_blacklist_add(c,
 748                                 le64_to_cpu(bl_entry->seq),
 749                                 le64_to_cpu(bl_entry->seq) + 1);
 750                 break;
 751         }
 752         case BCH_JSET_ENTRY_blacklist_v2: {
 753                 struct jset_entry_blacklist_v2 *bl_entry =
 754                         container_of(entry, struct jset_entry_blacklist_v2, entry);
 755
 756                 ret = bch2_journal_seq_blacklist_add(c,
 757                                 le64_to_cpu(bl_entry->start),
 758                                 le64_to_cpu(bl_entry->end) + 1);
 759                 break;
 760         }
 761         case BCH_JSET_ENTRY_clock: {
 762                 struct jset_entry_clock *clock =
 763                         container_of(entry, struct jset_entry_clock, entry);
 764
 765                 atomic64_set(&c->io_clock[clock->rw].now, le64_to_cpu(clock->time));
 766         }
 767         }
 768
 769         return ret;
 770 }
 771
 772 static int journal_replay_early(struct bch_fs *c,
 773                                 struct bch_sb_field_clean *clean,
 774                                 struct list_head *journal)
 775 {
 776         struct journal_replay *i;
 777         struct jset_entry *entry;
 778         int ret;
 779
 780         if (clean) {
 781                 for (entry = clean->start;
 782                      entry != vstruct_end(&clean->field);
 783                      entry = vstruct_next(entry)) {
 784                         ret = journal_replay_entry_early(c, entry);
 785                         if (ret)
 786                                 return ret;
 787                 }
 788         } else {
 789                 list_for_each_entry(i, journal, list) {
 790                         if (i->ignore)
 791                                 continue;
 792
 793                         vstruct_for_each(&i->j, entry) {
 794                                 ret = journal_replay_entry_early(c, entry);
 795                                 if (ret)
 796                                         return ret;
 797                         }
 798                 }
 799         }
 800
 801         bch2_fs_usage_initialize(c);
 802
 803         return 0;
 804 }
 805
 806 /* sb clean section: */
 807
 808 static struct bkey_i *btree_root_find(struct bch_fs *c,
 809                                       struct bch_sb_field_clean *clean,
 810                                       struct jset *j,
 811                                       enum btree_id id, unsigned *level)
 812 {
 813         struct bkey_i *k;
 814         struct jset_entry *entry, *start, *end;
 815
 816         if (clean) {
 817                 start = clean->start;
 818                 end = vstruct_end(&clean->field);
 819         } else {
 820                 start = j->start;
 821                 end = vstruct_last(j);
 822         }
 823
 824         for (entry = start; entry < end; entry = vstruct_next(entry))
 825                 if (entry->type == BCH_JSET_ENTRY_btree_root &&
 826                     entry->btree_id == id)
 827                         goto found;
 828
 829         return NULL;
 830 found:
 831         if (!entry->u64s)
 832                 return ERR_PTR(-EINVAL);
 833
 834         k = entry->start;
 835         *level = entry->level;
 836         return k;
 837 }
 838
 839 static int verify_superblock_clean(struct bch_fs *c,
 840                                    struct bch_sb_field_clean **cleanp,
 841                                    struct jset *j)
 842 {
 843         unsigned i;
 844         struct bch_sb_field_clean *clean = *cleanp;
 845         int ret = 0;
 846
 847         if (mustfix_fsck_err_on(j->seq != clean->journal_seq, c,
 848                         "superblock journal seq (%llu) doesn't match journal (%llu) after clean shutdown",
 849                         le64_to_cpu(clean->journal_seq),
 850                         le64_to_cpu(j->seq))) {
 851                 kfree(clean);
 852                 *cleanp = NULL;
 853                 return 0;
 854         }
 855
 856         for (i = 0; i < BTREE_ID_NR; i++) {
 857                 char buf1[200], buf2[200];
 858                 struct bkey_i *k1, *k2;
 859                 unsigned l1 = 0, l2 = 0;
 860
 861                 k1 = btree_root_find(c, clean, NULL, i, &l1);
 862                 k2 = btree_root_find(c, NULL, j, i, &l2);
 863
 864                 if (!k1 && !k2)
 865                         continue;
 866
 867                 mustfix_fsck_err_on(!k1 || !k2 ||
 868                                     IS_ERR(k1) ||
 869                                     IS_ERR(k2) ||
 870                                     k1->k.u64s != k2->k.u64s ||
 871                                     memcmp(k1, k2, bkey_bytes(k1)) ||
 872                                     l1 != l2, c,
 873                         "superblock btree root %u doesn't match journal after clean shutdown\n"
 874                         "sb:      l=%u %s\n"
 875                         "journal: l=%u %s\n", i,
 876                         l1, (bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(k1)), buf1),
 877                         l2, (bch2_bkey_val_to_text(&PBUF(buf2), c, bkey_i_to_s_c(k2)), buf2));
 878         }
 879 fsck_err:
 880         return ret;
 881 }
 882
 883 static struct bch_sb_field_clean *read_superblock_clean(struct bch_fs *c)
 884 {
 885         struct bch_sb_field_clean *clean, *sb_clean;
 886         int ret;
 887
 888         mutex_lock(&c->sb_lock);
 889         sb_clean = bch2_sb_get_clean(c->disk_sb.sb);
 890
 891         if (fsck_err_on(!sb_clean, c,
 892                         "superblock marked clean but clean section not present")) {
 893                 SET_BCH_SB_CLEAN(c->disk_sb.sb, false);
 894                 c->sb.clean = false;
 895                 mutex_unlock(&c->sb_lock);
 896                 return NULL;
 897         }
 898
 899         clean = kmemdup(sb_clean, vstruct_bytes(&sb_clean->field),
 900                         GFP_KERNEL);
 901         if (!clean) {
 902                 mutex_unlock(&c->sb_lock);
 903                 return ERR_PTR(-ENOMEM);
 904         }
 905
 906         ret = bch2_sb_clean_validate(c, clean, READ);
 907         if (ret) {
 908                 mutex_unlock(&c->sb_lock);
 909                 return ERR_PTR(ret);
 910         }
 911
 912         mutex_unlock(&c->sb_lock);
 913
 914         return clean;
 915 fsck_err:
 916         mutex_unlock(&c->sb_lock);
 917         return ERR_PTR(ret);
 918 }
 919
 920 static int read_btree_roots(struct bch_fs *c)
 921 {
 922         unsigned i;
 923         int ret = 0;
 924
 925         for (i = 0; i < BTREE_ID_NR; i++) {
 926                 struct btree_root *r = &c->btree_roots[i];
 927
 928                 if (!r->alive)
 929                         continue;
 930
 931                 if (i == BTREE_ID_alloc &&
 932                     c->opts.reconstruct_alloc) {
 933                         c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
 934                         continue;
 935                 }
 936
 937                 if (r->error) {
 938                         __fsck_err(c, i == BTREE_ID_alloc
 939                                    ? FSCK_CAN_IGNORE : 0,
 940                                    "invalid btree root %s",
 941                                    bch2_btree_ids[i]);
 942                         if (i == BTREE_ID_alloc)
 943                                 c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
 944                 }
 945
 946                 ret = bch2_btree_root_read(c, i, &r->key, r->level);
 947                 if (ret) {
 948                         __fsck_err(c, i == BTREE_ID_alloc
 949                                    ? FSCK_CAN_IGNORE : 0,
 950                                    "error reading btree root %s",
 951                                    bch2_btree_ids[i]);
 952                         if (i == BTREE_ID_alloc)
 953                                 c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
 954                 }
 955         }
 956
 957         for (i = 0; i < BTREE_ID_NR; i++)
 958                 if (!c->btree_roots[i].b)
 959                         bch2_btree_root_alloc(c, i);
 960 fsck_err:
 961         return ret;
 962 }
 963
 964 int bch2_fs_recovery(struct bch_fs *c)
 965 {
 966         const char *err = "cannot allocate memory";
 967         struct bch_sb_field_clean *clean = NULL;
 968         struct jset *last_journal_entry = NULL;
 969         u64 blacklist_seq, journal_seq;
 970         bool write_sb = false;
 971         int ret = 0;
 972
 973         if (c->sb.clean)
 974                 clean = read_superblock_clean(c);
 975         ret = PTR_ERR_OR_ZERO(clean);
 976         if (ret)
 977                 goto err;
 978
 979         if (c->sb.clean)
 980                 bch_info(c, "recovering from clean shutdown, journal seq %llu",
 981                          le64_to_cpu(clean->journal_seq));
 982
 983         if (!(c->sb.features & (1ULL << BCH_FEATURE_new_extent_overwrite))) {
 984                 bch_err(c, "feature new_extent_overwrite not set, filesystem no longer supported");
 985                 ret = -EINVAL;
 986                 goto err;
 987         }
 988
 989         if (!c->sb.clean &&
 990             !(c->sb.features & (1ULL << BCH_FEATURE_extents_above_btree_updates))) {
 991                 bch_err(c, "filesystem needs recovery from older version; run fsck from older bcachefs-tools to fix");
 992                 ret = -EINVAL;
 993                 goto err;
 994         }
 995
 996         if (!(c->sb.compat & (1ULL << BCH_COMPAT_bformat_overflow_done))) {
 997                 bch_err(c, "filesystem may have incompatible bkey formats; run fsck from the compat branch to fix");
 998                 ret = -EINVAL;
 999                 goto err;
1000
1001         }
1002
1003         if (!(c->sb.features & (1ULL << BCH_FEATURE_alloc_v2))) {
1004                 bch_info(c, "alloc_v2 feature bit not set, fsck required");
1005                 c->opts.fsck = true;
1006                 c->opts.fix_errors = FSCK_OPT_YES;
1007         }
1008
1009         if (!c->replicas.entries ||
1010             c->opts.rebuild_replicas) {
1011                 bch_info(c, "building replicas info");
1012                 set_bit(BCH_FS_REBUILD_REPLICAS, &c->flags);
1013         }
1014
1015         if (c->sb.version < bcachefs_metadata_version_inode_backpointers) {
1016                 bch_info(c, "version prior to inode backpointers, upgrade and fsck required");
1017                 c->opts.version_upgrade = true;
1018                 c->opts.fsck            = true;
1019                 c->opts.fix_errors      = FSCK_OPT_YES;
1020         }
1021
1022         if (c->sb.version < bcachefs_metadata_version_btree_ptr_sectors_written) {
1023                 bch_info(c, "version prior to btree_ptr_sectors_written, upgrade required");
1024                 c->opts.version_upgrade = true;
1025         }
1026
1027         ret = bch2_blacklist_table_initialize(c);
1028         if (ret) {
1029                 bch_err(c, "error initializing blacklist table");
1030                 goto err;
1031         }
1032
1033         if (!c->sb.clean || c->opts.fsck || c->opts.keep_journal) {
1034                 struct journal_replay *i;
1035
1036                 ret = bch2_journal_read(c, &c->journal_entries,
1037                                         &blacklist_seq, &journal_seq);
1038                 if (ret)
1039                         goto err;
1040
1041                 list_for_each_entry_reverse(i, &c->journal_entries, list)
1042                         if (!i->ignore) {
1043                                 last_journal_entry = &i->j;
1044                                 break;
1045                         }
1046
1047                 if (mustfix_fsck_err_on(c->sb.clean &&
1048                                         last_journal_entry &&
1049                                         !journal_entry_empty(last_journal_entry), c,
1050                                 "filesystem marked clean but journal not empty")) {
1051                         c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
1052                         SET_BCH_SB_CLEAN(c->disk_sb.sb, false);
1053                         c->sb.clean = false;
1054                 }
1055
1056                 if (!last_journal_entry) {
1057                         fsck_err_on(!c->sb.clean, c, "no journal entries found");
1058                         goto use_clean;
1059                 }
1060
1061                 c->journal_keys = journal_keys_sort(&c->journal_entries);
1062                 if (!c->journal_keys.d) {
1063                         ret = -ENOMEM;
1064                         goto err;
1065                 }
1066
1067                 if (c->sb.clean && last_journal_entry) {
1068                         ret = verify_superblock_clean(c, &clean,
1069                                                       last_journal_entry);
1070                         if (ret)
1071                                 goto err;
1072                 }
1073         } else {
1074 use_clean:
1075                 if (!clean) {
1076                         bch_err(c, "no superblock clean section found");
1077                         ret = BCH_FSCK_REPAIR_IMPOSSIBLE;
1078                         goto err;
1079
1080                 }
1081                 blacklist_seq = journal_seq = le64_to_cpu(clean->journal_seq) + 1;
1082         }
1083
1084         if (c->opts.reconstruct_alloc) {
1085                 c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
1086                 drop_alloc_keys(&c->journal_keys);
1087         }
1088
1089         zero_out_btree_mem_ptr(&c->journal_keys);
1090
1091         ret = journal_replay_early(c, clean, &c->journal_entries);
1092         if (ret)
1093                 goto err;
1094
1095         /*
1096          * After an unclean shutdown, skip then next few journal sequence
1097          * numbers as they may have been referenced by btree writes that
1098          * happened before their corresponding journal writes - those btree
1099          * writes need to be ignored, by skipping and blacklisting the next few
1100          * journal sequence numbers:
1101          */
1102         if (!c->sb.clean)
1103                 journal_seq += 8;
1104
1105         if (blacklist_seq != journal_seq) {
1106                 ret = bch2_journal_seq_blacklist_add(c,
1107                                         blacklist_seq, journal_seq);
1108                 if (ret) {
1109                         bch_err(c, "error creating new journal seq blacklist entry");
1110                         goto err;
1111                 }
1112         }
1113
1114         ret = bch2_fs_journal_start(&c->journal, journal_seq,
1115                                     &c->journal_entries);
1116         if (ret)
1117                 goto err;
1118
1119         ret = read_btree_roots(c);
1120         if (ret)
1121                 goto err;
1122
1123         bch_verbose(c, "starting alloc read");
1124         err = "error reading allocation information";
1125         ret = bch2_alloc_read(c);
1126         if (ret)
1127                 goto err;
1128         bch_verbose(c, "alloc read done");
1129
1130         bch_verbose(c, "starting stripes_read");
1131         err = "error reading stripes";
1132         ret = bch2_stripes_read(c);
1133         if (ret)
1134                 goto err;
1135         bch_verbose(c, "stripes_read done");
1136
1137         set_bit(BCH_FS_ALLOC_READ_DONE, &c->flags);
1138
1139         if (c->opts.fsck ||
1140             !(c->sb.compat & (1ULL << BCH_COMPAT_alloc_info)) ||
1141             !(c->sb.compat & (1ULL << BCH_COMPAT_alloc_metadata)) ||
1142             test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags)) {
1143                 bool metadata_only = c->opts.norecovery;
1144
1145                 bch_info(c, "starting mark and sweep");
1146                 err = "error in mark and sweep";
1147                 ret = bch2_gc(c, true, metadata_only);
1148                 if (ret)
1149                         goto err;
1150                 bch_verbose(c, "mark and sweep done");
1151         }
1152
1153         bch2_stripes_heap_start(c);
1154
1155         clear_bit(BCH_FS_REBUILD_REPLICAS, &c->flags);
1156         set_bit(BCH_FS_INITIAL_GC_DONE, &c->flags);
1157
1158         /*
1159          * Skip past versions that might have possibly been used (as nonces),
1160          * but hadn't had their pointers written:
1161          */
1162         if (c->sb.encryption_type && !c->sb.clean)
1163                 atomic64_add(1 << 16, &c->key_version);
1164
1165         if (c->opts.norecovery)
1166                 goto out;
1167
1168         bch_verbose(c, "starting journal replay");
1169         err = "journal replay failed";
1170         ret = bch2_journal_replay(c, c->journal_keys);
1171         if (ret)
1172                 goto err;
1173         bch_verbose(c, "journal replay done");
1174
1175         if (test_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags) &&
1176             !c->opts.nochanges) {
1177                 /*
1178                  * note that even when filesystem was clean there might be work
1179                  * to do here, if we ran gc (because of fsck) which recalculated
1180                  * oldest_gen:
1181                  */
1182                 bch_verbose(c, "writing allocation info");
1183                 err = "error writing out alloc info";
1184                 ret = bch2_stripes_write(c, BTREE_INSERT_LAZY_RW) ?:
1185                         bch2_alloc_write(c, BTREE_INSERT_LAZY_RW);
1186                 if (ret) {
1187                         bch_err(c, "error writing alloc info");
1188                         goto err;
1189                 }
1190                 bch_verbose(c, "alloc write done");
1191         }
1192
1193         if (c->opts.fsck) {
1194                 bch_info(c, "starting fsck");
1195                 err = "error in fsck";
1196                 ret = bch2_fsck_full(c);
1197                 if (ret)
1198                         goto err;
1199                 bch_verbose(c, "fsck done");
1200         } else if (!c->sb.clean) {
1201                 bch_verbose(c, "checking for deleted inodes");
1202                 err = "error in recovery";
1203                 ret = bch2_fsck_walk_inodes_only(c);
1204                 if (ret)
1205                         goto err;
1206                 bch_verbose(c, "check inodes done");
1207         }
1208
1209         if (enabled_qtypes(c)) {
1210                 bch_verbose(c, "reading quotas");
1211                 ret = bch2_fs_quota_read(c);
1212                 if (ret)
1213                         goto err;
1214                 bch_verbose(c, "quotas done");
1215         }
1216
1217         if (!(c->sb.compat & (1ULL << BCH_COMPAT_extents_above_btree_updates_done)) ||
1218             !(c->sb.compat & (1ULL << BCH_COMPAT_bformat_overflow_done))) {
1219                 struct bch_move_stats stats = { 0 };
1220
1221                 bch_info(c, "scanning for old btree nodes");
1222                 ret = bch2_fs_read_write(c);
1223                 if (ret)
1224                         goto err;
1225
1226                 ret = bch2_scan_old_btree_nodes(c, &stats);
1227                 if (ret)
1228                         goto err;
1229                 bch_info(c, "scanning for old btree nodes done");
1230         }
1231
1232         mutex_lock(&c->sb_lock);
1233         if (c->opts.version_upgrade) {
1234                 c->disk_sb.sb->version = cpu_to_le16(bcachefs_metadata_version_current);
1235                 c->disk_sb.sb->features[0] |= cpu_to_le64(BCH_SB_FEATURES_ALL);
1236                 write_sb = true;
1237         }
1238
1239         if (!test_bit(BCH_FS_ERROR, &c->flags)) {
1240                 c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_alloc_info);
1241                 write_sb = true;
1242         }
1243
1244         if (c->opts.fsck &&
1245             !test_bit(BCH_FS_ERROR, &c->flags) &&
1246             !test_bit(BCH_FS_ERRORS_NOT_FIXED, &c->flags)) {
1247                 SET_BCH_SB_HAS_ERRORS(c->disk_sb.sb, 0);
1248                 SET_BCH_SB_HAS_TOPOLOGY_ERRORS(c->disk_sb.sb, 0);
1249                 write_sb = true;
1250         }
1251
1252         if (write_sb)
1253                 bch2_write_super(c);
1254         mutex_unlock(&c->sb_lock);
1255
1256         if (c->journal_seq_blacklist_table &&
1257             c->journal_seq_blacklist_table->nr > 128)
1258                 queue_work(system_long_wq, &c->journal_seq_blacklist_gc_work);
1259
1260         ret = 0;
1261 out:
1262         set_bit(BCH_FS_FSCK_DONE, &c->flags);
1263         bch2_flush_fsck_errs(c);
1264
1265         if (!c->opts.keep_journal) {
1266                 bch2_journal_keys_free(&c->journal_keys);
1267                 bch2_journal_entries_free(&c->journal_entries);
1268         }
1269         kfree(clean);
1270         if (ret)
1271                 bch_err(c, "Error in recovery: %s (%i)", err, ret);
1272         else
1273                 bch_verbose(c, "ret %i", ret);
1274         return ret;
1275 err:
1276 fsck_err:
1277         bch2_fs_emergency_read_only(c);
1278         goto out;
1279 }
1280
1281 int bch2_fs_initialize(struct bch_fs *c)
1282 {
1283         struct bch_inode_unpacked root_inode, lostfound_inode;
1284         struct bkey_inode_buf packed_inode;
1285         struct qstr lostfound = QSTR("lost+found");
1286         const char *err = "cannot allocate memory";
1287         struct bch_dev *ca;
1288         LIST_HEAD(journal);
1289         unsigned i;
1290         int ret;
1291
1292         bch_notice(c, "initializing new filesystem");
1293
1294         mutex_lock(&c->sb_lock);
1295         c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_extents_above_btree_updates_done);
1296         c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_bformat_overflow_done);
1297
1298         if (c->opts.version_upgrade) {
1299                 c->disk_sb.sb->version = cpu_to_le16(bcachefs_metadata_version_current);
1300                 c->disk_sb.sb->features[0] |= cpu_to_le64(BCH_SB_FEATURES_ALL);
1301                 bch2_write_super(c);
1302         }
1303
1304         for_each_online_member(ca, c, i)
1305                 bch2_mark_dev_superblock(c, ca, 0);
1306         mutex_unlock(&c->sb_lock);
1307
1308         set_bit(BCH_FS_ALLOC_READ_DONE, &c->flags);
1309         set_bit(BCH_FS_INITIAL_GC_DONE, &c->flags);
1310
1311         for (i = 0; i < BTREE_ID_NR; i++)
1312                 bch2_btree_root_alloc(c, i);
1313
1314         set_bit(BCH_FS_BTREE_INTERIOR_REPLAY_DONE, &c->flags);
1315         set_bit(JOURNAL_RECLAIM_STARTED, &c->journal.flags);
1316
1317         err = "unable to allocate journal buckets";
1318         for_each_online_member(ca, c, i) {
1319                 ret = bch2_dev_journal_alloc(ca);
1320                 if (ret) {
1321                         percpu_ref_put(&ca->io_ref);
1322                         goto err;
1323                 }
1324         }
1325
1326         /*
1327          * journal_res_get() will crash if called before this has
1328          * set up the journal.pin FIFO and journal.cur pointer:
1329          */
1330         bch2_fs_journal_start(&c->journal, 1, &journal);
1331         bch2_journal_set_replay_done(&c->journal);
1332
1333         err = "error going read-write";
1334         ret = bch2_fs_read_write_early(c);
1335         if (ret)
1336                 goto err;
1337
1338         /*
1339          * Write out the superblock and journal buckets, now that we can do
1340          * btree updates
1341          */
1342         err = "error marking superblock and journal";
1343         for_each_member_device(ca, c, i) {
1344                 ret = bch2_trans_mark_dev_sb(c, ca);
1345                 if (ret) {
1346                         percpu_ref_put(&ca->ref);
1347                         goto err;
1348                 }
1349         }
1350
1351         bch2_inode_init(c, &root_inode, 0, 0,
1352                         S_IFDIR|S_IRWXU|S_IRUGO|S_IXUGO, 0, NULL);
1353         root_inode.bi_inum = BCACHEFS_ROOT_INO;
1354         bch2_inode_pack(c, &packed_inode, &root_inode);
1355         packed_inode.inode.k.p.snapshot = U32_MAX;
1356
1357         err = "error creating root directory";
1358         ret = bch2_btree_insert(c, BTREE_ID_inodes,
1359                                 &packed_inode.inode.k_i,
1360                                 NULL, NULL, 0);
1361         if (ret)
1362                 goto err;
1363
1364         bch2_inode_init_early(c, &lostfound_inode);
1365
1366         err = "error creating lost+found";
1367         ret = bch2_trans_do(c, NULL, NULL, 0,
1368                 bch2_create_trans(&trans, BCACHEFS_ROOT_INO,
1369                                   &root_inode, &lostfound_inode,
1370                                   &lostfound,
1371                                   0, 0, S_IFDIR|0700, 0,
1372                                   NULL, NULL));
1373         if (ret) {
1374                 bch_err(c, "error creating lost+found");
1375                 goto err;
1376         }
1377
1378         if (enabled_qtypes(c)) {
1379                 ret = bch2_fs_quota_read(c);
1380                 if (ret)
1381                         goto err;
1382         }
1383
1384         err = "error writing first journal entry";
1385         ret = bch2_journal_meta(&c->journal);
1386         if (ret)
1387                 goto err;
1388
1389         mutex_lock(&c->sb_lock);
1390         SET_BCH_SB_INITIALIZED(c->disk_sb.sb, true);
1391         SET_BCH_SB_CLEAN(c->disk_sb.sb, false);
1392
1393         bch2_write_super(c);
1394         mutex_unlock(&c->sb_lock);
1395
1396         return 0;
1397 err:
1398         pr_err("Error initializing new filesystem: %s (%i)", err, ret);
1399         return ret;
1400 }