git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/journal_io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "bcachefs.h"
   3 #include "alloc_foreground.h"
   4 #include "buckets.h"
   5 #include "checksum.h"
   6 #include "error.h"
   7 #include "journal.h"
   8 #include "journal_io.h"
   9 #include "journal_reclaim.h"
  10 #include "replicas.h"
  11
  12 #include <trace/events/bcachefs.h>
  13
  14 struct journal_list {
  15         struct closure          cl;
  16         struct mutex            lock;
  17         struct list_head        *head;
  18         int                     ret;
  19 };
  20
  21 #define JOURNAL_ENTRY_ADD_OK            0
  22 #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE  5
  23
  24 /*
  25  * Given a journal entry we just read, add it to the list of journal entries to
  26  * be replayed:
  27  */
  28 static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca,
  29                              struct journal_list *jlist, struct jset *j)
  30 {
  31         struct journal_replay *i, *pos;
  32         struct list_head *where;
  33         size_t bytes = vstruct_bytes(j);
  34         __le64 last_seq;
  35         int ret;
  36
  37         last_seq = !list_empty(jlist->head)
  38                 ? list_last_entry(jlist->head, struct journal_replay,
  39                                   list)->j.last_seq
  40                 : 0;
  41
  42         /* Is this entry older than the range we need? */
  43         if (le64_to_cpu(j->seq) < le64_to_cpu(last_seq)) {
  44                 ret = JOURNAL_ENTRY_ADD_OUT_OF_RANGE;
  45                 goto out;
  46         }
  47
  48         /* Drop entries we don't need anymore */
  49         list_for_each_entry_safe(i, pos, jlist->head, list) {
  50                 if (le64_to_cpu(i->j.seq) >= le64_to_cpu(j->last_seq))
  51                         break;
  52                 list_del(&i->list);
  53                 kvpfree(i, offsetof(struct journal_replay, j) +
  54                         vstruct_bytes(&i->j));
  55         }
  56
  57         list_for_each_entry_reverse(i, jlist->head, list) {
  58                 /* Duplicate? */
  59                 if (le64_to_cpu(j->seq) == le64_to_cpu(i->j.seq)) {
  60                         fsck_err_on(bytes != vstruct_bytes(&i->j) ||
  61                                     memcmp(j, &i->j, bytes), c,
  62                                     "found duplicate but non identical journal entries (seq %llu)",
  63                                     le64_to_cpu(j->seq));
  64                         goto found;
  65                 }
  66
  67                 if (le64_to_cpu(j->seq) > le64_to_cpu(i->j.seq)) {
  68                         where = &i->list;
  69                         goto add;
  70                 }
  71         }
  72
  73         where = jlist->head;
  74 add:
  75         i = kvpmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL);
  76         if (!i) {
  77                 ret = -ENOMEM;
  78                 goto out;
  79         }
  80
  81         list_add(&i->list, where);
  82         i->devs.nr = 0;
  83         memcpy(&i->j, j, bytes);
  84 found:
  85         if (!bch2_dev_list_has_dev(i->devs, ca->dev_idx))
  86                 bch2_dev_list_add_dev(&i->devs, ca->dev_idx);
  87         else
  88                 fsck_err_on(1, c, "duplicate journal entries on same device");
  89         ret = JOURNAL_ENTRY_ADD_OK;
  90 out:
  91 fsck_err:
  92         return ret;
  93 }
  94
  95 static struct nonce journal_nonce(const struct jset *jset)
  96 {
  97         return (struct nonce) {{
  98                 [0] = 0,
  99                 [1] = ((__le32 *) &jset->seq)[0],
 100                 [2] = ((__le32 *) &jset->seq)[1],
 101                 [3] = BCH_NONCE_JOURNAL,
 102         }};
 103 }
 104
 105 /* this fills in a range with empty jset_entries: */
 106 static void journal_entry_null_range(void *start, void *end)
 107 {
 108         struct jset_entry *entry;
 109
 110         for (entry = start; entry != end; entry = vstruct_next(entry))
 111                 memset(entry, 0, sizeof(*entry));
 112 }
 113
 114 #define JOURNAL_ENTRY_REREAD    5
 115 #define JOURNAL_ENTRY_NONE      6
 116 #define JOURNAL_ENTRY_BAD       7
 117
 118 #define journal_entry_err(c, msg, ...)                                  \
 119 ({                                                                      \
 120         switch (write) {                                                \
 121         case READ:                                                      \
 122                 mustfix_fsck_err(c, msg, ##__VA_ARGS__);                \
 123                 break;                                                  \
 124         case WRITE:                                                     \
 125                 bch_err(c, "corrupt metadata before write:\n"           \
 126                         msg, ##__VA_ARGS__);                            \
 127                 if (bch2_fs_inconsistent(c)) {                          \
 128                         ret = BCH_FSCK_ERRORS_NOT_FIXED;                \
 129                         goto fsck_err;                                  \
 130                 }                                                       \
 131                 break;                                                  \
 132         }                                                               \
 133         true;                                                           \
 134 })
 135
 136 #define journal_entry_err_on(cond, c, msg, ...)                         \
 137         ((cond) ? journal_entry_err(c, msg, ##__VA_ARGS__) : false)
 138
 139 static int journal_validate_key(struct bch_fs *c, struct jset *jset,
 140                                 struct jset_entry *entry,
 141                                 struct bkey_i *k, enum btree_node_type key_type,
 142                                 const char *type, int write)
 143 {
 144         void *next = vstruct_next(entry);
 145         const char *invalid;
 146         unsigned version = le32_to_cpu(jset->version);
 147         int ret = 0;
 148
 149         if (journal_entry_err_on(!k->k.u64s, c,
 150                         "invalid %s in journal: k->u64s 0", type)) {
 151                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 152                 journal_entry_null_range(vstruct_next(entry), next);
 153                 return 0;
 154         }
 155
 156         if (journal_entry_err_on((void *) bkey_next(k) >
 157                                 (void *) vstruct_next(entry), c,
 158                         "invalid %s in journal: extends past end of journal entry",
 159                         type)) {
 160                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 161                 journal_entry_null_range(vstruct_next(entry), next);
 162                 return 0;
 163         }
 164
 165         if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT, c,
 166                         "invalid %s in journal: bad format %u",
 167                         type, k->k.format)) {
 168                 le16_add_cpu(&entry->u64s, -k->k.u64s);
 169                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 170                 journal_entry_null_range(vstruct_next(entry), next);
 171                 return 0;
 172         }
 173
 174         if (JSET_BIG_ENDIAN(jset) != CPU_BIG_ENDIAN)
 175                 bch2_bkey_swab(NULL, bkey_to_packed(k));
 176
 177         if (!write &&
 178             version < bcachefs_metadata_version_bkey_renumber)
 179                 bch2_bkey_renumber(key_type, bkey_to_packed(k), write);
 180
 181         invalid = bch2_bkey_invalid(c, bkey_i_to_s_c(k), key_type);
 182         if (invalid) {
 183                 char buf[160];
 184
 185                 bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(k));
 186                 mustfix_fsck_err(c, "invalid %s in journal: %s\n%s",
 187                                  type, invalid, buf);
 188
 189                 le16_add_cpu(&entry->u64s, -k->k.u64s);
 190                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 191                 journal_entry_null_range(vstruct_next(entry), next);
 192                 return 0;
 193         }
 194
 195         if (write &&
 196             version < bcachefs_metadata_version_bkey_renumber)
 197                 bch2_bkey_renumber(key_type, bkey_to_packed(k), write);
 198 fsck_err:
 199         return ret;
 200 }
 201
 202 static int journal_entry_validate_btree_keys(struct bch_fs *c,
 203                                              struct jset *jset,
 204                                              struct jset_entry *entry,
 205                                              int write)
 206 {
 207         struct bkey_i *k;
 208
 209         vstruct_for_each(entry, k) {
 210                 int ret = journal_validate_key(c, jset, entry, k,
 211                                 __btree_node_type(entry->level,
 212                                                   entry->btree_id),
 213                                 "key", write);
 214                 if (ret)
 215                         return ret;
 216         }
 217
 218         return 0;
 219 }
 220
 221 static int journal_entry_validate_btree_root(struct bch_fs *c,
 222                                              struct jset *jset,
 223                                              struct jset_entry *entry,
 224                                              int write)
 225 {
 226         struct bkey_i *k = entry->start;
 227         int ret = 0;
 228
 229         if (journal_entry_err_on(!entry->u64s ||
 230                                  le16_to_cpu(entry->u64s) != k->k.u64s, c,
 231                                  "invalid btree root journal entry: wrong number of keys")) {
 232                 void *next = vstruct_next(entry);
 233                 /*
 234                  * we don't want to null out this jset_entry,
 235                  * just the contents, so that later we can tell
 236                  * we were _supposed_ to have a btree root
 237                  */
 238                 entry->u64s = 0;
 239                 journal_entry_null_range(vstruct_next(entry), next);
 240                 return 0;
 241         }
 242
 243         return journal_validate_key(c, jset, entry, k, BKEY_TYPE_BTREE,
 244                                     "btree root", write);
 245 fsck_err:
 246         return ret;
 247 }
 248
 249 static int journal_entry_validate_prio_ptrs(struct bch_fs *c,
 250                                             struct jset *jset,
 251                                             struct jset_entry *entry,
 252                                             int write)
 253 {
 254         /* obsolete, don't care: */
 255         return 0;
 256 }
 257
 258 static int journal_entry_validate_blacklist(struct bch_fs *c,
 259                                             struct jset *jset,
 260                                             struct jset_entry *entry,
 261                                             int write)
 262 {
 263         int ret = 0;
 264
 265         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1, c,
 266                 "invalid journal seq blacklist entry: bad size")) {
 267                 journal_entry_null_range(entry, vstruct_next(entry));
 268         }
 269 fsck_err:
 270         return ret;
 271 }
 272
 273 static int journal_entry_validate_blacklist_v2(struct bch_fs *c,
 274                                                struct jset *jset,
 275                                                struct jset_entry *entry,
 276                                                int write)
 277 {
 278         struct jset_entry_blacklist_v2 *bl_entry;
 279         int ret = 0;
 280
 281         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2, c,
 282                 "invalid journal seq blacklist entry: bad size")) {
 283                 journal_entry_null_range(entry, vstruct_next(entry));
 284                 goto out;
 285         }
 286
 287         bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry);
 288
 289         if (journal_entry_err_on(le64_to_cpu(bl_entry->start) >
 290                                  le64_to_cpu(bl_entry->end), c,
 291                 "invalid journal seq blacklist entry: start > end")) {
 292                 journal_entry_null_range(entry, vstruct_next(entry));
 293         }
 294 out:
 295 fsck_err:
 296         return ret;
 297 }
 298
 299 static int journal_entry_validate_usage(struct bch_fs *c,
 300                                         struct jset *jset,
 301                                         struct jset_entry *entry,
 302                                         int write)
 303 {
 304         struct jset_entry_usage *u =
 305                 container_of(entry, struct jset_entry_usage, entry);
 306         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 307         int ret = 0;
 308
 309         if (journal_entry_err_on(bytes < sizeof(*u),
 310                                  c,
 311                                  "invalid journal entry usage: bad size")) {
 312                 journal_entry_null_range(entry, vstruct_next(entry));
 313                 return ret;
 314         }
 315
 316 fsck_err:
 317         return ret;
 318 }
 319
 320 static int journal_entry_validate_data_usage(struct bch_fs *c,
 321                                         struct jset *jset,
 322                                         struct jset_entry *entry,
 323                                         int write)
 324 {
 325         struct jset_entry_data_usage *u =
 326                 container_of(entry, struct jset_entry_data_usage, entry);
 327         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 328         int ret = 0;
 329
 330         if (journal_entry_err_on(bytes < sizeof(*u) ||
 331                                  bytes < sizeof(*u) + u->r.nr_devs,
 332                                  c,
 333                                  "invalid journal entry usage: bad size")) {
 334                 journal_entry_null_range(entry, vstruct_next(entry));
 335                 return ret;
 336         }
 337
 338 fsck_err:
 339         return ret;
 340 }
 341
 342 struct jset_entry_ops {
 343         int (*validate)(struct bch_fs *, struct jset *,
 344                         struct jset_entry *, int);
 345 };
 346
 347 static const struct jset_entry_ops bch2_jset_entry_ops[] = {
 348 #define x(f, nr)                                                \
 349         [BCH_JSET_ENTRY_##f]    = (struct jset_entry_ops) {     \
 350                 .validate       = journal_entry_validate_##f,   \
 351         },
 352         BCH_JSET_ENTRY_TYPES()
 353 #undef x
 354 };
 355
 356 static int journal_entry_validate(struct bch_fs *c, struct jset *jset,
 357                                   struct jset_entry *entry, int write)
 358 {
 359         return entry->type < BCH_JSET_ENTRY_NR
 360                 ? bch2_jset_entry_ops[entry->type].validate(c, jset,
 361                                                             entry, write)
 362                 : 0;
 363 }
 364
 365 static int jset_validate_entries(struct bch_fs *c, struct jset *jset,
 366                                  int write)
 367 {
 368         struct jset_entry *entry;
 369         int ret = 0;
 370
 371         vstruct_for_each(jset, entry) {
 372                 if (journal_entry_err_on(vstruct_next(entry) >
 373                                          vstruct_last(jset), c,
 374                                 "journal entry extends past end of jset")) {
 375                         jset->u64s = cpu_to_le32((u64 *) entry - jset->_data);
 376                         break;
 377                 }
 378
 379                 ret = journal_entry_validate(c, jset, entry, write);
 380                 if (ret)
 381                         break;
 382         }
 383 fsck_err:
 384         return ret;
 385 }
 386
 387 static int jset_validate(struct bch_fs *c,
 388                          struct jset *jset, u64 sector,
 389                          unsigned bucket_sectors_left,
 390                          unsigned sectors_read,
 391                          int write)
 392 {
 393         size_t bytes = vstruct_bytes(jset);
 394         struct bch_csum csum;
 395         unsigned version;
 396         int ret = 0;
 397
 398         if (le64_to_cpu(jset->magic) != jset_magic(c))
 399                 return JOURNAL_ENTRY_NONE;
 400
 401         version = le32_to_cpu(jset->version);
 402         if ((version != BCH_JSET_VERSION_OLD &&
 403              version < bcachefs_metadata_version_min) ||
 404             version >= bcachefs_metadata_version_max) {
 405                 bch_err(c, "unknown journal entry version %u", jset->version);
 406                 return BCH_FSCK_UNKNOWN_VERSION;
 407         }
 408
 409         if (journal_entry_err_on(bytes > bucket_sectors_left << 9, c,
 410                                  "journal entry too big (%zu bytes), sector %lluu",
 411                                  bytes, sector)) {
 412                 /* XXX: note we might have missing journal entries */
 413                 return JOURNAL_ENTRY_BAD;
 414         }
 415
 416         if (bytes > sectors_read << 9)
 417                 return JOURNAL_ENTRY_REREAD;
 418
 419         if (fsck_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)), c,
 420                         "journal entry with unknown csum type %llu sector %lluu",
 421                         JSET_CSUM_TYPE(jset), sector))
 422                 return JOURNAL_ENTRY_BAD;
 423
 424         csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), jset);
 425         if (journal_entry_err_on(bch2_crc_cmp(csum, jset->csum), c,
 426                                  "journal checksum bad, sector %llu", sector)) {
 427                 /* XXX: retry IO, when we start retrying checksum errors */
 428                 /* XXX: note we might have missing journal entries */
 429                 return JOURNAL_ENTRY_BAD;
 430         }
 431
 432         bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
 433                      jset->encrypted_start,
 434                      vstruct_end(jset) - (void *) jset->encrypted_start);
 435
 436         if (journal_entry_err_on(le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq), c,
 437                                  "invalid journal entry: last_seq > seq"))
 438                 jset->last_seq = jset->seq;
 439
 440         return 0;
 441 fsck_err:
 442         return ret;
 443 }
 444
 445 struct journal_read_buf {
 446         void            *data;
 447         size_t          size;
 448 };
 449
 450 static int journal_read_buf_realloc(struct journal_read_buf *b,
 451                                     size_t new_size)
 452 {
 453         void *n;
 454
 455         /* the bios are sized for this many pages, max: */
 456         if (new_size > JOURNAL_ENTRY_SIZE_MAX)
 457                 return -ENOMEM;
 458
 459         new_size = roundup_pow_of_two(new_size);
 460         n = kvpmalloc(new_size, GFP_KERNEL);
 461         if (!n)
 462                 return -ENOMEM;
 463
 464         kvpfree(b->data, b->size);
 465         b->data = n;
 466         b->size = new_size;
 467         return 0;
 468 }
 469
 470 static int journal_read_bucket(struct bch_dev *ca,
 471                                struct journal_read_buf *buf,
 472                                struct journal_list *jlist,
 473                                unsigned bucket)
 474 {
 475         struct bch_fs *c = ca->fs;
 476         struct journal_device *ja = &ca->journal;
 477         struct jset *j = NULL;
 478         unsigned sectors, sectors_read = 0;
 479         u64 offset = bucket_to_sector(ca, ja->buckets[bucket]),
 480             end = offset + ca->mi.bucket_size;
 481         bool saw_bad = false;
 482         int ret = 0;
 483
 484         pr_debug("reading %u", bucket);
 485
 486         while (offset < end) {
 487                 if (!sectors_read) {
 488                         struct bio *bio;
 489 reread:
 490                         sectors_read = min_t(unsigned,
 491                                 end - offset, buf->size >> 9);
 492
 493                         bio = bio_kmalloc(GFP_KERNEL,
 494                                           buf_pages(buf->data,
 495                                                     sectors_read << 9));
 496                         bio_set_dev(bio, ca->disk_sb.bdev);
 497                         bio->bi_iter.bi_sector  = offset;
 498                         bio_set_op_attrs(bio, REQ_OP_READ, 0);
 499                         bch2_bio_map(bio, buf->data, sectors_read << 9);
 500
 501                         ret = submit_bio_wait(bio);
 502                         bio_put(bio);
 503
 504                         if (bch2_dev_io_err_on(ret, ca,
 505                                                "journal read from sector %llu",
 506                                                offset) ||
 507                             bch2_meta_read_fault("journal"))
 508                                 return -EIO;
 509
 510                         j = buf->data;
 511                 }
 512
 513                 ret = jset_validate(c, j, offset,
 514                                     end - offset, sectors_read,
 515                                     READ);
 516                 switch (ret) {
 517                 case BCH_FSCK_OK:
 518                         break;
 519                 case JOURNAL_ENTRY_REREAD:
 520                         if (vstruct_bytes(j) > buf->size) {
 521                                 ret = journal_read_buf_realloc(buf,
 522                                                         vstruct_bytes(j));
 523                                 if (ret)
 524                                         return ret;
 525                         }
 526                         goto reread;
 527                 case JOURNAL_ENTRY_NONE:
 528                         if (!saw_bad)
 529                                 return 0;
 530                         sectors = c->opts.block_size;
 531                         goto next_block;
 532                 case JOURNAL_ENTRY_BAD:
 533                         saw_bad = true;
 534                         sectors = c->opts.block_size;
 535                         goto next_block;
 536                 default:
 537                         return ret;
 538                 }
 539
 540                 /*
 541                  * This happens sometimes if we don't have discards on -
 542                  * when we've partially overwritten a bucket with new
 543                  * journal entries. We don't need the rest of the
 544                  * bucket:
 545                  */
 546                 if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])
 547                         return 0;
 548
 549                 ja->bucket_seq[bucket] = le64_to_cpu(j->seq);
 550
 551                 mutex_lock(&jlist->lock);
 552                 ret = journal_entry_add(c, ca, jlist, j);
 553                 mutex_unlock(&jlist->lock);
 554
 555                 switch (ret) {
 556                 case JOURNAL_ENTRY_ADD_OK:
 557                         break;
 558                 case JOURNAL_ENTRY_ADD_OUT_OF_RANGE:
 559                         break;
 560                 default:
 561                         return ret;
 562                 }
 563
 564                 sectors = vstruct_sectors(j, c->block_bits);
 565 next_block:
 566                 pr_debug("next");
 567                 offset          += sectors;
 568                 sectors_read    -= sectors;
 569                 j = ((void *) j) + (sectors << 9);
 570         }
 571
 572         return 0;
 573 }
 574
 575 static void bch2_journal_read_device(struct closure *cl)
 576 {
 577         struct journal_device *ja =
 578                 container_of(cl, struct journal_device, read);
 579         struct bch_dev *ca = container_of(ja, struct bch_dev, journal);
 580         struct journal_list *jlist =
 581                 container_of(cl->parent, struct journal_list, cl);
 582         struct journal_read_buf buf = { NULL, 0 };
 583         u64 min_seq = U64_MAX;
 584         unsigned i;
 585         int ret;
 586
 587         if (!ja->nr)
 588                 goto out;
 589
 590         ret = journal_read_buf_realloc(&buf, PAGE_SIZE);
 591         if (ret)
 592                 goto err;
 593
 594         pr_debug("%u journal buckets", ja->nr);
 595
 596         for (i = 0; i < ja->nr; i++) {
 597                 ret = journal_read_bucket(ca, &buf, jlist, i);
 598                 if (ret)
 599                         goto err;
 600         }
 601
 602         /* Find the journal bucket with the highest sequence number: */
 603         for (i = 0; i < ja->nr; i++) {
 604                 if (ja->bucket_seq[i] > ja->bucket_seq[ja->cur_idx])
 605                         ja->cur_idx = i;
 606
 607                 min_seq = min(ja->bucket_seq[i], min_seq);
 608         }
 609
 610         /*
 611          * If there's duplicate journal entries in multiple buckets (which
 612          * definitely isn't supposed to happen, but...) - make sure to start
 613          * cur_idx at the last of those buckets, so we don't deadlock trying to
 614          * allocate
 615          */
 616         while (ja->bucket_seq[ja->cur_idx] > min_seq &&
 617                ja->bucket_seq[ja->cur_idx] >
 618                ja->bucket_seq[(ja->cur_idx + 1) % ja->nr])
 619                 ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
 620
 621         ja->sectors_free = 0;
 622
 623         /*
 624          * Set dirty_idx to indicate the entire journal is full and needs to be
 625          * reclaimed - journal reclaim will immediately reclaim whatever isn't
 626          * pinned when it first runs:
 627          */
 628         ja->discard_idx = ja->dirty_idx_ondisk =
 629                 ja->dirty_idx = (ja->cur_idx + 1) % ja->nr;
 630 out:
 631         kvpfree(buf.data, buf.size);
 632         percpu_ref_put(&ca->io_ref);
 633         closure_return(cl);
 634         return;
 635 err:
 636         mutex_lock(&jlist->lock);
 637         jlist->ret = ret;
 638         mutex_unlock(&jlist->lock);
 639         goto out;
 640 }
 641
 642 int bch2_journal_read(struct bch_fs *c, struct list_head *list)
 643 {
 644         struct journal_list jlist;
 645         struct journal_replay *i;
 646         struct bch_dev *ca;
 647         unsigned iter;
 648         size_t keys = 0, entries = 0;
 649         bool degraded = false;
 650         int ret = 0;
 651
 652         closure_init_stack(&jlist.cl);
 653         mutex_init(&jlist.lock);
 654         jlist.head = list;
 655         jlist.ret = 0;
 656
 657         for_each_member_device(ca, c, iter) {
 658                 if (!test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) &&
 659                     !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_JOURNAL)))
 660                         continue;
 661
 662                 if ((ca->mi.state == BCH_MEMBER_STATE_RW ||
 663                      ca->mi.state == BCH_MEMBER_STATE_RO) &&
 664                     percpu_ref_tryget(&ca->io_ref))
 665                         closure_call(&ca->journal.read,
 666                                      bch2_journal_read_device,
 667                                      system_unbound_wq,
 668                                      &jlist.cl);
 669                 else
 670                         degraded = true;
 671         }
 672
 673         closure_sync(&jlist.cl);
 674
 675         if (jlist.ret)
 676                 return jlist.ret;
 677
 678         list_for_each_entry(i, list, list) {
 679                 struct jset_entry *entry;
 680                 struct bkey_i *k, *_n;
 681                 struct bch_replicas_padded replicas;
 682                 char buf[80];
 683
 684                 ret = jset_validate_entries(c, &i->j, READ);
 685                 if (ret)
 686                         goto fsck_err;
 687
 688                 /*
 689                  * If we're mounting in degraded mode - if we didn't read all
 690                  * the devices - this is wrong:
 691                  */
 692
 693                 bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL, i->devs);
 694
 695                 if (!degraded &&
 696                     (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
 697                      fsck_err_on(!bch2_replicas_marked(c, &replicas.e, false), c,
 698                                  "superblock not marked as containing replicas %s",
 699                                  (bch2_replicas_entry_to_text(&PBUF(buf),
 700                                                               &replicas.e), buf)))) {
 701                         ret = bch2_mark_replicas(c, &replicas.e);
 702                         if (ret)
 703                                 return ret;
 704                 }
 705
 706                 for_each_jset_key(k, _n, entry, &i->j)
 707                         keys++;
 708                 entries++;
 709         }
 710
 711         if (!list_empty(list)) {
 712                 i = list_last_entry(list, struct journal_replay, list);
 713
 714                 bch_info(c, "journal read done, %zu keys in %zu entries, seq %llu",
 715                          keys, entries, le64_to_cpu(i->j.seq));
 716         }
 717 fsck_err:
 718         return ret;
 719 }
 720
 721 /* journal write: */
 722
 723 static void __journal_write_alloc(struct journal *j,
 724                                   struct journal_buf *w,
 725                                   struct dev_alloc_list *devs_sorted,
 726                                   unsigned sectors,
 727                                   unsigned *replicas,
 728                                   unsigned replicas_want)
 729 {
 730         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 731         struct journal_device *ja;
 732         struct bch_dev *ca;
 733         unsigned i;
 734
 735         if (*replicas >= replicas_want)
 736                 return;
 737
 738         for (i = 0; i < devs_sorted->nr; i++) {
 739                 ca = rcu_dereference(c->devs[devs_sorted->devs[i]]);
 740                 if (!ca)
 741                         continue;
 742
 743                 ja = &ca->journal;
 744
 745                 /*
 746                  * Check that we can use this device, and aren't already using
 747                  * it:
 748                  */
 749                 if (!ca->mi.durability ||
 750                     ca->mi.state != BCH_MEMBER_STATE_RW ||
 751                     !ja->nr ||
 752                     bch2_bkey_has_device(bkey_i_to_s_c(&w->key),
 753                                          ca->dev_idx) ||
 754                     sectors > ja->sectors_free)
 755                         continue;
 756
 757                 bch2_dev_stripe_increment(c, ca, &j->wp.stripe);
 758
 759                 bch2_bkey_append_ptr(&w->key,
 760                         (struct bch_extent_ptr) {
 761                                   .offset = bucket_to_sector(ca,
 762                                         ja->buckets[ja->cur_idx]) +
 763                                         ca->mi.bucket_size -
 764                                         ja->sectors_free,
 765                                   .dev = ca->dev_idx,
 766                 });
 767
 768                 ja->sectors_free -= sectors;
 769                 ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
 770
 771                 *replicas += ca->mi.durability;
 772
 773                 if (*replicas >= replicas_want)
 774                         break;
 775         }
 776 }
 777
 778 /**
 779  * journal_next_bucket - move on to the next journal bucket if possible
 780  */
 781 static int journal_write_alloc(struct journal *j, struct journal_buf *w,
 782                                unsigned sectors)
 783 {
 784         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 785         struct journal_device *ja;
 786         struct bch_dev *ca;
 787         struct dev_alloc_list devs_sorted;
 788         unsigned i, replicas = 0, replicas_want =
 789                 READ_ONCE(c->opts.metadata_replicas);
 790
 791         rcu_read_lock();
 792
 793         devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe,
 794                                           &c->rw_devs[BCH_DATA_JOURNAL]);
 795
 796         __journal_write_alloc(j, w, &devs_sorted,
 797                               sectors, &replicas, replicas_want);
 798
 799         if (replicas >= replicas_want)
 800                 goto done;
 801
 802         for (i = 0; i < devs_sorted.nr; i++) {
 803                 ca = rcu_dereference(c->devs[devs_sorted.devs[i]]);
 804                 if (!ca)
 805                         continue;
 806
 807                 ja = &ca->journal;
 808
 809                 if (sectors > ja->sectors_free &&
 810                     sectors <= ca->mi.bucket_size &&
 811                     bch2_journal_dev_buckets_available(j, ja,
 812                                         journal_space_discarded)) {
 813                         ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
 814                         ja->sectors_free = ca->mi.bucket_size;
 815
 816                         /*
 817                          * ja->bucket_seq[ja->cur_idx] must always have
 818                          * something sensible:
 819                          */
 820                         ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
 821                 }
 822         }
 823
 824         __journal_write_alloc(j, w, &devs_sorted,
 825                               sectors, &replicas, replicas_want);
 826 done:
 827         rcu_read_unlock();
 828
 829         return replicas >= c->opts.metadata_replicas_required ? 0 : -EROFS;
 830 }
 831
 832 static void journal_write_compact(struct jset *jset)
 833 {
 834         struct jset_entry *i, *next, *prev = NULL;
 835
 836         /*
 837          * Simple compaction, dropping empty jset_entries (from journal
 838          * reservations that weren't fully used) and merging jset_entries that
 839          * can be.
 840          *
 841          * If we wanted to be really fancy here, we could sort all the keys in
 842          * the jset and drop keys that were overwritten - probably not worth it:
 843          */
 844         vstruct_for_each_safe(jset, i, next) {
 845                 unsigned u64s = le16_to_cpu(i->u64s);
 846
 847                 /* Empty entry: */
 848                 if (!u64s)
 849                         continue;
 850
 851                 /* Can we merge with previous entry? */
 852                 if (prev &&
 853                     i->btree_id == prev->btree_id &&
 854                     i->level    == prev->level &&
 855                     i->type     == prev->type &&
 856                     i->type     == BCH_JSET_ENTRY_btree_keys &&
 857                     le16_to_cpu(prev->u64s) + u64s <= U16_MAX) {
 858                         memmove_u64s_down(vstruct_next(prev),
 859                                           i->_data,
 860                                           u64s);
 861                         le16_add_cpu(&prev->u64s, u64s);
 862                         continue;
 863                 }
 864
 865                 /* Couldn't merge, move i into new position (after prev): */
 866                 prev = prev ? vstruct_next(prev) : jset->start;
 867                 if (i != prev)
 868                         memmove_u64s_down(prev, i, jset_u64s(u64s));
 869         }
 870
 871         prev = prev ? vstruct_next(prev) : jset->start;
 872         jset->u64s = cpu_to_le32((u64 *) prev - jset->_data);
 873 }
 874
 875 static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)
 876 {
 877         /* we aren't holding j->lock: */
 878         unsigned new_size = READ_ONCE(j->buf_size_want);
 879         void *new_buf;
 880
 881         if (buf->buf_size >= new_size)
 882                 return;
 883
 884         new_buf = kvpmalloc(new_size, GFP_NOIO|__GFP_NOWARN);
 885         if (!new_buf)
 886                 return;
 887
 888         memcpy(new_buf, buf->data, buf->buf_size);
 889         kvpfree(buf->data, buf->buf_size);
 890         buf->data       = new_buf;
 891         buf->buf_size   = new_size;
 892 }
 893
 894 static void journal_write_done(struct closure *cl)
 895 {
 896         struct journal *j = container_of(cl, struct journal, io);
 897         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 898         struct journal_buf *w = journal_prev_buf(j);
 899         struct bch_devs_list devs =
 900                 bch2_bkey_devs(bkey_i_to_s_c(&w->key));
 901         struct bch_replicas_padded replicas;
 902         u64 seq = le64_to_cpu(w->data->seq);
 903         u64 last_seq = le64_to_cpu(w->data->last_seq);
 904
 905         bch2_time_stats_update(j->write_time, j->write_start_time);
 906
 907         if (!devs.nr) {
 908                 bch_err(c, "unable to write journal to sufficient devices");
 909                 goto err;
 910         }
 911
 912         bch2_devlist_to_replicas(&replicas.e, BCH_DATA_JOURNAL, devs);
 913
 914         if (bch2_mark_replicas(c, &replicas.e))
 915                 goto err;
 916
 917         spin_lock(&j->lock);
 918         if (seq >= j->pin.front)
 919                 journal_seq_pin(j, seq)->devs = devs;
 920
 921         j->seq_ondisk           = seq;
 922         j->last_seq_ondisk      = last_seq;
 923         bch2_journal_space_available(j);
 924
 925         /*
 926          * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard
 927          * more buckets:
 928          *
 929          * Must come before signaling write completion, for
 930          * bch2_fs_journal_stop():
 931          */
 932         mod_delayed_work(c->journal_reclaim_wq, &j->reclaim_work, 0);
 933 out:
 934         /* also must come before signalling write completion: */
 935         closure_debug_destroy(cl);
 936
 937         BUG_ON(!j->reservations.prev_buf_unwritten);
 938         atomic64_sub(((union journal_res_state) { .prev_buf_unwritten = 1 }).v,
 939                      &j->reservations.counter);
 940
 941         closure_wake_up(&w->wait);
 942         journal_wake(j);
 943
 944         if (test_bit(JOURNAL_NEED_WRITE, &j->flags))
 945                 mod_delayed_work(system_freezable_wq, &j->write_work, 0);
 946         spin_unlock(&j->lock);
 947         return;
 948 err:
 949         bch2_fatal_error(c);
 950         spin_lock(&j->lock);
 951         goto out;
 952 }
 953
 954 static void journal_write_endio(struct bio *bio)
 955 {
 956         struct bch_dev *ca = bio->bi_private;
 957         struct journal *j = &ca->fs->journal;
 958
 959         if (bch2_dev_io_err_on(bio->bi_status, ca, "journal write") ||
 960             bch2_meta_write_fault("journal")) {
 961                 struct journal_buf *w = journal_prev_buf(j);
 962                 unsigned long flags;
 963
 964                 spin_lock_irqsave(&j->err_lock, flags);
 965                 bch2_bkey_drop_device(bkey_i_to_s(&w->key), ca->dev_idx);
 966                 spin_unlock_irqrestore(&j->err_lock, flags);
 967         }
 968
 969         closure_put(&j->io);
 970         percpu_ref_put(&ca->io_ref);
 971 }
 972
 973 void bch2_journal_write(struct closure *cl)
 974 {
 975         struct journal *j = container_of(cl, struct journal, io);
 976         struct bch_fs *c = container_of(j, struct bch_fs, journal);
 977         struct bch_dev *ca;
 978         struct journal_buf *w = journal_prev_buf(j);
 979         struct jset_entry *start, *end;
 980         struct jset *jset;
 981         struct bio *bio;
 982         struct bch_extent_ptr *ptr;
 983         bool validate_before_checksum = false;
 984         unsigned i, sectors, bytes, u64s;
 985         int ret;
 986
 987         bch2_journal_pin_put(j, le64_to_cpu(w->data->seq));
 988
 989         journal_buf_realloc(j, w);
 990         jset = w->data;
 991
 992         j->write_start_time = local_clock();
 993
 994         start   = vstruct_last(jset);
 995         end     = bch2_journal_super_entries_add_common(c, start,
 996                                                 le64_to_cpu(jset->seq));
 997         u64s    = (u64 *) end - (u64 *) start;
 998         BUG_ON(u64s > j->entry_u64s_reserved);
 999
1000         le32_add_cpu(&jset->u64s, u64s);
1001         BUG_ON(vstruct_sectors(jset, c->block_bits) > w->sectors);
1002
1003         journal_write_compact(jset);
1004
1005         jset->read_clock        = cpu_to_le16(c->bucket_clock[READ].hand);
1006         jset->write_clock       = cpu_to_le16(c->bucket_clock[WRITE].hand);
1007         jset->magic             = cpu_to_le64(jset_magic(c));
1008
1009         jset->version           = c->sb.version < bcachefs_metadata_version_new_versioning
1010                 ? cpu_to_le32(BCH_JSET_VERSION_OLD)
1011                 : cpu_to_le32(c->sb.version);
1012
1013         SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN);
1014         SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c));
1015
1016         if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset)))
1017                 validate_before_checksum = true;
1018
1019         if (le32_to_cpu(jset->version) <
1020             bcachefs_metadata_version_bkey_renumber)
1021                 validate_before_checksum = true;
1022
1023         if (validate_before_checksum &&
1024             jset_validate_entries(c, jset, WRITE))
1025                 goto err;
1026
1027         bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
1028                     jset->encrypted_start,
1029                     vstruct_end(jset) - (void *) jset->encrypted_start);
1030
1031         jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset),
1032                                   journal_nonce(jset), jset);
1033
1034         if (!validate_before_checksum &&
1035             jset_validate_entries(c, jset, WRITE))
1036                 goto err;
1037
1038         sectors = vstruct_sectors(jset, c->block_bits);
1039         BUG_ON(sectors > w->sectors);
1040
1041         bytes = vstruct_bytes(jset);
1042         memset((void *) jset + bytes, 0, (sectors << 9) - bytes);
1043
1044         spin_lock(&j->lock);
1045         ret = journal_write_alloc(j, w, sectors);
1046
1047         /*
1048          * write is allocated, no longer need to account for it in
1049          * bch2_journal_space_available():
1050          */
1051         w->sectors = 0;
1052
1053         /*
1054          * journal entry has been compacted and allocated, recalculate space
1055          * available:
1056          */
1057         bch2_journal_space_available(j);
1058         spin_unlock(&j->lock);
1059
1060         if (ret) {
1061                 bch_err(c, "Unable to allocate journal write");
1062                 bch2_fatal_error(c);
1063                 continue_at(cl, journal_write_done, system_highpri_wq);
1064                 return;
1065         }
1066
1067         /*
1068          * XXX: we really should just disable the entire journal in nochanges
1069          * mode
1070          */
1071         if (c->opts.nochanges)
1072                 goto no_io;
1073
1074         extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) {
1075                 ca = bch_dev_bkey_exists(c, ptr->dev);
1076                 if (!percpu_ref_tryget(&ca->io_ref)) {
1077                         /* XXX: fix this */
1078                         bch_err(c, "missing device for journal write\n");
1079                         continue;
1080                 }
1081
1082                 this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_JOURNAL],
1083                              sectors);
1084
1085                 bio = ca->journal.bio;
1086                 bio_reset(bio);
1087                 bio_set_dev(bio, ca->disk_sb.bdev);
1088                 bio->bi_iter.bi_sector  = ptr->offset;
1089                 bio->bi_end_io          = journal_write_endio;
1090                 bio->bi_private         = ca;
1091                 bio_set_op_attrs(bio, REQ_OP_WRITE,
1092                                  REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA);
1093                 bch2_bio_map(bio, jset, sectors << 9);
1094
1095                 trace_journal_write(bio);
1096                 closure_bio_submit(bio, cl);
1097
1098                 ca->journal.bucket_seq[ca->journal.cur_idx] = le64_to_cpu(jset->seq);
1099         }
1100
1101         for_each_rw_member(ca, c, i)
1102                 if (journal_flushes_device(ca) &&
1103                     !bch2_extent_has_device(bkey_i_to_s_c_extent(&w->key), i)) {
1104                         percpu_ref_get(&ca->io_ref);
1105
1106                         bio = ca->journal.bio;
1107                         bio_reset(bio);
1108                         bio_set_dev(bio, ca->disk_sb.bdev);
1109                         bio->bi_opf             = REQ_OP_FLUSH;
1110                         bio->bi_end_io          = journal_write_endio;
1111                         bio->bi_private         = ca;
1112                         closure_bio_submit(bio, cl);
1113                 }
1114
1115 no_io:
1116         bch2_bucket_seq_cleanup(c);
1117
1118         continue_at(cl, journal_write_done, system_highpri_wq);
1119         return;
1120 err:
1121         bch2_inconsistent_error(c);
1122         continue_at(cl, journal_write_done, system_highpri_wq);
1123 }