git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/journal_io.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include "bcachefs.h"
   3 #include "alloc_foreground.h"
   4 #include "btree_io.h"
   5 #include "btree_update_interior.h"
   6 #include "buckets.h"
   7 #include "checksum.h"
   8 #include "disk_groups.h"
   9 #include "error.h"
  10 #include "io.h"
  11 #include "journal.h"
  12 #include "journal_io.h"
  13 #include "journal_reclaim.h"
  14 #include "journal_seq_blacklist.h"
  15 #include "replicas.h"
  16
  17 #include <trace/events/bcachefs.h>
  18
  19 static void __journal_replay_free(struct journal_replay *i)
  20 {
  21         list_del(&i->list);
  22         kvpfree(i, offsetof(struct journal_replay, j) +
  23                 vstruct_bytes(&i->j));
  24
  25 }
  26
  27 static void journal_replay_free(struct bch_fs *c, struct journal_replay *i)
  28 {
  29         i->ignore = true;
  30
  31         if (!c->opts.read_entire_journal)
  32                 __journal_replay_free(i);
  33 }
  34
  35 struct journal_list {
  36         struct closure          cl;
  37         struct mutex            lock;
  38         struct list_head        *head;
  39         int                     ret;
  40 };
  41
  42 #define JOURNAL_ENTRY_ADD_OK            0
  43 #define JOURNAL_ENTRY_ADD_OUT_OF_RANGE  5
  44
  45 /*
  46  * Given a journal entry we just read, add it to the list of journal entries to
  47  * be replayed:
  48  */
  49 static int journal_entry_add(struct bch_fs *c, struct bch_dev *ca,
  50                              struct journal_ptr entry_ptr,
  51                              struct journal_list *jlist, struct jset *j,
  52                              bool bad)
  53 {
  54         struct journal_replay *i, *pos, *dup = NULL;
  55         struct journal_ptr *ptr;
  56         struct list_head *where;
  57         size_t bytes = vstruct_bytes(j);
  58         u64 last_seq = 0;
  59         int ret = JOURNAL_ENTRY_ADD_OK;
  60
  61         list_for_each_entry_reverse(i, jlist->head, list) {
  62                 if (!JSET_NO_FLUSH(&i->j)) {
  63                         last_seq = le64_to_cpu(i->j.last_seq);
  64                         break;
  65                 }
  66         }
  67
  68         /* Is this entry older than the range we need? */
  69         if (!c->opts.read_entire_journal &&
  70             le64_to_cpu(j->seq) < last_seq) {
  71                 ret = JOURNAL_ENTRY_ADD_OUT_OF_RANGE;
  72                 goto out;
  73         }
  74
  75         /* Drop entries we don't need anymore */
  76         if (!JSET_NO_FLUSH(j)) {
  77                 list_for_each_entry_safe(i, pos, jlist->head, list) {
  78                         if (le64_to_cpu(i->j.seq) >= le64_to_cpu(j->last_seq))
  79                                 break;
  80                         journal_replay_free(c, i);
  81                 }
  82         }
  83
  84         list_for_each_entry_reverse(i, jlist->head, list) {
  85                 if (le64_to_cpu(j->seq) > le64_to_cpu(i->j.seq)) {
  86                         where = &i->list;
  87                         goto add;
  88                 }
  89         }
  90
  91         where = jlist->head;
  92 add:
  93         dup = where->next != jlist->head
  94                 ? container_of(where->next, struct journal_replay, list)
  95                 : NULL;
  96
  97         if (dup && le64_to_cpu(j->seq) != le64_to_cpu(dup->j.seq))
  98                 dup = NULL;
  99
 100         /*
 101          * Duplicate journal entries? If so we want the one that didn't have a
 102          * checksum error:
 103          */
 104         if (dup) {
 105                 if (dup->bad) {
 106                         /* we'll replace @dup: */
 107                 } else if (bad) {
 108                         i = dup;
 109                         goto found;
 110                 } else {
 111                         fsck_err_on(bytes != vstruct_bytes(&dup->j) ||
 112                                     memcmp(j, &dup->j, bytes), c,
 113                                     "found duplicate but non identical journal entries (seq %llu)",
 114                                     le64_to_cpu(j->seq));
 115                         i = dup;
 116                         goto found;
 117                 }
 118         }
 119
 120         i = kvpmalloc(offsetof(struct journal_replay, j) + bytes, GFP_KERNEL);
 121         if (!i) {
 122                 ret = -ENOMEM;
 123                 goto out;
 124         }
 125
 126         i->nr_ptrs       = 0;
 127         i->bad          = bad;
 128         i->ignore       = false;
 129         memcpy(&i->j, j, bytes);
 130
 131         if (dup) {
 132                 i->nr_ptrs = dup->nr_ptrs;
 133                 memcpy(i->ptrs, dup->ptrs, sizeof(dup->ptrs));
 134                 __journal_replay_free(dup);
 135         }
 136
 137         list_add(&i->list, where);
 138 found:
 139         for (ptr = i->ptrs; ptr < i->ptrs + i->nr_ptrs; ptr++) {
 140                 if (ptr->dev == ca->dev_idx) {
 141                         bch_err(c, "duplicate journal entry %llu on same device",
 142                                 le64_to_cpu(i->j.seq));
 143                         goto out;
 144                 }
 145         }
 146
 147         if (i->nr_ptrs >= ARRAY_SIZE(i->ptrs)) {
 148                 bch_err(c, "found too many copies of journal entry %llu",
 149                         le64_to_cpu(i->j.seq));
 150                 goto out;
 151         }
 152
 153         i->ptrs[i->nr_ptrs++] = entry_ptr;
 154 out:
 155 fsck_err:
 156         return ret;
 157 }
 158
 159 static struct nonce journal_nonce(const struct jset *jset)
 160 {
 161         return (struct nonce) {{
 162                 [0] = 0,
 163                 [1] = ((__le32 *) &jset->seq)[0],
 164                 [2] = ((__le32 *) &jset->seq)[1],
 165                 [3] = BCH_NONCE_JOURNAL,
 166         }};
 167 }
 168
 169 /* this fills in a range with empty jset_entries: */
 170 static void journal_entry_null_range(void *start, void *end)
 171 {
 172         struct jset_entry *entry;
 173
 174         for (entry = start; entry != end; entry = vstruct_next(entry))
 175                 memset(entry, 0, sizeof(*entry));
 176 }
 177
 178 #define JOURNAL_ENTRY_REREAD    5
 179 #define JOURNAL_ENTRY_NONE      6
 180 #define JOURNAL_ENTRY_BAD       7
 181
 182 #define journal_entry_err(c, msg, ...)                                  \
 183 ({                                                                      \
 184         switch (write) {                                                \
 185         case READ:                                                      \
 186                 mustfix_fsck_err(c, msg, ##__VA_ARGS__);                \
 187                 break;                                                  \
 188         case WRITE:                                                     \
 189                 bch_err(c, "corrupt metadata before write:\n"           \
 190                         msg, ##__VA_ARGS__);                            \
 191                 if (bch2_fs_inconsistent(c)) {                          \
 192                         ret = BCH_FSCK_ERRORS_NOT_FIXED;                \
 193                         goto fsck_err;                                  \
 194                 }                                                       \
 195                 break;                                                  \
 196         }                                                               \
 197         true;                                                           \
 198 })
 199
 200 #define journal_entry_err_on(cond, c, msg, ...)                         \
 201         ((cond) ? journal_entry_err(c, msg, ##__VA_ARGS__) : false)
 202
 203 #define FSCK_DELETED_KEY        5
 204
 205 static int journal_validate_key(struct bch_fs *c, const char *where,
 206                                 struct jset_entry *entry,
 207                                 unsigned level, enum btree_id btree_id,
 208                                 struct bkey_i *k, const char *type,
 209                                 unsigned version, int big_endian, int write)
 210 {
 211         void *next = vstruct_next(entry);
 212         const char *invalid;
 213         int ret = 0;
 214
 215         if (journal_entry_err_on(!k->k.u64s, c,
 216                         "invalid %s in %s entry offset %zi/%u: k->u64s 0",
 217                         type, where,
 218                         (u64 *) k - entry->_data,
 219                         le16_to_cpu(entry->u64s))) {
 220                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 221                 journal_entry_null_range(vstruct_next(entry), next);
 222                 return FSCK_DELETED_KEY;
 223         }
 224
 225         if (journal_entry_err_on((void *) bkey_next(k) >
 226                                 (void *) vstruct_next(entry), c,
 227                         "invalid %s in %s entry offset %zi/%u: extends past end of journal entry",
 228                         type, where,
 229                         (u64 *) k - entry->_data,
 230                         le16_to_cpu(entry->u64s))) {
 231                 entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
 232                 journal_entry_null_range(vstruct_next(entry), next);
 233                 return FSCK_DELETED_KEY;
 234         }
 235
 236         if (journal_entry_err_on(k->k.format != KEY_FORMAT_CURRENT, c,
 237                         "invalid %s in %s entry offset %zi/%u: bad format %u",
 238                         type, where,
 239                         (u64 *) k - entry->_data,
 240                         le16_to_cpu(entry->u64s),
 241                         k->k.format)) {
 242                 le16_add_cpu(&entry->u64s, -((u16) k->k.u64s));
 243                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 244                 journal_entry_null_range(vstruct_next(entry), next);
 245                 return FSCK_DELETED_KEY;
 246         }
 247
 248         if (!write)
 249                 bch2_bkey_compat(level, btree_id, version, big_endian,
 250                                  write, NULL, bkey_to_packed(k));
 251
 252         invalid = bch2_bkey_invalid(c, bkey_i_to_s_c(k),
 253                                     __btree_node_type(level, btree_id));
 254         if (invalid) {
 255                 char buf[160];
 256
 257                 bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(k));
 258                 mustfix_fsck_err(c, "invalid %s in %s entry offset %zi/%u: %s\n%s",
 259                                  type, where,
 260                                  (u64 *) k - entry->_data,
 261                                  le16_to_cpu(entry->u64s),
 262                                  invalid, buf);
 263
 264                 le16_add_cpu(&entry->u64s, -((u16) k->k.u64s));
 265                 memmove(k, bkey_next(k), next - (void *) bkey_next(k));
 266                 journal_entry_null_range(vstruct_next(entry), next);
 267                 return FSCK_DELETED_KEY;
 268         }
 269
 270         if (write)
 271                 bch2_bkey_compat(level, btree_id, version, big_endian,
 272                                  write, NULL, bkey_to_packed(k));
 273 fsck_err:
 274         return ret;
 275 }
 276
 277 static int journal_entry_btree_keys_validate(struct bch_fs *c,
 278                                              const char *where,
 279                                              struct jset_entry *entry,
 280                                              unsigned version, int big_endian, int write)
 281 {
 282         struct bkey_i *k = entry->start;
 283
 284         while (k != vstruct_last(entry)) {
 285                 int ret = journal_validate_key(c, where, entry,
 286                                                entry->level,
 287                                                entry->btree_id,
 288                                                k, "key", version, big_endian, write);
 289                 if (ret == FSCK_DELETED_KEY)
 290                         continue;
 291
 292                 k = bkey_next(k);
 293         }
 294
 295         return 0;
 296 }
 297
 298 static void journal_entry_btree_keys_to_text(struct printbuf *out, struct bch_fs *c,
 299                                              struct jset_entry *entry)
 300 {
 301         struct bkey_i *k;
 302         bool first = true;
 303
 304         vstruct_for_each(entry, k) {
 305                 if (!first) {
 306                         printbuf_newline(out);
 307                         pr_buf(out, "%s: ", bch2_jset_entry_types[entry->type]);
 308                 }
 309                 pr_buf(out, "btree=%s l=%u ", bch2_btree_ids[entry->btree_id], entry->level);
 310                 bch2_bkey_val_to_text(out, c, bkey_i_to_s_c(k));
 311                 first = false;
 312         }
 313 }
 314
 315 static int journal_entry_btree_root_validate(struct bch_fs *c,
 316                                              const char *where,
 317                                              struct jset_entry *entry,
 318                                              unsigned version, int big_endian, int write)
 319 {
 320         struct bkey_i *k = entry->start;
 321         int ret = 0;
 322
 323         if (journal_entry_err_on(!entry->u64s ||
 324                                  le16_to_cpu(entry->u64s) != k->k.u64s, c,
 325                                  "invalid btree root journal entry: wrong number of keys")) {
 326                 void *next = vstruct_next(entry);
 327                 /*
 328                  * we don't want to null out this jset_entry,
 329                  * just the contents, so that later we can tell
 330                  * we were _supposed_ to have a btree root
 331                  */
 332                 entry->u64s = 0;
 333                 journal_entry_null_range(vstruct_next(entry), next);
 334                 return 0;
 335         }
 336
 337         return journal_validate_key(c, where, entry, 1, entry->btree_id, k,
 338                                     "btree root", version, big_endian, write);
 339 fsck_err:
 340         return ret;
 341 }
 342
 343 static void journal_entry_btree_root_to_text(struct printbuf *out, struct bch_fs *c,
 344                                              struct jset_entry *entry)
 345 {
 346         journal_entry_btree_keys_to_text(out, c, entry);
 347 }
 348
 349 static int journal_entry_prio_ptrs_validate(struct bch_fs *c,
 350                                             const char *where,
 351                                             struct jset_entry *entry,
 352                                             unsigned version, int big_endian, int write)
 353 {
 354         /* obsolete, don't care: */
 355         return 0;
 356 }
 357
 358 static void journal_entry_prio_ptrs_to_text(struct printbuf *out, struct bch_fs *c,
 359                                             struct jset_entry *entry)
 360 {
 361 }
 362
 363 static int journal_entry_blacklist_validate(struct bch_fs *c,
 364                                             const char *where,
 365                                             struct jset_entry *entry,
 366                                             unsigned version, int big_endian, int write)
 367 {
 368         int ret = 0;
 369
 370         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 1, c,
 371                 "invalid journal seq blacklist entry: bad size")) {
 372                 journal_entry_null_range(entry, vstruct_next(entry));
 373         }
 374 fsck_err:
 375         return ret;
 376 }
 377
 378 static void journal_entry_blacklist_to_text(struct printbuf *out, struct bch_fs *c,
 379                                             struct jset_entry *entry)
 380 {
 381         struct jset_entry_blacklist *bl =
 382                 container_of(entry, struct jset_entry_blacklist, entry);
 383
 384         pr_buf(out, "seq=%llu", le64_to_cpu(bl->seq));
 385 }
 386
 387 static int journal_entry_blacklist_v2_validate(struct bch_fs *c,
 388                                                const char *where,
 389                                                struct jset_entry *entry,
 390                                                unsigned version, int big_endian, int write)
 391 {
 392         struct jset_entry_blacklist_v2 *bl_entry;
 393         int ret = 0;
 394
 395         if (journal_entry_err_on(le16_to_cpu(entry->u64s) != 2, c,
 396                 "invalid journal seq blacklist entry: bad size")) {
 397                 journal_entry_null_range(entry, vstruct_next(entry));
 398                 goto out;
 399         }
 400
 401         bl_entry = container_of(entry, struct jset_entry_blacklist_v2, entry);
 402
 403         if (journal_entry_err_on(le64_to_cpu(bl_entry->start) >
 404                                  le64_to_cpu(bl_entry->end), c,
 405                 "invalid journal seq blacklist entry: start > end")) {
 406                 journal_entry_null_range(entry, vstruct_next(entry));
 407         }
 408 out:
 409 fsck_err:
 410         return ret;
 411 }
 412
 413 static void journal_entry_blacklist_v2_to_text(struct printbuf *out, struct bch_fs *c,
 414                                                struct jset_entry *entry)
 415 {
 416         struct jset_entry_blacklist_v2 *bl =
 417                 container_of(entry, struct jset_entry_blacklist_v2, entry);
 418
 419         pr_buf(out, "start=%llu end=%llu",
 420                le64_to_cpu(bl->start),
 421                le64_to_cpu(bl->end));
 422 }
 423
 424 static int journal_entry_usage_validate(struct bch_fs *c,
 425                                         const char *where,
 426                                         struct jset_entry *entry,
 427                                         unsigned version, int big_endian, int write)
 428 {
 429         struct jset_entry_usage *u =
 430                 container_of(entry, struct jset_entry_usage, entry);
 431         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 432         int ret = 0;
 433
 434         if (journal_entry_err_on(bytes < sizeof(*u),
 435                                  c,
 436                                  "invalid journal entry usage: bad size")) {
 437                 journal_entry_null_range(entry, vstruct_next(entry));
 438                 return ret;
 439         }
 440
 441 fsck_err:
 442         return ret;
 443 }
 444
 445 static void journal_entry_usage_to_text(struct printbuf *out, struct bch_fs *c,
 446                                         struct jset_entry *entry)
 447 {
 448         struct jset_entry_usage *u =
 449                 container_of(entry, struct jset_entry_usage, entry);
 450
 451         pr_buf(out, "type=%s v=%llu",
 452                bch2_fs_usage_types[u->entry.btree_id],
 453                le64_to_cpu(u->v));
 454 }
 455
 456 static int journal_entry_data_usage_validate(struct bch_fs *c,
 457                                         const char *where,
 458                                         struct jset_entry *entry,
 459                                         unsigned version, int big_endian, int write)
 460 {
 461         struct jset_entry_data_usage *u =
 462                 container_of(entry, struct jset_entry_data_usage, entry);
 463         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 464         int ret = 0;
 465
 466         if (journal_entry_err_on(bytes < sizeof(*u) ||
 467                                  bytes < sizeof(*u) + u->r.nr_devs,
 468                                  c,
 469                                  "invalid journal entry usage: bad size")) {
 470                 journal_entry_null_range(entry, vstruct_next(entry));
 471                 return ret;
 472         }
 473
 474 fsck_err:
 475         return ret;
 476 }
 477
 478 static void journal_entry_data_usage_to_text(struct printbuf *out, struct bch_fs *c,
 479                                              struct jset_entry *entry)
 480 {
 481         struct jset_entry_data_usage *u =
 482                 container_of(entry, struct jset_entry_data_usage, entry);
 483
 484         bch2_replicas_entry_to_text(out, &u->r);
 485         pr_buf(out, "=%llu", le64_to_cpu(u->v));
 486 }
 487
 488 static int journal_entry_clock_validate(struct bch_fs *c,
 489                                         const char *where,
 490                                         struct jset_entry *entry,
 491                                         unsigned version, int big_endian, int write)
 492 {
 493         struct jset_entry_clock *clock =
 494                 container_of(entry, struct jset_entry_clock, entry);
 495         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 496         int ret = 0;
 497
 498         if (journal_entry_err_on(bytes != sizeof(*clock),
 499                                  c, "invalid journal entry clock: bad size")) {
 500                 journal_entry_null_range(entry, vstruct_next(entry));
 501                 return ret;
 502         }
 503
 504         if (journal_entry_err_on(clock->rw > 1,
 505                                  c, "invalid journal entry clock: bad rw")) {
 506                 journal_entry_null_range(entry, vstruct_next(entry));
 507                 return ret;
 508         }
 509
 510 fsck_err:
 511         return ret;
 512 }
 513
 514 static void journal_entry_clock_to_text(struct printbuf *out, struct bch_fs *c,
 515                                         struct jset_entry *entry)
 516 {
 517         struct jset_entry_clock *clock =
 518                 container_of(entry, struct jset_entry_clock, entry);
 519
 520         pr_buf(out, "%s=%llu", clock->rw ? "write" : "read", le64_to_cpu(clock->time));
 521 }
 522
 523 static int journal_entry_dev_usage_validate(struct bch_fs *c,
 524                                             const char *where,
 525                                             struct jset_entry *entry,
 526                                             unsigned version, int big_endian, int write)
 527 {
 528         struct jset_entry_dev_usage *u =
 529                 container_of(entry, struct jset_entry_dev_usage, entry);
 530         unsigned bytes = jset_u64s(le16_to_cpu(entry->u64s)) * sizeof(u64);
 531         unsigned expected = sizeof(*u);
 532         unsigned dev;
 533         int ret = 0;
 534
 535         if (journal_entry_err_on(bytes < expected,
 536                                  c, "invalid journal entry dev usage: bad size (%u < %u)",
 537                                  bytes, expected)) {
 538                 journal_entry_null_range(entry, vstruct_next(entry));
 539                 return ret;
 540         }
 541
 542         dev = le32_to_cpu(u->dev);
 543
 544         if (journal_entry_err_on(!bch2_dev_exists2(c, dev),
 545                                  c, "invalid journal entry dev usage: bad dev")) {
 546                 journal_entry_null_range(entry, vstruct_next(entry));
 547                 return ret;
 548         }
 549
 550         if (journal_entry_err_on(u->pad,
 551                                  c, "invalid journal entry dev usage: bad pad")) {
 552                 journal_entry_null_range(entry, vstruct_next(entry));
 553                 return ret;
 554         }
 555
 556 fsck_err:
 557         return ret;
 558 }
 559
 560 static void journal_entry_dev_usage_to_text(struct printbuf *out, struct bch_fs *c,
 561                                             struct jset_entry *entry)
 562 {
 563         struct jset_entry_dev_usage *u =
 564                 container_of(entry, struct jset_entry_dev_usage, entry);
 565         unsigned i, nr_types = jset_entry_dev_usage_nr_types(u);
 566
 567         pr_buf(out, "dev=%u", le32_to_cpu(u->dev));
 568
 569         for (i = 0; i < nr_types; i++) {
 570                 if (i < BCH_DATA_NR)
 571                         pr_buf(out, " %s", bch2_data_types[i]);
 572                 else
 573                         pr_buf(out, " (unknown data type %u)", i);
 574                 pr_buf(out, ": buckets=%llu sectors=%llu fragmented=%llu",
 575                        le64_to_cpu(u->d[i].buckets),
 576                        le64_to_cpu(u->d[i].sectors),
 577                        le64_to_cpu(u->d[i].fragmented));
 578         }
 579
 580         pr_buf(out, " buckets_ec: %llu buckets_unavailable: %llu",
 581                le64_to_cpu(u->buckets_ec),
 582                le64_to_cpu(u->buckets_unavailable));
 583 }
 584
 585 static int journal_entry_log_validate(struct bch_fs *c,
 586                                       const char *where,
 587                                       struct jset_entry *entry,
 588                                       unsigned version, int big_endian, int write)
 589 {
 590         return 0;
 591 }
 592
 593 static void journal_entry_log_to_text(struct printbuf *out, struct bch_fs *c,
 594                                       struct jset_entry *entry)
 595 {
 596         struct jset_entry_log *l = container_of(entry, struct jset_entry_log, entry);
 597         unsigned bytes = vstruct_bytes(entry) - offsetof(struct jset_entry_log, d);
 598
 599         bch_scnmemcpy(out, l->d, strnlen(l->d, bytes));
 600 }
 601
 602 struct jset_entry_ops {
 603         int (*validate)(struct bch_fs *, const char *,
 604                         struct jset_entry *, unsigned, int, int);
 605         void (*to_text)(struct printbuf *, struct bch_fs *, struct jset_entry *);
 606 };
 607
 608 static const struct jset_entry_ops bch2_jset_entry_ops[] = {
 609 #define x(f, nr)                                                \
 610         [BCH_JSET_ENTRY_##f]    = (struct jset_entry_ops) {     \
 611                 .validate       = journal_entry_##f##_validate, \
 612                 .to_text        = journal_entry_##f##_to_text,  \
 613         },
 614         BCH_JSET_ENTRY_TYPES()
 615 #undef x
 616 };
 617
 618 int bch2_journal_entry_validate(struct bch_fs *c, const char *where,
 619                                 struct jset_entry *entry,
 620                                 unsigned version, int big_endian, int write)
 621 {
 622         return entry->type < BCH_JSET_ENTRY_NR
 623                 ? bch2_jset_entry_ops[entry->type].validate(c, where, entry,
 624                                 version, big_endian, write)
 625                 : 0;
 626 }
 627
 628 void bch2_journal_entry_to_text(struct printbuf *out, struct bch_fs *c,
 629                                 struct jset_entry *entry)
 630 {
 631         if (entry->type < BCH_JSET_ENTRY_NR) {
 632                 pr_buf(out, "%s: ", bch2_jset_entry_types[entry->type]);
 633                 bch2_jset_entry_ops[entry->type].to_text(out, c, entry);
 634         } else {
 635                 pr_buf(out, "(unknown type %u)", entry->type);
 636         }
 637 }
 638
 639 static int jset_validate_entries(struct bch_fs *c, struct jset *jset,
 640                                  int write)
 641 {
 642         char buf[100];
 643         struct jset_entry *entry;
 644         int ret = 0;
 645
 646         vstruct_for_each(jset, entry) {
 647                 scnprintf(buf, sizeof(buf), "jset %llu entry offset %zi/%u",
 648                           le64_to_cpu(jset->seq),
 649                           (u64 *) entry - jset->_data,
 650                           le32_to_cpu(jset->u64s));
 651
 652                 if (journal_entry_err_on(vstruct_next(entry) >
 653                                          vstruct_last(jset), c,
 654                                 "journal entry extends past end of jset")) {
 655                         jset->u64s = cpu_to_le32((u64 *) entry - jset->_data);
 656                         break;
 657                 }
 658
 659                 ret = bch2_journal_entry_validate(c, buf, entry,
 660                                         le32_to_cpu(jset->version),
 661                                         JSET_BIG_ENDIAN(jset), write);
 662                 if (ret)
 663                         break;
 664         }
 665 fsck_err:
 666         return ret;
 667 }
 668
 669 static int jset_validate(struct bch_fs *c,
 670                          struct bch_dev *ca,
 671                          struct jset *jset, u64 sector,
 672                          unsigned bucket_sectors_left,
 673                          unsigned sectors_read,
 674                          int write)
 675 {
 676         size_t bytes = vstruct_bytes(jset);
 677         struct bch_csum csum;
 678         unsigned version;
 679         int ret = 0;
 680
 681         if (le64_to_cpu(jset->magic) != jset_magic(c))
 682                 return JOURNAL_ENTRY_NONE;
 683
 684         version = le32_to_cpu(jset->version);
 685         if (journal_entry_err_on((version != BCH_JSET_VERSION_OLD &&
 686                                   version < bcachefs_metadata_version_min) ||
 687                                  version >= bcachefs_metadata_version_max, c,
 688                         "%s sector %llu seq %llu: unknown journal entry version %u",
 689                         ca ? ca->name : c->name,
 690                         sector, le64_to_cpu(jset->seq),
 691                         version)) {
 692                 /* don't try to continue: */
 693                 return EINVAL;
 694         }
 695
 696         if (bytes > (sectors_read << 9) &&
 697             sectors_read < bucket_sectors_left)
 698                 return JOURNAL_ENTRY_REREAD;
 699
 700         if (journal_entry_err_on(bytes > bucket_sectors_left << 9, c,
 701                         "%s sector %llu seq %llu: journal entry too big (%zu bytes)",
 702                         ca ? ca->name : c->name,
 703                         sector, le64_to_cpu(jset->seq), bytes)) {
 704                 ret = JOURNAL_ENTRY_BAD;
 705                 le32_add_cpu(&jset->u64s,
 706                              -((bytes - (bucket_sectors_left << 9)) / 8));
 707         }
 708
 709         if (journal_entry_err_on(!bch2_checksum_type_valid(c, JSET_CSUM_TYPE(jset)), c,
 710                         "%s sector %llu seq %llu: journal entry with unknown csum type %llu",
 711                         ca ? ca->name : c->name,
 712                         sector, le64_to_cpu(jset->seq),
 713                         JSET_CSUM_TYPE(jset))) {
 714                 ret = JOURNAL_ENTRY_BAD;
 715                 goto csum_done;
 716         }
 717
 718         if (write)
 719                 goto csum_done;
 720
 721         csum = csum_vstruct(c, JSET_CSUM_TYPE(jset), journal_nonce(jset), jset);
 722         if (journal_entry_err_on(bch2_crc_cmp(csum, jset->csum), c,
 723                                  "%s sector %llu seq %llu: journal checksum bad",
 724                                  ca ? ca->name : c->name,
 725                                  sector, le64_to_cpu(jset->seq)))
 726                 ret = JOURNAL_ENTRY_BAD;
 727
 728         ret = bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
 729                      jset->encrypted_start,
 730                      vstruct_end(jset) - (void *) jset->encrypted_start);
 731         bch2_fs_fatal_err_on(ret, c,
 732                         "error decrypting journal entry: %i", ret);
 733 csum_done:
 734         /* last_seq is ignored when JSET_NO_FLUSH is true */
 735         if (journal_entry_err_on(!JSET_NO_FLUSH(jset) &&
 736                                  le64_to_cpu(jset->last_seq) > le64_to_cpu(jset->seq), c,
 737                                  "invalid journal entry: last_seq > seq (%llu > %llu)",
 738                                  le64_to_cpu(jset->last_seq),
 739                                  le64_to_cpu(jset->seq))) {
 740                 jset->last_seq = jset->seq;
 741                 return JOURNAL_ENTRY_BAD;
 742         }
 743 fsck_err:
 744         return ret;
 745 }
 746
 747 static int jset_validate_for_write(struct bch_fs *c, struct jset *jset)
 748 {
 749         unsigned sectors = vstruct_sectors(jset, c->block_bits);
 750
 751         return jset_validate(c, NULL, jset, 0, sectors, sectors, WRITE) ?:
 752                 jset_validate_entries(c, jset, WRITE);
 753 }
 754
 755 struct journal_read_buf {
 756         void            *data;
 757         size_t          size;
 758 };
 759
 760 static int journal_read_buf_realloc(struct journal_read_buf *b,
 761                                     size_t new_size)
 762 {
 763         void *n;
 764
 765         /* the bios are sized for this many pages, max: */
 766         if (new_size > JOURNAL_ENTRY_SIZE_MAX)
 767                 return -ENOMEM;
 768
 769         new_size = roundup_pow_of_two(new_size);
 770         n = kvpmalloc(new_size, GFP_KERNEL);
 771         if (!n)
 772                 return -ENOMEM;
 773
 774         kvpfree(b->data, b->size);
 775         b->data = n;
 776         b->size = new_size;
 777         return 0;
 778 }
 779
 780 static int journal_read_bucket(struct bch_dev *ca,
 781                                struct journal_read_buf *buf,
 782                                struct journal_list *jlist,
 783                                unsigned bucket)
 784 {
 785         struct bch_fs *c = ca->fs;
 786         struct journal_device *ja = &ca->journal;
 787         struct jset *j = NULL;
 788         unsigned sectors, sectors_read = 0;
 789         u64 offset = bucket_to_sector(ca, ja->buckets[bucket]),
 790             end = offset + ca->mi.bucket_size;
 791         bool saw_bad = false;
 792         int ret = 0;
 793
 794         pr_debug("reading %u", bucket);
 795
 796         while (offset < end) {
 797                 if (!sectors_read) {
 798                         struct bio *bio;
 799 reread:
 800                         sectors_read = min_t(unsigned,
 801                                 end - offset, buf->size >> 9);
 802
 803                         bio = bio_kmalloc(GFP_KERNEL,
 804                                           buf_pages(buf->data,
 805                                                     sectors_read << 9));
 806                         bio_set_dev(bio, ca->disk_sb.bdev);
 807                         bio->bi_iter.bi_sector  = offset;
 808                         bio_set_op_attrs(bio, REQ_OP_READ, 0);
 809                         bch2_bio_map(bio, buf->data, sectors_read << 9);
 810
 811                         ret = submit_bio_wait(bio);
 812                         bio_put(bio);
 813
 814                         if (bch2_dev_io_err_on(ret, ca,
 815                                                "journal read error: sector %llu",
 816                                                offset) ||
 817                             bch2_meta_read_fault("journal")) {
 818                                 /*
 819                                  * We don't error out of the recovery process
 820                                  * here, since the relevant journal entry may be
 821                                  * found on a different device, and missing or
 822                                  * no journal entries will be handled later
 823                                  */
 824                                 return 0;
 825                         }
 826
 827                         j = buf->data;
 828                 }
 829
 830                 ret = jset_validate(c, ca, j, offset,
 831                                     end - offset, sectors_read,
 832                                     READ);
 833                 switch (ret) {
 834                 case BCH_FSCK_OK:
 835                         sectors = vstruct_sectors(j, c->block_bits);
 836                         break;
 837                 case JOURNAL_ENTRY_REREAD:
 838                         if (vstruct_bytes(j) > buf->size) {
 839                                 ret = journal_read_buf_realloc(buf,
 840                                                         vstruct_bytes(j));
 841                                 if (ret)
 842                                         return ret;
 843                         }
 844                         goto reread;
 845                 case JOURNAL_ENTRY_NONE:
 846                         if (!saw_bad)
 847                                 return 0;
 848                         sectors = block_sectors(c);
 849                         goto next_block;
 850                 case JOURNAL_ENTRY_BAD:
 851                         saw_bad = true;
 852                         /*
 853                          * On checksum error we don't really trust the size
 854                          * field of the journal entry we read, so try reading
 855                          * again at next block boundary:
 856                          */
 857                         sectors = block_sectors(c);
 858                         break;
 859                 default:
 860                         return ret;
 861                 }
 862
 863                 /*
 864                  * This happens sometimes if we don't have discards on -
 865                  * when we've partially overwritten a bucket with new
 866                  * journal entries. We don't need the rest of the
 867                  * bucket:
 868                  */
 869                 if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])
 870                         return 0;
 871
 872                 ja->bucket_seq[bucket] = le64_to_cpu(j->seq);
 873
 874                 mutex_lock(&jlist->lock);
 875                 ret = journal_entry_add(c, ca, (struct journal_ptr) {
 876                                         .dev            = ca->dev_idx,
 877                                         .bucket         = bucket,
 878                                         .bucket_offset  = offset -
 879                                                 bucket_to_sector(ca, ja->buckets[bucket]),
 880                                         .sector         = offset,
 881                                         }, jlist, j, ret != 0);
 882                 mutex_unlock(&jlist->lock);
 883
 884                 switch (ret) {
 885                 case JOURNAL_ENTRY_ADD_OK:
 886                         break;
 887                 case JOURNAL_ENTRY_ADD_OUT_OF_RANGE:
 888                         break;
 889                 default:
 890                         return ret;
 891                 }
 892 next_block:
 893                 pr_debug("next");
 894                 offset          += sectors;
 895                 sectors_read    -= sectors;
 896                 j = ((void *) j) + (sectors << 9);
 897         }
 898
 899         return 0;
 900 }
 901
 902 static void bch2_journal_read_device(struct closure *cl)
 903 {
 904         struct journal_device *ja =
 905                 container_of(cl, struct journal_device, read);
 906         struct bch_dev *ca = container_of(ja, struct bch_dev, journal);
 907         struct bch_fs *c = ca->fs;
 908         struct journal_list *jlist =
 909                 container_of(cl->parent, struct journal_list, cl);
 910         struct journal_read_buf buf = { NULL, 0 };
 911         u64 min_seq = U64_MAX;
 912         unsigned i;
 913         int ret = 0;
 914
 915         if (!ja->nr)
 916                 goto out;
 917
 918         ret = journal_read_buf_realloc(&buf, PAGE_SIZE);
 919         if (ret)
 920                 goto err;
 921
 922         pr_debug("%u journal buckets", ja->nr);
 923
 924         for (i = 0; i < ja->nr; i++) {
 925                 ret = journal_read_bucket(ca, &buf, jlist, i);
 926                 if (ret)
 927                         goto err;
 928         }
 929
 930         /* Find the journal bucket with the highest sequence number: */
 931         for (i = 0; i < ja->nr; i++) {
 932                 if (ja->bucket_seq[i] > ja->bucket_seq[ja->cur_idx])
 933                         ja->cur_idx = i;
 934
 935                 min_seq = min(ja->bucket_seq[i], min_seq);
 936         }
 937
 938         /*
 939          * If there's duplicate journal entries in multiple buckets (which
 940          * definitely isn't supposed to happen, but...) - make sure to start
 941          * cur_idx at the last of those buckets, so we don't deadlock trying to
 942          * allocate
 943          */
 944         while (ja->bucket_seq[ja->cur_idx] > min_seq &&
 945                ja->bucket_seq[ja->cur_idx] >
 946                ja->bucket_seq[(ja->cur_idx + 1) % ja->nr])
 947                 ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
 948
 949         ja->sectors_free = 0;
 950
 951         /*
 952          * Set dirty_idx to indicate the entire journal is full and needs to be
 953          * reclaimed - journal reclaim will immediately reclaim whatever isn't
 954          * pinned when it first runs:
 955          */
 956         ja->discard_idx = ja->dirty_idx_ondisk =
 957                 ja->dirty_idx = (ja->cur_idx + 1) % ja->nr;
 958 out:
 959         bch_verbose(c, "journal read done on device %s, ret %i", ca->name, ret);
 960         kvpfree(buf.data, buf.size);
 961         percpu_ref_put(&ca->io_ref);
 962         closure_return(cl);
 963         return;
 964 err:
 965         mutex_lock(&jlist->lock);
 966         jlist->ret = ret;
 967         mutex_unlock(&jlist->lock);
 968         goto out;
 969 }
 970
 971 void bch2_journal_ptrs_to_text(struct printbuf *out, struct bch_fs *c,
 972                                struct journal_replay *j)
 973 {
 974         unsigned i;
 975
 976         for (i = 0; i < j->nr_ptrs; i++) {
 977                 struct bch_dev *ca = bch_dev_bkey_exists(c, j->ptrs[i].dev);
 978                 u64 offset;
 979
 980                 div64_u64_rem(j->ptrs[i].sector, ca->mi.bucket_size, &offset);
 981
 982                 if (i)
 983                         pr_buf(out, " ");
 984                 pr_buf(out, "%u:%u:%u (sector %llu)",
 985                        j->ptrs[i].dev,
 986                        j->ptrs[i].bucket,
 987                        j->ptrs[i].bucket_offset,
 988                        j->ptrs[i].sector);
 989         }
 990 }
 991
 992 int bch2_journal_read(struct bch_fs *c, struct list_head *list,
 993                       u64 *blacklist_seq, u64 *start_seq)
 994 {
 995         struct journal_list jlist;
 996         struct journal_replay *i, *t;
 997         struct bch_dev *ca;
 998         unsigned iter;
 999         size_t keys = 0, entries = 0;
1000         bool degraded = false;
1001         u64 seq, last_seq = 0;
1002         int ret = 0;
1003
1004         closure_init_stack(&jlist.cl);
1005         mutex_init(&jlist.lock);
1006         jlist.head = list;
1007         jlist.ret = 0;
1008
1009         for_each_member_device(ca, c, iter) {
1010                 if (!test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) &&
1011                     !(bch2_dev_has_data(c, ca) & (1 << BCH_DATA_journal)))
1012                         continue;
1013
1014                 if ((ca->mi.state == BCH_MEMBER_STATE_rw ||
1015                      ca->mi.state == BCH_MEMBER_STATE_ro) &&
1016                     percpu_ref_tryget(&ca->io_ref))
1017                         closure_call(&ca->journal.read,
1018                                      bch2_journal_read_device,
1019                                      system_unbound_wq,
1020                                      &jlist.cl);
1021                 else
1022                         degraded = true;
1023         }
1024
1025         closure_sync(&jlist.cl);
1026
1027         if (jlist.ret)
1028                 return jlist.ret;
1029
1030         if (list_empty(list)) {
1031                 bch_info(c, "journal read done, but no entries found");
1032                 return 0;
1033         }
1034
1035         i = list_last_entry(list, struct journal_replay, list);
1036         *start_seq = le64_to_cpu(i->j.seq) + 1;
1037
1038         /*
1039          * Find most recent flush entry, and ignore newer non flush entries -
1040          * those entries will be blacklisted:
1041          */
1042         list_for_each_entry_safe_reverse(i, t, list, list) {
1043                 if (i->ignore)
1044                         continue;
1045
1046                 if (!JSET_NO_FLUSH(&i->j)) {
1047                         last_seq        = le64_to_cpu(i->j.last_seq);
1048                         *blacklist_seq  = le64_to_cpu(i->j.seq) + 1;
1049                         break;
1050                 }
1051
1052                 journal_replay_free(c, i);
1053         }
1054
1055         if (!last_seq) {
1056                 fsck_err(c, "journal read done, but no entries found after dropping non-flushes");
1057                 return -1;
1058         }
1059
1060         /* Drop blacklisted entries and entries older than last_seq: */
1061         list_for_each_entry_safe(i, t, list, list) {
1062                 if (i->ignore)
1063                         continue;
1064
1065                 seq = le64_to_cpu(i->j.seq);
1066                 if (seq < last_seq) {
1067                         journal_replay_free(c, i);
1068                         continue;
1069                 }
1070
1071                 if (bch2_journal_seq_is_blacklisted(c, seq, true)) {
1072                         fsck_err_on(!JSET_NO_FLUSH(&i->j), c,
1073                                     "found blacklisted journal entry %llu", seq);
1074
1075                         journal_replay_free(c, i);
1076                 }
1077         }
1078
1079         /* Check for missing entries: */
1080         seq = last_seq;
1081         list_for_each_entry(i, list, list) {
1082                 if (i->ignore)
1083                         continue;
1084
1085                 BUG_ON(seq > le64_to_cpu(i->j.seq));
1086
1087                 while (seq < le64_to_cpu(i->j.seq)) {
1088                         u64 missing_start, missing_end;
1089                         char buf1[200], buf2[200];
1090
1091                         while (seq < le64_to_cpu(i->j.seq) &&
1092                                bch2_journal_seq_is_blacklisted(c, seq, false))
1093                                 seq++;
1094
1095                         if (seq == le64_to_cpu(i->j.seq))
1096                                 break;
1097
1098                         missing_start = seq;
1099
1100                         while (seq < le64_to_cpu(i->j.seq) &&
1101                                !bch2_journal_seq_is_blacklisted(c, seq, false))
1102                                 seq++;
1103
1104                         if (i->list.prev != list) {
1105                                 struct printbuf out = PBUF(buf1);
1106                                 struct journal_replay *p = list_prev_entry(i, list);
1107
1108                                 bch2_journal_ptrs_to_text(&out, c, p);
1109                                 pr_buf(&out, " size %llu", vstruct_sectors(&p->j, c->block_bits));
1110                         } else
1111                                 sprintf(buf1, "(none)");
1112                         bch2_journal_ptrs_to_text(&PBUF(buf2), c, i);
1113
1114                         missing_end = seq - 1;
1115                         fsck_err(c, "journal entries %llu-%llu missing! (replaying %llu-%llu)\n"
1116                                  "  prev at %s\n"
1117                                  "  next at %s",
1118                                  missing_start, missing_end,
1119                                  last_seq, *blacklist_seq - 1,
1120                                  buf1, buf2);
1121                 }
1122
1123                 seq++;
1124         }
1125
1126         list_for_each_entry(i, list, list) {
1127                 struct jset_entry *entry;
1128                 struct bkey_i *k, *_n;
1129                 struct bch_replicas_padded replicas = {
1130                         .e.data_type = BCH_DATA_journal,
1131                         .e.nr_required = 1,
1132                 };
1133                 unsigned ptr;
1134                 char buf[80];
1135
1136                 if (i->ignore)
1137                         continue;
1138
1139                 ret = jset_validate_entries(c, &i->j, READ);
1140                 if (ret)
1141                         goto fsck_err;
1142
1143                 for (ptr = 0; ptr < i->nr_ptrs; ptr++)
1144                         replicas.e.devs[replicas.e.nr_devs++] = i->ptrs[ptr].dev;
1145
1146                 bch2_replicas_entry_sort(&replicas.e);
1147
1148                 /*
1149                  * If we're mounting in degraded mode - if we didn't read all
1150                  * the devices - this is wrong:
1151                  */
1152
1153                 if (!degraded &&
1154                     (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
1155                      fsck_err_on(!bch2_replicas_marked(c, &replicas.e), c,
1156                                  "superblock not marked as containing replicas %s",
1157                                  (bch2_replicas_entry_to_text(&PBUF(buf),
1158                                                               &replicas.e), buf)))) {
1159                         ret = bch2_mark_replicas(c, &replicas.e);
1160                         if (ret)
1161                                 return ret;
1162                 }
1163
1164                 for_each_jset_key(k, _n, entry, &i->j)
1165                         keys++;
1166                 entries++;
1167         }
1168
1169         bch_info(c, "journal read done, %zu keys in %zu entries, seq %llu",
1170                  keys, entries, *start_seq);
1171
1172         if (*start_seq != *blacklist_seq)
1173                 bch_info(c, "dropped unflushed entries %llu-%llu",
1174                          *blacklist_seq, *start_seq - 1);
1175 fsck_err:
1176         return ret;
1177 }
1178
1179 /* journal write: */
1180
1181 static void __journal_write_alloc(struct journal *j,
1182                                   struct journal_buf *w,
1183                                   struct dev_alloc_list *devs_sorted,
1184                                   unsigned sectors,
1185                                   unsigned *replicas,
1186                                   unsigned replicas_want)
1187 {
1188         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1189         struct journal_device *ja;
1190         struct bch_dev *ca;
1191         unsigned i;
1192
1193         if (*replicas >= replicas_want)
1194                 return;
1195
1196         for (i = 0; i < devs_sorted->nr; i++) {
1197                 ca = rcu_dereference(c->devs[devs_sorted->devs[i]]);
1198                 if (!ca)
1199                         continue;
1200
1201                 ja = &ca->journal;
1202
1203                 /*
1204                  * Check that we can use this device, and aren't already using
1205                  * it:
1206                  */
1207                 if (!ca->mi.durability ||
1208                     ca->mi.state != BCH_MEMBER_STATE_rw ||
1209                     !ja->nr ||
1210                     bch2_bkey_has_device(bkey_i_to_s_c(&w->key),
1211                                          ca->dev_idx) ||
1212                     sectors > ja->sectors_free)
1213                         continue;
1214
1215                 bch2_dev_stripe_increment(ca, &j->wp.stripe);
1216
1217                 bch2_bkey_append_ptr(&w->key,
1218                         (struct bch_extent_ptr) {
1219                                   .offset = bucket_to_sector(ca,
1220                                         ja->buckets[ja->cur_idx]) +
1221                                         ca->mi.bucket_size -
1222                                         ja->sectors_free,
1223                                   .dev = ca->dev_idx,
1224                 });
1225
1226                 ja->sectors_free -= sectors;
1227                 ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
1228
1229                 *replicas += ca->mi.durability;
1230
1231                 if (*replicas >= replicas_want)
1232                         break;
1233         }
1234 }
1235
1236 /**
1237  * journal_next_bucket - move on to the next journal bucket if possible
1238  */
1239 static int journal_write_alloc(struct journal *j, struct journal_buf *w,
1240                                unsigned sectors)
1241 {
1242         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1243         struct bch_devs_mask devs;
1244         struct journal_device *ja;
1245         struct bch_dev *ca;
1246         struct dev_alloc_list devs_sorted;
1247         unsigned target = c->opts.metadata_target ?:
1248                 c->opts.foreground_target;
1249         unsigned i, replicas = 0, replicas_want =
1250                 READ_ONCE(c->opts.metadata_replicas);
1251
1252         rcu_read_lock();
1253 retry:
1254         devs = target_rw_devs(c, BCH_DATA_journal, target);
1255
1256         devs_sorted = bch2_dev_alloc_list(c, &j->wp.stripe, &devs);
1257
1258         __journal_write_alloc(j, w, &devs_sorted,
1259                               sectors, &replicas, replicas_want);
1260
1261         if (replicas >= replicas_want)
1262                 goto done;
1263
1264         for (i = 0; i < devs_sorted.nr; i++) {
1265                 ca = rcu_dereference(c->devs[devs_sorted.devs[i]]);
1266                 if (!ca)
1267                         continue;
1268
1269                 ja = &ca->journal;
1270
1271                 if (sectors > ja->sectors_free &&
1272                     sectors <= ca->mi.bucket_size &&
1273                     bch2_journal_dev_buckets_available(j, ja,
1274                                         journal_space_discarded)) {
1275                         ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
1276                         ja->sectors_free = ca->mi.bucket_size;
1277
1278                         /*
1279                          * ja->bucket_seq[ja->cur_idx] must always have
1280                          * something sensible:
1281                          */
1282                         ja->bucket_seq[ja->cur_idx] = le64_to_cpu(w->data->seq);
1283                 }
1284         }
1285
1286         __journal_write_alloc(j, w, &devs_sorted,
1287                               sectors, &replicas, replicas_want);
1288
1289         if (replicas < replicas_want && target) {
1290                 /* Retry from all devices: */
1291                 target = 0;
1292                 goto retry;
1293         }
1294 done:
1295         rcu_read_unlock();
1296
1297         BUG_ON(bkey_val_u64s(&w->key.k) > BCH_REPLICAS_MAX);
1298
1299         return replicas >= c->opts.metadata_replicas_required ? 0 : -EROFS;
1300 }
1301
1302 static void journal_write_compact(struct jset *jset)
1303 {
1304         struct jset_entry *i, *next, *prev = NULL;
1305
1306         /*
1307          * Simple compaction, dropping empty jset_entries (from journal
1308          * reservations that weren't fully used) and merging jset_entries that
1309          * can be.
1310          *
1311          * If we wanted to be really fancy here, we could sort all the keys in
1312          * the jset and drop keys that were overwritten - probably not worth it:
1313          */
1314         vstruct_for_each_safe(jset, i, next) {
1315                 unsigned u64s = le16_to_cpu(i->u64s);
1316
1317                 /* Empty entry: */
1318                 if (!u64s)
1319                         continue;
1320
1321                 /* Can we merge with previous entry? */
1322                 if (prev &&
1323                     i->btree_id == prev->btree_id &&
1324                     i->level    == prev->level &&
1325                     i->type     == prev->type &&
1326                     i->type     == BCH_JSET_ENTRY_btree_keys &&
1327                     le16_to_cpu(prev->u64s) + u64s <= U16_MAX) {
1328                         memmove_u64s_down(vstruct_next(prev),
1329                                           i->_data,
1330                                           u64s);
1331                         le16_add_cpu(&prev->u64s, u64s);
1332                         continue;
1333                 }
1334
1335                 /* Couldn't merge, move i into new position (after prev): */
1336                 prev = prev ? vstruct_next(prev) : jset->start;
1337                 if (i != prev)
1338                         memmove_u64s_down(prev, i, jset_u64s(u64s));
1339         }
1340
1341         prev = prev ? vstruct_next(prev) : jset->start;
1342         jset->u64s = cpu_to_le32((u64 *) prev - jset->_data);
1343 }
1344
1345 static void journal_buf_realloc(struct journal *j, struct journal_buf *buf)
1346 {
1347         /* we aren't holding j->lock: */
1348         unsigned new_size = READ_ONCE(j->buf_size_want);
1349         void *new_buf;
1350
1351         if (buf->buf_size >= new_size)
1352                 return;
1353
1354         new_buf = kvpmalloc(new_size, GFP_NOIO|__GFP_NOWARN);
1355         if (!new_buf)
1356                 return;
1357
1358         memcpy(new_buf, buf->data, buf->buf_size);
1359
1360         spin_lock(&j->lock);
1361         swap(buf->data,         new_buf);
1362         swap(buf->buf_size,     new_size);
1363         spin_unlock(&j->lock);
1364
1365         kvpfree(new_buf, new_size);
1366 }
1367
1368 static inline struct journal_buf *journal_last_unwritten_buf(struct journal *j)
1369 {
1370         return j->buf + j->reservations.unwritten_idx;
1371 }
1372
1373 static void journal_write_done(struct closure *cl)
1374 {
1375         struct journal *j = container_of(cl, struct journal, io);
1376         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1377         struct journal_buf *w = journal_last_unwritten_buf(j);
1378         struct bch_replicas_padded replicas;
1379         union journal_res_state old, new;
1380         u64 v, seq;
1381         int err = 0;
1382
1383         bch2_time_stats_update(!JSET_NO_FLUSH(w->data)
1384                                ? j->flush_write_time
1385                                : j->noflush_write_time, j->write_start_time);
1386
1387         if (!w->devs_written.nr) {
1388                 bch_err(c, "unable to write journal to sufficient devices");
1389                 err = -EIO;
1390         } else {
1391                 bch2_devlist_to_replicas(&replicas.e, BCH_DATA_journal,
1392                                          w->devs_written);
1393                 if (bch2_mark_replicas(c, &replicas.e))
1394                         err = -EIO;
1395         }
1396
1397         if (err)
1398                 bch2_fatal_error(c);
1399
1400         spin_lock(&j->lock);
1401         seq = le64_to_cpu(w->data->seq);
1402
1403         if (seq >= j->pin.front)
1404                 journal_seq_pin(j, seq)->devs = w->devs_written;
1405
1406         if (!err) {
1407                 j->seq_ondisk           = seq;
1408
1409                 if (!JSET_NO_FLUSH(w->data)) {
1410                         j->flushed_seq_ondisk = seq;
1411                         j->last_seq_ondisk = w->last_seq;
1412                 }
1413         } else if (!j->err_seq || seq < j->err_seq)
1414                 j->err_seq      = seq;
1415
1416         /*
1417          * Updating last_seq_ondisk may let bch2_journal_reclaim_work() discard
1418          * more buckets:
1419          *
1420          * Must come before signaling write completion, for
1421          * bch2_fs_journal_stop():
1422          */
1423         journal_reclaim_kick(&c->journal);
1424
1425         /* also must come before signalling write completion: */
1426         closure_debug_destroy(cl);
1427
1428         v = atomic64_read(&j->reservations.counter);
1429         do {
1430                 old.v = new.v = v;
1431                 BUG_ON(new.idx == new.unwritten_idx);
1432
1433                 new.unwritten_idx++;
1434         } while ((v = atomic64_cmpxchg(&j->reservations.counter,
1435                                        old.v, new.v)) != old.v);
1436
1437         bch2_journal_space_available(j);
1438
1439         closure_wake_up(&w->wait);
1440         journal_wake(j);
1441
1442         if (test_bit(JOURNAL_NEED_WRITE, &j->flags))
1443                 mod_delayed_work(c->io_complete_wq, &j->write_work, 0);
1444         spin_unlock(&j->lock);
1445
1446         if (new.unwritten_idx != new.idx &&
1447             !journal_state_count(new, new.unwritten_idx))
1448                 closure_call(&j->io, bch2_journal_write, c->io_complete_wq, NULL);
1449 }
1450
1451 static void journal_write_endio(struct bio *bio)
1452 {
1453         struct bch_dev *ca = bio->bi_private;
1454         struct journal *j = &ca->fs->journal;
1455         struct journal_buf *w = journal_last_unwritten_buf(j);
1456         unsigned long flags;
1457
1458         if (bch2_dev_io_err_on(bio->bi_status, ca, "error writing journal entry %llu: %s",
1459                                le64_to_cpu(w->data->seq),
1460                                bch2_blk_status_to_str(bio->bi_status)) ||
1461             bch2_meta_write_fault("journal")) {
1462                 spin_lock_irqsave(&j->err_lock, flags);
1463                 bch2_dev_list_drop_dev(&w->devs_written, ca->dev_idx);
1464                 spin_unlock_irqrestore(&j->err_lock, flags);
1465         }
1466
1467         closure_put(&j->io);
1468         percpu_ref_put(&ca->io_ref);
1469 }
1470
1471 static void do_journal_write(struct closure *cl)
1472 {
1473         struct journal *j = container_of(cl, struct journal, io);
1474         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1475         struct bch_dev *ca;
1476         struct journal_buf *w = journal_last_unwritten_buf(j);
1477         struct bch_extent_ptr *ptr;
1478         struct bio *bio;
1479         unsigned sectors = vstruct_sectors(w->data, c->block_bits);
1480
1481         extent_for_each_ptr(bkey_i_to_s_extent(&w->key), ptr) {
1482                 ca = bch_dev_bkey_exists(c, ptr->dev);
1483                 if (!percpu_ref_tryget(&ca->io_ref)) {
1484                         /* XXX: fix this */
1485                         bch_err(c, "missing device for journal write\n");
1486                         continue;
1487                 }
1488
1489                 this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_journal],
1490                              sectors);
1491
1492                 bio = ca->journal.bio;
1493                 bio_reset(bio);
1494                 bio_set_dev(bio, ca->disk_sb.bdev);
1495                 bio->bi_iter.bi_sector  = ptr->offset;
1496                 bio->bi_end_io          = journal_write_endio;
1497                 bio->bi_private         = ca;
1498                 bio->bi_opf             = REQ_OP_WRITE|REQ_SYNC|REQ_META;
1499
1500                 BUG_ON(bio->bi_iter.bi_sector == ca->prev_journal_sector);
1501                 ca->prev_journal_sector = bio->bi_iter.bi_sector;
1502
1503                 if (!JSET_NO_FLUSH(w->data))
1504                         bio->bi_opf    |= REQ_FUA;
1505                 if (!JSET_NO_FLUSH(w->data) && !w->separate_flush)
1506                         bio->bi_opf    |= REQ_PREFLUSH;
1507
1508                 bch2_bio_map(bio, w->data, sectors << 9);
1509
1510                 trace_journal_write(bio);
1511                 closure_bio_submit(bio, cl);
1512
1513                 ca->journal.bucket_seq[ca->journal.cur_idx] =
1514                         le64_to_cpu(w->data->seq);
1515         }
1516
1517         continue_at(cl, journal_write_done, c->io_complete_wq);
1518         return;
1519 }
1520
1521 void bch2_journal_write(struct closure *cl)
1522 {
1523         struct journal *j = container_of(cl, struct journal, io);
1524         struct bch_fs *c = container_of(j, struct bch_fs, journal);
1525         struct bch_dev *ca;
1526         struct journal_buf *w = journal_last_unwritten_buf(j);
1527         struct jset_entry *start, *end;
1528         struct jset *jset;
1529         struct bio *bio;
1530         char *journal_debug_buf = NULL;
1531         bool validate_before_checksum = false;
1532         unsigned i, sectors, bytes, u64s, nr_rw_members = 0;
1533         int ret;
1534
1535         BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb));
1536
1537         journal_buf_realloc(j, w);
1538         jset = w->data;
1539
1540         j->write_start_time = local_clock();
1541
1542         spin_lock(&j->lock);
1543         if (c->sb.features & (1ULL << BCH_FEATURE_journal_no_flush) &&
1544             (w->noflush ||
1545              (!w->must_flush &&
1546               (jiffies - j->last_flush_write) < msecs_to_jiffies(c->opts.journal_flush_delay) &&
1547               test_bit(JOURNAL_MAY_SKIP_FLUSH, &j->flags)))) {
1548                 w->noflush = true;
1549                 SET_JSET_NO_FLUSH(jset, true);
1550                 jset->last_seq  = 0;
1551                 w->last_seq     = 0;
1552
1553                 j->nr_noflush_writes++;
1554         } else {
1555                 j->last_flush_write = jiffies;
1556                 j->nr_flush_writes++;
1557         }
1558         spin_unlock(&j->lock);
1559
1560         /*
1561          * New btree roots are set by journalling them; when the journal entry
1562          * gets written we have to propagate them to c->btree_roots
1563          *
1564          * But, every journal entry we write has to contain all the btree roots
1565          * (at least for now); so after we copy btree roots to c->btree_roots we
1566          * have to get any missing btree roots and add them to this journal
1567          * entry:
1568          */
1569
1570         bch2_journal_entries_to_btree_roots(c, jset);
1571
1572         start = end = vstruct_last(jset);
1573
1574         end     = bch2_btree_roots_to_journal_entries(c, jset->start, end);
1575
1576         bch2_journal_super_entries_add_common(c, &end,
1577                                 le64_to_cpu(jset->seq));
1578         u64s    = (u64 *) end - (u64 *) start;
1579         BUG_ON(u64s > j->entry_u64s_reserved);
1580
1581         le32_add_cpu(&jset->u64s, u64s);
1582         BUG_ON(vstruct_sectors(jset, c->block_bits) > w->sectors);
1583
1584         journal_write_compact(jset);
1585
1586         jset->magic             = cpu_to_le64(jset_magic(c));
1587         jset->version           = c->sb.version < bcachefs_metadata_version_new_versioning
1588                 ? cpu_to_le32(BCH_JSET_VERSION_OLD)
1589                 : cpu_to_le32(c->sb.version);
1590
1591         SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN);
1592         SET_JSET_CSUM_TYPE(jset, bch2_meta_checksum_type(c));
1593
1594         if (!JSET_NO_FLUSH(jset) && journal_entry_empty(jset))
1595                 j->last_empty_seq = le64_to_cpu(jset->seq);
1596
1597         if (bch2_csum_type_is_encryption(JSET_CSUM_TYPE(jset)))
1598                 validate_before_checksum = true;
1599
1600         if (le32_to_cpu(jset->version) < bcachefs_metadata_version_current)
1601                 validate_before_checksum = true;
1602
1603         if (validate_before_checksum &&
1604             jset_validate_for_write(c, jset))
1605                 goto err;
1606
1607         ret = bch2_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
1608                     jset->encrypted_start,
1609                     vstruct_end(jset) - (void *) jset->encrypted_start);
1610         if (bch2_fs_fatal_err_on(ret, c,
1611                         "error decrypting journal entry: %i", ret))
1612                 goto err;
1613
1614         jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset),
1615                                   journal_nonce(jset), jset);
1616
1617         if (!validate_before_checksum &&
1618             jset_validate_for_write(c, jset))
1619                 goto err;
1620
1621         sectors = vstruct_sectors(jset, c->block_bits);
1622         BUG_ON(sectors > w->sectors);
1623
1624         bytes = vstruct_bytes(jset);
1625         memset((void *) jset + bytes, 0, (sectors << 9) - bytes);
1626
1627 retry_alloc:
1628         spin_lock(&j->lock);
1629         ret = journal_write_alloc(j, w, sectors);
1630
1631         if (ret && j->can_discard) {
1632                 spin_unlock(&j->lock);
1633                 bch2_journal_do_discards(j);
1634                 goto retry_alloc;
1635         }
1636
1637         if (ret) {
1638                 journal_debug_buf = kmalloc(4096, GFP_ATOMIC);
1639                 if (journal_debug_buf)
1640                         __bch2_journal_debug_to_text(&_PBUF(journal_debug_buf, 4096), j);
1641         }
1642
1643         /*
1644          * write is allocated, no longer need to account for it in
1645          * bch2_journal_space_available():
1646          */
1647         w->sectors = 0;
1648
1649         /*
1650          * journal entry has been compacted and allocated, recalculate space
1651          * available:
1652          */
1653         bch2_journal_space_available(j);
1654         spin_unlock(&j->lock);
1655
1656         if (ret) {
1657                 bch_err(c, "Unable to allocate journal write:\n%s",
1658                         journal_debug_buf);
1659                 kfree(journal_debug_buf);
1660                 bch2_fatal_error(c);
1661                 continue_at(cl, journal_write_done, c->io_complete_wq);
1662                 return;
1663         }
1664
1665         w->devs_written = bch2_bkey_devs(bkey_i_to_s_c(&w->key));
1666
1667         if (test_bit(JOURNAL_NOCHANGES, &j->flags))
1668                 goto no_io;
1669
1670         for_each_rw_member(ca, c, i)
1671                 nr_rw_members++;
1672
1673         if (nr_rw_members > 1)
1674                 w->separate_flush = true;
1675
1676         if (!JSET_NO_FLUSH(jset) && w->separate_flush) {
1677                 for_each_rw_member(ca, c, i) {
1678                         percpu_ref_get(&ca->io_ref);
1679
1680                         bio = ca->journal.bio;
1681                         bio_reset(bio);
1682                         bio_set_dev(bio, ca->disk_sb.bdev);
1683                         bio->bi_opf             = REQ_OP_FLUSH;
1684                         bio->bi_end_io          = journal_write_endio;
1685                         bio->bi_private         = ca;
1686                         closure_bio_submit(bio, cl);
1687                 }
1688         }
1689
1690         continue_at(cl, do_journal_write, c->io_complete_wq);
1691         return;
1692 no_io:
1693         continue_at(cl, journal_write_done, c->io_complete_wq);
1694         return;
1695 err:
1696         bch2_fatal_error(c);
1697         continue_at(cl, journal_write_done, c->io_complete_wq);
1698 }