git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_gc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2010 Kent Overstreet <kent.overstreet@gmail.com>
   4  * Copyright (C) 2014 Datera Inc.
   5  */
   6
   7 #include "bcachefs.h"
   8 #include "alloc_background.h"
   9 #include "alloc_foreground.h"
  10 #include "bkey_methods.h"
  11 #include "bkey_buf.h"
  12 #include "btree_locking.h"
  13 #include "btree_update_interior.h"
  14 #include "btree_io.h"
  15 #include "btree_gc.h"
  16 #include "buckets.h"
  17 #include "clock.h"
  18 #include "debug.h"
  19 #include "ec.h"
  20 #include "error.h"
  21 #include "extents.h"
  22 #include "journal.h"
  23 #include "keylist.h"
  24 #include "move.h"
  25 #include "recovery.h"
  26 #include "reflink.h"
  27 #include "replicas.h"
  28 #include "super-io.h"
  29
  30 #include <linux/slab.h>
  31 #include <linux/bitops.h>
  32 #include <linux/freezer.h>
  33 #include <linux/kthread.h>
  34 #include <linux/preempt.h>
  35 #include <linux/rcupdate.h>
  36 #include <linux/sched/task.h>
  37 #include <trace/events/bcachefs.h>
  38
  39 #define DROP_THIS_NODE          10
  40 #define DROP_PREV_NODE          11
  41
  42 static inline void __gc_pos_set(struct bch_fs *c, struct gc_pos new_pos)
  43 {
  44         preempt_disable();
  45         write_seqcount_begin(&c->gc_pos_lock);
  46         c->gc_pos = new_pos;
  47         write_seqcount_end(&c->gc_pos_lock);
  48         preempt_enable();
  49 }
  50
  51 static inline void gc_pos_set(struct bch_fs *c, struct gc_pos new_pos)
  52 {
  53         BUG_ON(gc_pos_cmp(new_pos, c->gc_pos) <= 0);
  54         __gc_pos_set(c, new_pos);
  55 }
  56
  57 /*
  58  * Missing: if an interior btree node is empty, we need to do something -
  59  * perhaps just kill it
  60  */
  61 static int bch2_gc_check_topology(struct bch_fs *c,
  62                                   struct btree *b,
  63                                   struct bkey_buf *prev,
  64                                   struct bkey_buf cur,
  65                                   bool is_last)
  66 {
  67         struct bpos node_start  = b->data->min_key;
  68         struct bpos node_end    = b->data->max_key;
  69         struct bpos expected_start = bkey_deleted(&prev->k->k)
  70                 ? node_start
  71                 : bpos_successor(prev->k->k.p);
  72         char buf1[200], buf2[200];
  73         int ret = 0;
  74
  75         if (cur.k->k.type == KEY_TYPE_btree_ptr_v2) {
  76                 struct bkey_i_btree_ptr_v2 *bp = bkey_i_to_btree_ptr_v2(cur.k);
  77
  78                 if (bkey_deleted(&prev->k->k)) {
  79                         struct printbuf out = PBUF(buf1);
  80                         pr_buf(&out, "start of node: ");
  81                         bch2_bpos_to_text(&out, node_start);
  82                 } else {
  83                         bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(prev->k));
  84                 }
  85
  86                 if (bpos_cmp(expected_start, bp->v.min_key)) {
  87                         bch2_topology_error(c);
  88
  89                         if (__fsck_err(c,
  90                                   FSCK_CAN_FIX|
  91                                   FSCK_CAN_IGNORE|
  92                                   FSCK_NO_RATELIMIT,
  93                                   "btree node with incorrect min_key at btree %s level %u:\n"
  94                                   "  prev %s\n"
  95                                   "  cur %s",
  96                                   bch2_btree_ids[b->c.btree_id], b->c.level,
  97                                   buf1,
  98                                   (bch2_bkey_val_to_text(&PBUF(buf2), c, bkey_i_to_s_c(cur.k)), buf2)) &&
  99                             !test_bit(BCH_FS_TOPOLOGY_REPAIR_DONE, &c->flags)) {
 100                                 bch_info(c, "Halting mark and sweep to start topology repair pass");
 101                                 return FSCK_ERR_START_TOPOLOGY_REPAIR;
 102                         } else {
 103                                 set_bit(BCH_FS_INITIAL_GC_UNFIXED, &c->flags);
 104                         }
 105                 }
 106         }
 107
 108         if (is_last && bpos_cmp(cur.k->k.p, node_end)) {
 109                 bch2_topology_error(c);
 110
 111                 if (__fsck_err(c,
 112                           FSCK_CAN_FIX|
 113                           FSCK_CAN_IGNORE|
 114                           FSCK_NO_RATELIMIT,
 115                           "btree node with incorrect max_key at btree %s level %u:\n"
 116                           "  %s\n"
 117                           "  expected %s",
 118                           bch2_btree_ids[b->c.btree_id], b->c.level,
 119                           (bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(cur.k)), buf1),
 120                           (bch2_bpos_to_text(&PBUF(buf2), node_end), buf2)) &&
 121                     !test_bit(BCH_FS_TOPOLOGY_REPAIR_DONE, &c->flags)) {
 122                         bch_info(c, "Halting mark and sweep to start topology repair pass");
 123                         return FSCK_ERR_START_TOPOLOGY_REPAIR;
 124                 } else {
 125                         set_bit(BCH_FS_INITIAL_GC_UNFIXED, &c->flags);
 126                 }
 127         }
 128
 129         bch2_bkey_buf_copy(prev, c, cur.k);
 130 fsck_err:
 131         return ret;
 132 }
 133
 134 static void btree_ptr_to_v2(struct btree *b, struct bkey_i_btree_ptr_v2 *dst)
 135 {
 136         switch (b->key.k.type) {
 137         case KEY_TYPE_btree_ptr: {
 138                 struct bkey_i_btree_ptr *src = bkey_i_to_btree_ptr(&b->key);
 139
 140                 dst->k.p                = src->k.p;
 141                 dst->v.mem_ptr          = 0;
 142                 dst->v.seq              = b->data->keys.seq;
 143                 dst->v.sectors_written  = 0;
 144                 dst->v.flags            = 0;
 145                 dst->v.min_key          = b->data->min_key;
 146                 set_bkey_val_bytes(&dst->k, sizeof(dst->v) + bkey_val_bytes(&src->k));
 147                 memcpy(dst->v.start, src->v.start, bkey_val_bytes(&src->k));
 148                 break;
 149         }
 150         case KEY_TYPE_btree_ptr_v2:
 151                 bkey_copy(&dst->k_i, &b->key);
 152                 break;
 153         default:
 154                 BUG();
 155         }
 156 }
 157
 158 static int set_node_min(struct bch_fs *c, struct btree *b, struct bpos new_min)
 159 {
 160         struct bkey_i_btree_ptr_v2 *new;
 161         int ret;
 162
 163         new = kmalloc(BKEY_BTREE_PTR_U64s_MAX * sizeof(u64), GFP_KERNEL);
 164         if (!new)
 165                 return -ENOMEM;
 166
 167         btree_ptr_to_v2(b, new);
 168         b->data->min_key        = new_min;
 169         new->v.min_key          = new_min;
 170         SET_BTREE_PTR_RANGE_UPDATED(&new->v, true);
 171
 172         ret = bch2_journal_key_insert(c, b->c.btree_id, b->c.level + 1, &new->k_i);
 173         if (ret) {
 174                 kfree(new);
 175                 return ret;
 176         }
 177
 178         bch2_btree_node_drop_keys_outside_node(b);
 179
 180         return 0;
 181 }
 182
 183 static int set_node_max(struct bch_fs *c, struct btree *b, struct bpos new_max)
 184 {
 185         struct bkey_i_btree_ptr_v2 *new;
 186         int ret;
 187
 188         ret = bch2_journal_key_delete(c, b->c.btree_id, b->c.level + 1, b->key.k.p);
 189         if (ret)
 190                 return ret;
 191
 192         new = kmalloc(BKEY_BTREE_PTR_U64s_MAX * sizeof(u64), GFP_KERNEL);
 193         if (!new)
 194                 return -ENOMEM;
 195
 196         btree_ptr_to_v2(b, new);
 197         b->data->max_key        = new_max;
 198         new->k.p                = new_max;
 199         SET_BTREE_PTR_RANGE_UPDATED(&new->v, true);
 200
 201         ret = bch2_journal_key_insert(c, b->c.btree_id, b->c.level + 1, &new->k_i);
 202         if (ret) {
 203                 kfree(new);
 204                 return ret;
 205         }
 206
 207         bch2_btree_node_drop_keys_outside_node(b);
 208
 209         mutex_lock(&c->btree_cache.lock);
 210         bch2_btree_node_hash_remove(&c->btree_cache, b);
 211
 212         bkey_copy(&b->key, &new->k_i);
 213         ret = __bch2_btree_node_hash_insert(&c->btree_cache, b);
 214         BUG_ON(ret);
 215         mutex_unlock(&c->btree_cache.lock);
 216         return 0;
 217 }
 218
 219 static int btree_repair_node_start(struct bch_fs *c, struct btree *b,
 220                                    struct btree *prev, struct btree *cur)
 221 {
 222         struct bpos expected_start = !prev
 223                 ? b->data->min_key
 224                 : bpos_successor(prev->key.k.p);
 225         char buf1[200], buf2[200];
 226         int ret = 0;
 227
 228         if (!prev) {
 229                 struct printbuf out = PBUF(buf1);
 230                 pr_buf(&out, "start of node: ");
 231                 bch2_bpos_to_text(&out, b->data->min_key);
 232         } else {
 233                 bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(&prev->key));
 234         }
 235
 236         if (mustfix_fsck_err_on(bpos_cmp(expected_start, cur->data->min_key), c,
 237                         "btree node with incorrect min_key at btree %s level %u:\n"
 238                         "  prev %s\n"
 239                         "  cur %s",
 240                         bch2_btree_ids[b->c.btree_id], b->c.level,
 241                         buf1,
 242                         (bch2_bkey_val_to_text(&PBUF(buf2), c, bkey_i_to_s_c(&cur->key)), buf2))) {
 243                 if (prev &&
 244                     bpos_cmp(expected_start, cur->data->min_key) > 0 &&
 245                     BTREE_NODE_SEQ(cur->data) > BTREE_NODE_SEQ(prev->data)) {
 246                         if (bkey_cmp(prev->data->min_key,
 247                                      cur->data->min_key) <= 0)
 248                                 return DROP_PREV_NODE;
 249
 250                         ret = set_node_max(c, prev,
 251                                            bpos_predecessor(cur->data->min_key));
 252                 } else {
 253                         if (bkey_cmp(expected_start, b->data->max_key) >= 0)
 254                                 return DROP_THIS_NODE;
 255
 256                         ret = set_node_min(c, cur, expected_start);
 257                 }
 258                 if (ret)
 259                         return ret;
 260         }
 261 fsck_err:
 262         return ret;
 263 }
 264
 265 static int btree_repair_node_end(struct bch_fs *c, struct btree *b,
 266                                  struct btree *child)
 267 {
 268         char buf1[200], buf2[200];
 269         int ret = 0;
 270
 271         if (mustfix_fsck_err_on(bpos_cmp(child->key.k.p, b->key.k.p), c,
 272                         "btree node with incorrect max_key at btree %s level %u:\n"
 273                         "  %s\n"
 274                         "  expected %s",
 275                         bch2_btree_ids[b->c.btree_id], b->c.level,
 276                         (bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(&child->key)), buf1),
 277                         (bch2_bpos_to_text(&PBUF(buf2), b->key.k.p), buf2))) {
 278                 ret = set_node_max(c, child, b->key.k.p);
 279                 if (ret)
 280                         return ret;
 281         }
 282 fsck_err:
 283         return ret;
 284 }
 285
 286 static int bch2_btree_repair_topology_recurse(struct bch_fs *c, struct btree *b)
 287 {
 288         struct btree_and_journal_iter iter;
 289         struct bkey_s_c k;
 290         struct bkey_buf prev_k, cur_k;
 291         struct btree *prev = NULL, *cur = NULL;
 292         bool have_child, dropped_children = false;
 293         char buf[200];
 294         int ret = 0;
 295
 296         if (!b->c.level)
 297                 return 0;
 298 again:
 299         prev = NULL;
 300         have_child = dropped_children = false;
 301         bch2_bkey_buf_init(&prev_k);
 302         bch2_bkey_buf_init(&cur_k);
 303         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 304
 305         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 306                 bch2_btree_and_journal_iter_advance(&iter);
 307                 bch2_bkey_buf_reassemble(&cur_k, c, k);
 308
 309                 cur = bch2_btree_node_get_noiter(c, cur_k.k,
 310                                         b->c.btree_id, b->c.level - 1,
 311                                         false);
 312                 ret = PTR_ERR_OR_ZERO(cur);
 313
 314                 if (mustfix_fsck_err_on(ret == -EIO, c,
 315                                 "Unreadable btree node at btree %s level %u:\n"
 316                                 "  %s",
 317                                 bch2_btree_ids[b->c.btree_id],
 318                                 b->c.level - 1,
 319                                 (bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(cur_k.k)), buf))) {
 320                         bch2_btree_node_evict(c, cur_k.k);
 321                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 322                                                       b->c.level, cur_k.k->k.p);
 323                         if (ret)
 324                                 break;
 325                         continue;
 326                 }
 327
 328                 if (ret) {
 329                         bch_err(c, "%s: error %i getting btree node",
 330                                 __func__, ret);
 331                         break;
 332                 }
 333
 334                 ret = btree_repair_node_start(c, b, prev, cur);
 335
 336                 if (ret == DROP_THIS_NODE) {
 337                         six_unlock_read(&cur->c.lock);
 338                         bch2_btree_node_evict(c, cur_k.k);
 339                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 340                                                       b->c.level, cur_k.k->k.p);
 341                         if (ret)
 342                                 break;
 343                         continue;
 344                 }
 345
 346                 if (prev)
 347                         six_unlock_read(&prev->c.lock);
 348                 prev = NULL;
 349
 350                 if (ret == DROP_PREV_NODE) {
 351                         bch2_btree_node_evict(c, prev_k.k);
 352                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 353                                                       b->c.level, prev_k.k->k.p);
 354                         if (ret)
 355                                 break;
 356
 357                         bch2_btree_and_journal_iter_exit(&iter);
 358                         bch2_bkey_buf_exit(&prev_k, c);
 359                         bch2_bkey_buf_exit(&cur_k, c);
 360                         goto again;
 361                 } else if (ret)
 362                         break;
 363
 364                 prev = cur;
 365                 cur = NULL;
 366                 bch2_bkey_buf_copy(&prev_k, c, cur_k.k);
 367         }
 368
 369         if (!ret && !IS_ERR_OR_NULL(prev)) {
 370                 BUG_ON(cur);
 371                 ret = btree_repair_node_end(c, b, prev);
 372         }
 373
 374         if (!IS_ERR_OR_NULL(prev))
 375                 six_unlock_read(&prev->c.lock);
 376         prev = NULL;
 377         if (!IS_ERR_OR_NULL(cur))
 378                 six_unlock_read(&cur->c.lock);
 379         cur = NULL;
 380
 381         if (ret)
 382                 goto err;
 383
 384         bch2_btree_and_journal_iter_exit(&iter);
 385         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 386
 387         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 388                 bch2_bkey_buf_reassemble(&cur_k, c, k);
 389                 bch2_btree_and_journal_iter_advance(&iter);
 390
 391                 cur = bch2_btree_node_get_noiter(c, cur_k.k,
 392                                         b->c.btree_id, b->c.level - 1,
 393                                         false);
 394                 ret = PTR_ERR_OR_ZERO(cur);
 395
 396                 if (ret) {
 397                         bch_err(c, "%s: error %i getting btree node",
 398                                 __func__, ret);
 399                         goto err;
 400                 }
 401
 402                 ret = bch2_btree_repair_topology_recurse(c, cur);
 403                 six_unlock_read(&cur->c.lock);
 404                 cur = NULL;
 405
 406                 if (ret == DROP_THIS_NODE) {
 407                         bch2_btree_node_evict(c, cur_k.k);
 408                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 409                                                       b->c.level, cur_k.k->k.p);
 410                         dropped_children = true;
 411                 }
 412
 413                 if (ret)
 414                         goto err;
 415
 416                 have_child = true;
 417         }
 418
 419         if (mustfix_fsck_err_on(!have_child, c,
 420                         "empty interior btree node at btree %s level %u\n"
 421                         "  %s",
 422                         bch2_btree_ids[b->c.btree_id],
 423                         b->c.level,
 424                         (bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(&b->key)), buf)))
 425                 ret = DROP_THIS_NODE;
 426 err:
 427 fsck_err:
 428         if (!IS_ERR_OR_NULL(prev))
 429                 six_unlock_read(&prev->c.lock);
 430         if (!IS_ERR_OR_NULL(cur))
 431                 six_unlock_read(&cur->c.lock);
 432
 433         bch2_btree_and_journal_iter_exit(&iter);
 434         bch2_bkey_buf_exit(&prev_k, c);
 435         bch2_bkey_buf_exit(&cur_k, c);
 436
 437         if (!ret && dropped_children)
 438                 goto again;
 439
 440         return ret;
 441 }
 442
 443 static int bch2_repair_topology(struct bch_fs *c)
 444 {
 445         struct btree *b;
 446         unsigned i;
 447         int ret = 0;
 448
 449         for (i = 0; i < BTREE_ID_NR && !ret; i++) {
 450                 b = c->btree_roots[i].b;
 451                 if (btree_node_fake(b))
 452                         continue;
 453
 454                 six_lock_read(&b->c.lock, NULL, NULL);
 455                 ret = bch2_btree_repair_topology_recurse(c, b);
 456                 six_unlock_read(&b->c.lock);
 457
 458                 if (ret == DROP_THIS_NODE) {
 459                         bch_err(c, "empty btree root - repair unimplemented");
 460                         ret = FSCK_ERR_EXIT;
 461                 }
 462         }
 463
 464         return ret;
 465 }
 466
 467 static int bch2_check_fix_ptrs(struct bch_fs *c, enum btree_id btree_id,
 468                                unsigned level, bool is_root,
 469                                struct bkey_s_c *k)
 470 {
 471         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(*k);
 472         const union bch_extent_entry *entry;
 473         struct extent_ptr_decoded p = { 0 };
 474         bool do_update = false;
 475         char buf[200];
 476         int ret = 0;
 477
 478         bkey_for_each_ptr_decode(k->k, ptrs, p, entry) {
 479                 struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev);
 480                 struct bucket *g = PTR_BUCKET(ca, &p.ptr, true);
 481                 struct bucket *g2 = PTR_BUCKET(ca, &p.ptr, false);
 482                 enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, &entry->ptr);
 483
 484                 if (fsck_err_on(g->mark.data_type &&
 485                                 g->mark.data_type != data_type, c,
 486                                 "bucket %u:%zu different types of data in same bucket: %s, %s\n"
 487                                 "while marking %s",
 488                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 489                                 bch2_data_types[g->mark.data_type],
 490                                 bch2_data_types[data_type],
 491                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf))) {
 492                         if (data_type == BCH_DATA_btree) {
 493                                 g2->_mark.data_type = g->_mark.data_type = data_type;
 494                                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);
 495                         } else {
 496                                 do_update = true;
 497                         }
 498                 }
 499
 500                 if (fsck_err_on(!g->gen_valid, c,
 501                                 "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n"
 502                                 "while marking %s",
 503                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 504                                 bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 505                                 p.ptr.gen,
 506                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf))) {
 507                         if (!p.ptr.cached) {
 508                                 g2->_mark.gen   = g->_mark.gen          = p.ptr.gen;
 509                                 g2->gen_valid   = g->gen_valid          = true;
 510                                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);
 511                         } else {
 512                                 do_update = true;
 513                         }
 514                 }
 515
 516                 if (fsck_err_on(gen_cmp(p.ptr.gen, g->mark.gen) > 0, c,
 517                                 "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n"
 518                                 "while marking %s",
 519                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 520                                 bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 521                                 p.ptr.gen, g->mark.gen,
 522                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf))) {
 523                         if (!p.ptr.cached) {
 524                                 g2->_mark.gen   = g->_mark.gen  = p.ptr.gen;
 525                                 g2->gen_valid   = g->gen_valid  = true;
 526                                 g2->_mark.data_type             = 0;
 527                                 g2->_mark.dirty_sectors         = 0;
 528                                 g2->_mark.cached_sectors        = 0;
 529                                 set_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
 530                                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);
 531                         } else {
 532                                 do_update = true;
 533                         }
 534                 }
 535
 536                 if (fsck_err_on(!p.ptr.cached &&
 537                                 gen_cmp(p.ptr.gen, g->mark.gen) < 0, c,
 538                                 "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n"
 539                                 "while marking %s",
 540                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 541                                 bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 542                                 p.ptr.gen, g->mark.gen,
 543                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf)))
 544                         do_update = true;
 545
 546                 if (p.has_ec) {
 547                         struct stripe *m = genradix_ptr(&c->stripes[true], p.ec.idx);
 548
 549                         if (fsck_err_on(!m || !m->alive, c,
 550                                         "pointer to nonexistent stripe %llu\n"
 551                                         "while marking %s",
 552                                         (u64) p.ec.idx,
 553                                         (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf)))
 554                                 do_update = true;
 555
 556                         if (fsck_err_on(!bch2_ptr_matches_stripe_m(m, p), c,
 557                                         "pointer does not match stripe %llu\n"
 558                                         "while marking %s",
 559                                         (u64) p.ec.idx,
 560                                         (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf)))
 561                                 do_update = true;
 562                 }
 563         }
 564
 565         if (do_update) {
 566                 struct bkey_ptrs ptrs;
 567                 union bch_extent_entry *entry;
 568                 struct bch_extent_ptr *ptr;
 569                 struct bkey_i *new;
 570
 571                 if (is_root) {
 572                         bch_err(c, "cannot update btree roots yet");
 573                         return -EINVAL;
 574                 }
 575
 576                 new = kmalloc(bkey_bytes(k->k), GFP_KERNEL);
 577                 if (!new) {
 578                         bch_err(c, "%s: error allocating new key", __func__);
 579                         return -ENOMEM;
 580                 }
 581
 582                 bkey_reassemble(new, *k);
 583
 584                 if (level) {
 585                         /*
 586                          * We don't want to drop btree node pointers - if the
 587                          * btree node isn't there anymore, the read path will
 588                          * sort it out:
 589                          */
 590                         ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
 591                         bkey_for_each_ptr(ptrs, ptr) {
 592                                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 593                                 struct bucket *g = PTR_BUCKET(ca, ptr, true);
 594
 595                                 ptr->gen = g->mark.gen;
 596                         }
 597                 } else {
 598                         bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, ({
 599                                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 600                                 struct bucket *g = PTR_BUCKET(ca, ptr, true);
 601                                 enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, ptr);
 602
 603                                 (ptr->cached &&
 604                                  (!g->gen_valid || gen_cmp(ptr->gen, g->mark.gen) > 0)) ||
 605                                 (!ptr->cached &&
 606                                  gen_cmp(ptr->gen, g->mark.gen) < 0) ||
 607                                 (g->mark.data_type &&
 608                                  g->mark.data_type != data_type);
 609                         }));
 610 again:
 611                         ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
 612                         bkey_extent_entry_for_each(ptrs, entry) {
 613                                 if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) {
 614                                         struct stripe *m = genradix_ptr(&c->stripes[true],
 615                                                                         entry->stripe_ptr.idx);
 616                                         union bch_extent_entry *next_ptr;
 617
 618                                         bkey_extent_entry_for_each_from(ptrs, next_ptr, entry)
 619                                                 if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr)
 620                                                         goto found;
 621                                         next_ptr = NULL;
 622 found:
 623                                         if (!next_ptr) {
 624                                                 bch_err(c, "aieee, found stripe ptr with no data ptr");
 625                                                 continue;
 626                                         }
 627
 628                                         if (!m || !m->alive ||
 629                                             !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block],
 630                                                                        &next_ptr->ptr,
 631                                                                        m->sectors)) {
 632                                                 bch2_bkey_extent_entry_drop(new, entry);
 633                                                 goto again;
 634                                         }
 635                                 }
 636                         }
 637                 }
 638
 639                 ret = bch2_journal_key_insert(c, btree_id, level, new);
 640                 if (ret)
 641                         kfree(new);
 642                 else
 643                         *k = bkey_i_to_s_c(new);
 644         }
 645 fsck_err:
 646         return ret;
 647 }
 648
 649 /* marking of btree keys/nodes: */
 650
 651 static int bch2_gc_mark_key(struct bch_fs *c, enum btree_id btree_id,
 652                             unsigned level, bool is_root,
 653                             struct bkey_s_c *k,
 654                             u8 *max_stale, bool initial)
 655 {
 656         struct bkey_ptrs_c ptrs;
 657         const struct bch_extent_ptr *ptr;
 658         unsigned flags =
 659                 BTREE_TRIGGER_INSERT|
 660                 BTREE_TRIGGER_GC|
 661                 (initial ? BTREE_TRIGGER_NOATOMIC : 0);
 662         int ret = 0;
 663
 664         if (initial) {
 665                 BUG_ON(bch2_journal_seq_verify &&
 666                        k->k->version.lo > journal_cur_seq(&c->journal));
 667
 668                 ret = bch2_check_fix_ptrs(c, btree_id, level, is_root, k);
 669                 if (ret)
 670                         goto err;
 671
 672                 if (fsck_err_on(k->k->version.lo > atomic64_read(&c->key_version), c,
 673                                 "key version number higher than recorded: %llu > %llu",
 674                                 k->k->version.lo,
 675                                 atomic64_read(&c->key_version)))
 676                         atomic64_set(&c->key_version, k->k->version.lo);
 677
 678                 if (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
 679                     fsck_err_on(!bch2_bkey_replicas_marked(c, *k), c,
 680                                 "superblock not marked as containing replicas (type %u)",
 681                                 k->k->type)) {
 682                         ret = bch2_mark_bkey_replicas(c, *k);
 683                         if (ret) {
 684                                 bch_err(c, "error marking bkey replicas: %i", ret);
 685                                 goto err;
 686                         }
 687                 }
 688         }
 689
 690         ptrs = bch2_bkey_ptrs_c(*k);
 691         bkey_for_each_ptr(ptrs, ptr) {
 692                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 693                 struct bucket *g = PTR_BUCKET(ca, ptr, true);
 694
 695                 if (gen_after(g->oldest_gen, ptr->gen))
 696                         g->oldest_gen = ptr->gen;
 697
 698                 *max_stale = max(*max_stale, ptr_stale(ca, ptr));
 699         }
 700
 701         bch2_mark_key(c, *k, flags);
 702 fsck_err:
 703 err:
 704         if (ret)
 705                 bch_err(c, "%s: ret %i", __func__, ret);
 706         return ret;
 707 }
 708
 709 static int btree_gc_mark_node(struct bch_fs *c, struct btree *b, u8 *max_stale,
 710                               bool initial)
 711 {
 712         struct btree_node_iter iter;
 713         struct bkey unpacked;
 714         struct bkey_s_c k;
 715         struct bkey_buf prev, cur;
 716         int ret = 0;
 717
 718         *max_stale = 0;
 719
 720         if (!btree_node_type_needs_gc(btree_node_type(b)))
 721                 return 0;
 722
 723         bch2_btree_node_iter_init_from_start(&iter, b);
 724         bch2_bkey_buf_init(&prev);
 725         bch2_bkey_buf_init(&cur);
 726         bkey_init(&prev.k->k);
 727
 728         while ((k = bch2_btree_node_iter_peek_unpack(&iter, b, &unpacked)).k) {
 729                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, false,
 730                                        &k, max_stale, initial);
 731                 if (ret)
 732                         break;
 733
 734                 bch2_btree_node_iter_advance(&iter, b);
 735
 736                 if (b->c.level) {
 737                         bch2_bkey_buf_reassemble(&cur, c, k);
 738
 739                         ret = bch2_gc_check_topology(c, b, &prev, cur,
 740                                         bch2_btree_node_iter_end(&iter));
 741                         if (ret)
 742                                 break;
 743                 }
 744         }
 745
 746         bch2_bkey_buf_exit(&cur, c);
 747         bch2_bkey_buf_exit(&prev, c);
 748         return ret;
 749 }
 750
 751 static int bch2_gc_btree(struct bch_fs *c, enum btree_id btree_id,
 752                          bool initial, bool metadata_only)
 753 {
 754         struct btree_trans trans;
 755         struct btree_iter *iter;
 756         struct btree *b;
 757         unsigned depth = metadata_only                  ? 1
 758                 : bch2_expensive_debug_checks           ? 0
 759                 : !btree_node_type_needs_gc(btree_id)   ? 1
 760                 : 0;
 761         u8 max_stale = 0;
 762         int ret = 0;
 763
 764         bch2_trans_init(&trans, c, 0, 0);
 765
 766         gc_pos_set(c, gc_pos_btree(btree_id, POS_MIN, 0));
 767
 768         __for_each_btree_node(&trans, iter, btree_id, POS_MIN,
 769                               0, depth, BTREE_ITER_PREFETCH, b) {
 770                 bch2_verify_btree_nr_keys(b);
 771
 772                 gc_pos_set(c, gc_pos_btree_node(b));
 773
 774                 ret = btree_gc_mark_node(c, b, &max_stale, initial);
 775                 if (ret)
 776                         break;
 777
 778                 if (!initial) {
 779                         if (max_stale > 64)
 780                                 bch2_btree_node_rewrite(c, iter,
 781                                                 b->data->keys.seq,
 782                                                 BTREE_INSERT_NOWAIT|
 783                                                 BTREE_INSERT_GC_LOCK_HELD);
 784                         else if (!bch2_btree_gc_rewrite_disabled &&
 785                                  (bch2_btree_gc_always_rewrite || max_stale > 16))
 786                                 bch2_btree_node_rewrite(c, iter,
 787                                                 b->data->keys.seq,
 788                                                 BTREE_INSERT_NOWAIT|
 789                                                 BTREE_INSERT_GC_LOCK_HELD);
 790                 }
 791
 792                 bch2_trans_cond_resched(&trans);
 793         }
 794         bch2_trans_iter_put(&trans, iter);
 795
 796         ret = bch2_trans_exit(&trans) ?: ret;
 797         if (ret)
 798                 return ret;
 799
 800         mutex_lock(&c->btree_root_lock);
 801         b = c->btree_roots[btree_id].b;
 802         if (!btree_node_fake(b)) {
 803                 struct bkey_s_c k = bkey_i_to_s_c(&b->key);
 804
 805                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, true,
 806                                        &k, &max_stale, initial);
 807         }
 808         gc_pos_set(c, gc_pos_btree_root(b->c.btree_id));
 809         mutex_unlock(&c->btree_root_lock);
 810
 811         return ret;
 812 }
 813
 814 static int bch2_gc_btree_init_recurse(struct bch_fs *c, struct btree *b,
 815                                       unsigned target_depth)
 816 {
 817         struct btree_and_journal_iter iter;
 818         struct bkey_s_c k;
 819         struct bkey_buf cur, prev;
 820         u8 max_stale = 0;
 821         char buf[200];
 822         int ret = 0;
 823
 824         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 825         bch2_bkey_buf_init(&prev);
 826         bch2_bkey_buf_init(&cur);
 827         bkey_init(&prev.k->k);
 828
 829         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 830                 BUG_ON(bpos_cmp(k.k->p, b->data->min_key) < 0);
 831                 BUG_ON(bpos_cmp(k.k->p, b->data->max_key) > 0);
 832
 833                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, false,
 834                                        &k, &max_stale, true);
 835                 if (ret) {
 836                         bch_err(c, "%s: error %i from bch2_gc_mark_key", __func__, ret);
 837                         goto fsck_err;
 838                 }
 839
 840                 if (b->c.level) {
 841                         bch2_bkey_buf_reassemble(&cur, c, k);
 842                         k = bkey_i_to_s_c(cur.k);
 843
 844                         bch2_btree_and_journal_iter_advance(&iter);
 845
 846                         ret = bch2_gc_check_topology(c, b,
 847                                         &prev, cur,
 848                                         !bch2_btree_and_journal_iter_peek(&iter).k);
 849                         if (ret)
 850                                 goto fsck_err;
 851                 } else {
 852                         bch2_btree_and_journal_iter_advance(&iter);
 853                 }
 854         }
 855
 856         if (b->c.level > target_depth) {
 857                 bch2_btree_and_journal_iter_exit(&iter);
 858                 bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 859
 860                 while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 861                         struct btree *child;
 862
 863                         bch2_bkey_buf_reassemble(&cur, c, k);
 864                         bch2_btree_and_journal_iter_advance(&iter);
 865
 866                         child = bch2_btree_node_get_noiter(c, cur.k,
 867                                                 b->c.btree_id, b->c.level - 1,
 868                                                 false);
 869                         ret = PTR_ERR_OR_ZERO(child);
 870
 871                         if (ret == -EIO) {
 872                                 bch2_topology_error(c);
 873
 874                                 if (__fsck_err(c,
 875                                           FSCK_CAN_FIX|
 876                                           FSCK_CAN_IGNORE|
 877                                           FSCK_NO_RATELIMIT,
 878                                           "Unreadable btree node at btree %s level %u:\n"
 879                                           "  %s",
 880                                           bch2_btree_ids[b->c.btree_id],
 881                                           b->c.level - 1,
 882                                           (bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(cur.k)), buf)) &&
 883                                     !test_bit(BCH_FS_TOPOLOGY_REPAIR_DONE, &c->flags)) {
 884                                         ret = FSCK_ERR_START_TOPOLOGY_REPAIR;
 885                                         bch_info(c, "Halting mark and sweep to start topology repair pass");
 886                                         goto fsck_err;
 887                                 } else {
 888                                         /* Continue marking when opted to not
 889                                          * fix the error: */
 890                                         ret = 0;
 891                                         set_bit(BCH_FS_INITIAL_GC_UNFIXED, &c->flags);
 892                                         continue;
 893                                 }
 894                         } else if (ret) {
 895                                 bch_err(c, "%s: error %i getting btree node",
 896                                         __func__, ret);
 897                                 break;
 898                         }
 899
 900                         ret = bch2_gc_btree_init_recurse(c, child,
 901                                                          target_depth);
 902                         six_unlock_read(&child->c.lock);
 903
 904                         if (ret)
 905                                 break;
 906                 }
 907         }
 908 fsck_err:
 909         bch2_bkey_buf_exit(&cur, c);
 910         bch2_bkey_buf_exit(&prev, c);
 911         bch2_btree_and_journal_iter_exit(&iter);
 912         return ret;
 913 }
 914
 915 static int bch2_gc_btree_init(struct bch_fs *c,
 916                               enum btree_id btree_id,
 917                               bool metadata_only)
 918 {
 919         struct btree *b;
 920         unsigned target_depth = metadata_only           ? 1
 921                 : bch2_expensive_debug_checks           ? 0
 922                 : !btree_node_type_needs_gc(btree_id)   ? 1
 923                 : 0;
 924         u8 max_stale = 0;
 925         char buf[100];
 926         int ret = 0;
 927
 928         b = c->btree_roots[btree_id].b;
 929
 930         if (btree_node_fake(b))
 931                 return 0;
 932
 933         six_lock_read(&b->c.lock, NULL, NULL);
 934         if (mustfix_fsck_err_on(bpos_cmp(b->data->min_key, POS_MIN), c,
 935                         "btree root with incorrect min_key: %s",
 936                         (bch2_bpos_to_text(&PBUF(buf), b->data->min_key), buf))) {
 937                 bch_err(c, "repair unimplemented");
 938                 ret = FSCK_ERR_EXIT;
 939                 goto fsck_err;
 940         }
 941
 942         if (mustfix_fsck_err_on(bpos_cmp(b->data->max_key, POS_MAX), c,
 943                         "btree root with incorrect max_key: %s",
 944                         (bch2_bpos_to_text(&PBUF(buf), b->data->max_key), buf))) {
 945                 bch_err(c, "repair unimplemented");
 946                 ret = FSCK_ERR_EXIT;
 947                 goto fsck_err;
 948         }
 949
 950         if (b->c.level >= target_depth)
 951                 ret = bch2_gc_btree_init_recurse(c, b, target_depth);
 952
 953         if (!ret) {
 954                 struct bkey_s_c k = bkey_i_to_s_c(&b->key);
 955
 956                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, true,
 957                                        &k, &max_stale, true);
 958         }
 959 fsck_err:
 960         six_unlock_read(&b->c.lock);
 961
 962         if (ret < 0)
 963                 bch_err(c, "%s: ret %i", __func__, ret);
 964         return ret;
 965 }
 966
 967 static inline int btree_id_gc_phase_cmp(enum btree_id l, enum btree_id r)
 968 {
 969         return  (int) btree_id_to_gc_phase(l) -
 970                 (int) btree_id_to_gc_phase(r);
 971 }
 972
 973 static int bch2_gc_btrees(struct bch_fs *c, bool initial, bool metadata_only)
 974 {
 975         enum btree_id ids[BTREE_ID_NR];
 976         unsigned i;
 977         int ret = 0;
 978
 979         for (i = 0; i < BTREE_ID_NR; i++)
 980                 ids[i] = i;
 981         bubble_sort(ids, BTREE_ID_NR, btree_id_gc_phase_cmp);
 982
 983         for (i = 0; i < BTREE_ID_NR && !ret; i++)
 984                 ret = initial
 985                         ? bch2_gc_btree_init(c, ids[i], metadata_only)
 986                         : bch2_gc_btree(c, ids[i], initial, metadata_only);
 987
 988         if (ret < 0)
 989                 bch_err(c, "%s: ret %i", __func__, ret);
 990         return ret;
 991 }
 992
 993 static void mark_metadata_sectors(struct bch_fs *c, struct bch_dev *ca,
 994                                   u64 start, u64 end,
 995                                   enum bch_data_type type,
 996                                   unsigned flags)
 997 {
 998         u64 b = sector_to_bucket(ca, start);
 999
1000         do {
1001                 unsigned sectors =
1002                         min_t(u64, bucket_to_sector(ca, b + 1), end) - start;
1003
1004                 bch2_mark_metadata_bucket(c, ca, b, type, sectors,
1005                                           gc_phase(GC_PHASE_SB), flags);
1006                 b++;
1007                 start += sectors;
1008         } while (start < end);
1009 }
1010
1011 void bch2_mark_dev_superblock(struct bch_fs *c, struct bch_dev *ca,
1012                               unsigned flags)
1013 {
1014         struct bch_sb_layout *layout = &ca->disk_sb.sb->layout;
1015         unsigned i;
1016         u64 b;
1017
1018         /*
1019          * This conditional is kind of gross, but we may be called from the
1020          * device add path, before the new device has actually been added to the
1021          * running filesystem:
1022          */
1023         if (c) {
1024                 lockdep_assert_held(&c->sb_lock);
1025                 percpu_down_read(&c->mark_lock);
1026         }
1027
1028         for (i = 0; i < layout->nr_superblocks; i++) {
1029                 u64 offset = le64_to_cpu(layout->sb_offset[i]);
1030
1031                 if (offset == BCH_SB_SECTOR)
1032                         mark_metadata_sectors(c, ca, 0, BCH_SB_SECTOR,
1033                                               BCH_DATA_sb, flags);
1034
1035                 mark_metadata_sectors(c, ca, offset,
1036                                       offset + (1 << layout->sb_max_size_bits),
1037                                       BCH_DATA_sb, flags);
1038         }
1039
1040         for (i = 0; i < ca->journal.nr; i++) {
1041                 b = ca->journal.buckets[i];
1042                 bch2_mark_metadata_bucket(c, ca, b, BCH_DATA_journal,
1043                                           ca->mi.bucket_size,
1044                                           gc_phase(GC_PHASE_SB), flags);
1045         }
1046
1047         if (c)
1048                 percpu_up_read(&c->mark_lock);
1049 }
1050
1051 static void bch2_mark_superblocks(struct bch_fs *c)
1052 {
1053         struct bch_dev *ca;
1054         unsigned i;
1055
1056         mutex_lock(&c->sb_lock);
1057         gc_pos_set(c, gc_phase(GC_PHASE_SB));
1058
1059         for_each_online_member(ca, c, i)
1060                 bch2_mark_dev_superblock(c, ca, BTREE_TRIGGER_GC);
1061         mutex_unlock(&c->sb_lock);
1062 }
1063
1064 #if 0
1065 /* Also see bch2_pending_btree_node_free_insert_done() */
1066 static void bch2_mark_pending_btree_node_frees(struct bch_fs *c)
1067 {
1068         struct btree_update *as;
1069         struct pending_btree_node_free *d;
1070
1071         mutex_lock(&c->btree_interior_update_lock);
1072         gc_pos_set(c, gc_phase(GC_PHASE_PENDING_DELETE));
1073
1074         for_each_pending_btree_node_free(c, as, d)
1075                 if (d->index_update_done)
1076                         bch2_mark_key(c, bkey_i_to_s_c(&d->key),
1077                                       BTREE_TRIGGER_INSERT|BTREE_TRIGGER_GC);
1078
1079         mutex_unlock(&c->btree_interior_update_lock);
1080 }
1081 #endif
1082
1083 static void bch2_gc_free(struct bch_fs *c)
1084 {
1085         struct bch_dev *ca;
1086         unsigned i;
1087
1088         genradix_free(&c->stripes[1]);
1089
1090         for_each_member_device(ca, c, i) {
1091                 kvpfree(rcu_dereference_protected(ca->buckets[1], 1),
1092                         sizeof(struct bucket_array) +
1093                         ca->mi.nbuckets * sizeof(struct bucket));
1094                 ca->buckets[1] = NULL;
1095
1096                 free_percpu(ca->usage_gc);
1097                 ca->usage_gc = NULL;
1098         }
1099
1100         free_percpu(c->usage_gc);
1101         c->usage_gc = NULL;
1102 }
1103
1104 static int bch2_gc_done(struct bch_fs *c,
1105                         bool initial, bool metadata_only)
1106 {
1107         struct bch_dev *ca = NULL;
1108         bool verify = !metadata_only && (!initial ||
1109                        (c->sb.compat & (1ULL << BCH_COMPAT_alloc_info)));
1110         unsigned i, dev;
1111         int ret = 0;
1112
1113 #define copy_field(_f, _msg, ...)                                       \
1114         if (dst->_f != src->_f) {                                       \
1115                 if (verify)                                             \
1116                         fsck_err(c, _msg ": got %llu, should be %llu"   \
1117                                 , ##__VA_ARGS__, dst->_f, src->_f);     \
1118                 dst->_f = src->_f;                                      \
1119                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);            \
1120         }
1121 #define copy_stripe_field(_f, _msg, ...)                                \
1122         if (dst->_f != src->_f) {                                       \
1123                 if (verify)                                             \
1124                         fsck_err(c, "stripe %zu has wrong "_msg         \
1125                                 ": got %u, should be %u",               \
1126                                 iter.pos, ##__VA_ARGS__,                \
1127                                 dst->_f, src->_f);                      \
1128                 dst->_f = src->_f;                                      \
1129                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);            \
1130         }
1131 #define copy_bucket_field(_f)                                           \
1132         if (dst->b[b].mark._f != src->b[b].mark._f) {                   \
1133                 if (verify)                                             \
1134                         fsck_err(c, "bucket %u:%zu gen %u data type %s has wrong " #_f  \
1135                                 ": got %u, should be %u", dev, b,       \
1136                                 dst->b[b].mark.gen,                     \
1137                                 bch2_data_types[dst->b[b].mark.data_type],\
1138                                 dst->b[b].mark._f, src->b[b].mark._f);  \
1139                 dst->b[b]._mark._f = src->b[b].mark._f;                 \
1140                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);            \
1141         }
1142 #define copy_dev_field(_f, _msg, ...)                                   \
1143         copy_field(_f, "dev %u has wrong " _msg, dev, ##__VA_ARGS__)
1144 #define copy_fs_field(_f, _msg, ...)                                    \
1145         copy_field(_f, "fs has wrong " _msg, ##__VA_ARGS__)
1146
1147         if (!metadata_only) {
1148                 struct genradix_iter iter = genradix_iter_init(&c->stripes[1], 0);
1149                 struct stripe *dst, *src;
1150
1151                 while ((src = genradix_iter_peek(&iter, &c->stripes[1]))) {
1152                         dst = genradix_ptr_alloc(&c->stripes[0], iter.pos, GFP_KERNEL);
1153
1154                         if (dst->alive          != src->alive ||
1155                             dst->sectors        != src->sectors ||
1156                             dst->algorithm      != src->algorithm ||
1157                             dst->nr_blocks      != src->nr_blocks ||
1158                             dst->nr_redundant   != src->nr_redundant) {
1159                                 bch_err(c, "unexpected stripe inconsistency at bch2_gc_done, confused");
1160                                 ret = -EINVAL;
1161                                 goto fsck_err;
1162                         }
1163
1164                         for (i = 0; i < ARRAY_SIZE(dst->block_sectors); i++)
1165                                 copy_stripe_field(block_sectors[i],
1166                                                   "block_sectors[%u]", i);
1167
1168                         dst->blocks_nonempty = 0;
1169                         for (i = 0; i < dst->nr_blocks; i++)
1170                                 dst->blocks_nonempty += dst->block_sectors[i] != 0;
1171
1172                         genradix_iter_advance(&iter, &c->stripes[1]);
1173                 }
1174         }
1175
1176         for (i = 0; i < ARRAY_SIZE(c->usage); i++)
1177                 bch2_fs_usage_acc_to_base(c, i);
1178
1179         for_each_member_device(ca, c, dev) {
1180                 struct bucket_array *dst = __bucket_array(ca, 0);
1181                 struct bucket_array *src = __bucket_array(ca, 1);
1182                 size_t b;
1183
1184                 for (b = 0; b < src->nbuckets; b++) {
1185                         copy_bucket_field(gen);
1186                         copy_bucket_field(data_type);
1187                         copy_bucket_field(stripe);
1188                         copy_bucket_field(dirty_sectors);
1189                         copy_bucket_field(cached_sectors);
1190
1191                         dst->b[b].oldest_gen = src->b[b].oldest_gen;
1192                 }
1193
1194                 {
1195                         struct bch_dev_usage *dst = ca->usage_base;
1196                         struct bch_dev_usage *src = (void *)
1197                                 bch2_acc_percpu_u64s((void *) ca->usage_gc,
1198                                                      dev_usage_u64s());
1199
1200                         copy_dev_field(buckets_ec,              "buckets_ec");
1201                         copy_dev_field(buckets_unavailable,     "buckets_unavailable");
1202
1203                         for (i = 0; i < BCH_DATA_NR; i++) {
1204                                 copy_dev_field(d[i].buckets,    "%s buckets", bch2_data_types[i]);
1205                                 copy_dev_field(d[i].sectors,    "%s sectors", bch2_data_types[i]);
1206                                 copy_dev_field(d[i].fragmented, "%s fragmented", bch2_data_types[i]);
1207                         }
1208                 }
1209         };
1210
1211         {
1212                 unsigned nr = fs_usage_u64s(c);
1213                 struct bch_fs_usage *dst = c->usage_base;
1214                 struct bch_fs_usage *src = (void *)
1215                         bch2_acc_percpu_u64s((void *) c->usage_gc, nr);
1216
1217                 copy_fs_field(hidden,           "hidden");
1218                 copy_fs_field(btree,            "btree");
1219
1220                 if (!metadata_only) {
1221                         copy_fs_field(data,     "data");
1222                         copy_fs_field(cached,   "cached");
1223                         copy_fs_field(reserved, "reserved");
1224                         copy_fs_field(nr_inodes,"nr_inodes");
1225
1226                         for (i = 0; i < BCH_REPLICAS_MAX; i++)
1227                                 copy_fs_field(persistent_reserved[i],
1228                                               "persistent_reserved[%i]", i);
1229                 }
1230
1231                 for (i = 0; i < c->replicas.nr; i++) {
1232                         struct bch_replicas_entry *e =
1233                                 cpu_replicas_entry(&c->replicas, i);
1234                         char buf[80];
1235
1236                         if (metadata_only &&
1237                             (e->data_type == BCH_DATA_user ||
1238                              e->data_type == BCH_DATA_cached))
1239                                 continue;
1240
1241                         bch2_replicas_entry_to_text(&PBUF(buf), e);
1242
1243                         copy_fs_field(replicas[i], "%s", buf);
1244                 }
1245         }
1246
1247 #undef copy_fs_field
1248 #undef copy_dev_field
1249 #undef copy_bucket_field
1250 #undef copy_stripe_field
1251 #undef copy_field
1252 fsck_err:
1253         if (ca)
1254                 percpu_ref_put(&ca->ref);
1255         if (ret)
1256                 bch_err(c, "%s: ret %i", __func__, ret);
1257         return ret;
1258 }
1259
1260 static int bch2_gc_start(struct bch_fs *c,
1261                          bool metadata_only)
1262 {
1263         struct bch_dev *ca = NULL;
1264         unsigned i;
1265         int ret;
1266
1267         BUG_ON(c->usage_gc);
1268
1269         c->usage_gc = __alloc_percpu_gfp(fs_usage_u64s(c) * sizeof(u64),
1270                                          sizeof(u64), GFP_KERNEL);
1271         if (!c->usage_gc) {
1272                 bch_err(c, "error allocating c->usage_gc");
1273                 return -ENOMEM;
1274         }
1275
1276         for_each_member_device(ca, c, i) {
1277                 BUG_ON(ca->buckets[1]);
1278                 BUG_ON(ca->usage_gc);
1279
1280                 ca->buckets[1] = kvpmalloc(sizeof(struct bucket_array) +
1281                                 ca->mi.nbuckets * sizeof(struct bucket),
1282                                 GFP_KERNEL|__GFP_ZERO);
1283                 if (!ca->buckets[1]) {
1284                         percpu_ref_put(&ca->ref);
1285                         bch_err(c, "error allocating ca->buckets[gc]");
1286                         return -ENOMEM;
1287                 }
1288
1289                 ca->usage_gc = alloc_percpu(struct bch_dev_usage);
1290                 if (!ca->usage_gc) {
1291                         bch_err(c, "error allocating ca->usage_gc");
1292                         percpu_ref_put(&ca->ref);
1293                         return -ENOMEM;
1294                 }
1295         }
1296
1297         ret = bch2_ec_mem_alloc(c, true);
1298         if (ret) {
1299                 bch_err(c, "error allocating ec gc mem");
1300                 return ret;
1301         }
1302
1303         percpu_down_write(&c->mark_lock);
1304
1305         /*
1306          * indicate to stripe code that we need to allocate for the gc stripes
1307          * radix tree, too
1308          */
1309         gc_pos_set(c, gc_phase(GC_PHASE_START));
1310
1311         for_each_member_device(ca, c, i) {
1312                 struct bucket_array *dst = __bucket_array(ca, 1);
1313                 struct bucket_array *src = __bucket_array(ca, 0);
1314                 size_t b;
1315
1316                 dst->first_bucket       = src->first_bucket;
1317                 dst->nbuckets           = src->nbuckets;
1318
1319                 for (b = 0; b < src->nbuckets; b++) {
1320                         struct bucket *d = &dst->b[b];
1321                         struct bucket *s = &src->b[b];
1322
1323                         d->_mark.gen = dst->b[b].oldest_gen = s->mark.gen;
1324                         d->gen_valid = s->gen_valid;
1325
1326                         if (metadata_only &&
1327                             (s->mark.data_type == BCH_DATA_user ||
1328                              s->mark.data_type == BCH_DATA_cached))
1329                                 d->_mark = s->mark;
1330                 }
1331         };
1332
1333         percpu_up_write(&c->mark_lock);
1334
1335         return 0;
1336 }
1337
1338 static int bch2_gc_reflink_done_initial_fn(struct bch_fs *c, struct bkey_s_c k)
1339 {
1340         struct reflink_gc *r;
1341         const __le64 *refcount = bkey_refcount_c(k);
1342         char buf[200];
1343         int ret = 0;
1344
1345         if (!refcount)
1346                 return 0;
1347
1348         r = genradix_ptr(&c->reflink_gc_table, c->reflink_gc_idx++);
1349         if (!r)
1350                 return -ENOMEM;
1351
1352         if (!r ||
1353             r->offset != k.k->p.offset ||
1354             r->size != k.k->size) {
1355                 bch_err(c, "unexpected inconsistency walking reflink table at gc finish");
1356                 return -EINVAL;
1357         }
1358
1359         if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), c,
1360                         "reflink key has wrong refcount:\n"
1361                         "  %s\n"
1362                         "  should be %u",
1363                         (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf),
1364                         r->refcount)) {
1365                 struct bkey_i *new;
1366
1367                 new = kmalloc(bkey_bytes(k.k), GFP_KERNEL);
1368                 if (!new) {
1369                         ret = -ENOMEM;
1370                         goto fsck_err;
1371                 }
1372
1373                 bkey_reassemble(new, k);
1374
1375                 if (!r->refcount) {
1376                         new->k.type = KEY_TYPE_deleted;
1377                         new->k.size = 0;
1378                 } else {
1379                         *bkey_refcount(new) = cpu_to_le64(r->refcount);
1380                 }
1381
1382                 ret = bch2_journal_key_insert(c, BTREE_ID_reflink, 0, new);
1383                 if (ret)
1384                         kfree(new);
1385         }
1386 fsck_err:
1387         return ret;
1388 }
1389
1390 static int bch2_gc_reflink_done(struct bch_fs *c, bool initial,
1391                                 bool metadata_only)
1392 {
1393         struct btree_trans trans;
1394         struct btree_iter *iter;
1395         struct bkey_s_c k;
1396         struct reflink_gc *r;
1397         size_t idx = 0;
1398         char buf[200];
1399         int ret = 0;
1400
1401         if (metadata_only)
1402                 return 0;
1403
1404         if (initial) {
1405                 c->reflink_gc_idx = 0;
1406
1407                 ret = bch2_btree_and_journal_walk(c, BTREE_ID_reflink,
1408                                 bch2_gc_reflink_done_initial_fn);
1409                 goto out;
1410         }
1411
1412         bch2_trans_init(&trans, c, 0, 0);
1413
1414         for_each_btree_key(&trans, iter, BTREE_ID_reflink, POS_MIN,
1415                            BTREE_ITER_PREFETCH, k, ret) {
1416                 const __le64 *refcount = bkey_refcount_c(k);
1417
1418                 if (!refcount)
1419                         continue;
1420
1421                 r = genradix_ptr(&c->reflink_gc_table, idx);
1422                 if (!r ||
1423                     r->offset != k.k->p.offset ||
1424                     r->size != k.k->size) {
1425                         bch_err(c, "unexpected inconsistency walking reflink table at gc finish");
1426                         ret = -EINVAL;
1427                         break;
1428                 }
1429
1430                 if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), c,
1431                                 "reflink key has wrong refcount:\n"
1432                                 "  %s\n"
1433                                 "  should be %u",
1434                                 (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf),
1435                                 r->refcount)) {
1436                         struct bkey_i *new;
1437
1438                         new = kmalloc(bkey_bytes(k.k), GFP_KERNEL);
1439                         if (!new) {
1440                                 ret = -ENOMEM;
1441                                 break;
1442                         }
1443
1444                         bkey_reassemble(new, k);
1445
1446                         if (!r->refcount)
1447                                 new->k.type = KEY_TYPE_deleted;
1448                         else
1449                                 *bkey_refcount(new) = cpu_to_le64(r->refcount);
1450
1451                         ret = __bch2_trans_do(&trans, NULL, NULL, 0,
1452                                         __bch2_btree_insert(&trans, BTREE_ID_reflink, new));
1453                         kfree(new);
1454
1455                         if (ret)
1456                                 break;
1457                 }
1458         }
1459 fsck_err:
1460         bch2_trans_iter_put(&trans, iter);
1461         bch2_trans_exit(&trans);
1462 out:
1463         genradix_free(&c->reflink_gc_table);
1464         c->reflink_gc_nr = 0;
1465         return ret;
1466 }
1467
1468 static int bch2_gc_reflink_start_initial_fn(struct bch_fs *c, struct bkey_s_c k)
1469 {
1470
1471         struct reflink_gc *r;
1472         const __le64 *refcount = bkey_refcount_c(k);
1473
1474         if (!refcount)
1475                 return 0;
1476
1477         r = genradix_ptr_alloc(&c->reflink_gc_table, c->reflink_gc_nr++,
1478                                GFP_KERNEL);
1479         if (!r)
1480                 return -ENOMEM;
1481
1482         r->offset       = k.k->p.offset;
1483         r->size         = k.k->size;
1484         r->refcount     = 0;
1485         return 0;
1486 }
1487
1488 static int bch2_gc_reflink_start(struct bch_fs *c, bool initial,
1489                                  bool metadata_only)
1490 {
1491         struct btree_trans trans;
1492         struct btree_iter *iter;
1493         struct bkey_s_c k;
1494         struct reflink_gc *r;
1495         int ret;
1496
1497         if (metadata_only)
1498                 return 0;
1499
1500         genradix_free(&c->reflink_gc_table);
1501         c->reflink_gc_nr = 0;
1502
1503         if (initial)
1504                 return bch2_btree_and_journal_walk(c, BTREE_ID_reflink,
1505                                 bch2_gc_reflink_start_initial_fn);
1506
1507         bch2_trans_init(&trans, c, 0, 0);
1508
1509         for_each_btree_key(&trans, iter, BTREE_ID_reflink, POS_MIN,
1510                            BTREE_ITER_PREFETCH, k, ret) {
1511                 const __le64 *refcount = bkey_refcount_c(k);
1512
1513                 if (!refcount)
1514                         continue;
1515
1516                 r = genradix_ptr_alloc(&c->reflink_gc_table, c->reflink_gc_nr++,
1517                                        GFP_KERNEL);
1518                 if (!r) {
1519                         ret = -ENOMEM;
1520                         break;
1521                 }
1522
1523                 r->offset       = k.k->p.offset;
1524                 r->size         = k.k->size;
1525                 r->refcount     = 0;
1526         }
1527         bch2_trans_iter_put(&trans, iter);
1528
1529         bch2_trans_exit(&trans);
1530         return 0;
1531 }
1532
1533 /**
1534  * bch2_gc - walk _all_ references to buckets, and recompute them:
1535  *
1536  * Order matters here:
1537  *  - Concurrent GC relies on the fact that we have a total ordering for
1538  *    everything that GC walks - see  gc_will_visit_node(),
1539  *    gc_will_visit_root()
1540  *
1541  *  - also, references move around in the course of index updates and
1542  *    various other crap: everything needs to agree on the ordering
1543  *    references are allowed to move around in - e.g., we're allowed to
1544  *    start with a reference owned by an open_bucket (the allocator) and
1545  *    move it to the btree, but not the reverse.
1546  *
1547  *    This is necessary to ensure that gc doesn't miss references that
1548  *    move around - if references move backwards in the ordering GC
1549  *    uses, GC could skip past them
1550  */
1551 int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only)
1552 {
1553         struct bch_dev *ca;
1554         u64 start_time = local_clock();
1555         unsigned i, iter = 0;
1556         int ret;
1557
1558         lockdep_assert_held(&c->state_lock);
1559         trace_gc_start(c);
1560
1561         down_write(&c->gc_lock);
1562
1563         /* flush interior btree updates: */
1564         closure_wait_event(&c->btree_interior_update_wait,
1565                            !bch2_btree_interior_updates_nr_pending(c));
1566 again:
1567         ret   = bch2_gc_start(c, metadata_only) ?:
1568                 bch2_gc_reflink_start(c, initial, metadata_only);
1569         if (ret)
1570                 goto out;
1571
1572         bch2_mark_superblocks(c);
1573
1574         if (test_bit(BCH_FS_TOPOLOGY_ERROR, &c->flags) &&
1575             !test_bit(BCH_FS_INITIAL_GC_DONE, &c->flags) &&
1576             c->opts.fix_errors != FSCK_OPT_NO) {
1577                 bch_info(c, "starting topology repair pass");
1578                 ret = bch2_repair_topology(c);
1579                 if (ret)
1580                         goto out;
1581                 bch_info(c, "topology repair pass done");
1582
1583                 set_bit(BCH_FS_TOPOLOGY_REPAIR_DONE, &c->flags);
1584         }
1585
1586         ret = bch2_gc_btrees(c, initial, metadata_only);
1587
1588         if (ret == FSCK_ERR_START_TOPOLOGY_REPAIR &&
1589             !test_bit(BCH_FS_TOPOLOGY_REPAIR_DONE, &c->flags) &&
1590             !test_bit(BCH_FS_INITIAL_GC_DONE, &c->flags)) {
1591                 set_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
1592                 ret = 0;
1593         }
1594
1595         if (ret == FSCK_ERR_START_TOPOLOGY_REPAIR)
1596                 ret = FSCK_ERR_EXIT;
1597
1598         if (ret)
1599                 goto out;
1600
1601 #if 0
1602         bch2_mark_pending_btree_node_frees(c);
1603 #endif
1604         c->gc_count++;
1605
1606         if (test_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags) ||
1607             (!iter && bch2_test_restart_gc)) {
1608                 /*
1609                  * XXX: make sure gens we fixed got saved
1610                  */
1611                 if (iter++ <= 2) {
1612                         bch_info(c, "Second GC pass needed, restarting:");
1613                         clear_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
1614                         __gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING));
1615
1616                         percpu_down_write(&c->mark_lock);
1617                         bch2_gc_free(c);
1618                         percpu_up_write(&c->mark_lock);
1619                         /* flush fsck errors, reset counters */
1620                         bch2_flush_fsck_errs(c);
1621
1622                         goto again;
1623                 }
1624
1625                 bch_info(c, "Unable to fix bucket gens, looping");
1626                 ret = -EINVAL;
1627         }
1628 out:
1629         if (!ret) {
1630                 bch2_journal_block(&c->journal);
1631
1632                 percpu_down_write(&c->mark_lock);
1633                 ret   = bch2_gc_reflink_done(c, initial, metadata_only) ?:
1634                         bch2_gc_done(c, initial, metadata_only);
1635
1636                 bch2_journal_unblock(&c->journal);
1637         } else {
1638                 percpu_down_write(&c->mark_lock);
1639         }
1640
1641         /* Indicates that gc is no longer in progress: */
1642         __gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING));
1643
1644         bch2_gc_free(c);
1645         percpu_up_write(&c->mark_lock);
1646
1647         up_write(&c->gc_lock);
1648
1649         trace_gc_end(c);
1650         bch2_time_stats_update(&c->times[BCH_TIME_btree_gc], start_time);
1651
1652         /*
1653          * Wake up allocator in case it was waiting for buckets
1654          * because of not being able to inc gens
1655          */
1656         for_each_member_device(ca, c, i)
1657                 bch2_wake_allocator(ca);
1658
1659         /*
1660          * At startup, allocations can happen directly instead of via the
1661          * allocator thread - issue wakeup in case they blocked on gc_lock:
1662          */
1663         closure_wake_up(&c->freelist_wait);
1664         return ret;
1665 }
1666
1667 static bool gc_btree_gens_key(struct bch_fs *c, struct bkey_s_c k)
1668 {
1669         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
1670         const struct bch_extent_ptr *ptr;
1671
1672         percpu_down_read(&c->mark_lock);
1673         bkey_for_each_ptr(ptrs, ptr) {
1674                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
1675                 struct bucket *g = PTR_BUCKET(ca, ptr, false);
1676
1677                 if (gen_after(g->mark.gen, ptr->gen) > 16) {
1678                         percpu_up_read(&c->mark_lock);
1679                         return true;
1680                 }
1681         }
1682
1683         bkey_for_each_ptr(ptrs, ptr) {
1684                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
1685                 struct bucket *g = PTR_BUCKET(ca, ptr, false);
1686
1687                 if (gen_after(g->gc_gen, ptr->gen))
1688                         g->gc_gen = ptr->gen;
1689         }
1690         percpu_up_read(&c->mark_lock);
1691
1692         return false;
1693 }
1694
1695 /*
1696  * For recalculating oldest gen, we only need to walk keys in leaf nodes; btree
1697  * node pointers currently never have cached pointers that can become stale:
1698  */
1699 static int bch2_gc_btree_gens(struct bch_fs *c, enum btree_id btree_id)
1700 {
1701         struct btree_trans trans;
1702         struct btree_iter *iter;
1703         struct bkey_s_c k;
1704         struct bkey_buf sk;
1705         int ret = 0, commit_err = 0;
1706
1707         bch2_bkey_buf_init(&sk);
1708         bch2_trans_init(&trans, c, 0, 0);
1709
1710         iter = bch2_trans_get_iter(&trans, btree_id, POS_MIN,
1711                                    BTREE_ITER_PREFETCH|
1712                                    BTREE_ITER_NOT_EXTENTS|
1713                                    BTREE_ITER_ALL_SNAPSHOTS);
1714
1715         while ((k = bch2_btree_iter_peek(iter)).k &&
1716                !(ret = bkey_err(k))) {
1717                 c->gc_gens_pos = iter->pos;
1718
1719                 if (gc_btree_gens_key(c, k) && !commit_err) {
1720                         bch2_bkey_buf_reassemble(&sk, c, k);
1721                         bch2_extent_normalize(c, bkey_i_to_s(sk.k));
1722
1723
1724                         commit_err =
1725                                 bch2_trans_update(&trans, iter, sk.k, 0) ?:
1726                                 bch2_trans_commit(&trans, NULL, NULL,
1727                                                        BTREE_INSERT_NOWAIT|
1728                                                        BTREE_INSERT_NOFAIL);
1729                         if (commit_err == -EINTR) {
1730                                 commit_err = 0;
1731                                 continue;
1732                         }
1733                 }
1734
1735                 bch2_btree_iter_advance(iter);
1736         }
1737         bch2_trans_iter_put(&trans, iter);
1738
1739         bch2_trans_exit(&trans);
1740         bch2_bkey_buf_exit(&sk, c);
1741
1742         return ret;
1743 }
1744
1745 int bch2_gc_gens(struct bch_fs *c)
1746 {
1747         struct bch_dev *ca;
1748         struct bucket_array *buckets;
1749         struct bucket *g;
1750         unsigned i;
1751         int ret;
1752
1753         /*
1754          * Ideally we would be using state_lock and not gc_lock here, but that
1755          * introduces a deadlock in the RO path - we currently take the state
1756          * lock at the start of going RO, thus the gc thread may get stuck:
1757          */
1758         down_read(&c->gc_lock);
1759
1760         for_each_member_device(ca, c, i) {
1761                 down_read(&ca->bucket_lock);
1762                 buckets = bucket_array(ca);
1763
1764                 for_each_bucket(g, buckets)
1765                         g->gc_gen = g->mark.gen;
1766                 up_read(&ca->bucket_lock);
1767         }
1768
1769         for (i = 0; i < BTREE_ID_NR; i++)
1770                 if ((1 << i) & BTREE_ID_HAS_PTRS) {
1771                         c->gc_gens_btree = i;
1772                         c->gc_gens_pos = POS_MIN;
1773                         ret = bch2_gc_btree_gens(c, i);
1774                         if (ret) {
1775                                 bch_err(c, "error recalculating oldest_gen: %i", ret);
1776                                 goto err;
1777                         }
1778                 }
1779
1780         for_each_member_device(ca, c, i) {
1781                 down_read(&ca->bucket_lock);
1782                 buckets = bucket_array(ca);
1783
1784                 for_each_bucket(g, buckets)
1785                         g->oldest_gen = g->gc_gen;
1786                 up_read(&ca->bucket_lock);
1787         }
1788
1789         c->gc_gens_btree        = 0;
1790         c->gc_gens_pos          = POS_MIN;
1791
1792         c->gc_count++;
1793 err:
1794         up_read(&c->gc_lock);
1795         return ret;
1796 }
1797
1798 static int bch2_gc_thread(void *arg)
1799 {
1800         struct bch_fs *c = arg;
1801         struct io_clock *clock = &c->io_clock[WRITE];
1802         unsigned long last = atomic64_read(&clock->now);
1803         unsigned last_kick = atomic_read(&c->kick_gc);
1804         int ret;
1805
1806         set_freezable();
1807
1808         while (1) {
1809                 while (1) {
1810                         set_current_state(TASK_INTERRUPTIBLE);
1811
1812                         if (kthread_should_stop()) {
1813                                 __set_current_state(TASK_RUNNING);
1814                                 return 0;
1815                         }
1816
1817                         if (atomic_read(&c->kick_gc) != last_kick)
1818                                 break;
1819
1820                         if (c->btree_gc_periodic) {
1821                                 unsigned long next = last + c->capacity / 16;
1822
1823                                 if (atomic64_read(&clock->now) >= next)
1824                                         break;
1825
1826                                 bch2_io_clock_schedule_timeout(clock, next);
1827                         } else {
1828                                 schedule();
1829                         }
1830
1831                         try_to_freeze();
1832                 }
1833                 __set_current_state(TASK_RUNNING);
1834
1835                 last = atomic64_read(&clock->now);
1836                 last_kick = atomic_read(&c->kick_gc);
1837
1838                 /*
1839                  * Full gc is currently incompatible with btree key cache:
1840                  */
1841 #if 0
1842                 ret = bch2_gc(c, false, false);
1843 #else
1844                 ret = bch2_gc_gens(c);
1845 #endif
1846                 if (ret < 0)
1847                         bch_err(c, "btree gc failed: %i", ret);
1848
1849                 debug_check_no_locks_held();
1850         }
1851
1852         return 0;
1853 }
1854
1855 void bch2_gc_thread_stop(struct bch_fs *c)
1856 {
1857         struct task_struct *p;
1858
1859         p = c->gc_thread;
1860         c->gc_thread = NULL;
1861
1862         if (p) {
1863                 kthread_stop(p);
1864                 put_task_struct(p);
1865         }
1866 }
1867
1868 int bch2_gc_thread_start(struct bch_fs *c)
1869 {
1870         struct task_struct *p;
1871
1872         if (c->gc_thread)
1873                 return 0;
1874
1875         p = kthread_create(bch2_gc_thread, c, "bch-gc/%s", c->name);
1876         if (IS_ERR(p)) {
1877                 bch_err(c, "error creating gc thread: %li", PTR_ERR(p));
1878                 return PTR_ERR(p);
1879         }
1880
1881         get_task_struct(p);
1882         c->gc_thread = p;
1883         wake_up_process(p);
1884         return 0;
1885 }