git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/btree_gc.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2010 Kent Overstreet <kent.overstreet@gmail.com>
   4  * Copyright (C) 2014 Datera Inc.
   5  */
   6
   7 #include "bcachefs.h"
   8 #include "alloc_background.h"
   9 #include "alloc_foreground.h"
  10 #include "bkey_methods.h"
  11 #include "bkey_buf.h"
  12 #include "btree_locking.h"
  13 #include "btree_update_interior.h"
  14 #include "btree_io.h"
  15 #include "btree_gc.h"
  16 #include "buckets.h"
  17 #include "clock.h"
  18 #include "debug.h"
  19 #include "ec.h"
  20 #include "error.h"
  21 #include "extents.h"
  22 #include "journal.h"
  23 #include "keylist.h"
  24 #include "move.h"
  25 #include "recovery.h"
  26 #include "reflink.h"
  27 #include "replicas.h"
  28 #include "super-io.h"
  29
  30 #include <linux/slab.h>
  31 #include <linux/bitops.h>
  32 #include <linux/freezer.h>
  33 #include <linux/kthread.h>
  34 #include <linux/preempt.h>
  35 #include <linux/rcupdate.h>
  36 #include <linux/sched/task.h>
  37 #include <trace/events/bcachefs.h>
  38
  39 #define DROP_THIS_NODE          10
  40 #define DROP_PREV_NODE          11
  41
  42 static inline void __gc_pos_set(struct bch_fs *c, struct gc_pos new_pos)
  43 {
  44         preempt_disable();
  45         write_seqcount_begin(&c->gc_pos_lock);
  46         c->gc_pos = new_pos;
  47         write_seqcount_end(&c->gc_pos_lock);
  48         preempt_enable();
  49 }
  50
  51 static inline void gc_pos_set(struct bch_fs *c, struct gc_pos new_pos)
  52 {
  53         BUG_ON(gc_pos_cmp(new_pos, c->gc_pos) <= 0);
  54         __gc_pos_set(c, new_pos);
  55 }
  56
  57 /*
  58  * Missing: if an interior btree node is empty, we need to do something -
  59  * perhaps just kill it
  60  */
  61 static int bch2_gc_check_topology(struct bch_fs *c,
  62                                   struct btree *b,
  63                                   struct bkey_buf *prev,
  64                                   struct bkey_buf cur,
  65                                   bool is_last)
  66 {
  67         struct bpos node_start  = b->data->min_key;
  68         struct bpos node_end    = b->data->max_key;
  69         struct bpos expected_start = bkey_deleted(&prev->k->k)
  70                 ? node_start
  71                 : bpos_successor(prev->k->k.p);
  72         char buf1[200], buf2[200];
  73         int ret = 0;
  74
  75         if (cur.k->k.type == KEY_TYPE_btree_ptr_v2) {
  76                 struct bkey_i_btree_ptr_v2 *bp = bkey_i_to_btree_ptr_v2(cur.k);
  77
  78                 if (bkey_deleted(&prev->k->k)) {
  79                         struct printbuf out = PBUF(buf1);
  80                         pr_buf(&out, "start of node: ");
  81                         bch2_bpos_to_text(&out, node_start);
  82                 } else {
  83                         bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(prev->k));
  84                 }
  85
  86                 if (bpos_cmp(expected_start, bp->v.min_key)) {
  87                         bch2_topology_error(c);
  88
  89                         if (fsck_err(c, "btree node with incorrect min_key at btree %s level %u:\n"
  90                                      "  prev %s\n"
  91                                      "  cur %s",
  92                                      bch2_btree_ids[b->c.btree_id], b->c.level,
  93                                      buf1,
  94                                      (bch2_bkey_val_to_text(&PBUF(buf2), c, bkey_i_to_s_c(cur.k)), buf2))) {
  95                                 bch_info(c, "Halting mark and sweep to start topology repair pass");
  96                                 return FSCK_ERR_START_TOPOLOGY_REPAIR;
  97                         } else {
  98                                 set_bit(BCH_FS_INITIAL_GC_UNFIXED, &c->flags);
  99                         }
 100                 }
 101         }
 102
 103         if (is_last && bpos_cmp(cur.k->k.p, node_end)) {
 104                 bch2_topology_error(c);
 105
 106                 if (fsck_err(c, "btree node with incorrect max_key at btree %s level %u:\n"
 107                              "  %s\n"
 108                              "  expected %s",
 109                              bch2_btree_ids[b->c.btree_id], b->c.level,
 110                              (bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(cur.k)), buf1),
 111                              (bch2_bpos_to_text(&PBUF(buf2), node_end), buf2))) {
 112                         bch_info(c, "Halting mark and sweep to start topology repair pass");
 113                         return FSCK_ERR_START_TOPOLOGY_REPAIR;
 114                 } else {
 115                         set_bit(BCH_FS_INITIAL_GC_UNFIXED, &c->flags);
 116                 }
 117         }
 118
 119         bch2_bkey_buf_copy(prev, c, cur.k);
 120 fsck_err:
 121         return ret;
 122 }
 123
 124 static void btree_ptr_to_v2(struct btree *b, struct bkey_i_btree_ptr_v2 *dst)
 125 {
 126         switch (b->key.k.type) {
 127         case KEY_TYPE_btree_ptr: {
 128                 struct bkey_i_btree_ptr *src = bkey_i_to_btree_ptr(&b->key);
 129
 130                 dst->k.p                = src->k.p;
 131                 dst->v.mem_ptr          = 0;
 132                 dst->v.seq              = b->data->keys.seq;
 133                 dst->v.sectors_written  = 0;
 134                 dst->v.flags            = 0;
 135                 dst->v.min_key          = b->data->min_key;
 136                 set_bkey_val_bytes(&dst->k, sizeof(dst->v) + bkey_val_bytes(&src->k));
 137                 memcpy(dst->v.start, src->v.start, bkey_val_bytes(&src->k));
 138                 break;
 139         }
 140         case KEY_TYPE_btree_ptr_v2:
 141                 bkey_copy(&dst->k_i, &b->key);
 142                 break;
 143         default:
 144                 BUG();
 145         }
 146 }
 147
 148 static int set_node_min(struct bch_fs *c, struct btree *b, struct bpos new_min)
 149 {
 150         struct bkey_i_btree_ptr_v2 *new;
 151         int ret;
 152
 153         new = kmalloc(BKEY_BTREE_PTR_U64s_MAX * sizeof(u64), GFP_KERNEL);
 154         if (!new)
 155                 return -ENOMEM;
 156
 157         btree_ptr_to_v2(b, new);
 158         b->data->min_key        = new_min;
 159         new->v.min_key          = new_min;
 160         SET_BTREE_PTR_RANGE_UPDATED(&new->v, true);
 161
 162         ret = bch2_journal_key_insert(c, b->c.btree_id, b->c.level + 1, &new->k_i);
 163         if (ret) {
 164                 kfree(new);
 165                 return ret;
 166         }
 167
 168         bch2_btree_node_drop_keys_outside_node(b);
 169
 170         return 0;
 171 }
 172
 173 static int set_node_max(struct bch_fs *c, struct btree *b, struct bpos new_max)
 174 {
 175         struct bkey_i_btree_ptr_v2 *new;
 176         int ret;
 177
 178         ret = bch2_journal_key_delete(c, b->c.btree_id, b->c.level + 1, b->key.k.p);
 179         if (ret)
 180                 return ret;
 181
 182         new = kmalloc(BKEY_BTREE_PTR_U64s_MAX * sizeof(u64), GFP_KERNEL);
 183         if (!new)
 184                 return -ENOMEM;
 185
 186         btree_ptr_to_v2(b, new);
 187         b->data->max_key        = new_max;
 188         new->k.p                = new_max;
 189         SET_BTREE_PTR_RANGE_UPDATED(&new->v, true);
 190
 191         ret = bch2_journal_key_insert(c, b->c.btree_id, b->c.level + 1, &new->k_i);
 192         if (ret) {
 193                 kfree(new);
 194                 return ret;
 195         }
 196
 197         bch2_btree_node_drop_keys_outside_node(b);
 198
 199         mutex_lock(&c->btree_cache.lock);
 200         bch2_btree_node_hash_remove(&c->btree_cache, b);
 201
 202         bkey_copy(&b->key, &new->k_i);
 203         ret = __bch2_btree_node_hash_insert(&c->btree_cache, b);
 204         BUG_ON(ret);
 205         mutex_unlock(&c->btree_cache.lock);
 206         return 0;
 207 }
 208
 209 static int btree_repair_node_start(struct bch_fs *c, struct btree *b,
 210                                    struct btree *prev, struct btree *cur)
 211 {
 212         struct bpos expected_start = !prev
 213                 ? b->data->min_key
 214                 : bpos_successor(prev->key.k.p);
 215         char buf1[200], buf2[200];
 216         int ret = 0;
 217
 218         if (!prev) {
 219                 struct printbuf out = PBUF(buf1);
 220                 pr_buf(&out, "start of node: ");
 221                 bch2_bpos_to_text(&out, b->data->min_key);
 222         } else {
 223                 bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(&prev->key));
 224         }
 225
 226         if (mustfix_fsck_err_on(bpos_cmp(expected_start, cur->data->min_key), c,
 227                         "btree node with incorrect min_key at btree %s level %u:\n"
 228                         "  prev %s\n"
 229                         "  cur %s",
 230                         bch2_btree_ids[b->c.btree_id], b->c.level,
 231                         buf1,
 232                         (bch2_bkey_val_to_text(&PBUF(buf2), c, bkey_i_to_s_c(&cur->key)), buf2))) {
 233                 if (prev &&
 234                     bpos_cmp(expected_start, cur->data->min_key) > 0 &&
 235                     BTREE_NODE_SEQ(cur->data) > BTREE_NODE_SEQ(prev->data)) {
 236                         if (bkey_cmp(prev->data->min_key,
 237                                      cur->data->min_key) <= 0)
 238                                 return DROP_PREV_NODE;
 239
 240                         ret = set_node_max(c, prev,
 241                                            bpos_predecessor(cur->data->min_key));
 242                 } else {
 243                         if (bkey_cmp(expected_start, b->data->max_key) >= 0)
 244                                 return DROP_THIS_NODE;
 245
 246                         ret = set_node_min(c, cur, expected_start);
 247                 }
 248                 if (ret)
 249                         return ret;
 250         }
 251 fsck_err:
 252         return ret;
 253 }
 254
 255 static int btree_repair_node_end(struct bch_fs *c, struct btree *b,
 256                                  struct btree *child)
 257 {
 258         char buf1[200], buf2[200];
 259         int ret = 0;
 260
 261         if (mustfix_fsck_err_on(bpos_cmp(child->key.k.p, b->key.k.p), c,
 262                         "btree node with incorrect max_key at btree %s level %u:\n"
 263                         "  %s\n"
 264                         "  expected %s",
 265                         bch2_btree_ids[b->c.btree_id], b->c.level,
 266                         (bch2_bkey_val_to_text(&PBUF(buf1), c, bkey_i_to_s_c(&child->key)), buf1),
 267                         (bch2_bpos_to_text(&PBUF(buf2), b->key.k.p), buf2))) {
 268                 ret = set_node_max(c, child, b->key.k.p);
 269                 if (ret)
 270                         return ret;
 271         }
 272 fsck_err:
 273         return ret;
 274 }
 275
 276 static int bch2_btree_repair_topology_recurse(struct bch_fs *c, struct btree *b)
 277 {
 278         struct btree_and_journal_iter iter;
 279         struct bkey_s_c k;
 280         struct bkey_buf prev_k, cur_k;
 281         struct btree *prev = NULL, *cur = NULL;
 282         bool have_child, dropped_children = false;
 283         char buf[200];
 284         int ret = 0;
 285
 286         if (!b->c.level)
 287                 return 0;
 288 again:
 289         have_child = dropped_children = false;
 290         bch2_bkey_buf_init(&prev_k);
 291         bch2_bkey_buf_init(&cur_k);
 292         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 293
 294         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 295                 bch2_btree_and_journal_iter_advance(&iter);
 296                 bch2_bkey_buf_reassemble(&cur_k, c, k);
 297
 298                 cur = bch2_btree_node_get_noiter(c, cur_k.k,
 299                                         b->c.btree_id, b->c.level - 1,
 300                                         false);
 301                 ret = PTR_ERR_OR_ZERO(cur);
 302
 303                 if (mustfix_fsck_err_on(ret == -EIO, c,
 304                                 "Unreadable btree node at btree %s level %u:\n"
 305                                 "  %s",
 306                                 bch2_btree_ids[b->c.btree_id],
 307                                 b->c.level - 1,
 308                                 (bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(cur_k.k)), buf))) {
 309                         bch2_btree_node_evict(c, cur_k.k);
 310                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 311                                                       b->c.level, cur_k.k->k.p);
 312                         if (ret)
 313                                 goto err;
 314                         continue;
 315                 }
 316
 317                 if (ret) {
 318                         bch_err(c, "%s: error %i getting btree node",
 319                                 __func__, ret);
 320                         break;
 321                 }
 322
 323                 ret = btree_repair_node_start(c, b, prev, cur);
 324                 if (prev)
 325                         six_unlock_read(&prev->c.lock);
 326
 327                 if (ret == DROP_PREV_NODE) {
 328                         bch2_btree_node_evict(c, prev_k.k);
 329                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 330                                                       b->c.level, prev_k.k->k.p);
 331                         if (ret)
 332                                 goto err;
 333                         goto again;
 334                 } else if (ret == DROP_THIS_NODE) {
 335                         bch2_btree_node_evict(c, cur_k.k);
 336                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 337                                                       b->c.level, cur_k.k->k.p);
 338                         if (ret)
 339                                 goto err;
 340                         continue;
 341                 } else if (ret)
 342                         break;
 343
 344                 prev = cur;
 345                 cur = NULL;
 346                 bch2_bkey_buf_copy(&prev_k, c, cur_k.k);
 347         }
 348
 349         if (!ret && !IS_ERR_OR_NULL(prev)) {
 350                 BUG_ON(cur);
 351                 ret = btree_repair_node_end(c, b, prev);
 352         }
 353
 354         if (!IS_ERR_OR_NULL(prev))
 355                 six_unlock_read(&prev->c.lock);
 356         prev = NULL;
 357         if (!IS_ERR_OR_NULL(cur))
 358                 six_unlock_read(&cur->c.lock);
 359         cur = NULL;
 360
 361         if (ret)
 362                 goto err;
 363
 364         bch2_btree_and_journal_iter_exit(&iter);
 365         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 366
 367         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 368                 bch2_bkey_buf_reassemble(&cur_k, c, k);
 369                 bch2_btree_and_journal_iter_advance(&iter);
 370
 371                 cur = bch2_btree_node_get_noiter(c, cur_k.k,
 372                                         b->c.btree_id, b->c.level - 1,
 373                                         false);
 374                 ret = PTR_ERR_OR_ZERO(cur);
 375
 376                 if (ret) {
 377                         bch_err(c, "%s: error %i getting btree node",
 378                                 __func__, ret);
 379                         goto err;
 380                 }
 381
 382                 ret = bch2_btree_repair_topology_recurse(c, cur);
 383                 six_unlock_read(&cur->c.lock);
 384                 cur = NULL;
 385
 386                 if (ret == DROP_THIS_NODE) {
 387                         bch2_btree_node_evict(c, cur_k.k);
 388                         ret = bch2_journal_key_delete(c, b->c.btree_id,
 389                                                       b->c.level, cur_k.k->k.p);
 390                         dropped_children = true;
 391                 }
 392
 393                 if (ret)
 394                         goto err;
 395
 396                 have_child = true;
 397         }
 398
 399         if (mustfix_fsck_err_on(!have_child, c,
 400                         "empty interior btree node at btree %s level %u\n"
 401                         "  %s",
 402                         bch2_btree_ids[b->c.btree_id],
 403                         b->c.level,
 404                         (bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(&b->key)), buf)))
 405                 ret = DROP_THIS_NODE;
 406 err:
 407 fsck_err:
 408         if (!IS_ERR_OR_NULL(prev))
 409                 six_unlock_read(&prev->c.lock);
 410         if (!IS_ERR_OR_NULL(cur))
 411                 six_unlock_read(&cur->c.lock);
 412
 413         bch2_btree_and_journal_iter_exit(&iter);
 414         bch2_bkey_buf_exit(&prev_k, c);
 415         bch2_bkey_buf_exit(&cur_k, c);
 416
 417         if (!ret && dropped_children)
 418                 goto again;
 419
 420         return ret;
 421 }
 422
 423 static int bch2_repair_topology(struct bch_fs *c)
 424 {
 425         struct btree *b;
 426         unsigned i;
 427         int ret = 0;
 428
 429         for (i = 0; i < BTREE_ID_NR && !ret; i++) {
 430                 b = c->btree_roots[i].b;
 431                 if (btree_node_fake(b))
 432                         continue;
 433
 434                 six_lock_read(&b->c.lock, NULL, NULL);
 435                 ret = bch2_btree_repair_topology_recurse(c, b);
 436                 six_unlock_read(&b->c.lock);
 437
 438                 if (ret == DROP_THIS_NODE) {
 439                         bch_err(c, "empty btree root - repair unimplemented");
 440                         ret = FSCK_ERR_EXIT;
 441                 }
 442         }
 443
 444         return ret;
 445 }
 446
 447 static int bch2_check_fix_ptrs(struct bch_fs *c, enum btree_id btree_id,
 448                                unsigned level, bool is_root,
 449                                struct bkey_s_c *k)
 450 {
 451         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(*k);
 452         const union bch_extent_entry *entry;
 453         struct extent_ptr_decoded p = { 0 };
 454         bool do_update = false;
 455         char buf[200];
 456         int ret = 0;
 457
 458         bkey_for_each_ptr_decode(k->k, ptrs, p, entry) {
 459                 struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev);
 460                 struct bucket *g = PTR_BUCKET(ca, &p.ptr, true);
 461                 struct bucket *g2 = PTR_BUCKET(ca, &p.ptr, false);
 462                 enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, &entry->ptr);
 463
 464                 if (fsck_err_on(g->mark.data_type &&
 465                                 g->mark.data_type != data_type, c,
 466                                 "bucket %u:%zu different types of data in same bucket: %s, %s\n"
 467                                 "while marking %s",
 468                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 469                                 bch2_data_types[g->mark.data_type],
 470                                 bch2_data_types[data_type],
 471                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf))) {
 472                         if (data_type == BCH_DATA_btree) {
 473                                 g2->_mark.data_type = g->_mark.data_type = data_type;
 474                                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);
 475                         } else {
 476                                 do_update = true;
 477                         }
 478                 }
 479
 480                 if (fsck_err_on(!g->gen_valid, c,
 481                                 "bucket %u:%zu data type %s ptr gen %u missing in alloc btree\n"
 482                                 "while marking %s",
 483                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 484                                 bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 485                                 p.ptr.gen,
 486                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf))) {
 487                         if (!p.ptr.cached) {
 488                                 g2->_mark.gen   = g->_mark.gen          = p.ptr.gen;
 489                                 g2->gen_valid   = g->gen_valid          = true;
 490                                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);
 491                         } else {
 492                                 do_update = true;
 493                         }
 494                 }
 495
 496                 if (fsck_err_on(gen_cmp(p.ptr.gen, g->mark.gen) > 0, c,
 497                                 "bucket %u:%zu data type %s ptr gen in the future: %u > %u\n"
 498                                 "while marking %s",
 499                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 500                                 bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 501                                 p.ptr.gen, g->mark.gen,
 502                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf))) {
 503                         if (!p.ptr.cached) {
 504                                 g2->_mark.gen   = g->_mark.gen  = p.ptr.gen;
 505                                 g2->gen_valid   = g->gen_valid  = true;
 506                                 g2->_mark.data_type             = 0;
 507                                 g2->_mark.dirty_sectors         = 0;
 508                                 g2->_mark.cached_sectors        = 0;
 509                                 set_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
 510                                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);
 511                         } else {
 512                                 do_update = true;
 513                         }
 514                 }
 515
 516                 if (fsck_err_on(!p.ptr.cached &&
 517                                 gen_cmp(p.ptr.gen, g->mark.gen) < 0, c,
 518                                 "bucket %u:%zu data type %s stale dirty ptr: %u < %u\n"
 519                                 "while marking %s",
 520                                 p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr),
 521                                 bch2_data_types[ptr_data_type(k->k, &p.ptr)],
 522                                 p.ptr.gen, g->mark.gen,
 523                                 (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf)))
 524                         do_update = true;
 525
 526                 if (p.has_ec) {
 527                         struct stripe *m = genradix_ptr(&c->stripes[true], p.ec.idx);
 528
 529                         if (fsck_err_on(!m || !m->alive, c,
 530                                         "pointer to nonexistent stripe %llu\n"
 531                                         "while marking %s",
 532                                         (u64) p.ec.idx,
 533                                         (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf)))
 534                                 do_update = true;
 535
 536                         if (fsck_err_on(!bch2_ptr_matches_stripe_m(m, p), c,
 537                                         "pointer does not match stripe %llu\n"
 538                                         "while marking %s",
 539                                         (u64) p.ec.idx,
 540                                         (bch2_bkey_val_to_text(&PBUF(buf), c, *k), buf)))
 541                                 do_update = true;
 542                 }
 543         }
 544
 545         if (do_update) {
 546                 struct bkey_ptrs ptrs;
 547                 union bch_extent_entry *entry;
 548                 struct bch_extent_ptr *ptr;
 549                 struct bkey_i *new;
 550
 551                 if (is_root) {
 552                         bch_err(c, "cannot update btree roots yet");
 553                         return -EINVAL;
 554                 }
 555
 556                 new = kmalloc(bkey_bytes(k->k), GFP_KERNEL);
 557                 if (!new) {
 558                         bch_err(c, "%s: error allocating new key", __func__);
 559                         return -ENOMEM;
 560                 }
 561
 562                 bkey_reassemble(new, *k);
 563
 564                 if (level) {
 565                         /*
 566                          * We don't want to drop btree node pointers - if the
 567                          * btree node isn't there anymore, the read path will
 568                          * sort it out:
 569                          */
 570                         ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
 571                         bkey_for_each_ptr(ptrs, ptr) {
 572                                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 573                                 struct bucket *g = PTR_BUCKET(ca, ptr, true);
 574
 575                                 ptr->gen = g->mark.gen;
 576                         }
 577                 } else {
 578                         bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, ({
 579                                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 580                                 struct bucket *g = PTR_BUCKET(ca, ptr, true);
 581                                 enum bch_data_type data_type = bch2_bkey_ptr_data_type(*k, ptr);
 582
 583                                 (ptr->cached &&
 584                                  (!g->gen_valid || gen_cmp(ptr->gen, g->mark.gen) > 0)) ||
 585                                 (!ptr->cached &&
 586                                  gen_cmp(ptr->gen, g->mark.gen) < 0) ||
 587                                 (g->mark.data_type &&
 588                                  g->mark.data_type != data_type);
 589                         }));
 590 again:
 591                         ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
 592                         bkey_extent_entry_for_each(ptrs, entry) {
 593                                 if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) {
 594                                         struct stripe *m = genradix_ptr(&c->stripes[true],
 595                                                                         entry->stripe_ptr.idx);
 596                                         union bch_extent_entry *next_ptr;
 597
 598                                         bkey_extent_entry_for_each_from(ptrs, next_ptr, entry)
 599                                                 if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr)
 600                                                         goto found;
 601                                         next_ptr = NULL;
 602 found:
 603                                         if (!next_ptr) {
 604                                                 bch_err(c, "aieee, found stripe ptr with no data ptr");
 605                                                 continue;
 606                                         }
 607
 608                                         if (!m || !m->alive ||
 609                                             !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block],
 610                                                                        &next_ptr->ptr,
 611                                                                        m->sectors)) {
 612                                                 bch2_bkey_extent_entry_drop(new, entry);
 613                                                 goto again;
 614                                         }
 615                                 }
 616                         }
 617                 }
 618
 619                 ret = bch2_journal_key_insert(c, btree_id, level, new);
 620                 if (ret)
 621                         kfree(new);
 622                 else
 623                         *k = bkey_i_to_s_c(new);
 624         }
 625 fsck_err:
 626         return ret;
 627 }
 628
 629 /* marking of btree keys/nodes: */
 630
 631 static int bch2_gc_mark_key(struct bch_fs *c, enum btree_id btree_id,
 632                             unsigned level, bool is_root,
 633                             struct bkey_s_c *k,
 634                             u8 *max_stale, bool initial)
 635 {
 636         struct bkey_ptrs_c ptrs;
 637         const struct bch_extent_ptr *ptr;
 638         unsigned flags =
 639                 BTREE_TRIGGER_GC|
 640                 (initial ? BTREE_TRIGGER_NOATOMIC : 0);
 641         int ret = 0;
 642
 643         if (initial) {
 644                 BUG_ON(bch2_journal_seq_verify &&
 645                        k->k->version.lo > journal_cur_seq(&c->journal));
 646
 647                 ret = bch2_check_fix_ptrs(c, btree_id, level, is_root, k);
 648                 if (ret)
 649                         goto err;
 650
 651                 if (fsck_err_on(k->k->version.lo > atomic64_read(&c->key_version), c,
 652                                 "key version number higher than recorded: %llu > %llu",
 653                                 k->k->version.lo,
 654                                 atomic64_read(&c->key_version)))
 655                         atomic64_set(&c->key_version, k->k->version.lo);
 656
 657                 if (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
 658                     fsck_err_on(!bch2_bkey_replicas_marked(c, *k), c,
 659                                 "superblock not marked as containing replicas (type %u)",
 660                                 k->k->type)) {
 661                         ret = bch2_mark_bkey_replicas(c, *k);
 662                         if (ret) {
 663                                 bch_err(c, "error marking bkey replicas: %i", ret);
 664                                 goto err;
 665                         }
 666                 }
 667         }
 668
 669         ptrs = bch2_bkey_ptrs_c(*k);
 670         bkey_for_each_ptr(ptrs, ptr) {
 671                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
 672                 struct bucket *g = PTR_BUCKET(ca, ptr, true);
 673
 674                 if (gen_after(g->oldest_gen, ptr->gen))
 675                         g->oldest_gen = ptr->gen;
 676
 677                 *max_stale = max(*max_stale, ptr_stale(ca, ptr));
 678         }
 679
 680         bch2_mark_key(c, *k, 0, k->k->size, NULL, 0, flags);
 681 fsck_err:
 682 err:
 683         if (ret)
 684                 bch_err(c, "%s: ret %i", __func__, ret);
 685         return ret;
 686 }
 687
 688 static int btree_gc_mark_node(struct bch_fs *c, struct btree *b, u8 *max_stale,
 689                               bool initial)
 690 {
 691         struct btree_node_iter iter;
 692         struct bkey unpacked;
 693         struct bkey_s_c k;
 694         struct bkey_buf prev, cur;
 695         int ret = 0;
 696
 697         *max_stale = 0;
 698
 699         if (!btree_node_type_needs_gc(btree_node_type(b)))
 700                 return 0;
 701
 702         bch2_btree_node_iter_init_from_start(&iter, b);
 703         bch2_bkey_buf_init(&prev);
 704         bch2_bkey_buf_init(&cur);
 705         bkey_init(&prev.k->k);
 706
 707         while ((k = bch2_btree_node_iter_peek_unpack(&iter, b, &unpacked)).k) {
 708                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, false,
 709                                        &k, max_stale, initial);
 710                 if (ret)
 711                         break;
 712
 713                 bch2_btree_node_iter_advance(&iter, b);
 714
 715                 if (b->c.level) {
 716                         bch2_bkey_buf_reassemble(&cur, c, k);
 717
 718                         ret = bch2_gc_check_topology(c, b, &prev, cur,
 719                                         bch2_btree_node_iter_end(&iter));
 720                         if (ret)
 721                                 break;
 722                 }
 723         }
 724
 725         bch2_bkey_buf_exit(&cur, c);
 726         bch2_bkey_buf_exit(&prev, c);
 727         return ret;
 728 }
 729
 730 static int bch2_gc_btree(struct bch_fs *c, enum btree_id btree_id,
 731                          bool initial, bool metadata_only)
 732 {
 733         struct btree_trans trans;
 734         struct btree_iter *iter;
 735         struct btree *b;
 736         unsigned depth = metadata_only                  ? 1
 737                 : bch2_expensive_debug_checks           ? 0
 738                 : !btree_node_type_needs_gc(btree_id)   ? 1
 739                 : 0;
 740         u8 max_stale = 0;
 741         int ret = 0;
 742
 743         bch2_trans_init(&trans, c, 0, 0);
 744
 745         gc_pos_set(c, gc_pos_btree(btree_id, POS_MIN, 0));
 746
 747         __for_each_btree_node(&trans, iter, btree_id, POS_MIN,
 748                               0, depth, BTREE_ITER_PREFETCH, b) {
 749                 bch2_verify_btree_nr_keys(b);
 750
 751                 gc_pos_set(c, gc_pos_btree_node(b));
 752
 753                 ret = btree_gc_mark_node(c, b, &max_stale, initial);
 754                 if (ret)
 755                         break;
 756
 757                 if (!initial) {
 758                         if (max_stale > 64)
 759                                 bch2_btree_node_rewrite(c, iter,
 760                                                 b->data->keys.seq,
 761                                                 BTREE_INSERT_NOWAIT|
 762                                                 BTREE_INSERT_GC_LOCK_HELD);
 763                         else if (!bch2_btree_gc_rewrite_disabled &&
 764                                  (bch2_btree_gc_always_rewrite || max_stale > 16))
 765                                 bch2_btree_node_rewrite(c, iter,
 766                                                 b->data->keys.seq,
 767                                                 BTREE_INSERT_NOWAIT|
 768                                                 BTREE_INSERT_GC_LOCK_HELD);
 769                 }
 770
 771                 bch2_trans_cond_resched(&trans);
 772         }
 773         bch2_trans_iter_put(&trans, iter);
 774
 775         ret = bch2_trans_exit(&trans) ?: ret;
 776         if (ret)
 777                 return ret;
 778
 779         mutex_lock(&c->btree_root_lock);
 780         b = c->btree_roots[btree_id].b;
 781         if (!btree_node_fake(b)) {
 782                 struct bkey_s_c k = bkey_i_to_s_c(&b->key);
 783
 784                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, true,
 785                                        &k, &max_stale, initial);
 786         }
 787         gc_pos_set(c, gc_pos_btree_root(b->c.btree_id));
 788         mutex_unlock(&c->btree_root_lock);
 789
 790         return ret;
 791 }
 792
 793 static int bch2_gc_btree_init_recurse(struct bch_fs *c, struct btree *b,
 794                                       unsigned target_depth)
 795 {
 796         struct btree_and_journal_iter iter;
 797         struct bkey_s_c k;
 798         struct bkey_buf cur, prev;
 799         u8 max_stale = 0;
 800         char buf[200];
 801         int ret = 0;
 802
 803         bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 804         bch2_bkey_buf_init(&prev);
 805         bch2_bkey_buf_init(&cur);
 806         bkey_init(&prev.k->k);
 807
 808         while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 809                 BUG_ON(bpos_cmp(k.k->p, b->data->min_key) < 0);
 810                 BUG_ON(bpos_cmp(k.k->p, b->data->max_key) > 0);
 811
 812                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, false,
 813                                        &k, &max_stale, true);
 814                 if (ret) {
 815                         bch_err(c, "%s: error %i from bch2_gc_mark_key", __func__, ret);
 816                         goto fsck_err;
 817                 }
 818
 819                 if (b->c.level) {
 820                         bch2_bkey_buf_reassemble(&cur, c, k);
 821                         k = bkey_i_to_s_c(cur.k);
 822
 823                         bch2_btree_and_journal_iter_advance(&iter);
 824
 825                         ret = bch2_gc_check_topology(c, b,
 826                                         &prev, cur,
 827                                         !bch2_btree_and_journal_iter_peek(&iter).k);
 828                         if (ret)
 829                                 goto fsck_err;
 830                 } else {
 831                         bch2_btree_and_journal_iter_advance(&iter);
 832                 }
 833         }
 834
 835         if (b->c.level > target_depth) {
 836                 bch2_btree_and_journal_iter_exit(&iter);
 837                 bch2_btree_and_journal_iter_init_node_iter(&iter, c, b);
 838
 839                 while ((k = bch2_btree_and_journal_iter_peek(&iter)).k) {
 840                         struct btree *child;
 841
 842                         bch2_bkey_buf_reassemble(&cur, c, k);
 843                         bch2_btree_and_journal_iter_advance(&iter);
 844
 845                         child = bch2_btree_node_get_noiter(c, cur.k,
 846                                                 b->c.btree_id, b->c.level - 1,
 847                                                 false);
 848                         ret = PTR_ERR_OR_ZERO(child);
 849
 850                         if (ret == -EIO) {
 851                                 bch2_topology_error(c);
 852
 853                                 if (fsck_err(c, "Unreadable btree node at btree %s level %u:\n"
 854                                         "  %s",
 855                                         bch2_btree_ids[b->c.btree_id],
 856                                         b->c.level - 1,
 857                                         (bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(cur.k)), buf))) {
 858                                         ret = FSCK_ERR_START_TOPOLOGY_REPAIR;
 859                                         bch_info(c, "Halting mark and sweep to start topology repair pass");
 860                                         goto fsck_err;
 861                                 } else {
 862                                         /* Continue marking when opted to not
 863                                          * fix the error: */
 864                                         ret = 0;
 865                                         set_bit(BCH_FS_INITIAL_GC_UNFIXED, &c->flags);
 866                                         continue;
 867                                 }
 868                         } else if (ret) {
 869                                 bch_err(c, "%s: error %i getting btree node",
 870                                         __func__, ret);
 871                                 break;
 872                         }
 873
 874                         ret = bch2_gc_btree_init_recurse(c, child,
 875                                                          target_depth);
 876                         six_unlock_read(&child->c.lock);
 877
 878                         if (ret)
 879                                 break;
 880                 }
 881         }
 882 fsck_err:
 883         bch2_bkey_buf_exit(&cur, c);
 884         bch2_bkey_buf_exit(&prev, c);
 885         bch2_btree_and_journal_iter_exit(&iter);
 886         return ret;
 887 }
 888
 889 static int bch2_gc_btree_init(struct bch_fs *c,
 890                               enum btree_id btree_id,
 891                               bool metadata_only)
 892 {
 893         struct btree *b;
 894         unsigned target_depth = metadata_only           ? 1
 895                 : bch2_expensive_debug_checks           ? 0
 896                 : !btree_node_type_needs_gc(btree_id)   ? 1
 897                 : 0;
 898         u8 max_stale = 0;
 899         char buf[100];
 900         int ret = 0;
 901
 902         b = c->btree_roots[btree_id].b;
 903
 904         if (btree_node_fake(b))
 905                 return 0;
 906
 907         six_lock_read(&b->c.lock, NULL, NULL);
 908         if (mustfix_fsck_err_on(bpos_cmp(b->data->min_key, POS_MIN), c,
 909                         "btree root with incorrect min_key: %s",
 910                         (bch2_bpos_to_text(&PBUF(buf), b->data->min_key), buf))) {
 911                 bch_err(c, "repair unimplemented");
 912                 ret = FSCK_ERR_EXIT;
 913                 goto fsck_err;
 914         }
 915
 916         if (mustfix_fsck_err_on(bpos_cmp(b->data->max_key, POS_MAX), c,
 917                         "btree root with incorrect max_key: %s",
 918                         (bch2_bpos_to_text(&PBUF(buf), b->data->max_key), buf))) {
 919                 bch_err(c, "repair unimplemented");
 920                 ret = FSCK_ERR_EXIT;
 921                 goto fsck_err;
 922         }
 923
 924         if (b->c.level >= target_depth)
 925                 ret = bch2_gc_btree_init_recurse(c, b, target_depth);
 926
 927         if (!ret) {
 928                 struct bkey_s_c k = bkey_i_to_s_c(&b->key);
 929
 930                 ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, true,
 931                                        &k, &max_stale, true);
 932         }
 933 fsck_err:
 934         six_unlock_read(&b->c.lock);
 935
 936         if (ret < 0)
 937                 bch_err(c, "%s: ret %i", __func__, ret);
 938         return ret;
 939 }
 940
 941 static inline int btree_id_gc_phase_cmp(enum btree_id l, enum btree_id r)
 942 {
 943         return  (int) btree_id_to_gc_phase(l) -
 944                 (int) btree_id_to_gc_phase(r);
 945 }
 946
 947 static int bch2_gc_btrees(struct bch_fs *c, bool initial, bool metadata_only)
 948 {
 949         enum btree_id ids[BTREE_ID_NR];
 950         unsigned i;
 951         int ret = 0;
 952
 953         for (i = 0; i < BTREE_ID_NR; i++)
 954                 ids[i] = i;
 955         bubble_sort(ids, BTREE_ID_NR, btree_id_gc_phase_cmp);
 956
 957         for (i = 0; i < BTREE_ID_NR && !ret; i++)
 958                 ret = initial
 959                         ? bch2_gc_btree_init(c, ids[i], metadata_only)
 960                         : bch2_gc_btree(c, ids[i], initial, metadata_only);
 961
 962         if (ret < 0)
 963                 bch_err(c, "%s: ret %i", __func__, ret);
 964         return ret;
 965 }
 966
 967 static void mark_metadata_sectors(struct bch_fs *c, struct bch_dev *ca,
 968                                   u64 start, u64 end,
 969                                   enum bch_data_type type,
 970                                   unsigned flags)
 971 {
 972         u64 b = sector_to_bucket(ca, start);
 973
 974         do {
 975                 unsigned sectors =
 976                         min_t(u64, bucket_to_sector(ca, b + 1), end) - start;
 977
 978                 bch2_mark_metadata_bucket(c, ca, b, type, sectors,
 979                                           gc_phase(GC_PHASE_SB), flags);
 980                 b++;
 981                 start += sectors;
 982         } while (start < end);
 983 }
 984
 985 void bch2_mark_dev_superblock(struct bch_fs *c, struct bch_dev *ca,
 986                               unsigned flags)
 987 {
 988         struct bch_sb_layout *layout = &ca->disk_sb.sb->layout;
 989         unsigned i;
 990         u64 b;
 991
 992         /*
 993          * This conditional is kind of gross, but we may be called from the
 994          * device add path, before the new device has actually been added to the
 995          * running filesystem:
 996          */
 997         if (c) {
 998                 lockdep_assert_held(&c->sb_lock);
 999                 percpu_down_read(&c->mark_lock);
1000         }
1001
1002         for (i = 0; i < layout->nr_superblocks; i++) {
1003                 u64 offset = le64_to_cpu(layout->sb_offset[i]);
1004
1005                 if (offset == BCH_SB_SECTOR)
1006                         mark_metadata_sectors(c, ca, 0, BCH_SB_SECTOR,
1007                                               BCH_DATA_sb, flags);
1008
1009                 mark_metadata_sectors(c, ca, offset,
1010                                       offset + (1 << layout->sb_max_size_bits),
1011                                       BCH_DATA_sb, flags);
1012         }
1013
1014         for (i = 0; i < ca->journal.nr; i++) {
1015                 b = ca->journal.buckets[i];
1016                 bch2_mark_metadata_bucket(c, ca, b, BCH_DATA_journal,
1017                                           ca->mi.bucket_size,
1018                                           gc_phase(GC_PHASE_SB), flags);
1019         }
1020
1021         if (c)
1022                 percpu_up_read(&c->mark_lock);
1023 }
1024
1025 static void bch2_mark_superblocks(struct bch_fs *c)
1026 {
1027         struct bch_dev *ca;
1028         unsigned i;
1029
1030         mutex_lock(&c->sb_lock);
1031         gc_pos_set(c, gc_phase(GC_PHASE_SB));
1032
1033         for_each_online_member(ca, c, i)
1034                 bch2_mark_dev_superblock(c, ca, BTREE_TRIGGER_GC);
1035         mutex_unlock(&c->sb_lock);
1036 }
1037
1038 #if 0
1039 /* Also see bch2_pending_btree_node_free_insert_done() */
1040 static void bch2_mark_pending_btree_node_frees(struct bch_fs *c)
1041 {
1042         struct btree_update *as;
1043         struct pending_btree_node_free *d;
1044
1045         mutex_lock(&c->btree_interior_update_lock);
1046         gc_pos_set(c, gc_phase(GC_PHASE_PENDING_DELETE));
1047
1048         for_each_pending_btree_node_free(c, as, d)
1049                 if (d->index_update_done)
1050                         bch2_mark_key(c, bkey_i_to_s_c(&d->key),
1051                                       0, 0, NULL, 0,
1052                                       BTREE_TRIGGER_GC);
1053
1054         mutex_unlock(&c->btree_interior_update_lock);
1055 }
1056 #endif
1057
1058 static void bch2_gc_free(struct bch_fs *c)
1059 {
1060         struct bch_dev *ca;
1061         unsigned i;
1062
1063         genradix_free(&c->stripes[1]);
1064
1065         for_each_member_device(ca, c, i) {
1066                 kvpfree(rcu_dereference_protected(ca->buckets[1], 1),
1067                         sizeof(struct bucket_array) +
1068                         ca->mi.nbuckets * sizeof(struct bucket));
1069                 ca->buckets[1] = NULL;
1070
1071                 free_percpu(ca->usage_gc);
1072                 ca->usage_gc = NULL;
1073         }
1074
1075         free_percpu(c->usage_gc);
1076         c->usage_gc = NULL;
1077 }
1078
1079 static int bch2_gc_done(struct bch_fs *c,
1080                         bool initial, bool metadata_only)
1081 {
1082         struct bch_dev *ca = NULL;
1083         bool verify = !metadata_only && (!initial ||
1084                        (c->sb.compat & (1ULL << BCH_COMPAT_alloc_info)));
1085         unsigned i, dev;
1086         int ret = 0;
1087
1088 #define copy_field(_f, _msg, ...)                                       \
1089         if (dst->_f != src->_f) {                                       \
1090                 if (verify)                                             \
1091                         fsck_err(c, _msg ": got %llu, should be %llu"   \
1092                                 , ##__VA_ARGS__, dst->_f, src->_f);     \
1093                 dst->_f = src->_f;                                      \
1094                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);            \
1095         }
1096 #define copy_stripe_field(_f, _msg, ...)                                \
1097         if (dst->_f != src->_f) {                                       \
1098                 if (verify)                                             \
1099                         fsck_err(c, "stripe %zu has wrong "_msg         \
1100                                 ": got %u, should be %u",               \
1101                                 iter.pos, ##__VA_ARGS__,                \
1102                                 dst->_f, src->_f);                      \
1103                 dst->_f = src->_f;                                      \
1104                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);            \
1105         }
1106 #define copy_bucket_field(_f)                                           \
1107         if (dst->b[b].mark._f != src->b[b].mark._f) {                   \
1108                 if (verify)                                             \
1109                         fsck_err(c, "bucket %u:%zu gen %u data type %s has wrong " #_f  \
1110                                 ": got %u, should be %u", dev, b,       \
1111                                 dst->b[b].mark.gen,                     \
1112                                 bch2_data_types[dst->b[b].mark.data_type],\
1113                                 dst->b[b].mark._f, src->b[b].mark._f);  \
1114                 dst->b[b]._mark._f = src->b[b].mark._f;                 \
1115                 set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags);            \
1116         }
1117 #define copy_dev_field(_f, _msg, ...)                                   \
1118         copy_field(_f, "dev %u has wrong " _msg, dev, ##__VA_ARGS__)
1119 #define copy_fs_field(_f, _msg, ...)                                    \
1120         copy_field(_f, "fs has wrong " _msg, ##__VA_ARGS__)
1121
1122         if (!metadata_only) {
1123                 struct genradix_iter iter = genradix_iter_init(&c->stripes[1], 0);
1124                 struct stripe *dst, *src;
1125
1126                 while ((src = genradix_iter_peek(&iter, &c->stripes[1]))) {
1127                         dst = genradix_ptr_alloc(&c->stripes[0], iter.pos, GFP_KERNEL);
1128
1129                         if (dst->alive          != src->alive ||
1130                             dst->sectors        != src->sectors ||
1131                             dst->algorithm      != src->algorithm ||
1132                             dst->nr_blocks      != src->nr_blocks ||
1133                             dst->nr_redundant   != src->nr_redundant) {
1134                                 bch_err(c, "unexpected stripe inconsistency at bch2_gc_done, confused");
1135                                 ret = -EINVAL;
1136                                 goto fsck_err;
1137                         }
1138
1139                         for (i = 0; i < ARRAY_SIZE(dst->block_sectors); i++)
1140                                 copy_stripe_field(block_sectors[i],
1141                                                   "block_sectors[%u]", i);
1142
1143                         dst->blocks_nonempty = 0;
1144                         for (i = 0; i < dst->nr_blocks; i++)
1145                                 dst->blocks_nonempty += dst->block_sectors[i] != 0;
1146
1147                         genradix_iter_advance(&iter, &c->stripes[1]);
1148                 }
1149         }
1150
1151         for (i = 0; i < ARRAY_SIZE(c->usage); i++)
1152                 bch2_fs_usage_acc_to_base(c, i);
1153
1154         for_each_member_device(ca, c, dev) {
1155                 struct bucket_array *dst = __bucket_array(ca, 0);
1156                 struct bucket_array *src = __bucket_array(ca, 1);
1157                 size_t b;
1158
1159                 for (b = 0; b < src->nbuckets; b++) {
1160                         copy_bucket_field(gen);
1161                         copy_bucket_field(data_type);
1162                         copy_bucket_field(stripe);
1163                         copy_bucket_field(dirty_sectors);
1164                         copy_bucket_field(cached_sectors);
1165
1166                         dst->b[b].oldest_gen = src->b[b].oldest_gen;
1167                 }
1168
1169                 {
1170                         struct bch_dev_usage *dst = ca->usage_base;
1171                         struct bch_dev_usage *src = (void *)
1172                                 bch2_acc_percpu_u64s((void *) ca->usage_gc,
1173                                                      dev_usage_u64s());
1174
1175                         copy_dev_field(buckets_ec,              "buckets_ec");
1176                         copy_dev_field(buckets_unavailable,     "buckets_unavailable");
1177
1178                         for (i = 0; i < BCH_DATA_NR; i++) {
1179                                 copy_dev_field(d[i].buckets,    "%s buckets", bch2_data_types[i]);
1180                                 copy_dev_field(d[i].sectors,    "%s sectors", bch2_data_types[i]);
1181                                 copy_dev_field(d[i].fragmented, "%s fragmented", bch2_data_types[i]);
1182                         }
1183                 }
1184         };
1185
1186         {
1187                 unsigned nr = fs_usage_u64s(c);
1188                 struct bch_fs_usage *dst = c->usage_base;
1189                 struct bch_fs_usage *src = (void *)
1190                         bch2_acc_percpu_u64s((void *) c->usage_gc, nr);
1191
1192                 copy_fs_field(hidden,           "hidden");
1193                 copy_fs_field(btree,            "btree");
1194
1195                 if (!metadata_only) {
1196                         copy_fs_field(data,     "data");
1197                         copy_fs_field(cached,   "cached");
1198                         copy_fs_field(reserved, "reserved");
1199                         copy_fs_field(nr_inodes,"nr_inodes");
1200
1201                         for (i = 0; i < BCH_REPLICAS_MAX; i++)
1202                                 copy_fs_field(persistent_reserved[i],
1203                                               "persistent_reserved[%i]", i);
1204                 }
1205
1206                 for (i = 0; i < c->replicas.nr; i++) {
1207                         struct bch_replicas_entry *e =
1208                                 cpu_replicas_entry(&c->replicas, i);
1209                         char buf[80];
1210
1211                         if (metadata_only &&
1212                             (e->data_type == BCH_DATA_user ||
1213                              e->data_type == BCH_DATA_cached))
1214                                 continue;
1215
1216                         bch2_replicas_entry_to_text(&PBUF(buf), e);
1217
1218                         copy_fs_field(replicas[i], "%s", buf);
1219                 }
1220         }
1221
1222 #undef copy_fs_field
1223 #undef copy_dev_field
1224 #undef copy_bucket_field
1225 #undef copy_stripe_field
1226 #undef copy_field
1227 fsck_err:
1228         if (ca)
1229                 percpu_ref_put(&ca->ref);
1230         if (ret)
1231                 bch_err(c, "%s: ret %i", __func__, ret);
1232         return ret;
1233 }
1234
1235 static int bch2_gc_start(struct bch_fs *c,
1236                          bool metadata_only)
1237 {
1238         struct bch_dev *ca = NULL;
1239         unsigned i;
1240         int ret;
1241
1242         BUG_ON(c->usage_gc);
1243
1244         c->usage_gc = __alloc_percpu_gfp(fs_usage_u64s(c) * sizeof(u64),
1245                                          sizeof(u64), GFP_KERNEL);
1246         if (!c->usage_gc) {
1247                 bch_err(c, "error allocating c->usage_gc");
1248                 return -ENOMEM;
1249         }
1250
1251         for_each_member_device(ca, c, i) {
1252                 BUG_ON(ca->buckets[1]);
1253                 BUG_ON(ca->usage_gc);
1254
1255                 ca->buckets[1] = kvpmalloc(sizeof(struct bucket_array) +
1256                                 ca->mi.nbuckets * sizeof(struct bucket),
1257                                 GFP_KERNEL|__GFP_ZERO);
1258                 if (!ca->buckets[1]) {
1259                         percpu_ref_put(&ca->ref);
1260                         bch_err(c, "error allocating ca->buckets[gc]");
1261                         return -ENOMEM;
1262                 }
1263
1264                 ca->usage_gc = alloc_percpu(struct bch_dev_usage);
1265                 if (!ca->usage_gc) {
1266                         bch_err(c, "error allocating ca->usage_gc");
1267                         percpu_ref_put(&ca->ref);
1268                         return -ENOMEM;
1269                 }
1270         }
1271
1272         ret = bch2_ec_mem_alloc(c, true);
1273         if (ret) {
1274                 bch_err(c, "error allocating ec gc mem");
1275                 return ret;
1276         }
1277
1278         percpu_down_write(&c->mark_lock);
1279
1280         /*
1281          * indicate to stripe code that we need to allocate for the gc stripes
1282          * radix tree, too
1283          */
1284         gc_pos_set(c, gc_phase(GC_PHASE_START));
1285
1286         for_each_member_device(ca, c, i) {
1287                 struct bucket_array *dst = __bucket_array(ca, 1);
1288                 struct bucket_array *src = __bucket_array(ca, 0);
1289                 size_t b;
1290
1291                 dst->first_bucket       = src->first_bucket;
1292                 dst->nbuckets           = src->nbuckets;
1293
1294                 for (b = 0; b < src->nbuckets; b++) {
1295                         struct bucket *d = &dst->b[b];
1296                         struct bucket *s = &src->b[b];
1297
1298                         d->_mark.gen = dst->b[b].oldest_gen = s->mark.gen;
1299                         d->gen_valid = s->gen_valid;
1300
1301                         if (metadata_only &&
1302                             (s->mark.data_type == BCH_DATA_user ||
1303                              s->mark.data_type == BCH_DATA_cached))
1304                                 d->_mark = s->mark;
1305                 }
1306         };
1307
1308         percpu_up_write(&c->mark_lock);
1309
1310         return 0;
1311 }
1312
1313 static int bch2_gc_reflink_done_initial_fn(struct bch_fs *c, struct bkey_s_c k)
1314 {
1315         struct reflink_gc *r;
1316         const __le64 *refcount = bkey_refcount_c(k);
1317         char buf[200];
1318         int ret = 0;
1319
1320         if (!refcount)
1321                 return 0;
1322
1323         r = genradix_ptr(&c->reflink_gc_table, c->reflink_gc_idx++);
1324         if (!r)
1325                 return -ENOMEM;
1326
1327         if (!r ||
1328             r->offset != k.k->p.offset ||
1329             r->size != k.k->size) {
1330                 bch_err(c, "unexpected inconsistency walking reflink table at gc finish");
1331                 return -EINVAL;
1332         }
1333
1334         if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), c,
1335                         "reflink key has wrong refcount:\n"
1336                         "  %s\n"
1337                         "  should be %u",
1338                         (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf),
1339                         r->refcount)) {
1340                 struct bkey_i *new;
1341
1342                 new = kmalloc(bkey_bytes(k.k), GFP_KERNEL);
1343                 if (!new) {
1344                         ret = -ENOMEM;
1345                         goto fsck_err;
1346                 }
1347
1348                 bkey_reassemble(new, k);
1349
1350                 if (!r->refcount) {
1351                         new->k.type = KEY_TYPE_deleted;
1352                         new->k.size = 0;
1353                 } else {
1354                         *bkey_refcount(new) = cpu_to_le64(r->refcount);
1355                 }
1356
1357                 ret = bch2_journal_key_insert(c, BTREE_ID_reflink, 0, new);
1358                 if (ret)
1359                         kfree(new);
1360         }
1361 fsck_err:
1362         return ret;
1363 }
1364
1365 static int bch2_gc_reflink_done(struct bch_fs *c, bool initial,
1366                                 bool metadata_only)
1367 {
1368         struct btree_trans trans;
1369         struct btree_iter *iter;
1370         struct bkey_s_c k;
1371         struct reflink_gc *r;
1372         size_t idx = 0;
1373         char buf[200];
1374         int ret = 0;
1375
1376         if (metadata_only)
1377                 return 0;
1378
1379         if (initial) {
1380                 c->reflink_gc_idx = 0;
1381
1382                 ret = bch2_btree_and_journal_walk(c, BTREE_ID_reflink,
1383                                 bch2_gc_reflink_done_initial_fn);
1384                 goto out;
1385         }
1386
1387         bch2_trans_init(&trans, c, 0, 0);
1388
1389         for_each_btree_key(&trans, iter, BTREE_ID_reflink, POS_MIN,
1390                            BTREE_ITER_PREFETCH, k, ret) {
1391                 const __le64 *refcount = bkey_refcount_c(k);
1392
1393                 if (!refcount)
1394                         continue;
1395
1396                 r = genradix_ptr(&c->reflink_gc_table, idx);
1397                 if (!r ||
1398                     r->offset != k.k->p.offset ||
1399                     r->size != k.k->size) {
1400                         bch_err(c, "unexpected inconsistency walking reflink table at gc finish");
1401                         ret = -EINVAL;
1402                         break;
1403                 }
1404
1405                 if (fsck_err_on(r->refcount != le64_to_cpu(*refcount), c,
1406                                 "reflink key has wrong refcount:\n"
1407                                 "  %s\n"
1408                                 "  should be %u",
1409                                 (bch2_bkey_val_to_text(&PBUF(buf), c, k), buf),
1410                                 r->refcount)) {
1411                         struct bkey_i *new;
1412
1413                         new = kmalloc(bkey_bytes(k.k), GFP_KERNEL);
1414                         if (!new) {
1415                                 ret = -ENOMEM;
1416                                 break;
1417                         }
1418
1419                         bkey_reassemble(new, k);
1420
1421                         if (!r->refcount)
1422                                 new->k.type = KEY_TYPE_deleted;
1423                         else
1424                                 *bkey_refcount(new) = cpu_to_le64(r->refcount);
1425
1426                         ret = __bch2_trans_do(&trans, NULL, NULL, 0,
1427                                         __bch2_btree_insert(&trans, BTREE_ID_reflink, new));
1428                         kfree(new);
1429
1430                         if (ret)
1431                                 break;
1432                 }
1433         }
1434 fsck_err:
1435         bch2_trans_iter_put(&trans, iter);
1436         bch2_trans_exit(&trans);
1437 out:
1438         genradix_free(&c->reflink_gc_table);
1439         c->reflink_gc_nr = 0;
1440         return ret;
1441 }
1442
1443 static int bch2_gc_reflink_start_initial_fn(struct bch_fs *c, struct bkey_s_c k)
1444 {
1445
1446         struct reflink_gc *r;
1447         const __le64 *refcount = bkey_refcount_c(k);
1448
1449         if (!refcount)
1450                 return 0;
1451
1452         r = genradix_ptr_alloc(&c->reflink_gc_table, c->reflink_gc_nr++,
1453                                GFP_KERNEL);
1454         if (!r)
1455                 return -ENOMEM;
1456
1457         r->offset       = k.k->p.offset;
1458         r->size         = k.k->size;
1459         r->refcount     = 0;
1460         return 0;
1461 }
1462
1463 static int bch2_gc_reflink_start(struct bch_fs *c, bool initial,
1464                                  bool metadata_only)
1465 {
1466         struct btree_trans trans;
1467         struct btree_iter *iter;
1468         struct bkey_s_c k;
1469         struct reflink_gc *r;
1470         int ret;
1471
1472         if (metadata_only)
1473                 return 0;
1474
1475         genradix_free(&c->reflink_gc_table);
1476         c->reflink_gc_nr = 0;
1477
1478         if (initial)
1479                 return bch2_btree_and_journal_walk(c, BTREE_ID_reflink,
1480                                 bch2_gc_reflink_start_initial_fn);
1481
1482         bch2_trans_init(&trans, c, 0, 0);
1483
1484         for_each_btree_key(&trans, iter, BTREE_ID_reflink, POS_MIN,
1485                            BTREE_ITER_PREFETCH, k, ret) {
1486                 const __le64 *refcount = bkey_refcount_c(k);
1487
1488                 if (!refcount)
1489                         continue;
1490
1491                 r = genradix_ptr_alloc(&c->reflink_gc_table, c->reflink_gc_nr++,
1492                                        GFP_KERNEL);
1493                 if (!r) {
1494                         ret = -ENOMEM;
1495                         break;
1496                 }
1497
1498                 r->offset       = k.k->p.offset;
1499                 r->size         = k.k->size;
1500                 r->refcount     = 0;
1501         }
1502         bch2_trans_iter_put(&trans, iter);
1503
1504         bch2_trans_exit(&trans);
1505         return 0;
1506 }
1507
1508 /**
1509  * bch2_gc - walk _all_ references to buckets, and recompute them:
1510  *
1511  * Order matters here:
1512  *  - Concurrent GC relies on the fact that we have a total ordering for
1513  *    everything that GC walks - see  gc_will_visit_node(),
1514  *    gc_will_visit_root()
1515  *
1516  *  - also, references move around in the course of index updates and
1517  *    various other crap: everything needs to agree on the ordering
1518  *    references are allowed to move around in - e.g., we're allowed to
1519  *    start with a reference owned by an open_bucket (the allocator) and
1520  *    move it to the btree, but not the reverse.
1521  *
1522  *    This is necessary to ensure that gc doesn't miss references that
1523  *    move around - if references move backwards in the ordering GC
1524  *    uses, GC could skip past them
1525  */
1526 int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only)
1527 {
1528         struct bch_dev *ca;
1529         u64 start_time = local_clock();
1530         unsigned i, iter = 0;
1531         int ret;
1532
1533         lockdep_assert_held(&c->state_lock);
1534         trace_gc_start(c);
1535
1536         down_write(&c->gc_lock);
1537
1538         /* flush interior btree updates: */
1539         closure_wait_event(&c->btree_interior_update_wait,
1540                            !bch2_btree_interior_updates_nr_pending(c));
1541 again:
1542         ret   = bch2_gc_start(c, metadata_only) ?:
1543                 bch2_gc_reflink_start(c, initial, metadata_only);
1544         if (ret)
1545                 goto out;
1546
1547         bch2_mark_superblocks(c);
1548
1549         if (test_bit(BCH_FS_TOPOLOGY_ERROR, &c->flags) &&
1550             !test_bit(BCH_FS_INITIAL_GC_DONE, &c->flags) &&
1551             c->opts.fix_errors != FSCK_OPT_NO) {
1552                 bch_info(c, "starting topology repair pass");
1553                 ret = bch2_repair_topology(c);
1554                 if (ret)
1555                         goto out;
1556                 bch_info(c, "topology repair pass done");
1557         }
1558
1559         ret = bch2_gc_btrees(c, initial, metadata_only);
1560
1561         if (ret == FSCK_ERR_START_TOPOLOGY_REPAIR &&
1562             !test_bit(BCH_FS_INITIAL_GC_DONE, &c->flags)) {
1563                 set_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
1564                 ret = 0;
1565         }
1566
1567         if (ret == FSCK_ERR_START_TOPOLOGY_REPAIR)
1568                 ret = FSCK_ERR_EXIT;
1569
1570         if (ret)
1571                 goto out;
1572
1573 #if 0
1574         bch2_mark_pending_btree_node_frees(c);
1575 #endif
1576         c->gc_count++;
1577
1578         if (test_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags) ||
1579             (!iter && bch2_test_restart_gc)) {
1580                 /*
1581                  * XXX: make sure gens we fixed got saved
1582                  */
1583                 if (iter++ <= 2) {
1584                         bch_info(c, "Second GC pass needed, restarting:");
1585                         clear_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags);
1586                         __gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING));
1587
1588                         percpu_down_write(&c->mark_lock);
1589                         bch2_gc_free(c);
1590                         percpu_up_write(&c->mark_lock);
1591                         /* flush fsck errors, reset counters */
1592                         bch2_flush_fsck_errs(c);
1593
1594                         goto again;
1595                 }
1596
1597                 bch_info(c, "Unable to fix bucket gens, looping");
1598                 ret = -EINVAL;
1599         }
1600 out:
1601         if (!ret) {
1602                 bch2_journal_block(&c->journal);
1603
1604                 percpu_down_write(&c->mark_lock);
1605                 ret   = bch2_gc_reflink_done(c, initial, metadata_only) ?:
1606                         bch2_gc_done(c, initial, metadata_only);
1607
1608                 bch2_journal_unblock(&c->journal);
1609         } else {
1610                 percpu_down_write(&c->mark_lock);
1611         }
1612
1613         /* Indicates that gc is no longer in progress: */
1614         __gc_pos_set(c, gc_phase(GC_PHASE_NOT_RUNNING));
1615
1616         bch2_gc_free(c);
1617         percpu_up_write(&c->mark_lock);
1618
1619         up_write(&c->gc_lock);
1620
1621         trace_gc_end(c);
1622         bch2_time_stats_update(&c->times[BCH_TIME_btree_gc], start_time);
1623
1624         /*
1625          * Wake up allocator in case it was waiting for buckets
1626          * because of not being able to inc gens
1627          */
1628         for_each_member_device(ca, c, i)
1629                 bch2_wake_allocator(ca);
1630
1631         /*
1632          * At startup, allocations can happen directly instead of via the
1633          * allocator thread - issue wakeup in case they blocked on gc_lock:
1634          */
1635         closure_wake_up(&c->freelist_wait);
1636         return ret;
1637 }
1638
1639 static bool gc_btree_gens_key(struct bch_fs *c, struct bkey_s_c k)
1640 {
1641         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
1642         const struct bch_extent_ptr *ptr;
1643
1644         percpu_down_read(&c->mark_lock);
1645         bkey_for_each_ptr(ptrs, ptr) {
1646                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
1647                 struct bucket *g = PTR_BUCKET(ca, ptr, false);
1648
1649                 if (gen_after(g->mark.gen, ptr->gen) > 16) {
1650                         percpu_up_read(&c->mark_lock);
1651                         return true;
1652                 }
1653         }
1654
1655         bkey_for_each_ptr(ptrs, ptr) {
1656                 struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
1657                 struct bucket *g = PTR_BUCKET(ca, ptr, false);
1658
1659                 if (gen_after(g->gc_gen, ptr->gen))
1660                         g->gc_gen = ptr->gen;
1661         }
1662         percpu_up_read(&c->mark_lock);
1663
1664         return false;
1665 }
1666
1667 /*
1668  * For recalculating oldest gen, we only need to walk keys in leaf nodes; btree
1669  * node pointers currently never have cached pointers that can become stale:
1670  */
1671 static int bch2_gc_btree_gens(struct bch_fs *c, enum btree_id btree_id)
1672 {
1673         struct btree_trans trans;
1674         struct btree_iter *iter;
1675         struct bkey_s_c k;
1676         struct bkey_buf sk;
1677         int ret = 0, commit_err = 0;
1678
1679         bch2_bkey_buf_init(&sk);
1680         bch2_trans_init(&trans, c, 0, 0);
1681
1682         iter = bch2_trans_get_iter(&trans, btree_id, POS_MIN,
1683                                    BTREE_ITER_PREFETCH|
1684                                    BTREE_ITER_NOT_EXTENTS|
1685                                    BTREE_ITER_ALL_SNAPSHOTS);
1686
1687         while ((k = bch2_btree_iter_peek(iter)).k &&
1688                !(ret = bkey_err(k))) {
1689                 c->gc_gens_pos = iter->pos;
1690
1691                 if (gc_btree_gens_key(c, k) && !commit_err) {
1692                         bch2_bkey_buf_reassemble(&sk, c, k);
1693                         bch2_extent_normalize(c, bkey_i_to_s(sk.k));
1694
1695
1696                         commit_err =
1697                                 bch2_trans_update(&trans, iter, sk.k, 0) ?:
1698                                 bch2_trans_commit(&trans, NULL, NULL,
1699                                                        BTREE_INSERT_NOWAIT|
1700                                                        BTREE_INSERT_NOFAIL);
1701                         if (commit_err == -EINTR) {
1702                                 commit_err = 0;
1703                                 continue;
1704                         }
1705                 }
1706
1707                 bch2_btree_iter_advance(iter);
1708         }
1709         bch2_trans_iter_put(&trans, iter);
1710
1711         bch2_trans_exit(&trans);
1712         bch2_bkey_buf_exit(&sk, c);
1713
1714         return ret;
1715 }
1716
1717 int bch2_gc_gens(struct bch_fs *c)
1718 {
1719         struct bch_dev *ca;
1720         struct bucket_array *buckets;
1721         struct bucket *g;
1722         unsigned i;
1723         int ret;
1724
1725         /*
1726          * Ideally we would be using state_lock and not gc_lock here, but that
1727          * introduces a deadlock in the RO path - we currently take the state
1728          * lock at the start of going RO, thus the gc thread may get stuck:
1729          */
1730         down_read(&c->gc_lock);
1731
1732         for_each_member_device(ca, c, i) {
1733                 down_read(&ca->bucket_lock);
1734                 buckets = bucket_array(ca);
1735
1736                 for_each_bucket(g, buckets)
1737                         g->gc_gen = g->mark.gen;
1738                 up_read(&ca->bucket_lock);
1739         }
1740
1741         for (i = 0; i < BTREE_ID_NR; i++)
1742                 if ((1 << i) & BTREE_ID_HAS_PTRS) {
1743                         c->gc_gens_btree = i;
1744                         c->gc_gens_pos = POS_MIN;
1745                         ret = bch2_gc_btree_gens(c, i);
1746                         if (ret) {
1747                                 bch_err(c, "error recalculating oldest_gen: %i", ret);
1748                                 goto err;
1749                         }
1750                 }
1751
1752         for_each_member_device(ca, c, i) {
1753                 down_read(&ca->bucket_lock);
1754                 buckets = bucket_array(ca);
1755
1756                 for_each_bucket(g, buckets)
1757                         g->oldest_gen = g->gc_gen;
1758                 up_read(&ca->bucket_lock);
1759         }
1760
1761         c->gc_gens_btree        = 0;
1762         c->gc_gens_pos          = POS_MIN;
1763
1764         c->gc_count++;
1765 err:
1766         up_read(&c->gc_lock);
1767         return ret;
1768 }
1769
1770 static int bch2_gc_thread(void *arg)
1771 {
1772         struct bch_fs *c = arg;
1773         struct io_clock *clock = &c->io_clock[WRITE];
1774         unsigned long last = atomic64_read(&clock->now);
1775         unsigned last_kick = atomic_read(&c->kick_gc);
1776         int ret;
1777
1778         set_freezable();
1779
1780         while (1) {
1781                 while (1) {
1782                         set_current_state(TASK_INTERRUPTIBLE);
1783
1784                         if (kthread_should_stop()) {
1785                                 __set_current_state(TASK_RUNNING);
1786                                 return 0;
1787                         }
1788
1789                         if (atomic_read(&c->kick_gc) != last_kick)
1790                                 break;
1791
1792                         if (c->btree_gc_periodic) {
1793                                 unsigned long next = last + c->capacity / 16;
1794
1795                                 if (atomic64_read(&clock->now) >= next)
1796                                         break;
1797
1798                                 bch2_io_clock_schedule_timeout(clock, next);
1799                         } else {
1800                                 schedule();
1801                         }
1802
1803                         try_to_freeze();
1804                 }
1805                 __set_current_state(TASK_RUNNING);
1806
1807                 last = atomic64_read(&clock->now);
1808                 last_kick = atomic_read(&c->kick_gc);
1809
1810                 /*
1811                  * Full gc is currently incompatible with btree key cache:
1812                  */
1813 #if 0
1814                 ret = bch2_gc(c, false, false);
1815 #else
1816                 ret = bch2_gc_gens(c);
1817 #endif
1818                 if (ret < 0)
1819                         bch_err(c, "btree gc failed: %i", ret);
1820
1821                 debug_check_no_locks_held();
1822         }
1823
1824         return 0;
1825 }
1826
1827 void bch2_gc_thread_stop(struct bch_fs *c)
1828 {
1829         struct task_struct *p;
1830
1831         p = c->gc_thread;
1832         c->gc_thread = NULL;
1833
1834         if (p) {
1835                 kthread_stop(p);
1836                 put_task_struct(p);
1837         }
1838 }
1839
1840 int bch2_gc_thread_start(struct bch_fs *c)
1841 {
1842         struct task_struct *p;
1843
1844         if (c->gc_thread)
1845                 return 0;
1846
1847         p = kthread_create(bch2_gc_thread, c, "bch-gc/%s", c->name);
1848         if (IS_ERR(p)) {
1849                 bch_err(c, "error creating gc thread: %li", PTR_ERR(p));
1850                 return PTR_ERR(p);
1851         }
1852
1853         get_task_struct(p);
1854         c->gc_thread = p;
1855         wake_up_process(p);
1856         return 0;
1857 }