git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/move.c

   1
   2 #include "bcachefs.h"
   3 #include "btree_gc.h"
   4 #include "btree_update.h"
   5 #include "buckets.h"
   6 #include "inode.h"
   7 #include "io.h"
   8 #include "journal_reclaim.h"
   9 #include "move.h"
  10 #include "replicas.h"
  11 #include "super-io.h"
  12 #include "keylist.h"
  13
  14 #include <linux/ioprio.h>
  15 #include <linux/kthread.h>
  16
  17 #include <trace/events/bcachefs.h>
  18
  19 #define SECTORS_IN_FLIGHT_PER_DEVICE    2048
  20
  21 struct moving_io {
  22         struct list_head        list;
  23         struct closure          cl;
  24         bool                    read_completed;
  25
  26         unsigned                read_sectors;
  27         unsigned                write_sectors;
  28
  29         struct bch_read_bio     rbio;
  30
  31         struct migrate_write    write;
  32         /* Must be last since it is variable size */
  33         struct bio_vec          bi_inline_vecs[0];
  34 };
  35
  36 struct moving_context {
  37         /* Closure for waiting on all reads and writes to complete */
  38         struct closure          cl;
  39
  40         struct bch_move_stats   *stats;
  41
  42         struct list_head        reads;
  43
  44         /* in flight sectors: */
  45         atomic_t                read_sectors;
  46         atomic_t                write_sectors;
  47
  48         wait_queue_head_t       wait;
  49 };
  50
  51 static int bch2_migrate_index_update(struct bch_write_op *op)
  52 {
  53         struct bch_fs *c = op->c;
  54         struct migrate_write *m =
  55                 container_of(op, struct migrate_write, op);
  56         struct keylist *keys = &op->insert_keys;
  57         struct btree_iter iter;
  58         int ret = 0;
  59
  60         bch2_btree_iter_init(&iter, c, BTREE_ID_EXTENTS,
  61                              bkey_start_pos(&bch2_keylist_front(keys)->k),
  62                              BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
  63
  64         while (1) {
  65                 struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter);
  66                 struct bkey_i_extent *insert, *new =
  67                         bkey_i_to_extent(bch2_keylist_front(keys));
  68                 BKEY_PADDED(k) _new, _insert;
  69                 struct bch_extent_ptr *ptr;
  70                 struct bch_extent_crc_unpacked crc;
  71                 bool did_work = false;
  72                 int nr;
  73
  74                 if (btree_iter_err(k)) {
  75                         ret = bch2_btree_iter_unlock(&iter);
  76                         break;
  77                 }
  78
  79                 if (bversion_cmp(k.k->version, new->k.version) ||
  80                     !bkey_extent_is_data(k.k) ||
  81                     !bch2_extent_matches_ptr(c, bkey_s_c_to_extent(k),
  82                                              m->ptr, m->offset))
  83                         goto nomatch;
  84
  85                 if (m->data_cmd == DATA_REWRITE &&
  86                     !bch2_extent_has_device(bkey_s_c_to_extent(k),
  87                                             m->data_opts.rewrite_dev))
  88                         goto nomatch;
  89
  90                 bkey_reassemble(&_insert.k, k);
  91                 insert = bkey_i_to_extent(&_insert.k);
  92
  93                 bkey_copy(&_new.k, bch2_keylist_front(keys));
  94                 new = bkey_i_to_extent(&_new.k);
  95
  96                 bch2_cut_front(iter.pos, &insert->k_i);
  97                 bch2_cut_back(new->k.p, &insert->k);
  98                 bch2_cut_back(insert->k.p, &new->k);
  99
 100                 if (m->data_cmd == DATA_REWRITE) {
 101                         ptr = (struct bch_extent_ptr *)
 102                                 bch2_extent_has_device(extent_i_to_s_c(insert),
 103                                                        m->data_opts.rewrite_dev);
 104                         bch2_extent_drop_ptr(extent_i_to_s(insert), ptr);
 105                 }
 106
 107                 extent_for_each_ptr_crc(extent_i_to_s(new), ptr, crc) {
 108                         if (bch2_extent_has_device(extent_i_to_s_c(insert), ptr->dev)) {
 109                                 /*
 110                                  * raced with another move op? extent already
 111                                  * has a pointer to the device we just wrote
 112                                  * data to
 113                                  */
 114                                 continue;
 115                         }
 116
 117                         bch2_extent_crc_append(insert, crc);
 118                         extent_ptr_append(insert, *ptr);
 119                         did_work = true;
 120                 }
 121
 122                 if (!did_work)
 123                         goto nomatch;
 124
 125                 bch2_extent_narrow_crcs(insert,
 126                                 (struct bch_extent_crc_unpacked) { 0 });
 127                 bch2_extent_normalize(c, extent_i_to_s(insert).s);
 128                 bch2_extent_mark_replicas_cached(c, extent_i_to_s(insert),
 129                                                  op->opts.background_target,
 130                                                  op->opts.data_replicas);
 131
 132                 /*
 133                  * It's possible we race, and for whatever reason the extent now
 134                  * has fewer replicas than when we last looked at it - meaning
 135                  * we need to get a disk reservation here:
 136                  */
 137                 nr = bch2_extent_nr_dirty_ptrs(bkey_i_to_s_c(&insert->k_i)) -
 138                         (bch2_extent_nr_dirty_ptrs(k) + m->nr_ptrs_reserved);
 139                 if (nr > 0) {
 140                         /*
 141                          * can't call bch2_disk_reservation_add() with btree
 142                          * locks held, at least not without a song and dance
 143                          */
 144                         bch2_btree_iter_unlock(&iter);
 145
 146                         ret = bch2_disk_reservation_add(c, &op->res,
 147                                         keylist_sectors(keys) * nr, 0);
 148                         if (ret)
 149                                 goto out;
 150
 151                         m->nr_ptrs_reserved += nr;
 152                         goto next;
 153                 }
 154
 155                 ret = bch2_mark_bkey_replicas(c, BCH_DATA_USER,
 156                                               extent_i_to_s_c(insert).s_c);
 157                 if (ret)
 158                         break;
 159
 160                 ret = bch2_btree_insert_at(c, &op->res,
 161                                 NULL, op_journal_seq(op),
 162                                 BTREE_INSERT_ATOMIC|
 163                                 BTREE_INSERT_NOFAIL|
 164                                 BTREE_INSERT_USE_RESERVE|
 165                                 m->data_opts.btree_insert_flags,
 166                                 BTREE_INSERT_ENTRY(&iter, &insert->k_i));
 167                 if (!ret)
 168                         atomic_long_inc(&c->extent_migrate_done);
 169                 if (ret == -EINTR)
 170                         ret = 0;
 171                 if (ret)
 172                         break;
 173 next:
 174                 while (bkey_cmp(iter.pos, bch2_keylist_front(keys)->k.p) >= 0) {
 175                         bch2_keylist_pop_front(keys);
 176                         if (bch2_keylist_empty(keys))
 177                                 goto out;
 178                 }
 179
 180                 bch2_cut_front(iter.pos, bch2_keylist_front(keys));
 181                 continue;
 182 nomatch:
 183                 if (m->ctxt)
 184                         atomic64_add(k.k->p.offset - iter.pos.offset,
 185                                      &m->ctxt->stats->sectors_raced);
 186                 atomic_long_inc(&c->extent_migrate_raced);
 187                 trace_move_race(&new->k);
 188                 bch2_btree_iter_next_slot(&iter);
 189                 goto next;
 190         }
 191 out:
 192         bch2_btree_iter_unlock(&iter);
 193         return ret;
 194 }
 195
 196 void bch2_migrate_read_done(struct migrate_write *m, struct bch_read_bio *rbio)
 197 {
 198         /* write bio must own pages: */
 199         BUG_ON(!m->op.wbio.bio.bi_vcnt);
 200
 201         m->ptr          = rbio->pick.ptr;
 202         m->offset       = rbio->pos.offset - rbio->pick.crc.offset;
 203         m->op.devs_have = rbio->devs_have;
 204         m->op.pos       = rbio->pos;
 205         m->op.version   = rbio->version;
 206         m->op.crc       = rbio->pick.crc;
 207         m->op.wbio.bio.bi_iter.bi_size = m->op.crc.compressed_size << 9;
 208
 209         if (bch2_csum_type_is_encryption(m->op.crc.csum_type)) {
 210                 m->op.nonce     = m->op.crc.nonce + m->op.crc.offset;
 211                 m->op.csum_type = m->op.crc.csum_type;
 212         }
 213
 214         if (m->data_cmd == DATA_REWRITE)
 215                 bch2_dev_list_drop_dev(&m->op.devs_have, m->data_opts.rewrite_dev);
 216 }
 217
 218 int bch2_migrate_write_init(struct bch_fs *c, struct migrate_write *m,
 219                             struct write_point_specifier wp,
 220                             struct bch_io_opts io_opts,
 221                             enum data_cmd data_cmd,
 222                             struct data_opts data_opts,
 223                             struct bkey_s_c k)
 224 {
 225         int ret;
 226
 227         m->data_cmd     = data_cmd;
 228         m->data_opts    = data_opts;
 229         m->nr_ptrs_reserved = 0;
 230
 231         bch2_write_op_init(&m->op, c, io_opts);
 232         m->op.compression_type =
 233                 bch2_compression_opt_to_type[io_opts.background_compression ?:
 234                                              io_opts.compression];
 235         m->op.target    = data_opts.target,
 236         m->op.write_point = wp;
 237
 238         if (m->data_opts.btree_insert_flags & BTREE_INSERT_USE_RESERVE)
 239                 m->op.alloc_reserve = RESERVE_MOVINGGC;
 240
 241         m->op.flags |= BCH_WRITE_ONLY_SPECIFIED_DEVS|
 242                 BCH_WRITE_PAGES_STABLE|
 243                 BCH_WRITE_PAGES_OWNED|
 244                 BCH_WRITE_DATA_ENCODED|
 245                 BCH_WRITE_NOMARK_REPLICAS;
 246
 247         m->op.nr_replicas       = 1;
 248         m->op.nr_replicas_required = 1;
 249         m->op.index_update_fn   = bch2_migrate_index_update;
 250
 251         switch (data_cmd) {
 252         case DATA_ADD_REPLICAS: {
 253                 int nr = (int) io_opts.data_replicas -
 254                         bch2_extent_nr_dirty_ptrs(k);
 255
 256                 if (nr > 0) {
 257                         m->op.nr_replicas = m->nr_ptrs_reserved = nr;
 258
 259                         ret = bch2_disk_reservation_get(c, &m->op.res,
 260                                         k.k->size, m->op.nr_replicas, 0);
 261                         if (ret)
 262                                 return ret;
 263                 }
 264                 break;
 265         }
 266         case DATA_REWRITE:
 267                 break;
 268         case DATA_PROMOTE:
 269                 m->op.flags     |= BCH_WRITE_ALLOC_NOWAIT;
 270                 m->op.flags     |= BCH_WRITE_CACHED;
 271                 break;
 272         default:
 273                 BUG();
 274         }
 275
 276         return 0;
 277 }
 278
 279 static void move_free(struct closure *cl)
 280 {
 281         struct moving_io *io = container_of(cl, struct moving_io, cl);
 282         struct moving_context *ctxt = io->write.ctxt;
 283         struct bio_vec *bv;
 284         int i;
 285
 286         bch2_disk_reservation_put(io->write.op.c, &io->write.op.res);
 287
 288         bio_for_each_segment_all(bv, &io->write.op.wbio.bio, i)
 289                 if (bv->bv_page)
 290                         __free_page(bv->bv_page);
 291
 292         wake_up(&ctxt->wait);
 293
 294         kfree(io);
 295 }
 296
 297 static void move_write_done(struct closure *cl)
 298 {
 299         struct moving_io *io = container_of(cl, struct moving_io, cl);
 300
 301         atomic_sub(io->write_sectors, &io->write.ctxt->write_sectors);
 302         closure_return_with_destructor(cl, move_free);
 303 }
 304
 305 static void move_write(struct closure *cl)
 306 {
 307         struct moving_io *io = container_of(cl, struct moving_io, cl);
 308
 309         if (likely(!io->rbio.bio.bi_status &&
 310                    !io->rbio.hole)) {
 311                 bch2_migrate_read_done(&io->write, &io->rbio);
 312
 313                 atomic_add(io->write_sectors, &io->write.ctxt->write_sectors);
 314                 closure_call(&io->write.op.cl, bch2_write, NULL, cl);
 315                 continue_at(cl, move_write_done, NULL);
 316         }
 317
 318         closure_return_with_destructor(cl, move_free);
 319 }
 320
 321 static inline struct moving_io *next_pending_write(struct moving_context *ctxt)
 322 {
 323         struct moving_io *io =
 324                 list_first_entry_or_null(&ctxt->reads, struct moving_io, list);
 325
 326         return io && io->read_completed ? io : NULL;
 327 }
 328
 329 static void move_read_endio(struct bio *bio)
 330 {
 331         struct moving_io *io = container_of(bio, struct moving_io, rbio.bio);
 332         struct moving_context *ctxt = io->write.ctxt;
 333
 334         atomic_sub(io->read_sectors, &ctxt->read_sectors);
 335         io->read_completed = true;
 336
 337         if (next_pending_write(ctxt))
 338                 wake_up(&ctxt->wait);
 339
 340         closure_put(&ctxt->cl);
 341 }
 342
 343 static void do_pending_writes(struct moving_context *ctxt)
 344 {
 345         struct moving_io *io;
 346
 347         while ((io = next_pending_write(ctxt))) {
 348                 list_del(&io->list);
 349                 closure_call(&io->cl, move_write, NULL, &ctxt->cl);
 350         }
 351 }
 352
 353 #define move_ctxt_wait_event(_ctxt, _cond)                      \
 354 do {                                                            \
 355         do_pending_writes(_ctxt);                               \
 356                                                                 \
 357         if (_cond)                                              \
 358                 break;                                          \
 359         __wait_event((_ctxt)->wait,                             \
 360                      next_pending_write(_ctxt) || (_cond));     \
 361 } while (1)
 362
 363 static void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt)
 364 {
 365         unsigned sectors_pending = atomic_read(&ctxt->write_sectors);
 366
 367         move_ctxt_wait_event(ctxt,
 368                 !atomic_read(&ctxt->write_sectors) ||
 369                 atomic_read(&ctxt->write_sectors) != sectors_pending);
 370 }
 371
 372 static int bch2_move_extent(struct bch_fs *c,
 373                             struct moving_context *ctxt,
 374                             struct write_point_specifier wp,
 375                             struct bch_io_opts io_opts,
 376                             struct bkey_s_c_extent e,
 377                             enum data_cmd data_cmd,
 378                             struct data_opts data_opts)
 379 {
 380         struct moving_io *io;
 381         const struct bch_extent_ptr *ptr;
 382         struct bch_extent_crc_unpacked crc;
 383         unsigned sectors = e.k->size, pages;
 384         int ret = -ENOMEM;
 385
 386         move_ctxt_wait_event(ctxt,
 387                 atomic_read(&ctxt->write_sectors) <
 388                 SECTORS_IN_FLIGHT_PER_DEVICE);
 389
 390         move_ctxt_wait_event(ctxt,
 391                 atomic_read(&ctxt->read_sectors) <
 392                 SECTORS_IN_FLIGHT_PER_DEVICE);
 393
 394         /* write path might have to decompress data: */
 395         extent_for_each_ptr_crc(e, ptr, crc)
 396                 sectors = max_t(unsigned, sectors, crc.uncompressed_size);
 397
 398         pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
 399         io = kzalloc(sizeof(struct moving_io) +
 400                      sizeof(struct bio_vec) * pages, GFP_KERNEL);
 401         if (!io)
 402                 goto err;
 403
 404         io->write.ctxt          = ctxt;
 405         io->read_sectors        = e.k->size;
 406         io->write_sectors       = e.k->size;
 407
 408         bio_init(&io->write.op.wbio.bio, io->bi_inline_vecs, pages);
 409         bio_set_prio(&io->write.op.wbio.bio,
 410                      IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 411         io->write.op.wbio.bio.bi_iter.bi_size = sectors << 9;
 412
 413         bch2_bio_map(&io->write.op.wbio.bio, NULL);
 414         if (bio_alloc_pages(&io->write.op.wbio.bio, GFP_KERNEL))
 415                 goto err_free;
 416
 417         io->rbio.opts = io_opts;
 418         bio_init(&io->rbio.bio, io->bi_inline_vecs, pages);
 419         io->rbio.bio.bi_vcnt = pages;
 420         bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 421         io->rbio.bio.bi_iter.bi_size = sectors << 9;
 422
 423         bio_set_op_attrs(&io->rbio.bio, REQ_OP_READ, 0);
 424         io->rbio.bio.bi_iter.bi_sector  = bkey_start_offset(e.k);
 425         io->rbio.bio.bi_end_io          = move_read_endio;
 426
 427         ret = bch2_migrate_write_init(c, &io->write, wp, io_opts,
 428                                       data_cmd, data_opts, e.s_c);
 429         if (ret)
 430                 goto err_free_pages;
 431
 432         atomic64_inc(&ctxt->stats->keys_moved);
 433         atomic64_add(e.k->size, &ctxt->stats->sectors_moved);
 434
 435         trace_move_extent(e.k);
 436
 437         atomic_add(io->read_sectors, &ctxt->read_sectors);
 438         list_add_tail(&io->list, &ctxt->reads);
 439
 440         /*
 441          * dropped by move_read_endio() - guards against use after free of
 442          * ctxt when doing wakeup
 443          */
 444         closure_get(&ctxt->cl);
 445         bch2_read_extent(c, &io->rbio, e.s_c,
 446                          BCH_READ_NODECODE|
 447                          BCH_READ_LAST_FRAGMENT);
 448         return 0;
 449 err_free_pages:
 450         bio_free_pages(&io->write.op.wbio.bio);
 451 err_free:
 452         kfree(io);
 453 err:
 454         trace_move_alloc_fail(e.k);
 455         return ret;
 456 }
 457
 458 int bch2_move_data(struct bch_fs *c,
 459                    struct bch_ratelimit *rate,
 460                    struct write_point_specifier wp,
 461                    struct bpos start,
 462                    struct bpos end,
 463                    move_pred_fn pred, void *arg,
 464                    struct bch_move_stats *stats)
 465 {
 466         bool kthread = (current->flags & PF_KTHREAD) != 0;
 467         struct moving_context ctxt = { .stats = stats };
 468         struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 469         BKEY_PADDED(k) tmp;
 470         struct bkey_s_c k;
 471         struct bkey_s_c_extent e;
 472         struct data_opts data_opts;
 473         enum data_cmd data_cmd;
 474         u64 cur_inum = U64_MAX;
 475         int ret = 0, ret2;
 476
 477         closure_init_stack(&ctxt.cl);
 478         INIT_LIST_HEAD(&ctxt.reads);
 479         init_waitqueue_head(&ctxt.wait);
 480
 481         stats->data_type = BCH_DATA_USER;
 482         bch2_btree_iter_init(&stats->iter, c, BTREE_ID_EXTENTS, start,
 483                              BTREE_ITER_PREFETCH);
 484
 485         if (rate)
 486                 bch2_ratelimit_reset(rate);
 487
 488         while (!kthread || !(ret = kthread_should_stop())) {
 489                 if (rate &&
 490                     bch2_ratelimit_delay(rate) &&
 491                     (bch2_btree_iter_unlock(&stats->iter),
 492                      (ret = bch2_ratelimit_wait_freezable_stoppable(rate))))
 493                         break;
 494 peek:
 495                 k = bch2_btree_iter_peek(&stats->iter);
 496                 if (!k.k)
 497                         break;
 498                 ret = btree_iter_err(k);
 499                 if (ret)
 500                         break;
 501                 if (bkey_cmp(bkey_start_pos(k.k), end) >= 0)
 502                         break;
 503
 504                 if (!bkey_extent_is_data(k.k))
 505                         goto next_nondata;
 506
 507                 e = bkey_s_c_to_extent(k);
 508
 509                 if (cur_inum != k.k->p.inode) {
 510                         struct bch_inode_unpacked inode;
 511
 512                         /* don't hold btree locks while looking up inode: */
 513                         bch2_btree_iter_unlock(&stats->iter);
 514
 515                         io_opts = bch2_opts_to_inode_opts(c->opts);
 516                         if (!bch2_inode_find_by_inum(c, k.k->p.inode, &inode))
 517                                 bch2_io_opts_apply(&io_opts, bch2_inode_opts_get(&inode));
 518                         cur_inum = k.k->p.inode;
 519                         goto peek;
 520                 }
 521
 522                 switch ((data_cmd = pred(c, arg, BKEY_TYPE_EXTENTS, e,
 523                                          &io_opts, &data_opts))) {
 524                 case DATA_SKIP:
 525                         goto next;
 526                 case DATA_SCRUB:
 527                         BUG();
 528                 case DATA_ADD_REPLICAS:
 529                 case DATA_REWRITE:
 530                 case DATA_PROMOTE:
 531                         break;
 532                 default:
 533                         BUG();
 534                 }
 535
 536                 /* unlock before doing IO: */
 537                 bkey_reassemble(&tmp.k, k);
 538                 k = bkey_i_to_s_c(&tmp.k);
 539                 bch2_btree_iter_unlock(&stats->iter);
 540
 541                 ret2 = bch2_move_extent(c, &ctxt, wp, io_opts,
 542                                         bkey_s_c_to_extent(k),
 543                                         data_cmd, data_opts);
 544                 if (ret2) {
 545                         if (ret2 == -ENOMEM) {
 546                                 /* memory allocation failure, wait for some IO to finish */
 547                                 bch2_move_ctxt_wait_for_io(&ctxt);
 548                                 continue;
 549                         }
 550
 551                         /* XXX signal failure */
 552                         goto next;
 553                 }
 554
 555                 if (rate)
 556                         bch2_ratelimit_increment(rate, k.k->size);
 557 next:
 558                 atomic64_add(k.k->size * bch2_extent_nr_dirty_ptrs(k),
 559                              &stats->sectors_seen);
 560 next_nondata:
 561                 bch2_btree_iter_next(&stats->iter);
 562                 bch2_btree_iter_cond_resched(&stats->iter);
 563         }
 564
 565         bch2_btree_iter_unlock(&stats->iter);
 566
 567         move_ctxt_wait_event(&ctxt, list_empty(&ctxt.reads));
 568         closure_sync(&ctxt.cl);
 569
 570         EBUG_ON(atomic_read(&ctxt.write_sectors));
 571
 572         trace_move_data(c,
 573                         atomic64_read(&stats->sectors_moved),
 574                         atomic64_read(&stats->keys_moved));
 575
 576         return ret;
 577 }
 578
 579 static int bch2_gc_data_replicas(struct bch_fs *c)
 580 {
 581         struct btree_iter iter;
 582         struct bkey_s_c k;
 583         int ret;
 584
 585         mutex_lock(&c->replicas_gc_lock);
 586         bch2_replicas_gc_start(c, (1 << BCH_DATA_USER)|(1 << BCH_DATA_CACHED));
 587
 588         for_each_btree_key(&iter, c, BTREE_ID_EXTENTS, POS_MIN,
 589                            BTREE_ITER_PREFETCH, k) {
 590                 ret = bch2_mark_bkey_replicas(c, BCH_DATA_USER, k);
 591                 if (ret)
 592                         break;
 593         }
 594         ret = bch2_btree_iter_unlock(&iter) ?: ret;
 595
 596         bch2_replicas_gc_end(c, ret);
 597         mutex_unlock(&c->replicas_gc_lock);
 598
 599         return ret;
 600 }
 601
 602 static int bch2_gc_btree_replicas(struct bch_fs *c)
 603 {
 604         struct btree_iter iter;
 605         struct btree *b;
 606         unsigned id;
 607         int ret = 0;
 608
 609         mutex_lock(&c->replicas_gc_lock);
 610         bch2_replicas_gc_start(c, 1 << BCH_DATA_BTREE);
 611
 612         for (id = 0; id < BTREE_ID_NR; id++) {
 613                 for_each_btree_node(&iter, c, id, POS_MIN, BTREE_ITER_PREFETCH, b) {
 614                         ret = bch2_mark_bkey_replicas(c, BCH_DATA_BTREE,
 615                                                       bkey_i_to_s_c(&b->key));
 616
 617                         bch2_btree_iter_cond_resched(&iter);
 618                 }
 619
 620                 ret = bch2_btree_iter_unlock(&iter) ?: ret;
 621         }
 622
 623         bch2_replicas_gc_end(c, ret);
 624         mutex_unlock(&c->replicas_gc_lock);
 625
 626         return ret;
 627 }
 628
 629 static int bch2_move_btree(struct bch_fs *c,
 630                            move_pred_fn pred,
 631                            void *arg,
 632                            struct bch_move_stats *stats)
 633 {
 634         struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 635         struct btree *b;
 636         unsigned id;
 637         struct data_opts data_opts;
 638         enum data_cmd cmd;
 639         int ret = 0;
 640
 641         stats->data_type = BCH_DATA_BTREE;
 642
 643         for (id = 0; id < BTREE_ID_NR; id++) {
 644                 for_each_btree_node(&stats->iter, c, id, POS_MIN, BTREE_ITER_PREFETCH, b) {
 645                         switch ((cmd = pred(c, arg, BKEY_TYPE_BTREE,
 646                                             bkey_i_to_s_c_extent(&b->key),
 647                                             &io_opts,
 648                                             &data_opts))) {
 649                         case DATA_SKIP:
 650                                 goto next;
 651                         case DATA_SCRUB:
 652                                 BUG();
 653                         case DATA_ADD_REPLICAS:
 654                         case DATA_REWRITE:
 655                                 break;
 656                         default:
 657                                 BUG();
 658                         }
 659
 660                         ret = bch2_btree_node_rewrite(c, &stats->iter,
 661                                         b->data->keys.seq, 0) ?: ret;
 662 next:
 663                         bch2_btree_iter_cond_resched(&stats->iter);
 664                 }
 665
 666                 ret = bch2_btree_iter_unlock(&stats->iter) ?: ret;
 667         }
 668
 669         return ret;
 670 }
 671
 672 #if 0
 673 static enum data_cmd scrub_pred(struct bch_fs *c, void *arg,
 674                                 enum bkey_type type,
 675                                 struct bkey_s_c_extent e,
 676                                 struct bch_io_opts *io_opts,
 677                                 struct data_opts *data_opts)
 678 {
 679         return DATA_SCRUB;
 680 }
 681 #endif
 682
 683 static enum data_cmd rereplicate_pred(struct bch_fs *c, void *arg,
 684                                       enum bkey_type type,
 685                                       struct bkey_s_c_extent e,
 686                                       struct bch_io_opts *io_opts,
 687                                       struct data_opts *data_opts)
 688 {
 689         unsigned nr_good = bch2_extent_durability(c, e);
 690         unsigned replicas = type == BKEY_TYPE_BTREE
 691                 ? c->opts.metadata_replicas
 692                 : io_opts->data_replicas;
 693
 694         if (!nr_good || nr_good >= replicas)
 695                 return DATA_SKIP;
 696
 697         data_opts->target               = 0;
 698         data_opts->btree_insert_flags = 0;
 699         return DATA_ADD_REPLICAS;
 700 }
 701
 702 static enum data_cmd migrate_pred(struct bch_fs *c, void *arg,
 703                                   enum bkey_type type,
 704                                   struct bkey_s_c_extent e,
 705                                   struct bch_io_opts *io_opts,
 706                                   struct data_opts *data_opts)
 707 {
 708         struct bch_ioctl_data *op = arg;
 709
 710         if (!bch2_extent_has_device(e, op->migrate.dev))
 711                 return DATA_SKIP;
 712
 713         data_opts->target               = 0;
 714         data_opts->btree_insert_flags   = 0;
 715         data_opts->rewrite_dev          = op->migrate.dev;
 716         return DATA_REWRITE;
 717 }
 718
 719 int bch2_data_job(struct bch_fs *c,
 720                   struct bch_move_stats *stats,
 721                   struct bch_ioctl_data op)
 722 {
 723         int ret = 0;
 724
 725         switch (op.op) {
 726         case BCH_DATA_OP_REREPLICATE:
 727                 stats->data_type = BCH_DATA_JOURNAL;
 728                 ret = bch2_journal_flush_device_pins(&c->journal, -1);
 729
 730                 ret = bch2_move_btree(c, rereplicate_pred, c, stats) ?: ret;
 731                 ret = bch2_gc_btree_replicas(c) ?: ret;
 732
 733                 ret = bch2_move_data(c, NULL,
 734                                      writepoint_hashed((unsigned long) current),
 735                                      op.start,
 736                                      op.end,
 737                                      rereplicate_pred, c, stats) ?: ret;
 738                 ret = bch2_gc_data_replicas(c) ?: ret;
 739                 break;
 740         case BCH_DATA_OP_MIGRATE:
 741                 if (op.migrate.dev >= c->sb.nr_devices)
 742                         return -EINVAL;
 743
 744                 stats->data_type = BCH_DATA_JOURNAL;
 745                 ret = bch2_journal_flush_device_pins(&c->journal, op.migrate.dev);
 746
 747                 ret = bch2_move_btree(c, migrate_pred, &op, stats) ?: ret;
 748                 ret = bch2_gc_btree_replicas(c) ?: ret;
 749
 750                 ret = bch2_move_data(c, NULL,
 751                                      writepoint_hashed((unsigned long) current),
 752                                      op.start,
 753                                      op.end,
 754                                      migrate_pred, &op, stats) ?: ret;
 755                 ret = bch2_gc_data_replicas(c) ?: ret;
 756                 break;
 757         default:
 758                 ret = -EINVAL;
 759         }
 760
 761         return ret;
 762 }