git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/move.c

   1
   2 #include "bcachefs.h"
   3 #include "btree_gc.h"
   4 #include "btree_update.h"
   5 #include "buckets.h"
   6 #include "inode.h"
   7 #include "io.h"
   8 #include "move.h"
   9 #include "super-io.h"
  10 #include "keylist.h"
  11
  12 #include <linux/ioprio.h>
  13 #include <linux/kthread.h>
  14
  15 #include <trace/events/bcachefs.h>
  16
  17 struct moving_io {
  18         struct list_head        list;
  19         struct closure          cl;
  20         bool                    read_completed;
  21         unsigned                sectors;
  22
  23         struct bch_read_bio     rbio;
  24
  25         struct migrate_write    write;
  26         /* Must be last since it is variable size */
  27         struct bio_vec          bi_inline_vecs[0];
  28 };
  29
  30 struct moving_context {
  31         /* Closure for waiting on all reads and writes to complete */
  32         struct closure          cl;
  33
  34         struct bch_move_stats   *stats;
  35
  36         struct list_head        reads;
  37         atomic_t                sectors_in_flight;
  38         wait_queue_head_t       wait;
  39 };
  40
  41 static int bch2_migrate_index_update(struct bch_write_op *op)
  42 {
  43         struct bch_fs *c = op->c;
  44         struct migrate_write *m =
  45                 container_of(op, struct migrate_write, op);
  46         struct keylist *keys = &op->insert_keys;
  47         struct btree_iter iter;
  48         int ret = 0;
  49
  50         bch2_btree_iter_init(&iter, c, BTREE_ID_EXTENTS,
  51                              bkey_start_pos(&bch2_keylist_front(keys)->k),
  52                              BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
  53
  54         while (1) {
  55                 struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter);
  56                 struct bkey_i_extent *insert, *new =
  57                         bkey_i_to_extent(bch2_keylist_front(keys));
  58                 BKEY_PADDED(k) _new, _insert;
  59                 struct bch_extent_ptr *ptr;
  60                 struct bch_extent_crc_unpacked crc;
  61                 unsigned nr_dirty;
  62                 bool did_work = false;
  63
  64                 if (btree_iter_err(k)) {
  65                         ret = bch2_btree_iter_unlock(&iter);
  66                         break;
  67                 }
  68
  69                 if (bversion_cmp(k.k->version, new->k.version) ||
  70                     !bkey_extent_is_data(k.k) ||
  71                     !bch2_extent_matches_ptr(c, bkey_s_c_to_extent(k),
  72                                              m->ptr, m->offset))
  73                         goto nomatch;
  74
  75                 if (m->data_cmd == DATA_REWRITE &&
  76                     !bch2_extent_has_device(bkey_s_c_to_extent(k),
  77                                             m->data_opts.rewrite_dev))
  78                         goto nomatch;
  79
  80                 bkey_reassemble(&_insert.k, k);
  81                 insert = bkey_i_to_extent(&_insert.k);
  82
  83                 bkey_copy(&_new.k, bch2_keylist_front(keys));
  84                 new = bkey_i_to_extent(&_new.k);
  85
  86                 bch2_cut_front(iter.pos, &insert->k_i);
  87                 bch2_cut_back(new->k.p, &insert->k);
  88                 bch2_cut_back(insert->k.p, &new->k);
  89
  90                 if (m->data_cmd == DATA_REWRITE) {
  91                         ptr = (struct bch_extent_ptr *)
  92                                 bch2_extent_has_device(extent_i_to_s_c(insert),
  93                                                        m->data_opts.rewrite_dev);
  94                         bch2_extent_drop_ptr(extent_i_to_s(insert), ptr);
  95                 }
  96
  97                 extent_for_each_ptr_crc(extent_i_to_s(new), ptr, crc) {
  98                         if (bch2_extent_has_device(extent_i_to_s_c(insert), ptr->dev)) {
  99                                 /*
 100                                  * raced with another move op? extent already
 101                                  * has a pointer to the device we just wrote
 102                                  * data to
 103                                  */
 104                                 continue;
 105                         }
 106
 107                         bch2_extent_crc_append(insert, crc);
 108                         extent_ptr_append(insert, *ptr);
 109                         did_work = true;
 110                 }
 111
 112                 if (!did_work)
 113                         goto nomatch;
 114
 115                 bch2_extent_narrow_crcs(insert,
 116                                 (struct bch_extent_crc_unpacked) { 0 });
 117                 bch2_extent_normalize(c, extent_i_to_s(insert).s);
 118                 bch2_extent_mark_replicas_cached(c, extent_i_to_s(insert),
 119                                                  c->opts.data_replicas);
 120
 121                 /*
 122                  * It's possible we race, and for whatever reason the extent now
 123                  * has fewer replicas than when we last looked at it - meaning
 124                  * we need to get a disk reservation here:
 125                  */
 126                 nr_dirty = bch2_extent_nr_dirty_ptrs(bkey_i_to_s_c(&insert->k_i));
 127                 if (m->nr_ptrs_reserved < nr_dirty) {
 128                         unsigned sectors = (nr_dirty - m->nr_ptrs_reserved) *
 129                                         keylist_sectors(keys);
 130
 131                         /*
 132                          * can't call bch2_disk_reservation_add() with btree
 133                          * locks held, at least not without a song and dance
 134                          */
 135                         bch2_btree_iter_unlock(&iter);
 136
 137                         ret = bch2_disk_reservation_add(c, &op->res, sectors, 0);
 138                         if (ret)
 139                                 goto out;
 140
 141                         m->nr_ptrs_reserved = nr_dirty;
 142                         goto next;
 143                 }
 144
 145                 ret = bch2_mark_bkey_replicas(c, BCH_DATA_USER,
 146                                               extent_i_to_s_c(insert).s_c);
 147                 if (ret)
 148                         break;
 149
 150                 ret = bch2_btree_insert_at(c, &op->res,
 151                                 NULL, op_journal_seq(op),
 152                                 BTREE_INSERT_ATOMIC|
 153                                 BTREE_INSERT_NOFAIL|
 154                                 BTREE_INSERT_USE_RESERVE|
 155                                 m->data_opts.btree_insert_flags,
 156                                 BTREE_INSERT_ENTRY(&iter, &insert->k_i));
 157                 if (!ret)
 158                         atomic_long_inc(&c->extent_migrate_done);
 159                 if (ret == -EINTR)
 160                         ret = 0;
 161                 if (ret)
 162                         break;
 163 next:
 164                 while (bkey_cmp(iter.pos, bch2_keylist_front(keys)->k.p) >= 0) {
 165                         bch2_keylist_pop_front(keys);
 166                         if (bch2_keylist_empty(keys))
 167                                 goto out;
 168                 }
 169
 170                 bch2_cut_front(iter.pos, bch2_keylist_front(keys));
 171                 continue;
 172 nomatch:
 173                 if (m->ctxt)
 174                         atomic64_add(k.k->p.offset - iter.pos.offset,
 175                                      &m->ctxt->stats->sectors_raced);
 176                 atomic_long_inc(&c->extent_migrate_raced);
 177                 trace_move_race(&new->k);
 178                 bch2_btree_iter_next_slot(&iter);
 179                 goto next;
 180         }
 181 out:
 182         bch2_btree_iter_unlock(&iter);
 183         return ret;
 184 }
 185
 186 void bch2_migrate_read_done(struct migrate_write *m, struct bch_read_bio *rbio)
 187 {
 188         /* write bio must own pages: */
 189         BUG_ON(!m->op.wbio.bio.bi_vcnt);
 190
 191         m->ptr          = rbio->pick.ptr;
 192         m->offset       = rbio->pos.offset - rbio->pick.crc.offset;
 193         m->op.devs_have = rbio->devs_have;
 194         m->op.pos       = rbio->pos;
 195         m->op.version   = rbio->version;
 196         m->op.crc       = rbio->pick.crc;
 197         m->op.wbio.bio.bi_iter.bi_size = m->op.crc.compressed_size << 9;
 198
 199         if (bch2_csum_type_is_encryption(m->op.crc.csum_type)) {
 200                 m->op.nonce     = m->op.crc.nonce + m->op.crc.offset;
 201                 m->op.csum_type = m->op.crc.csum_type;
 202         }
 203
 204         if (m->data_cmd == DATA_REWRITE)
 205                 bch2_dev_list_drop_dev(&m->op.devs_have, m->data_opts.rewrite_dev);
 206 }
 207
 208 int bch2_migrate_write_init(struct bch_fs *c, struct migrate_write *m,
 209                             struct bch_devs_mask *devs,
 210                             struct write_point_specifier wp,
 211                             struct bch_io_opts io_opts,
 212                             enum data_cmd data_cmd,
 213                             struct data_opts data_opts,
 214                             struct bkey_s_c k)
 215 {
 216         int ret;
 217
 218         m->data_cmd     = data_cmd;
 219         m->data_opts    = data_opts;
 220         m->nr_ptrs_reserved = bch2_extent_nr_dirty_ptrs(k);
 221
 222         bch2_write_op_init(&m->op, c);
 223         m->op.csum_type = bch2_data_checksum_type(c, io_opts.data_checksum);
 224         m->op.compression_type =
 225                 bch2_compression_opt_to_type[io_opts.compression];
 226         m->op.devs      = devs;
 227         m->op.write_point = wp;
 228
 229         if (m->data_opts.btree_insert_flags & BTREE_INSERT_USE_RESERVE)
 230                 m->op.alloc_reserve = RESERVE_MOVINGGC;
 231
 232         m->op.flags |= BCH_WRITE_ONLY_SPECIFIED_DEVS|
 233                 BCH_WRITE_PAGES_STABLE|
 234                 BCH_WRITE_PAGES_OWNED|
 235                 BCH_WRITE_DATA_ENCODED|
 236                 BCH_WRITE_NOMARK_REPLICAS;
 237
 238         m->op.nr_replicas       = 1;
 239         m->op.nr_replicas_required = 1;
 240         m->op.index_update_fn   = bch2_migrate_index_update;
 241
 242         switch (data_cmd) {
 243         case DATA_ADD_REPLICAS:
 244                 if (m->nr_ptrs_reserved < c->opts.data_replicas) {
 245                         m->op.nr_replicas = c->opts.data_replicas - m->nr_ptrs_reserved;
 246
 247                         ret = bch2_disk_reservation_get(c, &m->op.res,
 248                                                         k.k->size,
 249                                                         m->op.nr_replicas, 0);
 250                         if (ret)
 251                                 return ret;
 252
 253                         m->nr_ptrs_reserved = c->opts.data_replicas;
 254                 }
 255                 break;
 256         case DATA_REWRITE:
 257                 break;
 258         case DATA_PROMOTE:
 259                 m->op.flags     |= BCH_WRITE_ALLOC_NOWAIT;
 260                 m->op.flags     |= BCH_WRITE_CACHED;
 261                 break;
 262         default:
 263                 BUG();
 264         }
 265
 266         return 0;
 267 }
 268
 269 static void move_free(struct closure *cl)
 270 {
 271         struct moving_io *io = container_of(cl, struct moving_io, cl);
 272         struct moving_context *ctxt = io->write.ctxt;
 273         struct bio_vec *bv;
 274         int i;
 275
 276         bch2_disk_reservation_put(io->write.op.c, &io->write.op.res);
 277
 278         bio_for_each_segment_all(bv, &io->write.op.wbio.bio, i)
 279                 if (bv->bv_page)
 280                         __free_page(bv->bv_page);
 281
 282         atomic_sub(io->sectors, &ctxt->sectors_in_flight);
 283         wake_up(&ctxt->wait);
 284
 285         kfree(io);
 286 }
 287
 288 static void move_write(struct closure *cl)
 289 {
 290         struct moving_io *io = container_of(cl, struct moving_io, cl);
 291
 292         if (likely(!io->rbio.bio.bi_status)) {
 293                 bch2_migrate_read_done(&io->write, &io->rbio);
 294                 closure_call(&io->write.op.cl, bch2_write, NULL, cl);
 295         }
 296
 297         closure_return_with_destructor(cl, move_free);
 298 }
 299
 300 static inline struct moving_io *next_pending_write(struct moving_context *ctxt)
 301 {
 302         struct moving_io *io =
 303                 list_first_entry_or_null(&ctxt->reads, struct moving_io, list);
 304
 305         return io && io->read_completed ? io : NULL;
 306 }
 307
 308 static void move_read_endio(struct bio *bio)
 309 {
 310         struct moving_io *io = container_of(bio, struct moving_io, rbio.bio);
 311         struct moving_context *ctxt = io->write.ctxt;
 312
 313         io->read_completed = true;
 314         if (next_pending_write(ctxt))
 315                 wake_up(&ctxt->wait);
 316
 317         closure_put(&ctxt->cl);
 318 }
 319
 320 static int bch2_move_extent(struct bch_fs *c,
 321                             struct moving_context *ctxt,
 322                             struct bch_devs_mask *devs,
 323                             struct write_point_specifier wp,
 324                             struct bch_io_opts io_opts,
 325                             struct bkey_s_c_extent e,
 326                             enum data_cmd data_cmd,
 327                             struct data_opts data_opts)
 328 {
 329         struct extent_pick_ptr pick;
 330         struct moving_io *io;
 331         const struct bch_extent_ptr *ptr;
 332         struct bch_extent_crc_unpacked crc;
 333         unsigned sectors = e.k->size, pages;
 334         int ret = -ENOMEM;
 335
 336         bch2_extent_pick_ptr(c, e.s_c, NULL, &pick);
 337         if (IS_ERR_OR_NULL(pick.ca))
 338                 return pick.ca ? PTR_ERR(pick.ca) : 0;
 339
 340         /* write path might have to decompress data: */
 341         extent_for_each_ptr_crc(e, ptr, crc)
 342                 sectors = max_t(unsigned, sectors, crc.uncompressed_size);
 343
 344         pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
 345         io = kzalloc(sizeof(struct moving_io) +
 346                      sizeof(struct bio_vec) * pages, GFP_KERNEL);
 347         if (!io)
 348                 goto err;
 349
 350         io->write.ctxt  = ctxt;
 351         io->sectors     = e.k->size;
 352
 353         bio_init(&io->write.op.wbio.bio, io->bi_inline_vecs, pages);
 354         bio_set_prio(&io->write.op.wbio.bio,
 355                      IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 356         io->write.op.wbio.bio.bi_iter.bi_size = sectors << 9;
 357
 358         bch2_bio_map(&io->write.op.wbio.bio, NULL);
 359         if (bio_alloc_pages(&io->write.op.wbio.bio, GFP_KERNEL))
 360                 goto err_free;
 361
 362         io->rbio.opts = io_opts;
 363         bio_init(&io->rbio.bio, io->bi_inline_vecs, pages);
 364         bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 365         io->rbio.bio.bi_iter.bi_size = sectors << 9;
 366
 367         bio_set_op_attrs(&io->rbio.bio, REQ_OP_READ, 0);
 368         io->rbio.bio.bi_iter.bi_sector  = bkey_start_offset(e.k);
 369         io->rbio.bio.bi_end_io          = move_read_endio;
 370
 371         ret = bch2_migrate_write_init(c, &io->write, devs, wp,
 372                                       io_opts, data_cmd, data_opts, e.s_c);
 373         if (ret)
 374                 goto err_free_pages;
 375
 376         atomic64_inc(&ctxt->stats->keys_moved);
 377         atomic64_add(e.k->size, &ctxt->stats->sectors_moved);
 378
 379         trace_move_extent(e.k);
 380
 381         atomic_add(io->sectors, &ctxt->sectors_in_flight);
 382         list_add_tail(&io->list, &ctxt->reads);
 383
 384         /*
 385          * dropped by move_read_endio() - guards against use after free of
 386          * ctxt when doing wakeup
 387          */
 388         closure_get(&ctxt->cl);
 389         bch2_read_extent(c, &io->rbio, e, &pick, BCH_READ_NODECODE);
 390         return 0;
 391 err_free_pages:
 392         bio_free_pages(&io->write.op.wbio.bio);
 393 err_free:
 394         kfree(io);
 395 err:
 396         percpu_ref_put(&pick.ca->io_ref);
 397         trace_move_alloc_fail(e.k);
 398         return ret;
 399 }
 400
 401 static void do_pending_writes(struct moving_context *ctxt)
 402 {
 403         struct moving_io *io;
 404
 405         while ((io = next_pending_write(ctxt))) {
 406                 list_del(&io->list);
 407                 closure_call(&io->cl, move_write, NULL, &ctxt->cl);
 408         }
 409 }
 410
 411 #define move_ctxt_wait_event(_ctxt, _cond)                      \
 412 do {                                                            \
 413         do_pending_writes(_ctxt);                               \
 414                                                                 \
 415         if (_cond)                                              \
 416                 break;                                          \
 417         __wait_event((_ctxt)->wait,                             \
 418                      next_pending_write(_ctxt) || (_cond));     \
 419 } while (1)
 420
 421 static void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt)
 422 {
 423         unsigned sectors_pending = atomic_read(&ctxt->sectors_in_flight);
 424
 425         move_ctxt_wait_event(ctxt,
 426                 !atomic_read(&ctxt->sectors_in_flight) ||
 427                 atomic_read(&ctxt->sectors_in_flight) != sectors_pending);
 428 }
 429
 430 int bch2_move_data(struct bch_fs *c,
 431                    struct bch_ratelimit *rate,
 432                    unsigned sectors_in_flight,
 433                    struct bch_devs_mask *devs,
 434                    struct write_point_specifier wp,
 435                    struct bpos start,
 436                    struct bpos end,
 437                    move_pred_fn pred, void *arg,
 438                    struct bch_move_stats *stats)
 439 {
 440         bool kthread = (current->flags & PF_KTHREAD) != 0;
 441         struct moving_context ctxt = { .stats = stats };
 442         struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 443         BKEY_PADDED(k) tmp;
 444         struct bkey_s_c k;
 445         struct bkey_s_c_extent e;
 446         struct data_opts data_opts;
 447         enum data_cmd data_cmd;
 448         u64 cur_inum = U64_MAX;
 449         int ret = 0, ret2;
 450
 451         closure_init_stack(&ctxt.cl);
 452         INIT_LIST_HEAD(&ctxt.reads);
 453         init_waitqueue_head(&ctxt.wait);
 454
 455         stats->data_type = BCH_DATA_USER;
 456         bch2_btree_iter_init(&stats->iter, c, BTREE_ID_EXTENTS, start,
 457                              BTREE_ITER_PREFETCH);
 458
 459         if (rate)
 460                 bch2_ratelimit_reset(rate);
 461
 462         while (!kthread || !(ret = kthread_should_stop())) {
 463                 if (atomic_read(&ctxt.sectors_in_flight) >= sectors_in_flight) {
 464                         bch2_btree_iter_unlock(&stats->iter);
 465                         move_ctxt_wait_event(&ctxt,
 466                                              atomic_read(&ctxt.sectors_in_flight) <
 467                                              sectors_in_flight);
 468                 }
 469
 470                 if (rate &&
 471                     bch2_ratelimit_delay(rate) &&
 472                     (bch2_btree_iter_unlock(&stats->iter),
 473                      (ret = bch2_ratelimit_wait_freezable_stoppable(rate))))
 474                         break;
 475 peek:
 476                 k = bch2_btree_iter_peek(&stats->iter);
 477                 if (!k.k)
 478                         break;
 479                 ret = btree_iter_err(k);
 480                 if (ret)
 481                         break;
 482                 if (bkey_cmp(bkey_start_pos(k.k), end) >= 0)
 483                         break;
 484
 485                 if (!bkey_extent_is_data(k.k))
 486                         goto next_nondata;
 487
 488                 e = bkey_s_c_to_extent(k);
 489
 490                 if (cur_inum != k.k->p.inode) {
 491                         struct bch_inode_unpacked inode;
 492
 493                         /* don't hold btree locks while looking up inode: */
 494                         bch2_btree_iter_unlock(&stats->iter);
 495
 496                         io_opts = bch2_opts_to_inode_opts(c->opts);
 497                         if (!bch2_inode_find_by_inum(c, k.k->p.inode, &inode))
 498                                 bch2_io_opts_apply(&io_opts, bch2_inode_opts_get(&inode));
 499                         cur_inum = k.k->p.inode;
 500                         goto peek;
 501                 }
 502
 503                 switch ((data_cmd = pred(c, arg, BKEY_TYPE_EXTENTS, e,
 504                                          &io_opts, &data_opts))) {
 505                 case DATA_SKIP:
 506                         goto next;
 507                 case DATA_SCRUB:
 508                         BUG();
 509                 case DATA_ADD_REPLICAS:
 510                 case DATA_REWRITE:
 511                 case DATA_PROMOTE:
 512                         break;
 513                 default:
 514                         BUG();
 515                 }
 516
 517                 /* unlock before doing IO: */
 518                 bkey_reassemble(&tmp.k, k);
 519                 k = bkey_i_to_s_c(&tmp.k);
 520                 bch2_btree_iter_unlock(&stats->iter);
 521
 522                 ret2 = bch2_move_extent(c, &ctxt, devs, wp, io_opts,
 523                                         bkey_s_c_to_extent(k),
 524                                         data_cmd, data_opts);
 525                 if (ret2) {
 526                         if (ret2 == -ENOMEM) {
 527                                 /* memory allocation failure, wait for some IO to finish */
 528                                 bch2_move_ctxt_wait_for_io(&ctxt);
 529                                 continue;
 530                         }
 531
 532                         /* XXX signal failure */
 533                         goto next;
 534                 }
 535
 536                 if (rate)
 537                         bch2_ratelimit_increment(rate, k.k->size);
 538 next:
 539                 atomic64_add(k.k->size * bch2_extent_nr_dirty_ptrs(k),
 540                              &stats->sectors_seen);
 541 next_nondata:
 542                 bch2_btree_iter_next(&stats->iter);
 543                 bch2_btree_iter_cond_resched(&stats->iter);
 544         }
 545
 546         bch2_btree_iter_unlock(&stats->iter);
 547
 548         move_ctxt_wait_event(&ctxt, !atomic_read(&ctxt.sectors_in_flight));
 549         closure_sync(&ctxt.cl);
 550
 551         EBUG_ON(!list_empty(&ctxt.reads));
 552         EBUG_ON(atomic_read(&ctxt.sectors_in_flight));
 553
 554         trace_move_data(c,
 555                         atomic64_read(&stats->sectors_moved),
 556                         atomic64_read(&stats->keys_moved));
 557
 558         return ret;
 559 }
 560
 561 static int bch2_gc_data_replicas(struct bch_fs *c)
 562 {
 563         struct btree_iter iter;
 564         struct bkey_s_c k;
 565         int ret;
 566
 567         mutex_lock(&c->replicas_gc_lock);
 568         bch2_replicas_gc_start(c, (1 << BCH_DATA_USER)|(1 << BCH_DATA_CACHED));
 569
 570         for_each_btree_key(&iter, c, BTREE_ID_EXTENTS, POS_MIN,
 571                            BTREE_ITER_PREFETCH, k) {
 572                 ret = bch2_mark_bkey_replicas(c, BCH_DATA_USER, k);
 573                 if (ret)
 574                         break;
 575         }
 576         ret = bch2_btree_iter_unlock(&iter) ?: ret;
 577
 578         bch2_replicas_gc_end(c, ret);
 579         mutex_unlock(&c->replicas_gc_lock);
 580
 581         return ret;
 582 }
 583
 584 static int bch2_gc_btree_replicas(struct bch_fs *c)
 585 {
 586         struct btree_iter iter;
 587         struct btree *b;
 588         unsigned id;
 589         int ret = 0;
 590
 591         mutex_lock(&c->replicas_gc_lock);
 592         bch2_replicas_gc_start(c, 1 << BCH_DATA_BTREE);
 593
 594         for (id = 0; id < BTREE_ID_NR; id++) {
 595                 for_each_btree_node(&iter, c, id, POS_MIN, BTREE_ITER_PREFETCH, b) {
 596                         ret = bch2_mark_bkey_replicas(c, BCH_DATA_BTREE,
 597                                                       bkey_i_to_s_c(&b->key));
 598
 599                         bch2_btree_iter_cond_resched(&iter);
 600                 }
 601
 602                 ret = bch2_btree_iter_unlock(&iter) ?: ret;
 603         }
 604
 605         bch2_replicas_gc_end(c, ret);
 606         mutex_unlock(&c->replicas_gc_lock);
 607
 608         return ret;
 609 }
 610
 611 static int bch2_move_btree(struct bch_fs *c,
 612                            move_pred_fn pred,
 613                            void *arg,
 614                            struct bch_move_stats *stats)
 615 {
 616         struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 617         struct btree *b;
 618         unsigned id;
 619         struct data_opts data_opts;
 620         enum data_cmd cmd;
 621         int ret = 0;
 622
 623         stats->data_type = BCH_DATA_BTREE;
 624
 625         for (id = 0; id < BTREE_ID_NR; id++) {
 626                 for_each_btree_node(&stats->iter, c, id, POS_MIN, BTREE_ITER_PREFETCH, b) {
 627                         switch ((cmd = pred(c, arg, BKEY_TYPE_BTREE,
 628                                             bkey_i_to_s_c_extent(&b->key),
 629                                             &io_opts,
 630                                             &data_opts))) {
 631                         case DATA_SKIP:
 632                                 goto next;
 633                         case DATA_SCRUB:
 634                                 BUG();
 635                         case DATA_ADD_REPLICAS:
 636                         case DATA_REWRITE:
 637                                 break;
 638                         default:
 639                                 BUG();
 640                         }
 641
 642                         ret = bch2_btree_node_rewrite(c, &stats->iter,
 643                                         b->data->keys.seq, 0) ?: ret;
 644 next:
 645                         bch2_btree_iter_cond_resched(&stats->iter);
 646                 }
 647
 648                 ret = bch2_btree_iter_unlock(&stats->iter) ?: ret;
 649         }
 650
 651         return ret;
 652 }
 653
 654 #if 0
 655 static enum data_cmd scrub_pred(struct bch_fs *c, void *arg,
 656                                 enum bkey_type type,
 657                                 struct bkey_s_c_extent e,
 658                                 struct bch_io_opts *io_opts,
 659                                 struct data_opts *data_opts)
 660 {
 661         return DATA_SCRUB;
 662 }
 663 #endif
 664
 665 static enum data_cmd rereplicate_pred(struct bch_fs *c, void *arg,
 666                                       enum bkey_type type,
 667                                       struct bkey_s_c_extent e,
 668                                       struct bch_io_opts *io_opts,
 669                                       struct data_opts *data_opts)
 670 {
 671         unsigned nr_good = bch2_extent_nr_good_ptrs(c, e);
 672         unsigned replicas = type == BKEY_TYPE_BTREE
 673                 ? c->opts.metadata_replicas
 674                 : c->opts.data_replicas;
 675
 676         if (!nr_good || nr_good >= replicas)
 677                 return DATA_SKIP;
 678
 679         data_opts->btree_insert_flags = 0;
 680         return DATA_ADD_REPLICAS;
 681 }
 682
 683 static enum data_cmd migrate_pred(struct bch_fs *c, void *arg,
 684                                   enum bkey_type type,
 685                                   struct bkey_s_c_extent e,
 686                                   struct bch_io_opts *io_opts,
 687                                   struct data_opts *data_opts)
 688 {
 689         struct bch_ioctl_data *op = arg;
 690
 691         if (!bch2_extent_has_device(e, op->migrate.dev))
 692                 return DATA_SKIP;
 693
 694         data_opts->btree_insert_flags   = 0;
 695         data_opts->rewrite_dev          = op->migrate.dev;
 696         return DATA_REWRITE;
 697 }
 698
 699 int bch2_data_job(struct bch_fs *c,
 700                   struct bch_move_stats *stats,
 701                   struct bch_ioctl_data op)
 702 {
 703         int ret = 0;
 704
 705         switch (op.op) {
 706         case BCH_DATA_OP_REREPLICATE:
 707                 stats->data_type = BCH_DATA_JOURNAL;
 708                 ret = bch2_journal_flush_device(&c->journal, -1);
 709
 710                 ret = bch2_move_btree(c, rereplicate_pred, c, stats) ?: ret;
 711                 ret = bch2_gc_btree_replicas(c) ?: ret;
 712
 713                 ret = bch2_move_data(c, NULL, SECTORS_IN_FLIGHT_PER_DEVICE,
 714                                      NULL,
 715                                      writepoint_hashed((unsigned long) current),
 716                                      op.start,
 717                                      op.end,
 718                                      rereplicate_pred, c, stats) ?: ret;
 719                 ret = bch2_gc_data_replicas(c) ?: ret;
 720                 break;
 721         case BCH_DATA_OP_MIGRATE:
 722                 if (op.migrate.dev >= c->sb.nr_devices)
 723                         return -EINVAL;
 724
 725                 stats->data_type = BCH_DATA_JOURNAL;
 726                 ret = bch2_journal_flush_device(&c->journal, op.migrate.dev);
 727
 728                 ret = bch2_move_btree(c, migrate_pred, &op, stats) ?: ret;
 729                 ret = bch2_gc_btree_replicas(c) ?: ret;
 730
 731                 ret = bch2_move_data(c, NULL, SECTORS_IN_FLIGHT_PER_DEVICE,
 732                                      NULL,
 733                                      writepoint_hashed((unsigned long) current),
 734                                      op.start,
 735                                      op.end,
 736                                      migrate_pred, &op, stats) ?: ret;
 737                 ret = bch2_gc_data_replicas(c) ?: ret;
 738                 break;
 739         default:
 740                 ret = -EINVAL;
 741         }
 742
 743         return ret;
 744 }