git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/move.c

   1
   2 #include "bcachefs.h"
   3 #include "btree_gc.h"
   4 #include "btree_update.h"
   5 #include "buckets.h"
   6 #include "inode.h"
   7 #include "io.h"
   8 #include "move.h"
   9 #include "super-io.h"
  10 #include "keylist.h"
  11
  12 #include <linux/ioprio.h>
  13 #include <linux/kthread.h>
  14
  15 #include <trace/events/bcachefs.h>
  16
  17 struct moving_io {
  18         struct list_head        list;
  19         struct closure          cl;
  20         bool                    read_completed;
  21         unsigned                sectors;
  22
  23         struct bch_read_bio     rbio;
  24
  25         struct migrate_write    write;
  26         /* Must be last since it is variable size */
  27         struct bio_vec          bi_inline_vecs[0];
  28 };
  29
  30 struct moving_context {
  31         /* Closure for waiting on all reads and writes to complete */
  32         struct closure          cl;
  33
  34         struct bch_move_stats   *stats;
  35
  36         struct list_head        reads;
  37         atomic_t                sectors_in_flight;
  38         wait_queue_head_t       wait;
  39 };
  40
  41 static int bch2_migrate_index_update(struct bch_write_op *op)
  42 {
  43         struct bch_fs *c = op->c;
  44         struct migrate_write *m =
  45                 container_of(op, struct migrate_write, op);
  46         struct keylist *keys = &op->insert_keys;
  47         struct btree_iter iter;
  48         int ret = 0;
  49
  50         bch2_btree_iter_init(&iter, c, BTREE_ID_EXTENTS,
  51                              bkey_start_pos(&bch2_keylist_front(keys)->k),
  52                              BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
  53
  54         while (1) {
  55                 struct bkey_s_c k = bch2_btree_iter_peek_slot(&iter);
  56                 struct bkey_i_extent *insert, *new =
  57                         bkey_i_to_extent(bch2_keylist_front(keys));
  58                 BKEY_PADDED(k) _new, _insert;
  59                 struct bch_extent_ptr *ptr;
  60                 struct bch_extent_crc_unpacked crc;
  61                 bool did_work = false;
  62
  63                 if (btree_iter_err(k)) {
  64                         ret = bch2_btree_iter_unlock(&iter);
  65                         break;
  66                 }
  67
  68                 if (bversion_cmp(k.k->version, new->k.version) ||
  69                     !bkey_extent_is_data(k.k) ||
  70                     !bch2_extent_matches_ptr(c, bkey_s_c_to_extent(k),
  71                                              m->ptr, m->offset))
  72                         goto nomatch;
  73
  74                 bkey_reassemble(&_insert.k, k);
  75                 insert = bkey_i_to_extent(&_insert.k);
  76
  77                 bkey_copy(&_new.k, bch2_keylist_front(keys));
  78                 new = bkey_i_to_extent(&_new.k);
  79
  80                 bch2_cut_front(iter.pos, &insert->k_i);
  81                 bch2_cut_back(new->k.p, &insert->k);
  82                 bch2_cut_back(insert->k.p, &new->k);
  83
  84                 if (m->move_dev >= 0 &&
  85                     (ptr = (struct bch_extent_ptr *)
  86                      bch2_extent_has_device(extent_i_to_s_c(insert),
  87                                             m->move_dev)))
  88                         bch2_extent_drop_ptr(extent_i_to_s(insert), ptr);
  89
  90                 extent_for_each_ptr_crc(extent_i_to_s(new), ptr, crc) {
  91                         if (bch2_extent_has_device(extent_i_to_s_c(insert), ptr->dev)) {
  92                                 /*
  93                                  * raced with another move op? extent already
  94                                  * has a pointer to the device we just wrote
  95                                  * data to
  96                                  */
  97                                 continue;
  98                         }
  99
 100                         bch2_extent_crc_append(insert, crc);
 101                         extent_ptr_append(insert, *ptr);
 102                         did_work = true;
 103                 }
 104
 105                 if (!did_work)
 106                         goto nomatch;
 107
 108                 bch2_extent_narrow_crcs(insert,
 109                                 (struct bch_extent_crc_unpacked) { 0 });
 110                 bch2_extent_normalize(c, extent_i_to_s(insert).s);
 111                 bch2_extent_mark_replicas_cached(c, extent_i_to_s(insert));
 112
 113                 ret = bch2_check_mark_super(c, BCH_DATA_USER,
 114                                 bch2_extent_devs(extent_i_to_s_c(insert)));
 115                 if (ret)
 116                         break;
 117
 118                 ret = bch2_btree_insert_at(c, &op->res,
 119                                 NULL, op_journal_seq(op),
 120                                 BTREE_INSERT_ATOMIC|
 121                                 BTREE_INSERT_NOFAIL|
 122                                 m->btree_insert_flags,
 123                                 BTREE_INSERT_ENTRY(&iter, &insert->k_i));
 124                 if (!ret)
 125                         atomic_long_inc(&c->extent_migrate_done);
 126                 if (ret == -EINTR)
 127                         ret = 0;
 128                 if (ret)
 129                         break;
 130 next:
 131                 while (bkey_cmp(iter.pos, bch2_keylist_front(keys)->k.p) >= 0) {
 132                         bch2_keylist_pop_front(keys);
 133                         if (bch2_keylist_empty(keys))
 134                                 goto out;
 135                 }
 136
 137                 bch2_cut_front(iter.pos, bch2_keylist_front(keys));
 138                 continue;
 139 nomatch:
 140                 if (m->ctxt)
 141                         atomic64_add(k.k->p.offset - iter.pos.offset,
 142                                      &m->ctxt->stats->sectors_raced);
 143                 atomic_long_inc(&c->extent_migrate_raced);
 144                 trace_move_race(&new->k);
 145                 bch2_btree_iter_next_slot(&iter);
 146                 goto next;
 147         }
 148 out:
 149         bch2_btree_iter_unlock(&iter);
 150         return ret;
 151 }
 152
 153 void bch2_migrate_write_init(struct migrate_write *m,
 154                              struct bch_read_bio *rbio)
 155 {
 156         /* write bio must own pages: */
 157         BUG_ON(!m->op.wbio.bio.bi_vcnt);
 158
 159         m->ptr          = rbio->pick.ptr;
 160         m->offset       = rbio->pos.offset - rbio->pick.crc.offset;
 161         m->op.devs_have = rbio->devs_have;
 162         m->op.pos       = rbio->pos;
 163         m->op.version   = rbio->version;
 164         m->op.crc       = rbio->pick.crc;
 165
 166         if (bch2_csum_type_is_encryption(m->op.crc.csum_type)) {
 167                 m->op.nonce     = m->op.crc.nonce + m->op.crc.offset;
 168                 m->op.csum_type = m->op.crc.csum_type;
 169         }
 170
 171         if (m->move_dev >= 0)
 172                 bch2_dev_list_drop_dev(&m->op.devs_have, m->move_dev);
 173
 174         if (m->btree_insert_flags & BTREE_INSERT_USE_RESERVE)
 175                 m->op.alloc_reserve = RESERVE_MOVINGGC;
 176
 177         m->op.flags |= BCH_WRITE_ONLY_SPECIFIED_DEVS|
 178                 BCH_WRITE_PAGES_STABLE|
 179                 BCH_WRITE_PAGES_OWNED|
 180                 BCH_WRITE_DATA_ENCODED|
 181                 BCH_WRITE_NOMARK_REPLICAS;
 182
 183         m->op.wbio.bio.bi_iter.bi_size = m->op.crc.compressed_size << 9;
 184         m->op.nr_replicas       = 1;
 185         m->op.nr_replicas_required = 1;
 186         m->op.index_update_fn   = bch2_migrate_index_update;
 187 }
 188
 189 static void move_free(struct closure *cl)
 190 {
 191         struct moving_io *io = container_of(cl, struct moving_io, cl);
 192         struct moving_context *ctxt = io->write.ctxt;
 193         struct bio_vec *bv;
 194         int i;
 195
 196         bch2_disk_reservation_put(io->write.op.c, &io->write.op.res);
 197
 198         bio_for_each_segment_all(bv, &io->write.op.wbio.bio, i)
 199                 if (bv->bv_page)
 200                         __free_page(bv->bv_page);
 201
 202         atomic_sub(io->sectors, &ctxt->sectors_in_flight);
 203         wake_up(&ctxt->wait);
 204
 205         kfree(io);
 206 }
 207
 208 static void move_write(struct closure *cl)
 209 {
 210         struct moving_io *io = container_of(cl, struct moving_io, cl);
 211
 212         if (likely(!io->rbio.bio.bi_status)) {
 213                 bch2_migrate_write_init(&io->write, &io->rbio);
 214                 closure_call(&io->write.op.cl, bch2_write, NULL, cl);
 215         }
 216
 217         closure_return_with_destructor(cl, move_free);
 218 }
 219
 220 static inline struct moving_io *next_pending_write(struct moving_context *ctxt)
 221 {
 222         struct moving_io *io =
 223                 list_first_entry_or_null(&ctxt->reads, struct moving_io, list);
 224
 225         return io && io->read_completed ? io : NULL;
 226 }
 227
 228 static void move_read_endio(struct bio *bio)
 229 {
 230         struct moving_io *io = container_of(bio, struct moving_io, rbio.bio);
 231         struct moving_context *ctxt = io->write.ctxt;
 232
 233         io->read_completed = true;
 234         if (next_pending_write(ctxt))
 235                 wake_up(&ctxt->wait);
 236
 237         closure_put(&ctxt->cl);
 238 }
 239
 240 static int bch2_move_extent(struct bch_fs *c,
 241                           struct moving_context *ctxt,
 242                           struct bch_devs_mask *devs,
 243                           struct write_point_specifier wp,
 244                           int btree_insert_flags,
 245                           int move_device,
 246                           struct bch_io_opts opts,
 247                           struct bkey_s_c_extent e)
 248 {
 249         struct extent_pick_ptr pick;
 250         struct moving_io *io;
 251         const struct bch_extent_ptr *ptr;
 252         struct bch_extent_crc_unpacked crc;
 253         unsigned sectors = e.k->size, pages, nr_good;
 254         int ret = -ENOMEM;
 255
 256         bch2_extent_pick_ptr(c, e.s_c, NULL, &pick);
 257         if (IS_ERR_OR_NULL(pick.ca))
 258                 return pick.ca ? PTR_ERR(pick.ca) : 0;
 259
 260         /* write path might have to decompress data: */
 261         extent_for_each_ptr_crc(e, ptr, crc)
 262                 sectors = max_t(unsigned, sectors, crc.uncompressed_size);
 263
 264         pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
 265         io = kzalloc(sizeof(struct moving_io) +
 266                      sizeof(struct bio_vec) * pages, GFP_KERNEL);
 267         if (!io)
 268                 goto err;
 269
 270         io->write.ctxt  = ctxt;
 271         io->sectors     = e.k->size;
 272
 273         bio_init(&io->write.op.wbio.bio, io->bi_inline_vecs, pages);
 274         bio_set_prio(&io->write.op.wbio.bio,
 275                      IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 276         io->write.op.wbio.bio.bi_iter.bi_size = sectors << 9;
 277
 278         bch2_bio_map(&io->write.op.wbio.bio, NULL);
 279         if (bio_alloc_pages(&io->write.op.wbio.bio, GFP_KERNEL))
 280                 goto err_free;
 281
 282         io->rbio.opts = opts;
 283         bio_init(&io->rbio.bio, io->bi_inline_vecs, pages);
 284         bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 285         io->rbio.bio.bi_iter.bi_size = sectors << 9;
 286
 287         bio_set_op_attrs(&io->rbio.bio, REQ_OP_READ, 0);
 288         io->rbio.bio.bi_iter.bi_sector  = bkey_start_offset(e.k);
 289         io->rbio.bio.bi_end_io          = move_read_endio;
 290
 291         io->write.btree_insert_flags = btree_insert_flags;
 292         io->write.move_dev      = move_device;
 293
 294         bch2_write_op_init(&io->write.op, c);
 295         io->write.op.csum_type = bch2_data_checksum_type(c, opts.data_checksum);
 296         io->write.op.compression_type =
 297                 bch2_compression_opt_to_type(opts.compression);
 298         io->write.op.devs       = devs;
 299         io->write.op.write_point = wp;
 300
 301         if (move_device < 0 &&
 302             ((nr_good = bch2_extent_nr_good_ptrs(c, e)) <
 303              c->opts.data_replicas)) {
 304                 io->write.op.nr_replicas = c->opts.data_replicas - nr_good;
 305
 306                 ret = bch2_disk_reservation_get(c, &io->write.op.res,
 307                                                 e.k->size,
 308                                                 io->write.op.nr_replicas, 0);
 309                 if (ret)
 310                         goto err_free_pages;
 311         }
 312
 313         atomic64_inc(&ctxt->stats->keys_moved);
 314         atomic64_add(e.k->size, &ctxt->stats->sectors_moved);
 315
 316         trace_move_extent(e.k);
 317
 318         atomic_add(io->sectors, &ctxt->sectors_in_flight);
 319         list_add_tail(&io->list, &ctxt->reads);
 320
 321         /*
 322          * dropped by move_read_endio() - guards against use after free of
 323          * ctxt when doing wakeup
 324          */
 325         closure_get(&ctxt->cl);
 326         bch2_read_extent(c, &io->rbio, e, &pick, BCH_READ_NODECODE);
 327         return 0;
 328 err_free_pages:
 329         bio_free_pages(&io->write.op.wbio.bio);
 330 err_free:
 331         kfree(io);
 332 err:
 333         percpu_ref_put(&pick.ca->io_ref);
 334         trace_move_alloc_fail(e.k);
 335         return ret;
 336 }
 337
 338 static void do_pending_writes(struct moving_context *ctxt)
 339 {
 340         struct moving_io *io;
 341
 342         while ((io = next_pending_write(ctxt))) {
 343                 list_del(&io->list);
 344                 closure_call(&io->cl, move_write, NULL, &ctxt->cl);
 345         }
 346 }
 347
 348 #define move_ctxt_wait_event(_ctxt, _cond)                      \
 349 do {                                                            \
 350         do_pending_writes(_ctxt);                               \
 351                                                                 \
 352         if (_cond)                                              \
 353                 break;                                          \
 354         __wait_event((_ctxt)->wait,                             \
 355                      next_pending_write(_ctxt) || (_cond));     \
 356 } while (1)
 357
 358 static void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt)
 359 {
 360         unsigned sectors_pending = atomic_read(&ctxt->sectors_in_flight);
 361
 362         move_ctxt_wait_event(ctxt,
 363                 !atomic_read(&ctxt->sectors_in_flight) ||
 364                 atomic_read(&ctxt->sectors_in_flight) != sectors_pending);
 365 }
 366
 367 int bch2_move_data(struct bch_fs *c,
 368                    struct bch_ratelimit *rate,
 369                    unsigned sectors_in_flight,
 370                    struct bch_devs_mask *devs,
 371                    struct write_point_specifier wp,
 372                    int btree_insert_flags,
 373                    int move_device,
 374                    struct bpos start,
 375                    struct bpos end,
 376                    move_pred_fn pred, void *arg,
 377                    struct bch_move_stats *stats)
 378 {
 379         bool kthread = (current->flags & PF_KTHREAD) != 0;
 380         struct moving_context ctxt = { .stats = stats };
 381         struct bch_io_opts opts = bch2_opts_to_inode_opts(c->opts);
 382         BKEY_PADDED(k) tmp;
 383         struct bkey_s_c k;
 384         struct bkey_s_c_extent e;
 385         u64 cur_inum = U64_MAX;
 386         int ret = 0;
 387
 388         closure_init_stack(&ctxt.cl);
 389         INIT_LIST_HEAD(&ctxt.reads);
 390         init_waitqueue_head(&ctxt.wait);
 391
 392         stats->data_type = BCH_DATA_USER;
 393         bch2_btree_iter_init(&stats->iter, c, BTREE_ID_EXTENTS, start,
 394                              BTREE_ITER_PREFETCH);
 395
 396         if (rate)
 397                 bch2_ratelimit_reset(rate);
 398
 399         while (!kthread || !(ret = kthread_should_stop())) {
 400                 if (atomic_read(&ctxt.sectors_in_flight) >= sectors_in_flight) {
 401                         bch2_btree_iter_unlock(&stats->iter);
 402                         move_ctxt_wait_event(&ctxt,
 403                                              atomic_read(&ctxt.sectors_in_flight) <
 404                                              sectors_in_flight);
 405                 }
 406
 407                 if (rate &&
 408                     bch2_ratelimit_delay(rate) &&
 409                     (bch2_btree_iter_unlock(&stats->iter),
 410                      (ret = bch2_ratelimit_wait_freezable_stoppable(rate))))
 411                         break;
 412 peek:
 413                 k = bch2_btree_iter_peek(&stats->iter);
 414                 if (!k.k)
 415                         break;
 416                 ret = btree_iter_err(k);
 417                 if (ret)
 418                         break;
 419                 if (bkey_cmp(bkey_start_pos(k.k), end) >= 0)
 420                         break;
 421
 422                 if (!bkey_extent_is_data(k.k))
 423                         goto next_nondata;
 424
 425                 e = bkey_s_c_to_extent(k);
 426
 427                 if (cur_inum != k.k->p.inode) {
 428                         struct bch_inode_unpacked inode;
 429
 430                         /* don't hold btree locks while looking up inode: */
 431                         bch2_btree_iter_unlock(&stats->iter);
 432
 433                         opts = bch2_opts_to_inode_opts(c->opts);
 434                         if (!bch2_inode_find_by_inum(c, k.k->p.inode, &inode))
 435                                 bch2_io_opts_apply(&opts, bch2_inode_opts_get(&inode));
 436                         cur_inum = k.k->p.inode;
 437                         goto peek;
 438                 }
 439
 440                 if (!pred(arg, e))
 441                         goto next;
 442
 443                 /* unlock before doing IO: */
 444                 bkey_reassemble(&tmp.k, k);
 445                 k = bkey_i_to_s_c(&tmp.k);
 446                 bch2_btree_iter_unlock(&stats->iter);
 447
 448                 if (bch2_move_extent(c, &ctxt, devs, wp,
 449                                      btree_insert_flags,
 450                                      move_device, opts,
 451                                      bkey_s_c_to_extent(k))) {
 452                         /* memory allocation failure, wait for some IO to finish */
 453                         bch2_move_ctxt_wait_for_io(&ctxt);
 454                         continue;
 455                 }
 456
 457                 if (rate)
 458                         bch2_ratelimit_increment(rate, k.k->size);
 459 next:
 460                 atomic64_add(k.k->size * bch2_extent_nr_dirty_ptrs(k),
 461                              &stats->sectors_seen);
 462 next_nondata:
 463                 bch2_btree_iter_next(&stats->iter);
 464                 bch2_btree_iter_cond_resched(&stats->iter);
 465         }
 466
 467         bch2_btree_iter_unlock(&stats->iter);
 468
 469         move_ctxt_wait_event(&ctxt, !atomic_read(&ctxt.sectors_in_flight));
 470         closure_sync(&ctxt.cl);
 471
 472         EBUG_ON(!list_empty(&ctxt.reads));
 473         EBUG_ON(atomic_read(&ctxt.sectors_in_flight));
 474
 475         trace_move_data(c,
 476                         atomic64_read(&stats->sectors_moved),
 477                         atomic64_read(&stats->keys_moved));
 478
 479         return ret;
 480 }
 481
 482 static int bch2_gc_data_replicas(struct bch_fs *c)
 483 {
 484         struct btree_iter iter;
 485         struct bkey_s_c k;
 486         int ret;
 487
 488         mutex_lock(&c->replicas_gc_lock);
 489         bch2_replicas_gc_start(c, 1 << BCH_DATA_USER);
 490
 491         for_each_btree_key(&iter, c, BTREE_ID_EXTENTS, POS_MIN,
 492                            BTREE_ITER_PREFETCH, k) {
 493                 ret = bch2_check_mark_super(c, BCH_DATA_USER, bch2_bkey_devs(k));
 494                 if (ret)
 495                         break;
 496         }
 497         ret = bch2_btree_iter_unlock(&iter) ?: ret;
 498
 499         bch2_replicas_gc_end(c, ret);
 500         mutex_unlock(&c->replicas_gc_lock);
 501
 502         return ret;
 503 }
 504
 505 static int bch2_gc_btree_replicas(struct bch_fs *c)
 506 {
 507         struct btree_iter iter;
 508         struct btree *b;
 509         unsigned id;
 510         int ret = 0;
 511
 512         mutex_lock(&c->replicas_gc_lock);
 513         bch2_replicas_gc_start(c, 1 << BCH_DATA_BTREE);
 514
 515         for (id = 0; id < BTREE_ID_NR; id++) {
 516                 for_each_btree_node(&iter, c, id, POS_MIN, BTREE_ITER_PREFETCH, b) {
 517                         ret = bch2_check_mark_super(c, BCH_DATA_BTREE,
 518                                         bch2_bkey_devs(bkey_i_to_s_c(&b->key)));
 519
 520                         bch2_btree_iter_cond_resched(&iter);
 521                 }
 522
 523                 ret = bch2_btree_iter_unlock(&iter) ?: ret;
 524         }
 525
 526         bch2_replicas_gc_end(c, ret);
 527         mutex_unlock(&c->replicas_gc_lock);
 528
 529         return ret;
 530 }
 531
 532 static int bch2_move_btree(struct bch_fs *c,
 533                            move_pred_fn pred,
 534                            void *arg,
 535                            struct bch_move_stats *stats)
 536 {
 537         struct btree *b;
 538         unsigned id;
 539         int ret = 0;
 540
 541         stats->data_type = BCH_DATA_BTREE;
 542
 543         for (id = 0; id < BTREE_ID_NR; id++) {
 544                 for_each_btree_node(&stats->iter, c, id, POS_MIN, BTREE_ITER_PREFETCH, b) {
 545                         if (pred(arg, bkey_i_to_s_c_extent(&b->key)))
 546                                 ret = bch2_btree_node_rewrite(c, &stats->iter,
 547                                                 b->data->keys.seq, 0) ?: ret;
 548
 549                         bch2_btree_iter_cond_resched(&stats->iter);
 550                 }
 551
 552                 ret = bch2_btree_iter_unlock(&stats->iter) ?: ret;
 553         }
 554
 555         return ret;
 556 }
 557
 558 #if 0
 559 static bool scrub_data_pred(void *arg, struct bkey_s_c_extent e)
 560 {
 561 }
 562 #endif
 563
 564 static bool rereplicate_metadata_pred(void *arg, struct bkey_s_c_extent e)
 565 {
 566         struct bch_fs *c = arg;
 567         unsigned nr_good = bch2_extent_nr_good_ptrs(c, e);
 568
 569         return nr_good && nr_good < c->opts.metadata_replicas;
 570 }
 571
 572 static bool rereplicate_data_pred(void *arg, struct bkey_s_c_extent e)
 573 {
 574         struct bch_fs *c = arg;
 575         unsigned nr_good = bch2_extent_nr_good_ptrs(c, e);
 576
 577         return nr_good && nr_good < c->opts.data_replicas;
 578 }
 579
 580 static bool migrate_pred(void *arg, struct bkey_s_c_extent e)
 581 {
 582         struct bch_ioctl_data *op = arg;
 583
 584         return bch2_extent_has_device(e, op->migrate.dev);
 585 }
 586
 587 int bch2_data_job(struct bch_fs *c,
 588                   struct bch_move_stats *stats,
 589                   struct bch_ioctl_data op)
 590 {
 591         int ret = 0;
 592
 593         switch (op.op) {
 594         case BCH_DATA_OP_REREPLICATE:
 595                 stats->data_type = BCH_DATA_JOURNAL;
 596                 ret = bch2_journal_flush_device(&c->journal, -1);
 597
 598                 ret = bch2_move_btree(c, rereplicate_metadata_pred, c, stats) ?: ret;
 599                 ret = bch2_gc_btree_replicas(c) ?: ret;
 600
 601                 ret = bch2_move_data(c, NULL, SECTORS_IN_FLIGHT_PER_DEVICE,
 602                                      NULL,
 603                                      writepoint_hashed((unsigned long) current),
 604                                      0, -1,
 605                                      op.start,
 606                                      op.end,
 607                                      rereplicate_data_pred, c, stats) ?: ret;
 608                 ret = bch2_gc_data_replicas(c) ?: ret;
 609                 break;
 610         case BCH_DATA_OP_MIGRATE:
 611                 if (op.migrate.dev >= c->sb.nr_devices)
 612                         return -EINVAL;
 613
 614                 stats->data_type = BCH_DATA_JOURNAL;
 615                 ret = bch2_journal_flush_device(&c->journal, op.migrate.dev);
 616
 617                 ret = bch2_move_btree(c, migrate_pred, &op, stats) ?: ret;
 618                 ret = bch2_gc_btree_replicas(c) ?: ret;
 619
 620                 ret = bch2_move_data(c, NULL, SECTORS_IN_FLIGHT_PER_DEVICE,
 621                                      NULL,
 622                                      writepoint_hashed((unsigned long) current),
 623                                      0, -1,
 624                                      op.start,
 625                                      op.end,
 626                                      migrate_pred, &op, stats) ?: ret;
 627                 ret = bch2_gc_data_replicas(c) ?: ret;
 628                 break;
 629         default:
 630                 ret = -EINVAL;
 631         }
 632
 633         return ret;
 634 }