git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/move.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "alloc_background.h"
   5 #include "alloc_foreground.h"
   6 #include "backpointers.h"
   7 #include "bkey_buf.h"
   8 #include "btree_gc.h"
   9 #include "btree_update.h"
  10 #include "btree_update_interior.h"
  11 #include "btree_write_buffer.h"
  12 #include "disk_groups.h"
  13 #include "ec.h"
  14 #include "errcode.h"
  15 #include "error.h"
  16 #include "inode.h"
  17 #include "io_read.h"
  18 #include "io_write.h"
  19 #include "journal_reclaim.h"
  20 #include "keylist.h"
  21 #include "move.h"
  22 #include "replicas.h"
  23 #include "snapshot.h"
  24 #include "super-io.h"
  25 #include "trace.h"
  26
  27 #include <linux/ioprio.h>
  28 #include <linux/kthread.h>
  29
  30 const char * const bch2_data_ops_strs[] = {
  31 #define x(t, n, ...) [n] = #t,
  32         BCH_DATA_OPS()
  33 #undef x
  34         NULL
  35 };
  36
  37 static void trace_move_extent2(struct bch_fs *c, struct bkey_s_c k)
  38 {
  39         if (trace_move_extent_enabled()) {
  40                 struct printbuf buf = PRINTBUF;
  41
  42                 bch2_bkey_val_to_text(&buf, c, k);
  43                 trace_move_extent(c, buf.buf);
  44                 printbuf_exit(&buf);
  45         }
  46 }
  47
  48 static void trace_move_extent_read2(struct bch_fs *c, struct bkey_s_c k)
  49 {
  50         if (trace_move_extent_read_enabled()) {
  51                 struct printbuf buf = PRINTBUF;
  52
  53                 bch2_bkey_val_to_text(&buf, c, k);
  54                 trace_move_extent_read(c, buf.buf);
  55                 printbuf_exit(&buf);
  56         }
  57 }
  58
  59 static void trace_move_extent_alloc_mem_fail2(struct bch_fs *c, struct bkey_s_c k)
  60 {
  61         if (trace_move_extent_alloc_mem_fail_enabled()) {
  62                 struct printbuf buf = PRINTBUF;
  63
  64                 bch2_bkey_val_to_text(&buf, c, k);
  65                 trace_move_extent_alloc_mem_fail(c, buf.buf);
  66                 printbuf_exit(&buf);
  67         }
  68 }
  69
  70 struct moving_io {
  71         struct list_head                read_list;
  72         struct list_head                io_list;
  73         struct move_bucket_in_flight    *b;
  74         struct closure                  cl;
  75         bool                            read_completed;
  76
  77         unsigned                        read_sectors;
  78         unsigned                        write_sectors;
  79
  80         struct bch_read_bio             rbio;
  81
  82         struct data_update              write;
  83         /* Must be last since it is variable size */
  84         struct bio_vec                  bi_inline_vecs[0];
  85 };
  86
  87 static void move_free(struct moving_io *io)
  88 {
  89         struct moving_context *ctxt = io->write.ctxt;
  90
  91         if (io->b)
  92                 atomic_dec(&io->b->count);
  93
  94         bch2_data_update_exit(&io->write);
  95
  96         mutex_lock(&ctxt->lock);
  97         list_del(&io->io_list);
  98         wake_up(&ctxt->wait);
  99         mutex_unlock(&ctxt->lock);
 100
 101         kfree(io);
 102 }
 103
 104 static void move_write_done(struct bch_write_op *op)
 105 {
 106         struct moving_io *io = container_of(op, struct moving_io, write.op);
 107         struct moving_context *ctxt = io->write.ctxt;
 108
 109         if (io->write.op.error)
 110                 ctxt->write_error = true;
 111
 112         atomic_sub(io->write_sectors, &io->write.ctxt->write_sectors);
 113         atomic_dec(&io->write.ctxt->write_ios);
 114         move_free(io);
 115         closure_put(&ctxt->cl);
 116 }
 117
 118 static void move_write(struct moving_io *io)
 119 {
 120         if (unlikely(io->rbio.bio.bi_status || io->rbio.hole)) {
 121                 move_free(io);
 122                 return;
 123         }
 124
 125         closure_get(&io->write.ctxt->cl);
 126         atomic_add(io->write_sectors, &io->write.ctxt->write_sectors);
 127         atomic_inc(&io->write.ctxt->write_ios);
 128
 129         bch2_data_update_read_done(&io->write, io->rbio.pick.crc);
 130 }
 131
 132 struct moving_io *bch2_moving_ctxt_next_pending_write(struct moving_context *ctxt)
 133 {
 134         struct moving_io *io =
 135                 list_first_entry_or_null(&ctxt->reads, struct moving_io, read_list);
 136
 137         return io && io->read_completed ? io : NULL;
 138 }
 139
 140 static void move_read_endio(struct bio *bio)
 141 {
 142         struct moving_io *io = container_of(bio, struct moving_io, rbio.bio);
 143         struct moving_context *ctxt = io->write.ctxt;
 144
 145         atomic_sub(io->read_sectors, &ctxt->read_sectors);
 146         atomic_dec(&ctxt->read_ios);
 147         io->read_completed = true;
 148
 149         wake_up(&ctxt->wait);
 150         closure_put(&ctxt->cl);
 151 }
 152
 153 void bch2_moving_ctxt_do_pending_writes(struct moving_context *ctxt)
 154 {
 155         struct moving_io *io;
 156
 157         while ((io = bch2_moving_ctxt_next_pending_write(ctxt))) {
 158                 bch2_trans_unlock_long(ctxt->trans);
 159                 list_del(&io->read_list);
 160                 move_write(io);
 161         }
 162 }
 163
 164 void bch2_move_ctxt_wait_for_io(struct moving_context *ctxt)
 165 {
 166         unsigned sectors_pending = atomic_read(&ctxt->write_sectors);
 167
 168         move_ctxt_wait_event(ctxt,
 169                 !atomic_read(&ctxt->write_sectors) ||
 170                 atomic_read(&ctxt->write_sectors) != sectors_pending);
 171 }
 172
 173 static void bch2_moving_ctxt_flush_all(struct moving_context *ctxt)
 174 {
 175         move_ctxt_wait_event(ctxt, list_empty(&ctxt->reads));
 176         closure_sync(&ctxt->cl);
 177 }
 178
 179 void bch2_moving_ctxt_exit(struct moving_context *ctxt)
 180 {
 181         struct bch_fs *c = ctxt->trans->c;
 182
 183         bch2_moving_ctxt_flush_all(ctxt);
 184
 185         EBUG_ON(atomic_read(&ctxt->write_sectors));
 186         EBUG_ON(atomic_read(&ctxt->write_ios));
 187         EBUG_ON(atomic_read(&ctxt->read_sectors));
 188         EBUG_ON(atomic_read(&ctxt->read_ios));
 189
 190         mutex_lock(&c->moving_context_lock);
 191         list_del(&ctxt->list);
 192         mutex_unlock(&c->moving_context_lock);
 193
 194         bch2_trans_put(ctxt->trans);
 195         memset(ctxt, 0, sizeof(*ctxt));
 196 }
 197
 198 void bch2_moving_ctxt_init(struct moving_context *ctxt,
 199                            struct bch_fs *c,
 200                            struct bch_ratelimit *rate,
 201                            struct bch_move_stats *stats,
 202                            struct write_point_specifier wp,
 203                            bool wait_on_copygc)
 204 {
 205         memset(ctxt, 0, sizeof(*ctxt));
 206
 207         ctxt->trans     = bch2_trans_get(c);
 208         ctxt->fn        = (void *) _RET_IP_;
 209         ctxt->rate      = rate;
 210         ctxt->stats     = stats;
 211         ctxt->wp        = wp;
 212         ctxt->wait_on_copygc = wait_on_copygc;
 213
 214         closure_init_stack(&ctxt->cl);
 215
 216         mutex_init(&ctxt->lock);
 217         INIT_LIST_HEAD(&ctxt->reads);
 218         INIT_LIST_HEAD(&ctxt->ios);
 219         init_waitqueue_head(&ctxt->wait);
 220
 221         mutex_lock(&c->moving_context_lock);
 222         list_add(&ctxt->list, &c->moving_context_list);
 223         mutex_unlock(&c->moving_context_lock);
 224 }
 225
 226 void bch2_move_stats_exit(struct bch_move_stats *stats, struct bch_fs *c)
 227 {
 228         trace_move_data(c, stats);
 229 }
 230
 231 void bch2_move_stats_init(struct bch_move_stats *stats, const char *name)
 232 {
 233         memset(stats, 0, sizeof(*stats));
 234         stats->data_type = BCH_DATA_user;
 235         scnprintf(stats->name, sizeof(stats->name), "%s", name);
 236 }
 237
 238 static int bch2_extent_drop_ptrs(struct btree_trans *trans,
 239                                  struct btree_iter *iter,
 240                                  struct bkey_s_c k,
 241                                  struct data_update_opts data_opts)
 242 {
 243         struct bch_fs *c = trans->c;
 244         struct bkey_i *n;
 245         int ret;
 246
 247         n = bch2_bkey_make_mut_noupdate(trans, k);
 248         ret = PTR_ERR_OR_ZERO(n);
 249         if (ret)
 250                 return ret;
 251
 252         while (data_opts.kill_ptrs) {
 253                 unsigned i = 0, drop = __fls(data_opts.kill_ptrs);
 254                 struct bch_extent_ptr *ptr;
 255
 256                 bch2_bkey_drop_ptrs(bkey_i_to_s(n), ptr, i++ == drop);
 257                 data_opts.kill_ptrs ^= 1U << drop;
 258         }
 259
 260         /*
 261          * If the new extent no longer has any pointers, bch2_extent_normalize()
 262          * will do the appropriate thing with it (turning it into a
 263          * KEY_TYPE_error key, or just a discard if it was a cached extent)
 264          */
 265         bch2_extent_normalize(c, bkey_i_to_s(n));
 266
 267         /*
 268          * Since we're not inserting through an extent iterator
 269          * (BTREE_ITER_ALL_SNAPSHOTS iterators aren't extent iterators),
 270          * we aren't using the extent overwrite path to delete, we're
 271          * just using the normal key deletion path:
 272          */
 273         if (bkey_deleted(&n->k))
 274                 n->k.size = 0;
 275
 276         return bch2_trans_relock(trans) ?:
 277                 bch2_trans_update(trans, iter, n, BTREE_UPDATE_INTERNAL_SNAPSHOT_NODE) ?:
 278                 bch2_trans_commit(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc);
 279 }
 280
 281 int bch2_move_extent(struct moving_context *ctxt,
 282                      struct move_bucket_in_flight *bucket_in_flight,
 283                      struct btree_iter *iter,
 284                      struct bkey_s_c k,
 285                      struct bch_io_opts io_opts,
 286                      struct data_update_opts data_opts)
 287 {
 288         struct btree_trans *trans = ctxt->trans;
 289         struct bch_fs *c = trans->c;
 290         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
 291         struct moving_io *io;
 292         const union bch_extent_entry *entry;
 293         struct extent_ptr_decoded p;
 294         unsigned sectors = k.k->size, pages;
 295         int ret = -ENOMEM;
 296
 297         if (ctxt->stats)
 298                 ctxt->stats->pos = BBPOS(iter->btree_id, iter->pos);
 299         trace_move_extent2(c, k);
 300
 301         bch2_data_update_opts_normalize(k, &data_opts);
 302
 303         if (!data_opts.rewrite_ptrs &&
 304             !data_opts.extra_replicas) {
 305                 if (data_opts.kill_ptrs)
 306                         return bch2_extent_drop_ptrs(trans, iter, k, data_opts);
 307                 return 0;
 308         }
 309
 310         /*
 311          * Before memory allocations & taking nocow locks in
 312          * bch2_data_update_init():
 313          */
 314         bch2_trans_unlock(trans);
 315
 316         /* write path might have to decompress data: */
 317         bkey_for_each_ptr_decode(k.k, ptrs, p, entry)
 318                 sectors = max_t(unsigned, sectors, p.crc.uncompressed_size);
 319
 320         pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
 321         io = kzalloc(sizeof(struct moving_io) +
 322                      sizeof(struct bio_vec) * pages, GFP_KERNEL);
 323         if (!io)
 324                 goto err;
 325
 326         INIT_LIST_HEAD(&io->io_list);
 327         io->write.ctxt          = ctxt;
 328         io->read_sectors        = k.k->size;
 329         io->write_sectors       = k.k->size;
 330
 331         bio_init(&io->write.op.wbio.bio, NULL, io->bi_inline_vecs, pages, 0);
 332         bio_set_prio(&io->write.op.wbio.bio,
 333                      IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 334
 335         if (bch2_bio_alloc_pages(&io->write.op.wbio.bio, sectors << 9,
 336                                  GFP_KERNEL))
 337                 goto err_free;
 338
 339         io->rbio.c              = c;
 340         io->rbio.opts           = io_opts;
 341         bio_init(&io->rbio.bio, NULL, io->bi_inline_vecs, pages, 0);
 342         io->rbio.bio.bi_vcnt = pages;
 343         bio_set_prio(&io->rbio.bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
 344         io->rbio.bio.bi_iter.bi_size = sectors << 9;
 345
 346         io->rbio.bio.bi_opf             = REQ_OP_READ;
 347         io->rbio.bio.bi_iter.bi_sector  = bkey_start_offset(k.k);
 348         io->rbio.bio.bi_end_io          = move_read_endio;
 349
 350         ret = bch2_data_update_init(trans, ctxt, &io->write, ctxt->wp,
 351                                     io_opts, data_opts, iter->btree_id, k);
 352         if (ret && ret != -BCH_ERR_unwritten_extent_update)
 353                 goto err_free_pages;
 354
 355         if (ret == -BCH_ERR_unwritten_extent_update) {
 356                 bch2_update_unwritten_extent(trans, &io->write);
 357                 move_free(io);
 358                 return 0;
 359         }
 360
 361         BUG_ON(ret);
 362
 363         io->write.op.end_io = move_write_done;
 364
 365         if (ctxt->rate)
 366                 bch2_ratelimit_increment(ctxt->rate, k.k->size);
 367
 368         if (ctxt->stats) {
 369                 atomic64_inc(&ctxt->stats->keys_moved);
 370                 atomic64_add(k.k->size, &ctxt->stats->sectors_moved);
 371         }
 372
 373         if (bucket_in_flight) {
 374                 io->b = bucket_in_flight;
 375                 atomic_inc(&io->b->count);
 376         }
 377
 378         this_cpu_add(c->counters[BCH_COUNTER_io_move], k.k->size);
 379         this_cpu_add(c->counters[BCH_COUNTER_move_extent_read], k.k->size);
 380         trace_move_extent_read2(c, k);
 381
 382         mutex_lock(&ctxt->lock);
 383         atomic_add(io->read_sectors, &ctxt->read_sectors);
 384         atomic_inc(&ctxt->read_ios);
 385
 386         list_add_tail(&io->read_list, &ctxt->reads);
 387         list_add_tail(&io->io_list, &ctxt->ios);
 388         mutex_unlock(&ctxt->lock);
 389
 390         /*
 391          * dropped by move_read_endio() - guards against use after free of
 392          * ctxt when doing wakeup
 393          */
 394         closure_get(&ctxt->cl);
 395         bch2_read_extent(trans, &io->rbio,
 396                          bkey_start_pos(k.k),
 397                          iter->btree_id, k, 0,
 398                          BCH_READ_NODECODE|
 399                          BCH_READ_LAST_FRAGMENT);
 400         return 0;
 401 err_free_pages:
 402         bio_free_pages(&io->write.op.wbio.bio);
 403 err_free:
 404         kfree(io);
 405 err:
 406         this_cpu_inc(c->counters[BCH_COUNTER_move_extent_alloc_mem_fail]);
 407         trace_move_extent_alloc_mem_fail2(c, k);
 408         return ret;
 409 }
 410
 411 struct bch_io_opts *bch2_move_get_io_opts(struct btree_trans *trans,
 412                           struct per_snapshot_io_opts *io_opts,
 413                           struct bkey_s_c extent_k)
 414 {
 415         struct bch_fs *c = trans->c;
 416         u32 restart_count = trans->restart_count;
 417         int ret = 0;
 418
 419         if (io_opts->cur_inum != extent_k.k->p.inode) {
 420                 struct btree_iter iter;
 421                 struct bkey_s_c k;
 422
 423                 io_opts->d.nr = 0;
 424
 425                 for_each_btree_key(trans, iter, BTREE_ID_inodes, POS(0, extent_k.k->p.inode),
 426                                    BTREE_ITER_ALL_SNAPSHOTS, k, ret) {
 427                         if (k.k->p.offset != extent_k.k->p.inode)
 428                                 break;
 429
 430                         if (!bkey_is_inode(k.k))
 431                                 continue;
 432
 433                         struct bch_inode_unpacked inode;
 434                         BUG_ON(bch2_inode_unpack(k, &inode));
 435
 436                         struct snapshot_io_opts_entry e = { .snapshot = k.k->p.snapshot };
 437                         bch2_inode_opts_get(&e.io_opts, trans->c, &inode);
 438
 439                         ret = darray_push(&io_opts->d, e);
 440                         if (ret)
 441                                 break;
 442                 }
 443                 bch2_trans_iter_exit(trans, &iter);
 444                 io_opts->cur_inum = extent_k.k->p.inode;
 445         }
 446
 447         ret = ret ?: trans_was_restarted(trans, restart_count);
 448         if (ret)
 449                 return ERR_PTR(ret);
 450
 451         if (extent_k.k->p.snapshot) {
 452                 struct snapshot_io_opts_entry *i;
 453                 darray_for_each(io_opts->d, i)
 454                         if (bch2_snapshot_is_ancestor(c, extent_k.k->p.snapshot, i->snapshot))
 455                                 return &i->io_opts;
 456         }
 457
 458         return &io_opts->fs_io_opts;
 459 }
 460
 461 int bch2_move_get_io_opts_one(struct btree_trans *trans,
 462                               struct bch_io_opts *io_opts,
 463                               struct bkey_s_c extent_k)
 464 {
 465         struct btree_iter iter;
 466         struct bkey_s_c k;
 467         int ret;
 468
 469         /* reflink btree? */
 470         if (!extent_k.k->p.inode) {
 471                 *io_opts = bch2_opts_to_inode_opts(trans->c->opts);
 472                 return 0;
 473         }
 474
 475         k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes,
 476                                SPOS(0, extent_k.k->p.inode, extent_k.k->p.snapshot),
 477                                BTREE_ITER_CACHED);
 478         ret = bkey_err(k);
 479         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 480                 return ret;
 481
 482         if (!ret && bkey_is_inode(k.k)) {
 483                 struct bch_inode_unpacked inode;
 484                 bch2_inode_unpack(k, &inode);
 485                 bch2_inode_opts_get(io_opts, trans->c, &inode);
 486         } else {
 487                 *io_opts = bch2_opts_to_inode_opts(trans->c->opts);
 488         }
 489
 490         bch2_trans_iter_exit(trans, &iter);
 491         return 0;
 492 }
 493
 494 int bch2_move_ratelimit(struct moving_context *ctxt)
 495 {
 496         struct bch_fs *c = ctxt->trans->c;
 497         u64 delay;
 498
 499         if (ctxt->wait_on_copygc && c->copygc_running) {
 500                 bch2_moving_ctxt_flush_all(ctxt);
 501                 wait_event_killable(c->copygc_running_wq,
 502                                     !c->copygc_running ||
 503                                     kthread_should_stop());
 504         }
 505
 506         do {
 507                 delay = ctxt->rate ? bch2_ratelimit_delay(ctxt->rate) : 0;
 508
 509
 510                 if (delay) {
 511                         if (delay > HZ / 10)
 512                                 bch2_trans_unlock_long(ctxt->trans);
 513                         else
 514                                 bch2_trans_unlock(ctxt->trans);
 515                         set_current_state(TASK_INTERRUPTIBLE);
 516                 }
 517
 518                 if (kthread_should_stop()) {
 519                         __set_current_state(TASK_RUNNING);
 520                         return 1;
 521                 }
 522
 523                 if (delay)
 524                         schedule_timeout(delay);
 525
 526                 if (unlikely(freezing(current))) {
 527                         bch2_moving_ctxt_flush_all(ctxt);
 528                         try_to_freeze();
 529                 }
 530         } while (delay);
 531
 532         /*
 533          * XXX: these limits really ought to be per device, SSDs and hard drives
 534          * will want different limits
 535          */
 536         move_ctxt_wait_event(ctxt,
 537                 atomic_read(&ctxt->write_sectors) < c->opts.move_bytes_in_flight >> 9 &&
 538                 atomic_read(&ctxt->read_sectors) < c->opts.move_bytes_in_flight >> 9 &&
 539                 atomic_read(&ctxt->write_ios) < c->opts.move_ios_in_flight &&
 540                 atomic_read(&ctxt->read_ios) < c->opts.move_ios_in_flight);
 541
 542         return 0;
 543 }
 544
 545 static int bch2_move_data_btree(struct moving_context *ctxt,
 546                                 struct bpos start,
 547                                 struct bpos end,
 548                                 move_pred_fn pred, void *arg,
 549                                 enum btree_id btree_id)
 550 {
 551         struct btree_trans *trans = ctxt->trans;
 552         struct bch_fs *c = trans->c;
 553         struct per_snapshot_io_opts snapshot_io_opts;
 554         struct bch_io_opts *io_opts;
 555         struct bkey_buf sk;
 556         struct btree_iter iter;
 557         struct bkey_s_c k;
 558         struct data_update_opts data_opts;
 559         int ret = 0, ret2;
 560
 561         per_snapshot_io_opts_init(&snapshot_io_opts, c);
 562         bch2_bkey_buf_init(&sk);
 563
 564         if (ctxt->stats) {
 565                 ctxt->stats->data_type  = BCH_DATA_user;
 566                 ctxt->stats->pos        = BBPOS(btree_id, start);
 567         }
 568
 569         bch2_trans_iter_init(trans, &iter, btree_id, start,
 570                              BTREE_ITER_PREFETCH|
 571                              BTREE_ITER_ALL_SNAPSHOTS);
 572
 573         if (ctxt->rate)
 574                 bch2_ratelimit_reset(ctxt->rate);
 575
 576         while (!bch2_move_ratelimit(ctxt)) {
 577                 bch2_trans_begin(trans);
 578
 579                 k = bch2_btree_iter_peek(&iter);
 580                 if (!k.k)
 581                         break;
 582
 583                 ret = bkey_err(k);
 584                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 585                         continue;
 586                 if (ret)
 587                         break;
 588
 589                 if (bkey_ge(bkey_start_pos(k.k), end))
 590                         break;
 591
 592                 if (ctxt->stats)
 593                         ctxt->stats->pos = BBPOS(iter.btree_id, iter.pos);
 594
 595                 if (!bkey_extent_is_direct_data(k.k))
 596                         goto next_nondata;
 597
 598                 io_opts = bch2_move_get_io_opts(trans, &snapshot_io_opts, k);
 599                 ret = PTR_ERR_OR_ZERO(io_opts);
 600                 if (ret)
 601                         continue;
 602
 603                 memset(&data_opts, 0, sizeof(data_opts));
 604                 if (!pred(c, arg, k, io_opts, &data_opts))
 605                         goto next;
 606
 607                 /*
 608                  * The iterator gets unlocked by __bch2_read_extent - need to
 609                  * save a copy of @k elsewhere:
 610                  */
 611                 bch2_bkey_buf_reassemble(&sk, c, k);
 612                 k = bkey_i_to_s_c(sk.k);
 613
 614                 ret2 = bch2_move_extent(ctxt, NULL, &iter, k, *io_opts, data_opts);
 615                 if (ret2) {
 616                         if (bch2_err_matches(ret2, BCH_ERR_transaction_restart))
 617                                 continue;
 618
 619                         if (ret2 == -ENOMEM) {
 620                                 /* memory allocation failure, wait for some IO to finish */
 621                                 bch2_move_ctxt_wait_for_io(ctxt);
 622                                 continue;
 623                         }
 624
 625                         /* XXX signal failure */
 626                         goto next;
 627                 }
 628 next:
 629                 if (ctxt->stats)
 630                         atomic64_add(k.k->size, &ctxt->stats->sectors_seen);
 631 next_nondata:
 632                 bch2_btree_iter_advance(&iter);
 633         }
 634
 635         bch2_trans_iter_exit(trans, &iter);
 636         bch2_bkey_buf_exit(&sk, c);
 637         per_snapshot_io_opts_exit(&snapshot_io_opts);
 638
 639         return ret;
 640 }
 641
 642 int __bch2_move_data(struct moving_context *ctxt,
 643                      struct bbpos start,
 644                      struct bbpos end,
 645                      move_pred_fn pred, void *arg)
 646 {
 647         struct bch_fs *c = ctxt->trans->c;
 648         enum btree_id id;
 649         int ret = 0;
 650
 651         for (id = start.btree;
 652              id <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1);
 653              id++) {
 654                 ctxt->stats->pos = BBPOS(id, POS_MIN);
 655
 656                 if (!btree_type_has_ptrs(id) ||
 657                     !bch2_btree_id_root(c, id)->b)
 658                         continue;
 659
 660                 ret = bch2_move_data_btree(ctxt,
 661                                        id == start.btree ? start.pos : POS_MIN,
 662                                        id == end.btree   ? end.pos   : POS_MAX,
 663                                        pred, arg, id);
 664                 if (ret)
 665                         break;
 666         }
 667
 668         return ret;
 669 }
 670
 671 int bch2_move_data(struct bch_fs *c,
 672                    struct bbpos start,
 673                    struct bbpos end,
 674                    struct bch_ratelimit *rate,
 675                    struct bch_move_stats *stats,
 676                    struct write_point_specifier wp,
 677                    bool wait_on_copygc,
 678                    move_pred_fn pred, void *arg)
 679 {
 680
 681         struct moving_context ctxt;
 682         int ret;
 683
 684         bch2_moving_ctxt_init(&ctxt, c, rate, stats, wp, wait_on_copygc);
 685         ret = __bch2_move_data(&ctxt, start, end, pred, arg);
 686         bch2_moving_ctxt_exit(&ctxt);
 687
 688         return ret;
 689 }
 690
 691 int __bch2_evacuate_bucket(struct moving_context *ctxt,
 692                            struct move_bucket_in_flight *bucket_in_flight,
 693                            struct bpos bucket, int gen,
 694                            struct data_update_opts _data_opts)
 695 {
 696         struct btree_trans *trans = ctxt->trans;
 697         struct bch_fs *c = trans->c;
 698         struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 699         struct btree_iter iter;
 700         struct bkey_buf sk;
 701         struct bch_backpointer bp;
 702         struct bch_alloc_v4 a_convert;
 703         const struct bch_alloc_v4 *a;
 704         struct bkey_s_c k;
 705         struct data_update_opts data_opts;
 706         unsigned dirty_sectors, bucket_size;
 707         u64 fragmentation;
 708         struct bpos bp_pos = POS_MIN;
 709         int ret = 0;
 710
 711         trace_bucket_evacuate(c, &bucket);
 712
 713         bch2_bkey_buf_init(&sk);
 714
 715         /*
 716          * We're not run in a context that handles transaction restarts:
 717          */
 718         bch2_trans_begin(trans);
 719
 720         bch2_trans_iter_init(trans, &iter, BTREE_ID_alloc,
 721                              bucket, BTREE_ITER_CACHED);
 722         ret = lockrestart_do(trans,
 723                         bkey_err(k = bch2_btree_iter_peek_slot(&iter)));
 724         bch2_trans_iter_exit(trans, &iter);
 725
 726         if (ret) {
 727                 bch_err_msg(c, ret, "looking up alloc key");
 728                 goto err;
 729         }
 730
 731         a = bch2_alloc_to_v4(k, &a_convert);
 732         dirty_sectors = a->dirty_sectors;
 733         bucket_size = bch_dev_bkey_exists(c, bucket.inode)->mi.bucket_size;
 734         fragmentation = a->fragmentation_lru;
 735
 736         ret = bch2_btree_write_buffer_tryflush(trans);
 737         bch_err_msg(c, ret, "flushing btree write buffer");
 738         if (ret)
 739                 goto err;
 740
 741         while (!(ret = bch2_move_ratelimit(ctxt))) {
 742                 bch2_trans_begin(trans);
 743
 744                 ret = bch2_get_next_backpointer(trans, bucket, gen,
 745                                                 &bp_pos, &bp,
 746                                                 BTREE_ITER_CACHED);
 747                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 748                         continue;
 749                 if (ret)
 750                         goto err;
 751                 if (bkey_eq(bp_pos, POS_MAX))
 752                         break;
 753
 754                 if (!bp.level) {
 755                         const struct bch_extent_ptr *ptr;
 756                         unsigned i = 0;
 757
 758                         k = bch2_backpointer_get_key(trans, &iter, bp_pos, bp, 0);
 759                         ret = bkey_err(k);
 760                         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 761                                 continue;
 762                         if (ret)
 763                                 goto err;
 764                         if (!k.k)
 765                                 goto next;
 766
 767                         bch2_bkey_buf_reassemble(&sk, c, k);
 768                         k = bkey_i_to_s_c(sk.k);
 769
 770                         ret = bch2_move_get_io_opts_one(trans, &io_opts, k);
 771                         if (ret) {
 772                                 bch2_trans_iter_exit(trans, &iter);
 773                                 continue;
 774                         }
 775
 776                         data_opts = _data_opts;
 777                         data_opts.target        = io_opts.background_target;
 778                         data_opts.rewrite_ptrs = 0;
 779
 780                         bkey_for_each_ptr(bch2_bkey_ptrs_c(k), ptr) {
 781                                 if (ptr->dev == bucket.inode) {
 782                                         data_opts.rewrite_ptrs |= 1U << i;
 783                                         if (ptr->cached) {
 784                                                 bch2_trans_iter_exit(trans, &iter);
 785                                                 goto next;
 786                                         }
 787                                 }
 788                                 i++;
 789                         }
 790
 791                         ret = bch2_move_extent(ctxt, bucket_in_flight,
 792                                                &iter, k, io_opts, data_opts);
 793                         bch2_trans_iter_exit(trans, &iter);
 794
 795                         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 796                                 continue;
 797                         if (ret == -ENOMEM) {
 798                                 /* memory allocation failure, wait for some IO to finish */
 799                                 bch2_move_ctxt_wait_for_io(ctxt);
 800                                 continue;
 801                         }
 802                         if (ret)
 803                                 goto err;
 804
 805                         if (ctxt->stats)
 806                                 atomic64_add(k.k->size, &ctxt->stats->sectors_seen);
 807                 } else {
 808                         struct btree *b;
 809
 810                         b = bch2_backpointer_get_node(trans, &iter, bp_pos, bp);
 811                         ret = PTR_ERR_OR_ZERO(b);
 812                         if (ret == -BCH_ERR_backpointer_to_overwritten_btree_node)
 813                                 continue;
 814                         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 815                                 continue;
 816                         if (ret)
 817                                 goto err;
 818                         if (!b)
 819                                 goto next;
 820
 821                         ret = bch2_btree_node_rewrite(trans, &iter, b, 0);
 822                         bch2_trans_iter_exit(trans, &iter);
 823
 824                         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 825                                 continue;
 826                         if (ret)
 827                                 goto err;
 828
 829                         if (ctxt->rate)
 830                                 bch2_ratelimit_increment(ctxt->rate,
 831                                                          c->opts.btree_node_size >> 9);
 832                         if (ctxt->stats) {
 833                                 atomic64_add(c->opts.btree_node_size >> 9, &ctxt->stats->sectors_seen);
 834                                 atomic64_add(c->opts.btree_node_size >> 9, &ctxt->stats->sectors_moved);
 835                         }
 836                 }
 837 next:
 838                 bp_pos = bpos_nosnap_successor(bp_pos);
 839         }
 840
 841         trace_evacuate_bucket(c, &bucket, dirty_sectors, bucket_size, fragmentation, ret);
 842 err:
 843         bch2_bkey_buf_exit(&sk, c);
 844         return ret;
 845 }
 846
 847 int bch2_evacuate_bucket(struct bch_fs *c,
 848                          struct bpos bucket, int gen,
 849                          struct data_update_opts data_opts,
 850                          struct bch_ratelimit *rate,
 851                          struct bch_move_stats *stats,
 852                          struct write_point_specifier wp,
 853                          bool wait_on_copygc)
 854 {
 855         struct moving_context ctxt;
 856         int ret;
 857
 858         bch2_moving_ctxt_init(&ctxt, c, rate, stats, wp, wait_on_copygc);
 859         ret = __bch2_evacuate_bucket(&ctxt, NULL, bucket, gen, data_opts);
 860         bch2_moving_ctxt_exit(&ctxt);
 861
 862         return ret;
 863 }
 864
 865 typedef bool (*move_btree_pred)(struct bch_fs *, void *,
 866                                 struct btree *, struct bch_io_opts *,
 867                                 struct data_update_opts *);
 868
 869 static int bch2_move_btree(struct bch_fs *c,
 870                            struct bbpos start,
 871                            struct bbpos end,
 872                            move_btree_pred pred, void *arg,
 873                            struct bch_move_stats *stats)
 874 {
 875         struct bch_io_opts io_opts = bch2_opts_to_inode_opts(c->opts);
 876         struct moving_context ctxt;
 877         struct btree_trans *trans;
 878         struct btree_iter iter;
 879         struct btree *b;
 880         enum btree_id btree;
 881         struct data_update_opts data_opts;
 882         int ret = 0;
 883
 884         bch2_moving_ctxt_init(&ctxt, c, NULL, stats,
 885                               writepoint_ptr(&c->btree_write_point),
 886                               true);
 887         trans = ctxt.trans;
 888
 889         stats->data_type = BCH_DATA_btree;
 890
 891         for (btree = start.btree;
 892              btree <= min_t(unsigned, end.btree, btree_id_nr_alive(c) - 1);
 893              btree ++) {
 894                 stats->pos = BBPOS(btree, POS_MIN);
 895
 896                 if (!bch2_btree_id_root(c, btree)->b)
 897                         continue;
 898
 899                 bch2_trans_node_iter_init(trans, &iter, btree, POS_MIN, 0, 0,
 900                                           BTREE_ITER_PREFETCH);
 901 retry:
 902                 ret = 0;
 903                 while (bch2_trans_begin(trans),
 904                        (b = bch2_btree_iter_peek_node(&iter)) &&
 905                        !(ret = PTR_ERR_OR_ZERO(b))) {
 906                         if (kthread_should_stop())
 907                                 break;
 908
 909                         if ((cmp_int(btree, end.btree) ?:
 910                              bpos_cmp(b->key.k.p, end.pos)) > 0)
 911                                 break;
 912
 913                         stats->pos = BBPOS(iter.btree_id, iter.pos);
 914
 915                         if (!pred(c, arg, b, &io_opts, &data_opts))
 916                                 goto next;
 917
 918                         ret = bch2_btree_node_rewrite(trans, &iter, b, 0) ?: ret;
 919                         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 920                                 continue;
 921                         if (ret)
 922                                 break;
 923 next:
 924                         bch2_btree_iter_next_node(&iter);
 925                 }
 926                 if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 927                         goto retry;
 928
 929                 bch2_trans_iter_exit(trans, &iter);
 930
 931                 if (kthread_should_stop())
 932                         break;
 933         }
 934
 935         bch_err_fn(c, ret);
 936         bch2_moving_ctxt_exit(&ctxt);
 937         bch2_btree_interior_updates_flush(c);
 938
 939         return ret;
 940 }
 941
 942 static bool rereplicate_pred(struct bch_fs *c, void *arg,
 943                              struct bkey_s_c k,
 944                              struct bch_io_opts *io_opts,
 945                              struct data_update_opts *data_opts)
 946 {
 947         unsigned nr_good = bch2_bkey_durability(c, k);
 948         unsigned replicas = bkey_is_btree_ptr(k.k)
 949                 ? c->opts.metadata_replicas
 950                 : io_opts->data_replicas;
 951
 952         if (!nr_good || nr_good >= replicas)
 953                 return false;
 954
 955         data_opts->target               = 0;
 956         data_opts->extra_replicas       = replicas - nr_good;
 957         data_opts->btree_insert_flags   = 0;
 958         return true;
 959 }
 960
 961 static bool migrate_pred(struct bch_fs *c, void *arg,
 962                          struct bkey_s_c k,
 963                          struct bch_io_opts *io_opts,
 964                          struct data_update_opts *data_opts)
 965 {
 966         struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
 967         const struct bch_extent_ptr *ptr;
 968         struct bch_ioctl_data *op = arg;
 969         unsigned i = 0;
 970
 971         data_opts->rewrite_ptrs         = 0;
 972         data_opts->target               = 0;
 973         data_opts->extra_replicas       = 0;
 974         data_opts->btree_insert_flags   = 0;
 975
 976         bkey_for_each_ptr(ptrs, ptr) {
 977                 if (ptr->dev == op->migrate.dev)
 978                         data_opts->rewrite_ptrs |= 1U << i;
 979                 i++;
 980         }
 981
 982         return data_opts->rewrite_ptrs != 0;
 983 }
 984
 985 static bool rereplicate_btree_pred(struct bch_fs *c, void *arg,
 986                                    struct btree *b,
 987                                    struct bch_io_opts *io_opts,
 988                                    struct data_update_opts *data_opts)
 989 {
 990         return rereplicate_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
 991 }
 992
 993 static bool migrate_btree_pred(struct bch_fs *c, void *arg,
 994                                struct btree *b,
 995                                struct bch_io_opts *io_opts,
 996                                struct data_update_opts *data_opts)
 997 {
 998         return migrate_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
 999 }
1000
1001 static bool bformat_needs_redo(struct bkey_format *f)
1002 {
1003         unsigned i;
1004
1005         for (i = 0; i < f->nr_fields; i++) {
1006                 unsigned unpacked_bits = bch2_bkey_format_current.bits_per_field[i];
1007                 u64 unpacked_mask = ~((~0ULL << 1) << (unpacked_bits - 1));
1008                 u64 field_offset = le64_to_cpu(f->field_offset[i]);
1009
1010                 if (f->bits_per_field[i] > unpacked_bits)
1011                         return true;
1012
1013                 if ((f->bits_per_field[i] == unpacked_bits) && field_offset)
1014                         return true;
1015
1016                 if (((field_offset + ((1ULL << f->bits_per_field[i]) - 1)) &
1017                      unpacked_mask) <
1018                     field_offset)
1019                         return true;
1020         }
1021
1022         return false;
1023 }
1024
1025 static bool rewrite_old_nodes_pred(struct bch_fs *c, void *arg,
1026                                    struct btree *b,
1027                                    struct bch_io_opts *io_opts,
1028                                    struct data_update_opts *data_opts)
1029 {
1030         if (b->version_ondisk != c->sb.version ||
1031             btree_node_need_rewrite(b) ||
1032             bformat_needs_redo(&b->format)) {
1033                 data_opts->target               = 0;
1034                 data_opts->extra_replicas       = 0;
1035                 data_opts->btree_insert_flags   = 0;
1036                 return true;
1037         }
1038
1039         return false;
1040 }
1041
1042 int bch2_scan_old_btree_nodes(struct bch_fs *c, struct bch_move_stats *stats)
1043 {
1044         int ret;
1045
1046         ret = bch2_move_btree(c,
1047                               BBPOS_MIN,
1048                               BBPOS_MAX,
1049                               rewrite_old_nodes_pred, c, stats);
1050         if (!ret) {
1051                 mutex_lock(&c->sb_lock);
1052                 c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_extents_above_btree_updates_done);
1053                 c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_bformat_overflow_done);
1054                 c->disk_sb.sb->version_min = c->disk_sb.sb->version;
1055                 bch2_write_super(c);
1056                 mutex_unlock(&c->sb_lock);
1057         }
1058
1059         bch_err_fn(c, ret);
1060         return ret;
1061 }
1062
1063 static bool drop_extra_replicas_pred(struct bch_fs *c, void *arg,
1064                              struct bkey_s_c k,
1065                              struct bch_io_opts *io_opts,
1066                              struct data_update_opts *data_opts)
1067 {
1068         unsigned durability = bch2_bkey_durability(c, k);
1069         unsigned replicas = bkey_is_btree_ptr(k.k)
1070                 ? c->opts.metadata_replicas
1071                 : io_opts->data_replicas;
1072         const union bch_extent_entry *entry;
1073         struct extent_ptr_decoded p;
1074         unsigned i = 0;
1075
1076         bkey_for_each_ptr_decode(k.k, bch2_bkey_ptrs_c(k), p, entry) {
1077                 unsigned d = bch2_extent_ptr_durability(c, &p);
1078
1079                 if (d && durability - d >= replicas) {
1080                         data_opts->kill_ptrs |= BIT(i);
1081                         durability -= d;
1082                 }
1083
1084                 i++;
1085         }
1086
1087         return data_opts->kill_ptrs != 0;
1088 }
1089
1090 static bool drop_extra_replicas_btree_pred(struct bch_fs *c, void *arg,
1091                                    struct btree *b,
1092                                    struct bch_io_opts *io_opts,
1093                                    struct data_update_opts *data_opts)
1094 {
1095         return drop_extra_replicas_pred(c, arg, bkey_i_to_s_c(&b->key), io_opts, data_opts);
1096 }
1097
1098 int bch2_data_job(struct bch_fs *c,
1099                   struct bch_move_stats *stats,
1100                   struct bch_ioctl_data op)
1101 {
1102         struct bbpos start      = BBPOS(op.start_btree, op.start_pos);
1103         struct bbpos end        = BBPOS(op.end_btree, op.end_pos);
1104         int ret = 0;
1105
1106         if (op.op >= BCH_DATA_OP_NR)
1107                 return -EINVAL;
1108
1109         bch2_move_stats_init(stats, bch2_data_ops_strs[op.op]);
1110
1111         switch (op.op) {
1112         case BCH_DATA_OP_rereplicate:
1113                 stats->data_type = BCH_DATA_journal;
1114                 ret = bch2_journal_flush_device_pins(&c->journal, -1);
1115                 ret = bch2_move_btree(c, start, end,
1116                                       rereplicate_btree_pred, c, stats) ?: ret;
1117                 ret = bch2_move_data(c, start, end,
1118                                      NULL,
1119                                      stats,
1120                                      writepoint_hashed((unsigned long) current),
1121                                      true,
1122                                      rereplicate_pred, c) ?: ret;
1123                 ret = bch2_replicas_gc2(c) ?: ret;
1124                 break;
1125         case BCH_DATA_OP_migrate:
1126                 if (op.migrate.dev >= c->sb.nr_devices)
1127                         return -EINVAL;
1128
1129                 stats->data_type = BCH_DATA_journal;
1130                 ret = bch2_journal_flush_device_pins(&c->journal, op.migrate.dev);
1131                 ret = bch2_move_btree(c, start, end,
1132                                       migrate_btree_pred, &op, stats) ?: ret;
1133                 ret = bch2_move_data(c, start, end,
1134                                      NULL,
1135                                      stats,
1136                                      writepoint_hashed((unsigned long) current),
1137                                      true,
1138                                      migrate_pred, &op) ?: ret;
1139                 ret = bch2_replicas_gc2(c) ?: ret;
1140                 break;
1141         case BCH_DATA_OP_rewrite_old_nodes:
1142                 ret = bch2_scan_old_btree_nodes(c, stats);
1143                 break;
1144         case BCH_DATA_OP_drop_extra_replicas:
1145                 ret = bch2_move_btree(c, start, end,
1146                                 drop_extra_replicas_btree_pred, c, stats) ?: ret;
1147                 ret = bch2_move_data(c, start, end, NULL, stats,
1148                                 writepoint_hashed((unsigned long) current),
1149                                 true,
1150                                 drop_extra_replicas_pred, c) ?: ret;
1151                 ret = bch2_replicas_gc2(c) ?: ret;
1152                 break;
1153         default:
1154                 ret = -EINVAL;
1155         }
1156
1157         bch2_move_stats_exit(stats, c);
1158         return ret;
1159 }
1160
1161 void bch2_move_stats_to_text(struct printbuf *out, struct bch_move_stats *stats)
1162 {
1163         prt_printf(out, "%s: data type=%s pos=",
1164                    stats->name,
1165                    bch2_data_types[stats->data_type]);
1166         bch2_bbpos_to_text(out, stats->pos);
1167         prt_newline(out);
1168         printbuf_indent_add(out, 2);
1169
1170         prt_str(out, "keys moved:  ");
1171         prt_u64(out, atomic64_read(&stats->keys_moved));
1172         prt_newline(out);
1173
1174         prt_str(out, "keys raced:  ");
1175         prt_u64(out, atomic64_read(&stats->keys_raced));
1176         prt_newline(out);
1177
1178         prt_str(out, "bytes seen:  ");
1179         prt_human_readable_u64(out, atomic64_read(&stats->sectors_seen) << 9);
1180         prt_newline(out);
1181
1182         prt_str(out, "bytes moved: ");
1183         prt_human_readable_u64(out, atomic64_read(&stats->sectors_moved) << 9);
1184         prt_newline(out);
1185
1186         prt_str(out, "bytes raced: ");
1187         prt_human_readable_u64(out, atomic64_read(&stats->sectors_raced) << 9);
1188         prt_newline(out);
1189
1190         printbuf_indent_sub(out, 2);
1191 }
1192
1193 static void bch2_moving_ctxt_to_text(struct printbuf *out, struct bch_fs *c, struct moving_context *ctxt)
1194 {
1195         struct moving_io *io;
1196
1197         bch2_move_stats_to_text(out, ctxt->stats);
1198         printbuf_indent_add(out, 2);
1199
1200         prt_printf(out, "reads: ios %u/%u sectors %u/%u",
1201                    atomic_read(&ctxt->read_ios),
1202                    c->opts.move_ios_in_flight,
1203                    atomic_read(&ctxt->read_sectors),
1204                    c->opts.move_bytes_in_flight >> 9);
1205         prt_newline(out);
1206
1207         prt_printf(out, "writes: ios %u/%u sectors %u/%u",
1208                    atomic_read(&ctxt->write_ios),
1209                    c->opts.move_ios_in_flight,
1210                    atomic_read(&ctxt->write_sectors),
1211                    c->opts.move_bytes_in_flight >> 9);
1212         prt_newline(out);
1213
1214         printbuf_indent_add(out, 2);
1215
1216         mutex_lock(&ctxt->lock);
1217         list_for_each_entry(io, &ctxt->ios, io_list)
1218                 bch2_write_op_to_text(out, &io->write.op);
1219         mutex_unlock(&ctxt->lock);
1220
1221         printbuf_indent_sub(out, 4);
1222 }
1223
1224 void bch2_fs_moving_ctxts_to_text(struct printbuf *out, struct bch_fs *c)
1225 {
1226         struct moving_context *ctxt;
1227
1228         mutex_lock(&c->moving_context_lock);
1229         list_for_each_entry(ctxt, &c->moving_context_list, list)
1230                 bch2_moving_ctxt_to_text(out, c, ctxt);
1231         mutex_unlock(&c->moving_context_lock);
1232 }
1233
1234 void bch2_fs_move_init(struct bch_fs *c)
1235 {
1236         INIT_LIST_HEAD(&c->moving_context_list);
1237         mutex_init(&c->moving_context_lock);
1238 }