-struct moving_context {
- /* Closure for waiting on all reads and writes to complete */
- struct closure cl;
-
- struct bch_move_stats *stats;
-
- struct list_head reads;
-
- /* in flight sectors: */
- atomic_t read_sectors;
- atomic_t write_sectors;
-
- wait_queue_head_t wait;
-};
-
-static int bch2_migrate_index_update(struct bch_write_op *op)
-{
- struct bch_fs *c = op->c;
- struct btree_trans trans;
- struct btree_iter *iter;
- struct migrate_write *m =
- container_of(op, struct migrate_write, op);
- struct keylist *keys = &op->insert_keys;
- int ret = 0;
-
- bch2_trans_init(&trans, c, BTREE_ITER_MAX, 0);
-
- iter = bch2_trans_get_iter(&trans, m->btree_id,
- bkey_start_pos(&bch2_keylist_front(keys)->k),
- BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
-
- while (1) {
- struct bkey_s_c k = bch2_btree_iter_peek_slot(iter);
- struct bkey_i *insert;
- struct bkey_i_extent *new =
- bkey_i_to_extent(bch2_keylist_front(keys));
- BKEY_PADDED(k) _new, _insert;
- const union bch_extent_entry *entry;
- struct extent_ptr_decoded p;
- bool did_work = false;
- int nr;
-
- ret = bkey_err(k);
- if (ret)
- break;
-
- if (bversion_cmp(k.k->version, new->k.version) ||
- !bch2_bkey_matches_ptr(c, k, m->ptr, m->offset))
- goto nomatch;
-
- if (m->data_cmd == DATA_REWRITE &&
- !bch2_bkey_has_device(k, m->data_opts.rewrite_dev))
- goto nomatch;
-
- bkey_reassemble(&_insert.k, k);
- insert = &_insert.k;
-
- bkey_copy(&_new.k, bch2_keylist_front(keys));
- new = bkey_i_to_extent(&_new.k);
- bch2_cut_front(iter->pos, &new->k_i);
-
- bch2_cut_front(iter->pos, insert);
- bch2_cut_back(new->k.p, insert);
- bch2_cut_back(insert->k.p, &new->k_i);
-
- if (m->data_cmd == DATA_REWRITE)
- bch2_bkey_drop_device(bkey_i_to_s(insert),
- m->data_opts.rewrite_dev);
-
- extent_for_each_ptr_decode(extent_i_to_s(new), p, entry) {
- if (bch2_bkey_has_device(bkey_i_to_s_c(insert), p.ptr.dev)) {
- /*
- * raced with another move op? extent already
- * has a pointer to the device we just wrote
- * data to
- */
- continue;
- }
-
- bch2_extent_ptr_decoded_append(insert, &p);
- did_work = true;
- }
-
- if (!did_work)
- goto nomatch;
-
- bch2_bkey_narrow_crcs(insert,
- (struct bch_extent_crc_unpacked) { 0 });
- bch2_extent_normalize(c, bkey_i_to_s(insert));
- bch2_bkey_mark_replicas_cached(c, bkey_i_to_s(insert),
- op->opts.background_target,
- op->opts.data_replicas);
-
- /*
- * If we're not fully overwriting @k, and it's compressed, we
- * need a reservation for all the pointers in @insert
- */
- nr = bch2_bkey_nr_ptrs_allocated(bkey_i_to_s_c(insert)) -
- m->nr_ptrs_reserved;
-
- if (insert->k.size < k.k->size &&
- bch2_bkey_sectors_compressed(k) &&
- nr > 0) {
- ret = bch2_disk_reservation_add(c, &op->res,
- keylist_sectors(keys) * nr, 0);
- if (ret)
- goto out;
-
- m->nr_ptrs_reserved += nr;
- goto next;
- }
-
- bch2_trans_update(&trans, iter, insert, 0);
-
- ret = bch2_trans_commit(&trans, &op->res,
- op_journal_seq(op),
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_USE_RESERVE|
- m->data_opts.btree_insert_flags);
- if (!ret)
- atomic_long_inc(&c->extent_migrate_done);
- if (ret == -EINTR)
- ret = 0;
- if (ret)
- break;
-next:
- while (bkey_cmp(iter->pos, bch2_keylist_front(keys)->k.p) >= 0) {
- bch2_keylist_pop_front(keys);
- if (bch2_keylist_empty(keys))
- goto out;
- }
- continue;
-nomatch:
- if (m->ctxt)
- atomic64_add(k.k->p.offset - iter->pos.offset,
- &m->ctxt->stats->sectors_raced);
- atomic_long_inc(&c->extent_migrate_raced);
- trace_move_race(&new->k);
- bch2_btree_iter_next_slot(iter);
- goto next;
- }
-out:
- bch2_trans_exit(&trans);
- BUG_ON(ret == -EINTR);
- return ret;
-}
-
-void bch2_migrate_read_done(struct migrate_write *m, struct bch_read_bio *rbio)
-{
- /* write bio must own pages: */
- BUG_ON(!m->op.wbio.bio.bi_vcnt);
-
- m->ptr = rbio->pick.ptr;
- m->offset = rbio->pos.offset - rbio->pick.crc.offset;
- m->op.devs_have = rbio->devs_have;
- m->op.pos = rbio->pos;
- m->op.version = rbio->version;
- m->op.crc = rbio->pick.crc;
- m->op.wbio.bio.bi_iter.bi_size = m->op.crc.compressed_size << 9;
-
- if (bch2_csum_type_is_encryption(m->op.crc.csum_type)) {
- m->op.nonce = m->op.crc.nonce + m->op.crc.offset;
- m->op.csum_type = m->op.crc.csum_type;
- }
-
- if (m->data_cmd == DATA_REWRITE)
- bch2_dev_list_drop_dev(&m->op.devs_have, m->data_opts.rewrite_dev);
-}
-
-int bch2_migrate_write_init(struct bch_fs *c, struct migrate_write *m,
- struct write_point_specifier wp,
- struct bch_io_opts io_opts,
- enum data_cmd data_cmd,
- struct data_opts data_opts,
- enum btree_id btree_id,
- struct bkey_s_c k)
-{
- int ret;
-
- m->btree_id = btree_id;
- m->data_cmd = data_cmd;
- m->data_opts = data_opts;
- m->nr_ptrs_reserved = 0;
-
- bch2_write_op_init(&m->op, c, io_opts);
- m->op.compression_type =
- bch2_compression_opt_to_type[io_opts.background_compression ?:
- io_opts.compression];
- m->op.target = data_opts.target,
- m->op.write_point = wp;
-
- if (m->data_opts.btree_insert_flags & BTREE_INSERT_USE_RESERVE)
- m->op.alloc_reserve = RESERVE_MOVINGGC;
-
- m->op.flags |= BCH_WRITE_ONLY_SPECIFIED_DEVS|
- BCH_WRITE_PAGES_STABLE|
- BCH_WRITE_PAGES_OWNED|
- BCH_WRITE_DATA_ENCODED;
-
- m->op.nr_replicas = 1;
- m->op.nr_replicas_required = 1;
- m->op.index_update_fn = bch2_migrate_index_update;
-
- switch (data_cmd) {
- case DATA_ADD_REPLICAS: {
- /*
- * DATA_ADD_REPLICAS is used for moving data to a different
- * device in the background, and due to compression the new copy
- * might take up more space than the old copy:
- */
-#if 0
- int nr = (int) io_opts.data_replicas -
- bch2_bkey_nr_ptrs_allocated(k);
-#endif
- int nr = (int) io_opts.data_replicas;
-
- if (nr > 0) {
- m->op.nr_replicas = m->nr_ptrs_reserved = nr;
-
- ret = bch2_disk_reservation_get(c, &m->op.res,
- k.k->size, m->op.nr_replicas, 0);
- if (ret)
- return ret;
- }
- break;
- }
- case DATA_REWRITE: {
- struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
- const union bch_extent_entry *entry;
- struct extent_ptr_decoded p;
- unsigned compressed_sectors = 0;
-
- bkey_for_each_ptr_decode(k.k, ptrs, p, entry)
- if (!p.ptr.cached &&
- p.crc.compression_type != BCH_COMPRESSION_TYPE_none &&
- bch2_dev_in_target(c, p.ptr.dev, data_opts.target))
- compressed_sectors += p.crc.compressed_size;
-
- if (compressed_sectors) {
- ret = bch2_disk_reservation_add(c, &m->op.res,
- compressed_sectors,
- BCH_DISK_RESERVATION_NOFAIL);
- if (ret)
- return ret;
- }
- break;
- }
- case DATA_PROMOTE:
- m->op.flags |= BCH_WRITE_ALLOC_NOWAIT;
- m->op.flags |= BCH_WRITE_CACHED;
- break;
- default:
- BUG();
- }
-
- return 0;
-}
-