-8eca47e4d5c4e6817ad4c020be4280bd82104efd
+fe72e70682cd2430a099c08c3135253675030d28
__wait_event(wq, condition); \
} while (0)
+#define wait_event_freezable(wq, condition) ({wait_event(wq, condition); 0; })
#define wait_event_killable(wq, condition) ({wait_event(wq, condition); 0; })
#define wait_event_interruptible(wq, condition) ({wait_event(wq, condition); 0; })
/* Garbage collection */
-DEFINE_EVENT(btree_node, btree_gc_coalesce,
- TP_PROTO(struct bch_fs *c, struct btree *b),
- TP_ARGS(c, b)
-);
-
-TRACE_EVENT(btree_gc_coalesce_fail,
- TP_PROTO(struct bch_fs *c, int reason),
- TP_ARGS(c, reason),
-
- TP_STRUCT__entry(
- __field(u8, reason )
- __array(char, uuid, 16 )
- ),
-
- TP_fast_assign(
- __entry->reason = reason;
- memcpy(__entry->uuid, c->disk_sb.sb->user_uuid.b, 16);
- ),
-
- TP_printk("%pU: %u", __entry->uuid, __entry->reason)
-);
-
DEFINE_EVENT(btree_node, btree_gc_rewrite_node,
TP_PROTO(struct bch_fs *c, struct btree *b),
TP_ARGS(c, b)
TP_ARGS(c)
);
-DEFINE_EVENT(bch_fs, gc_coalesce_start,
- TP_PROTO(struct bch_fs *c),
- TP_ARGS(c)
-);
-
-DEFINE_EVENT(bch_fs, gc_coalesce_end,
- TP_PROTO(struct bch_fs *c),
- TP_ARGS(c)
-);
-
DEFINE_EVENT(bch_fs, gc_cannot_inc_gens,
TP_PROTO(struct bch_fs *c),
TP_ARGS(c)
/* Allocator */
-TRACE_EVENT(alloc_batch,
- TP_PROTO(struct bch_dev *ca, size_t free, size_t total),
- TP_ARGS(ca, free, total),
+TRACE_EVENT(alloc_scan,
+ TP_PROTO(struct bch_dev *ca, u64 found, u64 inc_gen, u64 inc_gen_skipped),
+ TP_ARGS(ca, found, inc_gen, inc_gen_skipped),
TP_STRUCT__entry(
- __array(char, uuid, 16 )
- __field(size_t, free )
- __field(size_t, total )
+ __field(dev_t, dev )
+ __field(u64, found )
+ __field(u64, inc_gen )
+ __field(u64, inc_gen_skipped )
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->uuid.b, 16);
- __entry->free = free;
- __entry->total = total;
+ __entry->dev = ca->disk_sb.bdev->bd_dev;
+ __entry->found = found;
+ __entry->inc_gen = inc_gen;
+ __entry->inc_gen_skipped = inc_gen_skipped;
),
- TP_printk("%pU free %zu total %zu",
- __entry->uuid, __entry->free, __entry->total)
+ TP_printk("%d,%d found %llu inc_gen %llu inc_gen_skipped %llu",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->found, __entry->inc_gen, __entry->inc_gen_skipped)
);
TRACE_EVENT(invalidate,
),
TP_printk("invalidated %u sectors at %d,%d sector=%llu",
- __entry->sectors, MAJOR(__entry->dev),
- MINOR(__entry->dev), __entry->offset)
-);
-
-DEFINE_EVENT(bch_fs, rescale_prios,
- TP_PROTO(struct bch_fs *c),
- TP_ARGS(c)
+ __entry->sectors,
+ MAJOR(__entry->dev),
+ MINOR(__entry->dev),
+ __entry->offset)
);
DECLARE_EVENT_CLASS(bucket_alloc,
TP_ARGS(ca, reserve),
TP_STRUCT__entry(
- __array(char, uuid, 16)
- __field(enum alloc_reserve, reserve )
+ __field(dev_t, dev )
+ __field(enum alloc_reserve, reserve )
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->uuid.b, 16);
- __entry->reserve = reserve;
+ __entry->dev = ca->disk_sb.bdev->bd_dev;
+ __entry->reserve = reserve;
),
- TP_printk("%pU reserve %d", __entry->uuid, __entry->reserve)
+ TP_printk("%d,%d reserve %d",
+ MAJOR(__entry->dev), MINOR(__entry->dev),
+ __entry->reserve)
);
DEFINE_EVENT(bucket_alloc, bucket_alloc,
TRACE_EVENT(trans_restart_would_deadlock,
TP_PROTO(unsigned long trans_ip,
unsigned long caller_ip,
+ bool in_traverse_all,
unsigned reason,
enum btree_id have_btree_id,
unsigned have_iter_type,
+ struct bpos *have_pos,
enum btree_id want_btree_id,
- unsigned want_iter_type),
- TP_ARGS(trans_ip, caller_ip, reason,
- have_btree_id, have_iter_type,
- want_btree_id, want_iter_type),
+ unsigned want_iter_type,
+ struct bpos *want_pos),
+ TP_ARGS(trans_ip, caller_ip, in_traverse_all, reason,
+ have_btree_id, have_iter_type, have_pos,
+ want_btree_id, want_iter_type, want_pos),
TP_STRUCT__entry(
__field(unsigned long, trans_ip )
__field(unsigned long, caller_ip )
+ __field(u8, in_traverse_all )
__field(u8, reason )
__field(u8, have_btree_id )
__field(u8, have_iter_type )
__field(u8, want_btree_id )
__field(u8, want_iter_type )
+
+ __field(u64, have_pos_inode )
+ __field(u64, have_pos_offset )
+ __field(u32, have_pos_snapshot)
+ __field(u32, want_pos_snapshot)
+ __field(u64, want_pos_inode )
+ __field(u64, want_pos_offset )
),
TP_fast_assign(
__entry->trans_ip = trans_ip;
__entry->caller_ip = caller_ip;
+ __entry->in_traverse_all = in_traverse_all;
__entry->reason = reason;
__entry->have_btree_id = have_btree_id;
__entry->have_iter_type = have_iter_type;
__entry->want_btree_id = want_btree_id;
__entry->want_iter_type = want_iter_type;
+
+ __entry->have_pos_inode = have_pos->inode;
+ __entry->have_pos_offset = have_pos->offset;
+ __entry->have_pos_snapshot = have_pos->snapshot;
+
+ __entry->want_pos_inode = want_pos->inode;
+ __entry->want_pos_offset = want_pos->offset;
+ __entry->want_pos_snapshot = want_pos->snapshot;
),
- TP_printk("%ps %pS because %u have %u:%u want %u:%u",
+ TP_printk("%ps %pS traverse_all %u because %u have %u:%u %llu:%llu:%u want %u:%u %llu:%llu:%u",
(void *) __entry->trans_ip,
(void *) __entry->caller_ip,
+ __entry->in_traverse_all,
__entry->reason,
__entry->have_btree_id,
__entry->have_iter_type,
+ __entry->have_pos_inode,
+ __entry->have_pos_offset,
+ __entry->have_pos_snapshot,
__entry->want_btree_id,
- __entry->want_iter_type)
-);
-
-TRACE_EVENT(trans_restart_iters_realloced,
- TP_PROTO(unsigned long ip, unsigned nr),
- TP_ARGS(ip, nr),
-
- TP_STRUCT__entry(
- __field(unsigned long, ip )
- __field(unsigned, nr )
- ),
-
- TP_fast_assign(
- __entry->ip = ip;
- __entry->nr = nr;
- ),
-
- TP_printk("%ps nr %u", (void *) __entry->ip, __entry->nr)
+ __entry->want_iter_type,
+ __entry->want_pos_inode,
+ __entry->want_pos_offset,
+ __entry->want_pos_snapshot)
);
TRACE_EVENT(trans_restart_mem_realloced,
- TP_PROTO(unsigned long ip, unsigned long bytes),
- TP_ARGS(ip, bytes),
+ TP_PROTO(unsigned long trans_ip, unsigned long caller_ip,
+ unsigned long bytes),
+ TP_ARGS(trans_ip, caller_ip, bytes),
TP_STRUCT__entry(
- __field(unsigned long, ip )
- __field(unsigned long, bytes )
+ __field(unsigned long, trans_ip )
+ __field(unsigned long, caller_ip )
+ __field(unsigned long, bytes )
),
TP_fast_assign(
- __entry->ip = ip;
- __entry->bytes = bytes;
+ __entry->trans_ip = trans_ip;
+ __entry->caller_ip = caller_ip;
+ __entry->bytes = bytes;
),
- TP_printk("%ps bytes %lu", (void *) __entry->ip, __entry->bytes)
+ TP_printk("%ps %pS bytes %lu",
+ (void *) __entry->trans_ip,
+ (void *) __entry->caller_ip,
+ __entry->bytes)
);
DEFINE_EVENT(transaction_restart, trans_restart_journal_res_get,
TP_ARGS(ip)
);
+DEFINE_EVENT(transaction_restart, trans_traverse_all,
+ TP_PROTO(unsigned long ip),
+ TP_ARGS(ip)
+);
+
DECLARE_EVENT_CLASS(node_lock_fail,
TP_PROTO(unsigned level, u32 iter_seq, unsigned node, u32 node_seq),
TP_ARGS(level, iter_seq, node, node_seq),
#include <linux/sort.h>
#include <trace/events/bcachefs.h>
+const char * const bch2_allocator_states[] = {
+#define x(n) #n,
+ ALLOC_THREAD_STATES()
+#undef x
+ NULL
+};
+
static const unsigned BCH_ALLOC_V1_FIELD_BYTES[] = {
#define x(name, bits) [BCH_ALLOC_FIELD_V1_##name] = bits / 8,
BCH_ALLOC_FIELDS_V1()
#undef x
};
-/* Ratelimiting/PD controllers */
-
-static void pd_controllers_update(struct work_struct *work)
-{
- struct bch_fs *c = container_of(to_delayed_work(work),
- struct bch_fs,
- pd_controllers_update);
- struct bch_dev *ca;
- s64 free = 0, fragmented = 0;
- unsigned i;
-
- for_each_member_device(ca, c, i) {
- struct bch_dev_usage stats = bch2_dev_usage_read(ca);
-
- free += bucket_to_sector(ca,
- __dev_buckets_available(ca, stats)) << 9;
- /*
- * Bytes of internal fragmentation, which can be
- * reclaimed by copy GC
- */
- fragmented += max_t(s64, 0, (bucket_to_sector(ca,
- stats.d[BCH_DATA_user].buckets +
- stats.d[BCH_DATA_cached].buckets) -
- (stats.d[BCH_DATA_user].sectors +
- stats.d[BCH_DATA_cached].sectors)) << 9);
- }
-
- bch2_pd_controller_update(&c->copygc_pd, free, fragmented, -1);
- schedule_delayed_work(&c->pd_controllers_update,
- c->pd_controllers_update_seconds * HZ);
-}
-
/* Persistent alloc info: */
static inline u64 alloc_field_v1_get(const struct bch_alloc *a,
bch2_alloc_pack_v2(dst, src);
}
-static unsigned bch_alloc_val_u64s(const struct bch_alloc *a)
+static unsigned bch_alloc_v1_val_u64s(const struct bch_alloc *a)
{
unsigned i, bytes = offsetof(struct bch_alloc, data);
return "invalid device";
/* allow for unknown fields */
- if (bkey_val_u64s(a.k) < bch_alloc_val_u64s(a.v))
+ if (bkey_val_u64s(a.k) < bch_alloc_v1_val_u64s(a.v))
return "incorrect value size";
return NULL;
{
struct bkey_alloc_unpacked u = bch2_alloc_unpack(k);
- pr_buf(out, "gen %u oldest_gen %u data_type %u",
- u.gen, u.oldest_gen, u.data_type);
-#define x(_name, ...) pr_buf(out, #_name " %llu ", (u64) u._name);
+ pr_buf(out, "gen %u oldest_gen %u data_type %s",
+ u.gen, u.oldest_gen, bch2_data_types[u.data_type]);
+#define x(_name, ...) pr_buf(out, " " #_name " %llu", (u64) u._name);
BCH_ALLOC_FIELDS_V2()
#undef x
}
ret = bch2_btree_and_journal_walk(c, journal_keys, BTREE_ID_alloc,
NULL, bch2_alloc_read_fn);
up_read(&c->gc_lock);
-
if (ret) {
bch_err(c, "error reading alloc info: %i", ret);
return ret;
* commands to the newly free buckets, then puts them on the various freelists.
*/
-/**
- * wait_buckets_available - wait on reclaimable buckets
- *
- * If there aren't enough available buckets to fill up free_inc, wait until
- * there are.
- */
-static int wait_buckets_available(struct bch_fs *c, struct bch_dev *ca)
-{
- unsigned long gc_count = c->gc_count;
- s64 available;
- int ret = 0;
-
- ca->allocator_state = ALLOCATOR_BLOCKED;
- closure_wake_up(&c->freelist_wait);
-
- while (1) {
- set_current_state(TASK_INTERRUPTIBLE);
- if (kthread_should_stop()) {
- ret = 1;
- break;
- }
-
- if (gc_count != c->gc_count)
- ca->inc_gen_really_needs_gc = 0;
-
- available = dev_buckets_reclaimable(ca);
- available -= ca->inc_gen_really_needs_gc;
-
- available = max(available, 0LL);
-
- if (available)
- break;
-
- up_read(&c->gc_lock);
- schedule();
- try_to_freeze();
- down_read(&c->gc_lock);
- }
-
- __set_current_state(TASK_RUNNING);
- ca->allocator_state = ALLOCATOR_RUNNING;
- closure_wake_up(&c->freelist_wait);
-
- return ret;
-}
-
static bool bch2_can_invalidate_bucket(struct bch_dev *ca, size_t b,
struct bucket_mark m)
{
gc_gen = bucket_gc_gen(bucket(ca, b));
- if (gc_gen >= BUCKET_GC_GEN_MAX / 2)
- ca->inc_gen_needs_gc++;
-
- if (gc_gen >= BUCKET_GC_GEN_MAX)
- ca->inc_gen_really_needs_gc++;
+ ca->inc_gen_needs_gc += gc_gen >= BUCKET_GC_GEN_MAX / 2;
+ ca->inc_gen_really_needs_gc += gc_gen >= BUCKET_GC_GEN_MAX;
return gc_gen < BUCKET_GC_GEN_MAX;
}
struct bucket_mark m = READ_ONCE(g->mark);
unsigned key = bucket_sort_key(g, m, now, last_seq_ondisk);
+ cond_resched();
+
if (!bch2_can_invalidate_bucket(ca, b, m))
continue;
.key = key,
};
}
-
- cond_resched();
}
if (e.nr)
size_t i, nr = 0;
ca->inc_gen_needs_gc = 0;
+ ca->inc_gen_really_needs_gc = 0;
switch (ca->mi.replacement) {
case BCH_CACHE_REPLACEMENT_lru:
return nr;
}
-static inline long next_alloc_bucket(struct bch_dev *ca)
-{
- struct alloc_heap_entry e, *top = ca->alloc_heap.data;
-
- while (ca->alloc_heap.used) {
- if (top->nr) {
- size_t b = top->bucket;
-
- top->bucket++;
- top->nr--;
- return b;
- }
-
- heap_pop(&ca->alloc_heap, e, bucket_alloc_cmp, NULL);
- }
-
- return -1;
-}
-
/*
* returns sequence number of most recent journal entry that updated this
* bucket:
}
}
-static int bch2_invalidate_one_bucket2(struct btree_trans *trans,
- struct bch_dev *ca,
- struct btree_iter *iter,
- u64 *journal_seq, unsigned flags)
+static int bucket_invalidate_btree(struct btree_trans *trans,
+ struct bch_dev *ca, u64 b)
{
struct bch_fs *c = trans->c;
- struct bkey_alloc_buf a;
+ struct bkey_alloc_buf *a;
struct bkey_alloc_unpacked u;
struct bucket *g;
struct bucket_mark m;
- bool invalidating_cached_data;
+ struct btree_iter *iter =
+ bch2_trans_get_iter(trans, BTREE_ID_alloc,
+ POS(ca->dev_idx, b),
+ BTREE_ITER_CACHED|
+ BTREE_ITER_CACHED_NOFILL|
+ BTREE_ITER_INTENT);
+ int ret;
+
+ a = bch2_trans_kmalloc(trans, sizeof(*a));
+ ret = PTR_ERR_OR_ZERO(a);
+ if (ret)
+ goto err;
+
+ ret = bch2_btree_iter_traverse(iter);
+ if (ret)
+ goto err;
+
+ percpu_down_read(&c->mark_lock);
+ g = bucket(ca, b);
+ m = READ_ONCE(g->mark);
+ u = alloc_mem_to_key(iter, g, m);
+ percpu_up_read(&c->mark_lock);
+
+ u.gen++;
+ u.data_type = 0;
+ u.dirty_sectors = 0;
+ u.cached_sectors = 0;
+ u.read_time = atomic64_read(&c->io_clock[READ].now);
+ u.write_time = atomic64_read(&c->io_clock[WRITE].now);
+
+ bch2_alloc_pack(c, a, u);
+ bch2_trans_update(trans, iter, &a->k, BTREE_TRIGGER_BUCKET_INVALIDATE);
+err:
+ bch2_trans_iter_put(trans, iter);
+ return ret;
+}
+
+static int bch2_invalidate_one_bucket(struct bch_fs *c, struct bch_dev *ca,
+ u64 *journal_seq, unsigned flags)
+{
+ struct bucket *g;
+ struct bucket_mark m;
size_t b;
int ret = 0;
BUG_ON(m.dirty_sectors);
- bch2_mark_alloc_bucket(c, ca, b, true, gc_pos_alloc(c, NULL), 0);
+ bch2_mark_alloc_bucket(c, ca, b, true);
spin_lock(&c->freelist_lock);
verify_not_on_freelist(c, ca, b);
goto out;
}
- bch2_btree_iter_set_pos(iter, POS(ca->dev_idx, b));
-retry:
- ret = bch2_btree_iter_traverse(iter);
- if (ret)
- return ret;
-
- percpu_down_read(&c->mark_lock);
- g = bucket(ca, iter->pos.offset);
- m = READ_ONCE(g->mark);
- u = alloc_mem_to_key(iter, g, m);
-
- percpu_up_read(&c->mark_lock);
-
- invalidating_cached_data = u.cached_sectors != 0;
-
- u.gen++;
- u.data_type = 0;
- u.dirty_sectors = 0;
- u.cached_sectors = 0;
- u.read_time = atomic64_read(&c->io_clock[READ].now);
- u.write_time = atomic64_read(&c->io_clock[WRITE].now);
-
- bch2_alloc_pack(c, &a, u);
- bch2_trans_update(trans, iter, &a.k,
- BTREE_TRIGGER_BUCKET_INVALIDATE);
-
- /*
- * XXX:
- * when using deferred btree updates, we have journal reclaim doing
- * btree updates and thus requiring the allocator to make forward
- * progress, and here the allocator is requiring space in the journal -
- * so we need a journal pre-reservation:
- */
- ret = bch2_trans_commit(trans, NULL,
- invalidating_cached_data ? journal_seq : NULL,
- BTREE_INSERT_NOUNLOCK|
- BTREE_INSERT_NOCHECK_RW|
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_JOURNAL_RESERVED|
- flags);
- if (ret == -EINTR)
- goto retry;
+ ret = bch2_trans_do(c, NULL, journal_seq,
+ BTREE_INSERT_NOCHECK_RW|
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_JOURNAL_RESERVED|
+ flags,
+ bucket_invalidate_btree(&trans, ca, b));
out:
if (!ret) {
/* remove from alloc_heap: */
percpu_down_read(&c->mark_lock);
spin_lock(&c->freelist_lock);
- bch2_mark_alloc_bucket(c, ca, b, false,
- gc_pos_alloc(c, NULL), 0);
+ bch2_mark_alloc_bucket(c, ca, b, false);
BUG_ON(!fifo_pop_back(&ca->free_inc, b2));
BUG_ON(b != b2);
*/
static int bch2_invalidate_buckets(struct bch_fs *c, struct bch_dev *ca)
{
- struct btree_trans trans;
- struct btree_iter *iter;
u64 journal_seq = 0;
int ret = 0;
- bch2_trans_init(&trans, c, 0, 0);
- iter = bch2_trans_get_iter(&trans, BTREE_ID_alloc,
- POS(ca->dev_idx, 0),
- BTREE_ITER_CACHED|
- BTREE_ITER_CACHED_NOFILL|
- BTREE_ITER_INTENT);
-
/* Only use nowait if we've already invalidated at least one bucket: */
while (!ret &&
!fifo_full(&ca->free_inc) &&
- ca->alloc_heap.used)
- ret = bch2_invalidate_one_bucket2(&trans, ca, iter, &journal_seq,
- BTREE_INSERT_GC_LOCK_HELD|
+ ca->alloc_heap.used) {
+ ret = bch2_invalidate_one_bucket(c, ca, &journal_seq,
(!fifo_empty(&ca->free_inc)
? BTREE_INSERT_NOWAIT : 0));
-
- bch2_trans_iter_put(&trans, iter);
- bch2_trans_exit(&trans);
+ /*
+ * We only want to batch up invalidates when they're going to
+ * require flushing the journal:
+ */
+ if (!journal_seq)
+ break;
+ }
/* If we used NOWAIT, don't return the error: */
if (!fifo_empty(&ca->free_inc))
return 0;
}
-static int push_invalidated_bucket(struct bch_fs *c, struct bch_dev *ca, size_t bucket)
+static void alloc_thread_set_state(struct bch_dev *ca, unsigned new_state)
+{
+ if (ca->allocator_state != new_state) {
+ ca->allocator_state = new_state;
+ closure_wake_up(&ca->fs->freelist_wait);
+ }
+}
+
+static int push_invalidated_bucket(struct bch_fs *c, struct bch_dev *ca, u64 b)
{
unsigned i;
int ret = 0;
- while (1) {
- set_current_state(TASK_INTERRUPTIBLE);
-
- spin_lock(&c->freelist_lock);
- for (i = 0; i < RESERVE_NR; i++) {
-
- /*
- * Don't strand buckets on the copygc freelist until
- * after recovery is finished:
- */
- if (!test_bit(BCH_FS_STARTED, &c->flags) &&
- i == RESERVE_MOVINGGC)
- continue;
-
- if (fifo_push(&ca->free[i], bucket)) {
- fifo_pop(&ca->free_inc, bucket);
-
- closure_wake_up(&c->freelist_wait);
- ca->allocator_state = ALLOCATOR_RUNNING;
-
- spin_unlock(&c->freelist_lock);
- goto out;
- }
- }
-
- if (ca->allocator_state != ALLOCATOR_BLOCKED_FULL) {
- ca->allocator_state = ALLOCATOR_BLOCKED_FULL;
- closure_wake_up(&c->freelist_wait);
- }
-
- spin_unlock(&c->freelist_lock);
+ spin_lock(&c->freelist_lock);
+ for (i = 0; i < RESERVE_NR; i++) {
+ /*
+ * Don't strand buckets on the copygc freelist until
+ * after recovery is finished:
+ */
+ if (i == RESERVE_MOVINGGC &&
+ !test_bit(BCH_FS_STARTED, &c->flags))
+ continue;
- if ((current->flags & PF_KTHREAD) &&
- kthread_should_stop()) {
+ if (fifo_push(&ca->free[i], b)) {
+ fifo_pop(&ca->free_inc, b);
ret = 1;
break;
}
-
- schedule();
- try_to_freeze();
}
-out:
- __set_current_state(TASK_RUNNING);
+ spin_unlock(&c->freelist_lock);
+
+ ca->allocator_state = ret
+ ? ALLOCATOR_running
+ : ALLOCATOR_blocked_full;
+ closure_wake_up(&c->freelist_wait);
return ret;
}
-/*
- * Pulls buckets off free_inc, discards them (if enabled), then adds them to
- * freelists, waiting until there's room if necessary:
- */
-static int discard_invalidated_buckets(struct bch_fs *c, struct bch_dev *ca)
+static void discard_one_bucket(struct bch_fs *c, struct bch_dev *ca, u64 b)
{
- while (!fifo_empty(&ca->free_inc)) {
- size_t bucket = fifo_peek(&ca->free_inc);
-
- if (ca->mi.discard &&
- blk_queue_discard(bdev_get_queue(ca->disk_sb.bdev)))
- blkdev_issue_discard(ca->disk_sb.bdev,
- bucket_to_sector(ca, bucket),
- ca->mi.bucket_size, GFP_NOIO, 0);
-
- if (push_invalidated_bucket(c, ca, bucket))
- return 1;
- }
+ if (ca->mi.discard &&
+ blk_queue_discard(bdev_get_queue(ca->disk_sb.bdev)))
+ blkdev_issue_discard(ca->disk_sb.bdev, bucket_to_sector(ca, b),
+ ca->mi.bucket_size, GFP_NOFS, 0);
+}
- return 0;
+static bool allocator_thread_running(struct bch_dev *ca)
+{
+ unsigned state = ca->mi.state == BCH_MEMBER_STATE_rw &&
+ test_bit(BCH_FS_ALLOCATOR_RUNNING, &ca->fs->flags)
+ ? ALLOCATOR_running
+ : ALLOCATOR_stopped;
+ alloc_thread_set_state(ca, state);
+ return state == ALLOCATOR_running;
}
-static inline bool allocator_thread_running(struct bch_dev *ca)
+static int buckets_available(struct bch_dev *ca, unsigned long gc_count)
{
- return ca->mi.state == BCH_MEMBER_STATE_rw &&
- test_bit(BCH_FS_ALLOCATOR_RUNNING, &ca->fs->flags);
+ s64 available = dev_buckets_reclaimable(ca) -
+ (gc_count == ca->fs->gc_count ? ca->inc_gen_really_needs_gc : 0);
+ bool ret = available > 0;
+
+ alloc_thread_set_state(ca, ret
+ ? ALLOCATOR_running
+ : ALLOCATOR_blocked);
+ return ret;
}
/**
{
struct bch_dev *ca = arg;
struct bch_fs *c = ca->fs;
+ unsigned long gc_count = c->gc_count;
size_t nr;
int ret;
set_freezable();
while (1) {
- if (!allocator_thread_running(ca)) {
- ca->allocator_state = ALLOCATOR_STOPPED;
- if (kthread_wait_freezable(allocator_thread_running(ca)))
- break;
- }
-
- ca->allocator_state = ALLOCATOR_RUNNING;
-
- cond_resched();
- if (kthread_should_stop())
- break;
-
- pr_debug("discarding %zu invalidated buckets",
- fifo_used(&ca->free_inc));
-
- ret = discard_invalidated_buckets(c, ca);
+ ret = kthread_wait_freezable(allocator_thread_running(ca));
if (ret)
goto stop;
- down_read(&c->gc_lock);
-
- ret = bch2_invalidate_buckets(c, ca);
- if (ret) {
- up_read(&c->gc_lock);
- goto stop;
- }
-
- if (!fifo_empty(&ca->free_inc)) {
- up_read(&c->gc_lock);
- continue;
- }
-
- pr_debug("free_inc now empty");
-
- while (1) {
+ while (!ca->alloc_heap.used) {
cond_resched();
- /*
- * Find some buckets that we can invalidate, either
- * they're completely unused, or only contain clean data
- * that's been written back to the backing device or
- * another cache tier
- */
- pr_debug("scanning for reclaimable buckets");
+ ret = kthread_wait_freezable(buckets_available(ca, gc_count));
+ if (ret)
+ goto stop;
+ gc_count = c->gc_count;
nr = find_reclaimable_buckets(c, ca);
- pr_debug("found %zu buckets", nr);
-
- trace_alloc_batch(ca, nr, ca->alloc_heap.size);
+ trace_alloc_scan(ca, nr, ca->inc_gen_needs_gc,
+ ca->inc_gen_really_needs_gc);
if ((ca->inc_gen_needs_gc >= ALLOC_SCAN_BATCH(ca) ||
ca->inc_gen_really_needs_gc) &&
atomic_inc(&c->kick_gc);
wake_up_process(c->gc_thread);
}
-
- if (nr)
- break;
-
- /*
- * If we found any buckets, we have to invalidate them
- * before we scan for more - but if we didn't find very
- * many we may want to wait on more buckets being
- * available so we don't spin:
- */
- ret = wait_buckets_available(c, ca);
- if (ret) {
- up_read(&c->gc_lock);
- goto stop;
- }
}
- up_read(&c->gc_lock);
+ ret = bch2_invalidate_buckets(c, ca);
+ if (ret)
+ goto stop;
- pr_debug("%zu buckets to invalidate", nr);
+ while (!fifo_empty(&ca->free_inc)) {
+ u64 b = fifo_peek(&ca->free_inc);
- /*
- * alloc_heap is now full of newly-invalidated buckets: next,
- * write out the new bucket gens:
- */
- }
+ discard_one_bucket(c, ca, b);
+ ret = kthread_wait_freezable(push_invalidated_bucket(c, ca, b));
+ if (ret)
+ goto stop;
+ }
+ }
stop:
- pr_debug("alloc thread stopping (ret %i)", ret);
- ca->allocator_state = ALLOCATOR_STOPPED;
- closure_wake_up(&c->freelist_wait);
+ alloc_thread_set_state(ca, ALLOCATOR_stopped);
return 0;
}
void bch2_recalc_capacity(struct bch_fs *c)
{
struct bch_dev *ca;
- u64 capacity = 0, reserved_sectors = 0, gc_reserve, copygc_threshold = 0;
+ u64 capacity = 0, reserved_sectors = 0, gc_reserve;
unsigned bucket_size_max = 0;
unsigned long ra_pages = 0;
unsigned i, j;
dev_reserve *= ca->mi.bucket_size;
- copygc_threshold += dev_reserve;
-
capacity += bucket_to_sector(ca, ca->mi.nbuckets -
ca->mi.first_bucket);
reserved_sectors = min(reserved_sectors, capacity);
- c->copygc_threshold = copygc_threshold;
c->capacity = capacity - reserved_sectors;
c->bucket_size_max = bucket_size_max;
{
if (ca->alloc_thread)
closure_wait_event(&c->freelist_wait,
- ca->allocator_state != ALLOCATOR_RUNNING);
+ ca->allocator_state != ALLOCATOR_running);
}
/* stop allocator thread: */
void bch2_fs_allocator_background_init(struct bch_fs *c)
{
spin_lock_init(&c->freelist_lock);
-
- c->pd_controllers_update_seconds = 5;
- INIT_DELAYED_WORK(&c->pd_controllers_update, pd_controllers_update);
}
#include "alloc_types.h"
#include "debug.h"
+extern const char * const bch2_allocator_states[];
+
struct bkey_alloc_unpacked {
u64 bucket;
u8 dev;
rcu_read_lock();
p = rcu_dereference(ca->alloc_thread);
- if (p) {
+ if (p)
wake_up_process(p);
- ca->allocator_state = ALLOCATOR_RUNNING;
- }
rcu_read_unlock();
}
// SPDX-License-Identifier: GPL-2.0
/*
- * Primary bucket allocation code
- *
* Copyright 2012 Google, Inc.
*
- * Allocation in bcache is done in terms of buckets:
- *
- * Each bucket has associated an 8 bit gen; this gen corresponds to the gen in
- * btree pointers - they must match for the pointer to be considered valid.
- *
- * Thus (assuming a bucket has no dirty data or metadata in it) we can reuse a
- * bucket simply by incrementing its gen.
- *
- * The gens (along with the priorities; it's really the gens are important but
- * the code is named as if it's the priorities) are written in an arbitrary list
- * of buckets on disk, with a pointer to them in the journal header.
- *
- * When we invalidate a bucket, we have to write its new gen to disk and wait
- * for that write to complete before we use it - otherwise after a crash we
- * could have pointers that appeared to be good but pointed to data that had
- * been overwritten.
- *
- * Since the gens and priorities are all stored contiguously on disk, we can
- * batch this up: We fill up the free_inc list with freshly invalidated buckets,
- * call prio_write(), and when prio_write() finishes we pull buckets off the
- * free_inc list and optionally discard them.
- *
- * free_inc isn't the only freelist - if it was, we'd often have to sleep while
- * priorities and gens were being written before we could allocate. c->free is a
- * smaller freelist, and buckets on that list are always ready to be used.
- *
- * If we've got discards enabled, that happens when a bucket moves from the
- * free_inc list to the free list.
- *
- * It's important to ensure that gens don't wrap around - with respect to
- * either the oldest gen in the btree or the gen on disk. This is quite
- * difficult to do in practice, but we explicitly guard against it anyways - if
- * a bucket is in danger of wrapping around we simply skip invalidating it that
- * time around, and we garbage collect or rewrite the priorities sooner than we
- * would have otherwise.
+ * Foreground allocator code: allocate buckets from freelist, and allocate in
+ * sector granularity from writepoints.
*
* bch2_bucket_alloc() allocates a single bucket from a specific device.
*
* bch2_bucket_alloc_set() allocates one or more buckets from different devices
* in a given filesystem.
- *
- * invalidate_buckets() drives all the processes described above. It's called
- * from bch2_bucket_alloc() and a few other places that need to make sure free
- * buckets are ready.
- *
- * invalidate_buckets_(lru|fifo)() find buckets that are available to be
- * invalidated, and then invalidate them and stick them on the free_inc list -
- * in either lru or fifo order.
*/
#include "bcachefs.h"
percpu_down_read(&c->mark_lock);
spin_lock(&ob->lock);
- bch2_mark_alloc_bucket(c, ca, PTR_BUCKET_NR(ca, &ob->ptr),
- false, gc_pos_alloc(c, ob), 0);
+ bch2_mark_alloc_bucket(c, ca, PTR_BUCKET_NR(ca, &ob->ptr), false);
ob->valid = false;
ob->type = 0;
spin_lock(&c->freelist_lock);
ob->freelist = c->open_buckets_freelist;
c->open_buckets_freelist = ob - c->open_buckets;
+
c->open_buckets_nr_free++;
+ ca->nr_open_buckets--;
spin_unlock(&c->freelist_lock);
closure_wake_up(&c->open_buckets_wait);
c->blocked_allocate = 0;
}
+ ca->nr_open_buckets++;
spin_unlock(&c->freelist_lock);
bch2_wake_allocator(ca);
{
struct write_point *wp;
+ rcu_read_lock();
hlist_for_each_entry_rcu(wp, head, node)
if (wp->write_point == write_point)
- return wp;
-
- return NULL;
+ goto out;
+ wp = NULL;
+out:
+ rcu_read_unlock();
+ return wp;
}
static inline bool too_many_writepoints(struct bch_fs *c, unsigned factor)
struct ec_bucket_buf;
+#define ALLOC_THREAD_STATES() \
+ x(stopped) \
+ x(running) \
+ x(blocked) \
+ x(blocked_full)
+
+enum allocator_states {
+#define x(n) ALLOCATOR_##n,
+ ALLOC_THREAD_STATES()
+#undef x
+};
+
enum alloc_reserve {
RESERVE_BTREE_MOVINGGC = -2,
RESERVE_BTREE = -1,
GC_PHASE_BTREE_reflink,
GC_PHASE_PENDING_DELETE,
- GC_PHASE_ALLOC,
};
struct gc_pos {
*/
alloc_fifo free[RESERVE_NR];
alloc_fifo free_inc;
+ unsigned nr_open_buckets;
open_bucket_idx_t open_buckets_partial[OPEN_BUCKETS_COUNT];
open_bucket_idx_t open_buckets_partial_nr;
size_t inc_gen_needs_gc;
size_t inc_gen_really_needs_gc;
- /*
- * XXX: this should be an enum for allocator state, so as to include
- * error state
- */
- enum {
- ALLOCATOR_STOPPED,
- ALLOCATOR_RUNNING,
- ALLOCATOR_BLOCKED,
- ALLOCATOR_BLOCKED_FULL,
- } allocator_state;
+ enum allocator_states allocator_state;
alloc_heap alloc_heap;
struct workqueue_struct *copygc_wq;
/* ALLOCATION */
- struct delayed_work pd_controllers_update;
- unsigned pd_controllers_update_seconds;
-
struct bch_devs_mask rw_devs[BCH_DATA_NR];
u64 capacity; /* sectors */
atomic_t kick_gc;
unsigned long gc_count;
+ enum btree_id gc_gens_btree;
+ struct bpos gc_gens_pos;
+
/*
* Tracks GC's progress - everything in the range [ZERO_KEY..gc_cur_pos]
* has been marked by GC.
/* COPYGC */
struct task_struct *copygc_thread;
copygc_heap copygc_heap;
- struct bch_pd_controller copygc_pd;
struct write_point copygc_write_point;
- u64 copygc_threshold;
+ s64 copygc_wait;
/* STRIPES: */
GENRADIX(struct stripe) stripes[2];
return bch2_bkey_ops[k.k->type].key_invalid(c, k);
}
+static unsigned bch2_key_types_allowed[] = {
+ [BKEY_TYPE_extents] =
+ (1U << KEY_TYPE_discard)|
+ (1U << KEY_TYPE_error)|
+ (1U << KEY_TYPE_extent)|
+ (1U << KEY_TYPE_reservation)|
+ (1U << KEY_TYPE_reflink_p)|
+ (1U << KEY_TYPE_inline_data),
+ [BKEY_TYPE_inodes] =
+ (1U << KEY_TYPE_inode)|
+ (1U << KEY_TYPE_inode_generation),
+ [BKEY_TYPE_dirents] =
+ (1U << KEY_TYPE_hash_whiteout)|
+ (1U << KEY_TYPE_dirent),
+ [BKEY_TYPE_xattrs] =
+ (1U << KEY_TYPE_hash_whiteout)|
+ (1U << KEY_TYPE_xattr),
+ [BKEY_TYPE_alloc] =
+ (1U << KEY_TYPE_alloc)|
+ (1U << KEY_TYPE_alloc_v2),
+ [BKEY_TYPE_quotas] =
+ (1U << KEY_TYPE_quota),
+ [BKEY_TYPE_stripes] =
+ (1U << KEY_TYPE_stripe),
+ [BKEY_TYPE_reflink] =
+ (1U << KEY_TYPE_reflink_v)|
+ (1U << KEY_TYPE_indirect_inline_data),
+ [BKEY_TYPE_btree] =
+ (1U << KEY_TYPE_btree_ptr)|
+ (1U << KEY_TYPE_btree_ptr_v2),
+};
+
const char *__bch2_bkey_invalid(struct bch_fs *c, struct bkey_s_c k,
enum btree_node_type type)
{
+ unsigned key_types_allowed = (1U << KEY_TYPE_deleted)|
+ bch2_key_types_allowed[type] ;
+
if (k.k->u64s < BKEY_U64s)
return "u64s too small";
+ if (!(key_types_allowed & (1U << k.k->type)))
+ return "invalid key type for this btree";
+
if (type == BKEY_TYPE_btree &&
bkey_val_u64s(k.k) > BKEY_BTREE_PTR_VAL_U64s_MAX)
return "value too big";
bkey_reassemble(new, *k);
- bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, ({
- struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
- struct bucket *g = PTR_BUCKET(ca, ptr, true);
-
- (ptr->cached &&
- (!g->gen_valid || gen_cmp(ptr->gen, g->mark.gen) > 0)) ||
- (!ptr->cached &&
- gen_cmp(ptr->gen, g->mark.gen) < 0);
- }));
+ if (level) {
+ /*
+ * We don't want to drop btree node pointers - if the
+ * btree node isn't there anymore, the read path will
+ * sort it out:
+ */
+ ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
+ bkey_for_each_ptr(ptrs, ptr) {
+ struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+ struct bucket *g = PTR_BUCKET(ca, ptr, true);
+
+ ptr->gen = g->mark.gen;
+ }
+ } else {
+ bch2_bkey_drop_ptrs(bkey_i_to_s(new), ptr, ({
+ struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
+ struct bucket *g = PTR_BUCKET(ca, ptr, true);
+
+ (ptr->cached &&
+ (!g->gen_valid || gen_cmp(ptr->gen, g->mark.gen) > 0)) ||
+ (!ptr->cached &&
+ gen_cmp(ptr->gen, g->mark.gen) < 0);
+ }));
again:
- ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
- bkey_extent_entry_for_each(ptrs, entry) {
- if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) {
- struct stripe *m = genradix_ptr(&c->stripes[true],
- entry->stripe_ptr.idx);
- union bch_extent_entry *next_ptr;
-
- bkey_extent_entry_for_each_from(ptrs, next_ptr, entry)
- if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr)
- goto found;
- next_ptr = NULL;
+ ptrs = bch2_bkey_ptrs(bkey_i_to_s(new));
+ bkey_extent_entry_for_each(ptrs, entry) {
+ if (extent_entry_type(entry) == BCH_EXTENT_ENTRY_stripe_ptr) {
+ struct stripe *m = genradix_ptr(&c->stripes[true],
+ entry->stripe_ptr.idx);
+ union bch_extent_entry *next_ptr;
+
+ bkey_extent_entry_for_each_from(ptrs, next_ptr, entry)
+ if (extent_entry_type(next_ptr) == BCH_EXTENT_ENTRY_ptr)
+ goto found;
+ next_ptr = NULL;
found:
- if (!next_ptr) {
- bch_err(c, "aieee, found stripe ptr with no data ptr");
- continue;
- }
-
- if (!m || !m->alive ||
- !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block],
- &next_ptr->ptr,
- m->sectors)) {
- bch2_bkey_extent_entry_drop(new, entry);
- goto again;
+ if (!next_ptr) {
+ bch_err(c, "aieee, found stripe ptr with no data ptr");
+ continue;
+ }
+
+ if (!m || !m->alive ||
+ !__bch2_ptr_matches_stripe(&m->ptrs[entry->stripe_ptr.block],
+ &next_ptr->ptr,
+ m->sectors)) {
+ bch2_bkey_extent_entry_drop(new, entry);
+ goto again;
+ }
}
}
}
static int bch2_gc_mark_key(struct bch_fs *c, enum btree_id btree_id,
unsigned level, bool is_root,
- struct bkey_s_c k,
+ struct bkey_s_c *k,
u8 *max_stale, bool initial)
{
- struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k);
+ struct bkey_ptrs_c ptrs;
const struct bch_extent_ptr *ptr;
unsigned flags =
BTREE_TRIGGER_GC|
if (initial) {
BUG_ON(bch2_journal_seq_verify &&
- k.k->version.lo > journal_cur_seq(&c->journal));
+ k->k->version.lo > journal_cur_seq(&c->journal));
- if (fsck_err_on(k.k->version.lo > atomic64_read(&c->key_version), c,
+ if (fsck_err_on(k->k->version.lo > atomic64_read(&c->key_version), c,
"key version number higher than recorded: %llu > %llu",
- k.k->version.lo,
+ k->k->version.lo,
atomic64_read(&c->key_version)))
- atomic64_set(&c->key_version, k.k->version.lo);
+ atomic64_set(&c->key_version, k->k->version.lo);
if (test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags) ||
- fsck_err_on(!bch2_bkey_replicas_marked(c, k), c,
+ fsck_err_on(!bch2_bkey_replicas_marked(c, *k), c,
"superblock not marked as containing replicas (type %u)",
- k.k->type)) {
- ret = bch2_mark_bkey_replicas(c, k);
+ k->k->type)) {
+ ret = bch2_mark_bkey_replicas(c, *k);
if (ret) {
bch_err(c, "error marking bkey replicas: %i", ret);
goto err;
}
}
- ret = bch2_check_fix_ptrs(c, btree_id, level, is_root, &k);
+ ret = bch2_check_fix_ptrs(c, btree_id, level, is_root, k);
}
+ ptrs = bch2_bkey_ptrs_c(*k);
bkey_for_each_ptr(ptrs, ptr) {
struct bch_dev *ca = bch_dev_bkey_exists(c, ptr->dev);
struct bucket *g = PTR_BUCKET(ca, ptr, true);
*max_stale = max(*max_stale, ptr_stale(ca, ptr));
}
- bch2_mark_key(c, k, 0, k.k->size, NULL, 0, flags);
+ bch2_mark_key(c, *k, 0, k->k->size, NULL, 0, flags);
fsck_err:
err:
if (ret)
while ((k = bch2_btree_node_iter_peek_unpack(&iter, b, &unpacked)).k) {
ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, false,
- k, max_stale, initial);
+ &k, max_stale, initial);
if (ret)
break;
}
static int bch2_gc_btree(struct bch_fs *c, enum btree_id btree_id,
- bool initial)
+ bool initial, bool metadata_only)
{
struct btree_trans trans;
struct btree_iter *iter;
struct btree *b;
- unsigned depth = bch2_expensive_debug_checks ? 0
+ unsigned depth = metadata_only ? 1
+ : bch2_expensive_debug_checks ? 0
: !btree_node_type_needs_gc(btree_id) ? 1
: 0;
u8 max_stale = 0;
mutex_lock(&c->btree_root_lock);
b = c->btree_roots[btree_id].b;
- if (!btree_node_fake(b))
+ if (!btree_node_fake(b)) {
+ struct bkey_s_c k = bkey_i_to_s_c(&b->key);
+
ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, true,
- bkey_i_to_s_c(&b->key),
- &max_stale, initial);
+ &k, &max_stale, initial);
+ }
gc_pos_set(c, gc_pos_btree_root(b->c.btree_id));
mutex_unlock(&c->btree_root_lock);
BUG_ON(bpos_cmp(k.k->p, b->data->max_key) > 0);
ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, false,
- k, &max_stale, true);
+ &k, &max_stale, true);
if (ret) {
bch_err(c, "%s: error %i from bch2_gc_mark_key", __func__, ret);
break;
}
static int bch2_gc_btree_init(struct bch_fs *c,
- enum btree_id btree_id)
+ enum btree_id btree_id,
+ bool metadata_only)
{
struct btree *b;
- unsigned target_depth = bch2_expensive_debug_checks ? 0
- : !btree_node_type_needs_gc(btree_id) ? 1
+ unsigned target_depth = metadata_only ? 1
+ : bch2_expensive_debug_checks ? 0
+ : !btree_node_type_needs_gc(btree_id) ? 1
: 0;
u8 max_stale = 0;
char buf[100];
if (b->c.level >= target_depth)
ret = bch2_gc_btree_init_recurse(c, b, target_depth);
- if (!ret)
+ if (!ret) {
+ struct bkey_s_c k = bkey_i_to_s_c(&b->key);
+
ret = bch2_gc_mark_key(c, b->c.btree_id, b->c.level, true,
- bkey_i_to_s_c(&b->key),
- &max_stale, true);
+ &k, &max_stale, true);
+ }
fsck_err:
six_unlock_read(&b->c.lock);
(int) btree_id_to_gc_phase(r);
}
-static int bch2_gc_btrees(struct bch_fs *c, bool initial)
+static int bch2_gc_btrees(struct bch_fs *c, bool initial, bool metadata_only)
{
enum btree_id ids[BTREE_ID_NR];
unsigned i;
for (i = 0; i < BTREE_ID_NR; i++) {
enum btree_id id = ids[i];
int ret = initial
- ? bch2_gc_btree_init(c, id)
- : bch2_gc_btree(c, id, initial);
+ ? bch2_gc_btree_init(c, id, metadata_only)
+ : bch2_gc_btree(c, id, initial, metadata_only);
if (ret) {
bch_err(c, "%s: ret %i", __func__, ret);
return ret;
}
#endif
-static void bch2_mark_allocator_buckets(struct bch_fs *c)
-{
- struct bch_dev *ca;
- struct open_bucket *ob;
- size_t i, j, iter;
- unsigned ci;
-
- percpu_down_read(&c->mark_lock);
-
- spin_lock(&c->freelist_lock);
- gc_pos_set(c, gc_pos_alloc(c, NULL));
-
- for_each_member_device(ca, c, ci) {
- fifo_for_each_entry(i, &ca->free_inc, iter)
- bch2_mark_alloc_bucket(c, ca, i, true,
- gc_pos_alloc(c, NULL),
- BTREE_TRIGGER_GC);
-
-
-
- for (j = 0; j < RESERVE_NR; j++)
- fifo_for_each_entry(i, &ca->free[j], iter)
- bch2_mark_alloc_bucket(c, ca, i, true,
- gc_pos_alloc(c, NULL),
- BTREE_TRIGGER_GC);
- }
-
- spin_unlock(&c->freelist_lock);
-
- for (ob = c->open_buckets;
- ob < c->open_buckets + ARRAY_SIZE(c->open_buckets);
- ob++) {
- spin_lock(&ob->lock);
- if (ob->valid) {
- gc_pos_set(c, gc_pos_alloc(c, ob));
- ca = bch_dev_bkey_exists(c, ob->ptr.dev);
- bch2_mark_alloc_bucket(c, ca, PTR_BUCKET_NR(ca, &ob->ptr), true,
- gc_pos_alloc(c, ob),
- BTREE_TRIGGER_GC);
- }
- spin_unlock(&ob->lock);
- }
-
- percpu_up_read(&c->mark_lock);
-}
-
static void bch2_gc_free(struct bch_fs *c)
{
struct bch_dev *ca;
}
static int bch2_gc_done(struct bch_fs *c,
- bool initial)
+ bool initial, bool metadata_only)
{
struct bch_dev *ca;
- bool verify = (!initial ||
+ bool verify = !metadata_only && (!initial ||
(c->sb.compat & (1ULL << BCH_COMPAT_alloc_info)));
unsigned i, dev;
int ret = 0;
if (dst->b[b].mark._f != src->b[b].mark._f) { \
if (verify) \
fsck_err(c, "bucket %u:%zu gen %u data type %s has wrong " #_f \
- ": got %u, should be %u", i, b, \
+ ": got %u, should be %u", dev, b, \
dst->b[b].mark.gen, \
bch2_data_types[dst->b[b].mark.data_type],\
dst->b[b].mark._f, src->b[b].mark._f); \
set_bit(BCH_FS_NEED_ALLOC_WRITE, &c->flags); \
}
#define copy_dev_field(_f, _msg, ...) \
- copy_field(_f, "dev %u has wrong " _msg, i, ##__VA_ARGS__)
+ copy_field(_f, "dev %u has wrong " _msg, dev, ##__VA_ARGS__)
#define copy_fs_field(_f, _msg, ...) \
copy_field(_f, "fs has wrong " _msg, ##__VA_ARGS__)
- {
+ if (!metadata_only) {
struct genradix_iter iter = genradix_iter_init(&c->stripes[1], 0);
struct stripe *dst, *src;
for (b = 0; b < src->nbuckets; b++) {
copy_bucket_field(gen);
copy_bucket_field(data_type);
- copy_bucket_field(owned_by_allocator);
copy_bucket_field(stripe);
copy_bucket_field(dirty_sectors);
copy_bucket_field(cached_sectors);
copy_fs_field(hidden, "hidden");
copy_fs_field(btree, "btree");
- copy_fs_field(data, "data");
- copy_fs_field(cached, "cached");
- copy_fs_field(reserved, "reserved");
- copy_fs_field(nr_inodes,"nr_inodes");
- for (i = 0; i < BCH_REPLICAS_MAX; i++)
- copy_fs_field(persistent_reserved[i],
- "persistent_reserved[%i]", i);
+ if (!metadata_only) {
+ copy_fs_field(data, "data");
+ copy_fs_field(cached, "cached");
+ copy_fs_field(reserved, "reserved");
+ copy_fs_field(nr_inodes,"nr_inodes");
+
+ for (i = 0; i < BCH_REPLICAS_MAX; i++)
+ copy_fs_field(persistent_reserved[i],
+ "persistent_reserved[%i]", i);
+ }
for (i = 0; i < c->replicas.nr; i++) {
struct bch_replicas_entry *e =
cpu_replicas_entry(&c->replicas, i);
char buf[80];
+ if (metadata_only &&
+ (e->data_type == BCH_DATA_user ||
+ e->data_type == BCH_DATA_cached))
+ continue;
+
bch2_replicas_entry_to_text(&PBUF(buf), e);
copy_fs_field(replicas[i], "%s", buf);
return ret;
}
-static int bch2_gc_start(struct bch_fs *c)
+static int bch2_gc_start(struct bch_fs *c,
+ bool metadata_only)
{
struct bch_dev *ca;
unsigned i;
d->_mark.gen = dst->b[b].oldest_gen = s->mark.gen;
d->gen_valid = s->gen_valid;
+
+ if (metadata_only &&
+ (s->mark.data_type == BCH_DATA_user ||
+ s->mark.data_type == BCH_DATA_cached))
+ d->_mark = s->mark;
}
};
* move around - if references move backwards in the ordering GC
* uses, GC could skip past them
*/
-int bch2_gc(struct bch_fs *c, bool initial)
+int bch2_gc(struct bch_fs *c, bool initial, bool metadata_only)
{
struct bch_dev *ca;
u64 start_time = local_clock();
closure_wait_event(&c->btree_interior_update_wait,
!bch2_btree_interior_updates_nr_pending(c));
again:
- ret = bch2_gc_start(c);
+ ret = bch2_gc_start(c, metadata_only);
if (ret)
goto out;
bch2_mark_superblocks(c);
- ret = bch2_gc_btrees(c, initial);
+ ret = bch2_gc_btrees(c, initial, metadata_only);
if (ret)
goto out;
#if 0
bch2_mark_pending_btree_node_frees(c);
#endif
- bch2_mark_allocator_buckets(c);
-
c->gc_count++;
if (test_bit(BCH_FS_NEED_ANOTHER_GC, &c->flags) ||
bch2_journal_block(&c->journal);
percpu_down_write(&c->mark_lock);
- ret = bch2_gc_done(c, initial);
+ ret = bch2_gc_done(c, initial, metadata_only);
bch2_journal_unblock(&c->journal);
} else {
struct btree_iter *iter;
struct bkey_s_c k;
struct bkey_buf sk;
- int ret = 0;
+ int ret = 0, commit_err = 0;
bch2_bkey_buf_init(&sk);
bch2_trans_init(&trans, c, 0, 0);
while ((k = bch2_btree_iter_peek(iter)).k &&
!(ret = bkey_err(k))) {
- if (gc_btree_gens_key(c, k)) {
+ c->gc_gens_pos = iter->pos;
+
+ if (gc_btree_gens_key(c, k) && !commit_err) {
bch2_bkey_buf_reassemble(&sk, c, k);
bch2_extent_normalize(c, bkey_i_to_s(sk.k));
bch2_trans_update(&trans, iter, sk.k, 0);
- ret = bch2_trans_commit(&trans, NULL, NULL,
- BTREE_INSERT_NOFAIL);
- if (ret == -EINTR)
+ commit_err = bch2_trans_commit(&trans, NULL, NULL,
+ BTREE_INSERT_NOWAIT|
+ BTREE_INSERT_NOFAIL);
+ if (commit_err == -EINTR) {
+ commit_err = 0;
continue;
- if (ret) {
- break;
}
}
for (i = 0; i < BTREE_ID_NR; i++)
if ((1 << i) & BTREE_ID_HAS_PTRS) {
+ c->gc_gens_btree = i;
+ c->gc_gens_pos = POS_MIN;
ret = bch2_gc_btree_gens(c, i);
if (ret) {
bch_err(c, "error recalculating oldest_gen: %i", ret);
up_read(&ca->bucket_lock);
}
+ c->gc_gens_btree = 0;
+ c->gc_gens_pos = POS_MIN;
+
c->gc_count++;
err:
up_read(&c->gc_lock);
return ret;
}
-/* Btree coalescing */
-
-static void recalc_packed_keys(struct btree *b)
-{
- struct bset *i = btree_bset_first(b);
- struct bkey_packed *k;
-
- memset(&b->nr, 0, sizeof(b->nr));
-
- BUG_ON(b->nsets != 1);
-
- vstruct_for_each(i, k)
- btree_keys_account_key_add(&b->nr, 0, k);
-}
-
-static void bch2_coalesce_nodes(struct bch_fs *c, struct btree_iter *iter,
- struct btree *old_nodes[GC_MERGE_NODES])
-{
- struct btree *parent = btree_node_parent(iter, old_nodes[0]);
- unsigned i, nr_old_nodes, nr_new_nodes, u64s = 0;
- unsigned blocks = btree_blocks(c) * 2 / 3;
- struct btree *new_nodes[GC_MERGE_NODES];
- struct btree_update *as;
- struct keylist keylist;
- struct bkey_format_state format_state;
- struct bkey_format new_format;
-
- memset(new_nodes, 0, sizeof(new_nodes));
- bch2_keylist_init(&keylist, NULL);
-
- /* Count keys that are not deleted */
- for (i = 0; i < GC_MERGE_NODES && old_nodes[i]; i++)
- u64s += old_nodes[i]->nr.live_u64s;
-
- nr_old_nodes = nr_new_nodes = i;
-
- /* Check if all keys in @old_nodes could fit in one fewer node */
- if (nr_old_nodes <= 1 ||
- __vstruct_blocks(struct btree_node, c->block_bits,
- DIV_ROUND_UP(u64s, nr_old_nodes - 1)) > blocks)
- return;
-
- /* Find a format that all keys in @old_nodes can pack into */
- bch2_bkey_format_init(&format_state);
-
- /*
- * XXX: this won't correctly take it account the new min/max keys:
- */
- for (i = 0; i < nr_old_nodes; i++)
- __bch2_btree_calc_format(&format_state, old_nodes[i]);
-
- new_format = bch2_bkey_format_done(&format_state);
-
- /* Check if repacking would make any nodes too big to fit */
- for (i = 0; i < nr_old_nodes; i++)
- if (!bch2_btree_node_format_fits(c, old_nodes[i], &new_format)) {
- trace_btree_gc_coalesce_fail(c,
- BTREE_GC_COALESCE_FAIL_FORMAT_FITS);
- return;
- }
-
- if (bch2_keylist_realloc(&keylist, NULL, 0,
- BKEY_BTREE_PTR_U64s_MAX * nr_old_nodes)) {
- trace_btree_gc_coalesce_fail(c,
- BTREE_GC_COALESCE_FAIL_KEYLIST_REALLOC);
- return;
- }
-
- as = bch2_btree_update_start(iter, old_nodes[0]->c.level,
- btree_update_reserve_required(c, parent) + nr_old_nodes,
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_USE_RESERVE);
- if (IS_ERR(as)) {
- trace_btree_gc_coalesce_fail(c,
- BTREE_GC_COALESCE_FAIL_RESERVE_GET);
- bch2_keylist_free(&keylist, NULL);
- return;
- }
-
- trace_btree_gc_coalesce(c, old_nodes[0]);
-
- for (i = 0; i < nr_old_nodes; i++)
- bch2_btree_interior_update_will_free_node(as, old_nodes[i]);
-
- /* Repack everything with @new_format and sort down to one bset */
- for (i = 0; i < nr_old_nodes; i++)
- new_nodes[i] =
- __bch2_btree_node_alloc_replacement(as, old_nodes[i],
- new_format);
-
- /*
- * Conceptually we concatenate the nodes together and slice them
- * up at different boundaries.
- */
- for (i = nr_new_nodes - 1; i > 0; --i) {
- struct btree *n1 = new_nodes[i];
- struct btree *n2 = new_nodes[i - 1];
-
- struct bset *s1 = btree_bset_first(n1);
- struct bset *s2 = btree_bset_first(n2);
- struct bkey_packed *k, *last = NULL;
-
- /* Calculate how many keys from @n2 we could fit inside @n1 */
- u64s = 0;
-
- for (k = s2->start;
- k < vstruct_last(s2) &&
- vstruct_blocks_plus(n1->data, c->block_bits,
- u64s + k->u64s) <= blocks;
- k = bkey_next(k)) {
- last = k;
- u64s += k->u64s;
- }
-
- if (u64s == le16_to_cpu(s2->u64s)) {
- /* n2 fits entirely in n1 */
- n1->key.k.p = n1->data->max_key = n2->data->max_key;
-
- memcpy_u64s(vstruct_last(s1),
- s2->start,
- le16_to_cpu(s2->u64s));
- le16_add_cpu(&s1->u64s, le16_to_cpu(s2->u64s));
-
- set_btree_bset_end(n1, n1->set);
-
- six_unlock_write(&n2->c.lock);
- bch2_btree_node_free_never_inserted(c, n2);
- six_unlock_intent(&n2->c.lock);
-
- memmove(new_nodes + i - 1,
- new_nodes + i,
- sizeof(new_nodes[0]) * (nr_new_nodes - i));
- new_nodes[--nr_new_nodes] = NULL;
- } else if (u64s) {
- /* move part of n2 into n1 */
- n1->key.k.p = n1->data->max_key =
- bkey_unpack_pos(n1, last);
-
- n2->data->min_key = bpos_successor(n1->data->max_key);
-
- memcpy_u64s(vstruct_last(s1),
- s2->start, u64s);
- le16_add_cpu(&s1->u64s, u64s);
-
- memmove(s2->start,
- vstruct_idx(s2, u64s),
- (le16_to_cpu(s2->u64s) - u64s) * sizeof(u64));
- s2->u64s = cpu_to_le16(le16_to_cpu(s2->u64s) - u64s);
-
- set_btree_bset_end(n1, n1->set);
- set_btree_bset_end(n2, n2->set);
- }
- }
-
- for (i = 0; i < nr_new_nodes; i++) {
- struct btree *n = new_nodes[i];
-
- recalc_packed_keys(n);
- btree_node_reset_sib_u64s(n);
-
- bch2_btree_build_aux_trees(n);
-
- bch2_btree_update_add_new_node(as, n);
- six_unlock_write(&n->c.lock);
-
- bch2_btree_node_write(c, n, SIX_LOCK_intent);
- }
-
- /*
- * The keys for the old nodes get deleted. We don't want to insert keys
- * that compare equal to the keys for the new nodes we'll also be
- * inserting - we can't because keys on a keylist must be strictly
- * greater than the previous keys, and we also don't need to since the
- * key for the new node will serve the same purpose (overwriting the key
- * for the old node).
- */
- for (i = 0; i < nr_old_nodes; i++) {
- struct bkey_i delete;
- unsigned j;
-
- for (j = 0; j < nr_new_nodes; j++)
- if (!bpos_cmp(old_nodes[i]->key.k.p,
- new_nodes[j]->key.k.p))
- goto next;
-
- bkey_init(&delete.k);
- delete.k.p = old_nodes[i]->key.k.p;
- bch2_keylist_add_in_order(&keylist, &delete);
-next:
- i = i;
- }
-
- /*
- * Keys for the new nodes get inserted: bch2_btree_insert_keys() only
- * does the lookup once and thus expects the keys to be in sorted order
- * so we have to make sure the new keys are correctly ordered with
- * respect to the deleted keys added in the previous loop
- */
- for (i = 0; i < nr_new_nodes; i++)
- bch2_keylist_add_in_order(&keylist, &new_nodes[i]->key);
-
- /* Insert the newly coalesced nodes */
- bch2_btree_insert_node(as, parent, iter, &keylist, 0);
-
- BUG_ON(!bch2_keylist_empty(&keylist));
-
- BUG_ON(iter->l[old_nodes[0]->c.level].b != old_nodes[0]);
-
- bch2_btree_iter_node_replace(iter, new_nodes[0]);
-
- for (i = 0; i < nr_new_nodes; i++)
- bch2_btree_update_get_open_buckets(as, new_nodes[i]);
-
- /* Free the old nodes and update our sliding window */
- for (i = 0; i < nr_old_nodes; i++) {
- bch2_btree_node_free_inmem(c, old_nodes[i], iter);
-
- /*
- * the index update might have triggered a split, in which case
- * the nodes we coalesced - the new nodes we just created -
- * might not be sibling nodes anymore - don't add them to the
- * sliding window (except the first):
- */
- if (!i) {
- old_nodes[i] = new_nodes[i];
- } else {
- old_nodes[i] = NULL;
- }
- }
-
- for (i = 0; i < nr_new_nodes; i++)
- six_unlock_intent(&new_nodes[i]->c.lock);
-
- bch2_btree_update_done(as);
- bch2_keylist_free(&keylist, NULL);
-}
-
-static int bch2_coalesce_btree(struct bch_fs *c, enum btree_id btree_id)
-{
- struct btree_trans trans;
- struct btree_iter *iter;
- struct btree *b;
- bool kthread = (current->flags & PF_KTHREAD) != 0;
- unsigned i;
- int ret = 0;
-
- /* Sliding window of adjacent btree nodes */
- struct btree *merge[GC_MERGE_NODES];
- u32 lock_seq[GC_MERGE_NODES];
-
- bch2_trans_init(&trans, c, 0, 0);
-
- /*
- * XXX: We don't have a good way of positively matching on sibling nodes
- * that have the same parent - this code works by handling the cases
- * where they might not have the same parent, and is thus fragile. Ugh.
- *
- * Perhaps redo this to use multiple linked iterators?
- */
- memset(merge, 0, sizeof(merge));
-
- __for_each_btree_node(&trans, iter, btree_id, POS_MIN,
- BTREE_MAX_DEPTH, 0,
- BTREE_ITER_PREFETCH, b) {
- memmove(merge + 1, merge,
- sizeof(merge) - sizeof(merge[0]));
- memmove(lock_seq + 1, lock_seq,
- sizeof(lock_seq) - sizeof(lock_seq[0]));
-
- merge[0] = b;
-
- for (i = 1; i < GC_MERGE_NODES; i++) {
- if (!merge[i] ||
- !six_relock_intent(&merge[i]->c.lock, lock_seq[i]))
- break;
-
- if (merge[i]->c.level != merge[0]->c.level) {
- six_unlock_intent(&merge[i]->c.lock);
- break;
- }
- }
- memset(merge + i, 0, (GC_MERGE_NODES - i) * sizeof(merge[0]));
-
- bch2_coalesce_nodes(c, iter, merge);
-
- for (i = 1; i < GC_MERGE_NODES && merge[i]; i++) {
- lock_seq[i] = merge[i]->c.lock.state.seq;
- six_unlock_intent(&merge[i]->c.lock);
- }
-
- lock_seq[0] = merge[0]->c.lock.state.seq;
-
- if (kthread && kthread_should_stop()) {
- ret = -ESHUTDOWN;
- break;
- }
-
- bch2_trans_cond_resched(&trans);
-
- /*
- * If the parent node wasn't relocked, it might have been split
- * and the nodes in our sliding window might not have the same
- * parent anymore - blow away the sliding window:
- */
- if (btree_iter_node(iter, iter->level + 1) &&
- !btree_node_intent_locked(iter, iter->level + 1))
- memset(merge + 1, 0,
- (GC_MERGE_NODES - 1) * sizeof(merge[0]));
- }
- bch2_trans_iter_put(&trans, iter);
-
- return bch2_trans_exit(&trans) ?: ret;
-}
-
-/**
- * bch_coalesce - coalesce adjacent nodes with low occupancy
- */
-void bch2_coalesce(struct bch_fs *c)
-{
- enum btree_id id;
-
- down_read(&c->gc_lock);
- trace_gc_coalesce_start(c);
-
- for (id = 0; id < BTREE_ID_NR; id++) {
- int ret = c->btree_roots[id].b
- ? bch2_coalesce_btree(c, id)
- : 0;
-
- if (ret) {
- if (ret != -ESHUTDOWN)
- bch_err(c, "btree coalescing failed: %d", ret);
- return;
- }
- }
-
- trace_gc_coalesce_end(c);
- up_read(&c->gc_lock);
-}
-
static int bch2_gc_thread(void *arg)
{
struct bch_fs *c = arg;
#include "btree_types.h"
-void bch2_coalesce(struct bch_fs *);
-
-int bch2_gc(struct bch_fs *, bool);
+int bch2_gc(struct bch_fs *, bool, bool);
int bch2_gc_gens(struct bch_fs *);
void bch2_gc_thread_stop(struct bch_fs *);
int bch2_gc_thread_start(struct bch_fs *);
return gc_pos_btree(id, POS_MAX, BTREE_MAX_DEPTH);
}
-static inline struct gc_pos gc_pos_alloc(struct bch_fs *c, struct open_bucket *ob)
-{
- return (struct gc_pos) {
- .phase = GC_PHASE_ALLOC,
- .pos = POS(ob ? ob - c->open_buckets : 0, 0),
- };
-}
-
static inline bool gc_visited(struct bch_fs *c, struct gc_pos pos)
{
unsigned seq;
struct btree_read_bio *rb;
struct bch_dev *ca;
struct bio *bio;
+ char buf[200];
int ret;
+ btree_pos_to_text(&PBUF(buf), c, b);
trace_btree_read(c, b);
ret = bch2_bkey_pick_read_device(c, bkey_i_to_s_c(&b->key),
NULL, &pick);
if (bch2_fs_fatal_err_on(ret <= 0, c,
- "btree node read error: no device to read from")) {
+ "btree node read error: no device to read from\n"
+ " at %s", buf)) {
set_btree_node_read_error(b);
return;
}
return ret;
}
-static void btree_write_submit(struct work_struct *work)
-{
- struct btree_write_bio *wbio = container_of(work, struct btree_write_bio, work);
-
- bch2_submit_wbio_replicas(&wbio->wbio, wbio->wbio.c, BCH_DATA_btree, &wbio->key);
-}
-
void __bch2_btree_node_write(struct bch_fs *c, struct btree *b)
{
struct btree_write_bio *wbio;
struct bset *i;
struct btree_node *bn = NULL;
struct btree_node_entry *bne = NULL;
+ struct bkey_buf k;
struct bch_extent_ptr *ptr;
struct sort_iter sort_iter;
struct nonce nonce;
bool validate_before_checksum = false;
void *data;
+ bch2_bkey_buf_init(&k);
+
if (test_bit(BCH_FS_HOLD_BTREE_WRITES, &c->flags))
return;
wbio_init(&wbio->wbio.bio);
wbio->data = data;
wbio->bytes = bytes;
- wbio->wbio.c = c;
wbio->wbio.used_mempool = used_mempool;
wbio->wbio.bio.bi_opf = REQ_OP_WRITE|REQ_META;
wbio->wbio.bio.bi_end_io = btree_node_write_endio;
* just make all btree node writes FUA to keep things sane.
*/
- bkey_copy(&wbio->key, &b->key);
+ bch2_bkey_buf_copy(&k, c, &b->key);
- bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(&wbio->key)), ptr)
+ bkey_for_each_ptr(bch2_bkey_ptrs(bkey_i_to_s(k.k)), ptr)
ptr->offset += b->written;
b->written += sectors_to_write;
atomic64_inc(&c->btree_writes_nr);
atomic64_add(sectors_to_write, &c->btree_writes_sectors);
- INIT_WORK(&wbio->work, btree_write_submit);
- schedule_work(&wbio->work);
+ /* XXX: submitting IO with btree locks held: */
+ bch2_submit_wbio_replicas(&wbio->wbio, c, BCH_DATA_btree, k.k);
+ bch2_bkey_buf_exit(&k, c);
return;
err:
set_btree_node_noevict(b);
struct btree_write_bio {
struct work_struct work;
- __BKEY_PADDED(key, BKEY_BTREE_PTR_VAL_U64s_MAX);
void *data;
unsigned bytes;
struct bch_write_bio wbio;
*/
if (type == SIX_LOCK_intent &&
linked->nodes_locked != linked->nodes_intent_locked) {
- linked->locks_want = max_t(unsigned,
- linked->locks_want,
- __fls(linked->nodes_locked) + 1);
- if (!btree_iter_get_locks(linked, true, false)) {
- deadlock_iter = linked;
- reason = 1;
- }
+ deadlock_iter = linked;
+ reason = 1;
}
if (linked->btree_id != iter->btree_id) {
* we're about to lock, it must have the ancestors locked too:
*/
if (level > __fls(linked->nodes_locked)) {
- linked->locks_want =
- max(level + 1, max_t(unsigned,
- linked->locks_want,
- iter->locks_want));
- if (!btree_iter_get_locks(linked, true, false)) {
- deadlock_iter = linked;
- reason = 5;
- }
+ deadlock_iter = linked;
+ reason = 5;
}
/* Must lock btree nodes in key order: */
btree_iter_type(linked))) <= 0) {
deadlock_iter = linked;
reason = 7;
- }
-
- /*
- * Recheck if this is a node we already have locked - since one
- * of the get_locks() calls might've successfully
- * upgraded/relocked it:
- */
- if (linked->l[level].b == b &&
- btree_node_locked_type(linked, level) >= type) {
- six_lock_increment(&b->c.lock, type);
- return true;
+ BUG_ON(trans->in_traverse_all);
}
}
if (unlikely(deadlock_iter)) {
trace_trans_restart_would_deadlock(iter->trans->ip, ip,
- reason,
+ trans->in_traverse_all, reason,
deadlock_iter->btree_id,
btree_iter_type(deadlock_iter),
+ &deadlock_iter->real_pos,
iter->btree_id,
- btree_iter_type(iter));
+ btree_iter_type(iter),
+ &pos);
return false;
}
return true;
/*
- * Ancestor nodes must be locked before child nodes, so set locks_want
- * on iterators that might lock ancestors before us to avoid getting
- * -EINTR later:
+ * XXX: this is ugly - we'd prefer to not be mucking with other
+ * iterators in the btree_trans here.
+ *
+ * On failure to upgrade the iterator, setting iter->locks_want and
+ * calling get_locks() is sufficient to make bch2_btree_iter_traverse()
+ * get the locks we want on transaction restart.
+ *
+ * But if this iterator was a clone, on transaction restart what we did
+ * to this iterator isn't going to be preserved.
+ *
+ * Possibly we could add an iterator field for the parent iterator when
+ * an iterator is a copy - for now, we'll just upgrade any other
+ * iterators with the same btree id.
+ *
+ * The code below used to be needed to ensure ancestor nodes get locked
+ * before interior nodes - now that's handled by
+ * bch2_btree_iter_traverse_all().
*/
trans_for_each_iter(iter->trans, linked)
if (linked != iter &&
+ btree_iter_type(linked) == btree_iter_type(iter) &&
linked->btree_id == iter->btree_id &&
linked->locks_want < new_locks_want) {
linked->locks_want = new_locks_want;
struct bch_fs *c = trans->c;
struct btree_iter *iter;
u8 sorted[BTREE_ITER_MAX];
- unsigned i, nr_sorted = 0;
+ int i, nr_sorted = 0;
+ bool relock_fail;
if (trans->in_traverse_all)
return -EINTR;
trans->in_traverse_all = true;
retry_all:
nr_sorted = 0;
+ relock_fail = false;
- trans_for_each_iter(trans, iter)
+ trans_for_each_iter(trans, iter) {
+ if (!bch2_btree_iter_relock(iter, true))
+ relock_fail = true;
sorted[nr_sorted++] = iter->idx;
+ }
+
+ if (!relock_fail) {
+ trans->in_traverse_all = false;
+ return 0;
+ }
#define btree_iter_cmp_by_idx(_l, _r) \
btree_iter_lock_cmp(&trans->iters[_l], &trans->iters[_r])
bubble_sort(sorted, nr_sorted, btree_iter_cmp_by_idx);
#undef btree_iter_cmp_by_idx
+
+ for (i = nr_sorted - 2; i >= 0; --i) {
+ struct btree_iter *iter1 = trans->iters + sorted[i];
+ struct btree_iter *iter2 = trans->iters + sorted[i + 1];
+
+ if (iter1->btree_id == iter2->btree_id &&
+ iter1->locks_want < iter2->locks_want)
+ __bch2_btree_iter_upgrade(iter1, iter2->locks_want);
+ else if (!iter1->locks_want && iter2->locks_want)
+ __bch2_btree_iter_upgrade(iter1, 1);
+ }
+
bch2_trans_unlock(trans);
cond_resched();
bch2_btree_cache_cannibalize_unlock(c);
trans->in_traverse_all = false;
+
+ trace_trans_traverse_all(trans->ip);
return ret;
}
if (iter->btree_id != btree_id)
continue;
- if (best &&
- bkey_cmp(bpos_diff(best->real_pos, pos),
- bpos_diff(iter->real_pos, pos)) > 0)
- continue;
+ if (best) {
+ int cmp = bkey_cmp(bpos_diff(best->real_pos, pos),
+ bpos_diff(iter->real_pos, pos));
+
+ if (cmp < 0 ||
+ ((cmp == 0 && btree_iter_keep(trans, iter))))
+ continue;
+ }
best = iter;
}
iter->snapshot = pos.snapshot;
- locks_want = min(locks_want, BTREE_MAX_DEPTH);
+ /*
+ * If the iterator has locks_want greater than requested, we explicitly
+ * do not downgrade it here - on transaction restart because btree node
+ * split needs to upgrade locks, we might be putting/getting the
+ * iterator again. Downgrading iterators only happens via an explicit
+ * bch2_trans_downgrade().
+ */
+ locks_want = min(locks_want, BTREE_MAX_DEPTH);
if (locks_want > iter->locks_want) {
iter->locks_want = locks_want;
btree_iter_get_locks(iter, true, false);
- } else if (locks_want < iter->locks_want) {
- __bch2_btree_iter_downgrade(iter, locks_want);
}
while (iter->level < depth) {
return iter;
}
-static int bch2_trans_preload_mem(struct btree_trans *trans, size_t size)
+void *bch2_trans_kmalloc(struct btree_trans *trans, size_t size)
{
- if (size > trans->mem_bytes) {
+ size_t new_top = trans->mem_top + size;
+ void *p;
+
+ if (new_top > trans->mem_bytes) {
size_t old_bytes = trans->mem_bytes;
- size_t new_bytes = roundup_pow_of_two(size);
+ size_t new_bytes = roundup_pow_of_two(new_top);
void *new_mem = krealloc(trans->mem, new_bytes, GFP_NOFS);
if (!new_mem)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
trans->mem = new_mem;
trans->mem_bytes = new_bytes;
if (old_bytes) {
- trace_trans_restart_mem_realloced(trans->ip, new_bytes);
- return -EINTR;
+ trace_trans_restart_mem_realloced(trans->ip, _RET_IP_, new_bytes);
+ return ERR_PTR(-EINTR);
}
}
- return 0;
-}
-
-void *bch2_trans_kmalloc(struct btree_trans *trans, size_t size)
-{
- void *p;
- int ret;
-
- ret = bch2_trans_preload_mem(trans, trans->mem_top + size);
- if (ret)
- return ERR_PTR(ret);
-
p = trans->mem + trans->mem_top;
trans->mem_top += size;
return p;
if (!(flags & TRANS_RESET_NOUNLOCK))
bch2_trans_cond_resched(trans);
- if (!(flags & TRANS_RESET_NOTRAVERSE))
+ if (!(flags & TRANS_RESET_NOTRAVERSE) &&
+ trans->iters_linked)
bch2_btree_iter_traverse_all(trans);
}
{
return cmp_int(l->btree_id, r->btree_id) ?:
-cmp_int(btree_iter_is_cached(l), btree_iter_is_cached(r)) ?:
- bkey_cmp(l->pos, r->pos);
+ bkey_cmp(l->real_pos, r->real_pos);
}
/*
static inline void btree_insert_entry_checks(struct btree_trans *trans,
struct btree_insert_entry *i)
{
- struct bch_fs *c = trans->c;
-
- if (bch2_debug_check_bkeys) {
- const char *invalid = bch2_bkey_invalid(c,
- bkey_i_to_s_c(i->k), i->bkey_type);
- if (invalid) {
- char buf[200];
-
- bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(i->k));
- panic("invalid bkey %s on insert: %s\n", buf, invalid);
- }
- }
BUG_ON(!i->is_extent && bpos_cmp(i->k->k.p, i->iter->real_pos));
BUG_ON(i->level != i->iter->level);
BUG_ON(i->btree_id != i->iter->btree_id);
}
static inline void do_btree_insert_one(struct btree_trans *trans,
- struct btree_iter *iter,
- struct bkey_i *insert)
+ struct btree_insert_entry *i)
{
struct bch_fs *c = trans->c;
struct journal *j = &c->journal;
EBUG_ON(trans->journal_res.ref !=
!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY));
- insert->k.needs_whiteout = false;
+ i->k->k.needs_whiteout = false;
- did_work = (btree_iter_type(iter) != BTREE_ITER_CACHED)
- ? btree_insert_key_leaf(trans, iter, insert)
- : bch2_btree_insert_key_cached(trans, iter, insert);
+ did_work = (btree_iter_type(i->iter) != BTREE_ITER_CACHED)
+ ? btree_insert_key_leaf(trans, i->iter, i->k)
+ : bch2_btree_insert_key_cached(trans, i->iter, i->k);
if (!did_work)
return;
if (likely(!(trans->flags & BTREE_INSERT_JOURNAL_REPLAY))) {
bch2_journal_add_keys(j, &trans->journal_res,
- iter->btree_id, insert);
+ i->btree_id,
+ i->level,
+ i->k);
bch2_journal_set_has_inode(j, &trans->journal_res,
- insert->k.p.inode);
+ i->k->k.p.inode);
if (trans->journal_seq)
*trans->journal_seq = trans->journal_res.seq;
bch2_trans_mark_gc(trans);
trans_for_each_update2(trans, i)
- do_btree_insert_one(trans, i->iter, i->k);
+ do_btree_insert_one(trans, i);
err:
if (marking) {
percpu_up_read(&c->mark_lock);
}
}
- if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG))
- trans_for_each_update2(trans, i)
- btree_insert_entry_checks(trans, i);
+ trans_for_each_update2(trans, i) {
+ const char *invalid = bch2_bkey_invalid(c,
+ bkey_i_to_s_c(i->k), i->bkey_type);
+ if (invalid) {
+ char buf[200];
+
+ bch2_bkey_val_to_text(&PBUF(buf), c, bkey_i_to_s_c(i->k));
+ bch_err(c, "invalid bkey %s on insert: %s\n", buf, invalid);
+ bch2_fatal_error(c);
+ }
+ btree_insert_entry_checks(trans, i);
+ }
bch2_btree_trans_verify_locks(trans);
trans_for_each_update2(trans, i)
static int journal_reclaim_wait_done(struct bch_fs *c)
{
- int ret;
-
- ret = bch2_journal_error(&c->journal);
- if (ret)
- return ret;
-
- ret = !bch2_btree_key_cache_must_wait(c);
- if (ret)
- return ret;
-
- journal_reclaim_kick(&c->journal);
-
- if (mutex_trylock(&c->journal.reclaim_lock)) {
- ret = bch2_journal_reclaim(&c->journal);
- mutex_unlock(&c->journal.reclaim_lock);
- }
+ int ret = bch2_journal_error(&c->journal) ?:
+ !bch2_btree_key_cache_must_wait(c);
if (!ret)
- ret = !bch2_btree_key_cache_must_wait(c);
+ journal_reclaim_kick(&c->journal);
return ret;
}
case BTREE_INSERT_NEED_JOURNAL_RECLAIM:
bch2_trans_unlock(trans);
- wait_event(c->journal.reclaim_wait,
- (ret = journal_reclaim_wait_done(c)));
+ wait_event_freezable(c->journal.reclaim_wait,
+ (ret = journal_reclaim_wait_done(c)));
+ if (ret < 0)
+ return ret;
- if (!ret && bch2_trans_relock(trans))
+ if (bch2_trans_relock(trans))
return 0;
trace_trans_restart_journal_reclaim(trans->ip);
iter = bch2_trans_get_iter(trans, id, bkey_start_pos(&k->k),
BTREE_ITER_INTENT);
- ret = bch2_btree_iter_traverse(iter) ?:
- bch2_trans_update(trans, iter, k, 0);
+ ret = bch2_trans_update(trans, iter, k, 0);
bch2_trans_iter_put(trans, iter);
return ret;
}
* Code for manipulating bucket marks for garbage collection.
*
* Copyright 2014 Datera, Inc.
- *
- * Bucket states:
- * - free bucket: mark == 0
- * The bucket contains no data and will not be read
- *
- * - allocator bucket: owned_by_allocator == 1
- * The bucket is on a free list, or it is an open bucket
- *
- * - cached bucket: owned_by_allocator == 0 &&
- * dirty_sectors == 0 &&
- * cached_sectors > 0
- * The bucket contains data but may be safely discarded as there are
- * enough replicas of the data on other cache devices, or it has been
- * written back to the backing device
- *
- * - dirty bucket: owned_by_allocator == 0 &&
- * dirty_sectors > 0
- * The bucket contains data that we must not discard (either only copy,
- * or one of the 'main copies' for data requiring multiple replicas)
- *
- * - metadata bucket: owned_by_allocator == 0 && is_metadata == 1
- * This is a btree node, journal or gen/prio bucket
- *
- * Lifecycle:
- *
- * bucket invalidated => bucket on freelist => open bucket =>
- * [dirty bucket =>] cached bucket => bucket invalidated => ...
- *
- * Note that cache promotion can skip the dirty bucket step, as data
- * is copied from a deeper tier to a shallower tier, onto a cached
- * bucket.
- * Note also that a cached bucket can spontaneously become dirty --
- * see below.
- *
- * Only a traversal of the key space can determine whether a bucket is
- * truly dirty or cached.
- *
- * Transitions:
- *
- * - free => allocator: bucket was invalidated
- * - cached => allocator: bucket was invalidated
- *
- * - allocator => dirty: open bucket was filled up
- * - allocator => cached: open bucket was filled up
- * - allocator => metadata: metadata was allocated
- *
- * - dirty => cached: dirty sectors were copied to a deeper tier
- * - dirty => free: dirty sectors were overwritten or moved (copy gc)
- * - cached => free: cached sectors were overwritten
- *
- * - metadata => free: metadata was freed
- *
- * Oddities:
- * - cached => dirty: a device was removed so formerly replicated data
- * is no longer sufficiently replicated
- * - free => cached: cannot happen
- * - free => dirty: cannot happen
- * - free => metadata: cannot happen
*/
#include "bcachefs.h"
percpu_down_read(&c->mark_lock);
ret = kmalloc(sizeof(struct bch_fs_usage_online) +
- sizeof(u64) + c->replicas.nr, GFP_NOFS);
+ sizeof(u64) * c->replicas.nr, GFP_NOFS);
if (unlikely(!ret)) {
percpu_up_read(&c->mark_lock);
return NULL;
ret; \
})
-static int __bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
- size_t b, bool owned_by_allocator,
- bool gc)
+void bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
+ size_t b, bool owned_by_allocator)
{
- struct bucket *g = __bucket(ca, b, gc);
+ struct bucket *g = bucket(ca, b);
struct bucket_mark old, new;
old = bucket_cmpxchg(g, new, ({
new.owned_by_allocator = owned_by_allocator;
}));
- BUG_ON(!gc &&
- !owned_by_allocator && !old.owned_by_allocator);
-
- return 0;
-}
-
-void bch2_mark_alloc_bucket(struct bch_fs *c, struct bch_dev *ca,
- size_t b, bool owned_by_allocator,
- struct gc_pos pos, unsigned flags)
-{
- preempt_disable();
-
- do_mark_fn(__bch2_mark_alloc_bucket, c, pos, flags,
- ca, b, owned_by_allocator);
-
- preempt_enable();
+ BUG_ON(owned_by_allocator == old.owned_by_allocator);
}
static int bch2_mark_alloc(struct bch_fs *c,
return 0;
if (!btree_node_type_is_extents(iter->btree_id)) {
- /* iterators should be uptodate, shouldn't get errors here: */
if (btree_iter_type(iter) != BTREE_ITER_CACHED) {
old = bch2_btree_iter_peek_slot(iter);
- BUG_ON(bkey_err(old));
+ ret = bkey_err(old);
+ if (ret)
+ return ret;
} else {
struct bkey_cached *ck = (void *) iter->l[0].b;
goto out;
}
- if ((unsigned) (u.dirty_sectors + sectors) > ca->mi.bucket_size) {
- bch2_fsck_err(c, FSCK_CAN_IGNORE|FSCK_NEED_FSCK,
- "bucket %llu:%llu gen %u data type %s sector count overflow: %u + %u > %u\n"
- "while marking %s",
- iter->pos.inode, iter->pos.offset, u.gen,
- bch2_data_types[u.data_type ?: type],
- u.dirty_sectors, sectors, ca->mi.bucket_size,
- bch2_data_types[type]);
- ret = -EIO;
- goto out;
- }
-
- if (u.data_type == type &&
- u.dirty_sectors == sectors)
- goto out;
-
u.data_type = type;
u.dirty_sectors = sectors;
}
int bch2_trans_mark_metadata_bucket(struct btree_trans *trans,
- struct disk_reservation *res,
struct bch_dev *ca, size_t b,
enum bch_data_type type,
unsigned sectors)
{
- return __bch2_trans_do(trans, res, NULL, 0,
- __bch2_trans_mark_metadata_bucket(trans, ca, b, BCH_DATA_journal,
- ca->mi.bucket_size));
-
+ return __bch2_trans_do(trans, NULL, NULL, 0,
+ __bch2_trans_mark_metadata_bucket(trans, ca, b, type, sectors));
}
static int bch2_trans_mark_metadata_sectors(struct btree_trans *trans,
- struct disk_reservation *res,
struct bch_dev *ca,
u64 start, u64 end,
enum bch_data_type type,
u64 *bucket, unsigned *bucket_sectors)
{
- int ret;
-
do {
u64 b = sector_to_bucket(ca, start);
unsigned sectors =
min_t(u64, bucket_to_sector(ca, b + 1), end) - start;
- if (b != *bucket) {
- if (*bucket_sectors) {
- ret = bch2_trans_mark_metadata_bucket(trans, res, ca,
- *bucket, type, *bucket_sectors);
- if (ret)
- return ret;
- }
+ if (b != *bucket && *bucket_sectors) {
+ int ret = bch2_trans_mark_metadata_bucket(trans, ca, *bucket,
+ type, *bucket_sectors);
+ if (ret)
+ return ret;
- *bucket = b;
- *bucket_sectors = 0;
+ *bucket_sectors = 0;
}
+ *bucket = b;
*bucket_sectors += sectors;
start += sectors;
- } while (!ret && start < end);
+ } while (start < end);
return 0;
}
static int __bch2_trans_mark_dev_sb(struct btree_trans *trans,
- struct disk_reservation *res,
- struct bch_dev *ca)
+ struct bch_dev *ca)
{
struct bch_sb_layout *layout = &ca->disk_sb.sb->layout;
u64 bucket = 0;
u64 offset = le64_to_cpu(layout->sb_offset[i]);
if (offset == BCH_SB_SECTOR) {
- ret = bch2_trans_mark_metadata_sectors(trans, res, ca,
+ ret = bch2_trans_mark_metadata_sectors(trans, ca,
0, BCH_SB_SECTOR,
BCH_DATA_sb, &bucket, &bucket_sectors);
if (ret)
return ret;
}
- ret = bch2_trans_mark_metadata_sectors(trans, res, ca, offset,
+ ret = bch2_trans_mark_metadata_sectors(trans, ca, offset,
offset + (1 << layout->sb_max_size_bits),
BCH_DATA_sb, &bucket, &bucket_sectors);
if (ret)
}
if (bucket_sectors) {
- ret = bch2_trans_mark_metadata_bucket(trans, res, ca,
+ ret = bch2_trans_mark_metadata_bucket(trans, ca,
bucket, BCH_DATA_sb, bucket_sectors);
if (ret)
return ret;
}
for (i = 0; i < ca->journal.nr; i++) {
- ret = bch2_trans_mark_metadata_bucket(trans, res, ca,
+ ret = bch2_trans_mark_metadata_bucket(trans, ca,
ca->journal.buckets[i],
BCH_DATA_journal, ca->mi.bucket_size);
if (ret)
return 0;
}
-int bch2_trans_mark_dev_sb(struct bch_fs *c,
- struct disk_reservation *res,
- struct bch_dev *ca)
+int bch2_trans_mark_dev_sb(struct bch_fs *c, struct bch_dev *ca)
{
- return bch2_trans_do(c, res, NULL, 0,
- __bch2_trans_mark_dev_sb(&trans, res, ca));
+ return bch2_trans_do(c, NULL, NULL, 0,
+ __bch2_trans_mark_dev_sb(&trans, ca));
}
/* Disk reservations: */
for (i = 0; i < RESERVE_NR; i++)
available -= fifo_used(&ca->free[i]);
available -= fifo_used(&ca->free_inc);
+ available -= ca->nr_open_buckets;
spin_unlock(&c->freelist_lock);
return max(available, 0LL);
void bch2_bucket_seq_cleanup(struct bch_fs *);
void bch2_fs_usage_initialize(struct bch_fs *);
-void bch2_mark_alloc_bucket(struct bch_fs *, struct bch_dev *,
- size_t, bool, struct gc_pos, unsigned);
+void bch2_mark_alloc_bucket(struct bch_fs *, struct bch_dev *, size_t, bool);
void bch2_mark_metadata_bucket(struct bch_fs *, struct bch_dev *,
size_t, enum bch_data_type, unsigned,
struct gc_pos, unsigned);
struct bkey_i *insert, unsigned);
void bch2_trans_fs_usage_apply(struct btree_trans *, struct replicas_delta_list *);
-int bch2_trans_mark_metadata_bucket(struct btree_trans *,
- struct disk_reservation *, struct bch_dev *,
- size_t, enum bch_data_type, unsigned);
-int bch2_trans_mark_dev_sb(struct bch_fs *, struct disk_reservation *,
- struct bch_dev *);
+int bch2_trans_mark_metadata_bucket(struct btree_trans *, struct bch_dev *,
+ size_t, enum bch_data_type, unsigned);
+int bch2_trans_mark_dev_sb(struct bch_fs *, struct bch_dev *);
/* disk reservations: */
struct {
u64 buckets;
u64 sectors; /* _compressed_ sectors: */
+ /*
+ * XXX
+ * Why do we have this? Isn't it just buckets * bucket_size -
+ * sectors?
+ */
u64 fragmented;
} d[BCH_DATA_NR];
};
return ret;
}
-static long bchfs_fallocate(struct bch_inode_info *inode, int mode,
- loff_t offset, loff_t len)
+static int __bchfs_fallocate(struct bch_inode_info *inode, int mode,
+ u64 start_sector, u64 end_sector)
{
- struct address_space *mapping = inode->v.i_mapping;
struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct btree_trans trans;
struct btree_iter *iter;
- struct bpos end_pos;
- loff_t end = offset + len;
- loff_t block_start = round_down(offset, block_bytes(c));
- loff_t block_end = round_up(end, block_bytes(c));
- unsigned sectors;
+ struct bpos end_pos = POS(inode->v.i_ino, end_sector);
unsigned replicas = io_opts(c, &inode->ei_inode).data_replicas;
- int ret;
+ int ret = 0;
bch2_trans_init(&trans, c, BTREE_ITER_MAX, 0);
- inode_lock(&inode->v);
- inode_dio_wait(&inode->v);
- bch2_pagecache_block_get(&inode->ei_pagecache_lock);
-
- if (!(mode & FALLOC_FL_KEEP_SIZE) && end > inode->v.i_size) {
- ret = inode_newsize_ok(&inode->v, end);
- if (ret)
- goto err;
- }
-
- if (mode & FALLOC_FL_ZERO_RANGE) {
- ret = __bch2_truncate_page(inode,
- offset >> PAGE_SHIFT,
- offset, end);
-
- if (!ret &&
- offset >> PAGE_SHIFT != end >> PAGE_SHIFT)
- ret = __bch2_truncate_page(inode,
- end >> PAGE_SHIFT,
- offset, end);
-
- if (unlikely(ret))
- goto err;
-
- truncate_pagecache_range(&inode->v, offset, end - 1);
- }
-
iter = bch2_trans_get_iter(&trans, BTREE_ID_extents,
- POS(inode->v.i_ino, block_start >> 9),
+ POS(inode->v.i_ino, start_sector),
BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
- end_pos = POS(inode->v.i_ino, block_end >> 9);
while (!ret && bkey_cmp(iter->pos, end_pos) < 0) {
s64 i_sectors_delta = 0;
struct quota_res quota_res = { 0 };
struct bkey_i_reservation reservation;
struct bkey_s_c k;
+ unsigned sectors;
bch2_trans_begin(&trans);
ret = 0;
}
bch2_trans_iter_put(&trans, iter);
+ bch2_trans_exit(&trans);
+ return ret;
+}
+static long bchfs_fallocate(struct bch_inode_info *inode, int mode,
+ loff_t offset, loff_t len)
+{
+ struct address_space *mapping = inode->v.i_mapping;
+ struct bch_fs *c = inode->v.i_sb->s_fs_info;
+ loff_t end = offset + len;
+ loff_t block_start = round_down(offset, block_bytes(c));
+ loff_t block_end = round_up(end, block_bytes(c));
+ int ret;
+
+ inode_lock(&inode->v);
+ inode_dio_wait(&inode->v);
+ bch2_pagecache_block_get(&inode->ei_pagecache_lock);
+
+ if (!(mode & FALLOC_FL_KEEP_SIZE) && end > inode->v.i_size) {
+ ret = inode_newsize_ok(&inode->v, end);
+ if (ret)
+ goto err;
+ }
+
+ if (mode & FALLOC_FL_ZERO_RANGE) {
+ ret = __bch2_truncate_page(inode,
+ offset >> PAGE_SHIFT,
+ offset, end);
+
+ if (!ret &&
+ offset >> PAGE_SHIFT != end >> PAGE_SHIFT)
+ ret = __bch2_truncate_page(inode,
+ end >> PAGE_SHIFT,
+ offset, end);
+
+ if (unlikely(ret))
+ goto err;
+
+ truncate_pagecache_range(&inode->v, offset, end - 1);
+ }
+
+ ret = __bchfs_fallocate(inode, mode, block_start >> 9, block_end >> 9);
if (ret)
goto err;
if (end >= inode->v.i_size &&
(!(mode & FALLOC_FL_KEEP_SIZE) ||
(mode & FALLOC_FL_ZERO_RANGE))) {
- struct btree_iter *inode_iter;
- struct bch_inode_unpacked inode_u;
-
- do {
- bch2_trans_begin(&trans);
- inode_iter = bch2_inode_peek(&trans, &inode_u,
- inode->v.i_ino, 0);
- ret = PTR_ERR_OR_ZERO(inode_iter);
- } while (ret == -EINTR);
-
- bch2_trans_iter_put(&trans, inode_iter);
- bch2_trans_unlock(&trans);
-
- if (ret)
- goto err;
/*
* Sync existing appends before extending i_size,
* as in bch2_extend():
*/
ret = filemap_write_and_wait_range(mapping,
- inode_u.bi_size, S64_MAX);
+ inode->ei_inode.bi_size, S64_MAX);
if (ret)
goto err;
mutex_unlock(&inode->ei_update_lock);
}
err:
- bch2_trans_exit(&trans);
bch2_pagecache_block_put(&inode->ei_pagecache_lock);
inode_unlock(&inode->v);
return ret;
return ret;
}
-static int __remove_dirent(struct btree_trans *trans,
- struct bkey_s_c_dirent dirent)
+static int __remove_dirent(struct btree_trans *trans, struct bpos pos)
{
struct bch_fs *c = trans->c;
- struct qstr name;
+ struct btree_iter *iter;
struct bch_inode_unpacked dir_inode;
struct bch_hash_info dir_hash_info;
- u64 dir_inum = dirent.k->p.inode;
int ret;
- char *buf;
-
- name.len = bch2_dirent_name_bytes(dirent);
- buf = bch2_trans_kmalloc(trans, name.len + 1);
- if (IS_ERR(buf))
- return PTR_ERR(buf);
- memcpy(buf, dirent.v->d_name, name.len);
- buf[name.len] = '\0';
- name.name = buf;
-
- ret = lookup_inode(trans, dir_inum, &dir_inode, NULL);
- if (ret && ret != -EINTR)
- bch_err(c, "remove_dirent: err %i looking up directory inode", ret);
+ ret = lookup_inode(trans, pos.inode, &dir_inode, NULL);
if (ret)
return ret;
dir_hash_info = bch2_hash_info_init(c, &dir_inode);
- ret = bch2_hash_delete(trans, bch2_dirent_hash_desc,
- &dir_hash_info, dir_inum, &name);
- if (ret && ret != -EINTR)
- bch_err(c, "remove_dirent: err %i deleting dirent", ret);
- if (ret)
- return ret;
+ iter = bch2_trans_get_iter(trans, BTREE_ID_dirents, pos, BTREE_ITER_INTENT);
- return 0;
+ ret = bch2_hash_delete_at(trans, bch2_dirent_hash_desc,
+ &dir_hash_info, iter);
+ bch2_trans_iter_put(trans, iter);
+ return ret;
}
-static int remove_dirent(struct btree_trans *trans,
- struct bkey_s_c_dirent dirent)
+static int remove_dirent(struct btree_trans *trans, struct bpos pos)
{
- return __bch2_trans_do(trans, NULL, NULL,
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_LAZY_RW,
- __remove_dirent(trans, dirent));
+ int ret = __bch2_trans_do(trans, NULL, NULL,
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_LAZY_RW,
+ __remove_dirent(trans, pos));
+ if (ret)
+ bch_err(trans->c, "remove_dirent: err %i deleting dirent", ret);
+ return ret;
}
static int __reattach_inode(struct btree_trans *trans,
struct bch_inode_unpacked *lostfound,
u64 inum)
{
- struct bch_fs *c = trans->c;
- int ret;
-
- ret = __bch2_trans_do(trans, NULL, NULL, BTREE_INSERT_LAZY_RW,
+ int ret = __bch2_trans_do(trans, NULL, NULL, BTREE_INSERT_LAZY_RW,
__reattach_inode(trans, lostfound, inum));
if (ret)
- bch_err(c, "error %i reattaching inode %llu", ret, inum);
+ bch_err(trans->c, "error %i reattaching inode %llu", ret, inum);
return ret;
}
goto out;
}
- ret = remove_dirent(trans, bkey_s_c_to_dirent(k));
+ ret = remove_dirent(trans, k.k->p);
out:
bch2_trans_iter_put(trans, iter);
return ret;
"dirent points to missing inode:\n%s",
(bch2_bkey_val_to_text(&PBUF(buf), c,
k), buf))) {
- ret = remove_dirent(&trans, d);
+ ret = remove_dirent(&trans, d.k->p);
if (ret)
goto err;
goto next;
backpointer_exists, c,
"directory %llu with multiple links",
target.bi_inum)) {
- ret = remove_dirent(&trans, d);
+ ret = remove_dirent(&trans, d.k->p);
if (ret)
goto err;
continue;
* We may be called from the device add path, before the new device has
* actually been added to the running filesystem:
*/
- if (c)
+ if (!new_fs)
spin_lock(&c->journal.lock);
memcpy(new_buckets, ja->buckets, ja->nr * sizeof(u64));
swap(new_buckets, ja->buckets);
swap(new_bucket_seq, ja->bucket_seq);
- if (c)
+ if (!new_fs)
spin_unlock(&c->journal.lock);
while (ja->nr < nr) {
struct open_bucket *ob = NULL;
unsigned pos;
- long bucket;
+ long b;
if (new_fs) {
- bucket = bch2_bucket_alloc_new_fs(ca);
- if (bucket < 0) {
+ b = bch2_bucket_alloc_new_fs(ca);
+ if (b < 0) {
ret = -ENOSPC;
goto err;
}
goto err;
}
- bucket = sector_to_bucket(ca, ob->ptr.offset);
- }
+ b = sector_to_bucket(ca, ob->ptr.offset);
- if (c) {
percpu_down_read(&c->mark_lock);
spin_lock(&c->journal.lock);
}
__array_insert_item(journal_buckets->buckets, ja->nr, pos);
ja->nr++;
- ja->buckets[pos] = bucket;
+ ja->buckets[pos] = b;
ja->bucket_seq[pos] = 0;
- journal_buckets->buckets[pos] = cpu_to_le64(bucket);
+ journal_buckets->buckets[pos] = cpu_to_le64(b);
if (pos <= ja->discard_idx)
ja->discard_idx = (ja->discard_idx + 1) % ja->nr;
if (pos <= ja->cur_idx)
ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
- if (!c || new_fs)
- bch2_mark_metadata_bucket(c, ca, bucket, BCH_DATA_journal,
+ if (new_fs) {
+ bch2_mark_metadata_bucket(c, ca, b, BCH_DATA_journal,
ca->mi.bucket_size,
gc_phase(GC_PHASE_SB),
0);
-
- if (c) {
+ } else {
spin_unlock(&c->journal.lock);
percpu_up_read(&c->mark_lock);
- }
- if (c && !new_fs)
ret = bch2_trans_do(c, NULL, NULL, BTREE_INSERT_NOFAIL,
- bch2_trans_mark_metadata_bucket(&trans, NULL, ca,
- bucket, BCH_DATA_journal,
+ bch2_trans_mark_metadata_bucket(&trans, ca,
+ b, BCH_DATA_journal,
ca->mi.bucket_size));
- if (!new_fs)
bch2_open_bucket_put(c, ob);
- if (ret)
- goto err;
+ if (ret)
+ goto err;
+ }
}
err:
bch2_sb_resize_journal(&ca->disk_sb,
}
static inline void bch2_journal_add_keys(struct journal *j, struct journal_res *res,
- enum btree_id id, const struct bkey_i *k)
+ enum btree_id id, unsigned level,
+ const struct bkey_i *k)
{
bch2_journal_add_entry(j, res, BCH_JSET_ENTRY_btree_keys,
- id, 0, k, k->k.u64s);
+ id, level, k, k->k.u64s);
}
static inline bool journal_entry_empty(struct jset *j)
struct bch_fs *c = container_of(j, struct bch_fs, journal);
bool kthread = (current->flags & PF_KTHREAD) != 0;
u64 seq_to_flush;
- size_t min_nr, nr_flushed;
+ size_t min_nr, min_key_cache, nr_flushed;
unsigned flags;
int ret = 0;
atomic_long_read(&c->btree_key_cache.nr_dirty),
atomic_long_read(&c->btree_key_cache.nr_keys));
+ min_key_cache = min(bch2_nr_btree_keys_need_flush(c), 128UL);
+
nr_flushed = journal_flush_pins(j, seq_to_flush,
- min_nr,
- min(bch2_nr_btree_keys_need_flush(c), 128UL));
+ min_nr, min_key_cache);
if (direct)
j->nr_direct_reclaim += nr_flushed;
if (nr_flushed)
wake_up(&j->reclaim_wait);
- } while (min_nr && nr_flushed && !direct);
+ } while ((min_nr || min_key_cache) && !direct);
memalloc_noreclaim_restore(flags);
bch2_bkey_buf_init(&_insert);
bch2_bkey_buf_realloc(&_insert, c, U8_MAX);
- bch2_trans_init(&trans, c, BTREE_ITER_MAX, 0);
+ bch2_trans_init(&trans, c, BTREE_ITER_MAX, 1024);
iter = bch2_trans_get_iter(&trans, m->btree_id,
bkey_start_pos(&bch2_keylist_front(keys)->k),
spin_lock(&ca->fs->freelist_lock);
ret = fifo_full(&ca->free[RESERVE_MOVINGGC]) ||
- ca->allocator_state != ALLOCATOR_RUNNING;
+ ca->allocator_state != ALLOCATOR_running;
spin_unlock(&ca->fs->freelist_lock);
return ret;
ret = bch2_move_data(c,
0, POS_MIN,
BTREE_ID_NR, POS_MAX,
- &c->copygc_pd.rate,
+ NULL,
writepoint_ptr(&c->copygc_write_point),
copygc_pred, NULL,
&move_stats);
{
struct bch_dev *ca;
unsigned dev_idx;
- u64 fragmented_allowed = c->copygc_threshold;
- u64 fragmented = 0;
+ u64 fragmented_allowed = 0, fragmented = 0;
for_each_rw_member(ca, c, dev_idx) {
struct bch_dev_usage usage = bch2_dev_usage_read(ca);
wait = bch2_copygc_wait_amount(c);
if (wait > clock->max_slop) {
+ c->copygc_wait = last + wait;
bch2_kthread_io_clock_wait(clock, last + wait,
MAX_SCHEDULE_TIMEOUT);
continue;
}
+ c->copygc_wait = 0;
+
if (bch2_copygc(c))
break;
}
void bch2_copygc_stop(struct bch_fs *c)
{
- c->copygc_pd.rate.rate = UINT_MAX;
- bch2_ratelimit_reset(&c->copygc_pd.rate);
-
if (c->copygc_thread) {
kthread_stop(c->copygc_thread);
put_task_struct(c->copygc_thread);
void bch2_fs_copygc_init(struct bch_fs *c)
{
- bch2_pd_controller_init(&c->copygc_pd);
- c->copygc_pd.d_term = 0;
}
}
- if (!c->sb.clean &&
- !(c->sb.features & (1 << BCH_FEATURE_atomic_nlink))) {
- bch_info(c, "BCH_FEATURE_atomic_nlink not set and filesystem dirty, fsck required");
- c->opts.fsck = true;
- c->opts.fix_errors = FSCK_OPT_YES;
- }
-
if (!(c->sb.features & (1ULL << BCH_FEATURE_alloc_v2))) {
bch_info(c, "alloc_v2 feature bit not set, fsck required");
c->opts.fsck = true;
!(c->sb.compat & (1ULL << BCH_COMPAT_alloc_info)) ||
!(c->sb.compat & (1ULL << BCH_COMPAT_alloc_metadata)) ||
test_bit(BCH_FS_REBUILD_REPLICAS, &c->flags)) {
+ bool metadata_only = c->opts.norecovery;
+
bch_info(c, "starting mark and sweep");
err = "error in mark and sweep";
- ret = bch2_gc(c, true);
+ ret = bch2_gc(c, true, metadata_only);
if (ret)
goto err;
bch_verbose(c, "mark and sweep done");
}
if (c->opts.fsck &&
- !test_bit(BCH_FS_ERROR, &c->flags)) {
- c->disk_sb.sb->features[0] |= 1ULL << BCH_FEATURE_atomic_nlink;
+ !test_bit(BCH_FS_ERROR, &c->flags) &&
+ BCH_SB_HAS_ERRORS(c->disk_sb.sb)) {
SET_BCH_SB_HAS_ERRORS(c->disk_sb.sb, 0);
write_sb = true;
}
* Write out the superblock and journal buckets, now that we can do
* btree updates
*/
- err = "error writing alloc info";
- ret = bch2_alloc_write(c, 0);
- if (ret)
- goto err;
+ err = "error marking superblock and journal";
+ for_each_member_device(ca, c, i) {
+ ret = bch2_trans_mark_dev_sb(c, ca);
+ if (ret)
+ goto err;
+ }
bch2_inode_init(c, &root_inode, 0, 0,
S_IFDIR|S_IRWXU|S_IRUGO|S_IXUGO, 0, NULL);
out:
free_percpu(new_gc);
kfree(new_scratch);
- free_percpu(new_usage[1]);
- free_percpu(new_usage[0]);
+ for (i = 0; i < ARRAY_SIZE(new_usage); i++)
+ free_percpu(new_usage[i]);
kfree(new_base);
return ret;
err:
percpu_ref_kill(&c->writes);
cancel_work_sync(&c->ec_stripe_delete_work);
- cancel_delayed_work(&c->pd_controllers_update);
/*
* If we're not doing an emergency shutdown, we want to wait on
return ret;
}
- schedule_delayed_work(&c->pd_controllers_update, 5 * HZ);
-
schedule_work(&c->ec_stripe_delete_work);
return 0;
cancel_work_sync(&ca->io_error_work);
cancel_work_sync(&c->btree_write_error_work);
- cancel_delayed_work_sync(&c->pd_controllers_update);
cancel_work_sync(&c->read_only_work);
for (i = 0; i < c->sb.nr_devices; i++)
/*
* Allocator threads don't start filling copygc reserve until after we
* set BCH_FS_STARTED - wake them now:
+ *
+ * XXX ugly hack:
+ * Need to set ca->allocator_state here instead of relying on the
+ * allocator threads to do it to avoid racing with the copygc threads
+ * checking it and thinking they have no alloc reserve:
*/
- for_each_online_member(ca, c, i)
+ for_each_online_member(ca, c, i) {
+ ca->allocator_state = ALLOCATOR_running;
bch2_wake_allocator(ca);
+ }
if (c->opts.read_only || c->opts.nochanges) {
bch2_fs_read_only(c);
bch2_dev_usage_journal_reserve(c);
err = "error marking superblock";
- ret = bch2_trans_mark_dev_sb(c, NULL, ca);
+ ret = bch2_trans_mark_dev_sb(c, ca);
if (ret)
goto err_late;
ca = bch_dev_locked(c, dev_idx);
- if (bch2_trans_mark_dev_sb(c, NULL, ca)) {
+ if (bch2_trans_mark_dev_sb(c, ca)) {
err = "bch2_trans_mark_dev_sb() error";
goto err;
}
} while (0)
write_attribute(trigger_journal_flush);
-write_attribute(trigger_btree_coalesce);
write_attribute(trigger_gc);
write_attribute(prune_cache);
rw_attribute(btree_gc_periodic);
+rw_attribute(gc_gens_pos);
read_attribute(uuid);
read_attribute(minor);
rw_attribute(label);
rw_attribute(copy_gc_enabled);
-sysfs_pd_controller_attribute(copy_gc);
+read_attribute(copy_gc_wait);
rw_attribute(rebalance_enabled);
sysfs_pd_controller_attribute(rebalance);
read_attribute(new_stripes);
-rw_attribute(pd_controllers_update_seconds);
-
read_attribute(io_timers_read);
read_attribute(io_timers_write);
return 0;
}
+void bch2_gc_gens_pos_to_text(struct printbuf *out, struct bch_fs *c)
+{
+ pr_buf(out, "%s: ", bch2_btree_ids[c->gc_gens_btree]);
+ bch2_bpos_to_text(out, c->gc_gens_pos);
+ pr_buf(out, "\n");
+}
+
SHOW(bch2_fs)
{
struct bch_fs *c = container_of(kobj, struct bch_fs, kobj);
sysfs_printf(btree_gc_periodic, "%u", (int) c->btree_gc_periodic);
- sysfs_printf(copy_gc_enabled, "%i", c->copy_gc_enabled);
+ if (attr == &sysfs_gc_gens_pos) {
+ bch2_gc_gens_pos_to_text(&out, c);
+ return out.pos - buf;
+ }
- sysfs_print(pd_controllers_update_seconds,
- c->pd_controllers_update_seconds);
+ sysfs_printf(copy_gc_enabled, "%i", c->copy_gc_enabled);
sysfs_printf(rebalance_enabled, "%i", c->rebalance.enabled);
sysfs_pd_controller_show(rebalance, &c->rebalance.pd); /* XXX */
- sysfs_pd_controller_show(copy_gc, &c->copygc_pd);
+ sysfs_hprint(copy_gc_wait,
+ max(0LL, c->copygc_wait -
+ atomic64_read(&c->io_clock[WRITE].now)) << 9);
if (attr == &sysfs_rebalance_work) {
bch2_rebalance_work_to_text(&out, c);
return ret;
}
- sysfs_strtoul(pd_controllers_update_seconds,
- c->pd_controllers_update_seconds);
sysfs_pd_controller_store(rebalance, &c->rebalance.pd);
- sysfs_pd_controller_store(copy_gc, &c->copygc_pd);
sysfs_strtoul(promote_whole_extents, c->promote_whole_extents);
if (attr == &sysfs_trigger_journal_flush)
bch2_journal_meta(&c->journal);
- if (attr == &sysfs_trigger_btree_coalesce)
- bch2_coalesce(c);
-
if (attr == &sysfs_trigger_gc) {
/*
* Full gc is currently incompatible with btree key cache:
&sysfs_extent_migrate_raced,
&sysfs_trigger_journal_flush,
- &sysfs_trigger_btree_coalesce,
&sysfs_trigger_gc,
+ &sysfs_gc_gens_pos,
&sysfs_prune_cache,
&sysfs_copy_gc_enabled,
+ &sysfs_copy_gc_wait,
&sysfs_rebalance_enabled,
&sysfs_rebalance_work,
sysfs_pd_controller_files(rebalance),
- sysfs_pd_controller_files(copy_gc),
&sysfs_new_stripes,
"free[RESERVE_MOVINGGC]\t%zu/%zu\n"
"free[RESERVE_NONE]\t%zu/%zu\n"
"freelist_wait\t\t%s\n"
- "open buckets\t\t%u/%u (reserved %u)\n"
+ "open buckets allocated\t%u\n"
+ "open buckets this dev\t%u\n"
+ "open buckets total\t%u\n"
"open_buckets_wait\t%s\n"
"open_buckets_btree\t%u\n"
"open_buckets_user\t%u\n"
- "btree reserve cache\t%u\n",
+ "btree reserve cache\t%u\n"
+ "thread state:\t\t%s\n",
stats.buckets_ec,
__dev_buckets_available(ca, stats),
fifo_used(&ca->free_inc), ca->free_inc.size,
fifo_used(&ca->free[RESERVE_MOVINGGC]), ca->free[RESERVE_MOVINGGC].size,
fifo_used(&ca->free[RESERVE_NONE]), ca->free[RESERVE_NONE].size,
c->freelist_wait.list.first ? "waiting" : "empty",
- c->open_buckets_nr_free, OPEN_BUCKETS_COUNT,
- BTREE_NODE_OPEN_BUCKET_RESERVE,
+ OPEN_BUCKETS_COUNT - c->open_buckets_nr_free,
+ ca->nr_open_buckets,
+ OPEN_BUCKETS_COUNT,
c->open_buckets_wait.list.first ? "waiting" : "empty",
nr[BCH_DATA_btree],
nr[BCH_DATA_user],
- c->btree_reserve_cache_nr);
+ c->btree_reserve_cache_nr,
+ bch2_allocator_states[ca->allocator_state]);
}
static const char * const bch2_rw[] = {
return ret;
}
+static int rand_insert_multi(struct bch_fs *c, u64 nr)
+{
+ struct btree_trans trans;
+ struct bkey_i_cookie k[8];
+ int ret = 0;
+ unsigned j;
+ u64 i;
+
+ bch2_trans_init(&trans, c, 0, 0);
+
+ for (i = 0; i < nr; i += ARRAY_SIZE(k)) {
+ for (j = 0; j < ARRAY_SIZE(k); j++) {
+ bkey_cookie_init(&k[j].k_i);
+ k[j].k.p.offset = test_rand();
+ k[j].k.p.snapshot = U32_MAX;
+ }
+
+ ret = __bch2_trans_do(&trans, NULL, NULL, 0,
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[0].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[1].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[2].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[3].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[4].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[5].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[6].k_i) ?:
+ __bch2_btree_insert(&trans, BTREE_ID_xattrs, &k[7].k_i));
+ if (ret) {
+ bch_err(c, "error in rand_insert_multi: %i", ret);
+ break;
+ }
+ }
+
+ bch2_trans_exit(&trans);
+ return ret;
+}
+
static int rand_lookup(struct bch_fs *c, u64 nr)
{
struct btree_trans trans;
if (!strcmp(testname, #_test)) j.fn = _test
perf_test(rand_insert);
+ perf_test(rand_insert_multi);
perf_test(rand_lookup);
perf_test(rand_mixed);
perf_test(rand_delete);
projects / bcachefs-tools-debian / commitdiff