-b1107114caf6aa6f725170f3d75b072badcfa573
+021e62a098d9fa7e558ae935180e2fb16bb50a3a
#ifndef _LINUX_SCHED_MM_H
#define _LINUX_SCHED_MM_H
-#define PF_MEMALLOC_NOFS 0
+#define PF_MEMALLOC 0x00000800 /* Allocating memory */
+#define PF_MEMALLOC_NOFS 0x00040000 /* All allocation requests will inherit GFP_NOFS */
static inline unsigned int memalloc_nofs_save(void)
{
current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags;
}
+static inline unsigned int memalloc_noreclaim_save(void)
+{
+ unsigned int flags = current->flags & PF_MEMALLOC;
+ current->flags |= PF_MEMALLOC;
+ return flags;
+}
+
+static inline void memalloc_noreclaim_restore(unsigned int flags)
+{
+ current->flags = (current->flags & ~PF_MEMALLOC) | flags;
+}
+
#endif /* _LINUX_SCHED_MM_H */
return p;
}
+struct kmem_cache {
+ size_t obj_size;
+};
+
+static inline void *kmem_cache_alloc(struct kmem_cache *c, gfp_t gfp)
+{
+ return kmalloc(c->obj_size, gfp);
+}
+
+static inline void kmem_cache_free(struct kmem_cache *c, void *p)
+{
+ kfree(p);
+}
+
+static inline void kmem_cache_destroy(struct kmem_cache *p)
+{
+ kfree(p);
+}
+
+static inline struct kmem_cache *kmem_cache_create(size_t obj_size)
+{
+ struct kmem_cache *p = kmalloc(sizeof(*p), GFP_KERNEL);
+ if (!p)
+ return NULL;
+
+ p->obj_size = obj_size;
+ return p;
+}
+
+#define KMEM_CACHE(_struct, _flags) kmem_cache_create(sizeof(struct _struct))
+
#endif /* __TOOLS_LINUX_SLAB_H */
TP_ARGS(bio)
);
+TRACE_EVENT(journal_reclaim_start,
+ TP_PROTO(struct bch_fs *c, u64 min_nr,
+ u64 prereserved, u64 prereserved_total,
+ u64 btree_cache_dirty, u64 btree_cache_total,
+ u64 btree_key_cache_dirty, u64 btree_key_cache_total),
+ TP_ARGS(c, min_nr, prereserved, prereserved_total,
+ btree_cache_dirty, btree_cache_total,
+ btree_key_cache_dirty, btree_key_cache_total),
+
+ TP_STRUCT__entry(
+ __array(char, uuid, 16 )
+ __field(u64, min_nr )
+ __field(u64, prereserved )
+ __field(u64, prereserved_total )
+ __field(u64, btree_cache_dirty )
+ __field(u64, btree_cache_total )
+ __field(u64, btree_key_cache_dirty )
+ __field(u64, btree_key_cache_total )
+ ),
+
+ TP_fast_assign(
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
+ __entry->min_nr = min_nr;
+ __entry->prereserved = prereserved;
+ __entry->prereserved_total = prereserved_total;
+ __entry->btree_cache_dirty = btree_cache_dirty;
+ __entry->btree_cache_total = btree_cache_total;
+ __entry->btree_key_cache_dirty = btree_key_cache_dirty;
+ __entry->btree_key_cache_total = btree_key_cache_total;
+ ),
+
+ TP_printk("%pU min %llu prereserved %llu/%llu btree cache %llu/%llu key cache %llu/%llu",
+ __entry->uuid,
+ __entry->min_nr,
+ __entry->prereserved,
+ __entry->prereserved_total,
+ __entry->btree_cache_dirty,
+ __entry->btree_cache_total,
+ __entry->btree_key_cache_dirty,
+ __entry->btree_key_cache_total)
+);
+
+TRACE_EVENT(journal_reclaim_finish,
+ TP_PROTO(struct bch_fs *c, u64 nr_flushed),
+ TP_ARGS(c, nr_flushed),
+
+ TP_STRUCT__entry(
+ __array(char, uuid, 16 )
+ __field(u64, nr_flushed )
+ ),
+
+ TP_fast_assign(
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
+ __entry->nr_flushed = nr_flushed;
+ ),
+
+ TP_printk("%pU flushed %llu", __entry->uuid, __entry->nr_flushed)
+);
+
/* bset.c: */
DEFINE_EVENT(bpos, bkey_pack_pos_fail,
TP_ARGS(ip)
);
+DEFINE_EVENT(transaction_restart, trans_restart_journal_reclaim,
+ TP_PROTO(unsigned long ip),
+ TP_ARGS(ip)
+);
+
DEFINE_EVENT(transaction_restart, trans_restart_mark_replicas,
TP_PROTO(unsigned long ip),
TP_ARGS(ip)
TP_ARGS(ip)
);
-DEFINE_EVENT(transaction_restart, trans_restart_atomic,
- TP_PROTO(unsigned long ip),
- TP_ARGS(ip)
-);
-
DECLARE_EVENT_CLASS(node_lock_fail,
TP_PROTO(unsigned level, u32 iter_seq, unsigned node, u32 node_seq),
TP_ARGS(level, iter_seq, node, node_seq),
return 0;
p = kthread_create(bch2_allocator_thread, ca,
- "bch_alloc[%s]", ca->name);
+ "bch-alloc/%s", ca->name);
if (IS_ERR(p))
return PTR_ERR(p);
struct workqueue_struct *wq;
/* copygc needs its own workqueue for index updates.. */
struct workqueue_struct *copygc_wq;
- struct workqueue_struct *journal_reclaim_wq;
/* ALLOCATION */
struct delayed_work pd_controllers_update;
stats.floats,
stats.failed);
}
+
+void bch2_btree_cache_to_text(struct printbuf *out, struct bch_fs *c)
+{
+ pr_buf(out, "nr nodes:\t%u\n", c->btree_cache.used);
+ pr_buf(out, "nr dirty:\t%u\n", atomic_read(&c->btree_cache.dirty));
+}
void bch2_btree_node_to_text(struct printbuf *, struct bch_fs *,
struct btree *);
+void bch2_btree_cache_to_text(struct printbuf *, struct bch_fs *);
#endif /* _BCACHEFS_BTREE_CACHE_H */
BUG_ON(c->gc_thread);
- p = kthread_create(bch2_gc_thread, c, "bch_gc");
+ p = kthread_create(bch2_gc_thread, c, "bch-gc/%s", c->name);
if (IS_ERR(p))
return PTR_ERR(p);
#include <linux/sched/mm.h>
#include <trace/events/bcachefs.h>
+static struct kmem_cache *bch2_key_cache;
+
static int bch2_btree_key_cache_cmp_fn(struct rhashtable_compare_arg *arg,
const void *obj)
{
{
struct bch_fs *c = container_of(bc, struct bch_fs, btree_key_cache);
+ BUG_ON(test_bit(BKEY_CACHED_DIRTY, &ck->flags));
+
ck->btree_trans_barrier_seq =
start_poll_synchronize_srcu(&c->btree_trans_barrier);
- list_move(&ck->list, &bc->freed);
+ list_move_tail(&ck->list, &bc->freed);
+ bc->nr_freed++;
kfree(ck->k);
ck->k = NULL;
{
struct bkey_cached *ck;
- list_for_each_entry(ck, &c->freed, list)
- if (bkey_cached_lock_for_evict(ck))
+ list_for_each_entry_reverse(ck, &c->freed, list)
+ if (bkey_cached_lock_for_evict(ck)) {
+ c->nr_freed--;
return ck;
+ }
+
+ ck = kmem_cache_alloc(bch2_key_cache, GFP_NOFS|__GFP_ZERO);
+ if (likely(ck)) {
+ INIT_LIST_HEAD(&ck->list);
+ six_lock_init(&ck->c.lock);
+ BUG_ON(!six_trylock_intent(&ck->c.lock));
+ BUG_ON(!six_trylock_write(&ck->c.lock));
+ return ck;
+ }
list_for_each_entry(ck, &c->clean, list)
if (bkey_cached_lock_for_evict(ck)) {
return ck;
}
- ck = kzalloc(sizeof(*ck), GFP_NOFS);
- if (!ck)
- return NULL;
-
- INIT_LIST_HEAD(&ck->list);
- six_lock_init(&ck->c.lock);
- BUG_ON(!six_trylock_intent(&ck->c.lock));
- BUG_ON(!six_trylock_write(&ck->c.lock));
-
- return ck;
+ return NULL;
}
static struct bkey_cached *
ck->key.btree_id = btree_id;
ck->key.pos = pos;
ck->valid = false;
-
- BUG_ON(ck->flags);
+ ck->flags = 1U << BKEY_CACHED_ACCESSED;
if (rhashtable_lookup_insert_fast(&c->table,
&ck->hash,
goto err;
}
+ if (!test_bit(BKEY_CACHED_ACCESSED, &ck->flags))
+ set_bit(BKEY_CACHED_ACCESSED, &ck->flags);
+
iter->uptodate = BTREE_ITER_NEED_PEEK;
bch2_btree_iter_downgrade(iter);
return ret;
{
struct bch_fs *c = trans->c;
struct bkey_cached *ck = (void *) iter->l[0].b;
+ bool kick_reclaim = false;
BUG_ON(insert->u64s > ck->u64s);
set_bit(BKEY_CACHED_DIRTY, &ck->flags);
c->btree_key_cache.nr_dirty++;
+
+ if (bch2_nr_btree_keys_need_flush(c))
+ kick_reclaim = true;
+
mutex_unlock(&c->btree_key_cache.lock);
}
bch2_journal_pin_update(&c->journal, trans->journal_res.seq,
&ck->journal, btree_key_cache_journal_flush);
+
+ if (kick_reclaim)
+ journal_reclaim_kick(&c->journal);
return true;
}
flags = memalloc_nofs_save();
+ /*
+ * Newest freed entries are at the end of the list - once we hit one
+ * that's too new to be freed, we can bail out:
+ */
list_for_each_entry_safe(ck, t, &bc->freed, list) {
- scanned++;
+ if (!poll_state_synchronize_srcu(&c->btree_trans_barrier,
+ ck->btree_trans_barrier_seq))
+ break;
- if (poll_state_synchronize_srcu(&c->btree_trans_barrier,
- ck->btree_trans_barrier_seq)) {
- list_del(&ck->list);
- kfree(ck);
- freed++;
- }
-
- if (scanned >= nr)
- goto out;
+ list_del(&ck->list);
+ kmem_cache_free(bch2_key_cache, ck);
+ bc->nr_freed--;
+ scanned++;
+ freed++;
}
- list_for_each_entry_safe(ck, t, &bc->clean, list) {
- scanned++;
+ if (scanned >= nr)
+ goto out;
- if (bkey_cached_lock_for_evict(ck)) {
+ list_for_each_entry_safe(ck, t, &bc->clean, list) {
+ if (test_bit(BKEY_CACHED_ACCESSED, &ck->flags))
+ clear_bit(BKEY_CACHED_ACCESSED, &ck->flags);
+ else if (bkey_cached_lock_for_evict(ck)) {
bkey_cached_evict(bc, ck);
bkey_cached_free(bc, ck);
}
+ scanned++;
if (scanned >= nr) {
if (&t->list != &bc->clean)
list_move_tail(&bc->clean, &t->list);
bch2_journal_preres_put(&c->journal, &ck->res);
kfree(ck->k);
- kfree(ck);
+ list_del(&ck->list);
+ kmem_cache_free(bch2_key_cache, ck);
bc->nr_keys--;
}
BUG_ON(bc->nr_dirty && !bch2_journal_error(&c->journal));
BUG_ON(bc->nr_keys);
- list_for_each_entry_safe(ck, n, &bc->freed, list)
- kfree(ck);
+ list_for_each_entry_safe(ck, n, &bc->freed, list) {
+ list_del(&ck->list);
+ kmem_cache_free(bch2_key_cache, ck);
+ }
mutex_unlock(&bc->lock);
- rhashtable_destroy(&bc->table);
+ if (bc->table_init_done)
+ rhashtable_destroy(&bc->table);
}
void bch2_fs_btree_key_cache_init_early(struct btree_key_cache *c)
int bch2_fs_btree_key_cache_init(struct btree_key_cache *c)
{
+ int ret;
+
+ c->shrink.seeks = 1;
c->shrink.count_objects = bch2_btree_key_cache_count;
c->shrink.scan_objects = bch2_btree_key_cache_scan;
- return register_shrinker(&c->shrink) ?:
- rhashtable_init(&c->table, &bch2_btree_key_cache_params);
+ ret = register_shrinker(&c->shrink);
+ if (ret)
+ return ret;
+
+ ret = rhashtable_init(&c->table, &bch2_btree_key_cache_params);
+ if (ret)
+ return ret;
+
+ c->table_init_done = true;
+ return 0;
}
void bch2_btree_key_cache_to_text(struct printbuf *out, struct btree_key_cache *c)
{
- struct bucket_table *tbl;
- struct bkey_cached *ck;
- struct rhash_head *pos;
- size_t i;
+ pr_buf(out, "nr_freed:\t%zu\n", c->nr_freed);
+ pr_buf(out, "nr_keys:\t%zu\n", c->nr_keys);
+ pr_buf(out, "nr_dirty:\t%zu\n", c->nr_dirty);
+}
- mutex_lock(&c->lock);
- tbl = rht_dereference_rcu(c->table.tbl, &c->table);
+void bch2_btree_key_cache_exit(void)
+{
+ if (bch2_key_cache)
+ kmem_cache_destroy(bch2_key_cache);
+}
- for (i = 0; i < tbl->size; i++) {
- rht_for_each_entry_rcu(ck, pos, tbl, i, hash) {
- pr_buf(out, "%s:",
- bch2_btree_ids[ck->key.btree_id]);
- bch2_bpos_to_text(out, ck->key.pos);
+int __init bch2_btree_key_cache_init(void)
+{
+ bch2_key_cache = KMEM_CACHE(bkey_cached, 0);
+ if (!bch2_key_cache)
+ return -ENOMEM;
- if (test_bit(BKEY_CACHED_DIRTY, &ck->flags))
- pr_buf(out, " journal seq %llu", ck->journal.seq);
- pr_buf(out, "\n");
- }
- }
- mutex_unlock(&c->lock);
+ return 0;
}
#ifndef _BCACHEFS_BTREE_KEY_CACHE_H
#define _BCACHEFS_BTREE_KEY_CACHE_H
+static inline size_t bch2_nr_btree_keys_need_flush(struct bch_fs *c)
+{
+ size_t nr_dirty = READ_ONCE(c->btree_key_cache.nr_dirty);
+ size_t nr_keys = READ_ONCE(c->btree_key_cache.nr_dirty);
+ size_t max_dirty = 4096 + nr_keys / 2;
+
+ return max_t(ssize_t, 0, nr_dirty - max_dirty);
+}
+
+static inline bool bch2_btree_key_cache_must_wait(struct bch_fs *c)
+{
+ size_t nr_dirty = READ_ONCE(c->btree_key_cache.nr_dirty);
+ size_t nr_keys = READ_ONCE(c->btree_key_cache.nr_dirty);
+ size_t max_dirty = 4096 + (nr_keys * 3) / 4;
+
+ return nr_dirty > max_dirty;
+}
+
struct bkey_cached *
bch2_btree_key_cache_find(struct bch_fs *, enum btree_id, struct bpos);
void bch2_btree_key_cache_to_text(struct printbuf *, struct btree_key_cache *);
+void bch2_btree_key_cache_exit(void);
+int __init bch2_btree_key_cache_init(void);
+
#endif /* _BCACHEFS_BTREE_KEY_CACHE_H */
struct btree_key_cache {
struct mutex lock;
struct rhashtable table;
+ bool table_init_done;
struct list_head freed;
struct list_head clean;
struct list_head dirty;
struct shrinker shrink;
+ size_t nr_freed;
size_t nr_keys;
size_t nr_dirty;
};
struct bpos pos;
} __attribute__((packed, aligned(4)));
-#define BKEY_CACHED_DIRTY 0
+#define BKEY_CACHED_ACCESSED 0
+#define BKEY_CACHED_DIRTY 1
struct bkey_cached {
struct btree_bkey_cached_common c;
BTREE_INSERT_ENOSPC,
BTREE_INSERT_NEED_MARK_REPLICAS,
BTREE_INSERT_NEED_JOURNAL_RES,
+ BTREE_INSERT_NEED_JOURNAL_RECLAIM,
};
enum btree_gc_coalesce_fail_reason {
int bch2_btree_insert(struct bch_fs *, enum btree_id, struct bkey_i *,
struct disk_reservation *, u64 *, int flags);
-int bch2_btree_delete_at_range(struct btree_trans *, struct btree_iter *,
- struct bpos, u64 *);
+int bch2_btree_delete_range_trans(struct btree_trans *, enum btree_id,
+ struct bpos, struct bpos, u64 *);
int bch2_btree_delete_range(struct bch_fs *, enum btree_id,
struct bpos, struct bpos, u64 *);
break;
bp = bkey_s_c_to_btree_ptr_v2(k);
- BUG_ON(bkey_cmp(next_node, bp.v->min_key));
+ if (bkey_cmp(next_node, bp.v->min_key)) {
+ bch2_dump_btree_node(c, b);
+ panic("expected next min_key %llu:%llu got %llu:%llu\n",
+ next_node.inode,
+ next_node.offset,
+ bp.v->min_key.inode,
+ bp.v->min_key.offset);
+ }
bch2_btree_node_iter_advance(&iter, b);
if (bch2_btree_node_iter_end(&iter)) {
- BUG_ON(bkey_cmp(k.k->p, b->key.k.p));
+
+ if (bkey_cmp(k.k->p, b->key.k.p)) {
+ bch2_dump_btree_node(c, b);
+ panic("expected end %llu:%llu got %llu:%llu\n",
+ b->key.k.p.inode,
+ b->key.k.p.offset,
+ k.k->p.inode,
+ k.k->p.offset);
+ }
break;
}
struct bkey_packed *k;
const char *invalid;
- invalid = bch2_bkey_invalid(c, bkey_i_to_s_c(insert), btree_node_type(b));
+ invalid = bch2_bkey_invalid(c, bkey_i_to_s_c(insert), btree_node_type(b)) ?:
+ bch2_bkey_in_btree_node(b, bkey_i_to_s_c(insert));
if (invalid) {
char buf[160];
BUG_ON(!as || as->b);
bch2_verify_keylist_sorted(keys);
- if (as->must_rewrite)
- goto split;
-
bch2_btree_node_lock_for_insert(c, b, iter);
if (!bch2_btree_node_insert_fits(c, b, bch2_keylist_u64s(keys))) {
goto split;
}
+ btree_node_interior_verify(c, b);
+
bch2_btree_insert_keys_interior(as, b, iter, keys);
live_u64s_added = (int) b->nr.live_u64s - old_live_u64s;
BTREE_INTERIOR_UPDATING_AS,
} mode;
- unsigned must_rewrite:1;
unsigned nodes_written:1;
enum btree_id btree_id;
BUG_ON(iter->level);
+ if (!test_bit(BKEY_CACHED_DIRTY, &ck->flags) &&
+ bch2_btree_key_cache_must_wait(trans->c))
+ return BTREE_INSERT_NEED_JOURNAL_RECLAIM;
+
if (u64s <= ck->u64s)
return BTREE_INSERT_OK;
trace_trans_restart_journal_res_get(trans->ip);
ret = -EINTR;
break;
- default:
- BUG_ON(ret >= 0);
- break;
- }
-
- if (ret == -EINTR) {
- int ret2 = bch2_btree_iter_traverse_all(trans);
+ case BTREE_INSERT_NEED_JOURNAL_RECLAIM:
+ bch2_trans_unlock(trans);
- if (ret2) {
- trace_trans_restart_traverse(trans->ip);
- return ret2;
+ while (bch2_btree_key_cache_must_wait(c)) {
+ mutex_lock(&c->journal.reclaim_lock);
+ bch2_journal_reclaim(&c->journal);
+ mutex_unlock(&c->journal.reclaim_lock);
}
- trace_trans_restart_atomic(trans->ip);
+ if (bch2_trans_relock(trans))
+ return 0;
+
+ trace_trans_restart_journal_reclaim(trans->ip);
+ ret = -EINTR;
+ break;
+ default:
+ BUG_ON(ret >= 0);
+ break;
}
return ret;
__bch2_btree_insert(&trans, id, k));
}
-int bch2_btree_delete_at_range(struct btree_trans *trans,
- struct btree_iter *iter,
- struct bpos end,
- u64 *journal_seq)
+int bch2_btree_delete_at(struct btree_trans *trans,
+ struct btree_iter *iter, unsigned flags)
+{
+ struct bkey_i k;
+
+ bkey_init(&k.k);
+ k.k.p = iter->pos;
+
+ bch2_trans_update(trans, iter, &k, 0);
+ return bch2_trans_commit(trans, NULL, NULL,
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_USE_RESERVE|flags);
+}
+
+int bch2_btree_delete_range_trans(struct btree_trans *trans, enum btree_id id,
+ struct bpos start, struct bpos end,
+ u64 *journal_seq)
{
+ struct btree_iter *iter;
struct bkey_s_c k;
int ret = 0;
+
+ iter = bch2_trans_get_iter(trans, id, start, BTREE_ITER_INTENT);
+ ret = PTR_ERR_OR_ZERO(iter);
+ if (ret)
+ return ret;
retry:
while ((k = bch2_btree_iter_peek(iter)).k &&
!(ret = bkey_err(k)) &&
bkey_init(&delete.k);
+ /*
+ * This could probably be more efficient for extents:
+ */
+
/*
* For extents, iter.pos won't necessarily be the same as
* bkey_start_pos(k.k) (for non extents they always will be the
goto retry;
}
+ bch2_trans_iter_put(trans, iter);
return ret;
-
-}
-
-int bch2_btree_delete_at(struct btree_trans *trans,
- struct btree_iter *iter, unsigned flags)
-{
- struct bkey_i k;
-
- bkey_init(&k.k);
- k.k.p = iter->pos;
-
- bch2_trans_update(trans, iter, &k, 0);
- return bch2_trans_commit(trans, NULL, NULL,
- BTREE_INSERT_NOFAIL|
- BTREE_INSERT_USE_RESERVE|flags);
}
/*
struct bpos start, struct bpos end,
u64 *journal_seq)
{
- struct btree_trans trans;
- struct btree_iter *iter;
- int ret = 0;
-
- /*
- * XXX: whether we need mem/more iters depends on whether this btree id
- * has triggers
- */
- bch2_trans_init(&trans, c, BTREE_ITER_MAX, 512);
-
- iter = bch2_trans_get_iter(&trans, id, start, BTREE_ITER_INTENT);
-
- ret = bch2_btree_delete_at_range(&trans, iter, end, journal_seq);
- ret = bch2_trans_exit(&trans) ?: ret;
-
- BUG_ON(ret == -EINTR);
- return ret;
+ return bch2_trans_do(c, NULL, journal_seq, 0,
+ bch2_btree_delete_range_trans(&trans, id, start, end, journal_seq));
}
return avail_factor(__bch2_fs_usage_read_short(c).free);
}
-void __bch2_disk_reservation_put(struct bch_fs *c, struct disk_reservation *res)
-{
- percpu_down_read(&c->mark_lock);
- this_cpu_sub(c->usage[0]->online_reserved,
- res->sectors);
- percpu_up_read(&c->mark_lock);
-
- res->sectors = 0;
-}
-
#define SECTORS_CACHE 1024
int bch2_disk_reservation_add(struct bch_fs *c, struct disk_reservation *res,
/* disk reservations: */
-void __bch2_disk_reservation_put(struct bch_fs *, struct disk_reservation *);
-
static inline void bch2_disk_reservation_put(struct bch_fs *c,
struct disk_reservation *res)
{
- if (res->sectors)
- __bch2_disk_reservation_put(c, res);
+ this_cpu_sub(c->usage[0]->online_reserved, res->sectors);
+ res->sectors = 0;
}
#define BCH_DISK_RESERVATION_NOFAIL (1 << 0)
ctx->c = c;
ctx->arg = arg;
- ctx->thread = kthread_create(bch2_data_thread, ctx, "[bcachefs]");
+ ctx->thread = kthread_create(bch2_data_thread, ctx,
+ "bch-data/%s", c->name);
if (IS_ERR(ctx->thread)) {
ret = PTR_ERR(ctx->thread);
goto err;
if (!iter->pages)
return -ENOMEM;
- __readahead_batch(ractl, iter->pages, nr_pages);
+ nr_pages = __readahead_batch(ractl, iter->pages, nr_pages);
for (i = 0; i < nr_pages; i++) {
__bch2_page_state_create(iter->pages[i], __GFP_NOFAIL);
put_page(iter->pages[i]);
KEY_TYPE_QUOTA_WARN);
bch2_quota_acct(c, inode->ei_qid, Q_INO, -1,
KEY_TYPE_QUOTA_WARN);
- bch2_inode_rm(c, inode->v.i_ino);
+ bch2_inode_rm(c, inode->v.i_ino, true);
}
}
bch2_fs_lazy_rw(c);
- ret = bch2_inode_rm(c, u.bi_inum);
+ ret = bch2_inode_rm(c, u.bi_inum, false);
if (ret)
bch_err(c, "error in fsck: error %i while deleting inode", ret);
return ret;
return ret;
}
-int bch2_inode_rm(struct bch_fs *c, u64 inode_nr)
+int bch2_inode_rm(struct bch_fs *c, u64 inode_nr, bool cached)
{
struct btree_trans trans;
struct btree_iter *iter;
u64 bi_generation;
int ret;
+ bch2_trans_init(&trans, c, 0, 0);
+
/*
* If this was a directory, there shouldn't be any real dirents left -
* but there could be whiteouts (from hash collisions) that we should
* XXX: the dirent could ideally would delete whiteouts when they're no
* longer needed
*/
- ret = bch2_btree_delete_range(c, BTREE_ID_EXTENTS,
- start, end, NULL) ?:
- bch2_btree_delete_range(c, BTREE_ID_XATTRS,
- start, end, NULL) ?:
- bch2_btree_delete_range(c, BTREE_ID_DIRENTS,
- start, end, NULL);
+ ret = bch2_btree_delete_range_trans(&trans, BTREE_ID_EXTENTS,
+ start, end, NULL) ?:
+ bch2_btree_delete_range_trans(&trans, BTREE_ID_XATTRS,
+ start, end, NULL) ?:
+ bch2_btree_delete_range_trans(&trans, BTREE_ID_DIRENTS,
+ start, end, NULL);
if (ret)
- return ret;
-
- bch2_trans_init(&trans, c, 0, 0);
+ goto err;
retry:
bch2_trans_begin(&trans);
bi_generation = 0;
- iter = bch2_trans_get_iter(&trans, BTREE_ID_INODES, POS(0, inode_nr),
- BTREE_ITER_CACHED|BTREE_ITER_INTENT);
- k = bch2_btree_iter_peek_cached(iter);
+ if (cached) {
+ iter = bch2_trans_get_iter(&trans, BTREE_ID_INODES, POS(0, inode_nr),
+ BTREE_ITER_CACHED|BTREE_ITER_INTENT);
+ k = bch2_btree_iter_peek_cached(iter);
+ } else {
+ iter = bch2_trans_get_iter(&trans, BTREE_ID_INODES, POS(0, inode_nr),
+ BTREE_ITER_SLOTS|BTREE_ITER_INTENT);
+ k = bch2_btree_iter_peek_slot(iter);
+ }
ret = bkey_err(k);
if (ret)
goto err;
- bch2_fs_inconsistent_on(k.k->type != KEY_TYPE_inode, c,
+ bch2_fs_inconsistent_on(k.k->type != KEY_TYPE_inode, trans.c,
"inode %llu not found when deleting",
inode_nr);
int bch2_inode_create(struct btree_trans *, struct bch_inode_unpacked *);
-int bch2_inode_rm(struct bch_fs *, u64);
+int bch2_inode_rm(struct bch_fs *, u64, bool);
int bch2_inode_find_by_inum_trans(struct btree_trans *, u64,
struct bch_inode_unpacked *);
*/
static int journal_entry_open(struct journal *j)
{
+ struct bch_fs *c = container_of(j, struct bch_fs, journal);
struct journal_buf *buf = journal_cur_buf(j);
union journal_res_state old, new;
int u64s;
u64 v;
+ BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb));
+
lockdep_assert_held(&j->lock);
BUG_ON(journal_entry_is_open(j));
if (j->blocked)
- return -EAGAIN;
+ return cur_entry_blocked;
if (j->cur_entry_error)
return j->cur_entry_error;
u64s = clamp_t(int, u64s, 0, JOURNAL_ENTRY_CLOSED_VAL - 1);
if (u64s <= le32_to_cpu(buf->data->u64s))
- return -ENOSPC;
+ return cur_entry_journal_full;
/*
* Must be set before marking the journal entry as open:
old.v = new.v = v;
if (old.cur_entry_offset == JOURNAL_ENTRY_ERROR_VAL)
- return -EROFS;
+ return cur_entry_insufficient_devices;
/* Handle any already added entries */
new.cur_entry_offset = le32_to_cpu(buf->data->u64s);
* Don't want to close current journal entry, just need to
* invoke reclaim:
*/
- ret = -ENOSPC;
+ ret = cur_entry_journal_full;
goto unlock;
}
* there's still a previous one in flight:
*/
trace_journal_entry_full(c);
- ret = -EAGAIN;
+ ret = cur_entry_blocked;
} else {
ret = journal_entry_open(j);
}
unlock:
- if ((ret == -EAGAIN || ret == -ENOSPC) &&
- !j->res_get_blocked_start)
+ if ((ret && ret != cur_entry_insufficient_devices) &&
+ !j->res_get_blocked_start) {
j->res_get_blocked_start = local_clock() ?: 1;
+ trace_journal_full(c);
+ }
can_discard = j->can_discard;
spin_unlock(&j->lock);
if (!ret)
goto retry;
- if (ret == -ENOSPC) {
- if (WARN_ONCE(!can_discard && (flags & JOURNAL_RES_GET_RESERVED),
- "JOURNAL_RES_GET_RESERVED set but journal full")) {
- char *buf;
-
- buf = kmalloc(4096, GFP_NOFS);
- if (buf) {
- bch2_journal_debug_to_text(&PBUF(buf), j);
- pr_err("\n%s", buf);
- kfree(buf);
- }
+ if (WARN_ONCE(ret == cur_entry_journal_full &&
+ !can_discard &&
+ (flags & JOURNAL_RES_GET_RESERVED),
+ "JOURNAL_RES_GET_RESERVED set but journal full")) {
+ char *buf;
+
+ buf = kmalloc(4096, GFP_NOFS);
+ if (buf) {
+ bch2_journal_debug_to_text(&_PBUF(buf, 4096), j);
+ pr_err("\n%s", buf);
+ kfree(buf);
}
+ }
- /*
- * Journal is full - can't rely on reclaim from work item due to
- * freezing:
- */
- trace_journal_full(c);
-
- if (!(flags & JOURNAL_RES_GET_NONBLOCK)) {
- if (can_discard) {
- bch2_journal_do_discards(j);
- goto retry;
- }
-
- if (mutex_trylock(&j->reclaim_lock)) {
- bch2_journal_reclaim(j);
- mutex_unlock(&j->reclaim_lock);
- }
+ /*
+ * Journal is full - can't rely on reclaim from work item due to
+ * freezing:
+ */
+ if ((ret == cur_entry_journal_full ||
+ ret == cur_entry_journal_pin_full) &&
+ !(flags & JOURNAL_RES_GET_NONBLOCK)) {
+ if (can_discard) {
+ bch2_journal_do_discards(j);
+ goto retry;
}
- ret = -EAGAIN;
+ if (mutex_trylock(&j->reclaim_lock)) {
+ bch2_journal_reclaim(j);
+ mutex_unlock(&j->reclaim_lock);
+ }
}
- return ret;
+ return ret == cur_entry_insufficient_devices ? -EROFS : -EAGAIN;
}
/*
{
bool ret = bch2_journal_preres_get_fast(j, res, new_u64s, flags);
- if (!ret)
- bch2_journal_reclaim_work(&j->reclaim_work.work);
+ if (!ret && mutex_trylock(&j->reclaim_lock)) {
+ bch2_journal_reclaim(j);
+ mutex_unlock(&j->reclaim_lock);
+ }
return ret;
}
* necessary
*/
int bch2_journal_flush_seq_async(struct journal *j, u64 seq,
- struct closure *parent)
+ struct closure *parent)
{
struct journal_buf *buf;
int ret = 0;
+ if (seq <= j->err_seq)
+ return -EIO;
+
+ if (seq <= j->seq_ondisk)
+ return 1;
+
spin_lock(&j->lock);
+
+ /* Recheck under lock: */
if (seq <= j->err_seq) {
ret = -EIO;
goto out;
if (nr <= ja->nr)
return 0;
- ret = -ENOMEM;
new_buckets = kzalloc(nr * sizeof(u64), GFP_KERNEL);
new_bucket_seq = kzalloc(nr * sizeof(u64), GFP_KERNEL);
- if (!new_buckets || !new_bucket_seq)
+ if (!new_buckets || !new_bucket_seq) {
+ ret = -ENOMEM;
goto err;
+ }
journal_buckets = bch2_sb_resize_journal(&ca->disk_sb,
nr + sizeof(*journal_buckets) / sizeof(u64));
- if (!journal_buckets)
+ if (!journal_buckets) {
+ ret = -ENOSPC;
goto err;
+ }
/*
* We may be called from the device add path, before the new device has
goto err;
}
} else {
+ rcu_read_lock();
ob = bch2_bucket_alloc(c, ca, RESERVE_ALLOC,
false, cl);
+ rcu_read_unlock();
if (IS_ERR(ob)) {
ret = cl ? -EAGAIN : -ENOSPC;
goto err;
if (!new_fs)
bch2_open_bucket_put(c, ob);
}
-
- ret = 0;
err:
bch2_sb_resize_journal(&ca->disk_sb,
ja->nr + sizeof(*journal_buckets) / sizeof(u64));
j->last_empty_seq + 1 != journal_cur_seq(j)));
cancel_delayed_work_sync(&j->write_work);
- cancel_delayed_work_sync(&j->reclaim_work);
+ bch2_journal_reclaim_stop(j);
}
int bch2_fs_journal_start(struct journal *j, u64 cur_seq,
spin_lock_init(&j->err_lock);
init_waitqueue_head(&j->wait);
INIT_DELAYED_WORK(&j->write_work, journal_write_work);
- INIT_DELAYED_WORK(&j->reclaim_work, bch2_journal_reclaim_work);
init_waitqueue_head(&j->pin_flush_wait);
mutex_init(&j->reclaim_lock);
mutex_init(&j->discard_lock);
"last_seq:\t\t%llu\n"
"last_seq_ondisk:\t%llu\n"
"prereserved:\t\t%u/%u\n"
+ "nr direct reclaim:\t%llu\n"
+ "nr background reclaim:\t%llu\n"
"current entry sectors:\t%u\n"
+ "current entry error:\t%u\n"
"current entry:\t\t",
fifo_used(&j->pin),
journal_cur_seq(j),
j->last_seq_ondisk,
j->prereserved.reserved,
j->prereserved.remaining,
- j->cur_entry_sectors);
+ j->nr_direct_reclaim,
+ j->nr_background_reclaim,
+ j->cur_entry_sectors,
+ j->cur_entry_error);
switch (s.cur_entry_offset) {
case JOURNAL_ENTRY_ERROR_VAL:
* Must come before signaling write completion, for
* bch2_fs_journal_stop():
*/
- mod_delayed_work(c->journal_reclaim_wq, &j->reclaim_work, 0);
+ journal_reclaim_kick(&c->journal);
/* also must come before signalling write completion: */
closure_debug_destroy(cl);
unsigned i, sectors, bytes, u64s;
int ret;
+ BUG_ON(BCH_SB_CLEAN(c->disk_sb.sb));
+
bch2_journal_pin_put(j, le64_to_cpu(w->data->seq));
journal_buf_realloc(j, w);
// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
+#include "btree_key_cache.h"
#include "journal.h"
#include "journal_io.h"
#include "journal_reclaim.h"
#include "replicas.h"
#include "super.h"
+#include <linux/kthread.h>
+#include <linux/sched/mm.h>
+#include <trace/events/bcachefs.h>
+
/* Free space calculations: */
static unsigned journal_space_from(struct journal_device *ja,
j->can_discard = can_discard;
if (nr_online < c->opts.metadata_replicas_required) {
- ret = -EROFS;
+ ret = cur_entry_insufficient_devices;
goto out;
}
if (!fifo_free(&j->pin)) {
- ret = -ENOSPC;
+ ret = cur_entry_journal_pin_full;
goto out;
}
clean = __journal_space_available(j, nr_devs_want, journal_space_clean);
if (!discarded.next_entry)
- ret = -ENOSPC;
+ ret = cur_entry_journal_full;
overhead = DIV_ROUND_UP(clean.remaining, max_entry_size) *
journal_entry_overhead(j);
list_move(&ret->list, &pin_list->flushed);
BUG_ON(j->flush_in_progress);
j->flush_in_progress = ret;
- j->last_flushed = jiffies;
}
spin_unlock(&j->lock);
}
/* returns true if we did work */
-static bool journal_flush_pins(struct journal *j, u64 seq_to_flush,
- unsigned min_nr)
+static u64 journal_flush_pins(struct journal *j, u64 seq_to_flush,
+ unsigned min_nr)
{
struct journal_entry_pin *pin;
- bool ret = false;
- u64 seq;
+ u64 seq, ret = 0;
lockdep_assert_held(&j->reclaim_lock);
- while ((pin = journal_get_next_pin(j, min_nr
- ? U64_MAX : seq_to_flush, &seq))) {
+ while (1) {
+ cond_resched();
+
+ j->last_flushed = jiffies;
+
+ pin = journal_get_next_pin(j, min_nr
+ ? U64_MAX : seq_to_flush, &seq);
+ if (!pin)
+ break;
+
if (min_nr)
min_nr--;
BUG_ON(j->flush_in_progress != pin);
j->flush_in_progress = NULL;
wake_up(&j->pin_flush_wait);
- ret = true;
+ ret++;
}
return ret;
* 512 journal entries or 25% of all journal buckets, then
* journal_next_bucket() should not stall.
*/
-void bch2_journal_reclaim(struct journal *j)
+static void __bch2_journal_reclaim(struct journal *j, bool direct)
{
struct bch_fs *c = container_of(j, struct bch_fs, journal);
- unsigned min_nr = 0;
- u64 seq_to_flush = 0;
+ bool kthread = (current->flags & PF_KTHREAD) != 0;
+ u64 seq_to_flush, nr_flushed = 0;
+ size_t min_nr;
+ unsigned flags;
+ /*
+ * We can't invoke memory reclaim while holding the reclaim_lock -
+ * journal reclaim is required to make progress for memory reclaim
+ * (cleaning the caches), so we can't get stuck in memory reclaim while
+ * we're holding the reclaim lock:
+ */
lockdep_assert_held(&j->reclaim_lock);
+ flags = memalloc_noreclaim_save();
do {
+ if (kthread && kthread_should_stop())
+ break;
+
bch2_journal_do_discards(j);
seq_to_flush = journal_seq_to_flush(j);
if (j->prereserved.reserved * 2 > j->prereserved.remaining)
min_nr = 1;
- if ((atomic_read(&c->btree_cache.dirty) * 4 >
- c->btree_cache.used * 3) ||
- (c->btree_key_cache.nr_dirty * 4 >
- c->btree_key_cache.nr_keys))
+ if (atomic_read(&c->btree_cache.dirty) * 4 >
+ c->btree_cache.used * 3)
min_nr = 1;
- } while (journal_flush_pins(j, seq_to_flush, min_nr));
- if (!bch2_journal_error(j))
- queue_delayed_work(c->journal_reclaim_wq, &j->reclaim_work,
- msecs_to_jiffies(j->reclaim_delay_ms));
+ min_nr = max(min_nr, bch2_nr_btree_keys_need_flush(c));
+
+ trace_journal_reclaim_start(c,
+ min_nr,
+ j->prereserved.reserved,
+ j->prereserved.remaining,
+ atomic_read(&c->btree_cache.dirty),
+ c->btree_cache.used,
+ c->btree_key_cache.nr_dirty,
+ c->btree_key_cache.nr_keys);
+
+ nr_flushed = journal_flush_pins(j, seq_to_flush, min_nr);
+
+ if (direct)
+ j->nr_direct_reclaim += nr_flushed;
+ else
+ j->nr_background_reclaim += nr_flushed;
+ trace_journal_reclaim_finish(c, nr_flushed);
+ } while (min_nr);
+
+ memalloc_noreclaim_restore(flags);
}
-void bch2_journal_reclaim_work(struct work_struct *work)
+void bch2_journal_reclaim(struct journal *j)
{
- struct journal *j = container_of(to_delayed_work(work),
- struct journal, reclaim_work);
+ __bch2_journal_reclaim(j, true);
+}
- mutex_lock(&j->reclaim_lock);
- bch2_journal_reclaim(j);
- mutex_unlock(&j->reclaim_lock);
+static int bch2_journal_reclaim_thread(void *arg)
+{
+ struct journal *j = arg;
+ unsigned long next;
+
+ set_freezable();
+
+ kthread_wait_freezable(test_bit(JOURNAL_RECLAIM_STARTED, &j->flags));
+
+ while (!kthread_should_stop()) {
+ j->reclaim_kicked = false;
+
+ mutex_lock(&j->reclaim_lock);
+ __bch2_journal_reclaim(j, false);
+ mutex_unlock(&j->reclaim_lock);
+
+ next = j->last_flushed + msecs_to_jiffies(j->reclaim_delay_ms);
+
+ while (1) {
+ set_current_state(TASK_INTERRUPTIBLE);
+ if (kthread_should_stop())
+ break;
+ if (j->reclaim_kicked)
+ break;
+ if (time_after_eq(jiffies, next))
+ break;
+ schedule_timeout(next - jiffies);
+ try_to_freeze();
+
+ }
+ __set_current_state(TASK_RUNNING);
+ }
+
+ return 0;
+}
+
+void bch2_journal_reclaim_stop(struct journal *j)
+{
+ struct task_struct *p = j->reclaim_thread;
+
+ j->reclaim_thread = NULL;
+
+ if (p) {
+ kthread_stop(p);
+ put_task_struct(p);
+ }
+}
+
+int bch2_journal_reclaim_start(struct journal *j)
+{
+ struct bch_fs *c = container_of(j, struct bch_fs, journal);
+ struct task_struct *p;
+
+ if (j->reclaim_thread)
+ return 0;
+
+ p = kthread_create(bch2_journal_reclaim_thread, j,
+ "bch-reclaim/%s", c->name);
+ if (IS_ERR(p))
+ return PTR_ERR(p);
+
+ get_task_struct(p);
+ j->reclaim_thread = p;
+ wake_up_process(p);
+ return 0;
}
static int journal_flush_done(struct journal *j, u64 seq_to_flush,
mutex_lock(&j->reclaim_lock);
- *did_work = journal_flush_pins(j, seq_to_flush, 0);
+ *did_work = journal_flush_pins(j, seq_to_flush, 0) != 0;
spin_lock(&j->lock);
/*
journal_space_clean,
};
+static inline void journal_reclaim_kick(struct journal *j)
+{
+ struct task_struct *p = READ_ONCE(j->reclaim_thread);
+
+ if (p && !j->reclaim_kicked) {
+ j->reclaim_kicked = true;
+ if (p)
+ wake_up_process(p);
+ }
+}
+
unsigned bch2_journal_dev_buckets_available(struct journal *,
struct journal_device *,
enum journal_space_from);
void bch2_journal_do_discards(struct journal *);
void bch2_journal_reclaim(struct journal *);
-void bch2_journal_reclaim_work(struct work_struct *);
+
+void bch2_journal_reclaim_stop(struct journal *);
+int bch2_journal_reclaim_start(struct journal *);
bool bch2_journal_flush_pins(struct journal *, u64);
* 0, or -ENOSPC if waiting on journal reclaim, or -EROFS if
* insufficient devices:
*/
- int cur_entry_error;
+ enum {
+ cur_entry_ok,
+ cur_entry_blocked,
+ cur_entry_journal_full,
+ cur_entry_journal_pin_full,
+ cur_entry_insufficient_devices,
+ } cur_entry_error;
union journal_preres_state prereserved;
struct write_point wp;
spinlock_t err_lock;
- struct delayed_work reclaim_work;
struct mutex reclaim_lock;
+ struct task_struct *reclaim_thread;
+ bool reclaim_kicked;
+ u64 nr_direct_reclaim;
+ u64 nr_background_reclaim;
+
unsigned long last_flushed;
struct journal_entry_pin *flush_in_progress;
wait_queue_head_t pin_flush_wait;
if (bch2_fs_init_fault("copygc_start"))
return -ENOMEM;
- t = kthread_create(bch2_copygc_thread, c, "bch_copygc");
+ t = kthread_create(bch2_copygc_thread, c, "bch-copygc/%s", c->name);
if (IS_ERR(t))
return PTR_ERR(t);
if (c->opts.nochanges)
return 0;
- p = kthread_create(bch2_rebalance_thread, c, "bch_rebalance");
+ p = kthread_create(bch2_rebalance_thread, c, "bch-rebalance/%s", c->name);
if (IS_ERR(p))
return PTR_ERR(p);
*/
set_bit(BCH_FS_BTREE_INTERIOR_REPLAY_DONE, &c->flags);
set_bit(JOURNAL_RECLAIM_STARTED, &j->flags);
+ journal_reclaim_kick(j);
j->replay_journal_seq = seq;
#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/idr.h>
-#include <linux/kthread.h>
#include <linux/module.h>
#include <linux/percpu.h>
#include <linux/random.h>
void bch2_fs_read_only(struct bch_fs *c)
{
if (!test_bit(BCH_FS_RW, &c->flags)) {
- cancel_delayed_work_sync(&c->journal.reclaim_work);
+ BUG_ON(c->journal.reclaim_thread);
return;
}
set_bit(BCH_FS_ALLOCATOR_RUNNING, &c->flags);
+ ret = bch2_journal_reclaim_start(&c->journal);
+ if (ret) {
+ bch_err(c, "error starting journal reclaim: %i", ret);
+ return ret;
+ }
+
if (!early) {
ret = bch2_fs_read_write_late(c);
if (ret)
percpu_ref_reinit(&c->writes);
set_bit(BCH_FS_RW, &c->flags);
-
- queue_delayed_work(c->journal_reclaim_wq,
- &c->journal.reclaim_work, 0);
return 0;
err:
__bch2_fs_read_only(c);
kfree(c->unused_inode_hints);
free_heap(&c->copygc_heap);
- if (c->journal_reclaim_wq)
- destroy_workqueue(c->journal_reclaim_wq);
if (c->copygc_wq)
destroy_workqueue(c->copygc_wq);
if (c->wq)
WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) ||
!(c->copygc_wq = alloc_workqueue("bcachefs_copygc",
WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE, 1)) ||
- !(c->journal_reclaim_wq = alloc_workqueue("bcachefs_journal_reclaim",
- WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_HIGHPRI, 1)) ||
percpu_ref_init(&c->writes, bch2_writes_disabled,
PERCPU_REF_INIT_DEAD, GFP_KERNEL) ||
mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) ||
bch2_debug_exit();
bch2_vfs_exit();
bch2_chardev_exit();
+ bch2_btree_key_cache_exit();
if (bcachefs_kset)
kset_unregister(bcachefs_kset);
}
bch2_bkey_pack_test();
if (!(bcachefs_kset = kset_create_and_add("bcachefs", NULL, fs_kobj)) ||
+ bch2_btree_key_cache_init() ||
bch2_chardev_init() ||
bch2_vfs_init() ||
bch2_debug_init())
read_attribute(journal_pins);
read_attribute(btree_updates);
read_attribute(dirty_btree_nodes);
+read_attribute(btree_cache);
read_attribute(btree_key_cache);
read_attribute(btree_transactions);
read_attribute(stripes_heap);
return out.pos - buf;
}
+ if (attr == &sysfs_btree_cache) {
+ bch2_btree_cache_to_text(&out, c);
+ return out.pos - buf;
+ }
+
if (attr == &sysfs_btree_key_cache) {
bch2_btree_key_cache_to_text(&out, &c->btree_key_cache);
return out.pos - buf;
&sysfs_journal_pins,
&sysfs_btree_updates,
&sysfs_dirty_btree_nodes,
+ &sysfs_btree_cache,
&sysfs_btree_key_cache,
&sysfs_btree_transactions,
&sysfs_stripes_heap,
projects / bcachefs-tools-debian / commitdiff