-43e3159567958ea70c8a95d98fdb6e881153a656
+14e9ac5016803fc63c1216608c866bef16b4053e
{
struct disk_reservation res;
struct bch_write_op op;
- struct bch_write_bio bio;
struct bio_vec bv;
struct closure cl;
closure_init_stack(&cl);
- bio_init(&bio.bio, &bv, 1);
- bio.bio.bi_iter.bi_size = len;
- bch2_bio_map(&bio.bio, buf);
+ bio_init(&op.wbio.bio, &bv, 1);
+ op.wbio.bio.bi_iter.bi_size = len;
+ bch2_bio_map(&op.wbio.bio, buf);
int ret = bch2_disk_reservation_get(c, &res, len >> 9, 0);
if (ret)
die("error reserving space in new filesystem: %s", strerror(-ret));
- bch2_write_op_init(&op, c, &bio, res, c->write_points,
+ bch2_write_op_init(&op, c, res, c->write_points,
POS(dst_inode->inum, dst_offset >> 9), NULL, 0);
closure_call(&op.cl, bch2_write, NULL, &cl);
closure_sync(&cl);
#define flush_cache_vmap(start, end) do { } while (0)
#define flush_cache_vunmap(start, end) do { } while (0)
+#ifdef __x86_64
+#define CONFIG_X86_64 y
+#endif
+
#endif /* _TOOLS_LINUX_COMPILER_H */
struct kmem_cache;
+typedef void * (mempool_alloc_t)(gfp_t gfp_mask, void *pool_data);
+typedef void (mempool_free_t)(void *element, void *pool_data);
+
typedef struct mempool_s {
- size_t elem_size;
+ size_t elem_size;
+ void *pool_data;
+ mempool_alloc_t *alloc;
+ mempool_free_t *free;
} mempool_t;
static inline bool mempool_initialized(mempool_t *pool)
return 0;
}
-static inline mempool_t *mempool_create_kmalloc_pool(int min_nr, size_t size)
-{
- mempool_t *pool = malloc(sizeof(*pool));
- pool->elem_size = size;
- return pool;
-}
-
static inline int mempool_init_page_pool(mempool_t *pool, int min_nr, int order)
{
pool->elem_size = PAGE_SIZE << order;
return 0;
}
-static inline mempool_t *mempool_create_page_pool(int min_nr, int order)
+static inline int mempool_init(mempool_t *pool, int min_nr,
+ mempool_alloc_t *alloc_fn,
+ mempool_free_t *free_fn,
+ void *pool_data)
{
- mempool_t *pool = malloc(sizeof(*pool));
- pool->elem_size = PAGE_SIZE << order;
- return pool;
+ pool->elem_size = (size_t) pool_data;
+ pool->pool_data = pool_data;
+ pool->alloc = alloc_fn;
+ pool->free = free_fn;
+ return 0;
}
#endif /* _LINUX_MEMPOOL_H */
#define kcalloc(n, size, flags) calloc(n, size)
#define kmalloc_array(n, size, flags) calloc(n, size)
-#define vmalloc(size) malloc(size)
-#define vzalloc(size) calloc(1, size)
-
#define kfree(p) free(p)
#define kvfree(p) free(p)
#define kzfree(p) free(p)
#define VM_NO_GUARD 0x00000040 /* don't add guard page */
#define VM_KASAN 0x00000080 /* has allocated kasan shadow memory */
-#define PAGE_KERNEL 0
-
static inline void vunmap(const void *addr) {}
static inline void *vmap(struct page **pages, unsigned int count,
#ifndef __TOOLS_LINUX_VMALLOC_H
#define __TOOLS_LINUX_VMALLOC_H
-#define vmalloc(size) malloc(size)
-#define __vmalloc(size, flags, prot) malloc(size)
+#include <stdlib.h>
+#include <sys/mman.h>
+
+#include "tools-util.h"
+
+#define PAGE_KERNEL 0
+#define PAGE_KERNEL_EXEC 1
+
#define vfree(p) free(p)
+static inline void *__vmalloc(unsigned long size, gfp_t gfp_mask, unsigned prot)
+{
+ void *p = aligned_alloc(PAGE_SIZE, size);
+
+ if (p && prot == PAGE_KERNEL_EXEC) {
+ if (mprotect(p, size, PROT_READ|PROT_WRITE|PROT_EXEC)) {
+ vfree(p);
+ p = NULL;
+ }
+ }
+
+ if (p && (gfp_mask & __GFP_ZERO))
+ memset(p, 0, size);
+
+ return p;
+}
+
+static inline void *vmalloc(unsigned long size)
+{
+ return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
+}
+
+static inline void *vzalloc(unsigned long size)
+{
+ return __vmalloc(size, GFP_KERNEL|__GFP_ZERO, PAGE_KERNEL);
+}
+
#endif /* __TOOLS_LINUX_VMALLOC_H */
struct bucket *g, struct btree_iter *iter,
u64 *journal_seq)
{
- struct bucket_mark m = READ_ONCE(g->mark);
+ struct bucket_mark m;
__BKEY_PADDED(k, DIV_ROUND_UP(sizeof(struct bch_alloc), 8)) alloc_key;
struct bkey_i_alloc *a;
u8 *d;
if (ret)
break;
+ /* read mark under btree node lock: */
+ m = READ_ONCE(g->mark);
a = bkey_alloc_init(&alloc_key.k);
a->k.p = iter->pos;
a->v.fields = 0;
struct btree_iter iter;
int ret;
- lockdep_assert_held(&c->state_lock);
-
if (pos.inode >= c->sb.nr_devices || !c->devs[pos.inode])
return 0;
struct work_struct read_retry_work;
spinlock_t read_retry_lock;
+ struct bio_list btree_write_error_list;
+ struct work_struct btree_write_error_work;
+ spinlock_t btree_write_error_lock;
+
/* ERRORS */
struct list_head fsck_errors;
struct mutex fsck_error_lock;
__le16 u64s;
__u8 btree_id;
__u8 level;
- __le32 flags; /* designates what this jset holds */
+ __u8 type; /* designates what this jset holds */
+ __u8 pad[3];
union {
struct bkey_i start[0];
#define JSET_KEYS_U64s (sizeof(struct jset_entry) / sizeof(__u64))
-LE32_BITMASK(JOURNAL_ENTRY_TYPE, struct jset_entry, flags, 0, 8);
enum {
JOURNAL_ENTRY_BTREE_KEYS = 0,
JOURNAL_ENTRY_BTREE_ROOT = 1,
unsigned dst_offset, unsigned dst_size,
bool *eax_zeroed)
{
- unsigned byte = format->key_u64s * sizeof(u64);
unsigned bits = format->bits_per_field[field];
u64 offset = format->field_offset[field];
- unsigned i, bit_offset = 0;
- unsigned shl, shr;
+ unsigned i, byte, bit_offset, align, shl, shr;
if (!bits && !offset) {
if (!*eax_zeroed) {
return out;
}
+ bit_offset = format->key_u64s * 64;
for (i = 0; i <= field; i++)
- bit_offset += format->bits_per_field[i];
+ bit_offset -= format->bits_per_field[i];
- byte -= DIV_ROUND_UP(bit_offset, 8);
- bit_offset = round_up(bit_offset, 8) - bit_offset;
+ byte = bit_offset / 8;
+ bit_offset -= byte * 8;
*eax_zeroed = false;
/* movzx eax, WORD PTR [rsi + imm8] */
I4(0x0f, 0xb7, 0x46, byte);
} else if (bit_offset + bits <= 32) {
+ align = min(4 - DIV_ROUND_UP(bit_offset + bits, 8), byte & 3);
+ byte -= align;
+ bit_offset += align * 8;
+
+ BUG_ON(bit_offset + bits > 32);
+
/* mov eax, [rsi + imm8] */
I3(0x8b, 0x46, byte);
out += 4;
}
} else if (bit_offset + bits <= 64) {
+ align = min(8 - DIV_ROUND_UP(bit_offset + bits, 8), byte & 7);
+ byte -= align;
+ bit_offset += align * 8;
+
+ BUG_ON(bit_offset + bits > 64);
+
/* mov rax, [rsi + imm8] */
I4(0x48, 0x8b, 0x46, byte);
I4(0x48, 0xc1, 0xe8, shr);
}
} else {
+ align = min(4 - DIV_ROUND_UP(bit_offset + bits, 8), byte & 3);
+ byte -= align;
+ bit_offset += align * 8;
+
+ BUG_ON(bit_offset + bits > 96);
+
/* mov rax, [rsi + byte] */
I4(0x48, 0x8b, 0x46, byte);
{
EBUG_ON(btree_node_write_in_flight(b));
- free_pages((unsigned long) b->data, btree_page_order(c));
+ kvpfree(b->data, btree_bytes(c));
b->data = NULL;
bch2_btree_keys_free(b);
}
list_move(&b->list, &c->btree_cache_freed);
}
-#define PTR_HASH(_k) (bkey_i_to_extent_c(_k)->v._data[0])
-
static const struct rhashtable_params bch_btree_cache_params = {
.head_offset = offsetof(struct btree, hash),
.key_offset = offsetof(struct btree, key.v),
static void mca_data_alloc(struct bch_fs *c, struct btree *b, gfp_t gfp)
{
- unsigned order = ilog2(btree_pages(c));
-
- b->data = (void *) __get_free_pages(gfp, order);
+ b->data = kvpmalloc(btree_bytes(c), gfp);
if (!b->data)
goto err;
- if (bch2_btree_keys_alloc(b, order, gfp))
+ if (bch2_btree_keys_alloc(b, btree_page_order(c), gfp))
goto err;
c->btree_cache_used++;
list_move(&b->list, &c->btree_cache_freeable);
return;
err:
- free_pages((unsigned long) b->data, order);
+ kvpfree(b->data, btree_bytes(c));
b->data = NULL;
list_move(&b->list, &c->btree_cache_freed);
}
six_lock_init(&b->lock);
INIT_LIST_HEAD(&b->list);
INIT_LIST_HEAD(&b->write_blocked);
- INIT_LIST_HEAD(&b->reachable);
mca_data_alloc(c, b, gfp);
return b->data ? b : NULL;
void bch2_btree_node_hash_remove(struct bch_fs *c, struct btree *b)
{
- BUG_ON(btree_node_dirty(b));
-
- b->nsets = 0;
-
rhashtable_remove_fast(&c->btree_cache_table, &b->hash,
bch_btree_cache_params);
bkey_i_to_extent(&b->key)->v._data[0] = 0;
}
+int __bch2_btree_node_hash_insert(struct bch_fs *c, struct btree *b)
+{
+ return rhashtable_lookup_insert_fast(&c->btree_cache_table, &b->hash,
+ bch_btree_cache_params);
+}
+
int bch2_btree_node_hash_insert(struct bch_fs *c, struct btree *b,
unsigned level, enum btree_id id)
{
int ret;
+
b->level = level;
b->btree_id = id;
- ret = rhashtable_lookup_insert_fast(&c->btree_cache_table, &b->hash,
- bch_btree_cache_params);
- if (ret)
- return ret;
-
mutex_lock(&c->btree_cache_lock);
- list_add(&b->list, &c->btree_cache);
+ ret = __bch2_btree_node_hash_insert(c, b);
+ if (!ret)
+ list_add(&b->list, &c->btree_cache);
mutex_unlock(&c->btree_cache_lock);
- return 0;
+ return ret;
}
__flatten
if (!six_trylock_write(&b->lock))
goto out_unlock_intent;
- if (btree_node_write_error(b) ||
- btree_node_noevict(b))
+ if (btree_node_noevict(b))
goto out_unlock;
if (!btree_node_may_write(b))
if (c->verify_data)
list_move(&c->verify_data->list, &c->btree_cache);
- free_pages((unsigned long) c->verify_ondisk, ilog2(btree_pages(c)));
+ kvpfree(c->verify_ondisk, btree_bytes(c));
#endif
for (i = 0; i < BTREE_ID_NR; i++)
#ifdef CONFIG_BCACHEFS_DEBUG
mutex_init(&c->verify_lock);
- c->verify_ondisk = (void *)
- __get_free_pages(GFP_KERNEL, ilog2(btree_pages(c)));
+ c->verify_ondisk = kvpmalloc(btree_bytes(c), GFP_KERNEL);
if (!c->verify_ondisk)
return -ENOMEM;
BUG_ON(!six_trylock_intent(&b->lock));
BUG_ON(!six_trylock_write(&b->lock));
out_unlock:
- BUG_ON(bkey_extent_is_data(&b->key.k) && PTR_HASH(&b->key));
+ BUG_ON(btree_node_hashed(b));
BUG_ON(btree_node_write_in_flight(b));
list_del_init(&b->list);
struct bch_fs *c = iter->c;
struct btree *b;
+ /*
+ * Parent node must be locked, else we could read in a btree node that's
+ * been freed:
+ */
+ BUG_ON(!btree_node_locked(iter, level + 1));
+
b = bch2_btree_node_mem_alloc(c);
if (IS_ERR(b))
return b;
#include "bcachefs.h"
#include "btree_types.h"
+#include "extents.h"
struct btree_iter;
void bch2_recalc_btree_reserve(struct bch_fs *);
void bch2_btree_node_hash_remove(struct bch_fs *, struct btree *);
+int __bch2_btree_node_hash_insert(struct bch_fs *, struct btree *);
int bch2_btree_node_hash_insert(struct bch_fs *, struct btree *,
unsigned, enum btree_id);
void bch2_fs_btree_exit(struct bch_fs *);
int bch2_fs_btree_init(struct bch_fs *);
+#define PTR_HASH(_k) (bkey_i_to_extent_c(_k)->v._data[0])
+
+/* is btree node in hash table? */
+static inline bool btree_node_hashed(struct btree *b)
+{
+ return bkey_extent_is_data(&b->key.k) && PTR_HASH(&b->key);
+}
+
#define for_each_cached_btree(_b, _c, _tbl, _iter, _pos) \
for ((_tbl) = rht_dereference_rcu((_c)->btree_cache_table.tbl, \
&(_c)->btree_cache_table), \
bch2_btree_interior_update_will_free_node(c, as, old_nodes[i]);
/* Repack everything with @new_format and sort down to one bset */
- for (i = 0; i < nr_old_nodes; i++) {
+ for (i = 0; i < nr_old_nodes; i++)
new_nodes[i] =
__bch2_btree_node_alloc_replacement(c, old_nodes[i],
- new_format, res);
- list_add(&new_nodes[i]->reachable, &as->reachable_list);
- }
+ new_format, as, res);
/*
* Conceptually we concatenate the nodes together and slice them
set_btree_bset_end(n1, n1->set);
- list_del_init(&n2->reachable);
six_unlock_write(&n2->lock);
bch2_btree_node_free_never_inserted(c, n2);
six_unlock_intent(&n2->lock);
memset(merge, 0, sizeof(merge));
__for_each_btree_node(&iter, c, btree_id, POS_MIN,
- U8_MAX, 0, BTREE_ITER_PREFETCH, b) {
+ BTREE_MAX_DEPTH, 0,
+ BTREE_ITER_PREFETCH, b) {
memmove(merge + 1, merge,
sizeof(merge) - sizeof(merge[0]));
memmove(lock_seq + 1, lock_seq,
bool used_mempool, void *p)
{
if (used_mempool)
- mempool_free(virt_to_page(p), &c->btree_bounce_pool);
+ mempool_free(p, &c->btree_bounce_pool);
else
- free_pages((unsigned long) p, order);
+ vpfree(p, PAGE_SIZE << order);
}
static void *btree_bounce_alloc(struct bch_fs *c, unsigned order,
{
void *p;
- BUG_ON(1 << order > btree_pages(c));
+ BUG_ON(order > btree_page_order(c));
*used_mempool = false;
p = (void *) __get_free_pages(__GFP_NOWARN|GFP_NOWAIT, order);
return p;
*used_mempool = true;
- return page_address(mempool_alloc(&c->btree_bounce_pool, GFP_NOIO));
+ return mempool_alloc(&c->btree_bounce_pool, GFP_NOIO);
}
typedef int (*sort_cmp_fn)(struct btree *,
if (bne->keys.seq == b->data->keys.seq)
goto err;
- sorted = btree_bounce_alloc(c, ilog2(btree_pages(c)), &used_mempool);
+ sorted = btree_bounce_alloc(c, btree_page_order(c), &used_mempool);
sorted->keys.u64s = 0;
b->nr = btree_node_is_extents(b)
BUG_ON(b->nr.live_u64s != u64s);
- btree_bounce_free(c, ilog2(btree_pages(c)), used_mempool, sorted);
+ btree_bounce_free(c, btree_page_order(c), used_mempool, sorted);
bch2_bset_build_aux_tree(b, b->set, false);
{
struct btree_write *w = btree_prev_write(b);
- /*
- * Before calling bch2_btree_complete_write() - if the write errored, we
- * have to halt new journal writes before they see this btree node
- * write as completed:
- */
- if (btree_node_write_error(b))
- bch2_journal_halt(&c->journal);
-
bch2_btree_complete_write(c, b, w);
btree_node_io_unlock(b);
}
-static void btree_node_write_endio(struct bio *bio)
+static void bch2_btree_node_write_error(struct bch_fs *c,
+ struct bch_write_bio *wbio)
{
- struct btree *b = bio->bi_private;
- struct bch_write_bio *wbio = to_wbio(bio);
- struct bch_fs *c = wbio->c;
- struct bio *orig = wbio->split ? wbio->orig : NULL;
- struct closure *cl = !wbio->split ? wbio->cl : NULL;
- struct bch_dev *ca = wbio->ca;
+ struct btree *b = wbio->bio.bi_private;
+ struct closure *cl = wbio->cl;
+ __BKEY_PADDED(k, BKEY_BTREE_PTR_VAL_U64s_MAX) tmp;
+ struct bkey_i_extent *new_key;
+
+ bkey_copy(&tmp.k, &b->key);
+ new_key = bkey_i_to_extent(&tmp.k);
+
+ while (wbio->replicas_failed) {
+ unsigned idx = __fls(wbio->replicas_failed);
+
+ bch2_extent_drop_ptr_idx(extent_i_to_s(new_key), idx);
+ wbio->replicas_failed ^= 1 << idx;
+ }
+
+ if (!bch2_extent_nr_ptrs(extent_i_to_s_c(new_key)) ||
+ bch2_btree_node_update_key(c, b, new_key)) {
+ set_btree_node_noevict(b);
+ bch2_fatal_error(c);
+ }
+
+ bio_put(&wbio->bio);
+ btree_node_write_done(c, b);
+ if (cl)
+ closure_put(cl);
+}
+
+void bch2_btree_write_error_work(struct work_struct *work)
+{
+ struct bch_fs *c = container_of(work, struct bch_fs,
+ btree_write_error_work);
+ struct bio *bio;
+
+ while (1) {
+ spin_lock_irq(&c->read_retry_lock);
+ bio = bio_list_pop(&c->read_retry_list);
+ spin_unlock_irq(&c->read_retry_lock);
- if (bch2_dev_fatal_io_err_on(bio->bi_error, ca, "btree write") ||
+ if (!bio)
+ break;
+
+ bch2_btree_node_write_error(c, to_wbio(bio));
+ }
+}
+
+static void btree_node_write_endio(struct bio *bio)
+{
+ struct btree *b = bio->bi_private;
+ struct bch_write_bio *wbio = to_wbio(bio);
+ struct bch_write_bio *parent = wbio->split ? wbio->parent : NULL;
+ struct bch_write_bio *orig = parent ?: wbio;
+ struct closure *cl = !wbio->split ? wbio->cl : NULL;
+ struct bch_fs *c = wbio->c;
+ struct bch_dev *ca = wbio->ca;
+
+ if (bch2_dev_nonfatal_io_err_on(bio->bi_error, ca, "btree write") ||
bch2_meta_write_fault("btree"))
- set_btree_node_write_error(b);
+ set_bit(wbio->ptr_idx, (unsigned long *) &orig->replicas_failed);
if (wbio->have_io_ref)
percpu_ref_put(&ca->io_ref);
- if (wbio->bounce)
- btree_bounce_free(c,
- wbio->order,
- wbio->used_mempool,
- page_address(bio->bi_io_vec[0].bv_page));
-
- if (wbio->put_bio)
+ if (parent) {
bio_put(bio);
+ bio_endio(&parent->bio);
+ return;
+ }
- if (orig) {
- bio_endio(orig);
- } else {
- btree_node_write_done(c, b);
- if (cl)
- closure_put(cl);
+ btree_bounce_free(c,
+ wbio->order,
+ wbio->used_mempool,
+ wbio->data);
+
+ if (wbio->replicas_failed) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&c->btree_write_error_lock, flags);
+ bio_list_add(&c->read_retry_list, &wbio->bio);
+ spin_unlock_irqrestore(&c->btree_write_error_lock, flags);
+ queue_work(c->wq, &c->btree_write_error_work);
+ return;
}
+
+ bio_put(bio);
+ btree_node_write_done(c, b);
+ if (cl)
+ closure_put(cl);
}
static int validate_bset_for_write(struct bch_fs *c, struct btree *b,
struct closure *parent,
enum six_lock_type lock_type_held)
{
- struct bio *bio;
struct bch_write_bio *wbio;
struct bset_tree *t;
struct bset *i;
} while (cmpxchg_acquire(&b->flags, old, new) != old);
BUG_ON(!list_empty(&b->write_blocked));
- BUG_ON(!list_empty_careful(&b->reachable) != !b->written);
+ BUG_ON((b->will_make_reachable != NULL) != !b->written);
BUG_ON(b->written >= c->sb.btree_node_size);
BUG_ON(bset_written(b, btree_bset_last(b)));
trace_btree_write(b, bytes_to_write, sectors_to_write);
- bio = bio_alloc_bioset(GFP_NOIO, 1 << order, &c->bio_write);
-
- wbio = to_wbio(bio);
+ wbio = wbio_init(bio_alloc_bioset(GFP_NOIO, 1 << order, &c->bio_write));
wbio->cl = parent;
- wbio->bounce = true;
- wbio->put_bio = true;
wbio->order = order;
wbio->used_mempool = used_mempool;
- bio->bi_opf = REQ_OP_WRITE|REQ_META|REQ_FUA;
- bio->bi_iter.bi_size = sectors_to_write << 9;
- bio->bi_end_io = btree_node_write_endio;
- bio->bi_private = b;
+ wbio->data = data;
+ wbio->bio.bi_opf = REQ_OP_WRITE|REQ_META|REQ_FUA;
+ wbio->bio.bi_iter.bi_size = sectors_to_write << 9;
+ wbio->bio.bi_end_io = btree_node_write_endio;
+ wbio->bio.bi_private = b;
if (parent)
closure_get(parent);
- bch2_bio_map(bio, data);
+ bch2_bio_map(&wbio->bio, data);
/*
* If we're appending to a leaf node, we don't technically need FUA -
static inline bool btree_node_may_write(struct btree *b)
{
return list_empty_careful(&b->write_blocked) &&
- list_empty_careful(&b->reachable);
+ !b->will_make_reachable;
}
enum compact_mode {
void bch2_btree_complete_write(struct bch_fs *, struct btree *,
struct btree_write *);
+void bch2_btree_write_error_work(struct work_struct *);
void __bch2_btree_node_write(struct bch_fs *, struct btree *,
struct closure *, enum six_lock_type);
while (iter->nodes_locked)
btree_node_unlock(iter, __ffs(iter->nodes_locked));
+ iter->flags &= ~BTREE_ITER_UPTODATE;
+
return iter->flags & BTREE_ITER_ERROR ? -EIO : 0;
}
iter->flags |= BTREE_ITER_AT_END_OF_LEAF;
iter->pos = new_pos;
+ iter->flags &= ~BTREE_ITER_UPTODATE;
}
void bch2_btree_iter_set_pos(struct btree_iter *iter, struct bpos new_pos)
{
EBUG_ON(bkey_cmp(new_pos, iter->pos) < 0); /* XXX handle this */
iter->pos = new_pos;
+ iter->flags &= ~BTREE_ITER_UPTODATE;
}
void bch2_btree_iter_advance_pos(struct btree_iter *iter)
{
+ if (iter->flags & BTREE_ITER_UPTODATE &&
+ !(iter->flags & BTREE_ITER_WITH_HOLES)) {
+ struct bkey_s_c k;
+
+ __btree_iter_advance(iter);
+ k = __btree_iter_peek(iter);
+ if (likely(k.k)) {
+ iter->pos = bkey_start_pos(k.k);
+ return;
+ }
+ }
+
/*
* We use iter->k instead of iter->pos for extents: iter->pos will be
* equal to the start of the extent we returned, but we need to advance
BUG_ON(bkey_cmp(pos, iter->nodes[iter->level]->data->min_key) < 0);
iter->pos = pos;
+ iter->flags &= ~BTREE_ITER_UPTODATE;
__btree_iter_init(iter, iter->nodes[iter->level]);
}
EBUG_ON(!!(iter->flags & BTREE_ITER_IS_EXTENTS) !=
(iter->btree_id == BTREE_ID_EXTENTS));
+ if (iter->flags & BTREE_ITER_UPTODATE) {
+ struct btree *b = iter->nodes[0];
+ struct bkey_packed *k =
+ __bch2_btree_node_iter_peek_all(&iter->node_iters[0], b);
+
+ return (struct bkey_s_c) {
+ .k = &iter->k,
+ .v = bkeyp_val(&b->format, k)
+ };
+ }
+
while (1) {
ret = bch2_btree_iter_traverse(iter);
if (unlikely(ret)) {
*/
if (!(iter->flags & BTREE_ITER_IS_EXTENTS) ||
bkey_cmp(bkey_start_pos(k.k), iter->pos) > 0)
- bch2_btree_iter_set_pos(iter, bkey_start_pos(k.k));
+ iter->pos = bkey_start_pos(k.k);
+
+ iter->flags |= BTREE_ITER_UPTODATE;
return k;
}
EBUG_ON(!!(iter->flags & BTREE_ITER_IS_EXTENTS) !=
(iter->btree_id == BTREE_ID_EXTENTS));
+ iter->flags &= ~BTREE_ITER_UPTODATE;
+
while (1) {
ret = bch2_btree_iter_traverse(iter);
if (unlikely(ret)) {
unsigned locks_want, unsigned depth,
unsigned flags)
{
+ EBUG_ON(depth >= BTREE_MAX_DEPTH);
+ EBUG_ON(locks_want > BTREE_MAX_DEPTH);
+
iter->c = c;
iter->pos = pos;
iter->flags = flags;
iter->btree_id = btree_id;
iter->level = depth;
- iter->locks_want = min(locks_want, BTREE_MAX_DEPTH);
+ iter->locks_want = locks_want;
iter->nodes_locked = 0;
iter->nodes_intent_locked = 0;
memset(iter->nodes, 0, sizeof(iter->nodes));
#include "btree_types.h"
-#define BTREE_ITER_INTENT (1 << 0)
+#define BTREE_ITER_UPTODATE (1 << 0)
#define BTREE_ITER_WITH_HOLES (1 << 1)
-#define BTREE_ITER_PREFETCH (1 << 2)
+#define BTREE_ITER_INTENT (1 << 2)
+#define BTREE_ITER_PREFETCH (1 << 3)
/*
* Used in bch2_btree_iter_traverse(), to indicate whether we're searching for
* @pos or the first key strictly greater than @pos
*/
-#define BTREE_ITER_IS_EXTENTS (1 << 3)
+#define BTREE_ITER_IS_EXTENTS (1 << 4)
/*
* indicates we need to call bch2_btree_iter_traverse() to revalidate iterator:
*/
-#define BTREE_ITER_AT_END_OF_LEAF (1 << 4)
-#define BTREE_ITER_ERROR (1 << 5)
+#define BTREE_ITER_AT_END_OF_LEAF (1 << 5)
+#define BTREE_ITER_ERROR (1 << 6)
/*
* @pos - iterator's current position
#define for_each_btree_node(_iter, _c, _btree_id, _start, _flags, _b) \
__for_each_btree_node(_iter, _c, _btree_id, _start, 0, 0, _flags, _b)
+static inline struct bkey_s_c __bch2_btree_iter_peek(struct btree_iter *iter,
+ unsigned flags)
+{
+ return flags & BTREE_ITER_WITH_HOLES
+ ? bch2_btree_iter_peek_with_holes(iter)
+ : bch2_btree_iter_peek(iter);
+}
+
#define for_each_btree_key(_iter, _c, _btree_id, _start, _flags, _k) \
- for (bch2_btree_iter_init((_iter), (_c), (_btree_id), \
- (_start), (_flags)); \
- !IS_ERR_OR_NULL(((_k) = (((_flags) & BTREE_ITER_WITH_HOLES)\
- ? bch2_btree_iter_peek_with_holes(_iter)\
- : bch2_btree_iter_peek(_iter))).k); \
+ for (bch2_btree_iter_init((_iter), (_c), (_btree_id), \
+ (_start), (_flags)); \
+ !IS_ERR_OR_NULL(((_k) = __bch2_btree_iter_peek(_iter, _flags)).k);\
bch2_btree_iter_advance_pos(_iter))
static inline int btree_iter_err(struct bkey_s_c k)
{
- return IS_ERR(k.k) ? PTR_ERR(k.k) : 0;
+ return PTR_ERR_OR_ZERO(k.k);
}
/*
* another write - because that write also won't yet be reachable and
* marking it as completed before it's reachable would be incorrect:
*/
- struct list_head reachable;
+ struct btree_interior_update *will_make_reachable;
struct open_bucket *ob;
enum btree_flags {
BTREE_NODE_read_in_flight,
BTREE_NODE_read_error,
- BTREE_NODE_write_error,
BTREE_NODE_dirty,
BTREE_NODE_need_write,
BTREE_NODE_noevict,
BTREE_FLAG(read_in_flight);
BTREE_FLAG(read_error);
-BTREE_FLAG(write_error);
BTREE_FLAG(dirty);
BTREE_FLAG(need_write);
BTREE_FLAG(noevict);
static void btree_interior_update_updated_root(struct bch_fs *,
struct btree_interior_update *,
enum btree_id);
+static void btree_interior_update_will_make_reachable(struct bch_fs *,
+ struct btree_interior_update *,
+ struct btree *);
+static void btree_interior_update_drop_new_node(struct bch_fs *,
+ struct btree *);
/* Calculate ideal packed bkey format for new btree nodes: */
BUG_ON(b == btree_node_root(c, b));
BUG_ON(b->ob);
BUG_ON(!list_empty(&b->write_blocked));
- BUG_ON(!list_empty(&b->reachable));
+ BUG_ON(b->will_make_reachable);
clear_btree_node_noevict(b);
{
struct open_bucket *ob = b->ob;
+ btree_interior_update_drop_new_node(c, b);
+
b->ob = NULL;
clear_btree_node_dirty(b);
static struct btree *bch2_btree_node_alloc(struct bch_fs *c,
unsigned level, enum btree_id id,
+ struct btree_interior_update *as,
struct btree_reserve *reserve)
{
struct btree *b;
bch2_btree_build_aux_trees(b);
- bch2_check_mark_super(c, bkey_i_to_s_c_extent(&b->key), BCH_DATA_BTREE);
+ btree_interior_update_will_make_reachable(c, as, b);
trace_btree_node_alloc(c, b);
return b;
struct btree *__bch2_btree_node_alloc_replacement(struct bch_fs *c,
struct btree *b,
struct bkey_format format,
+ struct btree_interior_update *as,
struct btree_reserve *reserve)
{
struct btree *n;
- n = bch2_btree_node_alloc(c, b->level, b->btree_id, reserve);
+ n = bch2_btree_node_alloc(c, b->level, b->btree_id, as, reserve);
n->data->min_key = b->data->min_key;
n->data->max_key = b->data->max_key;
static struct btree *bch2_btree_node_alloc_replacement(struct bch_fs *c,
struct btree *b,
+ struct btree_interior_update *as,
struct btree_reserve *reserve)
{
struct bkey_format new_f = bch2_btree_calc_format(b);
if (!bch2_btree_node_format_fits(c, b, &new_f))
new_f = b->format;
- return __bch2_btree_node_alloc_replacement(c, b, new_f, reserve);
+ return __bch2_btree_node_alloc_replacement(c, b, new_f, as, reserve);
}
static void bch2_btree_set_root_inmem(struct bch_fs *c, struct btree *b,
static struct btree *__btree_root_alloc(struct bch_fs *c, unsigned level,
enum btree_id id,
+ struct btree_interior_update *as,
struct btree_reserve *reserve)
{
- struct btree *b = bch2_btree_node_alloc(c, level, id, reserve);
+ struct btree *b = bch2_btree_node_alloc(c, level, id, as, reserve);
b->data->min_key = POS_MIN;
b->data->max_key = POS_MAX;
goto err_free;
}
+ ret = bch2_check_mark_super(c, bkey_i_to_s_c_extent(&b->key),
+ BCH_DATA_BTREE);
+ if (ret)
+ goto err_free;
+
reserve->b[reserve->nr++] = b;
}
int bch2_btree_root_alloc(struct bch_fs *c, enum btree_id id,
struct closure *writes)
{
- struct closure cl;
+ struct btree_interior_update as;
struct btree_reserve *reserve;
+ struct closure cl;
struct btree *b;
- LIST_HEAD(reachable_list);
+ memset(&as, 0, sizeof(as));
closure_init_stack(&cl);
while (1) {
closure_sync(&cl);
}
- b = __btree_root_alloc(c, 0, id, reserve);
- list_add(&b->reachable, &reachable_list);
+ b = __btree_root_alloc(c, 0, id, &as, reserve);
bch2_btree_node_write(c, b, writes, SIX_LOCK_intent);
bch2_btree_set_root_initial(c, b, reserve);
- bch2_btree_open_bucket_put(c, b);
- list_del_init(&b->reachable);
+ btree_interior_update_drop_new_node(c, b);
+ bch2_btree_open_bucket_put(c, b);
six_unlock_intent(&b->lock);
bch2_btree_reserve_put(c, reserve);
/* ick */
insert->k.needs_whiteout = false;
bch2_journal_add_keys(j, &trans->journal_res,
- b->btree_id, insert);
+ b->btree_id, insert);
insert->k.needs_whiteout = needs_whiteout;
+ bch2_journal_set_has_inode(j, &trans->journal_res,
+ insert->k.p.inode);
+
if (trans->journal_seq)
*trans->journal_seq = seq;
btree_bset_last(b)->journal_seq = cpu_to_le64(seq);
as->c = c;
as->mode = BTREE_INTERIOR_NO_UPDATE;
INIT_LIST_HEAD(&as->write_blocked_list);
- INIT_LIST_HEAD(&as->reachable_list);
bch2_keylist_init(&as->parent_keys, as->inline_keys,
ARRAY_SIZE(as->inline_keys));
struct btree_interior_update *as =
container_of(cl, struct btree_interior_update, cl);
struct bch_fs *c = as->c;
- unsigned i;
bch2_journal_pin_drop(&c->journal, &as->journal);
mutex_lock(&c->btree_interior_update_lock);
- while (!list_empty(&as->reachable_list)) {
- struct btree *b = list_first_entry(&as->reachable_list,
- struct btree, reachable);
- list_del_init(&b->reachable);
+ while (as->nr_new_nodes) {
+ struct btree *b = as->new_nodes[--as->nr_new_nodes];
+
+ BUG_ON(b->will_make_reachable != as);
+ b->will_make_reachable = NULL;
mutex_unlock(&c->btree_interior_update_lock);
six_lock_read(&b->lock);
mutex_lock(&c->btree_interior_update_lock);
}
- for (i = 0; i < as->nr_pending; i++)
- bch2_btree_node_free_ondisk(c, &as->pending[i]);
- as->nr_pending = 0;
+ while (as->nr_pending)
+ bch2_btree_node_free_ondisk(c, &as->pending[--as->nr_pending]);
list_del(&as->list);
mutex_unlock(&c->btree_interior_update_lock);
system_freezable_wq);
}
+static void btree_interior_update_will_make_reachable(struct bch_fs *c,
+ struct btree_interior_update *as,
+ struct btree *b)
+{
+ mutex_lock(&c->btree_interior_update_lock);
+ BUG_ON(as->nr_new_nodes >= ARRAY_SIZE(as->new_nodes));
+ BUG_ON(b->will_make_reachable);
+
+ as->new_nodes[as->nr_new_nodes++] = b;
+ b->will_make_reachable = as;
+ mutex_unlock(&c->btree_interior_update_lock);
+}
+
+static void __btree_interior_update_drop_new_node(struct btree *b)
+{
+ struct btree_interior_update *as = b->will_make_reachable;
+ unsigned i;
+
+ BUG_ON(!as);
+
+ for (i = 0; i < as->nr_new_nodes; i++)
+ if (as->new_nodes[i] == b)
+ goto found;
+
+ BUG();
+found:
+ as->nr_new_nodes--;
+ memmove(&as->new_nodes[i],
+ &as->new_nodes[i + 1],
+ sizeof(struct btree *) * (as->nr_new_nodes - i));
+ b->will_make_reachable = NULL;
+}
+
+static void btree_interior_update_drop_new_node(struct bch_fs *c,
+ struct btree *b)
+{
+ mutex_lock(&c->btree_interior_update_lock);
+ __btree_interior_update_drop_new_node(b);
+ mutex_unlock(&c->btree_interior_update_lock);
+}
+
+static void bch2_btree_interior_update_add_node_reference(struct bch_fs *c,
+ struct btree_interior_update *as,
+ struct btree *b)
+{
+ struct pending_btree_node_free *d;
+
+ mutex_lock(&c->btree_interior_update_lock);
+
+ /* Add this node to the list of nodes being freed: */
+ BUG_ON(as->nr_pending >= ARRAY_SIZE(as->pending));
+
+ d = &as->pending[as->nr_pending++];
+ d->index_update_done = false;
+ d->seq = b->data->keys.seq;
+ d->btree_id = b->btree_id;
+ d->level = b->level;
+ bkey_copy(&d->key, &b->key);
+
+ mutex_unlock(&c->btree_interior_update_lock);
+}
+
/*
* @b is being split/rewritten: it may have pointers to not-yet-written btree
* nodes and thus outstanding btree_interior_updates - redirect @b's
{
struct closure *cl, *cl_n;
struct btree_interior_update *p, *n;
- struct pending_btree_node_free *d;
struct btree_write *w;
struct bset_tree *t;
+ bch2_btree_interior_update_add_node_reference(c, as, b);
+
/*
* Does this node have data that hasn't been written in the journal?
*
mutex_lock(&c->btree_interior_update_lock);
- /* Add this node to the list of nodes being freed: */
- BUG_ON(as->nr_pending >= ARRAY_SIZE(as->pending));
-
- d = &as->pending[as->nr_pending++];
- d->index_update_done = false;
- d->seq = b->data->keys.seq;
- d->btree_id = b->btree_id;
- d->level = b->level;
- bkey_copy(&d->key, &b->key);
-
/*
* Does this node have any btree_interior_update operations preventing
* it from being written?
&as->journal, interior_update_flush);
bch2_journal_pin_drop(&c->journal, &w->journal);
- if (!list_empty(&b->reachable))
- list_del_init(&b->reachable);
+ w = btree_prev_write(b);
+ bch2_journal_pin_add_if_older(&c->journal, &w->journal,
+ &as->journal, interior_update_flush);
+ bch2_journal_pin_drop(&c->journal, &w->journal);
+
+ if (b->will_make_reachable)
+ __btree_interior_update_drop_new_node(b);
mutex_unlock(&c->btree_interior_update_lock);
}
#endif
}
-static enum btree_insert_ret
+static int
bch2_btree_insert_keys_interior(struct btree *b,
struct btree_iter *iter,
struct keylist *insert_keys,
if (bch_keylist_u64s(insert_keys) >
bch_btree_keys_u64s_remaining(c, b)) {
bch2_btree_node_unlock_write(b, iter);
- return BTREE_INSERT_BTREE_NODE_FULL;
+ return -1;
}
/* Don't screw up @iter's position: */
bch2_btree_node_unlock_write(b, iter);
btree_node_interior_verify(b);
- return BTREE_INSERT_OK;
+ return 0;
}
/*
struct btree_reserve *reserve,
struct btree_interior_update *as)
{
+ struct bch_fs *c = iter->c;
size_t nr_packed = 0, nr_unpacked = 0;
struct btree *n2;
struct bset *set1, *set2;
struct bkey_packed *k, *prev = NULL;
- n2 = bch2_btree_node_alloc(iter->c, n1->level, iter->btree_id, reserve);
- list_add(&n2->reachable, &as->reachable_list);
+ n2 = bch2_btree_node_alloc(c, n1->level, iter->btree_id, as, reserve);
n2->data->max_key = n1->data->max_key;
n2->data->format = n1->format;
bch2_btree_interior_update_will_free_node(c, as, b);
- n1 = bch2_btree_node_alloc_replacement(c, b, reserve);
- list_add(&n1->reachable, &as->reachable_list);
+ n1 = bch2_btree_node_alloc_replacement(c, b, as, reserve);
if (b->level)
btree_split_insert_keys(iter, n1, insert_keys, reserve);
/* Depth increases, make a new root */
n3 = __btree_root_alloc(c, b->level + 1,
iter->btree_id,
- reserve);
- list_add(&n3->reachable, &as->reachable_list);
+ as, reserve);
n3->sib_u64s[0] = U16_MAX;
n3->sib_u64s[1] = U16_MAX;
BUG_ON(!b->level);
BUG_ON(!reserve || !as);
- switch (bch2_btree_insert_keys_interior(b, iter, insert_keys,
- as, reserve)) {
- case BTREE_INSERT_OK:
- break;
- case BTREE_INSERT_BTREE_NODE_FULL:
+ if ((as->flags & BTREE_INTERIOR_UPDATE_MUST_REWRITE) ||
+ bch2_btree_insert_keys_interior(b, iter, insert_keys,
+ as, reserve))
btree_split(b, iter, insert_keys, reserve, as);
- break;
- default:
- BUG();
- }
}
static int bch2_btree_split_leaf(struct btree_iter *iter, unsigned flags)
bch2_btree_interior_update_will_free_node(c, as, b);
bch2_btree_interior_update_will_free_node(c, as, m);
- n = bch2_btree_node_alloc(c, b->level, b->btree_id, reserve);
- list_add(&n->reachable, &as->reachable_list);
+ n = bch2_btree_node_alloc(c, b->level, b->btree_id, as, reserve);
n->data->min_key = prev->data->min_key;
n->data->max_key = next->data->max_key;
int old_live_u64s = b->nr.live_u64s;
int live_u64s_added, u64s_added;
+ iter->flags &= ~BTREE_ITER_UPTODATE;
+
ret = !btree_node_is_extents(b)
? bch2_insert_fixup_key(trans, insert)
: bch2_insert_fixup_extent(trans, insert);
bch2_btree_interior_update_will_free_node(c, as, b);
- n = bch2_btree_node_alloc_replacement(c, b, reserve);
- list_add(&n->reachable, &as->reachable_list);
+ n = bch2_btree_node_alloc_replacement(c, b, as, reserve);
bch2_btree_build_aux_trees(n);
six_unlock_write(&n->lock);
closure_sync(&cl);
return ret;
}
+
+int bch2_btree_node_update_key(struct bch_fs *c, struct btree *b,
+ struct bkey_i_extent *new_key)
+{
+ struct btree_interior_update *as;
+ struct btree_reserve *reserve = NULL;
+ struct btree *parent, *new_hash = NULL;
+ struct btree_iter iter;
+ struct closure cl;
+ bool must_rewrite_parent = false;
+ int ret;
+
+ __bch2_btree_iter_init(&iter, c, b->btree_id, b->key.k.p,
+ BTREE_MAX_DEPTH,
+ b->level, 0);
+ closure_init_stack(&cl);
+
+ if (PTR_HASH(&new_key->k_i) != PTR_HASH(&b->key)) {
+ /* bch2_btree_reserve_get will unlock */
+ do {
+ ret = bch2_btree_node_cannibalize_lock(c, &cl);
+ closure_sync(&cl);
+ } while (ret == -EAGAIN);
+
+ BUG_ON(ret);
+
+ new_hash = bch2_btree_node_mem_alloc(c);
+ }
+retry:
+ reserve = bch2_btree_reserve_get(c, b, 0,
+ BTREE_INSERT_NOFAIL|
+ BTREE_INSERT_USE_RESERVE|
+ BTREE_INSERT_USE_ALLOC_RESERVE,
+ &cl);
+ closure_sync(&cl);
+ if (IS_ERR(reserve)) {
+ ret = PTR_ERR(reserve);
+ if (ret == -EAGAIN || ret == -EINTR)
+ goto retry;
+ goto err;
+ }
+
+ down_read(&c->gc_lock);
+
+ ret = bch2_btree_iter_traverse(&iter);
+ if (ret)
+ goto err;
+
+ mutex_lock(&c->btree_interior_update_lock);
+
+ /*
+ * Two corner cases that need to be thought about here:
+ *
+ * @b may not be reachable yet - there might be another interior update
+ * operation waiting on @b to be written, and we're gonna deliver the
+ * write completion to that interior update operation _before_
+ * persisting the new_key update
+ *
+ * That ends up working without us having to do anything special here:
+ * the reason is, we do kick off (and do the in memory updates) for the
+ * update for @new_key before we return, creating a new interior_update
+ * operation here.
+ *
+ * The new interior update operation here will in effect override the
+ * previous one. The previous one was going to terminate - make @b
+ * reachable - in one of two ways:
+ * - updating the btree root pointer
+ * In that case,
+ * no, this doesn't work. argh.
+ */
+
+ if (b->will_make_reachable)
+ must_rewrite_parent = true;
+
+ /* other case: btree node being freed */
+ if (iter.nodes[b->level] != b) {
+ /* node has been freed: */
+ BUG_ON(btree_node_hashed(b));
+ mutex_unlock(&c->btree_interior_update_lock);
+ goto err;
+ }
+
+ mutex_unlock(&c->btree_interior_update_lock);
+
+ ret = bch2_check_mark_super(c, extent_i_to_s_c(new_key), BCH_DATA_BTREE);
+ if (ret)
+ goto err;
+
+ as = bch2_btree_interior_update_alloc(c);
+
+ if (must_rewrite_parent)
+ as->flags |= BTREE_INTERIOR_UPDATE_MUST_REWRITE;
+
+ bch2_btree_interior_update_add_node_reference(c, as, b);
+
+ if (new_hash) {
+ bkey_copy(&new_hash->key, &new_key->k_i);
+ BUG_ON(bch2_btree_node_hash_insert(c, new_hash,
+ b->level, b->btree_id));
+ }
+
+ parent = iter.nodes[b->level + 1];
+ if (parent) {
+ bch2_btree_insert_node(parent, &iter,
+ &keylist_single(&b->key),
+ reserve, as);
+ } else {
+ bch2_btree_set_root(&iter, b, as, reserve);
+ }
+
+ if (new_hash) {
+ mutex_lock(&c->btree_cache_lock);
+ bch2_btree_node_hash_remove(c, b);
+
+ bkey_copy(&b->key, &new_key->k_i);
+ __bch2_btree_node_hash_insert(c, b);
+
+ bch2_btree_node_hash_remove(c, new_hash);
+ mutex_unlock(&c->btree_cache_lock);
+ } else {
+ bkey_copy(&b->key, &new_key->k_i);
+ }
+err:
+ if (!IS_ERR_OR_NULL(reserve))
+ bch2_btree_reserve_put(c, reserve);
+ if (new_hash) {
+ mutex_lock(&c->btree_cache_lock);
+ list_move(&b->list, &c->btree_cache_freeable);
+ mutex_unlock(&c->btree_cache_lock);
+
+ six_unlock_write(&new_hash->lock);
+ six_unlock_intent(&new_hash->lock);
+ }
+ bch2_btree_iter_unlock(&iter);
+ up_read(&c->gc_lock);
+ return ret;
+}
BTREE_INTERIOR_UPDATING_AS,
} mode;
+ unsigned flags;
+ struct btree_reserve *reserve;
+
/*
* BTREE_INTERIOR_UPDATING_NODE:
* The update that made the new nodes visible was a regular update to an
*/
struct btree *b;
struct list_head write_blocked_list;
- struct list_head reachable_list;
/*
* BTREE_INTERIOR_UPDATING_AS: btree node we updated was freed, so now
struct pending_btree_node_free pending[BTREE_MAX_DEPTH + GC_MERGE_NODES];
unsigned nr_pending;
+ /* New nodes, that will be made reachable by this update: */
+ struct btree *new_nodes[BTREE_MAX_DEPTH * 2 + GC_MERGE_NODES];
+ unsigned nr_new_nodes;
+
/* Only here to reduce stack usage on recursive splits: */
struct keylist parent_keys;
/*
u64 inline_keys[BKEY_BTREE_PTR_U64s_MAX * 3];
};
+#define BTREE_INTERIOR_UPDATE_MUST_REWRITE (1 << 0)
+
#define for_each_pending_btree_node_free(c, as, p) \
list_for_each_entry(as, &c->btree_interior_update_list, list) \
for (p = as->pending; p < as->pending + as->nr_pending; p++)
struct btree *__bch2_btree_node_alloc_replacement(struct bch_fs *,
struct btree *,
struct bkey_format,
+ struct btree_interior_update *,
struct btree_reserve *);
struct btree_interior_update *
int bch2_btree_node_rewrite(struct bch_fs *c, struct btree_iter *,
__le64, unsigned);
+int bch2_btree_node_update_key(struct bch_fs *, struct btree *,
+ struct bkey_i_extent *);
#endif /* _BCACHE_BTREE_INSERT_H */
return nr_ptrs;
}
+/* Doesn't cleanup redundant crcs */
+void __bch2_extent_drop_ptr(struct bkey_s_extent e, struct bch_extent_ptr *ptr)
+{
+ EBUG_ON(ptr < &e.v->start->ptr ||
+ ptr >= &extent_entry_last(e)->ptr);
+ EBUG_ON(ptr->type != 1 << BCH_EXTENT_ENTRY_ptr);
+ memmove_u64s_down(ptr, ptr + 1,
+ (u64 *) extent_entry_last(e) - (u64 *) (ptr + 1));
+ e.k->u64s -= sizeof(*ptr) / sizeof(u64);
+}
+
+void bch2_extent_drop_ptr(struct bkey_s_extent e, struct bch_extent_ptr *ptr)
+{
+ __bch2_extent_drop_ptr(e, ptr);
+ bch2_extent_drop_redundant_crcs(e);
+}
+
+void bch2_extent_drop_ptr_idx(struct bkey_s_extent e, unsigned idx)
+{
+ struct bch_extent_ptr *ptr;
+ unsigned i = 0;
+
+ extent_for_each_ptr(e, ptr)
+ if (i++ == idx)
+ goto found;
+
+ BUG();
+found:
+ bch2_extent_drop_ptr(e, ptr);
+}
+
/* returns true if equal */
static bool crc_cmp(union bch_extent_crc *l, union bch_extent_crc *r)
{
void bch2_extent_narrow_crcs(struct bkey_s_extent);
void bch2_extent_drop_redundant_crcs(struct bkey_s_extent);
-/* Doesn't cleanup redundant crcs */
-static inline void __bch2_extent_drop_ptr(struct bkey_s_extent e,
- struct bch_extent_ptr *ptr)
-{
- EBUG_ON(ptr < &e.v->start->ptr ||
- ptr >= &extent_entry_last(e)->ptr);
- EBUG_ON(ptr->type != 1 << BCH_EXTENT_ENTRY_ptr);
- memmove_u64s_down(ptr, ptr + 1,
- (u64 *) extent_entry_last(e) - (u64 *) (ptr + 1));
- e.k->u64s -= sizeof(*ptr) / sizeof(u64);
-}
-
-static inline void bch2_extent_drop_ptr(struct bkey_s_extent e,
- struct bch_extent_ptr *ptr)
-{
- __bch2_extent_drop_ptr(e, ptr);
- bch2_extent_drop_redundant_crcs(e);
-}
+void __bch2_extent_drop_ptr(struct bkey_s_extent, struct bch_extent_ptr *);
+void bch2_extent_drop_ptr(struct bkey_s_extent, struct bch_extent_ptr *);
+void bch2_extent_drop_ptr_idx(struct bkey_s_extent, unsigned);
const struct bch_extent_ptr *
bch2_extent_has_device(struct bkey_s_c_extent, unsigned);
{
struct bch_writepage_io *io = container_of(cl,
struct bch_writepage_io, cl);
- struct bio *bio = &io->bio.bio;
- bio_put(bio);
+ bio_put(&io->op.op.wbio.bio);
}
static void bch2_writepage_io_done(struct closure *cl)
struct bch_writepage_io *io = container_of(cl,
struct bch_writepage_io, cl);
struct bch_fs *c = io->op.op.c;
- struct bio *bio = &io->bio.bio;
+ struct bio *bio = &io->op.op.wbio.bio;
struct bio_vec *bvec;
unsigned i;
static void bch2_writepage_do_io(struct bch_writepage_state *w)
{
struct bch_writepage_io *io = w->io;
+ struct bio *bio = &io->op.op.wbio.bio;
w->io = NULL;
- atomic_add(io->bio.bio.bi_vcnt, &io->op.op.c->writeback_pages);
+ atomic_add(bio->bi_vcnt, &io->op.op.c->writeback_pages);
- io->op.op.pos.offset = io->bio.bio.bi_iter.bi_sector;
+ io->op.op.pos.offset = bio->bi_iter.bi_sector;
closure_call(&io->op.op.cl, bch2_write, NULL, &io->cl);
continue_at(&io->cl, bch2_writepage_io_done, NULL);
w->io = container_of(bio_alloc_bioset(GFP_NOFS,
BIO_MAX_PAGES,
bch2_writepage_bioset),
- struct bch_writepage_io, bio.bio);
+ struct bch_writepage_io, op.op.wbio.bio);
closure_init(&w->io->cl, NULL);
w->io->op.ei = ei;
w->io->op.sectors_added = 0;
w->io->op.is_dio = false;
- bch2_write_op_init(&w->io->op.op, c, &w->io->bio,
+ bch2_write_op_init(&w->io->op.op, c,
(struct disk_reservation) {
.nr_replicas = c->opts.data_replicas,
},
}
if (w->io->op.op.res.nr_replicas != nr_replicas ||
- bio_add_page_contig(&w->io->bio.bio, page)) {
+ bio_add_page_contig(&w->io->op.op.wbio.bio, page)) {
bch2_writepage_do_io(w);
goto alloc_io;
}
w->io->op.new_i_size = i_size;
if (wbc->sync_mode == WB_SYNC_ALL)
- w->io->bio.bio.bi_opf |= REQ_SYNC;
+ w->io->op.op.wbio.bio.bi_opf |= REQ_SYNC;
/* Before unlocking the page, transfer reservation to w->io: */
old = page_state_cmpxchg(page_state(page), new, {
done_index = page->index;
if (w.io &&
- !bio_can_add_page_contig(&w.io->bio.bio, page))
+ !bio_can_add_page_contig(&w.io->op.op.wbio.bio, page))
bch2_writepage_do_io(&w);
if (!w.io &&
if (dio->iovec && dio->iovec != dio->inline_vecs)
kfree(dio->iovec);
- bio_put(&dio->bio.bio);
+ bio_put(&dio->iop.op.wbio.bio);
return ret;
}
if (dio->iop.op.error)
dio->error = dio->iop.op.error;
- bio_for_each_segment_all(bv, &dio->bio.bio, i)
+ bio_for_each_segment_all(bv, &dio->iop.op.wbio.bio, i)
put_page(bv->bv_page);
if (dio->iter.count)
- bio_reset(&dio->bio.bio);
+ bio_reset(&dio->iop.op.wbio.bio);
}
static void bch2_do_direct_IO_write(struct dio_write *dio)
struct file *file = dio->req->ki_filp;
struct inode *inode = file->f_inode;
struct bch_inode_info *ei = to_bch_ei(inode);
- struct bio *bio = &dio->bio.bio;
+ struct bio *bio = &dio->iop.op.wbio.bio;
unsigned flags = 0;
int ret;
!dio->c->opts.journal_flush_disabled)
flags |= BCH_WRITE_FLUSH;
- bio->bi_iter.bi_sector = (dio->offset + dio->written) >> 9;
-
ret = bio_iov_iter_get_pages(bio, &dio->iter);
if (ret < 0) {
/*
dio->iop.sectors_added = 0;
dio->iop.is_dio = true;
dio->iop.new_i_size = U64_MAX;
- bch2_write_op_init(&dio->iop.op, dio->c, &dio->bio,
- dio->res,
+ bch2_write_op_init(&dio->iop.op, dio->c, dio->res,
foreground_write_point(dio->c, inode->i_ino),
- POS(inode->i_ino, bio->bi_iter.bi_sector),
+ POS(inode->i_ino, (dio->offset + dio->written) >> 9),
&ei->journal_seq, flags);
dio->iop.op.index_update_fn = bchfs_write_index_update;
bio = bio_alloc_bioset(GFP_KERNEL,
iov_iter_npages(iter, BIO_MAX_PAGES),
bch2_dio_write_bioset);
- dio = container_of(bio, struct dio_write, bio.bio);
+ dio = container_of(bio, struct dio_write, iop.op.wbio.bio);
dio->req = req;
dio->c = c;
dio->written = 0;
s64 sectors_added;
bool is_dio;
u64 new_i_size;
+
+ /* must be last: */
struct bch_write_op op;
};
struct bch_writepage_io {
struct closure cl;
+ /* must be last: */
struct bchfs_write_op op;
-
- /* must come last: */
- struct bch_write_bio bio;
};
extern struct bio_set *bch2_writepage_bioset;
struct mm_struct *mm;
- struct bchfs_write_op iop;
-
/* must be last: */
- struct bch_write_bio bio;
+ struct bchfs_write_op iop;
};
extern struct bio_set *bch2_dio_write_bioset;
goto err;
bch2_writepage_bioset =
- bioset_create(4, offsetof(struct bch_writepage_io, bio.bio));
+ bioset_create(4, offsetof(struct bch_writepage_io, op.op.wbio.bio));
if (!bch2_writepage_bioset)
goto err;
if (!bch2_dio_read_bioset)
goto err;
- bch2_dio_write_bioset = bioset_create(4, offsetof(struct dio_write, bio.bio));
+ bch2_dio_write_bioset =
+ bioset_create(4, offsetof(struct dio_write, iop.op.wbio.bio));
if (!bch2_dio_write_bioset)
goto err;
const struct bch_extent_ptr *ptr;
struct bch_write_bio *n;
struct bch_dev *ca;
+ unsigned ptr_idx = 0;
BUG_ON(c->opts.nochanges);
- wbio->split = false;
- wbio->c = c;
-
extent_for_each_ptr(e, ptr) {
ca = c->devs[ptr->dev];
n->bio.bi_end_io = wbio->bio.bi_end_io;
n->bio.bi_private = wbio->bio.bi_private;
- n->c = c;
- n->orig = &wbio->bio;
- n->bounce = false;
+ n->parent = wbio;
n->split = true;
+ n->bounce = false;
n->put_bio = true;
n->bio.bi_opf = wbio->bio.bi_opf;
- __bio_inc_remaining(n->orig);
+ __bio_inc_remaining(&wbio->bio);
} else {
n = wbio;
+ n->split = false;
}
- if (!journal_flushes_device(ca))
- n->bio.bi_opf |= REQ_FUA;
-
+ n->c = c;
n->ca = ca;
+ n->ptr_idx = ptr_idx++;
n->submit_time_us = local_clock_us();
n->bio.bi_iter.bi_sector = ptr->offset;
+ if (!journal_flushes_device(ca))
+ n->bio.bi_opf |= REQ_FUA;
+
if (likely(percpu_ref_tryget(&ca->io_ref))) {
n->have_io_ref = true;
n->bio.bi_bdev = ca->disk_sb.bdev;
static void bch2_write_discard(struct closure *cl)
{
struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
- struct bio *bio = &op->bio->bio;
struct bpos end = op->pos;
- end.offset += bio_sectors(bio);
+ end.offset += bio_sectors(&op->wbio.bio);
op->error = bch2_discard(op->c, op->pos, end, op->version,
&op->res, NULL, NULL);
static void bch2_write_endio(struct bio *bio)
{
- struct closure *cl = bio->bi_private;
- struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
- struct bch_write_bio *wbio = to_wbio(bio);
- struct bch_fs *c = wbio->c;
- struct bio *orig = wbio->orig;
- struct bch_dev *ca = wbio->ca;
+ struct closure *cl = bio->bi_private;
+ struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
+ struct bch_write_bio *wbio = to_wbio(bio);
+ struct bch_write_bio *parent = wbio->split ? wbio->parent : NULL;
+ struct bch_fs *c = wbio->c;
+ struct bch_dev *ca = wbio->ca;
if (bch2_dev_nonfatal_io_err_on(bio->bi_error, ca,
- "data write"))
+ "data write"))
set_closure_fn(cl, bch2_write_io_error, index_update_wq(op));
if (wbio->have_io_ref)
percpu_ref_put(&ca->io_ref);
- if (bio->bi_error && orig)
- orig->bi_error = bio->bi_error;
-
if (wbio->bounce)
bch2_bio_free_pages_pool(c, bio);
if (wbio->put_bio)
bio_put(bio);
- if (orig)
- bio_endio(orig);
+ if (parent)
+ bio_endio(&parent->bio);
else
closure_put(cl);
}
bch2_keylist_push(&op->insert_keys);
}
-static int bch2_write_extent(struct bch_write_op *op,
- struct open_bucket *ob,
- struct bio *orig)
+static int bch2_write_extent(struct bch_write_op *op, struct open_bucket *ob)
{
struct bch_fs *c = op->c;
+ struct bio *orig = &op->wbio.bio;
struct bio *bio;
struct bch_write_bio *wbio;
unsigned key_to_write_offset = op->insert_keys.top_p -
struct bkey_i *key_to_write;
unsigned csum_type = op->csum_type;
unsigned compression_type = op->compression_type;
- int ret;
+ int ret, more;
/* don't refetch csum type/compression type */
barrier();
+ BUG_ON(!bio_sectors(orig));
+
/* Need to decompress data? */
if ((op->flags & BCH_WRITE_DATA_COMPRESSED) &&
(crc_uncompressed_size(NULL, &op->crc) != op->size ||
ob);
bio = orig;
- wbio = to_wbio(bio);
- wbio->orig = NULL;
- wbio->bounce = false;
- wbio->put_bio = false;
- ret = 0;
+ wbio = wbio_init(bio);
+ more = 0;
} else if (csum_type != BCH_CSUM_NONE ||
compression_type != BCH_COMPRESSION_NONE) {
/* all units here in bytes */
bio = bio_alloc_bioset(GFP_NOIO,
DIV_ROUND_UP(output_available, PAGE_SIZE),
&c->bio_write);
+ wbio = wbio_init(bio);
+ wbio->bounce = true;
+ wbio->put_bio = true;
+ /* copy WRITE_SYNC flag */
+ wbio->bio.bi_opf = orig->bi_opf;
+
/*
* XXX: can't use mempool for more than
* BCH_COMPRESSED_EXTENT_MAX worth of pages
*/
bch2_bio_alloc_pages_pool(c, bio, output_available);
- /* copy WRITE_SYNC flag */
- bio->bi_opf = orig->bi_opf;
- wbio = to_wbio(bio);
- wbio->orig = NULL;
- wbio->bounce = true;
- wbio->put_bio = true;
-
do {
unsigned fragment_compression_type = compression_type;
size_t dst_len, src_len;
mempool_free(bio->bi_io_vec[--bio->bi_vcnt].bv_page,
&c->bio_bounce_pages);
- ret = orig->bi_iter.bi_size != 0;
+ more = orig->bi_iter.bi_size != 0;
} else {
bio = bio_next_split(orig, ob->sectors_free, GFP_NOIO,
&c->bio_write);
-
- wbio = to_wbio(bio);
- wbio->orig = NULL;
- wbio->bounce = false;
+ wbio = wbio_init(bio);
wbio->put_bio = bio != orig;
init_append_extent(op, bio_sectors(bio), bio_sectors(bio),
compression_type, 0,
(struct bch_csum) { 0 }, csum_type, ob);
- ret = bio != orig;
+ more = bio != orig;
}
+ /* might have done a realloc... */
+
+ key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
+
+ ret = bch2_check_mark_super(c, bkey_i_to_s_c_extent(key_to_write),
+ BCH_DATA_USER);
+ if (ret)
+ return ret;
+
bio->bi_end_io = bch2_write_endio;
bio->bi_private = &op->cl;
bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
closure_get(bio->bi_private);
- /* might have done a realloc... */
-
- key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
-
- bch2_check_mark_super(c, bkey_i_to_s_c_extent(key_to_write),
- BCH_DATA_USER);
-
bch2_submit_wbio_replicas(to_wbio(bio), c, key_to_write);
- return ret;
+ return more;
}
static void __bch2_write(struct closure *cl)
{
struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
struct bch_fs *c = op->c;
- struct bio *bio = &op->bio->bio;
unsigned open_bucket_nr = 0;
struct open_bucket *b;
int ret;
memset(op->open_buckets, 0, sizeof(op->open_buckets));
if (op->flags & BCH_WRITE_DISCARD) {
- op->flags |= BCH_WRITE_DONE;
bch2_write_discard(cl);
- bio_put(bio);
+ op->flags |= BCH_WRITE_DONE;
continue_at(cl, bch2_write_done, index_update_wq(op));
}
- /*
- * Journal writes are marked REQ_PREFLUSH; if the original write was a
- * flush, it'll wait on the journal write.
- */
- bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA);
-
do {
- EBUG_ON(bio->bi_iter.bi_sector != op->pos.offset);
- EBUG_ON(!bio_sectors(bio));
-
if (open_bucket_nr == ARRAY_SIZE(op->open_buckets))
continue_at(cl, bch2_write_index, index_update_wq(op));
b - c->open_buckets > U8_MAX);
op->open_buckets[open_bucket_nr++] = b - c->open_buckets;
- ret = bch2_write_extent(op, b, bio);
+ ret = bch2_write_extent(op, b);
bch2_alloc_sectors_done(c, op->wp, b);
* after the data is written it calls bch_journal, and after the keys have been
* added to the next journal write they're inserted into the btree.
*
- * It inserts the data in op->bio; bi_sector is used for the key offset, and
- * op->inode is used for the key inode.
- *
* If op->discard is true, instead of inserting the data it invalidates the
* region of the cache represented by op->bio and op->inode.
*/
void bch2_write(struct closure *cl)
{
struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
- struct bio *bio = &op->bio->bio;
+ struct bio *bio = &op->wbio.bio;
struct bch_fs *c = op->c;
u64 inode = op->pos.inode;
spin_lock_irqsave(&c->foreground_write_pd_lock, flags);
bch2_ratelimit_increment(&c->foreground_write_pd.rate,
- bio->bi_iter.bi_size);
+ bio->bi_iter.bi_size);
delay = bch2_ratelimit_delay(&c->foreground_write_pd.rate);
}
void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
- struct bch_write_bio *bio, struct disk_reservation res,
- struct write_point *wp, struct bpos pos,
- u64 *journal_seq, unsigned flags)
+ struct disk_reservation res,
+ struct write_point *wp, struct bpos pos,
+ u64 *journal_seq, unsigned flags)
{
EBUG_ON(res.sectors && !res.nr_replicas);
op->c = c;
op->io_wq = index_update_wq(op);
- op->bio = bio;
op->written = 0;
op->error = 0;
op->flags = flags;
struct cache_promote_op *op =
container_of(cl, struct cache_promote_op, cl);
- bch2_bio_free_pages_pool(op->write.op.c, &op->write.wbio.bio);
+ bch2_bio_free_pages_pool(op->write.op.c, &op->write.op.wbio.bio);
kfree(op);
}
trace_promote(&rbio->bio);
/* we now own pages: */
- swap(promote->write.wbio.bio.bi_vcnt, rbio->bio.bi_vcnt);
+ swap(promote->write.op.wbio.bio.bi_vcnt, rbio->bio.bi_vcnt);
rbio->promote = NULL;
bch2_rbio_done(rbio);
promote_op = kmalloc(sizeof(*promote_op) +
sizeof(struct bio_vec) * pages, GFP_NOIO);
if (promote_op) {
- struct bio *promote_bio = &promote_op->write.wbio.bio;
+ struct bio *promote_bio = &promote_op->write.op.wbio.bio;
bio_init(promote_bio,
promote_bio->bi_inline_vecs,
rbio->bio.bi_end_io = bch2_read_endio;
if (promote_op) {
- struct bio *promote_bio = &promote_op->write.wbio.bio;
+ struct bio *promote_bio = &promote_op->write.op.wbio.bio;
promote_bio->bi_iter = rbio->bio.bi_iter;
memcpy(promote_bio->bi_io_vec, rbio->bio.bi_io_vec,
read_retry_work);
struct bch_read_bio *rbio;
struct bio *bio;
- unsigned long flags;
while (1) {
- spin_lock_irqsave(&c->read_retry_lock, flags);
+ spin_lock_irq(&c->read_retry_lock);
bio = bio_list_pop(&c->read_retry_list);
- spin_unlock_irqrestore(&c->read_retry_lock, flags);
+ spin_unlock_irq(&c->read_retry_lock);
if (!bio)
break;
}
void bch2_write_op_init(struct bch_write_op *, struct bch_fs *,
- struct bch_write_bio *,
struct disk_reservation, struct write_point *,
struct bpos, u64 *, unsigned);
void bch2_write(struct closure *);
+static inline struct bch_write_bio *wbio_init(struct bio *bio)
+{
+ struct bch_write_bio *wbio = to_wbio(bio);
+
+ memset(wbio, 0, offsetof(struct bch_write_bio, bio));
+ return wbio;
+}
+
struct cache_promote_op;
struct extent_pick_ptr;
struct bch_fs *c;
struct bch_dev *ca;
union {
- struct bio *orig;
- struct closure *cl;
+ struct bch_write_bio *parent;
+ struct closure *cl;
};
- unsigned submit_time_us;
+ u8 ptr_idx;
+ u8 replicas_failed;
+ u8 order;
+
unsigned split:1,
bounce:1,
put_bio:1,
- have_io_ref:1;
+ have_io_ref:1,
+ used_mempool:1;
- /* Only for btree writes: */
- unsigned used_mempool:1;
- u8 order;
+ unsigned submit_time_us;
+ void *data;
struct bio bio;
};
-struct bch_replace_info {
- struct extent_insert_hook hook;
- /* How many insertions succeeded */
- unsigned successes;
- /* How many insertions failed */
- unsigned failures;
- BKEY_PADDED(key);
-};
-
struct bch_write_op {
struct closure cl;
- struct bch_fs *c;
+ struct bch_fs *c;
struct workqueue_struct *io_wq;
- struct bch_write_bio *bio;
unsigned written; /* sectors */
struct keylist insert_keys;
u64 inline_keys[BKEY_EXTENT_U64s_MAX * 2];
+
+ /* Must be last: */
+ struct bch_write_bio wbio;
};
#endif /* _BCACHE_IO_TYPES_H */
return last_seq(j) + fifo_entry_idx(&j->pin, pin_list);
}
-static inline void bch2_journal_add_entry(struct journal_buf *buf,
- const void *data, size_t u64s,
- unsigned type, enum btree_id id,
- unsigned level)
+static inline void bch2_journal_add_entry_noreservation(struct journal_buf *buf,
+ unsigned type, enum btree_id id,
+ unsigned level,
+ const void *data, size_t u64s)
{
struct jset *jset = buf->data;
- bch2_journal_add_entry_at(buf, data, u64s, type, id, level,
- le32_to_cpu(jset->u64s));
+ bch2_journal_add_entry_at(buf, le32_to_cpu(jset->u64s),
+ type, id, level, data, u64s);
le32_add_cpu(&jset->u64s, jset_u64s(u64s));
}
enum btree_id id, struct bkey_i *k,
unsigned level)
{
- bch2_journal_add_entry(buf, k, k->k.u64s,
- JOURNAL_ENTRY_BTREE_ROOT, id, level);
+ bch2_journal_add_entry_noreservation(buf,
+ JOURNAL_ENTRY_BTREE_ROOT, id, level,
+ k, k->k.u64s);
}
static void journal_seq_blacklist_flush(struct journal *j,
{
struct jset_entry *entry;
- for (entry = start; entry != end; entry = vstruct_next(entry)) {
- entry->u64s = 0;
- entry->btree_id = 0;
- entry->level = 0;
- entry->flags = 0;
- SET_JOURNAL_ENTRY_TYPE(entry, 0);
- }
+ for (entry = start; entry != end; entry = vstruct_next(entry))
+ memset(entry, 0, sizeof(*entry));
}
static int journal_validate_key(struct bch_fs *c, struct jset *j,
break;
}
- switch (JOURNAL_ENTRY_TYPE(entry)) {
+ switch (entry->type) {
case JOURNAL_ENTRY_BTREE_KEYS:
vstruct_for_each(entry, k) {
ret = journal_validate_key(c, j, entry, k,
break;
default:
- journal_entry_err(c, "invalid journal entry type %llu",
- JOURNAL_ENTRY_TYPE(entry));
+ journal_entry_err(c, "invalid journal entry type %u",
+ entry->type);
journal_entry_null_range(entry, vstruct_next(entry));
break;
}
*/
list_for_each_entry(bl, &j->seq_blacklist, list)
if (!bl->written) {
- bch2_journal_add_entry(journal_cur_buf(j), &bl->seq, 1,
+ bch2_journal_add_entry_noreservation(journal_cur_buf(j),
JOURNAL_ENTRY_JOURNAL_SEQ_BLACKLISTED,
- 0, 0);
+ 0, 0, &bl->seq, 1);
journal_pin_add_entry(j,
&fifo_peek_back(&j->pin),
if (prev &&
i->btree_id == prev->btree_id &&
i->level == prev->level &&
- JOURNAL_ENTRY_TYPE(i) == JOURNAL_ENTRY_TYPE(prev) &&
- JOURNAL_ENTRY_TYPE(i) == JOURNAL_ENTRY_BTREE_KEYS &&
+ i->type == prev->type &&
+ i->type == JOURNAL_ENTRY_BTREE_KEYS &&
le16_to_cpu(prev->u64s) + u64s <= U16_MAX) {
memmove_u64s_down(vstruct_next(prev),
i->_data,
closure_return_with_destructor(cl, journal_write_done);
}
- bch2_check_mark_super(c, bkey_i_to_s_c_extent(&j->key),
- BCH_DATA_JOURNAL);
+ if (bch2_check_mark_super(c, bkey_i_to_s_c_extent(&j->key),
+ BCH_DATA_JOURNAL))
+ goto err;
/*
* XXX: we really should just disable the entire journal in nochanges
struct jset_entry *entry, unsigned type)
{
while (entry < vstruct_last(jset)) {
- if (JOURNAL_ENTRY_TYPE(entry) == type)
+ if (entry->type == type)
return entry;
entry = vstruct_next(entry);
s->buf1_count += s->idx == 1;
}
-static inline void bch2_journal_set_has_inode(struct journal_buf *buf, u64 inum)
+static inline void bch2_journal_set_has_inode(struct journal *j,
+ struct journal_res *res,
+ u64 inum)
{
+ struct journal_buf *buf = &j->buf[res->idx];
+
set_bit(hash_64(inum, ilog2(sizeof(buf->has_inode) * 8)), buf->has_inode);
}
}
static inline void bch2_journal_add_entry_at(struct journal_buf *buf,
- const void *data, size_t u64s,
+ unsigned offset,
unsigned type, enum btree_id id,
- unsigned level, unsigned offset)
+ unsigned level,
+ const void *data, size_t u64s)
{
struct jset_entry *entry = vstruct_idx(buf->data, offset);
- entry->u64s = cpu_to_le16(u64s);
+ memset(entry, 0, sizeof(*entry));
+ entry->u64s = cpu_to_le16(u64s);
entry->btree_id = id;
- entry->level = level;
- entry->flags = 0;
- SET_JOURNAL_ENTRY_TYPE(entry, type);
+ entry->level = level;
+ entry->type = type;
memcpy_u64s(entry->_data, data, u64s);
}
-static inline void bch2_journal_add_keys(struct journal *j, struct journal_res *res,
- enum btree_id id, const struct bkey_i *k)
+static inline void bch2_journal_add_entry(struct journal *j, struct journal_res *res,
+ unsigned type, enum btree_id id,
+ unsigned level,
+ const void *data, unsigned u64s)
{
struct journal_buf *buf = &j->buf[res->idx];
- unsigned actual = jset_u64s(k->k.u64s);
+ unsigned actual = jset_u64s(u64s);
EBUG_ON(!res->ref);
BUG_ON(actual > res->u64s);
- bch2_journal_set_has_inode(buf, k->k.p.inode);
-
- bch2_journal_add_entry_at(buf, k, k->k.u64s,
- JOURNAL_ENTRY_BTREE_KEYS, id,
- 0, res->offset);
-
+ bch2_journal_add_entry_at(buf, res->offset, type,
+ id, level, data, u64s);
res->offset += actual;
res->u64s -= actual;
}
+static inline void bch2_journal_add_keys(struct journal *j, struct journal_res *res,
+ enum btree_id id, const struct bkey_i *k)
+{
+ bch2_journal_add_entry(j, res, JOURNAL_ENTRY_BTREE_KEYS,
+ id, 0, k, k->k.u64s);
+}
+
void bch2_journal_buf_put_slowpath(struct journal *, bool);
static inline void bch2_journal_buf_put(struct journal *j, unsigned idx,
lock_release(&j->res_map, 0, _RET_IP_);
- while (res->u64s) {
- bch2_journal_add_entry_at(&j->buf[res->idx], NULL, 0,
- JOURNAL_ENTRY_BTREE_KEYS,
- 0, 0, res->offset);
- res->offset += jset_u64s(0);
- res->u64s -= jset_u64s(0);
- }
+ while (res->u64s)
+ bch2_journal_add_entry(j, res,
+ JOURNAL_ENTRY_BTREE_KEYS,
+ 0, 0, NULL, 0);
bch2_journal_buf_put(j, res->idx, false);
seen_key_count++;
continue;
next:
- if (bkey_extent_is_data(k.k))
- bch2_check_mark_super(c, bkey_s_c_to_extent(k),
- BCH_DATA_USER);
+ if (bkey_extent_is_data(k.k)) {
+ ret = bch2_check_mark_super(c, bkey_s_c_to_extent(k),
+ BCH_DATA_USER);
+ if (ret)
+ break;
+ }
bch2_btree_iter_advance_pos(&iter);
bch2_btree_iter_cond_resched(&iter);
*/
continue;
advance:
- if (bkey_extent_is_data(k.k))
- bch2_check_mark_super(c, bkey_s_c_to_extent(k),
- BCH_DATA_USER);
+ if (bkey_extent_is_data(k.k)) {
+ ret = bch2_check_mark_super(c, bkey_s_c_to_extent(k),
+ BCH_DATA_USER);
+ if (ret)
+ break;
+ }
bch2_btree_iter_advance_pos(&iter);
}
(move_ptr && move_ptr->cached))
flags |= BCH_WRITE_CACHED;
- bch2_write_op_init(&m->op, c, &m->wbio,
- (struct disk_reservation) { 0 },
- wp,
- bkey_start_pos(k.k),
- NULL, flags);
+ bch2_write_op_init(&m->op, c, (struct disk_reservation) { 0 }, wp,
+ bkey_start_pos(k.k), NULL, flags);
if (m->move)
m->op.alloc_reserve = RESERVE_MOVINGGC;
atomic_sub(io->write.key.k.size, &ctxt->sectors_in_flight);
wake_up(&ctxt->wait);
- bio_for_each_segment_all(bv, &io->write.wbio.bio, i)
+ bio_for_each_segment_all(bv, &io->write.op.wbio.bio, i)
if (bv->bv_page)
__free_page(bv->bv_page);
return -ENOMEM;
}
- migrate_bio_init(io, &io->write.wbio.bio, k.k->size);
- bio_get(&io->write.wbio.bio);
- io->write.wbio.bio.bi_iter.bi_sector = bkey_start_offset(k.k);
+ migrate_bio_init(io, &io->write.op.wbio.bio, k.k->size);
bch2_migrate_write_init(c, &io->write, wp, k, move_ptr, 0);
bool move;
struct bch_extent_ptr move_ptr;
struct bch_write_op op;
- struct bch_write_bio wbio;
};
void bch2_migrate_write_init(struct bch_fs *,
/* replica information: */
+static inline struct bch_replicas_cpu_entry *
+cpu_replicas_entry(struct bch_replicas_cpu *r, unsigned i)
+{
+ return (void *) r->entries + r->entry_size * i;
+}
+
static inline struct bch_replicas_entry *
replicas_entry_next(struct bch_replicas_entry *i)
{
(void *) (_i) < vstruct_end(&(_r)->field) && (_i)->data_type;\
(_i) = replicas_entry_next(_i))
+static inline bool replicas_test_dev(struct bch_replicas_cpu_entry *e,
+ unsigned dev)
+{
+ return (e->devs[dev >> 3] & (1 << (dev & 7))) != 0;
+}
+
+static inline void replicas_set_dev(struct bch_replicas_cpu_entry *e,
+ unsigned dev)
+{
+ e->devs[dev >> 3] |= 1 << (dev & 7);
+}
+
+static inline unsigned replicas_dev_slots(struct bch_replicas_cpu *r)
+{
+ return (r->entry_size -
+ offsetof(struct bch_replicas_cpu_entry, devs)) * 8;
+}
+
static void bch2_sb_replicas_nr_entries(struct bch_sb_field_replicas *r,
unsigned *nr,
unsigned *bytes,
return 0;
}
+static void bkey_to_replicas(struct bkey_s_c_extent e,
+ enum bch_data_types data_type,
+ struct bch_replicas_cpu_entry *r,
+ unsigned *max_dev)
+{
+ const struct bch_extent_ptr *ptr;
+
+ BUG_ON(!data_type ||
+ data_type == BCH_DATA_SB ||
+ data_type >= BCH_DATA_NR);
+
+ memset(r, 0, sizeof(*r));
+ r->data_type = data_type;
+
+ *max_dev = 0;
+
+ extent_for_each_ptr(e, ptr)
+ if (!ptr->cached) {
+ *max_dev = max_t(unsigned, *max_dev, ptr->dev);
+ replicas_set_dev(r, ptr->dev);
+ }
+}
+
/*
* for when gc of replica information is in progress:
*/
struct bkey_s_c_extent e,
enum bch_data_types data_type)
{
- const struct bch_extent_ptr *ptr;
- struct bch_replicas_cpu_entry *new_e;
+ struct bch_replicas_cpu_entry new_e;
struct bch_replicas_cpu *new;
- unsigned i, nr, entry_size, max_dev = 0;
+ unsigned i, nr, entry_size, max_dev;
- extent_for_each_ptr(e, ptr)
- if (!ptr->cached)
- max_dev = max_t(unsigned, max_dev, ptr->dev);
+ bkey_to_replicas(e, data_type, &new_e, &max_dev);
entry_size = offsetof(struct bch_replicas_cpu_entry, devs) +
DIV_ROUND_UP(max_dev + 1, 8);
cpu_replicas_entry(gc_r, i),
gc_r->entry_size);
- new_e = cpu_replicas_entry(new, nr - 1);
- new_e->data_type = data_type;
-
- extent_for_each_ptr(e, ptr)
- if (!ptr->cached)
- replicas_set_dev(new_e, ptr->dev);
+ memcpy(cpu_replicas_entry(new, nr - 1),
+ &new_e,
+ new->entry_size);
eytzinger0_sort(new->entries,
new->nr,
return 0;
}
-int bch2_check_mark_super_slowpath(struct bch_fs *c, struct bkey_s_c_extent e,
- enum bch_data_types data_type)
+static bool replicas_has_extent(struct bch_replicas_cpu *r,
+ struct bkey_s_c_extent e,
+ enum bch_data_types data_type)
+{
+ struct bch_replicas_cpu_entry search;
+ unsigned max_dev;
+
+ bkey_to_replicas(e, data_type, &search, &max_dev);
+
+ return max_dev < replicas_dev_slots(r) &&
+ eytzinger0_find(r->entries, r->nr,
+ r->entry_size,
+ memcmp, &search) < r->nr;
+}
+
+bool bch2_sb_has_replicas(struct bch_fs *c, struct bkey_s_c_extent e,
+ enum bch_data_types data_type)
+{
+ bool ret;
+
+ rcu_read_lock();
+ ret = replicas_has_extent(rcu_dereference(c->replicas),
+ e, data_type);
+ rcu_read_unlock();
+
+ return ret;
+}
+
+noinline
+static int bch2_check_mark_super_slowpath(struct bch_fs *c,
+ struct bkey_s_c_extent e,
+ enum bch_data_types data_type)
{
struct bch_replicas_cpu *gc_r;
const struct bch_extent_ptr *ptr;
return ret;
}
+int bch2_check_mark_super(struct bch_fs *c, struct bkey_s_c_extent e,
+ enum bch_data_types data_type)
+{
+ struct bch_replicas_cpu *gc_r;
+ bool marked;
+
+ rcu_read_lock();
+ marked = replicas_has_extent(rcu_dereference(c->replicas),
+ e, data_type) &&
+ (!(gc_r = rcu_dereference(c->replicas_gc)) ||
+ replicas_has_extent(gc_r, e, data_type));
+ rcu_read_unlock();
+
+ if (marked)
+ return 0;
+
+ return bch2_check_mark_super_slowpath(c, e, data_type);
+}
+
struct replicas_status __bch2_replicas_status(struct bch_fs *c,
struct bch_dev *dev_to_offline)
{
struct bch_opts, const char *);
void bch2_write_super(struct bch_fs *);
-static inline bool replicas_test_dev(struct bch_replicas_cpu_entry *e,
- unsigned dev)
-{
- return (e->devs[dev >> 3] & (1 << (dev & 7))) != 0;
-}
-
-static inline void replicas_set_dev(struct bch_replicas_cpu_entry *e,
- unsigned dev)
-{
- e->devs[dev >> 3] |= 1 << (dev & 7);
-}
-
-static inline unsigned replicas_dev_slots(struct bch_replicas_cpu *r)
-{
- return (r->entry_size -
- offsetof(struct bch_replicas_cpu_entry, devs)) * 8;
-}
-
-static inline struct bch_replicas_cpu_entry *
-cpu_replicas_entry(struct bch_replicas_cpu *r, unsigned i)
-{
- return (void *) r->entries + r->entry_size * i;
-}
-
-int bch2_check_mark_super_slowpath(struct bch_fs *, struct bkey_s_c_extent,
- enum bch_data_types);
-
-static inline bool replicas_has_extent(struct bch_replicas_cpu *r,
- struct bkey_s_c_extent e,
- enum bch_data_types data_type)
-{
- const struct bch_extent_ptr *ptr;
- struct bch_replicas_cpu_entry search = {
- .data_type = data_type,
- };
- unsigned max_dev = 0;
-
- BUG_ON(!data_type ||
- data_type == BCH_DATA_SB ||
- data_type >= BCH_DATA_NR);
-
- extent_for_each_ptr(e, ptr)
- if (!ptr->cached) {
- max_dev = max_t(unsigned, max_dev, ptr->dev);
- replicas_set_dev(&search, ptr->dev);
- }
-
- return max_dev < replicas_dev_slots(r) &&
- eytzinger0_find(r->entries, r->nr,
- r->entry_size,
- memcmp, &search) < r->nr;
-}
-
-static inline bool bch2_sb_has_replicas(struct bch_fs *c,
- struct bkey_s_c_extent e,
- enum bch_data_types data_type)
-{
- bool ret;
-
- rcu_read_lock();
- ret = replicas_has_extent(rcu_dereference(c->replicas),
- e, data_type);
- rcu_read_unlock();
-
- return ret;
-}
-
-static inline int bch2_check_mark_super(struct bch_fs *c,
- struct bkey_s_c_extent e,
- enum bch_data_types data_type)
-{
- struct bch_replicas_cpu *gc_r;
- bool marked;
-
- rcu_read_lock();
- marked = replicas_has_extent(rcu_dereference(c->replicas),
- e, data_type) &&
- (!(gc_r = rcu_dereference(c->replicas_gc)) ||
- replicas_has_extent(gc_r, e, data_type));
- rcu_read_unlock();
-
- if (marked)
- return 0;
-
- return bch2_check_mark_super_slowpath(c, e, data_type);
-}
+bool bch2_sb_has_replicas(struct bch_fs *, struct bkey_s_c_extent,
+ enum bch_data_types);
+int bch2_check_mark_super(struct bch_fs *, struct bkey_s_c_extent,
+ enum bch_data_types);
struct replicas_status {
struct {
mutex_init(&c->btree_interior_update_lock);
mutex_init(&c->bio_bounce_pages_lock);
+ mutex_init(&c->zlib_workspace_lock);
+
bio_list_init(&c->read_retry_list);
spin_lock_init(&c->read_retry_lock);
INIT_WORK(&c->read_retry_work, bch2_read_retry_work);
- mutex_init(&c->zlib_workspace_lock);
+
+ bio_list_init(&c->btree_write_error_list);
+ spin_lock_init(&c->btree_write_error_lock);
+ INIT_WORK(&c->btree_write_error_work, bch2_btree_write_error_work);
INIT_LIST_HEAD(&c->fsck_errors);
mutex_init(&c->fsck_error_lock);
PAGE_SECTORS, 0) ||
!(c->usage_percpu = alloc_percpu(struct bch_fs_usage)) ||
lg_lock_init(&c->usage_lock) ||
- mempool_init_page_pool(&c->btree_bounce_pool, 1,
- ilog2(btree_pages(c))) ||
+ mempool_init_vp_pool(&c->btree_bounce_pool, 1, btree_bytes(c)) ||
bdi_setup_and_register(&c->bdi, "bcachefs") ||
bch2_io_clock_init(&c->io_clock[READ]) ||
bch2_io_clock_init(&c->io_clock[WRITE]) ||
}
}
-static bool bch2_fs_may_start(struct bch_fs *c, int flags)
+static bool bch2_fs_may_start(struct bch_fs *c)
{
struct replicas_status s;
struct bch_sb_field_members *mi;
- unsigned i;
+ unsigned i, flags = c->opts.degraded
+ ? BCH_FORCE_IF_DEGRADED
+ : 0;
if (!c->opts.degraded) {
mutex_lock(&c->sb_lock);
mutex_unlock(&c->sb_lock);
err = "insufficient devices";
- if (!bch2_fs_may_start(c, 0))
+ if (!bch2_fs_may_start(c))
goto err;
if (!c->opts.nostart) {
}
mutex_unlock(&c->sb_lock);
- if (!c->opts.nostart && bch2_fs_may_start(c, 0)) {
+ if (!c->opts.nostart && bch2_fs_may_start(c)) {
err = __bch2_fs_start(c);
if (err)
goto err;
}
}
}
+
+void mempool_free_vp(void *element, void *pool_data)
+{
+ size_t size = (size_t) pool_data;
+
+ vpfree(element, size);
+}
+
+void *mempool_alloc_vp(gfp_t gfp_mask, void *pool_data)
+{
+ size_t size = (size_t) pool_data;
+
+ return vpmalloc(size, gfp_mask);
+}
(__builtin_types_compatible_p(typeof(_val), _type) || \
__builtin_types_compatible_p(typeof(_val), const _type))
-static inline void kvpfree(void *p, size_t size)
+static inline void vpfree(void *p, size_t size)
{
- if (size < PAGE_SIZE)
- kfree(p);
- else if (is_vmalloc_addr(p))
+ if (is_vmalloc_addr(p))
vfree(p);
else
free_pages((unsigned long) p, get_order(size));
+}
+static inline void *vpmalloc(size_t size, gfp_t gfp_mask)
+{
+ return (void *) __get_free_pages(gfp_mask|__GFP_NOWARN,
+ get_order(size)) ?:
+ __vmalloc(size, gfp_mask, PAGE_KERNEL);
+}
+
+static inline void kvpfree(void *p, size_t size)
+{
+ if (size < PAGE_SIZE)
+ kfree(p);
+ else
+ vpfree(p, size);
}
static inline void *kvpmalloc(size_t size, gfp_t gfp_mask)
{
- return size < PAGE_SIZE ? kmalloc(size, gfp_mask)
- : (void *) __get_free_pages(gfp_mask|__GFP_NOWARN,
- get_order(size))
- ?: __vmalloc(size, gfp_mask, PAGE_KERNEL);
+ return size < PAGE_SIZE
+ ? kmalloc(size, gfp_mask)
+ : vpmalloc(size, gfp_mask);
+}
+
+void mempool_free_vp(void *element, void *pool_data);
+void *mempool_alloc_vp(gfp_t gfp_mask, void *pool_data);
+
+static inline int mempool_init_vp_pool(mempool_t *pool, int min_nr, size_t size)
+{
+ return mempool_init(pool, min_nr, mempool_alloc_vp,
+ mempool_free_vp, (void *) size);
}
#define HEAP(type) \
#include <string.h>
+#include <sys/mman.h>
#include <linux/math64.h>
#include <linux/printk.h>
{
struct task_struct *p = malloc(sizeof(*p));
+ mlockall(MCL_CURRENT|MCL_FUTURE);
+
memset(p, 0, sizeof(*p));
p->state = TASK_RUNNING;
projects / bcachefs-tools-debian / commitdiff