-7e42539c80470cb655bbc46cd0f144de6c644523
+b91a514413ecdd15e0f9d8290761d24663a93425
closure_init_stack(&cl);
bio_init(&o.op.wbio.bio, o.bv, ARRAY_SIZE(o.bv));
- o.op.wbio.bio.bi_iter.bi_size = len;
- bch2_bio_map(&o.op.wbio.bio, buf);
+ bch2_bio_map(&o.op.wbio.bio, buf, len);
bch2_write_op_init(&o.op, c, bch2_opts_to_inode_opts(c->opts));
o.op.write_point = writepoint_hashed(0);
#define __bio_kunmap_atomic(addr) kunmap_atomic(addr)
-#define bio_for_each_segment_all(bvl, bio, i) \
- for (i = 0, bvl = (bio)->bi_io_vec; i < (bio)->bi_vcnt; i++, bvl++)
+struct bvec_iter_all {
+ unsigned done;
+};
+
+#define bio_for_each_segment_all(bvl, bio, i, iter) \
+ for (i = 0, bvl = (bio)->bi_io_vec, iter = (struct bvec_iter_all) { 0 }; \
+ i < (bio)->bi_vcnt; i++, bvl++)
static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
unsigned bytes)
#define bio_for_each_segment(bvl, bio, iter) \
__bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
+#define __bio_for_each_bvec(bvl, bio, iter, start) \
+ __bio_for_each_segment(bvl, bio, iter, start)
+
#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
static inline unsigned bio_segments(struct bio *bio)
extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
extern void bio_put(struct bio *);
+int bio_add_page(struct bio *, struct page *, unsigned, unsigned);
+
extern void __bio_clone_fast(struct bio *, struct bio *);
extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs);
--- /dev/null
+#ifndef _LINUX_SCHED_MM_H
+#define _LINUX_SCHED_MM_H
+
+#define PF_MEMALLOC_NOFS 0
+
+static inline unsigned int memalloc_nofs_save(void)
+{
+ unsigned int flags = current->flags & PF_MEMALLOC_NOFS;
+ current->flags |= PF_MEMALLOC_NOFS;
+ return flags;
+}
+
+static inline void memalloc_nofs_restore(unsigned int flags)
+{
+ current->flags = (current->flags & ~PF_MEMALLOC_NOFS) | flags;
+}
+
+#endif /* _LINUX_SCHED_MM_H */
+/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM bcachefs
+// SPDX-License-Identifier: GPL-2.0
#ifdef CONFIG_BCACHEFS_POSIX_ACL
#include "bcachefs.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_ACL_H
#define _BCACHEFS_ACL_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_background.h"
#include "alloc_foreground.h"
old_u = bch2_alloc_unpack(k);
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
g = bucket(ca, b);
m = READ_ONCE(g->mark);
new_u = alloc_mem_to_key(g, m);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
if (!m.dirty)
continue;
b = ca->alloc_heap.data[0].bucket;
/* first, put on free_inc and mark as owned by allocator: */
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
spin_lock(&c->freelist_lock);
verify_not_on_freelist(c, ca, b);
bch2_mark_alloc_bucket(c, ca, b, true, gc_pos_alloc(c, NULL), 0);
spin_unlock(&c->freelist_lock);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
BUG_ON(BKEY_ALLOC_VAL_U64s_MAX > 8);
* we have to trust the in memory bucket @m, not the version in the
* btree:
*/
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
g = bucket(ca, b);
m = READ_ONCE(g->mark);
u = alloc_mem_to_key(g, m);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
invalidating_cached_data = m.cached_sectors != 0;
size_t b2;
/* remove from free_inc: */
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
spin_lock(&c->freelist_lock);
bch2_mark_alloc_bucket(c, ca, b, false,
BUG_ON(b != b2);
spin_unlock(&c->freelist_lock);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
}
return ret;
{
struct bucket_mark m;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
spin_lock(&c->freelist_lock);
bch2_invalidate_bucket(c, ca, bucket, &m);
bucket_io_clock_reset(c, ca, bucket, READ);
bucket_io_clock_reset(c, ca, bucket, WRITE);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
*flush_seq = max(*flush_seq, bucket_journal_seq(c, m));
test_bit(bu, ca->buckets_nouse)))
continue;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
bch2_mark_alloc_bucket(c, ca, bu, true,
gc_pos_alloc(c, NULL), 0);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
fifo_push(&ca->free_inc, bu);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_ALLOC_BACKGROUND_H
#define _BCACHEFS_ALLOC_BACKGROUND_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Primary bucket allocation code
*
return;
}
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
spin_lock(&ob->lock);
bch2_mark_alloc_bucket(c, ca, PTR_BUCKET_NR(ca, &ob->ptr),
ob->type = 0;
spin_unlock(&ob->lock);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
spin_lock(&c->freelist_lock);
ob->freelist = c->open_buckets_freelist;
#define BUCKET_MAY_ALLOC_PARTIAL (1 << 0)
#define BUCKET_ALLOC_USE_DURABILITY (1 << 1)
+static void add_new_bucket(struct bch_fs *c,
+ struct open_buckets *ptrs,
+ struct bch_devs_mask *devs_may_alloc,
+ unsigned *nr_effective,
+ bool *have_cache,
+ unsigned flags,
+ struct open_bucket *ob)
+{
+ unsigned durability =
+ bch_dev_bkey_exists(c, ob->ptr.dev)->mi.durability;
+
+ __clear_bit(ob->ptr.dev, devs_may_alloc->d);
+ *nr_effective += (flags & BUCKET_ALLOC_USE_DURABILITY)
+ ? durability : 1;
+ *have_cache |= !durability;
+
+ ob_push(c, ptrs, ob);
+}
+
static int bch2_bucket_alloc_set(struct bch_fs *c,
struct open_buckets *ptrs,
struct dev_stripe_state *stripe,
bch2_dev_alloc_list(c, stripe, devs_may_alloc);
struct bch_dev *ca;
bool alloc_failure = false;
- unsigned i, durability;
+ unsigned i;
BUG_ON(*nr_effective >= nr_replicas);
continue;
}
- durability = (flags & BUCKET_ALLOC_USE_DURABILITY)
- ? ca->mi.durability : 1;
-
- __clear_bit(ca->dev_idx, devs_may_alloc->d);
- *nr_effective += durability;
- *have_cache |= !durability;
-
- ob_push(c, ptrs, ob);
+ add_new_bucket(c, ptrs, devs_may_alloc,
+ nr_effective, have_cache, flags, ob);
bch2_dev_stripe_increment(c, ca, stripe);
open_bucket_for_each(c, &h->blocks, ob, i)
__clear_bit(ob->ptr.dev, devs.d);
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
rcu_read_lock();
if (h->parity.nr < h->redundancy) {
}
rcu_read_unlock();
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return bch2_ec_stripe_new_alloc(c, h);
err:
rcu_read_unlock();
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return -1;
}
unsigned erasure_code,
unsigned nr_replicas,
unsigned *nr_effective,
- bool *have_cache)
+ bool *have_cache,
+ unsigned flags)
{
struct dev_alloc_list devs_sorted;
struct ec_stripe_head *h;
ob->ec_idx = ec_idx;
ob->ec = h->s;
- __clear_bit(ob->ptr.dev, devs_may_alloc->d);
- *nr_effective += ca->mi.durability;
- *have_cache |= !ca->mi.durability;
-
- ob_push(c, ptrs, ob);
+ add_new_bucket(c, ptrs, devs_may_alloc,
+ nr_effective, have_cache, flags, ob);
atomic_inc(&h->s->pin);
out_put_head:
bch2_ec_stripe_head_put(h);
unsigned nr_replicas,
unsigned *nr_effective,
bool *have_cache,
+ unsigned flags,
bool need_ec)
{
struct open_buckets ptrs_skip = { .nr = 0 };
(ca->mi.durability ||
(wp->type == BCH_DATA_USER && !*have_cache)) &&
(ob->ec || !need_ec)) {
- __clear_bit(ob->ptr.dev, devs_may_alloc->d);
- *nr_effective += ca->mi.durability;
- *have_cache |= !ca->mi.durability;
-
- ob_push(c, ptrs, ob);
+ add_new_bucket(c, ptrs, devs_may_alloc,
+ nr_effective, have_cache,
+ flags, ob);
} else {
ob_push(c, &ptrs_skip, ob);
}
unsigned *nr_effective,
bool *have_cache,
enum alloc_reserve reserve,
+ unsigned flags,
struct closure *_cl)
{
struct bch_devs_mask devs;
struct open_bucket *ob;
struct closure *cl = NULL;
- unsigned i, flags = BUCKET_ALLOC_USE_DURABILITY;
+ unsigned i;
int ret;
- if (wp->type == BCH_DATA_USER)
- flags |= BUCKET_MAY_ALLOC_PARTIAL;
-
rcu_read_lock();
devs = target_rw_devs(c, wp->type, target);
rcu_read_unlock();
if (erasure_code) {
get_buckets_from_writepoint(c, ptrs, wp, &devs,
nr_replicas, nr_effective,
- have_cache, true);
+ have_cache, flags, true);
if (*nr_effective >= nr_replicas)
return 0;
bucket_alloc_from_stripe(c, ptrs, wp, &devs,
target, erasure_code,
nr_replicas, nr_effective,
- have_cache);
+ have_cache, flags);
if (*nr_effective >= nr_replicas)
return 0;
}
get_buckets_from_writepoint(c, ptrs, wp, &devs,
nr_replicas, nr_effective,
- have_cache, false);
+ have_cache, flags, false);
if (*nr_effective >= nr_replicas)
return 0;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
rcu_read_lock();
retry_blocking:
}
rcu_read_unlock();
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return ret;
}
struct open_bucket *ob;
struct open_buckets ptrs;
unsigned nr_effective, write_points_nr;
+ unsigned ob_flags = 0;
bool have_cache;
int ret, i;
+ if (!(flags & BCH_WRITE_ONLY_SPECIFIED_DEVS))
+ ob_flags |= BUCKET_ALLOC_USE_DURABILITY;
+
BUG_ON(!nr_replicas || !nr_replicas_required);
retry:
ptrs.nr = 0;
wp = writepoint_find(c, write_point.v);
+ if (wp->type == BCH_DATA_USER)
+ ob_flags |= BUCKET_MAY_ALLOC_PARTIAL;
+
/* metadata may not allocate on cache devices: */
if (wp->type != BCH_DATA_USER)
have_cache = true;
ret = open_bucket_add_buckets(c, &ptrs, wp, devs_have,
target, erasure_code,
nr_replicas, &nr_effective,
- &have_cache, reserve, cl);
+ &have_cache, reserve,
+ ob_flags, cl);
} else {
ret = open_bucket_add_buckets(c, &ptrs, wp, devs_have,
target, erasure_code,
nr_replicas, &nr_effective,
- &have_cache, reserve, NULL);
+ &have_cache, reserve,
+ ob_flags, NULL);
if (!ret)
goto alloc_done;
ret = open_bucket_add_buckets(c, &ptrs, wp, devs_have,
0, erasure_code,
nr_replicas, &nr_effective,
- &have_cache, reserve, cl);
+ &have_cache, reserve,
+ ob_flags, cl);
}
alloc_done:
BUG_ON(!ret && nr_effective < nr_replicas);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_ALLOC_FOREGROUND_H
#define _BCACHEFS_ALLOC_FOREGROUND_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_ALLOC_TYPES_H
#define _BCACHEFS_ALLOC_TYPES_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_H
#define _BCACHEFS_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FORMAT_H
#define _BCACHEFS_FORMAT_H
BCH_ALLOC_FIELDS(), sizeof(u64));
#undef x
+#define BKEY_ALLOC_U64s_MAX (BKEY_U64s + BKEY_ALLOC_VAL_U64s_MAX)
+
/* Quotas: */
enum quota_types {
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_IOCTL_H
#define _BCACHEFS_IOCTL_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BKEY_H
#define _BCACHEFS_BKEY_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_methods.h"
if (k.k->u64s < BKEY_U64s)
return "u64s too small";
+ if ((btree_node_type_is_extents(type) ||
+ type == BKEY_TYPE_BTREE) &&
+ bkey_val_u64s(k.k) > BKEY_EXTENT_VAL_U64s_MAX)
+ return "value too big";
+
if (btree_node_type_is_extents(type)) {
if ((k.k->size == 0) != bkey_deleted(k.k))
return "bad size field";
+
+ if (k.k->size > k.k->p.offset)
+ return "size greater than offset";
} else {
if (k.k->size)
return "nonzero size field";
}
enum merge_result bch2_bkey_merge(struct bch_fs *c,
- struct bkey_i *l, struct bkey_i *r)
+ struct bkey_s l, struct bkey_s r)
{
- const struct bkey_ops *ops = &bch2_bkey_ops[l->k.type];
+ const struct bkey_ops *ops = &bch2_bkey_ops[l.k->type];
enum merge_result ret;
if (key_merging_disabled(c) ||
!ops->key_merge ||
- l->k.type != r->k.type ||
- bversion_cmp(l->k.version, r->k.version) ||
- bkey_cmp(l->k.p, bkey_start_pos(&r->k)))
+ l.k->type != r.k->type ||
+ bversion_cmp(l.k->version, r.k->version) ||
+ bkey_cmp(l.k->p, bkey_start_pos(r.k)))
return BCH_MERGE_NOMERGE;
ret = ops->key_merge(c, l, r);
if (ret != BCH_MERGE_NOMERGE)
- l->k.needs_whiteout |= r->k.needs_whiteout;
+ l.k->needs_whiteout |= r.k->needs_whiteout;
return ret;
}
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BKEY_METHODS_H
#define _BCACHEFS_BKEY_METHODS_H
void (*swab)(const struct bkey_format *, struct bkey_packed *);
bool (*key_normalize)(struct bch_fs *, struct bkey_s);
enum merge_result (*key_merge)(struct bch_fs *,
- struct bkey_i *, struct bkey_i *);
+ struct bkey_s, struct bkey_s);
};
const char *bch2_bkey_val_invalid(struct bch_fs *, struct bkey_s_c);
bool bch2_bkey_normalize(struct bch_fs *, struct bkey_s);
enum merge_result bch2_bkey_merge(struct bch_fs *,
- struct bkey_i *, struct bkey_i *);
+ struct bkey_s, struct bkey_s);
void bch2_bkey_renumber(enum btree_node_type, struct bkey_packed *, int);
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_sort.h"
#include "bset.h"
heap_sift_down(iter, i - iter->data, extent_sort_cmp, NULL);
}
+static void extent_sort_advance_prev(struct bkey_format *f,
+ struct btree_nr_keys *nr,
+ struct bkey_packed *start,
+ struct bkey_packed **prev)
+{
+ if (*prev) {
+ bch2_bkey_pack(*prev, (void *) *prev, f);
+
+ btree_keys_account_key_add(nr, 0, *prev);
+ *prev = bkey_next(*prev);
+ } else {
+ *prev = start;
+ }
+}
+
static void extent_sort_append(struct bch_fs *c,
- struct btree *b,
+ struct bkey_format *f,
struct btree_nr_keys *nr,
struct bkey_packed *start,
struct bkey_packed **prev,
- struct bkey_packed *k)
+ struct bkey_s k)
{
- struct bkey_format *f = &b->format;
- BKEY_PADDED(k) tmp;
-
- if (bkey_whiteout(k))
+ if (bkey_whiteout(k.k))
return;
- bch2_bkey_unpack(b, &tmp.k, k);
+ /*
+ * prev is always unpacked, for key merging - until right before we
+ * advance it:
+ */
if (*prev &&
- bch2_bkey_merge(c, (void *) *prev, &tmp.k))
+ bch2_bkey_merge(c, bkey_i_to_s((void *) *prev), k) ==
+ BCH_MERGE_MERGE)
return;
- if (*prev) {
- bch2_bkey_pack(*prev, (void *) *prev, f);
-
- btree_keys_account_key_add(nr, 0, *prev);
- *prev = bkey_next(*prev);
- } else {
- *prev = start;
- }
+ extent_sort_advance_prev(f, nr, start, prev);
- bkey_copy(*prev, &tmp.k);
+ bkey_reassemble((void *) *prev, k.s_c);
}
struct btree_nr_keys bch2_extent_sort_fix_overlapping(struct bch_fs *c,
{
struct bkey_format *f = &b->format;
struct btree_node_iter_set *_l = iter->data, *_r;
- struct bkey_packed *prev = NULL, *out, *lk, *rk;
+ struct bkey_packed *prev = NULL, *lk, *rk;
struct bkey l_unpacked, r_unpacked;
struct bkey_s l, r;
struct btree_nr_keys nr;
while (!bch2_btree_node_iter_large_end(iter)) {
lk = __btree_node_offset_to_key(b, _l->k);
+ l = __bkey_disassemble(b, lk, &l_unpacked);
if (iter->used == 1) {
- extent_sort_append(c, b, &nr, dst->start, &prev, lk);
+ extent_sort_append(c, f, &nr, dst->start, &prev, l);
extent_sort_next(iter, b, _l);
continue;
}
_r++;
rk = __btree_node_offset_to_key(b, _r->k);
-
- l = __bkey_disassemble(b, lk, &l_unpacked);
r = __bkey_disassemble(b, rk, &r_unpacked);
/* If current key and next key don't overlap, just append */
if (bkey_cmp(l.k->p, bkey_start_pos(r.k)) <= 0) {
- extent_sort_append(c, b, &nr, dst->start, &prev, lk);
+ extent_sort_append(c, f, &nr, dst->start, &prev, l);
extent_sort_next(iter, b, _l);
continue;
}
extent_sort_sift(iter, b, 0);
- extent_sort_append(c, b, &nr, dst->start, &prev,
- bkey_to_packed(&tmp.k));
+ extent_sort_append(c, f, &nr, dst->start,
+ &prev, bkey_i_to_s(&tmp.k));
} else {
bch2_cut_back(bkey_start_pos(r.k), l.k);
extent_save(b, lk, l.k);
}
}
- if (prev) {
- bch2_bkey_pack(prev, (void *) prev, f);
- btree_keys_account_key_add(&nr, 0, prev);
- out = bkey_next(prev);
- } else {
- out = dst->start;
- }
+ extent_sort_advance_prev(f, &nr, dst->start, &prev);
- dst->u64s = cpu_to_le16((u64 *) out - dst->_data);
+ dst->u64s = cpu_to_le16((u64 *) prev - dst->_data);
return nr;
}
struct bkey_format *out_f,
bool filter_whiteouts)
{
- struct bkey_packed *k, *prev = NULL, *out;
+ struct bkey_packed *prev = NULL, *k_packed, *next;
+ struct bkey k_unpacked;
+ struct bkey_s k;
struct btree_nr_keys nr;
- BKEY_PADDED(k) tmp;
memset(&nr, 0, sizeof(nr));
- while ((k = bch2_btree_node_iter_next_all(iter, src))) {
- if (filter_whiteouts && bkey_whiteout(k))
- continue;
-
+ next = bch2_btree_node_iter_next_all(iter, src);
+ while ((k_packed = next)) {
/*
- * The filter might modify pointers, so we have to unpack the
- * key and values to &tmp.k:
+ * The filter might modify the size of @k's value, so advance
+ * the iterator first:
*/
- bch2_bkey_unpack(src, &tmp.k, k);
+ next = bch2_btree_node_iter_next_all(iter, src);
- if (filter_whiteouts &&
- bch2_bkey_normalize(c, bkey_i_to_s(&tmp.k)))
+ if (filter_whiteouts && bkey_whiteout(k_packed))
continue;
- /* prev is always unpacked, for key merging: */
+ k = __bkey_disassemble(src, k_packed, &k_unpacked);
- if (prev &&
- bch2_bkey_merge(c, (void *) prev, &tmp.k) ==
- BCH_MERGE_MERGE)
+ if (filter_whiteouts &&
+ bch2_bkey_normalize(c, k))
continue;
- /*
- * the current key becomes the new prev: advance prev, then
- * copy the current key - but first pack prev (in place):
- */
- if (prev) {
- bch2_bkey_pack(prev, (void *) prev, out_f);
-
- btree_keys_account_key_add(&nr, 0, prev);
- prev = bkey_next(prev);
- } else {
- prev = vstruct_last(dst);
- }
-
- bkey_copy(prev, &tmp.k);
+ extent_sort_append(c, out_f, &nr, vstruct_last(dst), &prev, k);
}
- if (prev) {
- bch2_bkey_pack(prev, (void *) prev, out_f);
- btree_keys_account_key_add(&nr, 0, prev);
- out = bkey_next(prev);
- } else {
- out = vstruct_last(dst);
- }
+ extent_sort_advance_prev(out_f, &nr, vstruct_last(dst), &prev);
- dst->u64s = cpu_to_le16((u64 *) out - dst->_data);
+ dst->u64s = cpu_to_le16((u64 *) prev - dst->_data);
return nr;
}
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BKEY_SORT_H
#define _BCACHEFS_BKEY_SORT_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Code for working with individual keys, and sorted sets of keys with in a
* btree node
#include "util.h"
#include <asm/unaligned.h>
-#include <linux/dynamic_fault.h>
#include <linux/console.h>
#include <linux/random.h>
#include <linux/prefetch.h>
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BSET_H
#define _BCACHEFS_BSET_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "btree_cache.h"
#include "debug.h"
#include <linux/prefetch.h>
+#include <linux/sched/mm.h>
#include <trace/events/bcachefs.h>
const char * const bch2_btree_ids[] = {
struct btree_cache *bc = &c->btree_cache;
struct btree *b;
u64 start_time = local_clock();
+ unsigned flags;
+ flags = memalloc_nofs_save();
mutex_lock(&bc->lock);
/*
list_del_init(&b->list);
mutex_unlock(&bc->lock);
+ memalloc_nofs_restore(flags);
out:
b->flags = 0;
b->written = 0;
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_CACHE_H
#define _BCACHEFS_BTREE_CACHE_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2010 Kent Overstreet <kent.overstreet@gmail.com>
* Copyright (C) 2014 Datera Inc.
for_each_btree_key(&trans, iter, id, bkey_start_pos(&insert->k),
BTREE_ITER_SLOTS, k, ret) {
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
ret = bch2_mark_overwrite(&trans, iter, k, insert, NULL,
BCH_BUCKET_MARK_GC|
BCH_BUCKET_MARK_NOATOMIC);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
if (!ret)
break;
*/
if (c) {
lockdep_assert_held(&c->sb_lock);
- percpu_down_read_preempt_disable(&c->mark_lock);
- } else {
- preempt_disable();
+ percpu_down_read(&c->mark_lock);
}
for (i = 0; i < layout->nr_superblocks; i++) {
gc_phase(GC_PHASE_SB), flags);
}
- if (c) {
- percpu_up_read_preempt_enable(&c->mark_lock);
- } else {
- preempt_enable();
- }
+ if (c)
+ percpu_up_read(&c->mark_lock);
}
static void bch2_mark_superblocks(struct bch_fs *c)
size_t i, j, iter;
unsigned ci;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
spin_lock(&c->freelist_lock);
gc_pos_set(c, gc_pos_alloc(c, NULL));
spin_unlock(&ob->lock);
}
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
}
static void bch2_gc_free(struct bch_fs *c)
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_GC_H
#define _BCACHEFS_BTREE_GC_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_methods.h"
INIT_WORK(&rb->work, btree_node_read_work);
bio->bi_opf = REQ_OP_READ|REQ_SYNC|REQ_META;
bio->bi_iter.bi_sector = pick.ptr.offset;
- bio->bi_iter.bi_size = btree_bytes(c);
bio->bi_end_io = btree_node_read_endio;
bio->bi_private = b;
- bch2_bio_map(bio, b->data);
+ bch2_bio_map(bio, b->data, btree_bytes(c));
set_btree_node_read_in_flight(b);
wbio->wbio.order = order;
wbio->wbio.used_mempool = used_mempool;
wbio->wbio.bio.bi_opf = REQ_OP_WRITE|REQ_META|REQ_FUA;
- wbio->wbio.bio.bi_iter.bi_size = sectors_to_write << 9;
wbio->wbio.bio.bi_end_io = btree_node_write_endio;
wbio->wbio.bio.bi_private = b;
- bch2_bio_map(&wbio->wbio.bio, data);
+ bch2_bio_map(&wbio->wbio.bio, data, sectors_to_write << 9);
/*
* If we're appending to a leaf node, we don't technically need FUA -
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_IO_H
#define _BCACHEFS_BTREE_IO_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_methods.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_ITER_H
#define _BCACHEFS_BTREE_ITER_H
-#include <linux/dynamic_fault.h>
-
#include "bset.h"
#include "btree_types.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_LOCKING_H
#define _BCACHEFS_BTREE_LOCKING_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_TYPES_H
#define _BCACHEFS_BTREE_TYPES_H
bool deferred;
bool triggered;
+ bool marked;
};
#define BTREE_ITER_MAX 64
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_UPDATE_H
#define _BCACHEFS_BTREE_UPDATE_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_foreground.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_BTREE_UPDATE_INTERIOR_H
#define _BCACHEFS_BTREE_UPDATE_INTERIOR_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "btree_update.h"
struct bch_fs *c = trans->c;
struct bch_fs_usage *fs_usage = NULL;
struct btree_insert_entry *i;
+ bool saw_non_marked;
unsigned mark_flags = trans->flags & BTREE_INSERT_BUCKET_INVALIDATE
? BCH_BUCKET_MARK_BUCKET_INVALIDATE
: 0;
BUG_ON(i->iter->uptodate >= BTREE_ITER_NEED_RELOCK);
trans_for_each_update_iter(trans, i)
- if (update_has_triggers(trans, i) &&
- update_triggers_transactional(trans, i)) {
- ret = bch2_trans_mark_update(trans, i);
- if (ret == -EINTR)
- trace_trans_restart_mark(trans->ip);
- if (ret)
- goto out_clear_replicas;
+ i->marked = false;
+
+ do {
+ saw_non_marked = false;
+
+ trans_for_each_update_iter(trans, i) {
+ if (i->marked)
+ continue;
+
+ saw_non_marked = true;
+ i->marked = true;
+
+ if (update_has_triggers(trans, i) &&
+ update_triggers_transactional(trans, i)) {
+ ret = bch2_trans_mark_update(trans, i->iter, i->k);
+ if (ret == -EINTR)
+ trace_trans_restart_mark(trans->ip);
+ if (ret)
+ goto out_clear_replicas;
+ }
}
+ } while (saw_non_marked);
btree_trans_lock_write(c, trans);
+// SPDX-License-Identifier: GPL-2.0
/*
* Code for manipulating bucket marks for garbage collection.
*
if (unlikely(!ret))
return NULL;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
v = fs_usage_u64s(c);
if (unlikely(u64s != v)) {
u64s = v;
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
kfree(ret);
goto retry;
}
{
struct bch_fs_usage_short ret;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
ret = __bch2_fs_usage_read_short(c);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return ret;
}
bch2_data_types[old.data_type],
bch2_data_types[new.data_type]);
+ preempt_disable();
dev_usage = this_cpu_ptr(ca->usage[gc]);
if (bucket_type(old))
(int) new.cached_sectors - (int) old.cached_sectors;
dev_usage->sectors_fragmented +=
is_fragmented_bucket(new, ca) - is_fragmented_bucket(old, ca);
+ preempt_enable();
if (!is_available_bucket(old) && is_available_bucket(new))
bch2_wake_allocator(ca);
buckets = bucket_array(ca);
- preempt_disable();
for_each_bucket(g, buckets)
bch2_dev_usage_update(c, ca, c->usage_base,
old, g->mark, false);
- preempt_enable();
}
}
size_t b, bool owned_by_allocator,
struct gc_pos pos, unsigned flags)
{
+ preempt_disable();
+
do_mark_fn(__bch2_mark_alloc_bucket, c, pos, flags,
ca, b, owned_by_allocator);
+
+ preempt_enable();
}
static int bch2_mark_alloc(struct bch_fs *c, struct bkey_s_c k,
BUG_ON(type != BCH_DATA_SB &&
type != BCH_DATA_JOURNAL);
+ preempt_disable();
+
if (likely(c)) {
do_mark_fn(__bch2_mark_metadata_bucket, c, pos, flags,
ca, b, type, sectors);
} else {
__bch2_mark_metadata_bucket(c, ca, b, type, sectors, 0);
}
+
+ preempt_enable();
}
static s64 ptr_disk_sectors_delta(struct extent_ptr_decoded p,
{
int ret;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
ret = bch2_mark_key_locked(c, k, sectors,
fs_usage, journal_seq, flags);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return ret;
}
static int trans_get_key(struct btree_trans *trans,
enum btree_id btree_id, struct bpos pos,
- struct btree_insert_entry **insert,
struct btree_iter **iter,
struct bkey_s_c *k)
{
unsigned i;
int ret;
- *insert = NULL;
-
for (i = 0; i < trans->nr_updates; i++)
if (!trans->updates[i].deferred &&
trans->updates[i].iter->btree_id == btree_id &&
!bkey_cmp(pos, trans->updates[i].iter->pos)) {
- *insert = &trans->updates[i];
- *iter = (*insert)->iter;
- *k = bkey_i_to_s_c((*insert)->k);
+ *iter = trans->updates[i].iter;
+ *k = bkey_i_to_s_c(trans->updates[i].k);
return 0;
}
return ret;
}
-static int trans_update_key(struct btree_trans *trans,
- struct btree_insert_entry **insert,
- struct btree_iter *iter,
- struct bkey_s_c k,
- unsigned extra_u64s)
+static void *trans_update_key(struct btree_trans *trans,
+ struct btree_iter *iter,
+ unsigned u64s)
{
struct bkey_i *new_k;
+ unsigned i;
- if (*insert)
- return 0;
-
- new_k = bch2_trans_kmalloc(trans, bkey_bytes(k.k) +
- extra_u64s * sizeof(u64));
+ new_k = bch2_trans_kmalloc(trans, u64s * sizeof(u64));
if (IS_ERR(new_k))
- return PTR_ERR(new_k);
+ return new_k;
+
+ bkey_init(&new_k->k);
+ new_k->k.p = iter->pos;
- *insert = bch2_trans_update(trans, ((struct btree_insert_entry) {
- .iter = iter,
- .k = new_k,
- .triggered = true,
+ for (i = 0; i < trans->nr_updates; i++)
+ if (!trans->updates[i].deferred &&
+ trans->updates[i].iter == iter) {
+ trans->updates[i].k = new_k;
+ return new_k;
+ }
+
+ bch2_trans_update(trans, ((struct btree_insert_entry) {
+ .iter = iter,
+ .k = new_k,
+ .triggered = true,
}));
- bkey_reassemble((*insert)->k, k);
- return 0;
+ return new_k;
}
static int bch2_trans_mark_pointer(struct btree_trans *trans,
{
struct bch_fs *c = trans->c;
struct bch_dev *ca = bch_dev_bkey_exists(c, p.ptr.dev);
- struct btree_insert_entry *insert;
struct btree_iter *iter;
struct bkey_s_c k;
struct bkey_alloc_unpacked u;
ret = trans_get_key(trans, BTREE_ID_ALLOC,
POS(p.ptr.dev, PTR_BUCKET_NR(ca, &p.ptr)),
- &insert, &iter, &k);
+ &iter, &k);
if (ret)
return ret;
? u.dirty_sectors
: u.cached_sectors, sectors);
- ret = trans_update_key(trans, &insert, iter, k, 1);
+ a = trans_update_key(trans, iter, BKEY_ALLOC_U64s_MAX);
+ ret = PTR_ERR_OR_ZERO(a);
if (ret)
goto out;
- a = bkey_alloc_init(insert->k);
+ bkey_alloc_init(&a->k_i);
a->k.p = iter->pos;
bch2_alloc_pack(a, u);
out:
s64 sectors, enum bch_data_type data_type)
{
struct bch_replicas_padded r;
- struct btree_insert_entry *insert;
struct btree_iter *iter;
+ struct bkey_i *new_k;
struct bkey_s_c k;
struct bkey_s_stripe s;
unsigned nr_data;
BUG_ON(!sectors);
- ret = trans_get_key(trans, BTREE_ID_EC, POS(0, p.idx),
- &insert, &iter, &k);
+ ret = trans_get_key(trans, BTREE_ID_EC, POS(0, p.idx), &iter, &k);
if (ret)
return ret;
goto out;
}
- ret = trans_update_key(trans, &insert, iter, k, 1);
+ new_k = trans_update_key(trans, iter, k.k->u64s);
+ ret = PTR_ERR_OR_ZERO(new_k);
if (ret)
goto out;
- s = bkey_i_to_s_stripe(insert->k);
+ bkey_reassemble(new_k, k);
+ s = bkey_i_to_s_stripe(new_k);
nr_data = s.v->nr_blocks - s.v->nr_redundant;
}
int bch2_trans_mark_update(struct btree_trans *trans,
- struct btree_insert_entry *insert)
+ struct btree_iter *iter,
+ struct bkey_i *insert)
{
- struct btree_iter *iter = insert->iter;
struct btree *b = iter->l[0].b;
struct btree_node_iter node_iter = iter->l[0].iter;
struct bkey_packed *_k;
return 0;
ret = bch2_trans_mark_key(trans,
- bkey_i_to_s_c(insert->k),
- bpos_min(insert->k->k.p, b->key.k.p).offset -
- bkey_start_offset(&insert->k->k),
+ bkey_i_to_s_c(insert),
+ bpos_min(insert->k.p, b->key.k.p).offset -
+ bkey_start_offset(&insert->k),
BCH_BUCKET_MARK_INSERT);
if (ret)
return ret;
k = bkey_disassemble(b, _k, &unpacked);
if (btree_node_is_extents(b)
- ? bkey_cmp(insert->k->k.p, bkey_start_pos(k.k)) <= 0
- : bkey_cmp(insert->k->k.p, k.k->p))
+ ? bkey_cmp(insert->k.p, bkey_start_pos(k.k)) <= 0
+ : bkey_cmp(insert->k.p, k.k->p))
break;
if (btree_node_is_extents(b)) {
- switch (bch2_extent_overlap(&insert->k->k, k.k)) {
+ switch (bch2_extent_overlap(&insert->k, k.k)) {
case BCH_EXTENT_OVERLAP_ALL:
sectors = -((s64) k.k->size);
break;
case BCH_EXTENT_OVERLAP_BACK:
- sectors = bkey_start_offset(&insert->k->k) -
+ sectors = bkey_start_offset(&insert->k) -
k.k->p.offset;
break;
case BCH_EXTENT_OVERLAP_FRONT:
sectors = bkey_start_offset(k.k) -
- insert->k->k.p.offset;
+ insert->k.p.offset;
break;
case BCH_EXTENT_OVERLAP_MIDDLE:
- sectors = k.k->p.offset - insert->k->k.p.offset;
+ sectors = k.k->p.offset - insert->k.p.offset;
BUG_ON(sectors <= 0);
ret = bch2_trans_mark_key(trans, k, sectors,
if (ret)
return ret;
- sectors = bkey_start_offset(&insert->k->k) -
+ sectors = bkey_start_offset(&insert->k) -
k.k->p.offset;
break;
}
void __bch2_disk_reservation_put(struct bch_fs *c, struct disk_reservation *res)
{
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
this_cpu_sub(c->usage[0]->online_reserved,
res->sectors);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
res->sectors = 0;
}
s64 sectors_available;
int ret;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
+ preempt_disable();
pcpu = this_cpu_ptr(c->pcpu);
if (sectors <= pcpu->sectors_available)
get = min((u64) sectors + SECTORS_CACHE, old);
if (get < sectors) {
- percpu_up_read_preempt_enable(&c->mark_lock);
+ preempt_enable();
+ percpu_up_read(&c->mark_lock);
goto recalculate;
}
} while ((v = atomic64_cmpxchg(&c->sectors_available,
this_cpu_add(c->usage[0]->online_reserved, sectors);
res->sectors += sectors;
- percpu_up_read_preempt_enable(&c->mark_lock);
+ preempt_enable();
+ percpu_up_read(&c->mark_lock);
return 0;
recalculate:
+/* SPDX-License-Identifier: GPL-2.0 */
/*
* Code for manipulating bucket marks for garbage collection.
*
struct replicas_delta_list *);
int bch2_trans_mark_key(struct btree_trans *, struct bkey_s_c, s64, unsigned);
int bch2_trans_mark_update(struct btree_trans *,
- struct btree_insert_entry *);
+ struct btree_iter *iter,
+ struct bkey_i *insert);
void bch2_trans_fs_usage_apply(struct btree_trans *, struct bch_fs_usage *);
/* disk reservations: */
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BUCKETS_TYPES_H
#define _BUCKETS_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
#ifndef NO_BCACHEFS_CHARDEV
#include "bcachefs.h"
dst.used = bch2_fs_sectors_used(c, src);
dst.online_reserved = src->online_reserved;
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
for (i = 0; i < BCH_REPLICAS_MAX; i++) {
dst.persistent_reserved[i] =
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_CHARDEV_H
#define _BCACHEFS_CHARDEV_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "checksum.h"
#include "super.h"
return crc32c(crc, data, len);
case BCH_CSUM_CRC64_NONZERO:
case BCH_CSUM_CRC64:
- return bch2_crc64_update(crc, data, len);
+ return crc64_be(crc, data, len);
default:
BUG();
}
kunmap_atomic(p);
}
#else
- __bio_for_each_contig_segment(bv, bio, *iter, *iter)
+ __bio_for_each_bvec(bv, bio, *iter, *iter)
crc = bch2_checksum_update(type, crc,
page_address(bv.bv_page) + bv.bv_offset,
bv.bv_len);
kunmap_atomic(p);
}
#else
- __bio_for_each_contig_segment(bv, bio, *iter, *iter)
+ __bio_for_each_bvec(bv, bio, *iter, *iter)
crypto_shash_update(desc,
page_address(bv.bv_page) + bv.bv_offset,
bv.bv_len);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_CHECKSUM_H
#define _BCACHEFS_CHECKSUM_H
struct bch_csum bch2_checksum_merge(unsigned, struct bch_csum,
struct bch_csum, size_t);
-static inline u64 bch2_crc64_update(u64 crc, const void *p, size_t len)
-{
- return crc64_be(crc, p, len);
-}
-
#define BCH_NONCE_EXTENT cpu_to_le32(1 << 28)
#define BCH_NONCE_BTREE cpu_to_le32(2 << 28)
#define BCH_NONCE_JOURNAL cpu_to_le32(3 << 28)
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "clock.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_CLOCK_H
#define _BCACHEFS_CLOCK_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_CLOCK_TYPES_H
#define _BCACHEFS_CLOCK_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "checksum.h"
#include "compress.h"
#include "io.h"
#include "super-io.h"
-#include "lz4.h"
#include <linux/lz4.h>
#include <linux/zlib.h>
#include <linux/zstd.h>
BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
#ifndef CONFIG_HIGHMEM
- __bio_for_each_contig_segment(bv, bio, iter, start) {
+ __bio_for_each_bvec(bv, bio, iter, start) {
if (bv.bv_len == start.bi_size)
return (struct bbuf) {
.b = page_address(bv.bv_page) + bv.bv_offset,
switch (crc.compression_type) {
case BCH_COMPRESSION_LZ4_OLD:
- ret = bch2_lz4_decompress(src_data.b, &src_len,
- dst_data, dst_len);
- if (ret)
- goto err;
- break;
case BCH_COMPRESSION_LZ4:
ret = LZ4_decompress_safe_partial(src_data.b, dst_data,
src_len, dst_len, dst_len);
}
/*
- * might have to free existing pages and retry allocation from mempool -
- * do this _after_ decompressing:
+ * XXX: don't have a good way to assert that the bio was allocated with
+ * enough space, we depend on bch2_move_extent doing the right thing
*/
- bch2_bio_alloc_more_pages_pool(c, bio, crc->live_size << 9);
+ bio->bi_iter.bi_size = crc->live_size << 9;
memcpy_to_bio(bio, bio->bi_iter, data.b + (crc->offset << 9));
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_COMPRESS_H
#define _BCACHEFS_COMPRESS_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Assorted bcachefs debug code
*
bio_set_dev(bio, ca->disk_sb.bdev);
bio->bi_opf = REQ_OP_READ|REQ_META;
bio->bi_iter.bi_sector = pick.ptr.offset;
- bio->bi_iter.bi_size = btree_bytes(c);
- bch2_bio_map(bio, n_sorted);
+ bch2_bio_map(bio, n_sorted, btree_bytes(c));
submit_bio_wait(bio);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_DEBUG_H
#define _BCACHEFS_DEBUG_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_methods.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_DIRENT_H
#define _BCACHEFS_DIRENT_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "disk_groups.h"
#include "super-io.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_DISK_GROUPS_H
#define _BCACHEFS_DISK_GROUPS_H
+// SPDX-License-Identifier: GPL-2.0
/* erasure coding */
bio_set_op_attrs(&ec_bio->bio, rw, 0);
ec_bio->bio.bi_iter.bi_sector = ptr->offset + buf->offset + (offset >> 9);
- ec_bio->bio.bi_iter.bi_size = b;
ec_bio->bio.bi_end_io = ec_block_endio;
ec_bio->bio.bi_private = cl;
- bch2_bio_map(&ec_bio->bio, buf->data[idx] + offset);
+ bch2_bio_map(&ec_bio->bio, buf->data[idx] + offset, b);
closure_get(cl);
percpu_ref_get(&ca->io_ref);
bch2_stripes_heap_insert(c, m, idx);
}
- if (stripe_idx_to_delete(c) >= 0)
+ if (stripe_idx_to_delete(c) >= 0 &&
+ !percpu_ref_is_dying(&c->writes))
schedule_work(&c->ec_stripe_delete_work);
}
if (idx < 0)
break;
- ec_stripe_delete(c, idx);
+ if (ec_stripe_delete(c, idx))
+ break;
}
mutex_unlock(&c->ec_stripe_create_lock);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_EC_H
#define _BCACHEFS_EC_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_EC_TYPES_H
#define _BCACHEFS_EC_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "error.h"
#include "io.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_ERROR_H
#define _BCACHEFS_ERROR_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2010 Kent Overstreet <kent.overstreet@gmail.com>
*
const char *bch2_extent_invalid(const struct bch_fs *c, struct bkey_s_c k)
{
- if (bkey_val_u64s(k.k) > BKEY_EXTENT_VAL_U64s_MAX)
- return "value too big";
-
return bch2_bkey_ptrs_invalid(c, k);
}
}
enum merge_result bch2_extent_merge(struct bch_fs *c,
- struct bkey_i *l, struct bkey_i *r)
+ struct bkey_s _l, struct bkey_s _r)
{
- struct bkey_s_extent el = bkey_i_to_s_extent(l);
- struct bkey_s_extent er = bkey_i_to_s_extent(r);
- union bch_extent_entry *en_l = el.v->start;
- union bch_extent_entry *en_r = er.v->start;
+ struct bkey_s_extent l = bkey_s_to_extent(_l);
+ struct bkey_s_extent r = bkey_s_to_extent(_r);
+ union bch_extent_entry *en_l = l.v->start;
+ union bch_extent_entry *en_r = r.v->start;
struct bch_extent_crc_unpacked crc_l, crc_r;
- if (bkey_val_u64s(&l->k) != bkey_val_u64s(&r->k))
+ if (bkey_val_u64s(l.k) != bkey_val_u64s(r.k))
return BCH_MERGE_NOMERGE;
- crc_l = bch2_extent_crc_unpack(el.k, NULL);
+ crc_l = bch2_extent_crc_unpack(l.k, NULL);
- extent_for_each_entry(el, en_l) {
- en_r = vstruct_idx(er.v, (u64 *) en_l - el.v->_data);
+ extent_for_each_entry(l, en_l) {
+ en_r = vstruct_idx(r.v, (u64 *) en_l - l.v->_data);
if (extent_entry_type(en_l) != extent_entry_type(en_r))
return BCH_MERGE_NOMERGE;
case BCH_EXTENT_ENTRY_crc32:
case BCH_EXTENT_ENTRY_crc64:
case BCH_EXTENT_ENTRY_crc128:
- crc_l = bch2_extent_crc_unpack(el.k, entry_to_crc(en_l));
- crc_r = bch2_extent_crc_unpack(er.k, entry_to_crc(en_r));
+ crc_l = bch2_extent_crc_unpack(l.k, entry_to_crc(en_l));
+ crc_r = bch2_extent_crc_unpack(r.k, entry_to_crc(en_r));
if (crc_l.csum_type != crc_r.csum_type ||
crc_l.compression_type != crc_r.compression_type ||
}
}
- extent_for_each_entry(el, en_l) {
+ extent_for_each_entry(l, en_l) {
struct bch_extent_crc_unpacked crc_l, crc_r;
- en_r = vstruct_idx(er.v, (u64 *) en_l - el.v->_data);
+ en_r = vstruct_idx(r.v, (u64 *) en_l - l.v->_data);
if (!extent_entry_is_crc(en_l))
continue;
- crc_l = bch2_extent_crc_unpack(el.k, entry_to_crc(en_l));
- crc_r = bch2_extent_crc_unpack(er.k, entry_to_crc(en_r));
+ crc_l = bch2_extent_crc_unpack(l.k, entry_to_crc(en_l));
+ crc_r = bch2_extent_crc_unpack(r.k, entry_to_crc(en_r));
crc_l.csum = bch2_checksum_merge(crc_l.csum_type,
crc_l.csum,
bch2_extent_crc_pack(entry_to_crc(en_l), crc_l);
}
- bch2_key_resize(&l->k, l->k.size + r->k.size);
+ bch2_key_resize(l.k, l.k->size + r.k->size);
return BCH_MERGE_MERGE;
}
bch2_bkey_unpack(b, &li.k, l);
bch2_bkey_unpack(b, &ri.k, r);
- ret = bch2_bkey_merge(c, &li.k, &ri.k);
+ ret = bch2_bkey_merge(c,
+ bkey_i_to_s(&li.k),
+ bkey_i_to_s(&ri.k));
if (ret == BCH_MERGE_NOMERGE)
return false;
}
enum merge_result bch2_reservation_merge(struct bch_fs *c,
- struct bkey_i *l, struct bkey_i *r)
+ struct bkey_s _l, struct bkey_s _r)
{
- struct bkey_i_reservation *li = bkey_i_to_reservation(l);
- struct bkey_i_reservation *ri = bkey_i_to_reservation(r);
+ struct bkey_s_reservation l = bkey_s_to_reservation(_l);
+ struct bkey_s_reservation r = bkey_s_to_reservation(_r);
- if (li->v.generation != ri->v.generation ||
- li->v.nr_replicas != ri->v.nr_replicas)
+ if (l.v->generation != r.v->generation ||
+ l.v->nr_replicas != r.v->nr_replicas)
return BCH_MERGE_NOMERGE;
- if ((u64) l->k.size + r->k.size > KEY_SIZE_MAX) {
- bch2_key_resize(&l->k, KEY_SIZE_MAX);
- bch2_cut_front(l->k.p, r);
+ if ((u64) l.k->size + r.k->size > KEY_SIZE_MAX) {
+ bch2_key_resize(l.k, KEY_SIZE_MAX);
+ __bch2_cut_front(l.k->p, r.s);
return BCH_MERGE_PARTIAL;
}
- bch2_key_resize(&l->k, l->k.size + r->k.size);
+ bch2_key_resize(l.k, l.k->size + r.k->size);
return BCH_MERGE_MERGE;
}
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_EXTENTS_H
#define _BCACHEFS_EXTENTS_H
void bch2_extent_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);
bool bch2_extent_normalize(struct bch_fs *, struct bkey_s);
enum merge_result bch2_extent_merge(struct bch_fs *,
- struct bkey_i *, struct bkey_i *);
+ struct bkey_s, struct bkey_s);
#define bch2_bkey_ops_extent (struct bkey_ops) { \
.key_invalid = bch2_extent_invalid, \
const char *bch2_reservation_invalid(const struct bch_fs *, struct bkey_s_c);
void bch2_reservation_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c);
enum merge_result bch2_reservation_merge(struct bch_fs *,
- struct bkey_i *, struct bkey_i *);
+ struct bkey_s, struct bkey_s);
#define bch2_bkey_ops_reservation (struct bkey_ops) { \
.key_invalid = bch2_reservation_invalid, \
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_EXTENTS_TYPES_H
#define _BCACHEFS_EXTENTS_TYPES_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _EYTZINGER_H
#define _EYTZINGER_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FIFO_H
#define _BCACHEFS_FIFO_H
+// SPDX-License-Identifier: GPL-2.0
#ifndef NO_BCACHEFS_FS
#include "bcachefs.h"
/* stored in page->private: */
-/*
- * bch_page_state has to (unfortunately) be manipulated with cmpxchg - we could
- * almost protected it with the page lock, except that bch2_writepage_io_done has
- * to update the sector counts (and from interrupt/bottom half context).
- */
-struct bch_page_state {
-union { struct {
- /* existing data: */
- unsigned sectors:PAGE_SECTOR_SHIFT + 1;
-
+struct bch_page_sector {
/* Uncompressed, fully allocated replicas: */
- unsigned nr_replicas:4;
+ unsigned nr_replicas:3;
/* Owns PAGE_SECTORS * replicas_reserved sized reservation: */
- unsigned replicas_reserved:4;
-
- /* Owns PAGE_SECTORS sized quota reservation: */
- unsigned quota_reserved:1;
-
- /*
- * Number of sectors on disk - for i_blocks
- * Uncompressed size, not compressed size:
- */
- unsigned dirty_sectors:PAGE_SECTOR_SHIFT + 1;
-};
- /* for cmpxchg: */
- unsigned long v;
+ unsigned replicas_reserved:3;
+
+ /* i_sectors: */
+ enum {
+ SECTOR_UNALLOCATED,
+ SECTOR_QUOTA_RESERVED,
+ SECTOR_DIRTY,
+ SECTOR_ALLOCATED,
+ } state:2;
};
+
+struct bch_page_state {
+ struct bch_page_sector s[PAGE_SECTORS];
};
-#define page_state_cmpxchg(_ptr, _new, _expr) \
-({ \
- unsigned long _v = READ_ONCE((_ptr)->v); \
- struct bch_page_state _old; \
- \
- do { \
- _old.v = _new.v = _v; \
- _expr; \
- \
- EBUG_ON(_new.sectors + _new.dirty_sectors > PAGE_SECTORS);\
- } while (_old.v != _new.v && \
- (_v = cmpxchg(&(_ptr)->v, _old.v, _new.v)) != _old.v); \
- \
- _old; \
-})
+static inline struct bch_page_state *__bch2_page_state(struct page *page)
+{
+ return page_has_private(page)
+ ? (struct bch_page_state *) page_private(page)
+ : NULL;
+}
-static inline struct bch_page_state *page_state(struct page *page)
+static inline struct bch_page_state *bch2_page_state(struct page *page)
{
- struct bch_page_state *s = (void *) &page->private;
+ EBUG_ON(!PageLocked(page));
- BUILD_BUG_ON(sizeof(*s) > sizeof(page->private));
+ return __bch2_page_state(page);
+}
- if (!PagePrivate(page))
- SetPagePrivate(page);
+/* for newly allocated pages: */
+static void __bch2_page_state_release(struct page *page)
+{
+ struct bch_page_state *s = __bch2_page_state(page);
- return s;
+ if (!s)
+ return;
+
+ ClearPagePrivate(page);
+ set_page_private(page, 0);
+ put_page(page);
+ kfree(s);
}
-static inline unsigned page_res_sectors(struct bch_page_state s)
+static void bch2_page_state_release(struct page *page)
{
+ struct bch_page_state *s = bch2_page_state(page);
+
+ if (!s)
+ return;
- return s.replicas_reserved * PAGE_SECTORS;
+ ClearPagePrivate(page);
+ set_page_private(page, 0);
+ put_page(page);
+ kfree(s);
}
-static void __bch2_put_page_reservation(struct bch_fs *c, struct bch_inode_info *inode,
- struct bch_page_state s)
+/* for newly allocated pages: */
+static struct bch_page_state *__bch2_page_state_create(struct page *page,
+ gfp_t gfp)
{
- struct disk_reservation res = { .sectors = page_res_sectors(s) };
- struct quota_res quota_res = { .sectors = s.quota_reserved ? PAGE_SECTORS : 0 };
+ struct bch_page_state *s;
- bch2_quota_reservation_put(c, inode, "a_res);
- bch2_disk_reservation_put(c, &res);
+ s = kzalloc(sizeof(*s), GFP_NOFS|gfp);
+ if (!s)
+ return NULL;
+
+ /*
+ * migrate_page_move_mapping() assumes that pages with private data
+ * have their count elevated by 1.
+ */
+ get_page(page);
+ set_page_private(page, (unsigned long) s);
+ SetPagePrivate(page);
+ return s;
+}
+
+static struct bch_page_state *bch2_page_state_create(struct page *page,
+ gfp_t gfp)
+{
+ return bch2_page_state(page) ?: __bch2_page_state_create(page, gfp);
}
static void bch2_put_page_reservation(struct bch_fs *c, struct bch_inode_info *inode,
struct page *page)
{
- struct bch_page_state s;
+ struct bch_page_state *s = bch2_page_state(page);
+ struct disk_reservation disk_res = { 0 };
+ struct quota_res quota_res = { 0 };
+ unsigned i;
- EBUG_ON(!PageLocked(page));
+ if (!s)
+ return;
- s = page_state_cmpxchg(page_state(page), s, {
- s.replicas_reserved = 0;
- s.quota_reserved = 0;
- });
+ for (i = 0; i < ARRAY_SIZE(s->s); i++) {
+ disk_res.sectors += s->s[i].replicas_reserved;
+ s->s[i].replicas_reserved = 0;
- __bch2_put_page_reservation(c, inode, s);
+ if (s->s[i].state == SECTOR_QUOTA_RESERVED) {
+ quota_res.sectors++;
+ s->s[i].state = SECTOR_UNALLOCATED;
+ }
+ }
+
+ bch2_quota_reservation_put(c, inode, "a_res);
+ bch2_disk_reservation_put(c, &disk_res);
}
-static int bch2_get_page_reservation(struct bch_fs *c, struct bch_inode_info *inode,
- struct page *page, bool check_enospc)
+static inline unsigned inode_nr_replicas(struct bch_fs *c, struct bch_inode_info *inode)
{
- struct bch_page_state *s = page_state(page), new;
-
/* XXX: this should not be open coded */
- unsigned nr_replicas = inode->ei_inode.bi_data_replicas
+ return inode->ei_inode.bi_data_replicas
? inode->ei_inode.bi_data_replicas - 1
: c->opts.data_replicas;
- struct disk_reservation disk_res;
+}
+
+static inline unsigned sectors_to_reserve(struct bch_page_sector *s,
+ unsigned nr_replicas)
+{
+ return max(0, (int) nr_replicas -
+ s->nr_replicas -
+ s->replicas_reserved);
+}
+
+static int bch2_get_page_disk_reservation(struct bch_fs *c,
+ struct bch_inode_info *inode,
+ struct page *page, bool check_enospc)
+{
+ struct bch_page_state *s = bch2_page_state_create(page, 0);
+ unsigned nr_replicas = inode_nr_replicas(c, inode);
+ struct disk_reservation disk_res = { 0 };
+ unsigned i, disk_res_sectors = 0;
+ int ret;
+
+ if (!s)
+ return -ENOMEM;
+
+ for (i = 0; i < ARRAY_SIZE(s->s); i++)
+ disk_res_sectors += sectors_to_reserve(&s->s[i], nr_replicas);
+
+ if (!disk_res_sectors)
+ return 0;
+
+ ret = bch2_disk_reservation_get(c, &disk_res,
+ disk_res_sectors, 1,
+ !check_enospc
+ ? BCH_DISK_RESERVATION_NOFAIL
+ : 0);
+ if (unlikely(ret))
+ return ret;
+
+ for (i = 0; i < ARRAY_SIZE(s->s); i++)
+ s->s[i].replicas_reserved +=
+ sectors_to_reserve(&s->s[i], nr_replicas);
+
+ return 0;
+}
+
+static int bch2_get_page_quota_reservation(struct bch_fs *c,
+ struct bch_inode_info *inode,
+ struct page *page, bool check_enospc)
+{
+ struct bch_page_state *s = bch2_page_state_create(page, 0);
struct quota_res quota_res = { 0 };
+ unsigned i, quota_res_sectors = 0;
int ret;
- EBUG_ON(!PageLocked(page));
+ if (!s)
+ return -ENOMEM;
- if (s->replicas_reserved < nr_replicas) {
- ret = bch2_disk_reservation_get(c, &disk_res, PAGE_SECTORS,
- nr_replicas - s->replicas_reserved,
- !check_enospc ? BCH_DISK_RESERVATION_NOFAIL : 0);
- if (unlikely(ret))
- return ret;
+ for (i = 0; i < ARRAY_SIZE(s->s); i++)
+ quota_res_sectors += s->s[i].state == SECTOR_UNALLOCATED;
- page_state_cmpxchg(s, new, ({
- BUG_ON(new.replicas_reserved +
- disk_res.nr_replicas != nr_replicas);
- new.replicas_reserved += disk_res.nr_replicas;
- }));
- }
+ if (!quota_res_sectors)
+ return 0;
- if (!s->quota_reserved &&
- s->sectors + s->dirty_sectors < PAGE_SECTORS) {
- ret = bch2_quota_reservation_add(c, inode, "a_res,
- PAGE_SECTORS,
- check_enospc);
- if (unlikely(ret))
- return ret;
+ ret = bch2_quota_reservation_add(c, inode, "a_res,
+ quota_res_sectors,
+ check_enospc);
+ if (unlikely(ret))
+ return ret;
- page_state_cmpxchg(s, new, ({
- BUG_ON(new.quota_reserved);
- new.quota_reserved = 1;
- }));
- }
+ for (i = 0; i < ARRAY_SIZE(s->s); i++)
+ if (s->s[i].state == SECTOR_UNALLOCATED)
+ s->s[i].state = SECTOR_QUOTA_RESERVED;
- return ret;
+ return 0;
+}
+
+static int bch2_get_page_reservation(struct bch_fs *c, struct bch_inode_info *inode,
+ struct page *page, bool check_enospc)
+{
+ return bch2_get_page_disk_reservation(c, inode, page, check_enospc) ?:
+ bch2_get_page_quota_reservation(c, inode, page, check_enospc);
}
static void bch2_clear_page_bits(struct page *page)
{
struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
- struct bch_page_state s;
+ struct bch_page_state *s = bch2_page_state(page);
+ int i, dirty_sectors = 0;
- EBUG_ON(!PageLocked(page));
-
- if (!PagePrivate(page))
+ if (!s)
return;
- s.v = xchg(&page_state(page)->v, 0);
- ClearPagePrivate(page);
+ for (i = 0; i < ARRAY_SIZE(s->s); i++) {
+ if (s->s[i].state == SECTOR_DIRTY) {
+ dirty_sectors++;
+ s->s[i].state = SECTOR_UNALLOCATED;
+ }
+ }
- if (s.dirty_sectors)
- i_sectors_acct(c, inode, NULL, -s.dirty_sectors);
+ if (dirty_sectors)
+ i_sectors_acct(c, inode, NULL, -dirty_sectors);
+ bch2_put_page_reservation(c, inode, page);
- __bch2_put_page_reservation(c, inode, s);
+ bch2_page_state_release(page);
}
-int bch2_set_page_dirty(struct page *page)
+static void __bch2_set_page_dirty(struct page *page)
{
struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
+ struct bch_page_state *s = bch2_page_state(page);
struct quota_res quota_res = { 0 };
- struct bch_page_state old, new;
+ unsigned i, dirty_sectors = 0;
- old = page_state_cmpxchg(page_state(page), new,
- new.dirty_sectors = PAGE_SECTORS - new.sectors;
- new.quota_reserved = 0;
- );
+ BUG_ON(!s);
- quota_res.sectors += old.quota_reserved * PAGE_SECTORS;
+ for (i = 0; i < ARRAY_SIZE(s->s); i++) {
+ if (s->s[i].state == SECTOR_QUOTA_RESERVED)
+ quota_res.sectors++;
+
+ if (s->s[i].state == SECTOR_UNALLOCATED ||
+ s->s[i].state == SECTOR_QUOTA_RESERVED) {
+ s->s[i].state = SECTOR_DIRTY;
+ dirty_sectors++;
+ }
+ }
- if (old.dirty_sectors != new.dirty_sectors)
- i_sectors_acct(c, inode, "a_res,
- new.dirty_sectors - old.dirty_sectors);
+ if (dirty_sectors)
+ i_sectors_acct(c, inode, "a_res, dirty_sectors);
bch2_quota_reservation_put(c, inode, "a_res);
+}
- return __set_page_dirty_nobuffers(page);
+static void bch2_set_page_dirty(struct page *page)
+{
+ __bch2_set_page_dirty(page);
+ __set_page_dirty_nobuffers(page);
}
-int bch2_page_mkwrite(struct vm_fault *vmf)
+vm_fault_t bch2_page_mkwrite(struct vm_fault *vmf)
{
struct page *page = vmf->page;
struct file *file = vmf->vma->vm_file;
}
if (!PageDirty(page))
- set_page_dirty(page);
+ bch2_set_page_dirty(page);
wait_for_stable_page(page);
out:
if (current->pagecache_lock != &mapping->add_lock)
return ret;
if (PagePrivate(page)) {
- *page_state(newpage) = *page_state(page);
ClearPagePrivate(page);
+ get_page(newpage);
+ set_page_private(newpage, page_private(page));
+ set_page_private(page, 0);
+ put_page(page);
+ SetPagePrivate(newpage);
}
- migrate_page_copy(newpage, page);
+ if (mode != MIGRATE_SYNC_NO_COPY)
+ migrate_page_copy(newpage, page);
+ else
+ migrate_page_states(newpage, page);
return MIGRATEPAGE_SUCCESS;
}
#endif
else if (!bio_can_add_page_contig(bio, page))
return -1;
- __bio_add_page(bio, page, PAGE_SIZE, 0);
+ BUG_ON(!bio_add_page(bio, page, PAGE_SIZE, 0));
return 0;
}
static void bch2_readpages_end_io(struct bio *bio)
{
+ struct bvec_iter_all iter;
struct bio_vec *bv;
int i;
- bio_for_each_segment_all(bv, bio, i) {
+ bio_for_each_segment_all(bv, bio, i, iter) {
struct page *page = bv->bv_page;
if (!bio->bi_status) {
while (!list_empty(pages)) {
struct page *page = list_last_entry(pages, struct page, lru);
- prefetchw(&page->flags);
+ __bch2_page_state_create(page, __GFP_NOFAIL);
+
iter->pages[iter->nr_pages++] = page;
list_del(&page->lru);
}
iter->idx++;
iter->nr_added++;
+ __bch2_page_state_release(page);
put_page(page);
}
out:
EBUG_ON(iter->pages[iter->idx]->index != iter->offset + iter->idx);
- page_state_init_for_read(iter->pages[iter->idx]);
return iter->pages[iter->idx];
}
struct bio_vec bv;
unsigned nr_ptrs = bch2_bkey_nr_ptrs_allocated(k);
- bio_for_each_segment(bv, bio, iter) {
- /* brand new pages, don't need to be locked: */
-
- struct bch_page_state *s = page_state(bv.bv_page);
-
- /* sectors in @k from the start of this page: */
- unsigned k_sectors = k.k->size - (iter.bi_sector - k.k->p.offset);
-
- unsigned page_sectors = min(bv.bv_len >> 9, k_sectors);
+ BUG_ON(bio->bi_iter.bi_sector < bkey_start_offset(k.k));
+ BUG_ON(bio_end_sector(bio) > k.k->p.offset);
- s->nr_replicas = page_sectors == PAGE_SECTORS
- ? nr_ptrs : 0;
- BUG_ON(s->sectors + page_sectors > PAGE_SECTORS);
- s->sectors += page_sectors;
+ bio_for_each_segment(bv, bio, iter) {
+ struct bch_page_state *s = bch2_page_state(bv.bv_page);
+ unsigned i;
+
+ for (i = bv.bv_offset >> 9;
+ i < (bv.bv_offset + bv.bv_len) >> 9;
+ i++) {
+ s->s[i].nr_replicas = nr_ptrs;
+ s->s[i].state = SECTOR_ALLOCATED;
+ }
}
}
if (!page)
break;
- page_state_init_for_read(page);
+ if (!__bch2_page_state_create(page, 0)) {
+ put_page(page);
+ break;
+ }
ret = add_to_page_cache_lru(page, iter->mapping,
page_offset, GFP_NOFS);
if (ret) {
- ClearPagePrivate(page);
+ __bch2_page_state_release(page);
put_page(page);
break;
}
put_page(page);
}
- __bio_add_page(bio, page, PAGE_SIZE, 0);
+ BUG_ON(!bio_add_page(bio, page, PAGE_SIZE, 0));
}
}
bio_set_op_attrs(&rbio->bio, REQ_OP_READ, 0);
rbio->bio.bi_iter.bi_sector = (sector_t) index << PAGE_SECTOR_SHIFT;
rbio->bio.bi_end_io = bch2_readpages_end_io;
- __bio_add_page(&rbio->bio, page, PAGE_SIZE, 0);
+ BUG_ON(!bio_add_page(&rbio->bio, page, PAGE_SIZE, 0));
bchfs_read(&trans, iter, rbio, inode->v.i_ino,
&readpages_iter);
struct btree_trans trans;
struct btree_iter *iter;
- page_state_init_for_read(page);
+ bch2_page_state_create(page, __GFP_NOFAIL);
bio_set_op_attrs(&rbio->bio, REQ_OP_READ, REQ_SYNC);
bio_add_page_contig(&rbio->bio, page);
struct bch_writepage_io, cl);
struct bch_fs *c = io->op.op.c;
struct bio *bio = &io->op.op.wbio.bio;
+ struct bvec_iter_all iter;
struct bio_vec *bvec;
unsigned i;
if (io->op.op.error) {
- bio_for_each_segment_all(bvec, bio, i) {
+ bio_for_each_segment_all(bvec, bio, i, iter) {
SetPageError(bvec->bv_page);
mapping_set_error(bvec->bv_page->mapping, -EIO);
}
i_sectors_acct(c, io->op.inode, NULL,
io->op.sectors_added - (s64) io->new_sectors);
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all(bvec, bio, i, iter)
end_page_writeback(bvec->bv_page);
closure_return_with_destructor(&io->cl, bch2_writepage_io_free);
struct bch_inode_info *inode = to_bch_ei(page->mapping->host);
struct bch_fs *c = inode->v.i_sb->s_fs_info;
struct bch_writepage_state *w = data;
- struct bch_page_state new, old;
- unsigned offset, nr_replicas_this_write;
+ struct bch_page_state *s;
+ unsigned offset, nr_replicas_this_write = U32_MAX;
+ unsigned dirty_sectors = 0, reserved_sectors = 0;
loff_t i_size = i_size_read(&inode->v);
pgoff_t end_index = i_size >> PAGE_SHIFT;
+ unsigned i;
+ int ret;
EBUG_ON(!PageUptodate(page));
*/
zero_user_segment(page, offset, PAGE_SIZE);
do_io:
- EBUG_ON(!PageLocked(page));
+ s = bch2_page_state_create(page, __GFP_NOFAIL);
- /* Before unlocking the page, transfer reservation to w->io: */
- old = page_state_cmpxchg(page_state(page), new, {
- /*
- * If we didn't get a reservation, we can only write out the
- * number of (fully allocated) replicas that currently exist,
- * and only if the entire page has been written:
- */
+ ret = bch2_get_page_disk_reservation(c, inode, page, true);
+ if (ret) {
+ SetPageError(page);
+ mapping_set_error(page->mapping, ret);
+ unlock_page(page);
+ return 0;
+ }
+
+ for (i = 0; i < PAGE_SECTORS; i++)
nr_replicas_this_write =
- max_t(unsigned,
- new.replicas_reserved,
- (new.sectors == PAGE_SECTORS
- ? new.nr_replicas : 0));
+ min_t(unsigned, nr_replicas_this_write,
+ s->s[i].nr_replicas +
+ s->s[i].replicas_reserved);
- BUG_ON(!nr_replicas_this_write);
+ /* Before unlocking the page, transfer reservation to w->io: */
- new.nr_replicas = w->opts.compression
- ? 0
- : nr_replicas_this_write;
+ for (i = 0; i < PAGE_SECTORS; i++) {
+ s->s[i].nr_replicas = w->opts.compression
+ ? 0 : nr_replicas_this_write;
- new.replicas_reserved = 0;
+ reserved_sectors += s->s[i].replicas_reserved;
+ s->s[i].replicas_reserved = 0;
- new.sectors += new.dirty_sectors;
- BUG_ON(new.sectors != PAGE_SECTORS);
- new.dirty_sectors = 0;
- });
+ dirty_sectors += s->s[i].state == SECTOR_DIRTY;
+ s->s[i].state = SECTOR_ALLOCATED;
+ }
BUG_ON(PageWriteback(page));
set_page_writeback(page);
bch2_writepage_io_alloc(c, w, inode, page,
nr_replicas_this_write);
- w->io->new_sectors += new.sectors - old.sectors;
+ w->io->new_sectors += dirty_sectors;
BUG_ON(inode != w->io->op.inode);
BUG_ON(bio_add_page_contig(&w->io->op.op.wbio.bio, page));
- w->io->op.op.res.sectors += old.replicas_reserved * PAGE_SECTORS;
+ w->io->op.op.res.sectors += reserved_sectors;
w->io->op.new_i_size = i_size;
if (wbc->sync_mode == WB_SYNC_ALL)
if (!PageUptodate(page))
SetPageUptodate(page);
if (!PageDirty(page))
- set_page_dirty(page);
+ bch2_set_page_dirty(page);
inode->ei_last_dirtied = (unsigned long) current;
} else {
if (!PageUptodate(pages[i]))
SetPageUptodate(pages[i]);
if (!PageDirty(pages[i]))
- set_page_dirty(pages[i]);
+ bch2_set_page_dirty(pages[i]);
unlock_page(pages[i]);
put_page(pages[i]);
}
struct address_space *mapping = req->ki_filp->f_mapping;
struct bch_inode_info *inode = dio->iop.inode;
struct bio *bio = &dio->iop.op.wbio.bio;
+ struct bvec_iter_all iter;
struct bio_vec *bv;
loff_t offset;
bool sync;
closure_sync(&dio->cl);
loop:
- bio_for_each_segment_all(bv, bio, i)
+ bio_for_each_segment_all(bv, bio, i, iter)
put_page(bv->bv_page);
if (!dio->iter.count || dio->iop.op.error)
break;
zero_user_segment(page, 0, end_offset);
if (!PageDirty(page))
- set_page_dirty(page);
+ bch2_set_page_dirty(page);
unlock:
unlock_page(page);
put_page(page);
static bool page_is_data(struct page *page)
{
- EBUG_ON(!PageLocked(page));
+ struct bch_page_state *s = bch2_page_state(page);
+ unsigned i;
+
+ if (!s)
+ return false;
+
+ for (i = 0; i < PAGE_SECTORS; i++)
+ if (s->s[i].state >= SECTOR_DIRTY)
+ return true;
- /* XXX: should only have to check PageDirty */
- return PagePrivate(page) &&
- (page_state(page)->sectors ||
- page_state(page)->dirty_sectors);
+ return false;
}
static loff_t bch2_next_pagecache_data(struct inode *vinode,
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FS_IO_H
#define _BCACHEFS_FS_IO_H
#include <linux/uio.h>
-int bch2_set_page_dirty(struct page *);
-
int bch2_writepage(struct page *, struct writeback_control *);
int bch2_readpage(struct file *, struct page *);
loff_t bch2_llseek(struct file *, loff_t, int);
-int bch2_page_mkwrite(struct vm_fault *);
+vm_fault_t bch2_page_mkwrite(struct vm_fault *);
void bch2_invalidatepage(struct page *, unsigned int, unsigned int);
int bch2_releasepage(struct page *, gfp_t);
int bch2_migrate_page(struct address_space *, struct page *,
+// SPDX-License-Identifier: GPL-2.0
#ifndef NO_BCACHEFS_FS
#include "bcachefs.h"
if (ret)
goto err2;
- bch2_lock_inodes(src, dst);
+ bch2_lock_inodes(INODE_UPDATE_LOCK, src, dst);
if (inode_attr_changing(src, dst, Inode_opt_project)) {
ret = bch2_fs_quota_transfer(c, dst,
ret = bch2_write_inode(c, dst, bch2_reinherit_attrs_fn, src, 0);
err3:
- bch2_unlock_inodes(src, dst);
+ bch2_unlock_inodes(INODE_UPDATE_LOCK, src, dst);
/* return true if we did work */
if (ret >= 0)
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FS_IOCTL_H
#define _BCACHEFS_FS_IOCTL_H
+// SPDX-License-Identifier: GPL-2.0
#ifndef NO_BCACHEFS_FS
#include "bcachefs.h"
struct btree_trans trans;
int ret;
- bch2_lock_inodes(dir, inode);
+ bch2_lock_inodes(INODE_UPDATE_LOCK, dir, inode);
bch2_trans_init(&trans, c, 4, 1024);
retry:
bch2_trans_begin(&trans);
ATTR_MTIME);
err:
bch2_trans_exit(&trans);
- bch2_unlock_inodes(dir, inode);
+ bch2_unlock_inodes(INODE_UPDATE_LOCK, dir, inode);
return ret;
}
bch2_trans_init(&trans, c, 8, 2048);
- bch2_lock_inodes(i.src_dir,
+ bch2_lock_inodes(INODE_UPDATE_LOCK,
+ i.src_dir,
i.dst_dir,
i.src_inode,
i.dst_inode);
1 << QTYP_PRJ,
KEY_TYPE_QUOTA_NOCHECK);
- bch2_unlock_inodes(i.src_dir,
+ bch2_unlock_inodes(INODE_UPDATE_LOCK,
+ i.src_dir,
i.dst_dir,
i.src_inode,
i.dst_inode);
.readpage = bch2_readpage,
.writepages = bch2_writepages,
.readpages = bch2_readpages,
- .set_page_dirty = bch2_set_page_dirty,
+ .set_page_dirty = __set_page_dirty_nobuffers,
.write_begin = bch2_write_begin,
.write_end = bch2_write_end,
.invalidatepage = bch2_invalidatepage,
sb->s_bdi->congested_fn = bch2_congested;
sb->s_bdi->congested_data = c;
- sb->s_bdi->ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_SIZE;
+ sb->s_bdi->ra_pages = VM_READAHEAD_PAGES;
for_each_online_member(ca, c, i) {
struct block_device *bdev = ca->disk_sb.bdev;
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FS_H
#define _BCACHEFS_FS_H
return cmp_int(l, r);
}
-#define __bch2_lock_inodes(_lock, ...) \
+enum bch_inode_lock_op {
+ INODE_LOCK = (1U << 0),
+ INODE_UPDATE_LOCK = (1U << 1),
+};
+
+#define bch2_lock_inodes(_locks, ...) \
do { \
struct bch_inode_info *a[] = { NULL, __VA_ARGS__ }; \
unsigned i; \
\
- bubble_sort(&a[1], ARRAY_SIZE(a) - 1 , ptrcmp); \
+ bubble_sort(&a[1], ARRAY_SIZE(a) - 1, ptrcmp); \
\
- for (i = ARRAY_SIZE(a) - 1; a[i]; --i) \
+ for (i = 1; i < ARRAY_SIZE(a); i++) \
if (a[i] != a[i - 1]) { \
- if (_lock) \
+ if (_locks & INODE_LOCK) \
+ down_write_nested(&a[i]->v.i_rwsem, i); \
+ if (_locks & INODE_UPDATE_LOCK) \
mutex_lock_nested(&a[i]->ei_update_lock, i);\
- else \
- mutex_unlock(&a[i]->ei_update_lock); \
} \
} while (0)
-#define bch2_lock_inodes(...) __bch2_lock_inodes(true, __VA_ARGS__)
-#define bch2_unlock_inodes(...) __bch2_lock_inodes(false, __VA_ARGS__)
+#define bch2_unlock_inodes(_locks, ...) \
+do { \
+ struct bch_inode_info *a[] = { NULL, __VA_ARGS__ }; \
+ unsigned i; \
+ \
+ bubble_sort(&a[1], ARRAY_SIZE(a) - 1, ptrcmp); \
+ \
+ for (i = 1; i < ARRAY_SIZE(a); i++) \
+ if (a[i] != a[i - 1]) { \
+ if (_locks & INODE_LOCK) \
+ up_write(&a[i]->v.i_rwsem); \
+ if (_locks & INODE_UPDATE_LOCK) \
+ mutex_unlock(&a[i]->ei_update_lock); \
+ } \
+} while (0)
static inline struct bch_inode_info *file_bch_inode(struct file *file)
{
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "btree_update.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_FSCK_H
#define _BCACHEFS_FSCK_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_methods.h"
void bch2_inode_generation_to_text(struct printbuf *out, struct bch_fs *c,
struct bkey_s_c k)
{
+ struct bkey_s_c_inode_generation gen = bkey_s_c_to_inode_generation(k);
+
+ pr_buf(out, "generation: %u", le32_to_cpu(gen.v->bi_generation));
}
void bch2_inode_init(struct bch_fs *c, struct bch_inode_unpacked *inode_u,
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_INODE_H
#define _BCACHEFS_INODE_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Some low level IO code, and hacks for various block layer limitations
*
void bch2_bio_free_pages_pool(struct bch_fs *c, struct bio *bio)
{
+ struct bvec_iter_all iter;
struct bio_vec *bv;
unsigned i;
- bio_for_each_segment_all(bv, bio, i)
+ bio_for_each_segment_all(bv, bio, i, iter)
if (bv->bv_page != ZERO_PAGE(0))
mempool_free(bv->bv_page, &c->bio_bounce_pages);
bio->bi_vcnt = 0;
}
-static void bch2_bio_alloc_page_pool(struct bch_fs *c, struct bio *bio,
- bool *using_mempool)
+static struct page *__bio_alloc_page_pool(struct bch_fs *c, bool *using_mempool)
{
- struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt++];
+ struct page *page;
if (likely(!*using_mempool)) {
- bv->bv_page = alloc_page(GFP_NOIO);
- if (unlikely(!bv->bv_page)) {
+ page = alloc_page(GFP_NOIO);
+ if (unlikely(!page)) {
mutex_lock(&c->bio_bounce_pages_lock);
*using_mempool = true;
goto pool_alloc;
}
} else {
pool_alloc:
- bv->bv_page = mempool_alloc(&c->bio_bounce_pages, GFP_NOIO);
+ page = mempool_alloc(&c->bio_bounce_pages, GFP_NOIO);
}
- bv->bv_len = PAGE_SIZE;
- bv->bv_offset = 0;
+ return page;
}
void bch2_bio_alloc_pages_pool(struct bch_fs *c, struct bio *bio,
- size_t bytes)
+ size_t size)
{
bool using_mempool = false;
- BUG_ON(DIV_ROUND_UP(bytes, PAGE_SIZE) > bio->bi_max_vecs);
+ while (size) {
+ struct page *page = __bio_alloc_page_pool(c, &using_mempool);
+ unsigned len = min(PAGE_SIZE, size);
- bio->bi_iter.bi_size = bytes;
-
- while (bio->bi_vcnt < DIV_ROUND_UP(bytes, PAGE_SIZE))
- bch2_bio_alloc_page_pool(c, bio, &using_mempool);
+ BUG_ON(!bio_add_page(bio, page, len, 0));
+ size -= len;
+ }
if (using_mempool)
mutex_unlock(&c->bio_bounce_pages_lock);
}
-void bch2_bio_alloc_more_pages_pool(struct bch_fs *c, struct bio *bio,
- size_t bytes)
-{
- while (bio->bi_vcnt < DIV_ROUND_UP(bytes, PAGE_SIZE)) {
- struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
-
- BUG_ON(bio->bi_vcnt >= bio->bi_max_vecs);
-
- bv->bv_page = alloc_page(GFP_NOIO);
- if (!bv->bv_page) {
- /*
- * We already allocated from mempool, we can't allocate from it again
- * without freeing the pages we already allocated or else we could
- * deadlock:
- */
- bch2_bio_free_pages_pool(c, bio);
- bch2_bio_alloc_pages_pool(c, bio, bytes);
- return;
- }
-
- bv->bv_len = PAGE_SIZE;
- bv->bv_offset = 0;
- bio->bi_vcnt++;
- }
-
- bio->bi_iter.bi_size = bytes;
-}
-
/* Writes */
void bch2_submit_wbio_replicas(struct bch_write_bio *wbio, struct bch_fs *c,
wbio->bio.bi_opf = src->bi_opf;
if (buf) {
- bio->bi_iter.bi_size = output_available;
- bch2_bio_map(bio, buf);
+ bch2_bio_map(bio, buf, output_available);
return bio;
}
* We can't use mempool for more than c->sb.encoded_extent_max
* worth of pages, but we'd like to allocate more if we can:
*/
- while (bio->bi_iter.bi_size < output_available) {
- unsigned len = min_t(unsigned, PAGE_SIZE,
- output_available - bio->bi_iter.bi_size);
- struct page *p;
-
- p = alloc_page(GFP_NOIO);
- if (!p) {
- unsigned pool_max =
- min_t(unsigned, output_available,
- c->sb.encoded_extent_max << 9);
-
- if (bio_sectors(bio) < pool_max)
- bch2_bio_alloc_pages_pool(c, bio, pool_max);
- break;
- }
+ bch2_bio_alloc_pages_pool(c, bio,
+ min_t(unsigned, output_available,
+ c->sb.encoded_extent_max << 9));
- bio->bi_io_vec[bio->bi_vcnt++] = (struct bio_vec) {
- .bv_page = p,
- .bv_len = len,
- .bv_offset = 0,
- };
- bio->bi_iter.bi_size += len;
- }
+ if (bio->bi_iter.bi_size < output_available)
+ *page_alloc_failed =
+ bch2_bio_alloc_pages(bio,
+ output_available -
+ bio->bi_iter.bi_size,
+ GFP_NOFS) != 0;
- *page_alloc_failed = bio->bi_vcnt < pages;
return bio;
}
}
dst->bi_iter.bi_size = total_output;
-
- /* Free unneeded pages after compressing: */
- if (to_wbio(dst)->bounce)
- while (dst->bi_vcnt > DIV_ROUND_UP(dst->bi_iter.bi_size, PAGE_SIZE))
- mempool_free(dst->bi_io_vec[--dst->bi_vcnt].bv_page,
- &c->bio_bounce_pages);
do_write:
/* might have done a realloc... */
BUG_ON(!op->nr_replicas);
BUG_ON(!op->write_point.v);
BUG_ON(!bkey_cmp(op->pos, POS_MAX));
- BUG_ON(bio_sectors(&op->wbio.bio) > U16_MAX);
op->start_time = local_clock();
struct bch_io_opts opts,
unsigned flags)
{
- if (!opts.promote_target)
+ if (!bkey_extent_is_data(k.k))
return false;
if (!(flags & BCH_READ_MAY_PROMOTE))
return false;
- if (percpu_ref_is_dying(&c->writes))
- return false;
-
- if (!bkey_extent_is_data(k.k))
+ if (!opts.promote_target)
return false;
- if (bch2_extent_has_target(c, bkey_s_c_to_extent(k), opts.promote_target))
+ if (bch2_extent_has_target(c, bkey_s_c_to_extent(k),
+ opts.promote_target))
return false;
- if (bch2_target_congested(c, opts.promote_target))
+ if (bch2_target_congested(c, opts.promote_target)) {
+ /* XXX trace this */
return false;
+ }
if (rhashtable_lookup_fast(&c->promote_table, &pos,
bch_promote_params))
struct bpos pos,
struct extent_ptr_decoded *pick,
struct bch_io_opts opts,
- unsigned rbio_sectors,
+ unsigned sectors,
struct bch_read_bio **rbio)
{
struct promote_op *op = NULL;
struct bio *bio;
- unsigned rbio_pages = DIV_ROUND_UP(rbio_sectors, PAGE_SECTORS);
- /* data might have to be decompressed in the write path: */
- unsigned wbio_pages = DIV_ROUND_UP(pick->crc.uncompressed_size,
- PAGE_SECTORS);
+ unsigned pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
int ret;
if (!percpu_ref_tryget(&c->writes))
return NULL;
- op = kzalloc(sizeof(*op) + sizeof(struct bio_vec) * wbio_pages,
- GFP_NOIO);
+ op = kzalloc(sizeof(*op) + sizeof(struct bio_vec) * pages, GFP_NOIO);
if (!op)
goto err;
op->pos = pos;
/*
- * promotes require bouncing, but if the extent isn't
- * checksummed/compressed it might be too big for the mempool:
+ * We don't use the mempool here because extents that aren't
+ * checksummed or compressed can be too big for the mempool:
*/
- if (rbio_sectors > c->sb.encoded_extent_max) {
- *rbio = kzalloc(sizeof(struct bch_read_bio) +
- sizeof(struct bio_vec) * rbio_pages,
- GFP_NOIO);
- if (!*rbio)
- goto err;
+ *rbio = kzalloc(sizeof(struct bch_read_bio) +
+ sizeof(struct bio_vec) * pages,
+ GFP_NOIO);
+ if (!*rbio)
+ goto err;
- rbio_init(&(*rbio)->bio, opts);
- bio_init(&(*rbio)->bio, (*rbio)->bio.bi_inline_vecs,
- rbio_pages);
+ rbio_init(&(*rbio)->bio, opts);
+ bio_init(&(*rbio)->bio, (*rbio)->bio.bi_inline_vecs, pages);
- (*rbio)->bio.bi_iter.bi_size = rbio_sectors << 9;
- bch2_bio_map(&(*rbio)->bio, NULL);
+ if (bch2_bio_alloc_pages(&(*rbio)->bio, sectors << 9,
+ GFP_NOIO))
+ goto err;
- if (bch2_bio_alloc_pages(&(*rbio)->bio, GFP_NOIO))
- goto err;
-
- (*rbio)->bounce = true;
- (*rbio)->split = true;
- (*rbio)->kmalloc = true;
- }
+ (*rbio)->bounce = true;
+ (*rbio)->split = true;
+ (*rbio)->kmalloc = true;
if (rhashtable_lookup_insert_fast(&c->promote_table, &op->hash,
bch_promote_params))
goto err;
bio = &op->write.op.wbio.bio;
- bio_init(bio, bio->bi_inline_vecs, wbio_pages);
+ bio_init(bio, bio->bi_inline_vecs, pages);
ret = bch2_migrate_write_init(c, &op->write,
writepoint_hashed((unsigned long) current),
bool *read_full)
{
bool promote_full = *read_full || READ_ONCE(c->promote_whole_extents);
+ /* data might have to be decompressed in the write path: */
unsigned sectors = promote_full
- ? pick->crc.compressed_size
+ ? max(pick->crc.compressed_size, pick->crc.live_size)
: bvec_iter_sectors(iter);
struct bpos pos = promote_full
? bkey_start_pos(k.k)
}
if (rbio) {
- /* promote already allocated bounce rbio */
+ /*
+ * promote already allocated bounce rbio:
+ * promote needs to allocate a bio big enough for uncompressing
+ * data in the write path, but we're not going to use it all
+ * here:
+ */
+ BUG_ON(rbio->bio.bi_iter.bi_size <
+ pick.crc.compressed_size << 9);
+ rbio->bio.bi_iter.bi_size =
+ pick.crc.compressed_size << 9;
} else if (bounce) {
unsigned sectors = pick.crc.compressed_size;
bch2_increment_clock(c, bio_sectors(&rbio->bio), READ);
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
bucket_io_clock_reset(c, ca, PTR_BUCKET_NR(ca, &pick.ptr), READ);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
if (likely(!(flags & (BCH_READ_IN_RETRY|BCH_READ_LAST_FRAGMENT)))) {
bio_inc_remaining(&orig->bio);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_IO_H
#define _BCACHEFS_IO_H
void bch2_bio_free_pages_pool(struct bch_fs *, struct bio *);
void bch2_bio_alloc_pages_pool(struct bch_fs *, struct bio *, size_t);
-void bch2_bio_alloc_more_pages_pool(struct bch_fs *, struct bio *, size_t);
void bch2_latency_acct(struct bch_dev *, u64, int);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_IO_TYPES_H
#define _BCACHEFS_IO_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
/*
* bcachefs journalling code, for btree insertions
*
}
if (c) {
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
spin_lock(&c->journal.lock);
- } else {
- preempt_disable();
}
pos = ja->nr ? (ja->cur_idx + 1) % ja->nr : 0;
if (c) {
spin_unlock(&c->journal.lock);
- percpu_up_read_preempt_enable(&c->mark_lock);
- } else {
- preempt_enable();
+ percpu_up_read(&c->mark_lock);
}
if (!new_fs)
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_JOURNAL_H
#define _BCACHEFS_JOURNAL_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_foreground.h"
#include "buckets.h"
sectors_read << 9));
bio_set_dev(bio, ca->disk_sb.bdev);
bio->bi_iter.bi_sector = offset;
- bio->bi_iter.bi_size = sectors_read << 9;
bio_set_op_attrs(bio, REQ_OP_READ, 0);
- bch2_bio_map(bio, buf->data);
+ bch2_bio_map(bio, buf->data, sectors_read << 9);
ret = submit_bio_wait(bio);
bio_put(bio);
bio_reset(bio);
bio_set_dev(bio, ca->disk_sb.bdev);
bio->bi_iter.bi_sector = ptr->offset;
- bio->bi_iter.bi_size = sectors << 9;
bio->bi_end_io = journal_write_endio;
bio->bi_private = ca;
bio_set_op_attrs(bio, REQ_OP_WRITE,
REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA);
- bch2_bio_map(bio, jset);
+ bch2_bio_map(bio, jset, sectors << 9);
trace_journal_write(bio);
closure_bio_submit(bio, cl);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_JOURNAL_IO_H
#define _BCACHEFS_JOURNAL_IO_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "journal.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_JOURNAL_RECLAIM_H
#define _BCACHEFS_JOURNAL_RECLAIM_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "btree_iter.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_JOURNAL_SEQ_BLACKLIST_H
#define _BCACHEFS_JOURNAL_SEQ_BLACKLIST_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_JOURNAL_TYPES_H
#define _BCACHEFS_JOURNAL_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "keylist.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_KEYLIST_H
#define _BCACHEFS_KEYLIST_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_KEYLIST_TYPES_H
#define _BCACHEFS_KEYLIST_TYPES_H
+++ /dev/null
-#ifndef __BCH_LZ4_H__
-#define __BCH_LZ4_H__
-
-int bch2_lz4_decompress(const unsigned char *src, size_t *src_len,
- unsigned char *dest, size_t actual_dest_len);
-
-#endif
+++ /dev/null
-/*
- * LZ4 Decompressor for Linux kernel
- *
- * Copyright (C) 2013, LG Electronics, Kyungsik Lee <kyungsik.lee@lge.com>
- *
- * Based on LZ4 implementation by Yann Collet.
- *
- * LZ4 - Fast LZ compression algorithm
- * Copyright (C) 2011-2012, Yann Collet.
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * You can contact the author at :
- * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- * - LZ4 source repository : http://code.google.com/p/lz4/
- */
-
-#ifndef STATIC
-#include <linux/module.h>
-#include <linux/kernel.h>
-#endif
-
-#include "lz4.h"
-
-/*
- * Detects 64 bits mode
- */
-#if defined(CONFIG_64BIT)
-#define LZ4_ARCH64 1
-#else
-#define LZ4_ARCH64 0
-#endif
-
-#include <asm/unaligned.h>
-#include <linux/log2.h>
-#include <linux/string.h>
-
-#define A32(_p) get_unaligned((u32 *) (_p))
-#define A16(_p) get_unaligned((u16 *) (_p))
-
-#define GET_LE16_ADVANCE(_src) \
-({ \
- u16 _r = get_unaligned_le16(_src); \
- (_src) += 2; \
- _r; \
-})
-
-#define PUT_LE16_ADVANCE(_dst, _v) \
-do { \
- put_unaligned_le16((_v), (_dst)); \
- (_dst) += 2; \
-} while (0)
-
-#define LENGTH_LONG 15
-#define COPYLENGTH 8
-#define ML_BITS 4
-#define ML_MASK ((1U << ML_BITS) - 1)
-#define RUN_BITS (8 - ML_BITS)
-#define RUN_MASK ((1U << RUN_BITS) - 1)
-#define MEMORY_USAGE 14
-#define MINMATCH 4
-#define SKIPSTRENGTH 6
-#define LASTLITERALS 5
-#define MFLIMIT (COPYLENGTH + MINMATCH)
-#define MINLENGTH (MFLIMIT + 1)
-#define MAXD_LOG 16
-#define MAXD (1 << MAXD_LOG)
-#define MAXD_MASK (u32)(MAXD - 1)
-#define MAX_DISTANCE (MAXD - 1)
-#define HASH_LOG (MAXD_LOG - 1)
-#define HASHTABLESIZE (1 << HASH_LOG)
-#define MAX_NB_ATTEMPTS 256
-#define OPTIMAL_ML (int)((ML_MASK-1)+MINMATCH)
-#define LZ4_64KLIMIT ((1<<16) + (MFLIMIT - 1))
-
-#define __HASH_VALUE(p, bits) \
- (((A32(p)) * 2654435761U) >> (32 - (bits)))
-
-#define HASH_VALUE(p) __HASH_VALUE(p, HASH_LOG)
-
-#define MEMCPY_ADVANCE(_dst, _src, length) \
-do { \
- typeof(length) _length = (length); \
- memcpy(_dst, _src, _length); \
- _src += _length; \
- _dst += _length; \
-} while (0)
-
-#define MEMCPY_ADVANCE_BYTES(_dst, _src, _length) \
-do { \
- const u8 *_end = (_src) + (_length); \
- while ((_src) < _end) \
- *_dst++ = *_src++; \
-} while (0)
-
-#define STEPSIZE __SIZEOF_LONG__
-
-#define LZ4_COPYPACKET(_src, _dst) \
-do { \
- MEMCPY_ADVANCE(_dst, _src, STEPSIZE); \
- MEMCPY_ADVANCE(_dst, _src, COPYLENGTH - STEPSIZE);\
-} while (0)
-
-/*
- * Equivalent to MEMCPY_ADVANCE - except may overrun @_dst and @_src by
- * COPYLENGTH:
- *
- * Note: src and dst may overlap (with src < dst) - we must do the copy in
- * STEPSIZE chunks for correctness
- *
- * Note also: length may be negative - we must not call memcpy if length is
- * negative, but still adjust dst and src by length
- */
-#define MEMCPY_ADVANCE_CHUNKED(_dst, _src, _length) \
-do { \
- u8 *_end = (_dst) + (_length); \
- while ((_dst) < _end) \
- LZ4_COPYPACKET(_src, _dst); \
- _src -= (_dst) - _end; \
- _dst = _end; \
-} while (0)
-
-#define MEMCPY_ADVANCE_CHUNKED_NOFIXUP(_dst, _src, _end)\
-do { \
- while ((_dst) < (_end)) \
- LZ4_COPYPACKET((_src), (_dst)); \
-} while (0)
-
-static const int dec32table[8] = {0, 3, 2, 3, 0, 0, 0, 0};
-#if LZ4_ARCH64
-static const int dec64table[8] = {0, 0, 0, -1, 0, 1, 2, 3};
-#else
-static const int dec64table[8] = {0, 0, 0, 0, 0, 0, 0, 0};
-#endif
-
-static inline size_t get_length(const u8 **ip, size_t length)
-{
- if (length == LENGTH_LONG) {
- size_t len;
-
- do {
- length += (len = *(*ip)++);
- } while (len == 255);
- }
-
- return length;
-}
-
-static int lz4_uncompress(const u8 *source, u8 *dest, int osize)
-{
- const u8 *ip = source;
- const u8 *ref;
- u8 *op = dest;
- u8 * const oend = op + osize;
- u8 *cpy;
- unsigned token, offset;
- ssize_t length;
-
- while (1) {
- /* get runlength */
- token = *ip++;
- length = get_length(&ip, token >> ML_BITS);
-
- /* copy literals */
- if (unlikely(op + length > oend - COPYLENGTH)) {
- /*
- * Error: not enough place for another match
- * (min 4) + 5 literals
- */
- if (op + length != oend)
- goto _output_error;
-
- MEMCPY_ADVANCE(op, ip, length);
- break; /* EOF */
- }
- MEMCPY_ADVANCE_CHUNKED(op, ip, length);
-
- /* get match offset */
- offset = GET_LE16_ADVANCE(ip);
- ref = op - offset;
-
- /* Error: offset create reference outside destination buffer */
- if (unlikely(ref < (u8 *const) dest))
- goto _output_error;
-
- /* get match length */
- length = get_length(&ip, token & ML_MASK);
- length += MINMATCH;
-
- /* copy first STEPSIZE bytes of match: */
- if (unlikely(offset < STEPSIZE)) {
- MEMCPY_ADVANCE_BYTES(op, ref, 4);
- ref -= dec32table[offset];
-
- memcpy(op, ref, 4);
- op += STEPSIZE - 4;
- ref -= dec64table[offset];
- } else {
- MEMCPY_ADVANCE(op, ref, STEPSIZE);
- }
- length -= STEPSIZE;
- /*
- * Note - length could have been < STEPSIZE; that's ok, length
- * will now be negative and we'll just end up rewinding op:
- */
-
- /* copy rest of match: */
- cpy = op + length;
- if (cpy > oend - COPYLENGTH) {
- /* Error: request to write beyond destination buffer */
- if (cpy > oend ||
- ref + COPYLENGTH > oend)
- goto _output_error;
-#if !LZ4_ARCH64
- if (op + COPYLENGTH > oend)
- goto _output_error;
-#endif
- MEMCPY_ADVANCE_CHUNKED_NOFIXUP(op, ref, oend - COPYLENGTH);
- /* op could be > cpy here */
- while (op < cpy)
- *op++ = *ref++;
- op = cpy;
- /*
- * Check EOF (should never happen, since last 5 bytes
- * are supposed to be literals)
- */
- if (op == oend)
- goto _output_error;
- } else {
- MEMCPY_ADVANCE_CHUNKED(op, ref, length);
- }
- }
- /* end of decoding */
- return ip - source;
-
- /* write overflow error detected */
-_output_error:
- return -1;
-}
-
-int bch2_lz4_decompress(const unsigned char *src, size_t *src_len,
- unsigned char *dest, size_t actual_dest_len)
-{
- int ret = -1;
- int input_len = 0;
-
- input_len = lz4_uncompress(src, dest, actual_dest_len);
- if (input_len < 0)
- goto exit_0;
- *src_len = input_len;
-
- return 0;
-exit_0:
- return ret;
-}
+// SPDX-License-Identifier: GPL-2.0
/*
* Code for moving data off a device.
*/
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_MIGRATE_H
#define _BCACHEFS_MIGRATE_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_foreground.h"
{
struct moving_io *io = container_of(cl, struct moving_io, cl);
struct moving_context *ctxt = io->write.ctxt;
+ struct bvec_iter_all iter;
struct bio_vec *bv;
int i;
bch2_disk_reservation_put(io->write.op.c, &io->write.op.res);
- bio_for_each_segment_all(bv, &io->write.op.wbio.bio, i)
+ bio_for_each_segment_all(bv, &io->write.op.wbio.bio, i, iter)
if (bv->bv_page)
__free_page(bv->bv_page);
bio_init(&io->write.op.wbio.bio, io->bi_inline_vecs, pages);
bio_set_prio(&io->write.op.wbio.bio,
IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
- io->write.op.wbio.bio.bi_iter.bi_size = sectors << 9;
- bch2_bio_map(&io->write.op.wbio.bio, NULL);
- if (bch2_bio_alloc_pages(&io->write.op.wbio.bio, GFP_KERNEL))
+ if (bch2_bio_alloc_pages(&io->write.op.wbio.bio, sectors << 9,
+ GFP_KERNEL))
goto err_free;
io->rbio.opts = io_opts;
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_MOVE_H
#define _BCACHEFS_MOVE_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_MOVE_TYPES_H
#define _BCACHEFS_MOVE_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
/*
* Moving/copying garbage collector
*
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_MOVINGGC_H
#define _BCACHEFS_MOVINGGC_H
+// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_OPTS_H
#define _BCACHEFS_OPTS_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "btree_update.h"
#include "inode.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_QUOTA_H
#define _BCACHEFS_QUOTA_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_QUOTA_TYPES_H
#define _BCACHEFS_QUOTA_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_foreground.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_REBALANCE_H
#define _BCACHEFS_REBALANCE_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_REBALANCE_TYPES_H
#define _BCACHEFS_REBALANCE_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_background.h"
switch (entry->type) {
case BCH_JSET_ENTRY_btree_root: {
- struct btree_root *r = &c->btree_roots[entry->btree_id];
+ struct btree_root *r;
+
+ if (entry->btree_id >= BTREE_ID_NR) {
+ bch_err(c, "filesystem has unknown btree type %u",
+ entry->btree_id);
+ return -EINVAL;
+ }
+
+ r = &c->btree_roots[entry->btree_id];
if (entry->u64s) {
r->level = entry->level;
ret = bch2_blacklist_table_initialize(c);
- ret = verify_journal_entries_not_blacklisted_or_missing(c,
- &journal_entries);
- if (ret)
- goto err;
+ if (!list_empty(&journal_entries)) {
+ ret = verify_journal_entries_not_blacklisted_or_missing(c,
+ &journal_entries);
+ if (ret)
+ goto err;
+ }
ret = bch2_fs_journal_start(&c->journal, journal_seq,
&journal_entries);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_RECOVERY_H
#define _BCACHEFS_RECOVERY_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "buckets.h"
{
bool marked;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
marked = bch2_replicas_marked_locked(c, search, check_gc_replicas);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return marked;
}
{
bool marked;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
marked = bch2_bkey_replicas_marked_locked(c, k, check_gc_replicas);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return marked;
}
mi = bch2_sb_get_members(c->disk_sb.sb);
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
for_each_cpu_replicas_entry(&c->replicas, e) {
if (e->data_type >= ARRAY_SIZE(ret.replicas))
nr_offline);
}
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
for (i = 0; i < ARRAY_SIZE(ret.replicas); i++)
if (ret.replicas[i].redundancy == INT_MAX)
struct bch_replicas_entry *e;
unsigned i, ret = 0;
- percpu_down_read_preempt_disable(&c->mark_lock);
+ percpu_down_read(&c->mark_lock);
for_each_cpu_replicas_entry(&c->replicas, e)
for (i = 0; i < e->nr_devs; i++)
if (e->devs[i] == ca->dev_idx)
ret |= 1 << e->data_type;
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
return ret;
}
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_REPLICAS_H
#define _BCACHEFS_REPLICAS_H
+// SPDX-License-Identifier: BSD-3-Clause
/* $OpenBSD: siphash.c,v 1.3 2015/02/20 11:51:03 tedu Exp $ */
/*-
+/* SPDX-License-Identifier: BSD-3-Clause */
/* $OpenBSD: siphash.h,v 1.5 2015/02/20 11:51:03 tedu Exp $ */
/*-
* Copyright (c) 2013 Andre Oppermann <andre@FreeBSD.org>
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_STR_HASH_H
#define _BCACHEFS_STR_HASH_H
ctx->crc32c = crc32c(~0, &info->crc_key, sizeof(info->crc_key));
break;
case BCH_STR_HASH_CRC64:
- ctx->crc64 = bch2_crc64_update(~0, &info->crc_key, sizeof(info->crc_key));
+ ctx->crc64 = crc64_be(~0, &info->crc_key, sizeof(info->crc_key));
break;
case BCH_STR_HASH_SIPHASH:
SipHash24_Init(&ctx->siphash, &info->siphash_key);
ctx->crc32c = crc32c(ctx->crc32c, data, len);
break;
case BCH_STR_HASH_CRC64:
- ctx->crc64 = bch2_crc64_update(ctx->crc64, data, len);
+ ctx->crc64 = crc64_be(ctx->crc64, data, len);
break;
case BCH_STR_HASH_SIPHASH:
SipHash24_Update(&ctx->siphash, data, len);
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "buckets.h"
bio_reset(sb->bio);
bio_set_dev(sb->bio, sb->bdev);
sb->bio->bi_iter.bi_sector = offset;
- sb->bio->bi_iter.bi_size = PAGE_SIZE << sb->page_order;
bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
- bch2_bio_map(sb->bio, sb->sb);
+ bch2_bio_map(sb->bio, sb->sb, PAGE_SIZE << sb->page_order);
if (submit_bio_wait(sb->bio))
return "IO error";
bio_reset(sb->bio);
bio_set_dev(sb->bio, sb->bdev);
sb->bio->bi_iter.bi_sector = BCH_SB_LAYOUT_SECTOR;
- sb->bio->bi_iter.bi_size = sizeof(struct bch_sb_layout);
bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
/*
* use sb buffer to read layout, since sb buffer is page aligned but
* layout won't be:
*/
- bch2_bio_map(sb->bio, sb->sb);
+ bch2_bio_map(sb->bio, sb->sb, sizeof(struct bch_sb_layout));
err = "IO error";
if (submit_bio_wait(sb->bio))
bio_reset(bio);
bio_set_dev(bio, ca->disk_sb.bdev);
bio->bi_iter.bi_sector = le64_to_cpu(sb->layout.sb_offset[0]);
- bio->bi_iter.bi_size = PAGE_SIZE;
bio->bi_end_io = write_super_endio;
bio->bi_private = ca;
bio_set_op_attrs(bio, REQ_OP_READ, REQ_SYNC|REQ_META);
- bch2_bio_map(bio, ca->sb_read_scratch);
+ bch2_bio_map(bio, ca->sb_read_scratch, PAGE_SIZE);
this_cpu_add(ca->io_done->sectors[READ][BCH_DATA_SB],
bio_sectors(bio));
bio_reset(bio);
bio_set_dev(bio, ca->disk_sb.bdev);
bio->bi_iter.bi_sector = le64_to_cpu(sb->offset);
- bio->bi_iter.bi_size =
- roundup((size_t) vstruct_bytes(sb),
- bdev_logical_block_size(ca->disk_sb.bdev));
bio->bi_end_io = write_super_endio;
bio->bi_private = ca;
bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META);
- bch2_bio_map(bio, sb);
+ bch2_bio_map(bio, sb,
+ roundup((size_t) vstruct_bytes(sb),
+ bdev_logical_block_size(ca->disk_sb.bdev)));
this_cpu_add(ca->io_done->sectors[WRITE][BCH_DATA_SB],
bio_sectors(bio));
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_SUPER_IO_H
#define _BCACHEFS_SUPER_IO_H
+// SPDX-License-Identifier: GPL-2.0
/*
* bcachefs setup/teardown code, and some metadata io - read a superblock and
* figure out what to do with it.
schedule_delayed_work(&c->pd_controllers_update, 5 * HZ);
+ schedule_work(&c->ec_stripe_delete_work);
+
return 0;
}
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_SUPER_H
#define _BCACHEFS_SUPER_H
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_SUPER_TYPES_H
#define _BCACHEFS_SUPER_TYPES_H
+// SPDX-License-Identifier: GPL-2.0
/*
* bcache sysfs interfaces
*
bch2_fs_usage_to_text(&out, c, fs_usage);
- percpu_up_read_preempt_enable(&c->mark_lock);
+ percpu_up_read(&c->mark_lock);
kfree(fs_usage);
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_SYSFS_H_
#define _BCACHEFS_SYSFS_H_
+// SPDX-License-Identifier: GPL-2.0
#ifdef CONFIG_BCACHEFS_TESTS
#include "bcachefs.h"
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_TEST_H
#define _BCACHEFS_TEST_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "alloc_types.h"
#include "buckets.h"
+// SPDX-License-Identifier: GPL-2.0
/*
* random utiility code, for bcache but in theory not specific to bcache
*
/* misc: */
-void bch2_bio_map(struct bio *bio, void *base)
+void bch2_bio_map(struct bio *bio, void *base, size_t size)
{
- size_t size = bio->bi_iter.bi_size;
- struct bio_vec *bv = bio->bi_io_vec;
-
- BUG_ON(!bio->bi_iter.bi_size);
- BUG_ON(bio->bi_vcnt);
- BUG_ON(!bio->bi_max_vecs);
-
- bv->bv_offset = base ? offset_in_page(base) : 0;
- goto start;
-
- for (; size; bio->bi_vcnt++, bv++) {
- BUG_ON(bio->bi_vcnt >= bio->bi_max_vecs);
-
- bv->bv_offset = 0;
-start: bv->bv_len = min_t(size_t, PAGE_SIZE - bv->bv_offset,
- size);
- if (base) {
- bv->bv_page = is_vmalloc_addr(base)
+ while (size) {
+ struct page *page = is_vmalloc_addr(base)
? vmalloc_to_page(base)
: virt_to_page(base);
+ unsigned offset = offset_in_page(base);
+ unsigned len = min_t(size_t, PAGE_SIZE - offset, size);
- base += bv->bv_len;
- }
-
- size -= bv->bv_len;
+ BUG_ON(!bio_add_page(bio, page, len, offset));
+ size -= len;
+ base += len;
}
}
-int bch2_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask)
+int bch2_bio_alloc_pages(struct bio *bio, size_t size, gfp_t gfp_mask)
{
- int i;
- struct bio_vec *bv;
+ while (size) {
+ struct page *page = alloc_page(gfp_mask);
+ unsigned len = min(PAGE_SIZE, size);
- bio_for_each_segment_all(bv, bio, i) {
- bv->bv_page = alloc_page(gfp_mask);
- if (!bv->bv_page) {
- while (--bv >= bio->bi_io_vec)
- __free_page(bv->bv_page);
+ if (!page)
return -ENOMEM;
- }
+
+ BUG_ON(!bio_add_page(bio, page, len, 0));
+ size -= len;
}
return 0;
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_UTIL_H
#define _BCACHEFS_UTIL_H
return x;
}
-void bch2_bio_map(struct bio *bio, void *base);
-int bch2_bio_alloc_pages(struct bio *bio, gfp_t gfp_mask);
+void bch2_bio_map(struct bio *bio, void *base, size_t);
+int bch2_bio_alloc_pages(struct bio *, size_t, gfp_t);
static inline sector_t bdev_sectors(struct block_device *bdev)
{
__memmove_u64s_up(dst, src, u64s);
}
-static inline struct bio_vec next_contig_bvec(struct bio *bio,
- struct bvec_iter *iter)
-{
- struct bio_vec bv = bio_iter_iovec(bio, *iter);
-
- bio_advance_iter(bio, iter, bv.bv_len);
-#ifndef CONFIG_HIGHMEM
- while (iter->bi_size) {
- struct bio_vec next = bio_iter_iovec(bio, *iter);
-
- if (page_address(bv.bv_page) + bv.bv_offset + bv.bv_len !=
- page_address(next.bv_page) + next.bv_offset)
- break;
-
- bv.bv_len += next.bv_len;
- bio_advance_iter(bio, iter, next.bv_len);
- }
-#endif
- return bv;
-}
-
-#define __bio_for_each_contig_segment(bv, bio, iter, start) \
- for (iter = (start); \
- (iter).bi_size && \
- ((bv = next_contig_bvec((bio), &(iter))), 1);)
-
-#define bio_for_each_contig_segment(bv, bio, iter) \
- __bio_for_each_contig_segment(bv, bio, iter, (bio)->bi_iter)
-
void sort_cmp_size(void *base, size_t num, size_t size,
int (*cmp_func)(const void *, const void *, size_t),
void (*swap_func)(void *, void *, size_t));
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _VSTRUCTS_H
#define _VSTRUCTS_H
+// SPDX-License-Identifier: GPL-2.0
#include "bcachefs.h"
#include "bkey_methods.h"
bch2_inode_opts_to_opts(bch2_inode_opts_get(&inode->ei_inode));
const struct bch_option *opt;
int id, inode_opt_id;
+ char buf[512];
+ struct printbuf out = PBUF(buf);
+ unsigned val_len;
u64 v;
id = bch2_opt_lookup(name);
return -ENODATA;
v = bch2_opt_get_by_id(&opts, id);
+ bch2_opt_to_text(&out, c, opt, v, 0);
- if (!buffer) {
- char buf[512];
- struct printbuf out = PBUF(buf);
+ val_len = out.pos - buf;
- bch2_opt_to_text(&out, c, opt, v, 0);
+ if (buffer && val_len > size)
+ return -ERANGE;
- return out.pos - buf;
- } else {
- struct printbuf out = _PBUF(buffer, size);
-
- bch2_opt_to_text(&out, c, opt, v, 0);
-
- return printbuf_remaining(&out)
- ? (void *) out.pos - buffer
- : -ERANGE;
- }
+ if (buffer)
+ memcpy(buffer, buf, val_len);
+ return val_len;
}
static int bch2_xattr_bcachefs_get(const struct xattr_handler *handler,
+/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _BCACHEFS_XATTR_H
#define _BCACHEFS_XATTR_H
void bio_free_pages(struct bio *bio)
{
+ struct bvec_iter_all iter;
struct bio_vec *bvec;
int i;
- bio_for_each_segment_all(bvec, bio, i)
+ bio_for_each_segment_all(bvec, bio, i, iter)
__free_page(bvec->bv_page);
}
}
}
+int bio_add_page(struct bio *bio, struct page *page,
+ unsigned int len, unsigned int off)
+{
+ struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
+
+ WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
+ WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs);
+
+ bv->bv_page = page;
+ bv->bv_offset = off;
+ bv->bv_len = len;
+
+ bio->bi_iter.bi_size += len;
+ bio->bi_vcnt++;
+ return len;
+}
+
static inline bool bio_remaining_done(struct bio *bio)
{
/*
projects / bcachefs-tools-debian / commitdiff