-BCACHE_REVISION=76e3b2312705df2cb5adb8834bc6df56a288932e
+BCACHE_REVISION=561f3067172cbfc63a680cfb670d558724441123
LDFLAGS+=-flto
endif
-PKGCONFIG_LIBS="blkid uuid liburcu"
+PKGCONFIG_LIBS="blkid uuid liburcu libsodium"
CFLAGS+=`pkg-config --cflags ${PKGCONFIG_LIBS}`
-LDLIBS+=`pkg-config --libs ${PKGCONFIG_LIBS}` -lm -lpthread -lrt
+LDLIBS+=`pkg-config --libs ${PKGCONFIG_LIBS}` \
+ -lm -lpthread -lrt -lscrypt -lkeyutils
ifeq ($(PREFIX),/usr)
ROOT_SBINDIR=/sbin
cmd_fs.o \
cmd_fsck.o \
cmd_format.o \
+ cmd_key.o \
cmd_run.o \
+ crypto.o \
libbcache.o \
qcow2.o \
tools-util.o \
#include "six.c"
//#include "stats.c"
#include "super.c"
+#include "super-io.c"
//#include "sysfs.c"
#include "tier.c"
#include "trace.c"
"\n"
"Commands for formatting, startup and shutdown:\n"
" format Format a new filesystem\n"
+ " unlock Unlock an encrypted filesystem prior to running/mounting\n"
" assemble Assemble an existing multi device filesystem\n"
" incremental Incrementally assemble an existing multi device filesystem\n"
" run Start a partially assembled filesystem\n"
"\n"
"Repair:\n"
" bcache fsck Check an existing filesystem for errors\n"
+ "\n"
"Debug:\n"
" bcache dump Dump filesystem metadata to a qcow2 image\n"
" bcache list List filesystem metadata in textual form\n");
if (!strcmp(cmd, "fsck"))
return cmd_fsck(argc, argv);
+ if (!strcmp(cmd, "unlock"))
+ return cmd_unlock(argc, argv);
+
if (!strcmp(cmd, "dump"))
return cmd_dump(argc, argv);
if (!strcmp(cmd, "list"))
"Report bugs to <linux-bcache@vger.kernel.org>");
}
-void dump_one_device(struct cache_set *c, struct cache *ca, int fd)
+static void dump_one_device(struct cache_set *c, struct cache *ca, int fd)
{
- struct cache_sb *sb = ca->disk_sb.sb;
+ struct bch_sb *sb = ca->disk_sb.sb;
sparse_data data;
unsigned i;
darray_init(data);
/* Superblock: */
- data_add(&data, SB_SECTOR << 9, __set_bytes(sb, le16_to_cpu(sb->u64s)));
+ data_add(&data, BCH_SB_LAYOUT_SECTOR << 9,
+ sizeof(struct bch_sb_layout));
+
+ for (i = 0; i < sb->layout.nr_superblocks; i++)
+ data_add(&data,
+ le64_to_cpu(sb->layout.sb_offset[i]) << 9,
+ vstruct_bytes(sb));
/* Journal: */
- for (i = 0; i < bch_nr_journal_buckets(ca->disk_sb.sb); i++)
+ for (i = 0; i < ca->journal.nr; i++)
if (ca->journal.bucket_seq[i] >= c->journal.last_seq_ondisk) {
- u64 bucket = journal_bucket(ca->disk_sb.sb, i);
+ u64 bucket = ca->journal.buckets[i];
data_add(&data,
bucket_bytes(ca) * bucket,
struct bkey_s_c_extent e = bkey_i_to_s_c_extent(&b->key);
extent_for_each_ptr(e, ptr)
- if (ptr->dev == ca->sb.nr_this_dev)
+ if (ptr->dev == ca->dev_idx)
data_add(&data,
ptr->offset << 9,
b->written << 9);
down_read(&c->gc_lock);
- for (i = 0; i < c->sb.nr_in_set; i++)
+ for (i = 0; i < c->sb.nr_devices; i++)
if (c->cache[i])
nr_devices++;
BUG_ON(!nr_devices);
- for (i = 0; i < c->sb.nr_in_set; i++) {
+ for (i = 0; i < c->sb.nr_devices; i++) {
int mode = O_WRONLY|O_CREAT|O_TRUNC;
if (!force)
return 0;
}
-void list_keys(struct cache_set *c, enum btree_id btree_id,
- struct bpos start, struct bpos end, int mode)
+static void list_keys(struct cache_set *c, enum btree_id btree_id,
+ struct bpos start, struct bpos end, int mode)
{
struct btree_iter iter;
struct bkey_s_c k;
bch_btree_iter_unlock(&iter);
}
-void list_btree_formats(struct cache_set *c, enum btree_id btree_id,
- struct bpos start, struct bpos end, int mode)
+static void list_btree_formats(struct cache_set *c, enum btree_id btree_id,
+ struct bpos start, struct bpos end, int mode)
{
struct btree_iter iter;
struct btree *b;
bch_btree_iter_unlock(&iter);
}
-struct bpos parse_pos(char *buf)
+static struct bpos parse_pos(char *buf)
{
char *s = buf;
char *inode = strsep(&s, ":");
struct bcache_dev devices[256];
unsigned i, j, nr_devices = 0, nr_active_tiers = 0;
- unsigned tiers[CACHE_TIERS]; /* number of devices in each tier */
+ unsigned tiers[BCH_TIER_MAX]; /* number of devices in each tier */
memset(tiers, 0, sizeof(tiers));
while ((entry = readdir(fs.sysfs))) {
close(fd);
}
- for (i = 0; i < CACHE_TIERS; i++)
+ for (i = 0; i < BCH_TIER_MAX; i++)
if (tiers[i])
nr_active_tiers++;
/* Print out devices sorted by tier: */
bool first = true;
- for (i = 0; i < CACHE_TIERS; i++) {
+ for (i = 0; i < BCH_TIER_MAX; i++) {
if (!tiers[i])
continue;
int cmd_device_show(int argc, char *argv[])
{
- struct cache_sb *sb;
+ struct bch_sb *sb;
if (argc != 2)
die("please supply a single device");
#include "cmds.h"
#include "libbcache.h"
+#include "crypto.h"
#include "opts.h"
#include "util.h"
" --metadata_checksum_type=(none|crc32c|crc64)\n"
" --data_checksum_type=(none|crc32c|crc64)\n"
" --compression_type=(none|lz4|gzip)\n"
+ " --encrypted\n"
" --error_action=(continue|readonly|panic)\n"
" Action to take on filesystem error\n"
" --max_journal_entry_size=size\n"
OPT(0, metadata_checksum_type, required_argument) \
OPT(0, data_checksum_type, required_argument) \
OPT(0, compression_type, required_argument) \
+ OPT(0, encrypted, no_argument) \
OPT('e', error_action, required_argument) \
OPT(0, max_journal_entry_size, required_argument) \
OPT('L', label, required_argument) \
unsigned meta_csum_type = BCH_CSUM_CRC32C;
unsigned data_csum_type = BCH_CSUM_CRC32C;
unsigned compression_type = BCH_COMPRESSION_NONE;
+ bool encrypted = false;
unsigned on_error_action = BCH_ON_ERROR_RO;
char *label = NULL;
uuid_le uuid;
bch_compression_types,
"compression type");
break;
+ case Opt_encrypted:
+ encrypted = true;
+ break;
case Opt_error_action:
case 'e':
on_error_action = read_string_list_or_die(optarg,
case Opt_tier:
case 't':
if (kstrtouint(optarg, 10, &tier) ||
- tier >= CACHE_TIERS)
+ tier >= BCH_TIER_MAX)
die("invalid tier");
break;
case Opt_discard:
if (uuid_is_null(uuid.b))
uuid_generate(uuid.b);
+ if (encrypted) {
+ passphrase = read_passphrase("Enter passphrase: ");
+
+ if (isatty(STDIN_FILENO)) {
+ char *pass2 =
+ read_passphrase("Enter same passphrase again: ");
+
+ if (strcmp(passphrase, pass2)) {
+ memzero_explicit(passphrase, strlen(passphrase));
+ memzero_explicit(pass2, strlen(pass2));
+ die("Passphrases do not match");
+ }
+
+ memzero_explicit(pass2, strlen(pass2));
+ free(pass2);
+ }
+ }
+
darray_foreach(dev, devices)
dev->fd = open_for_format(dev->path, force);
meta_csum_type,
data_csum_type,
compression_type,
+ passphrase,
1,
1,
on_error_action,
--- /dev/null
+#include <errno.h>
+#include <unistd.h>
+#include <keyutils.h>
+#include <uuid/uuid.h>
+
+#include "cmds.h"
+#include "checksum.h"
+#include "crypto.h"
+#include "libbcache.h"
+
+int cmd_unlock(int argc, char *argv[])
+{
+ struct bch_encrypted_key sb_key;
+ struct bch_key passphrase_key;
+ struct bch_sb *sb;
+ struct bch_sb_field_crypt *crypt;
+ char *passphrase;
+ char uuid[40];
+ char description[60];
+
+ if (argc != 2)
+ die("please supply a single device");
+
+ sb = bcache_super_read(argv[1]);
+
+ crypt = bch_sb_get_crypt(sb);
+ if (!crypt)
+ die("filesystem is not encrypted");
+
+ sb_key = crypt->key;
+
+ if (!bch_key_is_encrypted(&sb_key))
+ die("filesystem does not have encryption key");
+
+ passphrase = read_passphrase("Enter passphrase: ");
+ derive_passphrase(crypt, &passphrase_key, passphrase);
+
+ /* Check if the user supplied the correct passphrase: */
+ if (bch_chacha_encrypt_key(&passphrase_key, __bch_sb_key_nonce(sb),
+ &sb_key, sizeof(sb_key)))
+ die("error encrypting key");
+
+ if (bch_key_is_encrypted(&sb_key))
+ die("incorrect passphrase");
+
+ uuid_unparse_lower(sb->user_uuid.b, uuid);
+ sprintf(description, "bcache:%s", uuid);
+
+ if (add_key("logon", description,
+ &passphrase_key, sizeof(passphrase_key),
+ KEY_SPEC_USER_KEYRING) < 0 ||
+ add_key("user", description,
+ &passphrase_key, sizeof(passphrase_key),
+ KEY_SPEC_USER_KEYRING) < 0)
+ die("add_key error: %s", strerror(errno));
+
+ memzero_explicit(&sb_key, sizeof(sb_key));
+ memzero_explicit(&passphrase_key, sizeof(passphrase_key));
+ memzero_explicit(passphrase, strlen(passphrase));
+ free(passphrase);
+ return 0;
+}
int cmd_format(int argc, char *argv[]);
+int cmd_unlock(int argc, char *argv[]);
int cmd_assemble(int argc, char *argv[]);
int cmd_incremental(int argc, char *argv[]);
int cmd_run(int argc, char *argv[]);
--- /dev/null
+#include <errno.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <termios.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <linux/random.h>
+#include <libscrypt.h>
+
+#include "checksum.h"
+#include "crypto.h"
+
+char *read_passphrase(const char *prompt)
+{
+ char *buf = NULL;
+ size_t buflen = 0;
+ ssize_t len;
+
+ if (isatty(STDIN_FILENO)) {
+ struct termios old, new;
+
+ fprintf(stderr, "%s", prompt);
+ fflush(stderr);
+
+ if (tcgetattr(STDIN_FILENO, &old))
+ die("error getting terminal attrs");
+
+ new = old;
+ new.c_lflag &= ~ECHO;
+ if (tcsetattr(STDIN_FILENO, TCSAFLUSH, &new))
+ die("error setting terminal attrs");
+
+ len = getline(&buf, &buflen, stdin);
+
+ tcsetattr(STDIN_FILENO, TCSAFLUSH, &old);
+ fprintf(stderr, "\n");
+ } else {
+ len = getline(&buf, &buflen, stdin);
+ }
+
+ if (len < 0)
+ die("error reading passphrase");
+ if (len && buf[len - 1] == '\n')
+ buf[len - 1] = '\0';
+
+ return buf;
+}
+
+void derive_passphrase(struct bch_sb_field_crypt *crypt,
+ struct bch_key *key,
+ const char *passphrase)
+{
+ const unsigned char salt[] = "bcache";
+ int ret;
+
+ switch (BCH_CRYPT_KDF_TYPE(crypt)) {
+ case BCH_KDF_SCRYPT:
+ ret = libscrypt_scrypt((void *) passphrase, strlen(passphrase),
+ salt, sizeof(salt),
+ 1ULL << BCH_KDF_SCRYPT_N(crypt),
+ 1ULL << BCH_KDF_SCRYPT_R(crypt),
+ 1ULL << BCH_KDF_SCRYPT_P(crypt),
+ (void *) key, sizeof(*key));
+ if (ret)
+ die("scrypt error: %i", ret);
+ break;
+ default:
+ die("unknown kdf type %llu", BCH_CRYPT_KDF_TYPE(crypt));
+ }
+}
+
+void bch_sb_crypt_init(struct bch_sb *sb,
+ struct bch_sb_field_crypt *crypt,
+ const char *passphrase)
+{
+ struct bch_key passphrase_key;
+
+ SET_BCH_CRYPT_KDF_TYPE(crypt, BCH_KDF_SCRYPT);
+ SET_BCH_KDF_SCRYPT_N(crypt, ilog2(SCRYPT_N));
+ SET_BCH_KDF_SCRYPT_R(crypt, ilog2(SCRYPT_r));
+ SET_BCH_KDF_SCRYPT_P(crypt, ilog2(SCRYPT_p));
+
+ derive_passphrase(crypt, &passphrase_key, passphrase);
+
+ crypt->key.magic = BCH_KEY_MAGIC;
+ get_random_bytes(&crypt->key.key, sizeof(crypt->key.key));
+
+ assert(!bch_key_is_encrypted(&crypt->key));
+
+ if (bch_chacha_encrypt_key(&passphrase_key, __bch_sb_key_nonce(sb),
+ &crypt->key, sizeof(crypt->key)))
+ die("error encrypting key");
+
+ assert(bch_key_is_encrypted(&crypt->key));
+
+ memzero_explicit(&passphrase_key, sizeof(passphrase_key));
+}
--- /dev/null
+#ifndef _CRYPTO_H
+#define _CRYPTO_H
+
+#include "super-io.h"
+#include "tools-util.h"
+
+char *read_passphrase(const char *);
+void derive_passphrase(struct bch_sb_field_crypt *,
+ struct bch_key *, const char *);
+void bch_sb_crypt_init(struct bch_sb *sb, struct bch_sb_field_crypt *,
+ const char *);
+
+#endif /* _CRYPTO_H */
#define _CRYPTO_ALGAPI_H
#include <linux/crypto.h>
-#include <linux/device.h>
-#include <linux/list.h>
-#include <linux/kernel.h>
-#include <linux/kthread.h>
-
-struct crypto_aead;
-struct crypto_instance;
-struct module;
-struct rtattr;
-struct seq_file;
-struct sk_buff;
struct crypto_type {
unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask);
unsigned int (*extsize)(struct crypto_alg *alg);
int (*init)(struct crypto_tfm *tfm, u32 type, u32 mask);
int (*init_tfm)(struct crypto_tfm *tfm);
- void (*show)(struct seq_file *m, struct crypto_alg *alg);
- struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask);
- void (*free)(struct crypto_instance *inst);
-
- unsigned int type;
- unsigned int maskclear;
- unsigned int maskset;
- unsigned int tfmsize;
-};
-
-struct crypto_instance {
- struct crypto_alg alg;
-
- struct crypto_template *tmpl;
- struct hlist_node list;
-
- void *__ctx[] CRYPTO_MINALIGN_ATTR;
-};
-
-struct crypto_template {
- struct list_head list;
- struct hlist_head instances;
- struct module *module;
-
- struct crypto_instance *(*alloc)(struct rtattr **tb);
- void (*free)(struct crypto_instance *inst);
- int (*create)(struct crypto_template *tmpl, struct rtattr **tb);
-
- char name[CRYPTO_MAX_ALG_NAME];
-};
-
-struct scatter_walk {
- struct scatterlist *sg;
- unsigned int offset;
-};
-
-struct blkcipher_walk {
- union {
- struct {
- struct page *page;
- unsigned long offset;
- } phys;
-
- struct {
- u8 *page;
- u8 *addr;
- } virt;
- } src, dst;
- struct scatter_walk in;
- unsigned int nbytes;
-
- struct scatter_walk out;
- unsigned int total;
-
- void *page;
- u8 *buffer;
- u8 *iv;
- unsigned int ivsize;
-
- int flags;
- unsigned int walk_blocksize;
- unsigned int cipher_blocksize;
- unsigned int alignmask;
+ unsigned type;
+ unsigned maskclear;
+ unsigned maskset;
+ unsigned tfmsize;
};
extern const struct crypto_type crypto_blkcipher_type;
-struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb);
-int crypto_check_attr_type(struct rtattr **tb, u32 type);
-const char *crypto_attr_alg_name(struct rtattr *rta);
-struct crypto_alg *crypto_attr_alg2(struct rtattr *rta,
- const struct crypto_type *frontend,
- u32 type, u32 mask);
-
-static inline struct crypto_alg *crypto_attr_alg(struct rtattr *rta,
- u32 type, u32 mask)
-{
- return crypto_attr_alg2(rta, NULL, type, mask);
-}
-
-int crypto_attr_u32(struct rtattr *rta, u32 *num);
-
-/* These functions require the input/output to be aligned as u32. */
-void crypto_inc(u8 *a, unsigned int size);
-void crypto_xor(u8 *dst, const u8 *src, unsigned int size);
-
-int blkcipher_walk_done(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk, int err);
-int blkcipher_walk_virt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk);
-int blkcipher_walk_phys(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk);
-int blkcipher_walk_virt_block(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk,
- unsigned int blocksize);
-int blkcipher_aead_walk_virt_block(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk,
- struct crypto_aead *tfm,
- unsigned int blocksize);
-
-static inline void *crypto_tfm_ctx_aligned(struct crypto_tfm *tfm)
-{
- return PTR_ALIGN(crypto_tfm_ctx(tfm),
- crypto_tfm_alg_alignmask(tfm) + 1);
-}
-
-static inline struct crypto_instance *crypto_tfm_alg_instance(
- struct crypto_tfm *tfm)
-{
- return container_of(tfm->__crt_alg, struct crypto_instance, alg);
-}
-
-static inline void *crypto_instance_ctx(struct crypto_instance *inst)
-{
- return inst->__ctx;
-}
-
static inline void *crypto_blkcipher_ctx(struct crypto_blkcipher *tfm)
{
return crypto_tfm_ctx(&tfm->base);
}
-static inline void *crypto_blkcipher_ctx_aligned(struct crypto_blkcipher *tfm)
-{
- return crypto_tfm_ctx_aligned(&tfm->base);
-}
-
-static inline struct cipher_alg *crypto_cipher_alg(struct crypto_cipher *tfm)
-{
- return &crypto_cipher_tfm(tfm)->__crt_alg->cra_cipher;
-}
-
-static inline void blkcipher_walk_init(struct blkcipher_walk *walk,
- struct scatterlist *dst,
- struct scatterlist *src,
- unsigned int nbytes)
-{
- walk->in.sg = src;
- walk->out.sg = dst;
- walk->total = nbytes;
-}
-
-static inline struct crypto_alg *crypto_get_attr_alg(struct rtattr **tb,
- u32 type, u32 mask)
-{
- return crypto_attr_alg(tb[1], type, mask);
-}
-
-static inline int crypto_requires_sync(u32 type, u32 mask)
-{
- return (type ^ CRYPTO_ALG_ASYNC) & mask & CRYPTO_ALG_ASYNC;
-}
-
-noinline unsigned long __crypto_memneq(const void *a, const void *b, size_t size);
-
-/**
- * crypto_memneq - Compare two areas of memory without leaking
- * timing information.
- *
- * @a: One area of memory
- * @b: Another area of memory
- * @size: The size of the area.
- *
- * Returns 0 when data is equal, 1 otherwise.
- */
-static inline int crypto_memneq(const void *a, const void *b, size_t size)
-{
- return __crypto_memneq(a, b, size) != 0UL ? 1 : 0;
-}
-
-static inline void crypto_yield(u32 flags)
-{
-#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PREEMPT_VOLUNTARY)
- if (flags & CRYPTO_TFM_REQ_MAY_SLEEP)
- cond_resched();
-#endif
-}
-
#endif /* _CRYPTO_ALGAPI_H */
#define CHACHA20_KEY_SIZE 32
#define CHACHA20_BLOCK_SIZE 64
-struct chacha20_ctx {
- u32 key[8];
-};
-
-void chacha20_block(u32 *state, void *stream);
-void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv);
-int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keysize);
-int crypto_chacha20_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes);
-
#endif
#include <linux/crypto.h>
#include <linux/string.h>
-struct hash_alg_common {
- unsigned int digestsize;
- unsigned int statesize;
-
- struct crypto_alg base;
-};
-
struct shash_desc {
struct crypto_shash *tfm;
u32 flags;
struct shash_alg {
int (*init)(struct shash_desc *desc);
- int (*update)(struct shash_desc *desc, const u8 *data,
- unsigned int len);
+ int (*update)(struct shash_desc *desc, const u8 *data, unsigned len);
int (*final)(struct shash_desc *desc, u8 *out);
int (*finup)(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out);
+ unsigned len, u8 *out);
int (*digest)(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out);
- int (*export)(struct shash_desc *desc, void *out);
- int (*import)(struct shash_desc *desc, const void *in);
- int (*setkey)(struct crypto_shash *tfm, const u8 *key,
- unsigned int keylen);
-
- unsigned int descsize;
-
- /* These fields must match hash_alg_common. */
- unsigned int digestsize
- __attribute__ ((aligned(__alignof__(struct hash_alg_common))));
- unsigned int statesize;
+ unsigned len, u8 *out);
- struct crypto_alg base;
+ unsigned descsize;
+ unsigned digestsize;
+ struct crypto_alg base;
};
struct crypto_shash {
- unsigned int descsize;
- struct crypto_tfm base;
+ unsigned descsize;
+ struct crypto_tfm base;
};
struct crypto_shash *crypto_alloc_shash(const char *alg_name, u32 type,
crypto_destroy_tfm(tfm, crypto_shash_tfm(tfm));
}
-static inline const char *crypto_shash_alg_name(struct crypto_shash *tfm)
-{
- return crypto_tfm_alg_name(crypto_shash_tfm(tfm));
-}
-
-static inline const char *crypto_shash_driver_name(struct crypto_shash *tfm)
-{
- return crypto_tfm_alg_driver_name(crypto_shash_tfm(tfm));
-}
-
-static inline unsigned int crypto_shash_alignmask(
- struct crypto_shash *tfm)
-{
- return crypto_tfm_alg_alignmask(crypto_shash_tfm(tfm));
-}
-
-static inline unsigned int crypto_shash_blocksize(struct crypto_shash *tfm)
-{
- return crypto_tfm_alg_blocksize(crypto_shash_tfm(tfm));
-}
-
static inline struct shash_alg *__crypto_shash_alg(struct crypto_alg *alg)
{
return container_of(alg, struct shash_alg, base);
return __crypto_shash_alg(crypto_shash_tfm(tfm)->__crt_alg);
}
-static inline unsigned int crypto_shash_digestsize(struct crypto_shash *tfm)
+static inline unsigned crypto_shash_digestsize(struct crypto_shash *tfm)
{
return crypto_shash_alg(tfm)->digestsize;
}
-static inline unsigned int crypto_shash_statesize(struct crypto_shash *tfm)
-{
- return crypto_shash_alg(tfm)->statesize;
-}
-
-static inline u32 crypto_shash_get_flags(struct crypto_shash *tfm)
-{
- return crypto_tfm_get_flags(crypto_shash_tfm(tfm));
-}
-
-static inline void crypto_shash_set_flags(struct crypto_shash *tfm, u32 flags)
-{
- crypto_tfm_set_flags(crypto_shash_tfm(tfm), flags);
-}
-
-static inline void crypto_shash_clear_flags(struct crypto_shash *tfm, u32 flags)
-{
- crypto_tfm_clear_flags(crypto_shash_tfm(tfm), flags);
-}
-
-static inline unsigned int crypto_shash_descsize(struct crypto_shash *tfm)
+static inline unsigned crypto_shash_descsize(struct crypto_shash *tfm)
{
return tfm->descsize;
}
return desc->__ctx;
}
-int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key,
- unsigned int keylen);
-
-int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out);
-
-static inline int crypto_shash_export(struct shash_desc *desc, void *out)
+static inline int crypto_shash_init(struct shash_desc *desc)
{
- return crypto_shash_alg(desc->tfm)->export(desc, out);
+ return crypto_shash_alg(desc->tfm)->init(desc);
}
-static inline int crypto_shash_import(struct shash_desc *desc, const void *in)
+static inline int crypto_shash_update(struct shash_desc *desc,
+ const u8 *data, unsigned len)
{
- return crypto_shash_alg(desc->tfm)->import(desc, in);
+ return crypto_shash_alg(desc->tfm)->update(desc, data, len);
}
-static inline int crypto_shash_init(struct shash_desc *desc)
+static inline int crypto_shash_final(struct shash_desc *desc, u8 *out)
{
- return crypto_shash_alg(desc->tfm)->init(desc);
+ return crypto_shash_alg(desc->tfm)->final(desc, out);
}
-int crypto_shash_update(struct shash_desc *desc, const u8 *data,
- unsigned int len);
-
-int crypto_shash_final(struct shash_desc *desc, u8 *out);
-
-int crypto_shash_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out);
+static inline int crypto_shash_finup(struct shash_desc *desc, const u8 *data,
+ unsigned len, u8 *out)
+{
+ return crypto_shash_alg(desc->tfm)->finup(desc, data, len, out);
+}
-static inline void shash_desc_zero(struct shash_desc *desc)
+static inline int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
+ unsigned len, u8 *out)
{
- memzero_explicit(desc,
- sizeof(*desc) + crypto_shash_descsize(desc->tfm));
+ return crypto_shash_alg(desc->tfm)->digest(desc, data, len, out);
}
#endif /* _CRYPTO_HASH_H */
#include <crypto/hash.h>
int crypto_register_shash(struct shash_alg *alg);
-int crypto_unregister_shash(struct shash_alg *alg);
-int crypto_register_shashes(struct shash_alg *algs, int count);
-int crypto_unregister_shashes(struct shash_alg *algs, int count);
static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm)
{
#ifndef _CRYPTO_POLY1305_H
#define _CRYPTO_POLY1305_H
-#include <linux/types.h>
-#include <linux/crypto.h>
+#include <sodium/crypto_onetimeauth_poly1305.h>
-#define POLY1305_BLOCK_SIZE 16
-#define POLY1305_KEY_SIZE 32
-#define POLY1305_DIGEST_SIZE 16
-
-struct poly1305_desc_ctx {
- /* key */
- u32 r[5];
- /* finalize key */
- u32 s[4];
- /* accumulator */
- u32 h[5];
- /* partial buffer */
- u8 buf[POLY1305_BLOCK_SIZE];
- /* bytes used in partial buffer */
- unsigned int buflen;
- /* r key has been set */
- bool rset;
- /* s key has been set */
- bool sset;
-};
-
-int crypto_poly1305_init(struct shash_desc *desc);
-int crypto_poly1305_setkey(struct crypto_shash *tfm,
- const u8 *key, unsigned int keylen);
-unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx,
- const u8 *src, unsigned int srclen);
-int crypto_poly1305_update(struct shash_desc *desc,
- const u8 *src, unsigned int srclen);
-int crypto_poly1305_final(struct shash_desc *desc, u8 *dst);
+#define POLY1305_KEY_SIZE crypto_onetimeauth_poly1305_KEYBYTES
+#define POLY1305_DIGEST_SIZE crypto_onetimeauth_poly1305_BYTES
#endif
+++ /dev/null
-/*
- * Common values for SHA algorithms
- */
-
-#ifndef _CRYPTO_SHA_H
-#define _CRYPTO_SHA_H
-
-#include <linux/types.h>
-
-#define SHA1_DIGEST_SIZE 20
-#define SHA1_BLOCK_SIZE 64
-
-#define SHA224_DIGEST_SIZE 28
-#define SHA224_BLOCK_SIZE 64
-
-#define SHA256_DIGEST_SIZE 32
-#define SHA256_BLOCK_SIZE 64
-
-#define SHA384_DIGEST_SIZE 48
-#define SHA384_BLOCK_SIZE 128
-
-#define SHA512_DIGEST_SIZE 64
-#define SHA512_BLOCK_SIZE 128
-
-#define SHA1_H0 0x67452301UL
-#define SHA1_H1 0xefcdab89UL
-#define SHA1_H2 0x98badcfeUL
-#define SHA1_H3 0x10325476UL
-#define SHA1_H4 0xc3d2e1f0UL
-
-#define SHA224_H0 0xc1059ed8UL
-#define SHA224_H1 0x367cd507UL
-#define SHA224_H2 0x3070dd17UL
-#define SHA224_H3 0xf70e5939UL
-#define SHA224_H4 0xffc00b31UL
-#define SHA224_H5 0x68581511UL
-#define SHA224_H6 0x64f98fa7UL
-#define SHA224_H7 0xbefa4fa4UL
-
-#define SHA256_H0 0x6a09e667UL
-#define SHA256_H1 0xbb67ae85UL
-#define SHA256_H2 0x3c6ef372UL
-#define SHA256_H3 0xa54ff53aUL
-#define SHA256_H4 0x510e527fUL
-#define SHA256_H5 0x9b05688cUL
-#define SHA256_H6 0x1f83d9abUL
-#define SHA256_H7 0x5be0cd19UL
-
-#define SHA384_H0 0xcbbb9d5dc1059ed8ULL
-#define SHA384_H1 0x629a292a367cd507ULL
-#define SHA384_H2 0x9159015a3070dd17ULL
-#define SHA384_H3 0x152fecd8f70e5939ULL
-#define SHA384_H4 0x67332667ffc00b31ULL
-#define SHA384_H5 0x8eb44a8768581511ULL
-#define SHA384_H6 0xdb0c2e0d64f98fa7ULL
-#define SHA384_H7 0x47b5481dbefa4fa4ULL
-
-#define SHA512_H0 0x6a09e667f3bcc908ULL
-#define SHA512_H1 0xbb67ae8584caa73bULL
-#define SHA512_H2 0x3c6ef372fe94f82bULL
-#define SHA512_H3 0xa54ff53a5f1d36f1ULL
-#define SHA512_H4 0x510e527fade682d1ULL
-#define SHA512_H5 0x9b05688c2b3e6c1fULL
-#define SHA512_H6 0x1f83d9abfb41bd6bULL
-#define SHA512_H7 0x5be0cd19137e2179ULL
-
-extern const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE];
-
-extern const u8 sha224_zero_message_hash[SHA224_DIGEST_SIZE];
-
-extern const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE];
-
-struct sha1_state {
- u32 state[SHA1_DIGEST_SIZE / 4];
- u64 count;
- u8 buffer[SHA1_BLOCK_SIZE];
-};
-
-struct sha256_state {
- u32 state[SHA256_DIGEST_SIZE / 4];
- u64 count;
- u8 buf[SHA256_BLOCK_SIZE];
-};
-
-struct sha512_state {
- u64 state[SHA512_DIGEST_SIZE / 8];
- u64 count[2];
- u8 buf[SHA512_BLOCK_SIZE];
-};
-
-struct shash_desc;
-
-extern int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
- unsigned int len);
-
-extern int crypto_sha1_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *hash);
-
-extern int crypto_sha256_update(struct shash_desc *desc, const u8 *data,
- unsigned int len);
-
-extern int crypto_sha256_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *hash);
-
-extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data,
- unsigned int len);
-
-extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *hash);
-#endif
+++ /dev/null
-/*
- * sha1_base.h - core logic for SHA-1 implementations
- *
- * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <crypto/internal/hash.h>
-#include <crypto/sha.h>
-#include <linux/byteorder.h>
-#include <linux/crypto.h>
-#include <linux/module.h>
-
-#include <asm/unaligned.h>
-
-typedef void (sha1_block_fn)(struct sha1_state *sst, u8 const *src, int blocks);
-
-static inline int sha1_base_init(struct shash_desc *desc)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
-
- sctx->state[0] = SHA1_H0;
- sctx->state[1] = SHA1_H1;
- sctx->state[2] = SHA1_H2;
- sctx->state[3] = SHA1_H3;
- sctx->state[4] = SHA1_H4;
- sctx->count = 0;
-
- return 0;
-}
-
-static inline int sha1_base_do_update(struct shash_desc *desc,
- const u8 *data,
- unsigned int len,
- sha1_block_fn *block_fn)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
- unsigned int partial = sctx->count % SHA1_BLOCK_SIZE;
-
- sctx->count += len;
-
- if (unlikely((partial + len) >= SHA1_BLOCK_SIZE)) {
- int blocks;
-
- if (partial) {
- int p = SHA1_BLOCK_SIZE - partial;
-
- memcpy(sctx->buffer + partial, data, p);
- data += p;
- len -= p;
-
- block_fn(sctx, sctx->buffer, 1);
- }
-
- blocks = len / SHA1_BLOCK_SIZE;
- len %= SHA1_BLOCK_SIZE;
-
- if (blocks) {
- block_fn(sctx, data, blocks);
- data += blocks * SHA1_BLOCK_SIZE;
- }
- partial = 0;
- }
- if (len)
- memcpy(sctx->buffer + partial, data, len);
-
- return 0;
-}
-
-static inline int sha1_base_do_finalize(struct shash_desc *desc,
- sha1_block_fn *block_fn)
-{
- const int bit_offset = SHA1_BLOCK_SIZE - sizeof(__be64);
- struct sha1_state *sctx = shash_desc_ctx(desc);
- __be64 *bits = (__be64 *)(sctx->buffer + bit_offset);
- unsigned int partial = sctx->count % SHA1_BLOCK_SIZE;
-
- sctx->buffer[partial++] = 0x80;
- if (partial > bit_offset) {
- memset(sctx->buffer + partial, 0x0, SHA1_BLOCK_SIZE - partial);
- partial = 0;
-
- block_fn(sctx, sctx->buffer, 1);
- }
-
- memset(sctx->buffer + partial, 0x0, bit_offset - partial);
- *bits = cpu_to_be64(sctx->count << 3);
- block_fn(sctx, sctx->buffer, 1);
-
- return 0;
-}
-
-static inline int sha1_base_finish(struct shash_desc *desc, u8 *out)
-{
- struct sha1_state *sctx = shash_desc_ctx(desc);
- __be32 *digest = (__be32 *)out;
- int i;
-
- for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(__be32); i++)
- put_unaligned_be32(sctx->state[i], digest++);
-
- *sctx = (struct sha1_state){};
- return 0;
-}
--- /dev/null
+#ifndef _KEYS_USER_TYPE_H
+#define _KEYS_USER_TYPE_H
+
+#include <linux/key.h>
+
+#endif /* _KEYS_USER_TYPE_H */
__u64 __nothing[0];
};
-struct bkey {
- __u64 _data[0];
+struct bversion {
+#if defined(__LITTLE_ENDIAN)
+ __u64 lo;
+ __u32 hi;
+#elif defined(__BIG_ENDIAN)
+ __u32 hi;
+ __u64 lo;
+#endif
+} __attribute__((packed, aligned(4)));
+struct bkey {
/* Size of combined key and value, in u64s */
__u8 u64s;
#if defined(__LITTLE_ENDIAN)
__u8 pad[1];
- __u32 version;
+ struct bversion version;
__u32 size; /* extent size, in sectors */
struct bpos p;
#elif defined(__BIG_ENDIAN)
struct bpos p;
__u32 size; /* extent size, in sectors */
- __u32 version;
+ struct bversion version;
__u8 pad[1];
#endif
BKEY_FIELD_OFFSET,
BKEY_FIELD_SNAPSHOT,
BKEY_FIELD_SIZE,
- BKEY_FIELD_VERSION,
+ BKEY_FIELD_VERSION_HI,
+ BKEY_FIELD_VERSION_LO,
BKEY_NR_FIELDS,
};
bkey_format_field(OFFSET, p.offset), \
bkey_format_field(SNAPSHOT, p.snapshot), \
bkey_format_field(SIZE, size), \
- bkey_format_field(VERSION, version), \
+ bkey_format_field(VERSION_HI, version.hi), \
+ bkey_format_field(VERSION_LO, version.lo), \
}, \
})
/* bkey with inline value */
struct bkey_i {
- struct bkey k;
- struct bch_val v;
+ __u64 _data[0];
+
+ union {
+ struct {
+ /* Size of combined key and value, in u64s */
+ __u8 u64s;
+ };
+ struct {
+ struct bkey k;
+ struct bch_val v;
+ };
+ };
};
#ifndef __cplusplus
* is neither checksummed nor compressed.
*/
+/* 128 bits, sufficient for cryptographic MACs: */
+struct bch_csum {
+ __le64 lo;
+ __le64 hi;
+} __attribute__((packed, aligned(8)));
+
+#define BCH_CSUM_NONE 0U
+#define BCH_CSUM_CRC32C 1U
+#define BCH_CSUM_CRC64 2U
+#define BCH_CSUM_CHACHA20_POLY1305_80 3U
+#define BCH_CSUM_CHACHA20_POLY1305_128 4U
+#define BCH_CSUM_NR 5U
+
+static inline _Bool bch_csum_type_is_encryption(unsigned type)
+{
+ switch (type) {
+ case BCH_CSUM_CHACHA20_POLY1305_80:
+ case BCH_CSUM_CHACHA20_POLY1305_128:
+ return true;
+ default:
+ return false;
+ }
+}
+
enum bch_extent_entry_type {
- BCH_EXTENT_ENTRY_crc32 = 0,
- BCH_EXTENT_ENTRY_ptr = 1,
+ BCH_EXTENT_ENTRY_ptr = 0,
+ BCH_EXTENT_ENTRY_crc32 = 1,
BCH_EXTENT_ENTRY_crc64 = 2,
+ BCH_EXTENT_ENTRY_crc128 = 3,
};
-#define BCH_EXTENT_ENTRY_MAX 3
+#define BCH_EXTENT_ENTRY_MAX 4
+/* Compressed/uncompressed size are stored biased by 1: */
struct bch_extent_crc32 {
#if defined(__LITTLE_ENDIAN_BITFIELD)
- __u32 type:1,
+ __u32 type:2,
+ _compressed_size:7,
+ _uncompressed_size:7,
offset:7,
- compressed_size:8,
- uncompressed_size:8,
+ _unused:1,
csum_type:4,
compression_type:4;
__u32 csum;
__u32 csum;
__u32 compression_type:4,
csum_type:4,
- uncompressed_size:8,
- compressed_size:8,
+ _unused:1,
offset:7,
- type:1;
+ _uncompressed_size:7,
+ _compressed_size:7,
+ type:2;
#endif
} __attribute__((packed, aligned(8)));
-#define CRC32_EXTENT_SIZE_MAX (1U << 7)
-
-/* 64k */
-#define BCH_COMPRESSED_EXTENT_MAX 128U
+#define CRC32_SIZE_MAX (1U << 7)
+#define CRC32_NONCE_MAX 0
struct bch_extent_crc64 {
#if defined(__LITTLE_ENDIAN_BITFIELD)
__u64 type:3,
- offset:17,
- compressed_size:18,
- uncompressed_size:18,
+ _compressed_size:9,
+ _uncompressed_size:9,
+ offset:9,
+ nonce:10,
+ csum_type:4,
+ compression_type:4,
+ csum_hi:16;
+#elif defined (__BIG_ENDIAN_BITFIELD)
+ __u64 csum_hi:16,
+ compression_type:4,
+ csum_type:4,
+ nonce:10,
+ offset:9,
+ _uncompressed_size:9,
+ _compressed_size:9,
+ type:3;
+#endif
+ __u64 csum_lo;
+} __attribute__((packed, aligned(8)));
+
+#define CRC64_SIZE_MAX (1U << 9)
+#define CRC64_NONCE_MAX ((1U << 10) - 1)
+
+struct bch_extent_crc128 {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+ __u64 type:4,
+ _compressed_size:13,
+ _uncompressed_size:13,
+ offset:13,
+ nonce:13,
csum_type:4,
compression_type:4;
#elif defined (__BIG_ENDIAN_BITFIELD)
__u64 compression_type:4,
csum_type:4,
- uncompressed_size:18,
- compressed_size:18,
- offset:17,
+ nonce:14,
+ offset:13,
+ _uncompressed_size:13,
+ _compressed_size:13,
type:3;
#endif
- __u64 csum;
+ struct bch_csum csum;
} __attribute__((packed, aligned(8)));
-#define CRC64_EXTENT_SIZE_MAX (1U << 17)
+#define CRC128_SIZE_MAX (1U << 13)
+#define CRC128_NONCE_MAX ((1U << 13) - 1)
+
+/*
+ * Max size of an extent that may require bouncing to read or write
+ * (checksummed, compressed): 64k
+ */
+#define BCH_ENCODED_EXTENT_MAX 128U
/*
* @reservation - pointer hasn't been written to, just reserved
*/
struct bch_extent_ptr {
#if defined(__LITTLE_ENDIAN_BITFIELD)
- __u64 type:2,
+ __u64 type:1,
+ cached:1,
erasure_coded:1,
reservation:1,
offset:44, /* 8 petabytes */
offset:44,
reservation:1,
erasure_coded:1,
- type:2;
+ cached:1,
+ type:1;
#endif
} __attribute__((packed, aligned(8)));
+struct bch_extent_reservation {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+ __u64 type:5,
+ unused:23,
+ replicas:4,
+ generation:32;
+#elif defined (__BIG_ENDIAN_BITFIELD)
+ __u64 generation:32,
+ replicas:4,
+ unused:23,
+ type:5;
+#endif
+};
+
union bch_extent_entry {
#if defined(__LITTLE_ENDIAN) || __BITS_PER_LONG == 64
unsigned long type;
#endif
struct bch_extent_crc32 crc32;
struct bch_extent_crc64 crc64;
+ struct bch_extent_crc128 crc128;
struct bch_extent_ptr ptr;
};
} __attribute__((packed, aligned(8)));
BKEY_VAL_TYPE(extent, BCH_EXTENT);
+struct bch_reservation {
+ struct bch_val v;
+
+ __le32 generation;
+ __u8 nr_replicas;
+ __u8 pad[3];
+} __attribute__((packed, aligned(8)));
+BKEY_VAL_TYPE(reservation, BCH_RESERVATION);
+
/* Maximum size (in u64s) a single pointer could be: */
#define BKEY_EXTENT_PTR_U64s_MAX\
- ((sizeof(struct bch_extent_crc64) + \
+ ((sizeof(struct bch_extent_crc128) + \
sizeof(struct bch_extent_ptr)) / sizeof(u64))
/* Maximum possible size of an entire extent value: */
struct bch_inode {
struct bch_val v;
- __le16 i_mode;
- __le16 pad;
- __le32 i_flags;
-
- /* Nanoseconds */
- __le64 i_atime;
- __le64 i_ctime;
- __le64 i_mtime;
-
- __le64 i_size;
- __le64 i_sectors;
-
- __le32 i_uid;
- __le32 i_gid;
- __le32 i_nlink;
-
- __le32 i_dev;
-
__le64 i_hash_seed;
+ __le32 i_flags;
+ __le16 i_mode;
+ __u8 fields[0];
} __attribute__((packed));
BKEY_VAL_TYPE(inode, BCH_INODE_FS);
+#define BCH_INODE_FIELDS() \
+ BCH_INODE_FIELD(i_atime, 64) \
+ BCH_INODE_FIELD(i_ctime, 64) \
+ BCH_INODE_FIELD(i_mtime, 64) \
+ BCH_INODE_FIELD(i_otime, 64) \
+ BCH_INODE_FIELD(i_size, 64) \
+ BCH_INODE_FIELD(i_sectors, 64) \
+ BCH_INODE_FIELD(i_uid, 32) \
+ BCH_INODE_FIELD(i_gid, 32) \
+ BCH_INODE_FIELD(i_nlink, 32) \
+ BCH_INODE_FIELD(i_generation, 32) \
+ BCH_INODE_FIELD(i_dev, 32)
+
enum {
/*
* User flags (get/settable with FS_IOC_*FLAGS, correspond to FS_*_FL
/* not implemented yet: */
__BCH_INODE_HAS_XATTRS = 7, /* has xattrs in xattr btree */
-};
-LE32_BITMASK(INODE_STR_HASH_TYPE, struct bch_inode, i_flags, 28, 32);
+ /* bits 20+ reserved for packed fields below: */
+};
#define BCH_INODE_SYNC (1 << __BCH_INODE_SYNC)
#define BCH_INODE_IMMUTABLE (1 << __BCH_INODE_IMMUTABLE)
#define BCH_INODE_I_SECTORS_DIRTY (1 << __BCH_INODE_I_SECTORS_DIRTY)
#define BCH_INODE_HAS_XATTRS (1 << __BCH_INODE_HAS_XATTRS)
+LE32_BITMASK(INODE_STR_HASH, struct bch_inode, i_flags, 20, 24);
+LE32_BITMASK(INODE_NR_FIELDS, struct bch_inode, i_flags, 24, 32);
+
struct bch_inode_blockdev {
struct bch_val v;
/* Thin provisioned volume, or cache for another block device? */
LE64_BITMASK(CACHED_DEV, struct bch_inode_blockdev, i_flags, 0, 1)
+
/* Dirents */
/*
* Version 4: Backing device with data offset
* Version 5: All the incompat changes
* Version 6: Cache device UUIDs all in superblock, another incompat bset change
+ * Version 7: Encryption (expanded checksum fields), other random things
*/
#define BCACHE_SB_VERSION_CDEV_V0 0
#define BCACHE_SB_VERSION_BDEV 1
#define BCACHE_SB_VERSION_BDEV_WITH_OFFSET 4
#define BCACHE_SB_VERSION_CDEV_V2 5
#define BCACHE_SB_VERSION_CDEV_V3 6
-#define BCACHE_SB_VERSION_CDEV 6
-#define BCACHE_SB_MAX_VERSION 6
+#define BCACHE_SB_VERSION_CDEV_V4 7
+#define BCACHE_SB_VERSION_CDEV 7
+#define BCACHE_SB_MAX_VERSION 7
-#define SB_SECTOR 8
-#define SB_LABEL_SIZE 32
-#define MAX_CACHES_PER_SET 64
-
-#define BDEV_DATA_START_DEFAULT 16 /* sectors */
+#define BCH_SB_SECTOR 8
+#define BCH_SB_LABEL_SIZE 32
+#define BCH_SB_MEMBERS_MAX 64 /* XXX kill */
-struct cache_member {
+struct bch_member {
uuid_le uuid;
__le64 nbuckets; /* device size */
__le16 first_bucket; /* index of first bucket used */
__le32 pad;
__le64 last_mount; /* time_t */
- __le64 f1;
- __le64 f2;
+ __le64 flags[2];
};
-LE64_BITMASK(CACHE_STATE, struct cache_member, f1, 0, 4)
-#define CACHE_ACTIVE 0U
-#define CACHE_RO 1U
-#define CACHE_FAILED 2U
-#define CACHE_SPARE 3U
-#define CACHE_STATE_NR 4U
+LE64_BITMASK(BCH_MEMBER_STATE, struct bch_member, flags[0], 0, 4)
+LE64_BITMASK(BCH_MEMBER_TIER, struct bch_member, flags[0], 4, 8)
+LE64_BITMASK(BCH_MEMBER_HAS_METADATA, struct bch_member, flags[0], 8, 9)
+LE64_BITMASK(BCH_MEMBER_HAS_DATA, struct bch_member, flags[0], 9, 10)
+LE64_BITMASK(BCH_MEMBER_REPLACEMENT, struct bch_member, flags[0], 10, 14)
+LE64_BITMASK(BCH_MEMBER_DISCARD, struct bch_member, flags[0], 14, 15);
-LE64_BITMASK(CACHE_TIER, struct cache_member, f1, 4, 8)
-#define CACHE_TIERS 4U
+#if 0
+LE64_BITMASK(BCH_MEMBER_NR_READ_ERRORS, struct bch_member, flags[1], 0, 20);
+LE64_BITMASK(BCH_MEMBER_NR_WRITE_ERRORS,struct bch_member, flags[1], 20, 40);
+#endif
-LE64_BITMASK(CACHE_REPLICATION_SET, struct cache_member, f1, 8, 16)
+enum bch_member_state {
+ BCH_MEMBER_STATE_ACTIVE = 0,
+ BCH_MEMBER_STATE_RO = 1,
+ BCH_MEMBER_STATE_FAILED = 2,
+ BCH_MEMBER_STATE_SPARE = 3,
+ BCH_MEMBER_STATE_NR = 4,
+};
-LE64_BITMASK(CACHE_HAS_METADATA, struct cache_member, f1, 24, 25)
-LE64_BITMASK(CACHE_HAS_DATA, struct cache_member, f1, 25, 26)
+#define BCH_TIER_MAX 4U
-LE64_BITMASK(CACHE_REPLACEMENT, struct cache_member, f1, 26, 30)
-#define CACHE_REPLACEMENT_LRU 0U
-#define CACHE_REPLACEMENT_FIFO 1U
-#define CACHE_REPLACEMENT_RANDOM 2U
-#define CACHE_REPLACEMENT_NR 3U
+enum cache_replacement {
+ CACHE_REPLACEMENT_LRU = 0,
+ CACHE_REPLACEMENT_FIFO = 1,
+ CACHE_REPLACEMENT_RANDOM = 2,
+ CACHE_REPLACEMENT_NR = 3,
+};
-LE64_BITMASK(CACHE_DISCARD, struct cache_member, f1, 30, 31);
+struct bch_sb_layout {
+ uuid_le magic; /* bcache superblock UUID */
+ __u8 layout_type;
+ __u8 sb_max_size_bits; /* base 2 of 512 byte sectors */
+ __u8 nr_superblocks;
+ __u8 pad[5];
+ __u64 sb_offset[61];
+} __attribute__((packed));
-LE64_BITMASK(CACHE_NR_READ_ERRORS, struct cache_member, f2, 0, 20);
-LE64_BITMASK(CACHE_NR_WRITE_ERRORS, struct cache_member, f2, 20, 40);
+#define BCH_SB_LAYOUT_SECTOR 7
-struct cache_sb {
- __le64 csum;
- __le64 offset; /* sector where this sb was written */
- __le64 version; /* of on disk format */
+struct bch_sb_field {
+ __u64 _data[0];
+ __le32 u64s;
+ __le32 type;
+};
- uuid_le magic; /* bcache superblock UUID */
+enum bch_sb_field_types {
+ BCH_SB_FIELD_journal = 0,
+ BCH_SB_FIELD_members = 1,
+ BCH_SB_FIELD_crypt = 2,
+ BCH_SB_FIELD_NR = 3,
+};
- /* Identifies this disk within the cache set: */
- uuid_le disk_uuid;
+struct bch_sb_field_journal {
+ struct bch_sb_field field;
+ __le64 buckets[0];
+};
- /*
- * Internal cache set UUID - xored with various magic numbers and thus
- * must never change:
- */
- union {
- uuid_le set_uuid;
- __le64 set_magic;
- };
+struct bch_sb_field_members {
+ struct bch_sb_field field;
+ struct bch_member members[0];
+};
+
+/* Crypto: */
- __u8 label[SB_LABEL_SIZE];
+struct nonce {
+ __le32 d[4];
+};
+
+struct bch_key {
+ __le64 key[4];
+};
+
+#define BCH_KEY_MAGIC \
+ (((u64) 'b' << 0)|((u64) 'c' << 8)| \
+ ((u64) 'h' << 16)|((u64) '*' << 24)| \
+ ((u64) '*' << 32)|((u64) 'k' << 40)| \
+ ((u64) 'e' << 48)|((u64) 'y' << 56))
+
+struct bch_encrypted_key {
+ __le64 magic;
+ struct bch_key key;
+};
+
+/*
+ * If this field is present in the superblock, it stores an encryption key which
+ * is used encrypt all other data/metadata. The key will normally be encrypted
+ * with the key userspace provides, but if encryption has been turned off we'll
+ * just store the master key unencrypted in the superblock so we can access the
+ * previously encrypted data.
+ */
+struct bch_sb_field_crypt {
+ struct bch_sb_field field;
__le64 flags;
+ __le64 kdf_flags;
+ struct bch_encrypted_key key;
+};
- /* Incremented each time superblock is written: */
- __le64 seq;
+LE64_BITMASK(BCH_CRYPT_KDF_TYPE, struct bch_sb_field_crypt, flags, 0, 4);
- /*
- * User visible UUID for identifying the cache set the user is allowed
- * to change:
- */
- uuid_le user_uuid;
+enum bch_kdf_types {
+ BCH_KDF_SCRYPT = 0,
+ BCH_KDF_NR = 1,
+};
- __le64 flags2;
- __le64 pad1[5];
+/* stored as base 2 log of scrypt params: */
+LE64_BITMASK(BCH_KDF_SCRYPT_N, struct bch_sb_field_crypt, kdf_flags, 0, 16);
+LE64_BITMASK(BCH_KDF_SCRYPT_R, struct bch_sb_field_crypt, kdf_flags, 16, 32);
+LE64_BITMASK(BCH_KDF_SCRYPT_P, struct bch_sb_field_crypt, kdf_flags, 32, 48);
- /* Number of cache_member entries: */
- __u8 nr_in_set;
+/*
+ * @offset - sector where this sb was written
+ * @version - on disk format version
+ * @magic - identifies as a bcache superblock (BCACHE_MAGIC)
+ * @seq - incremented each time superblock is written
+ * @uuid - used for generating various magic numbers and identifying
+ * member devices, never changes
+ * @user_uuid - user visible UUID, may be changed
+ * @label - filesystem label
+ * @seq - identifies most recent superblock, incremented each time
+ * superblock is written
+ * @features - enabled incompatible features
+ */
+struct bch_sb {
+ struct bch_csum csum;
+ __le64 version;
+ uuid_le magic;
+ uuid_le uuid;
+ uuid_le user_uuid;
+ __u8 label[BCH_SB_LABEL_SIZE];
+ __le64 offset;
+ __le64 seq;
- /*
- * Index of this device - for PTR_DEV(), and also this device's
- * slot in the cache_member array:
- */
- __u8 nr_this_dev;
- __le16 pad2[3];
+ __le16 block_size;
+ __u8 dev_idx;
+ __u8 nr_devices;
+ __le32 u64s;
- __le16 block_size; /* sectors */
- __le16 pad3[6];
+ __le64 time_base_lo;
+ __le32 time_base_hi;
+ __le32 time_precision;
+
+ __le64 flags[8];
+ __le64 features[2];
+ __le64 compat[2];
- __le16 u64s; /* size of variable length portion */
+ struct bch_sb_layout layout;
union {
- struct cache_member members[0];
- /*
- * Journal buckets also in the variable length portion, after
- * the member info:
- */
- __le64 _data[0];
+ struct bch_sb_field start[0];
+ __le64 _data[0];
};
-};
+} __attribute__((packed, aligned(8)));
-/* XXX: rename CACHE_SET -> BCH_FS or something? */
+/*
+ * Flags:
+ * BCH_SB_INITALIZED - set on first mount
+ * BCH_SB_CLEAN - did we shut down cleanly? Just a hint, doesn't affect
+ * behaviour of mount/recovery path:
+ * BCH_SB_INODE_32BIT - limit inode numbers to 32 bits
+ * BCH_SB_128_BIT_MACS - 128 bit macs instead of 80
+ * BCH_SB_ENCRYPTION_TYPE - if nonzero encryption is enabled; overrides
+ * DATA/META_CSUM_TYPE. Also indicates encryption
+ * algorithm in use, if/when we get more than one
+ */
-LE64_BITMASK(CACHE_SET_SYNC, struct cache_sb, flags, 0, 1);
+LE64_BITMASK(BCH_SB_INITIALIZED, struct bch_sb, flags[0], 0, 1);
+LE64_BITMASK(BCH_SB_CLEAN, struct bch_sb, flags[0], 1, 2);
+LE64_BITMASK(BCH_SB_CSUM_TYPE, struct bch_sb, flags[0], 2, 8);
+LE64_BITMASK(BCH_SB_ERROR_ACTION, struct bch_sb, flags[0], 8, 12);
-LE64_BITMASK(CACHE_SET_ERROR_ACTION, struct cache_sb, flags, 1, 4);
-#define BCH_ON_ERROR_CONTINUE 0U
-#define BCH_ON_ERROR_RO 1U
-#define BCH_ON_ERROR_PANIC 2U
-#define BCH_NR_ERROR_ACTIONS 3U
+LE64_BITMASK(BCH_SB_BTREE_NODE_SIZE, struct bch_sb, flags[0], 12, 28);
-LE64_BITMASK(CACHE_SET_META_REPLICAS_WANT,struct cache_sb, flags, 4, 8);
-LE64_BITMASK(CACHE_SET_DATA_REPLICAS_WANT,struct cache_sb, flags, 8, 12);
+LE64_BITMASK(BCH_SB_GC_RESERVE, struct bch_sb, flags[0], 28, 33);
+LE64_BITMASK(BCH_SB_ROOT_RESERVE, struct bch_sb, flags[0], 33, 40);
-#define BCH_REPLICAS_MAX 4U
+LE64_BITMASK(BCH_SB_META_CSUM_TYPE, struct bch_sb, flags[0], 40, 44);
+LE64_BITMASK(BCH_SB_DATA_CSUM_TYPE, struct bch_sb, flags[0], 44, 48);
-LE64_BITMASK(CACHE_SB_CSUM_TYPE, struct cache_sb, flags, 12, 16);
+LE64_BITMASK(BCH_SB_META_REPLICAS_WANT, struct bch_sb, flags[0], 48, 52);
+LE64_BITMASK(BCH_SB_DATA_REPLICAS_WANT, struct bch_sb, flags[0], 52, 56);
-LE64_BITMASK(CACHE_SET_META_PREFERRED_CSUM_TYPE,struct cache_sb, flags, 16, 20);
-#define BCH_CSUM_NONE 0U
-#define BCH_CSUM_CRC32C 1U
-#define BCH_CSUM_CRC64 2U
-#define BCH_CSUM_NR 3U
+LE64_BITMASK(BCH_SB_META_REPLICAS_HAVE, struct bch_sb, flags[0], 56, 60);
+LE64_BITMASK(BCH_SB_DATA_REPLICAS_HAVE, struct bch_sb, flags[0], 60, 64);
-LE64_BITMASK(CACHE_SET_BTREE_NODE_SIZE, struct cache_sb, flags, 20, 36);
+LE64_BITMASK(BCH_SB_STR_HASH_TYPE, struct bch_sb, flags[1], 0, 4);
+LE64_BITMASK(BCH_SB_COMPRESSION_TYPE, struct bch_sb, flags[1], 4, 8);
+LE64_BITMASK(BCH_SB_INODE_32BIT, struct bch_sb, flags[1], 8, 9);
-LE64_BITMASK(CACHE_SET_META_REPLICAS_HAVE,struct cache_sb, flags, 36, 40);
-LE64_BITMASK(CACHE_SET_DATA_REPLICAS_HAVE,struct cache_sb, flags, 40, 44);
+LE64_BITMASK(BCH_SB_128_BIT_MACS, struct bch_sb, flags[1], 9, 10);
+LE64_BITMASK(BCH_SB_ENCRYPTION_TYPE, struct bch_sb, flags[1], 10, 14);
+LE64_BITMASK(BCH_SB_JOURNAL_ENTRY_SIZE, struct bch_sb, flags[1], 14, 20);
-LE64_BITMASK(CACHE_SET_STR_HASH_TYPE,struct cache_sb, flags, 44, 48);
-enum bch_str_hash_type {
- BCH_STR_HASH_CRC32C = 0,
- BCH_STR_HASH_CRC64 = 1,
- BCH_STR_HASH_SIPHASH = 2,
- BCH_STR_HASH_SHA1 = 3,
+/* Features: */
+enum bch_sb_features {
+ BCH_FEATURE_LZ4 = 0,
+ BCH_FEATURE_GZIP = 1,
};
-#define BCH_STR_HASH_NR 4
+/* options: */
-LE64_BITMASK(CACHE_SET_DATA_PREFERRED_CSUM_TYPE, struct cache_sb, flags, 48, 52);
+#define BCH_REPLICAS_MAX 4U
-LE64_BITMASK(CACHE_SET_COMPRESSION_TYPE, struct cache_sb, flags, 52, 56);
-enum {
- BCH_COMPRESSION_NONE = 0,
- BCH_COMPRESSION_LZ4 = 1,
- BCH_COMPRESSION_GZIP = 2,
+#if 0
+#define BCH_ERROR_ACTIONS() \
+ x(BCH_ON_ERROR_CONTINUE, 0, "continue") \
+ x(BCH_ON_ERROR_RO, 1, "remount-ro") \
+ x(BCH_ON_ERROR_PANIC, 2, "panic") \
+ x(BCH_NR_ERROR_ACTIONS, 3, NULL)
+
+enum bch_error_actions {
+#define x(_opt, _nr, _str) _opt = _nr,
+ BCH_ERROR_ACTIONS()
+#undef x
};
+#endif
-#define BCH_COMPRESSION_NR 3U
-
-/* Limit inode numbers to 32 bits: */
-LE64_BITMASK(CACHE_INODE_32BIT, struct cache_sb, flags, 56, 57);
-
-LE64_BITMASK(CACHE_SET_GC_RESERVE, struct cache_sb, flags, 57, 63);
-
-LE64_BITMASK(CACHE_SET_ROOT_RESERVE, struct cache_sb, flags2, 0, 6);
+enum bch_error_actions {
+ BCH_ON_ERROR_CONTINUE = 0,
+ BCH_ON_ERROR_RO = 1,
+ BCH_ON_ERROR_PANIC = 2,
+ BCH_NR_ERROR_ACTIONS = 3,
+};
-/*
- * Did we shut down cleanly? Just a hint, doesn't affect behaviour of
- * mount/recovery path:
- */
-LE64_BITMASK(CACHE_SET_CLEAN, struct cache_sb, flags2, 6, 7);
+enum bch_csum_opts {
+ BCH_CSUM_OPT_NONE = 0,
+ BCH_CSUM_OPT_CRC32C = 1,
+ BCH_CSUM_OPT_CRC64 = 2,
+ BCH_CSUM_OPT_NR = 3,
+};
-LE64_BITMASK(CACHE_SET_JOURNAL_ENTRY_SIZE, struct cache_sb, flags2, 7, 15);
+enum bch_str_hash_opts {
+ BCH_STR_HASH_CRC32C = 0,
+ BCH_STR_HASH_CRC64 = 1,
+ BCH_STR_HASH_SIPHASH = 2,
+ BCH_STR_HASH_NR = 3,
+};
-/* options: */
+enum bch_compression_opts {
+ BCH_COMPRESSION_NONE = 0,
+ BCH_COMPRESSION_LZ4 = 1,
+ BCH_COMPRESSION_GZIP = 2,
+ BCH_COMPRESSION_NR = 3,
+};
/**
- * CACHE_SET_OPT(name, choices, min, max, sb_option, sysfs_writeable)
+ * BCH_OPT(name, choices, min, max, sb_option, sysfs_writeable)
*
* @name - name of mount option, sysfs attribute, and struct cache_set_opts
* member
* @sysfs_writeable - if true, option will be modifiable at runtime via sysfs
*/
-#define CACHE_SET_SB_OPTS() \
- CACHE_SET_OPT(errors, \
- bch_error_actions, \
- 0, BCH_NR_ERROR_ACTIONS, \
- CACHE_SET_ERROR_ACTION, \
- true) \
- CACHE_SET_OPT(metadata_replicas, \
- bch_uint_opt, \
- 0, BCH_REPLICAS_MAX, \
- CACHE_SET_META_REPLICAS_WANT, \
- false) \
- CACHE_SET_OPT(data_replicas, \
- bch_uint_opt, \
- 0, BCH_REPLICAS_MAX, \
- CACHE_SET_DATA_REPLICAS_WANT, \
- false) \
- CACHE_SET_OPT(metadata_checksum, \
- bch_csum_types, \
- 0, BCH_CSUM_NR, \
- CACHE_SET_META_PREFERRED_CSUM_TYPE, \
- true) \
- CACHE_SET_OPT(data_checksum, \
- bch_csum_types, \
- 0, BCH_CSUM_NR, \
- CACHE_SET_DATA_PREFERRED_CSUM_TYPE, \
- true) \
- CACHE_SET_OPT(compression, \
- bch_compression_types, \
- 0, BCH_COMPRESSION_NR, \
- CACHE_SET_COMPRESSION_TYPE, \
- true) \
- CACHE_SET_OPT(str_hash, \
- bch_str_hash_types, \
- 0, BCH_STR_HASH_NR, \
- CACHE_SET_STR_HASH_TYPE, \
- true) \
- CACHE_SET_OPT(inodes_32bit, \
- bch_bool_opt, 0, 2, \
- CACHE_INODE_32BIT, \
- true) \
- CACHE_SET_OPT(gc_reserve_percent, \
- bch_uint_opt, \
- 5, 21, \
- CACHE_SET_GC_RESERVE, \
- false) \
- CACHE_SET_OPT(root_reserve_percent, \
- bch_uint_opt, \
- 0, 21, \
- CACHE_SET_ROOT_RESERVE, \
- false)
+#define BCH_SB_OPTS() \
+ BCH_OPT(errors, \
+ bch_error_actions, \
+ 0, BCH_NR_ERROR_ACTIONS, \
+ BCH_SB_ERROR_ACTION, \
+ true) \
+ BCH_OPT(metadata_replicas, \
+ bch_uint_opt, \
+ 0, BCH_REPLICAS_MAX, \
+ BCH_SB_META_REPLICAS_WANT, \
+ false) \
+ BCH_OPT(data_replicas, \
+ bch_uint_opt, \
+ 0, BCH_REPLICAS_MAX, \
+ BCH_SB_DATA_REPLICAS_WANT, \
+ false) \
+ BCH_OPT(metadata_checksum, \
+ bch_csum_types, \
+ 0, BCH_CSUM_OPT_NR, \
+ BCH_SB_META_CSUM_TYPE, \
+ true) \
+ BCH_OPT(data_checksum, \
+ bch_csum_types, \
+ 0, BCH_CSUM_OPT_NR, \
+ BCH_SB_DATA_CSUM_TYPE, \
+ true) \
+ BCH_OPT(compression, \
+ bch_compression_types, \
+ 0, BCH_COMPRESSION_NR, \
+ BCH_SB_COMPRESSION_TYPE, \
+ true) \
+ BCH_OPT(str_hash, \
+ bch_str_hash_types, \
+ 0, BCH_STR_HASH_NR, \
+ BCH_SB_STR_HASH_TYPE, \
+ true) \
+ BCH_OPT(inodes_32bit, \
+ bch_bool_opt, 0, 2, \
+ BCH_SB_INODE_32BIT, \
+ true) \
+ BCH_OPT(gc_reserve_percent, \
+ bch_uint_opt, \
+ 5, 21, \
+ BCH_SB_GC_RESERVE, \
+ false) \
+ BCH_OPT(root_reserve_percent, \
+ bch_uint_opt, \
+ 0, 100, \
+ BCH_SB_ROOT_RESERVE, \
+ false) \
+ BCH_OPT(wide_macs, \
+ bch_bool_opt, 0, 2, \
+ BCH_SB_128_BIT_MACS, \
+ true)
/* backing device specific stuff: */
uuid_le set_uuid;
__le64 set_magic;
};
- __u8 label[SB_LABEL_SIZE];
+ __u8 label[BCH_SB_LABEL_SIZE];
__le64 flags;
#define BDEV_STATE_DIRTY 2U
#define BDEV_STATE_STALE 3U
-static inline unsigned bch_journal_buckets_offset(struct cache_sb *sb)
-{
- return sb->nr_in_set * (sizeof(struct cache_member) / sizeof(__u64));
-}
-
-static inline unsigned bch_nr_journal_buckets(struct cache_sb *sb)
-{
- return __le16_to_cpu(sb->u64s) - bch_journal_buckets_offset(sb);
-}
+#define BDEV_DATA_START_DEFAULT 16 /* sectors */
static inline _Bool __SB_IS_BDEV(__u64 version)
{
|| version == BCACHE_SB_VERSION_BDEV_WITH_OFFSET;
}
-static inline _Bool SB_IS_BDEV(const struct cache_sb *sb)
+static inline _Bool SB_IS_BDEV(const struct bch_sb *sb)
{
return __SB_IS_BDEV(sb->version);
}
#define BCACHE_STATFS_MAGIC 0xca451a4e
-#define BCACHE_SB_MAGIC 0xca451a4ef67385c6ULL
-#define BCACHE_SB_MAGIC2 0x816dba487ff56582ULL
-#define JSET_MAGIC 0x245235c1a3625032ULL
-#define PSET_MAGIC 0x6750e15f87337f91ULL
-#define BSET_MAGIC 0x90135c78b99e07f5ULL
+#define JSET_MAGIC __cpu_to_le64(0x245235c1a3625032ULL)
+#define PSET_MAGIC __cpu_to_le64(0x6750e15f87337f91ULL)
+#define BSET_MAGIC __cpu_to_le64(0x90135c78b99e07f5ULL)
-static inline __u64 jset_magic(struct cache_sb *sb)
+static inline __le64 __bch_sb_magic(struct bch_sb *sb)
{
- return __le64_to_cpu(sb->set_magic) ^ JSET_MAGIC;
+ __le64 ret;
+ memcpy(&ret, &sb->uuid, sizeof(ret));
+ return ret;
}
-static inline __u64 pset_magic(struct cache_sb *sb)
+static inline __u64 __jset_magic(struct bch_sb *sb)
{
- return __le64_to_cpu(sb->set_magic) ^ PSET_MAGIC;
+ return __le64_to_cpu(__bch_sb_magic(sb) ^ JSET_MAGIC);
}
-static inline __u64 bset_magic(struct cache_sb *sb)
+static inline __u64 __pset_magic(struct bch_sb *sb)
{
- return __le64_to_cpu(sb->set_magic) ^ BSET_MAGIC;
+ return __le64_to_cpu(__bch_sb_magic(sb) ^ PSET_MAGIC);
}
-/* Journal */
+static inline __u64 __bset_magic(struct bch_sb *sb)
+{
+ return __le64_to_cpu(__bch_sb_magic(sb) ^ BSET_MAGIC);
+}
+/* Journal */
#define BCACHE_JSET_VERSION_UUIDv1 1
#define BCACHE_JSET_VERSION_UUID 1 /* Always latest UUID format */
* version is for on disk format changes.
*/
struct jset {
- __le64 csum;
+ struct bch_csum csum;
+
__le64 magic;
+ __le64 seq;
__le32 version;
__le32 flags;
- /* Sequence number of oldest dirty journal entry */
- __le64 seq;
- __le64 last_seq;
+ __le32 u64s; /* size of d[] in u64s */
+
+ __u8 encrypted_start[0];
__le16 read_clock;
__le16 write_clock;
- __le32 u64s; /* size of d[] in u64s */
+
+ /* Sequence number of oldest dirty journal entry */
+ __le64 last_seq;
+
union {
struct jset_entry start[0];
__u64 _data[0];
};
-};
+} __attribute__((packed));
LE32_BITMASK(JSET_CSUM_TYPE, struct jset, flags, 0, 4);
LE32_BITMASK(JSET_BIG_ENDIAN, struct jset, flags, 4, 5);
/* Bucket prios/gens */
struct prio_set {
- __le64 csum;
+ struct bch_csum csum;
+
__le64 magic;
- __le32 version;
- __le32 flags;
+ __le32 nonce[3];
+ __le16 version;
+ __le16 flags;
+
+ __u8 encrypted_start[0];
__le64 next_bucket;
__le16 write_prio;
__u8 gen;
} __attribute__((packed)) data[];
-};
+} __attribute__((packed));
LE32_BITMASK(PSET_CSUM_TYPE, struct prio_set, flags, 0, 4);
LE32_BITMASK(BSET_CSUM_TYPE, struct bset, flags, 0, 4);
-/* Only used in first bset */
-LE32_BITMASK(BSET_BTREE_LEVEL, struct bset, flags, 4, 8);
-
-LE32_BITMASK(BSET_BIG_ENDIAN, struct bset, flags, 8, 9);
+LE32_BITMASK(BSET_BIG_ENDIAN, struct bset, flags, 4, 5);
LE32_BITMASK(BSET_SEPARATE_WHITEOUTS,
- struct bset, flags, 9, 10);
+ struct bset, flags, 5, 6);
struct btree_node {
- __le64 csum;
+ struct bch_csum csum;
__le64 magic;
+ /* this flags field is encrypted, unlike bset->flags: */
+ __le64 flags;
+
/* Closed interval: */
struct bpos min_key;
struct bpos max_key;
+ struct bch_extent_ptr ptr;
struct bkey_format format;
+ union {
struct bset keys;
+ struct {
+ __u8 pad[22];
+ __le16 u64s;
+ __u64 _data[0];
+
+ };
+ };
} __attribute__((packed));
+LE64_BITMASK(BTREE_NODE_ID, struct btree_node, flags, 0, 4);
+LE64_BITMASK(BTREE_NODE_LEVEL, struct btree_node, flags, 4, 8);
+
struct btree_node_entry {
- __le64 csum;
+ struct bch_csum csum;
+
+ union {
struct bset keys;
+ struct {
+ __u8 pad[22];
+ __le16 u64s;
+ __u64 _data[0];
+
+ };
+ };
} __attribute__((packed));
/* OBSOLETE */
__u16 btree_level;
__u16 pad[3];
- __u64 prio_bucket[MAX_CACHES_PER_SET];
+ __u64 prio_bucket[64];
union {
struct bkey start[0];
#include <linux/slab.h>
#include <linux/string.h>
-/*
- * Autoloaded crypto modules should only use a prefixed name to avoid allowing
- * arbitrary modules to be loaded. Loading from userspace may still need the
- * unprefixed names, so retains those aliases as well.
- * This uses __MODULE_INFO directly instead of MODULE_ALIAS because pre-4.3
- * gcc (e.g. avr32 toolchain) uses __LINE__ for uniqueness, and this macro
- * expands twice on the same line. Instead, use a separate base name for the
- * alias.
- */
-#define MODULE_ALIAS_CRYPTO(name) \
- __MODULE_INFO(alias, alias_userspace, name); \
- __MODULE_INFO(alias, alias_crypto, "crypto-" name)
-
-/*
- * Algorithm masks and types.
- */
#define CRYPTO_ALG_TYPE_MASK 0x0000000f
-#define CRYPTO_ALG_TYPE_CIPHER 0x00000001
-#define CRYPTO_ALG_TYPE_AEAD 0x00000003
#define CRYPTO_ALG_TYPE_BLKCIPHER 0x00000004
-#define CRYPTO_ALG_TYPE_ABLKCIPHER 0x00000005
-#define CRYPTO_ALG_TYPE_SKCIPHER 0x00000005
-#define CRYPTO_ALG_TYPE_GIVCIPHER 0x00000006
-#define CRYPTO_ALG_TYPE_KPP 0x00000008
-#define CRYPTO_ALG_TYPE_RNG 0x0000000c
-#define CRYPTO_ALG_TYPE_AKCIPHER 0x0000000d
-#define CRYPTO_ALG_TYPE_DIGEST 0x0000000e
-#define CRYPTO_ALG_TYPE_HASH 0x0000000e
#define CRYPTO_ALG_TYPE_SHASH 0x0000000e
-#define CRYPTO_ALG_TYPE_AHASH 0x0000000f
-
-#define CRYPTO_ALG_TYPE_HASH_MASK 0x0000000e
-#define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e
#define CRYPTO_ALG_TYPE_BLKCIPHER_MASK 0x0000000c
-
#define CRYPTO_ALG_ASYNC 0x00000080
-/*
- * Set this bit if and only if the algorithm requires another algorithm of
- * the same type to handle corner cases.
- */
-#define CRYPTO_ALG_NEED_FALLBACK 0x00000100
-
-/*
- * This bit is set for symmetric key ciphers that have already been wrapped
- * with a generic IV generator to prevent them from being wrapped again.
- */
-#define CRYPTO_ALG_GENIV 0x00000200
-
-/*
- * Set if the algorithm is an instance that is build from templates.
- */
-#define CRYPTO_ALG_INSTANCE 0x00000800
-
-/* Set this bit if the algorithm provided is hardware accelerated but
- * not available to userspace via instruction set or so.
- */
-#define CRYPTO_ALG_KERN_DRIVER_ONLY 0x00001000
-
-/*
- * Mark a cipher as a service implementation only usable by another
- * cipher and never by a normal user of the kernel crypto API
- */
-#define CRYPTO_ALG_INTERNAL 0x00002000
-
-/*
- * Transform masks and values (for crt_flags).
- */
-#define CRYPTO_TFM_REQ_MASK 0x000fff00
-#define CRYPTO_TFM_RES_MASK 0xfff00000
-
-#define CRYPTO_TFM_REQ_WEAK_KEY 0x00000100
-#define CRYPTO_TFM_REQ_MAY_SLEEP 0x00000200
-#define CRYPTO_TFM_REQ_MAY_BACKLOG 0x00000400
-#define CRYPTO_TFM_RES_WEAK_KEY 0x00100000
-#define CRYPTO_TFM_RES_BAD_KEY_LEN 0x00200000
-#define CRYPTO_TFM_RES_BAD_KEY_SCHED 0x00400000
-#define CRYPTO_TFM_RES_BAD_BLOCK_LEN 0x00800000
-#define CRYPTO_TFM_RES_BAD_FLAGS 0x01000000
-
-/*
- * Miscellaneous stuff.
- */
#define CRYPTO_MAX_ALG_NAME 64
-/*
- * The macro CRYPTO_MINALIGN_ATTR (along with the void * type in the actual
- * declaration) is used to ensure that the crypto_tfm context structure is
- * aligned correctly for the given architecture so that there are no alignment
- * faults for C data types. In particular, this is required on platforms such
- * as arm where pointers are 32-bit aligned but there are data types such as
- * u64 which require 64-bit alignment.
- */
#define CRYPTO_MINALIGN ARCH_KMALLOC_MINALIGN
-
#define CRYPTO_MINALIGN_ATTR __attribute__ ((__aligned__(CRYPTO_MINALIGN)))
struct scatterlist;
struct crypto_blkcipher;
struct crypto_tfm;
struct crypto_type;
-struct skcipher_givcrypt_request;
struct blkcipher_desc {
- struct crypto_blkcipher *tfm;
- void *info;
- u32 flags;
-};
-
-struct cipher_desc {
- struct crypto_tfm *tfm;
- void (*crfn)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
- unsigned int (*prfn)(const struct cipher_desc *desc, u8 *dst,
- const u8 *src, unsigned int nbytes);
- void *info;
+ struct crypto_blkcipher *tfm;
+ void *info;
+ u32 flags;
};
struct blkcipher_alg {
int (*setkey)(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen);
+ unsigned keylen);
int (*encrypt)(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
- unsigned int nbytes);
+ unsigned nbytes);
int (*decrypt)(struct blkcipher_desc *desc,
struct scatterlist *dst, struct scatterlist *src,
- unsigned int nbytes);
-
- const char *geniv;
-
- unsigned int min_keysize;
- unsigned int max_keysize;
- unsigned int ivsize;
-};
-
-struct cipher_alg {
- unsigned int cia_min_keysize;
- unsigned int cia_max_keysize;
- int (*cia_setkey)(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen);
- void (*cia_encrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
- void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
-};
-
-struct compress_alg {
- int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src,
- unsigned int slen, u8 *dst, unsigned int *dlen);
- int (*coa_decompress)(struct crypto_tfm *tfm, const u8 *src,
- unsigned int slen, u8 *dst, unsigned int *dlen);
+ unsigned nbytes);
};
-
#define cra_blkcipher cra_u.blkcipher
-#define cra_cipher cra_u.cipher
-#define cra_compress cra_u.compress
struct crypto_alg {
- struct list_head cra_list;
- struct list_head cra_users;
-
- u32 cra_flags;
- unsigned int cra_blocksize;
- unsigned int cra_ctxsize;
- unsigned int cra_alignmask;
-
- int cra_priority;
- atomic_t cra_refcnt;
+ struct list_head cra_list;
+ struct list_head cra_users;
- char cra_name[CRYPTO_MAX_ALG_NAME];
- char cra_driver_name[CRYPTO_MAX_ALG_NAME];
+ u32 cra_flags;
+ unsigned cra_ctxsize;
+ char cra_name[CRYPTO_MAX_ALG_NAME];
const struct crypto_type *cra_type;
union {
struct blkcipher_alg blkcipher;
- struct cipher_alg cipher;
- struct compress_alg compress;
} cra_u;
int (*cra_init)(struct crypto_tfm *tfm);
void (*cra_exit)(struct crypto_tfm *tfm);
- void (*cra_destroy)(struct crypto_alg *alg);
-
- struct module *cra_module;
} CRYPTO_MINALIGN_ATTR;
-/*
- * Algorithm registration interface.
- */
int crypto_register_alg(struct crypto_alg *alg);
-int crypto_unregister_alg(struct crypto_alg *alg);
-int crypto_register_algs(struct crypto_alg *algs, int count);
-int crypto_unregister_algs(struct crypto_alg *algs, int count);
-
-/*
- * Algorithm query interface.
- */
-int crypto_has_alg(const char *name, u32 type, u32 mask);
-
-/*
- * Transforms: user-instantiated objects which encapsulate algorithms
- * and core processing logic. Managed via crypto_alloc_*() and
- * crypto_free_*(), as well as the various helpers below.
- */
struct blkcipher_tfm {
- void *iv;
int (*setkey)(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen);
+ unsigned keylen);
int (*encrypt)(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes);
+ struct scatterlist *src, unsigned nbytes);
int (*decrypt)(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes);
+ struct scatterlist *src, unsigned nbytes);
};
-struct cipher_tfm {
- int (*cit_setkey)(struct crypto_tfm *tfm,
- const u8 *key, unsigned int keylen);
- void (*cit_encrypt_one)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
- void (*cit_decrypt_one)(struct crypto_tfm *tfm, u8 *dst, const u8 *src);
-};
-
-struct compress_tfm {
- int (*cot_compress)(struct crypto_tfm *tfm,
- const u8 *src, unsigned int slen,
- u8 *dst, unsigned int *dlen);
- int (*cot_decompress)(struct crypto_tfm *tfm,
- const u8 *src, unsigned int slen,
- u8 *dst, unsigned int *dlen);
-};
-
-#define crt_blkcipher crt_u.blkcipher
-#define crt_cipher crt_u.cipher
-#define crt_compress crt_u.compress
-
struct crypto_tfm {
+ u32 crt_flags;
- u32 crt_flags;
-
- union {
- struct blkcipher_tfm blkcipher;
- struct cipher_tfm cipher;
- struct compress_tfm compress;
- } crt_u;
+ struct blkcipher_tfm crt_blkcipher;
void (*exit)(struct crypto_tfm *tfm);
- struct crypto_alg *__crt_alg;
-
- void *__crt_ctx[] CRYPTO_MINALIGN_ATTR;
-};
-
-struct crypto_blkcipher {
- struct crypto_tfm base;
-};
-
-struct crypto_cipher {
- struct crypto_tfm base;
-};
-
-struct crypto_comp {
- struct crypto_tfm base;
+ struct crypto_alg *__crt_alg;
+ void *__crt_ctx[] CRYPTO_MINALIGN_ATTR;
};
-enum {
- CRYPTOA_UNSPEC,
- CRYPTOA_ALG,
- CRYPTOA_TYPE,
- CRYPTOA_U32,
- __CRYPTOA_MAX,
-};
-
-#define CRYPTOA_MAX (__CRYPTOA_MAX - 1)
-
-/* Maximum number of (rtattr) parameters for each template. */
-#define CRYPTO_MAX_ATTRS 32
-
-struct crypto_attr_alg {
- char name[CRYPTO_MAX_ALG_NAME];
-};
-
-struct crypto_attr_type {
- u32 type;
- u32 mask;
-};
-
-struct crypto_attr_u32 {
- u32 num;
-};
-
-/*
- * Transform user interface.
- */
-
struct crypto_tfm *crypto_alloc_base(const char *alg_name, u32 type, u32 mask);
void crypto_destroy_tfm(void *mem, struct crypto_tfm *tfm);
return crypto_destroy_tfm(tfm, tfm);
}
-int alg_test(const char *driver, const char *alg, u32 type, u32 mask);
-
-/*
- * Transform helpers which query the underlying algorithm.
- */
-static inline const char *crypto_tfm_alg_name(struct crypto_tfm *tfm)
-{
- return tfm->__crt_alg->cra_name;
-}
-
-static inline const char *crypto_tfm_alg_driver_name(struct crypto_tfm *tfm)
-{
- return tfm->__crt_alg->cra_driver_name;
-}
-
-static inline int crypto_tfm_alg_priority(struct crypto_tfm *tfm)
-{
- return tfm->__crt_alg->cra_priority;
-}
-
static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm)
{
return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK;
}
-static inline unsigned int crypto_tfm_alg_blocksize(struct crypto_tfm *tfm)
-{
- return tfm->__crt_alg->cra_blocksize;
-}
-
-static inline unsigned int crypto_tfm_alg_alignmask(struct crypto_tfm *tfm)
-{
- return tfm->__crt_alg->cra_alignmask;
-}
-
-static inline u32 crypto_tfm_get_flags(struct crypto_tfm *tfm)
-{
- return tfm->crt_flags;
-}
-
-static inline void crypto_tfm_set_flags(struct crypto_tfm *tfm, u32 flags)
-{
- tfm->crt_flags |= flags;
-}
-
-static inline void crypto_tfm_clear_flags(struct crypto_tfm *tfm, u32 flags)
-{
- tfm->crt_flags &= ~flags;
-}
-
static inline void *crypto_tfm_ctx(struct crypto_tfm *tfm)
{
return tfm->__crt_ctx;
}
-static inline unsigned int crypto_tfm_ctx_alignment(void)
-{
- struct crypto_tfm *tfm;
- return __alignof__(tfm->__crt_ctx);
-}
-
-static inline u32 crypto_skcipher_type(u32 type)
-{
- type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
- type |= CRYPTO_ALG_TYPE_BLKCIPHER;
- return type;
-}
-
-static inline u32 crypto_skcipher_mask(u32 mask)
-{
- mask &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV);
- mask |= CRYPTO_ALG_TYPE_BLKCIPHER_MASK;
- return mask;
-}
-
-/**
- * DOC: Synchronous Block Cipher API
- *
- * The synchronous block cipher API is used with the ciphers of type
- * CRYPTO_ALG_TYPE_BLKCIPHER (listed as type "blkcipher" in /proc/crypto)
- *
- * Synchronous calls, have a context in the tfm. But since a single tfm can be
- * used in multiple calls and in parallel, this info should not be changeable
- * (unless a lock is used). This applies, for example, to the symmetric key.
- * However, the IV is changeable, so there is an iv field in blkcipher_tfm
- * structure for synchronous blkcipher api. So, its the only state info that can
- * be kept for synchronous calls without using a big lock across a tfm.
- *
- * The block cipher API allows the use of a complete cipher, i.e. a cipher
- * consisting of a template (a block chaining mode) and a single block cipher
- * primitive (e.g. AES).
- *
- * The plaintext data buffer and the ciphertext data buffer are pointed to
- * by using scatter/gather lists. The cipher operation is performed
- * on all segments of the provided scatter/gather lists.
- *
- * The kernel crypto API supports a cipher operation "in-place" which means that
- * the caller may provide the same scatter/gather list for the plaintext and
- * cipher text. After the completion of the cipher operation, the plaintext
- * data is replaced with the ciphertext data in case of an encryption and vice
- * versa for a decryption. The caller must ensure that the scatter/gather lists
- * for the output data point to sufficiently large buffers, i.e. multiples of
- * the block size of the cipher.
- */
+struct crypto_blkcipher {
+ struct crypto_tfm base;
+};
static inline struct crypto_blkcipher *__crypto_blkcipher_cast(
struct crypto_tfm *tfm)
return __crypto_blkcipher_cast(tfm);
}
-/**
- * crypto_alloc_blkcipher() - allocate synchronous block cipher handle
- * @alg_name: is the cra_name / name or cra_driver_name / driver name of the
- * blkcipher cipher
- * @type: specifies the type of the cipher
- * @mask: specifies the mask for the cipher
- *
- * Allocate a cipher handle for a block cipher. The returned struct
- * crypto_blkcipher is the cipher handle that is required for any subsequent
- * API invocation for that block cipher.
- *
- * Return: allocated cipher handle in case of success; IS_ERR() is true in case
- * of an error, PTR_ERR() returns the error code.
- */
static inline struct crypto_blkcipher *crypto_alloc_blkcipher(
const char *alg_name, u32 type, u32 mask)
{
return __crypto_blkcipher_cast(crypto_alloc_base(alg_name, type, mask));
}
-static inline struct crypto_tfm *crypto_blkcipher_tfm(
- struct crypto_blkcipher *tfm)
-{
- return &tfm->base;
-}
-
-/**
- * crypto_free_blkcipher() - zeroize and free the block cipher handle
- * @tfm: cipher handle to be freed
- */
static inline void crypto_free_blkcipher(struct crypto_blkcipher *tfm)
{
- crypto_free_tfm(crypto_blkcipher_tfm(tfm));
-}
-
-/**
- * crypto_has_blkcipher() - Search for the availability of a block cipher
- * @alg_name: is the cra_name / name or cra_driver_name / driver name of the
- * block cipher
- * @type: specifies the type of the cipher
- * @mask: specifies the mask for the cipher
- *
- * Return: true when the block cipher is known to the kernel crypto API; false
- * otherwise
- */
-static inline int crypto_has_blkcipher(const char *alg_name, u32 type, u32 mask)
-{
- type &= ~CRYPTO_ALG_TYPE_MASK;
- type |= CRYPTO_ALG_TYPE_BLKCIPHER;
- mask |= CRYPTO_ALG_TYPE_MASK;
-
- return crypto_has_alg(alg_name, type, mask);
-}
-
-/**
- * crypto_blkcipher_name() - return the name / cra_name from the cipher handle
- * @tfm: cipher handle
- *
- * Return: The character string holding the name of the cipher
- */
-static inline const char *crypto_blkcipher_name(struct crypto_blkcipher *tfm)
-{
- return crypto_tfm_alg_name(crypto_blkcipher_tfm(tfm));
+ crypto_free_tfm(&tfm->base);
}
static inline struct blkcipher_tfm *crypto_blkcipher_crt(
struct crypto_blkcipher *tfm)
{
- return &crypto_blkcipher_tfm(tfm)->crt_blkcipher;
+ return &tfm->base.crt_blkcipher;
}
-static inline struct blkcipher_alg *crypto_blkcipher_alg(
- struct crypto_blkcipher *tfm)
-{
- return &crypto_blkcipher_tfm(tfm)->__crt_alg->cra_blkcipher;
-}
-
-/**
- * crypto_blkcipher_ivsize() - obtain IV size
- * @tfm: cipher handle
- *
- * The size of the IV for the block cipher referenced by the cipher handle is
- * returned. This IV size may be zero if the cipher does not need an IV.
- *
- * Return: IV size in bytes
- */
-static inline unsigned int crypto_blkcipher_ivsize(struct crypto_blkcipher *tfm)
-{
- return crypto_blkcipher_alg(tfm)->ivsize;
-}
-
-/**
- * crypto_blkcipher_blocksize() - obtain block size of cipher
- * @tfm: cipher handle
- *
- * The block size for the block cipher referenced with the cipher handle is
- * returned. The caller may use that information to allocate appropriate
- * memory for the data returned by the encryption or decryption operation.
- *
- * Return: block size of cipher
- */
-static inline unsigned int crypto_blkcipher_blocksize(
- struct crypto_blkcipher *tfm)
-{
- return crypto_tfm_alg_blocksize(crypto_blkcipher_tfm(tfm));
-}
-
-static inline unsigned int crypto_blkcipher_alignmask(
- struct crypto_blkcipher *tfm)
-{
- return crypto_tfm_alg_alignmask(crypto_blkcipher_tfm(tfm));
-}
-
-static inline u32 crypto_blkcipher_get_flags(struct crypto_blkcipher *tfm)
-{
- return crypto_tfm_get_flags(crypto_blkcipher_tfm(tfm));
-}
-
-static inline void crypto_blkcipher_set_flags(struct crypto_blkcipher *tfm,
- u32 flags)
-{
- crypto_tfm_set_flags(crypto_blkcipher_tfm(tfm), flags);
-}
-
-static inline void crypto_blkcipher_clear_flags(struct crypto_blkcipher *tfm,
- u32 flags)
-{
- crypto_tfm_clear_flags(crypto_blkcipher_tfm(tfm), flags);
-}
-
-/**
- * crypto_blkcipher_setkey() - set key for cipher
- * @tfm: cipher handle
- * @key: buffer holding the key
- * @keylen: length of the key in bytes
- *
- * The caller provided key is set for the block cipher referenced by the cipher
- * handle.
- *
- * Note, the key length determines the cipher type. Many block ciphers implement
- * different cipher modes depending on the key size, such as AES-128 vs AES-192
- * vs. AES-256. When providing a 16 byte key for an AES cipher handle, AES-128
- * is performed.
- *
- * Return: 0 if the setting of the key was successful; < 0 if an error occurred
- */
static inline int crypto_blkcipher_setkey(struct crypto_blkcipher *tfm,
- const u8 *key, unsigned int keylen)
+ const u8 *key, unsigned keylen)
{
- return crypto_blkcipher_crt(tfm)->setkey(crypto_blkcipher_tfm(tfm),
- key, keylen);
+ return crypto_blkcipher_crt(tfm)->setkey(&tfm->base, key, keylen);
}
-/**
- * crypto_blkcipher_encrypt() - encrypt plaintext
- * @desc: reference to the block cipher handle with meta data
- * @dst: scatter/gather list that is filled by the cipher operation with the
- * ciphertext
- * @src: scatter/gather list that holds the plaintext
- * @nbytes: number of bytes of the plaintext to encrypt.
- *
- * Encrypt plaintext data using the IV set by the caller with a preceding
- * call of crypto_blkcipher_set_iv.
- *
- * The blkcipher_desc data structure must be filled by the caller and can
- * reside on the stack. The caller must fill desc as follows: desc.tfm is filled
- * with the block cipher handle; desc.flags is filled with either
- * CRYPTO_TFM_REQ_MAY_SLEEP or 0.
- *
- * Return: 0 if the cipher operation was successful; < 0 if an error occurred
- */
-static inline int crypto_blkcipher_encrypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src,
- unsigned int nbytes)
-{
- desc->info = crypto_blkcipher_crt(desc->tfm)->iv;
- return crypto_blkcipher_crt(desc->tfm)->encrypt(desc, dst, src, nbytes);
-}
-
-/**
- * crypto_blkcipher_encrypt_iv() - encrypt plaintext with dedicated IV
- * @desc: reference to the block cipher handle with meta data
- * @dst: scatter/gather list that is filled by the cipher operation with the
- * ciphertext
- * @src: scatter/gather list that holds the plaintext
- * @nbytes: number of bytes of the plaintext to encrypt.
- *
- * Encrypt plaintext data with the use of an IV that is solely used for this
- * cipher operation. Any previously set IV is not used.
- *
- * The blkcipher_desc data structure must be filled by the caller and can
- * reside on the stack. The caller must fill desc as follows: desc.tfm is filled
- * with the block cipher handle; desc.info is filled with the IV to be used for
- * the current operation; desc.flags is filled with either
- * CRYPTO_TFM_REQ_MAY_SLEEP or 0.
- *
- * Return: 0 if the cipher operation was successful; < 0 if an error occurred
- */
static inline int crypto_blkcipher_encrypt_iv(struct blkcipher_desc *desc,
struct scatterlist *dst,
struct scatterlist *src,
- unsigned int nbytes)
+ unsigned nbytes)
{
return crypto_blkcipher_crt(desc->tfm)->encrypt(desc, dst, src, nbytes);
}
-/**
- * crypto_blkcipher_decrypt() - decrypt ciphertext
- * @desc: reference to the block cipher handle with meta data
- * @dst: scatter/gather list that is filled by the cipher operation with the
- * plaintext
- * @src: scatter/gather list that holds the ciphertext
- * @nbytes: number of bytes of the ciphertext to decrypt.
- *
- * Decrypt ciphertext data using the IV set by the caller with a preceding
- * call of crypto_blkcipher_set_iv.
- *
- * The blkcipher_desc data structure must be filled by the caller as documented
- * for the crypto_blkcipher_encrypt call above.
- *
- * Return: 0 if the cipher operation was successful; < 0 if an error occurred
- *
- */
-static inline int crypto_blkcipher_decrypt(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src,
- unsigned int nbytes)
-{
- desc->info = crypto_blkcipher_crt(desc->tfm)->iv;
- return crypto_blkcipher_crt(desc->tfm)->decrypt(desc, dst, src, nbytes);
-}
-
-/**
- * crypto_blkcipher_decrypt_iv() - decrypt ciphertext with dedicated IV
- * @desc: reference to the block cipher handle with meta data
- * @dst: scatter/gather list that is filled by the cipher operation with the
- * plaintext
- * @src: scatter/gather list that holds the ciphertext
- * @nbytes: number of bytes of the ciphertext to decrypt.
- *
- * Decrypt ciphertext data with the use of an IV that is solely used for this
- * cipher operation. Any previously set IV is not used.
- *
- * The blkcipher_desc data structure must be filled by the caller as documented
- * for the crypto_blkcipher_encrypt_iv call above.
- *
- * Return: 0 if the cipher operation was successful; < 0 if an error occurred
- */
-static inline int crypto_blkcipher_decrypt_iv(struct blkcipher_desc *desc,
- struct scatterlist *dst,
- struct scatterlist *src,
- unsigned int nbytes)
-{
- return crypto_blkcipher_crt(desc->tfm)->decrypt(desc, dst, src, nbytes);
-}
-
-/**
- * crypto_blkcipher_set_iv() - set IV for cipher
- * @tfm: cipher handle
- * @src: buffer holding the IV
- * @len: length of the IV in bytes
- *
- * The caller provided IV is set for the block cipher referenced by the cipher
- * handle.
- */
-static inline void crypto_blkcipher_set_iv(struct crypto_blkcipher *tfm,
- const u8 *src, unsigned int len)
-{
- memcpy(crypto_blkcipher_crt(tfm)->iv, src, len);
-}
-
-/**
- * crypto_blkcipher_get_iv() - obtain IV from cipher
- * @tfm: cipher handle
- * @dst: buffer filled with the IV
- * @len: length of the buffer dst
- *
- * The caller can obtain the IV set for the block cipher referenced by the
- * cipher handle and store it into the user-provided buffer. If the buffer
- * has an insufficient space, the IV is truncated to fit the buffer.
- */
-static inline void crypto_blkcipher_get_iv(struct crypto_blkcipher *tfm,
- u8 *dst, unsigned int len)
-{
- memcpy(dst, crypto_blkcipher_crt(tfm)->iv, len);
-}
-
-/**
- * DOC: Single Block Cipher API
- *
- * The single block cipher API is used with the ciphers of type
- * CRYPTO_ALG_TYPE_CIPHER (listed as type "cipher" in /proc/crypto).
- *
- * Using the single block cipher API calls, operations with the basic cipher
- * primitive can be implemented. These cipher primitives exclude any block
- * chaining operations including IV handling.
- *
- * The purpose of this single block cipher API is to support the implementation
- * of templates or other concepts that only need to perform the cipher operation
- * on one block at a time. Templates invoke the underlying cipher primitive
- * block-wise and process either the input or the output data of these cipher
- * operations.
- */
-
-static inline struct crypto_cipher *__crypto_cipher_cast(struct crypto_tfm *tfm)
-{
- return (struct crypto_cipher *)tfm;
-}
-
-static inline struct crypto_cipher *crypto_cipher_cast(struct crypto_tfm *tfm)
-{
- BUG_ON(crypto_tfm_alg_type(tfm) != CRYPTO_ALG_TYPE_CIPHER);
- return __crypto_cipher_cast(tfm);
-}
-
-/**
- * crypto_alloc_cipher() - allocate single block cipher handle
- * @alg_name: is the cra_name / name or cra_driver_name / driver name of the
- * single block cipher
- * @type: specifies the type of the cipher
- * @mask: specifies the mask for the cipher
- *
- * Allocate a cipher handle for a single block cipher. The returned struct
- * crypto_cipher is the cipher handle that is required for any subsequent API
- * invocation for that single block cipher.
- *
- * Return: allocated cipher handle in case of success; IS_ERR() is true in case
- * of an error, PTR_ERR() returns the error code.
- */
-static inline struct crypto_cipher *crypto_alloc_cipher(const char *alg_name,
- u32 type, u32 mask)
-{
- type &= ~CRYPTO_ALG_TYPE_MASK;
- type |= CRYPTO_ALG_TYPE_CIPHER;
- mask |= CRYPTO_ALG_TYPE_MASK;
-
- return __crypto_cipher_cast(crypto_alloc_base(alg_name, type, mask));
-}
-
-static inline struct crypto_tfm *crypto_cipher_tfm(struct crypto_cipher *tfm)
-{
- return &tfm->base;
-}
-
-/**
- * crypto_free_cipher() - zeroize and free the single block cipher handle
- * @tfm: cipher handle to be freed
- */
-static inline void crypto_free_cipher(struct crypto_cipher *tfm)
-{
- crypto_free_tfm(crypto_cipher_tfm(tfm));
-}
-
-/**
- * crypto_has_cipher() - Search for the availability of a single block cipher
- * @alg_name: is the cra_name / name or cra_driver_name / driver name of the
- * single block cipher
- * @type: specifies the type of the cipher
- * @mask: specifies the mask for the cipher
- *
- * Return: true when the single block cipher is known to the kernel crypto API;
- * false otherwise
- */
-static inline int crypto_has_cipher(const char *alg_name, u32 type, u32 mask)
-{
- type &= ~CRYPTO_ALG_TYPE_MASK;
- type |= CRYPTO_ALG_TYPE_CIPHER;
- mask |= CRYPTO_ALG_TYPE_MASK;
-
- return crypto_has_alg(alg_name, type, mask);
-}
-
-static inline struct cipher_tfm *crypto_cipher_crt(struct crypto_cipher *tfm)
-{
- return &crypto_cipher_tfm(tfm)->crt_cipher;
-}
-
-/**
- * crypto_cipher_blocksize() - obtain block size for cipher
- * @tfm: cipher handle
- *
- * The block size for the single block cipher referenced with the cipher handle
- * tfm is returned. The caller may use that information to allocate appropriate
- * memory for the data returned by the encryption or decryption operation
- *
- * Return: block size of cipher
- */
-static inline unsigned int crypto_cipher_blocksize(struct crypto_cipher *tfm)
-{
- return crypto_tfm_alg_blocksize(crypto_cipher_tfm(tfm));
-}
-
-static inline unsigned int crypto_cipher_alignmask(struct crypto_cipher *tfm)
-{
- return crypto_tfm_alg_alignmask(crypto_cipher_tfm(tfm));
-}
-
-static inline u32 crypto_cipher_get_flags(struct crypto_cipher *tfm)
-{
- return crypto_tfm_get_flags(crypto_cipher_tfm(tfm));
-}
-
-static inline void crypto_cipher_set_flags(struct crypto_cipher *tfm,
- u32 flags)
-{
- crypto_tfm_set_flags(crypto_cipher_tfm(tfm), flags);
-}
-
-static inline void crypto_cipher_clear_flags(struct crypto_cipher *tfm,
- u32 flags)
-{
- crypto_tfm_clear_flags(crypto_cipher_tfm(tfm), flags);
-}
-
-/**
- * crypto_cipher_setkey() - set key for cipher
- * @tfm: cipher handle
- * @key: buffer holding the key
- * @keylen: length of the key in bytes
- *
- * The caller provided key is set for the single block cipher referenced by the
- * cipher handle.
- *
- * Note, the key length determines the cipher type. Many block ciphers implement
- * different cipher modes depending on the key size, such as AES-128 vs AES-192
- * vs. AES-256. When providing a 16 byte key for an AES cipher handle, AES-128
- * is performed.
- *
- * Return: 0 if the setting of the key was successful; < 0 if an error occurred
- */
-static inline int crypto_cipher_setkey(struct crypto_cipher *tfm,
- const u8 *key, unsigned int keylen)
-{
- return crypto_cipher_crt(tfm)->cit_setkey(crypto_cipher_tfm(tfm),
- key, keylen);
-}
-
-/**
- * crypto_cipher_encrypt_one() - encrypt one block of plaintext
- * @tfm: cipher handle
- * @dst: points to the buffer that will be filled with the ciphertext
- * @src: buffer holding the plaintext to be encrypted
- *
- * Invoke the encryption operation of one block. The caller must ensure that
- * the plaintext and ciphertext buffers are at least one block in size.
- */
-static inline void crypto_cipher_encrypt_one(struct crypto_cipher *tfm,
- u8 *dst, const u8 *src)
-{
- crypto_cipher_crt(tfm)->cit_encrypt_one(crypto_cipher_tfm(tfm),
- dst, src);
-}
-
-/**
- * crypto_cipher_decrypt_one() - decrypt one block of ciphertext
- * @tfm: cipher handle
- * @dst: points to the buffer that will be filled with the plaintext
- * @src: buffer holding the ciphertext to be decrypted
- *
- * Invoke the decryption operation of one block. The caller must ensure that
- * the plaintext and ciphertext buffers are at least one block in size.
- */
-static inline void crypto_cipher_decrypt_one(struct crypto_cipher *tfm,
- u8 *dst, const u8 *src)
-{
- crypto_cipher_crt(tfm)->cit_decrypt_one(crypto_cipher_tfm(tfm),
- dst, src);
-}
-
#endif /* _LINUX_CRYPTO_H */
+++ /dev/null
-#ifndef __CRYPTOHASH_H
-#define __CRYPTOHASH_H
-
-#include <linux/types.h>
-
-#define SHA_DIGEST_WORDS 5
-#define SHA_MESSAGE_BYTES (512 /*bits*/ / 8)
-#define SHA_WORKSPACE_WORDS 16
-
-void sha_init(__u32 *buf);
-void sha_transform(__u32 *digest, const char *data, __u32 *W);
-
-#define MD5_DIGEST_WORDS 4
-#define MD5_MESSAGE_BYTES 64
-
-void md5_transform(__u32 *hash, __u32 const *in);
-
-__u32 half_md4_transform(__u32 buf[4], __u32 const in[8]);
-
-#endif
BUILD_BUG_ON_ZERO((perms) & 2) + \
(perms))
-#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
-
#endif
--- /dev/null
+#ifndef _LINUX_KEY_H
+#define _LINUX_KEY_H
+
+#include <linux/types.h>
+#include <linux/list.h>
+#include <linux/rbtree.h>
+#include <linux/rcupdate.h>
+#include <linux/sysctl.h>
+#include <linux/rwsem.h>
+#include <linux/atomic.h>
+
+#include <keyutils.h>
+
+struct key;
+
+struct user_key_payload {
+ size_t datalen; /* length of this data */
+ char data[0]; /* actual data */
+};
+
+struct key {
+ atomic_t usage; /* number of references */
+ key_serial_t serial; /* key serial number */
+ struct rw_semaphore sem; /* change vs change sem */
+ struct user_key_payload payload;
+};
+
+static inline const struct user_key_payload *user_key_payload(const struct key *key)
+{
+ return &key->payload;
+}
+
+static inline void key_put(struct key *key)
+{
+ if (atomic_dec_and_test(&key->usage))
+ free(key);
+}
+
+static inline struct key *__key_get(struct key *key)
+{
+ atomic_inc(&key->usage);
+ return key;
+}
+
+static inline struct key *key_get(struct key *key)
+{
+ return key ? __key_get(key) : key;
+}
+
+#endif /* _LINUX_KEY_H */
size_t elem_size;
} mempool_t;
+static inline bool mempool_initialized(mempool_t *pool)
+{
+ return true;
+}
+
extern int mempool_resize(mempool_t *pool, int new_min_nr);
static inline void mempool_free(void *element, mempool_t *pool)
struct page;
-#define virt_to_page(kaddr) ((struct page *) (kaddr))
-#define page_address(kaddr) ((void *) (kaddr))
+#define virt_to_page(p) \
+ ((struct page *) (((unsigned long) (p)) & PAGE_MASK))
+#define offset_in_page(p) ((unsigned long) (p) & ~PAGE_MASK)
+
+#define page_address(p) ((void *) (p))
#define kmap_atomic(page) page_address(page)
#define kunmap_atomic(addr) do {} while (0)
--- /dev/null
+#ifndef _LINUX_SCATTERLIST_H
+#define _LINUX_SCATTERLIST_H
+
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/bug.h>
+#include <linux/mm.h>
+
+struct scatterlist {
+ unsigned long page_link;
+ unsigned int offset;
+ unsigned int length;
+};
+
+#define sg_is_chain(sg) ((sg)->page_link & 0x01)
+#define sg_is_last(sg) ((sg)->page_link & 0x02)
+#define sg_chain_ptr(sg) \
+ ((struct scatterlist *) ((sg)->page_link & ~0x03))
+
+static inline void sg_assign_page(struct scatterlist *sg, struct page *page)
+{
+ unsigned long page_link = sg->page_link & 0x3;
+
+ /*
+ * In order for the low bit stealing approach to work, pages
+ * must be aligned at a 32-bit boundary as a minimum.
+ */
+ BUG_ON((unsigned long) page & 0x03);
+ sg->page_link = page_link | (unsigned long) page;
+}
+
+static inline void sg_set_page(struct scatterlist *sg, struct page *page,
+ unsigned int len, unsigned int offset)
+{
+ sg_assign_page(sg, page);
+ sg->offset = offset;
+ sg->length = len;
+}
+
+static inline struct page *sg_page(struct scatterlist *sg)
+{
+ return (struct page *)((sg)->page_link & ~0x3);
+}
+
+static inline void sg_set_buf(struct scatterlist *sg, const void *buf,
+ unsigned int buflen)
+{
+ sg_set_page(sg, virt_to_page(buf), buflen, offset_in_page(buf));
+}
+
+static inline struct scatterlist *sg_next(struct scatterlist *sg)
+{
+ if (sg_is_last(sg))
+ return NULL;
+
+ sg++;
+ if (unlikely(sg_is_chain(sg)))
+ sg = sg_chain_ptr(sg);
+
+ return sg;
+}
+
+#define for_each_sg(sglist, sg, nr, __i) \
+ for (__i = 0, sg = (sglist); __i < (nr); __i++, sg = sg_next(sg))
+
+static inline void sg_chain(struct scatterlist *prv, unsigned int prv_nents,
+ struct scatterlist *sgl)
+{
+ /*
+ * offset and length are unused for chain entry. Clear them.
+ */
+ prv[prv_nents - 1].offset = 0;
+ prv[prv_nents - 1].length = 0;
+
+ /*
+ * Set lowest bit to indicate a link pointer, and make sure to clear
+ * the termination bit if it happens to be set.
+ */
+ prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02;
+}
+
+static inline void sg_mark_end(struct scatterlist *sg)
+{
+ sg->page_link |= 0x02;
+ sg->page_link &= ~0x01;
+}
+
+static inline void sg_unmark_end(struct scatterlist *sg)
+{
+ sg->page_link &= ~0x02;
+}
+
+static inline void *sg_virt(struct scatterlist *sg)
+{
+ return page_address(sg_page(sg)) + sg->offset;
+}
+
+static inline void sg_init_table(struct scatterlist *sgl, unsigned int nents)
+{
+ memset(sgl, 0, sizeof(*sgl) * nents);
+ sg_mark_end(&sgl[nents - 1]);
+}
+
+static inline void sg_init_one(struct scatterlist *sg, const void *buf,
+ unsigned int buflen)
+{
+ sg_init_table(sg, 1);
+ sg_set_buf(sg, buf, buflen);
+}
+
+#endif /* _LINUX_SCATTERLIST_H */
#define KTIME_MAX ((s64)~((u64)1 << 63))
#define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC)
+static inline struct timespec ns_to_timespec(const u64 nsec)
+{
+ return (struct timespec) {
+ .tv_sec = nsec / NSEC_PER_SEC,
+ .tv_nsec = nsec % NSEC_PER_SEC,
+ };
+}
+
+static inline s64 timespec_to_ns(const struct timespec *ts)
+{
+ return ((s64) ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec;
+}
+
#if __BITS_PER_LONG == 64
static inline struct timespec timespec64_to_timespec(const struct timespec64 ts64)
# define ns_to_timespec64 ns_to_timespec
# define timespec64_add_ns timespec_add_ns
-static inline s64 timespec_to_ns(const struct timespec *ts)
-{
- return ((s64) ts->tv_sec * NSEC_PER_SEC) + ts->tv_nsec;
-}
-
#else
static inline struct timespec timespec64_to_timespec(const struct timespec64 ts64)
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->inode = inode;
__entry->sector = bio->bi_iter.bi_sector;
__entry->nr_sector = bio->bi_iter.bi_size >> 9;
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->inode = inode;
__entry->sector = bio->bi_iter.bi_sector;
__entry->nr_sector = bio->bi_iter.bi_size >> 9;
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->size = size;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
),
TP_printk("%pU", __entry->uuid)
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->disk_sb.sb->disk_uuid.b, 16);
+ memcpy(__entry->uuid, ca->uuid.b, 16);
__entry->cur_idx = cur_idx;
__entry->last_idx = last_idx;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->seq = seq;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->seq = seq;
__entry->written = written;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->disk_sb.sb->disk_uuid.b, 16);
+ memcpy(__entry->uuid, ca->uuid.b, 16);
__entry->tier = ca->mi.tier;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->bucket = PTR_BUCKET_NR_TRACE(c, &b->key, 0);
__entry->level = b->level;
__entry->id = b->btree_id;
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->id = id;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->touched = touched;
__entry->freed = freed;
__entry->can_free = can_free;
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
),
TP_printk("%pU", __entry->uuid)
TP_fast_assign(
__entry->reason = reason;
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->disk_sb->user_uuid.b, 16);
),
TP_printk("%pU: %u", __entry->uuid, __entry->reason)
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->old_bucket = PTR_BUCKET_NR_TRACE(c,
&old->key, 0);
__entry->bucket = PTR_BUCKET_NR_TRACE(c, &b->key, 0);
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->disk_sb.sb->disk_uuid.b, 16);
+ memcpy(__entry->uuid, ca->uuid.b, 16);
__entry->inode = k->p.inode;
__entry->offset = k->p.offset;
__entry->sectors = sectors;
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->disk_sb.sb->disk_uuid.b, 16);
+ memcpy(__entry->uuid, ca->uuid.b, 16);
__entry->free = free;
__entry->total = total;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->required = required;
__entry->cl = cl;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->disk_sb.sb->disk_uuid.b, 16);
+ memcpy(__entry->uuid, ca->uuid.b, 16);
__entry->reserve = reserve;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->reserve = reserve;
__entry->cl = cl;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->cl = cl;
),
),
TP_fast_assign(
- memcpy(__entry->uuid, ca->disk_sb.sb->disk_uuid.b, 16);
+ memcpy(__entry->uuid, ca->uuid.b, 16);
__entry->sectors_moved = sectors_moved;
__entry->keys_moved = keys_moved;
__entry->buckets_moved = buckets_moved;
),
TP_fast_assign(
- memcpy(__entry->uuid, c->disk_sb.user_uuid.b, 16);
+ memcpy(__entry->uuid, c->sb.user_uuid.b, 16);
__entry->sectors_moved = sectors_moved;
__entry->keys_moved = keys_moved;
),
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <time.h>
#include <unistd.h>
#include <uuid/uuid.h>
#include "linux/bcache.h"
#include "libbcache.h"
#include "checksum.h"
+#include "crypto.h"
#include "opts.h"
+#include "super-io.h"
+
+#define NSEC_PER_SEC 1000000000L
#define BCH_MIN_NR_NBUCKETS (1 << 10)
/* first bucket should start 1 mb in, in sectors: */
#define FIRST_BUCKET_OFFSET (1 << 11)
-void __do_write_sb(int fd, void *sb, size_t bytes)
-{
- char zeroes[SB_SECTOR << 9] = {0};
-
- /* Zero start of disk */
- xpwrite(fd, zeroes, SB_SECTOR << 9, 0);
-
- /* Write superblock */
- xpwrite(fd, sb, bytes, SB_SECTOR << 9);
-
- fsync(fd);
- close(fd);
-}
-
-#define do_write_sb(_fd, _sb) \
- __do_write_sb(_fd, _sb, ((void *) __bset_bkey_last(_sb)) - (void *) _sb);
-
/* minimum size filesystem we can create, given a bucket size: */
static u64 min_size(unsigned bucket_size)
{
BCH_MIN_NR_NBUCKETS) * bucket_size;
}
+static void init_layout(struct bch_sb_layout *l)
+{
+ memset(l, 0, sizeof(*l));
+
+ l->magic = BCACHE_MAGIC;
+ l->layout_type = 0;
+ l->nr_superblocks = 2;
+ l->sb_max_size_bits = 7;
+ l->sb_offset[0] = cpu_to_le64(BCH_SB_SECTOR);
+ l->sb_offset[1] = cpu_to_le64(BCH_SB_SECTOR +
+ (1 << l->sb_max_size_bits));
+}
+
void bcache_format(struct dev_opts *devs, size_t nr_devs,
unsigned block_size,
unsigned btree_node_size,
unsigned meta_csum_type,
unsigned data_csum_type,
unsigned compression_type,
+ const char *passphrase,
unsigned meta_replicas,
unsigned data_replicas,
unsigned on_error_action,
char *label,
uuid_le uuid)
{
- struct cache_sb *sb;
+ struct bch_sb *sb;
struct dev_opts *i;
+ struct bch_sb_field_members *mi;
+ unsigned u64s, j;
/* calculate block size: */
if (!block_size)
max_journal_entry_size = roundup_pow_of_two(max_journal_entry_size);
- sb = calloc(1, sizeof(*sb) + sizeof(struct cache_member) * nr_devs);
+ sb = calloc(1, sizeof(*sb) +
+ sizeof(struct bch_sb_field_members) +
+ sizeof(struct bch_member) * nr_devs +
+ sizeof(struct bch_sb_field_crypt));
- sb->offset = __cpu_to_le64(SB_SECTOR);
- sb->version = __cpu_to_le64(BCACHE_SB_VERSION_CDEV_V3);
+ sb->version = cpu_to_le64(BCACHE_SB_VERSION_CDEV_V4);
sb->magic = BCACHE_MAGIC;
- sb->block_size = __cpu_to_le16(block_size);
+ sb->block_size = cpu_to_le16(block_size);
sb->user_uuid = uuid;
- sb->nr_in_set = nr_devs;
+ sb->nr_devices = nr_devs;
+
+ init_layout(&sb->layout);
- uuid_generate(sb->set_uuid.b);
+ uuid_generate(sb->uuid.b);
if (label)
strncpy((char *) sb->label, label, sizeof(sb->label));
* don't have a userspace crc32c implementation handy, just always use
* crc64
*/
- SET_CACHE_SB_CSUM_TYPE(sb, BCH_CSUM_CRC64);
- SET_CACHE_SET_META_PREFERRED_CSUM_TYPE(sb, meta_csum_type);
- SET_CACHE_SET_DATA_PREFERRED_CSUM_TYPE(sb, data_csum_type);
- SET_CACHE_SET_COMPRESSION_TYPE(sb, compression_type);
-
- SET_CACHE_SET_BTREE_NODE_SIZE(sb, btree_node_size);
- SET_CACHE_SET_META_REPLICAS_WANT(sb, meta_replicas);
- SET_CACHE_SET_META_REPLICAS_HAVE(sb, meta_replicas);
- SET_CACHE_SET_DATA_REPLICAS_WANT(sb, data_replicas);
- SET_CACHE_SET_DATA_REPLICAS_HAVE(sb, data_replicas);
- SET_CACHE_SET_ERROR_ACTION(sb, on_error_action);
- SET_CACHE_SET_STR_HASH_TYPE(sb, BCH_STR_HASH_SIPHASH);
- SET_CACHE_SET_JOURNAL_ENTRY_SIZE(sb, ilog2(max_journal_entry_size));
+ SET_BCH_SB_CSUM_TYPE(sb, BCH_CSUM_CRC64);
+ SET_BCH_SB_META_CSUM_TYPE(sb, meta_csum_type);
+ SET_BCH_SB_DATA_CSUM_TYPE(sb, data_csum_type);
+ SET_BCH_SB_COMPRESSION_TYPE(sb, compression_type);
+
+ SET_BCH_SB_BTREE_NODE_SIZE(sb, btree_node_size);
+ SET_BCH_SB_GC_RESERVE(sb, 8);
+ SET_BCH_SB_META_REPLICAS_WANT(sb, meta_replicas);
+ SET_BCH_SB_META_REPLICAS_HAVE(sb, meta_replicas);
+ SET_BCH_SB_DATA_REPLICAS_WANT(sb, data_replicas);
+ SET_BCH_SB_DATA_REPLICAS_HAVE(sb, data_replicas);
+ SET_BCH_SB_ERROR_ACTION(sb, on_error_action);
+ SET_BCH_SB_STR_HASH_TYPE(sb, BCH_STR_HASH_SIPHASH);
+ SET_BCH_SB_JOURNAL_ENTRY_SIZE(sb, ilog2(max_journal_entry_size));
+
+ struct timespec now;
+ if (clock_gettime(CLOCK_REALTIME, &now))
+ die("error getting current time: %s", strerror(errno));
+
+ sb->time_base_lo = cpu_to_le64(now.tv_sec * NSEC_PER_SEC + now.tv_nsec);
+ sb->time_precision = cpu_to_le32(1);
+
+ if (passphrase) {
+ struct bch_sb_field_crypt *crypt = vstruct_end(sb);
+
+ u64s = sizeof(struct bch_sb_field_crypt) / sizeof(u64);
+
+ le32_add_cpu(&sb->u64s, u64s);
+ crypt->field.u64s = cpu_to_le32(u64s);
+ crypt->field.type = BCH_SB_FIELD_crypt;
+
+ bch_sb_crypt_init(sb, crypt, passphrase);
+ SET_BCH_SB_ENCRYPTION_TYPE(sb, 1);
+ }
+
+ mi = vstruct_end(sb);
+ u64s = (sizeof(struct bch_sb_field_members) +
+ sizeof(struct bch_member) * nr_devs) / sizeof(u64);
+
+ le32_add_cpu(&sb->u64s, u64s);
+ mi->field.u64s = cpu_to_le32(u64s);
+ mi->field.type = BCH_SB_FIELD_members;
for (i = devs; i < devs + nr_devs; i++) {
- struct cache_member *m = sb->members + (i - devs);
+ struct bch_member *m = mi->members + (i - devs);
uuid_generate(m->uuid.b);
- m->nbuckets = __cpu_to_le64(i->nbuckets);
- m->first_bucket = __cpu_to_le16(i->first_bucket);
- m->bucket_size = __cpu_to_le16(i->bucket_size);
+ m->nbuckets = cpu_to_le64(i->nbuckets);
+ m->first_bucket = cpu_to_le16(i->first_bucket);
+ m->bucket_size = cpu_to_le16(i->bucket_size);
- SET_CACHE_TIER(m, i->tier);
- SET_CACHE_REPLACEMENT(m, CACHE_REPLACEMENT_LRU);
- SET_CACHE_DISCARD(m, i->discard);
+ SET_BCH_MEMBER_TIER(m, i->tier);
+ SET_BCH_MEMBER_REPLACEMENT(m, CACHE_REPLACEMENT_LRU);
+ SET_BCH_MEMBER_DISCARD(m, i->discard);
}
- sb->u64s = __cpu_to_le16(bch_journal_buckets_offset(sb));
-
for (i = devs; i < devs + nr_devs; i++) {
- struct cache_member *m = sb->members + (i - devs);
+ sb->dev_idx = i - devs;
+
+ static const char zeroes[BCH_SB_SECTOR << 9];
+ struct nonce nonce = { 0 };
+
+ /* Zero start of disk */
+ xpwrite(i->fd, zeroes, BCH_SB_SECTOR << 9, 0);
+
+ xpwrite(i->fd, &sb->layout, sizeof(sb->layout),
+ BCH_SB_LAYOUT_SECTOR << 9);
- sb->disk_uuid = m->uuid;
- sb->nr_this_dev = i - devs;
- sb->csum = __cpu_to_le64(__csum_set(sb, __le16_to_cpu(sb->u64s),
- CACHE_SB_CSUM_TYPE(sb)));
+ for (j = 0; j < sb->layout.nr_superblocks; j++) {
+ sb->offset = sb->layout.sb_offset[j];
- do_write_sb(i->fd, sb);
+ sb->csum = csum_vstruct(NULL, BCH_SB_CSUM_TYPE(sb),
+ nonce, sb);
+ xpwrite(i->fd, sb, vstruct_bytes(sb),
+ le64_to_cpu(sb->offset) << 9);
+ }
+
+ fsync(i->fd);
+ close(i->fd);
}
bcache_super_print(sb, HUMAN_READABLE);
free(sb);
}
-void bcache_super_print(struct cache_sb *sb, int units)
+struct bch_sb *bcache_super_read(const char *path)
{
- unsigned i;
+ struct bch_sb sb, *ret;
+
+ int fd = open(path, O_RDONLY);
+ if (fd < 0)
+ die("couldn't open %s", path);
+
+ xpread(fd, &sb, sizeof(sb), BCH_SB_SECTOR << 9);
+
+ if (memcmp(&sb.magic, &BCACHE_MAGIC, sizeof(sb.magic)))
+ die("not a bcache superblock");
+
+ size_t bytes = vstruct_bytes(&sb);
+
+ ret = malloc(bytes);
+
+ xpread(fd, ret, bytes, BCH_SB_SECTOR << 9);
+
+ return ret;
+}
+
+void bcache_super_print(struct bch_sb *sb, int units)
+{
+ struct bch_sb_field_members *mi;
char user_uuid_str[40], internal_uuid_str[40], member_uuid_str[40];
- char label[SB_LABEL_SIZE + 1];
+ char label[BCH_SB_LABEL_SIZE + 1];
+ unsigned i;
memset(label, 0, sizeof(label));
memcpy(label, sb->label, sizeof(sb->label));
uuid_unparse(sb->user_uuid.b, user_uuid_str);
- uuid_unparse(sb->set_uuid.b, internal_uuid_str);
+ uuid_unparse(sb->uuid.b, internal_uuid_str);
printf("External UUID: %s\n"
"Internal UUID: %s\n"
label,
le64_to_cpu(sb->version),
pr_units(le16_to_cpu(sb->block_size), units),
- pr_units(CACHE_SET_BTREE_NODE_SIZE(sb), units),
- pr_units(1U << CACHE_SET_JOURNAL_ENTRY_SIZE(sb), units),
+ pr_units(BCH_SB_BTREE_NODE_SIZE(sb), units),
+ pr_units(1U << BCH_SB_JOURNAL_ENTRY_SIZE(sb), units),
- CACHE_SET_ERROR_ACTION(sb) < BCH_NR_ERROR_ACTIONS
- ? bch_error_actions[CACHE_SET_ERROR_ACTION(sb)]
+ BCH_SB_ERROR_ACTION(sb) < BCH_NR_ERROR_ACTIONS
+ ? bch_error_actions[BCH_SB_ERROR_ACTION(sb)]
: "unknown",
- CACHE_SET_CLEAN(sb),
+ BCH_SB_CLEAN(sb),
- CACHE_SET_META_REPLICAS_HAVE(sb),
- CACHE_SET_META_REPLICAS_WANT(sb),
- CACHE_SET_DATA_REPLICAS_HAVE(sb),
- CACHE_SET_DATA_REPLICAS_WANT(sb),
+ BCH_SB_META_REPLICAS_HAVE(sb),
+ BCH_SB_META_REPLICAS_WANT(sb),
+ BCH_SB_DATA_REPLICAS_HAVE(sb),
+ BCH_SB_DATA_REPLICAS_WANT(sb),
- CACHE_SET_META_PREFERRED_CSUM_TYPE(sb) < BCH_CSUM_NR
- ? bch_csum_types[CACHE_SET_META_PREFERRED_CSUM_TYPE(sb)]
+ BCH_SB_META_CSUM_TYPE(sb) < BCH_CSUM_NR
+ ? bch_csum_types[BCH_SB_META_CSUM_TYPE(sb)]
: "unknown",
- CACHE_SET_DATA_PREFERRED_CSUM_TYPE(sb) < BCH_CSUM_NR
- ? bch_csum_types[CACHE_SET_DATA_PREFERRED_CSUM_TYPE(sb)]
+ BCH_SB_DATA_CSUM_TYPE(sb) < BCH_CSUM_NR
+ ? bch_csum_types[BCH_SB_DATA_CSUM_TYPE(sb)]
: "unknown",
- CACHE_SET_COMPRESSION_TYPE(sb) < BCH_COMPRESSION_NR
- ? bch_compression_types[CACHE_SET_COMPRESSION_TYPE(sb)]
+ BCH_SB_COMPRESSION_TYPE(sb) < BCH_COMPRESSION_NR
+ ? bch_compression_types[BCH_SB_COMPRESSION_TYPE(sb)]
: "unknown",
- CACHE_SET_STR_HASH_TYPE(sb) < BCH_STR_HASH_NR
- ? bch_str_hash_types[CACHE_SET_STR_HASH_TYPE(sb)]
+ BCH_SB_STR_HASH_TYPE(sb) < BCH_STR_HASH_NR
+ ? bch_str_hash_types[BCH_SB_STR_HASH_TYPE(sb)]
: "unknown",
- CACHE_INODE_32BIT(sb),
- CACHE_SET_GC_RESERVE(sb),
- CACHE_SET_ROOT_RESERVE(sb),
+ BCH_SB_INODE_32BIT(sb),
+ BCH_SB_GC_RESERVE(sb),
+ BCH_SB_ROOT_RESERVE(sb),
- sb->nr_in_set);
+ sb->nr_devices);
- for (i = 0; i < sb->nr_in_set; i++) {
- struct cache_member *m = sb->members + i;
+ mi = bch_sb_get_members(sb);
+ if (!mi) {
+ printf("Member info section missing\n");
+ return;
+ }
+
+ for (i = 0; i < sb->nr_devices; i++) {
+ struct bch_member *m = mi->members + i;
time_t last_mount = le64_to_cpu(m->last_mount);
uuid_unparse(m->uuid.b, member_uuid_str);
le64_to_cpu(m->nbuckets),
last_mount ? ctime(&last_mount) : "(never)",
- CACHE_STATE(m) < CACHE_STATE_NR
- ? bch_cache_state[CACHE_STATE(m)]
+ BCH_MEMBER_STATE(m) < BCH_MEMBER_STATE_NR
+ ? bch_cache_state[BCH_MEMBER_STATE(m)]
: "unknown",
- CACHE_TIER(m),
- CACHE_HAS_METADATA(m),
- CACHE_HAS_DATA(m),
+ BCH_MEMBER_TIER(m),
+ BCH_MEMBER_HAS_METADATA(m),
+ BCH_MEMBER_HAS_DATA(m),
- CACHE_REPLACEMENT(m) < CACHE_REPLACEMENT_NR
- ? bch_cache_replacement_policies[CACHE_REPLACEMENT(m)]
+ BCH_MEMBER_REPLACEMENT(m) < CACHE_REPLACEMENT_NR
+ ? bch_cache_replacement_policies[BCH_MEMBER_REPLACEMENT(m)]
: "unknown",
- CACHE_DISCARD(m));
+ BCH_MEMBER_DISCARD(m));
}
}
-
-struct cache_sb *bcache_super_read(const char *path)
-{
- struct cache_sb sb, *ret;
- size_t bytes;
-
- int fd = open(path, O_RDONLY);
- if (fd < 0)
- die("couldn't open %s", path);
-
- xpread(fd, &sb, sizeof(sb), SB_SECTOR << 9);
-
- if (memcmp(&sb.magic, &BCACHE_MAGIC, sizeof(sb.magic)))
- die("not a bcache superblock");
-
- bytes = sizeof(sb) + le16_to_cpu(sb.u64s) * sizeof(u64);
-
- ret = calloc(1, bytes);
-
- xpread(fd, ret, bytes, SB_SECTOR << 9);
-
- return ret;
-}
#define _LIBBCACHE_H
#include <linux/uuid.h>
+#include "tools-util.h"
+#include "vstructs.h"
#include "stdbool.h"
#include "tools-util.h"
unsigned meta_csum_type,
unsigned data_csum_type,
unsigned compression_type,
+ const char *passphrase,
unsigned meta_replicas,
unsigned data_replicas,
unsigned on_error_action,
char *label,
uuid_le uuid);
-void bcache_super_print(struct cache_sb *, int);
+struct bch_sb *bcache_super_read(const char *);
-struct cache_sb *bcache_super_read(const char *);
+void bcache_super_print(struct bch_sb *, int);
#endif /* _LIBBCACHE_H */
if (ret < 0)
return ret;
else {
- inode->i_ctime = CURRENT_TIME_SEC;
+ inode->i_ctime = current_fs_time(inode->i_sb);
mark_inode_dirty(inode);
if (ret == 0)
acl = NULL;
#include "extents.h"
#include "io.h"
#include "journal.h"
-#include "super.h"
+#include "super-io.h"
#include <linux/blkdev.h>
#include <linux/kthread.h>
if (rcu_access_pointer(grp->d[i].dev) == ca)
goto out;
- BUG_ON(grp->nr_devices >= MAX_CACHES_PER_SET);
+ BUG_ON(grp->nr_devices >= BCH_SB_MEMBERS_MAX);
rcu_assign_pointer(grp->d[grp->nr_devices++].dev, ca);
out:
int i;
/* All units are in bytes */
- u64 tier_size[CACHE_TIERS];
- u64 tier_free[CACHE_TIERS];
- u64 tier_dirty[CACHE_TIERS];
+ u64 tier_size[BCH_TIER_MAX];
+ u64 tier_free[BCH_TIER_MAX];
+ u64 tier_dirty[BCH_TIER_MAX];
u64 tier0_can_free = 0;
memset(tier_size, 0, sizeof(tier_size));
memset(tier_dirty, 0, sizeof(tier_dirty));
rcu_read_lock();
- for (i = CACHE_TIERS - 1; i >= 0; --i)
+ for (i = BCH_TIER_MAX - 1; i >= 0; --i)
group_for_each_cache_rcu(ca, &c->cache_tiers[i], iter) {
struct bucket_stats_cache stats = bch_bucket_stats_read_cache(ca);
unsigned bucket_bits = ca->bucket_bits + 9;
return submit_bio_wait(ca->bio_prio);
}
+static struct nonce prio_nonce(struct prio_set *p)
+{
+ return (struct nonce) {{
+ [0] = 0,
+ [1] = p->nonce[0],
+ [2] = p->nonce[1],
+ [3] = p->nonce[2]^BCH_NONCE_PRIO,
+ }};
+}
+
static int bch_prio_write(struct cache *ca)
{
struct cache_set *c = ca->set;
}
p->next_bucket = cpu_to_le64(ca->prio_buckets[i + 1]);
- p->magic = cpu_to_le64(pset_magic(&c->disk_sb));
-
- SET_PSET_CSUM_TYPE(p, c->opts.metadata_checksum);
- p->csum = cpu_to_le64(bch_checksum(PSET_CSUM_TYPE(p),
- &p->magic,
- bucket_bytes(ca) - 8));
+ p->magic = cpu_to_le64(pset_magic(c));
+ get_random_bytes(&p->nonce, sizeof(p->nonce));
spin_lock(&ca->prio_buckets_lock);
r = bch_bucket_alloc(ca, RESERVE_PRIO);
bch_mark_metadata_bucket(ca, ca->buckets + r, false);
spin_unlock(&ca->prio_buckets_lock);
+ SET_PSET_CSUM_TYPE(p, bch_meta_checksum_type(c));
+
+ bch_encrypt(c, PSET_CSUM_TYPE(p),
+ prio_nonce(p),
+ p->encrypted_start,
+ bucket_bytes(ca) -
+ offsetof(struct prio_set, encrypted_start));
+
+ p->csum = bch_checksum(c, PSET_CSUM_TYPE(p),
+ prio_nonce(p),
+ (void *) p + sizeof(p->csum),
+ bucket_bytes(ca) - sizeof(p->csum));
+
ret = prio_io(ca, r, REQ_OP_WRITE);
if (cache_fatal_io_err_on(ret, ca,
"prio write to bucket %zu", r) ||
}
spin_lock(&j->lock);
- j->prio_buckets[ca->sb.nr_this_dev] = cpu_to_le64(ca->prio_buckets[0]);
+ j->prio_buckets[ca->dev_idx] = cpu_to_le64(ca->prio_buckets[0]);
j->nr_prio_buckets = max_t(unsigned,
- ca->sb.nr_this_dev + 1,
+ ca->dev_idx + 1,
j->nr_prio_buckets);
spin_unlock(&j->lock);
return ret;
need_new_journal_entry = j->buf[res.idx].nr_prio_buckets <
- ca->sb.nr_this_dev + 1;
+ ca->dev_idx + 1;
bch_journal_res_put(j, &res);
ret = bch_journal_flush_seq(j, res.seq);
struct prio_set *p = ca->disk_buckets;
struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;
struct bucket_mark new;
+ struct bch_csum csum;
unsigned bucket_nr = 0;
u64 bucket, expect, got;
size_t b;
int ret = 0;
spin_lock(&c->journal.lock);
- bucket = le64_to_cpu(c->journal.prio_buckets[ca->sb.nr_this_dev]);
+ bucket = le64_to_cpu(c->journal.prio_buckets[ca->dev_idx]);
spin_unlock(&c->journal.lock);
/*
return -EIO;
got = le64_to_cpu(p->magic);
- expect = pset_magic(&c->disk_sb);
+ expect = pset_magic(c);
unfixable_fsck_err_on(got != expect, c,
"bad magic (got %llu expect %llu) while reading prios from bucket %llu",
got, expect, bucket);
- got = le64_to_cpu(p->csum);
- expect = bch_checksum(PSET_CSUM_TYPE(p),
- &p->magic,
- bucket_bytes(ca) - 8);
- unfixable_fsck_err_on(got != expect, c,
- "bad checksum (got %llu expect %llu) while reading prios from bucket %llu",
- got, expect, bucket);
+ unfixable_fsck_err_on(PSET_CSUM_TYPE(p) >= BCH_CSUM_NR, c,
+ "prio bucket with unknown csum type %llu bucket %lluu",
+ PSET_CSUM_TYPE(p), bucket);
+
+ csum = bch_checksum(c, PSET_CSUM_TYPE(p),
+ prio_nonce(p),
+ (void *) p + sizeof(p->csum),
+ bucket_bytes(ca) - sizeof(p->csum));
+ unfixable_fsck_err_on(bch_crc_cmp(csum, p->csum), c,
+ "bad checksum reading prios from bucket %llu",
+ bucket);
+
+ bch_encrypt(c, PSET_CSUM_TYPE(p),
+ prio_nonce(p),
+ p->encrypted_start,
+ bucket_bytes(ca) -
+ offsetof(struct prio_set, encrypted_start));
bucket = le64_to_cpu(p->next_bucket);
d = p->data;
spin_lock(&devs->lock);
for (i = 0; i < devs->nr_devices; i++)
- available += !test_bit(devs->d[i].dev->sb.nr_this_dev,
+ available += !test_bit(devs->d[i].dev->dev_idx,
caches_used);
recalc_alloc_group_weights(c, devs);
ca = devs->d[i].dev;
- if (test_bit(ca->sb.nr_this_dev, caches_used))
+ if (test_bit(ca->dev_idx, caches_used))
continue;
if (fail_idx == -1 &&
ob->ptrs[0] = (struct bch_extent_ptr) {
.gen = ca->buckets[bucket].mark.gen,
.offset = bucket_to_sector(ca, bucket),
- .dev = ca->sb.nr_this_dev,
+ .dev = ca->dev_idx,
};
ob->ptr_offset[0] = 0;
- __set_bit(ca->sb.nr_this_dev, caches_used);
+ __set_bit(ca->dev_idx, caches_used);
available--;
devs->cur_device = i;
}
enum alloc_reserve reserve,
struct closure *cl)
{
- long caches_used[BITS_TO_LONGS(MAX_CACHES_PER_SET)];
+ long caches_used[BITS_TO_LONGS(BCH_SB_MEMBERS_MAX)];
int i, dst;
/*
EBUG_ON(bch_extent_has_device(extent_i_to_s_c(e), ob->ptrs[i].dev));
tmp = ob->ptrs[i];
+ tmp.cached = bkey_extent_is_cached(&e->k);
tmp.offset += ob->ptr_offset[i];
extent_ptr_append(e, tmp);
return;
for (ptr = ob->ptrs; ptr < ob->ptrs + ob->nr_ptrs; ptr++)
- if (ptr->dev == ca->sb.nr_this_dev)
+ if (ptr->dev == ca->dev_idx)
goto found;
mutex_unlock(&ob->lock);
if (atomic_read(&ob->pin)) {
mutex_lock(&ob->lock);
for (ptr = ob->ptrs; ptr < ob->ptrs + ob->nr_ptrs; ptr++)
- if (ptr->dev == ca->sb.nr_this_dev) {
+ if (ptr->dev == ca->dev_idx) {
mutex_unlock(&ob->lock);
return true;
}
struct {
u64 weight;
struct cache *dev;
- } d[MAX_CACHES_PER_SET];
+ } d[BCH_SB_MEMBERS_MAX];
};
/* Enough for 16 cache devices, 2 tiers and some left over for pipelining */
struct btree;
struct cache;
+struct crypto_blkcipher;
+struct crypto_ahash;
enum gc_phase {
GC_PHASE_PENDING_DELETE = BTREE_ID_NR + 1,
u16 bucket_size; /* sectors */
u8 state;
u8 tier;
- u8 replication_set;
u8 has_metadata;
u8 has_data;
u8 replacement;
struct cache_member_rcu {
struct rcu_head rcu;
- unsigned nr_in_set;
+ unsigned nr_devices;
struct cache_member_cpu m[];
};
struct cache_group self;
+ u8 dev_idx;
/*
* Cached version of this device's member info from superblock
- * Committed by write_super()
+ * Committed by bch_write_super() -> bch_cache_set_mi_update()
*/
- struct {
- u8 nr_this_dev;
- } sb;
struct cache_member_cpu mi;
+ uuid_le uuid;
struct bcache_superblock disk_sb;
struct percpu_ref writes;
struct work_struct read_only_work;
- struct cache __rcu *cache[MAX_CACHES_PER_SET];
-
- struct mutex mi_lock;
- struct cache_member_rcu __rcu *members;
- struct cache_member *disk_mi; /* protected by register_lock */
+ struct cache __rcu *cache[BCH_SB_MEMBERS_MAX];
struct cache_set_opts opts;
/*
* Cached copy in native endianness:
- * Set by cache_sb_to_cache_set:
+ * Set by bch_cache_set_mi_update():
*/
+ struct cache_member_rcu __rcu *members;
+
+ /* Updated by bch_sb_update():*/
struct {
+ uuid_le uuid;
+ uuid_le user_uuid;
+
u16 block_size;
u16 btree_node_size;
- u8 nr_in_set;
+ u8 nr_devices;
u8 clean;
u8 meta_replicas_have;
u8 data_replicas_have;
u8 str_hash_type;
+ u8 encryption_type;
+
+ u64 time_base_lo;
+ u32 time_base_hi;
+ u32 time_precision;
} sb;
- struct cache_sb disk_sb;
+ struct bch_sb *disk_sb;
+ unsigned disk_sb_order;
+
unsigned short block_bits; /* ilog2(block_size) */
struct closure sb_write;
- struct semaphore sb_write_mutex;
+ struct mutex sb_lock;
struct backing_dev_info bdi;
* allocate from:
*/
struct cache_group cache_all;
- struct cache_group cache_tiers[CACHE_TIERS];
+ struct cache_group cache_tiers[BCH_TIER_MAX];
u64 capacity; /* sectors */
struct bio_decompress_worker __percpu
*bio_decompress_worker;
+ struct crypto_blkcipher *chacha20;
+ struct crypto_shash *poly1305;
+
+ atomic64_t key_version;
+
/* For punting bio submissions to workqueue, io.c */
struct bio_list bio_submit_list;
struct work_struct bio_submit_work;
#define p(...) (out += scnprintf(out, end - out, __VA_ARGS__))
- p("u64s %u type %u %llu:%llu snap %u len %u ver %u",
+ p("u64s %u type %u %llu:%llu snap %u len %u ver %llu",
k->u64s, k->type, k->p.inode, k->p.offset,
- k->p.snapshot, k->size, k->version);
+ k->p.snapshot, k->size, k->version.lo);
BUG_ON(bkey_packed(k));
return true;
}
+#define bkey_fields() \
+ x(BKEY_FIELD_INODE, p.inode) \
+ x(BKEY_FIELD_OFFSET, p.offset) \
+ x(BKEY_FIELD_SNAPSHOT, p.snapshot) \
+ x(BKEY_FIELD_SIZE, size) \
+ x(BKEY_FIELD_VERSION_HI, version.hi) \
+ x(BKEY_FIELD_VERSION_LO, version.lo)
+
struct bkey __bkey_unpack_key(const struct bkey_format *format,
const struct bkey_packed *in)
{
struct unpack_state state = unpack_state_init(format, in);
struct bkey out;
- EBUG_ON(format->nr_fields != 5);
+ EBUG_ON(format->nr_fields != BKEY_NR_FIELDS);
EBUG_ON(in->u64s < format->key_u64s);
EBUG_ON(in->format != KEY_FORMAT_LOCAL_BTREE);
EBUG_ON(in->u64s - format->key_u64s + BKEY_U64s > U8_MAX);
out.needs_whiteout = in->needs_whiteout;
out.type = in->type;
out.pad[0] = 0;
- out.p.inode = get_inc_field(&state, BKEY_FIELD_INODE);
- out.p.offset = get_inc_field(&state, BKEY_FIELD_OFFSET);
- out.p.snapshot = get_inc_field(&state, BKEY_FIELD_SNAPSHOT);
- out.size = get_inc_field(&state, BKEY_FIELD_SIZE);
- out.version = get_inc_field(&state, BKEY_FIELD_VERSION);
+
+#define x(id, field) out.field = get_inc_field(&state, id);
+ bkey_fields()
+#undef x
return out;
}
struct unpack_state state = unpack_state_init(format, in);
struct bpos out;
- EBUG_ON(format->nr_fields != 5);
+ EBUG_ON(format->nr_fields != BKEY_NR_FIELDS);
EBUG_ON(in->u64s < format->key_u64s);
EBUG_ON(in->format != KEY_FORMAT_LOCAL_BTREE);
struct pack_state state = pack_state_init(format, out);
EBUG_ON((void *) in == (void *) out);
- EBUG_ON(format->nr_fields != 5);
+ EBUG_ON(format->nr_fields != BKEY_NR_FIELDS);
EBUG_ON(in->format != KEY_FORMAT_CURRENT);
out->_data[0] = 0;
- if (!set_inc_field(&state, BKEY_FIELD_INODE, in->p.inode) ||
- !set_inc_field(&state, BKEY_FIELD_OFFSET, in->p.offset) ||
- !set_inc_field(&state, BKEY_FIELD_SNAPSHOT, in->p.snapshot) ||
- !set_inc_field(&state, BKEY_FIELD_SIZE, in->size) ||
- !set_inc_field(&state, BKEY_FIELD_VERSION, in->version))
- return false;
+#define x(id, field) if (!set_inc_field(&state, id, in->field)) return false;
+ bkey_fields()
+#undef x
/*
* Extents - we have to guarantee that if an extent is packed, a trimmed
return true;
}
-/*
- * Alternate implementations using bch_bkey_transform_key() - unfortunately, too
- * slow
- */
-#if 0
-struct bkey __bkey_unpack_key(const struct bkey_format *format,
- const struct bkey_packed *in)
-{
- struct bkey out;
- bool s;
-
- EBUG_ON(format->nr_fields != 5);
- EBUG_ON(in->u64s < format->key_u64s);
- EBUG_ON(in->format != KEY_FORMAT_LOCAL_BTREE);
-
- s = bch_bkey_transform_key(&bch_bkey_format_current, (void *) &out,
- format, in);
- EBUG_ON(!s);
-
- out.format = KEY_FORMAT_CURRENT;
-
- return out;
-}
-
-bool bkey_pack_key(struct bkey_packed *out, const struct bkey *in,
- const struct bkey_format *format)
-{
- EBUG_ON(format->nr_fields != 5);
- EBUG_ON(in->format != KEY_FORMAT_CURRENT);
-
- if (!bch_bkey_transform_key(format, out,
- &bch_bkey_format_current, (void *) in))
- return false;
-
- out->format = KEY_FORMAT_LOCAL_BTREE;
-
- bch_bkey_pack_verify(out, in, format);
- return true;
-}
-#endif
-
/**
* bkey_unpack -- unpack the key and the value
*/
*/
void bch_bkey_format_add_key(struct bkey_format_state *s, const struct bkey *k)
{
- __bkey_format_add(s, BKEY_FIELD_INODE, k->p.inode);
- __bkey_format_add(s, BKEY_FIELD_OFFSET, k->p.offset);
+#define x(id, field) __bkey_format_add(s, id, k->field);
+ bkey_fields()
+#undef x
__bkey_format_add(s, BKEY_FIELD_OFFSET, bkey_start_offset(k));
- __bkey_format_add(s, BKEY_FIELD_SNAPSHOT, k->p.snapshot);
- __bkey_format_add(s, BKEY_FIELD_SIZE, k->size);
- __bkey_format_add(s, BKEY_FIELD_VERSION, k->version);
}
void bch_bkey_format_add_pos(struct bkey_format_state *s, struct bpos p)
bits += ret.bits_per_field[i];
}
+ /* allow for extent merging: */
+ if (ret.bits_per_field[BKEY_FIELD_SIZE]) {
+ ret.bits_per_field[BKEY_FIELD_SIZE] += 4;
+ bits += 4;
+ }
+
ret.key_u64s = DIV_ROUND_UP(bits, 64);
/* if we have enough spare bits, round fields up to nearest byte */
/* mov [rdi], eax */
I2(0x89, 0x07);
- out = compile_bkey_field(format, out, BKEY_FIELD_INODE,
- offsetof(struct bkey, p.inode), 8,
- &eax_zeroed);
-
- out = compile_bkey_field(format, out, BKEY_FIELD_OFFSET,
- offsetof(struct bkey, p.offset), 8,
- &eax_zeroed);
-
- out = compile_bkey_field(format, out, BKEY_FIELD_SNAPSHOT,
- offsetof(struct bkey, p.snapshot), 4,
- &eax_zeroed);
-
- out = compile_bkey_field(format, out, BKEY_FIELD_SIZE,
- offsetof(struct bkey, size), 4,
- &eax_zeroed);
-
- out = compile_bkey_field(format, out, BKEY_FIELD_VERSION,
- offsetof(struct bkey, version), 4,
+#define x(id, field) \
+ out = compile_bkey_field(format, out, id, \
+ offsetof(struct bkey, field), \
+ sizeof(((struct bkey *) NULL)->field), \
&eax_zeroed);
+ bkey_fields()
+#undef x
/* retq */
I1(0xc3);
}
#endif
-/*
- * Would like to use this if we can make __bkey_cmp_bits() fast enough, it'll be
- * a decent reduction in code size
- */
-#if 0
-static int bkey_cmp_verify(const struct bkey *l, const struct bkey *r)
-{
- if (l->p.inode != r->p.inode)
- return l->p.inode < r->p.inode ? -1 : 1;
-
- if (l->p.offset != r->p.offset)
- return l->p.offset < r->p.offset ? -1 : 1;
-
- if (l->p.snapshot != r->p.snapshot)
- return l->p.snapshot < r->p.snapshot ? -1 : 1;
-
- return 0;
-}
-
-int bkey_cmp(const struct bkey *l, const struct bkey *r)
-{
- int ret;
-
- EBUG_ON(bkey_packed(l) || bkey_packed(r));
-
- ret = __bkey_cmp_bits((sizeof(l->inode) +
- sizeof(l->offset) +
- sizeof(l->snapshot)) * BITS_PER_BYTE,
- __high_word(BKEY_U64s, l),
- __high_word(BKEY_U64s, r));
-
- BUG_ON(ret != bkey_cmp_verify(l, r));
-
- return ret;
-}
-#endif
-
__pure
int __bkey_cmp_packed_format_checked(const struct bkey_packed *l,
const struct bkey_packed *r,
struct bkey_format test_format = {
.key_u64s = 2,
- .nr_fields = 5,
+ .nr_fields = BKEY_NR_FIELDS,
.bits_per_field = {
13,
64,
u64 a, v = get_inc_field(&in_s, i);
switch (i) {
- case 0:
- a = t.p.inode;
- break;
- case 1:
- a = t.p.offset;
- break;
- case 2:
- a = t.p.snapshot;
- break;
- case 3:
- a = t.size;
- break;
- case 4:
- a = t.version;
- break;
+#define x(id, field) case id: a = t.field; break;
+ bkey_fields()
+#undef x
default:
BUG();
}
#include <linux/bcache.h>
#include "util.h"
+#include "vstructs.h"
void bch_to_binary(char *, const u64 *, unsigned);
int bch_bkey_to_text(char *, size_t, const struct bkey *);
};
};
-#define bkey_next(_k) \
-({ \
- BUILD_BUG_ON(!type_is(_k, struct bkey *) && \
- !type_is(_k, struct bkey_i *) && \
- !type_is(_k, struct bkey_packed *)); \
- \
- ((typeof(_k)) __bkey_idx(((struct bkey *) (_k)), \
- ((struct bkey *) (_k))->u64s)); \
-})
+#define bkey_next(_k) vstruct_next(_k)
static inline unsigned bkey_val_u64s(const struct bkey *k)
{
void bch_bpos_swab(struct bpos *);
void bch_bkey_swab_key(const struct bkey_format *, struct bkey_packed *);
+static __always_inline int bversion_cmp(struct bversion l, struct bversion r)
+{
+ if (l.hi != r.hi)
+ return l.hi < r.hi ? -1 : 1;
+ if (l.lo != r.lo)
+ return l.lo < r.lo ? -1 : 1;
+ return 0;
+}
+
+#define ZERO_VERSION ((struct bversion) { .hi = 0, .lo = 0 })
+
+static __always_inline int bversion_zero(struct bversion v)
+{
+ return !bversion_cmp(v, ZERO_VERSION);
+}
+
#ifdef CONFIG_BCACHE_DEBUG
/* statement expressions confusing unlikely()? */
#define bkey_packed(_k) \
}
__BKEY_VAL_ACCESSORS(extent, BCH_EXTENT, __bch_extent_assert);
+BKEY_VAL_ACCESSORS(reservation, BCH_RESERVATION);
BKEY_VAL_ACCESSORS(inode, BCH_INODE_FS);
BKEY_VAL_ACCESSORS(inode_blockdev, BCH_INODE_BLOCKDEV);
#include "bcache.h"
#include "blockdev.h"
#include "btree_iter.h"
+#include "btree_update.h"
#include "checksum.h"
#include "error.h"
#include "inode.h"
#include "request.h"
-#include "super.h"
+#include "super-io.h"
#include "writeback.h"
#include <linux/kthread.h>
down(&dc->sb_write_mutex);
closure_init(cl, parent);
+ sb->csum = csum_vstruct(NULL, BCH_CSUM_CRC64,
+ (struct nonce) { 0 }, sb).lo;
+
bio_reset(bio);
- bio->bi_end_io = write_bdev_super_endio;
- bio->bi_private = dc;
+ bio->bi_bdev = dc->disk_sb.bdev;
+ bio->bi_iter.bi_sector = le64_to_cpu(sb->offset);
+ bio->bi_iter.bi_size =
+ roundup(vstruct_bytes(sb),
+ bdev_logical_block_size(dc->disk_sb.bdev));
+ bio->bi_end_io = write_bdev_super_endio;
+ bio->bi_private = dc;
+ bio_set_op_attrs(bio, REQ_OP_WRITE, WRITE_FUA|REQ_META);
+ bch_bio_map(bio, sb);
closure_get(cl);
- sb->csum = cpu_to_le64(__csum_set(sb, 0, BCH_CSUM_CRC64));
- __write_super(dc->disk.c, (void *) &dc->disk_sb);
-
closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
}
void bch_cached_dev_run(struct cached_dev *dc)
{
struct bcache_device *d = &dc->disk;
- char buf[SB_LABEL_SIZE + 1];
+ char buf[BCH_SB_LABEL_SIZE + 1];
char *env[] = {
"DRIVER=bcache",
kasprintf(GFP_KERNEL, "CACHED_UUID=%pU",
NULL,
};
- memcpy(buf, dc->disk_sb.sb->label, SB_LABEL_SIZE);
- buf[SB_LABEL_SIZE] = '\0';
+ memcpy(buf, dc->disk_sb.sb->label, BCH_SB_LABEL_SIZE);
+ buf[BCH_SB_LABEL_SIZE] = '\0';
env[2] = kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf);
if (atomic_xchg(&dc->running, 1)) {
bdevname(dc->disk_sb.bdev, buf);
if (memcmp(&dc->disk_sb.sb->set_uuid,
- &c->disk_sb.set_uuid,
- sizeof(c->disk_sb.set_uuid)))
+ &c->sb.uuid,
+ sizeof(c->sb.uuid)))
return -ENOENT;
if (dc->disk.c) {
SET_CACHED_DEV(&dc->disk.inode.v, true);
dc->disk.inode.v.i_uuid = dc->disk_sb.sb->disk_uuid;
memcpy(dc->disk.inode.v.i_label,
- dc->disk_sb.sb->label, SB_LABEL_SIZE);
+ dc->disk_sb.sb->label, BCH_SB_LABEL_SIZE);
dc->disk.inode.v.i_ctime = rtime;
dc->disk.inode.v.i_mtime = rtime;
pr_info("attached inode %llu", bcache_dev_inum(&dc->disk));
- dc->disk_sb.sb->set_uuid = c->disk_sb.set_uuid;
+ dc->disk_sb.sb->set_uuid = c->sb.uuid;
SET_BDEV_STATE(dc->disk_sb.sb, BDEV_STATE_CLEAN);
bch_write_bdev_super(dc, &cl);
closure_sync(&cl);
} else {
dc->disk.inode.v.i_mtime = rtime;
- bch_inode_update(c, &dc->disk.inode.k_i, NULL);
+ bch_btree_update(c, BTREE_ID_INODES,
+ &dc->disk.inode.k_i, NULL);
}
/* Count dirty sectors before attaching */
pr_info("Caching %s as %s on set %pU",
bdevname(dc->disk_sb.bdev, buf), dc->disk.disk->disk_name,
- dc->disk.c->disk_sb.set_uuid.b);
+ dc->disk.c->sb.uuid.b);
return 0;
}
mutex_unlock(&bch_register_lock);
- free_super((void *) &dc->disk_sb);
+ bch_free_super((void *) &dc->disk_sb);
kobject_put(&dc->disk.kobj);
}
return;
for (_k = i->start, k = bkey_unpack_key(b, _k);
- _k < bset_bkey_last(i);
+ _k < vstruct_last(i);
_k = _n, k = n) {
_n = bkey_next(_k);
printk(KERN_ERR "block %u key %zi/%u: %s\n", set,
_k->_data - i->_data, i->u64s, buf);
- if (_n == bset_bkey_last(i))
+ if (_n == vstruct_last(i))
continue;
n = bkey_unpack_key(b, _n);
#include "bkey_methods.h"
#include "btree_types.h"
#include "util.h" /* for time_stats */
+#include "vstructs.h"
/*
* BKEYS:
bch_bset_set_no_aux_tree(b, b->set);
}
-#define __set_bytes(_i, _u64s) (sizeof(*(_i)) + (_u64s) * sizeof(u64))
-#define set_bytes(_i) __set_bytes(_i, (_i)->u64s)
-
-#define __set_blocks(_i, _u64s, _block_bytes) \
- DIV_ROUND_UP((size_t) __set_bytes((_i), (_u64s)), (_block_bytes))
-
-#define set_blocks(_i, _block_bytes) \
- __set_blocks((_i), (_i)->u64s, (_block_bytes))
-
static inline struct bset *bset_next_set(struct btree *b,
unsigned block_bytes)
{
EBUG_ON(!is_power_of_2(block_bytes));
- return ((void *) i) + round_up(set_bytes(i), block_bytes);
+ return ((void *) i) + round_up(vstruct_bytes(i), block_bytes);
}
void bch_btree_keys_free(struct btree *);
(cmp == 0 && !strictly_greater && !bkey_deleted(k));
}
-static inline struct bkey_packed *bset_bkey_idx(struct bset *i, unsigned idx)
-{
- return bkey_idx(i, idx);
-}
-
struct bset_tree *bch_bkey_to_bset(struct btree *, struct bkey_packed *);
struct bkey_packed *bkey_prev_all(struct btree *, struct bset_tree *,
struct bkey_packed *);
EBUG_ON(!b->written);
EBUG_ON(b->btree_id != iter->btree_id ||
- BSET_BTREE_LEVEL(&b->data->keys) != level ||
+ BTREE_NODE_LEVEL(b->data) != level ||
bkey_cmp(b->data->max_key, k->k.p));
return b;
#include "journal.h"
#include "keylist.h"
#include "move.h"
+#include "super-io.h"
#include "writeback.h"
#include <linux/slab.h>
/*
* For runtime mark and sweep:
*/
-u8 __bch_btree_mark_key(struct cache_set *c, enum bkey_type type,
- struct bkey_s_c k)
+static u8 bch_btree_mark_key(struct cache_set *c, enum bkey_type type,
+ struct bkey_s_c k)
{
switch (type) {
case BKEY_TYPE_BTREE:
}
}
-static u8 btree_mark_key(struct cache_set *c, struct btree *b,
- struct bkey_s_c k)
+u8 bch_btree_mark_key_initial(struct cache_set *c, enum bkey_type type,
+ struct bkey_s_c k)
{
- return __bch_btree_mark_key(c, btree_node_type(b), k);
+ atomic64_set(&c->key_version,
+ max_t(u64, k.k->version.lo,
+ atomic64_read(&c->key_version)));
+
+ return bch_btree_mark_key(c, type, k);
}
static bool btree_gc_mark_node(struct cache_set *c, struct btree *b)
btree_node_is_extents(b),
&unpacked) {
bkey_debugcheck(c, b, k);
- stale = max(stale, btree_mark_key(c, b, k));
+ stale = max(stale, bch_btree_mark_key(c,
+ btree_node_type(b), k));
}
if (btree_gc_rewrite_disabled(c))
mutex_lock(&c->btree_root_lock);
b = c->btree_roots[btree_id].b;
- __bch_btree_mark_key(c, BKEY_TYPE_BTREE, bkey_i_to_s_c(&b->key));
+ bch_btree_mark_key(c, BKEY_TYPE_BTREE, bkey_i_to_s_c(&b->key));
gc_pos_set(c, gc_pos_btree_root(b->btree_id));
mutex_unlock(&c->btree_root_lock);
static void bch_mark_metadata(struct cache_set *c)
{
struct cache *ca;
- unsigned i;
+ unsigned i, j;
+ u64 b;
for_each_cache(ca, c, i) {
- unsigned j;
- u64 *i;
-
- for (j = 0; j < bch_nr_journal_buckets(ca->disk_sb.sb); j++)
- bch_mark_metadata_bucket(ca,
- &ca->buckets[journal_bucket(ca->disk_sb.sb, j)],
- true);
+ for (j = 0; j < ca->journal.nr; j++) {
+ b = ca->journal.buckets[j];
+ bch_mark_metadata_bucket(ca, ca->buckets + b, true);
+ }
spin_lock(&ca->prio_buckets_lock);
- for (i = ca->prio_buckets;
- i < ca->prio_buckets + prio_buckets(ca) * 2; i++)
- bch_mark_metadata_bucket(ca, &ca->buckets[*i], true);
+ for (j = 0; j < prio_buckets(ca) * 2; j++) {
+ b = ca->prio_buckets[j];
+ bch_mark_metadata_bucket(ca, ca->buckets + b, true);
+ }
spin_unlock(&ca->prio_buckets_lock);
}
/* Check if all keys in @old_nodes could fit in one fewer node */
if (nr_old_nodes <= 1 ||
- __set_blocks(old_nodes[0]->data,
- DIV_ROUND_UP(u64s, nr_old_nodes - 1),
- block_bytes(c)) > blocks)
+ __vstruct_blocks(struct btree_node, c->block_bits,
+ DIV_ROUND_UP(u64s, nr_old_nodes - 1)) > blocks)
return;
res = bch_btree_reserve_get(c, parent, nr_old_nodes,
u64s = 0;
for (k = s2->start;
- k < bset_bkey_last(s2) &&
- __set_blocks(n1->data, le16_to_cpu(s1->u64s) + u64s + k->u64s,
- block_bytes(c)) <= blocks;
+ k < vstruct_last(s2) &&
+ vstruct_blocks_plus(n1->data, c->block_bits,
+ u64s + k->u64s) <= blocks;
k = bkey_next(k)) {
last = k;
u64s += k->u64s;
/* n2 fits entirely in n1 */
n1->key.k.p = n1->data->max_key = n2->data->max_key;
- memcpy_u64s(bset_bkey_last(s1),
+ memcpy_u64s(vstruct_last(s1),
s2->start,
le16_to_cpu(s2->u64s));
le16_add_cpu(&s1->u64s, le16_to_cpu(s2->u64s));
btree_type_successor(iter->btree_id,
n1->data->max_key);
- memcpy_u64s(bset_bkey_last(s1),
+ memcpy_u64s(vstruct_last(s1),
s2->start, u64s);
le16_add_cpu(&s1->u64s, u64s);
memmove(s2->start,
- bset_bkey_idx(s2, u64s),
+ vstruct_idx(s2, u64s),
(le16_to_cpu(s2->u64s) - u64s) * sizeof(u64));
s2->u64s = cpu_to_le16(le16_to_cpu(s2->u64s) - u64s);
for_each_btree_node_key_unpack(b, k, &node_iter,
btree_node_is_extents(b),
&unpacked)
- btree_mark_key(c, b, k);
+ bch_btree_mark_key_initial(c, btree_node_type(b), k);
}
bch_btree_iter_cond_resched(&iter);
bch_btree_iter_unlock(&iter);
- __bch_btree_mark_key(c, BKEY_TYPE_BTREE,
- bkey_i_to_s_c(&c->btree_roots[id].b->key));
+ bch_btree_mark_key(c, BKEY_TYPE_BTREE,
+ bkey_i_to_s_c(&c->btree_roots[id].b->key));
}
int bch_initial_gc(struct cache_set *c, struct list_head *journal)
bch_journal_mark(c, journal);
}
+ /*
+ * Skip past versions that might have possibly been used (as nonces),
+ * but hadn't had their pointers written:
+ */
+ if (c->sb.encryption_type)
+ atomic64_add(1 << 16, &c->key_version);
+
bch_mark_metadata(c);
gc_pos_set(c, gc_phase(GC_PHASE_DONE));
int bch_gc_thread_start(struct cache_set *);
int bch_initial_gc(struct cache_set *, struct list_head *);
u8 bch_btree_key_recalc_oldest_gen(struct cache_set *, struct bkey_s_c);
-u8 __bch_btree_mark_key(struct cache_set *, enum bkey_type,
+u8 bch_btree_mark_key_initial(struct cache_set *, enum bkey_type,
struct bkey_s_c);
/*
#include "extents.h"
#include "io.h"
#include "journal.h"
+#include "super-io.h"
#include <trace/events/bcache.h>
{
struct bkey_packed *k;
- for (k = i->start; k != bset_bkey_last(i); k = bkey_next(k))
+ for (k = i->start; k != vstruct_last(i); k = bkey_next(k))
k->needs_whiteout = false;
}
{
struct bkey_packed *k;
- for (k = i->start; k != bset_bkey_last(i); k = bkey_next(k))
+ for (k = i->start; k != vstruct_last(i); k = bkey_next(k))
k->needs_whiteout = true;
}
compacting = true;
u_start = u_pos;
start = i->start;
- end = bset_bkey_last(i);
+ end = vstruct_last(i);
if (src != dst) {
memmove(dst, src, sizeof(*src));
order = sorting_entire_node
? btree_page_order(c)
- : get_order(__set_bytes(b->data, u64s));
+ : get_order(__vstruct_bytes(struct btree_node, u64s));
out = btree_bounce_alloc(c, order, &used_mempool);
out->keys.u64s = cpu_to_le16(u64s);
- BUG_ON((void *) bset_bkey_last(&out->keys) >
- (void *) out + (PAGE_SIZE << order));
+ BUG_ON(vstruct_end(&out->keys) > (void *) out + (PAGE_SIZE << order));
if (sorting_entire_node)
bch_time_stats_update(&c->btree_sort_time, start_time);
bool filter_whiteouts)
{
struct bkey_format *in_f = &src->format;
- struct bkey_packed *in, *out = bset_bkey_last(dst);
+ struct bkey_packed *in, *out = vstruct_last(dst);
struct btree_nr_keys nr;
memset(&nr, 0, sizeof(nr));
btree_keys_account_key_add(&nr, 0, prev);
prev = bkey_next(prev);
} else {
- prev = bset_bkey_last(dst);
+ prev = vstruct_last(dst);
}
bkey_copy(prev, &tmp.k);
btree_keys_account_key_add(&nr, 0, prev);
out = bkey_next(prev);
} else {
- out = bset_bkey_last(dst);
+ out = vstruct_last(dst);
}
dst->u64s = cpu_to_le16((u64 *) out - dst->_data);
bch_btree_iter_reinit_node(iter, b);
}
-/*
- * We seed the checksum with the entire first pointer (dev, gen and offset),
- * since for btree nodes we have to store the checksum with the data instead of
- * the pointer - this helps guard against reading a valid btree node that is not
- * the node we actually wanted:
- */
-#define btree_csum_set(_b, _i) \
-({ \
- void *_data = (void *) (_i) + 8; \
- void *_end = bset_bkey_last(&(_i)->keys); \
- \
- bch_checksum_update(BSET_CSUM_TYPE(&(_i)->keys), \
- bkey_i_to_extent_c(&(_b)->key)->v._data[0], \
- _data, \
- _end - _data) ^ 0xffffffffffffffffULL; \
-})
+static struct nonce btree_nonce(struct btree *b,
+ struct bset *i,
+ unsigned offset)
+{
+ return (struct nonce) {{
+ [0] = cpu_to_le32(offset),
+ [1] = ((__le32 *) &i->seq)[0],
+ [2] = ((__le32 *) &i->seq)[1],
+ [3] = ((__le32 *) &i->journal_seq)[0]^BCH_NONCE_BTREE,
+ }};
+}
+
+static void bset_encrypt(struct cache_set *c, struct bset *i, struct nonce nonce)
+{
+ bch_encrypt(c, BSET_CSUM_TYPE(i), nonce, i->_data,
+ vstruct_end(i) - (void *) i->_data);
+}
#define btree_node_error(b, c, ptr, fmt, ...) \
cache_set_inconsistent(c, \
(b)->btree_id, (b)->level, btree_node_root(c, b) \
? btree_node_root(c, b)->level : -1, \
PTR_BUCKET_NR(ca, ptr), (b)->written, \
- (i)->u64s, ##__VA_ARGS__)
+ le16_to_cpu((i)->u64s), ##__VA_ARGS__)
static const char *validate_bset(struct cache_set *c, struct btree *b,
struct cache *ca,
unsigned *whiteout_u64s)
{
struct bkey_packed *k, *prev = NULL;
+ struct bpos prev_pos = POS_MIN;
bool seen_non_whiteout = false;
if (le16_to_cpu(i->version) != BCACHE_BSET_VERSION)
}
for (k = i->start;
- k != bset_bkey_last(i);) {
+ k != vstruct_last(i);) {
struct bkey_s_c u;
struct bkey tmp;
const char *invalid;
if (!k->u64s) {
btree_node_error(b, c, ptr,
"KEY_U64s 0: %zu bytes of metadata lost",
- (void *) bset_bkey_last(i) - (void *) k);
+ vstruct_end(i) - (void *) k);
i->u64s = cpu_to_le16((u64 *) k - i->_data);
break;
}
- if (bkey_next(k) > bset_bkey_last(i)) {
+ if (bkey_next(k) > vstruct_last(i)) {
btree_node_error(b, c, ptr,
"key extends past end of bset");
i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
memmove_u64s_down(k, bkey_next(k),
- (u64 *) bset_bkey_last(i) - (u64 *) k);
+ (u64 *) vstruct_end(i) - (u64 *) k);
continue;
}
i->u64s = cpu_to_le16(le16_to_cpu(i->u64s) - k->u64s);
memmove_u64s_down(k, bkey_next(k),
- (u64 *) bset_bkey_last(i) - (u64 *) k);
+ (u64 *) vstruct_end(i) - (u64 *) k);
continue;
}
if (!seen_non_whiteout &&
(!bkey_whiteout(k) ||
- (prev && bkey_cmp_left_packed_byval(b, prev,
- bkey_start_pos(u.k)) > 0))) {
+ (bkey_cmp(prev_pos, bkey_start_pos(u.k)) > 0))) {
*whiteout_u64s = k->_data - i->_data;
seen_non_whiteout = true;
+ } else if (bkey_cmp(prev_pos, bkey_start_pos(u.k)) > 0) {
+ btree_node_error(b, c, ptr,
+ "keys out of order: %llu:%llu > %llu:%llu",
+ prev_pos.inode,
+ prev_pos.offset,
+ u.k->p.inode,
+ bkey_start_offset(u.k));
+ /* XXX: repair this */
}
+ prev_pos = u.k->p;
prev = k;
k = bkey_next(k);
}
SET_BSET_BIG_ENDIAN(i, CPU_BIG_ENDIAN);
-
- b->written += sectors;
return NULL;
}
+static bool extent_contains_ptr(struct bkey_s_c_extent e,
+ struct bch_extent_ptr match)
+{
+ const struct bch_extent_ptr *ptr;
+
+ extent_for_each_ptr(e, ptr)
+ if (!memcmp(ptr, &match, sizeof(*ptr)))
+ return true;
+
+ return false;
+}
+
void bch_btree_node_read_done(struct cache_set *c, struct btree *b,
struct cache *ca,
const struct bch_extent_ptr *ptr)
bool used_mempool;
unsigned u64s;
const char *err;
+ struct bch_csum csum;
+ struct nonce nonce;
int ret;
iter = mempool_alloc(&c->fill_iter, GFP_NOIO);
if (!b->written) {
i = &b->data->keys;
+ err = "bad magic";
+ if (le64_to_cpu(b->data->magic) != bset_magic(c))
+ goto err;
+
+ err = "bad btree header";
+ if (!b->data->keys.seq)
+ goto err;
+
err = "unknown checksum type";
- if (BSET_CSUM_TYPE(i) >= BCH_CSUM_NR)
+ if (!bch_checksum_type_valid(c, BSET_CSUM_TYPE(i)))
goto err;
/* XXX: retry checksum errors */
+ nonce = btree_nonce(b, i, b->written << 9);
+ csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, b->data);
+
err = "bad checksum";
- if (le64_to_cpu(b->data->csum) !=
- btree_csum_set(b, b->data))
+ if (bch_crc_cmp(csum, b->data->csum))
goto err;
- sectors = __set_blocks(b->data,
- le16_to_cpu(b->data->keys.u64s),
- block_bytes(c)) << c->block_bits;
+ bch_encrypt(c, BSET_CSUM_TYPE(i), nonce,
+ &b->data->flags,
+ (void *) &b->data->keys -
+ (void *) &b->data->flags);
+ nonce = nonce_add(nonce,
+ round_up((void *) &b->data->keys -
+ (void *) &b->data->flags,
+ CHACHA20_BLOCK_SIZE));
+ bset_encrypt(c, i, nonce);
- err = "bad magic";
- if (le64_to_cpu(b->data->magic) != bset_magic(&c->disk_sb))
- goto err;
-
- err = "bad btree header";
- if (!b->data->keys.seq)
- goto err;
+ sectors = vstruct_sectors(b->data, c->block_bits);
if (BSET_BIG_ENDIAN(i) != CPU_BIG_ENDIAN) {
+ u64 *p = (u64 *) &b->data->ptr;
+
+ *p = swab64(*p);
bch_bpos_swab(&b->data->min_key);
bch_bpos_swab(&b->data->max_key);
}
+ err = "incorrect btree id";
+ if (BTREE_NODE_ID(b->data) != b->btree_id)
+ goto err;
+
+ err = "incorrect level";
+ if (BTREE_NODE_LEVEL(b->data) != b->level)
+ goto err;
+
err = "incorrect max key";
if (bkey_cmp(b->data->max_key, b->key.k.p))
goto err;
- err = "incorrect level";
- if (BSET_BTREE_LEVEL(i) != b->level)
+ err = "incorrect backpointer";
+ if (!extent_contains_ptr(bkey_i_to_s_c_extent(&b->key),
+ b->data->ptr))
goto err;
err = bch_bkey_format_validate(&b->data->format);
break;
err = "unknown checksum type";
- if (BSET_CSUM_TYPE(i) >= BCH_CSUM_NR)
+ if (!bch_checksum_type_valid(c, BSET_CSUM_TYPE(i)))
goto err;
+ nonce = btree_nonce(b, i, b->written << 9);
+ csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bne);
+
err = "bad checksum";
- if (le64_to_cpu(bne->csum) !=
- btree_csum_set(b, bne))
+ if (memcmp(&csum, &bne->csum, sizeof(csum)))
goto err;
- sectors = __set_blocks(bne,
- le16_to_cpu(bne->keys.u64s),
- block_bytes(c)) << c->block_bits;
+ bset_encrypt(c, i, nonce);
+
+ sectors = vstruct_sectors(bne, c->block_bits);
}
err = validate_bset(c, b, ca, ptr, i, sectors, &whiteout_u64s);
if (err)
goto err;
+ b->written += sectors;
+
err = "insufficient memory";
ret = bch_journal_seq_should_ignore(c, le64_to_cpu(i->journal_seq), b);
if (ret < 0)
__bch_btree_node_iter_push(iter, b,
i->start,
- bkey_idx(i, whiteout_u64s));
+ vstruct_idx(i, whiteout_u64s));
__bch_btree_node_iter_push(iter, b,
- bkey_idx(i, whiteout_u64s),
- bset_bkey_last(i));
+ vstruct_idx(i, whiteout_u64s),
+ vstruct_last(i));
}
err = "corrupted btree";
struct bch_extent_ptr *ptr;
struct cache *ca;
struct sort_iter sort_iter;
+ struct nonce nonce;
unsigned bytes_to_write, sectors_to_write, order, bytes, u64s;
u64 seq = 0;
bool used_mempool;
BUG_ON(b->written >= c->sb.btree_node_size);
BUG_ON(bset_written(b, btree_bset_last(b)));
- BUG_ON(le64_to_cpu(b->data->magic) != bset_magic(&c->disk_sb));
+ BUG_ON(le64_to_cpu(b->data->magic) != bset_magic(c));
BUG_ON(memcmp(&b->data->format, &b->format, sizeof(b->format)));
if (lock_type_held == SIX_LOCK_intent) {
b->whiteout_u64s = 0;
u64s = btree_node_is_extents(b)
- ? sort_extents(bset_bkey_last(i), &sort_iter, false)
+ ? sort_extents(vstruct_last(i), &sort_iter, false)
: sort_keys(i->start, &sort_iter, false);
le16_add_cpu(&i->u64s, u64s);
BUG_ON(i->seq != b->data->keys.seq);
i->version = cpu_to_le16(BCACHE_BSET_VERSION);
- SET_BSET_CSUM_TYPE(i, c->opts.metadata_checksum);
+ SET_BSET_CSUM_TYPE(i, bch_meta_checksum_type(c));
+
+ nonce = btree_nonce(b, i, b->written << 9);
+
+ if (bn) {
+ bch_encrypt(c, BSET_CSUM_TYPE(i), nonce,
+ &bn->flags,
+ (void *) &b->data->keys -
+ (void *) &b->data->flags);
+ nonce = nonce_add(nonce,
+ round_up((void *) &b->data->keys -
+ (void *) &b->data->flags,
+ CHACHA20_BLOCK_SIZE));
+ bset_encrypt(c, i, nonce);
+
+ nonce = btree_nonce(b, i, b->written << 9);
+ bn->csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bn);
+ } else {
+ bset_encrypt(c, i, nonce);
- if (bn)
- bn->csum = cpu_to_le64(btree_csum_set(b, bn));
- else
- bne->csum = cpu_to_le64(btree_csum_set(b, bne));
+ bne->csum = csum_vstruct(c, BSET_CSUM_TYPE(i), nonce, bne);
+ }
- bytes_to_write = (void *) bset_bkey_last(i) - data;
+ bytes_to_write = vstruct_end(i) - data;
sectors_to_write = round_up(bytes_to_write, block_bytes(c)) >> 9;
memset(data + bytes_to_write, 0,
* If later we don't unconditionally sort down to a single bset, we have
* to ensure this is still true:
*/
- BUG_ON((void *) bset_bkey_last(btree_bset_last(b)) > write_block(b));
+ BUG_ON((void *) btree_bkey_last(b, bset_tree_last(b)) > write_block(b));
bne = want_new_bset(c, b);
if (bne)
return (void *) ((u64 *) b->data + k + 1);
}
-#define __bkey_idx(_set, _offset) \
- ((_set)->_data + (_offset))
-
-#define bkey_idx(_set, _offset) \
- ((typeof(&(_set)->start[0])) __bkey_idx((_set), (_offset)))
-
-#define __bset_bkey_last(_set) \
- __bkey_idx((_set), (_set)->u64s)
-
-#define bset_bkey_last(_set) \
- bkey_idx((_set), le16_to_cpu((_set)->u64s))
-
#define btree_bkey_first(_b, _t) (bset(_b, _t)->start)
#define btree_bkey_last(_b, _t) \
({ \
EBUG_ON(__btree_node_offset_to_key(_b, (_t)->end_offset) != \
- bset_bkey_last(bset(_b, _t))); \
+ vstruct_last(bset(_b, _t))); \
\
__btree_node_offset_to_key(_b, (_t)->end_offset); \
})
static inline void set_btree_bset_end(struct btree *b, struct bset_tree *t)
{
t->end_offset =
- __btree_node_key_to_offset(b, bset_bkey_last(bset(b, t)));
+ __btree_node_key_to_offset(b, vstruct_last(bset(b, t)));
btree_bkey_last(b, t);
}
#include "extents.h"
#include "journal.h"
#include "keylist.h"
-#include "super.h"
+#include "super-io.h"
#include <linux/random.h>
#include <linux/sort.h>
{
size_t u64s = btree_node_u64s_with_format(b, new_f);
- return __set_bytes(b->data, u64s) < btree_bytes(c);
+ return __vstruct_bytes(struct btree_node, u64s) < btree_bytes(c);
}
/* Btree node freeing/allocation: */
bch_bset_init_first(b, &b->data->keys);
memset(&b->nr, 0, sizeof(b->nr));
- b->data->magic = cpu_to_le64(bset_magic(&c->disk_sb));
- SET_BSET_BTREE_LEVEL(&b->data->keys, level);
+ b->data->magic = cpu_to_le64(bset_magic(c));
+ b->data->flags = 0;
+ SET_BTREE_NODE_ID(b->data, id);
+ SET_BTREE_NODE_LEVEL(b->data, level);
+ b->data->ptr = bkey_i_to_extent(&b->key)->v.start->ptr;
bch_btree_build_aux_trees(b);
*/
k = set1->start;
while (1) {
- if (bkey_next(k) == bset_bkey_last(set1))
+ if (bkey_next(k) == vstruct_last(set1))
break;
if (k->_data - set1->_data >= (le16_to_cpu(set1->u64s) * 3) / 5)
break;
n2->data->min_key =
btree_type_successor(n1->btree_id, n1->key.k.p);
- set2->u64s = cpu_to_le16((u64 *) bset_bkey_last(set1) - (u64 *) k);
+ set2->u64s = cpu_to_le16((u64 *) vstruct_end(set1) - (u64 *) k);
set1->u64s = cpu_to_le16(le16_to_cpu(set1->u64s) - le16_to_cpu(set2->u64s));
set_btree_bset_end(n1, n1->set);
BUG_ON(!set2->u64s);
memcpy_u64s(set2->start,
- bset_bkey_last(set1),
+ vstruct_end(set1),
le16_to_cpu(set2->u64s));
btree_node_reset_sib_u64s(n1);
*/
i = btree_bset_first(b);
p = i->start;
- while (p != bset_bkey_last(i))
+ while (p != vstruct_last(i))
if (bkey_deleted(p)) {
le16_add_cpu(&i->u64s, -p->u64s);
set_btree_bset_end(b, b->set);
memmove_u64s_down(p, bkey_next(p),
- (u64 *) bset_bkey_last(i) -
+ (u64 *) vstruct_last(i) -
(u64 *) p);
} else
p = bkey_next(p);
if (b->level)
btree_split_insert_keys(iter, n1, insert_keys, reserve);
- if (__set_blocks(n1->data,
- le16_to_cpu(n1->data->keys.u64s),
- block_bytes(c)) > BTREE_SPLIT_THRESHOLD(c)) {
+ if (vstruct_blocks(n1->data, c->block_bits) > BTREE_SPLIT_THRESHOLD(c)) {
trace_bcache_btree_node_split(c, b, b->nr.live_u64s);
n2 = __btree_split_node(iter, n1, reserve);
u64s = 0;
trans_for_each_entry(trans, i)
if (!i->done)
- u64s += jset_u64s(i->k->k.u64s);
+ u64s += jset_u64s(i->k->k.u64s + i->extra_res);
memset(&trans->journal_res, 0, sizeof(trans->journal_res));
* written one
*/
if (!i->done) {
- u64s += i->k->k.u64s;
+ u64s += i->k->k.u64s + i->extra_res;
if (!bch_btree_node_insert_fits(c,
i->iter->nodes[0], u64s)) {
split = i->iter;
int bch_btree_delete_range(struct cache_set *c, enum btree_id id,
struct bpos start,
struct bpos end,
- u64 version,
+ struct bversion version,
struct disk_reservation *disk_res,
struct extent_insert_hook *hook,
u64 *journal_seq)
#include "btree_iter.h"
#include "buckets.h"
#include "journal.h"
+#include "vstructs.h"
struct cache_set;
struct bkey_format_state;
static inline unsigned bset_end_sector(struct cache_set *c, struct btree *b,
struct bset *i)
{
- return round_up(bset_byte_offset(b, bset_bkey_last(i)),
+ return round_up(bset_byte_offset(b, vstruct_end(i)),
block_bytes(c)) >> 9;
}
struct btree *b)
{
struct bset *i = btree_bset_last(b);
- unsigned used = bset_byte_offset(b, bset_bkey_last(i)) / sizeof(u64) +
+ unsigned used = bset_byte_offset(b, vstruct_end(i)) / sizeof(u64) +
b->whiteout_u64s +
b->uncompacted_whiteout_u64s;
unsigned total = c->sb.btree_node_size << 6;
{
struct bset *i = btree_bset_last(b);
unsigned offset = max_t(unsigned, b->written << 9,
- bset_byte_offset(b, bset_bkey_last(i)));
+ bset_byte_offset(b, vstruct_end(i)));
ssize_t n = (ssize_t) btree_bytes(c) - (ssize_t)
(offset + sizeof(struct btree_node_entry) +
b->whiteout_u64s * sizeof(u64) +
EBUG_ON(offset > btree_bytes(c));
if ((unlikely(bset_written(b, i)) && n > 0) ||
- (unlikely(__set_bytes(i, le16_to_cpu(i->u64s)) >
- btree_write_set_buffer(b)) && n > btree_write_set_buffer(b)))
+ (unlikely(vstruct_bytes(i) > btree_write_set_buffer(b)) &&
+ n > btree_write_set_buffer(b)))
return (void *) b->data + offset;
return NULL;
struct btree_insert_entry {
struct btree_iter *iter;
struct bkey_i *k;
+ unsigned extra_res;
/*
* true if entire key was inserted - can only be false for
* extents
.done = false, \
})
+#define BTREE_INSERT_ENTRY_EXTRA_RES(_iter, _k, _extra) \
+ ((struct btree_insert_entry) { \
+ .iter = (_iter), \
+ .k = (_k), \
+ .extra_res = (_extra), \
+ .done = false, \
+ })
+
/**
* bch_btree_insert_at - insert one or more keys at iterator positions
* @iter: btree iterator
return true;
for (i = insert; i < trans->entries + trans->nr; i++)
- u64s += jset_u64s(i->k->k.u64s);
+ u64s += jset_u64s(i->k->k.u64s + i->extra_res);
return u64s <= trans->journal_res.u64s;
}
struct bkey_i *, u64 *);
int bch_btree_delete_range(struct cache_set *, enum btree_id,
- struct bpos, struct bpos, u64,
+ struct bpos, struct bpos, struct bversion,
struct disk_reservation *,
struct extent_insert_hook *, u64 *);
rcu_read_lock();
extent_for_each_online_device_crc(c, e, crc, ptr, ca) {
- bool dirty = bch_extent_ptr_is_dirty(c, e, ptr);
-
- trace_bcache_mark_bucket(ca, e.k, ptr, sectors, dirty);
+ trace_bcache_mark_bucket(ca, e.k, ptr, sectors, !ptr->cached);
bch_mark_pointer(c, e, ca, crc, ptr, sectors,
- dirty ? type : S_CACHED,
+ ptr->cached ? S_CACHED : type,
may_make_unavailable,
stats, gc_will_visit, journal_seq);
}
may_make_unavailable, stats,
gc_will_visit, journal_seq);
break;
- case BCH_RESERVATION:
- stats->persistent_reserved += sectors;
+ case BCH_RESERVATION: {
+ struct bkey_s_c_reservation r = bkey_s_c_to_reservation(k);
+
+ stats->persistent_reserved += r.v->nr_replicas * sectors;
break;
}
+ }
}
void __bch_gc_mark_key(struct cache_set *c, struct bkey_s_c k,
static inline struct cache *PTR_CACHE(const struct cache_set *c,
const struct bch_extent_ptr *ptr)
{
- EBUG_ON(ptr->dev > rcu_dereference(c->members)->nr_in_set);
+ EBUG_ON(ptr->dev > rcu_dereference(c->members)->nr_devices);
return rcu_dereference(c->cache[ptr->dev]);
}
#include "bcache.h"
#include "super.h"
+#include "super-io.h"
#include <linux/module.h>
#include <linux/fs.h>
return ret;
}
-static struct cache_member *bch_uuid_lookup(struct cache_set *c, uuid_le uuid)
+static struct bch_member *bch_uuid_lookup(struct cache_set *c, uuid_le uuid)
{
- struct cache_member *mi = c->disk_mi;
+ struct bch_sb_field_members *mi = bch_sb_get_members(c->disk_sb);
unsigned i;
- lockdep_assert_held(&bch_register_lock);
+ lockdep_assert_held(&c->sb_lock);
- for (i = 0; i < c->disk_sb.nr_in_set; i++)
- if (!memcmp(&mi[i].uuid, &uuid, sizeof(uuid)))
- return &mi[i];
+ for (i = 0; i < c->disk_sb->nr_devices; i++)
+ if (!memcmp(&mi->members[i].uuid, &uuid, sizeof(uuid)))
+ return &mi->members[i];
return NULL;
}
struct bch_ioctl_disk_remove_by_uuid __user *user_arg)
{
struct bch_ioctl_disk_fail_by_uuid arg;
- struct cache_member *m;
+ struct bch_member *m;
int ret = -ENOENT;
if (copy_from_user(&arg, user_arg, sizeof(arg)))
return -EFAULT;
- mutex_lock(&bch_register_lock);
+ mutex_lock(&c->sb_lock);
if ((m = bch_uuid_lookup(c, arg.dev))) {
/* XXX: */
- SET_CACHE_STATE(m, CACHE_FAILED);
- bcache_write_super(c);
+ SET_BCH_MEMBER_STATE(m, BCH_MEMBER_STATE_FAILED);
+ bch_write_super(c);
ret = 0;
}
- mutex_unlock(&bch_register_lock);
+ mutex_unlock(&c->sb_lock);
return ret;
}
struct bch_ioctl_disk_fail_by_uuid __user *user_arg)
{
struct bch_ioctl_disk_fail_by_uuid arg;
- struct cache_member *m;
+ struct bch_member *m;
int ret = -ENOENT;
if (copy_from_user(&arg, user_arg, sizeof(arg)))
return -EFAULT;
- mutex_lock(&bch_register_lock);
+ mutex_lock(&c->sb_lock);
if ((m = bch_uuid_lookup(c, arg.dev))) {
- SET_CACHE_STATE(m, CACHE_FAILED);
- bcache_write_super(c);
+ SET_BCH_MEMBER_STATE(m, BCH_MEMBER_STATE_FAILED);
+ bch_write_super(c);
ret = 0;
}
- mutex_unlock(&bch_register_lock);
+ mutex_unlock(&c->sb_lock);
return ret;
}
struct bch_ioctl_query_uuid __user *user_arg)
{
return copy_to_user(&user_arg->uuid,
- &c->disk_sb.user_uuid,
- sizeof(c->disk_sb.user_uuid));
+ &c->sb.user_uuid,
+ sizeof(c->sb.user_uuid));
}
long bch_cache_set_ioctl(struct cache_set *c, unsigned cmd, void __user *arg)
#include "bcache.h"
#include "checksum.h"
+#include "super.h"
+#include "super-io.h"
#include <linux/crc32c.h>
+#include <linux/crypto.h>
+#include <linux/key.h>
+#include <linux/random.h>
+#include <linux/scatterlist.h>
+#include <crypto/algapi.h>
#include <crypto/chacha20.h>
#include <crypto/hash.h>
#include <crypto/poly1305.h>
+#include <keys/user-type.h>
/*
* Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group (Any
return crc;
}
-u64 bch_checksum_update(unsigned type, u64 crc, const void *data, size_t len)
+static u64 bch_checksum_init(unsigned type)
+{
+ switch (type) {
+ case BCH_CSUM_NONE:
+ return 0;
+ case BCH_CSUM_CRC32C:
+ return U32_MAX;
+ case BCH_CSUM_CRC64:
+ return U64_MAX;
+ default:
+ BUG();
+ }
+}
+
+static u64 bch_checksum_final(unsigned type, u64 crc)
+{
+ switch (type) {
+ case BCH_CSUM_NONE:
+ return 0;
+ case BCH_CSUM_CRC32C:
+ return crc ^ U32_MAX;
+ case BCH_CSUM_CRC64:
+ return crc ^ U64_MAX;
+ default:
+ BUG();
+ }
+}
+
+static u64 bch_checksum_update(unsigned type, u64 crc, const void *data, size_t len)
{
switch (type) {
case BCH_CSUM_NONE:
}
}
-u64 bch_checksum(unsigned type, const void *data, size_t len)
+static inline void do_encrypt_sg(struct crypto_blkcipher *tfm,
+ struct nonce nonce,
+ struct scatterlist *sg, size_t len)
+{
+ struct blkcipher_desc desc = { .tfm = tfm, .info = nonce.d };
+ int ret;
+
+ ret = crypto_blkcipher_encrypt_iv(&desc, sg, sg, len);
+ BUG_ON(ret);
+}
+
+static inline void do_encrypt(struct crypto_blkcipher *tfm,
+ struct nonce nonce,
+ void *buf, size_t len)
+{
+ struct scatterlist sg;
+
+ sg_init_one(&sg, buf, len);
+ do_encrypt_sg(tfm, nonce, &sg, len);
+}
+
+int bch_chacha_encrypt_key(struct bch_key *key, struct nonce nonce,
+ void *buf, size_t len)
+{
+ struct crypto_blkcipher *chacha20 =
+ crypto_alloc_blkcipher("chacha20", 0, CRYPTO_ALG_ASYNC);
+ int ret;
+
+ if (!chacha20)
+ return PTR_ERR(chacha20);
+
+ ret = crypto_blkcipher_setkey(chacha20, (void *) key, sizeof(*key));
+ if (ret)
+ goto err;
+
+ do_encrypt(chacha20, nonce, buf, len);
+err:
+ crypto_free_blkcipher(chacha20);
+ return ret;
+}
+
+static void gen_poly_key(struct cache_set *c, struct shash_desc *desc,
+ struct nonce nonce)
+{
+ u8 key[POLY1305_KEY_SIZE];
+
+ nonce.d[3] ^= BCH_NONCE_POLY;
+
+ memset(key, 0, sizeof(key));
+ do_encrypt(c->chacha20, nonce, key, sizeof(key));
+
+ desc->tfm = c->poly1305;
+ desc->flags = 0;
+ crypto_shash_init(desc);
+ crypto_shash_update(desc, key, sizeof(key));
+}
+
+struct bch_csum bch_checksum(struct cache_set *c, unsigned type,
+ struct nonce nonce, const void *data, size_t len)
{
- u64 crc = 0xffffffffffffffffULL;
+ switch (type) {
+ case BCH_CSUM_NONE:
+ case BCH_CSUM_CRC32C:
+ case BCH_CSUM_CRC64: {
+ u64 crc = bch_checksum_init(type);
+
+ crc = bch_checksum_update(type, crc, data, len);
+ crc = bch_checksum_final(type, crc);
+
+ return (struct bch_csum) { .lo = crc };
+ }
+
+ case BCH_CSUM_CHACHA20_POLY1305_80:
+ case BCH_CSUM_CHACHA20_POLY1305_128: {
+ SHASH_DESC_ON_STACK(desc, c->poly1305);
+ u8 digest[POLY1305_DIGEST_SIZE];
+ struct bch_csum ret = { 0 };
+
+ gen_poly_key(c, desc, nonce);
+
+ crypto_shash_update(desc, data, len);
+ crypto_shash_final(desc, digest);
+
+ memcpy(&ret, digest, bch_crc_bytes[type]);
+ return ret;
+ }
+ default:
+ BUG();
+ }
+}
- crc = bch_checksum_update(type, crc, data, len);
+void bch_encrypt(struct cache_set *c, unsigned type,
+ struct nonce nonce, void *data, size_t len)
+{
+ if (!bch_csum_type_is_encryption(type))
+ return;
- return crc ^ 0xffffffffffffffffULL;
+ do_encrypt(c->chacha20, nonce, data, len);
}
-u32 bch_checksum_bio(struct bio *bio, unsigned type)
+struct bch_csum bch_checksum_bio(struct cache_set *c, unsigned type,
+ struct nonce nonce, struct bio *bio)
{
struct bio_vec bv;
struct bvec_iter iter;
- u32 csum = U32_MAX;
- if (type == BCH_CSUM_NONE)
- return 0;
+ switch (type) {
+ case BCH_CSUM_NONE:
+ return (struct bch_csum) { 0 };
+ case BCH_CSUM_CRC32C:
+ case BCH_CSUM_CRC64: {
+ u64 crc = bch_checksum_init(type);
+
+ bio_for_each_segment(bv, bio, iter) {
+ void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
+
+ crc = bch_checksum_update(type,
+ crc, p, bv.bv_len);
+ kunmap_atomic(p);
+ }
+
+ crc = bch_checksum_final(type, crc);
+ return (struct bch_csum) { .lo = crc };
+ }
+
+ case BCH_CSUM_CHACHA20_POLY1305_80:
+ case BCH_CSUM_CHACHA20_POLY1305_128: {
+ SHASH_DESC_ON_STACK(desc, c->poly1305);
+ u8 digest[POLY1305_DIGEST_SIZE];
+ struct bch_csum ret = { 0 };
+
+ gen_poly_key(c, desc, nonce);
+
+ bio_for_each_segment(bv, bio, iter) {
+ void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
+
+ crypto_shash_update(desc, p, bv.bv_len);
+ kunmap_atomic(p);
+ }
+
+ crypto_shash_final(desc, digest);
+
+ memcpy(&ret, digest, bch_crc_bytes[type]);
+ return ret;
+ }
+ default:
+ BUG();
+ }
+}
+
+void bch_encrypt_bio(struct cache_set *c, unsigned type,
+ struct nonce nonce, struct bio *bio)
+{
+ struct bio_vec bv;
+ struct bvec_iter iter;
+ struct scatterlist sgl[16], *sg = sgl;
+ size_t bytes = 0;
+
+ if (!bch_csum_type_is_encryption(type))
+ return;
+
+ sg_init_table(sgl, ARRAY_SIZE(sgl));
bio_for_each_segment(bv, bio, iter) {
- void *p = kmap_atomic(bv.bv_page);
+ if (sg == sgl + ARRAY_SIZE(sgl)) {
+ sg_mark_end(sg - 1);
+ do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
+
+ le32_add_cpu(nonce.d, bytes / CHACHA20_BLOCK_SIZE);
+ bytes = 0;
+
+ sg_init_table(sgl, ARRAY_SIZE(sgl));
+ sg = sgl;
+ }
+
+ sg_set_page(sg++, bv.bv_page, bv.bv_len, bv.bv_offset);
+ bytes += bv.bv_len;
+
+ }
+
+ sg_mark_end(sg - 1);
+ do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
+}
+
+#ifdef __KERNEL__
+int bch_request_key(struct bch_sb *sb, struct bch_key *key)
+{
+ char key_description[60];
+ struct key *keyring_key;
+ const struct user_key_payload *ukp;
+ int ret;
+
+ snprintf(key_description, sizeof(key_description),
+ "bcache:%pUb", &sb->user_uuid);
+
+ keyring_key = request_key(&key_type_logon, key_description, NULL);
+ if (IS_ERR(keyring_key))
+ return PTR_ERR(keyring_key);
+
+ down_read(&keyring_key->sem);
+ ukp = user_key_payload(keyring_key);
+ if (ukp->datalen == sizeof(*key)) {
+ memcpy(key, ukp->data, ukp->datalen);
+ ret = 0;
+ } else {
+ ret = -EINVAL;
+ }
+ up_read(&keyring_key->sem);
+ key_put(keyring_key);
+
+ return ret;
+}
+#else
+#include <keyutils.h>
+#include <uuid/uuid.h>
+
+int bch_request_key(struct bch_sb *sb, struct bch_key *key)
+{
+ key_serial_t key_id;
+ char key_description[60];
+ char uuid[40];
+
+ uuid_unparse_lower(sb->user_uuid.b, uuid);
+ sprintf(key_description, "bcache:%s", uuid);
+
+ key_id = request_key("user", key_description, NULL,
+ KEY_SPEC_USER_KEYRING);
+ if (key_id < 0)
+ return -errno;
+
+ if (keyctl_read(key_id, (void *) key, sizeof(*key)) != sizeof(*key))
+ return -1;
+
+ return 0;
+}
+#endif
- csum = bch_checksum_update(type, csum,
- p + bv.bv_offset,
- bv.bv_len);
- kunmap_atomic(p);
+static int bch_decrypt_sb_key(struct cache_set *c,
+ struct bch_sb_field_crypt *crypt,
+ struct bch_key *key)
+{
+ struct bch_encrypted_key sb_key = crypt->key;
+ struct bch_key user_key;
+ int ret = 0;
+
+ /* is key encrypted? */
+ if (!bch_key_is_encrypted(&sb_key))
+ goto out;
+
+ ret = bch_request_key(c->disk_sb, &user_key);
+ if (ret) {
+ bch_err(c, "error requesting encryption key");
+ goto err;
}
- return csum ^= U32_MAX;
+ /* decrypt real key: */
+ ret = bch_chacha_encrypt_key(&user_key, bch_sb_key_nonce(c),
+ &sb_key, sizeof(sb_key));
+ if (ret)
+ goto err;
+
+ if (bch_key_is_encrypted(&sb_key)) {
+ bch_err(c, "incorrect encryption key");
+ ret = -EINVAL;
+ goto err;
+ }
+out:
+ *key = sb_key.key;
+err:
+ memzero_explicit(&sb_key, sizeof(sb_key));
+ memzero_explicit(&user_key, sizeof(user_key));
+ return ret;
+}
+
+static int bch_alloc_ciphers(struct cache_set *c)
+{
+ if (!c->chacha20)
+ c->chacha20 = crypto_alloc_blkcipher("chacha20", 0,
+ CRYPTO_ALG_ASYNC);
+ if (IS_ERR(c->chacha20))
+ return PTR_ERR(c->chacha20);
+
+ if (!c->poly1305)
+ c->poly1305 = crypto_alloc_shash("poly1305", 0, 0);
+ if (IS_ERR(c->poly1305))
+ return PTR_ERR(c->poly1305);
+
+ return 0;
+}
+
+int bch_disable_encryption(struct cache_set *c)
+{
+ struct bch_sb_field_crypt *crypt;
+ struct bch_key key;
+ int ret = -EINVAL;
+
+ mutex_lock(&c->sb_lock);
+
+ crypt = bch_sb_get_crypt(c->disk_sb);
+ if (!crypt)
+ goto out;
+
+ /* is key encrypted? */
+ ret = 0;
+ if (bch_key_is_encrypted(&crypt->key))
+ goto out;
+
+ ret = bch_decrypt_sb_key(c, crypt, &key);
+ if (ret)
+ goto out;
+
+ crypt->key.magic = BCH_KEY_MAGIC;
+ crypt->key.key = key;
+
+ SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 0);
+ bch_write_super(c);
+out:
+ mutex_unlock(&c->sb_lock);
+
+ return ret;
+}
+
+int bch_enable_encryption(struct cache_set *c, bool keyed)
+{
+ struct bch_encrypted_key key;
+ struct bch_key user_key;
+ struct bch_sb_field_crypt *crypt;
+ int ret = -EINVAL;
+
+ mutex_lock(&c->sb_lock);
+
+ /* Do we already have an encryption key? */
+ if (bch_sb_get_crypt(c->disk_sb))
+ goto err;
+
+ ret = bch_alloc_ciphers(c);
+ if (ret)
+ goto err;
+
+ key.magic = BCH_KEY_MAGIC;
+ get_random_bytes(&key.key, sizeof(key.key));
+
+ if (keyed) {
+ ret = bch_request_key(c->disk_sb, &user_key);
+ if (ret) {
+ bch_err(c, "error requesting encryption key");
+ goto err;
+ }
+
+ ret = bch_chacha_encrypt_key(&user_key, bch_sb_key_nonce(c),
+ &key, sizeof(key));
+ if (ret)
+ goto err;
+ }
+
+ ret = crypto_blkcipher_setkey(c->chacha20,
+ (void *) &key.key, sizeof(key.key));
+ if (ret)
+ goto err;
+
+ crypt = container_of_or_null(bch_fs_sb_field_resize(c, NULL,
+ sizeof(*crypt) / sizeof(u64)),
+ struct bch_sb_field_crypt, field);
+ if (!crypt) {
+ ret = -ENOMEM; /* XXX this technically could be -ENOSPC */
+ goto err;
+ }
+
+ crypt->field.type = BCH_SB_FIELD_crypt;
+ crypt->key = key;
+
+ /* write superblock */
+ SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 1);
+ bch_write_super(c);
+err:
+ mutex_unlock(&c->sb_lock);
+ memzero_explicit(&user_key, sizeof(user_key));
+ memzero_explicit(&key, sizeof(key));
+ return ret;
+}
+
+void bch_cache_set_encryption_free(struct cache_set *c)
+{
+ if (!IS_ERR_OR_NULL(c->poly1305))
+ crypto_free_shash(c->poly1305);
+ if (!IS_ERR_OR_NULL(c->chacha20))
+ crypto_free_blkcipher(c->chacha20);
+}
+
+int bch_cache_set_encryption_init(struct cache_set *c)
+{
+ struct bch_sb_field_crypt *crypt;
+ struct bch_key key;
+ int ret;
+
+ crypt = bch_sb_get_crypt(c->disk_sb);
+ if (!crypt)
+ return 0;
+
+ ret = bch_alloc_ciphers(c);
+ if (ret)
+ return ret;
+
+ ret = bch_decrypt_sb_key(c, crypt, &key);
+ if (ret)
+ goto err;
+
+ ret = crypto_blkcipher_setkey(c->chacha20,
+ (void *) &key.key, sizeof(key.key));
+err:
+ memzero_explicit(&key, sizeof(key));
+ return ret;
}
#ifndef _BCACHE_CHECKSUM_H
#define _BCACHE_CHECKSUM_H
-#include "btree_types.h"
+#include "bcache.h"
+#include "super-io.h"
+
+#include <crypto/chacha20.h>
u64 bch_crc64_update(u64, const void *, size_t);
-u64 bch_checksum_update(unsigned, u64, const void *, size_t);
-u64 bch_checksum(unsigned, const void *, size_t);
-u32 bch_checksum_bio(struct bio *, unsigned);
+#define BCH_NONCE_EXTENT cpu_to_le32(1 << 28)
+#define BCH_NONCE_BTREE cpu_to_le32(2 << 28)
+#define BCH_NONCE_JOURNAL cpu_to_le32(3 << 28)
+#define BCH_NONCE_PRIO cpu_to_le32(4 << 28)
+#define BCH_NONCE_POLY cpu_to_le32(1 << 31)
+
+struct bch_csum bch_checksum(struct cache_set *, unsigned, struct nonce,
+ const void *, size_t);
/*
- * This is used for various on disk data structures - cache_sb, prio_set, bset,
- * jset: The checksum is _always_ the first 8 bytes of these structs
+ * This is used for various on disk data structures - bch_sb, prio_set, bset,
+ * jset: The checksum is _always_ the first field of these structs
*/
-#define __csum_set(i, u64s, type) \
+#define csum_vstruct(_c, _type, _nonce, _i) \
({ \
- const void *start = ((const void *) (i)) + sizeof(u64); \
- const void *end = __bkey_idx(i, u64s); \
+ const void *start = ((const void *) (_i)) + sizeof((_i)->csum); \
+ const void *end = vstruct_end(_i); \
\
- bch_checksum(type, start, end - start); \
+ bch_checksum(_c, _type, _nonce, start, end - start); \
})
+int bch_chacha_encrypt_key(struct bch_key *, struct nonce, void *, size_t);
+int bch_request_key(struct bch_sb *, struct bch_key *);
+
+void bch_encrypt(struct cache_set *, unsigned, struct nonce,
+ void *data, size_t);
+
+struct bch_csum bch_checksum_bio(struct cache_set *, unsigned,
+ struct nonce, struct bio *);
+void bch_encrypt_bio(struct cache_set *, unsigned,
+ struct nonce, struct bio *);
+
+int bch_disable_encryption(struct cache_set *);
+int bch_enable_encryption(struct cache_set *, bool);
+
+void bch_cache_set_encryption_free(struct cache_set *);
+int bch_cache_set_encryption_init(struct cache_set *);
+
+static inline unsigned bch_data_checksum_type(struct cache_set *c)
+{
+ if (c->sb.encryption_type)
+ return c->opts.wide_macs
+ ? BCH_CSUM_CHACHA20_POLY1305_128
+ : BCH_CSUM_CHACHA20_POLY1305_80;
+
+ return c->opts.data_checksum;
+}
+
+static inline unsigned bch_meta_checksum_type(struct cache_set *c)
+{
+ return c->sb.encryption_type
+ ? BCH_CSUM_CHACHA20_POLY1305_128
+ : c->opts.metadata_checksum;
+}
+
+static inline bool bch_checksum_type_valid(const struct cache_set *c,
+ unsigned type)
+{
+ if (type >= BCH_CSUM_NR)
+ return false;
+
+ if (bch_csum_type_is_encryption(type) && !c->chacha20)
+ return false;
+
+ return true;
+}
+
+static const unsigned bch_crc_bytes[] = {
+ [BCH_CSUM_NONE] = 0,
+ [BCH_CSUM_CRC32C] = 4,
+ [BCH_CSUM_CRC64] = 8,
+ [BCH_CSUM_CHACHA20_POLY1305_80] = 10,
+ [BCH_CSUM_CHACHA20_POLY1305_128] = 16,
+};
+
+static inline bool bch_crc_cmp(struct bch_csum l, struct bch_csum r)
+{
+ /*
+ * XXX: need some way of preventing the compiler from optimizing this
+ * into a form that isn't constant time..
+ */
+ return ((l.lo ^ r.lo) | (l.hi ^ r.hi)) != 0;
+}
+
+/* for skipping ahead and encrypting/decrypting at an offset: */
+static inline struct nonce nonce_add(struct nonce nonce, unsigned offset)
+{
+ EBUG_ON(offset & (CHACHA20_BLOCK_SIZE - 1));
+
+ le32_add_cpu(&nonce.d[0], offset / CHACHA20_BLOCK_SIZE);
+ return nonce;
+}
+
+static inline bool bch_key_is_encrypted(struct bch_encrypted_key *key)
+{
+ return le64_to_cpu(key->magic) != BCH_KEY_MAGIC;
+}
+
+static inline struct nonce __bch_sb_key_nonce(struct bch_sb *sb)
+{
+ __le64 magic = __bch_sb_magic(sb);
+
+ return (struct nonce) {{
+ [0] = 0,
+ [1] = 0,
+ [2] = ((__le32 *) &magic)[0],
+ [3] = ((__le32 *) &magic)[1],
+ }};
+}
+
+static inline struct nonce bch_sb_key_nonce(struct cache_set *c)
+{
+ __le64 magic = bch_sb_magic(c);
+
+ return (struct nonce) {{
+ [0] = 0,
+ [1] = 0,
+ [2] = ((__le32 *) &magic)[0],
+ [3] = ((__le32 *) &magic)[1],
+ }};
+}
+
#endif /* _BCACHE_CHECKSUM_H */
#include "bcache.h"
#include "compress.h"
+#include "extents.h"
#include "io.h"
+#include "super-io.h"
#include <linux/lz4.h>
#include <linux/zlib.h>
unsigned prev_end = PAGE_SIZE;
void *data;
- BUG_ON(bvec_iter_sectors(start) > BCH_COMPRESSED_EXTENT_MAX);
+ BUG_ON(bvec_iter_sectors(start) > BCH_ENCODED_EXTENT_MAX);
*bounced = BOUNCED_MAPPED;
}
static int __bio_uncompress(struct cache_set *c, struct bio *src,
- void *dst_data, struct bch_extent_crc64 crc)
+ void *dst_data, struct bch_extent_crc128 crc)
{
void *src_data = NULL;
unsigned src_bounced;
size_t src_len = src->bi_iter.bi_size;
- size_t dst_len = crc.uncompressed_size << 9;
+ size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
int ret;
src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
int bch_bio_uncompress_inplace(struct cache_set *c, struct bio *bio,
unsigned live_data_sectors,
- struct bch_extent_crc64 crc)
+ struct bch_extent_crc128 crc)
{
void *dst_data = NULL;
- size_t dst_len = crc.uncompressed_size << 9;
+ size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
int ret = -ENOMEM;
BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
int bch_bio_uncompress(struct cache_set *c, struct bio *src,
struct bio *dst, struct bvec_iter dst_iter,
- struct bch_extent_crc64 crc)
+ struct bch_extent_crc128 crc)
{
void *dst_data = NULL;
unsigned dst_bounced;
- size_t dst_len = crc.uncompressed_size << 9;
+ size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
int ret = -ENOMEM;
dst_data = dst_len == dst_iter.bi_size
*src_len = src->bi_iter.bi_size;
workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
-retry_compress:
- ret = lz4_compress(src_data, *src_len,
- dst_data, dst_len,
- workspace);
- /*
- * On error, the compressed data was bigger than dst_len, and
- * -ret is the amount of data we were able to compress - round
- * down to nearest block and try again:
- */
- if (ret && round_down(-ret, block_bytes(c)) > *dst_len) {
- BUG_ON(ret > 0);
- /* not supposed to happen */
- if (WARN_ON(-ret >= *src_len))
- goto err;
+ while (*src_len > block_bytes(c) &&
+ (ret = lz4_compress(src_data, *src_len,
+ dst_data, dst_len,
+ workspace))) {
+ /*
+ * On error, the compressed data was bigger than
+ * dst_len, and -ret is the amount of data we were able
+ * to compress - round down to nearest block and try
+ * again:
+ */
+ BUG_ON(ret > 0);
+ BUG_ON(-ret >= *src_len);
*src_len = round_down(-ret, block_bytes(c));
- if (!*src_len)
- goto err;
-
- goto retry_compress;
}
+
mempool_free(workspace, &c->lz4_workspace_pool);
if (ret)
}
BUG_ON(!*dst_len);
+ BUG_ON(*dst_len > dst->bi_iter.bi_size);
+
+ BUG_ON(*src_len & (block_bytes(c) - 1));
+ BUG_ON(*src_len > src->bi_iter.bi_size);
/* Didn't get smaller: */
if (round_up(*dst_len, block_bytes(c)) >= *src_len) {
unsigned orig_dst = dst->bi_iter.bi_size;
unsigned orig_src = src->bi_iter.bi_size;
- /* Don't consume more than BCH_COMPRESSED_EXTENT_MAX from @src: */
+ /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
src->bi_iter.bi_size =
- min(src->bi_iter.bi_size, BCH_COMPRESSED_EXTENT_MAX << 9);
+ min(src->bi_iter.bi_size, BCH_ENCODED_EXTENT_MAX << 9);
/* Don't generate a bigger output than input: */
dst->bi_iter.bi_size =
src->bi_iter.bi_size = orig_src;
}
+/* doesn't write superblock: */
+int bch_check_set_has_compressed_data(struct cache_set *c,
+ unsigned compression_type)
+{
+ switch (compression_type) {
+ case BCH_COMPRESSION_NONE:
+ return 0;
+ case BCH_COMPRESSION_LZ4:
+ if (bch_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
+ return 0;
+
+ bch_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
+ break;
+ case BCH_COMPRESSION_GZIP:
+ if (bch_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
+ return 0;
+
+ bch_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
+ break;
+ }
+
+ return bch_compress_init(c);
+}
+
void bch_compress_free(struct cache_set *c)
{
vfree(c->zlib_workspace);
int bch_compress_init(struct cache_set *c)
{
+ unsigned order = get_order(BCH_ENCODED_EXTENT_MAX << 9);
int ret, cpu;
- c->bio_decompress_worker = alloc_percpu(*c->bio_decompress_worker);
- if (!c->bio_decompress_worker)
- return -ENOMEM;
+ if (!bch_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
+ !bch_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
+ return 0;
- for_each_possible_cpu(cpu) {
- struct bio_decompress_worker *d =
- per_cpu_ptr(c->bio_decompress_worker, cpu);
+ if (!c->bio_decompress_worker) {
+ c->bio_decompress_worker = alloc_percpu(*c->bio_decompress_worker);
+ if (!c->bio_decompress_worker)
+ return -ENOMEM;
- d->c = c;
- INIT_WORK(&d->work, bch_bio_decompress_work);
- init_llist_head(&d->bio_list);
+ for_each_possible_cpu(cpu) {
+ struct bio_decompress_worker *d =
+ per_cpu_ptr(c->bio_decompress_worker, cpu);
+
+ d->c = c;
+ INIT_WORK(&d->work, bch_bio_decompress_work);
+ init_llist_head(&d->bio_list);
+ }
}
- ret = mempool_init_page_pool(&c->compression_bounce[READ], 1,
- get_order(BCH_COMPRESSED_EXTENT_MAX << 9));
- if (ret)
- return ret;
+ if (!mempool_initialized(&c->compression_bounce[READ])) {
+ ret = mempool_init_page_pool(&c->compression_bounce[READ],
+ 1, order);
+ if (ret)
+ return ret;
+ }
- ret = mempool_init_page_pool(&c->compression_bounce[WRITE], 1,
- get_order(BCH_COMPRESSED_EXTENT_MAX << 9));
- if (ret)
- return ret;
+ if (!mempool_initialized(&c->compression_bounce[WRITE])) {
+ ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
+ 1, order);
+ if (ret)
+ return ret;
+ }
- ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool, 1,
- LZ4_MEM_COMPRESS);
- if (ret)
- return ret;
+ if (!mempool_initialized(&c->lz4_workspace_pool) &&
+ bch_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
+ ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
+ 1, LZ4_MEM_COMPRESS);
+ if (ret)
+ return ret;
+ }
- c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
- if (!c->zlib_workspace)
- return -ENOMEM;
+ if (!c->zlib_workspace &&
+ bch_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
+ c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
+ if (!c->zlib_workspace)
+ return -ENOMEM;
+ }
return 0;
}
#define _BCACHE_COMPRESS_H
int bch_bio_uncompress_inplace(struct cache_set *, struct bio *,
- unsigned, struct bch_extent_crc64);
+ unsigned, struct bch_extent_crc128);
int bch_bio_uncompress(struct cache_set *, struct bio *, struct bio *,
- struct bvec_iter, struct bch_extent_crc64);
+ struct bvec_iter, struct bch_extent_crc128);
void bch_bio_compress(struct cache_set *, struct bio *, size_t *,
struct bio *, size_t *, unsigned *);
+int bch_check_set_has_compressed_data(struct cache_set *, unsigned);
void bch_compress_free(struct cache_set *);
int bch_compress_init(struct cache_set *);
if (inmemory->u64s != sorted->u64s ||
memcmp(inmemory->start,
sorted->start,
- (void *) bset_bkey_last(inmemory) - (void *) inmemory->start)) {
+ vstruct_end(inmemory) - (void *) inmemory->start)) {
unsigned offset = 0, sectors;
struct bset *i;
unsigned j;
while (offset < b->written) {
if (!offset ) {
i = &n_ondisk->keys;
- sectors = __set_blocks(n_ondisk,
- le16_to_cpu(n_ondisk->keys.u64s),
- block_bytes(c)) <<
+ sectors = vstruct_blocks(n_ondisk, c->block_bits) <<
c->block_bits;
} else {
struct btree_node_entry *bne =
(void *) n_ondisk + (offset << 9);
i = &bne->keys;
- sectors = __set_blocks(bne,
- le16_to_cpu(bne->keys.u64s),
- block_bytes(c)) <<
+ sectors = vstruct_blocks(bne, c->block_bits) <<
c->block_bits;
}
if (IS_ERR_OR_NULL(bch_debug))
return;
- snprintf(name, sizeof(name), "%pU", c->disk_sb.user_uuid.b);
+ snprintf(name, sizeof(name), "%pU", c->sb.user_uuid.b);
c->debug = debugfs_create_dir(name, bch_debug);
if (IS_ERR_OR_NULL(c->debug))
return;
static u64 bch_dirent_hash(const struct bch_hash_info *info,
const struct qstr *name)
{
- switch (info->type) {
- case BCH_STR_HASH_SHA1: {
- SHASH_DESC_ON_STACK(desc, bch_sha1);
- u8 digest[SHA1_DIGEST_SIZE];
- u64 ret;
- desc->tfm = bch_sha1;
- desc->flags = 0;
- crypto_shash_init(desc);
-
- crypto_shash_update(desc, (void *) &info->seed, sizeof(info->seed));
-
- crypto_shash_update(desc, (void *) name->name, name->len);
- crypto_shash_final(desc, digest);
- memcpy(&ret, &digest, sizeof(ret));
- return max_t(u64, ret >> 1, 2);
- }
- default: {
- struct bch_str_hash_ctx ctx;
-
- bch_str_hash_init(&ctx, info->type);
- bch_str_hash_update(&ctx, info->type, &info->seed, sizeof(info->seed));
+ struct bch_str_hash_ctx ctx;
- bch_str_hash_update(&ctx, info->type, name->name, name->len);
+ bch_str_hash_init(&ctx, info);
+ bch_str_hash_update(&ctx, info, name->name, name->len);
- /* [0,2) reserved for dots */
- return max_t(u64, bch_str_hash_end(&ctx, info->type), 2);
- }
- }
+ /* [0,2) reserved for dots */
+ return max_t(u64, bch_str_hash_end(&ctx, info), 2);
}
static u64 dirent_hash_key(const struct bch_hash_info *info, const void *key)
#include "bkey_methods.h"
#include "btree_gc.h"
#include "btree_update.h"
+#include "checksum.h"
#include "debug.h"
#include "dirent.h"
#include "error.h"
#include "extents.h"
#include "inode.h"
#include "journal.h"
-#include "super.h"
+#include "super-io.h"
#include "writeback.h"
#include "xattr.h"
#include <trace/events/bcache.h>
-static bool __bch_extent_normalize(struct cache_set *, struct bkey_s, bool);
static enum merge_result bch_extent_merge(struct cache_set *, struct btree *,
struct bkey_i *, struct bkey_i *);
return NULL;
}
-unsigned bch_extent_nr_ptrs_from(struct bkey_s_c_extent e,
- const struct bch_extent_ptr *start)
+unsigned bch_extent_nr_ptrs(struct bkey_s_c_extent e)
{
const struct bch_extent_ptr *ptr;
unsigned nr_ptrs = 0;
- extent_for_each_ptr_from(e, ptr, start)
+ extent_for_each_ptr(e, ptr)
nr_ptrs++;
return nr_ptrs;
}
-unsigned bch_extent_nr_ptrs(struct bkey_s_c_extent e)
+unsigned bch_extent_nr_dirty_ptrs(struct bkey_s_c k)
{
- return bch_extent_nr_ptrs_from(e, &e.v->start->ptr);
+ struct bkey_s_c_extent e;
+ const struct bch_extent_ptr *ptr;
+ unsigned nr_ptrs = 0;
+
+ switch (k.k->type) {
+ case BCH_EXTENT:
+ case BCH_EXTENT_CACHED:
+ e = bkey_s_c_to_extent(k);
+
+ extent_for_each_ptr(e, ptr)
+ nr_ptrs += !ptr->cached;
+ break;
+
+ case BCH_RESERVATION:
+ nr_ptrs = bkey_s_c_to_reservation(k).v->nr_replicas;
+ break;
+ }
+
+ return nr_ptrs;
}
/* returns true if equal */
*
* and then verify that crc_dead1 + crc_live + crc_dead2 == orig_crc, and then
* use crc_live here (that we verified was correct earlier)
+ *
+ * note: doesn't work with encryption
*/
void bch_extent_narrow_crcs(struct bkey_s_extent e)
{
union bch_extent_crc *crc;
bool have_wide = false, have_narrow = false;
- u64 csum = 0;
+ struct bch_csum csum = { 0 };
unsigned csum_type = 0;
extent_for_each_crc(e, crc) {
- if (crc_compression_type(crc))
+ if (crc_compression_type(crc) ||
+ bch_csum_type_is_encryption(crc_csum_type(crc)))
continue;
if (crc_uncompressed_size(e.k, crc) != e.k->size) {
case BCH_EXTENT_CRC_NONE:
BUG();
case BCH_EXTENT_CRC32:
- if (bch_crc_size[csum_type] > sizeof(crc->crc32.csum))
+ if (bch_crc_bytes[csum_type] > 4)
continue;
bch_extent_crc_narrow_pointers(e, crc);
- crc->crc32.compressed_size = e.k->size;
- crc->crc32.uncompressed_size = e.k->size;
+ crc->crc32._compressed_size = e.k->size - 1;
+ crc->crc32._uncompressed_size = e.k->size - 1;
crc->crc32.offset = 0;
crc->crc32.csum_type = csum_type;
- crc->crc32.csum = csum;
+ crc->crc32.csum = csum.lo;
break;
case BCH_EXTENT_CRC64:
- if (bch_crc_size[csum_type] > sizeof(crc->crc64.csum))
+ if (bch_crc_bytes[csum_type] > 10)
continue;
bch_extent_crc_narrow_pointers(e, crc);
- crc->crc64.compressed_size = e.k->size;
- crc->crc64.uncompressed_size = e.k->size;
+ crc->crc64._compressed_size = e.k->size - 1;
+ crc->crc64._uncompressed_size = e.k->size - 1;
crc->crc64.offset = 0;
crc->crc64.csum_type = csum_type;
- crc->crc64.csum = csum;
+ crc->crc64.csum_lo = csum.lo;
+ crc->crc64.csum_hi = csum.hi;
+ break;
+ case BCH_EXTENT_CRC128:
+ if (bch_crc_bytes[csum_type] > 16)
+ continue;
+
+ bch_extent_crc_narrow_pointers(e, crc);
+ crc->crc128._compressed_size = e.k->size - 1;
+ crc->crc128._uncompressed_size = e.k->size - 1;
+ crc->crc128.offset = 0;
+ crc->crc128.csum_type = csum_type;
+ crc->crc128.csum = csum;
break;
}
}
struct bch_extent_ptr *ptr = &e.v->start->ptr;
bool dropped = false;
- /*
- * We don't want to change which pointers are considered cached/dirty,
- * so don't remove pointers that are considered dirty:
- */
rcu_read_lock();
- while ((ptr = extent_ptr_next(e, ptr)) &&
- !bch_extent_ptr_is_dirty(c, e.c, ptr))
+ while ((ptr = extent_ptr_next(e, ptr)))
if (should_drop_ptr(c, e.c, ptr)) {
__bch_extent_drop_ptr(e, ptr);
dropped = true;
static bool bch_ptr_normalize(struct cache_set *c, struct btree *bk,
struct bkey_s k)
{
- return __bch_extent_normalize(c, k, false);
+ return bch_extent_normalize(c, k);
}
static void bch_ptr_swab(const struct bkey_format *f, struct bkey_packed *k)
{
- u64 *d = (u64 *) bkeyp_val(f, k);
- unsigned i;
+ switch (k->type) {
+ case BCH_EXTENT:
+ case BCH_EXTENT_CACHED: {
+ union bch_extent_entry *entry;
+ u64 *d = (u64 *) bkeyp_val(f, k);
+ unsigned i;
- for (i = 0; i < bkeyp_val_u64s(f, k); i++)
- d[i] = swab64(d[i]);
+ for (i = 0; i < bkeyp_val_u64s(f, k); i++)
+ d[i] = swab64(d[i]);
+
+ for (entry = (union bch_extent_entry *) d;
+ entry < (union bch_extent_entry *) (d + bkeyp_val_u64s(f, k));
+ entry = extent_entry_next(entry)) {
+ switch (extent_entry_type(entry)) {
+ case BCH_EXTENT_ENTRY_crc32:
+ entry->crc32.csum = swab32(entry->crc32.csum);
+ break;
+ case BCH_EXTENT_ENTRY_crc64:
+ entry->crc64.csum_hi = swab16(entry->crc64.csum_hi);
+ entry->crc64.csum_lo = swab64(entry->crc64.csum_lo);
+ break;
+ case BCH_EXTENT_ENTRY_crc128:
+ entry->crc128.csum.hi = swab64(entry->crc64.csum_hi);
+ entry->crc128.csum.lo = swab64(entry->crc64.csum_lo);
+ break;
+ case BCH_EXTENT_ENTRY_ptr:
+ break;
+ }
+ }
+ break;
+ }
+ }
}
static const char *extent_ptr_invalid(struct bkey_s_c_extent e,
const struct bch_extent_ptr *ptr2;
const struct cache_member_cpu *m = mi->m + ptr->dev;
- if (ptr->dev > mi->nr_in_set || !m->valid)
+ if (ptr->dev > mi->nr_devices || !m->valid)
return "pointer to invalid device";
extent_for_each_ptr(e, ptr2)
switch (__extent_entry_type(entry)) {
case BCH_EXTENT_ENTRY_crc32:
case BCH_EXTENT_ENTRY_crc64:
+ case BCH_EXTENT_ENTRY_crc128:
crc = entry_to_crc(entry);
+
p("crc: c_size %u size %u offset %u csum %u compress %u",
crc_compressed_size(e.k, crc),
crc_uncompressed_size(e.k, crc),
crc_compression_type(crc));
break;
case BCH_EXTENT_ENTRY_ptr:
- ptr = &entry->ptr;
+ ptr = entry_to_ptr(entry);
+
p("ptr: %u:%llu gen %u%s", ptr->dev,
(u64) ptr->offset, ptr->gen,
(ca = PTR_CACHE(c, ptr)) && ptr_stale(ca, ptr)
if (prev_crc != crc)
crc->crc64.offset += e.k->size - len;
break;
+ case BCH_EXTENT_CRC128:
+ if (prev_crc != crc)
+ crc->crc128.offset += e.k->size - len;
+ break;
}
prev_crc = crc;
}
BUG_ON(!l.k->size || !r.k->size);
if (l.k->type != r.k->type ||
- l.k->version != r.k->version)
+ bversion_cmp(l.k->version, r.k->version))
return false;
switch (l.k->type) {
extent_for_each_ptr(le, lp) {
const union bch_extent_entry *entry =
- bkey_idx(re.v, (u64 *) lp - le.v->_data);
+ vstruct_idx(re.v, (u64 *) lp - le.v->_data);
if (!extent_entry_is_ptr(entry))
return false;
if (!(s->trans->flags & BTREE_INSERT_JOURNAL_REPLAY) &&
bkey_cmp(s->committed, insert->k.p) &&
- bkey_extent_is_compressed(c, bkey_i_to_s_c(insert))) {
+ bkey_extent_is_compressed(bkey_i_to_s_c(insert))) {
/* XXX: possibly need to increase our reservation? */
bch_cut_subtract_back(s, s->committed,
bkey_i_to_s(&split.k));
{
struct extent_insert_hook *hook = s->trans->hook;
enum extent_insert_hook_ret ret;
-
+#if 0
+ /*
+ * Currently disabled for encryption - broken with fcollapse. Will have
+ * to reenable when versions are exposed for send/receive - versions
+ * will have to be monotonic then:
+ */
if (k.k && k.k->size &&
- s->insert->k->k.version &&
- k.k->version > s->insert->k->k.version)
+ !bversion_zero(s->insert->k->k.version) &&
+ bversion_cmp(k.k->version, s->insert->k->k.version) > 0) {
ret = BTREE_HOOK_NO_INSERT;
- else if (hook)
+ } else
+#endif
+ if (hook)
ret = hook->fn(hook, s->committed, next_pos, k, s->insert->k);
else
ret = BTREE_HOOK_DO_INSERT;
unsigned sectors;
if (overlap == BCH_EXTENT_OVERLAP_MIDDLE &&
- (sectors = bkey_extent_is_compressed(c, k))) {
+ (sectors = bkey_extent_is_compressed(k))) {
int flags = BCH_DISK_RESERVATION_BTREE_LOCKS_HELD;
if (s->trans->flags & BTREE_INSERT_NOFAIL)
struct bkey_s_c_extent e = bkey_s_c_to_extent(k);
const union bch_extent_entry *entry;
const union bch_extent_crc *crc;
+ const struct bch_extent_ptr *ptr;
struct cache_member_rcu *mi = cache_member_info_get(c);
unsigned size_ondisk = e.k->size;
const char *reason;
if (__extent_entry_type(entry) >= BCH_EXTENT_ENTRY_MAX)
goto invalid;
- switch (extent_entry_type(entry)) {
- case BCH_EXTENT_ENTRY_crc32:
- case BCH_EXTENT_ENTRY_crc64:
+ if (extent_entry_is_crc(entry)) {
crc = entry_to_crc(entry);
reason = "checksum offset + key size > uncompressed size";
size_ondisk = crc_compressed_size(e.k, crc);
reason = "invalid checksum type";
- if (crc_csum_type(crc) >= BCH_CSUM_NR)
+ if (!bch_checksum_type_valid(c, crc_csum_type(crc)))
goto invalid;
reason = "invalid compression type";
if (crc_compression_type(crc) >= BCH_COMPRESSION_NR)
goto invalid;
- break;
- case BCH_EXTENT_ENTRY_ptr:
+ } else {
+ ptr = entry_to_ptr(entry);
+
reason = extent_ptr_invalid(e, mi,
&entry->ptr, size_ondisk);
if (reason)
goto invalid;
- break;
}
}
return reason;
}
- case BCH_RESERVATION:
+ case BCH_RESERVATION: {
+ struct bkey_s_c_reservation r = bkey_s_c_to_reservation(k);
+
+ if (bkey_val_bytes(k.k) != sizeof(struct bch_reservation))
+ return "incorrect value size";
+
+ if (!r.v->nr_replicas || r.v->nr_replicas > BCH_REPLICAS_MAX)
+ return "invalid nr_replicas";
+
return NULL;
+ }
default:
return "invalid value type";
unsigned seq, stale;
char buf[160];
bool bad;
- unsigned ptrs_per_tier[CACHE_TIERS];
+ unsigned ptrs_per_tier[BCH_TIER_MAX];
unsigned tier, replicas = 0;
/*
mi = cache_member_info_get(c);
extent_for_each_ptr(e, ptr) {
- bool dirty = bch_extent_ptr_is_dirty(c, e, ptr);
-
replicas++;
- if (ptr->dev >= mi->nr_in_set)
+ if (ptr->dev >= mi->nr_devices)
goto bad_device;
/*
stale = ptr_stale(ca, ptr);
- cache_set_bug_on(stale && dirty, c,
+ cache_set_bug_on(stale && !ptr->cached, c,
"stale dirty pointer");
cache_set_bug_on(stale > 96, c,
bad = (mark.is_metadata ||
(gc_pos_cmp(c->gc_pos, gc_pos_btree_node(b)) > 0 &&
!mark.owned_by_allocator &&
- !(dirty
- ? mark.dirty_sectors
- : mark.cached_sectors)));
+ !(ptr->cached
+ ? mark.cached_sectors
+ : mark.dirty_sectors)));
} while (read_seqcount_retry(&c->gc_pos_lock, seq));
if (bad)
case BCH_EXTENT:
case BCH_EXTENT_CACHED:
bch_extent_debugcheck_extent(c, b, bkey_s_c_to_extent(k));
+ break;
case BCH_RESERVATION:
break;
default:
static unsigned PTR_TIER(struct cache_member_rcu *mi,
const struct bch_extent_ptr *ptr)
{
- return ptr->dev < mi->nr_in_set
+ return ptr->dev < mi->nr_devices
? mi->m[ptr->dev].tier
: UINT_MAX;
}
-void bch_extent_entry_append(struct bkey_i_extent *e,
- union bch_extent_entry *entry)
-{
- BUG_ON(bkey_val_u64s(&e->k) + extent_entry_u64s(entry) >
- BKEY_EXTENT_VAL_U64s_MAX);
-
- memcpy_u64s(extent_entry_last(extent_i_to_s(e)),
- entry,
- extent_entry_u64s(entry));
- e->k.u64s += extent_entry_u64s(entry);
-}
-
-const unsigned bch_crc_size[] = {
- [BCH_CSUM_NONE] = 0,
- [BCH_CSUM_CRC32C] = 4,
- [BCH_CSUM_CRC64] = 8,
-};
-
static void bch_extent_crc_init(union bch_extent_crc *crc,
unsigned compressed_size,
unsigned uncompressed_size,
unsigned compression_type,
- u64 csum, unsigned csum_type)
+ unsigned nonce,
+ struct bch_csum csum, unsigned csum_type)
{
- if (bch_crc_size[csum_type] <= 4 &&
- uncompressed_size <= CRC32_EXTENT_SIZE_MAX) {
+ if (bch_crc_bytes[csum_type] <= 4 &&
+ uncompressed_size <= CRC32_SIZE_MAX &&
+ nonce <= CRC32_NONCE_MAX) {
crc->crc32 = (struct bch_extent_crc32) {
.type = 1 << BCH_EXTENT_ENTRY_crc32,
- .compressed_size = compressed_size,
- .uncompressed_size = uncompressed_size,
+ ._compressed_size = compressed_size - 1,
+ ._uncompressed_size = uncompressed_size - 1,
.offset = 0,
.compression_type = compression_type,
.csum_type = csum_type,
- .csum = csum,
+ .csum = *((__le32 *) &csum.lo),
};
- } else {
- BUG_ON(uncompressed_size > CRC64_EXTENT_SIZE_MAX);
+ return;
+ }
+ if (bch_crc_bytes[csum_type] <= 10 &&
+ uncompressed_size <= CRC64_SIZE_MAX &&
+ nonce <= CRC64_NONCE_MAX) {
crc->crc64 = (struct bch_extent_crc64) {
.type = 1 << BCH_EXTENT_ENTRY_crc64,
- .compressed_size = compressed_size,
- .uncompressed_size = uncompressed_size,
+ ._compressed_size = compressed_size - 1,
+ ._uncompressed_size = uncompressed_size - 1,
+ .offset = 0,
+ .nonce = nonce,
+ .compression_type = compression_type,
+ .csum_type = csum_type,
+ .csum_lo = csum.lo,
+ .csum_hi = *((__le16 *) &csum.hi),
+ };
+ return;
+ }
+
+ if (bch_crc_bytes[csum_type] <= 16 &&
+ uncompressed_size <= CRC128_SIZE_MAX &&
+ nonce <= CRC128_NONCE_MAX) {
+ crc->crc128 = (struct bch_extent_crc128) {
+ .type = 1 << BCH_EXTENT_ENTRY_crc128,
+ ._compressed_size = compressed_size - 1,
+ ._uncompressed_size = uncompressed_size - 1,
.offset = 0,
+ .nonce = nonce,
.compression_type = compression_type,
.csum_type = csum_type,
.csum = csum,
};
+ return;
}
+
+ BUG();
}
void bch_extent_crc_append(struct bkey_i_extent *e,
unsigned compressed_size,
unsigned uncompressed_size,
unsigned compression_type,
- u64 csum, unsigned csum_type)
+ unsigned nonce,
+ struct bch_csum csum, unsigned csum_type)
{
union bch_extent_crc *crc;
- union bch_extent_crc new;
BUG_ON(compressed_size > uncompressed_size);
BUG_ON(uncompressed_size != e->k.size);
extent_for_each_crc(extent_i_to_s(e), crc)
;
- switch (extent_crc_type(crc)) {
- case BCH_EXTENT_CRC_NONE:
- if (!csum_type && !compression_type)
- return;
- break;
- case BCH_EXTENT_CRC32:
- case BCH_EXTENT_CRC64:
- if (crc_compressed_size(&e->k, crc) == compressed_size &&
- crc_uncompressed_size(&e->k, crc) == uncompressed_size &&
- crc_offset(crc) == 0 &&
- crc_compression_type(crc) == compression_type &&
- crc_csum_type(crc) == csum_type &&
- crc_csum(crc) == csum)
- return;
- break;
- }
+ if (!crc && !csum_type && !compression_type)
+ return;
+
+ if (crc &&
+ crc_compressed_size(&e->k, crc) == compressed_size &&
+ crc_uncompressed_size(&e->k, crc) == uncompressed_size &&
+ crc_offset(crc) == 0 &&
+ crc_nonce(crc) == nonce &&
+ crc_csum_type(crc) == csum_type &&
+ crc_compression_type(crc) == compression_type &&
+ crc_csum(crc).lo == csum.lo &&
+ crc_csum(crc).hi == csum.hi)
+ return;
- bch_extent_crc_init(&new,
+ bch_extent_crc_init((void *) extent_entry_last(extent_i_to_s(e)),
compressed_size,
uncompressed_size,
compression_type,
- csum, csum_type);
- bch_extent_entry_append(e, to_entry(&new));
-}
-
-static void __extent_sort_ptrs(struct cache_member_rcu *mi,
- struct bkey_s_extent src)
-{
- struct bch_extent_ptr *src_ptr, *dst_ptr;
- union bch_extent_crc *src_crc, *dst_crc;
- union bch_extent_crc _src;
- BKEY_PADDED(k) tmp;
- struct bkey_s_extent dst;
- size_t u64s, crc_u64s;
- u64 *p;
-
- /*
- * Insertion sort:
- *
- * Note: this sort needs to be stable, because pointer order determines
- * pointer dirtyness.
- */
-
- tmp.k.k = *src.k;
- dst = bkey_i_to_s_extent(&tmp.k);
- set_bkey_val_u64s(dst.k, 0);
-
- extent_for_each_ptr_crc(src, src_ptr, src_crc) {
- extent_for_each_ptr_crc(dst, dst_ptr, dst_crc)
- if (PTR_TIER(mi, src_ptr) < PTR_TIER(mi, dst_ptr))
- goto found;
-
- dst_ptr = &extent_entry_last(dst)->ptr;
- dst_crc = NULL;
-found:
- /* found insert position: */
-
- /*
- * we're making sure everything has a crc at this point, if
- * dst_ptr points to a pointer it better have a crc:
- */
- BUG_ON(dst_ptr != &extent_entry_last(dst)->ptr && !dst_crc);
- BUG_ON(dst_crc &&
- (extent_entry_next(to_entry(dst_crc)) !=
- to_entry(dst_ptr)));
-
- if (!src_crc) {
- bch_extent_crc_init(&_src, src.k->size,
- src.k->size, 0, 0, 0);
- src_crc = &_src;
- }
-
- p = dst_ptr != &extent_entry_last(dst)->ptr
- ? (void *) dst_crc
- : (void *) dst_ptr;
-
- crc_u64s = extent_entry_u64s(to_entry(src_crc));
- u64s = crc_u64s + sizeof(*dst_ptr) / sizeof(u64);
-
- memmove_u64s_up(p + u64s, p,
- (u64 *) extent_entry_last(dst) - (u64 *) p);
- set_bkey_val_u64s(dst.k, bkey_val_u64s(dst.k) + u64s);
-
- memcpy_u64s(p, src_crc, crc_u64s);
- memcpy_u64s(p + crc_u64s, src_ptr,
- sizeof(*src_ptr) / sizeof(u64));
- }
-
- /* Sort done - now drop redundant crc entries: */
- bch_extent_drop_redundant_crcs(dst);
-
- memcpy_u64s(src.v, dst.v, bkey_val_u64s(dst.k));
- set_bkey_val_u64s(src.k, bkey_val_u64s(dst.k));
-}
-
-static void extent_sort_ptrs(struct cache_set *c, struct bkey_s_extent e)
-{
- struct cache_member_rcu *mi;
- struct bch_extent_ptr *ptr, *prev = NULL;
- union bch_extent_crc *crc;
-
- /*
- * First check if any pointers are out of order before doing the actual
- * sort:
- */
- mi = cache_member_info_get(c);
-
- extent_for_each_ptr_crc(e, ptr, crc) {
- if (prev &&
- PTR_TIER(mi, ptr) < PTR_TIER(mi, prev)) {
- __extent_sort_ptrs(mi, e);
- break;
- }
- prev = ptr;
- }
-
- cache_member_info_put();
+ nonce, csum, csum_type);
+ __extent_entry_push(e);
}
/*
* For existing keys, only called when btree nodes are being rewritten, not when
* they're merely being compacted/resorted in memory.
*/
-static bool __bch_extent_normalize(struct cache_set *c, struct bkey_s k,
- bool sort)
+bool bch_extent_normalize(struct cache_set *c, struct bkey_s k)
{
struct bkey_s_extent e;
return true;
case KEY_TYPE_DISCARD:
- return !k.k->version;
+ return bversion_zero(k.k->version);
case BCH_EXTENT:
case BCH_EXTENT_CACHED:
bch_extent_drop_stale(c, e);
- if (sort)
- extent_sort_ptrs(c, e);
-
if (!bkey_val_u64s(e.k)) {
if (bkey_extent_is_cached(e.k)) {
k.k->type = KEY_TYPE_DISCARD;
- if (!k.k->version)
+ if (bversion_zero(k.k->version))
return true;
} else {
k.k->type = KEY_TYPE_ERROR;
}
}
-bool bch_extent_normalize(struct cache_set *c, struct bkey_s k)
+void bch_extent_mark_replicas_cached(struct cache_set *c,
+ struct bkey_s_extent e,
+ unsigned nr_cached)
{
- return __bch_extent_normalize(c, k, true);
+ struct bch_extent_ptr *ptr;
+ struct cache_member_rcu *mi;
+ bool have_higher_tier;
+ unsigned tier = 0;
+
+ if (!nr_cached)
+ return;
+
+ mi = cache_member_info_get(c);
+
+ do {
+ have_higher_tier = false;
+
+ extent_for_each_ptr(e, ptr) {
+ if (!ptr->cached &&
+ PTR_TIER(mi, ptr) == tier) {
+ ptr->cached = true;
+ nr_cached--;
+ if (!nr_cached)
+ goto out;
+ }
+
+ if (PTR_TIER(mi, ptr) > tier)
+ have_higher_tier = true;
+ }
+
+ tier++;
+ } while (have_higher_tier);
+out:
+ cache_member_info_put();
}
/*
extent_for_each_online_device_crc(c, e, crc, ptr, ca)
if (!ptr_stale(ca, ptr)) {
*ret = (struct extent_pick_ptr) {
- .crc = crc_to_64(e.k, crc),
+ .crc = crc_to_128(e.k, crc),
.ptr = *ptr,
.ca = ca,
};
if (l->k.u64s != r->k.u64s ||
l->k.type != r->k.type ||
- l->k.version != r->k.version ||
+ bversion_cmp(l->k.version, r->k.version) ||
bkey_cmp(l->k.p, bkey_start_pos(&r->k)))
return BCH_MERGE_NOMERGE;
case KEY_TYPE_DELETED:
case KEY_TYPE_DISCARD:
case KEY_TYPE_ERROR:
- case BCH_RESERVATION:
/* These types are mergeable, and no val to check */
break;
struct bch_extent_ptr *lp, *rp;
struct cache_member_cpu *m;
- en_r = bkey_idx(er.v, (u64 *) en_l - el.v->_data);
+ en_r = vstruct_idx(er.v, (u64 *) en_l - el.v->_data);
if ((extent_entry_type(en_l) !=
extent_entry_type(en_r)) ||
}
break;
+ case BCH_RESERVATION: {
+ struct bkey_i_reservation *li = bkey_i_to_reservation(l);
+ struct bkey_i_reservation *ri = bkey_i_to_reservation(r);
+
+ if (li->v.generation != ri->v.generation ||
+ li->v.nr_replicas != ri->v.nr_replicas)
+ return BCH_MERGE_NOMERGE;
+ break;
+ }
default:
return BCH_MERGE_NOMERGE;
}
struct journal_res;
struct extent_pick_ptr {
- struct bch_extent_crc64 crc;
+ struct bch_extent_crc128 crc;
struct bch_extent_ptr ptr;
struct cache *ca;
};
struct btree_insert_entry *);
bool bch_extent_normalize(struct cache_set *, struct bkey_s);
+void bch_extent_mark_replicas_cached(struct cache_set *,
+ struct bkey_s_extent, unsigned);
-unsigned bch_extent_nr_ptrs_from(struct bkey_s_c_extent,
- const struct bch_extent_ptr *);
unsigned bch_extent_nr_ptrs(struct bkey_s_c_extent);
+unsigned bch_extent_nr_dirty_ptrs(struct bkey_s_c);
static inline bool bkey_extent_is_data(const struct bkey *k)
{
return sizeof(struct bch_extent_crc32);
case BCH_EXTENT_ENTRY_crc64:
return sizeof(struct bch_extent_crc64);
+ case BCH_EXTENT_ENTRY_crc128:
+ return sizeof(struct bch_extent_crc128);
case BCH_EXTENT_ENTRY_ptr:
return sizeof(struct bch_extent_ptr);
default:
u8 type;
struct bch_extent_crc32 crc32;
struct bch_extent_crc64 crc64;
+ struct bch_extent_crc128 crc128;
};
/* downcast, preserves const */
BCH_EXTENT_CRC_NONE,
BCH_EXTENT_CRC32,
BCH_EXTENT_CRC64,
+ BCH_EXTENT_CRC128,
};
static inline enum bch_extent_crc_type
-extent_crc_type(const union bch_extent_crc *crc)
+__extent_crc_type(const union bch_extent_crc *crc)
{
if (!crc)
return BCH_EXTENT_CRC_NONE;
return BCH_EXTENT_CRC32;
case BCH_EXTENT_ENTRY_crc64:
return BCH_EXTENT_CRC64;
+ case BCH_EXTENT_ENTRY_crc128:
+ return BCH_EXTENT_CRC128;
default:
BUG();
}
}
+#define extent_crc_type(_crc) \
+({ \
+ BUILD_BUG_ON(!type_is(_crc, struct bch_extent_crc32 *) && \
+ !type_is(_crc, struct bch_extent_crc64 *) && \
+ !type_is(_crc, struct bch_extent_crc128 *) && \
+ !type_is(_crc, union bch_extent_crc *)); \
+ \
+ type_is(_crc, struct bch_extent_crc32 *) ? BCH_EXTENT_CRC32 \
+ : type_is(_crc, struct bch_extent_crc64 *) ? BCH_EXTENT_CRC64 \
+ : type_is(_crc, struct bch_extent_crc128 *) ? BCH_EXTENT_CRC128 \
+ : __extent_crc_type((union bch_extent_crc *) _crc); \
+})
+
#define extent_entry_next(_entry) \
((typeof(_entry)) ((void *) (_entry) + extent_entry_bytes(_entry)))
#define extent_entry_last(_e) \
- bkey_idx((_e).v, bkey_val_u64s((_e).k))
+ vstruct_idx((_e).v, bkey_val_u64s((_e).k))
/* Iterate over all entries: */
#define extent_ptr_next(_e, _ptr) \
extent_ptr_next_filter(_e, _ptr, true)
-#define extent_for_each_ptr_from_filter(_e, _ptr, _start, _filter) \
- for ((_ptr) = (_start); \
+#define extent_for_each_ptr_filter(_e, _ptr, _filter) \
+ for ((_ptr) = &(_e).v->start->ptr; \
((_ptr) = extent_ptr_next_filter(_e, _ptr, _filter)); \
(_ptr)++)
-#define extent_for_each_ptr_from(_e, _ptr, _start) \
- extent_for_each_ptr_from_filter(_e, _ptr, _start, true)
-
#define extent_for_each_ptr(_e, _ptr) \
- extent_for_each_ptr_from_filter(_e, _ptr, &(_e).v->start->ptr, true)
+ extent_for_each_ptr_filter(_e, _ptr, true)
#define extent_for_each_online_device(_c, _e, _ptr, _ca) \
- extent_for_each_ptr_from_filter(_e, _ptr, &(_e).v->start->ptr, \
- ((_ca) = PTR_CACHE(_c, _ptr)))
+ extent_for_each_ptr_filter(_e, _ptr, ((_ca) = PTR_CACHE(_c, _ptr)))
#define extent_ptr_prev(_e, _ptr) \
({ \
(_ptr); \
(_ptr) = extent_ptr_prev(_e, _ptr))
-void bch_extent_entry_append(struct bkey_i_extent *, union bch_extent_entry *);
void bch_extent_crc_append(struct bkey_i_extent *, unsigned, unsigned,
- unsigned, u64, unsigned);
+ unsigned, unsigned, struct bch_csum, unsigned);
+
+static inline void __extent_entry_push(struct bkey_i_extent *e)
+{
+ union bch_extent_entry *entry = extent_entry_last(extent_i_to_s(e));
+
+ EBUG_ON(bkey_val_u64s(&e->k) + extent_entry_u64s(entry) >
+ BKEY_EXTENT_VAL_U64s_MAX);
+
+ e->k.u64s += extent_entry_u64s(entry);
+}
static inline void extent_ptr_append(struct bkey_i_extent *e,
struct bch_extent_ptr ptr)
{
ptr.type = 1 << BCH_EXTENT_ENTRY_ptr;
- bch_extent_entry_append(e, to_entry(&ptr));
+ extent_entry_last(extent_i_to_s(e))->ptr = ptr;
+ __extent_entry_push(e);
}
-/* XXX: inefficient */
-static inline bool bch_extent_ptr_is_dirty(const struct cache_set *c,
- struct bkey_s_c_extent e,
- const struct bch_extent_ptr *ptr)
+static inline struct bch_extent_crc128 crc_to_128(const struct bkey *k,
+ const union bch_extent_crc *crc)
{
- if (bkey_extent_is_cached(e.k))
- return false;
-
- /* Dirty pointers come last */
- return bch_extent_nr_ptrs_from(e, ptr) <= c->opts.data_replicas;
-}
-
-extern const unsigned bch_crc_size[];
+ EBUG_ON(!k->size);
-static inline struct bch_extent_crc64 crc_to_64(const struct bkey *k,
- const union bch_extent_crc *crc)
-{
switch (extent_crc_type(crc)) {
case BCH_EXTENT_CRC_NONE:
- return (struct bch_extent_crc64) {
- .compressed_size = k->size,
- .uncompressed_size = k->size,
+ return (struct bch_extent_crc128) {
+ ._compressed_size = k->size - 1,
+ ._uncompressed_size = k->size - 1,
};
case BCH_EXTENT_CRC32:
- return (struct bch_extent_crc64) {
- .compressed_size = crc->crc32.compressed_size,
- .uncompressed_size = crc->crc32.uncompressed_size,
+ return (struct bch_extent_crc128) {
+ .type = 1 << BCH_EXTENT_ENTRY_crc128,
+ ._compressed_size = crc->crc32._compressed_size,
+ ._uncompressed_size = crc->crc32._uncompressed_size,
.offset = crc->crc32.offset,
.csum_type = crc->crc32.csum_type,
.compression_type = crc->crc32.compression_type,
- .csum = crc->crc32.csum,
+ .csum.lo = crc->crc32.csum,
};
case BCH_EXTENT_CRC64:
- return crc->crc64;
+ return (struct bch_extent_crc128) {
+ .type = 1 << BCH_EXTENT_ENTRY_crc128,
+ ._compressed_size = crc->crc64._compressed_size,
+ ._uncompressed_size = crc->crc64._uncompressed_size,
+ .offset = crc->crc64.offset,
+ .nonce = crc->crc64.nonce,
+ .csum_type = crc->crc64.csum_type,
+ .compression_type = crc->crc64.compression_type,
+ .csum.lo = crc->crc64.csum_lo,
+ .csum.hi = crc->crc64.csum_hi,
+ };
+ case BCH_EXTENT_CRC128:
+ return crc->crc128;
default:
BUG();
}
}
-static inline unsigned crc_compressed_size(const struct bkey *k,
- const union bch_extent_crc *crc)
-{
- return crc_to_64(k, crc).compressed_size;
-}
+#define crc_compressed_size(_k, _crc) \
+({ \
+ unsigned _size = 0; \
+ \
+ switch (extent_crc_type(_crc)) { \
+ case BCH_EXTENT_CRC_NONE: \
+ _size = ((const struct bkey *) (_k))->size; \
+ break; \
+ case BCH_EXTENT_CRC32: \
+ _size = ((struct bch_extent_crc32 *) _crc) \
+ ->_compressed_size + 1; \
+ break; \
+ case BCH_EXTENT_CRC64: \
+ _size = ((struct bch_extent_crc64 *) _crc) \
+ ->_compressed_size + 1; \
+ break; \
+ case BCH_EXTENT_CRC128: \
+ _size = ((struct bch_extent_crc128 *) _crc) \
+ ->_compressed_size + 1; \
+ break; \
+ } \
+ _size; \
+})
-static inline unsigned crc_uncompressed_size(const struct bkey *k,
- const union bch_extent_crc *crc)
-{
- return crc_to_64(k, crc).uncompressed_size;
-}
+#define crc_uncompressed_size(_k, _crc) \
+({ \
+ unsigned _size = 0; \
+ \
+ switch (extent_crc_type(_crc)) { \
+ case BCH_EXTENT_CRC_NONE: \
+ _size = ((const struct bkey *) (_k))->size; \
+ break; \
+ case BCH_EXTENT_CRC32: \
+ _size = ((struct bch_extent_crc32 *) _crc) \
+ ->_uncompressed_size + 1; \
+ break; \
+ case BCH_EXTENT_CRC64: \
+ _size = ((struct bch_extent_crc64 *) _crc) \
+ ->_uncompressed_size + 1; \
+ break; \
+ case BCH_EXTENT_CRC128: \
+ _size = ((struct bch_extent_crc128 *) _crc) \
+ ->_uncompressed_size + 1; \
+ break; \
+ } \
+ _size; \
+})
static inline unsigned crc_offset(const union bch_extent_crc *crc)
{
return crc->crc32.offset;
case BCH_EXTENT_CRC64:
return crc->crc64.offset;
+ case BCH_EXTENT_CRC128:
+ return crc->crc128.offset;
+ default:
+ BUG();
+ }
+}
+
+static inline unsigned crc_nonce(const union bch_extent_crc *crc)
+{
+ switch (extent_crc_type(crc)) {
+ case BCH_EXTENT_CRC_NONE:
+ case BCH_EXTENT_CRC32:
+ return 0;
+ case BCH_EXTENT_CRC64:
+ return crc->crc64.nonce;
+ case BCH_EXTENT_CRC128:
+ return crc->crc128.nonce;
default:
BUG();
}
return crc->crc32.csum_type;
case BCH_EXTENT_CRC64:
return crc->crc64.csum_type;
+ case BCH_EXTENT_CRC128:
+ return crc->crc128.csum_type;
default:
BUG();
}
return crc->crc32.compression_type;
case BCH_EXTENT_CRC64:
return crc->crc64.compression_type;
+ case BCH_EXTENT_CRC128:
+ return crc->crc128.compression_type;
default:
BUG();
}
}
-static inline u64 crc_csum(const union bch_extent_crc *crc)
+static inline struct bch_csum crc_csum(const union bch_extent_crc *crc)
{
switch (extent_crc_type(crc)) {
case BCH_EXTENT_CRC_NONE:
- return 0;
+ return (struct bch_csum) { 0 };
case BCH_EXTENT_CRC32:
- return crc->crc32.csum;
+ return (struct bch_csum) { .lo = crc->crc32.csum };
case BCH_EXTENT_CRC64:
- return crc->crc64.csum;
+ return (struct bch_csum) {
+ .lo = crc->crc64.csum_lo,
+ .hi = crc->crc64.csum_hi,
+ };
+ case BCH_EXTENT_CRC128:
+ return crc->crc128.csum;
default:
BUG();
}
}
-static inline unsigned bkey_extent_is_compressed(struct cache_set *c,
- struct bkey_s_c k)
+static inline unsigned bkey_extent_is_compressed(struct bkey_s_c k)
{
struct bkey_s_c_extent e;
const struct bch_extent_ptr *ptr;
e = bkey_s_c_to_extent(k);
extent_for_each_ptr_crc(e, ptr, crc)
- if (bch_extent_ptr_is_dirty(c, e, ptr) &&
+ if (!ptr->cached &&
crc_compression_type(crc) != BCH_COMPRESSION_NONE &&
crc_compressed_size(e.k, crc) < k.k->size)
ret = max_t(unsigned, ret,
return ret;
}
+static inline unsigned extent_current_nonce(struct bkey_s_c_extent e)
+{
+ const union bch_extent_crc *crc;
+
+ extent_for_each_crc(e, crc)
+ if (bch_csum_type_is_encryption(crc_csum_type(crc)))
+ return crc_offset(crc) + crc_nonce(crc);
+
+ return 0;
+}
+
void bch_extent_narrow_crcs(struct bkey_s_extent);
void bch_extent_drop_redundant_crcs(struct bkey_s_extent);
struct bkey_s_c_dirent dirent)
{
struct qstr name;
- struct bkey_i_inode dir_inode;
+ struct bch_inode_unpacked dir_inode;
struct bch_hash_info dir_hash_info;
u64 dir_inum = dirent.k->p.inode;
int ret;
if (ret)
goto err;
- dir_hash_info = bch_hash_info_init(&dir_inode.v);
+ dir_hash_info = bch_hash_info_init(&dir_inode);
ret = bch_dirent_delete(c, dir_inum, &dir_hash_info, &name, NULL);
err:
}
static int reattach_inode(struct cache_set *c,
- struct bkey_i_inode *lostfound_inode,
+ struct bch_inode_unpacked *lostfound_inode,
u64 inum)
{
struct bch_hash_info lostfound_hash_info =
- bch_hash_info_init(&lostfound_inode->v);
+ bch_hash_info_init(lostfound_inode);
+ struct bkey_inode_buf packed;
char name_buf[20];
struct qstr name;
int ret;
snprintf(name_buf, sizeof(name_buf), "%llu", inum);
name = (struct qstr) QSTR(name_buf);
- le32_add_cpu(&lostfound_inode->v.i_nlink, 1);
+ lostfound_inode->i_nlink++;
- ret = bch_btree_insert(c, BTREE_ID_INODES, &lostfound_inode->k_i,
+ bch_inode_pack(&packed, lostfound_inode);
+
+ ret = bch_btree_insert(c, BTREE_ID_INODES, &packed.inode.k_i,
NULL, NULL, NULL, 0);
if (ret)
return ret;
- return bch_dirent_create(c, lostfound_inode->k.p.inode,
+ return bch_dirent_create(c, lostfound_inode->inum,
&lostfound_hash_info,
DT_DIR, &name, inum, NULL, 0);
}
struct inode_walker {
bool first_this_inode;
bool have_inode;
- u16 i_mode;
- u64 i_size;
u64 cur_inum;
- struct bkey_i_inode inode;
+ struct bch_inode_unpacked inode;
};
static struct inode_walker inode_walker_init(void)
return ret;
w->have_inode = !ret;
-
- if (w->have_inode) {
- w->i_mode = le16_to_cpu(w->inode.v.i_mode);
- w->i_size = le64_to_cpu(w->inode.v.i_size);
- }
}
return 0;
k.k->type, k.k->p.inode);
unfixable_fsck_err_on(w.first_this_inode && w.have_inode &&
- le64_to_cpu(w.inode.v.i_sectors) !=
+ w.inode.i_sectors !=
(i_sectors = bch_count_inode_sectors(c, w.cur_inum)),
c, "i_sectors wrong: got %llu, should be %llu",
- le64_to_cpu(w.inode.v.i_sectors), i_sectors);
+ w.inode.i_sectors, i_sectors);
unfixable_fsck_err_on(w.have_inode &&
- !S_ISREG(w.i_mode) && !S_ISLNK(w.i_mode), c,
+ !S_ISREG(w.inode.i_mode) && !S_ISLNK(w.inode.i_mode), c,
"extent type %u for non regular file, inode %llu mode %o",
- k.k->type, k.k->p.inode, w.i_mode);
+ k.k->type, k.k->p.inode, w.inode.i_mode);
unfixable_fsck_err_on(k.k->type != BCH_RESERVATION &&
- k.k->p.offset > round_up(w.i_size, PAGE_SIZE) >> 9, c,
+ k.k->p.offset > round_up(w.inode.i_size, PAGE_SIZE) >> 9, c,
"extent type %u offset %llu past end of inode %llu, i_size %llu",
- k.k->type, k.k->p.offset, k.k->p.inode, w.i_size);
+ k.k->type, k.k->p.offset, k.k->p.inode, w.inode.i_size);
}
fsck_err:
return bch_btree_iter_unlock(&iter) ?: ret;
for_each_btree_key(&iter, c, BTREE_ID_DIRENTS,
POS(BCACHE_ROOT_INO, 0), k) {
struct bkey_s_c_dirent d;
- struct bkey_i_inode target;
+ struct bch_inode_unpacked target;
bool have_target;
u64 d_inum;
"dirent in nonexisting directory %llu",
k.k->p.inode);
- unfixable_fsck_err_on(!S_ISDIR(w.i_mode), c,
+ unfixable_fsck_err_on(!S_ISDIR(w.inode.i_mode), c,
"dirent in non directory inode %llu, type %u",
- k.k->p.inode, mode_to_type(w.i_mode));
+ k.k->p.inode, mode_to_type(w.inode.i_mode));
if (k.k->type != BCH_DIRENT)
continue;
if (fsck_err_on(have_target &&
d.v->d_type !=
- mode_to_type(le16_to_cpu(target.v.i_mode)), c,
+ mode_to_type(le16_to_cpu(target.i_mode)), c,
"incorrect d_type: got %u should be %u, filename %s",
d.v->d_type,
- mode_to_type(le16_to_cpu(target.v.i_mode)),
+ mode_to_type(le16_to_cpu(target.i_mode)),
d.v->d_name)) {
struct bkey_i_dirent *n;
}
bkey_reassemble(&n->k_i, d.s_c);
- n->v.d_type = mode_to_type(le16_to_cpu(target.v.i_mode));
+ n->v.d_type = mode_to_type(le16_to_cpu(target.i_mode));
ret = bch_btree_insert_at(c, NULL, NULL, NULL,
BTREE_INSERT_NOFAIL,
}
/* Get root directory, create if it doesn't exist: */
-static int check_root(struct cache_set *c, struct bkey_i_inode *root_inode)
+static int check_root(struct cache_set *c, struct bch_inode_unpacked *root_inode)
{
+ struct bkey_inode_buf packed;
int ret;
ret = bch_inode_find_by_inum(c, BCACHE_ROOT_INO, root_inode);
if (fsck_err_on(ret, c, "root directory missing"))
goto create_root;
- if (fsck_err_on(!S_ISDIR(le16_to_cpu(root_inode->v.i_mode)), c,
+ if (fsck_err_on(!S_ISDIR(root_inode->i_mode), c,
"root inode not a directory"))
goto create_root;
return ret;
create_root:
bch_inode_init(c, root_inode, 0, 0, S_IFDIR|S_IRWXU|S_IRUGO|S_IXUGO, 0);
- root_inode->k.p.inode = BCACHE_ROOT_INO;
+ root_inode->inum = BCACHE_ROOT_INO;
+
+ bch_inode_pack(&packed, root_inode);
- return bch_btree_insert(c, BTREE_ID_INODES, &root_inode->k_i,
+ return bch_btree_insert(c, BTREE_ID_INODES, &packed.inode.k_i,
NULL, NULL, NULL, 0);
}
/* Get lost+found, create if it doesn't exist: */
static int check_lostfound(struct cache_set *c,
- struct bkey_i_inode *root_inode,
- struct bkey_i_inode *lostfound_inode)
+ struct bch_inode_unpacked *root_inode,
+ struct bch_inode_unpacked *lostfound_inode)
{
struct qstr lostfound = QSTR("lost+found");
- struct bch_hash_info root_hash_info = bch_hash_info_init(&root_inode->v);
+ struct bch_hash_info root_hash_info =
+ bch_hash_info_init(root_inode);
+ struct bkey_inode_buf packed;
u64 inum;
int ret;
if (fsck_err_on(ret, c, "lost+found missing"))
goto create_lostfound;
- if (fsck_err_on(!S_ISDIR(le16_to_cpu(lostfound_inode->v.i_mode)), c,
+ if (fsck_err_on(!S_ISDIR(lostfound_inode->i_mode), c,
"lost+found inode not a directory"))
goto create_lostfound;
fsck_err:
return ret;
create_lostfound:
- le32_add_cpu(&root_inode->v.i_nlink, 1);
+ root_inode->i_nlink++;
- ret = bch_btree_insert(c, BTREE_ID_INODES, &root_inode->k_i,
+ bch_inode_pack(&packed, root_inode);
+
+ ret = bch_btree_insert(c, BTREE_ID_INODES, &packed.inode.k_i,
NULL, NULL, NULL, 0);
if (ret)
return ret;
bch_inode_init(c, lostfound_inode, 0, 0, S_IFDIR|S_IRWXU|S_IRUGO|S_IXUGO, 0);
+ bch_inode_pack(&packed, lostfound_inode);
- ret = bch_inode_create(c, &lostfound_inode->k_i, BLOCKDEV_INODE_MAX, 0,
+ ret = bch_inode_create(c, &packed.inode.k_i, BLOCKDEV_INODE_MAX, 0,
&c->unused_inode_hint);
if (ret)
return ret;
+ lostfound_inode->inum = packed.inode.k.p.inode;
+
ret = bch_dirent_create(c, BCACHE_ROOT_INO, &root_hash_info, DT_DIR,
- &lostfound, lostfound_inode->k.p.inode, NULL, 0);
+ &lostfound, lostfound_inode->inum, NULL, 0);
if (ret)
return ret;
noinline_for_stack
static int check_directory_structure(struct cache_set *c,
- struct bkey_i_inode *lostfound_inode)
+ struct bch_inode_unpacked *lostfound_inode)
{
struct inode_bitmap dirs_done = { NULL, 0 };
struct pathbuf path = { 0, 0, NULL };
}
static int bch_gc_do_inode(struct cache_set *c,
- struct bkey_i_inode *lostfound_inode,
+ struct bch_inode_unpacked *lostfound_inode,
struct btree_iter *iter,
struct bkey_s_c_inode inode, struct nlink link)
{
- u16 i_mode = le16_to_cpu(inode.v->i_mode);
- u32 i_flags = le32_to_cpu(inode.v->i_flags);
- u32 i_nlink = le32_to_cpu(inode.v->i_nlink);
- u64 i_size = le64_to_cpu(inode.v->i_size);
- s64 i_sectors = 0;
+ struct bch_inode_unpacked u;
int ret = 0;
- u32 real_i_nlink;
+ u32 i_nlink, real_i_nlink;
+ bool do_update = false;
+
+ ret = bch_inode_unpack(inode, &u);
+ if (cache_set_inconsistent_on(ret, c,
+ "error unpacking inode %llu in fs-gc",
+ inode.k->p.inode))
+ return ret;
+
+ i_nlink = u.i_nlink + nlink_bias(u.i_mode);
fsck_err_on(i_nlink < link.count, c,
"inode %llu i_link too small (%u < %u, type %i)",
inode.k->p.inode, i_nlink,
- link.count, mode_to_type(i_mode));
+ link.count, mode_to_type(u.i_mode));
/* These should have been caught/fixed by earlier passes: */
- if (S_ISDIR(i_mode)) {
+ if (S_ISDIR(u.i_mode)) {
need_fsck_err_on(link.count > 1, c,
"directory %llu with multiple hardlinks: %u",
inode.k->p.inode, link.count);
"but found orphaned inode %llu",
inode.k->p.inode);
- if (fsck_err_on(S_ISDIR(i_mode) &&
+ if (fsck_err_on(S_ISDIR(u.i_mode) &&
bch_empty_dir(c, inode.k->p.inode), c,
"non empty directory with link count 0, "
"inode nlink %u, dir links found %u",
return ret;
}
- if (i_flags & BCH_INODE_I_SIZE_DIRTY) {
+ if (u.i_flags & BCH_INODE_I_SIZE_DIRTY) {
fsck_err_on(c->sb.clean, c,
"filesystem marked clean, "
"but inode %llu has i_size dirty",
*/
ret = bch_inode_truncate(c, inode.k->p.inode,
- round_up(i_size, PAGE_SIZE) >> 9,
+ round_up(u.i_size, PAGE_SIZE) >> 9,
NULL, NULL);
if (ret) {
bch_err(c, "error in fs gc: error %i "
* We truncated without our normal sector accounting hook, just
* make sure we recalculate it:
*/
- i_flags |= BCH_INODE_I_SECTORS_DIRTY;
+ u.i_flags |= BCH_INODE_I_SECTORS_DIRTY;
+
+ u.i_flags &= ~BCH_INODE_I_SIZE_DIRTY;
+ do_update = true;
}
- if (i_flags & BCH_INODE_I_SECTORS_DIRTY) {
+ if (u.i_flags & BCH_INODE_I_SECTORS_DIRTY) {
+ s64 sectors;
+
fsck_err_on(c->sb.clean, c,
"filesystem marked clean, "
"but inode %llu has i_sectors dirty",
bch_verbose(c, "recounting sectors for inode %llu",
inode.k->p.inode);
- i_sectors = bch_count_inode_sectors(c, inode.k->p.inode);
- if (i_sectors < 0) {
+ sectors = bch_count_inode_sectors(c, inode.k->p.inode);
+ if (sectors < 0) {
bch_err(c, "error in fs gc: error %i "
"recounting inode sectors",
- (int) i_sectors);
- return i_sectors;
+ (int) sectors);
+ return sectors;
}
+
+ u.i_sectors = sectors;
+ u.i_flags &= ~BCH_INODE_I_SECTORS_DIRTY;
+ do_update = true;
}
if (i_nlink != real_i_nlink) {
"filesystem marked clean, "
"but inode %llu has wrong i_nlink "
"(type %u i_nlink %u, should be %u)",
- inode.k->p.inode, mode_to_type(i_mode),
+ inode.k->p.inode, mode_to_type(u.i_mode),
i_nlink, real_i_nlink);
bch_verbose(c, "setting inode %llu nlinks from %u to %u",
inode.k->p.inode, i_nlink, real_i_nlink);
+ u.i_nlink = real_i_nlink - nlink_bias(u.i_mode);;
+ do_update = true;
}
- if (i_nlink != real_i_nlink||
- i_flags & BCH_INODE_I_SECTORS_DIRTY ||
- i_flags & BCH_INODE_I_SIZE_DIRTY) {
- struct bkey_i_inode update;
-
- bkey_reassemble(&update.k_i, inode.s_c);
- update.v.i_nlink = cpu_to_le32(real_i_nlink);
- update.v.i_flags = cpu_to_le32(i_flags &
- ~(BCH_INODE_I_SIZE_DIRTY|
- BCH_INODE_I_SECTORS_DIRTY));
+ if (do_update) {
+ struct bkey_inode_buf p;
- if (i_flags & BCH_INODE_I_SECTORS_DIRTY)
- update.v.i_sectors = cpu_to_le64(i_sectors);
+ bch_inode_pack(&p, &u);
ret = bch_btree_insert_at(c, NULL, NULL, NULL,
BTREE_INSERT_NOFAIL,
- BTREE_INSERT_ENTRY(iter, &update.k_i));
+ BTREE_INSERT_ENTRY(iter, &p.inode.k_i));
if (ret && ret != -EINTR)
bch_err(c, "error in fs gc: error %i "
"updating inode", ret);
noinline_for_stack
static int bch_gc_walk_inodes(struct cache_set *c,
- struct bkey_i_inode *lostfound_inode,
+ struct bch_inode_unpacked *lostfound_inode,
struct nlinks *links,
u64 range_start, u64 range_end)
{
noinline_for_stack
static int check_inode_nlinks(struct cache_set *c,
- struct bkey_i_inode *lostfound_inode)
+ struct bch_inode_unpacked *lostfound_inode)
{
struct nlinks links;
u64 this_iter_range_start, next_iter_range_start = 0;
*/
int bch_fsck(struct cache_set *c, bool full_fsck)
{
- struct bkey_i_inode root_inode, lostfound_inode;
+ struct bch_inode_unpacked root_inode, lostfound_inode;
int ret;
ret = check_root(c, &root_inode);
/* i_size updates: */
-static int inode_set_size(struct bch_inode_info *ei, struct bch_inode *bi,
+static int inode_set_size(struct bch_inode_info *ei,
+ struct bch_inode_unpacked *bi,
void *p)
{
loff_t *new_i_size = p;
- unsigned i_flags = le32_to_cpu(bi->i_flags);
lockdep_assert_held(&ei->update_lock);
- bi->i_size = cpu_to_le64(*new_i_size);
+ bi->i_size = *new_i_size;
if (atomic_long_read(&ei->i_size_dirty_count))
- i_flags |= BCH_INODE_I_SIZE_DIRTY;
+ bi->i_flags |= BCH_INODE_I_SIZE_DIRTY;
else
- i_flags &= ~BCH_INODE_I_SIZE_DIRTY;
-
- bi->i_flags = cpu_to_le32(i_flags);
+ bi->i_flags &= ~BCH_INODE_I_SIZE_DIRTY;
return 0;
}
}
static int inode_set_i_sectors_dirty(struct bch_inode_info *ei,
- struct bch_inode *bi, void *p)
+ struct bch_inode_unpacked *bi, void *p)
{
- BUG_ON(le32_to_cpu(bi->i_flags) & BCH_INODE_I_SECTORS_DIRTY);
+ BUG_ON(bi->i_flags & BCH_INODE_I_SECTORS_DIRTY);
- bi->i_flags = cpu_to_le32(le32_to_cpu(bi->i_flags)|
- BCH_INODE_I_SECTORS_DIRTY);
+ bi->i_flags |= BCH_INODE_I_SECTORS_DIRTY;
return 0;
}
static int inode_clear_i_sectors_dirty(struct bch_inode_info *ei,
- struct bch_inode *bi, void *p)
+ struct bch_inode_unpacked *bi,
+ void *p)
{
- BUG_ON(!(le32_to_cpu(bi->i_flags) & BCH_INODE_I_SECTORS_DIRTY));
+ BUG_ON(!(bi->i_flags & BCH_INODE_I_SECTORS_DIRTY));
- bi->i_sectors = cpu_to_le64(atomic64_read(&ei->i_sectors));
- bi->i_flags = cpu_to_le32(le32_to_cpu(bi->i_flags) &
- ~BCH_INODE_I_SECTORS_DIRTY);
+ bi->i_sectors = atomic64_read(&ei->i_sectors);
+ bi->i_flags &= ~BCH_INODE_I_SECTORS_DIRTY;
return 0;
}
struct bchfs_extent_trans_hook {
struct bchfs_write_op *op;
struct extent_insert_hook hook;
- struct bkey_i_inode new_inode;
+
+ struct bch_inode_unpacked inode_u;
+ struct bkey_inode_buf inode_p;
+
bool need_inode_update;
};
(k.k && bkey_extent_is_allocation(k.k));
s64 sectors = (s64) (next_pos.offset - committed_pos.offset) * sign;
u64 offset = min(next_pos.offset << 9, h->op->new_i_size);
+ bool do_pack = false;
BUG_ON((next_pos.offset << 9) > round_up(offset, PAGE_SIZE));
return BTREE_HOOK_RESTART_TRANS;
}
- h->new_inode.v.i_size = cpu_to_le64(offset);
+ h->inode_u.i_size = offset;
+ do_pack = true;
+
ei->i_size = offset;
if (h->op->is_dio)
return BTREE_HOOK_RESTART_TRANS;
}
- le64_add_cpu(&h->new_inode.v.i_sectors, sectors);
+ h->inode_u.i_sectors += sectors;
+ do_pack = true;
+
atomic64_add(sectors, &ei->i_sectors);
h->op->sectors_added += sectors;
}
}
+ if (do_pack)
+ bch_inode_pack(&h->inode_p, &h->inode_u);
+
return BTREE_HOOK_DO_INSERT;
}
break;
}
- bkey_reassemble(&hook.new_inode.k_i, inode);
+ if (WARN_ONCE(bkey_bytes(inode.k) >
+ sizeof(hook.inode_p),
+ "inode %llu too big (%zu bytes, buf %zu)",
+ extent_iter.pos.inode,
+ bkey_bytes(inode.k),
+ sizeof(hook.inode_p))) {
+ ret = -ENOENT;
+ break;
+ }
+
+ bkey_reassemble(&hook.inode_p.inode.k_i, inode);
+ ret = bch_inode_unpack(bkey_s_c_to_inode(inode),
+ &hook.inode_u);
+ if (WARN_ONCE(ret,
+ "error %i unpacking inode %llu",
+ ret, extent_iter.pos.inode)) {
+ ret = -ENOENT;
+ break;
+ }
ret = bch_btree_insert_at(wop->c, &wop->res,
&hook.hook, op_journal_seq(wop),
BTREE_INSERT_NOFAIL|BTREE_INSERT_ATOMIC,
BTREE_INSERT_ENTRY(&extent_iter, k),
- BTREE_INSERT_ENTRY(&inode_iter, &hook.new_inode.k_i));
+ BTREE_INSERT_ENTRY_EXTRA_RES(&inode_iter,
+ &hook.inode_p.inode.k_i, 2));
} else {
ret = bch_btree_insert_at(wop->c, &wop->res,
&hook.hook, op_journal_seq(wop),
struct bch_page_state {
union { struct {
/*
- * BCH_PAGE_ALLOCATED: page is _fully_ written on disk, and not
- * compressed - which means to write this page we don't have to reserve
- * space (the new write will never take up more space on disk than what
- * it's overwriting)
- *
- * BCH_PAGE_UNALLOCATED: page is not fully written on disk, or is
- * compressed - before writing we have to reserve space with
- * bch_reserve_sectors()
- *
- * BCH_PAGE_RESERVED: page has space reserved on disk (reservation will
- * be consumed when the page is written).
+ * page is _fully_ written on disk, and not compressed - which means to
+ * write this page we don't have to reserve space (the new write will
+ * never take up more space on disk than what it's overwriting)
*/
- enum {
- BCH_PAGE_UNALLOCATED = 0,
- BCH_PAGE_ALLOCATED,
- } alloc_state:2;
+ unsigned allocated:1;
/* Owns PAGE_SECTORS sized reservation: */
unsigned reserved:1;
+ unsigned nr_replicas:4;
/*
* Number of sectors on disk - for i_blocks
struct disk_reservation res;
int ret = 0;
- BUG_ON(s->alloc_state == BCH_PAGE_ALLOCATED &&
- s->sectors != PAGE_SECTORS);
+ BUG_ON(s->allocated && s->sectors != PAGE_SECTORS);
- if (s->reserved ||
- s->alloc_state == BCH_PAGE_ALLOCATED)
+ if (s->allocated || s->reserved)
return 0;
ret = bch_disk_reservation_get(c, &res, PAGE_SECTORS, !check_enospc
bch_disk_reservation_put(c, &res);
return 0;
}
- new.reserved = 1;
+ new.reserved = 1;
+ new.nr_replicas = res.nr_replicas;
});
return 0;
struct bio_vec bv;
bio_for_each_segment(bv, bio, iter)
- page_state(bv.bv_page)->alloc_state = BCH_PAGE_UNALLOCATED;
+ page_state(bv.bv_page)->allocated = 0;
}
-static void bch_add_page_sectors(struct bio *bio, const struct bkey *k)
+static void bch_add_page_sectors(struct bio *bio, struct bkey_s_c k)
{
struct bvec_iter iter;
struct bio_vec bv;
struct bch_page_state *s = page_state(bv.bv_page);
/* sectors in @k from the start of this page: */
- unsigned k_sectors = k->size - (iter.bi_sector - k->p.offset);
+ unsigned k_sectors = k.k->size - (iter.bi_sector - k.k->p.offset);
unsigned page_sectors = min(bv.bv_len >> 9, k_sectors);
- BUG_ON(s->sectors + page_sectors > PAGE_SECTORS);
+ if (!s->sectors)
+ s->nr_replicas = bch_extent_nr_dirty_ptrs(k);
+ else
+ s->nr_replicas = min_t(unsigned, s->nr_replicas,
+ bch_extent_nr_dirty_ptrs(k));
+ BUG_ON(s->sectors + page_sectors > PAGE_SECTORS);
s->sectors += page_sectors;
}
}
EBUG_ON(s->reserved);
- s->alloc_state = BCH_PAGE_ALLOCATED;
+ s->allocated = 1;
s->sectors = 0;
}
k = bkey_i_to_s_c(&tmp.k);
if (!bkey_extent_is_allocation(k.k) ||
- bkey_extent_is_compressed(c, k))
+ bkey_extent_is_compressed(k))
bch_mark_pages_unalloc(bio);
bch_extent_pick_ptr(c, k, &pick);
swap(bio->bi_iter.bi_size, bytes);
if (bkey_extent_is_allocation(k.k))
- bch_add_page_sectors(bio, k.k);
+ bch_add_page_sectors(bio, k);
if (pick.ca) {
PTR_BUCKET(pick.ca, &pick.ptr)->read_prio =
struct page *page)
{
u64 inum = ei->vfs_inode.i_ino;
+ unsigned nr_replicas = page_state(page)->nr_replicas;
+
+ EBUG_ON(!nr_replicas);
+ /* XXX: disk_reservation->gen isn't plumbed through */
if (!w->io) {
alloc_io:
w->io->op.op.index_update_fn = bchfs_write_index_update;
}
- if (bio_add_page_contig(&w->io->bio.bio, page)) {
+ if (w->io->op.op.res.nr_replicas != nr_replicas ||
+ bio_add_page_contig(&w->io->bio.bio, page)) {
bch_writepage_do_io(w);
goto alloc_io;
}
/* Before unlocking the page, transfer reservation to w->io: */
old = page_state_cmpxchg(page_state(page), new, {
- BUG_ON(!new.reserved &&
- (new.sectors != PAGE_SECTORS ||
- new.alloc_state != BCH_PAGE_ALLOCATED));
+ EBUG_ON(!new.reserved &&
+ (new.sectors != PAGE_SECTORS ||
+ !new.allocated));
- if (new.alloc_state == BCH_PAGE_ALLOCATED &&
+ if (new.allocated &&
w->io->op.op.compression_type != BCH_COMPRESSION_NONE)
- new.alloc_state = BCH_PAGE_UNALLOCATED;
+ new.allocated = 0;
else if (!new.reserved)
goto out;
new.reserved = 0;
mutex_lock(&ei->update_lock);
setattr_copy(inode, iattr);
- inode->i_mtime = inode->i_ctime = CURRENT_TIME;
+ inode->i_mtime = inode->i_ctime = current_fs_time(inode->i_sb);
/* clear I_SIZE_DIRTY: */
i_size_dirty_put(ei);
ret = bch_discard(c,
POS(ino, discard_start),
POS(ino, discard_end),
- 0,
+ ZERO_VERSION,
&disk_res,
&i_sectors_hook.hook,
&ei->journal_seq);
struct cache_set *c = inode->i_sb->s_fs_info;
struct i_sectors_hook i_sectors_hook;
struct btree_iter iter;
- struct bkey_i reservation;
- struct bkey_s_c k;
struct bpos end;
loff_t block_start, block_end;
loff_t new_size = offset + len;
unsigned sectors;
+ unsigned replicas = READ_ONCE(c->opts.data_replicas);
int ret;
bch_btree_iter_init_intent(&iter, c, BTREE_ID_EXTENTS, POS_MIN);
while (bkey_cmp(iter.pos, end) < 0) {
struct disk_reservation disk_res = { 0 };
+ struct bkey_i_reservation reservation;
+ struct bkey_s_c k;
k = bch_btree_iter_peek_with_holes(&iter);
if ((ret = btree_iter_err(k)))
goto btree_iter_err;
/* already reserved */
- if (k.k->type == BCH_RESERVATION) {
+ if (k.k->type == BCH_RESERVATION &&
+ bkey_s_c_to_reservation(k).v->nr_replicas >= replicas) {
bch_btree_iter_advance_pos(&iter);
continue;
}
}
}
- bkey_init(&reservation.k);
+ bkey_reservation_init(&reservation.k_i);
reservation.k.type = BCH_RESERVATION;
reservation.k.p = k.k->p;
reservation.k.size = k.k->size;
- bch_cut_front(iter.pos, &reservation);
+ bch_cut_front(iter.pos, &reservation.k_i);
bch_cut_back(end, &reservation.k);
sectors = reservation.k.size;
+ reservation.v.nr_replicas = bch_extent_nr_dirty_ptrs(k);
- if (!bkey_extent_is_allocation(k.k) ||
- bkey_extent_is_compressed(c, k)) {
+ if (reservation.v.nr_replicas < replicas ||
+ bkey_extent_is_compressed(k)) {
ret = bch_disk_reservation_get(c, &disk_res,
sectors, 0);
if (ret)
goto err_put_sectors_dirty;
+
+ reservation.v.nr_replicas = disk_res.nr_replicas;
}
ret = bch_btree_insert_at(c, &disk_res, &i_sectors_hook.hook,
&ei->journal_seq,
BTREE_INSERT_ATOMIC|
BTREE_INSERT_NOFAIL,
- BTREE_INSERT_ENTRY(&iter, &reservation));
+ BTREE_INSERT_ENTRY(&iter, &reservation.k_i));
bch_disk_reservation_put(c, &disk_res);
btree_iter_err:
if (ret < 0 && ret != -EINTR)
static struct kmem_cache *bch_inode_cache;
-static void bch_vfs_inode_init(struct bch_inode_info *, struct bkey_s_c_inode);
+static void bch_vfs_inode_init(struct cache_set *,
+ struct bch_inode_info *,
+ struct bch_inode_unpacked *);
/*
* I_SIZE_DIRTY requires special handling:
{
struct btree_iter iter;
struct inode *inode = &ei->vfs_inode;
- struct bkey_i_inode new_inode;
- struct bch_inode *bi;
+ struct bch_inode_unpacked inode_u;
+ struct bkey_inode_buf inode_p;
u64 inum = inode->i_ino;
+ unsigned i_nlink = READ_ONCE(inode->i_nlink);
int ret;
+ /*
+ * We can't write an inode with i_nlink == 0 because it's stored biased;
+ * however, we don't need to because if i_nlink is 0 the inode is
+ * getting deleted when it's evicted.
+ */
+ if (!i_nlink)
+ return 0;
+
lockdep_assert_held(&ei->update_lock);
bch_btree_iter_init_intent(&iter, c, BTREE_ID_INODES, POS(inum, 0));
return -ENOENT;
}
- bkey_reassemble(&new_inode.k_i, k);
- bi = &new_inode.v;
+ ret = bch_inode_unpack(bkey_s_c_to_inode(k), &inode_u);
+ if (WARN_ONCE(ret,
+ "error %i unpacking inode %llu", ret, inum)) {
+ ret = -ENOENT;
+ break;
+ }
if (set) {
- ret = set(ei, bi, p);
+ ret = set(ei, &inode_u, p);
if (ret)
goto out;
}
- bi->i_mode = cpu_to_le16(inode->i_mode);
- bi->i_uid = cpu_to_le32(i_uid_read(inode));
- bi->i_gid = cpu_to_le32(i_gid_read(inode));
- bi->i_nlink = cpu_to_le32(inode->i_nlink);
- bi->i_dev = cpu_to_le32(inode->i_rdev);
- bi->i_atime = cpu_to_le64(timespec_to_ns(&inode->i_atime));
- bi->i_mtime = cpu_to_le64(timespec_to_ns(&inode->i_mtime));
- bi->i_ctime = cpu_to_le64(timespec_to_ns(&inode->i_ctime));
+ BUG_ON(i_nlink < nlink_bias(inode->i_mode));
+
+ inode_u.i_mode = inode->i_mode;
+ inode_u.i_uid = i_uid_read(inode);
+ inode_u.i_gid = i_gid_read(inode);
+ inode_u.i_nlink = i_nlink - nlink_bias(inode->i_mode);
+ inode_u.i_dev = inode->i_rdev;
+ inode_u.i_atime = timespec_to_bch_time(c, inode->i_atime);
+ inode_u.i_mtime = timespec_to_bch_time(c, inode->i_mtime);
+ inode_u.i_ctime = timespec_to_bch_time(c, inode->i_ctime);
+
+ bch_inode_pack(&inode_p, &inode_u);
ret = bch_btree_insert_at(c, NULL, NULL, &ei->journal_seq,
BTREE_INSERT_ATOMIC|
BTREE_INSERT_NOFAIL,
- BTREE_INSERT_ENTRY(&iter, &new_inode.k_i));
+ BTREE_INSERT_ENTRY(&iter, &inode_p.inode.k_i));
} while (ret == -EINTR);
if (!ret) {
- ei->i_size = le64_to_cpu(bi->i_size);
- ei->i_flags = le32_to_cpu(bi->i_flags);
+ ei->i_size = inode_u.i_size;
+ ei->i_flags = inode_u.i_flags;
}
out:
bch_btree_iter_unlock(&iter);
int bch_dec_nlink(struct cache_set *c, struct bch_inode_info *ei)
{
- int ret;
+ int ret = 0;
mutex_lock(&ei->update_lock);
drop_nlink(&ei->vfs_inode);
{
struct cache_set *c = sb->s_fs_info;
struct inode *inode;
+ struct bch_inode_unpacked inode_u;
struct bch_inode_info *ei;
- struct btree_iter iter;
- struct bkey_s_c k;
int ret;
pr_debug("inum %llu", inum);
if (!(inode->i_state & I_NEW))
return inode;
- bch_btree_iter_init(&iter, c, BTREE_ID_INODES, POS(inum, 0));
- k = bch_btree_iter_peek_with_holes(&iter);
-
- if ((ret = btree_iter_err(k)) || k.k->type != BCH_INODE_FS) {
- ret = bch_btree_iter_unlock(&iter);
+ ret = bch_inode_find_by_inum(c, inum, &inode_u);
+ if (ret) {
iget_failed(inode);
- return ERR_PTR(ret ?: -ENOENT);
+ return ERR_PTR(ret);
}
ei = to_bch_ei(inode);
- bch_vfs_inode_init(ei, bkey_s_c_to_inode(k));
+ bch_vfs_inode_init(c, ei, &inode_u);
ei->journal_seq = bch_inode_journal_seq(&c->journal, inum);
unlock_new_inode(inode);
- bch_btree_iter_unlock(&iter);
-
return inode;
}
struct inode *inode;
struct posix_acl *default_acl = NULL, *acl = NULL;
struct bch_inode_info *ei;
- struct bkey_i_inode bkey_inode;
+ struct bch_inode_unpacked inode_u;
+ struct bkey_inode_buf inode_p;
int ret;
inode = new_inode(parent->i_sb);
ei = to_bch_ei(inode);
- bch_inode_init(c, &bkey_inode, i_uid_read(inode),
+ bch_inode_init(c, &inode_u, i_uid_read(inode),
i_gid_read(inode), inode->i_mode, rdev);
+ bch_inode_pack(&inode_p, &inode_u);
- ret = bch_inode_create(c, &bkey_inode.k_i,
+ ret = bch_inode_create(c, &inode_p.inode.k_i,
BLOCKDEV_INODE_MAX, 0,
&c->unused_inode_hint);
if (unlikely(ret)) {
goto err;
}
- bch_vfs_inode_init(ei, inode_i_to_s_c(&bkey_inode));
+ inode_u.inum = inode_p.inode.k.p.inode;
+ bch_vfs_inode_init(c, ei, &inode_u);
if (default_acl) {
ret = bch_set_acl(inode, default_acl, ACL_TYPE_DEFAULT);
if (unlikely(ret))
return ret;
- dir->i_mtime = dir->i_ctime = CURRENT_TIME;
+ dir->i_mtime = dir->i_ctime = current_fs_time(dir->i_sb);
mark_inode_dirty_sync(dir);
return 0;
}
lockdep_assert_held(&inode->i_rwsem);
- inode->i_ctime = CURRENT_TIME;
+ inode->i_ctime = current_fs_time(dir->i_sb);
ret = bch_inc_nlink(c, ei);
if (ret)
drop_nlink(inode);
}
- drop_nlink(inode);
- if (inode->i_nlink) {
- mutex_lock(&ei->update_lock);
- ret = bch_write_inode(c, ei);
- mutex_unlock(&ei->update_lock);
- }
+ bch_dec_nlink(c, ei);
return 0;
}
struct inode *old_inode = old_dentry->d_inode;
struct bch_inode_info *ei = to_bch_ei(old_inode);
struct inode *new_inode = new_dentry->d_inode;
- struct timespec now = CURRENT_TIME;
+ struct timespec now = current_fs_time(old_dir->i_sb);
int ret;
lockdep_assert_held(&old_dir->i_rwsem);
struct inode *old_inode = old_dentry->d_inode;
struct inode *new_inode = new_dentry->d_inode;
struct bch_inode_info *ei = to_bch_ei(old_inode);
- struct timespec now = CURRENT_TIME;
+ struct timespec now = current_fs_time(old_dir->i_sb);
int ret;
ret = bch_dirent_rename(c,
}
static int bch_inode_user_flags_set(struct bch_inode_info *ei,
- struct bch_inode *bi,
+ struct bch_inode_unpacked *bi,
void *p)
{
/*
* We're relying on btree locking here for exclusion with other ioctl
* calls - use the flags in the btree (@bi), not ei->i_flags:
*/
- unsigned bch_flags = le32_to_cpu(bi->i_flags);
+ unsigned bch_flags = bi->i_flags;
unsigned oldflags = bch_inode_flags_to_user_flags(bch_flags);
unsigned newflags = *((unsigned *) p);
unsigned i;
if (oldflags != newflags)
return -EOPNOTSUPP;
- bi->i_flags = cpu_to_le32(bch_flags);
- ei->vfs_inode.i_ctime = CURRENT_TIME;
+ bi->i_flags = bch_flags;
+ ei->vfs_inode.i_ctime = current_fs_time(ei->vfs_inode.i_sb);
return 0;
}
.error_remove_page = generic_error_remove_page,
};
-static void bch_vfs_inode_init(struct bch_inode_info *ei,
- struct bkey_s_c_inode bkey_inode)
+static void bch_vfs_inode_init(struct cache_set *c,
+ struct bch_inode_info *ei,
+ struct bch_inode_unpacked *bi)
{
struct inode *inode = &ei->vfs_inode;
- const struct bch_inode *bi = bkey_inode.v;
pr_debug("init inode %llu with mode %o",
- bkey_inode.k->p.inode, bi->i_mode);
-
- ei->i_flags = le32_to_cpu(bi->i_flags);
- ei->i_size = le64_to_cpu(bi->i_size);
-
- inode->i_mode = le16_to_cpu(bi->i_mode);
- i_uid_write(inode, le32_to_cpu(bi->i_uid));
- i_gid_write(inode, le32_to_cpu(bi->i_gid));
-
- atomic64_set(&ei->i_sectors, le64_to_cpu(bi->i_sectors));
- inode->i_blocks = atomic64_read(&ei->i_sectors);
-
- inode->i_ino = bkey_inode.k->p.inode;
- set_nlink(inode, le32_to_cpu(bi->i_nlink));
- inode->i_rdev = le32_to_cpu(bi->i_dev);
- inode->i_size = le64_to_cpu(bi->i_size);
- inode->i_atime = ns_to_timespec(le64_to_cpu(bi->i_atime));
- inode->i_mtime = ns_to_timespec(le64_to_cpu(bi->i_mtime));
- inode->i_ctime = ns_to_timespec(le64_to_cpu(bi->i_ctime));
+ bi->inum, bi->i_mode);
+
+ ei->i_flags = bi->i_flags;
+ ei->i_size = bi->i_size;
+
+ inode->i_mode = bi->i_mode;
+ i_uid_write(inode, bi->i_uid);
+ i_gid_write(inode, bi->i_gid);
+
+ atomic64_set(&ei->i_sectors, bi->i_sectors);
+ inode->i_blocks = bi->i_sectors;
+
+ inode->i_ino = bi->inum;
+ set_nlink(inode, bi->i_nlink + nlink_bias(inode->i_mode));
+ inode->i_rdev = bi->i_dev;
+ inode->i_generation = bi->i_generation;
+ inode->i_size = bi->i_size;
+ inode->i_atime = bch_time_to_timespec(c, bi->i_atime);
+ inode->i_mtime = bch_time_to_timespec(c, bi->i_mtime);
+ inode->i_ctime = bch_time_to_timespec(c, bi->i_ctime);
bch_inode_flags_to_vfs(inode);
ei->str_hash = bch_hash_info_init(bi);
buf->f_files = atomic_long_read(&c->nr_inodes);
buf->f_ffree = U64_MAX;
- fsid = le64_to_cpup((void *) c->disk_sb.user_uuid.b) ^
- le64_to_cpup((void *) c->disk_sb.user_uuid.b + sizeof(u64));
+ fsid = le64_to_cpup((void *) c->sb.user_uuid.b) ^
+ le64_to_cpup((void *) c->sb.user_uuid.b + sizeof(u64));
buf->f_fsid.val[0] = fsid & 0xFFFFFFFFUL;
buf->f_fsid.val[1] = (fsid >> 32) & 0xFFFFFFFFUL;
buf->f_namelen = NAME_MAX;
sb->s_op = &bch_super_operations;
sb->s_xattr = bch_xattr_handlers;
sb->s_magic = BCACHE_STATFS_MAGIC;
- sb->s_time_gran = 1;
+ sb->s_time_gran = c->sb.time_precision;
c->vfs_sb = sb;
sb->s_bdi = &c->bdi;
return (mode >> 12) & 15;
}
+static inline unsigned nlink_bias(umode_t mode)
+{
+ return S_ISDIR(mode) ? 2 : 1;
+}
+
+struct bch_inode_unpacked;
+
/* returns 0 if we want to do the update, or error is passed up */
typedef int (*inode_set_fn)(struct bch_inode_info *,
- struct bch_inode *, void *);
+ struct bch_inode_unpacked *, void *);
int __must_check __bch_write_inode(struct cache_set *, struct bch_inode_info *,
inode_set_fn, void *);
#include <linux/random.h>
-ssize_t bch_inode_status(char *buf, size_t len, const struct bkey *k)
+#include <asm/unaligned.h>
+
+#define FIELD_BYTES() \
+
+static const u8 byte_table[8] = { 1, 2, 3, 4, 6, 8, 10, 13 };
+static const u8 bits_table[8] = {
+ 1 * 8 - 1,
+ 2 * 8 - 2,
+ 3 * 8 - 3,
+ 4 * 8 - 4,
+ 6 * 8 - 5,
+ 8 * 8 - 6,
+ 10 * 8 - 7,
+ 13 * 8 - 8,
+};
+
+static int inode_encode_field(u8 *out, u8 *end, const u64 in[2])
{
- if (k->p.offset)
- return scnprintf(buf, len, "offset nonzero: %llu", k->p.offset);
-
- if (k->size)
- return scnprintf(buf, len, "size nonzero: %u", k->size);
-
- switch (k->type) {
- case KEY_TYPE_DELETED:
- return scnprintf(buf, len, "deleted");
- case KEY_TYPE_DISCARD:
- return scnprintf(buf, len, "discarded");
- case KEY_TYPE_ERROR:
- return scnprintf(buf, len, "error");
- case KEY_TYPE_COOKIE:
- return scnprintf(buf, len, "cookie");
+ unsigned bytes, bits, shift;
- case BCH_INODE_FS:
- if (bkey_val_bytes(k) != sizeof(struct bch_inode))
- return scnprintf(buf, len, "bad size: %zu",
- bkey_val_bytes(k));
+ if (likely(!in[1]))
+ bits = fls64(in[0]);
+ else
+ bits = fls64(in[1]) + 64;
- if (k->p.inode < BLOCKDEV_INODE_MAX)
- return scnprintf(buf, len,
- "fs inode in blockdev range: %llu",
- k->p.inode);
- return 0;
+ for (shift = 1; shift <= 8; shift++)
+ if (bits < bits_table[shift - 1])
+ goto got_shift;
- case BCH_INODE_BLOCKDEV:
- if (bkey_val_bytes(k) != sizeof(struct bch_inode_blockdev))
- return scnprintf(buf, len, "bad size: %zu",
- bkey_val_bytes(k));
+ BUG();
+got_shift:
+ bytes = byte_table[shift - 1];
- if (k->p.inode >= BLOCKDEV_INODE_MAX)
- return scnprintf(buf, len,
- "blockdev inode in fs range: %llu",
- k->p.inode);
- return 0;
+ BUG_ON(out + bytes > end);
- default:
- return scnprintf(buf, len, "unknown inode type: %u", k->type);
+ if (likely(bytes <= 8)) {
+ u64 b = cpu_to_be64(in[0]);
+
+ memcpy(out, (void *) &b + 8 - bytes, bytes);
+ } else {
+ u64 b = cpu_to_be64(in[1]);
+
+ memcpy(out, (void *) &b + 16 - bytes, bytes);
+ put_unaligned_be64(in[0], out + bytes - 8);
+ }
+
+ *out |= (1 << 8) >> shift;
+
+ return bytes;
+}
+
+static int inode_decode_field(const u8 *in, const u8 *end,
+ u64 out[2], unsigned *out_bits)
+{
+ unsigned bytes, bits, shift;
+
+ if (in >= end)
+ return -1;
+
+ if (!*in)
+ return -1;
+
+ /*
+ * position of highest set bit indicates number of bytes:
+ * shift = number of bits to remove in high byte:
+ */
+ shift = 8 - __fls(*in); /* 1 <= shift <= 8 */
+ bytes = byte_table[shift - 1];
+ bits = bytes * 8 - shift;
+
+ if (in + bytes > end)
+ return -1;
+
+ /*
+ * we're assuming it's safe to deref up to 7 bytes < in; this will work
+ * because keys always start quite a bit more than 7 bytes after the
+ * start of the btree node header:
+ */
+ if (likely(bytes <= 8)) {
+ out[0] = get_unaligned_be64(in + bytes - 8);
+ out[0] <<= 64 - bits;
+ out[0] >>= 64 - bits;
+ out[1] = 0;
+ } else {
+ out[0] = get_unaligned_be64(in + bytes - 8);
+ out[1] = get_unaligned_be64(in + bytes - 16);
+ out[1] <<= 128 - bits;
+ out[1] >>= 128 - bits;
+ }
+
+ *out_bits = out[1] ? 64 + fls64(out[1]) : fls64(out[0]);
+ return bytes;
+}
+
+void bch_inode_pack(struct bkey_inode_buf *packed,
+ const struct bch_inode_unpacked *inode)
+{
+ u8 *out = packed->inode.v.fields;
+ u8 *end = (void *) &packed[1];
+ u8 *last_nonzero_field = out;
+ u64 field[2];
+ unsigned nr_fields = 0, last_nonzero_fieldnr = 0;
+
+ bkey_inode_init(&packed->inode.k_i);
+ packed->inode.k.p.inode = inode->inum;
+ packed->inode.v.i_hash_seed = inode->i_hash_seed;
+ packed->inode.v.i_flags = cpu_to_le32(inode->i_flags);
+ packed->inode.v.i_mode = cpu_to_le16(inode->i_mode);
+
+#define BCH_INODE_FIELD(_name, _bits) \
+ field[0] = inode->_name; \
+ field[1] = 0; \
+ out += inode_encode_field(out, end, field); \
+ nr_fields++; \
+ \
+ if (field[0] | field[1]) { \
+ last_nonzero_field = out; \
+ last_nonzero_fieldnr = nr_fields; \
+ }
+
+ BCH_INODE_FIELDS()
+#undef BCH_INODE_FIELD
+
+ out = last_nonzero_field;
+ nr_fields = last_nonzero_fieldnr;
+
+ set_bkey_val_bytes(&packed->inode.k, out - (u8 *) &packed->inode.v);
+ memset(out, 0,
+ (u8 *) &packed->inode.v +
+ bkey_val_bytes(&packed->inode.k) - out);
+
+ SET_INODE_NR_FIELDS(&packed->inode.v, nr_fields);
+
+ if (IS_ENABLED(CONFIG_BCACHE_DEBUG)) {
+ struct bch_inode_unpacked unpacked;
+
+ int ret = bch_inode_unpack(inode_i_to_s_c(&packed->inode),
+ &unpacked);
+ BUG_ON(ret);
+ BUG_ON(unpacked.inum != inode->inum);
+ BUG_ON(unpacked.i_hash_seed != inode->i_hash_seed);
+ BUG_ON(unpacked.i_mode != inode->i_mode);
+
+#define BCH_INODE_FIELD(_name, _bits) BUG_ON(unpacked._name != inode->_name);
+ BCH_INODE_FIELDS()
+#undef BCH_INODE_FIELD
}
}
+int bch_inode_unpack(struct bkey_s_c_inode inode,
+ struct bch_inode_unpacked *unpacked)
+{
+ const u8 *in = inode.v->fields;
+ const u8 *end = (void *) inode.v + bkey_val_bytes(inode.k);
+ u64 field[2];
+ unsigned fieldnr = 0, field_bits;
+ int ret;
+
+ unpacked->inum = inode.k->p.inode;
+ unpacked->i_hash_seed = inode.v->i_hash_seed;
+ unpacked->i_flags = le32_to_cpu(inode.v->i_flags);
+ unpacked->i_mode = le16_to_cpu(inode.v->i_mode);
+
+#define BCH_INODE_FIELD(_name, _bits) \
+ if (fieldnr++ == INODE_NR_FIELDS(inode.v)) { \
+ memset(&unpacked->_name, 0, \
+ sizeof(*unpacked) - \
+ offsetof(struct bch_inode_unpacked, _name)); \
+ return 0; \
+ } \
+ \
+ ret = inode_decode_field(in, end, field, &field_bits); \
+ if (ret < 0) \
+ return ret; \
+ \
+ if (field_bits > sizeof(unpacked->_name) * 8) \
+ return -1; \
+ \
+ unpacked->_name = field[0]; \
+ in += ret;
+
+ BCH_INODE_FIELDS()
+#undef BCH_INODE_FIELD
+
+ /* XXX: signal if there were more fields than expected? */
+
+ return 0;
+}
+
static const char *bch_inode_invalid(const struct cache_set *c,
struct bkey_s_c k)
{
switch (k.k->type) {
case BCH_INODE_FS: {
struct bkey_s_c_inode inode = bkey_s_c_to_inode(k);
+ struct bch_inode_unpacked unpacked;
- if (bkey_val_bytes(k.k) != sizeof(struct bch_inode))
+ if (bkey_val_bytes(k.k) < sizeof(struct bch_inode))
return "incorrect value size";
if (k.k->p.inode < BLOCKDEV_INODE_MAX)
return "fs inode in blockdev range";
- if (INODE_STR_HASH_TYPE(inode.v) >= BCH_STR_HASH_NR)
+ if (INODE_STR_HASH(inode.v) >= BCH_STR_HASH_NR)
return "invalid str hash type";
+ if (bch_inode_unpack(inode, &unpacked))
+ return "invalid variable length fields";
+
return NULL;
}
case BCH_INODE_BLOCKDEV:
size_t size, struct bkey_s_c k)
{
struct bkey_s_c_inode inode;
+ struct bch_inode_unpacked unpacked;
switch (k.k->type) {
case BCH_INODE_FS:
inode = bkey_s_c_to_inode(k);
+ if (bch_inode_unpack(inode, &unpacked)) {
+ scnprintf(buf, size, "(unpack error)");
+ break;
+ }
- scnprintf(buf, size, "i_size %llu", inode.v->i_size);
+ scnprintf(buf, size, "i_size %llu", unpacked.i_size);
break;
}
}
.val_to_text = bch_inode_to_text,
};
-void bch_inode_init(struct cache_set *c, struct bkey_i_inode *inode,
+void bch_inode_init(struct cache_set *c, struct bch_inode_unpacked *inode_u,
uid_t uid, gid_t gid, umode_t mode, dev_t rdev)
{
- struct timespec ts = CURRENT_TIME;
- s64 now = timespec_to_ns(&ts);
- struct bch_inode *bi;
-
- bi = &bkey_inode_init(&inode->k_i)->v;
- bi->i_uid = cpu_to_le32(uid);
- bi->i_gid = cpu_to_le32(gid);
-
- bi->i_mode = cpu_to_le16(mode);
- bi->i_dev = cpu_to_le32(rdev);
- bi->i_atime = cpu_to_le64(now);
- bi->i_mtime = cpu_to_le64(now);
- bi->i_ctime = cpu_to_le64(now);
- bi->i_nlink = cpu_to_le32(S_ISDIR(mode) ? 2 : 1);
-
- get_random_bytes(&bi->i_hash_seed, sizeof(bi->i_hash_seed));
- SET_INODE_STR_HASH_TYPE(bi, c->sb.str_hash_type);
+ s64 now = timespec_to_bch_time(c, CURRENT_TIME);
+
+ memset(inode_u, 0, sizeof(*inode_u));
+
+ /* ick */
+ inode_u->i_flags |= c->sb.str_hash_type << INODE_STR_HASH_OFFSET;
+ get_random_bytes(&inode_u->i_hash_seed, sizeof(inode_u->i_hash_seed));
+
+ inode_u->i_mode = mode;
+ inode_u->i_uid = uid;
+ inode_u->i_gid = gid;
+ inode_u->i_dev = rdev;
+ inode_u->i_atime = now;
+ inode_u->i_mtime = now;
+ inode_u->i_ctime = now;
+ inode_u->i_otime = now;
}
int bch_inode_create(struct cache_set *c, struct bkey_i *inode,
struct extent_insert_hook *hook, u64 *journal_seq)
{
return bch_discard(c, POS(inode_nr, new_size), POS(inode_nr + 1, 0),
- 0, NULL, hook, journal_seq);
+ ZERO_VERSION, NULL, hook, journal_seq);
}
int bch_inode_rm(struct cache_set *c, u64 inode_nr)
ret = bch_btree_delete_range(c, BTREE_ID_XATTRS,
POS(inode_nr, 0),
POS(inode_nr + 1, 0),
- 0, NULL, NULL, NULL);
+ ZERO_VERSION, NULL, NULL, NULL);
if (ret < 0)
return ret;
ret = bch_btree_delete_range(c, BTREE_ID_DIRENTS,
POS(inode_nr, 0),
POS(inode_nr + 1, 0),
- 0, NULL, NULL, NULL);
+ ZERO_VERSION, NULL, NULL, NULL);
if (ret < 0)
return ret;
NULL, NULL, BTREE_INSERT_NOFAIL);
}
-int bch_inode_update(struct cache_set *c, struct bkey_i *inode,
- u64 *journal_seq)
-{
- return bch_btree_update(c, BTREE_ID_INODES, inode, journal_seq);
-}
-
int bch_inode_find_by_inum(struct cache_set *c, u64 inode_nr,
- struct bkey_i_inode *inode)
+ struct bch_inode_unpacked *inode)
{
struct btree_iter iter;
struct bkey_s_c k;
+ int ret = -ENOENT;
for_each_btree_key_with_holes(&iter, c, BTREE_ID_INODES,
POS(inode_nr, 0), k) {
switch (k.k->type) {
case BCH_INODE_FS:
- bkey_reassemble(&inode->k_i, k);
- bch_btree_iter_unlock(&iter);
- return 0;
+ ret = bch_inode_unpack(bkey_s_c_to_inode(k), inode);
+ break;
default:
/* hole, not found */
break;
}
- return bch_btree_iter_unlock(&iter) ?: -ENOENT;
+ return bch_btree_iter_unlock(&iter) ?: ret;
}
int bch_cached_dev_inode_find_by_uuid(struct cache_set *c, uuid_le *uuid,
extern const struct bkey_ops bch_bkey_inode_ops;
-ssize_t bch_inode_status(char *, size_t, const struct bkey *);
+struct bch_inode_unpacked {
+ u64 inum;
+ __le64 i_hash_seed;
+ u32 i_flags;
+ u16 i_mode;
-void bch_inode_init(struct cache_set *, struct bkey_i_inode *,
+#define BCH_INODE_FIELD(_name, _bits) u##_bits _name;
+ BCH_INODE_FIELDS()
+#undef BCH_INODE_FIELD
+};
+
+struct bkey_inode_buf {
+ struct bkey_i_inode inode;
+
+#define BCH_INODE_FIELD(_name, _bits) + 8 + _bits / 8
+ u8 _pad[0 + BCH_INODE_FIELDS()];
+#undef BCH_INODE_FIELD
+} __packed;
+
+void bch_inode_pack(struct bkey_inode_buf *, const struct bch_inode_unpacked *);
+int bch_inode_unpack(struct bkey_s_c_inode, struct bch_inode_unpacked *);
+
+void bch_inode_init(struct cache_set *, struct bch_inode_unpacked *,
uid_t, gid_t, umode_t, dev_t);
int bch_inode_create(struct cache_set *, struct bkey_i *, u64, u64, u64 *);
int bch_inode_truncate(struct cache_set *, u64, u64,
struct extent_insert_hook *, u64 *);
int bch_inode_rm(struct cache_set *, u64);
-int bch_inode_update(struct cache_set *, struct bkey_i *, u64 *);
-int bch_inode_find_by_inum(struct cache_set *, u64, struct bkey_i_inode *);
+int bch_inode_find_by_inum(struct cache_set *, u64,
+ struct bch_inode_unpacked *);
int bch_cached_dev_inode_find_by_uuid(struct cache_set *, uuid_le *,
struct bkey_i_inode_blockdev *);
+static inline struct timespec bch_time_to_timespec(struct cache_set *c, u64 time)
+{
+ return ns_to_timespec(time * c->sb.time_precision + c->sb.time_base_lo);
+}
+
+static inline u64 timespec_to_bch_time(struct cache_set *c, struct timespec ts)
+{
+ s64 ns = timespec_to_ns(&ts) - c->sb.time_base_lo;
+
+ if (c->sb.time_precision == 1)
+ return ns;
+
+ return div_s64(ns, c->sb.time_precision);
+}
+
#endif
#include "move.h"
#include "notify.h"
#include "stats.h"
-#include "super.h"
+#include "super-io.h"
#include <linux/blkdev.h>
#include <linux/random.h>
closure_put(cl);
}
+static struct nonce extent_nonce(struct bversion version,
+ unsigned nonce,
+ unsigned uncompressed_size,
+ unsigned compression_type)
+{
+ return (struct nonce) {{
+ [0] = cpu_to_le32((nonce << 12) |
+ (uncompressed_size << 22)),
+ [1] = cpu_to_le32(version.lo),
+ [2] = cpu_to_le32(version.lo >> 32),
+ [3] = cpu_to_le32(version.hi|
+ (compression_type << 24))^BCH_NONCE_EXTENT,
+ }};
+}
+
static void init_append_extent(struct bch_write_op *op,
unsigned compressed_size,
unsigned uncompressed_size,
unsigned compression_type,
- u64 csum, unsigned csum_type,
+ unsigned nonce,
+ struct bch_csum csum, unsigned csum_type,
struct open_bucket *ob)
{
struct bkey_i_extent *e = bkey_extent_init(op->insert_keys.top);
op->pos.offset += uncompressed_size;
e->k.p = op->pos;
e->k.size = uncompressed_size;
+ e->k.version = op->version;
+ bkey_extent_set_cached(&e->k, op->flags & BCH_WRITE_CACHED);
bch_extent_crc_append(e, compressed_size,
uncompressed_size,
compression_type,
- csum, csum_type);
+ nonce, csum, csum_type);
bch_alloc_sectors_append_ptrs(op->c, e, op->nr_replicas,
ob, compressed_size);
unsigned key_to_write_offset = op->insert_keys.top_p -
op->insert_keys.keys_p;
struct bkey_i *key_to_write;
- unsigned csum_type = c->opts.data_checksum;
+ unsigned csum_type = op->csum_type;
unsigned compression_type = op->compression_type;
int ret;
/* Need to decompress data? */
if ((op->flags & BCH_WRITE_DATA_COMPRESSED) &&
- (op->crc.uncompressed_size != op->size ||
- op->crc.compressed_size > ob->sectors_free)) {
+ (crc_uncompressed_size(NULL, &op->crc) != op->size ||
+ crc_compressed_size(NULL, &op->crc) > ob->sectors_free)) {
int ret;
ret = bch_bio_uncompress_inplace(c, orig, op->size, op->crc);
if (op->flags & BCH_WRITE_DATA_COMPRESSED) {
init_append_extent(op,
- op->crc.compressed_size,
- op->crc.uncompressed_size,
+ crc_compressed_size(NULL, &op->crc),
+ crc_uncompressed_size(NULL, &op->crc),
op->crc.compression_type,
+ op->crc.nonce,
op->crc.csum,
op->crc.csum_type,
ob);
/* all units here in bytes */
unsigned total_output = 0, output_available =
min(ob->sectors_free << 9, orig->bi_iter.bi_size);
- u64 csum;
+ unsigned crc_nonce = bch_csum_type_is_encryption(csum_type)
+ ? op->nonce : 0;
+ struct bch_csum csum;
+ struct nonce nonce;
bio = bio_alloc_bioset(GFP_NOIO,
DIV_ROUND_UP(output_available, PAGE_SIZE),
BUG_ON(src_len & (block_bytes(c) - 1));
swap(bio->bi_iter.bi_size, dst_len);
- csum = bch_checksum_bio(bio, csum_type);
+ nonce = extent_nonce(op->version,
+ crc_nonce,
+ src_len >> 9,
+ compression_type),
+
+ bch_encrypt_bio(c, csum_type, nonce, bio);
+
+ csum = bch_checksum_bio(c, csum_type, nonce, bio);
swap(bio->bi_iter.bi_size, dst_len);
init_append_extent(op,
dst_len >> 9, src_len >> 9,
fragment_compression_type,
- csum, csum_type, ob);
+ crc_nonce, csum, csum_type, ob);
total_output += dst_len;
bio_advance(bio, dst_len);
wbio->put_bio = bio != orig;
init_append_extent(op, bio_sectors(bio), bio_sectors(bio),
- compression_type, 0, csum_type, ob);
+ compression_type, 0,
+ (struct bch_csum) { 0 }, csum_type, ob);
ret = bio != orig;
}
key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
- if (!(op->flags & BCH_WRITE_CACHED))
- bch_check_mark_super(c, key_to_write, false);
+ bch_check_mark_super(c, key_to_write, false);
#ifndef CONFIG_BCACHE_NO_IO
bch_submit_wbio_replicas(to_wbio(bio), c, key_to_write, false);
closure_return(cl);
}
+ if (bversion_zero(op->version) &&
+ bch_csum_type_is_encryption(op->csum_type))
+ op->version.lo =
+ atomic64_inc_return(&c->key_version) + 1;
+
if (!(op->flags & BCH_WRITE_DISCARD))
bch_increment_clock(c, bio_sectors(bio), WRITE);
struct write_point *wp, struct bpos pos,
u64 *journal_seq, unsigned flags)
{
+ EBUG_ON(res.sectors && !res.nr_replicas);
+
op->c = c;
op->io_wq = index_update_wq(op);
op->bio = bio;
op->written = 0;
op->error = 0;
op->flags = flags;
+ op->csum_type = bch_data_checksum_type(c);
op->compression_type = c->opts.compression;
op->nr_replicas = res.nr_replicas;
op->alloc_reserve = RESERVE_NONE;
+ op->nonce = 0;
op->pos = pos;
- op->version = 0;
+ op->version = ZERO_VERSION;
op->res = res;
op->wp = wp;
* appropriately inode_truncate should call this
*/
int bch_discard(struct cache_set *c, struct bpos start,
- struct bpos end, u64 version,
+ struct bpos end, struct bversion version,
struct disk_reservation *disk_res,
struct extent_insert_hook *hook,
u64 *journal_seq)
struct bio *src = &rbio->bio;
struct bio *dst = &bch_rbio_parent(rbio)->bio;
struct bvec_iter dst_iter = rbio->parent_iter;
- u64 csum;
+ struct nonce nonce = extent_nonce(rbio->version,
+ rbio->crc.nonce,
+ crc_uncompressed_size(NULL, &rbio->crc),
+ rbio->crc.compression_type);
+ struct bch_csum csum;
int ret = 0;
/*
* in order to promote
*/
if (rbio->bounce) {
- src->bi_iter.bi_size = rbio->crc.compressed_size << 9;
- src->bi_iter.bi_idx = 0;
- src->bi_iter.bi_bvec_done = 0;
+ src->bi_iter.bi_size = crc_compressed_size(NULL, &rbio->crc) << 9;
+ src->bi_iter.bi_idx = 0;
+ src->bi_iter.bi_bvec_done = 0;
} else {
src->bi_iter = rbio->parent_iter;
}
- csum = bch_checksum_bio(src, rbio->crc.csum_type);
- if (cache_nonfatal_io_err_on(rbio->crc.csum != csum, rbio->ca,
- "data checksum error, inode %llu offset %llu: expected %0llx got %0llx (type %u)",
+ csum = bch_checksum_bio(c, rbio->crc.csum_type, nonce, src);
+ if (cache_nonfatal_io_err_on(bch_crc_cmp(rbio->crc.csum, csum), rbio->ca,
+ "data checksum error, inode %llu offset %llu: expected %0llx%0llx got %0llx%0llx (type %u)",
rbio->inode, (u64) rbio->parent_iter.bi_sector << 9,
- rbio->crc.csum, csum, rbio->crc.csum_type))
+ rbio->crc.csum.hi, rbio->crc.csum.lo, csum.hi, csum.lo,
+ rbio->crc.csum_type))
ret = -EIO;
/*
*/
if (rbio->crc.compression_type != BCH_COMPRESSION_NONE) {
if (!ret) {
+ bch_encrypt_bio(c, rbio->crc.csum_type, nonce, src);
ret = bch_bio_uncompress(c, src, dst,
dst_iter, rbio->crc);
if (ret)
}
} else if (rbio->bounce) {
bio_advance(src, rbio->crc.offset << 9);
+
+ /* don't need to decrypt the entire bio: */
+ BUG_ON(src->bi_iter.bi_size < dst_iter.bi_size);
+ src->bi_iter.bi_size = dst_iter.bi_size;
+
+ nonce = nonce_add(nonce, rbio->crc.offset << 9);
+
+ bch_encrypt_bio(c, rbio->crc.csum_type,
+ nonce, src);
+
bio_copy_data_iter(dst, dst_iter,
src, src->bi_iter);
+ } else {
+ bch_encrypt_bio(c, rbio->crc.csum_type, nonce, src);
}
return ret;
*/
unsigned sectors =
max_t(unsigned, k.k->size,
- pick->crc.uncompressed_size);
+ crc_uncompressed_size(NULL, &pick->crc));
unsigned pages = DIV_ROUND_UP(sectors, PAGE_SECTORS);
promote_op = kmalloc(sizeof(*promote_op) +
*/
if (pick->crc.compression_type != BCH_COMPRESSION_NONE ||
(pick->crc.csum_type != BCH_CSUM_NONE &&
- (bvec_iter_sectors(iter) != pick->crc.uncompressed_size ||
+ (bvec_iter_sectors(iter) != crc_uncompressed_size(NULL, &pick->crc) ||
(flags & BCH_READ_FORCE_BOUNCE)))) {
read_full = true;
bounce = true;
if (bounce) {
unsigned sectors = read_full
- ? (pick->crc.compressed_size ?: k.k->size)
+ ? (crc_compressed_size(NULL, &pick->crc) ?: k.k->size)
: bvec_iter_sectors(iter);
rbio = container_of(bio_alloc_bioset(GFP_NOIO,
rbio->flags = flags;
rbio->bounce = bounce;
rbio->split = split;
+ rbio->version = k.k->version;
rbio->crc = pick->crc;
/*
* crc.compressed_size will be 0 if there wasn't any checksum
* bounced (which isn't necessarily the original key size, if we bounced
* only for promoting)
*/
- rbio->crc.compressed_size = bio_sectors(&rbio->bio);
+ rbio->crc._compressed_size = bio_sectors(&rbio->bio) - 1;
rbio->ptr = pick->ptr;
rbio->ca = pick->ca;
rbio->promote = promote_op;
bch_migrate_write_init(c, &promote_op->write,
&c->promote_write_point,
k, NULL,
- BCH_WRITE_ALLOC_NOWAIT);
+ BCH_WRITE_ALLOC_NOWAIT|
+ BCH_WRITE_CACHED);
promote_op->write.promote = true;
if (rbio->crc.compression_type) {
const struct bkey_i *, bool);
int bch_discard(struct cache_set *, struct bpos, struct bpos,
- u64, struct disk_reservation *,
+ struct bversion, struct disk_reservation *,
struct extent_insert_hook *, u64 *);
void bch_read_retry_work(struct work_struct *);
u8 bounce:1,
split:1;
- struct bch_extent_crc64 crc;
+ struct bversion version;
+ struct bch_extent_crc128 crc;
struct bch_extent_ptr ptr;
struct cache *ca;
short error;
u16 flags;
+ unsigned csum_type:4;
unsigned compression_type:4;
unsigned nr_replicas:4;
unsigned alloc_reserve:4;
+ unsigned nonce:14;
struct bpos pos;
- unsigned version;
+ struct bversion version;
/* For BCH_WRITE_DATA_COMPRESSED: */
- struct bch_extent_crc64 crc;
+ struct bch_extent_crc128 crc;
unsigned size;
struct disk_reservation res;
#include "io.h"
#include "keylist.h"
#include "journal.h"
-#include "super.h"
+#include "super-io.h"
+#include "vstructs.h"
#include <trace/events/bcache.h>
return last_seq(j) + fifo_entry_idx(&j->pin, pin_list);
}
-#define for_each_jset_entry(entry, jset) \
- for (entry = (jset)->start; \
- entry < bkey_idx(jset, le32_to_cpu((jset)->u64s)); \
- entry = jset_keys_next(entry))
-
static inline struct jset_entry *__jset_entry_type_next(struct jset *jset,
struct jset_entry *entry, unsigned type)
{
- while (entry < bkey_idx(jset, le32_to_cpu(jset->u64s))) {
+ while (entry < vstruct_last(jset)) {
if (JOURNAL_ENTRY_TYPE(entry) == type)
return entry;
- entry = jset_keys_next(entry);
+ entry = vstruct_next(entry);
}
return NULL;
#define for_each_jset_entry_type(entry, jset, type) \
for (entry = (jset)->start; \
(entry = __jset_entry_type_next(jset, entry, type)); \
- entry = jset_keys_next(entry))
+ entry = vstruct_next(entry))
#define for_each_jset_key(k, _n, entry, jset) \
for_each_jset_entry_type(entry, jset, JOURNAL_ENTRY_BTREE_KEYS) \
- for (k = (entry)->start; \
- (k < bkey_idx(entry, le16_to_cpu((entry)->u64s)) &&\
- (_n = bkey_next(k), 1)); \
- k = _n)
+ vstruct_for_each_safe(entry, k, _n)
static inline void bch_journal_add_entry(struct journal_buf *buf,
const void *data, size_t u64s,
closure_sync(&cl);
- mutex_lock(&c->btree_interior_update_lock);
-
for (i = 0;; i++) {
struct btree_interior_update *as;
struct pending_btree_node_free *d;
}
n = bl->entries[i];
mutex_unlock(&j->blacklist_lock);
+redo_wait:
+ mutex_lock(&c->btree_interior_update_lock);
/*
* Is the node on the list of pending interior node updates -
closure_wait(&as->wait, &cl);
mutex_unlock(&c->btree_interior_update_lock);
closure_sync(&cl);
- break;
+ goto redo_wait;
}
- }
- mutex_unlock(&c->btree_interior_update_lock);
+ mutex_unlock(&c->btree_interior_update_lock);
+ }
mutex_lock(&j->blacklist_lock);
struct journal_list {
struct closure cl;
struct mutex lock;
- struct mutex cache_set_buffer_lock;
struct list_head *head;
int ret;
};
{
struct journal_replay *i, *pos;
struct list_head *where;
- size_t bytes = __set_bytes(j, le32_to_cpu(j->u64s));
+ size_t bytes = vstruct_bytes(j);
__le64 last_seq;
int ret;
list_for_each_entry_reverse(i, jlist->head, list) {
/* Duplicate? */
if (le64_to_cpu(j->seq) == le64_to_cpu(i->j.seq)) {
- fsck_err_on(bytes != __set_bytes(&i->j,
- le32_to_cpu(i->j.u64s)) ||
+ fsck_err_on(bytes != vstruct_bytes(&i->j) ||
memcmp(j, &i->j, bytes), c,
"found duplicate but non identical journal entries (seq %llu)",
le64_to_cpu(j->seq));
return ret;
}
+static struct nonce journal_nonce(const struct jset *jset)
+{
+ return (struct nonce) {{
+ [0] = 0,
+ [1] = ((__le32 *) &jset->seq)[0],
+ [2] = ((__le32 *) &jset->seq)[1],
+ [3] = BCH_NONCE_JOURNAL,
+ }};
+}
+
static void journal_entry_null_range(void *start, void *end)
{
struct jset_entry *entry;
- for (entry = start; entry != end; entry = jset_keys_next(entry)) {
+ for (entry = start; entry != end; entry = vstruct_next(entry)) {
entry->u64s = 0;
entry->btree_id = 0;
entry->level = 0;
struct bkey_i *k, enum bkey_type key_type,
const char *type)
{
- void *next = jset_keys_next(entry);
+ void *next = vstruct_next(entry);
const char *invalid;
char buf[160];
int ret = 0;
if (mustfix_fsck_err_on(!k->k.u64s, c,
"invalid %s in journal: k->u64s 0", type)) {
entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
- journal_entry_null_range(jset_keys_next(entry), next);
+ journal_entry_null_range(vstruct_next(entry), next);
return 0;
}
if (mustfix_fsck_err_on((void *) bkey_next(k) >
- (void *) jset_keys_next(entry), c,
+ (void *) vstruct_next(entry), c,
"invalid %s in journal: extends past end of journal entry",
type)) {
entry->u64s = cpu_to_le16((u64 *) k - entry->_data);
- journal_entry_null_range(jset_keys_next(entry), next);
+ journal_entry_null_range(vstruct_next(entry), next);
return 0;
}
type, k->k.format)) {
le16_add_cpu(&entry->u64s, -k->k.u64s);
memmove(k, bkey_next(k), next - (void *) bkey_next(k));
- journal_entry_null_range(jset_keys_next(entry), next);
+ journal_entry_null_range(vstruct_next(entry), next);
return 0;
}
le16_add_cpu(&entry->u64s, -k->k.u64s);
memmove(k, bkey_next(k), next - (void *) bkey_next(k));
- journal_entry_null_range(jset_keys_next(entry), next);
+ journal_entry_null_range(vstruct_next(entry), next);
return 0;
}
fsck_err:
#define JOURNAL_ENTRY_NONE 6
#define JOURNAL_ENTRY_BAD 7
-static int journal_entry_validate(struct cache_set *c, struct jset *j, u64 sector,
+static int journal_entry_validate(struct cache_set *c,
+ struct jset *j, u64 sector,
unsigned bucket_sectors_left,
unsigned sectors_read)
{
struct jset_entry *entry;
- size_t bytes = __set_bytes(j, le32_to_cpu(j->u64s));
- u64 got, expect;
+ size_t bytes = vstruct_bytes(j);
+ struct bch_csum csum;
int ret = 0;
- if (le64_to_cpu(j->magic) != jset_magic(&c->disk_sb))
+ if (le64_to_cpu(j->magic) != jset_magic(c))
return JOURNAL_ENTRY_NONE;
if (le32_to_cpu(j->version) != BCACHE_JSET_VERSION) {
if (bytes > sectors_read << 9)
return JOURNAL_ENTRY_REREAD;
- got = le64_to_cpu(j->csum);
- expect = __csum_set(j, le32_to_cpu(j->u64s), JSET_CSUM_TYPE(j));
- if (mustfix_fsck_err_on(got != expect, c,
- "journal checksum bad (got %llu expect %llu), sector %lluu",
- got, expect, sector)) {
+ if (fsck_err_on(!bch_checksum_type_valid(c, JSET_CSUM_TYPE(j)), c,
+ "journal entry with unknown csum type %llu sector %lluu",
+ JSET_CSUM_TYPE(j), sector))
+ return JOURNAL_ENTRY_BAD;
+
+ csum = csum_vstruct(c, JSET_CSUM_TYPE(j), journal_nonce(j), j);
+ if (mustfix_fsck_err_on(bch_crc_cmp(csum, j->csum), c,
+ "journal checksum bad, sector %llu", sector)) {
/* XXX: retry IO, when we start retrying checksum errors */
/* XXX: note we might have missing journal entries */
return JOURNAL_ENTRY_BAD;
}
- if (mustfix_fsck_err_on(le64_to_cpu(j->last_seq) > le64_to_cpu(j->seq),
- c, "invalid journal entry: last_seq > seq"))
+ bch_encrypt(c, JSET_CSUM_TYPE(j), journal_nonce(j),
+ j->encrypted_start,
+ vstruct_end(j) - (void *) j->encrypted_start);
+
+ if (mustfix_fsck_err_on(le64_to_cpu(j->last_seq) > le64_to_cpu(j->seq), c,
+ "invalid journal entry: last_seq > seq"))
j->last_seq = j->seq;
- for_each_jset_entry(entry, j) {
+ vstruct_for_each(j, entry) {
struct bkey_i *k;
- if (mustfix_fsck_err_on(jset_keys_next(entry) >
- bkey_idx(j, le32_to_cpu(j->u64s)), c,
+ if (mustfix_fsck_err_on(vstruct_next(entry) >
+ vstruct_last(j), c,
"journal entry extents past end of jset")) {
j->u64s = cpu_to_le64((u64 *) entry - j->_data);
break;
switch (JOURNAL_ENTRY_TYPE(entry)) {
case JOURNAL_ENTRY_BTREE_KEYS:
- for (k = entry->start;
- k < bkey_idx(entry, le16_to_cpu(entry->u64s));
- k = bkey_next(k)) {
+ vstruct_for_each(entry, k) {
ret = journal_validate_key(c, j, entry, k,
bkey_type(entry->level,
entry->btree_id),
le16_to_cpu(entry->u64s) != k->k.u64s, c,
"invalid btree root journal entry: wrong number of keys")) {
journal_entry_null_range(entry,
- jset_keys_next(entry));
+ vstruct_next(entry));
continue;
}
if (mustfix_fsck_err_on(le16_to_cpu(entry->u64s) != 1, c,
"invalid journal seq blacklist entry: bad size")) {
journal_entry_null_range(entry,
- jset_keys_next(entry));
+ vstruct_next(entry));
}
break;
default:
mustfix_fsck_err(c, "invalid journal entry type %llu",
JOURNAL_ENTRY_TYPE(entry));
- journal_entry_null_range(entry, jset_keys_next(entry));
+ journal_entry_null_range(entry, vstruct_next(entry));
break;
}
}
return ret;
}
-static int journal_read_bucket(struct cache *ca, struct journal_list *jlist,
+struct journal_read_buf {
+ void *data;
+ size_t size;
+};
+
+static int journal_read_buf_realloc(struct journal_read_buf *b,
+ size_t new_size)
+{
+ void *n;
+
+ new_size = roundup_pow_of_two(new_size);
+ n = (void *) __get_free_pages(GFP_KERNEL, get_order(new_size));
+ if (!n)
+ return -ENOMEM;
+
+ free_pages((unsigned long) b->data, get_order(b->size));
+ b->data = n;
+ b->size = new_size;
+ return 0;
+}
+
+static int journal_read_bucket(struct cache *ca,
+ struct journal_read_buf *buf,
+ struct journal_list *jlist,
unsigned bucket, u64 *seq, bool *entries_found)
{
struct cache_set *c = ca->set;
struct journal_device *ja = &ca->journal;
struct bio *bio = ja->bio;
- struct jset *j, *data;
- unsigned blocks, sectors_read, bucket_offset = 0;
- unsigned max_entry_sectors = c->journal.entry_size_max >> 9;
- u64 sector = bucket_to_sector(ca,
- journal_bucket(ca->disk_sb.sb, bucket));
+ struct jset *j = NULL;
+ unsigned sectors, sectors_read = 0;
+ u64 offset = bucket_to_sector(ca, ja->buckets[bucket]),
+ end = offset + ca->mi.bucket_size;
bool saw_bad = false;
int ret = 0;
- data = (void *) __get_free_pages(GFP_KERNEL,
- get_order(c->journal.entry_size_max));
- if (!data) {
- mutex_lock(&jlist->cache_set_buffer_lock);
- data = c->journal.buf[0].data;
- }
-
pr_debug("reading %u", bucket);
- while (bucket_offset < ca->mi.bucket_size) {
-reread:
- sectors_read = min_t(unsigned,
- ca->mi.bucket_size - bucket_offset,
- max_entry_sectors);
+ while (offset < end) {
+ if (!sectors_read) {
+reread: sectors_read = min_t(unsigned,
+ end - offset, buf->size >> 9);
- bio_reset(bio);
- bio->bi_bdev = ca->disk_sb.bdev;
- bio->bi_iter.bi_sector = sector + bucket_offset;
- bio->bi_iter.bi_size = sectors_read << 9;
- bio_set_op_attrs(bio, REQ_OP_READ, 0);
- bch_bio_map(bio, data);
-
- ret = submit_bio_wait(bio);
-
- if (cache_fatal_io_err_on(ret, ca,
- "journal read from sector %llu",
- sector + bucket_offset) ||
- bch_meta_read_fault("journal")) {
- ret = -EIO;
- goto err;
- }
+ bio_reset(bio);
+ bio->bi_bdev = ca->disk_sb.bdev;
+ bio->bi_iter.bi_sector = offset;
+ bio->bi_iter.bi_size = sectors_read << 9;
+ bio_set_op_attrs(bio, REQ_OP_READ, 0);
+ bch_bio_map(bio, buf->data);
- /* This function could be simpler now since we no longer write
- * journal entries that overlap bucket boundaries; this means
- * the start of a bucket will always have a valid journal entry
- * if it has any journal entries at all.
- */
+ ret = submit_bio_wait(bio);
- j = data;
- while (sectors_read) {
- ret = journal_entry_validate(c, j,
- sector + bucket_offset,
- ca->mi.bucket_size - bucket_offset,
- sectors_read);
- switch (ret) {
- case BCH_FSCK_OK:
- break;
- case JOURNAL_ENTRY_REREAD:
- goto reread;
- case JOURNAL_ENTRY_NONE:
- if (!saw_bad)
- goto out;
- blocks = 1;
- goto next_block;
- case JOURNAL_ENTRY_BAD:
- saw_bad = true;
- blocks = 1;
- goto next_block;
- default:
- goto err;
- }
+ if (cache_fatal_io_err_on(ret, ca,
+ "journal read from sector %llu",
+ offset) ||
+ bch_meta_read_fault("journal"))
+ return -EIO;
- /*
- * This happens sometimes if we don't have discards on -
- * when we've partially overwritten a bucket with new
- * journal entries. We don't need the rest of the
- * bucket:
- */
- if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])
- goto out;
-
- ja->bucket_seq[bucket] = le64_to_cpu(j->seq);
-
- ret = journal_entry_add(c, jlist, j);
- switch (ret) {
- case JOURNAL_ENTRY_ADD_OK:
- *entries_found = true;
- break;
- case JOURNAL_ENTRY_ADD_OUT_OF_RANGE:
- break;
- default:
- goto err;
+ j = buf->data;
+ }
+
+ ret = journal_entry_validate(c, j, offset,
+ end - offset, sectors_read);
+ switch (ret) {
+ case BCH_FSCK_OK:
+ break;
+ case JOURNAL_ENTRY_REREAD:
+ if (vstruct_bytes(j) > buf->size) {
+ ret = journal_read_buf_realloc(buf,
+ vstruct_bytes(j));
+ if (ret)
+ return ret;
}
+ goto reread;
+ case JOURNAL_ENTRY_NONE:
+ if (!saw_bad)
+ return 0;
+ sectors = c->sb.block_size;
+ goto next_block;
+ case JOURNAL_ENTRY_BAD:
+ saw_bad = true;
+ sectors = c->sb.block_size;
+ goto next_block;
+ default:
+ return ret;
+ }
- if (le64_to_cpu(j->seq) > *seq)
- *seq = le64_to_cpu(j->seq);
-next_block:
- blocks = __set_blocks(j, le32_to_cpu(j->u64s),
- block_bytes(c));
+ /*
+ * This happens sometimes if we don't have discards on -
+ * when we've partially overwritten a bucket with new
+ * journal entries. We don't need the rest of the
+ * bucket:
+ */
+ if (le64_to_cpu(j->seq) < ja->bucket_seq[bucket])
+ return 0;
+
+ ja->bucket_seq[bucket] = le64_to_cpu(j->seq);
- pr_debug("next");
- bucket_offset += blocks * c->sb.block_size;
- sectors_read -= blocks * c->sb.block_size;
- j = ((void *) j) + blocks * block_bytes(c);
+ ret = journal_entry_add(c, jlist, j);
+ switch (ret) {
+ case JOURNAL_ENTRY_ADD_OK:
+ *entries_found = true;
+ break;
+ case JOURNAL_ENTRY_ADD_OUT_OF_RANGE:
+ break;
+ default:
+ return ret;
}
+
+ if (le64_to_cpu(j->seq) > *seq)
+ *seq = le64_to_cpu(j->seq);
+
+ sectors = vstruct_sectors(j, c->block_bits);
+next_block:
+ pr_debug("next");
+ offset += sectors;
+ sectors_read -= sectors;
+ j = ((void *) j) + (sectors << 9);
}
-out:
- ret = 0;
-err:
- if (data == c->journal.buf[0].data)
- mutex_unlock(&jlist->cache_set_buffer_lock);
- else
- free_pages((unsigned long) data,
- get_order(c->journal.entry_size_max));
- return ret;
+ return 0;
}
static void bch_journal_read_device(struct closure *cl)
#define read_bucket(b) \
({ \
bool entries_found = false; \
- int ret = journal_read_bucket(ca, jlist, b, \
- &seq, &entries_found); \
+ ret = journal_read_bucket(ca, &buf, jlist, b, &seq, \
+ &entries_found); \
+ if (ret) \
+ goto err; \
__set_bit(b, bitmap); \
- if (ret) { \
- mutex_lock(&jlist->lock); \
- jlist->ret = ret; \
- mutex_unlock(&jlist->lock); \
- closure_return(cl); \
- } \
entries_found; \
})
struct journal_list *jlist =
container_of(cl->parent, struct journal_list, cl);
struct request_queue *q = bdev_get_queue(ca->disk_sb.bdev);
+ struct journal_read_buf buf = { NULL, 0 };
- unsigned nr_buckets = bch_nr_journal_buckets(ca->disk_sb.sb);
- DECLARE_BITMAP(bitmap, nr_buckets);
+ DECLARE_BITMAP(bitmap, ja->nr);
unsigned i, l, r;
u64 seq = 0;
+ int ret;
- if (!nr_buckets)
- closure_return(cl);
+ if (!ja->nr)
+ goto out;
+
+ bitmap_zero(bitmap, ja->nr);
+ ret = journal_read_buf_realloc(&buf, PAGE_SIZE);
+ if (ret)
+ goto err;
- bitmap_zero(bitmap, nr_buckets);
- pr_debug("%u journal buckets", nr_buckets);
+ pr_debug("%u journal buckets", ja->nr);
/*
* If the device supports discard but not secure discard, we can't do
* the fancy fibonacci hash/binary search because the live journal
* entries might not form a contiguous range:
*/
- for (i = 0; i < nr_buckets; i++)
+ for (i = 0; i < ja->nr; i++)
read_bucket(i);
goto search_done;
* Read journal buckets ordered by golden ratio hash to quickly
* find a sequence of buckets with valid journal entries
*/
- for (i = 0; i < nr_buckets; i++) {
- l = (i * 2654435769U) % nr_buckets;
+ for (i = 0; i < ja->nr; i++) {
+ l = (i * 2654435769U) % ja->nr;
if (test_bit(l, bitmap))
break;
*/
pr_debug("falling back to linear search");
linear_scan:
- for (l = find_first_zero_bit(bitmap, nr_buckets);
- l < nr_buckets;
- l = find_next_zero_bit(bitmap, nr_buckets, l + 1))
+ for (l = find_first_zero_bit(bitmap, ja->nr);
+ l < ja->nr;
+ l = find_next_zero_bit(bitmap, ja->nr, l + 1))
if (read_bucket(l))
goto bsearch;
/* no journal entries on this device? */
- if (l == nr_buckets)
- closure_return(cl);
+ if (l == ja->nr)
+ goto out;
bsearch:
/* Binary search */
- r = find_next_bit(bitmap, nr_buckets, l + 1);
+ r = find_next_bit(bitmap, ja->nr, l + 1);
pr_debug("starting binary search, l %u r %u", l, r);
while (l + 1 < r) {
*/
seq = 0;
- for (i = 0; i < nr_buckets; i++)
+ for (i = 0; i < ja->nr; i++)
if (ja->bucket_seq[i] >= seq &&
- ja->bucket_seq[i] != ja->bucket_seq[(i + 1) % nr_buckets]) {
+ ja->bucket_seq[i] != ja->bucket_seq[(i + 1) % ja->nr]) {
/*
* When journal_next_bucket() goes to allocate for
* the first time, it'll use the bucket after
* reclaimed - journal reclaim will immediately reclaim whatever isn't
* pinned when it first runs:
*/
- ja->last_idx = (ja->cur_idx + 1) % nr_buckets;
+ ja->last_idx = (ja->cur_idx + 1) % ja->nr;
/*
* Read buckets in reverse order until we stop finding more journal
* entries:
*/
- for (i = (ja->cur_idx + nr_buckets - 1) % nr_buckets;
+ for (i = (ja->cur_idx + ja->nr - 1) % ja->nr;
i != ja->cur_idx;
- i = (i + nr_buckets - 1) % nr_buckets)
+ i = (i + ja->nr - 1) % ja->nr)
if (!test_bit(i, bitmap) &&
!read_bucket(i))
break;
-
+out:
+ free_pages((unsigned long) buf.data, get_order(buf.size));
closure_return(cl);
+err:
+ mutex_lock(&jlist->lock);
+ jlist->ret = ret;
+ mutex_unlock(&jlist->lock);
+ goto out;
#undef read_bucket
}
return 0;
}
+static inline bool journal_has_keys(struct list_head *list)
+{
+ struct journal_replay *i;
+ struct jset_entry *entry;
+ struct bkey_i *k, *_n;
+
+ list_for_each_entry(i, list, list)
+ for_each_jset_key(k, _n, entry, &i->j)
+ return true;
+
+ return false;
+}
+
int bch_journal_read(struct cache_set *c, struct list_head *list)
{
struct jset_entry *prio_ptrs;
closure_init_stack(&jlist.cl);
mutex_init(&jlist.lock);
- mutex_init(&jlist.cache_set_buffer_lock);
jlist.head = list;
jlist.ret = 0;
return BCH_FSCK_REPAIR_IMPOSSIBLE;
}
+ fsck_err_on(c->sb.clean && journal_has_keys(list), c,
+ "filesystem marked clean but journal has keys to replay");
+
j = &list_entry(list->prev, struct journal_replay, list)->j;
unfixable_fsck_err_on(le64_to_cpu(j->seq) -
struct bkey_s_c k_s_c = bkey_i_to_s_c(k);
if (btree_type_has_ptrs(type))
- __bch_btree_mark_key(c, type, k_s_c);
+ bch_btree_mark_key_initial(c, type, k_s_c);
}
}
buf->data->last_seq = cpu_to_le64(last_seq(j));
j->prev_buf_sectors =
- __set_blocks(buf->data,
- le32_to_cpu(buf->data->u64s) +
- journal_entry_u64s_reserve(buf),
- block_bytes(c)) * c->sb.block_size;
+ vstruct_blocks_plus(buf->data, c->block_bits,
+ journal_entry_u64s_reserve(buf)) *
+ c->sb.block_size;
BUG_ON(j->prev_buf_sectors > j->cur_buf_sectors);
struct cache *ca)
{
struct journal_device *ja = &ca->journal;
- unsigned nr = bch_nr_journal_buckets(ca->disk_sb.sb);
- unsigned next = (ja->cur_idx + 1) % nr;
- unsigned available = (ja->last_idx + nr - next) % nr;
+ unsigned next = (ja->cur_idx + 1) % ja->nr;
+ unsigned available = (ja->last_idx + ja->nr - next) % ja->nr;
/*
* Hack to avoid a deadlock during journal replay:
* for the previous entry we have to make sure we have space for
* it too:
*/
- if (bch_extent_has_device(e.c, ca->sb.nr_this_dev)) {
+ if (bch_extent_has_device(e.c, ca->dev_idx)) {
if (j->prev_buf_sectors > ca->journal.sectors_free)
buckets_required++;
entries++;
}
+ if (keys) {
+ bch_btree_flush(c);
+
+ /*
+ * Write a new journal entry _before_ we start journalling new data -
+ * otherwise, we could end up with btree node bsets with journal seqs
+ * arbitrarily far in the future vs. the most recently written journal
+ * entry on disk, if we crash before writing the next journal entry:
+ */
+ ret = bch_journal_meta(&c->journal);
+ if (ret)
+ goto err;
+ }
+
bch_info(c, "journal replay done, %i keys in %i entries, seq %llu",
keys, entries, (u64) atomic64_read(&j->seq));
- fsck_err_on(c->sb.clean && keys, c,
- "filesystem marked clean, but journal had keys to replay");
-
bch_journal_set_replay_done(&c->journal);
err:
if (ret)
bch_err(c, "journal replay error: %d", ret);
-fsck_err:
+
bch_journal_entries_free(list);
return ret;
static int bch_set_nr_journal_buckets(struct cache *ca, unsigned nr)
{
- unsigned u64s = bch_journal_buckets_offset(ca->disk_sb.sb) + nr;
+ struct journal_device *ja = &ca->journal;
+ struct bch_sb_field_journal *journal_buckets =
+ bch_sb_get_journal(ca->disk_sb.sb);
+ struct bch_sb_field *f;
u64 *p;
- int ret;
- ret = bch_super_realloc(&ca->disk_sb, u64s);
- if (ret)
- return ret;
+ p = krealloc(ja->bucket_seq, nr * sizeof(u64),
+ GFP_KERNEL|__GFP_ZERO);
+ if (!p)
+ return -ENOMEM;
+
+ ja->bucket_seq = p;
- p = krealloc(ca->journal.bucket_seq,
- nr * sizeof(u64),
+ p = krealloc(ja->buckets, nr * sizeof(u64),
GFP_KERNEL|__GFP_ZERO);
if (!p)
return -ENOMEM;
- ca->journal.bucket_seq = p;
- ca->disk_sb.sb->u64s = cpu_to_le16(u64s);
+ ja->buckets = p;
+
+ f = bch_dev_sb_field_resize(&ca->disk_sb, &journal_buckets->field, nr +
+ sizeof(*journal_buckets) / sizeof(u64));
+ if (!f)
+ return -ENOMEM;
+ f->type = BCH_SB_FIELD_journal;
+ ja->nr = nr;
return 0;
}
int bch_cache_journal_alloc(struct cache *ca)
{
+ struct journal_device *ja = &ca->journal;
+ struct bch_sb_field_journal *journal_buckets;
int ret;
unsigned i;
if (ret)
return ret;
- for (i = 0; i < bch_nr_journal_buckets(ca->disk_sb.sb); i++) {
- unsigned long r = ca->mi.first_bucket + i;
+ journal_buckets = bch_sb_get_journal(ca->disk_sb.sb);
+
+ for (i = 0; i < ja->nr; i++) {
+ u64 bucket = ca->mi.first_bucket + i;
- bch_mark_metadata_bucket(ca, &ca->buckets[r], true);
- set_journal_bucket(ca->disk_sb.sb, i, r);
+ ja->buckets[i] = bucket;
+ journal_buckets->buckets[i] = cpu_to_le64(bucket);
+
+ bch_mark_metadata_bucket(ca, &ca->buckets[bucket], true);
}
return 0;
struct cache *ca;
struct journal_entry_pin *pin;
u64 seq_to_flush = 0;
- unsigned iter, nr, bucket_to_flush;
+ unsigned iter, bucket_to_flush;
unsigned long next_flush;
bool reclaim_lock_held = false, need_flush;
blk_queue_discard(bdev_get_queue(ca->disk_sb.bdev)))
blkdev_issue_discard(ca->disk_sb.bdev,
bucket_to_sector(ca,
- journal_bucket(ca->disk_sb.sb,
- ja->last_idx)),
+ ja->buckets[ja->last_idx]),
ca->mi.bucket_size, GFP_NOIO, 0);
spin_lock(&j->lock);
- ja->last_idx = (ja->last_idx + 1) %
- bch_nr_journal_buckets(ca->disk_sb.sb);
+ ja->last_idx = (ja->last_idx + 1) % ja->nr;
spin_unlock(&j->lock);
wake_up(&j->wait);
* buckets
*/
spin_lock(&j->lock);
- nr = bch_nr_journal_buckets(ca->disk_sb.sb),
- bucket_to_flush = (ja->cur_idx + (nr >> 1)) % nr;
+ bucket_to_flush = (ja->cur_idx + (ja->nr >> 1)) % ja->nr;
seq_to_flush = max_t(u64, seq_to_flush,
ja->bucket_seq[bucket_to_flush]);
spin_unlock(&j->lock);
*/
extent_for_each_ptr_backwards(e, ptr)
if (!(ca = PTR_CACHE(c, ptr)) ||
- ca->mi.state != CACHE_ACTIVE ||
+ ca->mi.state != BCH_MEMBER_STATE_ACTIVE ||
ca->journal.sectors_free <= sectors)
__bch_extent_drop_ptr(e, ptr);
else
*/
group_for_each_cache_rcu(ca, &j->devs, iter) {
struct journal_device *ja = &ca->journal;
- unsigned nr_buckets = bch_nr_journal_buckets(ca->disk_sb.sb);
if (replicas >= replicas_want)
break;
* Check that we can use this device, and aren't already using
* it:
*/
- if (bch_extent_has_device(e.c, ca->sb.nr_this_dev) ||
+ if (bch_extent_has_device(e.c, ca->dev_idx) ||
!journal_dev_buckets_available(j, ca) ||
sectors > ca->mi.bucket_size)
continue;
ja->sectors_free = ca->mi.bucket_size - sectors;
- ja->cur_idx = (ja->cur_idx + 1) % nr_buckets;
+ ja->cur_idx = (ja->cur_idx + 1) % ja->nr;
ja->bucket_seq[ja->cur_idx] = atomic64_read(&j->seq);
extent_ptr_append(bkey_i_to_extent(&j->key),
(struct bch_extent_ptr) {
.offset = bucket_to_sector(ca,
- journal_bucket(ca->disk_sb.sb,
- ja->cur_idx)),
- .dev = ca->sb.nr_this_dev,
+ ja->buckets[ja->cur_idx]),
+ .dev = ca->dev_idx,
});
replicas++;
* If we wanted to be really fancy here, we could sort all the keys in
* the jset and drop keys that were overwritten - probably not worth it:
*/
- for (i = jset->start;
- i < (struct jset_entry *) bkey_idx(jset, le32_to_cpu(jset->u64s)) &&
- (next = jset_keys_next(i), true);
- i = next) {
+ vstruct_for_each_safe(jset, i, next) {
unsigned u64s = le16_to_cpu(i->u64s);
/* Empty entry: */
JOURNAL_ENTRY_TYPE(i) == JOURNAL_ENTRY_TYPE(prev) &&
JOURNAL_ENTRY_TYPE(i) == JOURNAL_ENTRY_BTREE_KEYS &&
le16_to_cpu(prev->u64s) + u64s <= U16_MAX) {
- memmove_u64s_down(jset_keys_next(prev),
+ memmove_u64s_down(vstruct_next(prev),
i->_data,
u64s);
le16_add_cpu(&prev->u64s, u64s);
}
/* Couldn't merge, move i into new position (after prev): */
- prev = prev ? jset_keys_next(prev) : jset->start;
+ prev = prev ? vstruct_next(prev) : jset->start;
if (i != prev)
memmove_u64s_down(prev, i, jset_u64s(u64s));
}
- prev = prev ? jset_keys_next(prev) : jset->start;
+ prev = prev ? vstruct_next(prev) : jset->start;
jset->u64s = cpu_to_le32((u64 *) prev - jset->_data);
}
struct cache_set *c = container_of(j, struct cache_set, journal);
struct cache *ca;
struct journal_buf *w = journal_prev_buf(j);
+ struct jset *jset = w->data;
struct bio *bio;
struct bch_extent_ptr *ptr;
unsigned i, sectors, bytes;
}
mutex_unlock(&c->btree_root_lock);
- journal_write_compact(w->data);
+ journal_write_compact(jset);
+
+ jset->read_clock = cpu_to_le16(c->prio_clock[READ].hand);
+ jset->write_clock = cpu_to_le16(c->prio_clock[WRITE].hand);
+ jset->magic = cpu_to_le64(jset_magic(c));
+ jset->version = cpu_to_le32(BCACHE_JSET_VERSION);
- w->data->read_clock = cpu_to_le16(c->prio_clock[READ].hand);
- w->data->write_clock = cpu_to_le16(c->prio_clock[WRITE].hand);
- w->data->magic = cpu_to_le64(jset_magic(&c->disk_sb));
- w->data->version = cpu_to_le32(BCACHE_JSET_VERSION);
+ SET_JSET_BIG_ENDIAN(jset, CPU_BIG_ENDIAN);
+ SET_JSET_CSUM_TYPE(jset, bch_meta_checksum_type(c));
- SET_JSET_BIG_ENDIAN(w->data, CPU_BIG_ENDIAN);
- SET_JSET_CSUM_TYPE(w->data, c->opts.metadata_checksum);
- w->data->csum = cpu_to_le64(__csum_set(w->data,
- le32_to_cpu(w->data->u64s),
- JSET_CSUM_TYPE(w->data)));
+ bch_encrypt(c, JSET_CSUM_TYPE(jset), journal_nonce(jset),
+ jset->encrypted_start,
+ vstruct_end(jset) - (void *) jset->encrypted_start);
- sectors = __set_blocks(w->data, le32_to_cpu(w->data->u64s),
- block_bytes(c)) * c->sb.block_size;
+ jset->csum = csum_vstruct(c, JSET_CSUM_TYPE(jset),
+ journal_nonce(jset), jset);
+
+ sectors = vstruct_sectors(jset, c->block_bits);
BUG_ON(sectors > j->prev_buf_sectors);
- bytes = __set_bytes(w->data, le32_to_cpu(w->data->u64s));
+ bytes = vstruct_bytes(w->data);
memset((void *) w->data + bytes, 0, (sectors << 9) - bytes);
if (journal_write_alloc(j, sectors)) {
bio->bi_private = ca;
bio_set_op_attrs(bio, REQ_OP_WRITE,
REQ_SYNC|REQ_META|REQ_PREFLUSH|REQ_FUA);
- bch_bio_map(bio, w->data);
+ bch_bio_map(bio, jset);
trace_bcache_journal_write(bio);
closure_bio_submit_punt(bio, cl, c);
}
for_each_cache(ca, c, i)
- if (ca->mi.state == CACHE_ACTIVE &&
+ if (ca->mi.state == BCH_MEMBER_STATE_ACTIVE &&
journal_flushes_device(ca) &&
!bch_extent_has_device(bkey_i_to_s_c_extent(&j->key), i)) {
percpu_ref_get(&ca->ref);
"\tnr\t\t%u\n"
"\tcur_idx\t\t%u (seq %llu)\n"
"\tlast_idx\t%u (seq %llu)\n",
- iter, bch_nr_journal_buckets(ca->disk_sb.sb),
+ iter, ja->nr,
ja->cur_idx, ja->bucket_seq[ja->cur_idx],
ja->last_idx, ja->bucket_seq[ja->last_idx]);
}
spin_lock(&j->lock);
ret = bch_extent_has_device(bkey_i_to_s_c_extent(&j->key),
- ca->sb.nr_this_dev);
+ ca->dev_idx);
spin_unlock(&j->lock);
return ret;
int bch_journal_move(struct cache *ca)
{
- unsigned i, nr_buckets;
u64 last_flushed_seq;
+ struct journal_device *ja = &ca->journal;
struct cache_set *c = ca->set;
struct journal *j = &c->journal;
+ unsigned i;
int ret = 0; /* Success */
if (bch_journal_writing_to_device(ca)) {
last_flushed_seq = last_seq(j);
spin_unlock(&j->lock);
- nr_buckets = bch_nr_journal_buckets(ca->disk_sb.sb);
-
- for (i = 0; i < nr_buckets; i += 1)
- BUG_ON(ca->journal.bucket_seq[i] > last_flushed_seq);
+ for (i = 0; i < ja->nr; i += 1)
+ BUG_ON(ja->bucket_seq[i] > last_flushed_seq);
return ret;
}
+
+void bch_journal_free_cache(struct cache *ca)
+{
+ kfree(ca->journal.buckets);
+ kfree(ca->journal.bucket_seq);
+}
+
+int bch_journal_init_cache(struct cache *ca)
+{
+ struct journal_device *ja = &ca->journal;
+ struct bch_sb_field_journal *journal_buckets =
+ bch_sb_get_journal(ca->disk_sb.sb);
+ unsigned i, journal_entry_pages;
+
+ journal_entry_pages =
+ DIV_ROUND_UP(1U << BCH_SB_JOURNAL_ENTRY_SIZE(ca->disk_sb.sb),
+ PAGE_SECTORS);
+
+ ja->nr = bch_nr_journal_buckets(journal_buckets);
+
+ ja->bucket_seq = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL);
+ if (!ja->bucket_seq)
+ return -ENOMEM;
+
+ ca->journal.bio = bio_kmalloc(GFP_KERNEL, journal_entry_pages);
+ if (!ca->journal.bio)
+ return -ENOMEM;
+
+ ja->buckets = kcalloc(ja->nr, sizeof(u64), GFP_KERNEL);
+ if (!ja->buckets)
+ return -ENOMEM;
+
+ for (i = 0; i < ja->nr; i++)
+ ja->buckets[i] = le64_to_cpu(journal_buckets->buckets[i]);
+
+ return 0;
+}
#include <linux/hash.h>
#include "journal_types.h"
-
-static inline struct jset_entry *jset_keys_next(struct jset_entry *j)
-{
- return (void *) __bkey_idx(j, le16_to_cpu(j->u64s));
-}
+//#include "super-io.h"
/*
* Only used for holding the journal entries we read in btree_journal_read()
unsigned type, enum btree_id id,
unsigned level, unsigned offset)
{
- struct jset_entry *entry = bkey_idx(buf->data, offset);
+ struct jset_entry *entry = vstruct_idx(buf->data, offset);
entry->u64s = cpu_to_le16(u64s);
entry->btree_id = id;
static inline bool is_journal_device(struct cache *ca)
{
- return ca->mi.state == CACHE_ACTIVE && ca->mi.tier == 0;
+ return ca->mi.state == BCH_MEMBER_STATE_ACTIVE && ca->mi.tier == 0;
}
static inline bool journal_flushes_device(struct cache *ca)
int bch_cache_journal_alloc(struct cache *);
-static inline __le64 *__journal_buckets(struct cache_sb *sb)
-{
- return sb->_data + bch_journal_buckets_offset(sb);
-}
-
-static inline u64 journal_bucket(struct cache_sb *sb, unsigned nr)
+static inline unsigned bch_nr_journal_buckets(struct bch_sb_field_journal *j)
{
- return le64_to_cpu(__journal_buckets(sb)[nr]);
-}
-
-static inline void set_journal_bucket(struct cache_sb *sb, unsigned nr, u64 bucket)
-{
- __journal_buckets(sb)[nr] = cpu_to_le64(bucket);
+ return j
+ ? (__le64 *) vstruct_end(&j->field) - j->buckets
+ : 0;
}
int bch_journal_move(struct cache *);
+void bch_journal_free_cache(struct cache *);
+int bch_journal_init_cache(struct cache *);
+
#endif /* _BCACHE_JOURNAL_H */
* ugh: need to get prio_buckets converted over to the eventual new
* transaction machinery
*/
- __le64 prio_buckets[MAX_CACHES_PER_SET];
+ __le64 prio_buckets[BCH_SB_MEMBERS_MAX];
unsigned nr_prio_buckets;
unsigned write_delay_ms;
/*
* Embedded in struct cache. First three fields refer to the array of journal
- * buckets, in cache_sb.
+ * buckets, in bch_sb.
*/
struct journal_device {
/*
* sufficient to read:
*/
unsigned last_idx;
+ unsigned nr;
+ u64 *buckets;
/* Bio for journal reads/writes to this device */
struct bio *bio;
return -ENOSPC;
extent_for_each_ptr(bkey_s_c_to_extent(k), ptr)
- if (ptr->dev == ca->sb.nr_this_dev)
+ if (ptr->dev == ca->dev_idx)
goto found;
BUG();
u64 seen_key_count;
int ret = 0;
- BUG_ON(ca->mi.state == CACHE_ACTIVE);
+ BUG_ON(ca->mi.state == BCH_MEMBER_STATE_ACTIVE);
bch_move_ctxt_init(&ctxt, NULL, SECTORS_IN_FLIGHT_PER_DEVICE);
ctxt.avoid = ca;
!(ret = btree_iter_err(k))) {
if (!bkey_extent_is_data(k.k) ||
!bch_extent_has_device(bkey_s_c_to_extent(k),
- ca->sb.nr_this_dev))
+ ca->dev_idx))
goto next;
ret = issue_migration_move(ca, &ctxt, k);
struct btree *b;
int ret;
- BUG_ON(ca->mi.state == CACHE_ACTIVE);
+ BUG_ON(ca->mi.state == BCH_MEMBER_STATE_ACTIVE);
closure_init_stack(&cl);
for_each_btree_node(&iter, c, id, POS_MIN, 0, b) {
struct bkey_s_c_extent e = bkey_i_to_s_c_extent(&b->key);
retry:
- if (!bch_extent_has_device(e, ca->sb.nr_this_dev))
+ if (!bch_extent_has_device(e, ca->dev_idx))
continue;
ret = bch_btree_node_rewrite(&iter, b, &cl);
for_each_btree_node(&iter, c, id, POS_MIN, 0, b) {
struct bkey_s_c_extent e = bkey_i_to_s_c_extent(&b->key);
- BUG_ON(bch_extent_has_device(e, ca->sb.nr_this_dev));
+ BUG_ON(bch_extent_has_device(e, ca->dev_idx));
}
bch_btree_iter_unlock(&iter);
}
e = bkey_i_to_s_extent(&tmp.key);
extent_for_each_ptr_backwards(e, ptr)
- if (ptr->dev == ca->sb.nr_this_dev)
+ if (ptr->dev == ca->dev_idx)
bch_extent_drop_ptr(e, ptr);
/*
goto advance;
e = bkey_s_c_to_extent(k);
- if (!bch_extent_has_device(e, ca->sb.nr_this_dev))
+ if (!bch_extent_has_device(e, ca->dev_idx))
goto advance;
ret = bch_flag_key_bad(&iter, ca, e);
#include "buckets.h"
#include "io.h"
#include "move.h"
-#include "super.h"
+#include "super-io.h"
#include "keylist.h"
#include <linux/ioprio.h>
bkey_start_pos(&bch_keylist_front(keys)->k));
while (1) {
- struct bkey_i *insert = bch_keylist_front(keys);
+ struct bkey_s_extent insert =
+ bkey_i_to_s_extent(bch_keylist_front(keys));
struct bkey_s_c k = bch_btree_iter_peek_with_holes(&iter);
struct bch_extent_ptr *ptr;
struct bkey_s_extent e;
bkey_reassemble(&new.k, k);
bch_cut_front(iter.pos, &new.k);
- bch_cut_back(insert->k.p, &new.k.k);
+ bch_cut_back(insert.k->p, &new.k.k);
e = bkey_i_to_s_extent(&new.k);
/* hack - promotes can race: */
if (m->promote)
- extent_for_each_ptr(bkey_i_to_s_extent(insert), ptr)
+ extent_for_each_ptr(insert, ptr)
if (bch_extent_has_device(e.c, ptr->dev))
goto nomatch;
ptr = bch_migrate_matching_ptr(m, e);
if (ptr) {
+ int nr_new_dirty = bch_extent_nr_dirty_ptrs(insert.s_c);
unsigned insert_flags =
BTREE_INSERT_ATOMIC|
BTREE_INSERT_NOFAIL;
if (m->move)
insert_flags |= BTREE_INSERT_USE_RESERVE;
- if (m->move)
+ if (m->move) {
+ nr_new_dirty -= !ptr->cached;
__bch_extent_drop_ptr(e, ptr);
+ }
+
+ BUG_ON(nr_new_dirty < 0);
memcpy_u64s(extent_entry_last(e),
- &insert->v,
- bkey_val_u64s(&insert->k));
- e.k->u64s += bkey_val_u64s(&insert->k);
+ insert.v,
+ bkey_val_u64s(insert.k));
+ e.k->u64s += bkey_val_u64s(insert.k);
bch_extent_narrow_crcs(e);
bch_extent_drop_redundant_crcs(e);
bch_extent_normalize(c, e.s);
+ bch_extent_mark_replicas_cached(c, e, nr_new_dirty);
ret = bch_btree_insert_at(c, &op->res,
NULL, op_journal_seq(op),
if (move_ptr)
m->move_ptr = *move_ptr;
- if (bkey_extent_is_cached(k.k))
+ if (bkey_extent_is_cached(k.k) ||
+ (move_ptr && move_ptr->cached))
flags |= BCH_WRITE_CACHED;
bch_write_op_init(&m->op, c, &m->wbio,
if (m->move)
m->op.alloc_reserve = RESERVE_MOVINGGC;
+ m->op.nonce = extent_current_nonce(bkey_s_c_to_extent(k));
m->op.nr_replicas = 1;
m->op.index_update_fn = bch_migrate_index_update;
}
if (bkey_extent_is_data(k.k) &&
(ptr = bch_extent_has_device(bkey_s_c_to_extent(k),
- ca->sb.nr_this_dev)) &&
+ ca->dev_idx)) &&
PTR_BUCKET(ca, ptr)->mark.copygc)
return ptr;
env->envp_idx = 0;
env->buflen = 0;
- notify_var(c, "SET_UUID=%pU", c->disk_sb.user_uuid.b);
+ notify_var(c, "SET_UUID=%pU", c->sb.user_uuid.b);
}
static void notify_get_cache(struct cache *ca)
char buf[BDEVNAME_SIZE];
notify_get(c);
- notify_var(c, "UUID=%pU", ca->disk_sb.sb->disk_uuid.b);
+ notify_var(c, "UUID=%pU", ca->uuid.b);
notify_var(c, "BLOCKDEV=%s", bdevname(ca->disk_sb.bdev, buf));
}
"crc32c",
"crc64",
"siphash",
- "sha1",
NULL
};
};
enum bch_opts {
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
Opt_##_name,
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
Opt_bad_opt,
};
static struct bch_opt_result parse_one_opt(const char *opt)
{
static const struct bch_option opt_table[] = {
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
[Opt_##_name] = { \
.name = #_name, \
.opts = _choices, \
.min = _min, \
.max = _max, \
},
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
}, *i;
for (i = opt_table;
struct bch_opt_result res = parse_one_opt(p);
switch (res.opt) {
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
case Opt_##_name: \
opts->_name = res.val; \
break;
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
case Opt_bad_opt:
return -EINVAL;
extern const char * const bch_uint_opt[];
/* dummy option, for options that aren't stored in the superblock */
-LE64_BITMASK(NO_SB_OPT, struct cache_sb, flags, 0, 0);
-
-#define CACHE_SET_VISIBLE_OPTS() \
- CACHE_SET_OPT(verbose_recovery, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, false) \
- CACHE_SET_OPT(posix_acl, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, false) \
- CACHE_SET_OPT(journal_flush_disabled, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, true) \
- CACHE_SET_OPT(nofsck, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, true) \
- CACHE_SET_OPT(fix_errors, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, true) \
- CACHE_SET_OPT(nochanges, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, 0) \
- CACHE_SET_OPT(noreplay, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, 0) \
- CACHE_SET_OPT(norecovery, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, 0) \
- CACHE_SET_SB_OPTS()
-
-#define CACHE_SET_OPTS() \
- CACHE_SET_OPT(read_only, \
- bch_bool_opt, 0, 2, \
- NO_SB_OPT, 0) \
- CACHE_SET_VISIBLE_OPTS()
+LE64_BITMASK(NO_SB_OPT, struct bch_sb, flags[0], 0, 0);
+
+#define BCH_VISIBLE_OPTS() \
+ BCH_OPT(verbose_recovery, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, false) \
+ BCH_OPT(posix_acl, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, false) \
+ BCH_OPT(journal_flush_disabled, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, true) \
+ BCH_OPT(nofsck, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, true) \
+ BCH_OPT(fix_errors, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, true) \
+ BCH_OPT(nochanges, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, 0) \
+ BCH_OPT(noreplay, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, 0) \
+ BCH_OPT(norecovery, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, 0) \
+ BCH_SB_OPTS()
+
+#define BCH_OPTS() \
+ BCH_OPT(read_only, \
+ bch_bool_opt, 0, 2, \
+ NO_SB_OPT, 0) \
+ BCH_VISIBLE_OPTS()
struct cache_set_opts {
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm)\
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm)\
s8 _name;
- CACHE_SET_OPTS()
-#undef CACHE_SET_OPT
+ BCH_OPTS()
+#undef BCH_OPT
};
static inline struct cache_set_opts cache_set_opts_empty(void)
* Initial options from superblock - here we don't want any options undefined,
* any options the superblock doesn't specify are set to 0:
*/
-static inline struct cache_set_opts cache_superblock_opts(struct cache_sb *sb)
+static inline struct cache_set_opts cache_superblock_opts(struct bch_sb *sb)
{
return (struct cache_set_opts) {
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm)\
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm)\
._name = _sb_opt##_BITS ? _sb_opt(sb) : 0,
- CACHE_SET_OPTS()
-#undef CACHE_SET_OPT
+ BCH_SB_OPTS()
+#undef BCH_OPT
};
}
static inline void cache_set_opts_apply(struct cache_set_opts *dst,
struct cache_set_opts src)
{
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm)\
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm)\
BUILD_BUG_ON(_max > S8_MAX); \
if (src._name >= 0) \
dst->_name = src._name;
- CACHE_SET_OPTS()
-#undef CACHE_SET_OPT
+ BCH_SB_OPTS()
+#undef BCH_OPT
}
int bch_parse_options(struct cache_set_opts *, int, char *);
* https://131002.net/siphash/
*/
-//#include <sys/param.h>
-//#include <sys/systm.h>
-
#include <asm/byteorder.h>
+#include <asm/unaligned.h>
+#include <linux/bitops.h>
#include <linux/string.h>
#include "siphash.h"
-static void SipHash_CRounds(SIPHASH_CTX *, int);
-static void SipHash_Rounds(SIPHASH_CTX *, int);
+static void SipHash_Rounds(SIPHASH_CTX *ctx, int rounds)
+{
+ while (rounds--) {
+ ctx->v[0] += ctx->v[1];
+ ctx->v[2] += ctx->v[3];
+ ctx->v[1] = rol64(ctx->v[1], 13);
+ ctx->v[3] = rol64(ctx->v[3], 16);
+
+ ctx->v[1] ^= ctx->v[0];
+ ctx->v[3] ^= ctx->v[2];
+ ctx->v[0] = rol64(ctx->v[0], 32);
+
+ ctx->v[2] += ctx->v[1];
+ ctx->v[0] += ctx->v[3];
+ ctx->v[1] = rol64(ctx->v[1], 17);
+ ctx->v[3] = rol64(ctx->v[3], 21);
+
+ ctx->v[1] ^= ctx->v[2];
+ ctx->v[3] ^= ctx->v[0];
+ ctx->v[2] = rol64(ctx->v[2], 32);
+ }
+}
+
+static void SipHash_CRounds(SIPHASH_CTX *ctx, const void *ptr, int rounds)
+{
+ u64 m = get_unaligned_le64(ptr);
-void
-SipHash_Init(SIPHASH_CTX *ctx, const SIPHASH_KEY *key)
+ ctx->v[3] ^= m;
+ SipHash_Rounds(ctx, rounds);
+ ctx->v[0] ^= m;
+}
+
+void SipHash_Init(SIPHASH_CTX *ctx, const SIPHASH_KEY *key)
{
u64 k0, k1;
ctx->bytes = 0;
}
-void
-SipHash_Update(SIPHASH_CTX *ctx, int rc, int rf, const void *src, size_t len)
+void SipHash_Update(SIPHASH_CTX *ctx, int rc, int rf,
+ const void *src, size_t len)
{
const u8 *ptr = src;
size_t left, used;
if (len >= left) {
memcpy(&ctx->buf[used], ptr, left);
- SipHash_CRounds(ctx, rc);
+ SipHash_CRounds(ctx, ctx->buf, rc);
len -= left;
ptr += left;
} else {
}
while (len >= sizeof(ctx->buf)) {
- memcpy(ctx->buf, ptr, sizeof(ctx->buf));
- SipHash_CRounds(ctx, rc);
+ SipHash_CRounds(ctx, ptr, rc);
len -= sizeof(ctx->buf);
ptr += sizeof(ctx->buf);
}
memcpy(&ctx->buf[used], ptr, len);
}
-void
-SipHash_Final(void *dst, SIPHASH_CTX *ctx, int rc, int rf)
+void SipHash_Final(void *dst, SIPHASH_CTX *ctx, int rc, int rf)
{
u64 r;
*((__le64 *) dst) = cpu_to_le64(r);
}
-u64
-SipHash_End(SIPHASH_CTX *ctx, int rc, int rf)
+u64 SipHash_End(SIPHASH_CTX *ctx, int rc, int rf)
{
u64 r;
size_t left, used;
memset(&ctx->buf[used], 0, left - 1);
ctx->buf[7] = ctx->bytes;
- SipHash_CRounds(ctx, rc);
+ SipHash_CRounds(ctx, ctx->buf, rc);
ctx->v[2] ^= 0xff;
SipHash_Rounds(ctx, rf);
return (r);
}
-u64
-SipHash(const SIPHASH_KEY *key, int rc, int rf, const void *src, size_t len)
+u64 SipHash(const SIPHASH_KEY *key, int rc, int rf, const void *src, size_t len)
{
SIPHASH_CTX ctx;
SipHash_Init(&ctx, key);
SipHash_Update(&ctx, rc, rf, src, len);
- return (SipHash_End(&ctx, rc, rf));
-}
-
-#define SIP_ROTL(x, b) ((x) << (b)) | ( (x) >> (64 - (b)))
-
-static void
-SipHash_Rounds(SIPHASH_CTX *ctx, int rounds)
-{
- while (rounds--) {
- ctx->v[0] += ctx->v[1];
- ctx->v[2] += ctx->v[3];
- ctx->v[1] = SIP_ROTL(ctx->v[1], 13);
- ctx->v[3] = SIP_ROTL(ctx->v[3], 16);
-
- ctx->v[1] ^= ctx->v[0];
- ctx->v[3] ^= ctx->v[2];
- ctx->v[0] = SIP_ROTL(ctx->v[0], 32);
-
- ctx->v[2] += ctx->v[1];
- ctx->v[0] += ctx->v[3];
- ctx->v[1] = SIP_ROTL(ctx->v[1], 17);
- ctx->v[3] = SIP_ROTL(ctx->v[3], 21);
-
- ctx->v[1] ^= ctx->v[2];
- ctx->v[3] ^= ctx->v[0];
- ctx->v[2] = SIP_ROTL(ctx->v[2], 32);
- }
-}
-
-static void
-SipHash_CRounds(SIPHASH_CTX *ctx, int rounds)
-{
- u64 m = le64_to_cpu(*((__le64 *)ctx->buf));
-
- ctx->v[3] ^= m;
- SipHash_Rounds(ctx, rounds);
- ctx->v[0] ^= m;
+ return SipHash_End(&ctx, rc, rf);
}
#include "btree_iter.h"
#include "checksum.h"
+#include "inode.h"
#include "siphash.h"
#include "super.h"
-#include <crypto/sha1_base.h>
#include <linux/crc32c.h>
+#include <crypto/hash.h>
-static const SIPHASH_KEY bch_siphash_key = {
- .k0 = cpu_to_le64(0x5a9585fd80087730ULL),
- .k1 = cpu_to_le64(0xc8de666d50b45664ULL ),
+struct bch_hash_info {
+ u8 type;
+ union {
+ __le64 crc_key;
+ SIPHASH_KEY siphash_key;
+ };
};
+static inline struct bch_hash_info
+bch_hash_info_init(const struct bch_inode_unpacked *bi)
+{
+ /* XXX ick */
+ struct bch_hash_info info = {
+ .type = (bi->i_flags >> INODE_STR_HASH_OFFSET) &
+ ~(~0 << INODE_STR_HASH_BITS)
+ };
+
+ switch (info.type) {
+ case BCH_STR_HASH_CRC32C:
+ case BCH_STR_HASH_CRC64:
+ info.crc_key = bi->i_hash_seed;
+ break;
+ case BCH_STR_HASH_SIPHASH: {
+ SHASH_DESC_ON_STACK(desc, bch_sha256);
+ u8 digest[crypto_shash_digestsize(bch_sha256)];
+
+ desc->tfm = bch_sha256;
+ desc->flags = 0;
+
+ crypto_shash_digest(desc, (void *) &bi->i_hash_seed,
+ sizeof(bi->i_hash_seed), digest);
+ memcpy(&info.siphash_key, digest, sizeof(info.siphash_key));
+ break;
+ }
+ default:
+ BUG();
+ }
+
+ return info;
+}
+
struct bch_str_hash_ctx {
union {
- u32 crc32c;
- u64 crc64;
- SIPHASH_CTX siphash;
+ u32 crc32c;
+ u64 crc64;
+ SIPHASH_CTX siphash;
};
};
static inline void bch_str_hash_init(struct bch_str_hash_ctx *ctx,
- enum bch_str_hash_type type)
+ const struct bch_hash_info *info)
{
- switch (type) {
+ switch (info->type) {
case BCH_STR_HASH_CRC32C:
- ctx->crc32c = ~0;
+ ctx->crc32c = crc32c(~0, &info->crc_key, sizeof(info->crc_key));
break;
case BCH_STR_HASH_CRC64:
- ctx->crc64 = ~0;
+ ctx->crc64 = bch_crc64_update(~0, &info->crc_key, sizeof(info->crc_key));
break;
case BCH_STR_HASH_SIPHASH:
- SipHash24_Init(&ctx->siphash, &bch_siphash_key);
+ SipHash24_Init(&ctx->siphash, &info->siphash_key);
break;
default:
BUG();
}
static inline void bch_str_hash_update(struct bch_str_hash_ctx *ctx,
- enum bch_str_hash_type type,
- const void *data, size_t len)
+ const struct bch_hash_info *info,
+ const void *data, size_t len)
{
- switch (type) {
+ switch (info->type) {
case BCH_STR_HASH_CRC32C:
ctx->crc32c = crc32c(ctx->crc32c, data, len);
break;
}
static inline u64 bch_str_hash_end(struct bch_str_hash_ctx *ctx,
- enum bch_str_hash_type type)
+ const struct bch_hash_info *info)
{
- switch (type) {
+ switch (info->type) {
case BCH_STR_HASH_CRC32C:
return ctx->crc32c;
case BCH_STR_HASH_CRC64:
}
}
-struct bch_hash_info {
- u64 seed;
- u8 type;
-};
-
-static inline struct bch_hash_info bch_hash_info_init(const struct bch_inode *bi)
-{
- return (struct bch_hash_info) {
- .seed = le64_to_cpu(bi->i_hash_seed),
- .type = INODE_STR_HASH_TYPE(bi),
- };
-}
-
struct bch_hash_desc {
enum btree_id btree_id;
u8 key_type;
--- /dev/null
+
+#include "bcache.h"
+#include "blockdev.h"
+#include "checksum.h"
+#include "error.h"
+#include "io.h"
+#include "journal.h"
+#include "super-io.h"
+#include "super.h"
+#include "vstructs.h"
+
+#include <linux/backing-dev.h>
+
+static inline void __bch_sb_layout_size_assert(void)
+{
+ BUILD_BUG_ON(sizeof(struct bch_sb_layout) != 512);
+}
+
+struct bch_sb_field *bch_sb_field_get(struct bch_sb *sb,
+ enum bch_sb_field_types type)
+{
+ struct bch_sb_field *f;
+
+ /* XXX: need locking around superblock to access optional fields */
+
+ vstruct_for_each(sb, f)
+ if (le32_to_cpu(f->type) == type)
+ return f;
+ return NULL;
+}
+
+void bch_free_super(struct bcache_superblock *sb)
+{
+ if (sb->bio)
+ bio_put(sb->bio);
+ if (!IS_ERR_OR_NULL(sb->bdev))
+ blkdev_put(sb->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
+
+ free_pages((unsigned long) sb->sb, sb->page_order);
+ memset(sb, 0, sizeof(*sb));
+}
+
+static int __bch_super_realloc(struct bcache_superblock *sb, unsigned order)
+{
+ struct bch_sb *new_sb;
+ struct bio *bio;
+
+ if (sb->page_order >= order && sb->sb)
+ return 0;
+
+ if (dynamic_fault("bcache:add:super_realloc"))
+ return -ENOMEM;
+
+ bio = bio_kmalloc(GFP_KERNEL, 1 << order);
+ if (!bio)
+ return -ENOMEM;
+
+ if (sb->bio)
+ bio_put(sb->bio);
+ sb->bio = bio;
+
+ new_sb = (void *) __get_free_pages(GFP_KERNEL, order);
+ if (!new_sb)
+ return -ENOMEM;
+
+ if (sb->sb)
+ memcpy(new_sb, sb->sb, PAGE_SIZE << sb->page_order);
+
+ free_pages((unsigned long) sb->sb, sb->page_order);
+ sb->sb = new_sb;
+
+ sb->page_order = order;
+
+ return 0;
+}
+
+int bch_dev_sb_realloc(struct bcache_superblock *sb, unsigned u64s)
+{
+ u64 new_bytes = __vstruct_bytes(struct bch_sb, u64s);
+ u64 max_bytes = 512 << sb->sb->layout.sb_max_size_bits;
+
+ if (new_bytes > max_bytes) {
+ char buf[BDEVNAME_SIZE];
+
+ pr_err("%s: superblock too big: want %llu but have %llu",
+ bdevname(sb->bdev, buf), new_bytes, max_bytes);
+ return -ENOSPC;
+ }
+
+ return __bch_super_realloc(sb, get_order(new_bytes));
+}
+
+static int bch_fs_sb_realloc(struct cache_set *c, unsigned u64s)
+{
+ u64 bytes = __vstruct_bytes(struct bch_sb, u64s);
+ struct bch_sb *sb;
+ unsigned order = get_order(bytes);
+
+ if (c->disk_sb && order <= c->disk_sb_order)
+ return 0;
+
+ sb = (void *) __get_free_pages(GFP_KERNEL|__GFP_ZERO, order);
+ if (!sb)
+ return -ENOMEM;
+
+ if (c->disk_sb)
+ memcpy(sb, c->disk_sb, PAGE_SIZE << c->disk_sb_order);
+
+ free_pages((unsigned long) c->disk_sb, c->disk_sb_order);
+
+ c->disk_sb = sb;
+ c->disk_sb_order = order;
+ return 0;
+}
+
+static struct bch_sb_field *__bch_sb_field_resize(struct bch_sb *sb,
+ struct bch_sb_field *f,
+ unsigned u64s)
+{
+ unsigned old_u64s = f ? le32_to_cpu(f->u64s) : 0;
+
+ if (!f) {
+ f = vstruct_last(sb);
+ memset(f, 0, sizeof(u64) * u64s);
+ f->u64s = cpu_to_le32(u64s);
+ f->type = 0;
+ } else {
+ void *src, *dst;
+
+ src = vstruct_end(f);
+ f->u64s = cpu_to_le32(u64s);
+ dst = vstruct_end(f);
+
+ memmove(dst, src, vstruct_end(sb) - src);
+
+ if (dst > src)
+ memset(src, 0, dst - src);
+ }
+
+ le32_add_cpu(&sb->u64s, u64s - old_u64s);
+
+ return f;
+
+}
+
+struct bch_sb_field *bch_fs_sb_field_resize(struct cache_set *c,
+ struct bch_sb_field *f,
+ unsigned u64s)
+{
+ ssize_t old_u64s = f ? le32_to_cpu(f->u64s) : 0;
+ ssize_t d = -old_u64s + u64s;
+ struct cache *ca;
+ unsigned i;
+
+ lockdep_assert_held(&c->sb_lock);
+
+ if (bch_fs_sb_realloc(c, le32_to_cpu(c->disk_sb->u64s) + d))
+ return NULL;
+
+ for_each_cache(ca, c, i) {
+ struct bcache_superblock *sb = &ca->disk_sb;
+
+ if (bch_dev_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d)) {
+ percpu_ref_put(&ca->ref);
+ return NULL;
+ }
+ }
+
+ return __bch_sb_field_resize(c->disk_sb, f, u64s);
+}
+
+struct bch_sb_field *bch_dev_sb_field_resize(struct bcache_superblock *sb,
+ struct bch_sb_field *f,
+ unsigned u64s)
+{
+ ssize_t old_u64s = f ? le32_to_cpu(f->u64s) : 0;
+ ssize_t d = -old_u64s + u64s;
+
+ if (bch_dev_sb_realloc(sb, le32_to_cpu(sb->sb->u64s) + d))
+ return NULL;
+
+ return __bch_sb_field_resize(sb->sb, f, u64s);
+}
+
+static const char *validate_sb_layout(struct bch_sb_layout *layout)
+{
+ u64 offset, prev_offset, max_sectors;
+ unsigned i;
+
+ if (uuid_le_cmp(layout->magic, BCACHE_MAGIC))
+ return "Not a bcache superblock layout";
+
+ if (layout->layout_type != 0)
+ return "Invalid superblock layout type";
+
+ if (!layout->nr_superblocks)
+ return "Invalid superblock layout: no superblocks";
+
+ if (layout->nr_superblocks > ARRAY_SIZE(layout->sb_offset))
+ return "Invalid superblock layout: too many superblocks";
+
+ max_sectors = 1 << layout->sb_max_size_bits;
+
+ prev_offset = le64_to_cpu(layout->sb_offset[0]);
+
+ if (prev_offset != BCH_SB_SECTOR)
+ return "Invalid superblock layout: doesn't have default superblock location";
+
+ for (i = 1; i < layout->nr_superblocks; i++) {
+ offset = le64_to_cpu(layout->sb_offset[i]);
+
+ if (offset < prev_offset + max_sectors)
+ return "Invalid superblock layout: superblocks overlap";
+ prev_offset = offset;
+ }
+
+ return NULL;
+}
+
+const char *bch_validate_cache_super(struct bcache_superblock *disk_sb)
+{
+ struct bch_sb *sb = disk_sb->sb;
+ struct bch_sb_field *f;
+ struct bch_sb_field_members *sb_mi;
+ struct bch_sb_field_journal *journal;
+ struct cache_member_cpu mi;
+ const char *err;
+ u16 block_size;
+ unsigned i;
+
+ switch (le64_to_cpu(sb->version)) {
+ case BCACHE_SB_VERSION_CDEV_V4:
+ break;
+ default:
+ return"Unsupported superblock version";
+ }
+
+ if (BCH_SB_INITIALIZED(sb) &&
+ le64_to_cpu(sb->version) != BCACHE_SB_VERSION_CDEV_V4)
+ return "Unsupported superblock version";
+
+ block_size = le16_to_cpu(sb->block_size);
+
+ if (!is_power_of_2(block_size) ||
+ block_size > PAGE_SECTORS)
+ return "Bad block size";
+
+ if (bch_is_zero(sb->user_uuid.b, sizeof(uuid_le)))
+ return "Bad user UUID";
+
+ if (bch_is_zero(sb->uuid.b, sizeof(uuid_le)))
+ return "Bad internal UUID";
+
+ if (!sb->nr_devices ||
+ sb->nr_devices <= sb->dev_idx ||
+ sb->nr_devices > BCH_SB_MEMBERS_MAX)
+ return "Bad cache device number in set";
+
+ if (!BCH_SB_META_REPLICAS_WANT(sb) ||
+ BCH_SB_META_REPLICAS_WANT(sb) >= BCH_REPLICAS_MAX)
+ return "Invalid number of metadata replicas";
+
+ if (!BCH_SB_META_REPLICAS_HAVE(sb) ||
+ BCH_SB_META_REPLICAS_HAVE(sb) >
+ BCH_SB_META_REPLICAS_WANT(sb))
+ return "Invalid number of metadata replicas";
+
+ if (!BCH_SB_DATA_REPLICAS_WANT(sb) ||
+ BCH_SB_DATA_REPLICAS_WANT(sb) >= BCH_REPLICAS_MAX)
+ return "Invalid number of data replicas";
+
+ if (!BCH_SB_DATA_REPLICAS_HAVE(sb) ||
+ BCH_SB_DATA_REPLICAS_HAVE(sb) >
+ BCH_SB_DATA_REPLICAS_WANT(sb))
+ return "Invalid number of data replicas";
+
+ if (!BCH_SB_BTREE_NODE_SIZE(sb))
+ return "Btree node size not set";
+
+ if (!is_power_of_2(BCH_SB_BTREE_NODE_SIZE(sb)))
+ return "Btree node size not a power of two";
+
+ if (BCH_SB_BTREE_NODE_SIZE(sb) > BTREE_NODE_SIZE_MAX)
+ return "Btree node size too large";
+
+ if (BCH_SB_GC_RESERVE(sb) < 5)
+ return "gc reserve percentage too small";
+
+ if (1U << BCH_SB_JOURNAL_ENTRY_SIZE(sb) < block_size)
+ return "max journal entry size too small";
+
+ /* 4 mb max: */
+ if (512U << BCH_SB_JOURNAL_ENTRY_SIZE(sb) > JOURNAL_ENTRY_SIZE_MAX)
+ return "max journal entry size too big";
+
+ if (!sb->time_precision ||
+ le32_to_cpu(sb->time_precision) > NSEC_PER_SEC)
+ return "invalid time precision";
+
+ /* validate layout */
+ err = validate_sb_layout(&sb->layout);
+ if (err)
+ return err;
+
+ vstruct_for_each(sb, f) {
+ if (!f->u64s)
+ return "Invalid superblock: invalid optional field";
+
+ if (vstruct_next(f) > vstruct_last(sb))
+ return "Invalid superblock: invalid optional field";
+
+ if (le32_to_cpu(f->type) >= BCH_SB_FIELD_NR)
+ return "Invalid superblock: unknown optional field type";
+ }
+
+ /* Validate member info: */
+ sb_mi = bch_sb_get_members(sb);
+ if (!sb_mi)
+ return "Invalid superblock: member info area missing";
+
+ if ((void *) (sb_mi->members + sb->nr_devices) >
+ vstruct_end(&sb_mi->field))
+ return "Invalid superblock: bad member info";
+
+ mi = cache_mi_to_cpu_mi(sb_mi->members + sb->dev_idx);
+
+ for (i = 0; i < sb->layout.nr_superblocks; i++) {
+ u64 offset = le64_to_cpu(sb->layout.sb_offset[i]);
+ u64 max_size = 1 << sb->layout.sb_max_size_bits;
+
+ if (offset + max_size > mi.first_bucket * mi.bucket_size)
+ return "Invalid superblock: first bucket comes before end of super";
+ }
+
+ if (mi.nbuckets > LONG_MAX)
+ return "Too many buckets";
+
+ if (mi.nbuckets - mi.first_bucket < 1 << 10)
+ return "Not enough buckets";
+
+ if (!is_power_of_2(mi.bucket_size) ||
+ mi.bucket_size < PAGE_SECTORS ||
+ mi.bucket_size < block_size)
+ return "Bad bucket size";
+
+ if (get_capacity(disk_sb->bdev->bd_disk) <
+ mi.bucket_size * mi.nbuckets)
+ return "Invalid superblock: device too small";
+
+ /* Validate journal buckets: */
+ journal = bch_sb_get_journal(sb);
+ if (journal) {
+ for (i = 0; i < bch_nr_journal_buckets(journal); i++) {
+ u64 b = le64_to_cpu(journal->buckets[i]);
+
+ if (b < mi.first_bucket || b >= mi.nbuckets)
+ return "bad journal bucket";
+ }
+ }
+
+ return NULL;
+}
+
+/* device open: */
+
+static bool bch_is_open_cache(struct block_device *bdev)
+{
+ struct cache_set *c;
+ struct cache *ca;
+ unsigned i;
+
+ rcu_read_lock();
+ list_for_each_entry(c, &bch_cache_sets, list)
+ for_each_cache_rcu(ca, c, i)
+ if (ca->disk_sb.bdev == bdev) {
+ rcu_read_unlock();
+ return true;
+ }
+ rcu_read_unlock();
+ return false;
+}
+
+static bool bch_is_open(struct block_device *bdev)
+{
+ lockdep_assert_held(&bch_register_lock);
+
+ return bch_is_open_cache(bdev) || bch_is_open_backing_dev(bdev);
+}
+
+static const char *bch_blkdev_open(const char *path, void *holder,
+ struct cache_set_opts opts,
+ struct block_device **ret)
+{
+ struct block_device *bdev;
+ fmode_t mode = opts.nochanges > 0
+ ? FMODE_READ
+ : FMODE_READ|FMODE_WRITE|FMODE_EXCL;
+ const char *err;
+
+ *ret = NULL;
+ bdev = blkdev_get_by_path(path, mode, holder);
+
+ if (bdev == ERR_PTR(-EBUSY)) {
+ bdev = lookup_bdev(path);
+ if (IS_ERR(bdev))
+ return "device busy";
+
+ err = bch_is_open(bdev)
+ ? "device already registered"
+ : "device busy";
+
+ bdput(bdev);
+ return err;
+ }
+
+ if (IS_ERR(bdev))
+ return "failed to open device";
+
+ bdev_get_queue(bdev)->backing_dev_info.capabilities |= BDI_CAP_STABLE_WRITES;
+
+ *ret = bdev;
+ return NULL;
+}
+
+/* Update cached mi: */
+int bch_cache_set_mi_update(struct cache_set *c,
+ struct bch_member *mi,
+ unsigned nr_devices)
+{
+ struct cache_member_rcu *new, *old;
+ struct cache *ca;
+ unsigned i;
+
+ lockdep_assert_held(&c->sb_lock);
+
+ new = kzalloc(sizeof(struct cache_member_rcu) +
+ sizeof(struct cache_member_cpu) * nr_devices,
+ GFP_KERNEL);
+ if (!new)
+ return -ENOMEM;
+
+ new->nr_devices = nr_devices;
+
+ for (i = 0; i < nr_devices; i++)
+ new->m[i] = cache_mi_to_cpu_mi(&mi[i]);
+
+ rcu_read_lock();
+ for_each_cache(ca, c, i)
+ ca->mi = new->m[i];
+ rcu_read_unlock();
+
+ old = rcu_dereference_protected(c->members,
+ lockdep_is_held(&c->sb_lock));
+
+ rcu_assign_pointer(c->members, new);
+ if (old)
+ kfree_rcu(old, rcu);
+
+ return 0;
+}
+
+static void bch_sb_update(struct cache_set *c)
+{
+ struct bch_sb *src = c->disk_sb;
+
+ lockdep_assert_held(&c->sb_lock);
+
+ c->sb.uuid = src->uuid;
+ c->sb.user_uuid = src->user_uuid;
+ c->sb.block_size = le16_to_cpu(src->block_size);
+ c->sb.btree_node_size = BCH_SB_BTREE_NODE_SIZE(src);
+ c->sb.nr_devices = src->nr_devices;
+ c->sb.clean = BCH_SB_CLEAN(src);
+ c->sb.meta_replicas_have= BCH_SB_META_REPLICAS_HAVE(src);
+ c->sb.data_replicas_have= BCH_SB_DATA_REPLICAS_HAVE(src);
+ c->sb.str_hash_type = BCH_SB_STR_HASH_TYPE(src);
+ c->sb.encryption_type = BCH_SB_ENCRYPTION_TYPE(src);
+ c->sb.time_base_lo = le64_to_cpu(src->time_base_lo);
+ c->sb.time_base_hi = le32_to_cpu(src->time_base_hi);
+ c->sb.time_precision = le32_to_cpu(src->time_precision);
+}
+
+/* doesn't copy member info */
+static void __copy_super(struct bch_sb *dst, struct bch_sb *src)
+{
+ struct bch_sb_field *src_f, *dst_f;
+
+ dst->version = src->version;
+ dst->seq = src->seq;
+ dst->uuid = src->uuid;
+ dst->user_uuid = src->user_uuid;
+ memcpy(dst->label, src->label, sizeof(dst->label));
+
+ dst->block_size = src->block_size;
+ dst->nr_devices = src->nr_devices;
+
+ dst->time_base_lo = src->time_base_lo;
+ dst->time_base_hi = src->time_base_hi;
+ dst->time_precision = src->time_precision;
+
+ memcpy(dst->flags, src->flags, sizeof(dst->flags));
+ memcpy(dst->features, src->features, sizeof(dst->features));
+ memcpy(dst->compat, src->compat, sizeof(dst->compat));
+
+ vstruct_for_each(src, src_f) {
+ if (src_f->type == BCH_SB_FIELD_journal)
+ continue;
+
+ dst_f = bch_sb_field_get(dst, src_f->type);
+ dst_f = __bch_sb_field_resize(dst, dst_f,
+ le32_to_cpu(src_f->u64s));
+
+ memcpy(dst_f, src_f, vstruct_bytes(src_f));
+ }
+}
+
+int bch_sb_to_cache_set(struct cache_set *c, struct bch_sb *src)
+{
+ struct bch_sb_field_members *members =
+ bch_sb_get_members(src);
+ struct bch_sb_field_journal *journal_buckets =
+ bch_sb_get_journal(src);
+ unsigned journal_u64s = journal_buckets
+ ? le32_to_cpu(journal_buckets->field.u64s)
+ : 0;
+
+ lockdep_assert_held(&c->sb_lock);
+
+ if (bch_fs_sb_realloc(c, le32_to_cpu(src->u64s) - journal_u64s))
+ return -ENOMEM;
+
+ if (bch_cache_set_mi_update(c, members->members, src->nr_devices))
+ return -ENOMEM;
+
+ __copy_super(c->disk_sb, src);
+ bch_sb_update(c);
+
+ return 0;
+}
+
+int bch_sb_from_cache_set(struct cache_set *c, struct cache *ca)
+{
+ struct bch_sb *src = c->disk_sb, *dst = ca->disk_sb.sb;
+ struct bch_sb_field_journal *journal_buckets =
+ bch_sb_get_journal(dst);
+ unsigned journal_u64s = journal_buckets
+ ? le32_to_cpu(journal_buckets->field.u64s)
+ : 0;
+ unsigned u64s = le32_to_cpu(src->u64s) + journal_u64s;
+ int ret;
+
+ ret = bch_dev_sb_realloc(&ca->disk_sb, u64s);
+ if (ret)
+ return ret;
+
+ __copy_super(dst, src);
+
+ return 0;
+}
+
+/* read superblock: */
+
+static const char *read_one_super(struct bcache_superblock *sb, u64 offset)
+{
+ struct bch_csum csum;
+ size_t bytes;
+ unsigned order;
+reread:
+ bio_reset(sb->bio);
+ sb->bio->bi_bdev = sb->bdev;
+ sb->bio->bi_iter.bi_sector = BCH_SB_SECTOR;
+ sb->bio->bi_iter.bi_size = PAGE_SIZE << sb->page_order;
+ bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
+ bch_bio_map(sb->bio, sb->sb);
+
+ if (submit_bio_wait(sb->bio))
+ return "IO error";
+
+ if (uuid_le_cmp(sb->sb->magic, BCACHE_MAGIC))
+ return "Not a bcache superblock";
+
+ if (le64_to_cpu(sb->sb->version) != BCACHE_SB_VERSION_CDEV_V4)
+ return "Unsupported superblock version";
+
+ bytes = vstruct_bytes(sb->sb);
+
+ if (bytes > 512 << sb->sb->layout.sb_max_size_bits)
+ return "Bad superblock: too big";
+
+ order = get_order(bytes);
+ if (order > sb->page_order) {
+ if (__bch_super_realloc(sb, order))
+ return "cannot allocate memory";
+ goto reread;
+ }
+
+ if (BCH_SB_CSUM_TYPE(sb->sb) >= BCH_CSUM_NR)
+ return "unknown csum type";
+
+ /* XXX: verify MACs */
+ csum = csum_vstruct(NULL, BCH_SB_CSUM_TYPE(sb->sb),
+ (struct nonce) { 0 }, sb->sb);
+
+ if (bch_crc_cmp(csum, sb->sb->csum))
+ return "bad checksum reading superblock";
+
+ return NULL;
+}
+
+const char *bch_read_super(struct bcache_superblock *sb,
+ struct cache_set_opts opts,
+ const char *path)
+{
+ struct bch_sb_layout layout;
+ const char *err;
+ unsigned i;
+
+ lockdep_assert_held(&bch_register_lock);
+
+ memset(sb, 0, sizeof(*sb));
+
+ err = bch_blkdev_open(path, &sb, opts, &sb->bdev);
+ if (err)
+ return err;
+
+ err = "cannot allocate memory";
+ if (__bch_super_realloc(sb, 0))
+ goto err;
+
+ err = "dynamic fault";
+ if (cache_set_init_fault("read_super"))
+ goto err;
+
+ err = read_one_super(sb, BCH_SB_SECTOR);
+ if (!err)
+ goto got_super;
+
+ pr_err("error reading default super: %s", err);
+
+ /*
+ * Error reading primary superblock - read location of backup
+ * superblocks:
+ */
+ bio_reset(sb->bio);
+ sb->bio->bi_bdev = sb->bdev;
+ sb->bio->bi_iter.bi_sector = BCH_SB_LAYOUT_SECTOR;
+ sb->bio->bi_iter.bi_size = sizeof(struct bch_sb_layout);
+ bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
+ /*
+ * use sb buffer to read layout, since sb buffer is page aligned but
+ * layout won't be:
+ */
+ bch_bio_map(sb->bio, sb->sb);
+
+ err = "IO error";
+ if (submit_bio_wait(sb->bio))
+ goto err;
+
+ memcpy(&layout, sb->sb, sizeof(layout));
+ err = validate_sb_layout(&layout);
+ if (err)
+ goto err;
+
+ for (i = 0; i < layout.nr_superblocks; i++) {
+ u64 offset = le64_to_cpu(layout.sb_offset[i]);
+
+ if (offset == BCH_SB_SECTOR)
+ continue;
+
+ err = read_one_super(sb, offset);
+ if (!err)
+ goto got_super;
+ }
+ goto err;
+got_super:
+ pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
+ le64_to_cpu(sb->sb->version),
+ le64_to_cpu(sb->sb->flags),
+ le64_to_cpu(sb->sb->seq),
+ le16_to_cpu(sb->sb->u64s));
+
+ err = "Superblock block size smaller than device block size";
+ if (le16_to_cpu(sb->sb->block_size) << 9 <
+ bdev_logical_block_size(sb->bdev))
+ goto err;
+
+ return NULL;
+err:
+ bch_free_super(sb);
+ return err;
+}
+
+/* write superblock: */
+
+static void write_super_endio(struct bio *bio)
+{
+ struct cache *ca = bio->bi_private;
+
+ /* XXX: return errors directly */
+
+ cache_fatal_io_err_on(bio->bi_error, ca, "superblock write");
+
+ bch_account_io_completion(ca);
+
+ closure_put(&ca->set->sb_write);
+ percpu_ref_put(&ca->ref);
+}
+
+static bool write_one_super(struct cache_set *c, struct cache *ca, unsigned idx)
+{
+ struct bch_sb *sb = ca->disk_sb.sb;
+ struct bio *bio = ca->disk_sb.bio;
+
+ if (idx >= sb->layout.nr_superblocks)
+ return false;
+
+ sb->offset = sb->layout.sb_offset[idx];
+
+ SET_BCH_SB_CSUM_TYPE(sb, c->opts.metadata_checksum);
+ sb->csum = csum_vstruct(c, BCH_SB_CSUM_TYPE(sb),
+ (struct nonce) { 0 }, sb);
+
+ bio_reset(bio);
+ bio->bi_bdev = ca->disk_sb.bdev;
+ bio->bi_iter.bi_sector = le64_to_cpu(sb->offset);
+ bio->bi_iter.bi_size =
+ roundup(vstruct_bytes(sb),
+ bdev_logical_block_size(ca->disk_sb.bdev));
+ bio->bi_end_io = write_super_endio;
+ bio->bi_private = ca;
+ bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META);
+ bch_bio_map(bio, sb);
+
+ percpu_ref_get(&ca->ref);
+ closure_bio_submit_punt(bio, &c->sb_write, c);
+
+ return true;
+}
+
+void bch_write_super(struct cache_set *c)
+{
+ struct bch_sb_field_members *members =
+ bch_sb_get_members(c->disk_sb);
+ struct closure *cl = &c->sb_write;
+ struct cache *ca;
+ unsigned i, super_idx = 0;
+ bool wrote;
+
+ lockdep_assert_held(&c->sb_lock);
+
+ closure_init_stack(cl);
+
+ le64_add_cpu(&c->disk_sb->seq, 1);
+
+ for_each_cache(ca, c, i)
+ bch_sb_from_cache_set(c, ca);
+
+ do {
+ wrote = false;
+ for_each_cache(ca, c, i)
+ if (write_one_super(c, ca, super_idx))
+ wrote = true;
+
+ closure_sync(cl);
+ super_idx++;
+ } while (wrote);
+
+ /* Make new options visible after they're persistent: */
+ bch_cache_set_mi_update(c, members->members, c->sb.nr_devices);
+ bch_sb_update(c);
+}
+
+void bch_check_mark_super_slowpath(struct cache_set *c, const struct bkey_i *k,
+ bool meta)
+{
+ struct bch_member *mi;
+ struct bkey_s_c_extent e = bkey_i_to_s_c_extent(k);
+ const struct bch_extent_ptr *ptr;
+
+ mutex_lock(&c->sb_lock);
+
+ /* recheck, might have raced */
+ if (bch_check_super_marked(c, k, meta)) {
+ mutex_unlock(&c->sb_lock);
+ return;
+ }
+
+ mi = bch_sb_get_members(c->disk_sb)->members;
+
+ extent_for_each_ptr(e, ptr)
+ if (!ptr->cached)
+ (meta
+ ? SET_BCH_MEMBER_HAS_METADATA
+ : SET_BCH_MEMBER_HAS_DATA)(mi + ptr->dev, true);
+
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
+}
--- /dev/null
+#ifndef _BCACHE_SUPER_IO_H
+#define _BCACHE_SUPER_IO_H
+
+#include "extents.h"
+#include "super_types.h"
+
+#include <asm/byteorder.h>
+
+struct bch_sb_field *bch_sb_field_get(struct bch_sb *, enum bch_sb_field_types);
+
+#define BCH_SB_FIELD_TYPE(_name) \
+static inline struct bch_sb_field_##_name * \
+bch_sb_get_##_name(struct bch_sb *sb) \
+{ \
+ struct bch_sb_field *f = \
+ bch_sb_field_get(sb, BCH_SB_FIELD_##_name); \
+ \
+ return container_of_or_null(f, struct bch_sb_field_##_name, field);\
+}
+
+BCH_SB_FIELD_TYPE(journal);
+BCH_SB_FIELD_TYPE(members);
+BCH_SB_FIELD_TYPE(crypt);
+
+static inline bool bch_sb_test_feature(struct bch_sb *sb,
+ enum bch_sb_features f)
+{
+ unsigned w = f / 64;
+ unsigned b = f % 64;
+
+ return le64_to_cpu(sb->features[w]) & (1ULL << b);
+}
+
+static inline void bch_sb_set_feature(struct bch_sb *sb,
+ enum bch_sb_features f)
+{
+ if (!bch_sb_test_feature(sb, f)) {
+ unsigned w = f / 64;
+ unsigned b = f % 64;
+
+ le64_add_cpu(&sb->features[w], 1ULL << b);
+ }
+}
+
+static inline __le64 bch_sb_magic(struct cache_set *c)
+{
+ __le64 ret;
+ memcpy(&ret, &c->sb.uuid, sizeof(ret));
+ return ret;
+}
+
+static inline __u64 jset_magic(struct cache_set *c)
+{
+ return __le64_to_cpu(bch_sb_magic(c) ^ JSET_MAGIC);
+}
+
+static inline __u64 pset_magic(struct cache_set *c)
+{
+ return __le64_to_cpu(bch_sb_magic(c) ^ PSET_MAGIC);
+}
+
+static inline __u64 bset_magic(struct cache_set *c)
+{
+ return __le64_to_cpu(bch_sb_magic(c) ^ BSET_MAGIC);
+}
+
+static inline struct cache_member_cpu cache_mi_to_cpu_mi(struct bch_member *mi)
+{
+ return (struct cache_member_cpu) {
+ .nbuckets = le64_to_cpu(mi->nbuckets),
+ .first_bucket = le16_to_cpu(mi->first_bucket),
+ .bucket_size = le16_to_cpu(mi->bucket_size),
+ .state = BCH_MEMBER_STATE(mi),
+ .tier = BCH_MEMBER_TIER(mi),
+ .has_metadata = BCH_MEMBER_HAS_METADATA(mi),
+ .has_data = BCH_MEMBER_HAS_DATA(mi),
+ .replacement = BCH_MEMBER_REPLACEMENT(mi),
+ .discard = BCH_MEMBER_DISCARD(mi),
+ .valid = !bch_is_zero(mi->uuid.b, sizeof(uuid_le)),
+ };
+}
+
+int bch_cache_set_mi_update(struct cache_set *, struct bch_member *, unsigned);
+
+int bch_sb_to_cache_set(struct cache_set *, struct bch_sb *);
+int bch_sb_from_cache_set(struct cache_set *, struct cache *);
+
+struct bch_sb_field *bch_fs_sb_field_resize(struct cache_set *,
+ struct bch_sb_field *, unsigned);
+struct bch_sb_field *bch_dev_sb_field_resize(struct bcache_superblock *,
+ struct bch_sb_field *, unsigned);
+
+void bch_free_super(struct bcache_superblock *);
+int bch_super_realloc(struct bcache_superblock *, unsigned);
+
+const char *bch_validate_cache_super(struct bcache_superblock *);
+
+const char *bch_read_super(struct bcache_superblock *,
+ struct cache_set_opts, const char *);
+void bch_write_super(struct cache_set *);
+
+void bch_check_mark_super_slowpath(struct cache_set *,
+ const struct bkey_i *, bool);
+
+#define cache_member_info_get(_c) \
+ (rcu_read_lock(), rcu_dereference((_c)->members))
+
+#define cache_member_info_put() rcu_read_unlock()
+
+static inline bool bch_check_super_marked(struct cache_set *c,
+ const struct bkey_i *k, bool meta)
+{
+ struct bkey_s_c_extent e = bkey_i_to_s_c_extent(k);
+ const struct bch_extent_ptr *ptr;
+ struct cache_member_cpu *mi = cache_member_info_get(c)->m;
+ bool ret = true;
+
+ extent_for_each_ptr(e, ptr)
+ if (!ptr->cached &&
+ !(meta
+ ? mi[ptr->dev].has_metadata
+ : mi[ptr->dev].has_data)) {
+ ret = false;
+ break;
+ }
+
+ cache_member_info_put();
+
+ return ret;
+}
+
+static inline void bch_check_mark_super(struct cache_set *c,
+ const struct bkey_i *k, bool meta)
+{
+ if (bch_check_super_marked(c, k, meta))
+ return;
+
+ bch_check_mark_super_slowpath(c, k, meta);
+}
+
+#endif /* _BCACHE_SUPER_IO_H */
#include "notify.h"
#include "stats.h"
#include "super.h"
+#include "super-io.h"
#include "tier.h"
#include "writeback.h"
#include <linux/backing-dev.h>
#include <linux/blkdev.h>
#include <linux/debugfs.h>
+#include <linux/device.h>
#include <linux/genhd.h>
#include <linux/idr.h>
#include <linux/kthread.h>
static DEFINE_IDR(bch_chardev_minor);
static DECLARE_WAIT_QUEUE_HEAD(bch_read_only_wait);
struct workqueue_struct *bcache_io_wq;
-struct crypto_shash *bch_sha1;
+struct crypto_shash *bch_sha256;
static void bch_cache_stop(struct cache *);
static int bch_cache_online(struct cache *);
-static bool bch_is_open_cache(struct block_device *bdev)
-{
- struct cache_set *c;
- struct cache *ca;
- unsigned i;
-
- rcu_read_lock();
- list_for_each_entry(c, &bch_cache_sets, list)
- for_each_cache_rcu(ca, c, i)
- if (ca->disk_sb.bdev == bdev) {
- rcu_read_unlock();
- return true;
- }
- rcu_read_unlock();
- return false;
-}
-
-static bool bch_is_open(struct block_device *bdev)
-{
- lockdep_assert_held(&bch_register_lock);
-
- return bch_is_open_cache(bdev) || bch_is_open_backing_dev(bdev);
-}
-
-static const char *bch_blkdev_open(const char *path, void *holder,
- struct cache_set_opts opts,
- struct block_device **ret)
-{
- struct block_device *bdev;
- fmode_t mode = opts.nochanges > 0
- ? FMODE_READ
- : FMODE_READ|FMODE_WRITE|FMODE_EXCL;
- const char *err;
-
- *ret = NULL;
- bdev = blkdev_get_by_path(path, mode, holder);
-
- if (bdev == ERR_PTR(-EBUSY)) {
- bdev = lookup_bdev(path);
- if (IS_ERR(bdev))
- return "device busy";
-
- err = bch_is_open(bdev)
- ? "device already registered"
- : "device busy";
-
- bdput(bdev);
- return err;
- }
-
- if (IS_ERR(bdev))
- return "failed to open device";
-
- bdev_get_queue(bdev)->backing_dev_info.capabilities |= BDI_CAP_STABLE_WRITES;
-
- *ret = bdev;
- return NULL;
-}
-
static int bch_congested_fn(void *data, int bdi_bits)
{
struct backing_dev_info *bdi;
return ret;
}
-/* Superblock */
-
-static struct cache_member_cpu cache_mi_to_cpu_mi(struct cache_member *mi)
-{
- return (struct cache_member_cpu) {
- .nbuckets = le64_to_cpu(mi->nbuckets),
- .first_bucket = le16_to_cpu(mi->first_bucket),
- .bucket_size = le16_to_cpu(mi->bucket_size),
- .state = CACHE_STATE(mi),
- .tier = CACHE_TIER(mi),
- .replication_set= CACHE_REPLICATION_SET(mi),
- .has_metadata = CACHE_HAS_METADATA(mi),
- .has_data = CACHE_HAS_DATA(mi),
- .replacement = CACHE_REPLACEMENT(mi),
- .discard = CACHE_DISCARD(mi),
- .valid = !bch_is_zero(mi->uuid.b, sizeof(uuid_le)),
- };
-}
-
-static const char *validate_cache_super(struct bcache_superblock *disk_sb)
-{
- struct cache_sb *sb = disk_sb->sb;
- struct cache_member_cpu mi;
- u16 block_size;
- unsigned i;
-
- switch (le64_to_cpu(sb->version)) {
- case BCACHE_SB_VERSION_CDEV_V0:
- case BCACHE_SB_VERSION_CDEV_WITH_UUID:
- case BCACHE_SB_VERSION_CDEV_V2:
- case BCACHE_SB_VERSION_CDEV_V3:
- break;
- default:
- return"Unsupported superblock version";
- }
-
- if (CACHE_SET_SYNC(sb) &&
- le64_to_cpu(sb->version) != BCACHE_SB_VERSION_CDEV_V3)
- return "Unsupported superblock version";
-
- block_size = le16_to_cpu(sb->block_size);
-
- if (!is_power_of_2(block_size) ||
- block_size > PAGE_SECTORS)
- return "Bad block size";
-
- if (bch_is_zero(sb->disk_uuid.b, sizeof(uuid_le)))
- return "Bad disk UUID";
-
- if (bch_is_zero(sb->user_uuid.b, sizeof(uuid_le)))
- return "Bad user UUID";
-
- if (bch_is_zero(sb->set_uuid.b, sizeof(uuid_le)))
- return "Bad set UUID";
-
- if (!sb->nr_in_set ||
- sb->nr_in_set <= sb->nr_this_dev ||
- sb->nr_in_set > MAX_CACHES_PER_SET)
- return "Bad cache device number in set";
-
- if (!CACHE_SET_META_REPLICAS_WANT(sb) ||
- CACHE_SET_META_REPLICAS_WANT(sb) >= BCH_REPLICAS_MAX)
- return "Invalid number of metadata replicas";
-
- if (!CACHE_SET_META_REPLICAS_HAVE(sb) ||
- CACHE_SET_META_REPLICAS_HAVE(sb) >
- CACHE_SET_META_REPLICAS_WANT(sb))
- return "Invalid number of metadata replicas";
-
- if (!CACHE_SET_DATA_REPLICAS_WANT(sb) ||
- CACHE_SET_DATA_REPLICAS_WANT(sb) >= BCH_REPLICAS_MAX)
- return "Invalid number of data replicas";
-
- if (!CACHE_SET_DATA_REPLICAS_HAVE(sb) ||
- CACHE_SET_DATA_REPLICAS_HAVE(sb) >
- CACHE_SET_DATA_REPLICAS_WANT(sb))
- return "Invalid number of data replicas";
-
- if (CACHE_SB_CSUM_TYPE(sb) >= BCH_CSUM_NR)
- return "Invalid checksum type";
-
- if (!CACHE_SET_BTREE_NODE_SIZE(sb))
- return "Btree node size not set";
-
- if (!is_power_of_2(CACHE_SET_BTREE_NODE_SIZE(sb)))
- return "Btree node size not a power of two";
-
- if (CACHE_SET_BTREE_NODE_SIZE(sb) > BTREE_NODE_SIZE_MAX)
- return "Btree node size too large";
-
- /* Default value, for old filesystems: */
- if (!CACHE_SET_GC_RESERVE(sb))
- SET_CACHE_SET_GC_RESERVE(sb, 10);
-
- if (CACHE_SET_GC_RESERVE(sb) < 5)
- return "gc reserve percentage too small";
-
- if (!CACHE_SET_JOURNAL_ENTRY_SIZE(sb))
- SET_CACHE_SET_JOURNAL_ENTRY_SIZE(sb, 9);
-
- /* 4 mb max: */
- if (512U << CACHE_SET_JOURNAL_ENTRY_SIZE(sb) > JOURNAL_ENTRY_SIZE_MAX)
- return "max journal entry size too big";
-
- if (le16_to_cpu(sb->u64s) < bch_journal_buckets_offset(sb))
- return "Invalid superblock: member info area missing";
-
- mi = cache_mi_to_cpu_mi(sb->members + sb->nr_this_dev);
-
- if (mi.nbuckets > LONG_MAX)
- return "Too many buckets";
-
- if (mi.nbuckets < 1 << 8)
- return "Not enough buckets";
-
- if (!is_power_of_2(mi.bucket_size) ||
- mi.bucket_size < PAGE_SECTORS ||
- mi.bucket_size < block_size)
- return "Bad bucket size";
-
- if (get_capacity(disk_sb->bdev->bd_disk) <
- mi.bucket_size * mi.nbuckets)
- return "Invalid superblock: device too small";
-
- if (le64_to_cpu(sb->offset) +
- (__set_blocks(sb, le16_to_cpu(sb->u64s),
- block_size << 9) * block_size) >
- mi.first_bucket * mi.bucket_size)
- return "Invalid superblock: first bucket comes before end of super";
-
- for (i = 0; i < bch_nr_journal_buckets(sb); i++)
- if (journal_bucket(sb, i) < mi.first_bucket ||
- journal_bucket(sb, i) >= mi.nbuckets)
- return "bad journal bucket";
-
- return NULL;
-}
-
-void free_super(struct bcache_superblock *sb)
-{
- if (sb->bio)
- bio_put(sb->bio);
- if (!IS_ERR_OR_NULL(sb->bdev))
- blkdev_put(sb->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
-
- free_pages((unsigned long) sb->sb, sb->page_order);
- memset(sb, 0, sizeof(*sb));
-}
-
-static int __bch_super_realloc(struct bcache_superblock *sb, unsigned order)
-{
- struct cache_sb *new_sb;
- struct bio *bio;
-
- if (sb->page_order >= order && sb->sb)
- return 0;
-
- new_sb = (void *) __get_free_pages(GFP_KERNEL, order);
- if (!new_sb)
- return -ENOMEM;
-
- bio = (dynamic_fault("bcache:add:super_realloc")
- ? NULL
- : bio_kmalloc(GFP_KERNEL, 1 << order));
- if (!bio) {
- free_pages((unsigned long) new_sb, order);
- return -ENOMEM;
- }
-
- if (sb->sb)
- memcpy(new_sb, sb->sb, PAGE_SIZE << sb->page_order);
-
- free_pages((unsigned long) sb->sb, sb->page_order);
- sb->sb = new_sb;
-
- if (sb->bio)
- bio_put(sb->bio);
- sb->bio = bio;
-
- sb->page_order = order;
-
- return 0;
-}
-
-int bch_super_realloc(struct bcache_superblock *sb, unsigned u64s)
-{
- struct cache_member *mi = sb->sb->members + sb->sb->nr_this_dev;
- char buf[BDEVNAME_SIZE];
- size_t bytes = __set_bytes((struct cache_sb *) NULL, u64s);
- u64 want = bytes + (SB_SECTOR << 9);
-
- u64 first_bucket_offset = (u64) le16_to_cpu(mi->first_bucket) *
- ((u64) le16_to_cpu(mi->bucket_size) << 9);
-
- if (want > first_bucket_offset) {
- pr_err("%s: superblock too big: want %llu but have %llu",
- bdevname(sb->bdev, buf), want, first_bucket_offset);
- return -ENOSPC;
- }
-
- return __bch_super_realloc(sb, get_order(bytes));
-}
-
-static const char *read_super(struct bcache_superblock *sb,
- struct cache_set_opts opts,
- const char *path)
-{
- const char *err;
- unsigned order = 0;
-
- lockdep_assert_held(&bch_register_lock);
-
- memset(sb, 0, sizeof(*sb));
-
- err = bch_blkdev_open(path, &sb, opts, &sb->bdev);
- if (err)
- return err;
-retry:
- err = "cannot allocate memory";
- if (__bch_super_realloc(sb, order))
- goto err;
-
- err = "dynamic fault";
- if (cache_set_init_fault("read_super"))
- goto err;
-
- bio_reset(sb->bio);
- sb->bio->bi_bdev = sb->bdev;
- sb->bio->bi_iter.bi_sector = SB_SECTOR;
- sb->bio->bi_iter.bi_size = PAGE_SIZE << sb->page_order;
- bio_set_op_attrs(sb->bio, REQ_OP_READ, REQ_SYNC|REQ_META);
- bch_bio_map(sb->bio, sb->sb);
-
- err = "IO error";
- if (submit_bio_wait(sb->bio))
- goto err;
-
- err = "Not a bcache superblock";
- if (uuid_le_cmp(sb->sb->magic, BCACHE_MAGIC))
- goto err;
-
- err = "Superblock has incorrect offset";
- if (le64_to_cpu(sb->sb->offset) != SB_SECTOR)
- goto err;
-
- pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
- le64_to_cpu(sb->sb->version),
- le64_to_cpu(sb->sb->flags),
- le64_to_cpu(sb->sb->seq),
- le16_to_cpu(sb->sb->u64s));
-
- err = "Superblock block size smaller than device block size";
- if (le16_to_cpu(sb->sb->block_size) << 9 <
- bdev_logical_block_size(sb->bdev))
- goto err;
-
- order = get_order(__set_bytes(sb->sb, le16_to_cpu(sb->sb->u64s)));
- if (order > sb->page_order)
- goto retry;
-
- err = "bad checksum reading superblock";
- if (le64_to_cpu(sb->sb->csum) !=
- __csum_set(sb->sb, le16_to_cpu(sb->sb->u64s),
- le64_to_cpu(sb->sb->version) <
- BCACHE_SB_VERSION_CDEV_V3
- ? BCH_CSUM_CRC64
- : CACHE_SB_CSUM_TYPE(sb->sb)))
- goto err;
-
- return NULL;
-err:
- free_super(sb);
- return err;
-}
-
-void __write_super(struct cache_set *c, struct bcache_superblock *disk_sb)
-{
- struct cache_sb *sb = disk_sb->sb;
- struct bio *bio = disk_sb->bio;
-
- bio->bi_bdev = disk_sb->bdev;
- bio->bi_iter.bi_sector = SB_SECTOR;
- bio->bi_iter.bi_size =
- roundup(__set_bytes(sb, le16_to_cpu(sb->u64s)),
- bdev_logical_block_size(disk_sb->bdev));
- bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META);
- bch_bio_map(bio, sb);
-
- pr_debug("ver %llu, flags %llu, seq %llu",
- le64_to_cpu(sb->version),
- le64_to_cpu(sb->flags),
- le64_to_cpu(sb->seq));
-
- bch_generic_make_request(bio, c);
-}
-
-static void write_super_endio(struct bio *bio)
-{
- struct cache *ca = bio->bi_private;
-
- /* XXX: return errors directly */
-
- cache_fatal_io_err_on(bio->bi_error, ca, "superblock write");
-
- bch_account_io_completion(ca);
-
- closure_put(&ca->set->sb_write);
- percpu_ref_put(&ca->ref);
-}
-
-static void bcache_write_super_unlock(struct closure *cl)
-{
- struct cache_set *c = container_of(cl, struct cache_set, sb_write);
-
- up(&c->sb_write_mutex);
-}
-
-/* Update cached mi: */
-static int cache_set_mi_update(struct cache_set *c,
- struct cache_member *mi,
- unsigned nr_in_set)
-{
- struct cache_member_rcu *new, *old;
- struct cache *ca;
- unsigned i;
-
- mutex_lock(&c->mi_lock);
-
- new = kzalloc(sizeof(struct cache_member_rcu) +
- sizeof(struct cache_member_cpu) * nr_in_set,
- GFP_KERNEL);
- if (!new) {
- mutex_unlock(&c->mi_lock);
- return -ENOMEM;
- }
-
- new->nr_in_set = nr_in_set;
-
- for (i = 0; i < nr_in_set; i++)
- new->m[i] = cache_mi_to_cpu_mi(&mi[i]);
-
- rcu_read_lock();
- for_each_cache(ca, c, i)
- ca->mi = new->m[i];
- rcu_read_unlock();
-
- old = rcu_dereference_protected(c->members,
- lockdep_is_held(&c->mi_lock));
-
- rcu_assign_pointer(c->members, new);
- if (old)
- kfree_rcu(old, rcu);
-
- mutex_unlock(&c->mi_lock);
- return 0;
-}
-
-/* doesn't copy member info */
-static void __copy_super(struct cache_sb *dst, struct cache_sb *src)
-{
- dst->version = src->version;
- dst->seq = src->seq;
- dst->user_uuid = src->user_uuid;
- dst->set_uuid = src->set_uuid;
- memcpy(dst->label, src->label, SB_LABEL_SIZE);
- dst->flags = src->flags;
- dst->flags2 = src->flags2;
- dst->nr_in_set = src->nr_in_set;
- dst->block_size = src->block_size;
-}
-
-static int cache_sb_to_cache_set(struct cache_set *c, struct cache_sb *src)
-{
- struct cache_member *new;
-
- lockdep_assert_held(&bch_register_lock);
-
- new = kzalloc(sizeof(struct cache_member) * src->nr_in_set,
- GFP_KERNEL);
- if (!new)
- return -ENOMEM;
-
- memcpy(new, src->members,
- src->nr_in_set * sizeof(struct cache_member));
-
- if (cache_set_mi_update(c, new, src->nr_in_set)) {
- kfree(new);
- return -ENOMEM;
- }
-
- kfree(c->disk_mi);
- c->disk_mi = new;
-
- __copy_super(&c->disk_sb, src);
-
- c->sb.block_size = le16_to_cpu(src->block_size);
- c->sb.btree_node_size = CACHE_SET_BTREE_NODE_SIZE(src);
- c->sb.nr_in_set = src->nr_in_set;
- c->sb.clean = CACHE_SET_CLEAN(src);
- c->sb.meta_replicas_have= CACHE_SET_META_REPLICAS_HAVE(src);
- c->sb.data_replicas_have= CACHE_SET_DATA_REPLICAS_HAVE(src);
- c->sb.str_hash_type = CACHE_SET_STR_HASH_TYPE(src);
-
- return 0;
-}
-
-static int cache_sb_from_cache_set(struct cache_set *c, struct cache *ca)
-{
- struct cache_sb *src = &c->disk_sb, *dst = ca->disk_sb.sb;
-
- if (src->nr_in_set != dst->nr_in_set) {
- /*
- * We have to preserve the list of journal buckets on the
- * cache's superblock:
- */
- unsigned old_offset = bch_journal_buckets_offset(dst);
- unsigned u64s = bch_journal_buckets_offset(src)
- + bch_nr_journal_buckets(dst);
- int ret = bch_super_realloc(&ca->disk_sb, u64s);
-
- if (ret)
- return ret;
-
- dst->nr_in_set = src->nr_in_set;
- dst->u64s = cpu_to_le16(u64s);
-
- memmove(dst->_data + bch_journal_buckets_offset(dst),
- dst->_data + old_offset,
- bch_nr_journal_buckets(dst) * sizeof(u64));
- }
-
- memcpy(dst->_data,
- c->disk_mi,
- src->nr_in_set * sizeof(struct cache_member));
-
- __copy_super(dst, src);
-
- return 0;
-}
-
-static void __bcache_write_super(struct cache_set *c)
-{
- struct closure *cl = &c->sb_write;
- struct cache *ca;
- unsigned i;
-
- cache_set_mi_update(c, c->disk_mi, c->sb.nr_in_set);
-
- closure_init(cl, &c->cl);
-
- if (c->opts.nochanges)
- goto no_io;
-
- le64_add_cpu(&c->disk_sb.seq, 1);
-
- for_each_cache(ca, c, i) {
- struct cache_sb *sb = ca->disk_sb.sb;
- struct bio *bio = ca->disk_sb.bio;
-
- cache_sb_from_cache_set(c, ca);
-
- SET_CACHE_SB_CSUM_TYPE(sb, c->opts.metadata_checksum);
- sb->csum = cpu_to_le64(__csum_set(sb,
- le16_to_cpu(sb->u64s),
- CACHE_SB_CSUM_TYPE(sb)));
-
- bio_reset(bio);
- bio->bi_bdev = ca->disk_sb.bdev;
- bio->bi_end_io = write_super_endio;
- bio->bi_private = ca;
-
- closure_get(cl);
- percpu_ref_get(&ca->ref);
- __write_super(c, &ca->disk_sb);
- }
-no_io:
- closure_return_with_destructor(cl, bcache_write_super_unlock);
-}
-
-void bcache_write_super(struct cache_set *c)
-{
- down(&c->sb_write_mutex);
- __bcache_write_super(c);
-}
-
-void bch_check_mark_super_slowpath(struct cache_set *c, const struct bkey_i *k,
- bool meta)
-{
- struct cache_member *mi;
- struct bkey_s_c_extent e = bkey_i_to_s_c_extent(k);
- const struct bch_extent_ptr *ptr;
-
- if (!CACHE_SET_SYNC(&c->disk_sb))
- return;
-
- down(&c->sb_write_mutex);
-
- /* recheck, might have raced */
- if (bch_check_super_marked(c, k, meta)) {
- up(&c->sb_write_mutex);
- return;
- }
-
- mi = c->disk_mi;
-
- extent_for_each_ptr(e, ptr)
- if (bch_extent_ptr_is_dirty(c, e, ptr))
- (meta
- ? SET_CACHE_HAS_METADATA
- : SET_CACHE_HAS_DATA)(mi + ptr->dev, true);
-
- __bcache_write_super(c);
-}
-
/* Cache set RO/RW: */
/*
if (!bch_journal_error(&c->journal) &&
!test_bit(CACHE_SET_ERROR, &c->flags)) {
- SET_CACHE_SET_CLEAN(&c->disk_sb, true);
- bcache_write_super(c);
+ mutex_lock(&c->sb_lock);
+ SET_BCH_SB_CLEAN(c->disk_sb, true);
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
}
} else {
/*
err = "error starting allocator thread";
for_each_cache(ca, c, i)
- if (ca->mi.state == CACHE_ACTIVE &&
+ if (ca->mi.state == BCH_MEMBER_STATE_ACTIVE &&
bch_cache_allocator_start(ca)) {
percpu_ref_put(&ca->ref);
goto err;
goto err;
for_each_cache(ca, c, i) {
- if (ca->mi.state != CACHE_ACTIVE)
+ if (ca->mi.state != BCH_MEMBER_STATE_ACTIVE)
continue;
err = "error starting moving GC thread";
cancel_work_sync(&c->bio_submit_work);
cancel_work_sync(&c->read_retry_work);
+ bch_cache_set_encryption_free(c);
bch_btree_cache_free(c);
bch_journal_free(&c->journal);
bch_io_clock_exit(&c->io_clock[WRITE]);
destroy_workqueue(c->wq);
kfree_rcu(rcu_dereference_protected(c->members, 1), rcu); /* shutting down */
- kfree(c->disk_mi);
+ free_pages((unsigned long) c->disk_sb, c->disk_sb_order);
kfree(c);
module_put(THIS_MODULE);
}
static unsigned cache_set_nr_devices(struct cache_set *c)
{
+ struct bch_sb_field_members *mi;
unsigned i, nr = 0;
- struct cache_member *mi = c->disk_mi;
- lockdep_assert_held(&bch_register_lock);
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
- for (i = 0; i < c->disk_sb.nr_in_set; i++)
- if (!bch_is_zero(mi[i].uuid.b, sizeof(uuid_le)))
+ for (i = 0; i < c->disk_sb->nr_devices; i++)
+ if (!bch_is_zero(mi->members[i].uuid.b, sizeof(uuid_le)))
nr++;
+ mutex_unlock(&c->sb_lock);
+
return nr;
}
{
unsigned i, nr = 0;
- for (i = 0; i < c->sb.nr_in_set; i++)
+ for (i = 0; i < c->sb.nr_devices; i++)
if (c->cache[i])
nr++;
#define alloc_bucket_pages(gfp, ca) \
((void *) __get_free_pages(__GFP_ZERO|gfp, ilog2(bucket_pages(ca))))
-static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb,
+static struct cache_set *bch_cache_set_alloc(struct bch_sb *sb,
struct cache_set_opts opts)
{
struct cache_set *c;
c->minor = -1;
- sema_init(&c->sb_write_mutex, 1);
+ mutex_init(&c->sb_lock);
INIT_RADIX_TREE(&c->devices, GFP_KERNEL);
mutex_init(&c->btree_cache_lock);
mutex_init(&c->bucket_lock);
mutex_init(&c->btree_root_lock);
INIT_WORK(&c->read_only_work, bch_cache_set_read_only_work);
- mutex_init(&c->mi_lock);
init_rwsem(&c->gc_lock);
mutex_init(&c->uevent_lock);
- if (cache_sb_to_cache_set(c, sb))
+ mutex_lock(&c->sb_lock);
+
+ if (bch_sb_to_cache_set(c, sb)) {
+ mutex_unlock(&c->sb_lock);
goto err;
+ }
+
+ mutex_unlock(&c->sb_lock);
- scnprintf(c->name, sizeof(c->name), "%pU", &c->disk_sb.user_uuid);
+ scnprintf(c->name, sizeof(c->name), "%pU", &c->sb.user_uuid);
c->opts = cache_superblock_opts(sb);
cache_set_opts_apply(&c->opts, opts);
iter_size = (btree_blocks(c) + 1) * 2 *
sizeof(struct btree_node_iter_set);
- journal_entry_bytes = 512U << CACHE_SET_JOURNAL_ENTRY_SIZE(sb);
+ journal_entry_bytes = 512U << BCH_SB_JOURNAL_ENTRY_SIZE(sb);
if (!(c->wq = alloc_workqueue("bcache",
WQ_FREEZABLE|WQ_MEM_RECLAIM|WQ_HIGHPRI, 1)) ||
mempool_init_page_pool(&c->bio_bounce_pages,
max_t(unsigned,
c->sb.btree_node_size,
- CRC32_EXTENT_SIZE_MAX) /
+ BCH_ENCODED_EXTENT_MAX) /
PAGE_SECTORS, 0) ||
!(c->bucket_stats_percpu = alloc_percpu(struct bucket_stats_cache_set)) ||
lg_lock_init(&c->bucket_stats_lock) ||
bch_io_clock_init(&c->io_clock[WRITE]) ||
bch_journal_alloc(&c->journal, journal_entry_bytes) ||
bch_btree_cache_alloc(c) ||
- bch_compress_init(c))
+ bch_cache_set_encryption_init(c) ||
+ bch_compress_init(c) ||
+ bch_check_set_has_compressed_data(c, c->opts.compression))
goto err;
c->bdi.ra_pages = VM_MAX_READAHEAD * 1024 / PAGE_SIZE;
if (IS_ERR(c->chardev))
return PTR_ERR(c->chardev);
- if (kobject_add(&c->kobj, NULL, "%pU", c->disk_sb.user_uuid.b) ||
+ if (kobject_add(&c->kobj, NULL, "%pU", c->sb.user_uuid.b) ||
kobject_add(&c->internal, &c->kobj, "internal") ||
kobject_add(&c->opts_dir, &c->kobj, "options") ||
kobject_add(&c->time_stats, &c->kobj, "time_stats") ||
static const char *run_cache_set(struct cache_set *c)
{
const char *err = "cannot allocate memory";
+ struct bch_sb_field_members *mi;
struct cache *ca;
unsigned i, id;
time64_t now;
* we start testing it.
*/
for_each_cache(ca, c, i)
- cache_sb_from_cache_set(c, ca);
+ bch_sb_from_cache_set(c, ca);
- /*
- * CACHE_SET_SYNC is true if the cache set has already been run
- * and potentially has data.
- * It is false if it is the first time it is run.
- */
-
- if (CACHE_SET_SYNC(&c->disk_sb)) {
+ if (BCH_SB_INITIALIZED(c->disk_sb)) {
ret = bch_journal_read(c, &journal);
if (ret)
goto err;
err = "error starting allocator thread";
for_each_cache(ca, c, i)
- if (ca->mi.state == CACHE_ACTIVE &&
+ if (ca->mi.state == BCH_MEMBER_STATE_ACTIVE &&
bch_cache_allocator_start(ca)) {
percpu_ref_put(&ca->ref);
goto err;
if (c->opts.norecovery)
goto recovery_done;
- /*
- * Write a new journal entry _before_ we start journalling new
- * data - otherwise, we could end up with btree node bsets with
- * journal seqs arbitrarily far in the future vs. the most
- * recently written journal entry on disk, if we crash before
- * writing the next journal entry:
- */
- err = "error writing journal entry";
- if (bch_journal_meta(&c->journal))
- goto err;
-
bch_verbose(c, "starting fsck:");
err = "error in fsck";
ret = bch_fsck(c, !c->opts.nofsck);
if (ret)
goto err;
+
bch_verbose(c, "fsck done");
} else {
- struct bkey_i_inode inode;
+ struct bch_inode_unpacked inode;
+ struct bkey_inode_buf packed_inode;
struct closure cl;
closure_init_stack(&cl);
err = "error starting allocator thread";
for_each_cache(ca, c, i)
- if (ca->mi.state == CACHE_ACTIVE &&
+ if (ca->mi.state == BCH_MEMBER_STATE_ACTIVE &&
bch_cache_allocator_start(ca)) {
percpu_ref_put(&ca->ref);
goto err;
bch_inode_init(c, &inode, 0, 0,
S_IFDIR|S_IRWXU|S_IRUGO|S_IXUGO, 0);
- inode.k.p.inode = BCACHE_ROOT_INO;
+ inode.inum = BCACHE_ROOT_INO;
+
+ bch_inode_pack(&packed_inode, &inode);
err = "error creating root directory";
- if (bch_btree_insert(c, BTREE_ID_INODES, &inode.k_i,
+ if (bch_btree_insert(c, BTREE_ID_INODES,
+ &packed_inode.inode.k_i,
NULL, NULL, NULL, 0))
goto err;
goto err;
}
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
now = ktime_get_seconds();
+
rcu_read_lock();
for_each_cache_rcu(ca, c, i)
- c->disk_mi[ca->sb.nr_this_dev].last_mount = cpu_to_le64(now);
+ mi->members[ca->dev_idx].last_mount = cpu_to_le64(now);
rcu_read_unlock();
- /* Mark cache set as initialized: */
- SET_CACHE_SET_SYNC(&c->disk_sb, true);
- SET_CACHE_SET_CLEAN(&c->disk_sb, false);
- bcache_write_super(c);
+ SET_BCH_SB_INITIALIZED(c->disk_sb, true);
+ SET_BCH_SB_CLEAN(c->disk_sb, false);
+ c->disk_sb->version = BCACHE_SB_VERSION_CDEV;
+
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
err = "dynamic fault";
if (cache_set_init_fault("run_cache_set"))
goto out;
}
-static const char *can_add_cache(struct cache_sb *sb,
+static const char *can_add_cache(struct bch_sb *sb,
struct cache_set *c)
{
+ struct bch_sb_field_members *sb_mi;
+
+ sb_mi = bch_sb_get_members(sb);
+ if (!sb_mi)
+ return "Invalid superblock: member info area missing";
+
if (le16_to_cpu(sb->block_size) != c->sb.block_size)
return "mismatched block size";
- if (le16_to_cpu(sb->members[sb->nr_this_dev].bucket_size) <
- CACHE_SET_BTREE_NODE_SIZE(&c->disk_sb))
+ if (le16_to_cpu(sb_mi->members[sb->dev_idx].bucket_size) <
+ BCH_SB_BTREE_NODE_SIZE(c->disk_sb))
return "new cache bucket_size is too small";
return NULL;
}
-static const char *can_attach_cache(struct cache_sb *sb, struct cache_set *c)
+static const char *can_attach_cache(struct bch_sb *sb, struct cache_set *c)
{
+ struct bch_sb_field_members *mi = bch_sb_get_members(c->disk_sb);
+ struct bch_sb_field_members *dev_mi = bch_sb_get_members(sb);
+ uuid_le dev_uuid = dev_mi->members[sb->dev_idx].uuid;
const char *err;
- bool match;
err = can_add_cache(sb, c);
if (err)
return err;
+ if (bch_is_zero(&dev_uuid, sizeof(dev_uuid)))
+ return "device has been removed";
+
/*
* When attaching an existing device, the cache set superblock must
* already contain member_info with a matching UUID
*/
- match = le64_to_cpu(sb->seq) <= le64_to_cpu(c->disk_sb.seq)
- ? (sb->nr_this_dev < c->disk_sb.nr_in_set &&
- !memcmp(&c->disk_mi[sb->nr_this_dev].uuid,
- &sb->disk_uuid, sizeof(uuid_le)))
- : (sb->nr_this_dev < sb->nr_in_set &&
- !memcmp(&sb->members[sb->nr_this_dev].uuid,
- &sb->disk_uuid, sizeof(uuid_le)));
-
- if (!match)
+ if (sb->dev_idx >= c->disk_sb->nr_devices ||
+ memcmp(&mi->members[sb->dev_idx].uuid,
+ &dev_uuid, sizeof(uuid_le)))
return "cache sb does not match set";
return NULL;
bool bch_cache_read_only(struct cache *ca)
{
struct cache_set *c = ca->set;
+ struct bch_sb_field_members *mi;
char buf[BDEVNAME_SIZE];
bdevname(ca->disk_sb.bdev, buf);
lockdep_assert_held(&bch_register_lock);
- if (ca->mi.state != CACHE_ACTIVE)
+ if (ca->mi.state != BCH_MEMBER_STATE_ACTIVE)
return false;
if (!bch_cache_may_remove(ca)) {
bch_notice(c, "%s read only", bdevname(ca->disk_sb.bdev, buf));
bch_notify_cache_read_only(ca);
- SET_CACHE_STATE(&c->disk_mi[ca->sb.nr_this_dev], CACHE_RO);
- bcache_write_super(c);
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
+ SET_BCH_MEMBER_STATE(&mi->members[ca->dev_idx],
+ BCH_MEMBER_STATE_RO);
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
return true;
}
{
lockdep_assert_held(&bch_register_lock);
- if (ca->mi.state == CACHE_ACTIVE)
+ if (ca->mi.state == BCH_MEMBER_STATE_ACTIVE)
return NULL;
if (test_bit(CACHE_DEV_REMOVING, &ca->flags))
const char *bch_cache_read_write(struct cache *ca)
{
struct cache_set *c = ca->set;
+ struct bch_sb_field_members *mi;
const char *err;
err = __bch_cache_read_write(c, ca);
if (err)
return err;
- SET_CACHE_STATE(&c->disk_mi[ca->sb.nr_this_dev], CACHE_ACTIVE);
- bcache_write_super(c);
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
+ SET_BCH_MEMBER_STATE(&mi->members[ca->dev_idx],
+ BCH_MEMBER_STATE_ACTIVE);
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
return NULL;
}
if (c && c->kobj.state_in_sysfs) {
char buf[12];
- sprintf(buf, "cache%u", ca->sb.nr_this_dev);
+ sprintf(buf, "cache%u", ca->dev_idx);
sysfs_remove_link(&c->kobj, buf);
}
if (ca->kobj.state_in_sysfs)
kobject_del(&ca->kobj);
- free_super(&ca->disk_sb);
+ bch_free_super(&ca->disk_sb);
/*
* bch_cache_stop can be called in the middle of initialization
* However, they were zeroed when the object was allocated.
*/
+ bch_journal_free_cache(ca);
free_percpu(ca->sectors_written);
bioset_exit(&ca->replica_set);
free_percpu(ca->bucket_stats_percpu);
- kfree(ca->journal.bucket_seq);
free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));
kfree(ca->prio_buckets);
kfree(ca->bio_prio);
lockdep_assert_held(&bch_register_lock);
if (c) {
- BUG_ON(rcu_access_pointer(c->cache[ca->sb.nr_this_dev]) != ca);
- rcu_assign_pointer(c->cache[ca->sb.nr_this_dev], NULL);
+ BUG_ON(rcu_access_pointer(c->cache[ca->dev_idx]) != ca);
+ rcu_assign_pointer(c->cache[ca->dev_idx], NULL);
}
call_rcu(&ca->free_rcu, bch_cache_free_rcu);
static void bch_cache_remove_work(struct work_struct *work)
{
struct cache *ca = container_of(work, struct cache, remove_work);
+ struct bch_sb_field_members *mi;
struct cache_set *c = ca->set;
char name[BDEVNAME_SIZE];
bool force = test_bit(CACHE_DEV_FORCE_REMOVE, &ca->flags);
- unsigned dev = ca->sb.nr_this_dev;
+ unsigned dev_idx = ca->dev_idx;
bdevname(ca->disk_sb.bdev, name);
if (!ca->mi.has_data) {
/* Nothing to do: */
} else if (!bch_move_data_off_device(ca)) {
- lockdep_assert_held(&bch_register_lock);
- SET_CACHE_HAS_DATA(&c->disk_mi[ca->sb.nr_this_dev], false);
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
+ SET_BCH_MEMBER_HAS_DATA(&mi->members[ca->dev_idx], false);
- bcache_write_super(c);
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
} else if (force) {
bch_flag_data_bad(ca);
- lockdep_assert_held(&bch_register_lock);
- SET_CACHE_HAS_DATA(&c->disk_mi[ca->sb.nr_this_dev], false);
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
+ SET_BCH_MEMBER_HAS_DATA(&mi->members[ca->dev_idx], false);
- bcache_write_super(c);
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
} else {
bch_err(c, "Remove of %s failed, unable to migrate data off",
name);
if (!ca->mi.has_metadata) {
/* Nothing to do: */
} else if (!bch_move_meta_data_off_device(ca)) {
- lockdep_assert_held(&bch_register_lock);
- SET_CACHE_HAS_METADATA(&c->disk_mi[ca->sb.nr_this_dev], false);
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
+ SET_BCH_MEMBER_HAS_METADATA(&mi->members[ca->dev_idx], false);
- bcache_write_super(c);
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
} else {
bch_err(c, "Remove of %s failed, unable to migrate metadata off",
name);
bch_notify_cache_removed(ca);
spin_lock(&c->journal.lock);
- c->journal.prio_buckets[dev] = 0;
+ c->journal.prio_buckets[dev_idx] = 0;
spin_unlock(&c->journal.lock);
bch_journal_meta(&c->journal);
lockdep_assert_held(&bch_register_lock);
/*
- * Free this device's slot in the cache_member array - all pointers to
+ * Free this device's slot in the bch_member array - all pointers to
* this device must be gone:
*/
- memset(&c->disk_mi[dev].uuid, 0, sizeof(c->disk_mi[dev].uuid));
+ mutex_lock(&c->sb_lock);
+ mi = bch_sb_get_members(c->disk_sb);
+ memset(&mi->members[dev_idx].uuid, 0, sizeof(mi->members[dev_idx].uuid));
+
+ bch_write_super(c);
+ mutex_unlock(&c->sb_lock);
- bcache_write_super(c);
mutex_unlock(&bch_register_lock);
closure_put(&c->cl);
lockdep_assert_held(&bch_register_lock);
- sprintf(buf, "cache%u", ca->sb.nr_this_dev);
+ sprintf(buf, "cache%u", ca->dev_idx);
if (kobject_add(&ca->kobj,
&part_to_dev(ca->disk_sb.bdev->bd_part)->kobj,
struct cache_set *c,
struct cache **ret)
{
+ struct bch_member *member;
size_t reserve_none, movinggc_reserve, free_inc_reserve, total_reserve;
size_t heap_size;
- unsigned i, journal_entry_pages;
+ unsigned i;
const char *err = "cannot allocate memory";
struct cache *ca;
- if (c->sb.nr_in_set == 1)
+ if (c->sb.nr_devices == 1)
bdevname(sb->bdev, c->name);
if (cache_set_init_fault("cache_alloc"))
spin_lock_init(&ca->self.lock);
ca->self.nr_devices = 1;
rcu_assign_pointer(ca->self.d[0].dev, ca);
- ca->sb.nr_this_dev = sb->sb->nr_this_dev;
+ ca->dev_idx = sb->sb->dev_idx;
INIT_WORK(&ca->free_work, bch_cache_free_work);
INIT_WORK(&ca->remove_work, bch_cache_remove_work);
if (cache_set_init_fault("cache_alloc"))
goto err;
- ca->mi = cache_mi_to_cpu_mi(ca->disk_sb.sb->members +
- ca->disk_sb.sb->nr_this_dev);
+ member = bch_sb_get_members(ca->disk_sb.sb)->members +
+ ca->disk_sb.sb->dev_idx;
+
+ ca->mi = cache_mi_to_cpu_mi(member);
+ ca->uuid = member->uuid;
ca->bucket_bits = ilog2(ca->mi.bucket_size);
/* XXX: tune these */
free_inc_reserve = movinggc_reserve / 2;
heap_size = movinggc_reserve * 8;
- journal_entry_pages =
- DIV_ROUND_UP(1U << CACHE_SET_JOURNAL_ENTRY_SIZE(ca->disk_sb.sb),
- PAGE_SECTORS);
-
if (!init_fifo(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||
!init_fifo(&ca->free[RESERVE_BTREE], BTREE_NODE_RESERVE, GFP_KERNEL) ||
!init_fifo(&ca->free[RESERVE_MOVINGGC],
2, GFP_KERNEL)) ||
!(ca->disk_buckets = alloc_bucket_pages(GFP_KERNEL, ca)) ||
!(ca->bucket_stats_percpu = alloc_percpu(struct bucket_stats_cache)) ||
- !(ca->journal.bucket_seq = kcalloc(bch_nr_journal_buckets(ca->disk_sb.sb),
- sizeof(u64), GFP_KERNEL)) ||
- !(ca->journal.bio = bio_kmalloc(GFP_KERNEL, journal_entry_pages)) ||
- !(ca->bio_prio = bio_kmalloc(GFP_KERNEL, bucket_pages(ca))) ||
+ !(ca->bio_prio = bio_kmalloc(GFP_NOIO, bucket_pages(ca))) ||
bioset_init(&ca->replica_set, 4,
offsetof(struct bch_write_bio, bio)) ||
- !(ca->sectors_written = alloc_percpu(*ca->sectors_written)))
+ !(ca->sectors_written = alloc_percpu(*ca->sectors_written)) ||
+ bch_journal_init_cache(ca))
goto err;
ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
ca->copygc_write_point.group = &ca->self;
ca->tiering_write_point.group = &ca->self;
- kobject_get(&c->kobj);
- ca->set = c;
-
- kobject_get(&ca->kobj);
- rcu_assign_pointer(c->cache[ca->sb.nr_this_dev], ca);
-
- if (le64_to_cpu(ca->disk_sb.sb->seq) > le64_to_cpu(c->disk_sb.seq))
- cache_sb_to_cache_set(c, ca->disk_sb.sb);
-
/*
* Increase journal write timeout if flushes to this device are
* expensive:
c->journal.write_delay_ms =
max(c->journal.write_delay_ms, 1000U);
+ kobject_get(&c->kobj);
+ ca->set = c;
+
+ kobject_get(&ca->kobj);
+ rcu_assign_pointer(c->cache[ca->dev_idx], ca);
+
+ mutex_lock(&c->sb_lock);
+
+ if (le64_to_cpu(ca->disk_sb.sb->seq) > le64_to_cpu(c->disk_sb->seq))
+ bch_sb_to_cache_set(c, ca->disk_sb.sb);
+
+ mutex_unlock(&c->sb_lock);
+
err = "error creating kobject";
if (c->kobj.state_in_sysfs &&
bch_cache_online(ca))
lockdep_assert_held(&bch_register_lock);
list_for_each_entry(c, &bch_cache_sets, list)
- if (!memcmp(&c->disk_sb.set_uuid, &uuid, sizeof(uuid_le)))
+ if (!memcmp(&c->disk_sb->uuid, &uuid, sizeof(uuid_le)))
return c;
return NULL;
struct cache_set *c;
bool allocated_cache_set = false;
- err = validate_cache_super(sb);
+ err = bch_validate_cache_super(sb);
if (err)
return err;
bdevname(sb->bdev, name);
- c = cache_set_lookup(sb->sb->set_uuid);
+ c = cache_set_lookup(sb->sb->uuid);
if (c) {
err = can_attach_cache(sb->sb, c);
if (err)
struct bcache_superblock sb;
const char *err;
struct cache *ca;
- struct cache_member *new_mi = NULL;
- struct cache_member mi;
- unsigned nr_this_dev, nr_in_set, u64s;
+ struct bch_sb_field *f;
+ struct bch_sb_field_members *mi, *dev_mi;
+ struct bch_member saved_mi;
+ unsigned dev_idx, nr_devices, u64s;
int ret = -EINVAL;
mutex_lock(&bch_register_lock);
- err = read_super(&sb, c->opts, path);
+ err = bch_read_super(&sb, c->opts, path);
if (err)
- goto err_unlock;
+ goto err_unlock_register;
- err = validate_cache_super(&sb);
+ err = bch_validate_cache_super(&sb);
if (err)
- goto err_unlock;
+ goto err_unlock_register;
+
+ mutex_lock(&c->sb_lock);
err = can_add_cache(sb.sb, c);
if (err)
* Preserve the old cache member information (esp. tier)
* before we start bashing the disk stuff.
*/
- mi = sb.sb->members[sb.sb->nr_this_dev];
- mi.last_mount = cpu_to_le64(ktime_get_seconds());
+ dev_mi = bch_sb_get_members(sb.sb);
+ saved_mi = dev_mi->members[sb.sb->dev_idx];
+ saved_mi.last_mount = cpu_to_le64(ktime_get_seconds());
down_read(&c->gc_lock);
if (test_bit(CACHE_SET_GC_FAILURE, &c->flags))
goto no_slot;
- for (nr_this_dev = 0; nr_this_dev < MAX_CACHES_PER_SET; nr_this_dev++)
- if (nr_this_dev >= c->sb.nr_in_set ||
- bch_is_zero(c->disk_mi[nr_this_dev].uuid.b,
+ mi = bch_sb_get_members(c->disk_sb);
+ for (dev_idx = 0; dev_idx < BCH_SB_MEMBERS_MAX; dev_idx++)
+ if (dev_idx >= c->sb.nr_devices ||
+ bch_is_zero(mi->members[dev_idx].uuid.b,
sizeof(uuid_le)))
goto have_slot;
no_slot:
goto err_unlock;
have_slot:
- nr_in_set = max_t(unsigned, nr_this_dev + 1, c->sb.nr_in_set);
up_read(&c->gc_lock);
- u64s = nr_in_set * (sizeof(struct cache_member) / sizeof(u64));
+ nr_devices = max_t(unsigned, dev_idx + 1, c->sb.nr_devices);
+ u64s = (sizeof(struct bch_sb_field_members) +
+ sizeof(struct bch_member) * nr_devices) / sizeof(u64);
err = "no space in superblock for member info";
- if (bch_super_realloc(&sb, u64s))
+
+ f = bch_fs_sb_field_resize(c, &mi->field, u64s);
+ if (!f)
goto err_unlock;
- new_mi = dynamic_fault("bcache:add:member_info_realloc")
- ? NULL
- : kmalloc(sizeof(struct cache_member) * nr_in_set,
- GFP_KERNEL);
- if (!new_mi) {
- err = "cannot allocate memory";
- ret = -ENOMEM;
+ mi = container_of(f, struct bch_sb_field_members, field);
+
+ f = bch_dev_sb_field_resize(&sb, &dev_mi->field, u64s);
+ if (!f)
goto err_unlock;
- }
- memcpy(new_mi, c->disk_mi,
- sizeof(struct cache_member) * nr_in_set);
- new_mi[nr_this_dev] = mi;
+ dev_mi = container_of(f, struct bch_sb_field_members, field);
+ memcpy(dev_mi, mi, u64s * sizeof(u64));
+ dev_mi->members[dev_idx] = saved_mi;
- sb.sb->nr_this_dev = nr_this_dev;
- sb.sb->nr_in_set = nr_in_set;
- sb.sb->u64s = cpu_to_le16(u64s);
- memcpy(sb.sb->members, new_mi,
- sizeof(struct cache_member) * nr_in_set);
+ sb.sb->dev_idx = dev_idx;
+ sb.sb->nr_devices = nr_devices;
- if (cache_set_mi_update(c, new_mi, nr_in_set)) {
+ if (bch_cache_set_mi_update(c, dev_mi->members, nr_devices)) {
err = "cannot allocate memory";
ret = -ENOMEM;
goto err_unlock;
}
/* commit new member info */
- swap(c->disk_mi, new_mi);
- kfree(new_mi);
- new_mi = NULL;
- c->disk_sb.nr_in_set = nr_in_set;
- c->sb.nr_in_set = nr_in_set;
+ memcpy(mi, dev_mi, u64s * sizeof(u64));
+ c->disk_sb->nr_devices = nr_devices;
+ c->sb.nr_devices = nr_devices;
err = cache_alloc(&sb, c, &ca);
if (err)
goto err_unlock;
- bcache_write_super(c);
+ bch_write_super(c);
err = "journal alloc failed";
if (bch_cache_journal_alloc(ca))
bch_notify_cache_added(ca);
- if (ca->mi.state == CACHE_ACTIVE) {
+ if (ca->mi.state == BCH_MEMBER_STATE_ACTIVE) {
err = __bch_cache_read_write(c, ca);
if (err)
goto err_put;
}
kobject_put(&ca->kobj);
+ mutex_unlock(&c->sb_lock);
mutex_unlock(&bch_register_lock);
return 0;
err_put:
bch_cache_stop(ca);
err_unlock:
- kfree(new_mi);
- free_super(&sb);
+ mutex_unlock(&c->sb_lock);
+err_unlock_register:
mutex_unlock(&bch_register_lock);
+ bch_free_super(&sb);
bch_err(c, "Unable to add device: %s", err);
return ret ?: -EINVAL;
goto err;
/*
- * read_super() needs to happen under register_lock, so that the
+ * bch_read_super() needs to happen under register_lock, so that the
* exclusive open is atomic with adding the new cache set to the list of
* cache sets:
*/
mutex_lock(&bch_register_lock);
for (i = 0; i < nr_devices; i++) {
- err = read_super(&sb[i], opts, devices[i]);
+ err = bch_read_super(&sb[i], opts, devices[i]);
if (err)
goto err_unlock;
if (__SB_IS_BDEV(le64_to_cpu(sb[i].sb->version)))
goto err_unlock;
- err = validate_cache_super(&sb[i]);
+ err = bch_validate_cache_super(&sb[i]);
if (err)
goto err_unlock;
}
err = "cache set already registered";
- if (cache_set_lookup(sb->sb->set_uuid))
+ if (cache_set_lookup(sb->sb->uuid))
goto err_unlock;
err = "cannot allocate memory";
mutex_unlock(&bch_register_lock);
err:
for (i = 0; i < nr_devices; i++)
- free_super(&sb[i]);
+ bch_free_super(&sb[i]);
goto out;
}
mutex_lock(&bch_register_lock);
- err = read_super(&sb, opts, path);
+ err = bch_read_super(&sb, opts, path);
if (err)
goto err;
else
err = register_cache(&sb, opts);
- free_super(&sb);
+ bch_free_super(&sb);
err:
mutex_unlock(&bch_register_lock);
return err;
class_destroy(bch_chardev_class);
if (bch_chardev_major > 0)
unregister_chrdev(bch_chardev_major, "bcache");
- if (!IS_ERR_OR_NULL(bch_sha1))
- crypto_free_shash(bch_sha1);
+ if (!IS_ERR_OR_NULL(bch_sha256))
+ crypto_free_shash(bch_sha256);
unregister_reboot_notifier(&reboot);
}
closure_debug_init();
bkey_pack_test();
- bch_sha1 = crypto_alloc_shash("sha1", 0, 0);
- if (IS_ERR(bch_sha1))
+ bch_sha256 = crypto_alloc_shash("sha256", 0, 0);
+ if (IS_ERR(bch_sha256))
goto err;
bch_chardev_major = register_chrdev(0, "bcache-ctl", &bch_chardev_fops);
return s & (ca->mi.bucket_size - 1);
}
-#define cache_member_info_get(_c) \
- (rcu_read_lock(), rcu_dereference((_c)->members))
-
-#define cache_member_info_put() rcu_read_unlock()
-
static inline struct cache *bch_next_cache_rcu(struct cache_set *c,
unsigned *iter)
{
struct cache *ret = NULL;
- while (*iter < c->sb.nr_in_set &&
+ while (*iter < c->sb.nr_devices &&
!(ret = rcu_dereference(c->cache[*iter])))
(*iter)++;
(ca = bch_get_next_cache(c, &(iter))); \
percpu_ref_put(&ca->ref), (iter)++)
-void bch_check_mark_super_slowpath(struct cache_set *,
- const struct bkey_i *, bool);
-
-static inline bool bch_check_super_marked(struct cache_set *c,
- const struct bkey_i *k, bool meta)
-{
- struct bkey_s_c_extent e = bkey_i_to_s_c_extent(k);
- const struct bch_extent_ptr *ptr;
- struct cache_member_cpu *mi = cache_member_info_get(c)->m;
- bool ret = true;
-
- extent_for_each_ptr(e, ptr)
- if (!(meta
- ? mi[ptr->dev].has_metadata
- : mi[ptr->dev].has_data) &&
- bch_extent_ptr_is_dirty(c, e, ptr)) {
- ret = false;
- break;
- }
-
- cache_member_info_put();
-
- return ret;
-}
-
-static inline void bch_check_mark_super(struct cache_set *c,
- const struct bkey_i *k, bool meta)
-{
- if (bch_check_super_marked(c, k, meta))
- return;
-
- bch_check_mark_super_slowpath(c, k, meta);
-}
-
static inline bool bch_cache_may_remove(struct cache *ca)
{
struct cache_set *c = ca->set;
rcu_access_pointer(tier->d[0].dev) != ca;
}
-void free_super(struct bcache_superblock *);
-int bch_super_realloc(struct bcache_superblock *, unsigned);
-void bcache_write_super(struct cache_set *);
-void __write_super(struct cache_set *, struct bcache_superblock *);
-
void bch_cache_set_release(struct kobject *);
void bch_cache_release(struct kobject *);
extern struct list_head bch_cache_sets;
extern struct idr bch_cache_set_minor;
extern struct workqueue_struct *bcache_io_wq;
-extern struct crypto_shash *bch_sha1;
+extern struct crypto_shash *bch_sha256;
extern struct kobj_type bch_cache_set_ktype;
extern struct kobj_type bch_cache_set_internal_ktype;
#define _BCACHE_SUPER_TYPES_H
struct bcache_superblock {
- struct cache_sb *sb;
+ struct bch_sb *sb;
struct block_device *bdev;
struct bio *bio;
unsigned page_order;
#include "bcache.h"
#include "alloc.h"
#include "blockdev.h"
+#include "compress.h"
#include "sysfs.h"
#include "btree_cache.h"
#include "btree_iter.h"
+#include "btree_update.h"
#include "btree_gc.h"
#include "buckets.h"
#include "inode.h"
#include "move.h"
#include "opts.h"
#include "request.h"
+#include "super-io.h"
#include "writeback.h"
#include <linux/blkdev.h>
BCH_DEBUG_PARAMS()
#undef BCH_DEBUG_PARAM
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
static struct attribute sysfs_opt_##_name = { \
.name = #_name, \
.mode = S_IRUGO|(_perm ? S_IWUSR : 0) \
};
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
#define BCH_TIME_STAT(name, frequency_units, duration_units) \
sysfs_time_stats_attribute(name, frequency_units, duration_units);
sysfs_print(state, states[BDEV_STATE(dc->disk_sb.sb)]);
if (attr == &sysfs_label) {
- memcpy(buf, dc->disk_sb.sb->label, SB_LABEL_SIZE);
- buf[SB_LABEL_SIZE + 1] = '\0';
+ memcpy(buf, dc->disk_sb.sb->label, BCH_SB_LABEL_SIZE);
+ buf[BCH_SB_LABEL_SIZE + 1] = '\0';
strcat(buf, "\n");
return strlen(buf);
}
u64 journal_seq = 0;
int ret = 0;
- if (size > SB_LABEL_SIZE)
+ if (size > BCH_SB_LABEL_SIZE)
return -EINVAL;
mutex_lock(&dc->disk.inode_lock);
memcpy(dc->disk_sb.sb->label, buf, size);
- if (size < SB_LABEL_SIZE)
+ if (size < BCH_SB_LABEL_SIZE)
dc->disk_sb.sb->label[size] = '\0';
if (size && dc->disk_sb.sb->label[size - 1] == '\n')
dc->disk_sb.sb->label[size - 1] = '\0';
memcpy(dc->disk.inode.v.i_label,
- dc->disk_sb.sb->label, SB_LABEL_SIZE);
+ dc->disk_sb.sb->label, BCH_SB_LABEL_SIZE);
bch_write_bdev_super(dc, NULL);
if (dc->disk.c)
- ret = bch_inode_update(dc->disk.c, &dc->disk.inode.k_i,
+ ret = bch_btree_update(dc->disk.c, BTREE_ID_INODES,
+ &dc->disk.inode.k_i,
&journal_seq);
mutex_unlock(&dc->disk.inode_lock);
sysfs_hprint(size, le64_to_cpu(d->inode.v.i_size));
if (attr == &sysfs_label) {
- memcpy(buf, d->inode.v.i_label, SB_LABEL_SIZE);
- buf[SB_LABEL_SIZE + 1] = '\0';
+ memcpy(buf, d->inode.v.i_label, BCH_SB_LABEL_SIZE);
+ buf[BCH_SB_LABEL_SIZE + 1] = '\0';
strcat(buf, "\n");
return strlen(buf);
}
}
}
d->inode.v.i_size = cpu_to_le64(v);
- ret = bch_inode_update(d->c, &d->inode.k_i, &journal_seq);
+ ret = bch_btree_update(d->c, BTREE_ID_INODES,
+ &d->inode.k_i, &journal_seq);
mutex_unlock(&d->inode_lock);
mutex_lock(&d->inode_lock);
- memcpy(d->inode.v.i_label, buf, SB_LABEL_SIZE);
- ret = bch_inode_update(d->c, &d->inode.k_i, &journal_seq);
+ memcpy(d->inode.v.i_label, buf, BCH_SB_LABEL_SIZE);
+ ret = bch_btree_update(d->c, BTREE_ID_INODES,
+ &d->inode.k_i, &journal_seq);
mutex_unlock(&d->inode_lock);
sysfs_print(tiering_percent, c->tiering_percent);
sysfs_pd_controller_show(tiering, &c->tiering_pd);
- sysfs_printf(meta_replicas_have, "%llu",
- CACHE_SET_META_REPLICAS_HAVE(&c->disk_sb));
- sysfs_printf(data_replicas_have, "%llu",
- CACHE_SET_DATA_REPLICAS_HAVE(&c->disk_sb));
+ sysfs_printf(meta_replicas_have, "%u", c->sb.meta_replicas_have);
+ sysfs_printf(data_replicas_have, "%u", c->sb.data_replicas_have);
/* Debugging: */
if (attr == &sysfs_compression_stats)
return bch_compression_stats(c, buf);
- sysfs_printf(internal_uuid, "%pU", c->disk_sb.set_uuid.b);
+ sysfs_printf(internal_uuid, "%pU", c->sb.uuid.b);
return 0;
}
{
struct cache_set *c = container_of(kobj, struct cache_set, opts_dir);
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
if (attr == &sysfs_opt_##_name) \
return _choices == bch_bool_opt || _choices == bch_uint_opt\
? snprintf(buf, PAGE_SIZE, "%i\n", c->opts._name)\
: bch_snprint_string_list(buf, PAGE_SIZE, \
_choices, c->opts._name);\
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
return 0;
}
{
struct cache_set *c = container_of(kobj, struct cache_set, opts_dir);
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
if (attr == &sysfs_opt_##_name) { \
ssize_t v = (_choices == bch_bool_opt || \
_choices == bch_uint_opt) \
if (v < 0) \
return v; \
\
- c->opts._name = v; \
+ mutex_lock(&c->sb_lock); \
+ if (attr == &sysfs_opt_compression) { \
+ int ret = bch_check_set_has_compressed_data(c, v);\
+ if (ret) { \
+ mutex_unlock(&c->sb_lock); \
+ return ret; \
+ } \
+ } \
\
- if (_sb_opt##_BITS && v != _sb_opt(&c->disk_sb)) { \
- SET_##_sb_opt(&c->disk_sb, v); \
- bcache_write_super(c); \
+ if (_sb_opt##_BITS && v != _sb_opt(c->disk_sb)) { \
+ SET_##_sb_opt(c->disk_sb, v); \
+ bch_write_super(c); \
} \
\
+ c->opts._name = v; \
+ mutex_unlock(&c->sb_lock); \
+ \
return size; \
}
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
return size;
}
}
static struct attribute *bch_cache_set_opts_dir_files[] = {
-#define CACHE_SET_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
+#define BCH_OPT(_name, _choices, _min, _max, _sb_opt, _perm) \
&sysfs_opt_##_name,
- CACHE_SET_VISIBLE_OPTS()
-#undef CACHE_SET_OPT
+ BCH_VISIBLE_OPTS()
+#undef BCH_OPT
NULL
};
struct cache_set *c = ca->set;
struct bucket_stats_cache stats = bch_bucket_stats_read_cache(ca);
- sysfs_printf(uuid, "%pU\n", ca->disk_sb.sb->disk_uuid.b);
+ sysfs_printf(uuid, "%pU\n", ca->uuid.b);
sysfs_hprint(bucket_size, bucket_bytes(ca));
sysfs_print(bucket_size_bytes, bucket_bytes(ca));
{
struct cache *ca = container_of(kobj, struct cache, kobj);
struct cache_set *c = ca->set;
- struct cache_member *mi = &c->disk_mi[ca->sb.nr_this_dev];
+ struct bch_member *mi;
sysfs_pd_controller_store(copy_gc, &ca->moving_gc_pd);
if (attr == &sysfs_discard) {
bool v = strtoul_or_return(buf);
- if (v != CACHE_DISCARD(mi)) {
- SET_CACHE_DISCARD(mi, v);
- bcache_write_super(c);
+ mutex_lock(&c->sb_lock);
+ mi = &bch_sb_get_members(c->disk_sb)->members[ca->dev_idx];
+
+ if (v != BCH_MEMBER_DISCARD(mi)) {
+ SET_BCH_MEMBER_DISCARD(mi, v);
+ bch_write_super(c);
}
+ mutex_unlock(&c->sb_lock);
}
if (attr == &sysfs_cache_replacement_policy) {
if (v < 0)
return v;
- if ((unsigned) v != CACHE_REPLACEMENT(mi)) {
- SET_CACHE_REPLACEMENT(mi, v);
- bcache_write_super(c);
+ mutex_lock(&c->sb_lock);
+ mi = &bch_sb_get_members(c->disk_sb)->members[ca->dev_idx];
+
+ if ((unsigned) v != BCH_MEMBER_REPLACEMENT(mi)) {
+ SET_BCH_MEMBER_REPLACEMENT(mi, v);
+ bch_write_super(c);
}
+ mutex_unlock(&c->sb_lock);
}
if (attr == &sysfs_state_rw) {
return size;
switch (v) {
- case CACHE_ACTIVE:
+ case BCH_MEMBER_STATE_ACTIVE:
err = bch_cache_read_write(ca);
break;
- case CACHE_RO:
+ case BCH_MEMBER_STATE_RO:
bch_cache_read_only(ca);
break;
- case CACHE_FAILED:
- case CACHE_SPARE:
+ case BCH_MEMBER_STATE_FAILED:
+ case BCH_MEMBER_STATE_SPARE:
/*
* XXX: need to migrate data off and set correct state
*/
#include "io.h"
#include "keylist.h"
#include "move.h"
+#include "super-io.h"
#include "tier.h"
#include <linux/freezer.h>
mi = cache_member_info_get(c);
extent_for_each_ptr(e, ptr)
- if (ptr->dev < mi->nr_in_set &&
+ if (ptr->dev < mi->nr_devices &&
mi->m[ptr->dev].tier >= s->tier_idx)
replicas++;
cache_member_info_put();
--- /dev/null
+#ifndef _VSTRUCTS_H
+#define _VSTRUCTS_H
+
+#include "util.h"
+
+/*
+ * NOTE: we can't differentiate between __le64 and u64 with type_is - this
+ * assumes u64 is little endian:
+ */
+#define __vstruct_u64s(_s) \
+({ \
+ ( type_is((_s)->u64s, u64) ? le64_to_cpu((_s)->u64s) \
+ : type_is((_s)->u64s, u32) ? le32_to_cpu((_s)->u64s) \
+ : type_is((_s)->u64s, u16) ? le16_to_cpu((_s)->u64s) \
+ : ((_s)->u64s)); \
+})
+
+#define __vstruct_bytes(_type, _u64s) \
+({ \
+ BUILD_BUG_ON(offsetof(_type, _data) % sizeof(u64)); \
+ \
+ (offsetof(_type, _data) + (_u64s) * sizeof(u64)); \
+})
+
+#define vstruct_bytes(_s) \
+ __vstruct_bytes(typeof(*(_s)), __vstruct_u64s(_s))
+
+#define __vstruct_blocks(_type, _sector_block_bits, _u64s) \
+ (round_up(__vstruct_bytes(_type, _u64s), \
+ 512 << (_sector_block_bits)) >> (9 + (_sector_block_bits)))
+
+#define vstruct_blocks(_s, _sector_block_bits) \
+ __vstruct_blocks(typeof(*(_s)), _sector_block_bits, __vstruct_u64s(_s))
+
+#define vstruct_blocks_plus(_s, _sector_block_bits, _u64s) \
+ __vstruct_blocks(typeof(*(_s)), _sector_block_bits, \
+ __vstruct_u64s(_s) + (_u64s))
+
+#define vstruct_sectors(_s, _sector_block_bits) \
+ (round_up(vstruct_bytes(_s), 512 << (_sector_block_bits)) >> 9)
+
+#define vstruct_next(_s) \
+ ((typeof(_s)) ((_s)->_data + __vstruct_u64s(_s)))
+#define vstruct_last(_s) \
+ ((typeof(&(_s)->start[0])) ((_s)->_data + __vstruct_u64s(_s)))
+#define vstruct_end(_s) \
+ ((void *) ((_s)->_data + __vstruct_u64s(_s)))
+
+#define vstruct_for_each(_s, _i) \
+ for (_i = (_s)->start; \
+ _i < vstruct_last(_s); \
+ _i = vstruct_next(_i))
+
+#define vstruct_for_each_safe(_s, _i, _t) \
+ for (_i = (_s)->start; \
+ _i < vstruct_last(_s) && (_t = vstruct_next(_i), true); \
+ _i = _t)
+
+#define vstruct_idx(_s, _idx) \
+ ((typeof(&(_s)->start[0])) ((_s)->_data + (_idx)))
+
+#endif /* _VSTRUCTS_H */
#include <linux/posix_acl_xattr.h>
#include <linux/xattr.h>
-#include <crypto/hash.h>
struct xattr_search_key {
u8 type;
static u64 bch_xattr_hash(const struct bch_hash_info *info,
const struct xattr_search_key *key)
{
- switch (info->type) {
- case BCH_STR_HASH_SHA1: {
- SHASH_DESC_ON_STACK(desc, bch_sha1);
- u8 digest[SHA1_DIGEST_SIZE];
- u64 ret;
+ struct bch_str_hash_ctx ctx;
- desc->tfm = bch_sha1;
- desc->flags = 0;
- crypto_shash_init(desc);
+ bch_str_hash_init(&ctx, info);
+ bch_str_hash_update(&ctx, info, &key->type, sizeof(key->type));
+ bch_str_hash_update(&ctx, info, key->name.name, key->name.len);
- crypto_shash_update(desc, (void *) &info->seed, sizeof(info->seed));
-
- crypto_shash_update(desc, (void *) &key->type, sizeof(key->type));
- crypto_shash_update(desc, (void *) key->name.name, key->name.len);
-
- crypto_shash_final(desc, digest);
- memcpy(&ret, &digest, sizeof(ret));
- return ret >> 1;
- }
- default: {
- struct bch_str_hash_ctx ctx;
-
- bch_str_hash_init(&ctx, info->type);
- bch_str_hash_update(&ctx, info->type, &info->seed, sizeof(info->seed));
-
- bch_str_hash_update(&ctx, info->type, &key->type, sizeof(key->type));
- bch_str_hash_update(&ctx, info->type, key->name.name, key->name.len);
-
- return bch_str_hash_end(&ctx, info->type);
- }
- }
+ return bch_str_hash_end(&ctx, info);
}
#define xattr_val(_xattr) ((_xattr)->x_name + (_xattr)->x_name_len)
+++ /dev/null
-/*
- * Cryptographic API for algorithms (i.e., low-level API).
- *
- * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- */
-
-#include <linux/byteorder.h>
-#include <linux/err.h>
-#include <linux/errno.h>
-#include <linux/kernel.h>
-#include <linux/list.h>
-#include <linux/module.h>
-#include <linux/printk.h>
-#include <linux/rtnetlink.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-
-#include "internal.h"
-
-static inline int crypto_set_driver_name(struct crypto_alg *alg)
-{
- static const char suffix[] = "-generic";
- char *driver_name = alg->cra_driver_name;
- int len;
-
- if (*driver_name)
- return 0;
-
- len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
- if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME)
- return -ENAMETOOLONG;
-
- memcpy(driver_name + len, suffix, sizeof(suffix));
- return 0;
-}
-
-static int crypto_check_alg(struct crypto_alg *alg)
-{
- if (alg->cra_alignmask & (alg->cra_alignmask + 1))
- return -EINVAL;
-
- if (alg->cra_blocksize > PAGE_SIZE / 8)
- return -EINVAL;
-
- if (alg->cra_priority < 0)
- return -EINVAL;
-
- atomic_set(&alg->cra_refcnt, 1);
-
- return crypto_set_driver_name(alg);
-}
-
-static int __crypto_register_alg(struct crypto_alg *alg)
-{
- struct crypto_alg *q;
- int ret = -EAGAIN;
-
- INIT_LIST_HEAD(&alg->cra_users);
-
- ret = -EEXIST;
-
- list_for_each_entry(q, &crypto_alg_list, cra_list) {
- if (q == alg)
- goto err;
-
- if (!strcmp(q->cra_driver_name, alg->cra_name) ||
- !strcmp(q->cra_name, alg->cra_driver_name))
- goto err;
- }
-
- list_add(&alg->cra_list, &crypto_alg_list);
- return 0;
-err:
- return ret;
-}
-
-void crypto_remove_final(struct list_head *list)
-{
- struct crypto_alg *alg;
- struct crypto_alg *n;
-
- list_for_each_entry_safe(alg, n, list, cra_list) {
- list_del_init(&alg->cra_list);
- crypto_alg_put(alg);
- }
-}
-
-int crypto_register_alg(struct crypto_alg *alg)
-{
- int err;
-
- err = crypto_check_alg(alg);
- if (err)
- return err;
-
- down_write(&crypto_alg_sem);
- err = __crypto_register_alg(alg);
- up_write(&crypto_alg_sem);
-
- return err;
-}
-
-static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
-{
- if (unlikely(list_empty(&alg->cra_list)))
- return -ENOENT;
-
- list_del_init(&alg->cra_list);
- return 0;
-}
-
-int crypto_unregister_alg(struct crypto_alg *alg)
-{
- int ret;
- LIST_HEAD(list);
-
- down_write(&crypto_alg_sem);
- ret = crypto_remove_alg(alg, &list);
- up_write(&crypto_alg_sem);
-
- if (ret)
- return ret;
-
- BUG_ON(atomic_read(&alg->cra_refcnt) != 1);
- if (alg->cra_destroy)
- alg->cra_destroy(alg);
-
- crypto_remove_final(&list);
- return 0;
-}
-
-int crypto_register_algs(struct crypto_alg *algs, int count)
-{
- int i, ret;
-
- for (i = 0; i < count; i++) {
- ret = crypto_register_alg(&algs[i]);
- if (ret)
- goto err;
- }
-
- return 0;
-
-err:
- for (--i; i >= 0; --i)
- crypto_unregister_alg(&algs[i]);
-
- return ret;
-}
-
-int crypto_unregister_algs(struct crypto_alg *algs, int count)
-{
- int i, ret;
-
- for (i = 0; i < count; i++) {
- ret = crypto_unregister_alg(&algs[i]);
- if (ret)
- pr_err("Failed to unregister %s %s: %d\n",
- algs[i].cra_driver_name, algs[i].cra_name, ret);
- }
-
- return 0;
-}
-
-struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
-{
- struct rtattr *rta = tb[0];
- struct crypto_attr_type *algt;
-
- if (!rta)
- return ERR_PTR(-ENOENT);
- if (RTA_PAYLOAD(rta) < sizeof(*algt))
- return ERR_PTR(-EINVAL);
- if (rta->rta_type != CRYPTOA_TYPE)
- return ERR_PTR(-EINVAL);
-
- algt = RTA_DATA(rta);
-
- return algt;
-}
-
-int crypto_check_attr_type(struct rtattr **tb, u32 type)
-{
- struct crypto_attr_type *algt;
-
- algt = crypto_get_attr_type(tb);
- if (IS_ERR(algt))
- return PTR_ERR(algt);
-
- if ((algt->type ^ type) & algt->mask)
- return -EINVAL;
-
- return 0;
-}
-
-const char *crypto_attr_alg_name(struct rtattr *rta)
-{
- struct crypto_attr_alg *alga;
-
- if (!rta)
- return ERR_PTR(-ENOENT);
- if (RTA_PAYLOAD(rta) < sizeof(*alga))
- return ERR_PTR(-EINVAL);
- if (rta->rta_type != CRYPTOA_ALG)
- return ERR_PTR(-EINVAL);
-
- alga = RTA_DATA(rta);
- alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
-
- return alga->name;
-}
-
-struct crypto_alg *crypto_attr_alg2(struct rtattr *rta,
- const struct crypto_type *frontend,
- u32 type, u32 mask)
-{
- const char *name;
-
- name = crypto_attr_alg_name(rta);
- if (IS_ERR(name))
- return ERR_CAST(name);
-
- return crypto_find_alg(name, frontend, type, mask);
-}
-
-int crypto_attr_u32(struct rtattr *rta, u32 *num)
-{
- struct crypto_attr_u32 *nu32;
-
- if (!rta)
- return -ENOENT;
- if (RTA_PAYLOAD(rta) < sizeof(*nu32))
- return -EINVAL;
- if (rta->rta_type != CRYPTOA_U32)
- return -EINVAL;
-
- nu32 = RTA_DATA(rta);
- *num = nu32->num;
-
- return 0;
-}
-
-static inline void crypto_inc_byte(u8 *a, unsigned int size)
-{
- u8 *b = (a + size);
- u8 c;
-
- for (; size; size--) {
- c = *--b + 1;
- *b = c;
- if (c)
- break;
- }
-}
-
-void crypto_inc(u8 *a, unsigned int size)
-{
- __be32 *b = (__be32 *)(a + size);
- u32 c;
-
- for (; size >= 4; size -= 4) {
- c = be32_to_cpu(*--b) + 1;
- *b = cpu_to_be32(c);
- if (c)
- return;
- }
-
- crypto_inc_byte(a, size);
-}
-
-static inline void crypto_xor_byte(u8 *a, const u8 *b, unsigned int size)
-{
- for (; size; size--)
- *a++ ^= *b++;
-}
-
-void crypto_xor(u8 *dst, const u8 *src, unsigned int size)
-{
- u32 *a = (u32 *)dst;
- u32 *b = (u32 *)src;
-
- for (; size >= 4; size -= 4)
- *a++ ^= *b++;
-
- crypto_xor_byte((u8 *)a, (u8 *)b, size);
-}
-
-unsigned int crypto_alg_extsize(struct crypto_alg *alg)
-{
- return alg->cra_ctxsize +
- (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
-}
-
-int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
- u32 type, u32 mask)
-{
- int ret = 0;
- struct crypto_alg *alg = crypto_find_alg(name, frontend, type, mask);
-
- if (!IS_ERR(alg)) {
- crypto_alg_put(alg);
- ret = 1;
- }
-
- return ret;
-}
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Cryptographic algorithms API");
/*
- * Scatterlist Cryptographic API.
+ * Cryptographic API for algorithms (i.e., low-level API).
*
- * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
- * Copyright (c) 2002 David S. Miller (davem@redhat.com)
- * Copyright (c) 2005 Herbert Xu <herbert@gondor.apana.org.au>
- *
- * Portions derived from Cryptoapi, by Alexander Kjeldaas <astor@fast.no>
- * and Nettle, by Niels Möller.
+ * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the Free
#include <linux/err.h>
#include <linux/errno.h>
#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/param.h>
-#include <linux/sched.h>
+#include <linux/list.h>
+#include <linux/rwsem.h>
#include <linux/slab.h>
#include <linux/string.h>
+
+#include <crypto/algapi.h>
#include "internal.h"
-LIST_HEAD(crypto_alg_list);
-DECLARE_RWSEM(crypto_alg_sem);
+static LIST_HEAD(crypto_alg_list);
+static DECLARE_RWSEM(crypto_alg_sem);
-static struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type,
- u32 mask)
+static unsigned crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask)
{
- struct crypto_alg *q, *alg = NULL;
- int best = -2;
-
- list_for_each_entry(q, &crypto_alg_list, cra_list) {
- int exact, fuzzy;
-
- if ((q->cra_flags ^ type) & mask)
- continue;
-
- exact = !strcmp(q->cra_driver_name, name);
- fuzzy = !strcmp(q->cra_name, name);
- if (!exact && !(fuzzy && q->cra_priority > best))
- continue;
-
- if (unlikely(!crypto_alg_get(q)))
- continue;
-
- best = q->cra_priority;
- if (alg)
- crypto_alg_put(alg);
- alg = q;
-
- if (exact)
- break;
- }
+ return alg->cra_type->ctxsize(alg, type, mask);
+}
- return alg;
+unsigned crypto_alg_extsize(struct crypto_alg *alg)
+{
+ return alg->cra_ctxsize;
}
struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask)
{
struct crypto_alg *alg;
- /*
- * If the internal flag is set for a cipher, require a caller to
- * to invoke the cipher with the internal flag to use that cipher.
- * Also, if a caller wants to allocate a cipher that may or may
- * not be an internal cipher, use type | CRYPTO_ALG_INTERNAL and
- * !(mask & CRYPTO_ALG_INTERNAL).
- */
- if (!((type | mask) & CRYPTO_ALG_INTERNAL))
- mask |= CRYPTO_ALG_INTERNAL;
-
down_read(&crypto_alg_sem);
- alg = __crypto_alg_lookup(name, type, mask);
- up_read(&crypto_alg_sem);
+ list_for_each_entry(alg, &crypto_alg_list, cra_list)
+ if (!((alg->cra_flags ^ type) & mask) &&
+ !strcmp(alg->cra_name, name))
+ goto found;
- return alg ?: ERR_PTR(-ENOENT);
-}
-
-static int crypto_init_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
-{
- const struct crypto_type *type_obj = tfm->__crt_alg->cra_type;
-
- if (type_obj)
- return type_obj->init(tfm, type, mask);
-
- switch (crypto_tfm_alg_type(tfm)) {
- case CRYPTO_ALG_TYPE_CIPHER:
- return crypto_init_cipher_ops(tfm);
- default:
- break;
- }
+ alg = ERR_PTR(-ENOENT);
+found:
+ up_read(&crypto_alg_sem);
- BUG();
- return -EINVAL;
+ return alg;
}
static void crypto_exit_ops(struct crypto_tfm *tfm)
{
- const struct crypto_type *type = tfm->__crt_alg->cra_type;
-
- if (type) {
- if (tfm->exit)
- tfm->exit(tfm);
- return;
- }
-
- switch (crypto_tfm_alg_type(tfm)) {
- case CRYPTO_ALG_TYPE_CIPHER:
- crypto_exit_cipher_ops(tfm);
- break;
-
- default:
- BUG();
- }
-}
-
-static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask)
-{
- const struct crypto_type *type_obj = alg->cra_type;
- unsigned int len;
-
- len = alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1);
- if (type_obj)
- return len + type_obj->ctxsize(alg, type, mask);
-
- switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
- default:
- BUG();
-
- case CRYPTO_ALG_TYPE_CIPHER:
- len += crypto_cipher_ctxsize(alg);
- break;
- }
-
- return len;
+ if (tfm->exit)
+ tfm->exit(tfm);
}
-struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
- u32 mask)
+static struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg,
+ u32 type, u32 mask)
{
struct crypto_tfm *tfm = NULL;
- unsigned int tfm_size;
+ unsigned tfm_size;
int err = -ENOMEM;
tfm_size = sizeof(*tfm) + crypto_ctxsize(alg, type, mask);
tfm = kzalloc(tfm_size, GFP_KERNEL);
if (tfm == NULL)
- goto out_err;
+ return ERR_PTR(-ENOMEM);
tfm->__crt_alg = alg;
- err = crypto_init_ops(tfm, type, mask);
+ err = alg->cra_type->init(tfm, type, mask);
if (err)
goto out_free_tfm;
if (!tfm->exit && alg->cra_init && (err = alg->cra_init(tfm)))
goto cra_init_failed;
- goto out;
+ return tfm;
cra_init_failed:
crypto_exit_ops(tfm);
out_free_tfm:
kfree(tfm);
-out_err:
- tfm = ERR_PTR(err);
-out:
- return tfm;
+ return ERR_PTR(err);
}
-/*
- * crypto_alloc_base - Locate algorithm and allocate transform
- * @alg_name: Name of algorithm
- * @type: Type of algorithm
- * @mask: Mask for type comparison
- *
- * This function should not be used by new algorithm types.
- * Please use crypto_alloc_tfm instead.
- *
- * crypto_alloc_base() will first attempt to locate an already loaded
- * algorithm. If that fails and the kernel supports dynamically loadable
- * modules, it will then attempt to load a module of the same name or
- * alias. If that fails it will send a query to any loaded crypto manager
- * to construct an algorithm on the fly. A refcount is grabbed on the
- * algorithm which is then associated with the new transform.
- *
- * The returned transform is of a non-determinate type. Most people
- * should use one of the more specific allocation functions such as
- * crypto_alloc_blkcipher.
- *
- * In case of error the return value is an error pointer.
- */
struct crypto_tfm *crypto_alloc_base(const char *alg_name, u32 type, u32 mask)
{
struct crypto_alg *alg;
}
tfm = __crypto_alloc_tfm(alg, type, mask);
- if (IS_ERR(tfm)) {
- crypto_alg_put(alg);
+ if (IS_ERR(tfm))
return tfm;
- }
return tfm;
}
-void *crypto_create_tfm(struct crypto_alg *alg,
- const struct crypto_type *frontend)
+static void *crypto_create_tfm(struct crypto_alg *alg,
+ const struct crypto_type *frontend)
{
- char *mem;
struct crypto_tfm *tfm = NULL;
- unsigned int tfmsize;
- unsigned int total;
+ unsigned tfmsize;
+ unsigned total;
+ void *mem;
int err = -ENOMEM;
tfmsize = frontend->tfmsize;
total = tfmsize + sizeof(*tfm) + frontend->extsize(alg);
mem = kzalloc(total, GFP_KERNEL);
- if (mem == NULL)
+ if (!mem)
goto out_err;
- tfm = (struct crypto_tfm *)(mem + tfmsize);
+ tfm = mem + tfmsize;
tfm->__crt_alg = alg;
err = frontend->init_tfm(tfm);
return mem;
}
-struct crypto_alg *crypto_find_alg(const char *alg_name,
- const struct crypto_type *frontend,
- u32 type, u32 mask)
+static struct crypto_alg *crypto_find_alg(const char *alg_name,
+ const struct crypto_type *frontend,
+ u32 type, u32 mask)
{
- struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask) =
- crypto_alg_mod_lookup;
-
if (frontend) {
type &= frontend->maskclear;
mask &= frontend->maskclear;
type |= frontend->type;
mask |= frontend->maskset;
-
- if (frontend->lookup)
- lookup = frontend->lookup;
}
- return lookup(alg_name, type, mask);
+ return crypto_alg_mod_lookup(alg_name, type, mask);
}
void *crypto_alloc_tfm(const char *alg_name,
- const struct crypto_type *frontend, u32 type, u32 mask)
+ const struct crypto_type *frontend,
+ u32 type, u32 mask)
{
struct crypto_alg *alg;
void *tfm;
return ERR_CAST(alg);
tfm = crypto_create_tfm(alg, frontend);
- if (IS_ERR(tfm)) {
- crypto_alg_put(alg);
+ if (IS_ERR(tfm))
return tfm;
- }
return tfm;
}
if (!tfm->exit && alg->cra_exit)
alg->cra_exit(tfm);
crypto_exit_ops(tfm);
- crypto_alg_put(alg);
kzfree(mem);
}
-int crypto_has_alg(const char *name, u32 type, u32 mask)
+int crypto_register_alg(struct crypto_alg *alg)
{
- int ret = 0;
- struct crypto_alg *alg = crypto_alg_mod_lookup(name, type, mask);
+ INIT_LIST_HEAD(&alg->cra_users);
- if (!IS_ERR(alg)) {
- crypto_alg_put(alg);
- ret = 1;
- }
+ down_write(&crypto_alg_sem);
+ list_add(&alg->cra_list, &crypto_alg_list);
+ up_write(&crypto_alg_sem);
- return ret;
+ return 0;
}
-
-MODULE_DESCRIPTION("Cryptographic core API");
-MODULE_LICENSE("GPL");
--- /dev/null
+/*
+ * Block chaining cipher operations.
+ *
+ * Generic encrypt/decrypt wrapper for ciphers, handles operations across
+ * multiple page boundaries by using temporary blocks. In user context,
+ * the kernel is given a chance to schedule us once per page.
+ *
+ * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include <crypto/algapi.h>
+#include "internal.h"
+
+static unsigned crypto_blkcipher_ctxsize(struct crypto_alg *alg,
+ u32 type, u32 mask)
+{
+ return alg->cra_ctxsize;
+}
+
+static int crypto_init_blkcipher_ops(struct crypto_tfm *tfm, u32 type, u32 mask)
+{
+ struct blkcipher_tfm *crt = &tfm->crt_blkcipher;
+ struct blkcipher_alg *alg = &tfm->__crt_alg->cra_blkcipher;
+
+ BUG_ON((mask & CRYPTO_ALG_TYPE_MASK) != CRYPTO_ALG_TYPE_MASK);
+
+ crt->setkey = alg->setkey;
+ crt->encrypt = alg->encrypt;
+ crt->decrypt = alg->decrypt;
+ return 0;
+}
+
+const struct crypto_type crypto_blkcipher_type = {
+ .ctxsize = crypto_blkcipher_ctxsize,
+ .init = crypto_init_blkcipher_ops,
+};
--- /dev/null
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/byteorder.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include <asm/unaligned.h>
+
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+#include <crypto/chacha20.h>
+
+#include <sodium/crypto_stream_chacha20.h>
+
+struct chacha20_ctx {
+ u32 key[8];
+};
+
+static int crypto_chacha20_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keysize)
+{
+ struct chacha20_ctx *ctx = crypto_tfm_ctx(tfm);
+ int i;
+
+ if (keysize != CHACHA20_KEY_SIZE)
+ return -EINVAL;
+
+ for (i = 0; i < ARRAY_SIZE(ctx->key); i++)
+ ctx->key[i] = get_unaligned_le32(key + i * sizeof(u32));
+
+ return 0;
+}
+
+static int crypto_chacha20_crypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned nbytes)
+{
+ struct chacha20_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ struct scatterlist *sg = src;
+ u32 iv[4];
+ int ret;
+
+ BUG_ON(src != dst);
+
+ memcpy(iv, desc->info, sizeof(iv));
+
+ while (1) {
+ ret = crypto_stream_chacha20_xor_ic(sg_virt(sg),
+ sg_virt(sg),
+ sg->length,
+ (void *) &iv[2],
+ iv[0] | ((u64) iv[1] << 32),
+ (void *) ctx->key);
+ BUG_ON(ret);
+
+ nbytes -= sg->length;
+
+ if (sg_is_last(sg))
+ break;
+
+ BUG_ON(sg->length % CHACHA20_BLOCK_SIZE);
+ iv[0] += sg->length / CHACHA20_BLOCK_SIZE;
+ sg = sg_next(sg);
+ };
+
+ BUG_ON(nbytes);
+
+ return 0;
+}
+
+static struct crypto_alg alg = {
+ .cra_name = "chacha20",
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_ctxsize = sizeof(struct chacha20_ctx),
+ .cra_u = {
+ .blkcipher = {
+ .setkey = crypto_chacha20_setkey,
+ .encrypt = crypto_chacha20_crypt,
+ .decrypt = crypto_chacha20_crypt,
+ },
+ },
+};
+
+__attribute__((constructor(110)))
+static int chacha20_generic_mod_init(void)
+{
+ return crypto_register_alg(&alg);
+}
+++ /dev/null
-/*
- * Cryptographic API.
- *
- * Cipher operations.
- *
- * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
- * Copyright (c) 2005 Herbert Xu <herbert@gondor.apana.org.au>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- */
-
-#include <linux/kernel.h>
-#include <linux/crypto.h>
-#include <linux/errno.h>
-#include <linux/slab.h>
-#include <linux/string.h>
-#include "internal.h"
-
-static int setkey_unaligned(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct cipher_alg *cia = &tfm->__crt_alg->cra_cipher;
- unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
- int ret;
- u8 *buffer, *alignbuffer;
- unsigned long absize;
-
- absize = keylen + alignmask;
- buffer = kmalloc(absize, GFP_ATOMIC);
- if (!buffer)
- return -ENOMEM;
-
- alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
- memcpy(alignbuffer, key, keylen);
- ret = cia->cia_setkey(tfm, alignbuffer, keylen);
- memset(alignbuffer, 0, keylen);
- kfree(buffer);
- return ret;
-
-}
-
-static int setkey_default(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct cipher_alg *cia = &tfm->__crt_alg->cra_cipher;
- unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
-
- tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
- if (keylen < cia->cia_min_keysize || keylen > cia->cia_max_keysize) {
- tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
- return -EINVAL;
- }
-
- if ((unsigned long)key & alignmask)
- return setkey_unaligned(tfm, key, keylen);
-
- return cia->cia_setkey(tfm, key, keylen);
-}
-
-static void cipher_crypt_unaligned(void (*fn)(struct crypto_tfm *, u8 *,
- const u8 *),
- struct crypto_tfm *tfm,
- u8 *dst, const u8 *src)
-{
- unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
- unsigned int size = crypto_tfm_alg_blocksize(tfm);
- u8 buffer[size + alignmask];
- u8 *tmp = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
-
- memcpy(tmp, src, size);
- fn(tfm, tmp, tmp);
- memcpy(dst, tmp, size);
-}
-
-static void cipher_encrypt_unaligned(struct crypto_tfm *tfm,
- u8 *dst, const u8 *src)
-{
- unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
- struct cipher_alg *cipher = &tfm->__crt_alg->cra_cipher;
-
- if (unlikely(((unsigned long)dst | (unsigned long)src) & alignmask)) {
- cipher_crypt_unaligned(cipher->cia_encrypt, tfm, dst, src);
- return;
- }
-
- cipher->cia_encrypt(tfm, dst, src);
-}
-
-static void cipher_decrypt_unaligned(struct crypto_tfm *tfm,
- u8 *dst, const u8 *src)
-{
- unsigned long alignmask = crypto_tfm_alg_alignmask(tfm);
- struct cipher_alg *cipher = &tfm->__crt_alg->cra_cipher;
-
- if (unlikely(((unsigned long)dst | (unsigned long)src) & alignmask)) {
- cipher_crypt_unaligned(cipher->cia_decrypt, tfm, dst, src);
- return;
- }
-
- cipher->cia_decrypt(tfm, dst, src);
-}
-
-int crypto_init_cipher_ops(struct crypto_tfm *tfm)
-{
- struct cipher_tfm *ops = &tfm->crt_cipher;
- struct cipher_alg *cipher = &tfm->__crt_alg->cra_cipher;
-
- ops->cit_setkey = setkey_default;
- ops->cit_encrypt_one = crypto_tfm_alg_alignmask(tfm) ?
- cipher_encrypt_unaligned : cipher->cia_encrypt;
- ops->cit_decrypt_one = crypto_tfm_alg_alignmask(tfm) ?
- cipher_decrypt_unaligned : cipher->cia_decrypt;
-
- return 0;
-}
-
-void crypto_exit_cipher_ops(struct crypto_tfm *tfm)
-{
-}
#ifndef _CRYPTO_INTERNAL_H
#define _CRYPTO_INTERNAL_H
-#include <crypto/algapi.h>
-#include <linux/completion.h>
-#include <linux/mm.h>
-#include <linux/list.h>
-#include <linux/kernel.h>
-#include <linux/notifier.h>
-#include <linux/rwsem.h>
-#include <linux/slab.h>
+struct crypto_type;
+struct crypto_alg;
-struct crypto_instance;
-struct crypto_template;
-
-struct crypto_larval {
- struct crypto_alg alg;
- struct crypto_alg *adult;
- struct completion completion;
- u32 mask;
-};
-
-extern struct list_head crypto_alg_list;
-extern struct rw_semaphore crypto_alg_sem;
-
-static inline unsigned int crypto_cipher_ctxsize(struct crypto_alg *alg)
-{
- return alg->cra_ctxsize;
-}
-
-int crypto_init_cipher_ops(struct crypto_tfm *tfm);
-void crypto_exit_cipher_ops(struct crypto_tfm *tfm);
-
-void crypto_remove_final(struct list_head *list);
-struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type,
- u32 mask);
-void *crypto_create_tfm(struct crypto_alg *alg,
- const struct crypto_type *frontend);
-struct crypto_alg *crypto_find_alg(const char *alg_name,
- const struct crypto_type *frontend,
- u32 type, u32 mask);
-void *crypto_alloc_tfm(const char *alg_name,
- const struct crypto_type *frontend, u32 type, u32 mask);
-
-int crypto_register_notifier(struct notifier_block *nb);
-int crypto_unregister_notifier(struct notifier_block *nb);
-
-unsigned int crypto_alg_extsize(struct crypto_alg *alg);
-
-int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
- u32 type, u32 mask);
-
-static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg)
-{
- atomic_inc(&alg->cra_refcnt);
- return alg;
-}
-
-static inline void crypto_alg_put(struct crypto_alg *alg)
-{
- if (atomic_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy)
- alg->cra_destroy(alg);
-}
+void *crypto_alloc_tfm(const char *, const struct crypto_type *, u32, u32);
+unsigned int crypto_alg_extsize(struct crypto_alg *);
#endif /* _CRYPTO_INTERNAL_H */
--- /dev/null
+/*
+ * Poly1305 authenticator algorithm, RFC7539
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * Based on public domain code by Andrew Moon and Daniel J. Bernstein.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/byteorder.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <asm/unaligned.h>
+
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+#include <crypto/internal/hash.h>
+#include <crypto/poly1305.h>
+
+struct poly1305_desc_ctx {
+ bool key_done;
+ crypto_onetimeauth_poly1305_state s;
+};
+
+
+static int poly1305_init(struct shash_desc *desc)
+{
+ struct poly1305_desc_ctx *state = shash_desc_ctx(desc);
+
+ state->key_done = false;
+ return 0;
+}
+
+static int poly1305_update(struct shash_desc *desc,
+ const u8 *src, unsigned len)
+{
+ struct poly1305_desc_ctx *state = shash_desc_ctx(desc);
+
+ if (!state->key_done) {
+ BUG_ON(len != crypto_onetimeauth_poly1305_KEYBYTES);
+
+ state->key_done = true;
+ return crypto_onetimeauth_poly1305_init(&state->s, src);
+ }
+
+ return crypto_onetimeauth_poly1305_update(&state->s, src, len);
+}
+
+static int poly1305_final(struct shash_desc *desc, u8 *out)
+{
+ struct poly1305_desc_ctx *state = shash_desc_ctx(desc);
+
+ return crypto_onetimeauth_poly1305_final(&state->s, out);
+}
+
+static struct shash_alg poly1305_alg = {
+ .digestsize = crypto_onetimeauth_poly1305_BYTES,
+ .init = poly1305_init,
+ .update = poly1305_update,
+ .final = poly1305_final,
+ .descsize = sizeof(struct poly1305_desc_ctx),
+ .base = {
+ .cra_name = "poly1305",
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ },
+};
+
+__attribute__((constructor(110)))
+static int poly1305_mod_init(void)
+{
+ return crypto_register_shash(&poly1305_alg);
+}
+++ /dev/null
-/*
- * Cryptographic API.
- *
- * SHA1 Secure Hash Algorithm.
- *
- * Derived from cryptoapi implementation, adapted for in-place
- * scatterlist interface.
- *
- * Copyright (c) Alan Smithee.
- * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
- * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- */
-#include <crypto/internal/hash.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/cryptohash.h>
-#include <linux/types.h>
-#include <crypto/sha.h>
-#include <crypto/sha1_base.h>
-#include <asm/byteorder.h>
-
-const u8 sha1_zero_message_hash[SHA1_DIGEST_SIZE] = {
- 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
- 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
- 0xaf, 0xd8, 0x07, 0x09
-};
-
-static void sha1_generic_block_fn(struct sha1_state *sst, u8 const *src,
- int blocks)
-{
- u32 temp[SHA_WORKSPACE_WORDS];
-
- while (blocks--) {
- sha_transform(sst->state, src, temp);
- src += SHA1_BLOCK_SIZE;
- }
- memzero_explicit(temp, sizeof(temp));
-}
-
-int crypto_sha1_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha1_base_do_update(desc, data, len, sha1_generic_block_fn);
-}
-
-static int sha1_final(struct shash_desc *desc, u8 *out)
-{
- sha1_base_do_finalize(desc, sha1_generic_block_fn);
- return sha1_base_finish(desc, out);
-}
-
-int crypto_sha1_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- sha1_base_do_update(desc, data, len, sha1_generic_block_fn);
- return sha1_final(desc, out);
-}
-
-static struct shash_alg alg = {
- .digestsize = SHA1_DIGEST_SIZE,
- .init = sha1_base_init,
- .update = crypto_sha1_update,
- .final = sha1_final,
- .finup = crypto_sha1_finup,
- .descsize = sizeof(struct sha1_state),
- .base = {
- .cra_name = "sha1",
- .cra_driver_name= "sha1-generic",
- .cra_flags = CRYPTO_ALG_TYPE_SHASH,
- .cra_blocksize = SHA1_BLOCK_SIZE,
- .cra_module = THIS_MODULE,
- }
-};
-
-__attribute__((constructor(110)))
-static int __init sha1_generic_mod_init(void)
-{
- return crypto_register_shash(&alg);
-}
--- /dev/null
+/*
+ * Cryptographic API.
+ *
+ * SHA-256, as specified in
+ * http://csrc.nist.gov/groups/STM/cavp/documents/shs/sha256-384-512.pdf
+ *
+ * SHA-256 code by Jean-Luc Cooke <jlcooke@certainkey.com>.
+ *
+ * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
+ * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
+ * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
+ * SHA224 Support Copyright 2007 Intel Corporation <jonathan.lynch@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+
+#include <linux/bitops.h>
+#include <linux/byteorder.h>
+#include <linux/types.h>
+#include <asm/unaligned.h>
+
+#include <linux/crypto.h>
+#include <crypto/internal/hash.h>
+
+#include <sodium/crypto_hash_sha256.h>
+
+static int sha256_init(struct shash_desc *desc)
+{
+ crypto_hash_sha256_state *state = shash_desc_ctx(desc);
+
+ return crypto_hash_sha256_init(state);
+}
+
+static int sha256_update(struct shash_desc *desc, const u8 *data,
+ unsigned int len)
+{
+ crypto_hash_sha256_state *state = shash_desc_ctx(desc);
+
+ return crypto_hash_sha256_update(state, data, len);
+}
+
+static int sha256_final(struct shash_desc *desc, u8 *out)
+{
+ crypto_hash_sha256_state *state = shash_desc_ctx(desc);
+
+ return crypto_hash_sha256_final(state, out);
+}
+
+static struct shash_alg sha256_alg = {
+ .digestsize = crypto_hash_sha256_BYTES,
+ .init = sha256_init,
+ .update = sha256_update,
+ .final = sha256_final,
+ .descsize = sizeof(crypto_hash_sha256_state),
+ .base = {
+ .cra_name = "sha256",
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ }
+};
+
+__attribute__((constructor(110)))
+static int __init sha256_generic_mod_init(void)
+{
+ return crypto_register_shash(&sha256_alg);
+}
#include <crypto/internal/hash.h>
#include <linux/err.h>
#include <linux/kernel.h>
-#include <linux/module.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include "internal.h"
-static int shash_no_setkey(struct crypto_shash *tfm, const u8 *key,
- unsigned int keylen)
-{
- return -ENOSYS;
-}
-
-static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
- unsigned long absize;
- u8 *buffer, *alignbuffer;
- int err;
-
- absize = keylen + (alignmask & ~(crypto_tfm_ctx_alignment() - 1));
- buffer = kmalloc(absize, GFP_KERNEL);
- if (!buffer)
- return -ENOMEM;
-
- alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1);
- memcpy(alignbuffer, key, keylen);
- err = shash->setkey(tfm, alignbuffer, keylen);
- kzfree(buffer);
- return err;
-}
-
-int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key,
- unsigned int keylen)
-{
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
-
- if ((unsigned long)key & alignmask)
- return shash_setkey_unaligned(tfm, key, keylen);
-
- return shash->setkey(tfm, key, keylen);
-}
-
-static inline unsigned int shash_align_buffer_size(unsigned len,
- unsigned long mask)
-{
- typedef u8 __attribute__ ((aligned)) u8_aligned;
- return len + (mask & ~(__alignof__(u8_aligned) - 1));
-}
-
-static int shash_update_unaligned(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
- unsigned int unaligned_len = alignmask + 1 -
- ((unsigned long)data & alignmask);
- u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)]
- __attribute__ ((aligned));
- u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
- int err;
-
- if (unaligned_len > len)
- unaligned_len = len;
-
- memcpy(buf, data, unaligned_len);
- err = shash->update(desc, buf, unaligned_len);
- memset(buf, 0, unaligned_len);
-
- return err ?:
- shash->update(desc, data + unaligned_len, len - unaligned_len);
-}
-
-int crypto_shash_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
-
- if ((unsigned long)data & alignmask)
- return shash_update_unaligned(desc, data, len);
-
- return shash->update(desc, data, len);
-}
-
-static int shash_final_unaligned(struct shash_desc *desc, u8 *out)
-{
- struct crypto_shash *tfm = desc->tfm;
- unsigned long alignmask = crypto_shash_alignmask(tfm);
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned int ds = crypto_shash_digestsize(tfm);
- u8 ubuf[shash_align_buffer_size(ds, alignmask)]
- __attribute__ ((aligned));
- u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
- int err;
-
- err = shash->final(desc, buf);
- if (err)
- goto out;
-
- memcpy(out, buf, ds);
-
-out:
- memset(buf, 0, ds);
- return err;
-}
-
-int crypto_shash_final(struct shash_desc *desc, u8 *out)
-{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
-
- if ((unsigned long)out & alignmask)
- return shash_final_unaligned(desc, out);
-
- return shash->final(desc, out);
-}
-
-static int shash_finup_unaligned(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int shash_finup(struct shash_desc *desc, const u8 *data,
+ unsigned len, u8 *out)
{
return crypto_shash_update(desc, data, len) ?:
crypto_shash_final(desc, out);
}
-int crypto_shash_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
-
- if (((unsigned long)data | (unsigned long)out) & alignmask)
- return shash_finup_unaligned(desc, data, len, out);
-
- return shash->finup(desc, data, len, out);
-}
-
-static int shash_digest_unaligned(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
+static int shash_digest(struct shash_desc *desc, const u8 *data,
+ unsigned len, u8 *out)
{
return crypto_shash_init(desc) ?:
crypto_shash_finup(desc, data, len, out);
}
-int crypto_shash_digest(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- struct crypto_shash *tfm = desc->tfm;
- struct shash_alg *shash = crypto_shash_alg(tfm);
- unsigned long alignmask = crypto_shash_alignmask(tfm);
-
- if (((unsigned long)data | (unsigned long)out) & alignmask)
- return shash_digest_unaligned(desc, data, len, out);
-
- return shash->digest(desc, data, len, out);
-}
-
-static int shash_default_export(struct shash_desc *desc, void *out)
-{
- memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(desc->tfm));
- return 0;
-}
-
-static int shash_default_import(struct shash_desc *desc, const void *in)
-{
- memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(desc->tfm));
- return 0;
-}
-
static int crypto_shash_init_tfm(struct crypto_tfm *tfm)
{
struct crypto_shash *hash = __crypto_shash_cast(tfm);
}
static const struct crypto_type crypto_shash_type = {
- .extsize = crypto_alg_extsize,
- .init_tfm = crypto_shash_init_tfm,
- .maskclear = ~CRYPTO_ALG_TYPE_MASK,
- .maskset = CRYPTO_ALG_TYPE_MASK,
- .type = CRYPTO_ALG_TYPE_SHASH,
- .tfmsize = offsetof(struct crypto_shash, base),
+ .extsize = crypto_alg_extsize,
+ .init_tfm = crypto_shash_init_tfm,
+ .maskclear = ~CRYPTO_ALG_TYPE_MASK,
+ .maskset = CRYPTO_ALG_TYPE_MASK,
+ .type = CRYPTO_ALG_TYPE_SHASH,
+ .tfmsize = offsetof(struct crypto_shash, base),
};
-struct crypto_shash *crypto_alloc_shash(const char *alg_name, u32 type,
- u32 mask)
+struct crypto_shash *crypto_alloc_shash(const char *alg_name,
+ u32 type, u32 mask)
{
return crypto_alloc_tfm(alg_name, &crypto_shash_type, type, mask);
}
-static int shash_prepare_alg(struct shash_alg *alg)
+int crypto_register_shash(struct shash_alg *alg)
{
struct crypto_alg *base = &alg->base;
- if (alg->digestsize > PAGE_SIZE / 8 ||
- alg->descsize > PAGE_SIZE / 8 ||
- alg->statesize > PAGE_SIZE / 8)
- return -EINVAL;
-
base->cra_type = &crypto_shash_type;
base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
base->cra_flags |= CRYPTO_ALG_TYPE_SHASH;
if (!alg->finup)
- alg->finup = shash_finup_unaligned;
+ alg->finup = shash_finup;
if (!alg->digest)
- alg->digest = shash_digest_unaligned;
- if (!alg->export) {
- alg->export = shash_default_export;
- alg->import = shash_default_import;
- alg->statesize = alg->descsize;
- }
- if (!alg->setkey)
- alg->setkey = shash_no_setkey;
-
- return 0;
-}
-
-int crypto_register_shash(struct shash_alg *alg)
-{
- struct crypto_alg *base = &alg->base;
- int err;
-
- err = shash_prepare_alg(alg);
- if (err)
- return err;
+ alg->digest = shash_digest;
return crypto_register_alg(base);
}
-
-int crypto_unregister_shash(struct shash_alg *alg)
-{
- return crypto_unregister_alg(&alg->base);
-}
-
-int crypto_register_shashes(struct shash_alg *algs, int count)
-{
- int i, ret;
-
- for (i = 0; i < count; i++) {
- ret = crypto_register_shash(&algs[i]);
- if (ret)
- goto err;
- }
-
- return 0;
-
-err:
- for (--i; i >= 0; --i)
- crypto_unregister_shash(&algs[i]);
-
- return ret;
-}
-
-int crypto_unregister_shashes(struct shash_alg *algs, int count)
-{
- int i, ret;
-
- for (i = count - 1; i >= 0; --i) {
- ret = crypto_unregister_shash(&algs[i]);
- if (ret)
- pr_err("Failed to unregister %s %s: %d\n",
- algs[i].base.cra_driver_name,
- algs[i].base.cra_name, ret);
- }
-
- return 0;
-}
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Synchronous cryptographic hash type");
+++ /dev/null
-/*
- * LZ4 HC - High Compression Mode of LZ4
- * Copyright (C) 2011-2012, Yann Collet.
- * BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * You can contact the author at :
- * - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
- * - LZ4 source repository : http://code.google.com/p/lz4/
- *
- * Changed for kernel use by:
- * Chanho Min <chanho.min@lge.com>
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/lz4.h>
-#include <asm/unaligned.h>
-#include "lz4defs.h"
-
-struct lz4hc_data {
- const u8 *base;
- HTYPE hashtable[HASHTABLESIZE];
- u16 chaintable[MAXD];
- const u8 *nexttoupdate;
-} __attribute__((__packed__));
-
-static inline int lz4hc_init(struct lz4hc_data *hc4, const u8 *base)
-{
- memset((void *)hc4->hashtable, 0, sizeof(hc4->hashtable));
- memset(hc4->chaintable, 0xFF, sizeof(hc4->chaintable));
-
-#if LZ4_ARCH64
- hc4->nexttoupdate = base + 1;
-#else
- hc4->nexttoupdate = base;
-#endif
- hc4->base = base;
- return 1;
-}
-
-/* Update chains up to ip (excluded) */
-static inline void lz4hc_insert(struct lz4hc_data *hc4, const u8 *ip)
-{
- u16 *chaintable = hc4->chaintable;
- HTYPE *hashtable = hc4->hashtable;
-#if LZ4_ARCH64
- const u8 * const base = hc4->base;
-#else
- const int base = 0;
-#endif
-
- while (hc4->nexttoupdate < ip) {
- const u8 *p = hc4->nexttoupdate;
- size_t delta = p - (hashtable[HASH_VALUE(p)] + base);
- if (delta > MAX_DISTANCE)
- delta = MAX_DISTANCE;
- chaintable[(size_t)(p) & MAXD_MASK] = (u16)delta;
- hashtable[HASH_VALUE(p)] = (p) - base;
- hc4->nexttoupdate++;
- }
-}
-
-static inline int lz4hc_insertandfindbestmatch(struct lz4hc_data *hc4,
- const u8 *ip, const u8 *const matchlimit, const u8 **matchpos)
-{
- u16 *const chaintable = hc4->chaintable;
- HTYPE *const hashtable = hc4->hashtable;
- const u8 *ref;
-#if LZ4_ARCH64
- const u8 * const base = hc4->base;
-#else
- const int base = 0;
-#endif
- int nbattempts = MAX_NB_ATTEMPTS;
- size_t repl = 0, ml = 0;
- u16 delta;
-
- /* HC4 match finder */
- lz4hc_insert(hc4, ip);
- ref = hashtable[HASH_VALUE(ip)] + base;
-
- /* potential repetition */
- if (ref >= ip-4) {
- /* confirmed */
- if (A32(ref) == A32(ip)) {
- delta = (u16)(ip-ref);
- repl = ml = common_length(ip + MINMATCH,
- ref + MINMATCH, matchlimit) + MINMATCH;
- *matchpos = ref;
- }
- ref -= (size_t)chaintable[(size_t)(ref) & MAXD_MASK];
- }
-
- while ((ref >= ip - MAX_DISTANCE) && nbattempts) {
- nbattempts--;
- if (*(ref + ml) == *(ip + ml)) {
- if (A32(ref) == A32(ip)) {
- size_t mlt =
- common_length(ip + MINMATCH,
- ref + MINMATCH, matchlimit) + MINMATCH;
- if (mlt > ml) {
- ml = mlt;
- *matchpos = ref;
- }
- }
- }
- ref -= (size_t)chaintable[(size_t)(ref) & MAXD_MASK];
- }
-
- /* Complete table */
- if (repl) {
- const u8 *ptr = ip;
- const u8 *end;
- end = ip + repl - (MINMATCH-1);
- /* Pre-Load */
- while (ptr < end - delta) {
- chaintable[(size_t)(ptr) & MAXD_MASK] = delta;
- ptr++;
- }
- do {
- chaintable[(size_t)(ptr) & MAXD_MASK] = delta;
- /* Head of chain */
- hashtable[HASH_VALUE(ptr)] = (ptr) - base;
- ptr++;
- } while (ptr < end);
- hc4->nexttoupdate = end;
- }
-
- return (int)ml;
-}
-
-static inline int lz4hc_insertandgetwidermatch(struct lz4hc_data *hc4,
- const u8 *ip, const u8 *startlimit, const u8 *matchlimit, int longest,
- const u8 **matchpos, const u8 **startpos)
-{
- u16 *const chaintable = hc4->chaintable;
- HTYPE *const hashtable = hc4->hashtable;
-#if LZ4_ARCH64
- const u8 * const base = hc4->base;
-#else
- const int base = 0;
-#endif
- const u8 *ref;
- int nbattempts = MAX_NB_ATTEMPTS;
- int delta = (int)(ip - startlimit);
-
- /* First Match */
- lz4hc_insert(hc4, ip);
- ref = hashtable[HASH_VALUE(ip)] + base;
-
- while ((ref >= ip - MAX_DISTANCE) && (ref >= hc4->base)
- && (nbattempts)) {
- nbattempts--;
- if (*(startlimit + longest) == *(ref - delta + longest)) {
- if (A32(ref) == A32(ip)) {
- const u8 *reft = ref;
- const u8 *startt = ip;
- unsigned length =
- common_length(ip + MINMATCH,
- ref + MINMATCH,
- matchlimit);
-
- while ((startt > startlimit)
- && (reft > hc4->base)
- && (startt[-1] == reft[-1])) {
- startt--;
- reft--;
- length++;
- }
-
- if (length > longest) {
- longest = length;
- *matchpos = reft;
- *startpos = startt;
- }
- }
- }
- ref -= (size_t)chaintable[(size_t)(ref) & MAXD_MASK];
- }
- return longest;
-}
-
-static inline int lz4_encodesequence(const u8 **ip, u8 **op, const u8 **anchor,
- int ml, const u8 *ref)
-{
- unsigned length;
- u8 *token;
-
- /* Encode Literal length */
- length = *ip - *anchor;
- token = (*op)++;
- *token = encode_length(op, length) << ML_BITS;
-
- /* Copy Literals */
- MEMCPY_ADVANCE_CHUNKED(*op, *anchor, length);
-
- /* Encode Offset */
- PUT_LE16_ADVANCE(*op, (u16)(*ip - ref));
-
- *token += encode_length(op, ml - MINMATCH);
-
- /* Prepare next loop */
- *ip += ml;
- *anchor = *ip;
-
- return 0;
-}
-
-static int lz4_compresshcctx(struct lz4hc_data *ctx,
- const char *source,
- char *dest,
- int isize)
-{
- const u8 *ip = (const u8 *)source;
- const u8 *anchor = ip;
- const u8 *const iend = ip + isize;
- const u8 *const mflimit = iend - MFLIMIT;
- const u8 *const matchlimit = (iend - LASTLITERALS);
-
- u8 *op = (u8 *)dest;
-
- int ml, ml2, ml3, ml0;
- const u8 *ref = NULL;
- const u8 *start2 = NULL;
- const u8 *ref2 = NULL;
- const u8 *start3 = NULL;
- const u8 *ref3 = NULL;
- const u8 *start0;
- const u8 *ref0;
- int lastrun;
-
- ip++;
-
- /* Main Loop */
- while (ip < mflimit) {
- ml = lz4hc_insertandfindbestmatch(ctx, ip, matchlimit, (&ref));
- if (!ml) {
- ip++;
- continue;
- }
-
- /* saved, in case we would skip too much */
- start0 = ip;
- ref0 = ref;
- ml0 = ml;
-_search2:
- if (ip+ml < mflimit)
- ml2 = lz4hc_insertandgetwidermatch(ctx, ip + ml - 2,
- ip + 1, matchlimit, ml, &ref2, &start2);
- else
- ml2 = ml;
- /* No better match */
- if (ml2 == ml) {
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
- continue;
- }
-
- if (start0 < ip) {
- /* empirical */
- if (start2 < ip + ml0) {
- ip = start0;
- ref = ref0;
- ml = ml0;
- }
- }
- /*
- * Here, start0==ip
- * First Match too small : removed
- */
- if ((start2 - ip) < 3) {
- ml = ml2;
- ip = start2;
- ref = ref2;
- goto _search2;
- }
-
-_search3:
- /*
- * Currently we have :
- * ml2 > ml1, and
- * ip1+3 <= ip2 (usually < ip1+ml1)
- */
- if ((start2 - ip) < OPTIMAL_ML) {
- int correction;
- int new_ml = ml;
- if (new_ml > OPTIMAL_ML)
- new_ml = OPTIMAL_ML;
- if (ip + new_ml > start2 + ml2 - MINMATCH)
- new_ml = (int)(start2 - ip) + ml2 - MINMATCH;
- correction = new_ml - (int)(start2 - ip);
- if (correction > 0) {
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- }
- }
- /*
- * Now, we have start2 = ip+new_ml,
- * with new_ml=min(ml, OPTIMAL_ML=18)
- */
- if (start2 + ml2 < mflimit)
- ml3 = lz4hc_insertandgetwidermatch(ctx,
- start2 + ml2 - 3, start2, matchlimit,
- ml2, &ref3, &start3);
- else
- ml3 = ml2;
-
- /* No better match : 2 sequences to encode */
- if (ml3 == ml2) {
- /* ip & ref are known; Now for ml */
- if (start2 < ip+ml)
- ml = (int)(start2 - ip);
-
- /* Now, encode 2 sequences */
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
- ip = start2;
- lz4_encodesequence(&ip, &op, &anchor, ml2, ref2);
- continue;
- }
-
- /* Not enough space for match 2 : remove it */
- if (start3 < ip + ml + 3) {
- /*
- * can write Seq1 immediately ==> Seq2 is removed,
- * so Seq3 becomes Seq1
- */
- if (start3 >= (ip + ml)) {
- if (start2 < ip + ml) {
- int correction =
- (int)(ip + ml - start2);
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- if (ml2 < MINMATCH) {
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
- }
- }
-
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
- ip = start3;
- ref = ref3;
- ml = ml3;
-
- start0 = start2;
- ref0 = ref2;
- ml0 = ml2;
- goto _search2;
- }
-
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
- goto _search3;
- }
-
- /*
- * OK, now we have 3 ascending matches; let's write at least
- * the first one ip & ref are known; Now for ml
- */
- if (start2 < ip + ml) {
- if ((start2 - ip) < (int)ML_MASK) {
- int correction;
- if (ml > OPTIMAL_ML)
- ml = OPTIMAL_ML;
- if (ip + ml > start2 + ml2 - MINMATCH)
- ml = (int)(start2 - ip) + ml2
- - MINMATCH;
- correction = ml - (int)(start2 - ip);
- if (correction > 0) {
- start2 += correction;
- ref2 += correction;
- ml2 -= correction;
- }
- } else
- ml = (int)(start2 - ip);
- }
- lz4_encodesequence(&ip, &op, &anchor, ml, ref);
-
- ip = start2;
- ref = ref2;
- ml = ml2;
-
- start2 = start3;
- ref2 = ref3;
- ml2 = ml3;
-
- goto _search3;
- }
-
- /* Encode Last Literals */
- lastrun = (int)(iend - anchor);
- if (lastrun >= (int)RUN_MASK) {
- *op++ = (RUN_MASK << ML_BITS);
- lastrun -= RUN_MASK;
- for (; lastrun > 254 ; lastrun -= 255)
- *op++ = 255;
- *op++ = (u8) lastrun;
- } else
- *op++ = (lastrun << ML_BITS);
- memcpy(op, anchor, iend - anchor);
- op += iend - anchor;
- /* End */
- return (int) (((char *)op) - dest);
-}
-
-int lz4hc_compress(const unsigned char *src, size_t src_len,
- unsigned char *dst, size_t *dst_len, void *wrkmem)
-{
- int ret = -1;
- int out_len = 0;
-
- struct lz4hc_data *hc4 = (struct lz4hc_data *)wrkmem;
- lz4hc_init(hc4, (const u8 *)src);
- out_len = lz4_compresshcctx((struct lz4hc_data *)hc4, (const u8 *)src,
- (char *)dst, (int)src_len);
-
- if (out_len < 0)
- goto exit;
-
- *dst_len = out_len;
- return 0;
-
-exit:
- return ret;
-}
-EXPORT_SYMBOL(lz4hc_compress);
-
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_DESCRIPTION("LZ4HC compressor");
+++ /dev/null
-/*
- * SHA1 routine optimized to do word accesses rather than byte accesses,
- * and to avoid unnecessary copies into the context array.
- *
- * This was based on the git SHA1 implementation.
- */
-
-#include <linux/kernel.h>
-#include <linux/export.h>
-#include <linux/bitops.h>
-#include <linux/cryptohash.h>
-#include <asm/unaligned.h>
-
-/*
- * If you have 32 registers or more, the compiler can (and should)
- * try to change the array[] accesses into registers. However, on
- * machines with less than ~25 registers, that won't really work,
- * and at least gcc will make an unholy mess of it.
- *
- * So to avoid that mess which just slows things down, we force
- * the stores to memory to actually happen (we might be better off
- * with a 'W(t)=(val);asm("":"+m" (W(t))' there instead, as
- * suggested by Artur Skawina - that will also make gcc unable to
- * try to do the silly "optimize away loads" part because it won't
- * see what the value will be).
- *
- * Ben Herrenschmidt reports that on PPC, the C version comes close
- * to the optimized asm with this (ie on PPC you don't want that
- * 'volatile', since there are lots of registers).
- *
- * On ARM we get the best code generation by forcing a full memory barrier
- * between each SHA_ROUND, otherwise gcc happily get wild with spilling and
- * the stack frame size simply explode and performance goes down the drain.
- */
-
-#ifdef CONFIG_X86
- #define setW(x, val) (*(volatile __u32 *)&W(x) = (val))
-#elif defined(CONFIG_ARM)
- #define setW(x, val) do { W(x) = (val); __asm__("":::"memory"); } while (0)
-#else
- #define setW(x, val) (W(x) = (val))
-#endif
-
-/* This "rolls" over the 512-bit array */
-#define W(x) (array[(x)&15])
-
-/*
- * Where do we get the source from? The first 16 iterations get it from
- * the input data, the next mix it from the 512-bit array.
- */
-#define SHA_SRC(t) get_unaligned_be32((__u32 *)data + t)
-#define SHA_MIX(t) rol32(W(t+13) ^ W(t+8) ^ W(t+2) ^ W(t), 1)
-
-#define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \
- __u32 TEMP = input(t); setW(t, TEMP); \
- E += TEMP + rol32(A,5) + (fn) + (constant); \
- B = ror32(B, 2); } while (0)
-
-#define T_0_15(t, A, B, C, D, E) SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
-#define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
-#define T_20_39(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0x6ed9eba1, A, B, C, D, E )
-#define T_40_59(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, ((B&C)+(D&(B^C))) , 0x8f1bbcdc, A, B, C, D, E )
-#define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0xca62c1d6, A, B, C, D, E )
-
-/**
- * sha_transform - single block SHA1 transform
- *
- * @digest: 160 bit digest to update
- * @data: 512 bits of data to hash
- * @array: 16 words of workspace (see note)
- *
- * This function generates a SHA1 digest for a single 512-bit block.
- * Be warned, it does not handle padding and message digest, do not
- * confuse it with the full FIPS 180-1 digest algorithm for variable
- * length messages.
- *
- * Note: If the hash is security sensitive, the caller should be sure
- * to clear the workspace. This is left to the caller to avoid
- * unnecessary clears between chained hashing operations.
- */
-void sha_transform(__u32 *digest, const char *data, __u32 *array)
-{
- __u32 A, B, C, D, E;
-
- A = digest[0];
- B = digest[1];
- C = digest[2];
- D = digest[3];
- E = digest[4];
-
- /* Round 1 - iterations 0-16 take their input from 'data' */
- T_0_15( 0, A, B, C, D, E);
- T_0_15( 1, E, A, B, C, D);
- T_0_15( 2, D, E, A, B, C);
- T_0_15( 3, C, D, E, A, B);
- T_0_15( 4, B, C, D, E, A);
- T_0_15( 5, A, B, C, D, E);
- T_0_15( 6, E, A, B, C, D);
- T_0_15( 7, D, E, A, B, C);
- T_0_15( 8, C, D, E, A, B);
- T_0_15( 9, B, C, D, E, A);
- T_0_15(10, A, B, C, D, E);
- T_0_15(11, E, A, B, C, D);
- T_0_15(12, D, E, A, B, C);
- T_0_15(13, C, D, E, A, B);
- T_0_15(14, B, C, D, E, A);
- T_0_15(15, A, B, C, D, E);
-
- /* Round 1 - tail. Input from 512-bit mixing array */
- T_16_19(16, E, A, B, C, D);
- T_16_19(17, D, E, A, B, C);
- T_16_19(18, C, D, E, A, B);
- T_16_19(19, B, C, D, E, A);
-
- /* Round 2 */
- T_20_39(20, A, B, C, D, E);
- T_20_39(21, E, A, B, C, D);
- T_20_39(22, D, E, A, B, C);
- T_20_39(23, C, D, E, A, B);
- T_20_39(24, B, C, D, E, A);
- T_20_39(25, A, B, C, D, E);
- T_20_39(26, E, A, B, C, D);
- T_20_39(27, D, E, A, B, C);
- T_20_39(28, C, D, E, A, B);
- T_20_39(29, B, C, D, E, A);
- T_20_39(30, A, B, C, D, E);
- T_20_39(31, E, A, B, C, D);
- T_20_39(32, D, E, A, B, C);
- T_20_39(33, C, D, E, A, B);
- T_20_39(34, B, C, D, E, A);
- T_20_39(35, A, B, C, D, E);
- T_20_39(36, E, A, B, C, D);
- T_20_39(37, D, E, A, B, C);
- T_20_39(38, C, D, E, A, B);
- T_20_39(39, B, C, D, E, A);
-
- /* Round 3 */
- T_40_59(40, A, B, C, D, E);
- T_40_59(41, E, A, B, C, D);
- T_40_59(42, D, E, A, B, C);
- T_40_59(43, C, D, E, A, B);
- T_40_59(44, B, C, D, E, A);
- T_40_59(45, A, B, C, D, E);
- T_40_59(46, E, A, B, C, D);
- T_40_59(47, D, E, A, B, C);
- T_40_59(48, C, D, E, A, B);
- T_40_59(49, B, C, D, E, A);
- T_40_59(50, A, B, C, D, E);
- T_40_59(51, E, A, B, C, D);
- T_40_59(52, D, E, A, B, C);
- T_40_59(53, C, D, E, A, B);
- T_40_59(54, B, C, D, E, A);
- T_40_59(55, A, B, C, D, E);
- T_40_59(56, E, A, B, C, D);
- T_40_59(57, D, E, A, B, C);
- T_40_59(58, C, D, E, A, B);
- T_40_59(59, B, C, D, E, A);
-
- /* Round 4 */
- T_60_79(60, A, B, C, D, E);
- T_60_79(61, E, A, B, C, D);
- T_60_79(62, D, E, A, B, C);
- T_60_79(63, C, D, E, A, B);
- T_60_79(64, B, C, D, E, A);
- T_60_79(65, A, B, C, D, E);
- T_60_79(66, E, A, B, C, D);
- T_60_79(67, D, E, A, B, C);
- T_60_79(68, C, D, E, A, B);
- T_60_79(69, B, C, D, E, A);
- T_60_79(70, A, B, C, D, E);
- T_60_79(71, E, A, B, C, D);
- T_60_79(72, D, E, A, B, C);
- T_60_79(73, C, D, E, A, B);
- T_60_79(74, B, C, D, E, A);
- T_60_79(75, A, B, C, D, E);
- T_60_79(76, E, A, B, C, D);
- T_60_79(77, D, E, A, B, C);
- T_60_79(78, C, D, E, A, B);
- T_60_79(79, B, C, D, E, A);
-
- digest[0] += A;
- digest[1] += B;
- digest[2] += C;
- digest[3] += D;
- digest[4] += E;
-}
-EXPORT_SYMBOL(sha_transform);
-
-/**
- * sha_init - initialize the vectors for a SHA1 digest
- * @buf: vector to initialize
- */
-void sha_init(__u32 *buf)
-{
- buf[0] = 0x67452301;
- buf[1] = 0xefcdab89;
- buf[2] = 0x98badcfe;
- buf[3] = 0x10325476;
- buf[4] = 0xc3d2e1f0;
-}
-EXPORT_SYMBOL(sha_init);
projects / bcachefs-tools-debian / commitdiff