7 #include <linux/crc32c.h>
8 #include <linux/crypto.h>
10 #include <linux/random.h>
11 #include <linux/scatterlist.h>
12 #include <crypto/algapi.h>
13 #include <crypto/chacha20.h>
14 #include <crypto/hash.h>
15 #include <crypto/poly1305.h>
16 #include <keys/user-type.h>
19 * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group (Any
20 * use permitted, subject to terms of PostgreSQL license; see.)
22 * If we have a 64-bit integer type, then a 64-bit CRC looks just like the
23 * usual sort of implementation. (See Ross Williams' excellent introduction
24 * A PAINLESS GUIDE TO CRC ERROR DETECTION ALGORITHMS, available from
25 * ftp://ftp.rocksoft.com/papers/crc_v3.txt or several other net sites.)
26 * If we have no working 64-bit type, then fake it with two 32-bit registers.
28 * The present implementation is a normal (not "reflected", in Williams'
29 * terms) 64-bit CRC, using initial all-ones register contents and a final
30 * bit inversion. The chosen polynomial is borrowed from the DLT1 spec
31 * (ECMA-182, available from http://www.ecma.ch/ecma1/STAND/ECMA-182.HTM):
33 * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
34 * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
35 * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
39 static const u64 crc_table[256] = {
40 0x0000000000000000ULL, 0x42F0E1EBA9EA3693ULL, 0x85E1C3D753D46D26ULL,
41 0xC711223CFA3E5BB5ULL, 0x493366450E42ECDFULL, 0x0BC387AEA7A8DA4CULL,
42 0xCCD2A5925D9681F9ULL, 0x8E224479F47CB76AULL, 0x9266CC8A1C85D9BEULL,
43 0xD0962D61B56FEF2DULL, 0x17870F5D4F51B498ULL, 0x5577EEB6E6BB820BULL,
44 0xDB55AACF12C73561ULL, 0x99A54B24BB2D03F2ULL, 0x5EB4691841135847ULL,
45 0x1C4488F3E8F96ED4ULL, 0x663D78FF90E185EFULL, 0x24CD9914390BB37CULL,
46 0xE3DCBB28C335E8C9ULL, 0xA12C5AC36ADFDE5AULL, 0x2F0E1EBA9EA36930ULL,
47 0x6DFEFF5137495FA3ULL, 0xAAEFDD6DCD770416ULL, 0xE81F3C86649D3285ULL,
48 0xF45BB4758C645C51ULL, 0xB6AB559E258E6AC2ULL, 0x71BA77A2DFB03177ULL,
49 0x334A9649765A07E4ULL, 0xBD68D2308226B08EULL, 0xFF9833DB2BCC861DULL,
50 0x388911E7D1F2DDA8ULL, 0x7A79F00C7818EB3BULL, 0xCC7AF1FF21C30BDEULL,
51 0x8E8A101488293D4DULL, 0x499B3228721766F8ULL, 0x0B6BD3C3DBFD506BULL,
52 0x854997BA2F81E701ULL, 0xC7B97651866BD192ULL, 0x00A8546D7C558A27ULL,
53 0x4258B586D5BFBCB4ULL, 0x5E1C3D753D46D260ULL, 0x1CECDC9E94ACE4F3ULL,
54 0xDBFDFEA26E92BF46ULL, 0x990D1F49C77889D5ULL, 0x172F5B3033043EBFULL,
55 0x55DFBADB9AEE082CULL, 0x92CE98E760D05399ULL, 0xD03E790CC93A650AULL,
56 0xAA478900B1228E31ULL, 0xE8B768EB18C8B8A2ULL, 0x2FA64AD7E2F6E317ULL,
57 0x6D56AB3C4B1CD584ULL, 0xE374EF45BF6062EEULL, 0xA1840EAE168A547DULL,
58 0x66952C92ECB40FC8ULL, 0x2465CD79455E395BULL, 0x3821458AADA7578FULL,
59 0x7AD1A461044D611CULL, 0xBDC0865DFE733AA9ULL, 0xFF3067B657990C3AULL,
60 0x711223CFA3E5BB50ULL, 0x33E2C2240A0F8DC3ULL, 0xF4F3E018F031D676ULL,
61 0xB60301F359DBE0E5ULL, 0xDA050215EA6C212FULL, 0x98F5E3FE438617BCULL,
62 0x5FE4C1C2B9B84C09ULL, 0x1D14202910527A9AULL, 0x93366450E42ECDF0ULL,
63 0xD1C685BB4DC4FB63ULL, 0x16D7A787B7FAA0D6ULL, 0x5427466C1E109645ULL,
64 0x4863CE9FF6E9F891ULL, 0x0A932F745F03CE02ULL, 0xCD820D48A53D95B7ULL,
65 0x8F72ECA30CD7A324ULL, 0x0150A8DAF8AB144EULL, 0x43A04931514122DDULL,
66 0x84B16B0DAB7F7968ULL, 0xC6418AE602954FFBULL, 0xBC387AEA7A8DA4C0ULL,
67 0xFEC89B01D3679253ULL, 0x39D9B93D2959C9E6ULL, 0x7B2958D680B3FF75ULL,
68 0xF50B1CAF74CF481FULL, 0xB7FBFD44DD257E8CULL, 0x70EADF78271B2539ULL,
69 0x321A3E938EF113AAULL, 0x2E5EB66066087D7EULL, 0x6CAE578BCFE24BEDULL,
70 0xABBF75B735DC1058ULL, 0xE94F945C9C3626CBULL, 0x676DD025684A91A1ULL,
71 0x259D31CEC1A0A732ULL, 0xE28C13F23B9EFC87ULL, 0xA07CF2199274CA14ULL,
72 0x167FF3EACBAF2AF1ULL, 0x548F120162451C62ULL, 0x939E303D987B47D7ULL,
73 0xD16ED1D631917144ULL, 0x5F4C95AFC5EDC62EULL, 0x1DBC74446C07F0BDULL,
74 0xDAAD56789639AB08ULL, 0x985DB7933FD39D9BULL, 0x84193F60D72AF34FULL,
75 0xC6E9DE8B7EC0C5DCULL, 0x01F8FCB784FE9E69ULL, 0x43081D5C2D14A8FAULL,
76 0xCD2A5925D9681F90ULL, 0x8FDAB8CE70822903ULL, 0x48CB9AF28ABC72B6ULL,
77 0x0A3B7B1923564425ULL, 0x70428B155B4EAF1EULL, 0x32B26AFEF2A4998DULL,
78 0xF5A348C2089AC238ULL, 0xB753A929A170F4ABULL, 0x3971ED50550C43C1ULL,
79 0x7B810CBBFCE67552ULL, 0xBC902E8706D82EE7ULL, 0xFE60CF6CAF321874ULL,
80 0xE224479F47CB76A0ULL, 0xA0D4A674EE214033ULL, 0x67C58448141F1B86ULL,
81 0x253565A3BDF52D15ULL, 0xAB1721DA49899A7FULL, 0xE9E7C031E063ACECULL,
82 0x2EF6E20D1A5DF759ULL, 0x6C0603E6B3B7C1CAULL, 0xF6FAE5C07D3274CDULL,
83 0xB40A042BD4D8425EULL, 0x731B26172EE619EBULL, 0x31EBC7FC870C2F78ULL,
84 0xBFC9838573709812ULL, 0xFD39626EDA9AAE81ULL, 0x3A28405220A4F534ULL,
85 0x78D8A1B9894EC3A7ULL, 0x649C294A61B7AD73ULL, 0x266CC8A1C85D9BE0ULL,
86 0xE17DEA9D3263C055ULL, 0xA38D0B769B89F6C6ULL, 0x2DAF4F0F6FF541ACULL,
87 0x6F5FAEE4C61F773FULL, 0xA84E8CD83C212C8AULL, 0xEABE6D3395CB1A19ULL,
88 0x90C79D3FEDD3F122ULL, 0xD2377CD44439C7B1ULL, 0x15265EE8BE079C04ULL,
89 0x57D6BF0317EDAA97ULL, 0xD9F4FB7AE3911DFDULL, 0x9B041A914A7B2B6EULL,
90 0x5C1538ADB04570DBULL, 0x1EE5D94619AF4648ULL, 0x02A151B5F156289CULL,
91 0x4051B05E58BC1E0FULL, 0x87409262A28245BAULL, 0xC5B073890B687329ULL,
92 0x4B9237F0FF14C443ULL, 0x0962D61B56FEF2D0ULL, 0xCE73F427ACC0A965ULL,
93 0x8C8315CC052A9FF6ULL, 0x3A80143F5CF17F13ULL, 0x7870F5D4F51B4980ULL,
94 0xBF61D7E80F251235ULL, 0xFD913603A6CF24A6ULL, 0x73B3727A52B393CCULL,
95 0x31439391FB59A55FULL, 0xF652B1AD0167FEEAULL, 0xB4A25046A88DC879ULL,
96 0xA8E6D8B54074A6ADULL, 0xEA16395EE99E903EULL, 0x2D071B6213A0CB8BULL,
97 0x6FF7FA89BA4AFD18ULL, 0xE1D5BEF04E364A72ULL, 0xA3255F1BE7DC7CE1ULL,
98 0x64347D271DE22754ULL, 0x26C49CCCB40811C7ULL, 0x5CBD6CC0CC10FAFCULL,
99 0x1E4D8D2B65FACC6FULL, 0xD95CAF179FC497DAULL, 0x9BAC4EFC362EA149ULL,
100 0x158E0A85C2521623ULL, 0x577EEB6E6BB820B0ULL, 0x906FC95291867B05ULL,
101 0xD29F28B9386C4D96ULL, 0xCEDBA04AD0952342ULL, 0x8C2B41A1797F15D1ULL,
102 0x4B3A639D83414E64ULL, 0x09CA82762AAB78F7ULL, 0x87E8C60FDED7CF9DULL,
103 0xC51827E4773DF90EULL, 0x020905D88D03A2BBULL, 0x40F9E43324E99428ULL,
104 0x2CFFE7D5975E55E2ULL, 0x6E0F063E3EB46371ULL, 0xA91E2402C48A38C4ULL,
105 0xEBEEC5E96D600E57ULL, 0x65CC8190991CB93DULL, 0x273C607B30F68FAEULL,
106 0xE02D4247CAC8D41BULL, 0xA2DDA3AC6322E288ULL, 0xBE992B5F8BDB8C5CULL,
107 0xFC69CAB42231BACFULL, 0x3B78E888D80FE17AULL, 0x7988096371E5D7E9ULL,
108 0xF7AA4D1A85996083ULL, 0xB55AACF12C735610ULL, 0x724B8ECDD64D0DA5ULL,
109 0x30BB6F267FA73B36ULL, 0x4AC29F2A07BFD00DULL, 0x08327EC1AE55E69EULL,
110 0xCF235CFD546BBD2BULL, 0x8DD3BD16FD818BB8ULL, 0x03F1F96F09FD3CD2ULL,
111 0x41011884A0170A41ULL, 0x86103AB85A2951F4ULL, 0xC4E0DB53F3C36767ULL,
112 0xD8A453A01B3A09B3ULL, 0x9A54B24BB2D03F20ULL, 0x5D45907748EE6495ULL,
113 0x1FB5719CE1045206ULL, 0x919735E51578E56CULL, 0xD367D40EBC92D3FFULL,
114 0x1476F63246AC884AULL, 0x568617D9EF46BED9ULL, 0xE085162AB69D5E3CULL,
115 0xA275F7C11F7768AFULL, 0x6564D5FDE549331AULL, 0x279434164CA30589ULL,
116 0xA9B6706FB8DFB2E3ULL, 0xEB46918411358470ULL, 0x2C57B3B8EB0BDFC5ULL,
117 0x6EA7525342E1E956ULL, 0x72E3DAA0AA188782ULL, 0x30133B4B03F2B111ULL,
118 0xF7021977F9CCEAA4ULL, 0xB5F2F89C5026DC37ULL, 0x3BD0BCE5A45A6B5DULL,
119 0x79205D0E0DB05DCEULL, 0xBE317F32F78E067BULL, 0xFCC19ED95E6430E8ULL,
120 0x86B86ED5267CDBD3ULL, 0xC4488F3E8F96ED40ULL, 0x0359AD0275A8B6F5ULL,
121 0x41A94CE9DC428066ULL, 0xCF8B0890283E370CULL, 0x8D7BE97B81D4019FULL,
122 0x4A6ACB477BEA5A2AULL, 0x089A2AACD2006CB9ULL, 0x14DEA25F3AF9026DULL,
123 0x562E43B4931334FEULL, 0x913F6188692D6F4BULL, 0xD3CF8063C0C759D8ULL,
124 0x5DEDC41A34BBEEB2ULL, 0x1F1D25F19D51D821ULL, 0xD80C07CD676F8394ULL,
125 0x9AFCE626CE85B507ULL,
128 u64 bch_crc64_update(u64 crc, const void *_data, size_t len)
130 const unsigned char *data = _data;
133 int i = ((int) (crc >> 56) ^ *data++) & 0xFF;
134 crc = crc_table[i] ^ (crc << 8);
140 static u64 bch_checksum_init(unsigned type)
145 case BCH_CSUM_CRC32C:
154 static u64 bch_checksum_final(unsigned type, u64 crc)
159 case BCH_CSUM_CRC32C:
160 return crc ^ U32_MAX;
162 return crc ^ U64_MAX;
168 static u64 bch_checksum_update(unsigned type, u64 crc, const void *data, size_t len)
173 case BCH_CSUM_CRC32C:
174 return crc32c(crc, data, len);
176 return bch_crc64_update(crc, data, len);
182 static inline void do_encrypt_sg(struct crypto_blkcipher *tfm,
184 struct scatterlist *sg, size_t len)
186 struct blkcipher_desc desc = { .tfm = tfm, .info = nonce.d };
189 ret = crypto_blkcipher_encrypt_iv(&desc, sg, sg, len);
193 static inline void do_encrypt(struct crypto_blkcipher *tfm,
195 void *buf, size_t len)
197 struct scatterlist sg;
199 sg_init_one(&sg, buf, len);
200 do_encrypt_sg(tfm, nonce, &sg, len);
203 int bch_chacha_encrypt_key(struct bch_key *key, struct nonce nonce,
204 void *buf, size_t len)
206 struct crypto_blkcipher *chacha20 =
207 crypto_alloc_blkcipher("chacha20", 0, CRYPTO_ALG_ASYNC);
211 return PTR_ERR(chacha20);
213 ret = crypto_blkcipher_setkey(chacha20, (void *) key, sizeof(*key));
217 do_encrypt(chacha20, nonce, buf, len);
219 crypto_free_blkcipher(chacha20);
223 static void gen_poly_key(struct cache_set *c, struct shash_desc *desc,
226 u8 key[POLY1305_KEY_SIZE];
228 nonce.d[3] ^= BCH_NONCE_POLY;
230 memset(key, 0, sizeof(key));
231 do_encrypt(c->chacha20, nonce, key, sizeof(key));
233 desc->tfm = c->poly1305;
235 crypto_shash_init(desc);
236 crypto_shash_update(desc, key, sizeof(key));
239 struct bch_csum bch_checksum(struct cache_set *c, unsigned type,
240 struct nonce nonce, const void *data, size_t len)
244 case BCH_CSUM_CRC32C:
245 case BCH_CSUM_CRC64: {
246 u64 crc = bch_checksum_init(type);
248 crc = bch_checksum_update(type, crc, data, len);
249 crc = bch_checksum_final(type, crc);
251 return (struct bch_csum) { .lo = crc };
254 case BCH_CSUM_CHACHA20_POLY1305_80:
255 case BCH_CSUM_CHACHA20_POLY1305_128: {
256 SHASH_DESC_ON_STACK(desc, c->poly1305);
257 u8 digest[POLY1305_DIGEST_SIZE];
258 struct bch_csum ret = { 0 };
260 gen_poly_key(c, desc, nonce);
262 crypto_shash_update(desc, data, len);
263 crypto_shash_final(desc, digest);
265 memcpy(&ret, digest, bch_crc_bytes[type]);
273 void bch_encrypt(struct cache_set *c, unsigned type,
274 struct nonce nonce, void *data, size_t len)
276 if (!bch_csum_type_is_encryption(type))
279 do_encrypt(c->chacha20, nonce, data, len);
282 struct bch_csum bch_checksum_bio(struct cache_set *c, unsigned type,
283 struct nonce nonce, struct bio *bio)
286 struct bvec_iter iter;
290 return (struct bch_csum) { 0 };
291 case BCH_CSUM_CRC32C:
292 case BCH_CSUM_CRC64: {
293 u64 crc = bch_checksum_init(type);
295 bio_for_each_segment(bv, bio, iter) {
296 void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
298 crc = bch_checksum_update(type,
303 crc = bch_checksum_final(type, crc);
304 return (struct bch_csum) { .lo = crc };
307 case BCH_CSUM_CHACHA20_POLY1305_80:
308 case BCH_CSUM_CHACHA20_POLY1305_128: {
309 SHASH_DESC_ON_STACK(desc, c->poly1305);
310 u8 digest[POLY1305_DIGEST_SIZE];
311 struct bch_csum ret = { 0 };
313 gen_poly_key(c, desc, nonce);
315 bio_for_each_segment(bv, bio, iter) {
316 void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
318 crypto_shash_update(desc, p, bv.bv_len);
322 crypto_shash_final(desc, digest);
324 memcpy(&ret, digest, bch_crc_bytes[type]);
332 void bch_encrypt_bio(struct cache_set *c, unsigned type,
333 struct nonce nonce, struct bio *bio)
336 struct bvec_iter iter;
337 struct scatterlist sgl[16], *sg = sgl;
340 if (!bch_csum_type_is_encryption(type))
343 sg_init_table(sgl, ARRAY_SIZE(sgl));
345 bio_for_each_segment(bv, bio, iter) {
346 if (sg == sgl + ARRAY_SIZE(sgl)) {
348 do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
350 le32_add_cpu(nonce.d, bytes / CHACHA20_BLOCK_SIZE);
353 sg_init_table(sgl, ARRAY_SIZE(sgl));
357 sg_set_page(sg++, bv.bv_page, bv.bv_len, bv.bv_offset);
363 do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
367 int bch_request_key(struct bch_sb *sb, struct bch_key *key)
369 char key_description[60];
370 struct key *keyring_key;
371 const struct user_key_payload *ukp;
374 snprintf(key_description, sizeof(key_description),
375 "bcache:%pUb", &sb->user_uuid);
377 keyring_key = request_key(&key_type_logon, key_description, NULL);
378 if (IS_ERR(keyring_key))
379 return PTR_ERR(keyring_key);
381 down_read(&keyring_key->sem);
382 ukp = user_key_payload(keyring_key);
383 if (ukp->datalen == sizeof(*key)) {
384 memcpy(key, ukp->data, ukp->datalen);
389 up_read(&keyring_key->sem);
390 key_put(keyring_key);
395 #include <keyutils.h>
396 #include <uuid/uuid.h>
398 int bch_request_key(struct bch_sb *sb, struct bch_key *key)
401 char key_description[60];
404 uuid_unparse_lower(sb->user_uuid.b, uuid);
405 sprintf(key_description, "bcache:%s", uuid);
407 key_id = request_key("user", key_description, NULL,
408 KEY_SPEC_USER_KEYRING);
412 if (keyctl_read(key_id, (void *) key, sizeof(*key)) != sizeof(*key))
419 static int bch_decrypt_sb_key(struct cache_set *c,
420 struct bch_sb_field_crypt *crypt,
423 struct bch_encrypted_key sb_key = crypt->key;
424 struct bch_key user_key;
427 /* is key encrypted? */
428 if (!bch_key_is_encrypted(&sb_key))
431 ret = bch_request_key(c->disk_sb, &user_key);
433 bch_err(c, "error requesting encryption key");
437 /* decrypt real key: */
438 ret = bch_chacha_encrypt_key(&user_key, bch_sb_key_nonce(c),
439 &sb_key, sizeof(sb_key));
443 if (bch_key_is_encrypted(&sb_key)) {
444 bch_err(c, "incorrect encryption key");
451 memzero_explicit(&sb_key, sizeof(sb_key));
452 memzero_explicit(&user_key, sizeof(user_key));
456 static int bch_alloc_ciphers(struct cache_set *c)
459 c->chacha20 = crypto_alloc_blkcipher("chacha20", 0,
461 if (IS_ERR(c->chacha20))
462 return PTR_ERR(c->chacha20);
465 c->poly1305 = crypto_alloc_shash("poly1305", 0, 0);
466 if (IS_ERR(c->poly1305))
467 return PTR_ERR(c->poly1305);
472 int bch_disable_encryption(struct cache_set *c)
474 struct bch_sb_field_crypt *crypt;
478 mutex_lock(&c->sb_lock);
480 crypt = bch_sb_get_crypt(c->disk_sb);
484 /* is key encrypted? */
486 if (bch_key_is_encrypted(&crypt->key))
489 ret = bch_decrypt_sb_key(c, crypt, &key);
493 crypt->key.magic = BCH_KEY_MAGIC;
494 crypt->key.key = key;
496 SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 0);
499 mutex_unlock(&c->sb_lock);
504 int bch_enable_encryption(struct cache_set *c, bool keyed)
506 struct bch_encrypted_key key;
507 struct bch_key user_key;
508 struct bch_sb_field_crypt *crypt;
511 mutex_lock(&c->sb_lock);
513 /* Do we already have an encryption key? */
514 if (bch_sb_get_crypt(c->disk_sb))
517 ret = bch_alloc_ciphers(c);
521 key.magic = BCH_KEY_MAGIC;
522 get_random_bytes(&key.key, sizeof(key.key));
525 ret = bch_request_key(c->disk_sb, &user_key);
527 bch_err(c, "error requesting encryption key");
531 ret = bch_chacha_encrypt_key(&user_key, bch_sb_key_nonce(c),
537 ret = crypto_blkcipher_setkey(c->chacha20,
538 (void *) &key.key, sizeof(key.key));
542 crypt = container_of_or_null(bch_fs_sb_field_resize(c, NULL,
543 sizeof(*crypt) / sizeof(u64)),
544 struct bch_sb_field_crypt, field);
546 ret = -ENOMEM; /* XXX this technically could be -ENOSPC */
550 crypt->field.type = BCH_SB_FIELD_crypt;
553 /* write superblock */
554 SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 1);
557 mutex_unlock(&c->sb_lock);
558 memzero_explicit(&user_key, sizeof(user_key));
559 memzero_explicit(&key, sizeof(key));
563 void bch_fs_encryption_free(struct cache_set *c)
565 if (!IS_ERR_OR_NULL(c->poly1305))
566 crypto_free_shash(c->poly1305);
567 if (!IS_ERR_OR_NULL(c->chacha20))
568 crypto_free_blkcipher(c->chacha20);
571 int bch_fs_encryption_init(struct cache_set *c)
573 struct bch_sb_field_crypt *crypt;
577 crypt = bch_sb_get_crypt(c->disk_sb);
581 ret = bch_alloc_ciphers(c);
585 ret = bch_decrypt_sb_key(c, crypt, &key);
589 ret = crypto_blkcipher_setkey(c->chacha20,
590 (void *) &key.key, sizeof(key.key));
592 memzero_explicit(&key, sizeof(key));