git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/checksum.c

   1 #include "bcachefs.h"
   2 #include "checksum.h"
   3 #include "super.h"
   4 #include "super-io.h"
   5
   6 #include <linux/crc32c.h>
   7 #include <linux/crypto.h>
   8 #include <linux/key.h>
   9 #include <linux/random.h>
  10 #include <linux/scatterlist.h>
  11 #include <crypto/algapi.h>
  12 #include <crypto/chacha20.h>
  13 #include <crypto/hash.h>
  14 #include <crypto/poly1305.h>
  15 #include <keys/user-type.h>
  16
  17 /*
  18  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group (Any
  19  * use permitted, subject to terms of PostgreSQL license; see.)
  20
  21  * If we have a 64-bit integer type, then a 64-bit CRC looks just like the
  22  * usual sort of implementation. (See Ross Williams' excellent introduction
  23  * A PAINLESS GUIDE TO CRC ERROR DETECTION ALGORITHMS, available from
  24  * ftp://ftp.rocksoft.com/papers/crc_v3.txt or several other net sites.)
  25  * If we have no working 64-bit type, then fake it with two 32-bit registers.
  26  *
  27  * The present implementation is a normal (not "reflected", in Williams'
  28  * terms) 64-bit CRC, using initial all-ones register contents and a final
  29  * bit inversion. The chosen polynomial is borrowed from the DLT1 spec
  30  * (ECMA-182, available from http://www.ecma.ch/ecma1/STAND/ECMA-182.HTM):
  31  *
  32  * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
  33  * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
  34  * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
  35  * x^7 + x^4 + x + 1
  36 */
  37
  38 static const u64 crc_table[256] = {
  39         0x0000000000000000ULL, 0x42F0E1EBA9EA3693ULL, 0x85E1C3D753D46D26ULL,
  40         0xC711223CFA3E5BB5ULL, 0x493366450E42ECDFULL, 0x0BC387AEA7A8DA4CULL,
  41         0xCCD2A5925D9681F9ULL, 0x8E224479F47CB76AULL, 0x9266CC8A1C85D9BEULL,
  42         0xD0962D61B56FEF2DULL, 0x17870F5D4F51B498ULL, 0x5577EEB6E6BB820BULL,
  43         0xDB55AACF12C73561ULL, 0x99A54B24BB2D03F2ULL, 0x5EB4691841135847ULL,
  44         0x1C4488F3E8F96ED4ULL, 0x663D78FF90E185EFULL, 0x24CD9914390BB37CULL,
  45         0xE3DCBB28C335E8C9ULL, 0xA12C5AC36ADFDE5AULL, 0x2F0E1EBA9EA36930ULL,
  46         0x6DFEFF5137495FA3ULL, 0xAAEFDD6DCD770416ULL, 0xE81F3C86649D3285ULL,
  47         0xF45BB4758C645C51ULL, 0xB6AB559E258E6AC2ULL, 0x71BA77A2DFB03177ULL,
  48         0x334A9649765A07E4ULL, 0xBD68D2308226B08EULL, 0xFF9833DB2BCC861DULL,
  49         0x388911E7D1F2DDA8ULL, 0x7A79F00C7818EB3BULL, 0xCC7AF1FF21C30BDEULL,
  50         0x8E8A101488293D4DULL, 0x499B3228721766F8ULL, 0x0B6BD3C3DBFD506BULL,
  51         0x854997BA2F81E701ULL, 0xC7B97651866BD192ULL, 0x00A8546D7C558A27ULL,
  52         0x4258B586D5BFBCB4ULL, 0x5E1C3D753D46D260ULL, 0x1CECDC9E94ACE4F3ULL,
  53         0xDBFDFEA26E92BF46ULL, 0x990D1F49C77889D5ULL, 0x172F5B3033043EBFULL,
  54         0x55DFBADB9AEE082CULL, 0x92CE98E760D05399ULL, 0xD03E790CC93A650AULL,
  55         0xAA478900B1228E31ULL, 0xE8B768EB18C8B8A2ULL, 0x2FA64AD7E2F6E317ULL,
  56         0x6D56AB3C4B1CD584ULL, 0xE374EF45BF6062EEULL, 0xA1840EAE168A547DULL,
  57         0x66952C92ECB40FC8ULL, 0x2465CD79455E395BULL, 0x3821458AADA7578FULL,
  58         0x7AD1A461044D611CULL, 0xBDC0865DFE733AA9ULL, 0xFF3067B657990C3AULL,
  59         0x711223CFA3E5BB50ULL, 0x33E2C2240A0F8DC3ULL, 0xF4F3E018F031D676ULL,
  60         0xB60301F359DBE0E5ULL, 0xDA050215EA6C212FULL, 0x98F5E3FE438617BCULL,
  61         0x5FE4C1C2B9B84C09ULL, 0x1D14202910527A9AULL, 0x93366450E42ECDF0ULL,
  62         0xD1C685BB4DC4FB63ULL, 0x16D7A787B7FAA0D6ULL, 0x5427466C1E109645ULL,
  63         0x4863CE9FF6E9F891ULL, 0x0A932F745F03CE02ULL, 0xCD820D48A53D95B7ULL,
  64         0x8F72ECA30CD7A324ULL, 0x0150A8DAF8AB144EULL, 0x43A04931514122DDULL,
  65         0x84B16B0DAB7F7968ULL, 0xC6418AE602954FFBULL, 0xBC387AEA7A8DA4C0ULL,
  66         0xFEC89B01D3679253ULL, 0x39D9B93D2959C9E6ULL, 0x7B2958D680B3FF75ULL,
  67         0xF50B1CAF74CF481FULL, 0xB7FBFD44DD257E8CULL, 0x70EADF78271B2539ULL,
  68         0x321A3E938EF113AAULL, 0x2E5EB66066087D7EULL, 0x6CAE578BCFE24BEDULL,
  69         0xABBF75B735DC1058ULL, 0xE94F945C9C3626CBULL, 0x676DD025684A91A1ULL,
  70         0x259D31CEC1A0A732ULL, 0xE28C13F23B9EFC87ULL, 0xA07CF2199274CA14ULL,
  71         0x167FF3EACBAF2AF1ULL, 0x548F120162451C62ULL, 0x939E303D987B47D7ULL,
  72         0xD16ED1D631917144ULL, 0x5F4C95AFC5EDC62EULL, 0x1DBC74446C07F0BDULL,
  73         0xDAAD56789639AB08ULL, 0x985DB7933FD39D9BULL, 0x84193F60D72AF34FULL,
  74         0xC6E9DE8B7EC0C5DCULL, 0x01F8FCB784FE9E69ULL, 0x43081D5C2D14A8FAULL,
  75         0xCD2A5925D9681F90ULL, 0x8FDAB8CE70822903ULL, 0x48CB9AF28ABC72B6ULL,
  76         0x0A3B7B1923564425ULL, 0x70428B155B4EAF1EULL, 0x32B26AFEF2A4998DULL,
  77         0xF5A348C2089AC238ULL, 0xB753A929A170F4ABULL, 0x3971ED50550C43C1ULL,
  78         0x7B810CBBFCE67552ULL, 0xBC902E8706D82EE7ULL, 0xFE60CF6CAF321874ULL,
  79         0xE224479F47CB76A0ULL, 0xA0D4A674EE214033ULL, 0x67C58448141F1B86ULL,
  80         0x253565A3BDF52D15ULL, 0xAB1721DA49899A7FULL, 0xE9E7C031E063ACECULL,
  81         0x2EF6E20D1A5DF759ULL, 0x6C0603E6B3B7C1CAULL, 0xF6FAE5C07D3274CDULL,
  82         0xB40A042BD4D8425EULL, 0x731B26172EE619EBULL, 0x31EBC7FC870C2F78ULL,
  83         0xBFC9838573709812ULL, 0xFD39626EDA9AAE81ULL, 0x3A28405220A4F534ULL,
  84         0x78D8A1B9894EC3A7ULL, 0x649C294A61B7AD73ULL, 0x266CC8A1C85D9BE0ULL,
  85         0xE17DEA9D3263C055ULL, 0xA38D0B769B89F6C6ULL, 0x2DAF4F0F6FF541ACULL,
  86         0x6F5FAEE4C61F773FULL, 0xA84E8CD83C212C8AULL, 0xEABE6D3395CB1A19ULL,
  87         0x90C79D3FEDD3F122ULL, 0xD2377CD44439C7B1ULL, 0x15265EE8BE079C04ULL,
  88         0x57D6BF0317EDAA97ULL, 0xD9F4FB7AE3911DFDULL, 0x9B041A914A7B2B6EULL,
  89         0x5C1538ADB04570DBULL, 0x1EE5D94619AF4648ULL, 0x02A151B5F156289CULL,
  90         0x4051B05E58BC1E0FULL, 0x87409262A28245BAULL, 0xC5B073890B687329ULL,
  91         0x4B9237F0FF14C443ULL, 0x0962D61B56FEF2D0ULL, 0xCE73F427ACC0A965ULL,
  92         0x8C8315CC052A9FF6ULL, 0x3A80143F5CF17F13ULL, 0x7870F5D4F51B4980ULL,
  93         0xBF61D7E80F251235ULL, 0xFD913603A6CF24A6ULL, 0x73B3727A52B393CCULL,
  94         0x31439391FB59A55FULL, 0xF652B1AD0167FEEAULL, 0xB4A25046A88DC879ULL,
  95         0xA8E6D8B54074A6ADULL, 0xEA16395EE99E903EULL, 0x2D071B6213A0CB8BULL,
  96         0x6FF7FA89BA4AFD18ULL, 0xE1D5BEF04E364A72ULL, 0xA3255F1BE7DC7CE1ULL,
  97         0x64347D271DE22754ULL, 0x26C49CCCB40811C7ULL, 0x5CBD6CC0CC10FAFCULL,
  98         0x1E4D8D2B65FACC6FULL, 0xD95CAF179FC497DAULL, 0x9BAC4EFC362EA149ULL,
  99         0x158E0A85C2521623ULL, 0x577EEB6E6BB820B0ULL, 0x906FC95291867B05ULL,
 100         0xD29F28B9386C4D96ULL, 0xCEDBA04AD0952342ULL, 0x8C2B41A1797F15D1ULL,
 101         0x4B3A639D83414E64ULL, 0x09CA82762AAB78F7ULL, 0x87E8C60FDED7CF9DULL,
 102         0xC51827E4773DF90EULL, 0x020905D88D03A2BBULL, 0x40F9E43324E99428ULL,
 103         0x2CFFE7D5975E55E2ULL, 0x6E0F063E3EB46371ULL, 0xA91E2402C48A38C4ULL,
 104         0xEBEEC5E96D600E57ULL, 0x65CC8190991CB93DULL, 0x273C607B30F68FAEULL,
 105         0xE02D4247CAC8D41BULL, 0xA2DDA3AC6322E288ULL, 0xBE992B5F8BDB8C5CULL,
 106         0xFC69CAB42231BACFULL, 0x3B78E888D80FE17AULL, 0x7988096371E5D7E9ULL,
 107         0xF7AA4D1A85996083ULL, 0xB55AACF12C735610ULL, 0x724B8ECDD64D0DA5ULL,
 108         0x30BB6F267FA73B36ULL, 0x4AC29F2A07BFD00DULL, 0x08327EC1AE55E69EULL,
 109         0xCF235CFD546BBD2BULL, 0x8DD3BD16FD818BB8ULL, 0x03F1F96F09FD3CD2ULL,
 110         0x41011884A0170A41ULL, 0x86103AB85A2951F4ULL, 0xC4E0DB53F3C36767ULL,
 111         0xD8A453A01B3A09B3ULL, 0x9A54B24BB2D03F20ULL, 0x5D45907748EE6495ULL,
 112         0x1FB5719CE1045206ULL, 0x919735E51578E56CULL, 0xD367D40EBC92D3FFULL,
 113         0x1476F63246AC884AULL, 0x568617D9EF46BED9ULL, 0xE085162AB69D5E3CULL,
 114         0xA275F7C11F7768AFULL, 0x6564D5FDE549331AULL, 0x279434164CA30589ULL,
 115         0xA9B6706FB8DFB2E3ULL, 0xEB46918411358470ULL, 0x2C57B3B8EB0BDFC5ULL,
 116         0x6EA7525342E1E956ULL, 0x72E3DAA0AA188782ULL, 0x30133B4B03F2B111ULL,
 117         0xF7021977F9CCEAA4ULL, 0xB5F2F89C5026DC37ULL, 0x3BD0BCE5A45A6B5DULL,
 118         0x79205D0E0DB05DCEULL, 0xBE317F32F78E067BULL, 0xFCC19ED95E6430E8ULL,
 119         0x86B86ED5267CDBD3ULL, 0xC4488F3E8F96ED40ULL, 0x0359AD0275A8B6F5ULL,
 120         0x41A94CE9DC428066ULL, 0xCF8B0890283E370CULL, 0x8D7BE97B81D4019FULL,
 121         0x4A6ACB477BEA5A2AULL, 0x089A2AACD2006CB9ULL, 0x14DEA25F3AF9026DULL,
 122         0x562E43B4931334FEULL, 0x913F6188692D6F4BULL, 0xD3CF8063C0C759D8ULL,
 123         0x5DEDC41A34BBEEB2ULL, 0x1F1D25F19D51D821ULL, 0xD80C07CD676F8394ULL,
 124         0x9AFCE626CE85B507ULL,
 125 };
 126
 127 u64 bch2_crc64_update(u64 crc, const void *_data, size_t len)
 128 {
 129         const unsigned char *data = _data;
 130
 131         while (len--) {
 132                 int i = ((int) (crc >> 56) ^ *data++) & 0xFF;
 133                 crc = crc_table[i] ^ (crc << 8);
 134         }
 135
 136         return crc;
 137 }
 138
 139 static u64 bch2_checksum_init(unsigned type)
 140 {
 141         switch (type) {
 142         case BCH_CSUM_NONE:
 143                 return 0;
 144         case BCH_CSUM_CRC32C_NONZERO:
 145                 return U32_MAX;
 146         case BCH_CSUM_CRC64_NONZERO:
 147                 return U64_MAX;
 148         case BCH_CSUM_CRC32C:
 149                 return 0;
 150         case BCH_CSUM_CRC64:
 151                 return 0;
 152         default:
 153                 BUG();
 154         }
 155 }
 156
 157 static u64 bch2_checksum_final(unsigned type, u64 crc)
 158 {
 159         switch (type) {
 160         case BCH_CSUM_NONE:
 161                 return 0;
 162         case BCH_CSUM_CRC32C_NONZERO:
 163                 return crc ^ U32_MAX;
 164         case BCH_CSUM_CRC64_NONZERO:
 165                 return crc ^ U64_MAX;
 166         case BCH_CSUM_CRC32C:
 167                 return crc;
 168         case BCH_CSUM_CRC64:
 169                 return crc;
 170         default:
 171                 BUG();
 172         }
 173 }
 174
 175 static u64 bch2_checksum_update(unsigned type, u64 crc, const void *data, size_t len)
 176 {
 177         switch (type) {
 178         case BCH_CSUM_NONE:
 179                 return 0;
 180         case BCH_CSUM_CRC32C_NONZERO:
 181         case BCH_CSUM_CRC32C:
 182                 return crc32c(crc, data, len);
 183         case BCH_CSUM_CRC64_NONZERO:
 184         case BCH_CSUM_CRC64:
 185                 return bch2_crc64_update(crc, data, len);
 186         default:
 187                 BUG();
 188         }
 189 }
 190
 191 static inline void do_encrypt_sg(struct crypto_skcipher *tfm,
 192                                  struct nonce nonce,
 193                                  struct scatterlist *sg, size_t len)
 194 {
 195         SKCIPHER_REQUEST_ON_STACK(req, tfm);
 196         int ret;
 197
 198         skcipher_request_set_tfm(req, tfm);
 199         skcipher_request_set_crypt(req, sg, sg, len, nonce.d);
 200
 201         ret = crypto_skcipher_encrypt(req);
 202         BUG_ON(ret);
 203 }
 204
 205 static inline void do_encrypt(struct crypto_skcipher *tfm,
 206                               struct nonce nonce,
 207                               void *buf, size_t len)
 208 {
 209         struct scatterlist sg;
 210
 211         sg_init_one(&sg, buf, len);
 212         do_encrypt_sg(tfm, nonce, &sg, len);
 213 }
 214
 215 int bch2_chacha_encrypt_key(struct bch_key *key, struct nonce nonce,
 216                            void *buf, size_t len)
 217 {
 218         struct crypto_skcipher *chacha20 =
 219                 crypto_alloc_skcipher("chacha20", 0, 0);
 220         int ret;
 221
 222         if (!chacha20)
 223                 return PTR_ERR(chacha20);
 224
 225         ret = crypto_skcipher_setkey(chacha20, (void *) key, sizeof(*key));
 226         if (ret)
 227                 goto err;
 228
 229         do_encrypt(chacha20, nonce, buf, len);
 230 err:
 231         crypto_free_skcipher(chacha20);
 232         return ret;
 233 }
 234
 235 static void gen_poly_key(struct bch_fs *c, struct shash_desc *desc,
 236                          struct nonce nonce)
 237 {
 238         u8 key[POLY1305_KEY_SIZE];
 239
 240         nonce.d[3] ^= BCH_NONCE_POLY;
 241
 242         memset(key, 0, sizeof(key));
 243         do_encrypt(c->chacha20, nonce, key, sizeof(key));
 244
 245         desc->tfm = c->poly1305;
 246         desc->flags = 0;
 247         crypto_shash_init(desc);
 248         crypto_shash_update(desc, key, sizeof(key));
 249 }
 250
 251 struct bch_csum bch2_checksum(struct bch_fs *c, unsigned type,
 252                               struct nonce nonce, const void *data, size_t len)
 253 {
 254         switch (type) {
 255         case BCH_CSUM_NONE:
 256         case BCH_CSUM_CRC32C_NONZERO:
 257         case BCH_CSUM_CRC64_NONZERO:
 258         case BCH_CSUM_CRC32C:
 259         case BCH_CSUM_CRC64: {
 260                 u64 crc = bch2_checksum_init(type);
 261
 262                 crc = bch2_checksum_update(type, crc, data, len);
 263                 crc = bch2_checksum_final(type, crc);
 264
 265                 return (struct bch_csum) { .lo = cpu_to_le64(crc) };
 266         }
 267
 268         case BCH_CSUM_CHACHA20_POLY1305_80:
 269         case BCH_CSUM_CHACHA20_POLY1305_128: {
 270                 SHASH_DESC_ON_STACK(desc, c->poly1305);
 271                 u8 digest[POLY1305_DIGEST_SIZE];
 272                 struct bch_csum ret = { 0 };
 273
 274                 gen_poly_key(c, desc, nonce);
 275
 276                 crypto_shash_update(desc, data, len);
 277                 crypto_shash_final(desc, digest);
 278
 279                 memcpy(&ret, digest, bch_crc_bytes[type]);
 280                 return ret;
 281         }
 282         default:
 283                 BUG();
 284         }
 285 }
 286
 287 void bch2_encrypt(struct bch_fs *c, unsigned type,
 288                   struct nonce nonce, void *data, size_t len)
 289 {
 290         if (!bch2_csum_type_is_encryption(type))
 291                 return;
 292
 293         do_encrypt(c->chacha20, nonce, data, len);
 294 }
 295
 296 static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type,
 297                                            struct nonce nonce, struct bio *bio,
 298                                            struct bvec_iter *iter)
 299 {
 300         struct bio_vec bv;
 301
 302         switch (type) {
 303         case BCH_CSUM_NONE:
 304                 return (struct bch_csum) { 0 };
 305         case BCH_CSUM_CRC32C_NONZERO:
 306         case BCH_CSUM_CRC64_NONZERO:
 307         case BCH_CSUM_CRC32C:
 308         case BCH_CSUM_CRC64: {
 309                 u64 crc = bch2_checksum_init(type);
 310
 311 #ifdef CONFIG_HIGHMEM
 312                 __bio_for_each_segment(bv, bio, *iter, *iter) {
 313                         void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
 314                         crc = bch2_checksum_update(type,
 315                                 crc, p, bv.bv_len);
 316                         kunmap_atomic(p);
 317                 }
 318 #else
 319                 __bio_for_each_contig_segment(bv, bio, *iter, *iter)
 320                         crc = bch2_checksum_update(type, crc,
 321                                 page_address(bv.bv_page) + bv.bv_offset,
 322                                 bv.bv_len);
 323 #endif
 324                 crc = bch2_checksum_final(type, crc);
 325                 return (struct bch_csum) { .lo = cpu_to_le64(crc) };
 326         }
 327
 328         case BCH_CSUM_CHACHA20_POLY1305_80:
 329         case BCH_CSUM_CHACHA20_POLY1305_128: {
 330                 SHASH_DESC_ON_STACK(desc, c->poly1305);
 331                 u8 digest[POLY1305_DIGEST_SIZE];
 332                 struct bch_csum ret = { 0 };
 333
 334                 gen_poly_key(c, desc, nonce);
 335
 336 #ifdef CONFIG_HIGHMEM
 337                 __bio_for_each_segment(bv, bio, *iter, *iter) {
 338                         void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
 339
 340                         crypto_shash_update(desc, p, bv.bv_len);
 341                         kunmap_atomic(p);
 342                 }
 343 #else
 344                 __bio_for_each_contig_segment(bv, bio, *iter, *iter)
 345                         crypto_shash_update(desc,
 346                                 page_address(bv.bv_page) + bv.bv_offset,
 347                                 bv.bv_len);
 348 #endif
 349                 crypto_shash_final(desc, digest);
 350
 351                 memcpy(&ret, digest, bch_crc_bytes[type]);
 352                 return ret;
 353         }
 354         default:
 355                 BUG();
 356         }
 357 }
 358
 359 struct bch_csum bch2_checksum_bio(struct bch_fs *c, unsigned type,
 360                                   struct nonce nonce, struct bio *bio)
 361 {
 362         struct bvec_iter iter = bio->bi_iter;
 363
 364         return __bch2_checksum_bio(c, type, nonce, bio, &iter);
 365 }
 366
 367 void bch2_encrypt_bio(struct bch_fs *c, unsigned type,
 368                       struct nonce nonce, struct bio *bio)
 369 {
 370         struct bio_vec bv;
 371         struct bvec_iter iter;
 372         struct scatterlist sgl[16], *sg = sgl;
 373         size_t bytes = 0;
 374
 375         if (!bch2_csum_type_is_encryption(type))
 376                 return;
 377
 378         sg_init_table(sgl, ARRAY_SIZE(sgl));
 379
 380         bio_for_each_segment(bv, bio, iter) {
 381                 if (sg == sgl + ARRAY_SIZE(sgl)) {
 382                         sg_mark_end(sg - 1);
 383                         do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
 384
 385                         nonce = nonce_add(nonce, bytes);
 386                         bytes = 0;
 387
 388                         sg_init_table(sgl, ARRAY_SIZE(sgl));
 389                         sg = sgl;
 390                 }
 391
 392                 sg_set_page(sg++, bv.bv_page, bv.bv_len, bv.bv_offset);
 393                 bytes += bv.bv_len;
 394         }
 395
 396         sg_mark_end(sg - 1);
 397         do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
 398 }
 399
 400 static inline bool bch2_checksum_mergeable(unsigned type)
 401 {
 402
 403         switch (type) {
 404         case BCH_CSUM_NONE:
 405         case BCH_CSUM_CRC32C:
 406         case BCH_CSUM_CRC64:
 407                 return true;
 408         default:
 409                 return false;
 410         }
 411 }
 412
 413 static struct bch_csum bch2_checksum_merge(unsigned type,
 414                                            struct bch_csum a,
 415                                            struct bch_csum b, size_t b_len)
 416 {
 417         BUG_ON(!bch2_checksum_mergeable(type));
 418
 419         while (b_len) {
 420                 unsigned b = min(b_len, PAGE_SIZE);
 421
 422                 a.lo = bch2_checksum_update(type, a.lo,
 423                                 page_address(ZERO_PAGE(0)), b);
 424                 b_len -= b;
 425         }
 426
 427         a.lo ^= b.lo;
 428         a.hi ^= b.hi;
 429         return a;
 430 }
 431
 432 int bch2_rechecksum_bio(struct bch_fs *c, struct bio *bio,
 433                         struct bversion version,
 434                         struct bch_extent_crc_unpacked crc_old,
 435                         struct bch_extent_crc_unpacked *crc_a,
 436                         struct bch_extent_crc_unpacked *crc_b,
 437                         unsigned len_a, unsigned len_b,
 438                         unsigned new_csum_type)
 439 {
 440         struct bvec_iter iter = bio->bi_iter;
 441         struct nonce nonce = extent_nonce(version, crc_old);
 442         struct bch_csum merged = { 0 };
 443         struct crc_split {
 444                 struct bch_extent_crc_unpacked  *crc;
 445                 unsigned                        len;
 446                 unsigned                        csum_type;
 447                 struct bch_csum                 csum;
 448         } splits[3] = {
 449                 { crc_a, len_a, new_csum_type },
 450                 { crc_b, len_b, new_csum_type },
 451                 { NULL,  bio_sectors(bio) - len_a - len_b, new_csum_type },
 452         }, *i;
 453         bool mergeable = crc_old.csum_type == new_csum_type &&
 454                 bch2_checksum_mergeable(new_csum_type);
 455         unsigned crc_nonce = crc_old.nonce;
 456
 457         BUG_ON(len_a + len_b > bio_sectors(bio));
 458         BUG_ON(crc_old.uncompressed_size != bio_sectors(bio));
 459         BUG_ON(crc_old.compression_type);
 460         BUG_ON(bch2_csum_type_is_encryption(crc_old.csum_type) !=
 461                bch2_csum_type_is_encryption(new_csum_type));
 462
 463         for (i = splits; i < splits + ARRAY_SIZE(splits); i++) {
 464                 iter.bi_size = i->len << 9;
 465                 if (mergeable || i->crc)
 466                         i->csum = __bch2_checksum_bio(c, i->csum_type,
 467                                                       nonce, bio, &iter);
 468                 else
 469                         bio_advance_iter(bio, &iter, i->len << 9);
 470                 nonce = nonce_add(nonce, i->len << 9);
 471         }
 472
 473         if (mergeable)
 474                 for (i = splits; i < splits + ARRAY_SIZE(splits); i++)
 475                         merged = bch2_checksum_merge(new_csum_type, merged,
 476                                                      i->csum, i->len << 9);
 477         else
 478                 merged = bch2_checksum_bio(c, crc_old.csum_type,
 479                                 extent_nonce(version, crc_old), bio);
 480
 481         if (bch2_crc_cmp(merged, crc_old.csum))
 482                 return -EIO;
 483
 484         for (i = splits; i < splits + ARRAY_SIZE(splits); i++) {
 485                 if (i->crc)
 486                         *i->crc = (struct bch_extent_crc_unpacked) {
 487                                 .csum_type              = i->csum_type,
 488                                 .compressed_size        = i->len,
 489                                 .uncompressed_size      = i->len,
 490                                 .offset                 = 0,
 491                                 .live_size              = i->len,
 492                                 .nonce                  = crc_nonce,
 493                                 .csum                   = i->csum,
 494                         };
 495
 496                 if (bch2_csum_type_is_encryption(new_csum_type))
 497                         crc_nonce += i->len;
 498         }
 499
 500         return 0;
 501 }
 502
 503 #ifdef __KERNEL__
 504 int bch2_request_key(struct bch_sb *sb, struct bch_key *key)
 505 {
 506         char key_description[60];
 507         struct key *keyring_key;
 508         const struct user_key_payload *ukp;
 509         int ret;
 510
 511         snprintf(key_description, sizeof(key_description),
 512                  "bcachefs:%pUb", &sb->user_uuid);
 513
 514         keyring_key = request_key(&key_type_logon, key_description, NULL);
 515         if (IS_ERR(keyring_key))
 516                 return PTR_ERR(keyring_key);
 517
 518         down_read(&keyring_key->sem);
 519         ukp = dereference_key_locked(keyring_key);
 520         if (ukp->datalen == sizeof(*key)) {
 521                 memcpy(key, ukp->data, ukp->datalen);
 522                 ret = 0;
 523         } else {
 524                 ret = -EINVAL;
 525         }
 526         up_read(&keyring_key->sem);
 527         key_put(keyring_key);
 528
 529         return ret;
 530 }
 531 #else
 532 #include <keyutils.h>
 533 #include <uuid/uuid.h>
 534
 535 int bch2_request_key(struct bch_sb *sb, struct bch_key *key)
 536 {
 537         key_serial_t key_id;
 538         char key_description[60];
 539         char uuid[40];
 540
 541         uuid_unparse_lower(sb->user_uuid.b, uuid);
 542         sprintf(key_description, "bcachefs:%s", uuid);
 543
 544         key_id = request_key("user", key_description, NULL,
 545                              KEY_SPEC_USER_KEYRING);
 546         if (key_id < 0)
 547                 return -errno;
 548
 549         if (keyctl_read(key_id, (void *) key, sizeof(*key)) != sizeof(*key))
 550                 return -1;
 551
 552         return 0;
 553 }
 554 #endif
 555
 556 int bch2_decrypt_sb_key(struct bch_fs *c,
 557                         struct bch_sb_field_crypt *crypt,
 558                         struct bch_key *key)
 559 {
 560         struct bch_encrypted_key sb_key = crypt->key;
 561         struct bch_key user_key;
 562         int ret = 0;
 563
 564         /* is key encrypted? */
 565         if (!bch2_key_is_encrypted(&sb_key))
 566                 goto out;
 567
 568         ret = bch2_request_key(c->disk_sb, &user_key);
 569         if (ret) {
 570                 bch_err(c, "error requesting encryption key");
 571                 goto err;
 572         }
 573
 574         /* decrypt real key: */
 575         ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c),
 576                              &sb_key, sizeof(sb_key));
 577         if (ret)
 578                 goto err;
 579
 580         if (bch2_key_is_encrypted(&sb_key)) {
 581                 bch_err(c, "incorrect encryption key");
 582                 ret = -EINVAL;
 583                 goto err;
 584         }
 585 out:
 586         *key = sb_key.key;
 587 err:
 588         memzero_explicit(&sb_key, sizeof(sb_key));
 589         memzero_explicit(&user_key, sizeof(user_key));
 590         return ret;
 591 }
 592
 593 static int bch2_alloc_ciphers(struct bch_fs *c)
 594 {
 595         if (!c->chacha20)
 596                 c->chacha20 = crypto_alloc_skcipher("chacha20", 0, 0);
 597         if (IS_ERR(c->chacha20))
 598                 return PTR_ERR(c->chacha20);
 599
 600         if (!c->poly1305)
 601                 c->poly1305 = crypto_alloc_shash("poly1305", 0, 0);
 602         if (IS_ERR(c->poly1305))
 603                 return PTR_ERR(c->poly1305);
 604
 605         return 0;
 606 }
 607
 608 int bch2_disable_encryption(struct bch_fs *c)
 609 {
 610         struct bch_sb_field_crypt *crypt;
 611         struct bch_key key;
 612         int ret = -EINVAL;
 613
 614         mutex_lock(&c->sb_lock);
 615
 616         crypt = bch2_sb_get_crypt(c->disk_sb);
 617         if (!crypt)
 618                 goto out;
 619
 620         /* is key encrypted? */
 621         ret = 0;
 622         if (bch2_key_is_encrypted(&crypt->key))
 623                 goto out;
 624
 625         ret = bch2_decrypt_sb_key(c, crypt, &key);
 626         if (ret)
 627                 goto out;
 628
 629         crypt->key.magic        = BCH_KEY_MAGIC;
 630         crypt->key.key          = key;
 631
 632         SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 0);
 633         bch2_write_super(c);
 634 out:
 635         mutex_unlock(&c->sb_lock);
 636
 637         return ret;
 638 }
 639
 640 int bch2_enable_encryption(struct bch_fs *c, bool keyed)
 641 {
 642         struct bch_encrypted_key key;
 643         struct bch_key user_key;
 644         struct bch_sb_field_crypt *crypt;
 645         int ret = -EINVAL;
 646
 647         mutex_lock(&c->sb_lock);
 648
 649         /* Do we already have an encryption key? */
 650         if (bch2_sb_get_crypt(c->disk_sb))
 651                 goto err;
 652
 653         ret = bch2_alloc_ciphers(c);
 654         if (ret)
 655                 goto err;
 656
 657         key.magic = BCH_KEY_MAGIC;
 658         get_random_bytes(&key.key, sizeof(key.key));
 659
 660         if (keyed) {
 661                 ret = bch2_request_key(c->disk_sb, &user_key);
 662                 if (ret) {
 663                         bch_err(c, "error requesting encryption key");
 664                         goto err;
 665                 }
 666
 667                 ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c),
 668                                               &key, sizeof(key));
 669                 if (ret)
 670                         goto err;
 671         }
 672
 673         ret = crypto_skcipher_setkey(c->chacha20,
 674                         (void *) &key.key, sizeof(key.key));
 675         if (ret)
 676                 goto err;
 677
 678         crypt = bch2_fs_sb_resize_crypt(c, sizeof(*crypt) / sizeof(u64));
 679         if (!crypt) {
 680                 ret = -ENOMEM; /* XXX this technically could be -ENOSPC */
 681                 goto err;
 682         }
 683
 684         crypt->key = key;
 685
 686         /* write superblock */
 687         SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 1);
 688         bch2_write_super(c);
 689 err:
 690         mutex_unlock(&c->sb_lock);
 691         memzero_explicit(&user_key, sizeof(user_key));
 692         memzero_explicit(&key, sizeof(key));
 693         return ret;
 694 }
 695
 696 void bch2_fs_encryption_exit(struct bch_fs *c)
 697 {
 698         if (!IS_ERR_OR_NULL(c->poly1305))
 699                 crypto_free_shash(c->poly1305);
 700         if (!IS_ERR_OR_NULL(c->chacha20))
 701                 crypto_free_skcipher(c->chacha20);
 702         if (!IS_ERR_OR_NULL(c->sha256))
 703                 crypto_free_shash(c->sha256);
 704 }
 705
 706 int bch2_fs_encryption_init(struct bch_fs *c)
 707 {
 708         struct bch_sb_field_crypt *crypt;
 709         struct bch_key key;
 710         int ret;
 711
 712         c->sha256 = crypto_alloc_shash("sha256", 0, 0);
 713         if (IS_ERR(c->sha256))
 714                 return PTR_ERR(c->sha256);
 715
 716         crypt = bch2_sb_get_crypt(c->disk_sb);
 717         if (!crypt)
 718                 return 0;
 719
 720         ret = bch2_alloc_ciphers(c);
 721         if (ret)
 722                 return ret;
 723
 724         ret = bch2_decrypt_sb_key(c, crypt, &key);
 725         if (ret)
 726                 goto err;
 727
 728         ret = crypto_skcipher_setkey(c->chacha20,
 729                         (void *) &key.key, sizeof(key.key));
 730 err:
 731         memzero_explicit(&key, sizeof(key));
 732         return ret;
 733 }