git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/checksum.c

   1 #include "bcachefs.h"
   2 #include "checksum.h"
   3 #include "super.h"
   4 #include "super-io.h"
   5
   6 #include <linux/crc32c.h>
   7 #include <linux/crypto.h>
   8 #include <linux/key.h>
   9 #include <linux/random.h>
  10 #include <linux/scatterlist.h>
  11 #include <crypto/algapi.h>
  12 #include <crypto/chacha20.h>
  13 #include <crypto/hash.h>
  14 #include <crypto/poly1305.h>
  15 #include <keys/user-type.h>
  16
  17 /*
  18  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group (Any
  19  * use permitted, subject to terms of PostgreSQL license; see.)
  20
  21  * If we have a 64-bit integer type, then a 64-bit CRC looks just like the
  22  * usual sort of implementation. (See Ross Williams' excellent introduction
  23  * A PAINLESS GUIDE TO CRC ERROR DETECTION ALGORITHMS, available from
  24  * ftp://ftp.rocksoft.com/papers/crc_v3.txt or several other net sites.)
  25  * If we have no working 64-bit type, then fake it with two 32-bit registers.
  26  *
  27  * The present implementation is a normal (not "reflected", in Williams'
  28  * terms) 64-bit CRC, using initial all-ones register contents and a final
  29  * bit inversion. The chosen polynomial is borrowed from the DLT1 spec
  30  * (ECMA-182, available from http://www.ecma.ch/ecma1/STAND/ECMA-182.HTM):
  31  *
  32  * x^64 + x^62 + x^57 + x^55 + x^54 + x^53 + x^52 + x^47 + x^46 + x^45 +
  33  * x^40 + x^39 + x^38 + x^37 + x^35 + x^33 + x^32 + x^31 + x^29 + x^27 +
  34  * x^24 + x^23 + x^22 + x^21 + x^19 + x^17 + x^13 + x^12 + x^10 + x^9 +
  35  * x^7 + x^4 + x + 1
  36 */
  37
  38 static const u64 crc_table[256] = {
  39         0x0000000000000000ULL, 0x42F0E1EBA9EA3693ULL, 0x85E1C3D753D46D26ULL,
  40         0xC711223CFA3E5BB5ULL, 0x493366450E42ECDFULL, 0x0BC387AEA7A8DA4CULL,
  41         0xCCD2A5925D9681F9ULL, 0x8E224479F47CB76AULL, 0x9266CC8A1C85D9BEULL,
  42         0xD0962D61B56FEF2DULL, 0x17870F5D4F51B498ULL, 0x5577EEB6E6BB820BULL,
  43         0xDB55AACF12C73561ULL, 0x99A54B24BB2D03F2ULL, 0x5EB4691841135847ULL,
  44         0x1C4488F3E8F96ED4ULL, 0x663D78FF90E185EFULL, 0x24CD9914390BB37CULL,
  45         0xE3DCBB28C335E8C9ULL, 0xA12C5AC36ADFDE5AULL, 0x2F0E1EBA9EA36930ULL,
  46         0x6DFEFF5137495FA3ULL, 0xAAEFDD6DCD770416ULL, 0xE81F3C86649D3285ULL,
  47         0xF45BB4758C645C51ULL, 0xB6AB559E258E6AC2ULL, 0x71BA77A2DFB03177ULL,
  48         0x334A9649765A07E4ULL, 0xBD68D2308226B08EULL, 0xFF9833DB2BCC861DULL,
  49         0x388911E7D1F2DDA8ULL, 0x7A79F00C7818EB3BULL, 0xCC7AF1FF21C30BDEULL,
  50         0x8E8A101488293D4DULL, 0x499B3228721766F8ULL, 0x0B6BD3C3DBFD506BULL,
  51         0x854997BA2F81E701ULL, 0xC7B97651866BD192ULL, 0x00A8546D7C558A27ULL,
  52         0x4258B586D5BFBCB4ULL, 0x5E1C3D753D46D260ULL, 0x1CECDC9E94ACE4F3ULL,
  53         0xDBFDFEA26E92BF46ULL, 0x990D1F49C77889D5ULL, 0x172F5B3033043EBFULL,
  54         0x55DFBADB9AEE082CULL, 0x92CE98E760D05399ULL, 0xD03E790CC93A650AULL,
  55         0xAA478900B1228E31ULL, 0xE8B768EB18C8B8A2ULL, 0x2FA64AD7E2F6E317ULL,
  56         0x6D56AB3C4B1CD584ULL, 0xE374EF45BF6062EEULL, 0xA1840EAE168A547DULL,
  57         0x66952C92ECB40FC8ULL, 0x2465CD79455E395BULL, 0x3821458AADA7578FULL,
  58         0x7AD1A461044D611CULL, 0xBDC0865DFE733AA9ULL, 0xFF3067B657990C3AULL,
  59         0x711223CFA3E5BB50ULL, 0x33E2C2240A0F8DC3ULL, 0xF4F3E018F031D676ULL,
  60         0xB60301F359DBE0E5ULL, 0xDA050215EA6C212FULL, 0x98F5E3FE438617BCULL,
  61         0x5FE4C1C2B9B84C09ULL, 0x1D14202910527A9AULL, 0x93366450E42ECDF0ULL,
  62         0xD1C685BB4DC4FB63ULL, 0x16D7A787B7FAA0D6ULL, 0x5427466C1E109645ULL,
  63         0x4863CE9FF6E9F891ULL, 0x0A932F745F03CE02ULL, 0xCD820D48A53D95B7ULL,
  64         0x8F72ECA30CD7A324ULL, 0x0150A8DAF8AB144EULL, 0x43A04931514122DDULL,
  65         0x84B16B0DAB7F7968ULL, 0xC6418AE602954FFBULL, 0xBC387AEA7A8DA4C0ULL,
  66         0xFEC89B01D3679253ULL, 0x39D9B93D2959C9E6ULL, 0x7B2958D680B3FF75ULL,
  67         0xF50B1CAF74CF481FULL, 0xB7FBFD44DD257E8CULL, 0x70EADF78271B2539ULL,
  68         0x321A3E938EF113AAULL, 0x2E5EB66066087D7EULL, 0x6CAE578BCFE24BEDULL,
  69         0xABBF75B735DC1058ULL, 0xE94F945C9C3626CBULL, 0x676DD025684A91A1ULL,
  70         0x259D31CEC1A0A732ULL, 0xE28C13F23B9EFC87ULL, 0xA07CF2199274CA14ULL,
  71         0x167FF3EACBAF2AF1ULL, 0x548F120162451C62ULL, 0x939E303D987B47D7ULL,
  72         0xD16ED1D631917144ULL, 0x5F4C95AFC5EDC62EULL, 0x1DBC74446C07F0BDULL,
  73         0xDAAD56789639AB08ULL, 0x985DB7933FD39D9BULL, 0x84193F60D72AF34FULL,
  74         0xC6E9DE8B7EC0C5DCULL, 0x01F8FCB784FE9E69ULL, 0x43081D5C2D14A8FAULL,
  75         0xCD2A5925D9681F90ULL, 0x8FDAB8CE70822903ULL, 0x48CB9AF28ABC72B6ULL,
  76         0x0A3B7B1923564425ULL, 0x70428B155B4EAF1EULL, 0x32B26AFEF2A4998DULL,
  77         0xF5A348C2089AC238ULL, 0xB753A929A170F4ABULL, 0x3971ED50550C43C1ULL,
  78         0x7B810CBBFCE67552ULL, 0xBC902E8706D82EE7ULL, 0xFE60CF6CAF321874ULL,
  79         0xE224479F47CB76A0ULL, 0xA0D4A674EE214033ULL, 0x67C58448141F1B86ULL,
  80         0x253565A3BDF52D15ULL, 0xAB1721DA49899A7FULL, 0xE9E7C031E063ACECULL,
  81         0x2EF6E20D1A5DF759ULL, 0x6C0603E6B3B7C1CAULL, 0xF6FAE5C07D3274CDULL,
  82         0xB40A042BD4D8425EULL, 0x731B26172EE619EBULL, 0x31EBC7FC870C2F78ULL,
  83         0xBFC9838573709812ULL, 0xFD39626EDA9AAE81ULL, 0x3A28405220A4F534ULL,
  84         0x78D8A1B9894EC3A7ULL, 0x649C294A61B7AD73ULL, 0x266CC8A1C85D9BE0ULL,
  85         0xE17DEA9D3263C055ULL, 0xA38D0B769B89F6C6ULL, 0x2DAF4F0F6FF541ACULL,
  86         0x6F5FAEE4C61F773FULL, 0xA84E8CD83C212C8AULL, 0xEABE6D3395CB1A19ULL,
  87         0x90C79D3FEDD3F122ULL, 0xD2377CD44439C7B1ULL, 0x15265EE8BE079C04ULL,
  88         0x57D6BF0317EDAA97ULL, 0xD9F4FB7AE3911DFDULL, 0x9B041A914A7B2B6EULL,
  89         0x5C1538ADB04570DBULL, 0x1EE5D94619AF4648ULL, 0x02A151B5F156289CULL,
  90         0x4051B05E58BC1E0FULL, 0x87409262A28245BAULL, 0xC5B073890B687329ULL,
  91         0x4B9237F0FF14C443ULL, 0x0962D61B56FEF2D0ULL, 0xCE73F427ACC0A965ULL,
  92         0x8C8315CC052A9FF6ULL, 0x3A80143F5CF17F13ULL, 0x7870F5D4F51B4980ULL,
  93         0xBF61D7E80F251235ULL, 0xFD913603A6CF24A6ULL, 0x73B3727A52B393CCULL,
  94         0x31439391FB59A55FULL, 0xF652B1AD0167FEEAULL, 0xB4A25046A88DC879ULL,
  95         0xA8E6D8B54074A6ADULL, 0xEA16395EE99E903EULL, 0x2D071B6213A0CB8BULL,
  96         0x6FF7FA89BA4AFD18ULL, 0xE1D5BEF04E364A72ULL, 0xA3255F1BE7DC7CE1ULL,
  97         0x64347D271DE22754ULL, 0x26C49CCCB40811C7ULL, 0x5CBD6CC0CC10FAFCULL,
  98         0x1E4D8D2B65FACC6FULL, 0xD95CAF179FC497DAULL, 0x9BAC4EFC362EA149ULL,
  99         0x158E0A85C2521623ULL, 0x577EEB6E6BB820B0ULL, 0x906FC95291867B05ULL,
 100         0xD29F28B9386C4D96ULL, 0xCEDBA04AD0952342ULL, 0x8C2B41A1797F15D1ULL,
 101         0x4B3A639D83414E64ULL, 0x09CA82762AAB78F7ULL, 0x87E8C60FDED7CF9DULL,
 102         0xC51827E4773DF90EULL, 0x020905D88D03A2BBULL, 0x40F9E43324E99428ULL,
 103         0x2CFFE7D5975E55E2ULL, 0x6E0F063E3EB46371ULL, 0xA91E2402C48A38C4ULL,
 104         0xEBEEC5E96D600E57ULL, 0x65CC8190991CB93DULL, 0x273C607B30F68FAEULL,
 105         0xE02D4247CAC8D41BULL, 0xA2DDA3AC6322E288ULL, 0xBE992B5F8BDB8C5CULL,
 106         0xFC69CAB42231BACFULL, 0x3B78E888D80FE17AULL, 0x7988096371E5D7E9ULL,
 107         0xF7AA4D1A85996083ULL, 0xB55AACF12C735610ULL, 0x724B8ECDD64D0DA5ULL,
 108         0x30BB6F267FA73B36ULL, 0x4AC29F2A07BFD00DULL, 0x08327EC1AE55E69EULL,
 109         0xCF235CFD546BBD2BULL, 0x8DD3BD16FD818BB8ULL, 0x03F1F96F09FD3CD2ULL,
 110         0x41011884A0170A41ULL, 0x86103AB85A2951F4ULL, 0xC4E0DB53F3C36767ULL,
 111         0xD8A453A01B3A09B3ULL, 0x9A54B24BB2D03F20ULL, 0x5D45907748EE6495ULL,
 112         0x1FB5719CE1045206ULL, 0x919735E51578E56CULL, 0xD367D40EBC92D3FFULL,
 113         0x1476F63246AC884AULL, 0x568617D9EF46BED9ULL, 0xE085162AB69D5E3CULL,
 114         0xA275F7C11F7768AFULL, 0x6564D5FDE549331AULL, 0x279434164CA30589ULL,
 115         0xA9B6706FB8DFB2E3ULL, 0xEB46918411358470ULL, 0x2C57B3B8EB0BDFC5ULL,
 116         0x6EA7525342E1E956ULL, 0x72E3DAA0AA188782ULL, 0x30133B4B03F2B111ULL,
 117         0xF7021977F9CCEAA4ULL, 0xB5F2F89C5026DC37ULL, 0x3BD0BCE5A45A6B5DULL,
 118         0x79205D0E0DB05DCEULL, 0xBE317F32F78E067BULL, 0xFCC19ED95E6430E8ULL,
 119         0x86B86ED5267CDBD3ULL, 0xC4488F3E8F96ED40ULL, 0x0359AD0275A8B6F5ULL,
 120         0x41A94CE9DC428066ULL, 0xCF8B0890283E370CULL, 0x8D7BE97B81D4019FULL,
 121         0x4A6ACB477BEA5A2AULL, 0x089A2AACD2006CB9ULL, 0x14DEA25F3AF9026DULL,
 122         0x562E43B4931334FEULL, 0x913F6188692D6F4BULL, 0xD3CF8063C0C759D8ULL,
 123         0x5DEDC41A34BBEEB2ULL, 0x1F1D25F19D51D821ULL, 0xD80C07CD676F8394ULL,
 124         0x9AFCE626CE85B507ULL,
 125 };
 126
 127 u64 bch2_crc64_update(u64 crc, const void *_data, size_t len)
 128 {
 129         const unsigned char *data = _data;
 130
 131         while (len--) {
 132                 int i = ((int) (crc >> 56) ^ *data++) & 0xFF;
 133                 crc = crc_table[i] ^ (crc << 8);
 134         }
 135
 136         return crc;
 137 }
 138
 139 static u64 bch2_checksum_init(unsigned type)
 140 {
 141         switch (type) {
 142         case BCH_CSUM_NONE:
 143                 return 0;
 144         case BCH_CSUM_CRC32C_NONZERO:
 145                 return U32_MAX;
 146         case BCH_CSUM_CRC64_NONZERO:
 147                 return U64_MAX;
 148         case BCH_CSUM_CRC32C:
 149                 return 0;
 150         case BCH_CSUM_CRC64:
 151                 return 0;
 152         default:
 153                 BUG();
 154         }
 155 }
 156
 157 static u64 bch2_checksum_final(unsigned type, u64 crc)
 158 {
 159         switch (type) {
 160         case BCH_CSUM_NONE:
 161                 return 0;
 162         case BCH_CSUM_CRC32C_NONZERO:
 163                 return crc ^ U32_MAX;
 164         case BCH_CSUM_CRC64_NONZERO:
 165                 return crc ^ U64_MAX;
 166         case BCH_CSUM_CRC32C:
 167                 return crc;
 168         case BCH_CSUM_CRC64:
 169                 return crc;
 170         default:
 171                 BUG();
 172         }
 173 }
 174
 175 static u64 bch2_checksum_update(unsigned type, u64 crc, const void *data, size_t len)
 176 {
 177         switch (type) {
 178         case BCH_CSUM_NONE:
 179                 return 0;
 180         case BCH_CSUM_CRC32C_NONZERO:
 181         case BCH_CSUM_CRC32C:
 182                 return crc32c(crc, data, len);
 183         case BCH_CSUM_CRC64_NONZERO:
 184         case BCH_CSUM_CRC64:
 185                 return bch2_crc64_update(crc, data, len);
 186         default:
 187                 BUG();
 188         }
 189 }
 190
 191 static inline void do_encrypt_sg(struct crypto_skcipher *tfm,
 192                                  struct nonce nonce,
 193                                  struct scatterlist *sg, size_t len)
 194 {
 195         SKCIPHER_REQUEST_ON_STACK(req, tfm);
 196         int ret;
 197
 198         skcipher_request_set_tfm(req, tfm);
 199         skcipher_request_set_crypt(req, sg, sg, len, nonce.d);
 200
 201         ret = crypto_skcipher_encrypt(req);
 202         BUG_ON(ret);
 203 }
 204
 205 static inline void do_encrypt(struct crypto_skcipher *tfm,
 206                               struct nonce nonce,
 207                               void *buf, size_t len)
 208 {
 209         struct scatterlist sg;
 210
 211         sg_init_one(&sg, buf, len);
 212         do_encrypt_sg(tfm, nonce, &sg, len);
 213 }
 214
 215 int bch2_chacha_encrypt_key(struct bch_key *key, struct nonce nonce,
 216                            void *buf, size_t len)
 217 {
 218         struct crypto_skcipher *chacha20 =
 219                 crypto_alloc_skcipher("chacha20", 0, 0);
 220         int ret;
 221
 222         if (!chacha20) {
 223                 pr_err("error requesting chacha20 module: %li", PTR_ERR(chacha20));
 224                 return PTR_ERR(chacha20);
 225         }
 226
 227         ret = crypto_skcipher_setkey(chacha20, (void *) key, sizeof(*key));
 228         if (ret) {
 229                 pr_err("crypto_skcipher_setkey() error: %i", ret);
 230                 goto err;
 231         }
 232
 233         do_encrypt(chacha20, nonce, buf, len);
 234 err:
 235         crypto_free_skcipher(chacha20);
 236         return ret;
 237 }
 238
 239 static void gen_poly_key(struct bch_fs *c, struct shash_desc *desc,
 240                          struct nonce nonce)
 241 {
 242         u8 key[POLY1305_KEY_SIZE];
 243
 244         nonce.d[3] ^= BCH_NONCE_POLY;
 245
 246         memset(key, 0, sizeof(key));
 247         do_encrypt(c->chacha20, nonce, key, sizeof(key));
 248
 249         desc->tfm = c->poly1305;
 250         desc->flags = 0;
 251         crypto_shash_init(desc);
 252         crypto_shash_update(desc, key, sizeof(key));
 253 }
 254
 255 struct bch_csum bch2_checksum(struct bch_fs *c, unsigned type,
 256                               struct nonce nonce, const void *data, size_t len)
 257 {
 258         switch (type) {
 259         case BCH_CSUM_NONE:
 260         case BCH_CSUM_CRC32C_NONZERO:
 261         case BCH_CSUM_CRC64_NONZERO:
 262         case BCH_CSUM_CRC32C:
 263         case BCH_CSUM_CRC64: {
 264                 u64 crc = bch2_checksum_init(type);
 265
 266                 crc = bch2_checksum_update(type, crc, data, len);
 267                 crc = bch2_checksum_final(type, crc);
 268
 269                 return (struct bch_csum) { .lo = cpu_to_le64(crc) };
 270         }
 271
 272         case BCH_CSUM_CHACHA20_POLY1305_80:
 273         case BCH_CSUM_CHACHA20_POLY1305_128: {
 274                 SHASH_DESC_ON_STACK(desc, c->poly1305);
 275                 u8 digest[POLY1305_DIGEST_SIZE];
 276                 struct bch_csum ret = { 0 };
 277
 278                 gen_poly_key(c, desc, nonce);
 279
 280                 crypto_shash_update(desc, data, len);
 281                 crypto_shash_final(desc, digest);
 282
 283                 memcpy(&ret, digest, bch_crc_bytes[type]);
 284                 return ret;
 285         }
 286         default:
 287                 BUG();
 288         }
 289 }
 290
 291 void bch2_encrypt(struct bch_fs *c, unsigned type,
 292                   struct nonce nonce, void *data, size_t len)
 293 {
 294         if (!bch2_csum_type_is_encryption(type))
 295                 return;
 296
 297         do_encrypt(c->chacha20, nonce, data, len);
 298 }
 299
 300 static struct bch_csum __bch2_checksum_bio(struct bch_fs *c, unsigned type,
 301                                            struct nonce nonce, struct bio *bio,
 302                                            struct bvec_iter *iter)
 303 {
 304         struct bio_vec bv;
 305
 306         switch (type) {
 307         case BCH_CSUM_NONE:
 308                 return (struct bch_csum) { 0 };
 309         case BCH_CSUM_CRC32C_NONZERO:
 310         case BCH_CSUM_CRC64_NONZERO:
 311         case BCH_CSUM_CRC32C:
 312         case BCH_CSUM_CRC64: {
 313                 u64 crc = bch2_checksum_init(type);
 314
 315 #ifdef CONFIG_HIGHMEM
 316                 __bio_for_each_segment(bv, bio, *iter, *iter) {
 317                         void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
 318                         crc = bch2_checksum_update(type,
 319                                 crc, p, bv.bv_len);
 320                         kunmap_atomic(p);
 321                 }
 322 #else
 323                 __bio_for_each_contig_segment(bv, bio, *iter, *iter)
 324                         crc = bch2_checksum_update(type, crc,
 325                                 page_address(bv.bv_page) + bv.bv_offset,
 326                                 bv.bv_len);
 327 #endif
 328                 crc = bch2_checksum_final(type, crc);
 329                 return (struct bch_csum) { .lo = cpu_to_le64(crc) };
 330         }
 331
 332         case BCH_CSUM_CHACHA20_POLY1305_80:
 333         case BCH_CSUM_CHACHA20_POLY1305_128: {
 334                 SHASH_DESC_ON_STACK(desc, c->poly1305);
 335                 u8 digest[POLY1305_DIGEST_SIZE];
 336                 struct bch_csum ret = { 0 };
 337
 338                 gen_poly_key(c, desc, nonce);
 339
 340 #ifdef CONFIG_HIGHMEM
 341                 __bio_for_each_segment(bv, bio, *iter, *iter) {
 342                         void *p = kmap_atomic(bv.bv_page) + bv.bv_offset;
 343
 344                         crypto_shash_update(desc, p, bv.bv_len);
 345                         kunmap_atomic(p);
 346                 }
 347 #else
 348                 __bio_for_each_contig_segment(bv, bio, *iter, *iter)
 349                         crypto_shash_update(desc,
 350                                 page_address(bv.bv_page) + bv.bv_offset,
 351                                 bv.bv_len);
 352 #endif
 353                 crypto_shash_final(desc, digest);
 354
 355                 memcpy(&ret, digest, bch_crc_bytes[type]);
 356                 return ret;
 357         }
 358         default:
 359                 BUG();
 360         }
 361 }
 362
 363 struct bch_csum bch2_checksum_bio(struct bch_fs *c, unsigned type,
 364                                   struct nonce nonce, struct bio *bio)
 365 {
 366         struct bvec_iter iter = bio->bi_iter;
 367
 368         return __bch2_checksum_bio(c, type, nonce, bio, &iter);
 369 }
 370
 371 void bch2_encrypt_bio(struct bch_fs *c, unsigned type,
 372                       struct nonce nonce, struct bio *bio)
 373 {
 374         struct bio_vec bv;
 375         struct bvec_iter iter;
 376         struct scatterlist sgl[16], *sg = sgl;
 377         size_t bytes = 0;
 378
 379         if (!bch2_csum_type_is_encryption(type))
 380                 return;
 381
 382         sg_init_table(sgl, ARRAY_SIZE(sgl));
 383
 384         bio_for_each_segment(bv, bio, iter) {
 385                 if (sg == sgl + ARRAY_SIZE(sgl)) {
 386                         sg_mark_end(sg - 1);
 387                         do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
 388
 389                         nonce = nonce_add(nonce, bytes);
 390                         bytes = 0;
 391
 392                         sg_init_table(sgl, ARRAY_SIZE(sgl));
 393                         sg = sgl;
 394                 }
 395
 396                 sg_set_page(sg++, bv.bv_page, bv.bv_len, bv.bv_offset);
 397                 bytes += bv.bv_len;
 398         }
 399
 400         sg_mark_end(sg - 1);
 401         do_encrypt_sg(c->chacha20, nonce, sgl, bytes);
 402 }
 403
 404 static inline bool bch2_checksum_mergeable(unsigned type)
 405 {
 406
 407         switch (type) {
 408         case BCH_CSUM_NONE:
 409         case BCH_CSUM_CRC32C:
 410         case BCH_CSUM_CRC64:
 411                 return true;
 412         default:
 413                 return false;
 414         }
 415 }
 416
 417 static struct bch_csum bch2_checksum_merge(unsigned type,
 418                                            struct bch_csum a,
 419                                            struct bch_csum b, size_t b_len)
 420 {
 421         BUG_ON(!bch2_checksum_mergeable(type));
 422
 423         while (b_len) {
 424                 unsigned b = min(b_len, PAGE_SIZE);
 425
 426                 a.lo = bch2_checksum_update(type, a.lo,
 427                                 page_address(ZERO_PAGE(0)), b);
 428                 b_len -= b;
 429         }
 430
 431         a.lo ^= b.lo;
 432         a.hi ^= b.hi;
 433         return a;
 434 }
 435
 436 int bch2_rechecksum_bio(struct bch_fs *c, struct bio *bio,
 437                         struct bversion version,
 438                         struct bch_extent_crc_unpacked crc_old,
 439                         struct bch_extent_crc_unpacked *crc_a,
 440                         struct bch_extent_crc_unpacked *crc_b,
 441                         unsigned len_a, unsigned len_b,
 442                         unsigned new_csum_type)
 443 {
 444         struct bvec_iter iter = bio->bi_iter;
 445         struct nonce nonce = extent_nonce(version, crc_old);
 446         struct bch_csum merged = { 0 };
 447         struct crc_split {
 448                 struct bch_extent_crc_unpacked  *crc;
 449                 unsigned                        len;
 450                 unsigned                        csum_type;
 451                 struct bch_csum                 csum;
 452         } splits[3] = {
 453                 { crc_a, len_a, new_csum_type },
 454                 { crc_b, len_b, new_csum_type },
 455                 { NULL,  bio_sectors(bio) - len_a - len_b, new_csum_type },
 456         }, *i;
 457         bool mergeable = crc_old.csum_type == new_csum_type &&
 458                 bch2_checksum_mergeable(new_csum_type);
 459         unsigned crc_nonce = crc_old.nonce;
 460
 461         BUG_ON(len_a + len_b > bio_sectors(bio));
 462         BUG_ON(crc_old.uncompressed_size != bio_sectors(bio));
 463         BUG_ON(crc_old.compression_type);
 464         BUG_ON(bch2_csum_type_is_encryption(crc_old.csum_type) !=
 465                bch2_csum_type_is_encryption(new_csum_type));
 466
 467         for (i = splits; i < splits + ARRAY_SIZE(splits); i++) {
 468                 iter.bi_size = i->len << 9;
 469                 if (mergeable || i->crc)
 470                         i->csum = __bch2_checksum_bio(c, i->csum_type,
 471                                                       nonce, bio, &iter);
 472                 else
 473                         bio_advance_iter(bio, &iter, i->len << 9);
 474                 nonce = nonce_add(nonce, i->len << 9);
 475         }
 476
 477         if (mergeable)
 478                 for (i = splits; i < splits + ARRAY_SIZE(splits); i++)
 479                         merged = bch2_checksum_merge(new_csum_type, merged,
 480                                                      i->csum, i->len << 9);
 481         else
 482                 merged = bch2_checksum_bio(c, crc_old.csum_type,
 483                                 extent_nonce(version, crc_old), bio);
 484
 485         if (bch2_crc_cmp(merged, crc_old.csum))
 486                 return -EIO;
 487
 488         for (i = splits; i < splits + ARRAY_SIZE(splits); i++) {
 489                 if (i->crc)
 490                         *i->crc = (struct bch_extent_crc_unpacked) {
 491                                 .csum_type              = i->csum_type,
 492                                 .compressed_size        = i->len,
 493                                 .uncompressed_size      = i->len,
 494                                 .offset                 = 0,
 495                                 .live_size              = i->len,
 496                                 .nonce                  = crc_nonce,
 497                                 .csum                   = i->csum,
 498                         };
 499
 500                 if (bch2_csum_type_is_encryption(new_csum_type))
 501                         crc_nonce += i->len;
 502         }
 503
 504         return 0;
 505 }
 506
 507 #ifdef __KERNEL__
 508 int bch2_request_key(struct bch_sb *sb, struct bch_key *key)
 509 {
 510         char key_description[60];
 511         struct key *keyring_key;
 512         const struct user_key_payload *ukp;
 513         int ret;
 514
 515         snprintf(key_description, sizeof(key_description),
 516                  "bcachefs:%pUb", &sb->user_uuid);
 517
 518         keyring_key = request_key(&key_type_logon, key_description, NULL);
 519         if (IS_ERR(keyring_key))
 520                 return PTR_ERR(keyring_key);
 521
 522         down_read(&keyring_key->sem);
 523         ukp = dereference_key_locked(keyring_key);
 524         if (ukp->datalen == sizeof(*key)) {
 525                 memcpy(key, ukp->data, ukp->datalen);
 526                 ret = 0;
 527         } else {
 528                 ret = -EINVAL;
 529         }
 530         up_read(&keyring_key->sem);
 531         key_put(keyring_key);
 532
 533         return ret;
 534 }
 535 #else
 536 #include <keyutils.h>
 537 #include <uuid/uuid.h>
 538
 539 int bch2_request_key(struct bch_sb *sb, struct bch_key *key)
 540 {
 541         key_serial_t key_id;
 542         char key_description[60];
 543         char uuid[40];
 544
 545         uuid_unparse_lower(sb->user_uuid.b, uuid);
 546         sprintf(key_description, "bcachefs:%s", uuid);
 547
 548         key_id = request_key("user", key_description, NULL,
 549                              KEY_SPEC_USER_KEYRING);
 550         if (key_id < 0)
 551                 return -errno;
 552
 553         if (keyctl_read(key_id, (void *) key, sizeof(*key)) != sizeof(*key))
 554                 return -1;
 555
 556         return 0;
 557 }
 558 #endif
 559
 560 int bch2_decrypt_sb_key(struct bch_fs *c,
 561                         struct bch_sb_field_crypt *crypt,
 562                         struct bch_key *key)
 563 {
 564         struct bch_encrypted_key sb_key = crypt->key;
 565         struct bch_key user_key;
 566         int ret = 0;
 567
 568         /* is key encrypted? */
 569         if (!bch2_key_is_encrypted(&sb_key))
 570                 goto out;
 571
 572         ret = bch2_request_key(c->disk_sb, &user_key);
 573         if (ret) {
 574                 bch_err(c, "error requesting encryption key: %i", ret);
 575                 goto err;
 576         }
 577
 578         /* decrypt real key: */
 579         ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c),
 580                              &sb_key, sizeof(sb_key));
 581         if (ret)
 582                 goto err;
 583
 584         if (bch2_key_is_encrypted(&sb_key)) {
 585                 bch_err(c, "incorrect encryption key");
 586                 ret = -EINVAL;
 587                 goto err;
 588         }
 589 out:
 590         *key = sb_key.key;
 591 err:
 592         memzero_explicit(&sb_key, sizeof(sb_key));
 593         memzero_explicit(&user_key, sizeof(user_key));
 594         return ret;
 595 }
 596
 597 static int bch2_alloc_ciphers(struct bch_fs *c)
 598 {
 599         if (!c->chacha20)
 600                 c->chacha20 = crypto_alloc_skcipher("chacha20", 0, 0);
 601         if (IS_ERR(c->chacha20)) {
 602                 bch_err(c, "error requesting chacha20 module: %li",
 603                         PTR_ERR(c->chacha20));
 604                 return PTR_ERR(c->chacha20);
 605         }
 606
 607         if (!c->poly1305)
 608                 c->poly1305 = crypto_alloc_shash("poly1305", 0, 0);
 609         if (IS_ERR(c->poly1305)) {
 610                 bch_err(c, "error requesting poly1305 module: %li",
 611                         PTR_ERR(c->poly1305));
 612                 return PTR_ERR(c->poly1305);
 613         }
 614
 615         return 0;
 616 }
 617
 618 int bch2_disable_encryption(struct bch_fs *c)
 619 {
 620         struct bch_sb_field_crypt *crypt;
 621         struct bch_key key;
 622         int ret = -EINVAL;
 623
 624         mutex_lock(&c->sb_lock);
 625
 626         crypt = bch2_sb_get_crypt(c->disk_sb);
 627         if (!crypt)
 628                 goto out;
 629
 630         /* is key encrypted? */
 631         ret = 0;
 632         if (bch2_key_is_encrypted(&crypt->key))
 633                 goto out;
 634
 635         ret = bch2_decrypt_sb_key(c, crypt, &key);
 636         if (ret)
 637                 goto out;
 638
 639         crypt->key.magic        = BCH_KEY_MAGIC;
 640         crypt->key.key          = key;
 641
 642         SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 0);
 643         bch2_write_super(c);
 644 out:
 645         mutex_unlock(&c->sb_lock);
 646
 647         return ret;
 648 }
 649
 650 int bch2_enable_encryption(struct bch_fs *c, bool keyed)
 651 {
 652         struct bch_encrypted_key key;
 653         struct bch_key user_key;
 654         struct bch_sb_field_crypt *crypt;
 655         int ret = -EINVAL;
 656
 657         mutex_lock(&c->sb_lock);
 658
 659         /* Do we already have an encryption key? */
 660         if (bch2_sb_get_crypt(c->disk_sb))
 661                 goto err;
 662
 663         ret = bch2_alloc_ciphers(c);
 664         if (ret)
 665                 goto err;
 666
 667         key.magic = BCH_KEY_MAGIC;
 668         get_random_bytes(&key.key, sizeof(key.key));
 669
 670         if (keyed) {
 671                 ret = bch2_request_key(c->disk_sb, &user_key);
 672                 if (ret) {
 673                         bch_err(c, "error requesting encryption key: %i", ret);
 674                         goto err;
 675                 }
 676
 677                 ret = bch2_chacha_encrypt_key(&user_key, bch2_sb_key_nonce(c),
 678                                               &key, sizeof(key));
 679                 if (ret)
 680                         goto err;
 681         }
 682
 683         ret = crypto_skcipher_setkey(c->chacha20,
 684                         (void *) &key.key, sizeof(key.key));
 685         if (ret)
 686                 goto err;
 687
 688         crypt = bch2_fs_sb_resize_crypt(c, sizeof(*crypt) / sizeof(u64));
 689         if (!crypt) {
 690                 ret = -ENOMEM; /* XXX this technically could be -ENOSPC */
 691                 goto err;
 692         }
 693
 694         crypt->key = key;
 695
 696         /* write superblock */
 697         SET_BCH_SB_ENCRYPTION_TYPE(c->disk_sb, 1);
 698         bch2_write_super(c);
 699 err:
 700         mutex_unlock(&c->sb_lock);
 701         memzero_explicit(&user_key, sizeof(user_key));
 702         memzero_explicit(&key, sizeof(key));
 703         return ret;
 704 }
 705
 706 void bch2_fs_encryption_exit(struct bch_fs *c)
 707 {
 708         if (!IS_ERR_OR_NULL(c->poly1305))
 709                 crypto_free_shash(c->poly1305);
 710         if (!IS_ERR_OR_NULL(c->chacha20))
 711                 crypto_free_skcipher(c->chacha20);
 712         if (!IS_ERR_OR_NULL(c->sha256))
 713                 crypto_free_shash(c->sha256);
 714 }
 715
 716 int bch2_fs_encryption_init(struct bch_fs *c)
 717 {
 718         struct bch_sb_field_crypt *crypt;
 719         struct bch_key key;
 720         int ret = 0;
 721
 722         pr_verbose_init(c->opts, "");
 723
 724         c->sha256 = crypto_alloc_shash("sha256", 0, 0);
 725         if (IS_ERR(c->sha256)) {
 726                 bch_err(c, "error requesting sha256 module");
 727                 ret = PTR_ERR(c->sha256);
 728                 goto out;
 729         }
 730
 731         crypt = bch2_sb_get_crypt(c->disk_sb);
 732         if (!crypt)
 733                 goto out;
 734
 735         ret = bch2_alloc_ciphers(c);
 736         if (ret)
 737                 goto out;
 738
 739         ret = bch2_decrypt_sb_key(c, crypt, &key);
 740         if (ret)
 741                 goto out;
 742
 743         ret = crypto_skcipher_setkey(c->chacha20,
 744                         (void *) &key.key, sizeof(key.key));
 745         if (ret)
 746                 goto out;
 747 out:
 748         memzero_explicit(&key, sizeof(key));
 749         pr_verbose_init(c->opts, "ret %i", ret);
 750         return ret;
 751 }