git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/compress.c

   1 #include "bcachefs.h"
   2 #include "compress.h"
   3 #include "extents.h"
   4 #include "io.h"
   5 #include "super-io.h"
   6
   7 #include "lz4.h"
   8 #include <linux/lz4.h>
   9 #include <linux/zlib.h>
  10
  11 /* Bounce buffer: */
  12 struct bbuf {
  13         void            *b;
  14         enum {
  15                 BB_NONE,
  16                 BB_VMAP,
  17                 BB_KMALLOC,
  18                 BB_VMALLOC,
  19                 BB_MEMPOOL,
  20         }               type;
  21         int             rw;
  22 };
  23
  24 static struct bbuf __bounce_alloc(struct bch_fs *c, unsigned size, int rw)
  25 {
  26         void *b;
  27
  28         BUG_ON(size > c->sb.encoded_extent_max);
  29
  30         b = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
  31         if (b)
  32                 return (struct bbuf) { .b = b, .type = BB_KMALLOC, .rw = rw };
  33
  34         b = mempool_alloc(&c->compression_bounce[rw], GFP_NOWAIT);
  35         b = b ? page_address(b) : NULL;
  36         if (b)
  37                 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
  38
  39         b = vmalloc(size);
  40         if (b)
  41                 return (struct bbuf) { .b = b, .type = BB_VMALLOC, .rw = rw };
  42
  43         b = mempool_alloc(&c->compression_bounce[rw], GFP_NOIO);
  44         b = b ? page_address(b) : NULL;
  45         if (b)
  46                 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
  47
  48         BUG();
  49 }
  50
  51 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
  52                                        struct bvec_iter start, int rw)
  53 {
  54         struct bbuf ret;
  55         struct bio_vec bv;
  56         struct bvec_iter iter;
  57         unsigned nr_pages = 0;
  58         struct page *stack_pages[16];
  59         struct page **pages = NULL;
  60         bool first = true;
  61         unsigned prev_end = PAGE_SIZE;
  62         void *data;
  63
  64         BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
  65
  66 #ifndef CONFIG_HIGHMEM
  67         __bio_for_each_contig_segment(bv, bio, iter, start) {
  68                 if (bv.bv_len == start.bi_size)
  69                         return (struct bbuf) {
  70                                 .b = page_address(bv.bv_page) + bv.bv_offset,
  71                                 .type = BB_NONE, .rw = rw
  72                         };
  73         }
  74 #endif
  75         __bio_for_each_segment(bv, bio, iter, start) {
  76                 if ((!first && bv.bv_offset) ||
  77                     prev_end != PAGE_SIZE)
  78                         goto bounce;
  79
  80                 prev_end = bv.bv_offset + bv.bv_len;
  81                 nr_pages++;
  82         }
  83
  84         BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
  85
  86         pages = nr_pages > ARRAY_SIZE(stack_pages)
  87                 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
  88                 : stack_pages;
  89         if (!pages)
  90                 goto bounce;
  91
  92         nr_pages = 0;
  93         __bio_for_each_segment(bv, bio, iter, start)
  94                 pages[nr_pages++] = bv.bv_page;
  95
  96         data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
  97         if (pages != stack_pages)
  98                 kfree(pages);
  99
 100         if (data)
 101                 return (struct bbuf) {
 102                         .b = data + bio_iter_offset(bio, start),
 103                         .type = BB_VMAP, .rw = rw
 104                 };
 105 bounce:
 106         ret = __bounce_alloc(c, start.bi_size, rw);
 107
 108         if (rw == READ)
 109                 memcpy_from_bio(ret.b, bio, start);
 110
 111         return ret;
 112 }
 113
 114 static struct bbuf bio_map_or_bounce(struct bch_fs *c, struct bio *bio, int rw)
 115 {
 116         return __bio_map_or_bounce(c, bio, bio->bi_iter, rw);
 117 }
 118
 119 static void bio_unmap_or_unbounce(struct bch_fs *c, struct bbuf buf)
 120 {
 121         switch (buf.type) {
 122         case BB_NONE:
 123                 break;
 124         case BB_VMAP:
 125                 vunmap((void *) ((unsigned long) buf.b & PAGE_MASK));
 126                 break;
 127         case BB_KMALLOC:
 128                 kfree(buf.b);
 129                 break;
 130         case BB_VMALLOC:
 131                 vfree(buf.b);
 132                 break;
 133         case BB_MEMPOOL:
 134                 mempool_free(virt_to_page(buf.b),
 135                              &c->compression_bounce[buf.rw]);
 136                 break;
 137         }
 138 }
 139
 140 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
 141 {
 142 #ifdef __KERNEL__
 143         strm->workspace = workspace;
 144 #endif
 145 }
 146
 147 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
 148                             void *dst_data, struct bch_extent_crc128 crc)
 149 {
 150         struct bbuf src_data = { NULL };
 151         size_t src_len = src->bi_iter.bi_size;
 152         size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
 153         int ret;
 154
 155         src_data = bio_map_or_bounce(c, src, READ);
 156
 157         switch (crc.compression_type) {
 158         case BCH_COMPRESSION_LZ4_OLD:
 159                 ret = bch2_lz4_decompress(src_data.b, &src_len,
 160                                      dst_data, dst_len);
 161                 if (ret) {
 162                         ret = -EIO;
 163                         goto err;
 164                 }
 165                 break;
 166         case BCH_COMPRESSION_LZ4:
 167                 ret = LZ4_decompress_safe(src_data.b, dst_data,
 168                                           src_len, dst_len);
 169                 if (ret != dst_len) {
 170                         ret = -EIO;
 171                         goto err;
 172                 }
 173                 break;
 174         case BCH_COMPRESSION_GZIP: {
 175                 void *workspace;
 176                 z_stream strm;
 177
 178                 workspace = kmalloc(zlib_inflate_workspacesize(),
 179                                     GFP_NOIO|__GFP_NOWARN);
 180                 if (!workspace) {
 181                         mutex_lock(&c->zlib_workspace_lock);
 182                         workspace = c->zlib_workspace;
 183                 }
 184
 185                 strm.next_in    = src_data.b;
 186                 strm.avail_in   = src_len;
 187                 strm.next_out   = dst_data;
 188                 strm.avail_out  = dst_len;
 189                 zlib_set_workspace(&strm, workspace);
 190                 zlib_inflateInit2(&strm, -MAX_WBITS);
 191
 192                 ret = zlib_inflate(&strm, Z_FINISH);
 193
 194                 if (workspace == c->zlib_workspace)
 195                         mutex_unlock(&c->zlib_workspace_lock);
 196                 else
 197                         kfree(workspace);
 198
 199                 if (ret != Z_STREAM_END) {
 200                         ret = -EIO;
 201                         goto err;
 202                 }
 203                 break;
 204         }
 205         default:
 206                 BUG();
 207         }
 208         ret = 0;
 209 err:
 210         bio_unmap_or_unbounce(c, src_data);
 211         return ret;
 212 }
 213
 214 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
 215                                unsigned live_data_sectors,
 216                                struct bch_extent_crc128 crc)
 217 {
 218         struct bbuf dst_data = { NULL };
 219         size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
 220         int ret = -ENOMEM;
 221
 222         BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
 223
 224         if (crc_uncompressed_size(NULL, &crc) > c->sb.encoded_extent_max ||
 225             crc_compressed_size(NULL, &crc)   > c->sb.encoded_extent_max)
 226                 return -EIO;
 227
 228         dst_data = __bounce_alloc(c, dst_len, WRITE);
 229
 230         ret = __bio_uncompress(c, bio, dst_data.b, crc);
 231         if (ret)
 232                 goto err;
 233
 234         while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
 235                 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 236
 237                 bv->bv_page = alloc_page(GFP_NOIO);
 238                 if (!bv->bv_page)
 239                         goto use_mempool;
 240
 241                 bv->bv_len = PAGE_SIZE;
 242                 bv->bv_offset = 0;
 243                 bio->bi_vcnt++;
 244         }
 245
 246         bio->bi_iter.bi_size = live_data_sectors << 9;
 247 copy_data:
 248         memcpy_to_bio(bio, bio->bi_iter, dst_data.b + (crc.offset << 9));
 249 err:
 250         bio_unmap_or_unbounce(c, dst_data);
 251         return ret;
 252 use_mempool:
 253         /*
 254          * We already allocated from mempool, we can't allocate from it again
 255          * without freeing the pages we already allocated or else we could
 256          * deadlock:
 257          */
 258
 259         bch2_bio_free_pages_pool(c, bio);
 260         bch2_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
 261         goto copy_data;
 262 }
 263
 264 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
 265                        struct bio *dst, struct bvec_iter dst_iter,
 266                        struct bch_extent_crc128 crc)
 267 {
 268         struct bbuf dst_data = { NULL };
 269         size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
 270         int ret = -ENOMEM;
 271
 272         if (crc_uncompressed_size(NULL, &crc) < c->sb.encoded_extent_max)
 273                 return -EIO;
 274
 275         dst_data = dst_len == dst_iter.bi_size
 276                 ? __bio_map_or_bounce(c, dst, dst_iter, WRITE)
 277                 : __bounce_alloc(c, dst_len, WRITE);
 278
 279         ret = __bio_uncompress(c, src, dst_data.b, crc);
 280         if (ret)
 281                 goto err;
 282
 283         if (dst_data.type != BB_NONE)
 284                 memcpy_to_bio(dst, dst_iter, dst_data.b + (crc.offset << 9));
 285 err:
 286         bio_unmap_or_unbounce(c, dst_data);
 287         return ret;
 288 }
 289
 290 static int __bio_compress(struct bch_fs *c,
 291                           struct bio *dst, size_t *dst_len,
 292                           struct bio *src, size_t *src_len,
 293                           unsigned *compression_type)
 294 {
 295         struct bbuf src_data = { NULL }, dst_data = { NULL };
 296         unsigned pad;
 297         int ret;
 298
 299         dst_data = bio_map_or_bounce(c, dst, WRITE);
 300         src_data = bio_map_or_bounce(c, src, READ);
 301
 302         switch (*compression_type) {
 303         case BCH_COMPRESSION_LZ4_OLD:
 304                 *compression_type = BCH_COMPRESSION_LZ4;
 305
 306         case BCH_COMPRESSION_LZ4: {
 307                 void *workspace;
 308                 int len = src->bi_iter.bi_size;
 309
 310                 ret = 0;
 311
 312                 workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
 313
 314                 while (len > block_bytes(c) &&
 315                        (!(ret = LZ4_compress_destSize(
 316                                         src_data.b,     dst_data.b,
 317                                         &len,           dst->bi_iter.bi_size,
 318                                         workspace)) ||
 319                         (len & (block_bytes(c) - 1)))) {
 320                         /*
 321                          * On error, the compressed data was bigger than
 322                          * dst_len - round down to nearest block and try again:
 323                          */
 324                         len = round_down(len, block_bytes(c));
 325                 }
 326
 327                 mempool_free(workspace, &c->lz4_workspace_pool);
 328
 329                 if (!ret)
 330                         goto err;
 331
 332                 *src_len = len;
 333                 *dst_len = ret;
 334                 break;
 335         }
 336         case BCH_COMPRESSION_GZIP: {
 337                 void *workspace;
 338                 z_stream strm;
 339
 340                 workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
 341                                                                DEF_MEM_LEVEL),
 342                                     GFP_NOIO|__GFP_NOWARN);
 343                 if (!workspace) {
 344                         mutex_lock(&c->zlib_workspace_lock);
 345                         workspace = c->zlib_workspace;
 346                 }
 347
 348                 strm.next_in    = src_data.b;
 349                 strm.avail_in   = min(src->bi_iter.bi_size,
 350                                       dst->bi_iter.bi_size);
 351                 strm.next_out   = dst_data.b;
 352                 strm.avail_out  = dst->bi_iter.bi_size;
 353                 zlib_set_workspace(&strm, workspace);
 354                 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
 355                                   Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
 356                                   Z_DEFAULT_STRATEGY);
 357
 358                 ret = zlib_deflate(&strm, Z_FINISH);
 359                 if (ret != Z_STREAM_END) {
 360                         ret = -EIO;
 361                         goto zlib_err;
 362                 }
 363
 364                 ret = zlib_deflateEnd(&strm);
 365                 if (ret != Z_OK) {
 366                         ret = -EIO;
 367                         goto zlib_err;
 368                 }
 369
 370                 ret = 0;
 371 zlib_err:
 372                 if (workspace == c->zlib_workspace)
 373                         mutex_unlock(&c->zlib_workspace_lock);
 374                 else
 375                         kfree(workspace);
 376
 377                 if (ret)
 378                         goto err;
 379
 380                 *dst_len = strm.total_out;
 381                 *src_len = strm.total_in;
 382                 break;
 383         }
 384         default:
 385                 BUG();
 386         }
 387
 388         /* Didn't get smaller: */
 389         if (round_up(*dst_len, block_bytes(c)) >= *src_len)
 390                 goto err;
 391
 392         pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
 393
 394         memset(dst_data.b + *dst_len, 0, pad);
 395         *dst_len += pad;
 396
 397         if (dst_data.type != BB_NONE)
 398                 memcpy_to_bio(dst, dst->bi_iter, dst_data.b);
 399 out:
 400         bio_unmap_or_unbounce(c, src_data);
 401         bio_unmap_or_unbounce(c, dst_data);
 402         return ret;
 403 err:
 404         ret = -1;
 405         goto out;
 406 }
 407
 408 void bch2_bio_compress(struct bch_fs *c,
 409                        struct bio *dst, size_t *dst_len,
 410                        struct bio *src, size_t *src_len,
 411                        unsigned *compression_type)
 412 {
 413         unsigned orig_dst = dst->bi_iter.bi_size;
 414         unsigned orig_src = src->bi_iter.bi_size;
 415
 416         /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
 417         src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
 418                                      c->sb.encoded_extent_max << 9);
 419
 420         /* Don't generate a bigger output than input: */
 421         dst->bi_iter.bi_size =
 422                 min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
 423
 424         /* If it's only one block, don't bother trying to compress: */
 425         if (*compression_type != BCH_COMPRESSION_NONE &&
 426             bio_sectors(src) > c->sb.block_size &&
 427             !__bio_compress(c, dst, dst_len, src, src_len, compression_type))
 428                 goto out;
 429
 430         /* If compressing failed (didn't get smaller), just copy: */
 431         *compression_type = BCH_COMPRESSION_NONE;
 432         *dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
 433         bio_copy_data(dst, src);
 434 out:
 435         dst->bi_iter.bi_size = orig_dst;
 436         src->bi_iter.bi_size = orig_src;
 437
 438         BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
 439         BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
 440         BUG_ON(*dst_len & (block_bytes(c) - 1));
 441         BUG_ON(*src_len & (block_bytes(c) - 1));
 442 }
 443
 444 /* doesn't write superblock: */
 445 int bch2_check_set_has_compressed_data(struct bch_fs *c,
 446                                       unsigned compression_type)
 447 {
 448         switch (compression_type) {
 449         case BCH_COMPRESSION_NONE:
 450                 return 0;
 451         case BCH_COMPRESSION_LZ4:
 452                 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
 453                         return 0;
 454
 455                 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
 456                 break;
 457         case BCH_COMPRESSION_GZIP:
 458                 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
 459                         return 0;
 460
 461                 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
 462                 break;
 463         }
 464
 465         return bch2_fs_compress_init(c);
 466 }
 467
 468 void bch2_fs_compress_exit(struct bch_fs *c)
 469 {
 470         vfree(c->zlib_workspace);
 471         mempool_exit(&c->lz4_workspace_pool);
 472         mempool_exit(&c->compression_bounce[WRITE]);
 473         mempool_exit(&c->compression_bounce[READ]);
 474 }
 475
 476 #define COMPRESSION_WORKSPACE_SIZE                                      \
 477         max_t(size_t, zlib_inflate_workspacesize(),                     \
 478               zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
 479
 480 int bch2_fs_compress_init(struct bch_fs *c)
 481 {
 482         unsigned order = get_order(c->sb.encoded_extent_max << 9);
 483         int ret;
 484
 485         if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
 486             !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
 487                 return 0;
 488
 489         if (!mempool_initialized(&c->compression_bounce[READ])) {
 490                 ret = mempool_init_page_pool(&c->compression_bounce[READ],
 491                                              1, order);
 492                 if (ret)
 493                         return ret;
 494         }
 495
 496         if (!mempool_initialized(&c->compression_bounce[WRITE])) {
 497                 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
 498                                              1, order);
 499                 if (ret)
 500                         return ret;
 501         }
 502
 503         if (!mempool_initialized(&c->lz4_workspace_pool) &&
 504             bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
 505                 ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
 506                                                 1, LZ4_MEM_COMPRESS);
 507                 if (ret)
 508                         return ret;
 509         }
 510
 511         if (!c->zlib_workspace &&
 512             bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
 513                 c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
 514                 if (!c->zlib_workspace)
 515                         return -ENOMEM;
 516         }
 517
 518         return 0;
 519 }