git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/compress.c

   1 #include "bcachefs.h"
   2 #include "checksum.h"
   3 #include "compress.h"
   4 #include "extents.h"
   5 #include "io.h"
   6 #include "super-io.h"
   7
   8 #include "lz4.h"
   9 #include <linux/lz4.h>
  10 #include <linux/zlib.h>
  11
  12 /* Bounce buffer: */
  13 struct bbuf {
  14         void            *b;
  15         enum {
  16                 BB_NONE,
  17                 BB_VMAP,
  18                 BB_KMALLOC,
  19                 BB_VMALLOC,
  20                 BB_MEMPOOL,
  21         }               type;
  22         int             rw;
  23 };
  24
  25 static struct bbuf __bounce_alloc(struct bch_fs *c, unsigned size, int rw)
  26 {
  27         void *b;
  28
  29         BUG_ON(size > c->sb.encoded_extent_max << 9);
  30
  31         b = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
  32         if (b)
  33                 return (struct bbuf) { .b = b, .type = BB_KMALLOC, .rw = rw };
  34
  35         b = mempool_alloc(&c->compression_bounce[rw], GFP_NOWAIT);
  36         b = b ? page_address(b) : NULL;
  37         if (b)
  38                 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
  39
  40         b = vmalloc(size);
  41         if (b)
  42                 return (struct bbuf) { .b = b, .type = BB_VMALLOC, .rw = rw };
  43
  44         b = mempool_alloc(&c->compression_bounce[rw], GFP_NOIO);
  45         b = b ? page_address(b) : NULL;
  46         if (b)
  47                 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
  48
  49         BUG();
  50 }
  51
  52 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
  53                                        struct bvec_iter start, int rw)
  54 {
  55         struct bbuf ret;
  56         struct bio_vec bv;
  57         struct bvec_iter iter;
  58         unsigned nr_pages = 0;
  59         struct page *stack_pages[16];
  60         struct page **pages = NULL;
  61         bool first = true;
  62         unsigned prev_end = PAGE_SIZE;
  63         void *data;
  64
  65         BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
  66
  67 #ifndef CONFIG_HIGHMEM
  68         __bio_for_each_contig_segment(bv, bio, iter, start) {
  69                 if (bv.bv_len == start.bi_size)
  70                         return (struct bbuf) {
  71                                 .b = page_address(bv.bv_page) + bv.bv_offset,
  72                                 .type = BB_NONE, .rw = rw
  73                         };
  74         }
  75 #endif
  76         __bio_for_each_segment(bv, bio, iter, start) {
  77                 if ((!first && bv.bv_offset) ||
  78                     prev_end != PAGE_SIZE)
  79                         goto bounce;
  80
  81                 prev_end = bv.bv_offset + bv.bv_len;
  82                 nr_pages++;
  83         }
  84
  85         BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
  86
  87         pages = nr_pages > ARRAY_SIZE(stack_pages)
  88                 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
  89                 : stack_pages;
  90         if (!pages)
  91                 goto bounce;
  92
  93         nr_pages = 0;
  94         __bio_for_each_segment(bv, bio, iter, start)
  95                 pages[nr_pages++] = bv.bv_page;
  96
  97         data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
  98         if (pages != stack_pages)
  99                 kfree(pages);
 100
 101         if (data)
 102                 return (struct bbuf) {
 103                         .b = data + bio_iter_offset(bio, start),
 104                         .type = BB_VMAP, .rw = rw
 105                 };
 106 bounce:
 107         ret = __bounce_alloc(c, start.bi_size, rw);
 108
 109         if (rw == READ)
 110                 memcpy_from_bio(ret.b, bio, start);
 111
 112         return ret;
 113 }
 114
 115 static struct bbuf bio_map_or_bounce(struct bch_fs *c, struct bio *bio, int rw)
 116 {
 117         return __bio_map_or_bounce(c, bio, bio->bi_iter, rw);
 118 }
 119
 120 static void bio_unmap_or_unbounce(struct bch_fs *c, struct bbuf buf)
 121 {
 122         switch (buf.type) {
 123         case BB_NONE:
 124                 break;
 125         case BB_VMAP:
 126                 vunmap((void *) ((unsigned long) buf.b & PAGE_MASK));
 127                 break;
 128         case BB_KMALLOC:
 129                 kfree(buf.b);
 130                 break;
 131         case BB_VMALLOC:
 132                 vfree(buf.b);
 133                 break;
 134         case BB_MEMPOOL:
 135                 mempool_free(virt_to_page(buf.b),
 136                              &c->compression_bounce[buf.rw]);
 137                 break;
 138         }
 139 }
 140
 141 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
 142 {
 143 #ifdef __KERNEL__
 144         strm->workspace = workspace;
 145 #endif
 146 }
 147
 148 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
 149                             void *dst_data, struct bch_extent_crc_unpacked crc)
 150 {
 151         struct bbuf src_data = { NULL };
 152         size_t src_len = src->bi_iter.bi_size;
 153         size_t dst_len = crc.uncompressed_size << 9;
 154         int ret;
 155
 156         src_data = bio_map_or_bounce(c, src, READ);
 157
 158         switch (crc.compression_type) {
 159         case BCH_COMPRESSION_LZ4_OLD:
 160                 ret = bch2_lz4_decompress(src_data.b, &src_len,
 161                                      dst_data, dst_len);
 162                 if (ret) {
 163                         ret = -EIO;
 164                         goto err;
 165                 }
 166                 break;
 167         case BCH_COMPRESSION_LZ4:
 168                 ret = LZ4_decompress_safe_partial(src_data.b, dst_data,
 169                                                   src_len, dst_len, dst_len);
 170                 if (ret != dst_len) {
 171                         ret = -EIO;
 172                         goto err;
 173                 }
 174                 break;
 175         case BCH_COMPRESSION_GZIP: {
 176                 void *workspace;
 177                 z_stream strm;
 178
 179                 workspace = kmalloc(zlib_inflate_workspacesize(),
 180                                     GFP_NOIO|__GFP_NOWARN);
 181                 if (!workspace) {
 182                         mutex_lock(&c->zlib_workspace_lock);
 183                         workspace = c->zlib_workspace;
 184                 }
 185
 186                 strm.next_in    = src_data.b;
 187                 strm.avail_in   = src_len;
 188                 strm.next_out   = dst_data;
 189                 strm.avail_out  = dst_len;
 190                 zlib_set_workspace(&strm, workspace);
 191                 zlib_inflateInit2(&strm, -MAX_WBITS);
 192
 193                 ret = zlib_inflate(&strm, Z_FINISH);
 194
 195                 if (workspace == c->zlib_workspace)
 196                         mutex_unlock(&c->zlib_workspace_lock);
 197                 else
 198                         kfree(workspace);
 199
 200                 if (ret != Z_STREAM_END) {
 201                         ret = -EIO;
 202                         goto err;
 203                 }
 204                 break;
 205         }
 206         default:
 207                 BUG();
 208         }
 209         ret = 0;
 210 err:
 211         bio_unmap_or_unbounce(c, src_data);
 212         return ret;
 213 }
 214
 215 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
 216                                 struct bch_extent_crc_unpacked *crc)
 217 {
 218         struct bbuf data = { NULL };
 219         size_t dst_len = crc->uncompressed_size << 9;
 220
 221         /* bio must own its pages: */
 222         BUG_ON(!bio->bi_vcnt);
 223         BUG_ON(DIV_ROUND_UP(crc->live_size, PAGE_SECTORS) > bio->bi_max_vecs);
 224
 225         if (crc->uncompressed_size      > c->sb.encoded_extent_max ||
 226             crc->compressed_size        > c->sb.encoded_extent_max) {
 227                 bch_err(c, "error rewriting existing data: extent too big");
 228                 return -EIO;
 229         }
 230
 231         data = __bounce_alloc(c, dst_len, WRITE);
 232
 233         if (__bio_uncompress(c, bio, data.b, *crc)) {
 234                 bch_err(c, "error rewriting existing data: decompression error");
 235                 bio_unmap_or_unbounce(c, data);
 236                 return -EIO;
 237         }
 238
 239         /*
 240          * might have to free existing pages and retry allocation from mempool -
 241          * do this _after_ decompressing:
 242          */
 243         bch2_bio_alloc_more_pages_pool(c, bio, crc->live_size << 9);
 244
 245         memcpy_to_bio(bio, bio->bi_iter, data.b + (crc->offset << 9));
 246
 247         crc->csum_type          = 0;
 248         crc->compression_type   = 0;
 249         crc->compressed_size    = crc->live_size;
 250         crc->uncompressed_size  = crc->live_size;
 251         crc->offset             = 0;
 252         crc->csum               = (struct bch_csum) { 0, 0 };
 253
 254         bio_unmap_or_unbounce(c, data);
 255         return 0;
 256 }
 257
 258 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
 259                        struct bio *dst, struct bvec_iter dst_iter,
 260                        struct bch_extent_crc_unpacked crc)
 261 {
 262         struct bbuf dst_data = { NULL };
 263         size_t dst_len = crc.uncompressed_size << 9;
 264         int ret = -ENOMEM;
 265
 266         if (crc.uncompressed_size       > c->sb.encoded_extent_max ||
 267             crc.compressed_size         > c->sb.encoded_extent_max)
 268                 return -EIO;
 269
 270         dst_data = dst_len == dst_iter.bi_size
 271                 ? __bio_map_or_bounce(c, dst, dst_iter, WRITE)
 272                 : __bounce_alloc(c, dst_len, WRITE);
 273
 274         ret = __bio_uncompress(c, src, dst_data.b, crc);
 275         if (ret)
 276                 goto err;
 277
 278         if (dst_data.type != BB_NONE)
 279                 memcpy_to_bio(dst, dst_iter, dst_data.b + (crc.offset << 9));
 280 err:
 281         bio_unmap_or_unbounce(c, dst_data);
 282         return ret;
 283 }
 284
 285 static unsigned __bio_compress(struct bch_fs *c,
 286                                struct bio *dst, size_t *dst_len,
 287                                struct bio *src, size_t *src_len,
 288                                unsigned compression_type)
 289 {
 290         struct bbuf src_data = { NULL }, dst_data = { NULL };
 291         unsigned pad;
 292         int ret = 0;
 293
 294         /* If it's only one block, don't bother trying to compress: */
 295         if (bio_sectors(src) <= c->opts.block_size)
 296                 goto err;
 297
 298         dst_data = bio_map_or_bounce(c, dst, WRITE);
 299         src_data = bio_map_or_bounce(c, src, READ);
 300
 301         switch (compression_type) {
 302         case BCH_COMPRESSION_LZ4_OLD:
 303                 compression_type = BCH_COMPRESSION_LZ4;
 304
 305         case BCH_COMPRESSION_LZ4: {
 306                 void *workspace;
 307                 int len = src->bi_iter.bi_size;
 308
 309                 workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
 310
 311                 while (1) {
 312                         if (len <= block_bytes(c)) {
 313                                 ret = 0;
 314                                 break;
 315                         }
 316
 317                         ret = LZ4_compress_destSize(
 318                                         src_data.b,     dst_data.b,
 319                                         &len,           dst->bi_iter.bi_size,
 320                                         workspace);
 321                         if (ret >= len) {
 322                                 /* uncompressible: */
 323                                 ret = 0;
 324                                 break;
 325                         }
 326
 327                         if (!(len & (block_bytes(c) - 1)))
 328                                 break;
 329                         len = round_down(len, block_bytes(c));
 330                 }
 331                 mempool_free(workspace, &c->lz4_workspace_pool);
 332
 333                 if (!ret)
 334                         goto err;
 335
 336                 *src_len = len;
 337                 *dst_len = ret;
 338                 ret = 0;
 339                 break;
 340         }
 341         case BCH_COMPRESSION_GZIP: {
 342                 void *workspace;
 343                 z_stream strm;
 344
 345                 workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
 346                                                                DEF_MEM_LEVEL),
 347                                     GFP_NOIO|__GFP_NOWARN);
 348                 if (!workspace) {
 349                         mutex_lock(&c->zlib_workspace_lock);
 350                         workspace = c->zlib_workspace;
 351                 }
 352
 353                 strm.next_in    = src_data.b;
 354                 strm.avail_in   = min(src->bi_iter.bi_size,
 355                                       dst->bi_iter.bi_size);
 356                 strm.next_out   = dst_data.b;
 357                 strm.avail_out  = dst->bi_iter.bi_size;
 358                 zlib_set_workspace(&strm, workspace);
 359                 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
 360                                   Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
 361                                   Z_DEFAULT_STRATEGY);
 362
 363                 ret = zlib_deflate(&strm, Z_FINISH);
 364                 if (ret != Z_STREAM_END) {
 365                         ret = -EIO;
 366                         goto zlib_err;
 367                 }
 368
 369                 ret = zlib_deflateEnd(&strm);
 370                 if (ret != Z_OK) {
 371                         ret = -EIO;
 372                         goto zlib_err;
 373                 }
 374
 375                 ret = 0;
 376 zlib_err:
 377                 if (workspace == c->zlib_workspace)
 378                         mutex_unlock(&c->zlib_workspace_lock);
 379                 else
 380                         kfree(workspace);
 381
 382                 if (ret)
 383                         goto err;
 384
 385                 *dst_len = strm.total_out;
 386                 *src_len = strm.total_in;
 387                 break;
 388         }
 389         default:
 390                 BUG();
 391         }
 392
 393         /* Didn't get smaller: */
 394         if (round_up(*dst_len, block_bytes(c)) >= *src_len)
 395                 goto err;
 396
 397         pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
 398
 399         memset(dst_data.b + *dst_len, 0, pad);
 400         *dst_len += pad;
 401
 402         if (dst_data.type != BB_NONE)
 403                 memcpy_to_bio(dst, dst->bi_iter, dst_data.b);
 404
 405         BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
 406         BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
 407         BUG_ON(*dst_len & (block_bytes(c) - 1));
 408         BUG_ON(*src_len & (block_bytes(c) - 1));
 409 out:
 410         bio_unmap_or_unbounce(c, src_data);
 411         bio_unmap_or_unbounce(c, dst_data);
 412         return compression_type;
 413 err:
 414         compression_type = 0;
 415         goto out;
 416 }
 417
 418 unsigned bch2_bio_compress(struct bch_fs *c,
 419                            struct bio *dst, size_t *dst_len,
 420                            struct bio *src, size_t *src_len,
 421                            unsigned compression_type)
 422 {
 423         unsigned orig_dst = dst->bi_iter.bi_size;
 424         unsigned orig_src = src->bi_iter.bi_size;
 425
 426         /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
 427         src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
 428                                      c->sb.encoded_extent_max << 9);
 429         /* Don't generate a bigger output than input: */
 430         dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
 431
 432         compression_type =
 433                 __bio_compress(c, dst, dst_len, src, src_len, compression_type);
 434
 435         dst->bi_iter.bi_size = orig_dst;
 436         src->bi_iter.bi_size = orig_src;
 437         return compression_type;
 438 }
 439
 440 /* doesn't write superblock: */
 441 int bch2_check_set_has_compressed_data(struct bch_fs *c,
 442                                       unsigned compression_type)
 443 {
 444         switch (compression_type) {
 445         case BCH_COMPRESSION_OPT_NONE:
 446                 return 0;
 447         case BCH_COMPRESSION_OPT_LZ4:
 448                 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
 449                         return 0;
 450
 451                 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
 452                 break;
 453         case BCH_COMPRESSION_OPT_GZIP:
 454                 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
 455                         return 0;
 456
 457                 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
 458                 break;
 459         default:
 460                 BUG();
 461         }
 462
 463         return bch2_fs_compress_init(c);
 464 }
 465
 466 void bch2_fs_compress_exit(struct bch_fs *c)
 467 {
 468         vfree(c->zlib_workspace);
 469         mempool_exit(&c->lz4_workspace_pool);
 470         mempool_exit(&c->compression_bounce[WRITE]);
 471         mempool_exit(&c->compression_bounce[READ]);
 472 }
 473
 474 #define COMPRESSION_WORKSPACE_SIZE                                      \
 475         max_t(size_t, zlib_inflate_workspacesize(),                     \
 476               zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
 477
 478 int bch2_fs_compress_init(struct bch_fs *c)
 479 {
 480         unsigned order = get_order(c->sb.encoded_extent_max << 9);
 481         int ret;
 482
 483         if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
 484             !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
 485                 return 0;
 486
 487         if (!mempool_initialized(&c->compression_bounce[READ])) {
 488                 ret = mempool_init_page_pool(&c->compression_bounce[READ],
 489                                              1, order);
 490                 if (ret)
 491                         return ret;
 492         }
 493
 494         if (!mempool_initialized(&c->compression_bounce[WRITE])) {
 495                 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
 496                                              1, order);
 497                 if (ret)
 498                         return ret;
 499         }
 500
 501         if (!mempool_initialized(&c->lz4_workspace_pool) &&
 502             bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
 503                 ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
 504                                                 1, LZ4_MEM_COMPRESS);
 505                 if (ret)
 506                         return ret;
 507         }
 508
 509         if (!c->zlib_workspace &&
 510             bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
 511                 c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
 512                 if (!c->zlib_workspace)
 513                         return -ENOMEM;
 514         }
 515
 516         return 0;
 517 }