git.sesse.net Git - bcachefs-tools-debian/blob - libbcachefs/compress.c

   1 #include "bcachefs.h"
   2 #include "compress.h"
   3 #include "extents.h"
   4 #include "io.h"
   5 #include "super-io.h"
   6
   7 #include <linux/lz4.h>
   8 #include <linux/zlib.h>
   9
  10 enum bounced {
  11         BOUNCED_CONTIG,
  12         BOUNCED_MAPPED,
  13         BOUNCED_KMALLOCED,
  14         BOUNCED_VMALLOCED,
  15         BOUNCED_MEMPOOLED,
  16 };
  17
  18 static void *__bounce_alloc(struct bch_fs *c, unsigned size,
  19                             unsigned *bounced, int direction)
  20 {
  21         void *data;
  22
  23         *bounced = BOUNCED_KMALLOCED;
  24         data = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
  25         if (data)
  26                 return data;
  27
  28         *bounced = BOUNCED_MEMPOOLED;
  29         data = mempool_alloc(&c->compression_bounce[direction], GFP_NOWAIT);
  30         if (data)
  31                 return page_address(data);
  32
  33         *bounced = BOUNCED_VMALLOCED;
  34         data = vmalloc(size);
  35         if (data)
  36                 return data;
  37
  38         *bounced = BOUNCED_MEMPOOLED;
  39         data = mempool_alloc(&c->compression_bounce[direction], GFP_NOIO);
  40         return page_address(data);
  41 }
  42
  43 static void *__bio_map_or_bounce(struct bch_fs *c,
  44                                  struct bio *bio, struct bvec_iter start,
  45                                  unsigned *bounced, int direction)
  46 {
  47         struct bio_vec bv;
  48         struct bvec_iter iter;
  49         unsigned nr_pages = 0;
  50         struct page *stack_pages[16];
  51         struct page **pages = NULL;
  52         bool first = true;
  53         unsigned prev_end = PAGE_SIZE;
  54         void *data;
  55
  56         BUG_ON(bvec_iter_sectors(start) > BCH_ENCODED_EXTENT_MAX);
  57
  58 #ifndef CONFIG_HIGHMEM
  59         *bounced = BOUNCED_CONTIG;
  60
  61         __bio_for_each_contig_segment(bv, bio, iter, start) {
  62                 if (bv.bv_len == start.bi_size)
  63                         return page_address(bv.bv_page) + bv.bv_offset;
  64         }
  65 #endif
  66         *bounced = BOUNCED_MAPPED;
  67
  68         __bio_for_each_segment(bv, bio, iter, start) {
  69                 if ((!first && bv.bv_offset) ||
  70                     prev_end != PAGE_SIZE)
  71                         goto bounce;
  72
  73                 prev_end = bv.bv_offset + bv.bv_len;
  74                 nr_pages++;
  75         }
  76
  77         BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
  78
  79         pages = nr_pages > ARRAY_SIZE(stack_pages)
  80                 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
  81                 : stack_pages;
  82         if (!pages)
  83                 goto bounce;
  84
  85         nr_pages = 0;
  86         __bio_for_each_segment(bv, bio, iter, start)
  87                 pages[nr_pages++] = bv.bv_page;
  88
  89         data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
  90         if (pages != stack_pages)
  91                 kfree(pages);
  92
  93         return data + bio_iter_offset(bio, start);
  94 bounce:
  95         data = __bounce_alloc(c, start.bi_size, bounced, direction);
  96
  97         if (direction == READ)
  98                 memcpy_from_bio(data, bio, start);
  99
 100         return data;
 101 }
 102
 103 static void *bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
 104                                unsigned *bounced, int direction)
 105 {
 106         return __bio_map_or_bounce(c, bio, bio->bi_iter, bounced, direction);
 107 }
 108
 109 static void bio_unmap_or_unbounce(struct bch_fs *c, void *data,
 110                                   unsigned bounced, int direction)
 111 {
 112         if (!data)
 113                 return;
 114
 115         switch (bounced) {
 116         case BOUNCED_MAPPED:
 117                 vunmap((void *) ((unsigned long) data & PAGE_MASK));
 118                 return;
 119         case BOUNCED_KMALLOCED:
 120                 kfree(data);
 121                 return;
 122         case BOUNCED_VMALLOCED:
 123                 vfree(data);
 124                 return;
 125         case BOUNCED_MEMPOOLED:
 126                 mempool_free(virt_to_page(data), &c->compression_bounce[direction]);
 127                 return;
 128         }
 129 }
 130
 131 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
 132 {
 133 #ifdef __KERNEL__
 134         strm->workspace = workspace;
 135 #endif
 136 }
 137
 138 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
 139                             void *dst_data, struct bch_extent_crc128 crc)
 140 {
 141         void *src_data = NULL;
 142         unsigned src_bounced;
 143         size_t src_len = src->bi_iter.bi_size;
 144         size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
 145         int ret;
 146
 147         src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
 148
 149         switch (crc.compression_type) {
 150         case BCH_COMPRESSION_LZ4:
 151                 ret = lz4_decompress(src_data, &src_len,
 152                                      dst_data, dst_len);
 153                 if (ret) {
 154                         ret = -EIO;
 155                         goto err;
 156                 }
 157                 break;
 158         case BCH_COMPRESSION_GZIP: {
 159                 void *workspace;
 160                 z_stream strm;
 161
 162                 workspace = kmalloc(zlib_inflate_workspacesize(),
 163                                     GFP_NOIO|__GFP_NOWARN);
 164                 if (!workspace) {
 165                         mutex_lock(&c->zlib_workspace_lock);
 166                         workspace = c->zlib_workspace;
 167                 }
 168
 169                 strm.next_in    = src_data;
 170                 strm.avail_in   = src_len;
 171                 strm.next_out   = dst_data;
 172                 strm.avail_out  = dst_len;
 173                 zlib_set_workspace(&strm, workspace);
 174                 zlib_inflateInit2(&strm, -MAX_WBITS);
 175
 176                 ret = zlib_inflate(&strm, Z_FINISH);
 177
 178                 if (workspace == c->zlib_workspace)
 179                         mutex_unlock(&c->zlib_workspace_lock);
 180                 else
 181                         kfree(workspace);
 182
 183                 if (ret != Z_STREAM_END) {
 184                         ret = -EIO;
 185                         goto err;
 186                 }
 187                 break;
 188         }
 189         default:
 190                 BUG();
 191         }
 192         ret = 0;
 193 err:
 194         bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
 195         return ret;
 196 }
 197
 198 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
 199                                unsigned live_data_sectors,
 200                                struct bch_extent_crc128 crc)
 201 {
 202         void *dst_data = NULL;
 203         size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
 204         int ret = -ENOMEM;
 205
 206         BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
 207
 208         /* XXX mempoolify */
 209         dst_data = kmalloc(dst_len, GFP_NOIO|__GFP_NOWARN);
 210         if (!dst_data) {
 211                 dst_data = vmalloc(dst_len);
 212                 if (!dst_data)
 213                         goto err;
 214         }
 215
 216         ret = __bio_uncompress(c, bio, dst_data, crc);
 217         if (ret)
 218                 goto err;
 219
 220         while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
 221                 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 222
 223                 bv->bv_page = alloc_page(GFP_NOIO);
 224                 if (!bv->bv_page)
 225                         goto use_mempool;
 226
 227                 bv->bv_len = PAGE_SIZE;
 228                 bv->bv_offset = 0;
 229                 bio->bi_vcnt++;
 230         }
 231
 232         bio->bi_iter.bi_size = live_data_sectors << 9;
 233 copy_data:
 234         memcpy_to_bio(bio, bio->bi_iter, dst_data + (crc.offset << 9));
 235 err:
 236         kvfree(dst_data);
 237         return ret;
 238 use_mempool:
 239         /*
 240          * We already allocated from mempool, we can't allocate from it again
 241          * without freeing the pages we already allocated or else we could
 242          * deadlock:
 243          */
 244
 245         bch2_bio_free_pages_pool(c, bio);
 246         bch2_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
 247         goto copy_data;
 248 }
 249
 250 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
 251                        struct bio *dst, struct bvec_iter dst_iter,
 252                        struct bch_extent_crc128 crc)
 253 {
 254         void *dst_data = NULL;
 255         unsigned dst_bounced;
 256         size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
 257         int ret = -ENOMEM;
 258
 259         dst_data = dst_len == dst_iter.bi_size
 260                 ? __bio_map_or_bounce(c, dst, dst_iter, &dst_bounced, WRITE)
 261                 : __bounce_alloc(c, dst_len, &dst_bounced, WRITE);
 262
 263         ret = __bio_uncompress(c, src, dst_data, crc);
 264         if (ret)
 265                 goto err;
 266
 267         if (dst_bounced)
 268                 memcpy_to_bio(dst, dst_iter, dst_data + (crc.offset << 9));
 269 err:
 270         bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
 271         return ret;
 272 }
 273
 274 static int __bio_compress(struct bch_fs *c,
 275                           struct bio *dst, size_t *dst_len,
 276                           struct bio *src, size_t *src_len,
 277                           unsigned compression_type)
 278 {
 279         void *src_data = NULL, *dst_data = NULL;
 280         unsigned src_bounced, dst_bounced, pad;
 281         int ret = -1;
 282
 283         dst_data = bio_map_or_bounce(c, dst, &dst_bounced, WRITE);
 284         src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
 285
 286         switch (compression_type) {
 287         case BCH_COMPRESSION_LZ4: {
 288                 void *workspace;
 289
 290                 *dst_len = dst->bi_iter.bi_size;
 291                 *src_len = src->bi_iter.bi_size;
 292
 293                 workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
 294
 295                 while (*src_len > block_bytes(c) &&
 296                        (ret = lz4_compress(src_data, *src_len,
 297                                            dst_data, dst_len,
 298                                            workspace))) {
 299                         /*
 300                          * On error, the compressed data was bigger than
 301                          * dst_len, and -ret is the amount of data we were able
 302                          * to compress - round down to nearest block and try
 303                          * again:
 304                          */
 305                         BUG_ON(ret > 0);
 306                         BUG_ON(-ret >= *src_len);
 307
 308                         *src_len = round_down(-ret, block_bytes(c));
 309                 }
 310
 311                 mempool_free(workspace, &c->lz4_workspace_pool);
 312
 313                 if (ret)
 314                         goto err;
 315                 break;
 316         }
 317         case BCH_COMPRESSION_GZIP: {
 318                 void *workspace;
 319                 z_stream strm;
 320
 321                 workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
 322                                                                DEF_MEM_LEVEL),
 323                                     GFP_NOIO|__GFP_NOWARN);
 324                 if (!workspace) {
 325                         mutex_lock(&c->zlib_workspace_lock);
 326                         workspace = c->zlib_workspace;
 327                 }
 328
 329                 strm.next_in    = src_data;
 330                 strm.avail_in   = min(src->bi_iter.bi_size,
 331                                       dst->bi_iter.bi_size);
 332                 strm.next_out   = dst_data;
 333                 strm.avail_out  = dst->bi_iter.bi_size;
 334                 zlib_set_workspace(&strm, workspace);
 335                 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
 336                                   Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
 337                                   Z_DEFAULT_STRATEGY);
 338
 339                 ret = zlib_deflate(&strm, Z_FINISH);
 340                 if (ret != Z_STREAM_END) {
 341                         ret = -EIO;
 342                         goto zlib_err;
 343                 }
 344
 345                 ret = zlib_deflateEnd(&strm);
 346                 if (ret != Z_OK) {
 347                         ret = -EIO;
 348                         goto zlib_err;
 349                 }
 350
 351                 ret = 0;
 352 zlib_err:
 353                 if (workspace == c->zlib_workspace)
 354                         mutex_unlock(&c->zlib_workspace_lock);
 355                 else
 356                         kfree(workspace);
 357
 358                 if (ret)
 359                         goto err;
 360
 361                 *dst_len = strm.total_out;
 362                 *src_len = strm.total_in;
 363                 break;
 364         }
 365         default:
 366                 BUG();
 367         }
 368
 369         BUG_ON(!*dst_len);
 370         BUG_ON(*dst_len > dst->bi_iter.bi_size);
 371
 372         BUG_ON(*src_len & (block_bytes(c) - 1));
 373         BUG_ON(*src_len > src->bi_iter.bi_size);
 374
 375         /* Didn't get smaller: */
 376         if (round_up(*dst_len, block_bytes(c)) >= *src_len) {
 377                 ret = -1;
 378                 goto err;
 379         }
 380
 381         pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
 382
 383         memset(dst_data + *dst_len, 0, pad);
 384         *dst_len += pad;
 385
 386         if (dst_bounced)
 387                 memcpy_to_bio(dst, dst->bi_iter, dst_data);
 388 err:
 389         bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
 390         bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
 391         return ret;
 392 }
 393
 394 void bch2_bio_compress(struct bch_fs *c,
 395                       struct bio *dst, size_t *dst_len,
 396                       struct bio *src, size_t *src_len,
 397                       unsigned *compression_type)
 398 {
 399         unsigned orig_dst = dst->bi_iter.bi_size;
 400         unsigned orig_src = src->bi_iter.bi_size;
 401
 402         /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
 403         src->bi_iter.bi_size =
 404                 min(src->bi_iter.bi_size, BCH_ENCODED_EXTENT_MAX << 9);
 405
 406         /* Don't generate a bigger output than input: */
 407         dst->bi_iter.bi_size =
 408                 min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
 409
 410         /* If it's only one block, don't bother trying to compress: */
 411         if (*compression_type != BCH_COMPRESSION_NONE &&
 412             bio_sectors(src) > c->sb.block_size &&
 413             !__bio_compress(c, dst, dst_len, src, src_len, *compression_type))
 414                 goto out;
 415
 416         /* If compressing failed (didn't get smaller), just copy: */
 417         *compression_type = BCH_COMPRESSION_NONE;
 418         *dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
 419         bio_copy_data(dst, src);
 420 out:
 421         dst->bi_iter.bi_size = orig_dst;
 422         src->bi_iter.bi_size = orig_src;
 423 }
 424
 425 /* doesn't write superblock: */
 426 int bch2_check_set_has_compressed_data(struct bch_fs *c,
 427                                       unsigned compression_type)
 428 {
 429         switch (compression_type) {
 430         case BCH_COMPRESSION_NONE:
 431                 return 0;
 432         case BCH_COMPRESSION_LZ4:
 433                 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
 434                         return 0;
 435
 436                 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
 437                 break;
 438         case BCH_COMPRESSION_GZIP:
 439                 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
 440                         return 0;
 441
 442                 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
 443                 break;
 444         }
 445
 446         return bch2_fs_compress_init(c);
 447 }
 448
 449 void bch2_fs_compress_exit(struct bch_fs *c)
 450 {
 451         vfree(c->zlib_workspace);
 452         mempool_exit(&c->lz4_workspace_pool);
 453         mempool_exit(&c->compression_bounce[WRITE]);
 454         mempool_exit(&c->compression_bounce[READ]);
 455 }
 456
 457 #define COMPRESSION_WORKSPACE_SIZE                                      \
 458         max_t(size_t, zlib_inflate_workspacesize(),                     \
 459               zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
 460
 461 int bch2_fs_compress_init(struct bch_fs *c)
 462 {
 463         unsigned order = get_order(BCH_ENCODED_EXTENT_MAX << 9);
 464         int ret;
 465
 466         if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
 467             !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
 468                 return 0;
 469
 470         if (!mempool_initialized(&c->compression_bounce[READ])) {
 471                 ret = mempool_init_page_pool(&c->compression_bounce[READ],
 472                                              1, order);
 473                 if (ret)
 474                         return ret;
 475         }
 476
 477         if (!mempool_initialized(&c->compression_bounce[WRITE])) {
 478                 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
 479                                              1, order);
 480                 if (ret)
 481                         return ret;
 482         }
 483
 484         if (!mempool_initialized(&c->lz4_workspace_pool) &&
 485             bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
 486                 ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
 487                                                 1, LZ4_MEM_COMPRESS);
 488                 if (ret)
 489                         return ret;
 490         }
 491
 492         if (!c->zlib_workspace &&
 493             bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
 494                 c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
 495                 if (!c->zlib_workspace)
 496                         return -ENOMEM;
 497         }
 498
 499         return 0;
 500 }