10 #include <linux/zlib.h>
11 #include <linux/zstd.h>
26 static struct bbuf __bounce_alloc(struct bch_fs *c, unsigned size, int rw)
30 BUG_ON(size > c->sb.encoded_extent_max << 9);
32 b = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
34 return (struct bbuf) { .b = b, .type = BB_KMALLOC, .rw = rw };
36 b = mempool_alloc(&c->compression_bounce[rw], GFP_NOWAIT);
37 b = b ? page_address(b) : NULL;
39 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
43 return (struct bbuf) { .b = b, .type = BB_VMALLOC, .rw = rw };
45 b = mempool_alloc(&c->compression_bounce[rw], GFP_NOIO);
46 b = b ? page_address(b) : NULL;
48 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
53 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
54 struct bvec_iter start, int rw)
58 struct bvec_iter iter;
59 unsigned nr_pages = 0;
60 struct page *stack_pages[16];
61 struct page **pages = NULL;
63 unsigned prev_end = PAGE_SIZE;
66 BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
68 #ifndef CONFIG_HIGHMEM
69 __bio_for_each_contig_segment(bv, bio, iter, start) {
70 if (bv.bv_len == start.bi_size)
71 return (struct bbuf) {
72 .b = page_address(bv.bv_page) + bv.bv_offset,
73 .type = BB_NONE, .rw = rw
77 __bio_for_each_segment(bv, bio, iter, start) {
78 if ((!first && bv.bv_offset) ||
79 prev_end != PAGE_SIZE)
82 prev_end = bv.bv_offset + bv.bv_len;
86 BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
88 pages = nr_pages > ARRAY_SIZE(stack_pages)
89 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
95 __bio_for_each_segment(bv, bio, iter, start)
96 pages[nr_pages++] = bv.bv_page;
98 data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
99 if (pages != stack_pages)
103 return (struct bbuf) {
104 .b = data + bio_iter_offset(bio, start),
105 .type = BB_VMAP, .rw = rw
108 ret = __bounce_alloc(c, start.bi_size, rw);
111 memcpy_from_bio(ret.b, bio, start);
116 static struct bbuf bio_map_or_bounce(struct bch_fs *c, struct bio *bio, int rw)
118 return __bio_map_or_bounce(c, bio, bio->bi_iter, rw);
121 static void bio_unmap_or_unbounce(struct bch_fs *c, struct bbuf buf)
127 vunmap((void *) ((unsigned long) buf.b & PAGE_MASK));
136 mempool_free(virt_to_page(buf.b),
137 &c->compression_bounce[buf.rw]);
142 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
145 strm->workspace = workspace;
149 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
150 void *dst_data, struct bch_extent_crc_unpacked crc)
152 struct bbuf src_data = { NULL };
153 size_t src_len = src->bi_iter.bi_size;
154 size_t dst_len = crc.uncompressed_size << 9;
158 src_data = bio_map_or_bounce(c, src, READ);
160 switch (crc.compression_type) {
161 case BCH_COMPRESSION_LZ4_OLD:
162 ret = bch2_lz4_decompress(src_data.b, &src_len,
167 case BCH_COMPRESSION_LZ4:
168 ret = LZ4_decompress_safe_partial(src_data.b, dst_data,
169 src_len, dst_len, dst_len);
173 case BCH_COMPRESSION_GZIP: {
175 .next_in = src_data.b,
177 .next_out = dst_data,
178 .avail_out = dst_len,
181 workspace = mempool_alloc(&c->decompress_workspace, GFP_NOIO);
183 zlib_set_workspace(&strm, workspace);
184 zlib_inflateInit2(&strm, -MAX_WBITS);
185 ret = zlib_inflate(&strm, Z_FINISH);
187 mempool_free(workspace, &c->decompress_workspace);
189 if (ret != Z_STREAM_END)
193 case BCH_COMPRESSION_ZSTD: {
197 workspace = mempool_alloc(&c->decompress_workspace, GFP_NOIO);
198 ctx = ZSTD_initDCtx(workspace, ZSTD_DCtxWorkspaceBound());
200 src_len = le32_to_cpup(src_data.b);
202 len = ZSTD_decompressDCtx(ctx,
204 src_data.b + 4, src_len);
206 mempool_free(workspace, &c->decompress_workspace);
217 bio_unmap_or_unbounce(c, src_data);
224 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
225 struct bch_extent_crc_unpacked *crc)
227 struct bbuf data = { NULL };
228 size_t dst_len = crc->uncompressed_size << 9;
230 /* bio must own its pages: */
231 BUG_ON(!bio->bi_vcnt);
232 BUG_ON(DIV_ROUND_UP(crc->live_size, PAGE_SECTORS) > bio->bi_max_vecs);
234 if (crc->uncompressed_size > c->sb.encoded_extent_max ||
235 crc->compressed_size > c->sb.encoded_extent_max) {
236 bch_err(c, "error rewriting existing data: extent too big");
240 data = __bounce_alloc(c, dst_len, WRITE);
242 if (__bio_uncompress(c, bio, data.b, *crc)) {
243 bch_err(c, "error rewriting existing data: decompression error");
244 bio_unmap_or_unbounce(c, data);
249 * might have to free existing pages and retry allocation from mempool -
250 * do this _after_ decompressing:
252 bch2_bio_alloc_more_pages_pool(c, bio, crc->live_size << 9);
254 memcpy_to_bio(bio, bio->bi_iter, data.b + (crc->offset << 9));
257 crc->compression_type = 0;
258 crc->compressed_size = crc->live_size;
259 crc->uncompressed_size = crc->live_size;
261 crc->csum = (struct bch_csum) { 0, 0 };
263 bio_unmap_or_unbounce(c, data);
267 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
268 struct bio *dst, struct bvec_iter dst_iter,
269 struct bch_extent_crc_unpacked crc)
271 struct bbuf dst_data = { NULL };
272 size_t dst_len = crc.uncompressed_size << 9;
275 if (crc.uncompressed_size > c->sb.encoded_extent_max ||
276 crc.compressed_size > c->sb.encoded_extent_max)
279 dst_data = dst_len == dst_iter.bi_size
280 ? __bio_map_or_bounce(c, dst, dst_iter, WRITE)
281 : __bounce_alloc(c, dst_len, WRITE);
283 ret = __bio_uncompress(c, src, dst_data.b, crc);
287 if (dst_data.type != BB_NONE)
288 memcpy_to_bio(dst, dst_iter, dst_data.b + (crc.offset << 9));
290 bio_unmap_or_unbounce(c, dst_data);
294 static int attempt_compress(struct bch_fs *c,
296 void *dst, size_t dst_len,
297 void *src, size_t src_len,
298 unsigned compression_type)
300 switch (compression_type) {
301 case BCH_COMPRESSION_LZ4: {
303 int ret = LZ4_compress_destSize(
313 case BCH_COMPRESSION_GZIP: {
318 .avail_out = dst_len,
321 zlib_set_workspace(&strm, workspace);
322 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
323 Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
326 if (zlib_deflate(&strm, Z_FINISH) != Z_STREAM_END)
329 if (zlib_deflateEnd(&strm) != Z_OK)
332 return strm.total_out;
334 case BCH_COMPRESSION_ZSTD: {
335 ZSTD_CCtx *ctx = ZSTD_initCCtx(workspace,
336 ZSTD_CCtxWorkspaceBound(c->zstd_params.cParams));
338 size_t len = ZSTD_compressCCtx(ctx,
339 dst + 4, dst_len - 4,
342 if (ZSTD_isError(len))
345 *((__le32 *) dst) = cpu_to_le32(len);
353 static unsigned __bio_compress(struct bch_fs *c,
354 struct bio *dst, size_t *dst_len,
355 struct bio *src, size_t *src_len,
356 unsigned compression_type)
358 struct bbuf src_data = { NULL }, dst_data = { NULL };
363 BUG_ON(compression_type >= BCH_COMPRESSION_NR);
364 BUG_ON(!mempool_initialized(&c->compress_workspace[compression_type]));
366 /* If it's only one block, don't bother trying to compress: */
367 if (bio_sectors(src) <= c->opts.block_size)
370 dst_data = bio_map_or_bounce(c, dst, WRITE);
371 src_data = bio_map_or_bounce(c, src, READ);
373 workspace = mempool_alloc(&c->compress_workspace[compression_type], GFP_NOIO);
375 *src_len = src->bi_iter.bi_size;
376 *dst_len = dst->bi_iter.bi_size;
379 * XXX: this algorithm sucks when the compression code doesn't tell us
380 * how much would fit, like LZ4 does:
383 if (*src_len <= block_bytes(c)) {
388 ret = attempt_compress(c, workspace,
389 dst_data.b, *dst_len,
390 src_data.b, *src_len,
398 /* Didn't fit: should we retry with a smaller amount? */
399 if (*src_len <= *dst_len) {
405 * If ret is negative, it's a hint as to how much data would fit
407 BUG_ON(-ret >= *src_len);
412 *src_len -= (*src_len - *dst_len) / 2;
413 *src_len = round_down(*src_len, block_bytes(c));
416 mempool_free(workspace, &c->compress_workspace[compression_type]);
421 /* Didn't get smaller: */
422 if (round_up(*dst_len, block_bytes(c)) >= *src_len)
425 pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
427 memset(dst_data.b + *dst_len, 0, pad);
430 if (dst_data.type != BB_NONE)
431 memcpy_to_bio(dst, dst->bi_iter, dst_data.b);
433 BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
434 BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
435 BUG_ON(*dst_len & (block_bytes(c) - 1));
436 BUG_ON(*src_len & (block_bytes(c) - 1));
438 bio_unmap_or_unbounce(c, src_data);
439 bio_unmap_or_unbounce(c, dst_data);
440 return compression_type;
442 compression_type = 0;
446 unsigned bch2_bio_compress(struct bch_fs *c,
447 struct bio *dst, size_t *dst_len,
448 struct bio *src, size_t *src_len,
449 unsigned compression_type)
451 unsigned orig_dst = dst->bi_iter.bi_size;
452 unsigned orig_src = src->bi_iter.bi_size;
454 /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
455 src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
456 c->sb.encoded_extent_max << 9);
457 /* Don't generate a bigger output than input: */
458 dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
460 if (compression_type == BCH_COMPRESSION_LZ4_OLD)
461 compression_type = BCH_COMPRESSION_LZ4;
464 __bio_compress(c, dst, dst_len, src, src_len, compression_type);
466 dst->bi_iter.bi_size = orig_dst;
467 src->bi_iter.bi_size = orig_src;
468 return compression_type;
471 static int __bch2_fs_compress_init(struct bch_fs *, u64);
473 #define BCH_FEATURE_NONE 0
475 static const unsigned bch2_compression_opt_to_feature[] = {
476 #define x(t) [BCH_COMPRESSION_OPT_##t] = BCH_FEATURE_##t,
477 BCH_COMPRESSION_TYPES()
481 #undef BCH_FEATURE_NONE
483 int __bch2_check_set_has_compressed_data(struct bch_fs *c, u64 f)
487 if ((c->sb.features & f) == f)
490 mutex_lock(&c->sb_lock);
492 if ((c->sb.features & f) == f) {
493 mutex_unlock(&c->sb_lock);
497 ret = __bch2_fs_compress_init(c, c->sb.features|f);
499 mutex_unlock(&c->sb_lock);
503 c->disk_sb->features[0] |= cpu_to_le64(f);
505 mutex_unlock(&c->sb_lock);
510 int bch2_check_set_has_compressed_data(struct bch_fs *c,
511 unsigned compression_type)
513 BUG_ON(compression_type >= ARRAY_SIZE(bch2_compression_opt_to_feature));
515 return compression_type
516 ? __bch2_check_set_has_compressed_data(c,
517 1ULL << bch2_compression_opt_to_feature[compression_type])
521 void bch2_fs_compress_exit(struct bch_fs *c)
525 mempool_exit(&c->decompress_workspace);
526 for (i = 0; i < ARRAY_SIZE(c->compress_workspace); i++)
527 mempool_exit(&c->compress_workspace[i]);
528 mempool_exit(&c->compression_bounce[WRITE]);
529 mempool_exit(&c->compression_bounce[READ]);
532 static void *mempool_kvpmalloc(gfp_t gfp_mask, void *pool_data)
534 size_t size = (size_t)pool_data;
535 return kvpmalloc(size, gfp_mask);
538 void mempool_kvpfree(void *element, void *pool_data)
540 size_t size = (size_t)pool_data;
541 kvpfree(element, size);
544 static int mempool_init_kvpmalloc_pool(mempool_t *pool, int min_nr, size_t size)
546 return !mempool_initialized(pool)
547 ? mempool_init(pool, min_nr, mempool_kvpmalloc,
548 mempool_kvpfree, (void *) size)
552 static int __bch2_fs_compress_init(struct bch_fs *c, u64 features)
554 size_t max_extent = c->sb.encoded_extent_max << 9;
555 size_t order = get_order(max_extent);
556 size_t decompress_workspace_size = 0;
557 bool decompress_workspace_needed;
558 ZSTD_parameters params = ZSTD_getParams(0, max_extent, 0);
562 size_t compress_workspace;
563 size_t decompress_workspace;
564 } compression_types[] = {
565 { BCH_FEATURE_LZ4, BCH_COMPRESSION_LZ4, LZ4_MEM_COMPRESS, 0 },
566 { BCH_FEATURE_GZIP, BCH_COMPRESSION_GZIP,
567 zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL),
568 zlib_inflate_workspacesize(), },
569 { BCH_FEATURE_ZSTD, BCH_COMPRESSION_ZSTD,
570 ZSTD_CCtxWorkspaceBound(params.cParams),
571 ZSTD_DCtxWorkspaceBound() },
575 pr_verbose_init(c->opts, "");
577 c->zstd_params = params;
579 for (i = compression_types;
580 i < compression_types + ARRAY_SIZE(compression_types);
582 if (features & (1 << i->feature))
583 goto have_compressed;
588 if (!mempool_initialized(&c->compression_bounce[READ])) {
589 ret = mempool_init_page_pool(&c->compression_bounce[READ],
595 if (!mempool_initialized(&c->compression_bounce[WRITE])) {
596 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
602 for (i = compression_types;
603 i < compression_types + ARRAY_SIZE(compression_types);
605 decompress_workspace_size =
606 max(decompress_workspace_size, i->decompress_workspace);
608 if (!(features & (1 << i->feature)))
611 if (i->decompress_workspace)
612 decompress_workspace_needed = true;
614 ret = mempool_init_kvpmalloc_pool(
615 &c->compress_workspace[i->type],
616 1, i->compress_workspace);
621 ret = mempool_init_kmalloc_pool(
622 &c->decompress_workspace,
623 1, decompress_workspace_size);
627 pr_verbose_init(c->opts, "ret %i", ret);
631 int bch2_fs_compress_init(struct bch_fs *c)
633 u64 f = c->sb.features;
635 if (c->opts.compression)
636 f |= 1ULL << bch2_compression_opt_to_feature[c->opts.compression];
638 if (c->opts.background_compression)
639 f |= 1ULL << bch2_compression_opt_to_feature[c->opts.background_compression];
641 return __bch2_fs_compress_init(c, f);