9 #include <linux/zlib.h>
24 static struct bbuf __bounce_alloc(struct bch_fs *c, unsigned size, int rw)
28 BUG_ON(size > c->sb.encoded_extent_max << 9);
30 b = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
32 return (struct bbuf) { .b = b, .type = BB_KMALLOC, .rw = rw };
34 b = mempool_alloc(&c->compression_bounce[rw], GFP_NOWAIT);
35 b = b ? page_address(b) : NULL;
37 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
41 return (struct bbuf) { .b = b, .type = BB_VMALLOC, .rw = rw };
43 b = mempool_alloc(&c->compression_bounce[rw], GFP_NOIO);
44 b = b ? page_address(b) : NULL;
46 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
51 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
52 struct bvec_iter start, int rw)
56 struct bvec_iter iter;
57 unsigned nr_pages = 0;
58 struct page *stack_pages[16];
59 struct page **pages = NULL;
61 unsigned prev_end = PAGE_SIZE;
64 BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
66 #ifndef CONFIG_HIGHMEM
67 __bio_for_each_contig_segment(bv, bio, iter, start) {
68 if (bv.bv_len == start.bi_size)
69 return (struct bbuf) {
70 .b = page_address(bv.bv_page) + bv.bv_offset,
71 .type = BB_NONE, .rw = rw
75 __bio_for_each_segment(bv, bio, iter, start) {
76 if ((!first && bv.bv_offset) ||
77 prev_end != PAGE_SIZE)
80 prev_end = bv.bv_offset + bv.bv_len;
84 BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
86 pages = nr_pages > ARRAY_SIZE(stack_pages)
87 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
93 __bio_for_each_segment(bv, bio, iter, start)
94 pages[nr_pages++] = bv.bv_page;
96 data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
97 if (pages != stack_pages)
101 return (struct bbuf) {
102 .b = data + bio_iter_offset(bio, start),
103 .type = BB_VMAP, .rw = rw
106 ret = __bounce_alloc(c, start.bi_size, rw);
109 memcpy_from_bio(ret.b, bio, start);
114 static struct bbuf bio_map_or_bounce(struct bch_fs *c, struct bio *bio, int rw)
116 return __bio_map_or_bounce(c, bio, bio->bi_iter, rw);
119 static void bio_unmap_or_unbounce(struct bch_fs *c, struct bbuf buf)
125 vunmap((void *) ((unsigned long) buf.b & PAGE_MASK));
134 mempool_free(virt_to_page(buf.b),
135 &c->compression_bounce[buf.rw]);
140 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
143 strm->workspace = workspace;
147 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
148 void *dst_data, struct bch_extent_crc128 crc)
150 struct bbuf src_data = { NULL };
151 size_t src_len = src->bi_iter.bi_size;
152 size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
155 src_data = bio_map_or_bounce(c, src, READ);
157 switch (crc.compression_type) {
158 case BCH_COMPRESSION_LZ4_OLD:
159 ret = bch2_lz4_decompress(src_data.b, &src_len,
166 case BCH_COMPRESSION_LZ4:
167 ret = LZ4_decompress_safe_partial(src_data.b, dst_data,
168 src_len, dst_len, dst_len);
169 if (ret != dst_len) {
174 case BCH_COMPRESSION_GZIP: {
178 workspace = kmalloc(zlib_inflate_workspacesize(),
179 GFP_NOIO|__GFP_NOWARN);
181 mutex_lock(&c->zlib_workspace_lock);
182 workspace = c->zlib_workspace;
185 strm.next_in = src_data.b;
186 strm.avail_in = src_len;
187 strm.next_out = dst_data;
188 strm.avail_out = dst_len;
189 zlib_set_workspace(&strm, workspace);
190 zlib_inflateInit2(&strm, -MAX_WBITS);
192 ret = zlib_inflate(&strm, Z_FINISH);
194 if (workspace == c->zlib_workspace)
195 mutex_unlock(&c->zlib_workspace_lock);
199 if (ret != Z_STREAM_END) {
210 bio_unmap_or_unbounce(c, src_data);
214 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
215 unsigned live_data_sectors,
216 struct bch_extent_crc128 crc)
218 struct bbuf dst_data = { NULL };
219 size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
222 BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
224 if (crc_uncompressed_size(NULL, &crc) > c->sb.encoded_extent_max ||
225 crc_compressed_size(NULL, &crc) > c->sb.encoded_extent_max)
228 dst_data = __bounce_alloc(c, dst_len, WRITE);
230 ret = __bio_uncompress(c, bio, dst_data.b, crc);
234 while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
235 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
237 bv->bv_page = alloc_page(GFP_NOIO);
241 bv->bv_len = PAGE_SIZE;
246 bio->bi_iter.bi_size = live_data_sectors << 9;
248 memcpy_to_bio(bio, bio->bi_iter, dst_data.b + (crc.offset << 9));
250 bio_unmap_or_unbounce(c, dst_data);
254 * We already allocated from mempool, we can't allocate from it again
255 * without freeing the pages we already allocated or else we could
259 bch2_bio_free_pages_pool(c, bio);
260 bch2_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
264 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
265 struct bio *dst, struct bvec_iter dst_iter,
266 struct bch_extent_crc128 crc)
268 struct bbuf dst_data = { NULL };
269 size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
272 if (crc_uncompressed_size(NULL, &crc) > c->sb.encoded_extent_max ||
273 crc_compressed_size(NULL, &crc) > c->sb.encoded_extent_max)
276 dst_data = dst_len == dst_iter.bi_size
277 ? __bio_map_or_bounce(c, dst, dst_iter, WRITE)
278 : __bounce_alloc(c, dst_len, WRITE);
280 ret = __bio_uncompress(c, src, dst_data.b, crc);
284 if (dst_data.type != BB_NONE)
285 memcpy_to_bio(dst, dst_iter, dst_data.b + (crc.offset << 9));
287 bio_unmap_or_unbounce(c, dst_data);
291 static int __bio_compress(struct bch_fs *c,
292 struct bio *dst, size_t *dst_len,
293 struct bio *src, size_t *src_len,
294 unsigned *compression_type)
296 struct bbuf src_data = { NULL }, dst_data = { NULL };
300 dst_data = bio_map_or_bounce(c, dst, WRITE);
301 src_data = bio_map_or_bounce(c, src, READ);
303 switch (*compression_type) {
304 case BCH_COMPRESSION_LZ4_OLD:
305 *compression_type = BCH_COMPRESSION_LZ4;
307 case BCH_COMPRESSION_LZ4: {
309 int len = src->bi_iter.bi_size;
311 workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
314 if (len <= block_bytes(c)) {
319 ret = LZ4_compress_destSize(
320 src_data.b, dst_data.b,
321 &len, dst->bi_iter.bi_size,
324 /* uncompressible: */
329 if (!(len & (block_bytes(c) - 1)))
331 len = round_down(len, block_bytes(c));
333 mempool_free(workspace, &c->lz4_workspace_pool);
343 case BCH_COMPRESSION_GZIP: {
347 workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
349 GFP_NOIO|__GFP_NOWARN);
351 mutex_lock(&c->zlib_workspace_lock);
352 workspace = c->zlib_workspace;
355 strm.next_in = src_data.b;
356 strm.avail_in = min(src->bi_iter.bi_size,
357 dst->bi_iter.bi_size);
358 strm.next_out = dst_data.b;
359 strm.avail_out = dst->bi_iter.bi_size;
360 zlib_set_workspace(&strm, workspace);
361 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
362 Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
365 ret = zlib_deflate(&strm, Z_FINISH);
366 if (ret != Z_STREAM_END) {
371 ret = zlib_deflateEnd(&strm);
379 if (workspace == c->zlib_workspace)
380 mutex_unlock(&c->zlib_workspace_lock);
387 *dst_len = strm.total_out;
388 *src_len = strm.total_in;
395 /* Didn't get smaller: */
396 if (round_up(*dst_len, block_bytes(c)) >= *src_len)
399 pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
401 memset(dst_data.b + *dst_len, 0, pad);
404 if (dst_data.type != BB_NONE)
405 memcpy_to_bio(dst, dst->bi_iter, dst_data.b);
407 bio_unmap_or_unbounce(c, src_data);
408 bio_unmap_or_unbounce(c, dst_data);
415 void bch2_bio_compress(struct bch_fs *c,
416 struct bio *dst, size_t *dst_len,
417 struct bio *src, size_t *src_len,
418 unsigned *compression_type)
420 unsigned orig_dst = dst->bi_iter.bi_size;
421 unsigned orig_src = src->bi_iter.bi_size;
423 /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
424 src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
425 c->sb.encoded_extent_max << 9);
427 /* Don't generate a bigger output than input: */
428 dst->bi_iter.bi_size =
429 min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
431 /* If it's only one block, don't bother trying to compress: */
432 if (*compression_type != BCH_COMPRESSION_NONE &&
433 bio_sectors(src) > c->opts.block_size &&
434 !__bio_compress(c, dst, dst_len, src, src_len, compression_type))
437 /* If compressing failed (didn't get smaller), just copy: */
438 *compression_type = BCH_COMPRESSION_NONE;
439 *dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
440 bio_copy_data(dst, src);
442 dst->bi_iter.bi_size = orig_dst;
443 src->bi_iter.bi_size = orig_src;
445 BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
446 BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
447 BUG_ON(*dst_len & (block_bytes(c) - 1));
448 BUG_ON(*src_len & (block_bytes(c) - 1));
451 /* doesn't write superblock: */
452 int bch2_check_set_has_compressed_data(struct bch_fs *c,
453 unsigned compression_type)
455 switch (compression_type) {
456 case BCH_COMPRESSION_OPT_NONE:
458 case BCH_COMPRESSION_OPT_LZ4:
459 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
462 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
464 case BCH_COMPRESSION_OPT_GZIP:
465 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
468 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
474 return bch2_fs_compress_init(c);
477 void bch2_fs_compress_exit(struct bch_fs *c)
479 vfree(c->zlib_workspace);
480 mempool_exit(&c->lz4_workspace_pool);
481 mempool_exit(&c->compression_bounce[WRITE]);
482 mempool_exit(&c->compression_bounce[READ]);
485 #define COMPRESSION_WORKSPACE_SIZE \
486 max_t(size_t, zlib_inflate_workspacesize(), \
487 zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
489 int bch2_fs_compress_init(struct bch_fs *c)
491 unsigned order = get_order(c->sb.encoded_extent_max << 9);
494 if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
495 !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
498 if (!mempool_initialized(&c->compression_bounce[READ])) {
499 ret = mempool_init_page_pool(&c->compression_bounce[READ],
505 if (!mempool_initialized(&c->compression_bounce[WRITE])) {
506 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
512 if (!mempool_initialized(&c->lz4_workspace_pool) &&
513 bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
514 ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
515 1, LZ4_MEM_COMPRESS);
520 if (!c->zlib_workspace &&
521 bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
522 c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
523 if (!c->zlib_workspace)