10 #include <linux/zlib.h>
25 static struct bbuf __bounce_alloc(struct bch_fs *c, unsigned size, int rw)
29 BUG_ON(size > c->sb.encoded_extent_max << 9);
31 b = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
33 return (struct bbuf) { .b = b, .type = BB_KMALLOC, .rw = rw };
35 b = mempool_alloc(&c->compression_bounce[rw], GFP_NOWAIT);
36 b = b ? page_address(b) : NULL;
38 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
42 return (struct bbuf) { .b = b, .type = BB_VMALLOC, .rw = rw };
44 b = mempool_alloc(&c->compression_bounce[rw], GFP_NOIO);
45 b = b ? page_address(b) : NULL;
47 return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
52 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
53 struct bvec_iter start, int rw)
57 struct bvec_iter iter;
58 unsigned nr_pages = 0;
59 struct page *stack_pages[16];
60 struct page **pages = NULL;
62 unsigned prev_end = PAGE_SIZE;
65 BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
67 #ifndef CONFIG_HIGHMEM
68 __bio_for_each_contig_segment(bv, bio, iter, start) {
69 if (bv.bv_len == start.bi_size)
70 return (struct bbuf) {
71 .b = page_address(bv.bv_page) + bv.bv_offset,
72 .type = BB_NONE, .rw = rw
76 __bio_for_each_segment(bv, bio, iter, start) {
77 if ((!first && bv.bv_offset) ||
78 prev_end != PAGE_SIZE)
81 prev_end = bv.bv_offset + bv.bv_len;
85 BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
87 pages = nr_pages > ARRAY_SIZE(stack_pages)
88 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
94 __bio_for_each_segment(bv, bio, iter, start)
95 pages[nr_pages++] = bv.bv_page;
97 data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
98 if (pages != stack_pages)
102 return (struct bbuf) {
103 .b = data + bio_iter_offset(bio, start),
104 .type = BB_VMAP, .rw = rw
107 ret = __bounce_alloc(c, start.bi_size, rw);
110 memcpy_from_bio(ret.b, bio, start);
115 static struct bbuf bio_map_or_bounce(struct bch_fs *c, struct bio *bio, int rw)
117 return __bio_map_or_bounce(c, bio, bio->bi_iter, rw);
120 static void bio_unmap_or_unbounce(struct bch_fs *c, struct bbuf buf)
126 vunmap((void *) ((unsigned long) buf.b & PAGE_MASK));
135 mempool_free(virt_to_page(buf.b),
136 &c->compression_bounce[buf.rw]);
141 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
144 strm->workspace = workspace;
148 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
149 void *dst_data, struct bch_extent_crc_unpacked crc)
151 struct bbuf src_data = { NULL };
152 size_t src_len = src->bi_iter.bi_size;
153 size_t dst_len = crc.uncompressed_size << 9;
156 src_data = bio_map_or_bounce(c, src, READ);
158 switch (crc.compression_type) {
159 case BCH_COMPRESSION_LZ4_OLD:
160 ret = bch2_lz4_decompress(src_data.b, &src_len,
167 case BCH_COMPRESSION_LZ4:
168 ret = LZ4_decompress_safe_partial(src_data.b, dst_data,
169 src_len, dst_len, dst_len);
170 if (ret != dst_len) {
175 case BCH_COMPRESSION_GZIP: {
179 workspace = kmalloc(zlib_inflate_workspacesize(),
180 GFP_NOIO|__GFP_NOWARN);
182 mutex_lock(&c->zlib_workspace_lock);
183 workspace = c->zlib_workspace;
186 strm.next_in = src_data.b;
187 strm.avail_in = src_len;
188 strm.next_out = dst_data;
189 strm.avail_out = dst_len;
190 zlib_set_workspace(&strm, workspace);
191 zlib_inflateInit2(&strm, -MAX_WBITS);
193 ret = zlib_inflate(&strm, Z_FINISH);
195 if (workspace == c->zlib_workspace)
196 mutex_unlock(&c->zlib_workspace_lock);
200 if (ret != Z_STREAM_END) {
211 bio_unmap_or_unbounce(c, src_data);
215 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
216 struct bch_extent_crc_unpacked *crc)
218 struct bbuf data = { NULL };
219 size_t dst_len = crc->uncompressed_size << 9;
221 /* bio must own its pages: */
222 BUG_ON(!bio->bi_vcnt);
223 BUG_ON(DIV_ROUND_UP(crc->live_size, PAGE_SECTORS) > bio->bi_max_vecs);
225 if (crc->uncompressed_size > c->sb.encoded_extent_max ||
226 crc->compressed_size > c->sb.encoded_extent_max) {
227 bch_err(c, "error rewriting existing data: extent too big");
231 data = __bounce_alloc(c, dst_len, WRITE);
233 if (__bio_uncompress(c, bio, data.b, *crc)) {
234 bch_err(c, "error rewriting existing data: decompression error");
235 bio_unmap_or_unbounce(c, data);
240 * might have to free existing pages and retry allocation from mempool -
241 * do this _after_ decompressing:
243 bch2_bio_alloc_more_pages_pool(c, bio, crc->live_size << 9);
245 memcpy_to_bio(bio, bio->bi_iter, data.b + (crc->offset << 9));
248 crc->compression_type = 0;
249 crc->compressed_size = crc->live_size;
250 crc->uncompressed_size = crc->live_size;
252 crc->csum = (struct bch_csum) { 0, 0 };
254 bio_unmap_or_unbounce(c, data);
258 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
259 struct bio *dst, struct bvec_iter dst_iter,
260 struct bch_extent_crc_unpacked crc)
262 struct bbuf dst_data = { NULL };
263 size_t dst_len = crc.uncompressed_size << 9;
266 if (crc.uncompressed_size > c->sb.encoded_extent_max ||
267 crc.compressed_size > c->sb.encoded_extent_max)
270 dst_data = dst_len == dst_iter.bi_size
271 ? __bio_map_or_bounce(c, dst, dst_iter, WRITE)
272 : __bounce_alloc(c, dst_len, WRITE);
274 ret = __bio_uncompress(c, src, dst_data.b, crc);
278 if (dst_data.type != BB_NONE)
279 memcpy_to_bio(dst, dst_iter, dst_data.b + (crc.offset << 9));
281 bio_unmap_or_unbounce(c, dst_data);
285 static unsigned __bio_compress(struct bch_fs *c,
286 struct bio *dst, size_t *dst_len,
287 struct bio *src, size_t *src_len,
288 unsigned compression_type)
290 struct bbuf src_data = { NULL }, dst_data = { NULL };
294 /* If it's only one block, don't bother trying to compress: */
295 if (bio_sectors(src) <= c->opts.block_size)
298 dst_data = bio_map_or_bounce(c, dst, WRITE);
299 src_data = bio_map_or_bounce(c, src, READ);
301 switch (compression_type) {
302 case BCH_COMPRESSION_LZ4_OLD:
303 compression_type = BCH_COMPRESSION_LZ4;
305 case BCH_COMPRESSION_LZ4: {
307 int len = src->bi_iter.bi_size;
309 workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
312 if (len <= block_bytes(c)) {
317 ret = LZ4_compress_destSize(
318 src_data.b, dst_data.b,
319 &len, dst->bi_iter.bi_size,
322 /* uncompressible: */
327 if (!(len & (block_bytes(c) - 1)))
329 len = round_down(len, block_bytes(c));
331 mempool_free(workspace, &c->lz4_workspace_pool);
341 case BCH_COMPRESSION_GZIP: {
345 workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
347 GFP_NOIO|__GFP_NOWARN);
349 mutex_lock(&c->zlib_workspace_lock);
350 workspace = c->zlib_workspace;
353 strm.next_in = src_data.b;
354 strm.avail_in = min(src->bi_iter.bi_size,
355 dst->bi_iter.bi_size);
356 strm.next_out = dst_data.b;
357 strm.avail_out = dst->bi_iter.bi_size;
358 zlib_set_workspace(&strm, workspace);
359 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
360 Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
363 ret = zlib_deflate(&strm, Z_FINISH);
364 if (ret != Z_STREAM_END) {
369 ret = zlib_deflateEnd(&strm);
377 if (workspace == c->zlib_workspace)
378 mutex_unlock(&c->zlib_workspace_lock);
385 *dst_len = strm.total_out;
386 *src_len = strm.total_in;
393 /* Didn't get smaller: */
394 if (round_up(*dst_len, block_bytes(c)) >= *src_len)
397 pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
399 memset(dst_data.b + *dst_len, 0, pad);
402 if (dst_data.type != BB_NONE)
403 memcpy_to_bio(dst, dst->bi_iter, dst_data.b);
405 BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
406 BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
407 BUG_ON(*dst_len & (block_bytes(c) - 1));
408 BUG_ON(*src_len & (block_bytes(c) - 1));
410 bio_unmap_or_unbounce(c, src_data);
411 bio_unmap_or_unbounce(c, dst_data);
412 return compression_type;
414 compression_type = 0;
418 unsigned bch2_bio_compress(struct bch_fs *c,
419 struct bio *dst, size_t *dst_len,
420 struct bio *src, size_t *src_len,
421 unsigned compression_type)
423 unsigned orig_dst = dst->bi_iter.bi_size;
424 unsigned orig_src = src->bi_iter.bi_size;
426 /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
427 src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
428 c->sb.encoded_extent_max << 9);
429 /* Don't generate a bigger output than input: */
430 dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
433 __bio_compress(c, dst, dst_len, src, src_len, compression_type);
435 dst->bi_iter.bi_size = orig_dst;
436 src->bi_iter.bi_size = orig_src;
437 return compression_type;
440 /* doesn't write superblock: */
441 int bch2_check_set_has_compressed_data(struct bch_fs *c,
442 unsigned compression_type)
444 switch (compression_type) {
445 case BCH_COMPRESSION_OPT_NONE:
447 case BCH_COMPRESSION_OPT_LZ4:
448 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
451 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
453 case BCH_COMPRESSION_OPT_GZIP:
454 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
457 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
463 return bch2_fs_compress_init(c);
466 void bch2_fs_compress_exit(struct bch_fs *c)
468 vfree(c->zlib_workspace);
469 mempool_exit(&c->lz4_workspace_pool);
470 mempool_exit(&c->compression_bounce[WRITE]);
471 mempool_exit(&c->compression_bounce[READ]);
474 #define COMPRESSION_WORKSPACE_SIZE \
475 max_t(size_t, zlib_inflate_workspacesize(), \
476 zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
478 int bch2_fs_compress_init(struct bch_fs *c)
480 unsigned order = get_order(c->sb.encoded_extent_max << 9);
483 if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
484 !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
487 if (!mempool_initialized(&c->compression_bounce[READ])) {
488 ret = mempool_init_page_pool(&c->compression_bounce[READ],
494 if (!mempool_initialized(&c->compression_bounce[WRITE])) {
495 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
501 if (!mempool_initialized(&c->lz4_workspace_pool) &&
502 bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
503 ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
504 1, LZ4_MEM_COMPRESS);
509 if (!c->zlib_workspace &&
510 bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
511 c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
512 if (!c->zlib_workspace)