8 #include <linux/zlib.h>
18 static void *__bounce_alloc(struct bch_fs *c, unsigned size,
19 unsigned *bounced, int direction)
23 *bounced = BOUNCED_KMALLOCED;
24 data = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
28 *bounced = BOUNCED_MEMPOOLED;
29 data = mempool_alloc(&c->compression_bounce[direction], GFP_NOWAIT);
31 return page_address(data);
33 *bounced = BOUNCED_VMALLOCED;
38 *bounced = BOUNCED_MEMPOOLED;
39 data = mempool_alloc(&c->compression_bounce[direction], GFP_NOIO);
40 return page_address(data);
43 static void *__bio_map_or_bounce(struct bch_fs *c,
44 struct bio *bio, struct bvec_iter start,
45 unsigned *bounced, int direction)
48 struct bvec_iter iter;
49 unsigned nr_pages = 0;
50 struct page *stack_pages[16];
51 struct page **pages = NULL;
53 unsigned prev_end = PAGE_SIZE;
56 BUG_ON(bvec_iter_sectors(start) > BCH_ENCODED_EXTENT_MAX);
58 #ifndef CONFIG_HIGHMEM
59 *bounced = BOUNCED_CONTIG;
61 __bio_for_each_contig_segment(bv, bio, iter, start) {
62 if (bv.bv_len == start.bi_size)
63 return page_address(bv.bv_page) + bv.bv_offset;
66 *bounced = BOUNCED_MAPPED;
68 __bio_for_each_segment(bv, bio, iter, start) {
69 if ((!first && bv.bv_offset) ||
70 prev_end != PAGE_SIZE)
73 prev_end = bv.bv_offset + bv.bv_len;
77 BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
79 pages = nr_pages > ARRAY_SIZE(stack_pages)
80 ? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
86 __bio_for_each_segment(bv, bio, iter, start)
87 pages[nr_pages++] = bv.bv_page;
89 data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
90 if (pages != stack_pages)
93 return data + bio_iter_offset(bio, start);
95 data = __bounce_alloc(c, start.bi_size, bounced, direction);
97 if (direction == READ)
98 memcpy_from_bio(data, bio, start);
103 static void *bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
104 unsigned *bounced, int direction)
106 return __bio_map_or_bounce(c, bio, bio->bi_iter, bounced, direction);
109 static void bio_unmap_or_unbounce(struct bch_fs *c, void *data,
110 unsigned bounced, int direction)
117 vunmap((void *) ((unsigned long) data & PAGE_MASK));
119 case BOUNCED_KMALLOCED:
122 case BOUNCED_VMALLOCED:
125 case BOUNCED_MEMPOOLED:
126 mempool_free(virt_to_page(data), &c->compression_bounce[direction]);
131 static inline void zlib_set_workspace(z_stream *strm, void *workspace)
134 strm->workspace = workspace;
138 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
139 void *dst_data, struct bch_extent_crc128 crc)
141 void *src_data = NULL;
142 unsigned src_bounced;
143 size_t src_len = src->bi_iter.bi_size;
144 size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
147 src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
149 switch (crc.compression_type) {
150 case BCH_COMPRESSION_LZ4:
151 ret = lz4_decompress(src_data, &src_len,
158 case BCH_COMPRESSION_GZIP: {
162 workspace = kmalloc(zlib_inflate_workspacesize(),
163 GFP_NOIO|__GFP_NOWARN);
165 mutex_lock(&c->zlib_workspace_lock);
166 workspace = c->zlib_workspace;
169 strm.next_in = src_data;
170 strm.avail_in = src_len;
171 strm.next_out = dst_data;
172 strm.avail_out = dst_len;
173 zlib_set_workspace(&strm, workspace);
174 zlib_inflateInit2(&strm, -MAX_WBITS);
176 ret = zlib_inflate(&strm, Z_FINISH);
178 if (workspace == c->zlib_workspace)
179 mutex_unlock(&c->zlib_workspace_lock);
183 if (ret != Z_STREAM_END) {
194 bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
198 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
199 unsigned live_data_sectors,
200 struct bch_extent_crc128 crc)
202 void *dst_data = NULL;
203 size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
206 BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
209 dst_data = kmalloc(dst_len, GFP_NOIO|__GFP_NOWARN);
211 dst_data = vmalloc(dst_len);
216 ret = __bio_uncompress(c, bio, dst_data, crc);
220 while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
221 struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
223 bv->bv_page = alloc_page(GFP_NOIO);
227 bv->bv_len = PAGE_SIZE;
232 bio->bi_iter.bi_size = live_data_sectors << 9;
234 memcpy_to_bio(bio, bio->bi_iter, dst_data + (crc.offset << 9));
240 * We already allocated from mempool, we can't allocate from it again
241 * without freeing the pages we already allocated or else we could
245 bch2_bio_free_pages_pool(c, bio);
246 bch2_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
250 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
251 struct bio *dst, struct bvec_iter dst_iter,
252 struct bch_extent_crc128 crc)
254 void *dst_data = NULL;
255 unsigned dst_bounced;
256 size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
259 dst_data = dst_len == dst_iter.bi_size
260 ? __bio_map_or_bounce(c, dst, dst_iter, &dst_bounced, WRITE)
261 : __bounce_alloc(c, dst_len, &dst_bounced, WRITE);
263 ret = __bio_uncompress(c, src, dst_data, crc);
268 memcpy_to_bio(dst, dst_iter, dst_data + (crc.offset << 9));
270 bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
274 static int __bio_compress(struct bch_fs *c,
275 struct bio *dst, size_t *dst_len,
276 struct bio *src, size_t *src_len,
277 unsigned compression_type)
279 void *src_data = NULL, *dst_data = NULL;
280 unsigned src_bounced, dst_bounced, pad;
283 dst_data = bio_map_or_bounce(c, dst, &dst_bounced, WRITE);
284 src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
286 switch (compression_type) {
287 case BCH_COMPRESSION_LZ4: {
290 *dst_len = dst->bi_iter.bi_size;
291 *src_len = src->bi_iter.bi_size;
293 workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
295 while (*src_len > block_bytes(c) &&
296 (ret = lz4_compress(src_data, *src_len,
300 * On error, the compressed data was bigger than
301 * dst_len, and -ret is the amount of data we were able
302 * to compress - round down to nearest block and try
306 BUG_ON(-ret >= *src_len);
308 *src_len = round_down(-ret, block_bytes(c));
311 mempool_free(workspace, &c->lz4_workspace_pool);
317 case BCH_COMPRESSION_GZIP: {
321 workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
323 GFP_NOIO|__GFP_NOWARN);
325 mutex_lock(&c->zlib_workspace_lock);
326 workspace = c->zlib_workspace;
329 strm.next_in = src_data;
330 strm.avail_in = min(src->bi_iter.bi_size,
331 dst->bi_iter.bi_size);
332 strm.next_out = dst_data;
333 strm.avail_out = dst->bi_iter.bi_size;
334 zlib_set_workspace(&strm, workspace);
335 zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
336 Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
339 ret = zlib_deflate(&strm, Z_FINISH);
340 if (ret != Z_STREAM_END) {
345 ret = zlib_deflateEnd(&strm);
353 if (workspace == c->zlib_workspace)
354 mutex_unlock(&c->zlib_workspace_lock);
361 *dst_len = strm.total_out;
362 *src_len = strm.total_in;
370 BUG_ON(*dst_len > dst->bi_iter.bi_size);
372 BUG_ON(*src_len & (block_bytes(c) - 1));
373 BUG_ON(*src_len > src->bi_iter.bi_size);
375 /* Didn't get smaller: */
376 if (round_up(*dst_len, block_bytes(c)) >= *src_len) {
381 pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
383 memset(dst_data + *dst_len, 0, pad);
387 memcpy_to_bio(dst, dst->bi_iter, dst_data);
389 bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
390 bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
394 void bch2_bio_compress(struct bch_fs *c,
395 struct bio *dst, size_t *dst_len,
396 struct bio *src, size_t *src_len,
397 unsigned *compression_type)
399 unsigned orig_dst = dst->bi_iter.bi_size;
400 unsigned orig_src = src->bi_iter.bi_size;
402 /* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
403 src->bi_iter.bi_size =
404 min(src->bi_iter.bi_size, BCH_ENCODED_EXTENT_MAX << 9);
406 /* Don't generate a bigger output than input: */
407 dst->bi_iter.bi_size =
408 min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
410 /* If it's only one block, don't bother trying to compress: */
411 if (*compression_type != BCH_COMPRESSION_NONE &&
412 bio_sectors(src) > c->sb.block_size &&
413 !__bio_compress(c, dst, dst_len, src, src_len, *compression_type))
416 /* If compressing failed (didn't get smaller), just copy: */
417 *compression_type = BCH_COMPRESSION_NONE;
418 *dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
419 bio_copy_data(dst, src);
421 dst->bi_iter.bi_size = orig_dst;
422 src->bi_iter.bi_size = orig_src;
425 /* doesn't write superblock: */
426 int bch2_check_set_has_compressed_data(struct bch_fs *c,
427 unsigned compression_type)
429 switch (compression_type) {
430 case BCH_COMPRESSION_NONE:
432 case BCH_COMPRESSION_LZ4:
433 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
436 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
438 case BCH_COMPRESSION_GZIP:
439 if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
442 bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
446 return bch2_fs_compress_init(c);
449 void bch2_fs_compress_exit(struct bch_fs *c)
451 vfree(c->zlib_workspace);
452 mempool_exit(&c->lz4_workspace_pool);
453 mempool_exit(&c->compression_bounce[WRITE]);
454 mempool_exit(&c->compression_bounce[READ]);
457 #define COMPRESSION_WORKSPACE_SIZE \
458 max_t(size_t, zlib_inflate_workspacesize(), \
459 zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
461 int bch2_fs_compress_init(struct bch_fs *c)
463 unsigned order = get_order(BCH_ENCODED_EXTENT_MAX << 9);
466 if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
467 !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
470 if (!mempool_initialized(&c->compression_bounce[READ])) {
471 ret = mempool_init_page_pool(&c->compression_bounce[READ],
477 if (!mempool_initialized(&c->compression_bounce[WRITE])) {
478 ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
484 if (!mempool_initialized(&c->lz4_workspace_pool) &&
485 bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
486 ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
487 1, LZ4_MEM_COMPRESS);
492 if (!c->zlib_workspace &&
493 bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
494 c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
495 if (!c->zlib_workspace)