X-Git-Url: https://git.sesse.net/?a=blobdiff_plain;f=libbcachefs%2Fcompress.c;h=1410365a889156450c78da9165bdb146872370ed;hb=HEAD;hp=7b45bb789b3684633f3485271d3c132c32cee6bb;hpb=22291ae84a029d65334d1a90b67b5031f45cd540;p=bcachefs-tools-debian

diff --git a/libbcachefs/compress.c b/libbcachefs/compress.c
index 7b45bb7..1410365 100644
--- a/libbcachefs/compress.c
+++ b/libbcachefs/compress.c
@@ -1,12 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0
 #include "bcachefs.h"
+#include "checksum.h"
 #include "compress.h"
 #include "extents.h"
-#include "io.h"
 #include "super-io.h"
 
-#include "lz4.h"
 #include <linux/lz4.h>
 #include <linux/zlib.h>
+#include <linux/zstd.h>
 
 /* Bounce buffer: */
 struct bbuf {
@@ -15,7 +16,6 @@ struct bbuf {
 		BB_NONE,
 		BB_VMAP,
 		BB_KMALLOC,
-		BB_VMALLOC,
 		BB_MEMPOOL,
 	}		type;
 	int		rw;
@@ -25,27 +25,35 @@ static struct bbuf __bounce_alloc(struct bch_fs *c, unsigned size, int rw)
 {
 	void *b;
 
-	BUG_ON(size > c->sb.encoded_extent_max << 9);
+	BUG_ON(size > c->opts.encoded_extent_max);
 
-	b = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
+	b = kmalloc(size, GFP_NOFS|__GFP_NOWARN);
 	if (b)
 		return (struct bbuf) { .b = b, .type = BB_KMALLOC, .rw = rw };
 
-	b = mempool_alloc(&c->compression_bounce[rw], GFP_NOWAIT);
-	b = b ? page_address(b) : NULL;
+	b = mempool_alloc(&c->compression_bounce[rw], GFP_NOFS);
 	if (b)
 		return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
 
-	b = vmalloc(size);
-	if (b)
-		return (struct bbuf) { .b = b, .type = BB_VMALLOC, .rw = rw };
+	BUG();
+}
 
-	b = mempool_alloc(&c->compression_bounce[rw], GFP_NOIO);
-	b = b ? page_address(b) : NULL;
-	if (b)
-		return (struct bbuf) { .b = b, .type = BB_MEMPOOL, .rw = rw };
+static bool bio_phys_contig(struct bio *bio, struct bvec_iter start)
+{
+	struct bio_vec bv;
+	struct bvec_iter iter;
+	void *expected_start = NULL;
 
-	BUG();
+	__bio_for_each_bvec(bv, bio, iter, start) {
+		if (expected_start &&
+		    expected_start != page_address(bv.bv_page) + bv.bv_offset)
+			return false;
+
+		expected_start = page_address(bv.bv_page) +
+			bv.bv_offset + bv.bv_len;
+	}
+
+	return true;
 }
 
 static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
@@ -57,34 +65,35 @@ static struct bbuf __bio_map_or_bounce(struct bch_fs *c, struct bio *bio,
 	unsigned nr_pages = 0;
 	struct page *stack_pages[16];
 	struct page **pages = NULL;
-	bool first = true;
-	unsigned prev_end = PAGE_SIZE;
 	void *data;
 
-	BUG_ON(bvec_iter_sectors(start) > c->sb.encoded_extent_max);
+	BUG_ON(start.bi_size > c->opts.encoded_extent_max);
 
-#ifndef CONFIG_HIGHMEM
-	__bio_for_each_contig_segment(bv, bio, iter, start) {
-		if (bv.bv_len == start.bi_size)
-			return (struct bbuf) {
-				.b = page_address(bv.bv_page) + bv.bv_offset,
-				.type = BB_NONE, .rw = rw
-			};
-	}
-#endif
+	if (!PageHighMem(bio_iter_page(bio, start)) &&
+	    bio_phys_contig(bio, start))
+		return (struct bbuf) {
+			.b = page_address(bio_iter_page(bio, start)) +
+				bio_iter_offset(bio, start),
+			.type = BB_NONE, .rw = rw
+		};
+
+	/* check if we can map the pages contiguously: */
 	__bio_for_each_segment(bv, bio, iter, start) {
-		if ((!first && bv.bv_offset) ||
-		    prev_end != PAGE_SIZE)
+		if (iter.bi_size != start.bi_size &&
+		    bv.bv_offset)
+			goto bounce;
+
+		if (bv.bv_len < iter.bi_size &&
+		    bv.bv_offset + bv.bv_len < PAGE_SIZE)
 			goto bounce;
 
-		prev_end = bv.bv_offset + bv.bv_len;
 		nr_pages++;
 	}
 
 	BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
 
 	pages = nr_pages > ARRAY_SIZE(stack_pages)
-		? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
+		? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOFS)
 		: stack_pages;
 	if (!pages)
 		goto bounce;
@@ -127,12 +136,8 @@ static void bio_unmap_or_unbounce(struct bch_fs *c, struct bbuf buf)
 	case BB_KMALLOC:
 		kfree(buf.b);
 		break;
-	case BB_VMALLOC:
-		vfree(buf.b);
-		break;
 	case BB_MEMPOOL:
-		mempool_free(virt_to_page(buf.b),
-			     &c->compression_bounce[buf.rw]);
+		mempool_free(buf.b, &c->compression_bounce[buf.rw]);
 		break;
 	}
 }
@@ -145,132 +150,130 @@ static inline void zlib_set_workspace(z_stream *strm, void *workspace)
 }
 
 static int __bio_uncompress(struct bch_fs *c, struct bio *src,
-			    void *dst_data, struct bch_extent_crc128 crc)
+			    void *dst_data, struct bch_extent_crc_unpacked crc)
 {
 	struct bbuf src_data = { NULL };
 	size_t src_len = src->bi_iter.bi_size;
-	size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
+	size_t dst_len = crc.uncompressed_size << 9;
+	void *workspace;
 	int ret;
 
 	src_data = bio_map_or_bounce(c, src, READ);
 
 	switch (crc.compression_type) {
-	case BCH_COMPRESSION_LZ4_OLD:
-		ret = bch2_lz4_decompress(src_data.b, &src_len,
-				     dst_data, dst_len);
-		if (ret) {
-			ret = -EIO;
-			goto err;
-		}
-		break;
-	case BCH_COMPRESSION_LZ4:
+	case BCH_COMPRESSION_TYPE_lz4_old:
+	case BCH_COMPRESSION_TYPE_lz4:
 		ret = LZ4_decompress_safe_partial(src_data.b, dst_data,
 						  src_len, dst_len, dst_len);
-		if (ret != dst_len) {
-			ret = -EIO;
+		if (ret != dst_len)
 			goto err;
-		}
 		break;
-	case BCH_COMPRESSION_GZIP: {
-		void *workspace;
-		z_stream strm;
-
-		workspace = kmalloc(zlib_inflate_workspacesize(),
-				    GFP_NOIO|__GFP_NOWARN);
-		if (!workspace) {
-			mutex_lock(&c->zlib_workspace_lock);
-			workspace = c->zlib_workspace;
-		}
+	case BCH_COMPRESSION_TYPE_gzip: {
+		z_stream strm = {
+			.next_in	= src_data.b,
+			.avail_in	= src_len,
+			.next_out	= dst_data,
+			.avail_out	= dst_len,
+		};
+
+		workspace = mempool_alloc(&c->decompress_workspace, GFP_NOFS);
 
-		strm.next_in	= src_data.b;
-		strm.avail_in	= src_len;
-		strm.next_out	= dst_data;
-		strm.avail_out	= dst_len;
 		zlib_set_workspace(&strm, workspace);
 		zlib_inflateInit2(&strm, -MAX_WBITS);
-
 		ret = zlib_inflate(&strm, Z_FINISH);
 
-		if (workspace == c->zlib_workspace)
-			mutex_unlock(&c->zlib_workspace_lock);
-		else
-			kfree(workspace);
+		mempool_free(workspace, &c->decompress_workspace);
 
-		if (ret != Z_STREAM_END) {
-			ret = -EIO;
+		if (ret != Z_STREAM_END)
+			goto err;
+		break;
+	}
+	case BCH_COMPRESSION_TYPE_zstd: {
+		ZSTD_DCtx *ctx;
+		size_t real_src_len = le32_to_cpup(src_data.b);
+
+		if (real_src_len > src_len - 4)
+			goto err;
+
+		workspace = mempool_alloc(&c->decompress_workspace, GFP_NOFS);
+		ctx = zstd_init_dctx(workspace, zstd_dctx_workspace_bound());
+
+		ret = zstd_decompress_dctx(ctx,
+				dst_data,	dst_len,
+				src_data.b + 4, real_src_len);
+
+		mempool_free(workspace, &c->decompress_workspace);
+
+		if (ret != dst_len)
 			goto err;
-		}
 		break;
 	}
 	default:
 		BUG();
 	}
 	ret = 0;
-err:
+out:
 	bio_unmap_or_unbounce(c, src_data);
 	return ret;
+err:
+	ret = -EIO;
+	goto out;
 }
 
 int bch2_bio_uncompress_inplace(struct bch_fs *c, struct bio *bio,
-			       unsigned live_data_sectors,
-			       struct bch_extent_crc128 crc)
+				struct bch_extent_crc_unpacked *crc)
 {
-	struct bbuf dst_data = { NULL };
-	size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
-	int ret = -ENOMEM;
+	struct bbuf data = { NULL };
+	size_t dst_len = crc->uncompressed_size << 9;
 
-	BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
+	/* bio must own its pages: */
+	BUG_ON(!bio->bi_vcnt);
+	BUG_ON(DIV_ROUND_UP(crc->live_size, PAGE_SECTORS) > bio->bi_max_vecs);
 
-	if (crc_uncompressed_size(NULL, &crc) > c->sb.encoded_extent_max ||
-	    crc_compressed_size(NULL, &crc)   > c->sb.encoded_extent_max)
+	if (crc->uncompressed_size << 9	> c->opts.encoded_extent_max ||
+	    crc->compressed_size << 9	> c->opts.encoded_extent_max) {
+		bch_err(c, "error rewriting existing data: extent too big");
 		return -EIO;
+	}
 
-	dst_data = __bounce_alloc(c, dst_len, WRITE);
-
-	ret = __bio_uncompress(c, bio, dst_data.b, crc);
-	if (ret)
-		goto err;
-
-	while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
-		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
-
-		bv->bv_page = alloc_page(GFP_NOIO);
-		if (!bv->bv_page)
-			goto use_mempool;
+	data = __bounce_alloc(c, dst_len, WRITE);
 
-		bv->bv_len = PAGE_SIZE;
-		bv->bv_offset = 0;
-		bio->bi_vcnt++;
+	if (__bio_uncompress(c, bio, data.b, *crc)) {
+		if (!c->opts.no_data_io)
+			bch_err(c, "error rewriting existing data: decompression error");
+		bio_unmap_or_unbounce(c, data);
+		return -EIO;
 	}
 
-	bio->bi_iter.bi_size = live_data_sectors << 9;
-copy_data:
-	memcpy_to_bio(bio, bio->bi_iter, dst_data.b + (crc.offset << 9));
-err:
-	bio_unmap_or_unbounce(c, dst_data);
-	return ret;
-use_mempool:
 	/*
-	 * We already allocated from mempool, we can't allocate from it again
-	 * without freeing the pages we already allocated or else we could
-	 * deadlock:
+	 * XXX: don't have a good way to assert that the bio was allocated with
+	 * enough space, we depend on bch2_move_extent doing the right thing
 	 */
+	bio->bi_iter.bi_size = crc->live_size << 9;
 
-	bch2_bio_free_pages_pool(c, bio);
-	bch2_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
-	goto copy_data;
+	memcpy_to_bio(bio, bio->bi_iter, data.b + (crc->offset << 9));
+
+	crc->csum_type		= 0;
+	crc->compression_type	= 0;
+	crc->compressed_size	= crc->live_size;
+	crc->uncompressed_size	= crc->live_size;
+	crc->offset		= 0;
+	crc->csum		= (struct bch_csum) { 0, 0 };
+
+	bio_unmap_or_unbounce(c, data);
+	return 0;
 }
 
 int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
 		       struct bio *dst, struct bvec_iter dst_iter,
-		       struct bch_extent_crc128 crc)
+		       struct bch_extent_crc_unpacked crc)
 {
 	struct bbuf dst_data = { NULL };
-	size_t dst_len = crc_uncompressed_size(NULL, &crc) << 9;
-	int ret = -ENOMEM;
+	size_t dst_len = crc.uncompressed_size << 9;
+	int ret;
 
-	if (crc_uncompressed_size(NULL, &crc) > c->sb.encoded_extent_max ||
-	    crc_compressed_size(NULL, &crc)   > c->sb.encoded_extent_max)
+	if (crc.uncompressed_size << 9	> c->opts.encoded_extent_max ||
+	    crc.compressed_size << 9	> c->opts.encoded_extent_max)
 		return -EIO;
 
 	dst_data = dst_len == dst_iter.bi_size
@@ -281,116 +284,173 @@ int bch2_bio_uncompress(struct bch_fs *c, struct bio *src,
 	if (ret)
 		goto err;
 
-	if (dst_data.type != BB_NONE)
+	if (dst_data.type != BB_NONE &&
+	    dst_data.type != BB_VMAP)
 		memcpy_to_bio(dst, dst_iter, dst_data.b + (crc.offset << 9));
 err:
 	bio_unmap_or_unbounce(c, dst_data);
 	return ret;
 }
 
-static int __bio_compress(struct bch_fs *c,
-			  struct bio *dst, size_t *dst_len,
-			  struct bio *src, size_t *src_len,
-			  unsigned *compression_type)
+static int attempt_compress(struct bch_fs *c,
+			    void *workspace,
+			    void *dst, size_t dst_len,
+			    void *src, size_t src_len,
+			    struct bch_compression_opt compression)
+{
+	enum bch_compression_type compression_type =
+		__bch2_compression_opt_to_type[compression.type];
+
+	switch (compression_type) {
+	case BCH_COMPRESSION_TYPE_lz4:
+		if (compression.level < LZ4HC_MIN_CLEVEL) {
+			int len = src_len;
+			int ret = LZ4_compress_destSize(
+					src,		dst,
+					&len,		dst_len,
+					workspace);
+			if (len < src_len)
+				return -len;
+
+			return ret;
+		} else {
+			int ret = LZ4_compress_HC(
+					src,		dst,
+					src_len,	dst_len,
+					compression.level,
+					workspace);
+
+			return ret ?: -1;
+		}
+	case BCH_COMPRESSION_TYPE_gzip: {
+		z_stream strm = {
+			.next_in	= src,
+			.avail_in	= src_len,
+			.next_out	= dst,
+			.avail_out	= dst_len,
+		};
+
+		zlib_set_workspace(&strm, workspace);
+		zlib_deflateInit2(&strm,
+				  compression.level
+				  ? clamp_t(unsigned, compression.level,
+					    Z_BEST_SPEED, Z_BEST_COMPRESSION)
+				  : Z_DEFAULT_COMPRESSION,
+				  Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
+				  Z_DEFAULT_STRATEGY);
+
+		if (zlib_deflate(&strm, Z_FINISH) != Z_STREAM_END)
+			return 0;
+
+		if (zlib_deflateEnd(&strm) != Z_OK)
+			return 0;
+
+		return strm.total_out;
+	}
+	case BCH_COMPRESSION_TYPE_zstd: {
+		/*
+		 * rescale:
+		 * zstd max compression level is 22, our max level is 15
+		 */
+		unsigned level = min((compression.level * 3) / 2, zstd_max_clevel());
+		ZSTD_parameters params = zstd_get_params(level, c->opts.encoded_extent_max);
+		ZSTD_CCtx *ctx = zstd_init_cctx(workspace, c->zstd_workspace_size);
+
+		/*
+		 * ZSTD requires that when we decompress we pass in the exact
+		 * compressed size - rounding it up to the nearest sector
+		 * doesn't work, so we use the first 4 bytes of the buffer for
+		 * that.
+		 *
+		 * Additionally, the ZSTD code seems to have a bug where it will
+		 * write just past the end of the buffer - so subtract a fudge
+		 * factor (7 bytes) from the dst buffer size to account for
+		 * that.
+		 */
+		size_t len = zstd_compress_cctx(ctx,
+				dst + 4,	dst_len - 4 - 7,
+				src,		src_len,
+				&params);
+		if (zstd_is_error(len))
+			return 0;
+
+		*((__le32 *) dst) = cpu_to_le32(len);
+		return len + 4;
+	}
+	default:
+		BUG();
+	}
+}
+
+static unsigned __bio_compress(struct bch_fs *c,
+			       struct bio *dst, size_t *dst_len,
+			       struct bio *src, size_t *src_len,
+			       struct bch_compression_opt compression)
 {
 	struct bbuf src_data = { NULL }, dst_data = { NULL };
+	void *workspace;
+	enum bch_compression_type compression_type =
+		__bch2_compression_opt_to_type[compression.type];
 	unsigned pad;
 	int ret = 0;
 
-	dst_data = bio_map_or_bounce(c, dst, WRITE);
-	src_data = bio_map_or_bounce(c, src, READ);
+	BUG_ON(compression_type >= BCH_COMPRESSION_TYPE_NR);
+	BUG_ON(!mempool_initialized(&c->compress_workspace[compression_type]));
 
-	switch (*compression_type) {
-	case BCH_COMPRESSION_LZ4_OLD:
-		*compression_type = BCH_COMPRESSION_LZ4;
+	/* If it's only one block, don't bother trying to compress: */
+	if (src->bi_iter.bi_size <= c->opts.block_size)
+		return BCH_COMPRESSION_TYPE_incompressible;
 
-	case BCH_COMPRESSION_LZ4: {
-		void *workspace;
-		int len = src->bi_iter.bi_size;
+	dst_data = bio_map_or_bounce(c, dst, WRITE);
+	src_data = bio_map_or_bounce(c, src, READ);
 
-		workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
+	workspace = mempool_alloc(&c->compress_workspace[compression_type], GFP_NOFS);
 
-		while (1) {
-			if (len <= block_bytes(c)) {
-				ret = 0;
-				break;
-			}
+	*src_len = src->bi_iter.bi_size;
+	*dst_len = dst->bi_iter.bi_size;
 
-			ret = LZ4_compress_destSize(
-					src_data.b,	dst_data.b,
-					&len,		dst->bi_iter.bi_size,
-					workspace);
-			if (ret >= len) {
-				/* uncompressible: */
-				ret = 0;
-				break;
-			}
-
-			if (!(len & (block_bytes(c) - 1)))
-				break;
-			len = round_down(len, block_bytes(c));
+	/*
+	 * XXX: this algorithm sucks when the compression code doesn't tell us
+	 * how much would fit, like LZ4 does:
+	 */
+	while (1) {
+		if (*src_len <= block_bytes(c)) {
+			ret = -1;
+			break;
 		}
-		mempool_free(workspace, &c->lz4_workspace_pool);
-
-		if (!ret)
-			goto err;
 
-		*src_len = len;
-		*dst_len = ret;
-		ret = 0;
-		break;
-	}
-	case BCH_COMPRESSION_GZIP: {
-		void *workspace;
-		z_stream strm;
-
-		workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
-							       DEF_MEM_LEVEL),
-				    GFP_NOIO|__GFP_NOWARN);
-		if (!workspace) {
-			mutex_lock(&c->zlib_workspace_lock);
-			workspace = c->zlib_workspace;
+		ret = attempt_compress(c, workspace,
+				       dst_data.b,	*dst_len,
+				       src_data.b,	*src_len,
+				       compression);
+		if (ret > 0) {
+			*dst_len = ret;
+			ret = 0;
+			break;
 		}
 
-		strm.next_in	= src_data.b;
-		strm.avail_in	= min(src->bi_iter.bi_size,
-				      dst->bi_iter.bi_size);
-		strm.next_out	= dst_data.b;
-		strm.avail_out	= dst->bi_iter.bi_size;
-		zlib_set_workspace(&strm, workspace);
-		zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
-				  Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
-				  Z_DEFAULT_STRATEGY);
-
-		ret = zlib_deflate(&strm, Z_FINISH);
-		if (ret != Z_STREAM_END) {
-			ret = -EIO;
-			goto zlib_err;
+		/* Didn't fit: should we retry with a smaller amount?  */
+		if (*src_len <= *dst_len) {
+			ret = -1;
+			break;
 		}
 
-		ret = zlib_deflateEnd(&strm);
-		if (ret != Z_OK) {
-			ret = -EIO;
-			goto zlib_err;
-		}
+		/*
+		 * If ret is negative, it's a hint as to how much data would fit
+		 */
+		BUG_ON(-ret >= *src_len);
 
-		ret = 0;
-zlib_err:
-		if (workspace == c->zlib_workspace)
-			mutex_unlock(&c->zlib_workspace_lock);
+		if (ret < 0)
+			*src_len = -ret;
 		else
-			kfree(workspace);
+			*src_len -= (*src_len - *dst_len) / 2;
+		*src_len = round_down(*src_len, block_bytes(c));
+	}
 
-		if (ret)
-			goto err;
+	mempool_free(workspace, &c->compress_workspace[compression_type]);
 
-		*dst_len = strm.total_out;
-		*src_len = strm.total_in;
-		break;
-	}
-	default:
-		BUG();
-	}
+	if (ret)
+		goto err;
 
 	/* Didn't get smaller: */
 	if (round_up(*dst_len, block_bytes(c)) >= *src_len)
@@ -401,127 +461,267 @@ zlib_err:
 	memset(dst_data.b + *dst_len, 0, pad);
 	*dst_len += pad;
 
-	if (dst_data.type != BB_NONE)
+	if (dst_data.type != BB_NONE &&
+	    dst_data.type != BB_VMAP)
 		memcpy_to_bio(dst, dst->bi_iter, dst_data.b);
+
+	BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
+	BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
+	BUG_ON(*dst_len & (block_bytes(c) - 1));
+	BUG_ON(*src_len & (block_bytes(c) - 1));
+	ret = compression_type;
 out:
 	bio_unmap_or_unbounce(c, src_data);
 	bio_unmap_or_unbounce(c, dst_data);
 	return ret;
 err:
-	ret = -1;
+	ret = BCH_COMPRESSION_TYPE_incompressible;
 	goto out;
 }
 
-void bch2_bio_compress(struct bch_fs *c,
-		       struct bio *dst, size_t *dst_len,
-		       struct bio *src, size_t *src_len,
-		       unsigned *compression_type)
+unsigned bch2_bio_compress(struct bch_fs *c,
+			   struct bio *dst, size_t *dst_len,
+			   struct bio *src, size_t *src_len,
+			   unsigned compression_opt)
 {
 	unsigned orig_dst = dst->bi_iter.bi_size;
 	unsigned orig_src = src->bi_iter.bi_size;
+	unsigned compression_type;
 
 	/* Don't consume more than BCH_ENCODED_EXTENT_MAX from @src: */
 	src->bi_iter.bi_size = min_t(unsigned, src->bi_iter.bi_size,
-				     c->sb.encoded_extent_max << 9);
-
+				     c->opts.encoded_extent_max);
 	/* Don't generate a bigger output than input: */
-	dst->bi_iter.bi_size =
-		min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
+	dst->bi_iter.bi_size = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
+
+	compression_type =
+		__bio_compress(c, dst, dst_len, src, src_len,
+			       bch2_compression_decode(compression_opt));
 
-	/* If it's only one block, don't bother trying to compress: */
-	if (*compression_type != BCH_COMPRESSION_NONE &&
-	    bio_sectors(src) > c->opts.block_size &&
-	    !__bio_compress(c, dst, dst_len, src, src_len, compression_type))
-		goto out;
-
-	/* If compressing failed (didn't get smaller), just copy: */
-	*compression_type = BCH_COMPRESSION_NONE;
-	*dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
-	bio_copy_data(dst, src);
-out:
 	dst->bi_iter.bi_size = orig_dst;
 	src->bi_iter.bi_size = orig_src;
-
-	BUG_ON(!*dst_len || *dst_len > dst->bi_iter.bi_size);
-	BUG_ON(!*src_len || *src_len > src->bi_iter.bi_size);
-	BUG_ON(*dst_len & (block_bytes(c) - 1));
-	BUG_ON(*src_len & (block_bytes(c) - 1));
+	return compression_type;
 }
 
-/* doesn't write superblock: */
-int bch2_check_set_has_compressed_data(struct bch_fs *c,
-				      unsigned compression_type)
+static int __bch2_fs_compress_init(struct bch_fs *, u64);
+
+#define BCH_FEATURE_none	0
+
+static const unsigned bch2_compression_opt_to_feature[] = {
+#define x(t, n) [BCH_COMPRESSION_OPT_##t] = BCH_FEATURE_##t,
+	BCH_COMPRESSION_OPTS()
+#undef x
+};
+
+#undef BCH_FEATURE_none
+
+static int __bch2_check_set_has_compressed_data(struct bch_fs *c, u64 f)
 {
-	switch (compression_type) {
-	case BCH_COMPRESSION_OPT_NONE:
+	int ret = 0;
+
+	if ((c->sb.features & f) == f)
 		return 0;
-	case BCH_COMPRESSION_OPT_LZ4:
-		if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4))
-			return 0;
 
-		bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_LZ4);
-		break;
-	case BCH_COMPRESSION_OPT_GZIP:
-		if (bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
-			return 0;
+	mutex_lock(&c->sb_lock);
 
-		bch2_sb_set_feature(c->disk_sb, BCH_FEATURE_GZIP);
-		break;
-	default:
-		BUG();
+	if ((c->sb.features & f) == f) {
+		mutex_unlock(&c->sb_lock);
+		return 0;
 	}
 
-	return bch2_fs_compress_init(c);
+	ret = __bch2_fs_compress_init(c, c->sb.features|f);
+	if (ret) {
+		mutex_unlock(&c->sb_lock);
+		return ret;
+	}
+
+	c->disk_sb.sb->features[0] |= cpu_to_le64(f);
+	bch2_write_super(c);
+	mutex_unlock(&c->sb_lock);
+
+	return 0;
+}
+
+int bch2_check_set_has_compressed_data(struct bch_fs *c,
+				       unsigned compression_opt)
+{
+	unsigned compression_type = bch2_compression_decode(compression_opt).type;
+
+	BUG_ON(compression_type >= ARRAY_SIZE(bch2_compression_opt_to_feature));
+
+	return compression_type
+		? __bch2_check_set_has_compressed_data(c,
+				1ULL << bch2_compression_opt_to_feature[compression_type])
+		: 0;
 }
 
 void bch2_fs_compress_exit(struct bch_fs *c)
 {
-	vfree(c->zlib_workspace);
-	mempool_exit(&c->lz4_workspace_pool);
+	unsigned i;
+
+	mempool_exit(&c->decompress_workspace);
+	for (i = 0; i < ARRAY_SIZE(c->compress_workspace); i++)
+		mempool_exit(&c->compress_workspace[i]);
 	mempool_exit(&c->compression_bounce[WRITE]);
 	mempool_exit(&c->compression_bounce[READ]);
 }
 
-#define COMPRESSION_WORKSPACE_SIZE					\
-	max_t(size_t, zlib_inflate_workspacesize(),			\
-	      zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
+static int __bch2_fs_compress_init(struct bch_fs *c, u64 features)
+{
+	size_t decompress_workspace_size = 0;
+	ZSTD_parameters params = zstd_get_params(zstd_max_clevel(),
+						 c->opts.encoded_extent_max);
+
+	c->zstd_workspace_size = zstd_cctx_workspace_bound(&params.cParams);
+
+	struct {
+		unsigned			feature;
+		enum bch_compression_type	type;
+		size_t				compress_workspace;
+		size_t				decompress_workspace;
+	} compression_types[] = {
+		{ BCH_FEATURE_lz4, BCH_COMPRESSION_TYPE_lz4,
+			max_t(size_t, LZ4_MEM_COMPRESS, LZ4HC_MEM_COMPRESS),
+			0 },
+		{ BCH_FEATURE_gzip, BCH_COMPRESSION_TYPE_gzip,
+			zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL),
+			zlib_inflate_workspacesize(), },
+		{ BCH_FEATURE_zstd, BCH_COMPRESSION_TYPE_zstd,
+			c->zstd_workspace_size,
+			zstd_dctx_workspace_bound() },
+	}, *i;
+	bool have_compressed = false;
+
+	for (i = compression_types;
+	     i < compression_types + ARRAY_SIZE(compression_types);
+	     i++)
+		have_compressed |= (features & (1 << i->feature)) != 0;
+
+	if (!have_compressed)
+		return 0;
+
+	if (!mempool_initialized(&c->compression_bounce[READ]) &&
+	    mempool_init_kvmalloc_pool(&c->compression_bounce[READ],
+				       1, c->opts.encoded_extent_max))
+		return -BCH_ERR_ENOMEM_compression_bounce_read_init;
+
+	if (!mempool_initialized(&c->compression_bounce[WRITE]) &&
+	    mempool_init_kvmalloc_pool(&c->compression_bounce[WRITE],
+				       1, c->opts.encoded_extent_max))
+		return -BCH_ERR_ENOMEM_compression_bounce_write_init;
+
+	for (i = compression_types;
+	     i < compression_types + ARRAY_SIZE(compression_types);
+	     i++) {
+		decompress_workspace_size =
+			max(decompress_workspace_size, i->decompress_workspace);
+
+		if (!(features & (1 << i->feature)))
+			continue;
+
+		if (mempool_initialized(&c->compress_workspace[i->type]))
+			continue;
+
+		if (mempool_init_kvmalloc_pool(
+				&c->compress_workspace[i->type],
+				1, i->compress_workspace))
+			return -BCH_ERR_ENOMEM_compression_workspace_init;
+	}
+
+	if (!mempool_initialized(&c->decompress_workspace) &&
+	    mempool_init_kvmalloc_pool(&c->decompress_workspace,
+				       1, decompress_workspace_size))
+		return -BCH_ERR_ENOMEM_decompression_workspace_init;
+
+	return 0;
+}
+
+static u64 compression_opt_to_feature(unsigned v)
+{
+	unsigned type = bch2_compression_decode(v).type;
+
+	return BIT_ULL(bch2_compression_opt_to_feature[type]);
+}
 
 int bch2_fs_compress_init(struct bch_fs *c)
 {
-	unsigned order = get_order(c->sb.encoded_extent_max << 9);
+	u64 f = c->sb.features;
+
+	f |= compression_opt_to_feature(c->opts.compression);
+	f |= compression_opt_to_feature(c->opts.background_compression);
+
+	return __bch2_fs_compress_init(c, f);
+}
+
+int bch2_opt_compression_parse(struct bch_fs *c, const char *_val, u64 *res,
+			       struct printbuf *err)
+{
+	char *val = kstrdup(_val, GFP_KERNEL);
+	char *p = val, *type_str, *level_str;
+	struct bch_compression_opt opt = { 0 };
 	int ret;
 
-	if (!bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4) &&
-	    !bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP))
-		return 0;
+	if (!val)
+		return -ENOMEM;
 
-	if (!mempool_initialized(&c->compression_bounce[READ])) {
-		ret = mempool_init_page_pool(&c->compression_bounce[READ],
-					     1, order);
-		if (ret)
-			return ret;
-	}
+	type_str = strsep(&p, ":");
+	level_str = p;
 
-	if (!mempool_initialized(&c->compression_bounce[WRITE])) {
-		ret = mempool_init_page_pool(&c->compression_bounce[WRITE],
-					     1, order);
-		if (ret)
-			return ret;
-	}
+	ret = match_string(bch2_compression_opts, -1, type_str);
+	if (ret < 0 && err)
+		prt_str(err, "invalid compression type");
+	if (ret < 0)
+		goto err;
 
-	if (!mempool_initialized(&c->lz4_workspace_pool) &&
-	    bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_LZ4)) {
-		ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool,
-						1, LZ4_MEM_COMPRESS);
-		if (ret)
-			return ret;
+	opt.type = ret;
+
+	if (level_str) {
+		unsigned level;
+
+		ret = kstrtouint(level_str, 10, &level);
+		if (!ret && !opt.type && level)
+			ret = -EINVAL;
+		if (!ret && level > 15)
+			ret = -EINVAL;
+		if (ret < 0 && err)
+			prt_str(err, "invalid compression level");
+		if (ret < 0)
+			goto err;
+
+		opt.level = level;
 	}
 
-	if (!c->zlib_workspace &&
-	    bch2_sb_test_feature(c->disk_sb, BCH_FEATURE_GZIP)) {
-		c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
-		if (!c->zlib_workspace)
-			return -ENOMEM;
+	*res = bch2_compression_encode(opt);
+err:
+	kfree(val);
+	return ret;
+}
+
+void bch2_compression_opt_to_text(struct printbuf *out, u64 v)
+{
+	struct bch_compression_opt opt = bch2_compression_decode(v);
+
+	if (opt.type < BCH_COMPRESSION_OPT_NR)
+		prt_str(out, bch2_compression_opts[opt.type]);
+	else
+		prt_printf(out, "(unknown compression opt %u)", opt.type);
+	if (opt.level)
+		prt_printf(out, ":%u", opt.level);
+}
+
+void bch2_opt_compression_to_text(struct printbuf *out,
+				  struct bch_fs *c,
+				  struct bch_sb *sb,
+				  u64 v)
+{
+	return bch2_compression_opt_to_text(out, v);
+}
+
+int bch2_opt_compression_validate(u64 v, struct printbuf *err)
+{
+	if (!bch2_compression_opt_valid(v)) {
+		prt_printf(err, "invalid compression opt %llu", v);
+		return -BCH_ERR_invalid_sb_opt_compression;
 	}
 
 	return 0;