bcache: improve compression memory allocation

use vmalloc() for the zlib workspace, so as to avoid memory allocation failures
on mount

7 files changed, 520 insertions, 466 deletions

diff --git a/drivers/md/bcache/Makefile b/drivers/md/bcache/Makefile
index 390c167819c4..baf502a54c0e 100644
--- a/drivers/md/bcache/Makefile
+++ b/drivers/md/bcache/Makefile
@@ -3,8 +3,8 @@ obj-$(CONFIG_BCACHE)	+= bcache.o
 
 bcache-y		:= acl.o alloc.o bkey.o bkey_methods.o blockdev.o\
 	bset.o btree_cache.o btree_gc.o btree_io.o btree_iter.o btree_update.o\
-	buckets.o chardev.o checksum.o clock.o closure.o debug.o dirent.o\
-	error.o extents.o fs.o fs-gc.o fs-io.o inode.o io.o journal.o keybuf.o\
-	keylist.o migrate.o move.o movinggc.o notify.o opts.o request.o\
-	siphash.o six.o stats.o super.o sysfs.o tier.o trace.o util.o\
+	buckets.o chardev.o checksum.o clock.o closure.o compress.o debug.o\
+	dirent.o error.o extents.o fs.o fs-gc.o fs-io.o inode.o io.o journal.o\
+	keybuf.o keylist.o migrate.o move.o movinggc.o notify.o opts.o\
+	request.o siphash.o six.o stats.o super.o sysfs.o tier.o trace.o util.o\
 	writeback.o xattr.o
diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 2101c3b96981..d1d37fa8aa9d 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -730,7 +730,10 @@ struct cache_set {
 	struct bio_set		bio_write;
 	struct mutex		bio_bounce_pages_lock;
 	mempool_t		bio_bounce_pages;
-	mempool_t		compression_workspace_pool;
+
+	mempool_t		lz4_workspace_pool;
+	void			*zlib_workspace;
+	struct mutex		zlib_workspace_lock;
 	mempool_t		compression_bounce[2];
 	struct bio_decompress_worker __percpu
 				*bio_decompress_worker;
diff --git a/drivers/md/bcache/compress.c b/drivers/md/bcache/compress.c
new file mode 100644
index 000000000000..048ef7581b23
--- /dev/null
+++ b/drivers/md/bcache/compress.c
@@ -0,0 +1,487 @@
+#include "bcache.h"
+#include "compress.h"
+#include "io.h"
+
+#include <linux/lz4.h>
+#include <linux/zlib.h>
+
+enum bounced {
+	BOUNCED_MAPPED,
+	BOUNCED_KMALLOCED,
+	BOUNCED_VMALLOCED,
+	BOUNCED_MEMPOOLED,
+};
+
+static void *__bounce_alloc(struct cache_set *c, unsigned size,
+			    unsigned *bounced, int direction)
+{
+	void *data;
+
+	*bounced = BOUNCED_KMALLOCED;
+	data = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
+	if (data)
+		return data;
+
+	*bounced = BOUNCED_MEMPOOLED;
+	data = mempool_alloc(&c->compression_bounce[direction], GFP_NOWAIT);
+	if (data)
+		return page_address(data);
+
+	*bounced = BOUNCED_VMALLOCED;
+	data = vmalloc(size);
+	if (data)
+		return data;
+
+	*bounced = BOUNCED_MEMPOOLED;
+	data = mempool_alloc(&c->compression_bounce[direction], GFP_NOIO);
+	return page_address(data);
+}
+
+static void *__bio_map_or_bounce(struct cache_set *c,
+				 struct bio *bio, struct bvec_iter start,
+				 unsigned *bounced, int direction)
+{
+	struct bio_vec bv;
+	struct bvec_iter iter;
+	unsigned nr_pages = 0;
+	struct page *stack_pages[4];
+	struct page **pages = NULL;
+	bool first = true;
+	unsigned prev_end = PAGE_SIZE;
+	void *data;
+
+	BUG_ON(start.bi_size > (BCH_COMPRESSED_EXTENT_MAX << 9));
+
+	*bounced = BOUNCED_MAPPED;
+
+	__bio_for_each_segment(bv, bio, iter, start) {
+		if ((!first && bv.bv_offset) ||
+		    prev_end != PAGE_SIZE)
+			goto bounce;
+
+		prev_end = bv.bv_offset + bv.bv_len;
+		nr_pages++;
+	}
+
+	BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
+
+	pages = nr_pages > ARRAY_SIZE(stack_pages)
+		? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
+		: stack_pages;
+	if (!pages)
+		goto bounce;
+
+	nr_pages = 0;
+	__bio_for_each_segment(bv, bio, iter, start)
+		pages[nr_pages++] = bv.bv_page;
+
+	data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
+	if (pages != stack_pages)
+		kfree(pages);
+
+	return data + bio_iter_offset(bio, start);
+bounce:
+	data = __bounce_alloc(c, start.bi_size, bounced, direction);
+
+	if (direction == READ)
+		memcpy_from_bio(data, bio, start);
+
+	return data;
+}
+
+static void *bio_map_or_bounce(struct cache_set *c, struct bio *bio,
+			       unsigned *bounced, int direction)
+{
+	return __bio_map_or_bounce(c, bio, bio->bi_iter, bounced, direction);
+}
+
+static void bio_unmap_or_unbounce(struct cache_set *c, void *data,
+				  unsigned bounced, int direction)
+{
+	if (!data)
+		return;
+
+	switch (bounced) {
+	case BOUNCED_MAPPED:
+		vunmap((void *) ((unsigned long) data & PAGE_MASK));
+		return;
+	case BOUNCED_KMALLOCED:
+		kfree(data);
+		return;
+	case BOUNCED_VMALLOCED:
+		vfree(data);
+		return;
+	case BOUNCED_MEMPOOLED:
+		mempool_free(virt_to_page(data), &c->compression_bounce[direction]);
+		return;
+	}
+}
+
+static int __bio_uncompress(struct cache_set *c, struct bio *src,
+			    void *dst_data, struct bch_extent_crc64 crc)
+{
+	void *src_data = NULL;
+	unsigned src_bounced;
+	size_t src_len = src->bi_iter.bi_size;
+	size_t dst_len = crc.uncompressed_size << 9;
+	int ret;
+
+	src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
+
+	switch (crc.compression_type) {
+	case BCH_COMPRESSION_LZ4:
+		ret = lz4_decompress(src_data, &src_len,
+				     dst_data, dst_len);
+		if (ret) {
+			ret = -EIO;
+			goto err;
+		}
+		break;
+	case BCH_COMPRESSION_GZIP: {
+		void *workspace;
+		z_stream strm;
+
+		workspace = kmalloc(zlib_inflate_workspacesize(),
+				    GFP_NOIO|__GFP_NOWARN);
+		if (!workspace) {
+			mutex_lock(&c->zlib_workspace_lock);
+			workspace = c->zlib_workspace;
+		}
+
+		strm.workspace	= workspace;
+		strm.next_in	= src_data;
+		strm.avail_in	= src_len;
+		strm.next_out	= dst_data;
+		strm.avail_out	= dst_len;
+		zlib_inflateInit2(&strm, -MAX_WBITS);
+
+		ret = zlib_inflate(&strm, Z_FINISH);
+
+		if (workspace == c->zlib_workspace)
+			mutex_unlock(&c->zlib_workspace_lock);
+		else
+			kfree(workspace);
+
+		if (ret != Z_STREAM_END) {
+			ret = -EIO;
+			goto err;
+		}
+		break;
+	}
+	default:
+		BUG();
+	}
+	ret = 0;
+err:
+	bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
+	return ret;
+}
+
+int bch_bio_uncompress_inplace(struct cache_set *c, struct bio *bio,
+			       struct bkey *k, struct bch_extent_crc64 crc)
+{
+	void *dst_data = NULL;
+	size_t dst_len = crc.uncompressed_size << 9;
+	int ret = -ENOMEM;
+
+	BUG_ON(DIV_ROUND_UP(k->size, PAGE_SECTORS) > bio->bi_max_vecs);
+
+	/* XXX mempoolify */
+	dst_data = kmalloc(dst_len, GFP_NOIO|__GFP_NOWARN);
+	if (!dst_data) {
+		dst_data = vmalloc(dst_len);
+		if (!dst_data)
+			goto err;
+	}
+
+	ret = __bio_uncompress(c, bio, dst_data, crc);
+	if (ret)
+		goto err;
+
+	while (bio->bi_vcnt < DIV_ROUND_UP(k->size, PAGE_SECTORS)) {
+		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
+
+		bv->bv_page = alloc_page(GFP_NOIO);
+		if (!bv->bv_page)
+			goto use_mempool;
+
+		bv->bv_len = PAGE_SIZE;
+		bv->bv_offset = 0;
+		bio->bi_vcnt++;
+	}
+
+	bio->bi_iter.bi_size = k->size << 9;
+copy_data:
+	memcpy_to_bio(bio, bio->bi_iter, dst_data + (crc.offset << 9));
+err:
+	kvfree(dst_data);
+	return ret;
+use_mempool:
+	/*
+	 * We already allocated from mempool, we can't allocate from it again
+	 * without freeing the pages we already allocated or else we could
+	 * deadlock:
+	 */
+
+	bch_bio_free_pages_pool(c, bio);
+	bch_bio_alloc_pages_pool(c, bio, k->size << 9);
+	goto copy_data;
+}
+
+int bch_bio_uncompress(struct cache_set *c, struct bio *src,
+		       struct bio *dst, struct bvec_iter dst_iter,
+		       struct bch_extent_crc64 crc)
+{
+	void *dst_data = NULL;
+	unsigned dst_bounced;
+	size_t dst_len = crc.uncompressed_size << 9;
+	int ret = -ENOMEM;
+
+	dst_data = dst_len == dst_iter.bi_size
+		? __bio_map_or_bounce(c, dst, dst_iter, &dst_bounced, WRITE)
+		: __bounce_alloc(c, dst_len, &dst_bounced, WRITE);
+
+	ret = __bio_uncompress(c, src, dst_data, crc);
+	if (ret)
+		goto err;
+
+	if (dst_bounced)
+		memcpy_to_bio(dst, dst_iter, dst_data + (crc.offset << 9));
+err:
+	bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
+	return ret;
+}
+
+static struct bio *__bio_compress(struct cache_set *c,
+				  unsigned compression_type,
+				  struct bio *src,
+				  unsigned output_available,
+				  int *input_consumed)
+{
+	struct bio *dst;
+	void *src_data = NULL, *dst_data = NULL;
+	unsigned src_bounced, dst_bounced;
+	int ret = -1;
+
+	BUG_ON(output_available > src->bi_iter.bi_size);
+
+	output_available = min_t(unsigned, output_available,
+				 BCH_COMPRESSED_EXTENT_MAX << 9);
+
+	dst = bio_alloc_bioset(GFP_NOIO,
+		DIV_ROUND_UP(output_available, PAGE_SIZE),
+		&c->bio_write);
+
+	bch_bio_alloc_pages_pool(c, dst, output_available);
+
+	dst_data = bio_map_or_bounce(c, dst, &dst_bounced, WRITE);
+	src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
+
+	switch (compression_type) {
+	case BCH_COMPRESSION_LZ4: {
+		void *workspace;
+		size_t dst_size = dst->bi_iter.bi_size;
+
+		workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
+		/*
+		 * XXX: due to the way the interface to lz4_compress works, we
+		 * can't consume more than output_available bytes of input (even
+		 * though a lot more might fit after compressing)
+		 */
+		ret = lz4_compress(src_data, min(output_available,
+						 src->bi_iter.bi_size),
+				   dst_data, &dst_size, workspace);
+
+		mempool_free(workspace, &c->lz4_workspace_pool);
+		if (ret)
+			goto err;
+
+		dst->bi_iter.bi_size = dst_size;
+		*input_consumed = output_available;
+		break;
+	}
+	case BCH_COMPRESSION_GZIP: {
+		void *workspace;
+		z_stream strm;
+
+		workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL),
+				    GFP_NOIO|__GFP_NOWARN);
+		if (!workspace) {
+			mutex_lock(&c->zlib_workspace_lock);
+			workspace = c->zlib_workspace;
+		}
+
+		strm.workspace	= workspace;
+		strm.next_in	= src_data;
+		strm.avail_in	= output_available;
+		strm.next_out	= dst_data;
+		strm.avail_out	= output_available;
+		zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
+				  Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
+				  Z_DEFAULT_STRATEGY);
+
+		ret = zlib_deflate(&strm, Z_FINISH);
+
+		if (workspace == c->zlib_workspace)
+			mutex_unlock(&c->zlib_workspace_lock);
+		else
+			kfree(workspace);
+
+		if (ret != Z_STREAM_END) {
+			ret = -EIO;
+			goto err;
+		}
+
+		ret = zlib_deflateEnd(&strm);
+		if (ret != Z_OK) {
+			ret = -EIO;
+			goto err;
+		}
+
+		BUG_ON(strm.total_in != output_available);
+
+		dst->bi_iter.bi_size = strm.total_out;
+		*input_consumed = strm.total_in;
+		break;
+	}
+	default:
+		BUG();
+	}
+
+	BUG_ON(!dst->bi_iter.bi_size);
+
+	if (dst_bounced)
+		memcpy_to_bio(dst, dst->bi_iter, dst_data);
+out:
+	bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
+	bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
+
+	if (!ret)
+		while (dst->bi_vcnt * PAGE_SIZE >
+		       round_up(dst->bi_iter.bi_size, PAGE_SIZE))
+			mempool_free(dst->bi_io_vec[--dst->bi_vcnt].bv_page,
+				     &c->bio_bounce_pages);
+
+	return dst;
+err:
+	ret = -1;
+	*input_consumed = -1;
+	goto out;
+}
+
+struct bio *bch_bio_compress(struct cache_set *c, struct bio *src,
+			     unsigned *compression_type,
+			     unsigned output_available)
+{
+	struct bio *dst = NULL;
+	int input_consumed;
+
+	/* if it's only one block, don't bother trying to compress: */
+	if (bio_sectors(src) <= c->sb.block_size)
+		*compression_type = BCH_COMPRESSION_NONE;
+
+	switch (*compression_type) {
+		case BCH_COMPRESSION_NONE:
+			/* Just bounce it, for stable checksums: */
+copy:
+			if (!dst)
+				dst = bio_alloc_bioset(GFP_NOIO,
+						       DIV_ROUND_UP(output_available, PAGE_SIZE),
+						       &c->bio_write);
+			bch_bio_alloc_pages_pool(c, dst, output_available);
+			bio_copy_data(dst, src);
+			input_consumed = output_available;
+			goto advance;
+		case BCH_COMPRESSION_LZ4:
+		case BCH_COMPRESSION_GZIP:
+			dst = __bio_compress(c, *compression_type, src,
+					     output_available, &input_consumed);
+			break;
+		default:
+			BUG();
+	}
+
+	if ((int) round_up(dst->bi_iter.bi_size,
+			   block_bytes(c)) >= input_consumed) {
+		/* Failed to compress (didn't get smaller): */
+		*compression_type = BCH_COMPRESSION_NONE;
+		goto copy;
+	}
+
+	/* Pad to blocksize, and zero out padding: */
+	while (dst->bi_iter.bi_size & (block_bytes(c) - 1)) {
+		unsigned idx = dst->bi_iter.bi_size >> PAGE_SHIFT;
+		unsigned offset = dst->bi_iter.bi_size & (PAGE_SIZE - 1);
+		unsigned bytes = (PAGE_SIZE - offset) & (block_bytes(c) - 1);
+
+		if (idx < dst->bi_vcnt) {
+			struct bio_vec *bv = &dst->bi_io_vec[idx];
+
+			memset(page_address(bv->bv_page) + offset, 0, bytes);
+		} else {
+			dst->bi_io_vec[dst->bi_vcnt++] = (struct bio_vec) {
+				.bv_page	= ZERO_PAGE(0),
+					.bv_len		= PAGE_SIZE,
+					.bv_offset	= 0,
+			};
+		}
+
+		dst->bi_iter.bi_size += bytes;
+	}
+advance:
+	bio_advance(src, input_consumed);
+
+	return dst;
+}
+
+void bch_compress_free(struct cache_set *c)
+{
+	vfree(c->zlib_workspace);
+	mempool_exit(&c->lz4_workspace_pool);
+	mempool_exit(&c->compression_bounce[WRITE]);
+	mempool_exit(&c->compression_bounce[READ]);
+	free_percpu(c->bio_decompress_worker);
+}
+
+#define COMPRESSION_WORKSPACE_SIZE					\
+	max_t(size_t, zlib_inflate_workspacesize(),			\
+	      zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL))
+
+int bch_compress_init(struct cache_set *c)
+{
+	int ret, cpu;
+
+	c->bio_decompress_worker = alloc_percpu(*c->bio_decompress_worker);
+	if (!c->bio_decompress_worker)
+		return -ENOMEM;
+
+	for_each_possible_cpu(cpu) {
+		struct bio_decompress_worker *d =
+			per_cpu_ptr(c->bio_decompress_worker, cpu);
+
+		d->c = c;
+		INIT_WORK(&d->work, bch_bio_decompress_work);
+		init_llist_head(&d->bio_list);
+	}
+
+	ret = mempool_init_page_pool(&c->compression_bounce[READ], 1,
+				     get_order(BCH_COMPRESSED_EXTENT_MAX << 9));
+	if (ret)
+		return ret;
+
+	ret = mempool_init_page_pool(&c->compression_bounce[WRITE], 1,
+				     get_order(BCH_COMPRESSED_EXTENT_MAX << 9));
+	if (ret)
+		return ret;
+
+	ret = mempool_init_kmalloc_pool(&c->lz4_workspace_pool, 1,
+					LZ4_MEM_COMPRESS);
+	if (ret)
+		return ret;
+
+	c->zlib_workspace = vmalloc(COMPRESSION_WORKSPACE_SIZE);
+	if (!c->zlib_workspace)
+		return -ENOMEM;
+
+	return 0;
+}
diff --git a/drivers/md/bcache/compress.h b/drivers/md/bcache/compress.h
new file mode 100644
index 000000000000..2b5dded6f59f
--- /dev/null
+++ b/drivers/md/bcache/compress.h
@@ -0,0 +1,14 @@
+#ifndef _BCACHE_COMPRESS_H
+#define _BCACHE_COMPRESS_H
+
+int bch_bio_uncompress_inplace(struct cache_set *, struct bio *,
+			       struct bkey *, struct bch_extent_crc64);
+int bch_bio_uncompress(struct cache_set *, struct bio *, struct bio *,
+		       struct bvec_iter, struct bch_extent_crc64);
+struct bio *bch_bio_compress(struct cache_set *, struct bio *,
+			     unsigned *, unsigned);
+
+void bch_compress_free(struct cache_set *);
+int bch_compress_init(struct cache_set *);
+
+#endif /* _BCACHE_COMPRESS_H */
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index 019b3edbd266..165c640fe2cd 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -11,6 +11,7 @@
 #include "btree_update.h"
 #include "buckets.h"
 #include "checksum.h"
+#include "compress.h"
 #include "clock.h"
 #include "debug.h"
 #include "error.h"
@@ -24,9 +25,7 @@
 #include "super.h"
 
 #include <linux/blkdev.h>
-#include <linux/lz4.h>
 #include <linux/random.h>
-#include <linux/zlib.h>
 
 #include <trace/events/bcache.h>
 
@@ -195,430 +194,6 @@ void bch_bbio_endio(struct bbio *bio)
 
 /* Writes */
 
-enum bounced {
-	BOUNCED_MAPPED,
-	BOUNCED_KMALLOCED,
-	BOUNCED_VMALLOCED,
-	BOUNCED_MEMPOOLED,
-};
-
-static void *__bounce_alloc(struct cache_set *c, unsigned size,
-			    unsigned *bounced, int direction)
-{
-	void *data;
-
-	*bounced = BOUNCED_KMALLOCED;
-	data = kmalloc(size, GFP_NOIO|__GFP_NOWARN);
-	if (data)
-		return data;
-
-	*bounced = BOUNCED_MEMPOOLED;
-	data = mempool_alloc(&c->compression_bounce[direction], GFP_NOWAIT);
-	if (data)
-		return page_address(data);
-
-	*bounced = BOUNCED_VMALLOCED;
-	data = vmalloc(size);
-	if (data)
-		return data;
-
-	*bounced = BOUNCED_MEMPOOLED;
-	data = mempool_alloc(&c->compression_bounce[direction], GFP_NOIO);
-	return page_address(data);
-}
-
-static void *__bio_map_or_bounce(struct cache_set *c,
-				 struct bio *bio, struct bvec_iter start,
-				 unsigned *bounced, int direction)
-{
-	struct bio_vec bv;
-	struct bvec_iter iter;
-	unsigned nr_pages = 0;
-	struct page *stack_pages[4];
-	struct page **pages = NULL;
-	bool first = true;
-	unsigned prev_end = PAGE_SIZE;
-	void *data;
-
-	BUG_ON(start.bi_size > (BCH_COMPRESSED_EXTENT_MAX << 9));
-
-	*bounced = BOUNCED_MAPPED;
-
-	__bio_for_each_segment(bv, bio, iter, start) {
-		if ((!first && bv.bv_offset) ||
-		    prev_end != PAGE_SIZE)
-			goto bounce;
-
-		prev_end = bv.bv_offset + bv.bv_len;
-		nr_pages++;
-	}
-
-	BUG_ON(DIV_ROUND_UP(start.bi_size, PAGE_SIZE) > nr_pages);
-
-	pages = nr_pages > ARRAY_SIZE(stack_pages)
-		? kmalloc_array(nr_pages, sizeof(struct page *), GFP_NOIO)
-		: stack_pages;
-	if (!pages)
-		goto bounce;
-
-	nr_pages = 0;
-	__bio_for_each_segment(bv, bio, iter, start)
-		pages[nr_pages++] = bv.bv_page;
-
-	data = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
-	if (pages != stack_pages)
-		kfree(pages);
-
-	return data + bio_iter_offset(bio, start);
-bounce:
-	data = __bounce_alloc(c, start.bi_size, bounced, direction);
-
-	if (direction == READ)
-		memcpy_from_bio(data, bio, start);
-
-	return data;
-}
-
-static void *bio_map_or_bounce(struct cache_set *c, struct bio *bio,
-			       unsigned *bounced, int direction)
-{
-	return __bio_map_or_bounce(c, bio, bio->bi_iter, bounced, direction);
-}
-
-static void bio_unmap_or_unbounce(struct cache_set *c, void *data,
-				  unsigned bounced, int direction)
-{
-	if (!data)
-		return;
-
-	switch (bounced) {
-	case BOUNCED_MAPPED:
-		vunmap((void *) ((unsigned long) data & PAGE_MASK));
-		return;
-	case BOUNCED_KMALLOCED:
-		kfree(data);
-		return;
-	case BOUNCED_VMALLOCED:
-		vfree(data);
-		return;
-	case BOUNCED_MEMPOOLED:
-		mempool_free(virt_to_page(data), &c->compression_bounce[direction]);
-		return;
-	}
-}
-
-static int __bio_uncompress(struct cache_set *c, struct bio *src,
-			    void *dst_data, struct bch_extent_crc64 crc)
-{
-	void *src_data = NULL;
-	unsigned src_bounced;
-	size_t src_len = src->bi_iter.bi_size;
-	size_t dst_len = crc.uncompressed_size << 9;
-	int ret;
-
-	src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
-
-	switch (crc.compression_type) {
-	case BCH_COMPRESSION_LZ4:
-		ret = lz4_decompress(src_data, &src_len,
-				     dst_data, dst_len);
-		if (ret) {
-			ret = -EIO;
-			goto err;
-		}
-		break;
-	case BCH_COMPRESSION_GZIP: {
-		struct page *workspace;
-		z_stream strm;
-
-		workspace = mempool_alloc(&c->compression_workspace_pool,
-					  GFP_NOIO);
-		strm.workspace	= page_address(workspace);
-		strm.next_in	= src_data;
-		strm.avail_in	= src_len;
-		strm.next_out	= dst_data;
-		strm.avail_out	= dst_len;
-		zlib_inflateInit2(&strm, -MAX_WBITS);
-
-		ret = zlib_inflate(&strm, Z_FINISH);
-
-		mempool_free(workspace, &c->compression_workspace_pool);
-
-		if (ret != Z_STREAM_END) {
-			ret = -EIO;
-			goto err;
-		}
-		break;
-	}
-	default:
-		BUG();
-	}
-	ret = 0;
-err:
-	bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
-	return ret;
-}
-
-static int bio_uncompress_inplace(struct cache_set *c, struct bio *bio,
-				  struct bkey *k, struct bch_extent_crc64 crc)
-{
-	void *dst_data = NULL;
-	size_t dst_len = crc.uncompressed_size << 9;
-	int ret = -ENOMEM;
-
-	BUG_ON(DIV_ROUND_UP(k->size, PAGE_SECTORS) > bio->bi_max_vecs);
-
-	/* XXX mempoolify */
-	dst_data = kmalloc(dst_len, GFP_NOIO|__GFP_NOWARN);
-	if (!dst_data) {
-		dst_data = vmalloc(dst_len);
-		if (!dst_data)
-			goto err;
-	}
-
-	ret = __bio_uncompress(c, bio, dst_data, crc);
-	if (ret)
-		goto err;
-
-	while (bio->bi_vcnt < DIV_ROUND_UP(k->size, PAGE_SECTORS)) {
-		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
-
-		bv->bv_page = alloc_page(GFP_NOIO);
-		if (!bv->bv_page)
-			goto use_mempool;
-
-		bv->bv_len = PAGE_SIZE;
-		bv->bv_offset = 0;
-		bio->bi_vcnt++;
-	}
-
-	bio->bi_iter.bi_size = k->size << 9;
-copy_data:
-	memcpy_to_bio(bio, bio->bi_iter, dst_data + (crc.offset << 9));
-err:
-	kvfree(dst_data);
-	return ret;
-use_mempool:
-	/*
-	 * We already allocated from mempool, we can't allocate from it again
-	 * without freeing the pages we already allocated or else we could
-	 * deadlock:
-	 */
-
-	bch_bio_free_pages_pool(c, bio);
-	bch_bio_alloc_pages_pool(c, bio, k->size << 9);
-	goto copy_data;
-}
-
-static int bio_uncompress(struct cache_set *c, struct bio *src,
-			  struct bio *dst, struct bvec_iter dst_iter,
-			  struct bch_extent_crc64 crc)
-{
-	void *dst_data = NULL;
-	unsigned dst_bounced;
-	size_t dst_len = crc.uncompressed_size << 9;
-	int ret = -ENOMEM;
-
-	dst_data = dst_len == dst_iter.bi_size
-		? __bio_map_or_bounce(c, dst, dst_iter, &dst_bounced, WRITE)
-		: __bounce_alloc(c, dst_len, &dst_bounced, WRITE);
-
-	ret = __bio_uncompress(c, src, dst_data, crc);
-	if (ret)
-		goto err;
-
-	if (dst_bounced)
-		memcpy_to_bio(dst, dst_iter, dst_data + (crc.offset << 9));
-err:
-	bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
-	return ret;
-}
-
-static struct bio *__bio_compress(struct cache_set *c,
-				  unsigned compression_type,
-				  struct bio *src,
-				  unsigned output_available,
-				  int *input_consumed)
-{
-	struct bio *dst;
-	void *src_data = NULL, *dst_data = NULL;
-	unsigned src_bounced, dst_bounced;
-	int ret = -1;
-
-	BUG_ON(output_available > src->bi_iter.bi_size);
-
-	output_available = min_t(unsigned, output_available,
-				 BCH_COMPRESSED_EXTENT_MAX << 9);
-
-	dst = bio_alloc_bioset(GFP_NOIO,
-		DIV_ROUND_UP(output_available, PAGE_SIZE),
-		&c->bio_write);
-
-	bch_bio_alloc_pages_pool(c, dst, output_available);
-
-	dst_data = bio_map_or_bounce(c, dst, &dst_bounced, WRITE);
-	src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
-
-	switch (compression_type) {
-	case BCH_COMPRESSION_LZ4: {
-		struct page *workmem;
-		bool used_mempool = false;
-		unsigned order = get_order(LZ4_MEM_COMPRESS);
-		size_t dst_size = dst->bi_iter.bi_size;
-
-		workmem = alloc_pages(GFP_NOWAIT|__GFP_NOWARN, order);
-		if (!workmem) {
-			workmem = mempool_alloc(&c->compression_workspace_pool,
-						GFP_NOIO);
-			used_mempool = true;
-		}
-		/*
-		 * XXX: due to the way the interface to lz4_compress works, we
-		 * can't consume more than output_available bytes of input (even
-		 * though a lot more might fit after compressing)
-		 */
-		ret = lz4_compress(src_data, min(output_available,
-						 src->bi_iter.bi_size),
-				   dst_data, &dst_size,
-				   page_address(workmem));
-
-		if (used_mempool)
-			mempool_free(workmem, &c->compression_workspace_pool);
-		else
-			__free_pages(workmem, order);
-
-		if (ret)
-			goto err;
-
-		dst->bi_iter.bi_size = dst_size;
-		*input_consumed = output_available;
-		break;
-	}
-	case BCH_COMPRESSION_GZIP: {
-		struct page *workmem;
-		z_stream strm;
-
-		workmem = mempool_alloc(&c->compression_workspace_pool, GFP_NOIO);
-		strm.workspace	= page_address(workmem);
-		strm.next_in	= src_data;
-		strm.avail_in	= output_available;
-		strm.next_out	= dst_data;
-		strm.avail_out	= output_available;
-		zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
-				  Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
-				  Z_DEFAULT_STRATEGY);
-
-		ret = zlib_deflate(&strm, Z_FINISH);
-
-		mempool_free(workmem, &c->compression_workspace_pool);
-
-		if (ret != Z_STREAM_END) {
-			ret = -EIO;
-			goto err;
-		}
-
-		ret = zlib_deflateEnd(&strm);
-		if (ret != Z_OK) {
-			ret = -EIO;
-			goto err;
-		}
-
-		BUG_ON(strm.total_in != output_available);
-
-		dst->bi_iter.bi_size = strm.total_out;
-		*input_consumed = strm.total_in;
-		break;
-	}
-	default:
-		BUG();
-	}
-
-	BUG_ON(!dst->bi_iter.bi_size);
-
-	if (dst_bounced)
-		memcpy_to_bio(dst, dst->bi_iter, dst_data);
-out:
-	bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
-	bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
-
-	if (!ret)
-		while (dst->bi_vcnt * PAGE_SIZE >
-		       round_up(dst->bi_iter.bi_size, PAGE_SIZE))
-			mempool_free(dst->bi_io_vec[--dst->bi_vcnt].bv_page,
-				     &c->bio_bounce_pages);
-
-	return dst;
-err:
-	ret = -1;
-	*input_consumed = -1;
-	goto out;
-}
-
-static struct bio *bio_compress(struct cache_set *c, struct bio *src,
-				unsigned *compression_type,
-				unsigned output_available)
-{
-	struct bio *dst = NULL;
-	int input_consumed;
-
-	/* if it's only one block, don't bother trying to compress: */
-	if (bio_sectors(src) <= c->sb.block_size)
-		*compression_type = BCH_COMPRESSION_NONE;
-
-	switch (*compression_type) {
-	case BCH_COMPRESSION_NONE:
-		/* Just bounce it, for stable checksums: */
-copy:
-		if (!dst)
-			dst = bio_alloc_bioset(GFP_NOIO,
-				DIV_ROUND_UP(output_available, PAGE_SIZE),
-				&c->bio_write);
-		bch_bio_alloc_pages_pool(c, dst, output_available);
-		bio_copy_data(dst, src);
-		input_consumed = output_available;
-		goto advance;
-	case BCH_COMPRESSION_LZ4:
-	case BCH_COMPRESSION_GZIP:
-		dst = __bio_compress(c, *compression_type, src,
-				     output_available, &input_consumed);
-		break;
-	default:
-		BUG();
-	}
-
-	if ((int) round_up(dst->bi_iter.bi_size,
-			   block_bytes(c)) >= input_consumed) {
-		/* Failed to compress (didn't get smaller): */
-		*compression_type = BCH_COMPRESSION_NONE;
-		goto copy;
-	}
-
-	/* Pad to blocksize, and zero out padding: */
-	while (dst->bi_iter.bi_size & (block_bytes(c) - 1)) {
-		unsigned idx = dst->bi_iter.bi_size >> PAGE_SHIFT;
-		unsigned offset = dst->bi_iter.bi_size & (PAGE_SIZE - 1);
-		unsigned bytes = (PAGE_SIZE - offset) & (block_bytes(c) - 1);
-
-		if (idx < dst->bi_vcnt) {
-			struct bio_vec *bv = &dst->bi_io_vec[idx];
-
-			memset(page_address(bv->bv_page) + offset, 0, bytes);
-		} else {
-			dst->bi_io_vec[dst->bi_vcnt++] = (struct bio_vec) {
-				.bv_page	= ZERO_PAGE(0),
-				.bv_len		= PAGE_SIZE,
-				.bv_offset	= 0,
-			};
-		}
-
-		dst->bi_iter.bi_size += bytes;
-	}
-advance:
-	bio_advance(src, input_consumed);
-
-	return dst;
-}
-
 static void __bch_write(struct closure *);
 
 static void bch_write_done(struct closure *cl)
@@ -820,7 +395,7 @@ static int bch_write_extent(struct bch_write_op *op,
 	     op->crc.compressed_size > ob->sectors_free)) {
 		int ret;
 
-		ret = bio_uncompress_inplace(c, orig, &e->k, op->crc);
+		ret = bch_bio_uncompress_inplace(c, orig, &e->k, op->crc);
 		if (ret)
 			return ret;
 
@@ -865,9 +440,9 @@ static int bch_write_extent(struct bch_write_op *op,
 			: 0;
 		orig->bi_iter.bi_size -= extra_input;
 
-		bio = bio_compress(c, orig,
-				   &compression_type,
-				   output_available);
+		bio = bch_bio_compress(c, orig,
+				       &compression_type,
+				       output_available);
 		/* copy WRITE_SYNC flag */
 		bio->bi_opf		= orig->bi_opf;
 
@@ -1298,7 +873,7 @@ static int bio_checksum_uncompress(struct cache_set *c,
 	}
 
 	if (rbio->crc.compression_type != BCH_COMPRESSION_NONE) {
-		ret = bio_uncompress(c, src, dst, dst_iter, rbio->crc);
+		ret = bch_bio_uncompress(c, src, dst, dst_iter, rbio->crc);
 	} else if (rbio->bounce) {
 		bio_advance(src, rbio->crc.offset << 9);
 		bio_copy_data_iter(dst, dst_iter,
diff --git a/drivers/md/bcache/io.h b/drivers/md/bcache/io.h
index 3f611e652c64..6f9148c37230 100644
--- a/drivers/md/bcache/io.h
+++ b/drivers/md/bcache/io.h
@@ -3,14 +3,6 @@
 
 #include "io_types.h"
 
-#include <linux/lz4.h>
-#include <linux/zlib.h>
-
-#define COMPRESSION_WORKSPACE_SIZE					\
-	max_t(size_t, zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL),\
-	max_t(size_t, zlib_inflate_workspacesize(),			\
-		      LZ4HC_MEM_COMPRESS))
-
 #define to_bbio(_bio)		container_of((_bio), struct bbio, bio)
 
 #define to_wbio(_bio)			\
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index e7eb7e8bfc81..bc87ff457c2b 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -15,6 +15,7 @@
 #include "btree_io.h"
 #include "checksum.h"
 #include "clock.h"
+#include "compress.h"
 #include "debug.h"
 #include "error.h"
 #include "fs-gc.h"
@@ -881,13 +882,10 @@ static void cache_set_free(struct cache_set *c)
 	bch_journal_free(&c->journal);
 	bch_io_clock_exit(&c->io_clock[WRITE]);
 	bch_io_clock_exit(&c->io_clock[READ]);
+	bch_compress_free(c);
 	bdi_destroy(&c->bdi);
 	free_percpu(c->bucket_stats_lock.lock);
 	free_percpu(c->bucket_stats_percpu);
-	free_percpu(c->bio_decompress_worker);
-	mempool_exit(&c->compression_bounce[WRITE]);
-	mempool_exit(&c->compression_bounce[READ]);
-	mempool_exit(&c->compression_workspace_pool);
 	mempool_exit(&c->bio_bounce_pages);
 	bioset_exit(&c->bio_write);
 	bioset_exit(&c->bio_read_split);
@@ -1042,7 +1040,6 @@ static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb,
 {
 	struct cache_set *c;
 	unsigned iter_size;
-	int cpu;
 
 	c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);
 	if (!c)
@@ -1090,6 +1087,7 @@ static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb,
 	bio_list_init(&c->read_retry_list);
 	spin_lock_init(&c->read_retry_lock);
 	INIT_WORK(&c->read_retry_work, bch_read_retry_work);
+	mutex_init(&c->zlib_workspace_lock);
 
 	seqcount_init(&c->gc_pos_lock);
 
@@ -1151,13 +1149,6 @@ static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb,
 					 c->sb.btree_node_size,
 					 CRC32_EXTENT_SIZE_MAX) /
 				   PAGE_SECTORS, 0) ||
-	    mempool_init_page_pool(&c->compression_workspace_pool, 1,
-				   get_order(COMPRESSION_WORKSPACE_SIZE)) ||
-	    mempool_init_page_pool(&c->compression_bounce[READ], 1,
-				   get_order(BCH_COMPRESSED_EXTENT_MAX << 9)) ||
-	    mempool_init_page_pool(&c->compression_bounce[WRITE], 1,
-				   get_order(BCH_COMPRESSED_EXTENT_MAX << 9)) ||
-	    !(c->bio_decompress_worker = alloc_percpu(*c->bio_decompress_worker)) ||
 	    !(c->bucket_stats_percpu = alloc_percpu(struct bucket_stats_cache_set)) ||
 	    !(c->bucket_stats_lock.lock = alloc_percpu(*c->bucket_stats_lock.lock)) ||
 	    bdi_setup_and_register(&c->bdi, "bcache") ||
@@ -1166,18 +1157,10 @@ static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb,
 	    bch_journal_alloc(&c->journal) ||
 	    bch_btree_cache_alloc(c) ||
 	    bch_bset_sort_state_init(&c->sort, ilog2(btree_pages(c)),
-				     &c->btree_sort_time))
+				     &c->btree_sort_time) ||
+	    bch_compress_init(c))
 		goto err;
 
-	for_each_possible_cpu(cpu) {
-		struct bio_decompress_worker *d =
-			per_cpu_ptr(c->bio_decompress_worker, cpu);
-
-		d->c = c;
-		INIT_WORK(&d->work, bch_bio_decompress_work);
-		init_llist_head(&d->bio_list);
-	}
-
 	c->bdi.ra_pages		= VM_MAX_READAHEAD * 1024 / PAGE_SIZE;
 	c->bdi.congested_fn	= bch_congested_fn;
 	c->bdi.congested_data	= c;