bcache: Refactor write path to not split bios unnecessarily

compression code is a good deal saner now, too

7 files changed, 229 insertions, 248 deletions

diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c
index 376721461381..ac40a96b381e 100644
--- a/drivers/md/bcache/alloc.c
+++ b/drivers/md/bcache/alloc.c
@@ -1294,12 +1294,10 @@ static struct open_bucket *lock_writepoint(struct cache_set *c,
 static int open_bucket_add_buckets(struct cache_set *c,
 				   struct write_point *wp,
 				   struct open_bucket *ob,
-				   struct bkey_i_extent *e,
 				   unsigned nr_replicas,
 				   enum alloc_reserve reserve,
 				   struct closure *cl)
 {
-	const struct bch_extent_ptr *ptr;
 	long caches_used[BITS_TO_LONGS(MAX_CACHES_PER_SET)];
 	int i, dst;
 
@@ -1311,14 +1309,11 @@ static int open_bucket_add_buckets(struct cache_set *c,
 	 */
 
 	/* Short circuit all the fun stuff if posssible: */
-	if (!bkey_val_u64s(&e->k) && ob->nr_ptrs >= nr_replicas)
+	if (ob->nr_ptrs >= nr_replicas)
 		return 0;
 
 	memset(caches_used, 0, sizeof(caches_used));
 
-	extent_for_each_ptr(extent_i_to_s_c(e), ptr)
-		__set_bit(ptr->dev, caches_used);
-
 	/*
 	 * Shuffle pointers to devices we already have to the end:
 	 * bch_bucket_alloc_set() will add new pointers to the statr of @b, and
@@ -1343,7 +1338,6 @@ static int open_bucket_add_buckets(struct cache_set *c,
  */
 struct open_bucket *bch_alloc_sectors_start(struct cache_set *c,
 					    struct write_point *wp,
-					    struct bkey_i_extent *e,
 					    unsigned nr_replicas,
 					    enum alloc_reserve reserve,
 					    struct closure *cl)
@@ -1398,7 +1392,7 @@ retry:
 		ob = new_ob;
 	}
 
-	ret = open_bucket_add_buckets(c, wp, ob, e, nr_replicas,
+	ret = open_bucket_add_buckets(c, wp, ob, nr_replicas,
 				      reserve, cl);
 	if (ret) {
 		mutex_unlock(&ob->lock);
@@ -1417,11 +1411,10 @@ retry:
  * Append pointers to the space we just allocated to @k, and mark @sectors space
  * as allocated out of @ob
  */
-void bch_alloc_sectors_done(struct cache_set *c, struct write_point *wp,
-			    struct bkey_i_extent *e, unsigned nr_replicas,
-			    struct open_bucket *ob, unsigned sectors)
+void bch_alloc_sectors_append_ptrs(struct cache_set *c, struct bkey_i_extent *e,
+				   unsigned nr_replicas, struct open_bucket *ob,
+				   unsigned sectors)
 {
-	struct cache_member_rcu *mi = cache_member_info_get(c);
 	struct bch_extent_ptr tmp, *ptr;
 	struct cache *ca;
 	bool has_data = false;
@@ -1450,22 +1443,37 @@ void bch_alloc_sectors_done(struct cache_set *c, struct write_point *wp,
 		extent_ptr_append(e, tmp);
 
 		ob->ptr_offset[i] += sectors;
+	}
+
+	open_bucket_for_each_online_device(c, ob, ptr, ca)
+		this_cpu_add(*ca->sectors_written, sectors);
+}
 
+/*
+ * Append pointers to the space we just allocated to @k, and mark @sectors space
+ * as allocated out of @ob
+ */
+void bch_alloc_sectors_done(struct cache_set *c, struct write_point *wp,
+			    struct open_bucket *ob)
+{
+	struct cache_member_rcu *mi = cache_member_info_get(c);
+	bool has_data = false;
+	unsigned i;
+
+	for (i = 0; i < ob->nr_ptrs; i++) {
 		if (!ob_ptr_sectors_free(ob, mi, &ob->ptrs[i]))
 			ob->has_full_ptrs = true;
 		else
 			has_data = true;
 	}
 
+	cache_member_info_put();
+
 	if (likely(has_data))
 		atomic_inc(&ob->pin);
 	else
 		BUG_ON(xchg(&wp->b, NULL) != ob);
 
-	open_bucket_for_each_online_device(c, ob, ptr, ca)
-		this_cpu_add(*ca->sectors_written, sectors);
-	cache_member_info_put();
-
 	mutex_unlock(&ob->lock);
 }
 
@@ -1495,14 +1503,16 @@ struct open_bucket *bch_alloc_sectors(struct cache_set *c,
 {
 	struct open_bucket *ob;
 
-	ob = bch_alloc_sectors_start(c, wp, e, nr_replicas, reserve, cl);
+	ob = bch_alloc_sectors_start(c, wp, nr_replicas, reserve, cl);
 	if (IS_ERR_OR_NULL(ob))
 		return ob;
 
 	if (e->k.size > ob->sectors_free)
 		bch_key_resize(&e->k, ob->sectors_free);
 
-	bch_alloc_sectors_done(c, wp, e, nr_replicas, ob, e->k.size);
+	bch_alloc_sectors_append_ptrs(c, e, nr_replicas, ob, e->k.size);
+
+	bch_alloc_sectors_done(c, wp, ob);
 
 	return ob;
 }
diff --git a/drivers/md/bcache/alloc.h b/drivers/md/bcache/alloc.h
index d3d82fd994be..c39f43e02e80 100644
--- a/drivers/md/bcache/alloc.h
+++ b/drivers/md/bcache/alloc.h
@@ -17,12 +17,13 @@ void bch_open_bucket_put(struct cache_set *, struct open_bucket *);
 
 struct open_bucket *bch_alloc_sectors_start(struct cache_set *,
 					    struct write_point *,
-					    struct bkey_i_extent *,
 					    unsigned, enum alloc_reserve,
 					    struct closure *);
+
+void bch_alloc_sectors_append_ptrs(struct cache_set *, struct bkey_i_extent *,
+				   unsigned, struct open_bucket *, unsigned);
 void bch_alloc_sectors_done(struct cache_set *, struct write_point *,
-			    struct bkey_i_extent *, unsigned,
-			    struct open_bucket *, unsigned);
+			    struct open_bucket *);
 
 struct open_bucket *bch_alloc_sectors(struct cache_set *, struct write_point *,
 				      struct bkey_i_extent *, unsigned,
diff --git a/drivers/md/bcache/compress.c b/drivers/md/bcache/compress.c
index a4f30bc32e64..f7bfd57f9578 100644
--- a/drivers/md/bcache/compress.c
+++ b/drivers/md/bcache/compress.c
@@ -44,13 +44,13 @@ static void *__bio_map_or_bounce(struct cache_set *c,
 	struct bio_vec bv;
 	struct bvec_iter iter;
 	unsigned nr_pages = 0;
-	struct page *stack_pages[4];
+	struct page *stack_pages[16];
 	struct page **pages = NULL;
 	bool first = true;
 	unsigned prev_end = PAGE_SIZE;
 	void *data;
 
-	BUG_ON(start.bi_size > (BCH_COMPRESSED_EXTENT_MAX << 9));
+	BUG_ON(bvec_iter_sectors(start) > BCH_COMPRESSED_EXTENT_MAX);
 
 	*bounced = BOUNCED_MAPPED;
 
@@ -178,13 +178,14 @@ err:
 }
 
 int bch_bio_uncompress_inplace(struct cache_set *c, struct bio *bio,
-			       struct bkey *k, struct bch_extent_crc64 crc)
+			       unsigned live_data_sectors,
+			       struct bch_extent_crc64 crc)
 {
 	void *dst_data = NULL;
 	size_t dst_len = crc.uncompressed_size << 9;
 	int ret = -ENOMEM;
 
-	BUG_ON(DIV_ROUND_UP(k->size, PAGE_SECTORS) > bio->bi_max_vecs);
+	BUG_ON(DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS) > bio->bi_max_vecs);
 
 	/* XXX mempoolify */
 	dst_data = kmalloc(dst_len, GFP_NOIO|__GFP_NOWARN);
@@ -198,7 +199,7 @@ int bch_bio_uncompress_inplace(struct cache_set *c, struct bio *bio,
 	if (ret)
 		goto err;
 
-	while (bio->bi_vcnt < DIV_ROUND_UP(k->size, PAGE_SECTORS)) {
+	while (bio->bi_vcnt < DIV_ROUND_UP(live_data_sectors, PAGE_SECTORS)) {
 		struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt];
 
 		bv->bv_page = alloc_page(GFP_NOIO);
@@ -210,7 +211,7 @@ int bch_bio_uncompress_inplace(struct cache_set *c, struct bio *bio,
 		bio->bi_vcnt++;
 	}
 
-	bio->bi_iter.bi_size = k->size << 9;
+	bio->bi_iter.bi_size = live_data_sectors << 9;
 copy_data:
 	memcpy_to_bio(bio, bio->bi_iter, dst_data + (crc.offset << 9));
 err:
@@ -224,7 +225,7 @@ use_mempool:
 	 */
 
 	bch_bio_free_pages_pool(c, bio);
-	bch_bio_alloc_pages_pool(c, bio, k->size << 9);
+	bch_bio_alloc_pages_pool(c, bio, live_data_sectors << 9);
 	goto copy_data;
 }
 
@@ -252,59 +253,60 @@ err:
 	return ret;
 }
 
-static struct bio *__bio_compress(struct cache_set *c,
-				  unsigned compression_type,
-				  struct bio *src,
-				  unsigned output_available,
-				  int *input_consumed)
+static int __bio_compress(struct cache_set *c,
+			  struct bio *dst, size_t *dst_len,
+			  struct bio *src, size_t *src_len,
+			  unsigned compression_type)
 {
-	struct bio *dst;
 	void *src_data = NULL, *dst_data = NULL;
-	unsigned src_bounced, dst_bounced;
+	unsigned src_bounced, dst_bounced, pad;
 	int ret = -1;
 
-	BUG_ON(output_available > src->bi_iter.bi_size);
-
-	output_available = min_t(unsigned, output_available,
-				 BCH_COMPRESSED_EXTENT_MAX << 9);
-
-	dst = bio_alloc_bioset(GFP_NOIO,
-		DIV_ROUND_UP(output_available, PAGE_SIZE),
-		&c->bio_write);
-
-	bch_bio_alloc_pages_pool(c, dst, output_available);
-
 	dst_data = bio_map_or_bounce(c, dst, &dst_bounced, WRITE);
 	src_data = bio_map_or_bounce(c, src, &src_bounced, READ);
 
 	switch (compression_type) {
 	case BCH_COMPRESSION_LZ4: {
 		void *workspace;
-		size_t dst_size = dst->bi_iter.bi_size;
+
+		*dst_len = dst->bi_iter.bi_size;
+		*src_len = src->bi_iter.bi_size;
 
 		workspace = mempool_alloc(&c->lz4_workspace_pool, GFP_NOIO);
+retry_compress:
+		ret = lz4_compress(src_data, *src_len,
+				   dst_data, dst_len,
+				   workspace);
 		/*
-		 * XXX: due to the way the interface to lz4_compress works, we
-		 * can't consume more than output_available bytes of input (even
-		 * though a lot more might fit after compressing)
+		 * On error, the compressed data was bigger than dst_len, and
+		 * -ret is the amount of data we were able to compress - round
+		 * down to nearest block and try again:
 		 */
-		ret = lz4_compress(src_data, min(output_available,
-						 src->bi_iter.bi_size),
-				   dst_data, &dst_size, workspace);
+		if (ret && round_down(-ret, block_bytes(c)) > *dst_len) {
+			BUG_ON(ret > 0);
+
+			/* not supposed to happen */
+			if (WARN_ON(-ret >= *src_len))
+				goto err;
 
+			*src_len = round_down(-ret, block_bytes(c));
+			if (!*src_len)
+				goto err;
+
+			goto retry_compress;
+		}
 		mempool_free(workspace, &c->lz4_workspace_pool);
+
 		if (ret)
 			goto err;
-
-		dst->bi_iter.bi_size = dst_size;
-		*input_consumed = output_available;
 		break;
 	}
 	case BCH_COMPRESSION_GZIP: {
 		void *workspace;
 		z_stream strm;
 
-		workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS, DEF_MEM_LEVEL),
+		workspace = kmalloc(zlib_deflate_workspacesize(MAX_WBITS,
+							       DEF_MEM_LEVEL),
 				    GFP_NOIO|__GFP_NOWARN);
 		if (!workspace) {
 			mutex_lock(&c->zlib_workspace_lock);
@@ -313,9 +315,10 @@ static struct bio *__bio_compress(struct cache_set *c,
 
 		strm.workspace	= workspace;
 		strm.next_in	= src_data;
-		strm.avail_in	= output_available;
+		strm.avail_in	= min(src->bi_iter.bi_size,
+				      dst->bi_iter.bi_size);
 		strm.next_out	= dst_data;
-		strm.avail_out	= output_available;
+		strm.avail_out	= dst->bi_iter.bi_size;
 		zlib_deflateInit2(&strm, Z_DEFAULT_COMPRESSION,
 				  Z_DEFLATED, -MAX_WBITS, DEF_MEM_LEVEL,
 				  Z_DEFAULT_STRATEGY);
@@ -342,100 +345,64 @@ zlib_err:
 		if (ret)
 			goto err;
 
-		BUG_ON(strm.total_in != output_available);
-
-		dst->bi_iter.bi_size = strm.total_out;
-		*input_consumed = strm.total_in;
+		*dst_len = strm.total_out;
+		*src_len = strm.total_in;
 		break;
 	}
 	default:
 		BUG();
 	}
 
-	BUG_ON(!dst->bi_iter.bi_size);
+	BUG_ON(!*dst_len);
+
+	/* Didn't get smaller: */
+	if (round_up(*dst_len, block_bytes(c)) >= *src_len) {
+		ret = -1;
+		goto err;
+	}
+
+	pad = round_up(*dst_len, block_bytes(c)) - *dst_len;
+
+	memset(dst_data + *dst_len, 0, pad);
+	*dst_len += pad;
 
 	if (dst_bounced)
 		memcpy_to_bio(dst, dst->bi_iter, dst_data);
-out:
+err:
 	bio_unmap_or_unbounce(c, src_data, src_bounced, READ);
 	bio_unmap_or_unbounce(c, dst_data, dst_bounced, WRITE);
-
-	if (!ret)
-		while (dst->bi_vcnt * PAGE_SIZE >
-		       round_up(dst->bi_iter.bi_size, PAGE_SIZE))
-			mempool_free(dst->bi_io_vec[--dst->bi_vcnt].bv_page,
-				     &c->bio_bounce_pages);
-
-	return dst;
-err:
-	ret = -1;
-	*input_consumed = -1;
-	goto out;
+	return ret;
 }
 
-struct bio *bch_bio_compress(struct cache_set *c, struct bio *src,
-			     unsigned *compression_type,
-			     unsigned output_available)
+void bch_bio_compress(struct cache_set *c,
+		      struct bio *dst, size_t *dst_len,
+		      struct bio *src, size_t *src_len,
+		      unsigned *compression_type)
 {
-	struct bio *dst = NULL;
-	int input_consumed;
-
-	/* if it's only one block, don't bother trying to compress: */
-	if (bio_sectors(src) <= c->sb.block_size)
-		*compression_type = BCH_COMPRESSION_NONE;
-
-	switch (*compression_type) {
-		case BCH_COMPRESSION_NONE:
-			/* Just bounce it, for stable checksums: */
-copy:
-			if (!dst)
-				dst = bio_alloc_bioset(GFP_NOIO,
-						       DIV_ROUND_UP(output_available, PAGE_SIZE),
-						       &c->bio_write);
-			bch_bio_alloc_pages_pool(c, dst, output_available);
-			bio_copy_data(dst, src);
-			input_consumed = output_available;
-			goto advance;
-		case BCH_COMPRESSION_LZ4:
-		case BCH_COMPRESSION_GZIP:
-			dst = __bio_compress(c, *compression_type, src,
-					     output_available, &input_consumed);
-			break;
-		default:
-			BUG();
-	}
-
-	if ((int) round_up(dst->bi_iter.bi_size,
-			   block_bytes(c)) >= input_consumed) {
-		/* Failed to compress (didn't get smaller): */
-		*compression_type = BCH_COMPRESSION_NONE;
-		goto copy;
-	}
-
-	/* Pad to blocksize, and zero out padding: */
-	while (dst->bi_iter.bi_size & (block_bytes(c) - 1)) {
-		unsigned idx = dst->bi_iter.bi_size >> PAGE_SHIFT;
-		unsigned offset = dst->bi_iter.bi_size & (PAGE_SIZE - 1);
-		unsigned bytes = (PAGE_SIZE - offset) & (block_bytes(c) - 1);
-
-		if (idx < dst->bi_vcnt) {
-			struct bio_vec *bv = &dst->bi_io_vec[idx];
-
-			memset(page_address(bv->bv_page) + offset, 0, bytes);
-		} else {
-			dst->bi_io_vec[dst->bi_vcnt++] = (struct bio_vec) {
-				.bv_page	= ZERO_PAGE(0),
-					.bv_len		= PAGE_SIZE,
-					.bv_offset	= 0,
-			};
-		}
-
-		dst->bi_iter.bi_size += bytes;
-	}
-advance:
-	bio_advance(src, input_consumed);
-
-	return dst;
+	unsigned orig_dst = dst->bi_iter.bi_size;
+	unsigned orig_src = src->bi_iter.bi_size;
+
+	/* Don't consume more than BCH_COMPRESSED_EXTENT_MAX from @src: */
+	src->bi_iter.bi_size =
+		min(src->bi_iter.bi_size, BCH_COMPRESSED_EXTENT_MAX << 9);
+
+	/* Don't generate a bigger output than input: */
+	dst->bi_iter.bi_size =
+		min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
+
+	/* If it's only one block, don't bother trying to compress: */
+	if (*compression_type != BCH_COMPRESSION_NONE &&
+	    bio_sectors(src) > c->sb.block_size &&
+	    !__bio_compress(c, dst, dst_len, src, src_len, *compression_type))
+		goto out;
+
+	/* If compressing failed (didn't get smaller), just copy: */
+	*compression_type = BCH_COMPRESSION_NONE;
+	*dst_len = *src_len = min(dst->bi_iter.bi_size, src->bi_iter.bi_size);
+	bio_copy_data(dst, src);
+out:
+	dst->bi_iter.bi_size = orig_dst;
+	src->bi_iter.bi_size = orig_src;
 }
 
 void bch_compress_free(struct cache_set *c)
diff --git a/drivers/md/bcache/compress.h b/drivers/md/bcache/compress.h
index 2b5dded6f59f..02578ef73c9a 100644
--- a/drivers/md/bcache/compress.h
+++ b/drivers/md/bcache/compress.h
@@ -2,11 +2,11 @@
 #define _BCACHE_COMPRESS_H
 
 int bch_bio_uncompress_inplace(struct cache_set *, struct bio *,
-			       struct bkey *, struct bch_extent_crc64);
+			       unsigned, struct bch_extent_crc64);
 int bch_bio_uncompress(struct cache_set *, struct bio *, struct bio *,
 		       struct bvec_iter, struct bch_extent_crc64);
-struct bio *bch_bio_compress(struct cache_set *, struct bio *,
-			     unsigned *, unsigned);
+void bch_bio_compress(struct cache_set *, struct bio *, size_t *,
+		      struct bio *, size_t *, unsigned *);
 
 void bch_compress_free(struct cache_set *);
 int bch_compress_init(struct cache_set *);
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index d0dd6df459d6..7219b658dd05 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -380,14 +380,41 @@ static void bch_write_endio(struct bio *bio)
 		closure_put(cl);
 }
 
+static void init_append_extent(struct bch_write_op *op,
+			       unsigned compressed_size,
+			       unsigned uncompressed_size,
+			       unsigned compression_type,
+			       u64 csum, unsigned csum_type,
+			       struct open_bucket *ob)
+{
+	struct bkey_i_extent *e = bkey_extent_init(op->insert_keys.top);
+
+	op->pos.offset += uncompressed_size;
+	e->k.p = op->pos;
+	e->k.size = uncompressed_size;
+
+	bch_extent_crc_append(e, compressed_size,
+			      uncompressed_size,
+			      compression_type,
+			      csum, csum_type);
+
+	bch_alloc_sectors_append_ptrs(op->c, e, op->nr_replicas,
+				      ob, compressed_size);
+
+	bkey_extent_set_cached(&e->k, (op->flags & BCH_WRITE_CACHED));
+	bch_keylist_push(&op->insert_keys);
+}
+
 static int bch_write_extent(struct bch_write_op *op,
 			    struct open_bucket *ob,
-			    struct bkey_i_extent *e,
 			    struct bio *orig)
 {
 	struct cache_set *c = op->c;
 	struct bio *bio;
 	struct bch_write_bio *wbio;
+	unsigned key_to_write_offset = op->insert_keys.top_p -
+		op->insert_keys.keys_p;
+	struct bkey_i *key_to_write;
 	unsigned csum_type = c->opts.data_checksum;
 	unsigned compression_type = op->compression_type;
 	int ret;
@@ -397,11 +424,11 @@ static int bch_write_extent(struct bch_write_op *op,
 
 	/* Need to decompress data? */
 	if ((op->flags & BCH_WRITE_DATA_COMPRESSED) &&
-	    (op->crc.uncompressed_size != e->k.size ||
+	    (op->crc.uncompressed_size != op->size ||
 	     op->crc.compressed_size > ob->sectors_free)) {
 		int ret;
 
-		ret = bch_bio_uncompress_inplace(c, orig, &e->k, op->crc);
+		ret = bch_bio_uncompress_inplace(c, orig, op->size, op->crc);
 		if (ret)
 			return ret;
 
@@ -409,15 +436,13 @@ static int bch_write_extent(struct bch_write_op *op,
 	}
 
 	if (op->flags & BCH_WRITE_DATA_COMPRESSED) {
-		bch_extent_crc_append(e,
-				      op->crc.compressed_size,
-				      op->crc.uncompressed_size,
-				      op->crc.compression_type,
-				      op->crc.csum,
-				      op->crc.csum_type);
-		bch_alloc_sectors_done(op->c, op->wp,
-				       e, op->nr_replicas,
-				       ob, bio_sectors(orig));
+		init_append_extent(op,
+				   op->crc.compressed_size,
+				   op->crc.uncompressed_size,
+				   op->crc.compression_type,
+				   op->crc.csum,
+				   op->crc.csum_type,
+				   ob);
 
 		bio			= orig;
 		wbio			= to_wbio(bio);
@@ -428,90 +453,84 @@ static int bch_write_extent(struct bch_write_op *op,
 	} else if (csum_type != BCH_CSUM_NONE ||
 		   compression_type != BCH_COMPRESSION_NONE) {
 		/* all units here in bytes */
-		unsigned output_available, extra_input,
-			 orig_input = orig->bi_iter.bi_size;
+		unsigned total_output = 0, output_available =
+			min(ob->sectors_free << 9, orig->bi_iter.bi_size);
 		u64 csum;
 
-		/* XXX: decide extent size better: */
-		output_available = min(e->k.size,
-				   min(ob->sectors_free,
-				       CRC32_EXTENT_SIZE_MAX)) << 9;
-
+		bio = bio_alloc_bioset(GFP_NOIO,
+				       DIV_ROUND_UP(output_available, PAGE_SIZE),
+				       &c->bio_write);
 		/*
-		 * temporarily set input bio's size to the max we want to
-		 * consume from it, in order to avoid overflow in the crc info
+		 * XXX: can't use mempool for more than
+		 * BCH_COMPRESSED_EXTENT_MAX worth of pages
 		 */
-		extra_input = orig->bi_iter.bi_size > CRC32_EXTENT_SIZE_MAX << 9
-			? orig->bi_iter.bi_size - (CRC32_EXTENT_SIZE_MAX << 9)
-			: 0;
-		orig->bi_iter.bi_size -= extra_input;
-
-		bio = bch_bio_compress(c, orig,
-				       &compression_type,
-				       output_available);
+		bch_bio_alloc_pages_pool(c, bio, output_available);
+
 		/* copy WRITE_SYNC flag */
 		bio->bi_opf		= orig->bi_opf;
-
-		orig->bi_iter.bi_size += extra_input;
-
-		bio->bi_end_io		= bch_write_endio;
 		wbio			= to_wbio(bio);
 		wbio->orig		= NULL;
 		wbio->bounce		= true;
 		wbio->put_bio		= true;
 
-		/*
-		 * Set the (uncompressed) size of the key we're creating to the
-		 * number of sectors we consumed from orig:
-		 */
-		bch_key_resize(&e->k, (orig_input - orig->bi_iter.bi_size) >> 9);
+		do {
+			unsigned fragment_compression_type = compression_type;
+			size_t dst_len, src_len;
 
-		/*
-		 * XXX: could move checksumming out from under the open
-		 * bucket lock - but compression is also being done
-		 * under it
-		 */
-		csum = bch_checksum_bio(bio, csum_type);
-#if 0
-		if (compression_type != BCH_COMPRESSION_NONE)
-			pr_info("successfully compressed %u -> %u",
-				e->k.size, bio_sectors(bio));
-#endif
-		/*
-		 * Add a bch_extent_crc header for the pointers that
-		 * bch_alloc_sectors_done() is going to append:
-		 */
-		bch_extent_crc_append(e, bio_sectors(bio), e->k.size,
-				      compression_type,
-				      csum, csum_type);
-		bch_alloc_sectors_done(op->c, op->wp,
-				       e, op->nr_replicas,
-				       ob, bio_sectors(bio));
+			bch_bio_compress(c, bio, &dst_len,
+					 orig, &src_len,
+					 &fragment_compression_type);
 
-		ret = orig->bi_iter.bi_size != 0;
-	} else {
-		if (e->k.size > ob->sectors_free)
-			bch_key_resize(&e->k, ob->sectors_free);
+			BUG_ON(!dst_len || dst_len > bio->bi_iter.bi_size);
+			BUG_ON(!src_len || src_len > orig->bi_iter.bi_size);
+			BUG_ON(dst_len & (block_bytes(c) - 1));
+			BUG_ON(src_len & (block_bytes(c) - 1));
+
+			swap(bio->bi_iter.bi_size, dst_len);
+			csum = bch_checksum_bio(bio, csum_type);
+			swap(bio->bi_iter.bi_size, dst_len);
+
+			init_append_extent(op,
+					   dst_len >> 9, src_len >> 9,
+					   fragment_compression_type,
+					   csum, csum_type, ob);
+
+			total_output += dst_len;
+			bio_advance(bio, dst_len);
+			bio_advance(orig, src_len);
+		} while (bio->bi_iter.bi_size &&
+			 orig->bi_iter.bi_size &&
+			 !bch_keylist_realloc(&op->insert_keys,
+					      op->inline_keys,
+					      ARRAY_SIZE(op->inline_keys),
+					      BKEY_EXTENT_U64s_MAX));
+
+		BUG_ON(total_output > output_available);
+
+		memset(&bio->bi_iter, 0, sizeof(bio->bi_iter));
+		bio->bi_iter.bi_size = total_output;
 
-		BUG_ON(e->k.size > ob->sectors_free);
 		/*
-		 * We might need a checksum entry, if there's a previous
-		 * checksum entry we need to override:
+		 * Free unneeded pages after compressing:
 		 */
-		bch_extent_crc_append(e, e->k.size, e->k.size,
-				      compression_type, 0, csum_type);
-		bch_alloc_sectors_done(op->c, op->wp,
-				       e, op->nr_replicas,
-				       ob, e->k.size);
+		while (bio->bi_vcnt * PAGE_SIZE >
+		       round_up(bio->bi_iter.bi_size, PAGE_SIZE))
+			mempool_free(bio->bi_io_vec[--bio->bi_vcnt].bv_page,
+				     &c->bio_bounce_pages);
 
-		bio = bio_next_split(orig, e->k.size, GFP_NOIO,
-				     &op->c->bio_write);
+		ret = orig->bi_iter.bi_size != 0;
+	} else {
+		bio = bio_next_split(orig, ob->sectors_free, GFP_NOIO,
+				     &c->bio_write);
 
 		wbio			= to_wbio(bio);
 		wbio->orig		= NULL;
 		wbio->bounce		= false;
 		wbio->put_bio		= bio != orig;
 
+		init_append_extent(op, bio_sectors(bio), bio_sectors(bio),
+				   compression_type, 0, csum_type, ob);
+
 		ret = bio != orig;
 	}
 
@@ -520,10 +539,18 @@ static int bch_write_extent(struct bch_write_op *op,
 	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 
 	closure_get(bio->bi_private);
+
+	/* might have done a realloc... */
+
+	key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
+
+	if (!(op->flags & BCH_WRITE_CACHED))
+		bch_check_mark_super(c, key_to_write, false);
+
 #ifndef CONFIG_BCACHE_NO_IO
-	bch_submit_wbio_replicas(wbio, op->c, &e->k_i, false);
+	bch_submit_wbio_replicas(to_wbio(bio), c, key_to_write, false);
 #else
-	wbio->ca = NULL;
+	to_wbio(bio)->ca = NULL;
 	bio_endio(bio);
 #endif
 	return ret;
@@ -554,8 +581,6 @@ static void __bch_write(struct closure *cl)
 	bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA);
 
 	do {
-		struct bkey_i *k;
-
 		EBUG_ON(bio->bi_iter.bi_sector != op->pos.offset);
 		EBUG_ON(!bio_sectors(bio));
 
@@ -569,16 +594,7 @@ static void __bch_write(struct closure *cl)
 					BKEY_EXTENT_U64s_MAX))
 			continue_at(cl, bch_write_index, index_update_wq(op));
 
-		k = op->insert_keys.top;
-		bkey_extent_init(k);
-		k->k.p = op->pos;
-		bch_key_resize(&k->k,
-			       (op->flags & BCH_WRITE_DATA_COMPRESSED)
-			       ? op->size
-			       : bio_sectors(bio));
-
-		b = bch_alloc_sectors_start(c, op->wp,
-			bkey_i_to_extent(k), op->nr_replicas,
+		b = bch_alloc_sectors_start(c, op->wp, op->nr_replicas,
 			op->alloc_reserve,
 			(op->flags & BCH_WRITE_ALLOC_NOWAIT) ? NULL : cl);
 		EBUG_ON(!b);
@@ -622,20 +638,12 @@ static void __bch_write(struct closure *cl)
 		       b - c->open_buckets > U8_MAX);
 		op->open_buckets[open_bucket_nr++] = b - c->open_buckets;
 
-		ret = bch_write_extent(op, b, bkey_i_to_extent(k), bio);
-		if (ret < 0)
-			goto err;
-
-		op->pos.offset += k->k.size;
-
-		bkey_extent_set_cached(&k->k, (op->flags & BCH_WRITE_CACHED));
-
-		if (!(op->flags & BCH_WRITE_CACHED))
-			bch_check_mark_super(c, k, false);
+		ret = bch_write_extent(op, b, bio);
 
-		bch_keylist_push(&op->insert_keys);
+		bch_alloc_sectors_done(c, op->wp, b);
 
-		trace_bcache_cache_insert(&k->k);
+		if (ret < 0)
+			goto err;
 	} while (ret);
 
 	op->flags |= BCH_WRITE_DONE;
@@ -733,7 +741,7 @@ void bch_write(struct closure *cl)
 	if (!percpu_ref_tryget(&c->writes)) {
 		__bcache_io_error(c, "read only");
 		op->error = -EROFS;
-		bch_disk_reservation_put(op->c, &op->res);
+		bch_disk_reservation_put(c, &op->res);
 		closure_return(cl);
 	}
 
diff --git a/include/trace/events/bcache.h b/include/trace/events/bcache.h
index 4718009bf981..d4968c542849 100644
--- a/include/trace/events/bcache.h
+++ b/include/trace/events/bcache.h
@@ -235,11 +235,6 @@ DEFINE_EVENT(bcache_bio, bcache_read_retry,
 	TP_ARGS(bio)
 );
 
-DEFINE_EVENT(bkey, bcache_cache_insert,
-	TP_PROTO(const struct bkey *k),
-	TP_ARGS(k)
-);
-
 DECLARE_EVENT_CLASS(page_alloc_fail,
 	TP_PROTO(struct cache_set *c, u64 size),
 	TP_ARGS(c, size),
diff --git a/include/uapi/linux/bcache.h b/include/uapi/linux/bcache.h
index 2f1cc61be66e..44de20990b56 100644
--- a/include/uapi/linux/bcache.h
+++ b/include/uapi/linux/bcache.h
@@ -387,7 +387,7 @@ struct bch_extent_crc32 {
 #define CRC32_EXTENT_SIZE_MAX	(1U << 7)
 
 /* 64k */
-#define BCH_COMPRESSED_EXTENT_MAX 128
+#define BCH_COMPRESSED_EXTENT_MAX 128U
 
 struct bch_extent_crc64 {
 #if defined(__LITTLE_ENDIAN_BITFIELD)