bcache: strip another allocation out of the read path

15 files changed, 203 insertions, 112 deletions

diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index b53bcd7ae2dc..d5336b3e3691 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -709,6 +709,7 @@ struct cache_set {
 
 	/* IO PATH */
 	struct bio_set		bio_read;
+	struct bio_set		bio_read_split;
 	struct bio_set		bio_write;
 	struct mutex		bio_bounce_pages_lock;
 	mempool_t		bio_bounce_pages;
diff --git a/drivers/md/bcache/blockdev.h b/drivers/md/bcache/blockdev.h
index 2a2100cacc17..0fc0ed1b80c4 100644
--- a/drivers/md/bcache/blockdev.h
+++ b/drivers/md/bcache/blockdev.h
@@ -57,7 +57,10 @@ struct search {
 	/* Stack frame for bio_complete */
 	struct closure		cl;
 
-	struct bch_write_bio	bio;
+	union {
+	struct bch_read_bio	rbio;
+	struct bch_write_bio	wbio;
+	};
 	/* Not modified */
 	struct bio		*orig_bio;
 	struct bcache_device	*d;
diff --git a/drivers/md/bcache/fs-io.c b/drivers/md/bcache/fs-io.c
index 7cc0a5690fa8..8da0a551b1a0 100644
--- a/drivers/md/bcache/fs-io.c
+++ b/drivers/md/bcache/fs-io.c
@@ -635,8 +635,9 @@ static void bch_add_page_sectors(struct bio *bio, const struct bkey *k)
 	}
 }
 
-static void bchfs_read(struct cache_set *c, struct bio *bio, u64 inode)
+static void bchfs_read(struct cache_set *c, struct bch_read_bio *rbio, u64 inode)
 {
+	struct bio *bio = &rbio->bio;
 	struct btree_iter iter;
 	struct bkey_s_c k;
 	struct bio_vec *bv;
@@ -698,7 +699,7 @@ static void bchfs_read(struct cache_set *c, struct bio *bio, u64 inode)
 			PTR_BUCKET(pick.ca, &pick.ptr)->read_prio =
 				c->prio_clock[READ].hand;
 
-			bch_read_extent(c, bio, k, &pick,
+			bch_read_extent(c, rbio, k, &pick,
 					bio->bi_iter.bi_sector -
 					bkey_start_offset(k.k),
 					BCH_READ_FORCE_BOUNCE|
@@ -733,30 +734,32 @@ int bch_readpages(struct file *file, struct address_space *mapping,
 {
 	struct inode *inode = mapping->host;
 	struct cache_set *c = inode->i_sb->s_fs_info;
-	struct bio *bio = NULL;
+	struct bch_read_bio *rbio = NULL;
 	struct page *page;
 
 	pr_debug("reading %u pages", nr_pages);
 
 	for_each_readpage_page(mapping, pages, nr_pages, page) {
 again:
-		if (!bio) {
-			bio = bio_alloc(GFP_NOFS,
-					min_t(unsigned, nr_pages,
-					      BIO_MAX_PAGES));
-
-			bio->bi_end_io = bch_readpages_end_io;
+		if (!rbio) {
+			rbio = container_of(bio_alloc_bioset(GFP_NOFS,
+						min_t(unsigned, nr_pages,
+						      BIO_MAX_PAGES),
+						&c->bio_read),
+					   struct bch_read_bio, bio);
+
+			rbio->bio.bi_end_io = bch_readpages_end_io;
 		}
 
-		if (bch_bio_add_page(bio, page)) {
-			bchfs_read(c, bio, inode->i_ino);
-			bio = NULL;
+		if (bch_bio_add_page(&rbio->bio, page)) {
+			bchfs_read(c, rbio, inode->i_ino);
+			rbio = NULL;
 			goto again;
 		}
 	}
 
-	if (bio)
-		bchfs_read(c, bio, inode->i_ino);
+	if (rbio)
+		bchfs_read(c, rbio, inode->i_ino);
 
 	pr_debug("success");
 	return 0;
@@ -767,14 +770,16 @@ int bch_readpage(struct file *file, struct page *page)
 	struct address_space *mapping = page->mapping;
 	struct inode *inode = mapping->host;
 	struct cache_set *c = inode->i_sb->s_fs_info;
-	struct bio *bio;
+	struct bch_read_bio *rbio;
 
-	bio = bio_alloc(GFP_NOFS, 1);
-	bio_set_op_attrs(bio, REQ_OP_READ, REQ_SYNC);
-	bio->bi_end_io = bch_readpages_end_io;
+	rbio = container_of(bio_alloc_bioset(GFP_NOFS, 1,
+					    &c->bio_read),
+			   struct bch_read_bio, bio);
+	bio_set_op_attrs(&rbio->bio, REQ_OP_READ, REQ_SYNC);
+	rbio->bio.bi_end_io = bch_readpages_end_io;
 
-	bch_bio_add_page(bio, page);
-	bchfs_read(c, bio, inode->i_ino);
+	bch_bio_add_page(&rbio->bio, page);
+	bchfs_read(c, rbio, inode->i_ino);
 
 	return 0;
 }
@@ -1047,22 +1052,24 @@ static int bch_read_single_page(struct page *page,
 {
 	struct inode *inode = mapping->host;
 	struct cache_set *c = inode->i_sb->s_fs_info;
-	struct bio *bio;
+	struct bch_read_bio *rbio;
 	int ret = 0;
 	DECLARE_COMPLETION_ONSTACK(done);
 
-	bio = bio_alloc(GFP_NOFS, 1);
-	bio_set_op_attrs(bio, REQ_OP_READ, REQ_SYNC);
-	bio->bi_private = &done;
-	bio->bi_end_io = bch_read_single_page_end_io;
-	bch_bio_add_page(bio, page);
+	rbio = container_of(bio_alloc_bioset(GFP_NOFS, 1,
+					     &c->bio_read),
+			    struct bch_read_bio, bio);
+	bio_set_op_attrs(&rbio->bio, REQ_OP_READ, REQ_SYNC);
+	rbio->bio.bi_private = &done;
+	rbio->bio.bi_end_io = bch_read_single_page_end_io;
+	bch_bio_add_page(&rbio->bio, page);
 
-	bchfs_read(c, bio, inode->i_ino);
+	bchfs_read(c, rbio, inode->i_ino);
 	wait_for_completion(&done);
 
 	if (!ret)
-		ret = bio->bi_error;
-	bio_put(bio);
+		ret = rbio->bio.bi_error;
+	bio_put(&rbio->bio);
 
 	if (ret < 0)
 		return ret;
@@ -1218,7 +1225,7 @@ static void bch_dio_read_complete(struct closure *cl)
 	struct dio_read *dio = container_of(cl, struct dio_read, cl);
 
 	dio->req->ki_complete(dio->req, dio->ret, 0);
-	bio_put(&dio->bio);
+	bio_put(&dio->rbio.bio);
 }
 
 static void bch_direct_IO_read_endio(struct bio *bio)
@@ -1247,7 +1254,7 @@ static int bch_direct_IO_read(struct cache_set *c, struct kiocb *req,
 	bio = bio_alloc_bioset(GFP_KERNEL, pages, bch_dio_read_bioset);
 	bio_get(bio);
 
-	dio = container_of(bio, struct dio_read, bio);
+	dio = container_of(bio, struct dio_read, rbio.bio);
 	closure_init(&dio->cl, NULL);
 
 	/*
@@ -1282,7 +1289,7 @@ static int bch_direct_IO_read(struct cache_set *c, struct kiocb *req,
 	goto start;
 	while (iter->count) {
 		pages = iov_iter_npages(iter, BIO_MAX_PAGES);
-		bio = bio_alloc(GFP_KERNEL, pages);
+		bio = bio_alloc_bioset(GFP_KERNEL, pages, &c->bio_read);
 start:
 		bio_set_op_attrs(bio, REQ_OP_READ, REQ_SYNC);
 		bio->bi_iter.bi_sector	= offset >> 9;
@@ -1303,14 +1310,16 @@ start:
 		if (iter->count)
 			closure_get(&dio->cl);
 
-		bch_read(c, bio, inum);
+		bch_read(c, container_of(bio,
+				struct bch_read_bio, bio),
+			 inum);
 	}
 out:
 	if (sync) {
 		closure_sync(&dio->cl);
 		closure_debug_destroy(&dio->cl);
 		ret = dio->ret;
-		bio_put(&dio->bio);
+		bio_put(&dio->rbio.bio);
 		return ret;
 	} else {
 		return -EIOCBQUEUED;
diff --git a/drivers/md/bcache/fs-io.h b/drivers/md/bcache/fs-io.h
index c5681dd5cde4..bafbdd54fb45 100644
--- a/drivers/md/bcache/fs-io.h
+++ b/drivers/md/bcache/fs-io.h
@@ -87,7 +87,7 @@ struct dio_read {
 	struct closure		cl;
 	struct kiocb		*req;
 	long			ret;
-	struct bio		bio;
+	struct bch_read_bio	rbio;
 };
 
 extern struct bio_set *bch_dio_read_bioset;
diff --git a/drivers/md/bcache/fs.c b/drivers/md/bcache/fs.c
index b9919627245c..e1374d15dac8 100644
--- a/drivers/md/bcache/fs.c
+++ b/drivers/md/bcache/fs.c
@@ -1466,8 +1466,7 @@ int __init bch_fs_init(void)
 	if (!bch_writepage_bioset)
 		goto err;
 
-
-	bch_dio_read_bioset = bioset_create(4, offsetof(struct dio_read, bio));
+	bch_dio_read_bioset = bioset_create(4, offsetof(struct dio_read, rbio.bio));
 	if (!bch_dio_read_bioset)
 		goto err;
 
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index bd9304ccaaf3..792c31964ccd 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -29,7 +29,7 @@
 
 static inline void __bio_inc_remaining(struct bio *bio)
 {
-	bio->bi_flags |= (1 << BIO_CHAIN);
+	bio_set_flag(bio, BIO_CHAIN);
 	smp_mb__before_atomic();
 	atomic_inc(&bio->__bi_remaining);
 }
@@ -1479,7 +1479,7 @@ static int bio_checksum_uncompress(struct cache_set *c,
 				   struct bch_read_bio *rbio)
 {
 	struct bio *src = &rbio->bio;
-	struct bio *dst = rbio->parent;
+	struct bio *dst = &bch_rbio_parent(rbio)->bio;
 	struct bvec_iter dst_iter = rbio->parent_iter;
 	int ret = 0;
 
@@ -1489,9 +1489,13 @@ static int bio_checksum_uncompress(struct cache_set *c,
 	 * which was not necessarily the full extent if we were only bouncing
 	 * in order to promote
 	 */
-	src->bi_iter.bi_size		= rbio->crc.compressed_size << 9;
-	src->bi_iter.bi_idx		= 0;
-	src->bi_iter.bi_bvec_done	= 0;
+	if (rbio->bounce) {
+		src->bi_iter.bi_size		= rbio->crc.compressed_size << 9;
+		src->bi_iter.bi_idx		= 0;
+		src->bi_iter.bi_bvec_done	= 0;
+	} else {
+		src->bi_iter = rbio->parent_iter;
+	}
 
 	if (rbio->crc.csum_type != BCH_CSUM_NONE &&
 	    rbio->crc.csum != checksum_bio(src, rbio->crc.csum_type)) {
@@ -1535,6 +1539,7 @@ static void bch_rbio_free(struct cache_set *c, struct bch_read_bio *rbio)
 	struct bio *bio = &rbio->bio;
 
 	BUG_ON(rbio->ca);
+	BUG_ON(!rbio->split);
 
 	if (rbio->promote)
 		kfree(rbio->promote);
@@ -1546,10 +1551,21 @@ static void bch_rbio_free(struct cache_set *c, struct bch_read_bio *rbio)
 
 static void bch_rbio_done(struct cache_set *c, struct bch_read_bio *rbio)
 {
+	struct bio *orig = &bch_rbio_parent(rbio)->bio;
+
 	percpu_ref_put(&rbio->ca->ref);
 	rbio->ca = NULL;
-	bio_endio(rbio->parent);
-	bch_rbio_free(c, rbio);
+
+	if (rbio->split) {
+		bio_endio(orig);
+		bch_rbio_free(c, rbio);
+	} else {
+		if (rbio->promote)
+			kfree(rbio->promote);
+
+		orig->bi_end_io = rbio->orig_bi_end_io;
+		bio_endio_nodec(orig);
+	}
 }
 
 /*
@@ -1572,7 +1588,7 @@ static void bch_read_error_maybe_retry(struct cache_set *c,
 		/* io error - do we have another replica? */
 	}
 
-	rbio->parent->bi_error = error;
+	bch_rbio_parent(rbio)->bio.bi_error = error;
 	bch_rbio_done(c, rbio);
 	return;
 retry:
@@ -1609,10 +1625,12 @@ static void __bch_read_endio(struct cache_set *c, struct bch_read_bio *rbio)
 	    !test_bit(CACHE_SET_STOPPING, &c->flags)) {
 		struct closure *cl = &rbio->promote->cl;
 
+		BUG_ON(!rbio->split || !rbio->bounce);
+
 		percpu_ref_put(&rbio->ca->ref);
 		rbio->ca = NULL;
 
-		bio_endio(rbio->parent);
+		bio_endio(&rbio->parent->bio);
 		rbio->parent = NULL;
 
 		closure_init(cl, &c->cl);
@@ -1672,14 +1690,15 @@ static void bch_read_endio(struct bio *bio)
 	}
 }
 
-void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
+void bch_read_extent_iter(struct cache_set *c, struct bch_read_bio *orig,
 			  struct bvec_iter iter, struct bkey_s_c k,
 			  struct extent_pick_ptr *pick,
 			  unsigned skip, unsigned flags)
 {
 	struct bch_read_bio *rbio;
 	struct cache_promote_op *promote_op = NULL;
-	bool bounce = false, read_full = false;
+	unsigned orig_sectors = bio_sectors(&orig->bio);
+	bool bounce = false, split, read_full = false;
 
 	/* only promote if we're not reading from the fastest tier: */
 	if ((flags & BCH_READ_PROMOTE) && pick->ca->mi.tier) {
@@ -1695,7 +1714,7 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
 	 */
 	if (pick->crc.compression_type != BCH_COMPRESSION_NONE ||
 	    (pick->crc.csum_type != BCH_CSUM_NONE &&
-	     (bio_sectors(orig) != pick->crc.uncompressed_size ||
+	     (orig_sectors != pick->crc.uncompressed_size ||
 	      (flags & BCH_READ_FORCE_BOUNCE)))) {
 		read_full = true;
 		bounce = true;
@@ -1704,29 +1723,50 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
 	if (bounce) {
 		unsigned sectors = read_full
 			? (pick->crc.compressed_size ?: k.k->size)
-			: bio_sectors(orig);
+			: orig_sectors;
 
 		rbio = container_of(bio_alloc_bioset(GFP_NOIO,
 					DIV_ROUND_UP(sectors, PAGE_SECTORS),
-					&c->bio_read),
+					&c->bio_read_split),
 				    struct bch_read_bio, bio);
 
 		bch_bio_alloc_pages_pool(c, &rbio->bio, sectors << 9);
-	} else {
-		rbio = container_of(bio_clone_fast(orig,
-					GFP_NOIO, &c->bio_read),
+		split = true;
+	} else if (!(flags & BCH_READ_MAY_REUSE_BIO) ||
+		   !(flags & BCH_READ_IS_LAST)) {
+		/*
+		 * Have to clone if there were any splits, due to error
+		 * reporting issues (if a split errored, and retrying didn't
+		 * work, when it reports the error to its parent (us) we don't
+		 * know if the error was from our bio, and we should retry, or
+		 * from the whole bio, in which case we don't want to retry and
+		 * lose the error)
+		 */
+		rbio = container_of(bio_clone_fast(&orig->bio,
+					GFP_NOIO, &c->bio_read_split),
 				    struct bch_read_bio, bio);
 		rbio->bio.bi_iter = iter;
+		split = true;
+	} else {
+		rbio = orig;
+		rbio->bio.bi_iter = iter;
+		split = false;
+		BUG_ON(bio_flagged(&rbio->bio, BIO_CHAIN));
 	}
 
 	if (!(flags & BCH_READ_IS_LAST))
-		__bio_inc_remaining(orig);
+		__bio_inc_remaining(&orig->bio);
 
-	rbio->parent		= orig;
+	if (split)
+		rbio->parent	= orig;
+	else
+		rbio->orig_bi_end_io = orig->bio.bi_end_io;
 	rbio->parent_iter	= iter;
+
 	rbio->inode		= k.k->p.inode;
 	rbio->flags		= flags;
 	rbio->bounce		= bounce;
+	rbio->split		= split;
 	rbio->crc		= pick->crc;
 	/*
 	 * crc.compressed_size will be 0 if there wasn't any checksum
@@ -1740,7 +1780,7 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
 	rbio->promote		= promote_op;
 
 	rbio->bio.bi_bdev	= pick->ca->disk_sb.bdev;
-	rbio->bio.bi_opf	= orig->bi_opf;
+	rbio->bio.bi_opf	= orig->bio.bi_opf;
 	rbio->bio.bi_iter.bi_sector = pick->ptr.offset;
 	rbio->bio.bi_end_io	= bch_read_endio;
 
@@ -1764,7 +1804,7 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
 					  bkey_start_offset(k.k) + skip),
 				      &promote_op->iop.insert_key);
 			bch_key_resize(&promote_op->iop.insert_key.k,
-				       bio_sectors(orig));
+				       orig_sectors);
 		}
 
 		__bio_clone_fast(&promote_op->bio.bio.bio, &rbio->bio);
@@ -1779,9 +1819,11 @@ void bch_read_extent_iter(struct cache_set *c, struct bio *orig,
 #endif
 }
 
-static void bch_read_iter(struct cache_set *c, struct bio *bio,
-			  struct bvec_iter bvec_iter, u64 inode)
+static void bch_read_iter(struct cache_set *c, struct bch_read_bio *rbio,
+			  struct bvec_iter bvec_iter, u64 inode,
+			  unsigned flags)
 {
+	struct bio *bio = &rbio->bio;
 	struct btree_iter iter;
 	struct bkey_s_c k;
 
@@ -1818,17 +1860,18 @@ static void bch_read_iter(struct cache_set *c, struct bio *bio,
 		is_last = bytes == bvec_iter.bi_size;
 		swap(bvec_iter.bi_size, bytes);
 
+		if (is_last)
+			flags |= BCH_READ_IS_LAST;
+
 		if (pick.ca) {
 			PTR_BUCKET(pick.ca, &pick.ptr)->read_prio =
 				c->prio_clock[READ].hand;
 
-			bch_read_extent_iter(c, bio, bvec_iter, k, &pick,
+			bch_read_extent_iter(c, rbio, bvec_iter, k, &pick,
 					     bvec_iter.bi_sector -
-					     bkey_start_offset(k.k),
-					     BCH_READ_FORCE_BOUNCE|
-					     BCH_READ_RETRY_IF_STALE|
-					     BCH_READ_PROMOTE|
-					     (is_last ? BCH_READ_IS_LAST : 0));
+					     bkey_start_offset(k.k), flags);
+
+			flags &= ~BCH_READ_MAY_REUSE_BIO;
 		} else {
 			zero_fill_bio_iter(bio, bvec_iter);
 
@@ -1852,11 +1895,15 @@ static void bch_read_iter(struct cache_set *c, struct bio *bio,
 	bio_endio(bio);
 }
 
-void bch_read(struct cache_set *c, struct bio *bio, u64 inode)
+void bch_read(struct cache_set *c, struct bch_read_bio *bio, u64 inode)
 {
-	bch_increment_clock(c, bio_sectors(bio), READ);
+	bch_increment_clock(c, bio_sectors(&bio->bio), READ);
 
-	bch_read_iter(c, bio, bio->bi_iter, inode);
+	bch_read_iter(c, bio, bio->bio.bi_iter, inode,
+		      BCH_READ_FORCE_BOUNCE|
+		      BCH_READ_RETRY_IF_STALE|
+		      BCH_READ_PROMOTE|
+		      BCH_READ_MAY_REUSE_BIO);
 }
 EXPORT_SYMBOL(bch_read);
 
@@ -1865,14 +1912,21 @@ EXPORT_SYMBOL(bch_read);
  */
 static void bch_read_retry(struct cache_set *c, struct bch_read_bio *rbio)
 {
-	struct bio *parent = rbio->parent;
+	struct bch_read_bio *parent = bch_rbio_parent(rbio);
 	struct bvec_iter iter = rbio->parent_iter;
 	u64 inode = rbio->inode;
 
 	trace_bcache_read_retry(&rbio->bio);
 
-	bch_rbio_free(c, rbio);
-	bch_read_iter(c, parent, iter, inode);
+	if (rbio->split)
+		bch_rbio_free(c, rbio);
+	else
+		rbio->bio.bi_end_io = rbio->orig_bi_end_io;
+
+	bch_read_iter(c, parent, iter, inode,
+		      BCH_READ_FORCE_BOUNCE|
+		      BCH_READ_RETRY_IF_STALE|
+		      BCH_READ_PROMOTE);
 }
 
 void bch_read_retry_work(struct work_struct *work)
diff --git a/drivers/md/bcache/io.h b/drivers/md/bcache/io.h
index 8be6a2a9eed3..e36b1d7e5edf 100644
--- a/drivers/md/bcache/io.h
+++ b/drivers/md/bcache/io.h
@@ -41,17 +41,19 @@ struct cache_promote_op;
 
 struct extent_pick_ptr;
 
-void bch_read_extent_iter(struct cache_set *, struct bio *,
+void bch_read_extent_iter(struct cache_set *, struct bch_read_bio *,
 			  struct bvec_iter, struct bkey_s_c k,
 			  struct extent_pick_ptr *,
 			  unsigned, unsigned);
 
-static inline void bch_read_extent(struct cache_set *c, struct bio *orig,
+static inline void bch_read_extent(struct cache_set *c,
+				   struct bch_read_bio *orig,
 				   struct bkey_s_c k,
 				   struct extent_pick_ptr *pick,
 				   unsigned skip, unsigned flags)
 {
-	bch_read_extent_iter(c, orig, orig->bi_iter, k, pick, skip, flags);
+	bch_read_extent_iter(c, orig, orig->bio.bi_iter,
+			     k, pick, skip, flags);
 }
 
 enum bch_read_flags {
@@ -59,9 +61,10 @@ enum bch_read_flags {
 	BCH_READ_RETRY_IF_STALE		= 1 << 1,
 	BCH_READ_PROMOTE		= 1 << 2,
 	BCH_READ_IS_LAST		= 1 << 3,
+	BCH_READ_MAY_REUSE_BIO		= 1 << 4,
 };
 
-void bch_read(struct cache_set *, struct bio *, u64);
+void bch_read(struct cache_set *, struct bch_read_bio *, u64);
 
 void bch_bbio_endio(struct bbio *);
 
diff --git a/drivers/md/bcache/io_types.h b/drivers/md/bcache/io_types.h
index 15926177b520..30527c9914a3 100644
--- a/drivers/md/bcache/io_types.h
+++ b/drivers/md/bcache/io_types.h
@@ -22,10 +22,19 @@ struct bch_read_bio {
 	 * was checksummed or compressed we'll also have to allocate bounce
 	 * buffers and copy the data back into the original bio.
 	 *
-	 * parent denotes the original bio, and parent_iter is where in parent
-	 * we copy the data back to.
+	 * If we didn't have to split, we have to save and restore the original
+	 * bi_end_io - @split below indicates which:
+	 */
+	union {
+	struct bch_read_bio	*parent;
+	bio_end_io_t		*orig_bi_end_io;
+	};
+
+	/*
+	 * Saved copy of parent->bi_iter, from submission time - allows us to
+	 * resubmit on IO error, and also to copy data back to the original bio
+	 * when we're bouncing:
 	 */
-	struct bio		*parent;
 	struct bvec_iter	parent_iter;
 
 	/*
@@ -39,7 +48,8 @@ struct bch_read_bio {
 
 	unsigned		submit_time_us;
 	u16			flags;
-	u8			bounce:1;
+	u8			bounce:1,
+				split:1;
 
 	struct bch_extent_crc64	crc;
 	struct bch_extent_ptr	ptr;
@@ -53,6 +63,12 @@ struct bch_read_bio {
 	struct bio		bio;
 };
 
+static inline struct bch_read_bio *
+bch_rbio_parent(struct bch_read_bio *rbio)
+{
+	return rbio->split ? rbio->parent : rbio;
+}
+
 struct bch_write_bio {
 	struct bio		*orig;
 	unsigned		bounce:1;
diff --git a/drivers/md/bcache/migrate.c b/drivers/md/bcache/migrate.c
index 39e212a1c3b5..a23a9ed695d4 100644
--- a/drivers/md/bcache/migrate.c
+++ b/drivers/md/bcache/migrate.c
@@ -54,7 +54,7 @@ static int issue_migration_move(struct cache *ca,
 
 	bch_replace_init(&io->replace, k);
 
-	bch_write_op_init(&io->op, c, &io->bio, res,
+	bch_write_op_init(&io->op, c, &io->wbio, res,
 			  &c->migration_write_point,
 			  k, &io->replace.hook, NULL,
 			  0);
diff --git a/drivers/md/bcache/move.c b/drivers/md/bcache/move.c
index 3a20a03892e8..b8ae1b44ef3e 100644
--- a/drivers/md/bcache/move.c
+++ b/drivers/md/bcache/move.c
@@ -72,10 +72,8 @@ static void bch_queue_write(struct moving_queue *q)
 	queue_work(q->wq, &q->work);
 }
 
-static void moving_init(struct moving_io *io)
+static void moving_init(struct moving_io *io, struct bio *bio)
 {
-	struct bio *bio = &io->bio.bio.bio;
-
 	bio_init(bio);
 	bio_get(bio);
 	bio_set_prio(bio, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, 0));
@@ -84,7 +82,7 @@ static void moving_init(struct moving_io *io)
 	bio->bi_max_vecs	= DIV_ROUND_UP(io->key.k.size,
 					       PAGE_SECTORS);
 	bio->bi_private		= &io->cl;
-	bio->bi_io_vec		= bio->bi_inline_vecs;
+	bio->bi_io_vec		= io->bi_inline_vecs;
 	bch_bio_map(bio, NULL);
 }
 
@@ -100,9 +98,9 @@ struct moving_io *moving_io_alloc(struct bkey_s_c k)
 
 	bkey_reassemble(&io->key, k);
 
-	moving_init(io);
+	moving_init(io, &io->rbio.bio);
 
-	if (bio_alloc_pages(&io->bio.bio.bio, GFP_KERNEL)) {
+	if (bio_alloc_pages(&io->rbio.bio, GFP_KERNEL)) {
 		kfree(io);
 		return NULL;
 	}
@@ -115,7 +113,7 @@ void moving_io_free(struct moving_io *io)
 	struct bio_vec *bv;
 	int i;
 
-	bio_for_each_segment_all(bv, &io->bio.bio.bio, i)
+	bio_for_each_segment_all(bv, &io->wbio.bio.bio, i)
 		if (bv->bv_page)
 			__free_page(bv->bv_page);
 
@@ -203,7 +201,7 @@ static void write_moving(struct moving_io *io)
 	if (op->error || stopped)
 		closure_return_with_destructor(&io->cl, moving_io_destructor);
 	else {
-		moving_init(io);
+		moving_init(io, &io->wbio.bio.bio);
 
 		op->bio->bio.bio.bi_iter.bi_sector = bkey_start_offset(&io->key.k);
 
@@ -481,10 +479,10 @@ static void __bch_data_move(struct closure *cl)
 	if (io->context->rate)
 		bch_ratelimit_increment(io->context->rate, size);
 
-	bio_set_op_attrs(&io->bio.bio.bio, REQ_OP_READ, 0);
-	io->bio.bio.bio.bi_end_io	= read_moving_endio;
+	bio_set_op_attrs(&io->rbio.bio, REQ_OP_READ, 0);
+	io->rbio.bio.bi_end_io	= read_moving_endio;
 
-	bch_read_extent(io->op.c, &io->bio.bio.bio,
+	bch_read_extent(io->op.c, &io->rbio,
 			bkey_i_to_s_c(&io->key),
 			&pick, 0, BCH_READ_IS_LAST);
 }
diff --git a/drivers/md/bcache/move.h b/drivers/md/bcache/move.h
index 8b311a8b3676..8ce99204a264 100644
--- a/drivers/md/bcache/move.h
+++ b/drivers/md/bcache/move.h
@@ -93,8 +93,10 @@ struct moving_io {
 	unsigned		read_completed:1;
 	unsigned		write_issued:1;
 
+	struct bch_read_bio	rbio;
+	struct bch_write_bio	wbio;
 	/* Must be last since it is variable size */
-	struct bch_write_bio	bio;
+	struct bio_vec		bi_inline_vecs[0];
 };
 
 struct moving_io *moving_io_alloc(struct bkey_s_c);
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index f1a1aa3e49ea..3aeebeba7b97 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -60,7 +60,7 @@ static int issue_moving_gc_move(struct moving_queue *q,
 
 	bch_replace_init(&io->replace, bkey_i_to_s_c(k));
 
-	bch_write_op_init(&io->op, c, &io->bio,
+	bch_write_op_init(&io->op, c, &io->wbio,
 			  (struct disk_reservation) { 0 },
 			  NULL, bkey_i_to_s_c(k),
 			  &io->replace.hook, NULL,
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index 10732c4a8f52..f4ec2f3185ea 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -211,7 +211,8 @@ static void bio_complete(struct search *s)
 
 static void do_bio_hook(struct search *s, struct bio *orig_bio)
 {
-	struct bio *bio = &s->bio.bio.bio;
+	int rw = bio_data_dir(orig_bio);
+	struct bio *bio = rw ? &s->wbio.bio.bio : &s->rbio.bio;
 
 	bio_init(bio);
 	__bio_clone_fast(bio, orig_bio);
@@ -277,7 +278,7 @@ static void cached_dev_bio_complete(struct closure *cl)
 static void cached_dev_read_error(struct closure *cl)
 {
 	struct search *s = container_of(cl, struct search, cl);
-	struct bio *bio = &s->bio.bio.bio;
+	struct bio *bio = &s->rbio.bio;
 
 	if (s->recoverable) {
 		/* Read bucket invalidate races are handled here, also plain
@@ -393,7 +394,7 @@ nopromote:
 static void cached_dev_read(struct cached_dev *dc, struct search *s)
 {
 	struct closure *cl = &s->cl;
-	struct bio *bio = &s->bio.bio.bio;
+	struct bio *bio = &s->rbio.bio;
 	struct btree_iter iter;
 	struct bkey_s_c k;
 
@@ -439,7 +440,7 @@ retry:
 			if (!bkey_extent_is_cached(k.k))
 				s->read_dirty_data = true;
 
-			bch_read_extent(s->iop.c, bio, k, &pick,
+			bch_read_extent(s->iop.c, &s->rbio, k, &pick,
 					bio->bi_iter.bi_sector -
 					bkey_start_offset(k.k),
 					BCH_READ_FORCE_BOUNCE|
@@ -480,7 +481,7 @@ static void cached_dev_write_complete(struct closure *cl)
 static void cached_dev_write(struct cached_dev *dc, struct search *s)
 {
 	struct closure *cl = &s->cl;
-	struct bio *bio = &s->bio.bio.bio;
+	struct bio *bio = &s->wbio.bio.bio;
 	unsigned inode = bcache_dev_inum(&dc->disk);
 	bool writeback = false;
 	bool bypass = s->bypass;
@@ -556,7 +557,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 	if (bypass)
 		flags |= BCH_WRITE_DISCARD;
 
-	bch_write_op_init(&s->iop, dc->disk.c, &s->bio,
+	bch_write_op_init(&s->iop, dc->disk.c, &s->wbio,
 			  (struct disk_reservation) { 0 }, NULL,
 			  bkey_to_s_c(&insert_key),
 			  NULL, NULL, flags);
@@ -580,9 +581,13 @@ static void __cached_dev_make_request(struct request_queue *q, struct bio *bio)
 	bio->bi_iter.bi_sector += le64_to_cpu(dc->disk_sb.sb->data_offset);
 
 	if (cached_dev_get(dc)) {
+		struct bio *clone;
+
 		s = search_alloc(bio, d);
 		trace_bcache_request_start(s->d, bio);
 
+		clone = rw ? &s->wbio.bio.bio : &s->rbio.bio;
+
 		if (!bio->bi_iter.bi_size) {
 			if (s->orig_bio->bi_opf & (REQ_PREFLUSH|REQ_FUA))
 				bch_journal_flush_async(&s->iop.c->journal,
@@ -592,7 +597,7 @@ static void __cached_dev_make_request(struct request_queue *q, struct bio *bio)
 			 * If it's a flush, we send the flush to the backing
 			 * device too
 			 */
-			closure_bio_submit(&s->bio.bio.bio, &s->cl);
+			closure_bio_submit(clone, &s->cl);
 
 			continue_at(&s->cl, cached_dev_bio_complete, NULL);
 		} else {
@@ -673,9 +678,9 @@ static void __blockdev_volume_make_request(struct request_queue *q,
 
 	trace_bcache_request_start(d, bio);
 
-	if (!bio->bi_iter.bi_size) {
-		s = search_alloc(bio, d);
+	s = search_alloc(bio, d);
 
+	if (!bio->bi_iter.bi_size) {
 		if (s->orig_bio->bi_opf & (REQ_PREFLUSH|REQ_FUA))
 			bch_journal_flush_async(&s->iop.c->journal,
 						&s->cl);
@@ -697,17 +702,16 @@ static void __blockdev_volume_make_request(struct request_queue *q,
 		if (bio_op(bio) == REQ_OP_DISCARD)
 			flags |= BCH_WRITE_DISCARD;
 
-		s = search_alloc(bio, d);
-
-		bch_write_op_init(&s->iop, d->c, &s->bio, res, NULL,
+		bch_write_op_init(&s->iop, d->c, &s->wbio, res, NULL,
 				  bkey_to_s_c(&KEY(s->inode, 0, 0)),
 				  NULL, NULL, flags);
 
 		closure_call(&s->iop.cl, bch_write, NULL, &s->cl);
-		continue_at(&s->cl, search_free, NULL);
 	} else {
-		bch_read(d->c, bio, bcache_dev_inum(d));
+		closure_get(&s->cl);
+		bch_read(d->c, &s->rbio, bcache_dev_inum(d));
 	}
+	continue_at(&s->cl, search_free, NULL);
 }
 
 static blk_qc_t blockdev_volume_make_request(struct request_queue *q,
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 05c4abb4be32..ae33c6a05a1c 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -879,6 +879,7 @@ static void cache_set_free(struct cache_set *c)
 	mempool_exit(&c->compression_workspace_pool);
 	mempool_exit(&c->bio_bounce_pages);
 	bioset_exit(&c->bio_write);
+	bioset_exit(&c->bio_read_split);
 	bioset_exit(&c->bio_read);
 	bioset_exit(&c->btree_read_bio);
 	mempool_exit(&c->btree_async_split_pool);
@@ -1129,8 +1130,9 @@ static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb,
 				      sizeof(struct async_split)) ||
 	    mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) ||
 	    bioset_init(&c->btree_read_bio, 1, offsetof(struct bbio, bio)) ||
-	    bioset_init(&c->bio_read, 4, offsetof(struct bch_read_bio, bio)) ||
-	    bioset_init(&c->bio_write, 4, offsetof(struct bch_write_bio, bio.bio)) ||
+	    bioset_init(&c->bio_read, 1, offsetof(struct bch_read_bio, bio)) ||
+	    bioset_init(&c->bio_read_split, 1, offsetof(struct bch_read_bio, bio)) ||
+	    bioset_init(&c->bio_write, 1, offsetof(struct bch_write_bio, bio.bio)) ||
 	    mempool_init_page_pool(&c->bio_bounce_pages,
 				   CRC32_EXTENT_SIZE_MAX / PAGE_SECTORS, 0) ||
 	    mempool_init_page_pool(&c->compression_workspace_pool, 1,
diff --git a/drivers/md/bcache/tier.c b/drivers/md/bcache/tier.c
index 20cbbf9295ed..07a1456814dd 100644
--- a/drivers/md/bcache/tier.c
+++ b/drivers/md/bcache/tier.c
@@ -222,7 +222,7 @@ static int issue_tiering_move(struct moving_queue *q,
 
 	bch_replace_init(&io->replace, bkey_i_to_s_c(&io->key));
 
-	bch_write_op_init(&io->op, c, &io->bio,
+	bch_write_op_init(&io->op, c, &io->wbio,
 			  (struct disk_reservation) { 0 },
 			  &ca->tiering_write_point,
 			  bkey_i_to_s_c(&io->key),