bcache: bch_write_bio

15 files changed, 175 insertions, 162 deletions

diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 9dcc83142d4a..807278e80500 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -420,15 +420,10 @@ struct cache_set {
 
 	struct bio_set		bio_split;
 
-	/* For punting bio submissions to workqueue, io.c */
-	struct bio_list		bio_submit_list;
-	struct work_struct	bio_submit_work;
-	spinlock_t		bio_submit_lock;
-
 	struct backing_dev_info bdi;
 
 	/* BTREE CACHE */
-	struct bio_set		btree_bio;
+	struct bio_set		btree_read_bio;
 
 	spinlock_t		btree_root_lock;
 	struct btree		*btree_roots[BTREE_ID_NR];
@@ -568,6 +563,13 @@ struct cache_set {
 	struct rw_semaphore	gc_lock;
 
 	/* IO PATH */
+	struct bio_set		bio_write;
+
+	/* For punting bio submissions to workqueue, io.c */
+	struct bio_list		bio_submit_list;
+	struct work_struct	bio_submit_work;
+	spinlock_t		bio_submit_lock;
+
 	struct bio_list		read_race_list;
 	struct work_struct	read_race_work;
 	spinlock_t		read_race_lock;
@@ -659,22 +661,6 @@ struct cache_set {
 	unsigned		sector_reserve_percent;
 };
 
-struct bbio {
-	struct cache		*ca;
-
-	unsigned int		bi_idx;		/* current index into bvl_vec */
-
-	unsigned int            bi_bvec_done;	/* number of bytes completed in
-						   current bvec */
-	unsigned		submit_time_us;
-	struct bkey_i		key;
-	struct bch_extent_ptr	ptr;
-	/* Only ever have a single pointer (the one we're doing io to/from) */
-	struct bio		bio;
-};
-
-#define to_bbio(_bio)		container_of((_bio), struct bbio, bio)
-
 static inline unsigned bucket_pages(const struct cache *ca)
 {
 	return ca->mi.bucket_size / PAGE_SECTORS;
diff --git a/drivers/md/bcache/blockdev.h b/drivers/md/bcache/blockdev.h
index 8583201c0ccc..a4be96bd0eb6 100644
--- a/drivers/md/bcache/blockdev.h
+++ b/drivers/md/bcache/blockdev.h
@@ -55,7 +55,7 @@ struct search {
 	/* Stack frame for bio_complete */
 	struct closure		cl;
 
-	struct bbio		bio;
+	struct bch_write_bio	bio;
 	/* Not modified */
 	struct bio		*orig_bio;
 	struct bcache_device	*d;
diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c
index 5353dc94df42..eac7354c572f 100644
--- a/drivers/md/bcache/btree.c
+++ b/drivers/md/bcache/btree.c
@@ -564,7 +564,7 @@ static void bch_btree_node_read(struct cache_set *c, struct btree *b)
 
 	percpu_ref_get(&pick.ca->ref);
 
-	bio = bio_alloc_bioset(GFP_NOIO, btree_pages(c), &c->btree_bio);
+	bio = bio_alloc_bioset(GFP_NOIO, btree_pages(c), &c->btree_read_bio);
 	bio->bi_iter.bi_size	= btree_bytes(c);
 	bio->bi_end_io		= btree_node_read_endio;
 	bio->bi_private		= &cl;
@@ -618,14 +618,12 @@ static void btree_node_write_unlock(struct closure *cl)
 	up(&b->io_mutex);
 }
 
-static void __btree_node_write_done(struct closure *cl)
+static void btree_node_write_done(struct closure *cl)
 {
 	struct btree *b = container_of(cl, struct btree, io);
 	struct btree_write *w = btree_prev_write(b);
 	struct cache_set *c = b->c;
 
-	bio_put(b->bio);
-	b->bio = NULL;
 	btree_complete_write(c, b, w);
 
 	if (btree_node_dirty(b) && c->btree_flush_delay)
@@ -634,28 +632,28 @@ static void __btree_node_write_done(struct closure *cl)
 	closure_return_with_destructor(cl, btree_node_write_unlock);
 }
 
-static void btree_node_write_done(struct closure *cl)
-{
-	struct btree *b = container_of(cl, struct btree, io);
-
-	bch_bio_free_pages(b->bio);
-	__btree_node_write_done(cl);
-}
-
 static void btree_node_write_endio(struct bio *bio)
 {
 	struct closure *cl = bio->bi_private;
 	struct btree *b = container_of(cl, struct btree, io);
+	struct bch_write_bio *wbio = to_wbio(bio);
 
 	if (bio->bi_error || bch_meta_write_fault("btree"))
 		set_btree_node_io_error(b);
 
+	if (wbio->orig)
+		bio_endio(wbio->orig);
+	else if (wbio->bounce)
+		bch_bio_free_pages(bio);
+
 	bch_bbio_endio(to_bbio(bio), bio->bi_error, "writing btree");
 }
 
 static void do_btree_node_write(struct closure *cl)
 {
 	struct btree *b = container_of(cl, struct btree, io);
+	struct bio *bio;
+	struct bch_write_bio *wbio;
 	struct cache_set *c = b->c;
 	struct bset *i = btree_bset_last(b);
 	BKEY_PADDED(key) k;
@@ -702,20 +700,17 @@ static void do_btree_node_write(struct closure *cl)
 
 	BUG_ON(b->written + blocks_to_write > btree_blocks(c));
 
-	BUG_ON(b->bio);
-	b->bio = bio_alloc_bioset(GFP_NOIO, btree_pages(c), &c->btree_bio);
+	bio = bio_alloc_bioset(GFP_NOIO, btree_pages(c), &c->bio_write);
 
-	/*
-	 * Take an extra reference so that the bio_put() in
-	 * btree_node_write_endio() doesn't call bio_free()
-	 */
-	bio_get(b->bio);
+	wbio		= to_wbio(bio);
+	wbio->orig	= NULL;
+	wbio->bounce	= false;
 
-	b->bio->bi_end_io	= btree_node_write_endio;
-	b->bio->bi_private	= cl;
-	b->bio->bi_iter.bi_size	= blocks_to_write << (c->block_bits + 9);
-	bio_set_op_attrs(b->bio, REQ_OP_WRITE, REQ_META|WRITE_SYNC|REQ_FUA);
-	bch_bio_map(b->bio, data);
+	bio->bi_end_io		= btree_node_write_endio;
+	bio->bi_private		= cl;
+	bio->bi_iter.bi_size	= blocks_to_write << (c->block_bits + 9);
+	bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_META|WRITE_SYNC|REQ_FUA);
+	bch_bio_map(bio, data);
 
 	/*
 	 * If we're appending to a leaf node, we don't technically need FUA -
@@ -746,27 +741,29 @@ static void do_btree_node_write(struct closure *cl)
 
 	b->written += blocks_to_write;
 
-	if (!bio_alloc_pages(b->bio, __GFP_NOWARN|GFP_NOWAIT)) {
+	if (!bio_alloc_pages(bio, __GFP_NOWARN|GFP_NOWAIT)) {
 		int j;
 		struct bio_vec *bv;
 		void *base = (void *) ((unsigned long) data & ~(PAGE_SIZE - 1));
 
-		bio_for_each_segment_all(bv, b->bio, j)
+		bio_for_each_segment_all(bv, bio, j)
 			memcpy(page_address(bv->bv_page),
 			       base + (j << PAGE_SHIFT), PAGE_SIZE);
 
-		bch_submit_bbio_replicas(b->bio, c, &k.key, 0, true);
+		wbio->bounce = true;
+
+		bch_submit_bbio_replicas(wbio, c, &k.key, 0, true);
 		continue_at(cl, btree_node_write_done, NULL);
 	} else {
 		trace_bcache_btree_bounce_write_fail(b);
 
-		b->bio->bi_vcnt = 0;
-		bch_bio_map(b->bio, data);
+		bio->bi_vcnt = 0;
+		bch_bio_map(bio, data);
 
-		bch_submit_bbio_replicas(b->bio, c, &k.key, 0, true);
+		bch_submit_bbio_replicas(wbio, c, &k.key, 0, true);
 
 		closure_sync(cl);
-		continue_at_nobarrier(cl, __btree_node_write_done, NULL);
+		continue_at_nobarrier(cl, btree_node_write_done, NULL);
 	}
 }
 
diff --git a/drivers/md/bcache/btree.h b/drivers/md/bcache/btree.h
index 43b770111f58..ec467b1b56d6 100644
--- a/drivers/md/bcache/btree.h
+++ b/drivers/md/bcache/btree.h
@@ -129,7 +129,6 @@ struct btree {
 	struct delayed_work	work;
 
 	struct btree_write	writes[2];
-	struct bio		*bio;
 
 	struct list_head	journal_seq_blacklisted;
 };
diff --git a/drivers/md/bcache/debug.c b/drivers/md/bcache/debug.c
index 0a0b6ecd1f81..bb4d715c9b15 100644
--- a/drivers/md/bcache/debug.c
+++ b/drivers/md/bcache/debug.c
@@ -66,7 +66,7 @@ void bch_btree_verify(struct cache_set *c, struct btree *b)
 	if (IS_ERR_OR_NULL(pick.ca))
 		return;
 
-	bio = bio_alloc_bioset(GFP_NOIO, btree_pages(c), &c->btree_bio);
+	bio = bio_alloc_bioset(GFP_NOIO, btree_pages(c), &c->btree_read_bio);
 	bio->bi_bdev		= pick.ca->disk_sb.bdev;
 	bio->bi_iter.bi_size	= btree_bytes(c);
 	bio_set_op_attrs(bio, REQ_OP_READ, REQ_META|READ_SYNC);
diff --git a/drivers/md/bcache/fs.c b/drivers/md/bcache/fs.c
index aea81fa0fc27..502a983dcc3f 100644
--- a/drivers/md/bcache/fs.c
+++ b/drivers/md/bcache/fs.c
@@ -46,7 +46,7 @@ PAGEFLAG(Append, private_2, PF_ANY)
 TESTSCFLAG(Append, private_2, PF_ANY)
 #undef PF_ANY
 
-static struct bio_set *bch_fs_bioset;
+static struct bio_set *bch_writepage_bioset;
 static struct kmem_cache *bch_inode_cache;
 static DECLARE_WAIT_QUEUE_HEAD(bch_append_wait);
 
@@ -1015,7 +1015,7 @@ static int bch_readpage(struct file *file, struct page *page)
 struct bch_writepage_io {
 	struct closure		cl;
 	struct bch_write_op	op;
-	struct bbio		bio;
+	struct bch_write_bio	bio;
 };
 
 struct bch_writepage {
@@ -1029,17 +1029,18 @@ static void bch_writepage_io_free(struct closure *cl)
 	struct bch_writepage_io *io = container_of(cl,
 					struct bch_writepage_io, cl);
 	struct cache_set *c = io->op.c;
-	struct inode *inode = io->bio.bio.bi_io_vec[0].bv_page->mapping->host;
+	struct bio *bio = &io->bio.bio.bio;
+	struct inode *inode = bio->bi_io_vec[0].bv_page->mapping->host;
 	struct bch_inode_info *ei = to_bch_ei(inode);
 	struct bio_vec *bvec;
 	int i;
 
-	bio_for_each_segment_all(bvec, &io->bio.bio, i) {
+	bio_for_each_segment_all(bvec, bio, i) {
 		struct page *page = bvec->bv_page;
 
 		BUG_ON(!PageWriteback(page));
 
-		if (io->bio.bio.bi_error) {
+		if (io->bio.bio.bio.bi_error) {
 			SetPageError(page);
 			if (page->mapping)
 				set_bit(AS_EIO, &page->mapping->flags);
@@ -1049,15 +1050,15 @@ static void bch_writepage_io_free(struct closure *cl)
 		end_page_writeback(page);
 	}
 
-	bio_put(&io->bio.bio);
+	bio_put(bio);
 }
 
 static void bch_writepage_do_io(struct bch_writepage_io *io)
 {
 	pr_debug("writing %u sectors to %llu:%llu",
-		 bio_sectors(&io->bio.bio),
+		 bio_sectors(&io->bio.bio.bio),
 		 io->op.insert_key.k.p.inode,
-		 (u64) io->bio.bio.bi_iter.bi_sector);
+		 (u64) io->bio.bio.bio.bi_iter.bi_sector);
 
 	closure_call(&io->op.cl, bch_write, NULL, &io->cl);
 	closure_return_with_destructor(&io->cl, bch_writepage_io_free);
@@ -1107,17 +1108,18 @@ do_io:
 	mutex_unlock(&ei->update_lock);
 
 	if (!w->io) {
-		bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES, bch_fs_bioset);
-		w->io = container_of(bio, struct bch_writepage_io, bio.bio);
+		bio = bio_alloc_bioset(GFP_NOFS, BIO_MAX_PAGES,
+				       bch_writepage_bioset);
+		w->io = container_of(bio, struct bch_writepage_io, bio.bio.bio);
 
 		closure_init(&w->io->cl, NULL);
-		bch_write_op_init(&w->io->op, w->c, bio, NULL,
+		bch_write_op_init(&w->io->op, w->c, &w->io->bio, NULL,
 				  bkey_to_s_c(&KEY(w->inum, 0, 0)),
 				  bkey_s_c_null, 0);
 		w->io->op.journal_seq = &ei->journal_seq;
 	}
 
-	if (bch_bio_add_page(&w->io->bio.bio, page)) {
+	if (bch_bio_add_page(&w->io->bio.bio.bio, page)) {
 		bch_writepage_do_io(w->io);
 		w->io = NULL;
 		goto do_io;
@@ -1462,7 +1464,7 @@ struct dio_write_bio {
 	struct closure		cl;
 	struct dio_write	*dio;
 	struct bch_write_op	iop;
-	struct bbio		bio;
+	struct bch_write_bio	bio;
 };
 
 static void __bch_dio_write_complete(struct dio_write *dio)
@@ -1487,8 +1489,8 @@ static void bch_dio_write_complete(struct closure *cl)
 
 static void bch_direct_IO_write_done(struct closure *cl)
 {
-	struct dio_write_bio *op = container_of(cl,
-					struct dio_write_bio, cl);
+	struct dio_write_bio *op =
+		container_of(cl, struct dio_write_bio, cl);
 	struct bio_vec *bv;
 	int i;
 
@@ -1496,7 +1498,7 @@ static void bch_direct_IO_write_done(struct closure *cl)
 		op->dio->ret = op->iop.error;
 	closure_put(&op->dio->cl);
 
-	bio_for_each_segment_all(bv, &op->bio.bio, i)
+	bio_for_each_segment_all(bv, &op->bio.bio.bio, i)
 		put_page(bv->bv_page);
 	kfree(op);
 }
@@ -1552,7 +1554,7 @@ static int bch_direct_IO_write(struct cache_set *c, struct kiocb *req,
 			break;
 		}
 
-		bio = &op->bio.bio;
+		bio = &op->bio.bio.bio;
 		bio_init(bio);
 		bio->bi_iter.bi_sector	= offset >> 9;
 		bio->bi_max_vecs	= pages;
@@ -1570,7 +1572,7 @@ static int bch_direct_IO_write(struct cache_set *c, struct kiocb *req,
 		op->dio = dio;
 		closure_init(&op->cl, NULL);
 
-		bch_write_op_init(&op->iop, c, bio, NULL,
+		bch_write_op_init(&op->iop, c, &op->bio, NULL,
 				  bkey_to_s_c(&KEY(inum,
 						   bio_end_sector(bio),
 						   bio_sectors(bio))),
@@ -2052,8 +2054,8 @@ void bch_fs_exit(void)
 	unregister_filesystem(&bcache_fs_type);
 	if (bch_dio_read_bioset)
 		bioset_free(bch_dio_read_bioset);
-	if (bch_fs_bioset)
-		bioset_free(bch_fs_bioset);
+	if (bch_writepage_bioset)
+		bioset_free(bch_writepage_bioset);
 	if (bch_inode_cache)
 		kmem_cache_destroy(bch_inode_cache);
 }
@@ -2066,9 +2068,9 @@ int __init bch_fs_init(void)
 	if (!bch_inode_cache)
 		goto err;
 
-	bch_fs_bioset = bioset_create(4,
-				offsetof(struct bch_writepage_io, bio.bio));
-	if (!bch_fs_bioset)
+	bch_writepage_bioset =
+		bioset_create(4, offsetof(struct bch_writepage_io, bio.bio.bio));
+	if (!bch_writepage_bioset)
 		goto err;
 
 
diff --git a/drivers/md/bcache/io.c b/drivers/md/bcache/io.c
index 5d289ec2be86..d543344be4ab 100644
--- a/drivers/md/bcache/io.c
+++ b/drivers/md/bcache/io.c
@@ -24,6 +24,13 @@
 
 #include <trace/events/bcache.h>
 
+static inline void __bio_inc_remaining(struct bio *bio)
+{
+	bio->bi_flags |= (1 << BIO_CHAIN);
+	smp_mb__before_atomic();
+	atomic_inc(&bio->__bi_remaining);
+}
+
 void bch_generic_make_request(struct bio *bio, struct cache_set *c)
 {
 	if (current->bio_list) {
@@ -88,7 +95,7 @@ void bch_submit_bbio(struct bbio *b, struct cache *ca, const struct bkey_i *k,
 		closure_bio_submit(bio, bio->bi_private);
 }
 
-void bch_submit_bbio_replicas(struct bio *bio, struct cache_set *c,
+void bch_submit_bbio_replicas(struct bch_write_bio *bio, struct cache_set *c,
 			      const struct bkey_i *k, unsigned ptrs_from,
 			      bool punt)
 {
@@ -96,6 +103,8 @@ void bch_submit_bbio_replicas(struct bio *bio, struct cache_set *c,
 	struct cache *ca;
 	unsigned ptr;
 
+	BUG_ON(bio->orig);
+
 	for (ptr = ptrs_from;
 	     ptr < bch_extent_ptrs(e);
 	     ptr++) {
@@ -106,20 +115,24 @@ void bch_submit_bbio_replicas(struct bio *bio, struct cache_set *c,
 		rcu_read_unlock();
 
 		if (!ca) {
-			bch_submit_bbio(to_bbio(bio), ca, k,
+			bch_submit_bbio(&bio->bio, ca, k,
 					&e.v->ptr[ptr], punt);
 			break;
 		}
 
 		if (ptr + 1 < bch_extent_ptrs(e)) {
-			struct bio *n = bio_clone_fast(bio, GFP_NOIO,
-						       &ca->replica_set);
-			n->bi_end_io		= bio->bi_end_io;
-			n->bi_private		= bio->bi_private;
-			bch_submit_bbio(to_bbio(n), ca, k,
-					&e.v->ptr[ptr], punt);
+			struct bch_write_bio *n =
+				to_wbio(bio_clone_fast(&bio->bio.bio, GFP_NOIO,
+						       &ca->replica_set));
+
+			n->bio.bio.bi_end_io	= bio->bio.bio.bi_end_io;
+			n->bio.bio.bi_private	= bio->bio.bio.bi_private;
+			n->orig			= &bio->bio.bio;
+			__bio_inc_remaining(n->orig);
+
+			bch_submit_bbio(&n->bio, ca, k, &e.v->ptr[ptr], punt);
 		} else {
-			bch_submit_bbio(to_bbio(bio), ca, k,
+			bch_submit_bbio(&bio->bio, ca, k,
 					&e.v->ptr[ptr], punt);
 		}
 	}
@@ -323,7 +336,7 @@ static void bch_write_index(struct closure *cl)
 static void bch_write_discard(struct closure *cl)
 {
 	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
-	struct bio *bio = op->bio;
+	struct bio *bio = &op->bio->bio.bio;
 	u64 inode = op->insert_key.k.p.inode;
 
 	op->error = bch_discard(op->c,
@@ -386,7 +399,7 @@ static void bch_write_endio(struct bio *bio)
 static void __bch_write(struct closure *cl)
 {
 	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
-	struct bio *bio = op->bio, *n;
+	struct bio *bio = &op->bio->bio.bio, *n;
 	unsigned open_bucket_nr = 0, ptrs_from;
 	struct open_bucket *b;
 
@@ -448,7 +461,7 @@ static void __bch_write(struct closure *cl)
 		bch_cut_front(k->k.p, &op->insert_key);
 
 		n = bio_next_split(bio, k->k.size, GFP_NOIO,
-				   &op->c->bio_split);
+				   &op->c->bio_write);
 		if (n == bio)
 			bio_get(bio);
 
@@ -462,7 +475,8 @@ static void __bch_write(struct closure *cl)
 
 		bio_set_op_attrs(n, REQ_OP_WRITE, 0);
 #ifndef CONFIG_BCACHE_NO_IO
-		bch_submit_bbio_replicas(n, op->c, k, ptrs_from, false);
+		bch_submit_bbio_replicas(to_wbio(n), op->c, k,
+					 ptrs_from, false);
 #else
 		bch_bbio_prep(to_bbio(n), NULL);
 		closure_get(n->bi_private);
@@ -554,14 +568,15 @@ void bch_wake_delayed_writes(unsigned long data)
 void bch_write(struct closure *cl)
 {
 	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
+	struct bio *bio = &op->bio->bio.bio;
 	struct cache_set *c = op->c;
 	u64 inode = op->insert_key.k.p.inode;
 
-	trace_bcache_write(c, inode, op->bio,
+	trace_bcache_write(c, inode, bio,
 			   !bkey_extent_is_cached(&op->insert_key.k),
 			   op->discard);
 
-	if (!bio_sectors(op->bio)) {
+	if (!bio_sectors(bio)) {
 		WARN_ONCE(1, "bch_write() called with empty bio");
 		closure_return(cl);
 	}
@@ -583,22 +598,22 @@ void bch_write(struct closure *cl)
 		op->io_wq = op->c->wq;
 
 	if (!op->discard)
-		bch_increment_clock(c, bio_sectors(op->bio), WRITE);
+		bch_increment_clock(c, bio_sectors(bio), WRITE);
 
 	if (!op->discard)
-		bch_mark_foreground_write(c, bio_sectors(op->bio));
+		bch_mark_foreground_write(c, bio_sectors(bio));
 	else
-		bch_mark_discard(c, bio_sectors(op->bio));
+		bch_mark_discard(c, bio_sectors(bio));
 
-	if (atomic64_sub_return(bio_sectors(op->bio),
+	if (atomic64_sub_return(bio_sectors(bio),
 				&c->sectors_until_gc) < 0) {
 		trace_bcache_gc_periodic(c);
 		set_gc_sectors(c);
 		wake_up_process(c->gc_thread);
 	}
 
-	op->insert_key.k.p.offset	= bio_end_sector(op->bio);
-	op->insert_key.k.size		= bio_sectors(op->bio);
+	op->insert_key.k.p.offset	= bio_end_sector(bio);
+	op->insert_key.k.size		= bio_sectors(bio);
 
 	/* Don't call bch_next_delay() if rate is >= 1 GB/sec */
 
@@ -609,12 +624,12 @@ void bch_write(struct closure *cl)
 
 		spin_lock_irqsave(&c->foreground_write_pd_lock, flags);
 		bch_ratelimit_increment(&c->foreground_write_pd.rate,
-					op->bio->bi_iter.bi_size);
+					bio->bi_iter.bi_size);
 
 		delay = bch_ratelimit_delay(&c->foreground_write_pd.rate);
 
 		if (delay >= HZ / 100) {
-			trace_bcache_write_throttle(c, inode, op->bio, delay);
+			trace_bcache_write_throttle(c, inode, bio, delay);
 
 			closure_get(&op->cl); /* list takes a ref */
 
@@ -643,7 +658,7 @@ void bch_write(struct closure *cl)
 }
 
 void bch_write_op_init(struct bch_write_op *op, struct cache_set *c,
-		       struct bio *bio, struct write_point *wp,
+		       struct bch_write_bio *bio, struct write_point *wp,
 		       struct bkey_s_c insert_key,
 		       struct bkey_s_c replace_key,
 		       unsigned flags)
@@ -761,7 +776,7 @@ struct cache_promote_op {
 	struct bio		*orig_bio;
 	struct bch_write_op	iop;
 	bool			stale; /* was the ptr stale after the read? */
-	struct bbio		bio; /* must be last */
+	struct bch_write_bio	bio; /* must be last */
 };
 
 static void cache_promote_done(struct closure *cl)
@@ -775,7 +790,7 @@ static void cache_promote_done(struct closure *cl)
 		atomic_inc(&c->accounting.collector.cache_miss_collisions);
 	}
 
-	bch_bio_free_pages(op->iop.bio);
+	bch_bio_free_pages(&op->iop.bio->bio.bio);
 	kfree(op);
 }
 
@@ -783,7 +798,7 @@ static void cache_promote_write(struct closure *cl)
 {
 	struct cache_promote_op *op = container_of(cl,
 					struct cache_promote_op, cl);
-	struct bio *bio = op->iop.bio;
+	struct bio *bio = &op->iop.bio->bio.bio;
 
 	bio_reset(bio);
 	bio->bi_iter.bi_sector	= bkey_start_offset(&op->iop.insert_key.k);
@@ -807,8 +822,8 @@ static void cache_promote_write(struct closure *cl)
 static void cache_promote_endio(struct bio *bio)
 {
 	struct bbio *b = to_bbio(bio);
-	struct cache_promote_op *op = container_of(b,
-					struct cache_promote_op, bio);
+	struct cache_promote_op *op =
+		container_of(bio, struct cache_promote_op, bio.bio.bio);
 
 	/*
 	 * If the bucket was reused while our bio was in flight, we might have
@@ -850,7 +865,7 @@ void __cache_promote(struct cache_set *c, struct bbio *orig_bio,
 	/* clone the bbio */
 	memcpy(&op->bio, orig_bio, offsetof(struct bbio, bio));
 
-	bio = &op->bio.bio;
+	bio = &op->bio.bio.bio;
 	bio_init(bio);
 	bio_get(bio);
 	bio->bi_bdev		= orig_bio->bio.bi_bdev;
@@ -870,7 +885,7 @@ void __cache_promote(struct cache_set *c, struct bbio *orig_bio,
 	op->orig_bio		= &orig_bio->bio;
 	op->stale		= 0;
 
-	bch_write_op_init(&op->iop, c, bio, &c->promote_write_point,
+	bch_write_op_init(&op->iop, c, &op->bio, &c->promote_write_point,
 			  new, old, BCH_WRITE_CHECK_ENOSPC|write_flags);
 
 	bch_cut_front(bkey_start_pos(&orig_bio->key.k), &op->iop.insert_key);
@@ -878,7 +893,7 @@ void __cache_promote(struct cache_set *c, struct bbio *orig_bio,
 
 	trace_bcache_promote(&orig_bio->bio);
 
-	op->bio.submit_time_us = local_clock_us();
+	op->bio.bio.submit_time_us = local_clock_us();
 	closure_bio_submit(bio, &op->cl);
 
 	continue_at(&op->cl, cache_promote_write, c->wq);
@@ -944,13 +959,6 @@ static void bch_read_endio(struct bio *bio)
 		percpu_ref_put(&ca->ref);
 }
 
-static inline void __bio_inc_remaining(struct bio *bio)
-{
-	bio->bi_flags |= (1 << BIO_CHAIN);
-	smp_mb__before_atomic();
-	atomic_inc(&bio->__bi_remaining);
-}
-
 /* XXX: this looks a lot like cache_lookup_fn() */
 int bch_read(struct cache_set *c, struct bio *bio, u64 inode)
 {
diff --git a/drivers/md/bcache/io.h b/drivers/md/bcache/io.h
index 1d97fac2665c..fb024d2e5fa8 100644
--- a/drivers/md/bcache/io.h
+++ b/drivers/md/bcache/io.h
@@ -1,10 +1,30 @@
 #ifndef _BCACHE_IO_H
 #define _BCACHE_IO_H
 
-/*
- * Adding a wrapper around the replace_key allows easy addition of
- * statistics and other fields for debugging, etc.
- */
+struct bbio {
+	struct cache		*ca;
+
+	unsigned int		bi_idx;		/* current index into bvl_vec */
+
+	unsigned int            bi_bvec_done;	/* number of bytes completed in
+						   current bvec */
+	unsigned		submit_time_us;
+	struct bkey_i		key;
+	struct bch_extent_ptr	ptr;
+	/* Only ever have a single pointer (the one we're doing io to/from) */
+	struct bio		bio;
+};
+
+#define to_bbio(_bio)		container_of((_bio), struct bbio, bio)
+
+struct bch_write_bio {
+	struct bio		*orig;
+	unsigned		bounce:1;
+	struct bbio		bio;
+};
+
+#define to_wbio(_bio)			\
+	container_of((_bio), struct bch_write_bio, bio.bio)
 
 struct bch_replace_info {
 	unsigned successes;	/* How many insertions succeeded */
@@ -16,7 +36,7 @@ struct bch_write_op {
 	struct closure		cl;
 	struct cache_set	*c;
 	struct workqueue_struct	*io_wq;
-	struct bio		*bio;
+	struct bch_write_bio	*bio;
 
 	short			error;
 
@@ -71,7 +91,7 @@ enum bch_write_flags {
 };
 
 void bch_write_op_init(struct bch_write_op *, struct cache_set *,
-		       struct bio *, struct write_point *,
+		       struct bch_write_bio *, struct write_point *,
 		       struct bkey_s_c, struct bkey_s_c, unsigned);
 void bch_write(struct closure *);
 
@@ -87,7 +107,7 @@ void bch_bio_submit_work(struct work_struct *);
 void bch_bbio_prep(struct bbio *, struct cache *);
 void bch_submit_bbio(struct bbio *, struct cache *, const struct bkey_i *,
 		     const struct bch_extent_ptr *, bool);
-void bch_submit_bbio_replicas(struct bio *, struct cache_set *,
+void bch_submit_bbio_replicas(struct bch_write_bio *, struct cache_set *,
 			      const struct bkey_i *, unsigned, bool);
 
 int bch_discard(struct cache_set *, struct bpos, struct bpos, u64);
diff --git a/drivers/md/bcache/migrate.c b/drivers/md/bcache/migrate.c
index a0dadd51b766..66bf35c082c5 100644
--- a/drivers/md/bcache/migrate.c
+++ b/drivers/md/bcache/migrate.c
@@ -171,7 +171,7 @@ static int issue_migration_move(struct cache *ca,
 
 	/* This also copies k into the write op's replace_key and insert_key */
 
-	bch_write_op_init(&io->op, c, &io->bio.bio,
+	bch_write_op_init(&io->op, c, &io->bio,
 			  wp, k, k, BCH_WRITE_CHECK_ENOSPC);
 
 	BUG_ON(q->wq == NULL);
diff --git a/drivers/md/bcache/move.c b/drivers/md/bcache/move.c
index 776335d82baa..2aed02880a36 100644
--- a/drivers/md/bcache/move.c
+++ b/drivers/md/bcache/move.c
@@ -75,7 +75,7 @@ static void bch_queue_write(struct moving_queue *q)
 
 static void moving_init(struct moving_io *io)
 {
-	struct bio *bio = &io->bio.bio;
+	struct bio *bio = &io->bio.bio.bio;
 
 	bio_init(bio);
 	bio_get(bio);
@@ -103,7 +103,7 @@ struct moving_io *moving_io_alloc(struct bkey_s_c k)
 
 	moving_init(io);
 
-	if (bio_alloc_pages(&io->bio.bio, GFP_KERNEL)) {
+	if (bio_alloc_pages(&io->bio.bio.bio, GFP_KERNEL)) {
 		kfree(io);
 		return NULL;
 	}
@@ -113,7 +113,7 @@ struct moving_io *moving_io_alloc(struct bkey_s_c k)
 
 void moving_io_free(struct moving_io *io)
 {
-	bch_bio_free_pages(&io->bio.bio);
+	bch_bio_free_pages(&io->bio.bio.bio);
 	kfree(io);
 }
 
@@ -200,7 +200,7 @@ static void write_moving(struct moving_io *io)
 	else {
 		moving_init(io);
 
-		op->bio->bi_iter.bi_sector = bkey_start_offset(&io->key.k);
+		op->bio->bio.bio.bi_iter.bi_sector = bkey_start_offset(&io->key.k);
 
 		closure_call(&op->cl, bch_write, NULL, &io->cl);
 		closure_return_with_destructor(&io->cl, moving_io_after_write);
@@ -485,10 +485,10 @@ static void __bch_data_move(struct closure *cl)
 	if (io->context->rate)
 		bch_ratelimit_increment(io->context->rate, size);
 
-	bio_set_op_attrs(&io->bio.bio, REQ_OP_READ, 0);
-	io->bio.bio.bi_end_io	= read_moving_endio;
+	bio_set_op_attrs(&io->bio.bio.bio, REQ_OP_READ, 0);
+	io->bio.bio.bio.bi_end_io	= read_moving_endio;
 
-	bch_submit_bbio(&io->bio, pick.ca, &io->key, &pick.ptr, false);
+	bch_submit_bbio(&io->bio.bio, pick.ca, &io->key, &pick.ptr, false);
 }
 
 /*
diff --git a/drivers/md/bcache/move.h b/drivers/md/bcache/move.h
index 8d479c682109..564f813763e0 100644
--- a/drivers/md/bcache/move.h
+++ b/drivers/md/bcache/move.h
@@ -91,7 +91,7 @@ struct moving_io {
 	int			read_completed : 1;
 	int			write_issued : 1;
 	/* Must be last since it is variable size */
-	struct bbio		bio;
+	struct bch_write_bio	bio;
 };
 
 struct moving_io *moving_io_alloc(struct bkey_s_c);
diff --git a/drivers/md/bcache/movinggc.c b/drivers/md/bcache/movinggc.c
index 01938a76afde..91cba2715773 100644
--- a/drivers/md/bcache/movinggc.c
+++ b/drivers/md/bcache/movinggc.c
@@ -59,7 +59,7 @@ static int issue_moving_gc_move(struct moving_queue *q,
 		return -ENOMEM;
 	}
 
-	bch_write_op_init(&io->op, c, &io->bio.bio, NULL,
+	bch_write_op_init(&io->op, c, &io->bio, NULL,
 			  bkey_i_to_s_c(k), bkey_i_to_s_c(k),
 			  bkey_extent_is_cached(&k->k)
 			  ? BCH_WRITE_CACHED : 0);
diff --git a/drivers/md/bcache/request.c b/drivers/md/bcache/request.c
index f6092d3f20cb..17a910ef114e 100644
--- a/drivers/md/bcache/request.c
+++ b/drivers/md/bcache/request.c
@@ -209,7 +209,7 @@ static void bio_complete(struct search *s)
 
 static void do_bio_hook(struct search *s, struct bio *orig_bio)
 {
-	struct bio *bio = &s->bio.bio;
+	struct bio *bio = &s->bio.bio.bio;
 
 	bio_init(bio);
 	__bio_clone_fast(bio, orig_bio);
@@ -226,7 +226,7 @@ static void search_free(struct closure *cl)
 	bio_complete(s);
 
 	if (s->iop.bio)
-		bio_put(s->iop.bio);
+		bio_put(&s->iop.bio->bio.bio);
 
 	closure_debug_destroy(cl);
 	mempool_free(s, &s->d->c->search);
@@ -275,7 +275,7 @@ static void cached_dev_bio_complete(struct closure *cl)
 static void cached_dev_read_error(struct closure *cl)
 {
 	struct search *s = container_of(cl, struct search, cl);
-	struct bio *bio = &s->bio.bio;
+	struct bio *bio = &s->bio.bio.bio;
 
 	if (s->recoverable) {
 		/* Read bucket invalidate races are handled here, also plain
@@ -396,7 +396,7 @@ static void bch_cache_read_endio(struct bio *bio)
 
 	if (bio->bi_error)
 		s->iop.error = bio->bi_error;
-	else if (ptr_stale(b->ca, &bkey_i_to_extent_c(&b->key)->v.ptr[0])) {
+	else if (ptr_stale(b->ca, &b->ptr)) {
 		/* Read bucket invalidate race */
 		atomic_long_inc(&s->iop.c->cache_read_races);
 		s->iop.error = -EINTR;
@@ -408,11 +408,11 @@ static void bch_cache_read_endio(struct bio *bio)
 static void cached_dev_read(struct cached_dev *dc, struct search *s)
 {
 	struct closure *cl = &s->cl;
-	struct bio *bio = &s->bio.bio;
+	struct bio *bio = &s->bio.bio.bio;
 	struct btree_iter iter;
 	struct bkey_s_c k;
 
-	bch_increment_clock(s->iop.c, bio_sectors(&s->bio.bio), READ);
+	bch_increment_clock(s->iop.c, bio_sectors(bio), READ);
 
 	for_each_btree_key_with_holes(&iter, s->iop.c, BTREE_ID_EXTENTS,
 				POS(s->inode, bio->bi_iter.bi_sector), k) {
@@ -509,12 +509,11 @@ static void cached_dev_write_complete(struct closure *cl)
 static void cached_dev_write(struct cached_dev *dc, struct search *s)
 {
 	struct closure *cl = &s->cl;
-	struct bio *bio = &s->bio.bio;
+	struct bio *bio = &s->bio.bio.bio;
 	unsigned inode = bcache_dev_inum(&dc->disk);
 	bool writeback = false;
 	bool bypass = s->bypass;
 	struct bkey insert_key = KEY(s->inode, 0, 0);
-	struct bio *insert_bio;
 	unsigned flags = 0;
 
 	down_read_non_owner(&dc->writeback_lock);
@@ -546,19 +545,17 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 	}
 
 	if (bypass) {
-		insert_bio = s->orig_bio;
-		bio_get(insert_bio);
-
-		/* If this is a bypass-write (as opposed to a discard), send
+		/*
+		 * If this is a bypass-write (as opposed to a discard), send
 		 * it down to the backing device. If this is a discard, only
 		 * send it to the backing device if the backing device
 		 * supports discards. Otherwise, we simply discard the key
-		 * range from the cache and don't touch the backing device. */
+		 * range from the cache and don't touch the backing device.
+		 */
 		if ((bio_op(bio) != REQ_OP_DISCARD) ||
 		    blk_queue_discard(bdev_get_queue(dc->disk_sb.bdev)))
-			closure_bio_submit(bio, cl);
+			closure_bio_submit(s->orig_bio, cl);
 	} else if (writeback) {
-		insert_bio = bio;
 		bch_writeback_add(dc);
 
 		if (bio->bi_opf & REQ_PREFLUSH) {
@@ -574,8 +571,10 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 			closure_bio_submit(flush, cl);
 		}
 	} else {
-		insert_bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split);
-		closure_bio_submit(bio, cl);
+		struct bio *writethrough =
+			bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split);
+
+		closure_bio_submit(writethrough, cl);
 
 		flags |= BCH_WRITE_CACHED;
 		flags |= BCH_WRITE_ALLOC_NOWAIT;
@@ -586,7 +585,7 @@ static void cached_dev_write(struct cached_dev *dc, struct search *s)
 	if (bypass)
 		flags |= BCH_WRITE_DISCARD;
 
-	bch_write_op_init(&s->iop, dc->disk.c, insert_bio, NULL,
+	bch_write_op_init(&s->iop, dc->disk.c, &s->bio, NULL,
 			  bkey_to_s_c(&insert_key), bkey_s_c_null, flags);
 
 	closure_call(&s->iop.cl, bch_write, NULL, cl);
@@ -619,7 +618,7 @@ static void __cached_dev_make_request(struct request_queue *q, struct bio *bio)
 			 * If it's a flush, we send the flush to the backing
 			 * device too
 			 */
-			closure_bio_submit(&s->bio.bio, &s->cl);
+			closure_bio_submit(&s->bio.bio.bio, &s->cl);
 
 			continue_at(&s->cl, cached_dev_bio_complete, NULL);
 		} else {
@@ -709,15 +708,14 @@ static void __flash_dev_make_request(struct request_queue *q, struct bio *bio)
 	} else if (rw) {
 		unsigned flags = 0;
 
-		s = search_alloc(bio, d);
-		bio = &s->bio.bio;
-
 		if (bio->bi_opf & (REQ_PREFLUSH|REQ_FUA))
 			flags |= BCH_WRITE_FLUSH;
 		if (bio_op(bio) == REQ_OP_DISCARD)
 			flags |= BCH_WRITE_DISCARD;
 
-		bch_write_op_init(&s->iop, d->c, bio, NULL,
+		s = search_alloc(bio, d);
+
+		bch_write_op_init(&s->iop, d->c, &s->bio, NULL,
 				  bkey_to_s_c(&KEY(s->inode, 0, 0)),
 				  bkey_s_c_null, flags);
 
diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index 5d78a0b9ad3c..61047a66fb0c 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -764,7 +764,8 @@ static void cache_set_free(struct closure *cl)
 	bch_io_clock_exit(&c->io_clock[WRITE]);
 	bch_io_clock_exit(&c->io_clock[READ]);
 	bdi_destroy(&c->bdi);
-	bioset_exit(&c->btree_bio);
+	bioset_exit(&c->btree_read_bio);
+	bioset_exit(&c->bio_write);
 	bioset_exit(&c->bio_split);
 	mempool_exit(&c->btree_reserve_pool);
 	mempool_exit(&c->fill_iter);
@@ -992,7 +993,8 @@ static struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
 					BTREE_RESERVE_SIZE) ||
 	    mempool_init_kmalloc_pool(&c->fill_iter, 1, iter_size) ||
 	    bioset_init(&c->bio_split, 4, offsetof(struct bbio, bio)) ||
-	    bioset_init(&c->btree_bio, 1, offsetof(struct bbio, bio)) ||
+	    bioset_init(&c->bio_write, 4, offsetof(struct bch_write_bio, bio.bio)) ||
+	    bioset_init(&c->btree_read_bio, 1, offsetof(struct bbio, bio)) ||
 	    bdi_setup_and_register(&c->bdi, "bcache") ||
 	    bch_io_clock_init(&c->io_clock[READ]) ||
 	    bch_io_clock_init(&c->io_clock[WRITE]) ||
@@ -1906,7 +1908,8 @@ static const char *cache_alloc(struct bcache_superblock *sb,
 	    !(ca->journal.bucket_seq = kcalloc(bch_nr_journal_buckets(&ca->sb),
 					       sizeof(u64), GFP_KERNEL)) ||
 	    !(ca->bio_prio = bio_kmalloc(GFP_NOIO, bucket_pages(ca))) ||
-	    bioset_init(&ca->replica_set, 4, offsetof(struct bbio, bio)))
+	    bioset_init(&ca->replica_set, 4,
+			offsetof(struct bch_write_bio, bio.bio)))
 		goto err;
 
 	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);
diff --git a/drivers/md/bcache/tier.c b/drivers/md/bcache/tier.c
index 94437a297338..ed05c6f4d412 100644
--- a/drivers/md/bcache/tier.c
+++ b/drivers/md/bcache/tier.c
@@ -226,7 +226,7 @@ static int issue_tiering_move(struct moving_queue *q,
 		return -ENOMEM;
 	}
 
-	bch_write_op_init(&io->op, c, &io->bio.bio,
+	bch_write_op_init(&io->op, c, &io->bio,
 			  &ca->tiering_write_point,
 			  bkey_i_to_s_c(&io->key),
 			  bkey_i_to_s_c(&io->key), 0);