bcachefs: Avoid atomics in write fast path

This adds some horrible hacks, but the atomic ops for closures were
getting to be a pretty expensive part of the write path. We don't want
to rip out closures entirely from the write path, because they're used
for e.g. waiting on the allocator, or waiting on the journal flush, and
that stuff would get really ugly without closures.

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

4 files changed, 65 insertions, 50 deletions

diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c
index 6098ed7d8f96..b4a9f81fcc75 100644
--- a/fs/bcachefs/fs-io.c
+++ b/fs/bcachefs/fs-io.c
@@ -45,7 +45,7 @@ struct bch_writepage_io {
 };
 
 struct dio_write {
-	struct closure			cl;
+	struct completion		done;
 	struct kiocb			*req;
 	struct mm_struct		*mm;
 	unsigned			loop:1,
@@ -1732,8 +1732,6 @@ start:
 
 /* O_DIRECT writes */
 
-static void bch2_dio_write_loop_async(struct closure *);
-
 static long bch2_dio_write_loop(struct dio_write *dio)
 {
 	bool kthread = (current->flags & PF_KTHREAD) != 0;
@@ -1807,8 +1805,6 @@ static long bch2_dio_write_loop(struct dio_write *dio)
 
 		task_io_account_write(bio->bi_iter.bi_size);
 
-		closure_call(&dio->op.cl, bch2_write, NULL, &dio->cl);
-
 		if (!dio->sync && !dio->loop && dio->iter.count) {
 			struct iovec *iov = dio->inline_vecs;
 
@@ -1816,8 +1812,8 @@ static long bch2_dio_write_loop(struct dio_write *dio)
 				iov = kmalloc(dio->iter.nr_segs * sizeof(*iov),
 					      GFP_KERNEL);
 				if (unlikely(!iov)) {
-					dio->op.error = -ENOMEM;
-					goto err_wait_io;
+					dio->sync = true;
+					goto do_io;
 				}
 
 				dio->free_iov = true;
@@ -1826,15 +1822,14 @@ static long bch2_dio_write_loop(struct dio_write *dio)
 			memcpy(iov, dio->iter.iov, dio->iter.nr_segs * sizeof(*iov));
 			dio->iter.iov = iov;
 		}
-err_wait_io:
+do_io:
 		dio->loop = true;
+		closure_call(&dio->op.cl, bch2_write, NULL, NULL);
 
-		if (!dio->sync) {
-			continue_at(&dio->cl, bch2_dio_write_loop_async, NULL);
+		if (dio->sync)
+			wait_for_completion(&dio->done);
+		else
 			return -EIOCBQUEUED;
-		}
-
-		closure_sync(&dio->cl);
 loop:
 		i_sectors_acct(c, inode, &dio->quota_res,
 			       dio->op.i_sectors_delta);
@@ -1851,7 +1846,9 @@ loop:
 			put_page(bv->bv_page);
 		if (!dio->iter.count || dio->op.error)
 			break;
+
 		bio_reset(bio);
+		reinit_completion(&dio->done);
 	}
 
 	ret = dio->op.error ?: ((long) dio->op.written << 9);
@@ -1863,8 +1860,6 @@ err:
 	if (dio->free_iov)
 		kfree(dio->iter.iov);
 
-	closure_debug_destroy(&dio->cl);
-
 	sync = dio->sync;
 	bio_put(bio);
 
@@ -1878,11 +1873,14 @@ err:
 	return ret;
 }
 
-static void bch2_dio_write_loop_async(struct closure *cl)
+static void bch2_dio_write_loop_async(struct bch_write_op *op)
 {
-	struct dio_write *dio = container_of(cl, struct dio_write, cl);
+	struct dio_write *dio = container_of(op, struct dio_write, op);
 
-	bch2_dio_write_loop(dio);
+	if (dio->sync)
+		complete(&dio->done);
+	else
+		bch2_dio_write_loop(dio);
 }
 
 static int bch2_direct_IO_write(struct kiocb *req,
@@ -1909,7 +1907,7 @@ static int bch2_direct_IO_write(struct kiocb *req,
 			       iov_iter_npages(iter, BIO_MAX_PAGES),
 			       &c->dio_write_bioset);
 	dio = container_of(bio, struct dio_write, op.wbio.bio);
-	closure_init(&dio->cl, NULL);
+	init_completion(&dio->done);
 	dio->req		= req;
 	dio->mm			= current->mm;
 	dio->loop		= false;
@@ -1920,6 +1918,7 @@ static int bch2_direct_IO_write(struct kiocb *req,
 	dio->iter		= *iter;
 
 	bch2_write_op_init(&dio->op, c, opts);
+	dio->op.end_io		= bch2_dio_write_loop_async;
 	dio->op.target		= opts.foreground_target;
 	op_journal_seq_set(&dio->op, &inode->ei_journal_seq);
 	dio->op.write_point	= writepoint_hashed((unsigned long) current);
@@ -1949,7 +1948,6 @@ static int bch2_direct_IO_write(struct kiocb *req,
 err:
 	bch2_disk_reservation_put(c, &dio->op.res);
 	bch2_quota_reservation_put(c, inode, &dio->quota_res);
-	closure_debug_destroy(&dio->cl);
 	bio_put(bio);
 	return ret;
 }
diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c
index 139c565321a9..836004b128f0 100644
--- a/fs/bcachefs/io.c
+++ b/fs/bcachefs/io.c
@@ -497,7 +497,12 @@ static void bch2_write_done(struct closure *cl)
 
 	bch2_time_stats_update(&c->times[BCH_TIME_data_write], op->start_time);
 
-	closure_return(cl);
+	if (op->end_io)
+		op->end_io(op);
+	if (cl->parent)
+		closure_return(cl);
+	else
+		closure_debug_destroy(cl);
 }
 
 /**
@@ -606,8 +611,10 @@ static void bch2_write_endio(struct bio *bio)
 
 	if (parent)
 		bio_endio(&parent->bio);
-	else
+	else if (!(op->flags & BCH_WRITE_SKIP_CLOSURE_PUT))
 		closure_put(cl);
+	else
+		continue_at_nobarrier(cl, bch2_write_index, index_update_wq(op));
 }
 
 static void init_append_extent(struct bch_write_op *op,
@@ -826,15 +833,14 @@ static enum prep_encoded_ret {
 	return PREP_ENCODED_OK;
 }
 
-static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
+static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp,
+			     struct bio **_dst)
 {
 	struct bch_fs *c = op->c;
 	struct bio *src = &op->wbio.bio, *dst = src;
 	struct bvec_iter saved_iter;
-	struct bkey_i *key_to_write;
 	void *ec_buf;
-	unsigned key_to_write_offset = op->insert_keys.top_p -
-		op->insert_keys.keys_p;
+	struct bpos ec_pos = op->pos;
 	unsigned total_output = 0, total_input = 0;
 	bool bounce = false;
 	bool page_alloc_failed = false;
@@ -853,6 +859,7 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
 	case PREP_ENCODED_CHECKSUM_ERR:
 		goto csum_err;
 	case PREP_ENCODED_DO_WRITE:
+		/* XXX look for bug here */
 		if (ec_buf) {
 			dst = bch2_write_bio_alloc(c, wp, src,
 						   &page_alloc_failed,
@@ -1002,31 +1009,15 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
 	dst->bi_iter.bi_size = total_output;
 do_write:
 	/* might have done a realloc... */
+	bch2_ec_add_backpointer(c, wp, ec_pos, total_input >> 9);
 
-	key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
-
-	bch2_ec_add_backpointer(c, wp,
-				bkey_start_pos(&key_to_write->k),
-				total_input >> 9);
-
-	bch2_alloc_sectors_done(c, wp);
-
-	dst->bi_end_io	= bch2_write_endio;
-	dst->bi_private	= &op->cl;
-	bio_set_op_attrs(dst, REQ_OP_WRITE, 0);
-
-	closure_get(dst->bi_private);
-
-	bch2_submit_wbio_replicas(to_wbio(dst), c, BCH_DATA_USER,
-				  key_to_write);
+	*_dst = dst;
 	return more;
 csum_err:
 	bch_err(c, "error verifying existing checksum while "
 		"rewriting existing data (memory corruption?)");
 	ret = -EIO;
 err:
-	bch2_alloc_sectors_done(c, wp);
-
 	if (to_wbio(dst)->bounce)
 		bch2_bio_free_pages_pool(c, dst);
 	if (to_wbio(dst)->put_bio)
@@ -1040,11 +1031,17 @@ static void __bch2_write(struct closure *cl)
 	struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
 	struct bch_fs *c = op->c;
 	struct write_point *wp;
+	struct bio *bio;
+	bool skip_put = true;
 	int ret;
 again:
 	memset(&op->failed, 0, sizeof(op->failed));
 
 	do {
+		struct bkey_i *key_to_write;
+		unsigned key_to_write_offset = op->insert_keys.top_p -
+			op->insert_keys.keys_p;
+
 		/* +1 for possible cache device: */
 		if (op->open_buckets.nr + op->nr_replicas + 1 >
 		    ARRAY_SIZE(op->open_buckets.v))
@@ -1078,21 +1075,38 @@ again:
 		}
 
 		bch2_open_bucket_get(c, wp, &op->open_buckets);
-
-		ret = bch2_write_extent(op, wp);
+		ret = bch2_write_extent(op, wp, &bio);
+		bch2_alloc_sectors_done(c, wp);
 
 		if (ret < 0)
 			goto err;
+
+		if (ret)
+			skip_put = false;
+
+		bio->bi_end_io	= bch2_write_endio;
+		bio->bi_private	= &op->cl;
+		bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
+
+		if (!skip_put)
+			closure_get(bio->bi_private);
+		else
+			op->flags |= BCH_WRITE_SKIP_CLOSURE_PUT;
+
+		key_to_write = (void *) (op->insert_keys.keys_p +
+					 key_to_write_offset);
+
+		bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_USER,
+					  key_to_write);
 	} while (ret);
 
-	continue_at(cl, bch2_write_index, index_update_wq(op));
+	if (!skip_put)
+		continue_at(cl, bch2_write_index, index_update_wq(op));
 	return;
 err:
 	op->error = ret;
 
-	continue_at(cl, !bch2_keylist_empty(&op->insert_keys)
-		    ? bch2_write_index
-		    : bch2_write_done, index_update_wq(op));
+	continue_at(cl, bch2_write_index, index_update_wq(op));
 	return;
 flush_io:
 	closure_sync(cl);
diff --git a/fs/bcachefs/io.h b/fs/bcachefs/io.h
index bd3d74ddefa8..91aaa58fce4e 100644
--- a/fs/bcachefs/io.h
+++ b/fs/bcachefs/io.h
@@ -33,6 +33,7 @@ enum bch_write_flags {
 
 	/* Internal: */
 	BCH_WRITE_JOURNAL_SEQ_PTR	= (1 << 8),
+	BCH_WRITE_SKIP_CLOSURE_PUT	= (1 << 9),
 };
 
 static inline u64 *op_journal_seq(struct bch_write_op *op)
@@ -67,6 +68,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
 				      struct bch_io_opts opts)
 {
 	op->c			= c;
+	op->end_io		= NULL;
 	op->flags		= 0;
 	op->written		= 0;
 	op->error		= 0;
diff --git a/fs/bcachefs/io_types.h b/fs/bcachefs/io_types.h
index 77ea6b047463..c37b7d7401e9 100644
--- a/fs/bcachefs/io_types.h
+++ b/fs/bcachefs/io_types.h
@@ -93,6 +93,7 @@ struct bch_write_bio {
 struct bch_write_op {
 	struct closure		cl;
 	struct bch_fs		*c;
+	void			(*end_io)(struct bch_write_op *);
 	u64			start_time;
 
 	unsigned		written; /* sectors */