summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--fs/bcachefs/fs-io.c38
-rw-r--r--fs/bcachefs/io.c74
-rw-r--r--fs/bcachefs/io.h2
-rw-r--r--fs/bcachefs/io_types.h1
4 files changed, 65 insertions, 50 deletions
diff --git a/fs/bcachefs/fs-io.c b/fs/bcachefs/fs-io.c
index 6098ed7d8f96..b4a9f81fcc75 100644
--- a/fs/bcachefs/fs-io.c
+++ b/fs/bcachefs/fs-io.c
@@ -45,7 +45,7 @@ struct bch_writepage_io {
};
struct dio_write {
- struct closure cl;
+ struct completion done;
struct kiocb *req;
struct mm_struct *mm;
unsigned loop:1,
@@ -1732,8 +1732,6 @@ start:
/* O_DIRECT writes */
-static void bch2_dio_write_loop_async(struct closure *);
-
static long bch2_dio_write_loop(struct dio_write *dio)
{
bool kthread = (current->flags & PF_KTHREAD) != 0;
@@ -1807,8 +1805,6 @@ static long bch2_dio_write_loop(struct dio_write *dio)
task_io_account_write(bio->bi_iter.bi_size);
- closure_call(&dio->op.cl, bch2_write, NULL, &dio->cl);
-
if (!dio->sync && !dio->loop && dio->iter.count) {
struct iovec *iov = dio->inline_vecs;
@@ -1816,8 +1812,8 @@ static long bch2_dio_write_loop(struct dio_write *dio)
iov = kmalloc(dio->iter.nr_segs * sizeof(*iov),
GFP_KERNEL);
if (unlikely(!iov)) {
- dio->op.error = -ENOMEM;
- goto err_wait_io;
+ dio->sync = true;
+ goto do_io;
}
dio->free_iov = true;
@@ -1826,15 +1822,14 @@ static long bch2_dio_write_loop(struct dio_write *dio)
memcpy(iov, dio->iter.iov, dio->iter.nr_segs * sizeof(*iov));
dio->iter.iov = iov;
}
-err_wait_io:
+do_io:
dio->loop = true;
+ closure_call(&dio->op.cl, bch2_write, NULL, NULL);
- if (!dio->sync) {
- continue_at(&dio->cl, bch2_dio_write_loop_async, NULL);
+ if (dio->sync)
+ wait_for_completion(&dio->done);
+ else
return -EIOCBQUEUED;
- }
-
- closure_sync(&dio->cl);
loop:
i_sectors_acct(c, inode, &dio->quota_res,
dio->op.i_sectors_delta);
@@ -1851,7 +1846,9 @@ loop:
put_page(bv->bv_page);
if (!dio->iter.count || dio->op.error)
break;
+
bio_reset(bio);
+ reinit_completion(&dio->done);
}
ret = dio->op.error ?: ((long) dio->op.written << 9);
@@ -1863,8 +1860,6 @@ err:
if (dio->free_iov)
kfree(dio->iter.iov);
- closure_debug_destroy(&dio->cl);
-
sync = dio->sync;
bio_put(bio);
@@ -1878,11 +1873,14 @@ err:
return ret;
}
-static void bch2_dio_write_loop_async(struct closure *cl)
+static void bch2_dio_write_loop_async(struct bch_write_op *op)
{
- struct dio_write *dio = container_of(cl, struct dio_write, cl);
+ struct dio_write *dio = container_of(op, struct dio_write, op);
- bch2_dio_write_loop(dio);
+ if (dio->sync)
+ complete(&dio->done);
+ else
+ bch2_dio_write_loop(dio);
}
static int bch2_direct_IO_write(struct kiocb *req,
@@ -1909,7 +1907,7 @@ static int bch2_direct_IO_write(struct kiocb *req,
iov_iter_npages(iter, BIO_MAX_PAGES),
&c->dio_write_bioset);
dio = container_of(bio, struct dio_write, op.wbio.bio);
- closure_init(&dio->cl, NULL);
+ init_completion(&dio->done);
dio->req = req;
dio->mm = current->mm;
dio->loop = false;
@@ -1920,6 +1918,7 @@ static int bch2_direct_IO_write(struct kiocb *req,
dio->iter = *iter;
bch2_write_op_init(&dio->op, c, opts);
+ dio->op.end_io = bch2_dio_write_loop_async;
dio->op.target = opts.foreground_target;
op_journal_seq_set(&dio->op, &inode->ei_journal_seq);
dio->op.write_point = writepoint_hashed((unsigned long) current);
@@ -1949,7 +1948,6 @@ static int bch2_direct_IO_write(struct kiocb *req,
err:
bch2_disk_reservation_put(c, &dio->op.res);
bch2_quota_reservation_put(c, inode, &dio->quota_res);
- closure_debug_destroy(&dio->cl);
bio_put(bio);
return ret;
}
diff --git a/fs/bcachefs/io.c b/fs/bcachefs/io.c
index 139c565321a9..836004b128f0 100644
--- a/fs/bcachefs/io.c
+++ b/fs/bcachefs/io.c
@@ -497,7 +497,12 @@ static void bch2_write_done(struct closure *cl)
bch2_time_stats_update(&c->times[BCH_TIME_data_write], op->start_time);
- closure_return(cl);
+ if (op->end_io)
+ op->end_io(op);
+ if (cl->parent)
+ closure_return(cl);
+ else
+ closure_debug_destroy(cl);
}
/**
@@ -606,8 +611,10 @@ static void bch2_write_endio(struct bio *bio)
if (parent)
bio_endio(&parent->bio);
- else
+ else if (!(op->flags & BCH_WRITE_SKIP_CLOSURE_PUT))
closure_put(cl);
+ else
+ continue_at_nobarrier(cl, bch2_write_index, index_update_wq(op));
}
static void init_append_extent(struct bch_write_op *op,
@@ -826,15 +833,14 @@ static enum prep_encoded_ret {
return PREP_ENCODED_OK;
}
-static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
+static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp,
+ struct bio **_dst)
{
struct bch_fs *c = op->c;
struct bio *src = &op->wbio.bio, *dst = src;
struct bvec_iter saved_iter;
- struct bkey_i *key_to_write;
void *ec_buf;
- unsigned key_to_write_offset = op->insert_keys.top_p -
- op->insert_keys.keys_p;
+ struct bpos ec_pos = op->pos;
unsigned total_output = 0, total_input = 0;
bool bounce = false;
bool page_alloc_failed = false;
@@ -853,6 +859,7 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
case PREP_ENCODED_CHECKSUM_ERR:
goto csum_err;
case PREP_ENCODED_DO_WRITE:
+ /* XXX look for bug here */
if (ec_buf) {
dst = bch2_write_bio_alloc(c, wp, src,
&page_alloc_failed,
@@ -1002,31 +1009,15 @@ static int bch2_write_extent(struct bch_write_op *op, struct write_point *wp)
dst->bi_iter.bi_size = total_output;
do_write:
/* might have done a realloc... */
+ bch2_ec_add_backpointer(c, wp, ec_pos, total_input >> 9);
- key_to_write = (void *) (op->insert_keys.keys_p + key_to_write_offset);
-
- bch2_ec_add_backpointer(c, wp,
- bkey_start_pos(&key_to_write->k),
- total_input >> 9);
-
- bch2_alloc_sectors_done(c, wp);
-
- dst->bi_end_io = bch2_write_endio;
- dst->bi_private = &op->cl;
- bio_set_op_attrs(dst, REQ_OP_WRITE, 0);
-
- closure_get(dst->bi_private);
-
- bch2_submit_wbio_replicas(to_wbio(dst), c, BCH_DATA_USER,
- key_to_write);
+ *_dst = dst;
return more;
csum_err:
bch_err(c, "error verifying existing checksum while "
"rewriting existing data (memory corruption?)");
ret = -EIO;
err:
- bch2_alloc_sectors_done(c, wp);
-
if (to_wbio(dst)->bounce)
bch2_bio_free_pages_pool(c, dst);
if (to_wbio(dst)->put_bio)
@@ -1040,11 +1031,17 @@ static void __bch2_write(struct closure *cl)
struct bch_write_op *op = container_of(cl, struct bch_write_op, cl);
struct bch_fs *c = op->c;
struct write_point *wp;
+ struct bio *bio;
+ bool skip_put = true;
int ret;
again:
memset(&op->failed, 0, sizeof(op->failed));
do {
+ struct bkey_i *key_to_write;
+ unsigned key_to_write_offset = op->insert_keys.top_p -
+ op->insert_keys.keys_p;
+
/* +1 for possible cache device: */
if (op->open_buckets.nr + op->nr_replicas + 1 >
ARRAY_SIZE(op->open_buckets.v))
@@ -1078,21 +1075,38 @@ again:
}
bch2_open_bucket_get(c, wp, &op->open_buckets);
-
- ret = bch2_write_extent(op, wp);
+ ret = bch2_write_extent(op, wp, &bio);
+ bch2_alloc_sectors_done(c, wp);
if (ret < 0)
goto err;
+
+ if (ret)
+ skip_put = false;
+
+ bio->bi_end_io = bch2_write_endio;
+ bio->bi_private = &op->cl;
+ bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
+
+ if (!skip_put)
+ closure_get(bio->bi_private);
+ else
+ op->flags |= BCH_WRITE_SKIP_CLOSURE_PUT;
+
+ key_to_write = (void *) (op->insert_keys.keys_p +
+ key_to_write_offset);
+
+ bch2_submit_wbio_replicas(to_wbio(bio), c, BCH_DATA_USER,
+ key_to_write);
} while (ret);
- continue_at(cl, bch2_write_index, index_update_wq(op));
+ if (!skip_put)
+ continue_at(cl, bch2_write_index, index_update_wq(op));
return;
err:
op->error = ret;
- continue_at(cl, !bch2_keylist_empty(&op->insert_keys)
- ? bch2_write_index
- : bch2_write_done, index_update_wq(op));
+ continue_at(cl, bch2_write_index, index_update_wq(op));
return;
flush_io:
closure_sync(cl);
diff --git a/fs/bcachefs/io.h b/fs/bcachefs/io.h
index bd3d74ddefa8..91aaa58fce4e 100644
--- a/fs/bcachefs/io.h
+++ b/fs/bcachefs/io.h
@@ -33,6 +33,7 @@ enum bch_write_flags {
/* Internal: */
BCH_WRITE_JOURNAL_SEQ_PTR = (1 << 8),
+ BCH_WRITE_SKIP_CLOSURE_PUT = (1 << 9),
};
static inline u64 *op_journal_seq(struct bch_write_op *op)
@@ -67,6 +68,7 @@ static inline void bch2_write_op_init(struct bch_write_op *op, struct bch_fs *c,
struct bch_io_opts opts)
{
op->c = c;
+ op->end_io = NULL;
op->flags = 0;
op->written = 0;
op->error = 0;
diff --git a/fs/bcachefs/io_types.h b/fs/bcachefs/io_types.h
index 77ea6b047463..c37b7d7401e9 100644
--- a/fs/bcachefs/io_types.h
+++ b/fs/bcachefs/io_types.h
@@ -93,6 +93,7 @@ struct bch_write_bio {
struct bch_write_op {
struct closure cl;
struct bch_fs *c;
+ void (*end_io)(struct bch_write_op *);
u64 start_time;
unsigned written; /* sectors */
generated by cgit v1.2.3 (git 2.46.0) at 2025-07-31 10:09:43 +0000