summaryrefslogtreecommitdiff
path: root/io_uring
diff options
context:
space:
mode:
Diffstat (limited to 'io_uring')
-rw-r--r--io_uring/io_uring.c11
-rw-r--r--io_uring/kbuf.h8
-rw-r--r--io_uring/msg_ring.c3
-rw-r--r--io_uring/net.c23
-rw-r--r--io_uring/notif.c8
-rw-r--r--io_uring/opdef.c2
-rw-r--r--io_uring/poll.c2
-rw-r--r--io_uring/rw.c30
-rw-r--r--io_uring/xattr.c2
9 files changed, 52 insertions, 37 deletions
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index f9be9b7eb654..242d896c00f3 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1728,6 +1728,7 @@ static void io_queue_async(struct io_kiocb *req, int ret)
switch (io_arm_poll_handler(req, 0)) {
case IO_APOLL_READY:
+ io_kbuf_recycle(req, 0);
io_req_task_queue(req);
break;
case IO_APOLL_ABORTED:
@@ -2647,6 +2648,9 @@ static __cold void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
io_kill_timeouts(ctx, NULL, true);
/* if we failed setting up the ctx, we might not have any rings */
io_iopoll_try_reap_events(ctx);
+ /* drop cached put refs after potentially doing completions */
+ if (current->io_uring)
+ io_uring_drop_tctx_refs(current);
}
INIT_WORK(&ctx->exit_work, io_ring_exit_work);
@@ -3353,6 +3357,10 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
goto err;
}
+ if (ctx->flags & IORING_SETUP_SINGLE_ISSUER
+ && !(ctx->flags & IORING_SETUP_R_DISABLED))
+ ctx->submitter_task = get_task_struct(current);
+
file = io_uring_get_file(ctx);
if (IS_ERR(file)) {
ret = PTR_ERR(file);
@@ -3544,6 +3552,9 @@ static int io_register_enable_rings(struct io_ring_ctx *ctx)
if (!(ctx->flags & IORING_SETUP_R_DISABLED))
return -EBADFD;
+ if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task)
+ ctx->submitter_task = get_task_struct(current);
+
if (ctx->restrictions.registered)
ctx->restricted = 1;
diff --git a/io_uring/kbuf.h b/io_uring/kbuf.h
index d6af208d109f..746fbf31a703 100644
--- a/io_uring/kbuf.h
+++ b/io_uring/kbuf.h
@@ -91,9 +91,13 @@ static inline void io_kbuf_recycle(struct io_kiocb *req, unsigned issue_flags)
* buffer data. However if that buffer is recycled the original request
* data stored in addr is lost. Therefore forbid recycling for now.
*/
- if (req->opcode == IORING_OP_READV)
+ if (req->opcode == IORING_OP_READV) {
+ if ((req->flags & REQ_F_BUFFER_RING) && req->buf_list) {
+ req->buf_list->head++;
+ req->buf_list = NULL;
+ }
return;
-
+ }
if (req->flags & REQ_F_BUFFER_SELECTED)
io_kbuf_recycle_legacy(req, issue_flags);
if (req->flags & REQ_F_BUFFER_RING)
diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c
index 976c4ba68ee7..4a7e5d030c78 100644
--- a/io_uring/msg_ring.c
+++ b/io_uring/msg_ring.c
@@ -165,7 +165,8 @@ done:
req_set_fail(req);
io_req_set_res(req, ret, 0);
/* put file to avoid an attempt to IOPOLL the req */
- io_put_file(req->file);
+ if (!(req->flags & REQ_F_FIXED_FILE))
+ io_put_file(req->file);
req->file = NULL;
return IOU_OK;
}
diff --git a/io_uring/net.c b/io_uring/net.c
index 7047c1342541..60e392f7f2dc 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -905,6 +905,13 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST |
IORING_RECVSEND_FIXED_BUF))
return -EINVAL;
+ notif = zc->notif = io_alloc_notif(ctx);
+ if (!notif)
+ return -ENOMEM;
+ notif->cqe.user_data = req->cqe.user_data;
+ notif->cqe.res = 0;
+ notif->cqe.flags = IORING_CQE_F_NOTIF;
+ req->flags |= REQ_F_NEED_CLEANUP;
if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
unsigned idx = READ_ONCE(sqe->buf_index);
@@ -912,15 +919,8 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EFAULT;
idx = array_index_nospec(idx, ctx->nr_user_bufs);
req->imu = READ_ONCE(ctx->user_bufs[idx]);
- io_req_set_rsrc_node(req, ctx, 0);
+ io_req_set_rsrc_node(notif, ctx, 0);
}
- notif = zc->notif = io_alloc_notif(ctx);
- if (!notif)
- return -ENOMEM;
- notif->cqe.user_data = req->cqe.user_data;
- notif->cqe.res = 0;
- notif->cqe.flags = IORING_CQE_F_NOTIF;
- req->flags |= REQ_F_NEED_CLEANUP;
zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
zc->len = READ_ONCE(sqe->len);
@@ -1003,9 +1003,6 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
unsigned msg_flags, cflags;
int ret, min_ret = 0;
- if (!(req->flags & REQ_F_POLLED) &&
- (zc->flags & IORING_RECVSEND_POLL_FIRST))
- return -EAGAIN;
sock = sock_from_file(req->file);
if (unlikely(!sock))
return -ENOTSOCK;
@@ -1030,6 +1027,10 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags)
msg.msg_namelen = zc->addr_len;
}
+ if (!(req->flags & REQ_F_POLLED) &&
+ (zc->flags & IORING_RECVSEND_POLL_FIRST))
+ return io_setup_async_addr(req, addr, issue_flags);
+
if (zc->flags & IORING_RECVSEND_FIXED_BUF) {
ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu,
(u64)(uintptr_t)zc->buf, zc->len);
diff --git a/io_uring/notif.c b/io_uring/notif.c
index 38d77165edc3..e37c6569d82e 100644
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -21,14 +21,6 @@ static void __io_notif_complete_tw(struct io_kiocb *notif, bool *locked)
io_req_task_complete(notif, locked);
}
-static inline void io_notif_complete(struct io_kiocb *notif)
- __must_hold(&notif->ctx->uring_lock)
-{
- bool locked = true;
-
- __io_notif_complete_tw(notif, &locked);
-}
-
static void io_uring_tx_zerocopy_callback(struct sk_buff *skb,
struct ubuf_info *uarg,
bool success)
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index c61494e0a602..c4dddd0fd709 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -471,7 +471,7 @@ const struct io_op_def io_op_defs[] = {
.prep_async = io_uring_cmd_prep_async,
},
[IORING_OP_SEND_ZC] = {
- .name = "SENDZC_NOTIF",
+ .name = "SEND_ZC",
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
diff --git a/io_uring/poll.c b/io_uring/poll.c
index d5bad0bea6e4..0d9f49c575e0 100644
--- a/io_uring/poll.c
+++ b/io_uring/poll.c
@@ -857,7 +857,7 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
if (sqe->buf_index || sqe->off || sqe->addr)
return -EINVAL;
flags = READ_ONCE(sqe->len);
- if (flags & ~(IORING_POLL_ADD_MULTI|IORING_POLL_ADD_LEVEL))
+ if (flags & ~IORING_POLL_ADD_MULTI)
return -EINVAL;
if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))
return -EINVAL;
diff --git a/io_uring/rw.c b/io_uring/rw.c
index 1babd77da79c..76ebcfebc9a6 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -206,6 +206,20 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res)
return false;
}
+static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
+{
+ struct io_async_rw *io = req->async_data;
+
+ /* add previously done IO, if any */
+ if (req_has_async_data(req) && io->bytes_done > 0) {
+ if (res < 0)
+ res = io->bytes_done;
+ else
+ res += io->bytes_done;
+ }
+ return res;
+}
+
static void io_complete_rw(struct kiocb *kiocb, long res)
{
struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
@@ -213,7 +227,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res)
if (__io_complete_rw_common(req, res))
return;
- io_req_set_res(req, res, 0);
+ io_req_set_res(req, io_fixup_rw_res(req, res), 0);
req->io_task_work.func = io_req_task_complete;
io_req_task_work_add(req);
}
@@ -240,22 +254,14 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
static int kiocb_done(struct io_kiocb *req, ssize_t ret,
unsigned int issue_flags)
{
- struct io_async_rw *io = req->async_data;
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
-
- /* add previously done IO, if any */
- if (req_has_async_data(req) && io->bytes_done > 0) {
- if (ret < 0)
- ret = io->bytes_done;
- else
- ret += io->bytes_done;
- }
+ unsigned final_ret = io_fixup_rw_res(req, ret);
if (req->flags & REQ_F_CUR_POS)
req->file->f_pos = rw->kiocb.ki_pos;
if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) {
if (!__io_complete_rw_common(req, ret)) {
- io_req_set_res(req, req->cqe.res,
+ io_req_set_res(req, final_ret,
io_put_kbuf(req, issue_flags));
return IOU_OK;
}
@@ -268,7 +274,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,
if (io_resubmit_prep(req))
io_req_task_queue_reissue(req);
else
- io_req_task_queue_fail(req, ret);
+ io_req_task_queue_fail(req, final_ret);
}
return IOU_ISSUE_SKIP_COMPLETE;
}
diff --git a/io_uring/xattr.c b/io_uring/xattr.c
index 84180afd090b..99df641594d7 100644
--- a/io_uring/xattr.c
+++ b/io_uring/xattr.c
@@ -206,7 +206,7 @@ int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
}
static int __io_setxattr(struct io_kiocb *req, unsigned int issue_flags,
- struct path *path)
+ const struct path *path)
{
struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr);
int ret;
generated by cgit v1.2.3 (git 2.46.0) at 2025-08-22 11:11:07 +0000