diff options
| author | Kent Overstreet <kent.overstreet@linux.dev> | 2023-10-24 14:46:58 -0400 |
|---|---|---|
| committer | Kent Overstreet <kent.overstreet@linux.dev> | 2023-10-25 01:50:27 -0400 |
| commit | 0d63ed13ea3d867055ae5752e2e0514a227d1dcb (patch) | |
| tree | 77d245f872bb2dc2600d38dda0a64340716d59da | |
| parent | c8498e7253006090d0fdd680755ebfde24034fb1 (diff) | |
closures: Fix race in closure_sync()
As pointed out by Linus, closure_sync() was racy; we could skip blocking
immediately after a get() and a put(), but then that would skip any
barrier corresponding to the other thread's put() barrier.
To fix this, always do the full __closure_sync() sequence whenever any
get() has happened and the closure might have been used by other
threads.
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
| -rw-r--r-- | fs/bcachefs/fs-io-direct.c | 1 | ||||
| -rw-r--r-- | include/linux/closure.h | 10 | ||||
| -rw-r--r-- | lib/closure.c | 3 |
3 files changed, 13 insertions, 1 deletions
diff --git a/fs/bcachefs/fs-io-direct.c b/fs/bcachefs/fs-io-direct.c index 6a9557e7ecab..5b42a76c4796 100644 --- a/fs/bcachefs/fs-io-direct.c +++ b/fs/bcachefs/fs-io-direct.c @@ -113,6 +113,7 @@ static int bch2_direct_IO_read(struct kiocb *req, struct iov_iter *iter) } else { atomic_set(&dio->cl.remaining, CLOSURE_REMAINING_INITIALIZER + 1); + dio->cl.closure_get_happened = true; } dio->req = req; diff --git a/include/linux/closure.h b/include/linux/closure.h index bdab17050bc8..de7bb47d8a46 100644 --- a/include/linux/closure.h +++ b/include/linux/closure.h @@ -154,6 +154,7 @@ struct closure { struct closure *parent; atomic_t remaining; + bool closure_get_happened; #ifdef CONFIG_DEBUG_CLOSURES #define CLOSURE_MAGIC_DEAD 0xc054dead @@ -185,7 +186,11 @@ static inline unsigned closure_nr_remaining(struct closure *cl) */ static inline void closure_sync(struct closure *cl) { - if (closure_nr_remaining(cl) != 1) +#ifdef CONFIG_DEBUG_CLOSURES + BUG_ON(closure_nr_remaining(cl) != 1 && !cl->closure_get_happened); +#endif + + if (cl->closure_get_happened) __closure_sync(cl); } @@ -257,6 +262,8 @@ static inline void closure_queue(struct closure *cl) */ static inline void closure_get(struct closure *cl) { + cl->closure_get_happened = true; + #ifdef CONFIG_DEBUG_CLOSURES BUG_ON((atomic_inc_return(&cl->remaining) & CLOSURE_REMAINING_MASK) <= 1); @@ -279,6 +286,7 @@ static inline void closure_init(struct closure *cl, struct closure *parent) closure_get(parent); atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER); + cl->closure_get_happened = false; closure_debug_create(cl); closure_set_ip(cl); diff --git a/lib/closure.c b/lib/closure.c index 545b9726d1ea..1faa24d6400e 100644 --- a/lib/closure.c +++ b/lib/closure.c @@ -24,6 +24,8 @@ static inline void closure_put_after_sub(struct closure *cl, int flags) if (!r) { smp_acquire__after_ctrl_dep(); + cl->closure_get_happened = false; + if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) { atomic_set(&cl->remaining, CLOSURE_REMAINING_INITIALIZER); @@ -93,6 +95,7 @@ bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl) if (atomic_read(&cl->remaining) & CLOSURE_WAITING) return false; + cl->closure_get_happened = true; closure_set_waiting(cl, _RET_IP_); atomic_add(CLOSURE_WAITING + 1, &cl->remaining); llist_add(&cl->list, &waitlist->list); |