six locks: six_lock_waiter()

This allows passing in the wait list entry - to be used for a deadlock
cycle detector.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

2 files changed, 41 insertions, 13 deletions

diff --git a/include/linux/six.h b/include/linux/six.h
index 45097b3fdcb1..4d1853617ea7 100644
--- a/include/linux/six.h
+++ b/include/linux/six.h
@@ -151,6 +151,8 @@ do {									\
 bool six_trylock_##type(struct six_lock *);				\
 bool six_relock_##type(struct six_lock *, u32);				\
 int six_lock_##type(struct six_lock *, six_lock_should_sleep_fn, void *);\
+int six_lock_waiter_##type(struct six_lock *, struct six_lock_waiter *,	\
+			   six_lock_should_sleep_fn, void *);		\
 void six_unlock_##type(struct six_lock *);
 
 __SIX_LOCK(read)
@@ -187,6 +189,13 @@ static inline int six_lock_type(struct six_lock *lock, enum six_lock_type type,
 	SIX_LOCK_DISPATCH(type, six_lock, lock, should_sleep_fn, p);
 }
 
+static inline int six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
+				struct six_lock_waiter *wait,
+				six_lock_should_sleep_fn should_sleep_fn, void *p)
+{
+	SIX_LOCK_DISPATCH(type, six_lock_waiter, lock, wait, should_sleep_fn, p);
+}
+
 static inline void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 {
 	SIX_LOCK_DISPATCH(type, six_unlock, lock);
diff --git a/kernel/locking/six.c b/kernel/locking/six.c
index d1c8cf055ac1..54fe8b659ebd 100644
--- a/kernel/locking/six.c
+++ b/kernel/locking/six.c
@@ -460,13 +460,10 @@ static inline bool six_optimistic_spin(struct six_lock *lock, enum six_lock_type
 
 noinline
 static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type type,
+				    struct six_lock_waiter *wait,
 				    six_lock_should_sleep_fn should_sleep_fn, void *p)
 {
 	union six_lock_state old;
-	struct six_lock_waiter wait = {
-		.task = current,
-		.lock_want = type,
-	};
 	int ret = 0;
 
 	if (type == SIX_LOCK_write) {
@@ -480,6 +477,10 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty
 
 	lock_contended(&lock->dep_map, _RET_IP_);
 
+	wait->task		= current;
+	wait->lock_want		= type;
+	wait->lock_acquired	= false;
+
 	raw_spin_lock(&lock->wait_lock);
 	if (!(lock->state.waiters & (1 << type)))
 		set_bit(waitlist_bitnr(type), (unsigned long *) &lock->state.v);
@@ -489,7 +490,7 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty
 	 */
 	ret = __do_six_trylock_type(lock, type, current, false);
 	if (ret <= 0)
-		list_add_tail(&wait.list, &lock->wait_list);
+		list_add_tail(&wait->list, &lock->wait_list);
 	raw_spin_unlock(&lock->wait_lock);
 
 	if (unlikely(ret > 0)) {
@@ -505,17 +506,17 @@ static int __six_lock_type_slowpath(struct six_lock *lock, enum six_lock_type ty
 	while (1) {
 		set_current_state(TASK_UNINTERRUPTIBLE);
 
-		if (wait.lock_acquired)
+		if (wait->lock_acquired)
 			break;
 
 		ret = should_sleep_fn ? should_sleep_fn(lock, p) : 0;
 		if (unlikely(ret)) {
 			raw_spin_lock(&lock->wait_lock);
-			if (!wait.lock_acquired)
-				list_del(&wait.list);
+			if (!wait->lock_acquired)
+				list_del(&wait->list);
 			raw_spin_unlock(&lock->wait_lock);
 
-			if (wait.lock_acquired)
+			if (wait->lock_acquired)
 				do_six_unlock_type(lock, type);
 			break;
 		}
@@ -534,9 +535,10 @@ out:
 	return ret;
 }
 
-__always_inline
-static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
-			   six_lock_should_sleep_fn should_sleep_fn, void *p)
+__always_inline __flatten
+static int __six_lock_type_waiter(struct six_lock *lock, enum six_lock_type type,
+			 struct six_lock_waiter *wait,
+			 six_lock_should_sleep_fn should_sleep_fn, void *p)
 {
 	int ret;
 
@@ -544,7 +546,7 @@ static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
 		six_acquire(&lock->dep_map, 0);
 
 	ret = do_six_trylock_type(lock, type, true) ? 0
-		: __six_lock_type_slowpath(lock, type, should_sleep_fn, p);
+		: __six_lock_type_slowpath(lock, type, wait, should_sleep_fn, p);
 
 	if (ret && type != SIX_LOCK_write)
 		six_release(&lock->dep_map);
@@ -554,6 +556,15 @@ static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
 	return ret;
 }
 
+__always_inline
+static int __six_lock_type(struct six_lock *lock, enum six_lock_type type,
+			   six_lock_should_sleep_fn should_sleep_fn, void *p)
+{
+	struct six_lock_waiter wait;
+
+	return __six_lock_type_waiter(lock, type, &wait, should_sleep_fn, p);
+}
+
 __always_inline __flatten
 static void do_six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 {
@@ -619,6 +630,14 @@ int six_lock_##type(struct six_lock *lock,				\
 }									\
 EXPORT_SYMBOL_GPL(six_lock_##type);					\
 									\
+int six_lock_waiter_##type(struct six_lock *lock,			\
+			   struct six_lock_waiter *wait,		\
+			   six_lock_should_sleep_fn should_sleep_fn, void *p)\
+{									\
+	return __six_lock_type_waiter(lock, SIX_LOCK_##type, wait, should_sleep_fn, p);\
+}									\
+EXPORT_SYMBOL_GPL(six_lock_waiter_##type);				\
+									\
 void six_unlock_##type(struct six_lock *lock)				\
 {									\
 	__six_unlock_type(lock, SIX_LOCK_##type);			\