bcache: lockstat for six locks

2 files changed, 47 insertions, 77 deletions

diff --git a/drivers/md/bcache/six.c b/drivers/md/bcache/six.c
index d17fe10b3286..bb347b041d5b 100644
--- a/drivers/md/bcache/six.c
+++ b/drivers/md/bcache/six.c
@@ -1,6 +1,11 @@
 
+#include "linux/lockdep.h"
+
 #include "six.h"
 
+#define six_acquire(l, t)	lock_acquire(l, 0, t, 0, 0, NULL, _RET_IP_)
+#define six_release(l)		lock_release(l, 0, _RET_IP_)
+
 /* Number of times to trylock() before sleeping in six_lock(): */
 #define SIX_LOCK_SPIN_COUNT		1
 
@@ -43,7 +48,8 @@ struct six_lock_vals {
 	},								\
 }
 
-bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type)
+static inline bool __six_trylock_type(struct six_lock *lock,
+				      enum six_lock_type type)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
 	union six_lock_state old;
@@ -61,12 +67,21 @@ bool __six_trylock_type(struct six_lock *lock, enum six_lock_type type)
 	} while ((v = atomic64_cmpxchg_acquire(&lock->state.counter,
 				old.v,
 				old.v + l[type].lock_val)) != old.v);
-
 	return true;
 }
 
-bool __six_relock_type(struct six_lock *lock, enum six_lock_type type,
-		       unsigned seq)
+bool six_trylock_type(struct six_lock *lock, enum six_lock_type type)
+{
+	bool ret = __six_trylock_type(lock, type);
+
+	if (ret)
+		six_acquire(&lock->dep_map, 1);
+
+	return ret;
+}
+
+bool six_relock_type(struct six_lock *lock, enum six_lock_type type,
+		     unsigned seq)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
 	union six_lock_state old;
@@ -81,6 +96,7 @@ bool __six_relock_type(struct six_lock *lock, enum six_lock_type type,
 				old.v,
 				old.v + l[type].lock_val)) != old.v);
 
+	six_acquire(&lock->dep_map, 1);
 	return true;
 }
 
@@ -92,7 +108,7 @@ struct six_lock_waiter {
 /* This is probably up there with the more evil things I've done */
 #define waitlist_bitnr(id) ilog2((((union six_lock_state) { .waiters = 1 << (id) }).l))
 
-void __six_lock_type(struct six_lock *lock, enum six_lock_type type)
+void six_lock_type(struct six_lock *lock, enum six_lock_type type)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
 	union six_lock_state old, new;
@@ -100,21 +116,25 @@ void __six_lock_type(struct six_lock *lock, enum six_lock_type type)
 	unsigned i;
 	u64 v;
 
+	six_acquire(&lock->dep_map, 0);
+
 	for (i = 0; i < SIX_LOCK_SPIN_COUNT; i++) {
 		if (__six_trylock_type(lock, type))
 			return;
 		cpu_relax();
 	}
 
+	lock_contended(&lock->dep_map, _RET_IP_);
+
 	INIT_LIST_HEAD(&wait.list);
 	wait.task = current;
 
 	while (1) {
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		if (list_empty_careful(&wait.list)) {
-			spin_lock(&lock->wait_lock);
+			raw_spin_lock(&lock->wait_lock);
 			list_add_tail(&wait.list, &lock->wait_list[type]);
-			spin_unlock(&lock->wait_lock);
+			raw_spin_unlock(&lock->wait_lock);
 		}
 
 		v = lock->state.v;
@@ -139,10 +159,12 @@ void __six_lock_type(struct six_lock *lock, enum six_lock_type type)
 	__set_current_state(TASK_RUNNING);
 
 	if (!list_empty_careful(&wait.list)) {
-		spin_lock(&lock->wait_lock);
+		raw_spin_lock(&lock->wait_lock);
 		list_del_init(&wait.list);
-		spin_unlock(&lock->wait_lock);
+		raw_spin_unlock(&lock->wait_lock);
 	}
+
+	lock_acquired(&lock->dep_map, _RET_IP_);
 }
 
 static inline void six_lock_wakeup(struct six_lock *lock,
@@ -161,7 +183,7 @@ static inline void six_lock_wakeup(struct six_lock *lock,
 	clear_bit(waitlist_bitnr(waitlist_id),
 		  (unsigned long *) &lock->state.v);
 
-	spin_lock(&lock->wait_lock);
+	raw_spin_lock(&lock->wait_lock);
 
 	list_for_each_entry_safe(w, next, wait_list, list) {
 		list_del_init(&w->list);
@@ -175,17 +197,17 @@ static inline void six_lock_wakeup(struct six_lock *lock,
 		}
 	}
 
-	spin_unlock(&lock->wait_lock);
+	raw_spin_unlock(&lock->wait_lock);
 }
 
-void __six_unlock_type(struct six_lock *lock, enum six_lock_type type)
+void six_unlock_type(struct six_lock *lock, enum six_lock_type type)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
 	union six_lock_state state;
 
 	state.v = atomic64_add_return_release(l[type].unlock_val,
 					      &lock->state.counter);
-
+	six_release(&lock->dep_map);
 	six_lock_wakeup(lock, state, l[type].unlock_wakeup);
 }
 
@@ -216,11 +238,12 @@ bool six_trylock_convert(struct six_lock *lock,
  * Increment read/intent lock count, assuming we already have it read or intent
  * locked:
  */
-void __six_lock_increment(struct six_lock *lock, enum six_lock_type type)
+void six_lock_increment(struct six_lock *lock, enum six_lock_type type)
 {
 	const struct six_lock_vals l[] = LOCK_VALS;
 
 	EBUG_ON(type == SIX_LOCK_write);
+	six_acquire(&lock->dep_map, 0);
 
 	/* XXX: assert already locked, and that we don't overflow: */
 
diff --git a/drivers/md/bcache/six.h b/drivers/md/bcache/six.h
index 0feeb612993b..2c9b0b002da3 100644
--- a/drivers/md/bcache/six.h
+++ b/drivers/md/bcache/six.h
@@ -63,18 +63,19 @@ enum six_lock_type {
 struct six_lock {
 	union six_lock_state	state;
 
-	spinlock_t		wait_lock;
+	raw_spinlock_t		wait_lock;
 	struct list_head	wait_list[3];
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lockdep_map	dep_map;
 #endif
 };
 
-static __always_inline void __six_lock_init(struct six_lock *lock, const char *name,
-				   struct lock_class_key *key)
+static __always_inline void __six_lock_init(struct six_lock *lock,
+					    const char *name,
+					    struct lock_class_key *key)
 {
 	atomic64_set(&lock->state.counter, 0);
-	spin_lock_init(&lock->wait_lock);
+	raw_spin_lock_init(&lock->wait_lock);
 	INIT_LIST_HEAD(&lock->wait_list[SIX_LOCK_read]);
 	INIT_LIST_HEAD(&lock->wait_list[SIX_LOCK_intent]);
 	INIT_LIST_HEAD(&lock->wait_list[SIX_LOCK_write]);
@@ -91,70 +92,16 @@ do {									\
 	__six_lock_init((lock), #lock, &__key);				\
 } while (0)
 
-bool __six_trylock_type(struct six_lock *, enum six_lock_type);
-bool __six_relock_type(struct six_lock *, enum six_lock_type, unsigned);
-void __six_lock_type(struct six_lock *, enum six_lock_type);
-void __six_unlock_type(struct six_lock *, enum six_lock_type);
+bool six_trylock_type(struct six_lock *, enum six_lock_type);
+bool six_relock_type(struct six_lock *, enum six_lock_type, unsigned);
+void six_lock_type(struct six_lock *, enum six_lock_type);
+void six_unlock_type(struct six_lock *, enum six_lock_type);
 bool six_trylock_convert(struct six_lock *, enum six_lock_type,
 			 enum six_lock_type);
-void __six_lock_increment(struct six_lock *, enum six_lock_type);
-
-#ifdef CONFIG_DEBUG_LOCK_ALLOC
-
-#define six_acquire(l)	lock_acquire(l, 0, 0, 0, 0, NULL, _THIS_IP_)
-#define six_release(l)	lock_release(l, 0, _THIS_IP_)
-
-#else
-
-#define six_acquire(l)
-#define six_release(l)
-
-#endif
+void six_lock_increment(struct six_lock *, enum six_lock_type);
 
 #define __SIX_VAL(field, _v)	(((union six_lock_state) { .field = _v }).v)
 
-static __always_inline bool six_trylock_type(struct six_lock *lock,
-					     enum six_lock_type type)
-{
-	if (!__six_trylock_type(lock, type))
-		return false;
-
-	six_acquire(&lock->dep_map);
-	return true;
-}
-
-static __always_inline bool six_relock_type(struct six_lock *lock,
-					    enum six_lock_type type,
-					    u32 seq)
-{
-	if (!__six_relock_type(lock, type, seq))
-		return false;
-
-	six_acquire(&lock->dep_map);
-	return true;
-}
-
-static __always_inline void six_lock_type(struct six_lock *lock,
-					  enum six_lock_type type)
-{
-	__six_lock_type(lock, type);
-	six_acquire(&lock->dep_map);
-}
-
-static __always_inline void six_unlock_type(struct six_lock *lock,
-					    enum six_lock_type type)
-{
-	six_release(&lock->dep_map);
-	__six_unlock_type(lock, type);
-}
-
-static __always_inline void six_lock_increment(struct six_lock *lock,
-					       enum six_lock_type type)
-{
-	__six_lock_increment(lock, type);
-	six_acquire(&lock->dep_map);
-}
-
 #define __SIX_LOCK(type)						\
 static __always_inline bool six_trylock_##type(struct six_lock *lock)	\
 {									\