Code tagging based latency tracking: swait

This adds the ability to easily instrument code for measuring latency.
To use, add the following to calls to your code, at the start and end of
the event you wish to measure:

  code_tag_time_stats_start(start_time);
  code_tag_time_stats_finish(start_time);

Stastistics will then show up in debugfs under
/sys/kernel/debug/time_stats, listed by file and line number.

Stastics measured include weighted averages of frequency, duration, max
duration, as well as quantiles.

This patch also instruments all calls to init_wait and finish_wait,
which includes all calls to wait_event. Example debugfs output:

fs/xfs/xfs_trans_ail.c:746 module:xfs func:xfs_ail_push_all_sync
count:          17
rate:           0/sec
frequency:      2 sec
avg duration:   10 us
max duration:   232 us
quantiles (ns): 128 128 128 128 128 128 128 128 128 128 128 128 128 128 128

lib/sbitmap.c:813 module:sbitmap func:sbitmap_finish_wait
count:          3
rate:           0/sec
frequency:      4 sec
avg duration:   4 sec
max duration:   4 sec
quantiles (ns): 0 4288669120 4288669120 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048 5360836048

net/core/datagram.c:122 module:datagram func:__skb_wait_for_more_packets
count:          10
rate:           1/sec
frequency:      859 ms
avg duration:   472 ms
max duration:   30 sec
quantiles (ns): 0 12279 12279 15669 15669 15669 15669 17217 17217 17217 17217 17217 17217 17217 17217

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

2 files changed, 21 insertions, 5 deletions

diff --git a/include/linux/swait.h b/include/linux/swait.h
index 6a8c22b8c2a5..b935d3b027fd 100644
--- a/include/linux/swait.h
+++ b/include/linux/swait.h
@@ -48,11 +48,15 @@ struct swait_queue_head {
 struct swait_queue {
 	struct task_struct	*task;
 	struct list_head	task_list;
+#ifdef CONFIG_CODETAG_TIME_STATS
+	u64			start_time;
+#endif
 };
 
 #define __SWAITQUEUE_INITIALIZER(name) {				\
 	.task		= current,					\
 	.task_list	= LIST_HEAD_INIT((name).task_list),		\
+	WAIT_QUEUE_ENTRY_START_TIME_INITIALIZER				\
 }
 
 #define DECLARE_SWAITQUEUE(name)					\
@@ -151,8 +155,20 @@ extern void swake_up_locked(struct swait_queue_head *q);
 extern void prepare_to_swait_exclusive(struct swait_queue_head *q, struct swait_queue *wait, int state);
 extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state);
 
-extern void __finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
-extern void finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
+extern void __finish_swait_notrace(struct swait_queue_head *q, struct swait_queue *wait);
+extern void finish_swait_notrace(struct swait_queue_head *q, struct swait_queue *wait);
+
+#define __finish_swait(_q, _wait)					\
+do {									\
+	__finish_swait_notrace(_q, _wait);				\
+	codetag_time_stats_finish((_wait)->start_time);			\
+} while (0)
+
+#define finish_swait(_q, _wait)						\
+do {									\
+	finish_swait_notrace(_q, _wait);				\
+	codetag_time_stats_finish((_wait)->start_time);			\
+} while (0)
 
 /* as per ___wait_event() but for swait, therefore "exclusive == 1" */
 #define ___swait_event(wq, condition, state, ret, cmd)			\
diff --git a/kernel/sched/swait.c b/kernel/sched/swait.c
index 76b9b796e695..61d46ddfcd69 100644
--- a/kernel/sched/swait.c
+++ b/kernel/sched/swait.c
@@ -122,14 +122,14 @@ long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait
 }
 EXPORT_SYMBOL(prepare_to_swait_event);
 
-void __finish_swait(struct swait_queue_head *q, struct swait_queue *wait)
+void __finish_swait_notrace(struct swait_queue_head *q, struct swait_queue *wait)
 {
 	__set_current_state(TASK_RUNNING);
 	if (!list_empty(&wait->task_list))
 		list_del_init(&wait->task_list);
 }
 
-void finish_swait(struct swait_queue_head *q, struct swait_queue *wait)
+void finish_swait_notrace(struct swait_queue_head *q, struct swait_queue *wait)
 {
 	unsigned long flags;
 
@@ -141,4 +141,4 @@ void finish_swait(struct swait_queue_head *q, struct swait_queue *wait)
 		raw_spin_unlock_irqrestore(&q->lock, flags);
 	}
 }
-EXPORT_SYMBOL(finish_swait);
+EXPORT_SYMBOL(finish_swait_notrace);