qcacmn: Add Scheduler Watchdog Timer

In order to catch long running scheduler messages, start a watchdog
timer before processing each message, and stop the timer after each
message is processed. When the watchdog timer expires, immediately
crash the device in debug builds for easier debugging.

Change-Id: I4b43a9e12fc1f5b8a795fec790fe7548a100d9db
CRs-Fixed: 2037094

qdf/inc/qdf_trace.h

@@ -731,4 +731,15 @@ void qdf_logging_init(void);
  */
 void qdf_logging_exit(void);
 
+#define QDF_SYMBOL_LEN __QDF_SYMBOL_LEN
+
+/**
+ * qdf_sprint_symbol() - prints the name of a symbol into a string buffer
+ * @buffer: the string buffer to print into
+ * @addr: address of the symbol to lookup and print
+ *
+ * Return: number of characters printed
+ */
+int qdf_sprint_symbol(char *buffer, void *addr);
+
 #endif /* __QDF_TRACE_H */

qdf/linux/src/i_qdf_trace.h

@@ -35,6 +35,10 @@
 #if !defined(__I_QDF_TRACE_H)
 #define __I_QDF_TRACE_H
 
+/* older kernels have a bug in kallsyms, so ensure module.h is included */
+#include <module.h>
+#include <kallsyms.h>
+
 #if !defined(__printf)
 #define __printf(a, b)
 #endif
@@ -123,4 +127,10 @@ static inline void qdf_trace_msg(QDF_MODULE_ID module, ...)
 
 #endif
 
+#ifdef KSYM_SYMBOL_LEN
+#define __QDF_SYMBOL_LEN KSYM_SYMBOL_LEN
+#else
+#define __QDF_SYMBOL_LEN 0
+#endif
+
 #endif /* __I_QDF_TRACE_H */

qdf/linux/src/qdf_trace.c

@@ -2279,3 +2279,18 @@ void qdf_logging_exit(void)
 }
 #endif
 
+#ifdef CONFIG_KALLSYMS
+inline int qdf_sprint_symbol(char *buffer, void *addr)
+{
+	return sprint_symbol(buffer, (unsigned long)addr);
+}
+#else
+int qdf_sprint_symbol(char *buffer, void *addr)
+{
+	if (!buffer)
+		return 0;
+
+	buffer[0] = '\0';
+	return 1;
+}
+#endif

scheduler/inc/scheduler_core.h

@@ -28,12 +28,14 @@
 #define __SCHEDULER_CORE_H
 
 #include <qdf_threads.h>
+#include <qdf_timer.h>
 #include <scheduler_api.h>
 #include <qdf_list.h>
 
 #define SCHEDULER_CORE_MAX_MESSAGES 8000
 #define SCHEDULER_NUMBER_OF_MSG_QUEUE 5
 #define SCHEDULER_WRAPPER_MAX_FAIL_COUNT (SCHEDULER_CORE_MAX_MESSAGES * 3)
+#define SCHEDULER_WATCHDOG_TIMEOUT (10 * 1000) /* 10s */
 
 /**
  * struct scheduler_mq_type -  scheduler message queue
@@ -90,6 +92,9 @@ struct scheduler_mq_ctx {
  * @hdd_callback: os if suspend callback
  * @legacy_wma_handler: legacy wma message handler
  * @legacy_sys_handler: legacy sys message handler
+ * @watchdog_timer: timer for triggering a scheduler watchdog bite
+ * @watchdog_msg_type: 'type' of the current msg being processed
+ * @watchdog_callback: the callback of the current msg being processed
  */
 struct scheduler_ctx {
 	struct scheduler_mq_ctx queue_ctx;
@@ -104,6 +109,9 @@ struct scheduler_ctx {
 	hdd_suspend_callback hdd_callback;
 	scheduler_msg_process_fn_t legacy_wma_handler;
 	scheduler_msg_process_fn_t legacy_sys_handler;
+	qdf_timer_t watchdog_timer;
+	uint16_t watchdog_msg_type;
+	void *watchdog_callback;
 };
 
 

scheduler/src/scheduler_api.c

@@ -77,9 +77,39 @@ static QDF_STATUS scheduler_close(struct scheduler_ctx *sched_ctx)
 	/* Deinit all the queues */
 	scheduler_queues_deinit(sched_ctx);
 
+	qdf_timer_free(&sched_ctx->watchdog_timer);
+
 	return QDF_STATUS_SUCCESS;
 }
 
+static inline void scheduler_watchdog_notify(struct scheduler_ctx *sched)
+{
+	char symbol[QDF_SYMBOL_LEN] = "<null>";
+
+	if (sched->watchdog_callback)
+		qdf_sprint_symbol(symbol, sched->watchdog_callback);
+
+	QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR,
+		  "%s: Callback %s (type 0x%x) has exceeded its allotted time of %ds",
+		  __func__, symbol, sched->watchdog_msg_type,
+		  SCHEDULER_WATCHDOG_TIMEOUT / 1000);
+}
+
+#ifdef CONFIG_SLUB_DEBUG_ON
+static void scheduler_watchdog_bite(void *arg)
+{
+	scheduler_watchdog_notify((struct scheduler_ctx *)arg);
+	QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR,
+		  "%s: Going down for Scheduler Watchdog Bite!", __func__);
+	QDF_BUG(0);
+}
+#else
+static void scheduler_watchdog_bite(void *arg)
+{
+	scheduler_watchdog_notify((struct scheduler_ctx *)arg);
+}
+#endif
+
 static QDF_STATUS scheduler_open(struct scheduler_ctx *sched_ctx)
 {
 	QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_INFO_HIGH,
@@ -98,6 +128,12 @@ static QDF_STATUS scheduler_open(struct scheduler_ctx *sched_ctx)
 	qdf_spinlock_create(&sched_ctx->sch_thread_lock);
 	qdf_init_waitqueue_head(&sched_ctx->sch_wait_queue);
 	sched_ctx->sch_event_flag = 0;
+	qdf_timer_init(NULL,
+		       &sched_ctx->watchdog_timer,
+		       &scheduler_watchdog_bite,
+		       sched_ctx,
+		       QDF_TIMER_TYPE_SW);
+
 	/* Create the Scheduler Main Controller thread */
 	sched_ctx->sch_thread = qdf_create_thread(scheduler_thread,
 					sched_ctx, "scheduler_thread");

scheduler/src/scheduler_core.c

@@ -283,7 +283,7 @@ static void scheduler_core_return_msg(struct scheduler_ctx *sch_ctx,
 }
 
 static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx,
-						bool *shutdown)
+					    bool *shutdown)
 {
 	int i;
 	QDF_STATUS vStatus = QDF_STATUS_E_FAILURE;
@@ -331,9 +331,16 @@ static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx,
 			return;
 		}
 		if (sch_ctx->queue_ctx.scheduler_msg_process_fn[i]) {
+			struct scheduler_msg *msg = pMsgWrapper->msg_buf;
+
+			sch_ctx->watchdog_msg_type = msg->type;
+			sch_ctx->watchdog_callback = msg->callback;
+			qdf_timer_start(&sch_ctx->watchdog_timer,
+					SCHEDULER_WATCHDOG_TIMEOUT);
 			vStatus = sch_ctx->queue_ctx.
-					scheduler_msg_process_fn[i](
-							pMsgWrapper->msg_buf);
+					scheduler_msg_process_fn[i](msg);
+			qdf_timer_stop(&sch_ctx->watchdog_timer);
+
 			if (QDF_IS_STATUS_ERROR(vStatus)) {
 				QDF_TRACE(QDF_MODULE_ID_SCHEDULER,
 					QDF_TRACE_LEVEL_ERROR,