qcacmn: Add Scheduler Watchdog Timer

In order to catch long running scheduler messages, start a watchdog
timer before processing each message, and stop the timer after each
message is processed. When the watchdog timer expires, immediately
crash the device in debug builds for easier debugging.

Change-Id: I4b43a9e12fc1f5b8a795fec790fe7548a100d9db
CRs-Fixed: 2037094

scheduler/inc/scheduler_core.h

@@ -28,12 +28,14 @@
 #define __SCHEDULER_CORE_H
 
 #include <qdf_threads.h>
+#include <qdf_timer.h>
 #include <scheduler_api.h>
 #include <qdf_list.h>
 
 #define SCHEDULER_CORE_MAX_MESSAGES 8000
 #define SCHEDULER_NUMBER_OF_MSG_QUEUE 5
 #define SCHEDULER_WRAPPER_MAX_FAIL_COUNT (SCHEDULER_CORE_MAX_MESSAGES * 3)
+#define SCHEDULER_WATCHDOG_TIMEOUT (10 * 1000) /* 10s */
 
 /**
  * struct scheduler_mq_type -  scheduler message queue
@@ -90,6 +92,9 @@ struct scheduler_mq_ctx {
  * @hdd_callback: os if suspend callback
  * @legacy_wma_handler: legacy wma message handler
  * @legacy_sys_handler: legacy sys message handler
+ * @watchdog_timer: timer for triggering a scheduler watchdog bite
+ * @watchdog_msg_type: 'type' of the current msg being processed
+ * @watchdog_callback: the callback of the current msg being processed
  */
 struct scheduler_ctx {
 	struct scheduler_mq_ctx queue_ctx;
@@ -104,6 +109,9 @@ struct scheduler_ctx {
 	hdd_suspend_callback hdd_callback;
 	scheduler_msg_process_fn_t legacy_wma_handler;
 	scheduler_msg_process_fn_t legacy_sys_handler;
+	qdf_timer_t watchdog_timer;
+	uint16_t watchdog_msg_type;
+	void *watchdog_callback;
 };
 
 

scheduler/src/scheduler_api.c

@@ -77,9 +77,39 @@ static QDF_STATUS scheduler_close(struct scheduler_ctx *sched_ctx)
 	/* Deinit all the queues */
 	scheduler_queues_deinit(sched_ctx);
 
+	qdf_timer_free(&sched_ctx->watchdog_timer);
+
 	return QDF_STATUS_SUCCESS;
 }
 
+static inline void scheduler_watchdog_notify(struct scheduler_ctx *sched)
+{
+	char symbol[QDF_SYMBOL_LEN] = "<null>";
+
+	if (sched->watchdog_callback)
+		qdf_sprint_symbol(symbol, sched->watchdog_callback);
+
+	QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR,
+		  "%s: Callback %s (type 0x%x) has exceeded its allotted time of %ds",
+		  __func__, symbol, sched->watchdog_msg_type,
+		  SCHEDULER_WATCHDOG_TIMEOUT / 1000);
+}
+
+#ifdef CONFIG_SLUB_DEBUG_ON
+static void scheduler_watchdog_bite(void *arg)
+{
+	scheduler_watchdog_notify((struct scheduler_ctx *)arg);
+	QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR,
+		  "%s: Going down for Scheduler Watchdog Bite!", __func__);
+	QDF_BUG(0);
+}
+#else
+static void scheduler_watchdog_bite(void *arg)
+{
+	scheduler_watchdog_notify((struct scheduler_ctx *)arg);
+}
+#endif
+
 static QDF_STATUS scheduler_open(struct scheduler_ctx *sched_ctx)
 {
 	QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_INFO_HIGH,
@@ -98,6 +128,12 @@ static QDF_STATUS scheduler_open(struct scheduler_ctx *sched_ctx)
 	qdf_spinlock_create(&sched_ctx->sch_thread_lock);
 	qdf_init_waitqueue_head(&sched_ctx->sch_wait_queue);
 	sched_ctx->sch_event_flag = 0;
+	qdf_timer_init(NULL,
+		       &sched_ctx->watchdog_timer,
+		       &scheduler_watchdog_bite,
+		       sched_ctx,
+		       QDF_TIMER_TYPE_SW);
+
 	/* Create the Scheduler Main Controller thread */
 	sched_ctx->sch_thread = qdf_create_thread(scheduler_thread,
 					sched_ctx, "scheduler_thread");

scheduler/src/scheduler_core.c

@@ -283,7 +283,7 @@ static void scheduler_core_return_msg(struct scheduler_ctx *sch_ctx,
 }
 
 static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx,
-						bool *shutdown)
+					    bool *shutdown)
 {
 	int i;
 	QDF_STATUS vStatus = QDF_STATUS_E_FAILURE;
@@ -331,9 +331,16 @@ static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx,
 			return;
 		}
 		if (sch_ctx->queue_ctx.scheduler_msg_process_fn[i]) {
+			struct scheduler_msg *msg = pMsgWrapper->msg_buf;
+
+			sch_ctx->watchdog_msg_type = msg->type;
+			sch_ctx->watchdog_callback = msg->callback;
+			qdf_timer_start(&sch_ctx->watchdog_timer,
+					SCHEDULER_WATCHDOG_TIMEOUT);
 			vStatus = sch_ctx->queue_ctx.
-					scheduler_msg_process_fn[i](
-							pMsgWrapper->msg_buf);
+					scheduler_msg_process_fn[i](msg);
+			qdf_timer_stop(&sch_ctx->watchdog_timer);
+
 			if (QDF_IS_STATUS_ERROR(vStatus)) {
 				QDF_TRACE(QDF_MODULE_ID_SCHEDULER,
 					QDF_TRACE_LEVEL_ERROR,