From e226cebdd2ec2ac7fa21fdaaf5f22e7edcf3c812 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Fri, 21 Apr 2017 13:22:14 -0700 Subject: [PATCH] qcacmn: Add Scheduler Watchdog Timer In order to catch long running scheduler messages, start a watchdog timer before processing each message, and stop the timer after each message is processed. When the watchdog timer expires, immediately crash the device in debug builds for easier debugging. Change-Id: I4b43a9e12fc1f5b8a795fec790fe7548a100d9db CRs-Fixed: 2037094 --- qdf/inc/qdf_trace.h | 11 +++++++++++ qdf/linux/src/i_qdf_trace.h | 10 ++++++++++ qdf/linux/src/qdf_trace.c | 15 ++++++++++++++ scheduler/inc/scheduler_core.h | 8 ++++++++ scheduler/src/scheduler_api.c | 36 ++++++++++++++++++++++++++++++++++ scheduler/src/scheduler_core.c | 13 +++++++++--- 6 files changed, 90 insertions(+), 3 deletions(-) diff --git a/qdf/inc/qdf_trace.h b/qdf/inc/qdf_trace.h index 884f4d3cb3..454f385e73 100644 --- a/qdf/inc/qdf_trace.h +++ b/qdf/inc/qdf_trace.h @@ -731,4 +731,15 @@ void qdf_logging_init(void); */ void qdf_logging_exit(void); +#define QDF_SYMBOL_LEN __QDF_SYMBOL_LEN + +/** + * qdf_sprint_symbol() - prints the name of a symbol into a string buffer + * @buffer: the string buffer to print into + * @addr: address of the symbol to lookup and print + * + * Return: number of characters printed + */ +int qdf_sprint_symbol(char *buffer, void *addr); + #endif /* __QDF_TRACE_H */ diff --git a/qdf/linux/src/i_qdf_trace.h b/qdf/linux/src/i_qdf_trace.h index 66f1250cc4..1ec827fe39 100644 --- a/qdf/linux/src/i_qdf_trace.h +++ b/qdf/linux/src/i_qdf_trace.h @@ -35,6 +35,10 @@ #if !defined(__I_QDF_TRACE_H) #define __I_QDF_TRACE_H +/* older kernels have a bug in kallsyms, so ensure module.h is included */ +#include +#include + #if !defined(__printf) #define __printf(a, b) #endif @@ -123,4 +127,10 @@ static inline void qdf_trace_msg(QDF_MODULE_ID module, ...) #endif +#ifdef KSYM_SYMBOL_LEN +#define __QDF_SYMBOL_LEN KSYM_SYMBOL_LEN +#else +#define __QDF_SYMBOL_LEN 0 +#endif + #endif /* __I_QDF_TRACE_H */ diff --git a/qdf/linux/src/qdf_trace.c b/qdf/linux/src/qdf_trace.c index 54aabfa3c5..6b091ab9cf 100644 --- a/qdf/linux/src/qdf_trace.c +++ b/qdf/linux/src/qdf_trace.c @@ -2279,3 +2279,18 @@ void qdf_logging_exit(void) } #endif +#ifdef CONFIG_KALLSYMS +inline int qdf_sprint_symbol(char *buffer, void *addr) +{ + return sprint_symbol(buffer, (unsigned long)addr); +} +#else +int qdf_sprint_symbol(char *buffer, void *addr) +{ + if (!buffer) + return 0; + + buffer[0] = '\0'; + return 1; +} +#endif diff --git a/scheduler/inc/scheduler_core.h b/scheduler/inc/scheduler_core.h index 44a067c7b1..56cfd6cb9f 100644 --- a/scheduler/inc/scheduler_core.h +++ b/scheduler/inc/scheduler_core.h @@ -28,12 +28,14 @@ #define __SCHEDULER_CORE_H #include +#include #include #include #define SCHEDULER_CORE_MAX_MESSAGES 8000 #define SCHEDULER_NUMBER_OF_MSG_QUEUE 5 #define SCHEDULER_WRAPPER_MAX_FAIL_COUNT (SCHEDULER_CORE_MAX_MESSAGES * 3) +#define SCHEDULER_WATCHDOG_TIMEOUT (10 * 1000) /* 10s */ /** * struct scheduler_mq_type - scheduler message queue @@ -90,6 +92,9 @@ struct scheduler_mq_ctx { * @hdd_callback: os if suspend callback * @legacy_wma_handler: legacy wma message handler * @legacy_sys_handler: legacy sys message handler + * @watchdog_timer: timer for triggering a scheduler watchdog bite + * @watchdog_msg_type: 'type' of the current msg being processed + * @watchdog_callback: the callback of the current msg being processed */ struct scheduler_ctx { struct scheduler_mq_ctx queue_ctx; @@ -104,6 +109,9 @@ struct scheduler_ctx { hdd_suspend_callback hdd_callback; scheduler_msg_process_fn_t legacy_wma_handler; scheduler_msg_process_fn_t legacy_sys_handler; + qdf_timer_t watchdog_timer; + uint16_t watchdog_msg_type; + void *watchdog_callback; }; diff --git a/scheduler/src/scheduler_api.c b/scheduler/src/scheduler_api.c index 8ebae971d0..8d9fd5491b 100644 --- a/scheduler/src/scheduler_api.c +++ b/scheduler/src/scheduler_api.c @@ -77,9 +77,39 @@ static QDF_STATUS scheduler_close(struct scheduler_ctx *sched_ctx) /* Deinit all the queues */ scheduler_queues_deinit(sched_ctx); + qdf_timer_free(&sched_ctx->watchdog_timer); + return QDF_STATUS_SUCCESS; } +static inline void scheduler_watchdog_notify(struct scheduler_ctx *sched) +{ + char symbol[QDF_SYMBOL_LEN] = ""; + + if (sched->watchdog_callback) + qdf_sprint_symbol(symbol, sched->watchdog_callback); + + QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR, + "%s: Callback %s (type 0x%x) has exceeded its allotted time of %ds", + __func__, symbol, sched->watchdog_msg_type, + SCHEDULER_WATCHDOG_TIMEOUT / 1000); +} + +#ifdef CONFIG_SLUB_DEBUG_ON +static void scheduler_watchdog_bite(void *arg) +{ + scheduler_watchdog_notify((struct scheduler_ctx *)arg); + QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR, + "%s: Going down for Scheduler Watchdog Bite!", __func__); + QDF_BUG(0); +} +#else +static void scheduler_watchdog_bite(void *arg) +{ + scheduler_watchdog_notify((struct scheduler_ctx *)arg); +} +#endif + static QDF_STATUS scheduler_open(struct scheduler_ctx *sched_ctx) { QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_INFO_HIGH, @@ -98,6 +128,12 @@ static QDF_STATUS scheduler_open(struct scheduler_ctx *sched_ctx) qdf_spinlock_create(&sched_ctx->sch_thread_lock); qdf_init_waitqueue_head(&sched_ctx->sch_wait_queue); sched_ctx->sch_event_flag = 0; + qdf_timer_init(NULL, + &sched_ctx->watchdog_timer, + &scheduler_watchdog_bite, + sched_ctx, + QDF_TIMER_TYPE_SW); + /* Create the Scheduler Main Controller thread */ sched_ctx->sch_thread = qdf_create_thread(scheduler_thread, sched_ctx, "scheduler_thread"); diff --git a/scheduler/src/scheduler_core.c b/scheduler/src/scheduler_core.c index 487cf4a017..25193714a8 100644 --- a/scheduler/src/scheduler_core.c +++ b/scheduler/src/scheduler_core.c @@ -283,7 +283,7 @@ static void scheduler_core_return_msg(struct scheduler_ctx *sch_ctx, } static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx, - bool *shutdown) + bool *shutdown) { int i; QDF_STATUS vStatus = QDF_STATUS_E_FAILURE; @@ -331,9 +331,16 @@ static void scheduler_thread_process_queues(struct scheduler_ctx *sch_ctx, return; } if (sch_ctx->queue_ctx.scheduler_msg_process_fn[i]) { + struct scheduler_msg *msg = pMsgWrapper->msg_buf; + + sch_ctx->watchdog_msg_type = msg->type; + sch_ctx->watchdog_callback = msg->callback; + qdf_timer_start(&sch_ctx->watchdog_timer, + SCHEDULER_WATCHDOG_TIMEOUT); vStatus = sch_ctx->queue_ctx. - scheduler_msg_process_fn[i]( - pMsgWrapper->msg_buf); + scheduler_msg_process_fn[i](msg); + qdf_timer_stop(&sch_ctx->watchdog_timer); + if (QDF_IS_STATUS_ERROR(vStatus)) { QDF_TRACE(QDF_MODULE_ID_SCHEDULER, QDF_TRACE_LEVEL_ERROR,