From 045f3025eb320272350e2d61c315cfb99a59fef3 Mon Sep 17 00:00:00 2001 From: Dustin Brown Date: Thu, 4 Oct 2018 16:30:00 -0700 Subject: [PATCH] qcacld-3.0: Add transition wait timeout detection to DSC In order to catch and debug long waiting transitions, add a watchdog timer to Driver Synchronization Core (DSC) transition start wait calls. If the timer expires, panic the driver for offline debugging. Change-Id: I557f87ada182ced389e7d5e63fe8b78f47e1d6b5 CRs-Fixed: 2328594 --- components/dsc/src/__wlan_dsc.c | 77 ++++++++++++++++++++++++++-- components/dsc/src/__wlan_dsc.h | 20 ++++---- components/dsc/src/wlan_dsc_driver.c | 16 ++++-- components/dsc/src/wlan_dsc_psoc.c | 16 ++++-- components/dsc/src/wlan_dsc_vdev.c | 16 ++++-- 5 files changed, 116 insertions(+), 29 deletions(-) diff --git a/components/dsc/src/__wlan_dsc.c b/components/dsc/src/__wlan_dsc.c index e71b499b75..39b8e2870e 100644 --- a/components/dsc/src/__wlan_dsc.c +++ b/components/dsc/src/__wlan_dsc.c @@ -170,6 +170,57 @@ bool __dsc_ops_remove(struct dsc_ops *ops, const char *func) return ops->count == 0; } +#ifdef WLAN_DSC_DEBUG +static void __dsc_dbg_tran_wait_timeout(void *opaque_tran) +{ + struct dsc_tran *tran = opaque_tran; + + QDF_DEBUG_PANIC("Transition '%s' waited more than %ums", + tran->desc, DSC_TRANS_WAIT_TIMEOUT_MS); +} + +/** + * __dsc_dbg_tran_wait_timeout_start() - start a timeout timer for @tran + * @tran: the transition to start a timeout timer for + * + * Return: QDF_STATUS + */ +static QDF_STATUS __dsc_dbg_tran_wait_timeout_start(struct dsc_tran *tran) +{ + QDF_STATUS status; + + status = qdf_timer_init(NULL, &tran->timeout_timer, + __dsc_dbg_tran_wait_timeout, tran, + QDF_TIMER_TYPE_SW); + if (QDF_IS_STATUS_ERROR(status)) + return status; + + qdf_timer_start(&tran->timeout_timer, DSC_TRANS_WAIT_TIMEOUT_MS); + + return QDF_STATUS_SUCCESS; +} + +/** + * __dsc_dbg_tran_wait_timeout_stop() - stop the timeout timer for @tran + * @tran: the transition to stop the timeout timer for + * + * Return: None + */ +static void __dsc_dbg_tran_wait_timeout_stop(struct dsc_tran *tran) +{ + qdf_timer_stop(&tran->timeout_timer); + qdf_timer_free(&tran->timeout_timer); +} +#else +static inline QDF_STATUS +__dsc_dbg_tran_wait_timeout_start(struct dsc_tran *tran) +{ + return QDF_STATUS_SUCCESS; +} + +static inline void __dsc_dbg_tran_wait_timeout_stop(struct dsc_tran *tran) { } +#endif /* WLAN_DSC_DEBUG */ + void __dsc_trans_init(struct dsc_trans *trans) { trans->active_desc = NULL; @@ -182,13 +233,27 @@ void __dsc_trans_deinit(struct dsc_trans *trans) trans->active_desc = NULL; } -void __dsc_trans_queue(struct dsc_trans *trans, struct dsc_tran *tran, - const char *desc) +QDF_STATUS __dsc_trans_queue(struct dsc_trans *trans, struct dsc_tran *tran, + const char *desc) { + QDF_STATUS status; + tran->abort = false; tran->desc = desc; qdf_event_create(&tran->event); + + status = __dsc_dbg_tran_wait_timeout_start(tran); + if (QDF_IS_STATUS_ERROR(status)) + goto event_destroy; + qdf_list_insert_back(&trans->queue, &tran->node); + + return QDF_STATUS_SUCCESS; + +event_destroy: + qdf_event_destroy(&tran->event); + + return status; } /** @@ -201,12 +266,16 @@ static struct dsc_tran *__dsc_trans_dequeue(struct dsc_trans *trans) { QDF_STATUS status; qdf_list_node_t *node; + struct dsc_tran *tran; status = qdf_list_remove_front(&trans->queue, &node); if (QDF_IS_STATUS_ERROR(status)) return NULL; - return qdf_container_of(node, struct dsc_tran, node); + tran = qdf_container_of(node, struct dsc_tran, node); + __dsc_dbg_tran_wait_timeout_stop(tran); + + return tran; } bool __dsc_trans_abort(struct dsc_trans *trans) @@ -256,7 +325,7 @@ QDF_STATUS __dsc_tran_wait(struct dsc_tran *tran) { QDF_STATUS status; - status = qdf_wait_single_event(&tran->event, DSC_TRANS_TIMEOUT); + status = qdf_wait_single_event(&tran->event, 0); qdf_event_destroy(&tran->event); if (QDF_IS_STATUS_ERROR(status)) diff --git a/components/dsc/src/__wlan_dsc.h b/components/dsc/src/__wlan_dsc.h index 40ff29752e..93953de7f9 100644 --- a/components/dsc/src/__wlan_dsc.h +++ b/components/dsc/src/__wlan_dsc.h @@ -55,13 +55,9 @@ static inline bool __dsc_assert(const bool cond, const char *cond_str, #define dsc_assert_success(status) dsc_assert(QDF_IS_STATUS_SUCCESS(status)) #ifdef WLAN_DSC_DEBUG -#define DSC_TRANS_TIMEOUT 120000 /* 2 minutes */ -#define DSC_OP_TIMEOUT_MS (1 * 60 * 1000) /* 1 minute */ -#else -#define DSC_TRANS_TIMEOUT 0 /* no timeout */ -#endif +#define DSC_OP_TIMEOUT_MS (1 * 60 * 1000) /* 1 minute */ +#define DSC_TRANS_WAIT_TIMEOUT_MS (2 * 60 * 1000) /* 2 minutes */ -#ifdef WLAN_DSC_DEBUG /** * struct dsc_op - list node for operation tracking information * @node: list node @@ -90,17 +86,21 @@ struct dsc_ops { }; /** - * struct dsc_pending_trans - representation of a pending transition + * struct dsc_tran - representation of a pending transition * @abort: used to indicate if the transition stopped waiting due to an abort * @desc: unique description of the transition * @node: list node * @event: event used to wait in *_start_trans_wait() APIs + * @timeout_timer: a timer used to detect transition wait timeouts */ struct dsc_tran { bool abort; const char *desc; qdf_list_node_t node; qdf_event_t event; +#ifdef WLAN_DSC_DEBUG + qdf_timer_t timeout_timer; +#endif }; /** @@ -235,10 +235,10 @@ void __dsc_trans_deinit(struct dsc_trans *trans); * @tran: the transition to enqueue * @desc: unique description of the transition being queued * - * Return: None + * Return: QDF_STATUS */ -void __dsc_trans_queue(struct dsc_trans *trans, struct dsc_tran *tran, - const char *desc); +QDF_STATUS __dsc_trans_queue(struct dsc_trans *trans, struct dsc_tran *tran, + const char *desc); /** * __dsc_tran_wait() - block until @tran completes diff --git a/components/dsc/src/wlan_dsc_driver.c b/components/dsc/src/wlan_dsc_driver.c index 20ac4d74d6..ac82eb28b7 100644 --- a/components/dsc/src/wlan_dsc_driver.c +++ b/components/dsc/src/wlan_dsc_driver.c @@ -193,17 +193,23 @@ __dsc_driver_trans_start_wait(struct dsc_driver *driver, const char *desc) __dsc_lock(driver); + /* try to start without waiting */ status = __dsc_driver_trans_start_nolock(driver, desc); - if (QDF_IS_STATUS_SUCCESS(status)) { - __dsc_unlock(driver); - return QDF_STATUS_SUCCESS; - } + if (QDF_IS_STATUS_SUCCESS(status)) + goto unlock; - __dsc_trans_queue(&driver->trans, &tran, desc); + status = __dsc_trans_queue(&driver->trans, &tran, desc); + if (QDF_IS_STATUS_ERROR(status)) + goto unlock; __dsc_unlock(driver); return __dsc_tran_wait(&tran); + +unlock: + __dsc_unlock(driver); + + return status; } QDF_STATUS diff --git a/components/dsc/src/wlan_dsc_psoc.c b/components/dsc/src/wlan_dsc_psoc.c index 196a28677d..c5e6bf8ad3 100644 --- a/components/dsc/src/wlan_dsc_psoc.c +++ b/components/dsc/src/wlan_dsc_psoc.c @@ -198,17 +198,23 @@ __dsc_psoc_trans_start_wait(struct dsc_psoc *psoc, const char *desc) __dsc_driver_lock(psoc); + /* try to start without waiting */ status = __dsc_psoc_trans_start_nolock(psoc, desc); - if (QDF_IS_STATUS_SUCCESS(status)) { - __dsc_driver_unlock(psoc); - return QDF_STATUS_SUCCESS; - } + if (QDF_IS_STATUS_SUCCESS(status)) + goto unlock; - __dsc_trans_queue(&psoc->trans, &tran, desc); + status = __dsc_trans_queue(&psoc->trans, &tran, desc); + if (QDF_IS_STATUS_ERROR(status)) + goto unlock; __dsc_driver_unlock(psoc); return __dsc_tran_wait(&tran); + +unlock: + __dsc_driver_unlock(psoc); + + return status; } QDF_STATUS dsc_psoc_trans_start_wait(struct dsc_psoc *psoc, const char *desc) diff --git a/components/dsc/src/wlan_dsc_vdev.c b/components/dsc/src/wlan_dsc_vdev.c index 3ae6d44b75..e114aaf97a 100644 --- a/components/dsc/src/wlan_dsc_vdev.c +++ b/components/dsc/src/wlan_dsc_vdev.c @@ -172,17 +172,23 @@ __dsc_vdev_trans_start_wait(struct dsc_vdev *vdev, const char *desc) __dsc_driver_lock(vdev); + /* try to start without waiting */ status = __dsc_vdev_trans_start_nolock(vdev, desc); - if (QDF_IS_STATUS_SUCCESS(status)) { - __dsc_driver_unlock(vdev); - return QDF_STATUS_SUCCESS; - } + if (QDF_IS_STATUS_SUCCESS(status)) + goto unlock; - __dsc_trans_queue(&vdev->trans, &tran, desc); + status = __dsc_trans_queue(&vdev->trans, &tran, desc); + if (QDF_IS_STATUS_ERROR(status)) + goto unlock; __dsc_driver_unlock(vdev); return __dsc_tran_wait(&tran); + +unlock: + __dsc_driver_unlock(vdev); + + return status; } QDF_STATUS dsc_vdev_trans_start_wait(struct dsc_vdev *vdev, const char *desc)