qcacmn: Add excessive logging detection
One of the most common stability issues encountered by the stability team is watchdog bite due to excessive logging. Add a feature to track logging to dmesg using a tumbling window and counter, and panic if the configured maximum number of logs for that window is exceeded. Change-Id: If83c6dfccb9d191b02a3a7166b065c0a0704f969 CRs-Fixed: 2338385
This commit is contained in:
@@ -115,6 +115,8 @@ static inline qdf_time_t qdf_system_ticks(void)
|
||||
return __qdf_system_ticks();
|
||||
}
|
||||
|
||||
#define qdf_system_ticks_per_sec __qdf_system_ticks_per_sec
|
||||
|
||||
/**
|
||||
* qdf_system_ticks_to_msecs - convert ticks to milliseconds
|
||||
* @clock_ticks: Number of ticks
|
||||
|
@@ -122,6 +122,8 @@ static inline __qdf_time_t __qdf_system_ticks(void)
|
||||
return jiffies;
|
||||
}
|
||||
|
||||
#define __qdf_system_ticks_per_sec HZ
|
||||
|
||||
/**
|
||||
* __qdf_system_ticks_to_msecs() - convert system ticks into milli seconds
|
||||
* @ticks: System ticks
|
||||
|
@@ -299,21 +299,61 @@ static int wlan_add_user_log_time_stamp(char *tbuf, size_t tbuf_sz, uint64_t ts)
|
||||
#endif /* QCA_WIFI_3_0_ADRASTEA */
|
||||
|
||||
#ifdef CONFIG_MCL
|
||||
#ifdef WLAN_MAX_LOGS_PER_SEC
|
||||
static qdf_time_t __log_window_end_ticks;
|
||||
static qdf_atomic_t __log_window_count;
|
||||
|
||||
/**
|
||||
* assert_on_excessive_logging() - Check for and panic on excessive logging
|
||||
*
|
||||
* Track logging count using a quasi-tumbling window, 1 second long. If the max
|
||||
* logging count for a given window is exceeded, panic.
|
||||
*
|
||||
* Return: None
|
||||
*/
|
||||
static void assert_on_excessive_logging(void)
|
||||
{
|
||||
qdf_time_t now = qdf_system_ticks();
|
||||
|
||||
/*
|
||||
* If 'now' is more recent than the end of the window, reset.
|
||||
*
|
||||
* Note: This is not thread safe, and can result in more than one reset.
|
||||
* For our purposes, this is fine.
|
||||
*/
|
||||
if (qdf_system_time_after(now, __log_window_end_ticks)) {
|
||||
__log_window_end_ticks = now + qdf_system_ticks_per_sec;
|
||||
qdf_atomic_set(&__log_window_count, 0);
|
||||
}
|
||||
|
||||
/* this _is_ thread safe, and results in at most one panic */
|
||||
if (qdf_atomic_inc_return(&__log_window_count) == WLAN_MAX_LOGS_PER_SEC)
|
||||
QDF_DEBUG_PANIC("Exceeded %d logs per second",
|
||||
WLAN_MAX_LOGS_PER_SEC);
|
||||
}
|
||||
#else
|
||||
static inline void assert_on_excessive_logging(void) { }
|
||||
#endif /* WLAN_MAX_LOGS_PER_SEC */
|
||||
|
||||
static inline void
|
||||
log_to_console(QDF_TRACE_LEVEL level, const char *timestamp, const char *msg)
|
||||
{
|
||||
switch (level) {
|
||||
case QDF_TRACE_LEVEL_FATAL:
|
||||
pr_alert("%s %s\n", timestamp, msg);
|
||||
assert_on_excessive_logging();
|
||||
break;
|
||||
case QDF_TRACE_LEVEL_ERROR:
|
||||
pr_err("%s %s\n", timestamp, msg);
|
||||
assert_on_excessive_logging();
|
||||
break;
|
||||
case QDF_TRACE_LEVEL_WARN:
|
||||
pr_warn("%s %s\n", timestamp, msg);
|
||||
assert_on_excessive_logging();
|
||||
break;
|
||||
case QDF_TRACE_LEVEL_INFO:
|
||||
pr_info("%s %s\n", timestamp, msg);
|
||||
assert_on_excessive_logging();
|
||||
break;
|
||||
case QDF_TRACE_LEVEL_INFO_HIGH:
|
||||
case QDF_TRACE_LEVEL_INFO_MED:
|
||||
|
Reference in New Issue
Block a user