hif_irq_affinity.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533
  1. /*
  2. * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved.
  3. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. *
  5. * Permission to use, copy, modify, and/or distribute this software for
  6. * any purpose with or without fee is hereby granted, provided that the
  7. * above copyright notice and this permission notice appear in all
  8. * copies.
  9. *
  10. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
  11. * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  12. * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
  13. * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  14. * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  15. * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  16. * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  17. * PERFORMANCE OF THIS SOFTWARE.
  18. */
  19. /**
  20. * DOC: hif_irq_affinity.c
  21. *
  22. * This irq affinity implementation is os dependent, so this can be treated as
  23. * an abstraction layer... Should this be moved into a /linux folder?
  24. */
  25. #include <linux/string.h> /* memset */
  26. /* Linux headers */
  27. #include <linux/cpumask.h>
  28. #include <linux/cpufreq.h>
  29. #include <linux/cpu.h>
  30. #include <linux/topology.h>
  31. #include <linux/interrupt.h>
  32. #include <linux/pm.h>
  33. #include <hif_napi.h>
  34. #include <hif_irq_affinity.h>
  35. #include <hif_exec.h>
  36. #include <hif_main.h>
  37. #include "qdf_irq.h"
  38. #if defined(FEATURE_NAPI_DEBUG) && defined(HIF_IRQ_AFFINITY)
  39. /*
  40. * Local functions
  41. * - no argument checks, all internal/trusted callers
  42. */
  43. static void hnc_dump_cpus(struct qca_napi_data *napid)
  44. {
  45. hif_napi_stats(napid);
  46. }
  47. #else
  48. static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ };
  49. #endif /* FEATURE_NAPI_DEBUG */
  50. #ifdef HIF_IRQ_AFFINITY
  51. /**
  52. * hif_exec_event() - reacts to events that impact irq affinity
  53. * @hif_ctx: pointer to hif context
  54. * @event: event that has been detected
  55. * @data: more data regarding the event
  56. *
  57. * Description:
  58. * This function handles two types of events:
  59. * 1- Events that change the state of NAPI (enabled/disabled):
  60. * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE}
  61. * The state is retrievable by "hdd_napi_enabled(-1)"
  62. * - NAPI will be on if either INI file is on and it has not been disabled
  63. * by a subsequent vendor CMD,
  64. * or it has been enabled by a vendor CMD.
  65. * 2- Events that change the CPU affinity of a NAPI instance/IRQ:
  66. * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE}
  67. * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode
  68. * - NAPI will switch throughput mode based on hdd_napi_throughput_policy()
  69. * - In LO tput mode, NAPI will yield control if its interrupts to the system
  70. * management functions. However in HI throughput mode, NAPI will actively
  71. * manage its interrupts/instances (by trying to disperse them out to
  72. * separate performance cores).
  73. * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events.
  74. *
  75. * + In some cases (roaming peer management is the only case so far), a
  76. * a client can trigger a "SERIALIZE" event. Basically, this means that the
  77. * users is asking NAPI to go into a truly single execution context state.
  78. * So, NAPI indicates to msm-irqbalancer that it wants to be denylisted,
  79. * (if called for the first time) and then moves all IRQs (for NAPI
  80. * instances) to be collapsed to a single core. If called multiple times,
  81. * it will just re-collapse the CPUs. This is because denylist-on() API
  82. * is reference-counted, and because the API has already been called.
  83. *
  84. * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go
  85. * to its "normal" operation. Optionally, they can give a timeout value (in
  86. * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this
  87. * case, NAPI will just set the current throughput state to uninitialized
  88. * and set the delay period. Once policy handler is called, it would skip
  89. * applying the policy delay period times, and otherwise apply the policy.
  90. *
  91. * Return:
  92. * < 0: some error
  93. * = 0: event handled successfully
  94. */
  95. int hif_exec_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event,
  96. void *data)
  97. {
  98. int rc = 0;
  99. uint32_t prev_state;
  100. struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx);
  101. struct qca_napi_data *napid = &(hif->napi_data);
  102. enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED;
  103. enum {
  104. DENYLIST_NOT_PENDING,
  105. DENYLIST_ON_PENDING,
  106. DENYLIST_OFF_PENDING
  107. } denylist_pending = DENYLIST_NOT_PENDING;
  108. NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data);
  109. qdf_spin_lock_bh(&(napid->lock));
  110. prev_state = napid->state;
  111. switch (event) {
  112. case NAPI_EVT_INI_FILE:
  113. case NAPI_EVT_CMD_STATE:
  114. case NAPI_EVT_INT_STATE:
  115. /* deprecated */
  116. break;
  117. case NAPI_EVT_CPU_STATE: {
  118. int cpu = ((unsigned long int)data >> 16);
  119. int val = ((unsigned long int)data & 0x0ff);
  120. NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d",
  121. __func__, cpu, val);
  122. /* state has already been set by hnc_cpu_notify_cb */
  123. if ((val == QCA_NAPI_CPU_DOWN) &&
  124. (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */
  125. (napid->napi_cpu[cpu].napis != 0)) {
  126. NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d",
  127. __func__, cpu);
  128. rc = hif_exec_cpu_migrate(napid,
  129. cpu,
  130. HNC_ACT_RELOCATE);
  131. napid->napi_cpu[cpu].napis = 0;
  132. }
  133. /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */
  134. break;
  135. }
  136. case NAPI_EVT_TPUT_STATE: {
  137. tput_mode = (enum qca_napi_tput_state)data;
  138. if (tput_mode == QCA_NAPI_TPUT_LO) {
  139. /* from TPUT_HI -> TPUT_LO */
  140. NAPI_DEBUG("%s: Moving to napi_tput_LO state",
  141. __func__);
  142. denylist_pending = DENYLIST_OFF_PENDING;
  143. /*
  144. * Ideally we should "collapse" interrupts here, since
  145. * we are "dispersing" interrupts in the "else" case.
  146. * This allows the possibility that our interrupts may
  147. * still be on the perf cluster the next time we enter
  148. * high tput mode. However, the irq_balancer is free
  149. * to move our interrupts to power cluster once
  150. * denylisting has been turned off in the "else" case.
  151. */
  152. } else {
  153. /* from TPUT_LO -> TPUT->HI */
  154. NAPI_DEBUG("%s: Moving to napi_tput_HI state",
  155. __func__);
  156. rc = hif_exec_cpu_migrate(napid,
  157. HNC_ANY_CPU,
  158. HNC_ACT_DISPERSE);
  159. denylist_pending = DENYLIST_ON_PENDING;
  160. }
  161. napid->napi_mode = tput_mode;
  162. break;
  163. }
  164. case NAPI_EVT_USR_SERIAL: {
  165. unsigned long users = (unsigned long)data;
  166. NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld",
  167. __func__, users);
  168. rc = hif_exec_cpu_migrate(napid,
  169. HNC_ANY_CPU,
  170. HNC_ACT_COLLAPSE);
  171. if ((users == 0) && (rc == 0))
  172. denylist_pending = DENYLIST_ON_PENDING;
  173. break;
  174. }
  175. case NAPI_EVT_USR_NORMAL: {
  176. NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__);
  177. if (!napid->user_cpu_affin_mask)
  178. denylist_pending = DENYLIST_OFF_PENDING;
  179. /*
  180. * Deserialization timeout is handled at hdd layer;
  181. * just mark current mode to uninitialized to ensure
  182. * it will be set when the delay is over
  183. */
  184. napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED;
  185. break;
  186. }
  187. default: {
  188. hif_err("Unknown event: %d (data=0x%0lx)",
  189. event, (unsigned long) data);
  190. break;
  191. } /* default */
  192. }; /* switch */
  193. switch (denylist_pending) {
  194. case DENYLIST_ON_PENDING:
  195. /* assume the control of WLAN IRQs */
  196. hif_napi_cpu_denylist(napid, DENYLIST_ON);
  197. break;
  198. case DENYLIST_OFF_PENDING:
  199. /* yield the control of WLAN IRQs */
  200. hif_napi_cpu_denylist(napid, DENYLIST_OFF);
  201. break;
  202. default: /* nothing to do */
  203. break;
  204. } /* switch denylist_pending */
  205. qdf_spin_unlock_bh(&(napid->lock));
  206. NAPI_DEBUG("<--[rc=%d]", rc);
  207. return rc;
  208. }
  209. #endif
  210. /**
  211. * hncm_exec_migrate_to() - migrates a NAPI to a CPU
  212. * @napid: pointer to NAPI block
  213. * @ctx_id: CE_id of the NAPI instance
  214. * @didx: index in the CPU topology table for the CPU to migrate to
  215. *
  216. * Migrates NAPI (identified by the CE_id) to the destination core
  217. * Updates the napi_map of the destination entry
  218. *
  219. * Return:
  220. * =0 : success
  221. * <0 : error
  222. */
  223. static int hncm_exec_migrate_to(struct qca_napi_data *napid, uint8_t ctx_id,
  224. int didx)
  225. {
  226. struct hif_exec_context *exec_ctx;
  227. struct qdf_cpu_mask *cpumask;
  228. int rc = 0;
  229. int status = 0;
  230. int ind;
  231. NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, ctx_id, didx);
  232. exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, ctx_id);
  233. if (!exec_ctx)
  234. return -EINVAL;
  235. exec_ctx->cpumask.bits[0] = (1 << didx);
  236. for (ind = 0; ind < exec_ctx->numirq; ind++) {
  237. if (exec_ctx->os_irq[ind]) {
  238. qdf_dev_modify_irq_status(exec_ctx->os_irq[ind],
  239. QDF_IRQ_NO_BALANCING, 0);
  240. cpumask = (struct qdf_cpu_mask *)&exec_ctx->cpumask;
  241. rc = qdf_dev_set_irq_affinity(exec_ctx->os_irq[ind],
  242. cpumask);
  243. if (rc)
  244. status = rc;
  245. }
  246. }
  247. /* unmark the napis bitmap in the cpu table */
  248. napid->napi_cpu[exec_ctx->cpu].napis &= ~(0x01 << ctx_id);
  249. /* mark the napis bitmap for the new designated cpu */
  250. napid->napi_cpu[didx].napis |= (0x01 << ctx_id);
  251. exec_ctx->cpu = didx;
  252. NAPI_DEBUG("<--%s[%d]", __func__, rc);
  253. return status;
  254. }
  255. /**
  256. * hncm_dest_cpu() - finds a destination CPU for NAPI
  257. * @napid: pointer to NAPI block
  258. * @act : RELOCATE | COLLAPSE | DISPERSE
  259. *
  260. * Finds the designated destination for the next IRQ.
  261. * RELOCATE: translated to either COLLAPSE or DISPERSE based
  262. * on napid->napi_mode (throughput state)
  263. * COLLAPSE: All have the same destination: the first online CPU in lilcl
  264. * DISPERSE: One of the CPU in bigcl, which has the smallest number of
  265. * NAPIs on it
  266. *
  267. * Return: >=0 : index in the cpu topology table
  268. * : < 0 : error
  269. */
  270. static int hncm_dest_cpu(struct qca_napi_data *napid, int act)
  271. {
  272. int destidx = -1;
  273. int head, i;
  274. NAPI_DEBUG("-->%s(act=%d)", __func__, act);
  275. if (act == HNC_ACT_RELOCATE) {
  276. if (napid->napi_mode == QCA_NAPI_TPUT_LO)
  277. act = HNC_ACT_COLLAPSE;
  278. else
  279. act = HNC_ACT_DISPERSE;
  280. NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d",
  281. __func__, act);
  282. }
  283. if (act == HNC_ACT_COLLAPSE) {
  284. head = i = napid->lilcl_head;
  285. retry_collapse:
  286. while (i >= 0) {
  287. if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) {
  288. destidx = i;
  289. break;
  290. }
  291. i = napid->napi_cpu[i].cluster_nxt;
  292. }
  293. if ((destidx < 0) && (head == napid->lilcl_head)) {
  294. NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl",
  295. __func__);
  296. head = i = napid->bigcl_head;
  297. goto retry_collapse;
  298. }
  299. } else { /* HNC_ACT_DISPERSE */
  300. int smallest = 99; /* all 32 bits full */
  301. int smallidx = -1;
  302. head = i = napid->bigcl_head;
  303. retry_disperse:
  304. while (i >= 0) {
  305. if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) &&
  306. (hweight32(napid->napi_cpu[i].napis) <= smallest)) {
  307. smallest = napid->napi_cpu[i].napis;
  308. smallidx = i;
  309. }
  310. i = napid->napi_cpu[i].cluster_nxt;
  311. }
  312. destidx = smallidx;
  313. if ((destidx < 0) && (head == napid->bigcl_head)) {
  314. NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl",
  315. __func__);
  316. head = i = napid->lilcl_head;
  317. goto retry_disperse;
  318. }
  319. }
  320. NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx);
  321. return destidx;
  322. }
  323. /**
  324. * hif_exec_cpu_migrate() - migrate IRQs away
  325. * @napid: pointer to qca_napi_data structure
  326. * @cpu: -1: all CPUs <n> specific CPU
  327. * @action: COLLAPSE | DISPERSE
  328. *
  329. * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible
  330. * cores. Eligible cores are:
  331. * act=COLLAPSE -> the first online core of the little cluster
  332. * act=DISPERSE -> separate cores of the big cluster, so that each core will
  333. * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis)
  334. *
  335. * Note that this function is called with a spinlock acquired already.
  336. *
  337. * Return: =0: success
  338. * <0: error
  339. */
  340. int hif_exec_cpu_migrate(struct qca_napi_data *napid, int cpu, int action)
  341. {
  342. int rc = 0;
  343. struct qca_napi_cpu *cpup;
  344. int i, dind;
  345. uint32_t napis;
  346. NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)",
  347. __func__, cpu, action);
  348. if (napid->exec_map == 0) {
  349. NAPI_DEBUG("%s: datapath contexts to disperse", __func__);
  350. goto hncm_return;
  351. }
  352. cpup = napid->napi_cpu;
  353. switch (action) {
  354. case HNC_ACT_RELOCATE:
  355. case HNC_ACT_DISPERSE:
  356. case HNC_ACT_COLLAPSE: {
  357. /* first find the src napi set */
  358. if (cpu == HNC_ANY_CPU)
  359. napis = napid->exec_map;
  360. else
  361. napis = cpup[cpu].napis;
  362. /* then clear the napi bitmap on each CPU */
  363. for (i = 0; i < NR_CPUS; i++)
  364. cpup[i].napis = 0;
  365. /* then for each of the NAPIs to disperse: */
  366. for (i = 0; i < HIF_MAX_GROUP; i++)
  367. if (napis & (1 << i)) {
  368. /* find a destination CPU */
  369. dind = hncm_dest_cpu(napid, action);
  370. if (dind >= 0) {
  371. rc = hncm_exec_migrate_to(napid, i,
  372. dind);
  373. } else {
  374. NAPI_DEBUG("No dest for NAPI ce%d", i);
  375. hnc_dump_cpus(napid);
  376. rc = -1;
  377. }
  378. }
  379. break;
  380. }
  381. default: {
  382. NAPI_DEBUG("%s: bad action: %d\n", __func__, action);
  383. QDF_BUG(0);
  384. break;
  385. }
  386. } /* switch action */
  387. hncm_return:
  388. hnc_dump_cpus(napid);
  389. return rc;
  390. }
  391. /**
  392. * hif_exec_dl_irq() - calls irq_modify_status to enable/disable denylisting
  393. * @napid: pointer to qca_napi_data structure
  394. * @dl_flag: denylist flag to enable/disable denylisting
  395. *
  396. * The function enables/disables denylisting for all the copy engine
  397. * interrupts on which NAPI is enabled.
  398. *
  399. * Return: None
  400. */
  401. static inline void hif_exec_dl_irq(struct qca_napi_data *napid, bool dl_flag)
  402. {
  403. int i, j;
  404. struct hif_exec_context *exec_ctx;
  405. for (i = 0; i < HIF_MAX_GROUP; i++) {
  406. /* check if NAPI is enabled on the CE */
  407. if (!(napid->exec_map & (0x01 << i)))
  408. continue;
  409. /*double check that NAPI is allocated for the CE */
  410. exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, i);
  411. if (!(exec_ctx))
  412. continue;
  413. if (dl_flag == true)
  414. for (j = 0; j < exec_ctx->numirq; j++)
  415. qdf_dev_modify_irq_status(exec_ctx->os_irq[j],
  416. 0,
  417. QDF_IRQ_NO_BALANCING);
  418. else
  419. for (j = 0; j < exec_ctx->numirq; j++)
  420. qdf_dev_modify_irq_status(exec_ctx->os_irq[j],
  421. QDF_IRQ_NO_BALANCING,
  422. 0);
  423. hif_debug("dl_flag %d CE %d", dl_flag, i);
  424. }
  425. }
  426. /**
  427. * hif_exec_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts.
  428. * @napid: pointer to qca_napi_data structure
  429. * @op: denylist operation to perform
  430. *
  431. * The function enables/disables/queries denylisting for all CE RX
  432. * interrupts with NAPI enabled. Besides denylisting, it also enables/disables
  433. * core_ctl_set_boost.
  434. * Once denylisting is enabled, the interrupts will not be managed by the IRQ
  435. * balancer.
  436. *
  437. * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled
  438. * for DENYLIST_QUERY op - denylist refcount
  439. * for DENYLIST_ON op - return value from core_ctl_set_boost API
  440. * for DENYLIST_OFF op - return value from core_ctl_set_boost API
  441. */
  442. int hif_exec_cpu_denylist(struct qca_napi_data *napid,
  443. enum qca_denylist_op op)
  444. {
  445. int rc = 0;
  446. static int ref_count; /* = 0 by the compiler */
  447. uint8_t flags = napid->flags;
  448. bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING;
  449. bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST;
  450. NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op);
  451. if (!(dl_en && ccb_en)) {
  452. rc = -EINVAL;
  453. goto out;
  454. }
  455. switch (op) {
  456. case DENYLIST_QUERY:
  457. rc = ref_count;
  458. break;
  459. case DENYLIST_ON:
  460. ref_count++;
  461. rc = 0;
  462. if (ref_count == 1) {
  463. rc = hif_napi_core_ctl_set_boost(true);
  464. NAPI_DEBUG("boost_on() returns %d - refcnt=%d",
  465. rc, ref_count);
  466. hif_exec_dl_irq(napid, true);
  467. }
  468. break;
  469. case DENYLIST_OFF:
  470. if (ref_count)
  471. ref_count--;
  472. rc = 0;
  473. if (ref_count == 0) {
  474. rc = hif_napi_core_ctl_set_boost(false);
  475. NAPI_DEBUG("boost_off() returns %d - refcnt=%d",
  476. rc, ref_count);
  477. hif_exec_dl_irq(napid, false);
  478. }
  479. break;
  480. default:
  481. NAPI_DEBUG("Invalid denylist op: %d", op);
  482. rc = -EINVAL;
  483. } /* switch */
  484. out:
  485. NAPI_DEBUG("<--%s[%d]", __func__, rc);
  486. return rc;
  487. }