dp_rx_desc.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. /*
  2. * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved.
  3. * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. *
  5. * Permission to use, copy, modify, and/or distribute this software for
  6. * any purpose with or without fee is hereby granted, provided that the
  7. * above copyright notice and this permission notice appear in all
  8. * copies.
  9. *
  10. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
  11. * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  12. * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
  13. * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  14. * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  15. * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  16. * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  17. * PERFORMANCE OF THIS SOFTWARE.
  18. */
  19. #include "dp_types.h"
  20. #include "dp_rx.h"
  21. #include "dp_ipa.h"
  22. #include <qdf_module.h>
  23. #ifdef RX_DESC_MULTI_PAGE_ALLOC
  24. A_COMPILE_TIME_ASSERT(cookie_size_check,
  25. (DP_BLOCKMEM_SIZE /
  26. sizeof(union dp_rx_desc_list_elem_t))
  27. <= (1 << DP_RX_DESC_PAGE_ID_SHIFT));
  28. QDF_STATUS dp_rx_desc_pool_is_allocated(struct rx_desc_pool *rx_desc_pool)
  29. {
  30. if (!rx_desc_pool->desc_pages.num_pages) {
  31. dp_err("Multi page alloc fail, size=%d, elem=%d",
  32. rx_desc_pool->elem_size, rx_desc_pool->pool_size);
  33. return QDF_STATUS_E_NOMEM;
  34. }
  35. return QDF_STATUS_SUCCESS;
  36. }
  37. qdf_export_symbol(dp_rx_desc_pool_is_allocated);
  38. QDF_STATUS dp_rx_desc_pool_alloc(struct dp_soc *soc,
  39. uint32_t num_elem,
  40. struct rx_desc_pool *rx_desc_pool)
  41. {
  42. uint32_t desc_size;
  43. union dp_rx_desc_list_elem_t *rx_desc_elem;
  44. desc_size = sizeof(*rx_desc_elem);
  45. rx_desc_pool->elem_size = desc_size;
  46. rx_desc_pool->desc_pages.page_size = DP_BLOCKMEM_SIZE;
  47. dp_desc_multi_pages_mem_alloc(soc, rx_desc_pool->desc_type,
  48. &rx_desc_pool->desc_pages,
  49. desc_size, num_elem, 0, true);
  50. if (!rx_desc_pool->desc_pages.num_pages) {
  51. qdf_err("Multi page alloc fail,size=%d, elem=%d",
  52. desc_size, num_elem);
  53. return QDF_STATUS_E_NOMEM;
  54. }
  55. if (qdf_mem_multi_page_link(soc->osdev,
  56. &rx_desc_pool->desc_pages,
  57. desc_size, num_elem, true)) {
  58. qdf_err("overflow num link,size=%d, elem=%d",
  59. desc_size, num_elem);
  60. goto free_rx_desc_pool;
  61. }
  62. return QDF_STATUS_SUCCESS;
  63. free_rx_desc_pool:
  64. dp_rx_desc_pool_free(soc, rx_desc_pool);
  65. return QDF_STATUS_E_FAULT;
  66. }
  67. qdf_export_symbol(dp_rx_desc_pool_alloc);
  68. QDF_STATUS dp_rx_desc_pool_init_generic(struct dp_soc *soc,
  69. struct rx_desc_pool *rx_desc_pool,
  70. uint32_t pool_id)
  71. {
  72. uint32_t id, page_id, offset, num_desc_per_page;
  73. uint32_t count = 0;
  74. union dp_rx_desc_list_elem_t *rx_desc_elem;
  75. num_desc_per_page = rx_desc_pool->desc_pages.num_element_per_page;
  76. rx_desc_elem = rx_desc_pool->freelist;
  77. while (rx_desc_elem) {
  78. page_id = count / num_desc_per_page;
  79. offset = count % num_desc_per_page;
  80. /*
  81. * Below cookie size is from REO destination ring
  82. * reo_destination_ring -> buffer_addr_info -> sw_buffer_cookie
  83. * cookie size = 21 bits
  84. * 8 bits - offset
  85. * 8 bits - page ID
  86. * 4 bits - pool ID
  87. */
  88. id = ((pool_id << DP_RX_DESC_POOL_ID_SHIFT) |
  89. (page_id << DP_RX_DESC_PAGE_ID_SHIFT) |
  90. offset);
  91. rx_desc_elem->rx_desc.cookie = id;
  92. rx_desc_elem->rx_desc.pool_id = pool_id;
  93. rx_desc_elem->rx_desc.in_use = 0;
  94. rx_desc_elem = rx_desc_elem->next;
  95. count++;
  96. }
  97. return QDF_STATUS_SUCCESS;
  98. }
  99. void dp_rx_desc_pool_init(struct dp_soc *soc, uint32_t pool_id,
  100. uint32_t pool_size, struct rx_desc_pool *rx_desc_pool)
  101. {
  102. QDF_STATUS status;
  103. /* Initialize the lock */
  104. qdf_spinlock_create(&rx_desc_pool->lock);
  105. qdf_spin_lock_bh(&rx_desc_pool->lock);
  106. rx_desc_pool->pool_size = pool_size;
  107. rx_desc_pool->freelist = (union dp_rx_desc_list_elem_t *)
  108. *rx_desc_pool->desc_pages.cacheable_pages;
  109. status = soc->arch_ops.dp_rx_desc_pool_init(soc, rx_desc_pool,
  110. pool_id);
  111. if (!QDF_IS_STATUS_SUCCESS(status))
  112. dp_err("RX desc pool initialization failed");
  113. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  114. }
  115. qdf_export_symbol(dp_rx_desc_pool_init);
  116. union dp_rx_desc_list_elem_t *dp_rx_desc_find(uint16_t page_id, uint16_t offset,
  117. struct rx_desc_pool *rx_desc_pool)
  118. {
  119. return rx_desc_pool->desc_pages.cacheable_pages[page_id] +
  120. rx_desc_pool->elem_size * offset;
  121. }
  122. static QDF_STATUS dp_rx_desc_nbuf_collect(struct dp_soc *soc,
  123. struct rx_desc_pool *rx_desc_pool,
  124. qdf_nbuf_t *nbuf_unmap_list,
  125. qdf_nbuf_t *nbuf_free_list)
  126. {
  127. uint32_t i, num_desc, page_id, offset, num_desc_per_page;
  128. union dp_rx_desc_list_elem_t *rx_desc_elem;
  129. struct dp_rx_desc *rx_desc;
  130. if (qdf_unlikely(!(rx_desc_pool->desc_pages.cacheable_pages))) {
  131. qdf_err("No pages found on this desc pool");
  132. return QDF_STATUS_E_INVAL;
  133. }
  134. num_desc = rx_desc_pool->pool_size;
  135. num_desc_per_page = rx_desc_pool->desc_pages.num_element_per_page;
  136. for (i = 0; i < num_desc; i++) {
  137. page_id = i / num_desc_per_page;
  138. offset = i % num_desc_per_page;
  139. rx_desc_elem = dp_rx_desc_find(page_id, offset, rx_desc_pool);
  140. rx_desc = &rx_desc_elem->rx_desc;
  141. dp_rx_desc_free_dbg_info(rx_desc);
  142. if (rx_desc->in_use) {
  143. if (!rx_desc->unmapped) {
  144. DP_RX_HEAD_APPEND(*nbuf_unmap_list,
  145. rx_desc->nbuf);
  146. rx_desc->unmapped = 1;
  147. } else {
  148. DP_RX_HEAD_APPEND(*nbuf_free_list,
  149. rx_desc->nbuf);
  150. }
  151. }
  152. }
  153. return QDF_STATUS_SUCCESS;
  154. }
  155. static void dp_rx_desc_nbuf_cleanup(struct dp_soc *soc,
  156. qdf_nbuf_t nbuf_unmap_list,
  157. qdf_nbuf_t nbuf_free_list,
  158. uint16_t buf_size,
  159. bool is_mon_pool)
  160. {
  161. qdf_nbuf_t nbuf = nbuf_unmap_list;
  162. qdf_nbuf_t next;
  163. while (nbuf) {
  164. next = nbuf->next;
  165. if (!is_mon_pool)
  166. dp_audio_smmu_unmap(soc->osdev,
  167. QDF_NBUF_CB_PADDR(nbuf),
  168. buf_size);
  169. if (dp_ipa_handle_rx_buf_smmu_mapping(soc, nbuf, buf_size,
  170. false, __func__, __LINE__))
  171. dp_info_rl("Unable to unmap nbuf: %pK", nbuf);
  172. qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf,
  173. QDF_DMA_BIDIRECTIONAL, buf_size);
  174. dp_rx_nbuf_free(nbuf);
  175. nbuf = next;
  176. }
  177. nbuf = nbuf_free_list;
  178. while (nbuf) {
  179. next = nbuf->next;
  180. dp_rx_nbuf_free(nbuf);
  181. nbuf = next;
  182. }
  183. }
  184. void dp_rx_desc_nbuf_and_pool_free(struct dp_soc *soc, uint32_t pool_id,
  185. struct rx_desc_pool *rx_desc_pool)
  186. {
  187. qdf_nbuf_t nbuf_unmap_list = NULL;
  188. qdf_nbuf_t nbuf_free_list = NULL;
  189. qdf_spin_lock_bh(&rx_desc_pool->lock);
  190. dp_rx_desc_nbuf_collect(soc, rx_desc_pool,
  191. &nbuf_unmap_list, &nbuf_free_list);
  192. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  193. dp_rx_desc_nbuf_cleanup(soc, nbuf_unmap_list, nbuf_free_list,
  194. rx_desc_pool->buf_size, false);
  195. qdf_spinlock_destroy(&rx_desc_pool->lock);
  196. }
  197. void dp_rx_desc_nbuf_free(struct dp_soc *soc,
  198. struct rx_desc_pool *rx_desc_pool,
  199. bool is_mon_pool)
  200. {
  201. qdf_nbuf_t nbuf_unmap_list = NULL;
  202. qdf_nbuf_t nbuf_free_list = NULL;
  203. qdf_spin_lock_bh(&rx_desc_pool->lock);
  204. dp_rx_desc_nbuf_collect(soc, rx_desc_pool,
  205. &nbuf_unmap_list, &nbuf_free_list);
  206. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  207. dp_rx_desc_nbuf_cleanup(soc, nbuf_unmap_list, nbuf_free_list,
  208. rx_desc_pool->buf_size, is_mon_pool);
  209. }
  210. qdf_export_symbol(dp_rx_desc_nbuf_free);
  211. void dp_rx_desc_pool_free(struct dp_soc *soc,
  212. struct rx_desc_pool *rx_desc_pool)
  213. {
  214. if (qdf_unlikely(!(rx_desc_pool->desc_pages.cacheable_pages)))
  215. return;
  216. dp_desc_multi_pages_mem_free(soc, rx_desc_pool->desc_type,
  217. &rx_desc_pool->desc_pages, 0, true);
  218. }
  219. qdf_export_symbol(dp_rx_desc_pool_free);
  220. void dp_rx_desc_pool_deinit(struct dp_soc *soc,
  221. struct rx_desc_pool *rx_desc_pool,
  222. uint32_t pool_id)
  223. {
  224. qdf_spin_lock_bh(&rx_desc_pool->lock);
  225. rx_desc_pool->freelist = NULL;
  226. rx_desc_pool->pool_size = 0;
  227. /* Deinitialize rx mon desr frag flag */
  228. rx_desc_pool->rx_mon_dest_frag_enable = false;
  229. soc->arch_ops.dp_rx_desc_pool_deinit(soc, rx_desc_pool, pool_id);
  230. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  231. qdf_spinlock_destroy(&rx_desc_pool->lock);
  232. }
  233. qdf_export_symbol(dp_rx_desc_pool_deinit);
  234. #else
  235. QDF_STATUS dp_rx_desc_pool_is_allocated(struct rx_desc_pool *rx_desc_pool)
  236. {
  237. if (!rx_desc_pool->array) {
  238. dp_err("nss-wifi<4> skip Rx refil");
  239. return QDF_STATUS_E_NOMEM;
  240. }
  241. return QDF_STATUS_SUCCESS;
  242. }
  243. qdf_export_symbol(dp_rx_desc_pool_is_allocated);
  244. QDF_STATUS dp_rx_desc_pool_alloc(struct dp_soc *soc,
  245. uint32_t pool_size,
  246. struct rx_desc_pool *rx_desc_pool)
  247. {
  248. rx_desc_pool->array = qdf_mem_malloc(pool_size *
  249. sizeof(union dp_rx_desc_list_elem_t));
  250. if (!(rx_desc_pool->array)) {
  251. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_FATAL,
  252. "RX Desc Pool allocation failed");
  253. return QDF_STATUS_E_NOMEM;
  254. }
  255. return QDF_STATUS_SUCCESS;
  256. }
  257. qdf_export_symbol(dp_rx_desc_pool_alloc);
  258. QDF_STATUS dp_rx_desc_pool_init_generic(struct dp_soc *soc,
  259. struct rx_desc_pool *rx_desc_pool,
  260. uint32_t pool_id)
  261. {
  262. int i;
  263. for (i = 0; i <= rx_desc_pool->pool_size - 1; i++) {
  264. if (i == rx_desc_pool->pool_size - 1)
  265. rx_desc_pool->array[i].next = NULL;
  266. else
  267. rx_desc_pool->array[i].next =
  268. &rx_desc_pool->array[i + 1];
  269. rx_desc_pool->array[i].rx_desc.cookie = i | (pool_id << 18);
  270. rx_desc_pool->array[i].rx_desc.pool_id = pool_id;
  271. rx_desc_pool->array[i].rx_desc.in_use = 0;
  272. }
  273. return QDF_STATUS_SUCCESS;
  274. }
  275. void dp_rx_desc_pool_init(struct dp_soc *soc, uint32_t pool_id,
  276. uint32_t pool_size, struct rx_desc_pool *rx_desc_pool)
  277. {
  278. QDF_STATUS status;
  279. /* Initialize the lock */
  280. qdf_spinlock_create(&rx_desc_pool->lock);
  281. qdf_spin_lock_bh(&rx_desc_pool->lock);
  282. rx_desc_pool->pool_size = pool_size;
  283. /* link SW rx descs into a freelist */
  284. rx_desc_pool->freelist = &rx_desc_pool->array[0];
  285. qdf_mem_zero(rx_desc_pool->freelist, rx_desc_pool->pool_size);
  286. status = soc->arch_ops.dp_rx_desc_pool_init(soc, rx_desc_pool,
  287. pool_id);
  288. if (!QDF_IS_STATUS_SUCCESS(status))
  289. dp_err("RX desc pool initialization failed");
  290. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  291. }
  292. qdf_export_symbol(dp_rx_desc_pool_init);
  293. void dp_rx_desc_nbuf_and_pool_free(struct dp_soc *soc, uint32_t pool_id,
  294. struct rx_desc_pool *rx_desc_pool)
  295. {
  296. qdf_nbuf_t nbuf;
  297. int i;
  298. qdf_spin_lock_bh(&rx_desc_pool->lock);
  299. for (i = 0; i < rx_desc_pool->pool_size; i++) {
  300. if (rx_desc_pool->array[i].rx_desc.in_use) {
  301. nbuf = rx_desc_pool->array[i].rx_desc.nbuf;
  302. if (!(rx_desc_pool->array[i].rx_desc.unmapped)) {
  303. dp_rx_nbuf_unmap_pool(soc, rx_desc_pool, nbuf);
  304. rx_desc_pool->array[i].rx_desc.unmapped = 1;
  305. }
  306. dp_rx_nbuf_free(nbuf);
  307. }
  308. }
  309. qdf_mem_free(rx_desc_pool->array);
  310. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  311. qdf_spinlock_destroy(&rx_desc_pool->lock);
  312. }
  313. void dp_rx_desc_nbuf_free(struct dp_soc *soc,
  314. struct rx_desc_pool *rx_desc_pool,
  315. bool is_mon_pool)
  316. {
  317. qdf_nbuf_t nbuf;
  318. int i;
  319. qdf_spin_lock_bh(&rx_desc_pool->lock);
  320. for (i = 0; i < rx_desc_pool->pool_size; i++) {
  321. dp_rx_desc_free_dbg_info(&rx_desc_pool->array[i].rx_desc);
  322. if (rx_desc_pool->array[i].rx_desc.in_use) {
  323. nbuf = rx_desc_pool->array[i].rx_desc.nbuf;
  324. if (!(rx_desc_pool->array[i].rx_desc.unmapped)) {
  325. dp_rx_nbuf_unmap_pool(soc, rx_desc_pool, nbuf);
  326. rx_desc_pool->array[i].rx_desc.unmapped = 1;
  327. }
  328. dp_rx_nbuf_free(nbuf);
  329. }
  330. }
  331. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  332. }
  333. qdf_export_symbol(dp_rx_desc_nbuf_free);
  334. #ifdef DP_RX_MON_MEM_FRAG
  335. void dp_rx_desc_frag_free(struct dp_soc *soc,
  336. struct rx_desc_pool *rx_desc_pool)
  337. {
  338. qdf_dma_addr_t paddr;
  339. qdf_frag_t vaddr;
  340. int i;
  341. qdf_spin_lock_bh(&rx_desc_pool->lock);
  342. for (i = 0; i < rx_desc_pool->pool_size; i++) {
  343. if (rx_desc_pool->array[i].rx_desc.in_use) {
  344. paddr = rx_desc_pool->array[i].rx_desc.paddr_buf_start;
  345. vaddr = rx_desc_pool->array[i].rx_desc.rx_buf_start;
  346. dp_rx_desc_free_dbg_info(&rx_desc_pool->array[i].rx_desc);
  347. if (!(rx_desc_pool->array[i].rx_desc.unmapped)) {
  348. qdf_mem_unmap_page(soc->osdev, paddr,
  349. rx_desc_pool->buf_size,
  350. QDF_DMA_FROM_DEVICE);
  351. rx_desc_pool->array[i].rx_desc.unmapped = 1;
  352. }
  353. qdf_frag_free(vaddr);
  354. }
  355. }
  356. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  357. }
  358. qdf_export_symbol(dp_rx_desc_frag_free);
  359. #endif
  360. void dp_rx_desc_pool_free(struct dp_soc *soc,
  361. struct rx_desc_pool *rx_desc_pool)
  362. {
  363. qdf_mem_free(rx_desc_pool->array);
  364. }
  365. qdf_export_symbol(dp_rx_desc_pool_free);
  366. void dp_rx_desc_pool_deinit(struct dp_soc *soc,
  367. struct rx_desc_pool *rx_desc_pool,
  368. uint32_t pool_id)
  369. {
  370. qdf_spin_lock_bh(&rx_desc_pool->lock);
  371. rx_desc_pool->freelist = NULL;
  372. rx_desc_pool->pool_size = 0;
  373. /* Deinitialize rx mon dest frag flag */
  374. rx_desc_pool->rx_mon_dest_frag_enable = false;
  375. soc->arch_ops.dp_rx_desc_pool_deinit(soc, rx_desc_pool, pool_id);
  376. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  377. qdf_spinlock_destroy(&rx_desc_pool->lock);
  378. }
  379. qdf_export_symbol(dp_rx_desc_pool_deinit);
  380. #endif /* RX_DESC_MULTI_PAGE_ALLOC */
  381. void dp_rx_desc_pool_deinit_generic(struct dp_soc *soc,
  382. struct rx_desc_pool *rx_desc_pool,
  383. uint32_t pool_id)
  384. {
  385. }
  386. uint16_t dp_rx_get_free_desc_list(struct dp_soc *soc, uint32_t pool_id,
  387. struct rx_desc_pool *rx_desc_pool,
  388. uint16_t num_descs,
  389. union dp_rx_desc_list_elem_t **desc_list,
  390. union dp_rx_desc_list_elem_t **tail)
  391. {
  392. uint16_t count;
  393. qdf_spin_lock_bh(&rx_desc_pool->lock);
  394. *desc_list = *tail = rx_desc_pool->freelist;
  395. for (count = 0; count < num_descs; count++) {
  396. if (qdf_unlikely(!rx_desc_pool->freelist)) {
  397. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  398. return count;
  399. }
  400. *tail = rx_desc_pool->freelist;
  401. rx_desc_pool->freelist = rx_desc_pool->freelist->next;
  402. }
  403. (*tail)->next = NULL;
  404. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  405. return count;
  406. }
  407. qdf_export_symbol(dp_rx_get_free_desc_list);
  408. void dp_rx_add_desc_list_to_free_list(struct dp_soc *soc,
  409. union dp_rx_desc_list_elem_t **local_desc_list,
  410. union dp_rx_desc_list_elem_t **tail,
  411. uint16_t pool_id,
  412. struct rx_desc_pool *rx_desc_pool)
  413. {
  414. union dp_rx_desc_list_elem_t *temp_list = NULL;
  415. qdf_spin_lock_bh(&rx_desc_pool->lock);
  416. temp_list = rx_desc_pool->freelist;
  417. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_DEBUG,
  418. "temp_list: %pK, *local_desc_list: %pK, *tail: %pK (*tail)->next: %pK",
  419. temp_list, *local_desc_list, *tail, (*tail)->next);
  420. rx_desc_pool->freelist = *local_desc_list;
  421. (*tail)->next = temp_list;
  422. *tail = NULL;
  423. *local_desc_list = NULL;
  424. qdf_spin_unlock_bh(&rx_desc_pool->lock);
  425. }
  426. qdf_export_symbol(dp_rx_add_desc_list_to_free_list);