dp_be_tx.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476
  1. /*
  2. * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved.
  3. * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
  4. *
  5. * Permission to use, copy, modify, and/or distribute this software for
  6. * any purpose with or without fee is hereby granted, provided that the
  7. * above copyright notice and this permission notice appear in all
  8. * copies.
  9. *
  10. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
  11. * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  12. * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
  13. * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  14. * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  15. * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  16. * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  17. * PERFORMANCE OF THIS SOFTWARE.
  18. */
  19. #include "cdp_txrx_cmn_struct.h"
  20. #include "dp_types.h"
  21. #include "dp_tx.h"
  22. #include "dp_be_tx.h"
  23. #include "dp_tx_desc.h"
  24. #include "hal_tx.h"
  25. #include <hal_be_api.h>
  26. #include <hal_be_tx.h>
  27. #include <dp_htt.h>
  28. #ifdef FEATURE_WDS
  29. #include "dp_txrx_wds.h"
  30. #endif
  31. #if defined(WLAN_MAX_PDEVS) && (WLAN_MAX_PDEVS == 1)
  32. #define DP_TX_BANK_LOCK_CREATE(lock) qdf_mutex_create(lock)
  33. #define DP_TX_BANK_LOCK_DESTROY(lock) qdf_mutex_destroy(lock)
  34. #define DP_TX_BANK_LOCK_ACQUIRE(lock) qdf_mutex_acquire(lock)
  35. #define DP_TX_BANK_LOCK_RELEASE(lock) qdf_mutex_release(lock)
  36. #else
  37. #define DP_TX_BANK_LOCK_CREATE(lock) qdf_spinlock_create(lock)
  38. #define DP_TX_BANK_LOCK_DESTROY(lock) qdf_spinlock_destroy(lock)
  39. #define DP_TX_BANK_LOCK_ACQUIRE(lock) qdf_spin_lock_bh(lock)
  40. #define DP_TX_BANK_LOCK_RELEASE(lock) qdf_spin_unlock_bh(lock)
  41. #endif
  42. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP)
  43. #ifdef WLAN_MCAST_MLO
  44. /* MLO peer id for reinject*/
  45. #define DP_MLO_MCAST_REINJECT_PEER_ID 0XFFFD
  46. #define MAX_GSN_NUM 0x0FFF
  47. #ifdef QCA_MULTIPASS_SUPPORT
  48. #define INVALID_VLAN_ID 0xFFFF
  49. #define MULTIPASS_WITH_VLAN_ID 0xFFFE
  50. /**
  51. * struct dp_mlo_mpass_buf - Multipass buffer
  52. * @vlan_id: vlan_id of frame
  53. * @nbuf: pointer to skb buf
  54. */
  55. struct dp_mlo_mpass_buf {
  56. uint16_t vlan_id;
  57. qdf_nbuf_t nbuf;
  58. };
  59. #endif
  60. #endif
  61. #endif
  62. #define DP_TX_WBM_COMPLETION_V3_VDEV_ID_GET(_var) \
  63. HTT_TX_WBM_COMPLETION_V2_VDEV_ID_GET(_var)
  64. #define DP_TX_WBM_COMPLETION_V3_VALID_GET(_var) \
  65. HTT_TX_WBM_COMPLETION_V2_VALID_GET(_var)
  66. #define DP_TX_WBM_COMPLETION_V3_SW_PEER_ID_GET(_var) \
  67. HTT_TX_WBM_COMPLETION_V2_SW_PEER_ID_GET(_var)
  68. #define DP_TX_WBM_COMPLETION_V3_TID_NUM_GET(_var) \
  69. HTT_TX_WBM_COMPLETION_V2_TID_NUM_GET(_var)
  70. #define DP_TX_WBM_COMPLETION_V3_SCH_CMD_ID_GET(_var) \
  71. HTT_TX_WBM_COMPLETION_V2_SCH_CMD_ID_GET(_var)
  72. #define DP_TX_WBM_COMPLETION_V3_ACK_FRAME_RSSI_GET(_var) \
  73. HTT_TX_WBM_COMPLETION_V2_ACK_FRAME_RSSI_GET(_var)
  74. extern uint8_t sec_type_map[MAX_CDP_SEC_TYPE];
  75. #ifdef DP_USE_REDUCED_PEER_ID_FIELD_WIDTH
  76. static inline uint16_t dp_tx_comp_get_peer_id(struct dp_soc *soc,
  77. void *tx_comp_hal_desc)
  78. {
  79. uint16_t peer_id = hal_tx_comp_get_peer_id(tx_comp_hal_desc);
  80. struct dp_tx_comp_peer_id *tx_peer_id =
  81. (struct dp_tx_comp_peer_id *)&peer_id;
  82. return (tx_peer_id->peer_id |
  83. (tx_peer_id->ml_peer_valid << soc->peer_id_shift));
  84. }
  85. #else
  86. /* Combine ml_peer_valid and peer_id field */
  87. #define DP_BE_TX_COMP_PEER_ID_MASK 0x00003fff
  88. #define DP_BE_TX_COMP_PEER_ID_SHIFT 0
  89. static inline uint16_t dp_tx_comp_get_peer_id(struct dp_soc *soc,
  90. void *tx_comp_hal_desc)
  91. {
  92. uint16_t peer_id = hal_tx_comp_get_peer_id(tx_comp_hal_desc);
  93. return ((peer_id & DP_BE_TX_COMP_PEER_ID_MASK) >>
  94. DP_BE_TX_COMP_PEER_ID_SHIFT);
  95. }
  96. #endif
  97. #ifdef DP_FEATURE_HW_COOKIE_CONVERSION
  98. #ifdef DP_HW_COOKIE_CONVERT_EXCEPTION
  99. void dp_tx_comp_get_params_from_hal_desc_be(struct dp_soc *soc,
  100. void *tx_comp_hal_desc,
  101. struct dp_tx_desc_s **r_tx_desc)
  102. {
  103. uint32_t tx_desc_id;
  104. if (qdf_likely(
  105. hal_tx_comp_get_cookie_convert_done(tx_comp_hal_desc))) {
  106. /* HW cookie conversion done */
  107. *r_tx_desc = (struct dp_tx_desc_s *)
  108. hal_tx_comp_get_desc_va(tx_comp_hal_desc);
  109. } else {
  110. /* SW do cookie conversion to VA */
  111. tx_desc_id = hal_tx_comp_get_desc_id(tx_comp_hal_desc);
  112. *r_tx_desc =
  113. (struct dp_tx_desc_s *)dp_cc_desc_find(soc, tx_desc_id);
  114. }
  115. if (*r_tx_desc)
  116. (*r_tx_desc)->peer_id = dp_tx_comp_get_peer_id(soc,
  117. tx_comp_hal_desc);
  118. }
  119. #else
  120. void dp_tx_comp_get_params_from_hal_desc_be(struct dp_soc *soc,
  121. void *tx_comp_hal_desc,
  122. struct dp_tx_desc_s **r_tx_desc)
  123. {
  124. *r_tx_desc = (struct dp_tx_desc_s *)
  125. hal_tx_comp_get_desc_va(tx_comp_hal_desc);
  126. if (*r_tx_desc)
  127. (*r_tx_desc)->peer_id = dp_tx_comp_get_peer_id(soc,
  128. tx_comp_hal_desc);
  129. }
  130. #endif /* DP_HW_COOKIE_CONVERT_EXCEPTION */
  131. #else
  132. void dp_tx_comp_get_params_from_hal_desc_be(struct dp_soc *soc,
  133. void *tx_comp_hal_desc,
  134. struct dp_tx_desc_s **r_tx_desc)
  135. {
  136. uint32_t tx_desc_id;
  137. /* SW do cookie conversion to VA */
  138. tx_desc_id = hal_tx_comp_get_desc_id(tx_comp_hal_desc);
  139. *r_tx_desc =
  140. (struct dp_tx_desc_s *)dp_cc_desc_find(soc, tx_desc_id);
  141. if (*r_tx_desc)
  142. (*r_tx_desc)->peer_id = dp_tx_comp_get_peer_id(soc,
  143. tx_comp_hal_desc);
  144. }
  145. #endif /* DP_FEATURE_HW_COOKIE_CONVERSION */
  146. static inline
  147. void dp_tx_process_mec_notify_be(struct dp_soc *soc, uint8_t *status)
  148. {
  149. struct dp_vdev *vdev;
  150. uint8_t vdev_id;
  151. uint32_t *htt_desc = (uint32_t *)status;
  152. qdf_assert_always(!soc->mec_fw_offload);
  153. /*
  154. * Get vdev id from HTT status word in case of MEC
  155. * notification
  156. */
  157. vdev_id = DP_TX_WBM_COMPLETION_V3_VDEV_ID_GET(htt_desc[4]);
  158. if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
  159. return;
  160. vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  161. DP_MOD_ID_HTT_COMP);
  162. if (!vdev)
  163. return;
  164. dp_tx_mec_handler(vdev, status);
  165. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_HTT_COMP);
  166. }
  167. void dp_tx_process_htt_completion_be(struct dp_soc *soc,
  168. struct dp_tx_desc_s *tx_desc,
  169. uint8_t *status,
  170. uint8_t ring_id)
  171. {
  172. uint8_t tx_status;
  173. struct dp_pdev *pdev;
  174. struct dp_vdev *vdev = NULL;
  175. struct hal_tx_completion_status ts = {0};
  176. uint32_t *htt_desc = (uint32_t *)status;
  177. struct dp_txrx_peer *txrx_peer;
  178. dp_txrx_ref_handle txrx_ref_handle = NULL;
  179. struct cdp_tid_tx_stats *tid_stats = NULL;
  180. struct htt_soc *htt_handle;
  181. uint8_t vdev_id;
  182. tx_status = HTT_TX_WBM_COMPLETION_V3_TX_STATUS_GET(htt_desc[0]);
  183. htt_handle = (struct htt_soc *)soc->htt_handle;
  184. htt_wbm_event_record(htt_handle->htt_logger_handle, tx_status, status);
  185. /*
  186. * There can be scenario where WBM consuming descriptor enqueued
  187. * from TQM2WBM first and TQM completion can happen before MEC
  188. * notification comes from FW2WBM. Avoid access any field of tx
  189. * descriptor in case of MEC notify.
  190. */
  191. if (tx_status == HTT_TX_FW2WBM_TX_STATUS_MEC_NOTIFY)
  192. return dp_tx_process_mec_notify_be(soc, status);
  193. /*
  194. * If the descriptor is already freed in vdev_detach,
  195. * continue to next descriptor
  196. */
  197. if (qdf_unlikely(!tx_desc->flags)) {
  198. dp_tx_comp_info_rl("Descriptor freed in vdev_detach %d",
  199. tx_desc->id);
  200. return;
  201. }
  202. if (qdf_unlikely(tx_desc->vdev_id == DP_INVALID_VDEV_ID)) {
  203. dp_tx_comp_info_rl("Invalid vdev_id %d", tx_desc->id);
  204. tx_desc->flags |= DP_TX_DESC_FLAG_TX_COMP_ERR;
  205. goto release_tx_desc;
  206. }
  207. pdev = tx_desc->pdev;
  208. if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) {
  209. dp_tx_comp_info_rl("pdev in down state %d", tx_desc->id);
  210. tx_desc->flags |= DP_TX_DESC_FLAG_TX_COMP_ERR;
  211. goto release_tx_desc;
  212. }
  213. qdf_assert(tx_desc->pdev);
  214. vdev_id = tx_desc->vdev_id;
  215. vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  216. DP_MOD_ID_HTT_COMP);
  217. if (qdf_unlikely(!vdev)) {
  218. dp_tx_comp_info_rl("Unable to get vdev ref %d", tx_desc->id);
  219. tx_desc->flags |= DP_TX_DESC_FLAG_TX_COMP_ERR;
  220. goto release_tx_desc;
  221. }
  222. switch (tx_status) {
  223. case HTT_TX_FW2WBM_TX_STATUS_OK:
  224. case HTT_TX_FW2WBM_TX_STATUS_DROP:
  225. case HTT_TX_FW2WBM_TX_STATUS_TTL:
  226. {
  227. uint8_t tid;
  228. if (DP_TX_WBM_COMPLETION_V3_VALID_GET(htt_desc[3])) {
  229. ts.peer_id =
  230. DP_TX_WBM_COMPLETION_V3_SW_PEER_ID_GET(
  231. htt_desc[3]);
  232. ts.tid =
  233. DP_TX_WBM_COMPLETION_V3_TID_NUM_GET(
  234. htt_desc[3]);
  235. } else {
  236. ts.peer_id = HTT_INVALID_PEER;
  237. ts.tid = HTT_INVALID_TID;
  238. }
  239. ts.release_src = HAL_TX_COMP_RELEASE_SOURCE_FW;
  240. ts.ppdu_id =
  241. DP_TX_WBM_COMPLETION_V3_SCH_CMD_ID_GET(
  242. htt_desc[2]);
  243. ts.ack_frame_rssi =
  244. DP_TX_WBM_COMPLETION_V3_ACK_FRAME_RSSI_GET(
  245. htt_desc[2]);
  246. ts.tsf = htt_desc[4];
  247. ts.first_msdu = 1;
  248. ts.last_msdu = 1;
  249. ts.status = (tx_status == HTT_TX_FW2WBM_TX_STATUS_OK ?
  250. HAL_TX_TQM_RR_FRAME_ACKED :
  251. HAL_TX_TQM_RR_REM_CMD_REM);
  252. tid = ts.tid;
  253. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  254. tid = CDP_MAX_DATA_TIDS - 1;
  255. tid_stats = &pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
  256. if (qdf_unlikely(pdev->delay_stats_flag) ||
  257. qdf_unlikely(dp_is_vdev_tx_delay_stats_enabled(vdev)))
  258. dp_tx_compute_delay(vdev, tx_desc, tid, ring_id);
  259. if (tx_status < CDP_MAX_TX_HTT_STATUS)
  260. tid_stats->htt_status_cnt[tx_status]++;
  261. txrx_peer = dp_txrx_peer_get_ref_by_id(soc, ts.peer_id,
  262. &txrx_ref_handle,
  263. DP_MOD_ID_HTT_COMP);
  264. if (qdf_likely(txrx_peer))
  265. dp_tx_update_peer_basic_stats(
  266. txrx_peer,
  267. qdf_nbuf_len(tx_desc->nbuf),
  268. tx_status,
  269. pdev->enhanced_stats_en);
  270. dp_tx_comp_process_tx_status(soc, tx_desc, &ts, txrx_peer,
  271. ring_id);
  272. dp_tx_comp_process_desc(soc, tx_desc, &ts, txrx_peer);
  273. dp_tx_desc_release(tx_desc, tx_desc->pool_id);
  274. if (qdf_likely(txrx_peer))
  275. dp_txrx_peer_unref_delete(txrx_ref_handle,
  276. DP_MOD_ID_HTT_COMP);
  277. break;
  278. }
  279. case HTT_TX_FW2WBM_TX_STATUS_REINJECT:
  280. {
  281. uint8_t reinject_reason;
  282. reinject_reason =
  283. HTT_TX_WBM_COMPLETION_V3_REINJECT_REASON_GET(
  284. htt_desc[1]);
  285. dp_tx_reinject_handler(soc, vdev, tx_desc,
  286. status, reinject_reason);
  287. break;
  288. }
  289. case HTT_TX_FW2WBM_TX_STATUS_INSPECT:
  290. {
  291. dp_tx_inspect_handler(soc, vdev, tx_desc, status);
  292. break;
  293. }
  294. case HTT_TX_FW2WBM_TX_STATUS_VDEVID_MISMATCH:
  295. {
  296. DP_STATS_INC(vdev, tx_i.dropped.fail_per_pkt_vdev_id_check, 1);
  297. goto release_tx_desc;
  298. }
  299. default:
  300. dp_tx_comp_err("Invalid HTT tx_status %d\n",
  301. tx_status);
  302. goto release_tx_desc;
  303. }
  304. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_HTT_COMP);
  305. return;
  306. release_tx_desc:
  307. dp_tx_comp_free_buf(soc, tx_desc, false);
  308. dp_tx_desc_release(tx_desc, tx_desc->pool_id);
  309. if (vdev)
  310. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_HTT_COMP);
  311. }
  312. #ifdef QCA_OL_TX_MULTIQ_SUPPORT
  313. #ifdef DP_TX_IMPLICIT_RBM_MAPPING
  314. /*
  315. * dp_tx_get_rbm_id()- Get the RBM ID for data transmission completion.
  316. * @dp_soc - DP soc structure pointer
  317. * @ring_id - Transmit Queue/ring_id to be used when XPS is enabled
  318. *
  319. * Return - RBM ID corresponding to TCL ring_id
  320. */
  321. static inline uint8_t dp_tx_get_rbm_id_be(struct dp_soc *soc,
  322. uint8_t ring_id)
  323. {
  324. return 0;
  325. }
  326. #else
  327. static inline uint8_t dp_tx_get_rbm_id_be(struct dp_soc *soc,
  328. uint8_t ring_id)
  329. {
  330. return (ring_id ? soc->wbm_sw0_bm_id + (ring_id - 1) :
  331. HAL_WBM_SW2_BM_ID(soc->wbm_sw0_bm_id));
  332. }
  333. #endif /*DP_TX_IMPLICIT_RBM_MAPPING*/
  334. #else
  335. static inline uint8_t dp_tx_get_rbm_id_be(struct dp_soc *soc,
  336. uint8_t tcl_index)
  337. {
  338. uint8_t rbm;
  339. rbm = wlan_cfg_get_rbm_id_for_index(soc->wlan_cfg_ctx, tcl_index);
  340. dp_verbose_debug("tcl_id %u rbm %u", tcl_index, rbm);
  341. return rbm;
  342. }
  343. #endif
  344. #ifdef QCA_SUPPORT_TX_MIN_RATES_FOR_SPECIAL_FRAMES
  345. /*
  346. * dp_tx_set_min_rates_for_critical_frames()- sets min-rates for critical pkts
  347. * @dp_soc - DP soc structure pointer
  348. * @hal_tx_desc - HAL descriptor where fields are set
  349. * nbuf - skb to be considered for min rates
  350. *
  351. * The function relies on upper layers to set QDF_NBUF_CB_TX_EXTRA_IS_CRITICAL
  352. * and uses it to determine if the frame is critical. For a critical frame,
  353. * flow override bits are set to classify the frame into HW's high priority
  354. * queue. The HW will pick pre-configured min rates for such packets.
  355. *
  356. * Return - None
  357. */
  358. static void
  359. dp_tx_set_min_rates_for_critical_frames(struct dp_soc *soc,
  360. uint32_t *hal_tx_desc,
  361. qdf_nbuf_t nbuf)
  362. {
  363. /*
  364. * Critical frames should be queued to the high priority queue for the TID on
  365. * on which they are sent out (for the concerned peer).
  366. * FW is using HTT_MSDU_Q_IDX 2 for HOL (high priority) queue.
  367. * htt_msdu_idx = (2 * who_classify_info_sel) + flow_override
  368. * Hence, using who_classify_info_sel = 1, flow_override = 0 to select
  369. * HOL queue.
  370. */
  371. if (QDF_NBUF_CB_TX_EXTRA_IS_CRITICAL(nbuf)) {
  372. hal_tx_desc_set_flow_override_enable(hal_tx_desc, 1);
  373. hal_tx_desc_set_flow_override(hal_tx_desc, 0);
  374. hal_tx_desc_set_who_classify_info_sel(hal_tx_desc, 1);
  375. hal_tx_desc_set_tx_notify_frame(hal_tx_desc,
  376. TX_SEMI_HARD_NOTIFY_E);
  377. }
  378. }
  379. #else
  380. static inline void
  381. dp_tx_set_min_rates_for_critical_frames(struct dp_soc *soc,
  382. uint32_t *hal_tx_desc_cached,
  383. qdf_nbuf_t nbuf)
  384. {
  385. }
  386. #endif
  387. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  388. defined(WLAN_MCAST_MLO)
  389. #ifdef QCA_MULTIPASS_SUPPORT
  390. /**
  391. * dp_tx_mlo_mcast_multipass_lookup() - lookup vlan_id in mpass peer list
  392. * @be_vdev: Handle to DP be_vdev structure
  393. * @ptnr_vdev: DP ptnr_vdev handle
  394. * @arg: pointer to dp_mlo_mpass_ buf
  395. *
  396. * Return: None
  397. */
  398. static void
  399. dp_tx_mlo_mcast_multipass_lookup(struct dp_vdev_be *be_vdev,
  400. struct dp_vdev *ptnr_vdev,
  401. void *arg)
  402. {
  403. struct dp_mlo_mpass_buf *ptr = (struct dp_mlo_mpass_buf *)arg;
  404. struct dp_txrx_peer *txrx_peer = NULL;
  405. struct vlan_ethhdr *veh = NULL;
  406. qdf_ether_header_t *eh = (qdf_ether_header_t *)qdf_nbuf_data(ptr->nbuf);
  407. uint16_t vlan_id = 0;
  408. bool not_vlan = ((ptnr_vdev->tx_encap_type == htt_cmn_pkt_type_raw) ||
  409. (htons(eh->ether_type) != ETH_P_8021Q));
  410. if (qdf_unlikely(not_vlan))
  411. return;
  412. veh = (struct vlan_ethhdr *)eh;
  413. vlan_id = (ntohs(veh->h_vlan_TCI) & VLAN_VID_MASK);
  414. qdf_spin_lock_bh(&ptnr_vdev->mpass_peer_mutex);
  415. TAILQ_FOREACH(txrx_peer, &ptnr_vdev->mpass_peer_list,
  416. mpass_peer_list_elem) {
  417. if (vlan_id == txrx_peer->vlan_id) {
  418. qdf_spin_unlock_bh(&ptnr_vdev->mpass_peer_mutex);
  419. ptr->vlan_id = vlan_id;
  420. return;
  421. }
  422. }
  423. qdf_spin_unlock_bh(&ptnr_vdev->mpass_peer_mutex);
  424. }
  425. /**
  426. * dp_tx_mlo_mcast_multipass_send() - send multipass MLO Mcast packets
  427. * @be_vdev: Handle to DP be_vdev structure
  428. * @ptnr_vdev: DP ptnr_vdev handle
  429. * @arg: pointer to dp_mlo_mpass_ buf
  430. *
  431. * Return: None
  432. */
  433. static void
  434. dp_tx_mlo_mcast_multipass_send(struct dp_vdev_be *be_vdev,
  435. struct dp_vdev *ptnr_vdev,
  436. void *arg)
  437. {
  438. struct dp_mlo_mpass_buf *ptr = (struct dp_mlo_mpass_buf *)arg;
  439. struct dp_tx_msdu_info_s msdu_info;
  440. struct dp_vdev_be *be_ptnr_vdev = NULL;
  441. qdf_nbuf_t nbuf_clone;
  442. uint16_t group_key = 0;
  443. be_ptnr_vdev = dp_get_be_vdev_from_dp_vdev(ptnr_vdev);
  444. if (be_vdev != be_ptnr_vdev) {
  445. nbuf_clone = qdf_nbuf_clone(ptr->nbuf);
  446. if (qdf_unlikely(!nbuf_clone)) {
  447. dp_tx_debug("nbuf clone failed");
  448. return;
  449. }
  450. } else {
  451. nbuf_clone = ptr->nbuf;
  452. }
  453. qdf_mem_zero(&msdu_info, sizeof(msdu_info));
  454. dp_tx_get_queue(ptnr_vdev, nbuf_clone, &msdu_info.tx_queue);
  455. msdu_info.gsn = be_vdev->seq_num;
  456. be_ptnr_vdev->seq_num = be_vdev->seq_num;
  457. if (ptr->vlan_id == MULTIPASS_WITH_VLAN_ID) {
  458. msdu_info.tid = HTT_TX_EXT_TID_INVALID;
  459. HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_SET(
  460. msdu_info.meta_data[0], 1);
  461. } else {
  462. /* return when vlan map is not initialized */
  463. if (!ptnr_vdev->iv_vlan_map)
  464. return;
  465. group_key = ptnr_vdev->iv_vlan_map[ptr->vlan_id];
  466. /*
  467. * If group key is not installed, drop the frame.
  468. */
  469. if (!group_key)
  470. return;
  471. dp_tx_remove_vlan_tag(ptnr_vdev, nbuf_clone);
  472. dp_tx_add_groupkey_metadata(ptnr_vdev, &msdu_info, group_key);
  473. msdu_info.exception_fw = 1;
  474. }
  475. nbuf_clone = dp_tx_send_msdu_single(
  476. ptnr_vdev,
  477. nbuf_clone,
  478. &msdu_info,
  479. DP_MLO_MCAST_REINJECT_PEER_ID,
  480. NULL);
  481. if (qdf_unlikely(nbuf_clone)) {
  482. dp_info("pkt send failed");
  483. qdf_nbuf_free(nbuf_clone);
  484. return;
  485. }
  486. }
  487. /**
  488. * dp_tx_mlo_mcast_multipass_handler - If frame needs multipass processing
  489. * @soc: DP soc handle
  490. * @vdev: DP vdev handle
  491. * @nbuf: nbuf to be enqueued
  492. *
  493. * Return: true if handling is done else false
  494. */
  495. static bool
  496. dp_tx_mlo_mcast_multipass_handler(struct dp_soc *soc,
  497. struct dp_vdev *vdev,
  498. qdf_nbuf_t nbuf)
  499. {
  500. struct dp_vdev_be *be_vdev = dp_get_be_vdev_from_dp_vdev(vdev);
  501. struct dp_soc_be *be_soc = dp_get_be_soc_from_dp_soc(soc);
  502. qdf_nbuf_t nbuf_copy = NULL;
  503. struct dp_mlo_mpass_buf mpass_buf;
  504. memset(&mpass_buf, 0, sizeof(struct dp_mlo_mpass_buf));
  505. mpass_buf.vlan_id = INVALID_VLAN_ID;
  506. mpass_buf.nbuf = nbuf;
  507. dp_tx_mlo_mcast_multipass_lookup(be_vdev, vdev, &mpass_buf);
  508. if (mpass_buf.vlan_id == INVALID_VLAN_ID) {
  509. dp_mcast_mlo_iter_ptnr_vdev(be_soc, be_vdev,
  510. dp_tx_mlo_mcast_multipass_lookup,
  511. &mpass_buf, DP_MOD_ID_TX);
  512. /*
  513. * Do not drop the frame when vlan_id doesn't match.
  514. * Send the frame as it is.
  515. */
  516. if (mpass_buf.vlan_id == INVALID_VLAN_ID)
  517. return false;
  518. }
  519. /* AP can have classic clients, special clients &
  520. * classic repeaters.
  521. * 1. Classic clients & special client:
  522. * Remove vlan header, find corresponding group key
  523. * index, fill in metaheader and enqueue multicast
  524. * frame to TCL.
  525. * 2. Classic repeater:
  526. * Pass through to classic repeater with vlan tag
  527. * intact without any group key index. Hardware
  528. * will know which key to use to send frame to
  529. * repeater.
  530. */
  531. nbuf_copy = qdf_nbuf_copy(nbuf);
  532. /*
  533. * Send multicast frame to special peers even
  534. * if pass through to classic repeater fails.
  535. */
  536. if (nbuf_copy) {
  537. struct dp_mlo_mpass_buf mpass_buf_copy = {0};
  538. mpass_buf_copy.vlan_id = MULTIPASS_WITH_VLAN_ID;
  539. mpass_buf_copy.nbuf = nbuf_copy;
  540. /* send frame on partner vdevs */
  541. dp_mcast_mlo_iter_ptnr_vdev(be_soc, be_vdev,
  542. dp_tx_mlo_mcast_multipass_send,
  543. &mpass_buf_copy, DP_MOD_ID_TX);
  544. /* send frame on mcast primary vdev */
  545. dp_tx_mlo_mcast_multipass_send(be_vdev, vdev, &mpass_buf_copy);
  546. if (qdf_unlikely(be_vdev->seq_num > MAX_GSN_NUM))
  547. be_vdev->seq_num = 0;
  548. else
  549. be_vdev->seq_num++;
  550. }
  551. dp_mcast_mlo_iter_ptnr_vdev(be_soc, be_vdev,
  552. dp_tx_mlo_mcast_multipass_send,
  553. &mpass_buf, DP_MOD_ID_TX);
  554. dp_tx_mlo_mcast_multipass_send(be_vdev, vdev, &mpass_buf);
  555. if (qdf_unlikely(be_vdev->seq_num > MAX_GSN_NUM))
  556. be_vdev->seq_num = 0;
  557. else
  558. be_vdev->seq_num++;
  559. return true;
  560. }
  561. #else
  562. static bool
  563. dp_tx_mlo_mcast_multipass_handler(struct dp_soc *soc, struct dp_vdev *vdev,
  564. qdf_nbuf_t nbuf)
  565. {
  566. return false;
  567. }
  568. #endif
  569. void dp_tx_mcast_mlo_reinject_routing_set(struct dp_soc *soc, void *arg)
  570. {
  571. hal_soc_handle_t hal_soc = soc->hal_soc;
  572. uint8_t *cmd = (uint8_t *)arg;
  573. if (*cmd)
  574. hal_tx_mcast_mlo_reinject_routing_set(
  575. hal_soc,
  576. HAL_TX_MCAST_MLO_REINJECT_TQM_NOTIFY);
  577. else
  578. hal_tx_mcast_mlo_reinject_routing_set(
  579. hal_soc,
  580. HAL_TX_MCAST_MLO_REINJECT_FW_NOTIFY);
  581. }
  582. void
  583. dp_tx_mlo_mcast_pkt_send(struct dp_vdev_be *be_vdev,
  584. struct dp_vdev *ptnr_vdev,
  585. void *arg)
  586. {
  587. qdf_nbuf_t nbuf = (qdf_nbuf_t)arg;
  588. qdf_nbuf_t nbuf_clone;
  589. struct dp_vdev_be *be_ptnr_vdev = NULL;
  590. struct dp_tx_msdu_info_s msdu_info;
  591. be_ptnr_vdev = dp_get_be_vdev_from_dp_vdev(ptnr_vdev);
  592. if (be_vdev != be_ptnr_vdev) {
  593. nbuf_clone = qdf_nbuf_clone(nbuf);
  594. if (qdf_unlikely(!nbuf_clone)) {
  595. dp_tx_debug("nbuf clone failed");
  596. return;
  597. }
  598. } else {
  599. nbuf_clone = nbuf;
  600. }
  601. qdf_mem_zero(&msdu_info, sizeof(msdu_info));
  602. dp_tx_get_queue(ptnr_vdev, nbuf_clone, &msdu_info.tx_queue);
  603. msdu_info.gsn = be_vdev->seq_num;
  604. be_ptnr_vdev->seq_num = be_vdev->seq_num;
  605. nbuf_clone = dp_tx_send_msdu_single(
  606. ptnr_vdev,
  607. nbuf_clone,
  608. &msdu_info,
  609. DP_MLO_MCAST_REINJECT_PEER_ID,
  610. NULL);
  611. if (qdf_unlikely(nbuf_clone)) {
  612. dp_info("pkt send failed");
  613. qdf_nbuf_free(nbuf_clone);
  614. return;
  615. }
  616. }
  617. static inline void
  618. dp_tx_vdev_id_set_hal_tx_desc(uint32_t *hal_tx_desc_cached,
  619. struct dp_vdev *vdev,
  620. struct dp_tx_msdu_info_s *msdu_info)
  621. {
  622. hal_tx_desc_set_vdev_id(hal_tx_desc_cached, msdu_info->vdev_id);
  623. }
  624. void dp_tx_mlo_mcast_handler_be(struct dp_soc *soc,
  625. struct dp_vdev *vdev,
  626. qdf_nbuf_t nbuf)
  627. {
  628. struct dp_vdev_be *be_vdev = dp_get_be_vdev_from_dp_vdev(vdev);
  629. struct dp_soc_be *be_soc = dp_get_be_soc_from_dp_soc(soc);
  630. if (qdf_unlikely(vdev->multipass_en) &&
  631. dp_tx_mlo_mcast_multipass_handler(soc, vdev, nbuf))
  632. return;
  633. /* send frame on partner vdevs */
  634. dp_mcast_mlo_iter_ptnr_vdev(be_soc, be_vdev,
  635. dp_tx_mlo_mcast_pkt_send,
  636. nbuf, DP_MOD_ID_REINJECT);
  637. /* send frame on mcast primary vdev */
  638. dp_tx_mlo_mcast_pkt_send(be_vdev, vdev, nbuf);
  639. if (qdf_unlikely(be_vdev->seq_num > MAX_GSN_NUM))
  640. be_vdev->seq_num = 0;
  641. else
  642. be_vdev->seq_num++;
  643. }
  644. #else
  645. static inline void
  646. dp_tx_vdev_id_set_hal_tx_desc(uint32_t *hal_tx_desc_cached,
  647. struct dp_vdev *vdev,
  648. struct dp_tx_msdu_info_s *msdu_info)
  649. {
  650. hal_tx_desc_set_vdev_id(hal_tx_desc_cached, vdev->vdev_id);
  651. }
  652. #endif
  653. #if defined(WLAN_FEATURE_11BE_MLO) && !defined(WLAN_MLO_MULTI_CHIP) && \
  654. !defined(WLAN_MCAST_MLO)
  655. void dp_tx_mlo_mcast_handler_be(struct dp_soc *soc,
  656. struct dp_vdev *vdev,
  657. qdf_nbuf_t nbuf)
  658. {
  659. }
  660. #endif
  661. #ifdef CONFIG_SAWF
  662. void dp_sawf_config_be(struct dp_soc *soc, uint32_t *hal_tx_desc_cached,
  663. uint16_t *fw_metadata, qdf_nbuf_t nbuf)
  664. {
  665. uint8_t q_id = 0;
  666. if (!wlan_cfg_get_sawf_config(soc->wlan_cfg_ctx))
  667. return;
  668. dp_sawf_tcl_cmd(fw_metadata, nbuf);
  669. q_id = dp_sawf_queue_id_get(nbuf);
  670. if (q_id == DP_SAWF_DEFAULT_Q_INVALID)
  671. return;
  672. hal_tx_desc_set_hlos_tid(hal_tx_desc_cached, DP_TX_HLOS_TID_GET(q_id));
  673. hal_tx_desc_set_flow_override_enable(hal_tx_desc_cached,
  674. DP_TX_FLOW_OVERRIDE_ENABLE);
  675. hal_tx_desc_set_flow_override(hal_tx_desc_cached,
  676. DP_TX_FLOW_OVERRIDE_GET(q_id));
  677. hal_tx_desc_set_who_classify_info_sel(hal_tx_desc_cached,
  678. DP_TX_WHO_CLFY_INF_SEL_GET(q_id));
  679. }
  680. #else
  681. static inline
  682. void dp_sawf_config_be(struct dp_soc *soc, uint32_t *hal_tx_desc_cached,
  683. uint16_t *fw_metadata, qdf_nbuf_t nbuf)
  684. {
  685. }
  686. static inline
  687. QDF_STATUS dp_sawf_tx_enqueue_peer_stats(struct dp_soc *soc,
  688. struct dp_tx_desc_s *tx_desc)
  689. {
  690. return QDF_STATUS_SUCCESS;
  691. }
  692. static inline
  693. QDF_STATUS dp_sawf_tx_enqueue_fail_peer_stats(struct dp_soc *soc,
  694. struct dp_tx_desc_s *tx_desc)
  695. {
  696. return QDF_STATUS_SUCCESS;
  697. }
  698. #endif
  699. QDF_STATUS
  700. dp_tx_hw_enqueue_be(struct dp_soc *soc, struct dp_vdev *vdev,
  701. struct dp_tx_desc_s *tx_desc, uint16_t fw_metadata,
  702. struct cdp_tx_exception_metadata *tx_exc_metadata,
  703. struct dp_tx_msdu_info_s *msdu_info)
  704. {
  705. void *hal_tx_desc;
  706. uint32_t *hal_tx_desc_cached;
  707. int coalesce = 0;
  708. struct dp_tx_queue *tx_q = &msdu_info->tx_queue;
  709. uint8_t ring_id = tx_q->ring_id;
  710. uint8_t tid = msdu_info->tid;
  711. struct dp_vdev_be *be_vdev;
  712. uint8_t cached_desc[HAL_TX_DESC_LEN_BYTES] = { 0 };
  713. uint8_t bm_id = dp_tx_get_rbm_id_be(soc, ring_id);
  714. hal_ring_handle_t hal_ring_hdl = NULL;
  715. QDF_STATUS status = QDF_STATUS_E_RESOURCES;
  716. uint8_t num_desc_bytes = HAL_TX_DESC_LEN_BYTES;
  717. be_vdev = dp_get_be_vdev_from_dp_vdev(vdev);
  718. if (!dp_tx_is_desc_id_valid(soc, tx_desc->id)) {
  719. dp_err_rl("Invalid tx desc id:%d", tx_desc->id);
  720. return QDF_STATUS_E_RESOURCES;
  721. }
  722. if (qdf_unlikely(tx_exc_metadata)) {
  723. qdf_assert_always((tx_exc_metadata->tx_encap_type ==
  724. CDP_INVALID_TX_ENCAP_TYPE) ||
  725. (tx_exc_metadata->tx_encap_type ==
  726. vdev->tx_encap_type));
  727. if (tx_exc_metadata->tx_encap_type == htt_cmn_pkt_type_raw)
  728. qdf_assert_always((tx_exc_metadata->sec_type ==
  729. CDP_INVALID_SEC_TYPE) ||
  730. tx_exc_metadata->sec_type ==
  731. vdev->sec_type);
  732. }
  733. hal_tx_desc_cached = (void *)cached_desc;
  734. if (dp_sawf_tag_valid_get(tx_desc->nbuf)) {
  735. dp_sawf_config_be(soc, hal_tx_desc_cached,
  736. &fw_metadata, tx_desc->nbuf);
  737. dp_sawf_tx_enqueue_peer_stats(soc, tx_desc);
  738. }
  739. hal_tx_desc_set_buf_addr_be(soc->hal_soc, hal_tx_desc_cached,
  740. tx_desc->dma_addr, bm_id, tx_desc->id,
  741. (tx_desc->flags & DP_TX_DESC_FLAG_FRAG));
  742. hal_tx_desc_set_lmac_id_be(soc->hal_soc, hal_tx_desc_cached,
  743. vdev->lmac_id);
  744. hal_tx_desc_set_search_index_be(soc->hal_soc, hal_tx_desc_cached,
  745. vdev->bss_ast_idx);
  746. /*
  747. * Bank_ID is used as DSCP_TABLE number in beryllium
  748. * So there is no explicit field used for DSCP_TID_TABLE_NUM.
  749. */
  750. hal_tx_desc_set_cache_set_num(soc->hal_soc, hal_tx_desc_cached,
  751. (vdev->bss_ast_hash & 0xF));
  752. hal_tx_desc_set_fw_metadata(hal_tx_desc_cached, fw_metadata);
  753. hal_tx_desc_set_buf_length(hal_tx_desc_cached, tx_desc->length);
  754. hal_tx_desc_set_buf_offset(hal_tx_desc_cached, tx_desc->pkt_offset);
  755. if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)
  756. hal_tx_desc_set_to_fw(hal_tx_desc_cached, 1);
  757. /* verify checksum offload configuration*/
  758. if ((qdf_nbuf_get_tx_cksum(tx_desc->nbuf) ==
  759. QDF_NBUF_TX_CKSUM_TCP_UDP) ||
  760. qdf_nbuf_is_tso(tx_desc->nbuf)) {
  761. hal_tx_desc_set_l3_checksum_en(hal_tx_desc_cached, 1);
  762. hal_tx_desc_set_l4_checksum_en(hal_tx_desc_cached, 1);
  763. }
  764. hal_tx_desc_set_bank_id(hal_tx_desc_cached, vdev->bank_id);
  765. dp_tx_vdev_id_set_hal_tx_desc(hal_tx_desc_cached, vdev, msdu_info);
  766. if (tid != HTT_TX_EXT_TID_INVALID)
  767. hal_tx_desc_set_hlos_tid(hal_tx_desc_cached, tid);
  768. dp_tx_set_min_rates_for_critical_frames(soc, hal_tx_desc_cached,
  769. tx_desc->nbuf);
  770. dp_tx_desc_set_ktimestamp(vdev, tx_desc);
  771. hal_ring_hdl = dp_tx_get_hal_ring_hdl(soc, ring_id);
  772. if (qdf_unlikely(dp_tx_hal_ring_access_start(soc, hal_ring_hdl))) {
  773. dp_err("HAL RING Access Failed -- %pK", hal_ring_hdl);
  774. DP_STATS_INC(soc, tx.tcl_ring_full[ring_id], 1);
  775. DP_STATS_INC(vdev, tx_i.dropped.enqueue_fail, 1);
  776. dp_sawf_tx_enqueue_fail_peer_stats(soc, tx_desc);
  777. return status;
  778. }
  779. hal_tx_desc = hal_srng_src_get_next(soc->hal_soc, hal_ring_hdl);
  780. if (qdf_unlikely(!hal_tx_desc)) {
  781. dp_verbose_debug("TCL ring full ring_id:%d", ring_id);
  782. DP_STATS_INC(soc, tx.tcl_ring_full[ring_id], 1);
  783. DP_STATS_INC(vdev, tx_i.dropped.enqueue_fail, 1);
  784. dp_sawf_tx_enqueue_fail_peer_stats(soc, tx_desc);
  785. goto ring_access_fail;
  786. }
  787. tx_desc->flags |= DP_TX_DESC_FLAG_QUEUED_TX;
  788. dp_vdev_peer_stats_update_protocol_cnt_tx(vdev, tx_desc->nbuf);
  789. /* Sync cached descriptor with HW */
  790. hal_tx_desc_sync(hal_tx_desc_cached, hal_tx_desc, num_desc_bytes);
  791. coalesce = dp_tx_attempt_coalescing(soc, vdev, tx_desc, tid,
  792. msdu_info, ring_id);
  793. DP_STATS_INC_PKT(vdev, tx_i.processed, 1, tx_desc->length);
  794. DP_STATS_INC(soc, tx.tcl_enq[ring_id], 1);
  795. dp_tx_update_stats(soc, tx_desc, ring_id);
  796. status = QDF_STATUS_SUCCESS;
  797. dp_tx_hw_desc_update_evt((uint8_t *)hal_tx_desc_cached,
  798. hal_ring_hdl, soc);
  799. ring_access_fail:
  800. dp_tx_ring_access_end_wrapper(soc, hal_ring_hdl, coalesce);
  801. dp_pkt_add_timestamp(vdev, QDF_PKT_TX_DRIVER_EXIT,
  802. qdf_get_log_timestamp(), tx_desc->nbuf);
  803. return status;
  804. }
  805. #ifdef IPA_OFFLOAD
  806. static void
  807. dp_tx_get_ipa_bank_config(struct dp_soc_be *be_soc,
  808. union hal_tx_bank_config *bank_config)
  809. {
  810. bank_config->epd = 0;
  811. bank_config->encap_type = wlan_cfg_pkt_type(be_soc->soc.wlan_cfg_ctx);
  812. bank_config->encrypt_type = 0;
  813. bank_config->src_buffer_swap = 0;
  814. bank_config->link_meta_swap = 0;
  815. bank_config->index_lookup_enable = 0;
  816. bank_config->mcast_pkt_ctrl = HAL_TX_MCAST_CTRL_FW_EXCEPTION;
  817. bank_config->addrx_en = 1;
  818. bank_config->addry_en = 1;
  819. bank_config->mesh_enable = 0;
  820. bank_config->dscp_tid_map_id = 0;
  821. bank_config->vdev_id_check_en = 0;
  822. bank_config->pmac_id = 0;
  823. }
  824. static void dp_tx_init_ipa_bank_profile(struct dp_soc_be *be_soc)
  825. {
  826. union hal_tx_bank_config ipa_config = {0};
  827. int bid;
  828. if (!wlan_cfg_is_ipa_enabled(be_soc->soc.wlan_cfg_ctx)) {
  829. be_soc->ipa_bank_id = DP_BE_INVALID_BANK_ID;
  830. return;
  831. }
  832. dp_tx_get_ipa_bank_config(be_soc, &ipa_config);
  833. /* Let IPA use last HOST owned bank */
  834. bid = be_soc->num_bank_profiles - 1;
  835. be_soc->bank_profiles[bid].is_configured = true;
  836. be_soc->bank_profiles[bid].bank_config.val = ipa_config.val;
  837. hal_tx_populate_bank_register(be_soc->soc.hal_soc,
  838. &be_soc->bank_profiles[bid].bank_config,
  839. bid);
  840. qdf_atomic_inc(&be_soc->bank_profiles[bid].ref_count);
  841. dp_info("IPA bank at slot %d config:0x%x", bid,
  842. be_soc->bank_profiles[bid].bank_config.val);
  843. be_soc->ipa_bank_id = bid;
  844. }
  845. #else /* !IPA_OFFLOAD */
  846. static inline void dp_tx_init_ipa_bank_profile(struct dp_soc_be *be_soc)
  847. {
  848. }
  849. #endif /* IPA_OFFLOAD */
  850. QDF_STATUS dp_tx_init_bank_profiles(struct dp_soc_be *be_soc)
  851. {
  852. int i, num_tcl_banks;
  853. num_tcl_banks = hal_tx_get_num_tcl_banks(be_soc->soc.hal_soc);
  854. qdf_assert_always(num_tcl_banks);
  855. be_soc->num_bank_profiles = num_tcl_banks;
  856. be_soc->bank_profiles = qdf_mem_malloc(num_tcl_banks *
  857. sizeof(*be_soc->bank_profiles));
  858. if (!be_soc->bank_profiles) {
  859. dp_err("unable to allocate memory for DP TX Profiles!");
  860. return QDF_STATUS_E_NOMEM;
  861. }
  862. DP_TX_BANK_LOCK_CREATE(&be_soc->tx_bank_lock);
  863. for (i = 0; i < num_tcl_banks; i++) {
  864. be_soc->bank_profiles[i].is_configured = false;
  865. qdf_atomic_init(&be_soc->bank_profiles[i].ref_count);
  866. }
  867. dp_info("initialized %u bank profiles", be_soc->num_bank_profiles);
  868. dp_tx_init_ipa_bank_profile(be_soc);
  869. return QDF_STATUS_SUCCESS;
  870. }
  871. void dp_tx_deinit_bank_profiles(struct dp_soc_be *be_soc)
  872. {
  873. qdf_mem_free(be_soc->bank_profiles);
  874. DP_TX_BANK_LOCK_DESTROY(&be_soc->tx_bank_lock);
  875. }
  876. static
  877. void dp_tx_get_vdev_bank_config(struct dp_vdev_be *be_vdev,
  878. union hal_tx_bank_config *bank_config)
  879. {
  880. struct dp_vdev *vdev = &be_vdev->vdev;
  881. bank_config->epd = 0;
  882. bank_config->encap_type = vdev->tx_encap_type;
  883. /* Only valid for raw frames. Needs work for RAW mode */
  884. if (vdev->tx_encap_type == htt_cmn_pkt_type_raw) {
  885. bank_config->encrypt_type = sec_type_map[vdev->sec_type];
  886. } else {
  887. bank_config->encrypt_type = 0;
  888. }
  889. bank_config->src_buffer_swap = 0;
  890. bank_config->link_meta_swap = 0;
  891. if ((vdev->search_type == HAL_TX_ADDR_INDEX_SEARCH) &&
  892. vdev->opmode == wlan_op_mode_sta) {
  893. bank_config->index_lookup_enable = 1;
  894. bank_config->mcast_pkt_ctrl = HAL_TX_MCAST_CTRL_MEC_NOTIFY;
  895. bank_config->addrx_en = 0;
  896. bank_config->addry_en = 0;
  897. } else {
  898. bank_config->index_lookup_enable = 0;
  899. bank_config->mcast_pkt_ctrl = HAL_TX_MCAST_CTRL_FW_EXCEPTION;
  900. bank_config->addrx_en =
  901. (vdev->hal_desc_addr_search_flags &
  902. HAL_TX_DESC_ADDRX_EN) ? 1 : 0;
  903. bank_config->addry_en =
  904. (vdev->hal_desc_addr_search_flags &
  905. HAL_TX_DESC_ADDRY_EN) ? 1 : 0;
  906. }
  907. bank_config->mesh_enable = vdev->mesh_vdev ? 1 : 0;
  908. bank_config->dscp_tid_map_id = vdev->dscp_tid_map_id;
  909. /* Disabling vdev id check for now. Needs revist. */
  910. bank_config->vdev_id_check_en = be_vdev->vdev_id_check_en;
  911. bank_config->pmac_id = vdev->lmac_id;
  912. }
  913. int dp_tx_get_bank_profile(struct dp_soc_be *be_soc,
  914. struct dp_vdev_be *be_vdev)
  915. {
  916. char *temp_str = "";
  917. bool found_match = false;
  918. int bank_id = DP_BE_INVALID_BANK_ID;
  919. int i;
  920. int unconfigured_slot = DP_BE_INVALID_BANK_ID;
  921. int zero_ref_count_slot = DP_BE_INVALID_BANK_ID;
  922. union hal_tx_bank_config vdev_config = {0};
  923. /* convert vdev params into hal_tx_bank_config */
  924. dp_tx_get_vdev_bank_config(be_vdev, &vdev_config);
  925. DP_TX_BANK_LOCK_ACQUIRE(&be_soc->tx_bank_lock);
  926. /* go over all banks and find a matching/unconfigured/unsed bank */
  927. for (i = 0; i < be_soc->num_bank_profiles; i++) {
  928. if (be_soc->bank_profiles[i].is_configured &&
  929. (be_soc->bank_profiles[i].bank_config.val ^
  930. vdev_config.val) == 0) {
  931. found_match = true;
  932. break;
  933. }
  934. if (unconfigured_slot == DP_BE_INVALID_BANK_ID &&
  935. !be_soc->bank_profiles[i].is_configured)
  936. unconfigured_slot = i;
  937. else if (zero_ref_count_slot == DP_BE_INVALID_BANK_ID &&
  938. !qdf_atomic_read(&be_soc->bank_profiles[i].ref_count))
  939. zero_ref_count_slot = i;
  940. }
  941. if (found_match) {
  942. temp_str = "matching";
  943. bank_id = i;
  944. goto inc_ref_and_return;
  945. }
  946. if (unconfigured_slot != DP_BE_INVALID_BANK_ID) {
  947. temp_str = "unconfigured";
  948. bank_id = unconfigured_slot;
  949. goto configure_and_return;
  950. }
  951. if (zero_ref_count_slot != DP_BE_INVALID_BANK_ID) {
  952. temp_str = "zero_ref_count";
  953. bank_id = zero_ref_count_slot;
  954. }
  955. if (bank_id == DP_BE_INVALID_BANK_ID) {
  956. dp_alert("unable to find TX bank!");
  957. QDF_BUG(0);
  958. return bank_id;
  959. }
  960. configure_and_return:
  961. be_soc->bank_profiles[bank_id].is_configured = true;
  962. be_soc->bank_profiles[bank_id].bank_config.val = vdev_config.val;
  963. hal_tx_populate_bank_register(be_soc->soc.hal_soc,
  964. &be_soc->bank_profiles[bank_id].bank_config,
  965. bank_id);
  966. inc_ref_and_return:
  967. qdf_atomic_inc(&be_soc->bank_profiles[bank_id].ref_count);
  968. DP_TX_BANK_LOCK_RELEASE(&be_soc->tx_bank_lock);
  969. dp_info("found %s slot at index %d, input:0x%x match:0x%x ref_count %u",
  970. temp_str, bank_id, vdev_config.val,
  971. be_soc->bank_profiles[bank_id].bank_config.val,
  972. qdf_atomic_read(&be_soc->bank_profiles[bank_id].ref_count));
  973. dp_info("epd:%x encap:%x encryp:%x src_buf_swap:%x link_meta_swap:%x addrx_en:%x addry_en:%x mesh_en:%x vdev_id_check:%x pmac_id:%x mcast_pkt_ctrl:%x",
  974. be_soc->bank_profiles[bank_id].bank_config.epd,
  975. be_soc->bank_profiles[bank_id].bank_config.encap_type,
  976. be_soc->bank_profiles[bank_id].bank_config.encrypt_type,
  977. be_soc->bank_profiles[bank_id].bank_config.src_buffer_swap,
  978. be_soc->bank_profiles[bank_id].bank_config.link_meta_swap,
  979. be_soc->bank_profiles[bank_id].bank_config.addrx_en,
  980. be_soc->bank_profiles[bank_id].bank_config.addry_en,
  981. be_soc->bank_profiles[bank_id].bank_config.mesh_enable,
  982. be_soc->bank_profiles[bank_id].bank_config.vdev_id_check_en,
  983. be_soc->bank_profiles[bank_id].bank_config.pmac_id,
  984. be_soc->bank_profiles[bank_id].bank_config.mcast_pkt_ctrl);
  985. return bank_id;
  986. }
  987. void dp_tx_put_bank_profile(struct dp_soc_be *be_soc,
  988. struct dp_vdev_be *be_vdev)
  989. {
  990. DP_TX_BANK_LOCK_ACQUIRE(&be_soc->tx_bank_lock);
  991. qdf_atomic_dec(&be_soc->bank_profiles[be_vdev->bank_id].ref_count);
  992. DP_TX_BANK_LOCK_RELEASE(&be_soc->tx_bank_lock);
  993. }
  994. void dp_tx_update_bank_profile(struct dp_soc_be *be_soc,
  995. struct dp_vdev_be *be_vdev)
  996. {
  997. dp_tx_put_bank_profile(be_soc, be_vdev);
  998. be_vdev->bank_id = dp_tx_get_bank_profile(be_soc, be_vdev);
  999. be_vdev->vdev.bank_id = be_vdev->bank_id;
  1000. }
  1001. QDF_STATUS dp_tx_desc_pool_init_be(struct dp_soc *soc,
  1002. uint32_t num_elem,
  1003. uint8_t pool_id)
  1004. {
  1005. struct dp_tx_desc_pool_s *tx_desc_pool;
  1006. struct dp_hw_cookie_conversion_t *cc_ctx;
  1007. struct dp_soc_be *be_soc;
  1008. struct dp_spt_page_desc *page_desc;
  1009. struct dp_tx_desc_s *tx_desc;
  1010. uint32_t ppt_idx = 0;
  1011. uint32_t avail_entry_index = 0;
  1012. if (!num_elem) {
  1013. dp_err("desc_num 0 !!");
  1014. return QDF_STATUS_E_FAILURE;
  1015. }
  1016. be_soc = dp_get_be_soc_from_dp_soc(soc);
  1017. tx_desc_pool = &soc->tx_desc[pool_id];
  1018. cc_ctx = &be_soc->tx_cc_ctx[pool_id];
  1019. tx_desc = tx_desc_pool->freelist;
  1020. page_desc = &cc_ctx->page_desc_base[0];
  1021. while (tx_desc) {
  1022. if (avail_entry_index == 0) {
  1023. if (ppt_idx >= cc_ctx->total_page_num) {
  1024. dp_alert("insufficient secondary page tables");
  1025. qdf_assert_always(0);
  1026. }
  1027. page_desc = &cc_ctx->page_desc_base[ppt_idx++];
  1028. }
  1029. /* put each TX Desc VA to SPT pages and
  1030. * get corresponding ID
  1031. */
  1032. DP_CC_SPT_PAGE_UPDATE_VA(page_desc->page_v_addr,
  1033. avail_entry_index,
  1034. tx_desc);
  1035. tx_desc->id =
  1036. dp_cc_desc_id_generate(page_desc->ppt_index,
  1037. avail_entry_index);
  1038. tx_desc->pool_id = pool_id;
  1039. dp_tx_desc_set_magic(tx_desc, DP_TX_MAGIC_PATTERN_FREE);
  1040. tx_desc = tx_desc->next;
  1041. avail_entry_index = (avail_entry_index + 1) &
  1042. DP_CC_SPT_PAGE_MAX_ENTRIES_MASK;
  1043. }
  1044. return QDF_STATUS_SUCCESS;
  1045. }
  1046. void dp_tx_desc_pool_deinit_be(struct dp_soc *soc,
  1047. struct dp_tx_desc_pool_s *tx_desc_pool,
  1048. uint8_t pool_id)
  1049. {
  1050. struct dp_spt_page_desc *page_desc;
  1051. struct dp_soc_be *be_soc;
  1052. int i = 0;
  1053. struct dp_hw_cookie_conversion_t *cc_ctx;
  1054. be_soc = dp_get_be_soc_from_dp_soc(soc);
  1055. cc_ctx = &be_soc->tx_cc_ctx[pool_id];
  1056. for (i = 0; i < cc_ctx->total_page_num; i++) {
  1057. page_desc = &cc_ctx->page_desc_base[i];
  1058. qdf_mem_zero(page_desc->page_v_addr, qdf_page_size);
  1059. }
  1060. }
  1061. #ifdef WLAN_FEATURE_NEAR_FULL_IRQ
  1062. uint32_t dp_tx_comp_nf_handler(struct dp_intr *int_ctx, struct dp_soc *soc,
  1063. hal_ring_handle_t hal_ring_hdl, uint8_t ring_id,
  1064. uint32_t quota)
  1065. {
  1066. struct dp_srng *tx_comp_ring = &soc->tx_comp_ring[ring_id];
  1067. uint32_t work_done = 0;
  1068. if (dp_srng_get_near_full_level(soc, tx_comp_ring) <
  1069. DP_SRNG_THRESH_NEAR_FULL)
  1070. return 0;
  1071. qdf_atomic_set(&tx_comp_ring->near_full, 1);
  1072. work_done++;
  1073. return work_done;
  1074. }
  1075. #endif
  1076. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  1077. defined(WLAN_CONFIG_TX_DELAY)
  1078. #define PPDUID_GET_HW_LINK_ID(PPDU_ID, LINK_ID_OFFSET, LINK_ID_BITS) \
  1079. (((PPDU_ID) >> (LINK_ID_OFFSET)) & ((1 << (LINK_ID_BITS)) - 1))
  1080. #define HW_TX_DELAY_MAX 0x1000000
  1081. #define TX_COMPL_SHIFT_BUFFER_TIMESTAMP_US 10
  1082. #define HW_TX_DELAY_MASK 0x1FFFFFFF
  1083. #define TX_COMPL_BUFFER_TSTAMP_US(TSTAMP) \
  1084. (((TSTAMP) << TX_COMPL_SHIFT_BUFFER_TIMESTAMP_US) & \
  1085. HW_TX_DELAY_MASK)
  1086. static inline
  1087. QDF_STATUS dp_mlo_compute_hw_delay_us(struct dp_soc *soc,
  1088. struct dp_vdev *vdev,
  1089. struct hal_tx_completion_status *ts,
  1090. uint32_t *delay_us)
  1091. {
  1092. uint32_t ppdu_id;
  1093. uint8_t link_id_offset, link_id_bits;
  1094. uint8_t hw_link_id;
  1095. uint32_t msdu_tqm_enqueue_tstamp_us, final_msdu_tqm_enqueue_tstamp_us;
  1096. uint32_t msdu_compl_tsf_tstamp_us, final_msdu_compl_tsf_tstamp_us;
  1097. uint32_t delay;
  1098. int32_t delta_tsf2, delta_tqm;
  1099. if (!ts->valid)
  1100. return QDF_STATUS_E_INVAL;
  1101. link_id_offset = soc->link_id_offset;
  1102. link_id_bits = soc->link_id_bits;
  1103. ppdu_id = ts->ppdu_id;
  1104. hw_link_id = PPDUID_GET_HW_LINK_ID(ppdu_id, link_id_offset,
  1105. link_id_bits);
  1106. msdu_tqm_enqueue_tstamp_us =
  1107. TX_COMPL_BUFFER_TSTAMP_US(ts->buffer_timestamp);
  1108. msdu_compl_tsf_tstamp_us = ts->tsf;
  1109. delta_tsf2 = dp_mlo_get_delta_tsf2_wrt_mlo_offset(soc, hw_link_id);
  1110. delta_tqm = dp_mlo_get_delta_tqm_wrt_mlo_offset(soc);
  1111. final_msdu_tqm_enqueue_tstamp_us = (msdu_tqm_enqueue_tstamp_us +
  1112. delta_tqm) & HW_TX_DELAY_MASK;
  1113. final_msdu_compl_tsf_tstamp_us = (msdu_compl_tsf_tstamp_us +
  1114. delta_tsf2) & HW_TX_DELAY_MASK;
  1115. delay = (final_msdu_compl_tsf_tstamp_us -
  1116. final_msdu_tqm_enqueue_tstamp_us) & HW_TX_DELAY_MASK;
  1117. if (delay > HW_TX_DELAY_MAX)
  1118. return QDF_STATUS_E_FAILURE;
  1119. if (delay_us)
  1120. *delay_us = delay;
  1121. return QDF_STATUS_SUCCESS;
  1122. }
  1123. #else
  1124. static inline
  1125. QDF_STATUS dp_mlo_compute_hw_delay_us(struct dp_soc *soc,
  1126. struct dp_vdev *vdev,
  1127. struct hal_tx_completion_status *ts,
  1128. uint32_t *delay_us)
  1129. {
  1130. return QDF_STATUS_SUCCESS;
  1131. }
  1132. #endif
  1133. QDF_STATUS dp_tx_compute_tx_delay_be(struct dp_soc *soc,
  1134. struct dp_vdev *vdev,
  1135. struct hal_tx_completion_status *ts,
  1136. uint32_t *delay_us)
  1137. {
  1138. return dp_mlo_compute_hw_delay_us(soc, vdev, ts, delay_us);
  1139. }
  1140. static inline
  1141. qdf_dma_addr_t dp_tx_nbuf_map_be(struct dp_vdev *vdev,
  1142. struct dp_tx_desc_s *tx_desc,
  1143. qdf_nbuf_t nbuf)
  1144. {
  1145. qdf_nbuf_dma_clean_range_no_dsb((void *)nbuf->data,
  1146. (void *)(nbuf->data + 256));
  1147. return (qdf_dma_addr_t)qdf_mem_virt_to_phys(nbuf->data);
  1148. }
  1149. static inline
  1150. void dp_tx_nbuf_unmap_be(struct dp_soc *soc,
  1151. struct dp_tx_desc_s *desc)
  1152. {
  1153. }
  1154. /**
  1155. * dp_tx_fast_send_be() - Transmit a frame on a given VAP
  1156. * @soc: DP soc handle
  1157. * @vdev_id: id of DP vdev handle
  1158. * @nbuf: skb
  1159. *
  1160. * Entry point for Core Tx layer (DP_TX) invoked from
  1161. * hard_start_xmit in OSIF/HDD or from dp_rx_process for intravap forwarding
  1162. * cases
  1163. *
  1164. * Return: NULL on success,
  1165. * nbuf when it fails to send
  1166. */
  1167. qdf_nbuf_t dp_tx_fast_send_be(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  1168. qdf_nbuf_t nbuf)
  1169. {
  1170. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  1171. struct dp_vdev *vdev = NULL;
  1172. struct dp_pdev *pdev = NULL;
  1173. struct dp_tx_desc_s *tx_desc;
  1174. uint16_t desc_pool_id;
  1175. uint16_t pkt_len;
  1176. qdf_dma_addr_t paddr;
  1177. QDF_STATUS status = QDF_STATUS_E_RESOURCES;
  1178. uint8_t cached_desc[HAL_TX_DESC_LEN_BYTES] = { 0 };
  1179. hal_ring_handle_t hal_ring_hdl = NULL;
  1180. uint32_t *hal_tx_desc_cached;
  1181. void *hal_tx_desc;
  1182. if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
  1183. return nbuf;
  1184. vdev = soc->vdev_id_map[vdev_id];
  1185. if (qdf_unlikely(!vdev))
  1186. return nbuf;
  1187. desc_pool_id = qdf_nbuf_get_queue_mapping(nbuf) & DP_TX_QUEUE_MASK;
  1188. pkt_len = qdf_nbuf_headlen(nbuf);
  1189. DP_STATS_INC_PKT(vdev, tx_i.rcvd, 1, pkt_len);
  1190. DP_STATS_INC(vdev, tx_i.rcvd_in_fast_xmit_flow, 1);
  1191. DP_STATS_INC(vdev, tx_i.rcvd_per_core[desc_pool_id], 1);
  1192. pdev = vdev->pdev;
  1193. if (dp_tx_limit_check(vdev))
  1194. return nbuf;
  1195. tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
  1196. if (qdf_unlikely(!tx_desc)) {
  1197. DP_STATS_INC(vdev, tx_i.dropped.desc_na.num, 1);
  1198. DP_STATS_INC(vdev, tx_i.dropped.desc_na_exc_alloc_fail.num, 1);
  1199. return nbuf;
  1200. }
  1201. dp_tx_outstanding_inc(pdev);
  1202. /* Initialize the SW tx descriptor */
  1203. tx_desc->nbuf = nbuf;
  1204. tx_desc->shinfo_addr = skb_end_pointer(nbuf);
  1205. tx_desc->frm_type = dp_tx_frm_std;
  1206. tx_desc->tx_encap_type = vdev->tx_encap_type;
  1207. tx_desc->vdev_id = vdev_id;
  1208. tx_desc->pdev = pdev;
  1209. tx_desc->pkt_offset = 0;
  1210. tx_desc->length = pkt_len;
  1211. tx_desc->flags |= DP_TX_DESC_FLAG_SIMPLE;
  1212. paddr = dp_tx_nbuf_map_be(vdev, tx_desc, nbuf);
  1213. if (!paddr) {
  1214. /* Handle failure */
  1215. dp_err("qdf_nbuf_map failed");
  1216. DP_STATS_INC(vdev, tx_i.dropped.dma_error, 1);
  1217. goto release_desc;
  1218. }
  1219. tx_desc->dma_addr = paddr;
  1220. hal_tx_desc_cached = (void *)cached_desc;
  1221. hal_tx_desc_cached[0] = (uint32_t)tx_desc->dma_addr;
  1222. hal_tx_desc_cached[1] = tx_desc->id <<
  1223. TCL_DATA_CMD_BUF_ADDR_INFO_SW_BUFFER_COOKIE_LSB;
  1224. /* bank_id */
  1225. hal_tx_desc_cached[2] = vdev->bank_id << TCL_DATA_CMD_BANK_ID_LSB;
  1226. hal_tx_desc_cached[3] = vdev->htt_tcl_metadata <<
  1227. TCL_DATA_CMD_TCL_CMD_NUMBER_LSB;
  1228. hal_tx_desc_cached[4] = tx_desc->length;
  1229. /* l3 and l4 checksum enable */
  1230. hal_tx_desc_cached[4] |= DP_TX_L3_L4_CSUM_ENABLE <<
  1231. TCL_DATA_CMD_IPV4_CHECKSUM_EN_LSB;
  1232. hal_tx_desc_cached[5] = vdev->lmac_id << TCL_DATA_CMD_PMAC_ID_LSB;
  1233. hal_tx_desc_cached[5] |= vdev->vdev_id << TCL_DATA_CMD_VDEV_ID_LSB;
  1234. hal_ring_hdl = dp_tx_get_hal_ring_hdl(soc, desc_pool_id);
  1235. if (qdf_unlikely(dp_tx_hal_ring_access_start(soc, hal_ring_hdl))) {
  1236. dp_err("HAL RING Access Failed -- %pK", hal_ring_hdl);
  1237. DP_STATS_INC(soc, tx.tcl_ring_full[desc_pool_id], 1);
  1238. DP_STATS_INC(vdev, tx_i.dropped.enqueue_fail, 1);
  1239. goto ring_access_fail2;
  1240. }
  1241. hal_tx_desc = hal_srng_src_get_next(soc->hal_soc, hal_ring_hdl);
  1242. if (qdf_unlikely(!hal_tx_desc)) {
  1243. dp_verbose_debug("TCL ring full ring_id:%d", desc_pool_id);
  1244. DP_STATS_INC(soc, tx.tcl_ring_full[desc_pool_id], 1);
  1245. DP_STATS_INC(vdev, tx_i.dropped.enqueue_fail, 1);
  1246. goto ring_access_fail;
  1247. }
  1248. tx_desc->flags |= DP_TX_DESC_FLAG_QUEUED_TX;
  1249. /* Sync cached descriptor with HW */
  1250. qdf_mem_copy(hal_tx_desc, hal_tx_desc_cached, DP_TX_FAST_DESC_SIZE);
  1251. qdf_dsb();
  1252. DP_STATS_INC_PKT(vdev, tx_i.processed, 1, tx_desc->length);
  1253. DP_STATS_INC(soc, tx.tcl_enq[desc_pool_id], 1);
  1254. status = QDF_STATUS_SUCCESS;
  1255. ring_access_fail:
  1256. dp_tx_ring_access_end_wrapper(soc, hal_ring_hdl, 0);
  1257. ring_access_fail2:
  1258. if (status != QDF_STATUS_SUCCESS) {
  1259. dp_tx_nbuf_unmap_be(soc, tx_desc);
  1260. goto release_desc;
  1261. }
  1262. return NULL;
  1263. release_desc:
  1264. dp_tx_desc_release(tx_desc, desc_pool_id);
  1265. return nbuf;
  1266. }