hal_srng.c 55 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968
  1. /*
  2. * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved.
  3. *
  4. * Permission to use, copy, modify, and/or distribute this software for
  5. * any purpose with or without fee is hereby granted, provided that the
  6. * above copyright notice and this permission notice appear in all
  7. * copies.
  8. *
  9. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
  10. * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  11. * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
  12. * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  13. * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  14. * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  15. * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  16. * PERFORMANCE OF THIS SOFTWARE.
  17. */
  18. #include "hal_hw_headers.h"
  19. #include "hal_api.h"
  20. #include "target_type.h"
  21. #include "wcss_version.h"
  22. #include "qdf_module.h"
  23. #ifdef QCA_WIFI_QCA8074
  24. void hal_qca6290_attach(struct hal_soc *hal);
  25. #endif
  26. #ifdef QCA_WIFI_QCA8074
  27. void hal_qca8074_attach(struct hal_soc *hal);
  28. #endif
  29. #if defined(QCA_WIFI_QCA8074V2) || defined(QCA_WIFI_QCA6018)
  30. void hal_qca8074v2_attach(struct hal_soc *hal);
  31. #endif
  32. #ifdef QCA_WIFI_QCA6390
  33. void hal_qca6390_attach(struct hal_soc *hal);
  34. #endif
  35. #ifdef QCA_WIFI_QCA6490
  36. void hal_qca6490_attach(struct hal_soc *hal);
  37. #endif
  38. #ifdef QCA_WIFI_QCN9000
  39. void hal_qcn9000_attach(struct hal_soc *hal);
  40. #endif
  41. #ifdef QCA_WIFI_QCN6122
  42. void hal_qcn6122_attach(struct hal_soc *hal);
  43. #endif
  44. #ifdef QCA_WIFI_QCA6750
  45. void hal_qca6750_attach(struct hal_soc *hal);
  46. #endif
  47. #ifdef QCA_WIFI_QCA5018
  48. void hal_qca5018_attach(struct hal_soc *hal);
  49. #endif
  50. #ifdef ENABLE_VERBOSE_DEBUG
  51. bool is_hal_verbose_debug_enabled;
  52. #endif
  53. #ifdef ENABLE_HAL_REG_WR_HISTORY
  54. struct hal_reg_write_fail_history hal_reg_wr_hist;
  55. void hal_reg_wr_fail_history_add(struct hal_soc *hal_soc,
  56. uint32_t offset,
  57. uint32_t wr_val, uint32_t rd_val)
  58. {
  59. struct hal_reg_write_fail_entry *record;
  60. int idx;
  61. idx = hal_history_get_next_index(&hal_soc->reg_wr_fail_hist->index,
  62. HAL_REG_WRITE_HIST_SIZE);
  63. record = &hal_soc->reg_wr_fail_hist->record[idx];
  64. record->timestamp = qdf_get_log_timestamp();
  65. record->reg_offset = offset;
  66. record->write_val = wr_val;
  67. record->read_val = rd_val;
  68. }
  69. static void hal_reg_write_fail_history_init(struct hal_soc *hal)
  70. {
  71. hal->reg_wr_fail_hist = &hal_reg_wr_hist;
  72. qdf_atomic_set(&hal->reg_wr_fail_hist->index, -1);
  73. }
  74. #else
  75. static void hal_reg_write_fail_history_init(struct hal_soc *hal)
  76. {
  77. }
  78. #endif
  79. /**
  80. * hal_get_srng_ring_id() - get the ring id of a descriped ring
  81. * @hal: hal_soc data structure
  82. * @ring_type: type enum describing the ring
  83. * @ring_num: which ring of the ring type
  84. * @mac_id: which mac does the ring belong to (or 0 for non-lmac rings)
  85. *
  86. * Return: the ring id or -EINVAL if the ring does not exist.
  87. */
  88. static int hal_get_srng_ring_id(struct hal_soc *hal, int ring_type,
  89. int ring_num, int mac_id)
  90. {
  91. struct hal_hw_srng_config *ring_config =
  92. HAL_SRNG_CONFIG(hal, ring_type);
  93. int ring_id;
  94. if (ring_num >= ring_config->max_rings) {
  95. QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_INFO,
  96. "%s: ring_num exceeded maximum no. of supported rings",
  97. __func__);
  98. /* TODO: This is a programming error. Assert if this happens */
  99. return -EINVAL;
  100. }
  101. if (ring_config->lmac_ring) {
  102. ring_id = ring_config->start_ring_id + ring_num +
  103. (mac_id * HAL_MAX_RINGS_PER_LMAC);
  104. } else {
  105. ring_id = ring_config->start_ring_id + ring_num;
  106. }
  107. return ring_id;
  108. }
  109. static struct hal_srng *hal_get_srng(struct hal_soc *hal, int ring_id)
  110. {
  111. /* TODO: Should we allocate srng structures dynamically? */
  112. return &(hal->srng_list[ring_id]);
  113. }
  114. #define HP_OFFSET_IN_REG_START 1
  115. #define OFFSET_FROM_HP_TO_TP 4
  116. static void hal_update_srng_hp_tp_address(struct hal_soc *hal_soc,
  117. int shadow_config_index,
  118. int ring_type,
  119. int ring_num)
  120. {
  121. struct hal_srng *srng;
  122. int ring_id;
  123. struct hal_hw_srng_config *ring_config =
  124. HAL_SRNG_CONFIG(hal_soc, ring_type);
  125. ring_id = hal_get_srng_ring_id(hal_soc, ring_type, ring_num, 0);
  126. if (ring_id < 0)
  127. return;
  128. srng = hal_get_srng(hal_soc, ring_id);
  129. if (ring_config->ring_dir == HAL_SRNG_DST_RING) {
  130. srng->u.dst_ring.tp_addr = SHADOW_REGISTER(shadow_config_index)
  131. + hal_soc->dev_base_addr;
  132. hal_debug("tp_addr=%pK dev base addr %pK index %u",
  133. srng->u.dst_ring.tp_addr, hal_soc->dev_base_addr,
  134. shadow_config_index);
  135. } else {
  136. srng->u.src_ring.hp_addr = SHADOW_REGISTER(shadow_config_index)
  137. + hal_soc->dev_base_addr;
  138. hal_debug("hp_addr=%pK dev base addr %pK index %u",
  139. srng->u.src_ring.hp_addr,
  140. hal_soc->dev_base_addr, shadow_config_index);
  141. }
  142. }
  143. #ifdef GENERIC_SHADOW_REGISTER_ACCESS_ENABLE
  144. void hal_set_one_target_reg_config(struct hal_soc *hal,
  145. uint32_t target_reg_offset,
  146. int list_index)
  147. {
  148. int i = list_index;
  149. qdf_assert_always(i < MAX_GENERIC_SHADOW_REG);
  150. hal->list_shadow_reg_config[i].target_register =
  151. target_reg_offset;
  152. hal->num_generic_shadow_regs_configured++;
  153. }
  154. qdf_export_symbol(hal_set_one_target_reg_config);
  155. #define REO_R0_DESTINATION_RING_CTRL_ADDR_OFFSET 0x4
  156. #define MAX_REO_REMAP_SHADOW_REGS 4
  157. QDF_STATUS hal_set_shadow_regs(void *hal_soc)
  158. {
  159. uint32_t target_reg_offset;
  160. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  161. int i;
  162. struct hal_hw_srng_config *srng_config =
  163. &hal->hw_srng_table[WBM2SW_RELEASE];
  164. target_reg_offset =
  165. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_0_ADDR(
  166. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  167. for (i = 0; i < MAX_REO_REMAP_SHADOW_REGS; i++) {
  168. hal_set_one_target_reg_config(hal, target_reg_offset, i);
  169. target_reg_offset += REO_R0_DESTINATION_RING_CTRL_ADDR_OFFSET;
  170. }
  171. target_reg_offset = srng_config->reg_start[HP_OFFSET_IN_REG_START];
  172. target_reg_offset += (srng_config->reg_size[HP_OFFSET_IN_REG_START]
  173. * HAL_IPA_TX_COMP_RING_IDX);
  174. hal_set_one_target_reg_config(hal, target_reg_offset, i);
  175. return QDF_STATUS_SUCCESS;
  176. }
  177. qdf_export_symbol(hal_set_shadow_regs);
  178. QDF_STATUS hal_construct_shadow_regs(void *hal_soc)
  179. {
  180. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  181. int shadow_config_index = hal->num_shadow_registers_configured;
  182. int i;
  183. int num_regs = hal->num_generic_shadow_regs_configured;
  184. for (i = 0; i < num_regs; i++) {
  185. qdf_assert_always(shadow_config_index < MAX_SHADOW_REGISTERS);
  186. hal->shadow_config[shadow_config_index].addr =
  187. hal->list_shadow_reg_config[i].target_register;
  188. hal->list_shadow_reg_config[i].shadow_config_index =
  189. shadow_config_index;
  190. hal->list_shadow_reg_config[i].va =
  191. SHADOW_REGISTER(shadow_config_index) +
  192. (uintptr_t)hal->dev_base_addr;
  193. hal_debug("target_reg %x, shadow register 0x%x shadow_index 0x%x",
  194. hal->shadow_config[shadow_config_index].addr,
  195. SHADOW_REGISTER(shadow_config_index),
  196. shadow_config_index);
  197. shadow_config_index++;
  198. hal->num_shadow_registers_configured++;
  199. }
  200. return QDF_STATUS_SUCCESS;
  201. }
  202. qdf_export_symbol(hal_construct_shadow_regs);
  203. #endif
  204. QDF_STATUS hal_set_one_shadow_config(void *hal_soc,
  205. int ring_type,
  206. int ring_num)
  207. {
  208. uint32_t target_register;
  209. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  210. struct hal_hw_srng_config *srng_config = &hal->hw_srng_table[ring_type];
  211. int shadow_config_index = hal->num_shadow_registers_configured;
  212. if (shadow_config_index >= MAX_SHADOW_REGISTERS) {
  213. QDF_ASSERT(0);
  214. return QDF_STATUS_E_RESOURCES;
  215. }
  216. hal->num_shadow_registers_configured++;
  217. target_register = srng_config->reg_start[HP_OFFSET_IN_REG_START];
  218. target_register += (srng_config->reg_size[HP_OFFSET_IN_REG_START]
  219. *ring_num);
  220. /* if the ring is a dst ring, we need to shadow the tail pointer */
  221. if (srng_config->ring_dir == HAL_SRNG_DST_RING)
  222. target_register += OFFSET_FROM_HP_TO_TP;
  223. hal->shadow_config[shadow_config_index].addr = target_register;
  224. /* update hp/tp addr in the hal_soc structure*/
  225. hal_update_srng_hp_tp_address(hal_soc, shadow_config_index, ring_type,
  226. ring_num);
  227. hal_debug("target_reg %x, shadow register 0x%x shadow_index 0x%x, ring_type %d, ring num %d",
  228. target_register,
  229. SHADOW_REGISTER(shadow_config_index),
  230. shadow_config_index,
  231. ring_type, ring_num);
  232. return QDF_STATUS_SUCCESS;
  233. }
  234. qdf_export_symbol(hal_set_one_shadow_config);
  235. QDF_STATUS hal_construct_srng_shadow_regs(void *hal_soc)
  236. {
  237. int ring_type, ring_num;
  238. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  239. for (ring_type = 0; ring_type < MAX_RING_TYPES; ring_type++) {
  240. struct hal_hw_srng_config *srng_config =
  241. &hal->hw_srng_table[ring_type];
  242. if (ring_type == CE_SRC ||
  243. ring_type == CE_DST ||
  244. ring_type == CE_DST_STATUS)
  245. continue;
  246. if (srng_config->lmac_ring)
  247. continue;
  248. for (ring_num = 0; ring_num < srng_config->max_rings;
  249. ring_num++)
  250. hal_set_one_shadow_config(hal_soc, ring_type, ring_num);
  251. }
  252. return QDF_STATUS_SUCCESS;
  253. }
  254. qdf_export_symbol(hal_construct_srng_shadow_regs);
  255. void hal_get_shadow_config(void *hal_soc,
  256. struct pld_shadow_reg_v2_cfg **shadow_config,
  257. int *num_shadow_registers_configured)
  258. {
  259. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  260. *shadow_config = hal->shadow_config;
  261. *num_shadow_registers_configured =
  262. hal->num_shadow_registers_configured;
  263. }
  264. qdf_export_symbol(hal_get_shadow_config);
  265. static void hal_validate_shadow_register(struct hal_soc *hal,
  266. uint32_t *destination,
  267. uint32_t *shadow_address)
  268. {
  269. unsigned int index;
  270. uint32_t *shadow_0_offset = SHADOW_REGISTER(0) + hal->dev_base_addr;
  271. int destination_ba_offset =
  272. ((char *)destination) - (char *)hal->dev_base_addr;
  273. index = shadow_address - shadow_0_offset;
  274. if (index >= MAX_SHADOW_REGISTERS) {
  275. QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR,
  276. "%s: index %x out of bounds", __func__, index);
  277. goto error;
  278. } else if (hal->shadow_config[index].addr != destination_ba_offset) {
  279. QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR,
  280. "%s: sanity check failure, expected %x, found %x",
  281. __func__, destination_ba_offset,
  282. hal->shadow_config[index].addr);
  283. goto error;
  284. }
  285. return;
  286. error:
  287. qdf_print("baddr %pK, desination %pK, shadow_address %pK s0offset %pK index %x",
  288. hal->dev_base_addr, destination, shadow_address,
  289. shadow_0_offset, index);
  290. QDF_BUG(0);
  291. return;
  292. }
  293. static void hal_target_based_configure(struct hal_soc *hal)
  294. {
  295. /**
  296. * Indicate Initialization of srngs to avoid force wake
  297. * as umac power collapse is not enabled yet
  298. */
  299. hal->init_phase = true;
  300. switch (hal->target_type) {
  301. #ifdef QCA_WIFI_QCA6290
  302. case TARGET_TYPE_QCA6290:
  303. hal->use_register_windowing = true;
  304. hal_qca6290_attach(hal);
  305. break;
  306. #endif
  307. #ifdef QCA_WIFI_QCA6390
  308. case TARGET_TYPE_QCA6390:
  309. hal->use_register_windowing = true;
  310. hal_qca6390_attach(hal);
  311. break;
  312. #endif
  313. #ifdef QCA_WIFI_QCA6490
  314. case TARGET_TYPE_QCA6490:
  315. hal->use_register_windowing = true;
  316. hal_qca6490_attach(hal);
  317. break;
  318. #endif
  319. #ifdef QCA_WIFI_QCA6750
  320. case TARGET_TYPE_QCA6750:
  321. hal->use_register_windowing = true;
  322. hal->static_window_map = true;
  323. hal_qca6750_attach(hal);
  324. break;
  325. #endif
  326. #if defined(QCA_WIFI_QCA8074) && defined(WIFI_TARGET_TYPE_3_0)
  327. case TARGET_TYPE_QCA8074:
  328. hal_qca8074_attach(hal);
  329. break;
  330. #endif
  331. #if defined(QCA_WIFI_QCA8074V2)
  332. case TARGET_TYPE_QCA8074V2:
  333. hal_qca8074v2_attach(hal);
  334. break;
  335. #endif
  336. #if defined(QCA_WIFI_QCA6018)
  337. case TARGET_TYPE_QCA6018:
  338. hal_qca8074v2_attach(hal);
  339. break;
  340. #endif
  341. #if defined(QCA_WIFI_QCN6122)
  342. case TARGET_TYPE_QCN6122:
  343. hal->use_register_windowing = true;
  344. /*
  345. * Static window map is enabled for qcn9000 to use 2mb bar
  346. * size and use multiple windows to write into registers.
  347. */
  348. hal->static_window_map = true;
  349. hal_qcn6122_attach(hal);
  350. break;
  351. #endif
  352. #ifdef QCA_WIFI_QCN9000
  353. case TARGET_TYPE_QCN9000:
  354. hal->use_register_windowing = true;
  355. /*
  356. * Static window map is enabled for qcn9000 to use 2mb bar
  357. * size and use multiple windows to write into registers.
  358. */
  359. hal->static_window_map = true;
  360. hal_qcn9000_attach(hal);
  361. break;
  362. #endif
  363. #ifdef QCA_WIFI_QCA5018
  364. case TARGET_TYPE_QCA5018:
  365. hal->use_register_windowing = true;
  366. hal->static_window_map = true;
  367. hal_qca5018_attach(hal);
  368. break;
  369. #endif
  370. default:
  371. break;
  372. }
  373. }
  374. uint32_t hal_get_target_type(hal_soc_handle_t hal_soc_hdl)
  375. {
  376. struct hal_soc *hal_soc = (struct hal_soc *)hal_soc_hdl;
  377. struct hif_target_info *tgt_info =
  378. hif_get_target_info_handle(hal_soc->hif_handle);
  379. return tgt_info->target_type;
  380. }
  381. qdf_export_symbol(hal_get_target_type);
  382. #if defined(FEATURE_HAL_DELAYED_REG_WRITE) || \
  383. defined(FEATURE_HAL_DELAYED_REG_WRITE_V2)
  384. /**
  385. * hal_is_reg_write_tput_level_high() - throughput level for delayed reg writes
  386. * @hal: hal_soc pointer
  387. *
  388. * Return: true if throughput is high, else false.
  389. */
  390. static inline bool hal_is_reg_write_tput_level_high(struct hal_soc *hal)
  391. {
  392. int bw_level = hif_get_bandwidth_level(hal->hif_handle);
  393. return (bw_level >= PLD_BUS_WIDTH_MEDIUM) ? true : false;
  394. }
  395. static inline
  396. char *hal_fill_reg_write_srng_stats(struct hal_srng *srng,
  397. char *buf, qdf_size_t size)
  398. {
  399. qdf_scnprintf(buf, size, "enq %u deq %u coal %u direct %u",
  400. srng->wstats.enqueues, srng->wstats.dequeues,
  401. srng->wstats.coalesces, srng->wstats.direct);
  402. return buf;
  403. }
  404. /* bytes for local buffer */
  405. #define HAL_REG_WRITE_SRNG_STATS_LEN 100
  406. void hal_dump_reg_write_srng_stats(hal_soc_handle_t hal_soc_hdl)
  407. {
  408. struct hal_srng *srng;
  409. char buf[HAL_REG_WRITE_SRNG_STATS_LEN];
  410. struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl;
  411. srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1);
  412. hal_debug("SW2TCL1: %s",
  413. hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)));
  414. srng = hal_get_srng(hal, HAL_SRNG_WBM2SW0_RELEASE);
  415. hal_debug("WBM2SW0: %s",
  416. hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)));
  417. srng = hal_get_srng(hal, HAL_SRNG_REO2SW1);
  418. hal_debug("REO2SW1: %s",
  419. hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)));
  420. srng = hal_get_srng(hal, HAL_SRNG_REO2SW2);
  421. hal_debug("REO2SW2: %s",
  422. hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)));
  423. srng = hal_get_srng(hal, HAL_SRNG_REO2SW3);
  424. hal_debug("REO2SW3: %s",
  425. hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)));
  426. }
  427. #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2
  428. /**
  429. * hal_dump_tcl_stats() - dump the TCL reg write stats
  430. * @hal: hal_soc pointer
  431. *
  432. * Return: None
  433. */
  434. static inline void hal_dump_tcl_stats(struct hal_soc *hal)
  435. {
  436. struct hal_srng *srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1);
  437. uint32_t *hist = hal->tcl_stats.sched_delay;
  438. char buf[HAL_REG_WRITE_SRNG_STATS_LEN];
  439. hal_debug("TCL: %s sched-delay hist %u %u %u %u",
  440. hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)),
  441. hist[REG_WRITE_SCHED_DELAY_SUB_100us],
  442. hist[REG_WRITE_SCHED_DELAY_SUB_1000us],
  443. hist[REG_WRITE_SCHED_DELAY_SUB_5000us],
  444. hist[REG_WRITE_SCHED_DELAY_GT_5000us]);
  445. hal_debug("wq_dly %u wq_dir %u tim_enq %u tim_dir %u enq_tim_cnt %u dir_tim_cnt %u rst_tim_cnt %u",
  446. hal->tcl_stats.wq_delayed,
  447. hal->tcl_stats.wq_direct,
  448. hal->tcl_stats.timer_enq,
  449. hal->tcl_stats.timer_direct,
  450. hal->tcl_stats.enq_timer_set,
  451. hal->tcl_stats.direct_timer_set,
  452. hal->tcl_stats.timer_reset);
  453. }
  454. #else
  455. static inline void hal_dump_tcl_stats(struct hal_soc *hal)
  456. {
  457. }
  458. #endif
  459. void hal_dump_reg_write_stats(hal_soc_handle_t hal_soc_hdl)
  460. {
  461. uint32_t *hist;
  462. struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl;
  463. hist = hal->stats.wstats.sched_delay;
  464. hal_debug("wstats: enq %u deq %u coal %u direct %u q_depth %u max_q %u sched-delay hist %u %u %u %u",
  465. qdf_atomic_read(&hal->stats.wstats.enqueues),
  466. hal->stats.wstats.dequeues,
  467. qdf_atomic_read(&hal->stats.wstats.coalesces),
  468. qdf_atomic_read(&hal->stats.wstats.direct),
  469. qdf_atomic_read(&hal->stats.wstats.q_depth),
  470. hal->stats.wstats.max_q_depth,
  471. hist[REG_WRITE_SCHED_DELAY_SUB_100us],
  472. hist[REG_WRITE_SCHED_DELAY_SUB_1000us],
  473. hist[REG_WRITE_SCHED_DELAY_SUB_5000us],
  474. hist[REG_WRITE_SCHED_DELAY_GT_5000us]);
  475. hal_dump_tcl_stats(hal);
  476. }
  477. int hal_get_reg_write_pending_work(void *hal_soc)
  478. {
  479. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  480. return qdf_atomic_read(&hal->active_work_cnt);
  481. }
  482. #endif
  483. #ifdef FEATURE_HAL_DELAYED_REG_WRITE
  484. #ifdef MEMORY_DEBUG
  485. /*
  486. * Length of the queue(array) used to hold delayed register writes.
  487. * Must be a multiple of 2.
  488. */
  489. #define HAL_REG_WRITE_QUEUE_LEN 128
  490. #else
  491. #define HAL_REG_WRITE_QUEUE_LEN 32
  492. #endif
  493. #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2
  494. /**
  495. * hal_process_reg_write_q_elem() - process a regiter write queue element
  496. * @hal: hal_soc pointer
  497. * @q_elem: pointer to hal regiter write queue element
  498. *
  499. * Return: The value which was written to the address
  500. */
  501. static uint32_t
  502. hal_process_reg_write_q_elem(struct hal_soc *hal,
  503. struct hal_reg_write_q_elem *q_elem)
  504. {
  505. struct hal_srng *srng = q_elem->srng;
  506. uint32_t write_val;
  507. SRNG_LOCK(&srng->lock);
  508. srng->reg_write_in_progress = false;
  509. srng->wstats.dequeues++;
  510. if (srng->ring_dir == HAL_SRNG_SRC_RING) {
  511. write_val = srng->u.src_ring.hp;
  512. q_elem->dequeue_val = write_val;
  513. q_elem->valid = 0;
  514. SRNG_UNLOCK(&srng->lock);
  515. hal_write_address_32_mb(hal,
  516. srng->u.src_ring.hp_addr,
  517. write_val, false);
  518. } else {
  519. write_val = srng->u.dst_ring.tp;
  520. q_elem->dequeue_val = write_val;
  521. q_elem->valid = 0;
  522. SRNG_UNLOCK(&srng->lock);
  523. hal_write_address_32_mb(hal,
  524. srng->u.dst_ring.tp_addr,
  525. write_val, false);
  526. }
  527. return write_val;
  528. }
  529. #else
  530. /**
  531. * hal_process_reg_write_q_elem() - process a regiter write queue element
  532. * @hal: hal_soc pointer
  533. * @q_elem: pointer to hal regiter write queue element
  534. *
  535. * Return: The value which was written to the address
  536. */
  537. static uint32_t
  538. hal_process_reg_write_q_elem(struct hal_soc *hal,
  539. struct hal_reg_write_q_elem *q_elem)
  540. {
  541. struct hal_srng *srng = q_elem->srng;
  542. uint32_t write_val;
  543. SRNG_LOCK(&srng->lock);
  544. srng->reg_write_in_progress = false;
  545. srng->wstats.dequeues++;
  546. if (srng->ring_dir == HAL_SRNG_SRC_RING) {
  547. q_elem->dequeue_val = srng->u.src_ring.hp;
  548. hal_write_address_32_mb(hal,
  549. srng->u.src_ring.hp_addr,
  550. srng->u.src_ring.hp, false);
  551. write_val = srng->u.src_ring.hp;
  552. } else {
  553. q_elem->dequeue_val = srng->u.dst_ring.tp;
  554. hal_write_address_32_mb(hal,
  555. srng->u.dst_ring.tp_addr,
  556. srng->u.dst_ring.tp, false);
  557. write_val = srng->u.dst_ring.tp;
  558. }
  559. q_elem->valid = 0;
  560. SRNG_UNLOCK(&srng->lock);
  561. return write_val;
  562. }
  563. #endif
  564. /**
  565. * hal_reg_write_fill_sched_delay_hist() - fill reg write delay histogram in hal
  566. * @hal: hal_soc pointer
  567. * @delay: delay in us
  568. *
  569. * Return: None
  570. */
  571. static inline void hal_reg_write_fill_sched_delay_hist(struct hal_soc *hal,
  572. uint64_t delay_us)
  573. {
  574. uint32_t *hist;
  575. hist = hal->stats.wstats.sched_delay;
  576. if (delay_us < 100)
  577. hist[REG_WRITE_SCHED_DELAY_SUB_100us]++;
  578. else if (delay_us < 1000)
  579. hist[REG_WRITE_SCHED_DELAY_SUB_1000us]++;
  580. else if (delay_us < 5000)
  581. hist[REG_WRITE_SCHED_DELAY_SUB_5000us]++;
  582. else
  583. hist[REG_WRITE_SCHED_DELAY_GT_5000us]++;
  584. }
  585. /**
  586. * hal_reg_write_work() - Worker to process delayed writes
  587. * @arg: hal_soc pointer
  588. *
  589. * Return: None
  590. */
  591. static void hal_reg_write_work(void *arg)
  592. {
  593. int32_t q_depth, write_val;
  594. struct hal_soc *hal = arg;
  595. struct hal_reg_write_q_elem *q_elem;
  596. uint64_t delta_us;
  597. uint8_t ring_id;
  598. uint32_t *addr;
  599. uint32_t num_processed = 0;
  600. q_elem = &hal->reg_write_queue[(hal->read_idx)];
  601. q_elem->work_scheduled_time = qdf_get_log_timestamp();
  602. /* Make sure q_elem consistent in the memory for multi-cores */
  603. qdf_rmb();
  604. if (!q_elem->valid)
  605. return;
  606. q_depth = qdf_atomic_read(&hal->stats.wstats.q_depth);
  607. if (q_depth > hal->stats.wstats.max_q_depth)
  608. hal->stats.wstats.max_q_depth = q_depth;
  609. if (hif_prevent_link_low_power_states(hal->hif_handle)) {
  610. hal->stats.wstats.prevent_l1_fails++;
  611. return;
  612. }
  613. while (true) {
  614. qdf_rmb();
  615. if (!q_elem->valid)
  616. break;
  617. q_elem->dequeue_time = qdf_get_log_timestamp();
  618. ring_id = q_elem->srng->ring_id;
  619. addr = q_elem->addr;
  620. delta_us = qdf_log_timestamp_to_usecs(q_elem->dequeue_time -
  621. q_elem->enqueue_time);
  622. hal_reg_write_fill_sched_delay_hist(hal, delta_us);
  623. hal->stats.wstats.dequeues++;
  624. qdf_atomic_dec(&hal->stats.wstats.q_depth);
  625. write_val = hal_process_reg_write_q_elem(hal, q_elem);
  626. hal_verbose_debug("read_idx %u srng 0x%x, addr 0x%pK dequeue_val %u sched delay %llu us",
  627. hal->read_idx, ring_id, addr, write_val, delta_us);
  628. num_processed++;
  629. hal->read_idx = (hal->read_idx + 1) &
  630. (HAL_REG_WRITE_QUEUE_LEN - 1);
  631. q_elem = &hal->reg_write_queue[(hal->read_idx)];
  632. }
  633. hif_allow_link_low_power_states(hal->hif_handle);
  634. /*
  635. * Decrement active_work_cnt by the number of elements dequeued after
  636. * hif_allow_link_low_power_states.
  637. * This makes sure that hif_try_complete_tasks will wait till we make
  638. * the bus access in hif_allow_link_low_power_states. This will avoid
  639. * race condition between delayed register worker and bus suspend
  640. * (system suspend or runtime suspend).
  641. *
  642. * The following decrement should be done at the end!
  643. */
  644. qdf_atomic_sub(num_processed, &hal->active_work_cnt);
  645. }
  646. static void __hal_flush_reg_write_work(struct hal_soc *hal)
  647. {
  648. qdf_cancel_work(&hal->reg_write_work);
  649. }
  650. void hal_flush_reg_write_work(hal_soc_handle_t hal_handle)
  651. { __hal_flush_reg_write_work((struct hal_soc *)hal_handle);
  652. }
  653. /**
  654. * hal_reg_write_enqueue() - enqueue register writes into kworker
  655. * @hal_soc: hal_soc pointer
  656. * @srng: srng pointer
  657. * @addr: iomem address of regiter
  658. * @value: value to be written to iomem address
  659. *
  660. * This function executes from within the SRNG LOCK
  661. *
  662. * Return: None
  663. */
  664. static void hal_reg_write_enqueue(struct hal_soc *hal_soc,
  665. struct hal_srng *srng,
  666. void __iomem *addr,
  667. uint32_t value)
  668. {
  669. struct hal_reg_write_q_elem *q_elem;
  670. uint32_t write_idx;
  671. if (srng->reg_write_in_progress) {
  672. hal_verbose_debug("Already in progress srng ring id 0x%x addr 0x%pK val %u",
  673. srng->ring_id, addr, value);
  674. qdf_atomic_inc(&hal_soc->stats.wstats.coalesces);
  675. srng->wstats.coalesces++;
  676. return;
  677. }
  678. write_idx = qdf_atomic_inc_return(&hal_soc->write_idx);
  679. write_idx = write_idx & (HAL_REG_WRITE_QUEUE_LEN - 1);
  680. q_elem = &hal_soc->reg_write_queue[write_idx];
  681. if (q_elem->valid) {
  682. hal_err("queue full");
  683. QDF_BUG(0);
  684. return;
  685. }
  686. qdf_atomic_inc(&hal_soc->stats.wstats.enqueues);
  687. srng->wstats.enqueues++;
  688. qdf_atomic_inc(&hal_soc->stats.wstats.q_depth);
  689. q_elem->srng = srng;
  690. q_elem->addr = addr;
  691. q_elem->enqueue_val = value;
  692. q_elem->enqueue_time = qdf_get_log_timestamp();
  693. /*
  694. * Before the valid flag is set to true, all the other
  695. * fields in the q_elem needs to be updated in memory.
  696. * Else there is a chance that the dequeuing worker thread
  697. * might read stale entries and process incorrect srng.
  698. */
  699. qdf_wmb();
  700. q_elem->valid = true;
  701. /*
  702. * After all other fields in the q_elem has been updated
  703. * in memory successfully, the valid flag needs to be updated
  704. * in memory in time too.
  705. * Else there is a chance that the dequeuing worker thread
  706. * might read stale valid flag and the work will be bypassed
  707. * for this round. And if there is no other work scheduled
  708. * later, this hal register writing won't be updated any more.
  709. */
  710. qdf_wmb();
  711. srng->reg_write_in_progress = true;
  712. qdf_atomic_inc(&hal_soc->active_work_cnt);
  713. hal_verbose_debug("write_idx %u srng ring id 0x%x addr 0x%pK val %u",
  714. write_idx, srng->ring_id, addr, value);
  715. qdf_queue_work(hal_soc->qdf_dev, hal_soc->reg_write_wq,
  716. &hal_soc->reg_write_work);
  717. }
  718. /**
  719. * hal_delayed_reg_write_init() - Initialization function for delayed reg writes
  720. * @hal_soc: hal_soc pointer
  721. *
  722. * Initialize main data structures to process register writes in a delayed
  723. * workqueue.
  724. *
  725. * Return: QDF_STATUS_SUCCESS on success else a QDF error.
  726. */
  727. static QDF_STATUS hal_delayed_reg_write_init(struct hal_soc *hal)
  728. {
  729. hal->reg_write_wq =
  730. qdf_alloc_high_prior_ordered_workqueue("hal_register_write_wq");
  731. qdf_create_work(0, &hal->reg_write_work, hal_reg_write_work, hal);
  732. hal->reg_write_queue = qdf_mem_malloc(HAL_REG_WRITE_QUEUE_LEN *
  733. sizeof(*hal->reg_write_queue));
  734. if (!hal->reg_write_queue) {
  735. hal_err("unable to allocate memory");
  736. QDF_BUG(0);
  737. return QDF_STATUS_E_NOMEM;
  738. }
  739. /* Initial value of indices */
  740. hal->read_idx = 0;
  741. qdf_atomic_set(&hal->write_idx, -1);
  742. return QDF_STATUS_SUCCESS;
  743. }
  744. /**
  745. * hal_delayed_reg_write_deinit() - De-Initialize delayed reg write processing
  746. * @hal_soc: hal_soc pointer
  747. *
  748. * De-initialize main data structures to process register writes in a delayed
  749. * workqueue.
  750. *
  751. * Return: None
  752. */
  753. static void hal_delayed_reg_write_deinit(struct hal_soc *hal)
  754. {
  755. __hal_flush_reg_write_work(hal);
  756. qdf_flush_workqueue(0, hal->reg_write_wq);
  757. qdf_destroy_workqueue(0, hal->reg_write_wq);
  758. qdf_mem_free(hal->reg_write_queue);
  759. }
  760. #else
  761. static inline QDF_STATUS hal_delayed_reg_write_init(struct hal_soc *hal)
  762. {
  763. return QDF_STATUS_SUCCESS;
  764. }
  765. static inline void hal_delayed_reg_write_deinit(struct hal_soc *hal)
  766. {
  767. }
  768. #endif
  769. #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2
  770. #ifdef MEMORY_DEBUG
  771. /**
  772. * hal_reg_write_get_timestamp() - Function to get the timestamp
  773. *
  774. * Return: return present simestamp
  775. */
  776. static inline qdf_time_t hal_del_reg_write_get_ts(void)
  777. {
  778. return qdf_get_log_timestamp();
  779. }
  780. /**
  781. * hal_del_reg_write_ts_usecs() - Convert the timestamp to micro secs
  782. * @ts: timestamp value to be converted
  783. *
  784. * Return: return the timestamp in micro secs
  785. */
  786. static inline qdf_time_t hal_del_reg_write_ts_usecs(qdf_time_t ts)
  787. {
  788. return qdf_log_timestamp_to_usecs(ts);
  789. }
  790. /**
  791. * hal_tcl_write_fill_sched_delay_hist() - fill TCL reg write delay histogram
  792. * @hal: hal_soc pointer
  793. * @delay: delay in us
  794. *
  795. * Return: None
  796. */
  797. static inline void hal_tcl_write_fill_sched_delay_hist(struct hal_soc *hal)
  798. {
  799. uint32_t *hist;
  800. uint32_t delay_us;
  801. hal->tcl_stats.deq_time = hal_del_reg_write_get_ts();
  802. delay_us = hal_del_reg_write_ts_usecs(hal->tcl_stats.deq_time -
  803. hal->tcl_stats.enq_time);
  804. hist = hal->tcl_stats.sched_delay;
  805. if (delay_us < 100)
  806. hist[REG_WRITE_SCHED_DELAY_SUB_100us]++;
  807. else if (delay_us < 1000)
  808. hist[REG_WRITE_SCHED_DELAY_SUB_1000us]++;
  809. else if (delay_us < 5000)
  810. hist[REG_WRITE_SCHED_DELAY_SUB_5000us]++;
  811. else
  812. hist[REG_WRITE_SCHED_DELAY_GT_5000us]++;
  813. }
  814. #else
  815. static inline qdf_time_t hal_del_reg_write_get_ts(void)
  816. {
  817. return 0;
  818. }
  819. static inline qdf_time_t hal_del_reg_write_ts_usecs(qdf_time_t ts)
  820. {
  821. return 0;
  822. }
  823. static inline void hal_tcl_write_fill_sched_delay_hist(struct hal_soc *hal)
  824. {
  825. }
  826. #endif
  827. /**
  828. * hal_tcl_reg_write_work() - Worker to process delayed SW2TCL1 writes
  829. * @arg: hal_soc pointer
  830. *
  831. * Return: None
  832. */
  833. static void hal_tcl_reg_write_work(void *arg)
  834. {
  835. struct hal_soc *hal = arg;
  836. struct hal_srng *srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1);
  837. SRNG_LOCK(&srng->lock);
  838. srng->wstats.dequeues++;
  839. hal_tcl_write_fill_sched_delay_hist(hal);
  840. /*
  841. * During the tranition of low to high tput scenario, reg write moves
  842. * from delayed to direct write context, there is a little chance that
  843. * worker thread gets scheduled later than direct context write which
  844. * already wrote the latest HP value. This check can catch that case
  845. * and avoid the repetitive writing of the same HP value.
  846. */
  847. if (srng->last_reg_wr_val != srng->u.src_ring.hp) {
  848. srng->last_reg_wr_val = srng->u.src_ring.hp;
  849. if (hal->tcl_direct) {
  850. /*
  851. * TCL reg writes have been moved to direct context and
  852. * the assumption is that PCIe bus stays in Active state
  853. * during high tput, hence its fine to write the HP
  854. * while the SRNG_LOCK is being held.
  855. */
  856. hal->tcl_stats.wq_direct++;
  857. hal_write_address_32_mb(hal, srng->u.src_ring.hp_addr,
  858. srng->last_reg_wr_val, false);
  859. srng->reg_write_in_progress = false;
  860. SRNG_UNLOCK(&srng->lock);
  861. } else {
  862. /*
  863. * TCL reg write to happen in delayed context,
  864. * write operation might take time due to possibility of
  865. * PCIe bus stays in low power state during low tput,
  866. * Hence release the SRNG_LOCK before writing.
  867. */
  868. hal->tcl_stats.wq_delayed++;
  869. srng->reg_write_in_progress = false;
  870. SRNG_UNLOCK(&srng->lock);
  871. hal_write_address_32_mb(hal, srng->u.src_ring.hp_addr,
  872. srng->last_reg_wr_val, false);
  873. }
  874. } else {
  875. srng->reg_write_in_progress = false;
  876. SRNG_UNLOCK(&srng->lock);
  877. }
  878. /*
  879. * Decrement active_work_cnt to make sure that hif_try_complete_tasks
  880. * will wait. This will avoid race condition between delayed register
  881. * worker and bus suspend (system suspend or runtime suspend).
  882. *
  883. * The following decrement should be done at the end!
  884. */
  885. qdf_atomic_dec(&hal->active_work_cnt);
  886. qdf_atomic_set(&hal->tcl_work_active, false);
  887. }
  888. static void __hal_flush_tcl_reg_write_work(struct hal_soc *hal)
  889. {
  890. qdf_cancel_work(&hal->tcl_reg_write_work);
  891. }
  892. /**
  893. * hal_tcl_reg_write_enqueue() - enqueue TCL register writes into kworker
  894. * @hal_soc: hal_soc pointer
  895. * @srng: srng pointer
  896. * @addr: iomem address of regiter
  897. * @value: value to be written to iomem address
  898. *
  899. * This function executes from within the SRNG LOCK
  900. *
  901. * Return: None
  902. */
  903. static void hal_tcl_reg_write_enqueue(struct hal_soc *hal_soc,
  904. struct hal_srng *srng,
  905. void __iomem *addr,
  906. uint32_t value)
  907. {
  908. hal_soc->tcl_stats.enq_time = hal_del_reg_write_get_ts();
  909. if (qdf_queue_work(hal_soc->qdf_dev, hal_soc->tcl_reg_write_wq,
  910. &hal_soc->tcl_reg_write_work)) {
  911. srng->reg_write_in_progress = true;
  912. qdf_atomic_inc(&hal_soc->active_work_cnt);
  913. qdf_atomic_set(&hal_soc->tcl_work_active, true);
  914. srng->wstats.enqueues++;
  915. } else {
  916. hal_soc->tcl_stats.enq_timer_set++;
  917. qdf_timer_mod(&hal_soc->tcl_reg_write_timer, 1);
  918. }
  919. }
  920. /**
  921. * hal_tcl_reg_write_timer() - timer handler to take care of pending TCL writes
  922. * @arg: srng handle
  923. *
  924. * This function handles the pending TCL reg writes missed due to the previous
  925. * scheduled worker running.
  926. *
  927. * Return: None
  928. */
  929. static void hal_tcl_reg_write_timer(void *arg)
  930. {
  931. hal_ring_handle_t srng_hdl = arg;
  932. struct hal_srng *srng;
  933. struct hal_soc *hal;
  934. srng = (struct hal_srng *)srng_hdl;
  935. hal = srng->hal_soc;
  936. if (hif_pm_runtime_get(hal->hif_handle, RTPM_ID_DW_TX_HW_ENQUEUE,
  937. true)) {
  938. hal_srng_set_event(srng_hdl, HAL_SRNG_FLUSH_EVENT);
  939. hal_srng_inc_flush_cnt(srng_hdl);
  940. goto fail;
  941. }
  942. SRNG_LOCK(&srng->lock);
  943. if (hal->tcl_direct) {
  944. /*
  945. * Due to the previous scheduled worker still running,
  946. * direct reg write cannot be performed, so posted the
  947. * pending writes to timer context.
  948. */
  949. if (srng->last_reg_wr_val != srng->u.src_ring.hp) {
  950. srng->last_reg_wr_val = srng->u.src_ring.hp;
  951. srng->wstats.direct++;
  952. hal->tcl_stats.timer_direct++;
  953. hal_write_address_32_mb(hal, srng->u.src_ring.hp_addr,
  954. srng->last_reg_wr_val, false);
  955. }
  956. } else {
  957. /*
  958. * Due to the previous scheduled worker still running,
  959. * queue_work from delayed context would fail,
  960. * so retry from timer context.
  961. */
  962. if (qdf_queue_work(hal->qdf_dev, hal->tcl_reg_write_wq,
  963. &hal->tcl_reg_write_work)) {
  964. srng->reg_write_in_progress = true;
  965. qdf_atomic_inc(&hal->active_work_cnt);
  966. qdf_atomic_set(&hal->tcl_work_active, true);
  967. srng->wstats.enqueues++;
  968. hal->tcl_stats.timer_enq++;
  969. } else {
  970. if (srng->last_reg_wr_val != srng->u.src_ring.hp) {
  971. hal->tcl_stats.timer_reset++;
  972. qdf_timer_mod(&hal->tcl_reg_write_timer, 1);
  973. }
  974. }
  975. }
  976. SRNG_UNLOCK(&srng->lock);
  977. hif_pm_runtime_put(hal->hif_handle, RTPM_ID_DW_TX_HW_ENQUEUE);
  978. fail:
  979. return;
  980. }
  981. /**
  982. * hal_delayed_tcl_reg_write_init() - Initialization for delayed TCL reg writes
  983. * @hal_soc: hal_soc pointer
  984. *
  985. * Initialize main data structures to process TCL register writes in a delayed
  986. * workqueue.
  987. *
  988. * Return: QDF_STATUS_SUCCESS on success else a QDF error.
  989. */
  990. static QDF_STATUS hal_delayed_tcl_reg_write_init(struct hal_soc *hal)
  991. {
  992. struct hal_srng *srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1);
  993. QDF_STATUS status;
  994. hal->tcl_reg_write_wq =
  995. qdf_alloc_high_prior_ordered_workqueue("hal_tcl_reg_write_wq");
  996. if (!hal->tcl_reg_write_wq) {
  997. hal_err("hal_tcl_reg_write_wq alloc failed");
  998. return QDF_STATUS_E_NOMEM;
  999. }
  1000. status = qdf_create_work(0, &hal->tcl_reg_write_work,
  1001. hal_tcl_reg_write_work, hal);
  1002. if (status != QDF_STATUS_SUCCESS) {
  1003. hal_err("tcl_reg_write_work create failed");
  1004. goto fail;
  1005. }
  1006. status = qdf_timer_init(hal->qdf_dev, &hal->tcl_reg_write_timer,
  1007. hal_tcl_reg_write_timer, (void *)srng,
  1008. QDF_TIMER_TYPE_WAKE_APPS);
  1009. if (status != QDF_STATUS_SUCCESS) {
  1010. hal_err("tcl_reg_write_timer init failed");
  1011. goto fail;
  1012. }
  1013. qdf_atomic_init(&hal->tcl_work_active);
  1014. return QDF_STATUS_SUCCESS;
  1015. fail:
  1016. qdf_destroy_workqueue(0, hal->tcl_reg_write_wq);
  1017. return status;
  1018. }
  1019. /**
  1020. * hal_delayed_tcl_reg_write_deinit() - De-Initialize delayed TCL reg writes
  1021. * @hal_soc: hal_soc pointer
  1022. *
  1023. * De-initialize main data structures to process TCL register writes in a
  1024. * delayed workqueue.
  1025. *
  1026. * Return: None
  1027. */
  1028. static void hal_delayed_tcl_reg_write_deinit(struct hal_soc *hal)
  1029. {
  1030. qdf_timer_stop(&hal->tcl_reg_write_timer);
  1031. qdf_timer_free(&hal->tcl_reg_write_timer);
  1032. __hal_flush_tcl_reg_write_work(hal);
  1033. qdf_flush_workqueue(0, hal->tcl_reg_write_wq);
  1034. qdf_destroy_workqueue(0, hal->tcl_reg_write_wq);
  1035. }
  1036. #else
  1037. static inline QDF_STATUS hal_delayed_tcl_reg_write_init(struct hal_soc *hal)
  1038. {
  1039. return QDF_STATUS_SUCCESS;
  1040. }
  1041. static inline void hal_delayed_tcl_reg_write_deinit(struct hal_soc *hal)
  1042. {
  1043. }
  1044. #endif
  1045. #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2
  1046. #ifdef FEATURE_HAL_DELAYED_REG_WRITE
  1047. static inline void hal_reg_write_enqueue_v2(struct hal_soc *hal_soc,
  1048. struct hal_srng *srng,
  1049. void __iomem *addr,
  1050. uint32_t value)
  1051. {
  1052. hal_reg_write_enqueue(hal_soc, srng, addr, value);
  1053. }
  1054. #else
  1055. static inline void hal_reg_write_enqueue_v2(struct hal_soc *hal_soc,
  1056. struct hal_srng *srng,
  1057. void __iomem *addr,
  1058. uint32_t value)
  1059. {
  1060. qdf_atomic_inc(&hal_soc->stats.wstats.direct);
  1061. srng->wstats.direct++;
  1062. hal_write_address_32_mb(hal_soc, addr, value, false);
  1063. }
  1064. #endif
  1065. void hal_delayed_reg_write(struct hal_soc *hal_soc,
  1066. struct hal_srng *srng,
  1067. void __iomem *addr,
  1068. uint32_t value)
  1069. {
  1070. switch (srng->ring_type) {
  1071. case TCL_DATA:
  1072. if (hal_is_reg_write_tput_level_high(hal_soc)) {
  1073. hal_soc->tcl_direct = true;
  1074. if (srng->reg_write_in_progress ||
  1075. !qdf_atomic_read(&hal_soc->tcl_work_active)) {
  1076. /*
  1077. * Now the delayed work have either completed
  1078. * the writing or not even scheduled and would
  1079. * be blocked by SRNG_LOCK, hence it is fine to
  1080. * do direct write here.
  1081. */
  1082. srng->last_reg_wr_val = srng->u.src_ring.hp;
  1083. srng->wstats.direct++;
  1084. hal_write_address_32_mb(hal_soc, addr,
  1085. srng->last_reg_wr_val,
  1086. false);
  1087. } else {
  1088. hal_soc->tcl_stats.direct_timer_set++;
  1089. qdf_timer_mod(&hal_soc->tcl_reg_write_timer, 1);
  1090. }
  1091. } else {
  1092. hal_soc->tcl_direct = false;
  1093. if (srng->reg_write_in_progress) {
  1094. srng->wstats.coalesces++;
  1095. } else {
  1096. hal_tcl_reg_write_enqueue(hal_soc, srng,
  1097. addr, value);
  1098. }
  1099. }
  1100. break;
  1101. case CE_SRC:
  1102. case CE_DST:
  1103. case CE_DST_STATUS:
  1104. hal_reg_write_enqueue_v2(hal_soc, srng, addr, value);
  1105. break;
  1106. default:
  1107. qdf_atomic_inc(&hal_soc->stats.wstats.direct);
  1108. srng->wstats.direct++;
  1109. hal_write_address_32_mb(hal_soc, addr, value, false);
  1110. break;
  1111. }
  1112. }
  1113. #else
  1114. #ifdef FEATURE_HAL_DELAYED_REG_WRITE
  1115. #ifdef QCA_WIFI_QCA6750
  1116. void hal_delayed_reg_write(struct hal_soc *hal_soc,
  1117. struct hal_srng *srng,
  1118. void __iomem *addr,
  1119. uint32_t value)
  1120. {
  1121. uint8_t vote_access;
  1122. switch (srng->ring_type) {
  1123. case CE_SRC:
  1124. case CE_DST:
  1125. case CE_DST_STATUS:
  1126. vote_access = hif_get_ep_vote_access(hal_soc->hif_handle,
  1127. HIF_EP_VOTE_NONDP_ACCESS);
  1128. if ((vote_access == HIF_EP_VOTE_ACCESS_DISABLE) ||
  1129. (vote_access == HIF_EP_VOTE_INTERMEDIATE_ACCESS &&
  1130. PLD_MHI_STATE_L0 ==
  1131. pld_get_mhi_state(hal_soc->qdf_dev->dev))) {
  1132. hal_write_address_32_mb(hal_soc, addr, value, false);
  1133. qdf_atomic_inc(&hal_soc->stats.wstats.direct);
  1134. srng->wstats.direct++;
  1135. } else {
  1136. hal_reg_write_enqueue(hal_soc, srng, addr, value);
  1137. }
  1138. break;
  1139. default:
  1140. if (hif_get_ep_vote_access(hal_soc->hif_handle,
  1141. HIF_EP_VOTE_DP_ACCESS) ==
  1142. HIF_EP_VOTE_ACCESS_DISABLE ||
  1143. hal_is_reg_write_tput_level_high(hal_soc) ||
  1144. PLD_MHI_STATE_L0 ==
  1145. pld_get_mhi_state(hal_soc->qdf_dev->dev)) {
  1146. hal_write_address_32_mb(hal_soc, addr, value, false);
  1147. qdf_atomic_inc(&hal_soc->stats.wstats.direct);
  1148. srng->wstats.direct++;
  1149. } else {
  1150. hal_reg_write_enqueue(hal_soc, srng, addr, value);
  1151. }
  1152. break;
  1153. }
  1154. }
  1155. #else
  1156. void hal_delayed_reg_write(struct hal_soc *hal_soc,
  1157. struct hal_srng *srng,
  1158. void __iomem *addr,
  1159. uint32_t value)
  1160. {
  1161. if (pld_is_device_awake(hal_soc->qdf_dev->dev) ||
  1162. hal_is_reg_write_tput_level_high(hal_soc)) {
  1163. qdf_atomic_inc(&hal_soc->stats.wstats.direct);
  1164. srng->wstats.direct++;
  1165. hal_write_address_32_mb(hal_soc, addr, value, false);
  1166. } else {
  1167. hal_reg_write_enqueue(hal_soc, srng, addr, value);
  1168. }
  1169. }
  1170. #endif
  1171. #endif
  1172. #endif
  1173. /**
  1174. * hal_attach - Initialize HAL layer
  1175. * @hif_handle: Opaque HIF handle
  1176. * @qdf_dev: QDF device
  1177. *
  1178. * Return: Opaque HAL SOC handle
  1179. * NULL on failure (if given ring is not available)
  1180. *
  1181. * This function should be called as part of HIF initialization (for accessing
  1182. * copy engines). DP layer will get hal_soc handle using hif_get_hal_handle()
  1183. *
  1184. */
  1185. void *hal_attach(struct hif_opaque_softc *hif_handle, qdf_device_t qdf_dev)
  1186. {
  1187. struct hal_soc *hal;
  1188. int i;
  1189. hal = qdf_mem_malloc(sizeof(*hal));
  1190. if (!hal) {
  1191. QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR,
  1192. "%s: hal_soc allocation failed", __func__);
  1193. goto fail0;
  1194. }
  1195. hal->hif_handle = hif_handle;
  1196. hal->dev_base_addr = hif_get_dev_ba(hif_handle); /* UMAC */
  1197. hal->dev_base_addr_ce = hif_get_dev_ba_ce(hif_handle); /* CE */
  1198. hal->qdf_dev = qdf_dev;
  1199. hal->shadow_rdptr_mem_vaddr = (uint32_t *)qdf_mem_alloc_consistent(
  1200. qdf_dev, qdf_dev->dev, sizeof(*(hal->shadow_rdptr_mem_vaddr)) *
  1201. HAL_SRNG_ID_MAX, &(hal->shadow_rdptr_mem_paddr));
  1202. if (!hal->shadow_rdptr_mem_paddr) {
  1203. QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR,
  1204. "%s: hal->shadow_rdptr_mem_paddr allocation failed",
  1205. __func__);
  1206. goto fail1;
  1207. }
  1208. qdf_mem_zero(hal->shadow_rdptr_mem_vaddr,
  1209. sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX);
  1210. hal->shadow_wrptr_mem_vaddr =
  1211. (uint32_t *)qdf_mem_alloc_consistent(qdf_dev, qdf_dev->dev,
  1212. sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS,
  1213. &(hal->shadow_wrptr_mem_paddr));
  1214. if (!hal->shadow_wrptr_mem_vaddr) {
  1215. QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR,
  1216. "%s: hal->shadow_wrptr_mem_vaddr allocation failed",
  1217. __func__);
  1218. goto fail2;
  1219. }
  1220. qdf_mem_zero(hal->shadow_wrptr_mem_vaddr,
  1221. sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS);
  1222. for (i = 0; i < HAL_SRNG_ID_MAX; i++) {
  1223. hal->srng_list[i].initialized = 0;
  1224. hal->srng_list[i].ring_id = i;
  1225. }
  1226. qdf_spinlock_create(&hal->register_access_lock);
  1227. hal->register_window = 0;
  1228. hal->target_type = hal_get_target_type(hal_soc_to_hal_soc_handle(hal));
  1229. hal->ops = qdf_mem_malloc(sizeof(*hal->ops));
  1230. if (!hal->ops) {
  1231. hal_err("unable to allocable memory for HAL ops");
  1232. goto fail3;
  1233. }
  1234. hal_target_based_configure(hal);
  1235. hal_reg_write_fail_history_init(hal);
  1236. qdf_minidump_log(hal, sizeof(*hal), "hal_soc");
  1237. qdf_atomic_init(&hal->active_work_cnt);
  1238. hal_delayed_reg_write_init(hal);
  1239. hal_delayed_tcl_reg_write_init(hal);
  1240. return (void *)hal;
  1241. fail3:
  1242. qdf_mem_free_consistent(qdf_dev, qdf_dev->dev,
  1243. sizeof(*hal->shadow_wrptr_mem_vaddr) *
  1244. HAL_MAX_LMAC_RINGS,
  1245. hal->shadow_wrptr_mem_vaddr,
  1246. hal->shadow_wrptr_mem_paddr, 0);
  1247. fail2:
  1248. qdf_mem_free_consistent(qdf_dev, qdf_dev->dev,
  1249. sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX,
  1250. hal->shadow_rdptr_mem_vaddr, hal->shadow_rdptr_mem_paddr, 0);
  1251. fail1:
  1252. qdf_mem_free(hal);
  1253. fail0:
  1254. return NULL;
  1255. }
  1256. qdf_export_symbol(hal_attach);
  1257. /**
  1258. * hal_mem_info - Retrieve hal memory base address
  1259. *
  1260. * @hal_soc: Opaque HAL SOC handle
  1261. * @mem: pointer to structure to be updated with hal mem info
  1262. */
  1263. void hal_get_meminfo(hal_soc_handle_t hal_soc_hdl, struct hal_mem_info *mem)
  1264. {
  1265. struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl;
  1266. mem->dev_base_addr = (void *)hal->dev_base_addr;
  1267. mem->shadow_rdptr_mem_vaddr = (void *)hal->shadow_rdptr_mem_vaddr;
  1268. mem->shadow_wrptr_mem_vaddr = (void *)hal->shadow_wrptr_mem_vaddr;
  1269. mem->shadow_rdptr_mem_paddr = (void *)hal->shadow_rdptr_mem_paddr;
  1270. mem->shadow_wrptr_mem_paddr = (void *)hal->shadow_wrptr_mem_paddr;
  1271. hif_read_phy_mem_base((void *)hal->hif_handle,
  1272. (qdf_dma_addr_t *)&mem->dev_base_paddr);
  1273. return;
  1274. }
  1275. qdf_export_symbol(hal_get_meminfo);
  1276. /**
  1277. * hal_detach - Detach HAL layer
  1278. * @hal_soc: HAL SOC handle
  1279. *
  1280. * Return: Opaque HAL SOC handle
  1281. * NULL on failure (if given ring is not available)
  1282. *
  1283. * This function should be called as part of HIF initialization (for accessing
  1284. * copy engines). DP layer will get hal_soc handle using hif_get_hal_handle()
  1285. *
  1286. */
  1287. extern void hal_detach(void *hal_soc)
  1288. {
  1289. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  1290. hal_delayed_reg_write_deinit(hal);
  1291. hal_delayed_tcl_reg_write_deinit(hal);
  1292. qdf_mem_free(hal->ops);
  1293. qdf_mem_free_consistent(hal->qdf_dev, hal->qdf_dev->dev,
  1294. sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX,
  1295. hal->shadow_rdptr_mem_vaddr, hal->shadow_rdptr_mem_paddr, 0);
  1296. qdf_mem_free_consistent(hal->qdf_dev, hal->qdf_dev->dev,
  1297. sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS,
  1298. hal->shadow_wrptr_mem_vaddr, hal->shadow_wrptr_mem_paddr, 0);
  1299. qdf_minidump_remove(hal, sizeof(*hal), "hal_soc");
  1300. qdf_mem_free(hal);
  1301. return;
  1302. }
  1303. qdf_export_symbol(hal_detach);
  1304. /**
  1305. * hal_ce_dst_setup - Initialize CE destination ring registers
  1306. * @hal_soc: HAL SOC handle
  1307. * @srng: SRNG ring pointer
  1308. */
  1309. static inline void hal_ce_dst_setup(struct hal_soc *hal, struct hal_srng *srng,
  1310. int ring_num)
  1311. {
  1312. uint32_t reg_val = 0;
  1313. uint32_t reg_addr;
  1314. struct hal_hw_srng_config *ring_config =
  1315. HAL_SRNG_CONFIG(hal, CE_DST);
  1316. /* set DEST_MAX_LENGTH according to ce assignment */
  1317. reg_addr = HWIO_WFSS_CE_CHANNEL_DST_R0_DEST_CTRL_ADDR(
  1318. ring_config->reg_start[R0_INDEX] +
  1319. (ring_num * ring_config->reg_size[R0_INDEX]));
  1320. reg_val = HAL_REG_READ(hal, reg_addr);
  1321. reg_val &= ~HWIO_WFSS_CE_CHANNEL_DST_R0_DEST_CTRL_DEST_MAX_LENGTH_BMSK;
  1322. reg_val |= srng->u.dst_ring.max_buffer_length &
  1323. HWIO_WFSS_CE_CHANNEL_DST_R0_DEST_CTRL_DEST_MAX_LENGTH_BMSK;
  1324. HAL_REG_WRITE(hal, reg_addr, reg_val);
  1325. if (srng->prefetch_timer) {
  1326. reg_addr = HWIO_WFSS_CE_CHANNEL_DST_R0_DEST_RING_CONSUMER_PREFETCH_TIMER_ADDR(
  1327. ring_config->reg_start[R0_INDEX] +
  1328. (ring_num * ring_config->reg_size[R0_INDEX]));
  1329. reg_val = HAL_REG_READ(hal, reg_addr);
  1330. reg_val &= ~HWIO_WFSS_CE_CHANNEL_DST_R0_DEST_RING_CONSUMER_PREFETCH_TIMER_RMSK;
  1331. reg_val |= srng->prefetch_timer;
  1332. HAL_REG_WRITE(hal, reg_addr, reg_val);
  1333. reg_val = HAL_REG_READ(hal, reg_addr);
  1334. }
  1335. }
  1336. /**
  1337. * hal_reo_read_write_ctrl_ix - Read or write REO_DESTINATION_RING_CTRL_IX
  1338. * @hal: HAL SOC handle
  1339. * @read: boolean value to indicate if read or write
  1340. * @ix0: pointer to store IX0 reg value
  1341. * @ix1: pointer to store IX1 reg value
  1342. * @ix2: pointer to store IX2 reg value
  1343. * @ix3: pointer to store IX3 reg value
  1344. */
  1345. void hal_reo_read_write_ctrl_ix(hal_soc_handle_t hal_soc_hdl, bool read,
  1346. uint32_t *ix0, uint32_t *ix1,
  1347. uint32_t *ix2, uint32_t *ix3)
  1348. {
  1349. uint32_t reg_offset;
  1350. struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl;
  1351. if (read) {
  1352. if (ix0) {
  1353. reg_offset =
  1354. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_0_ADDR(
  1355. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1356. *ix0 = HAL_REG_READ(hal, reg_offset);
  1357. }
  1358. if (ix1) {
  1359. reg_offset =
  1360. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_1_ADDR(
  1361. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1362. *ix1 = HAL_REG_READ(hal, reg_offset);
  1363. }
  1364. if (ix2) {
  1365. reg_offset =
  1366. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_2_ADDR(
  1367. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1368. *ix2 = HAL_REG_READ(hal, reg_offset);
  1369. }
  1370. if (ix3) {
  1371. reg_offset =
  1372. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_3_ADDR(
  1373. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1374. *ix3 = HAL_REG_READ(hal, reg_offset);
  1375. }
  1376. } else {
  1377. if (ix0) {
  1378. reg_offset =
  1379. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_0_ADDR(
  1380. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1381. HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset,
  1382. *ix0, true);
  1383. }
  1384. if (ix1) {
  1385. reg_offset =
  1386. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_1_ADDR(
  1387. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1388. HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset,
  1389. *ix1, true);
  1390. }
  1391. if (ix2) {
  1392. reg_offset =
  1393. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_2_ADDR(
  1394. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1395. HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset,
  1396. *ix2, true);
  1397. }
  1398. if (ix3) {
  1399. reg_offset =
  1400. HWIO_REO_R0_DESTINATION_RING_CTRL_IX_3_ADDR(
  1401. SEQ_WCSS_UMAC_REO_REG_OFFSET);
  1402. HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset,
  1403. *ix3, true);
  1404. }
  1405. }
  1406. }
  1407. /**
  1408. * hal_srng_dst_set_hp_paddr_confirm() - Set physical address to dest ring head
  1409. * pointer and confirm that write went through by reading back the value
  1410. * @srng: sring pointer
  1411. * @paddr: physical address
  1412. *
  1413. * Return: None
  1414. */
  1415. void hal_srng_dst_set_hp_paddr_confirm(struct hal_srng *srng, uint64_t paddr)
  1416. {
  1417. SRNG_DST_REG_WRITE_CONFIRM(srng, HP_ADDR_LSB, paddr & 0xffffffff);
  1418. SRNG_DST_REG_WRITE_CONFIRM(srng, HP_ADDR_MSB, paddr >> 32);
  1419. }
  1420. /**
  1421. * hal_srng_dst_init_hp() - Initialize destination ring head
  1422. * pointer
  1423. * @hal_soc: hal_soc handle
  1424. * @srng: sring pointer
  1425. * @vaddr: virtual address
  1426. */
  1427. void hal_srng_dst_init_hp(struct hal_soc_handle *hal_soc,
  1428. struct hal_srng *srng,
  1429. uint32_t *vaddr)
  1430. {
  1431. uint32_t reg_offset;
  1432. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  1433. if (!srng)
  1434. return;
  1435. srng->u.dst_ring.hp_addr = vaddr;
  1436. reg_offset = SRNG_DST_ADDR(srng, HP) - hal->dev_base_addr;
  1437. HAL_REG_WRITE_CONFIRM_RETRY(
  1438. hal, reg_offset, srng->u.dst_ring.cached_hp, true);
  1439. if (vaddr) {
  1440. *srng->u.dst_ring.hp_addr = srng->u.dst_ring.cached_hp;
  1441. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  1442. "hp_addr=%pK, cached_hp=%d, hp=%d",
  1443. (void *)srng->u.dst_ring.hp_addr,
  1444. srng->u.dst_ring.cached_hp,
  1445. *srng->u.dst_ring.hp_addr);
  1446. }
  1447. }
  1448. /**
  1449. * hal_srng_hw_init - Private function to initialize SRNG HW
  1450. * @hal_soc: HAL SOC handle
  1451. * @srng: SRNG ring pointer
  1452. */
  1453. static inline void hal_srng_hw_init(struct hal_soc *hal,
  1454. struct hal_srng *srng)
  1455. {
  1456. if (srng->ring_dir == HAL_SRNG_SRC_RING)
  1457. hal_srng_src_hw_init(hal, srng);
  1458. else
  1459. hal_srng_dst_hw_init(hal, srng);
  1460. }
  1461. #ifdef CONFIG_SHADOW_V2
  1462. #define ignore_shadow false
  1463. #define CHECK_SHADOW_REGISTERS true
  1464. #else
  1465. #define ignore_shadow true
  1466. #define CHECK_SHADOW_REGISTERS false
  1467. #endif
  1468. /**
  1469. * hal_srng_setup - Initialize HW SRNG ring.
  1470. * @hal_soc: Opaque HAL SOC handle
  1471. * @ring_type: one of the types from hal_ring_type
  1472. * @ring_num: Ring number if there are multiple rings of same type (staring
  1473. * from 0)
  1474. * @mac_id: valid MAC Id should be passed if ring type is one of lmac rings
  1475. * @ring_params: SRNG ring params in hal_srng_params structure.
  1476. * Callers are expected to allocate contiguous ring memory of size
  1477. * 'num_entries * entry_size' bytes and pass the physical and virtual base
  1478. * addresses through 'ring_base_paddr' and 'ring_base_vaddr' in
  1479. * hal_srng_params structure. Ring base address should be 8 byte aligned
  1480. * and size of each ring entry should be queried using the API
  1481. * hal_srng_get_entrysize
  1482. *
  1483. * Return: Opaque pointer to ring on success
  1484. * NULL on failure (if given ring is not available)
  1485. */
  1486. void *hal_srng_setup(void *hal_soc, int ring_type, int ring_num,
  1487. int mac_id, struct hal_srng_params *ring_params)
  1488. {
  1489. int ring_id;
  1490. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  1491. struct hal_srng *srng;
  1492. struct hal_hw_srng_config *ring_config =
  1493. HAL_SRNG_CONFIG(hal, ring_type);
  1494. void *dev_base_addr;
  1495. int i;
  1496. ring_id = hal_get_srng_ring_id(hal_soc, ring_type, ring_num, mac_id);
  1497. if (ring_id < 0)
  1498. return NULL;
  1499. hal_verbose_debug("mac_id %d ring_id %d", mac_id, ring_id);
  1500. srng = hal_get_srng(hal_soc, ring_id);
  1501. if (srng->initialized) {
  1502. hal_verbose_debug("Ring (ring_type, ring_num) already initialized");
  1503. return NULL;
  1504. }
  1505. dev_base_addr = hal->dev_base_addr;
  1506. srng->ring_id = ring_id;
  1507. srng->ring_type = ring_type;
  1508. srng->ring_dir = ring_config->ring_dir;
  1509. srng->ring_base_paddr = ring_params->ring_base_paddr;
  1510. srng->ring_base_vaddr = ring_params->ring_base_vaddr;
  1511. srng->entry_size = ring_config->entry_size;
  1512. srng->num_entries = ring_params->num_entries;
  1513. srng->ring_size = srng->num_entries * srng->entry_size;
  1514. srng->ring_size_mask = srng->ring_size - 1;
  1515. srng->msi_addr = ring_params->msi_addr;
  1516. srng->msi_data = ring_params->msi_data;
  1517. srng->intr_timer_thres_us = ring_params->intr_timer_thres_us;
  1518. srng->intr_batch_cntr_thres_entries =
  1519. ring_params->intr_batch_cntr_thres_entries;
  1520. srng->prefetch_timer = ring_params->prefetch_timer;
  1521. srng->hal_soc = hal_soc;
  1522. for (i = 0 ; i < MAX_SRNG_REG_GROUPS; i++) {
  1523. srng->hwreg_base[i] = dev_base_addr + ring_config->reg_start[i]
  1524. + (ring_num * ring_config->reg_size[i]);
  1525. }
  1526. /* Zero out the entire ring memory */
  1527. qdf_mem_zero(srng->ring_base_vaddr, (srng->entry_size *
  1528. srng->num_entries) << 2);
  1529. srng->flags = ring_params->flags;
  1530. #ifdef BIG_ENDIAN_HOST
  1531. /* TODO: See if we should we get these flags from caller */
  1532. srng->flags |= HAL_SRNG_DATA_TLV_SWAP;
  1533. srng->flags |= HAL_SRNG_MSI_SWAP;
  1534. srng->flags |= HAL_SRNG_RING_PTR_SWAP;
  1535. #endif
  1536. if (srng->ring_dir == HAL_SRNG_SRC_RING) {
  1537. srng->u.src_ring.hp = 0;
  1538. srng->u.src_ring.reap_hp = srng->ring_size -
  1539. srng->entry_size;
  1540. srng->u.src_ring.tp_addr =
  1541. &(hal->shadow_rdptr_mem_vaddr[ring_id]);
  1542. srng->u.src_ring.low_threshold =
  1543. ring_params->low_threshold * srng->entry_size;
  1544. if (ring_config->lmac_ring) {
  1545. /* For LMAC rings, head pointer updates will be done
  1546. * through FW by writing to a shared memory location
  1547. */
  1548. srng->u.src_ring.hp_addr =
  1549. &(hal->shadow_wrptr_mem_vaddr[ring_id -
  1550. HAL_SRNG_LMAC1_ID_START]);
  1551. srng->flags |= HAL_SRNG_LMAC_RING;
  1552. } else if (ignore_shadow || (srng->u.src_ring.hp_addr == 0)) {
  1553. srng->u.src_ring.hp_addr =
  1554. hal_get_window_address(hal,
  1555. SRNG_SRC_ADDR(srng, HP));
  1556. if (CHECK_SHADOW_REGISTERS) {
  1557. QDF_TRACE(QDF_MODULE_ID_TXRX,
  1558. QDF_TRACE_LEVEL_ERROR,
  1559. "%s: Ring (%d, %d) missing shadow config",
  1560. __func__, ring_type, ring_num);
  1561. }
  1562. } else {
  1563. hal_validate_shadow_register(hal,
  1564. SRNG_SRC_ADDR(srng, HP),
  1565. srng->u.src_ring.hp_addr);
  1566. }
  1567. } else {
  1568. /* During initialization loop count in all the descriptors
  1569. * will be set to zero, and HW will set it to 1 on completing
  1570. * descriptor update in first loop, and increments it by 1 on
  1571. * subsequent loops (loop count wraps around after reaching
  1572. * 0xffff). The 'loop_cnt' in SW ring state is the expected
  1573. * loop count in descriptors updated by HW (to be processed
  1574. * by SW).
  1575. */
  1576. srng->u.dst_ring.loop_cnt = 1;
  1577. srng->u.dst_ring.tp = 0;
  1578. srng->u.dst_ring.hp_addr =
  1579. &(hal->shadow_rdptr_mem_vaddr[ring_id]);
  1580. if (ring_config->lmac_ring) {
  1581. /* For LMAC rings, tail pointer updates will be done
  1582. * through FW by writing to a shared memory location
  1583. */
  1584. srng->u.dst_ring.tp_addr =
  1585. &(hal->shadow_wrptr_mem_vaddr[ring_id -
  1586. HAL_SRNG_LMAC1_ID_START]);
  1587. srng->flags |= HAL_SRNG_LMAC_RING;
  1588. } else if (ignore_shadow || srng->u.dst_ring.tp_addr == 0) {
  1589. srng->u.dst_ring.tp_addr =
  1590. hal_get_window_address(hal,
  1591. SRNG_DST_ADDR(srng, TP));
  1592. if (CHECK_SHADOW_REGISTERS) {
  1593. QDF_TRACE(QDF_MODULE_ID_TXRX,
  1594. QDF_TRACE_LEVEL_ERROR,
  1595. "%s: Ring (%d, %d) missing shadow config",
  1596. __func__, ring_type, ring_num);
  1597. }
  1598. } else {
  1599. hal_validate_shadow_register(hal,
  1600. SRNG_DST_ADDR(srng, TP),
  1601. srng->u.dst_ring.tp_addr);
  1602. }
  1603. }
  1604. if (!(ring_config->lmac_ring)) {
  1605. hal_srng_hw_init(hal, srng);
  1606. if (ring_type == CE_DST) {
  1607. srng->u.dst_ring.max_buffer_length = ring_params->max_buffer_length;
  1608. hal_ce_dst_setup(hal, srng, ring_num);
  1609. }
  1610. }
  1611. SRNG_LOCK_INIT(&srng->lock);
  1612. srng->srng_event = 0;
  1613. srng->initialized = true;
  1614. return (void *)srng;
  1615. }
  1616. qdf_export_symbol(hal_srng_setup);
  1617. /**
  1618. * hal_srng_cleanup - Deinitialize HW SRNG ring.
  1619. * @hal_soc: Opaque HAL SOC handle
  1620. * @hal_srng: Opaque HAL SRNG pointer
  1621. */
  1622. void hal_srng_cleanup(void *hal_soc, hal_ring_handle_t hal_ring_hdl)
  1623. {
  1624. struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl;
  1625. SRNG_LOCK_DESTROY(&srng->lock);
  1626. srng->initialized = 0;
  1627. }
  1628. qdf_export_symbol(hal_srng_cleanup);
  1629. /**
  1630. * hal_srng_get_entrysize - Returns size of ring entry in bytes
  1631. * @hal_soc: Opaque HAL SOC handle
  1632. * @ring_type: one of the types from hal_ring_type
  1633. *
  1634. */
  1635. uint32_t hal_srng_get_entrysize(void *hal_soc, int ring_type)
  1636. {
  1637. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  1638. struct hal_hw_srng_config *ring_config =
  1639. HAL_SRNG_CONFIG(hal, ring_type);
  1640. return ring_config->entry_size << 2;
  1641. }
  1642. qdf_export_symbol(hal_srng_get_entrysize);
  1643. /**
  1644. * hal_srng_max_entries - Returns maximum possible number of ring entries
  1645. * @hal_soc: Opaque HAL SOC handle
  1646. * @ring_type: one of the types from hal_ring_type
  1647. *
  1648. * Return: Maximum number of entries for the given ring_type
  1649. */
  1650. uint32_t hal_srng_max_entries(void *hal_soc, int ring_type)
  1651. {
  1652. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  1653. struct hal_hw_srng_config *ring_config =
  1654. HAL_SRNG_CONFIG(hal, ring_type);
  1655. return ring_config->max_size / ring_config->entry_size;
  1656. }
  1657. qdf_export_symbol(hal_srng_max_entries);
  1658. enum hal_srng_dir hal_srng_get_dir(void *hal_soc, int ring_type)
  1659. {
  1660. struct hal_soc *hal = (struct hal_soc *)hal_soc;
  1661. struct hal_hw_srng_config *ring_config =
  1662. HAL_SRNG_CONFIG(hal, ring_type);
  1663. return ring_config->ring_dir;
  1664. }
  1665. /**
  1666. * hal_srng_dump - Dump ring status
  1667. * @srng: hal srng pointer
  1668. */
  1669. void hal_srng_dump(struct hal_srng *srng)
  1670. {
  1671. if (srng->ring_dir == HAL_SRNG_SRC_RING) {
  1672. hal_debug("=== SRC RING %d ===", srng->ring_id);
  1673. hal_debug("hp %u, reap_hp %u, tp %u, cached tp %u",
  1674. srng->u.src_ring.hp,
  1675. srng->u.src_ring.reap_hp,
  1676. *srng->u.src_ring.tp_addr,
  1677. srng->u.src_ring.cached_tp);
  1678. } else {
  1679. hal_debug("=== DST RING %d ===", srng->ring_id);
  1680. hal_debug("tp %u, hp %u, cached tp %u, loop_cnt %u",
  1681. srng->u.dst_ring.tp,
  1682. *srng->u.dst_ring.hp_addr,
  1683. srng->u.dst_ring.cached_hp,
  1684. srng->u.dst_ring.loop_cnt);
  1685. }
  1686. }
  1687. /**
  1688. * hal_get_srng_params - Retrieve SRNG parameters for a given ring from HAL
  1689. *
  1690. * @hal_soc: Opaque HAL SOC handle
  1691. * @hal_ring: Ring pointer (Source or Destination ring)
  1692. * @ring_params: SRNG parameters will be returned through this structure
  1693. */
  1694. extern void hal_get_srng_params(hal_soc_handle_t hal_soc_hdl,
  1695. hal_ring_handle_t hal_ring_hdl,
  1696. struct hal_srng_params *ring_params)
  1697. {
  1698. struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl;
  1699. int i =0;
  1700. ring_params->ring_id = srng->ring_id;
  1701. ring_params->ring_dir = srng->ring_dir;
  1702. ring_params->entry_size = srng->entry_size;
  1703. ring_params->ring_base_paddr = srng->ring_base_paddr;
  1704. ring_params->ring_base_vaddr = srng->ring_base_vaddr;
  1705. ring_params->num_entries = srng->num_entries;
  1706. ring_params->msi_addr = srng->msi_addr;
  1707. ring_params->msi_data = srng->msi_data;
  1708. ring_params->intr_timer_thres_us = srng->intr_timer_thres_us;
  1709. ring_params->intr_batch_cntr_thres_entries =
  1710. srng->intr_batch_cntr_thres_entries;
  1711. ring_params->low_threshold = srng->u.src_ring.low_threshold;
  1712. ring_params->flags = srng->flags;
  1713. ring_params->ring_id = srng->ring_id;
  1714. for (i = 0 ; i < MAX_SRNG_REG_GROUPS; i++)
  1715. ring_params->hwreg_base[i] = srng->hwreg_base[i];
  1716. }
  1717. qdf_export_symbol(hal_get_srng_params);
  1718. void hal_set_low_threshold(hal_ring_handle_t hal_ring_hdl,
  1719. uint32_t low_threshold)
  1720. {
  1721. struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl;
  1722. srng->u.src_ring.low_threshold = low_threshold * srng->entry_size;
  1723. }
  1724. qdf_export_symbol(hal_set_low_threshold);
  1725. #ifdef FORCE_WAKE
  1726. void hal_set_init_phase(hal_soc_handle_t soc, bool init_phase)
  1727. {
  1728. struct hal_soc *hal_soc = (struct hal_soc *)soc;
  1729. hal_soc->init_phase = init_phase;
  1730. }
  1731. #endif /* FORCE_WAKE */