hns3_pmu.c 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * This driver adds support for HNS3 PMU iEP device. Related perf events are
  4. * bandwidth, latency, packet rate, interrupt rate etc.
  5. *
  6. * Copyright (C) 2022 HiSilicon Limited
  7. */
  8. #include <linux/bitfield.h>
  9. #include <linux/bitmap.h>
  10. #include <linux/bug.h>
  11. #include <linux/cpuhotplug.h>
  12. #include <linux/cpumask.h>
  13. #include <linux/delay.h>
  14. #include <linux/device.h>
  15. #include <linux/err.h>
  16. #include <linux/interrupt.h>
  17. #include <linux/iopoll.h>
  18. #include <linux/io-64-nonatomic-hi-lo.h>
  19. #include <linux/irq.h>
  20. #include <linux/kernel.h>
  21. #include <linux/list.h>
  22. #include <linux/module.h>
  23. #include <linux/pci.h>
  24. #include <linux/pci-epf.h>
  25. #include <linux/perf_event.h>
  26. #include <linux/smp.h>
  27. /* registers offset address */
  28. #define HNS3_PMU_REG_GLOBAL_CTRL 0x0000
  29. #define HNS3_PMU_REG_CLOCK_FREQ 0x0020
  30. #define HNS3_PMU_REG_BDF 0x0fe0
  31. #define HNS3_PMU_REG_VERSION 0x0fe4
  32. #define HNS3_PMU_REG_DEVICE_ID 0x0fe8
  33. #define HNS3_PMU_REG_EVENT_OFFSET 0x1000
  34. #define HNS3_PMU_REG_EVENT_SIZE 0x1000
  35. #define HNS3_PMU_REG_EVENT_CTRL_LOW 0x00
  36. #define HNS3_PMU_REG_EVENT_CTRL_HIGH 0x04
  37. #define HNS3_PMU_REG_EVENT_INTR_STATUS 0x08
  38. #define HNS3_PMU_REG_EVENT_INTR_MASK 0x0c
  39. #define HNS3_PMU_REG_EVENT_COUNTER 0x10
  40. #define HNS3_PMU_REG_EVENT_EXT_COUNTER 0x18
  41. #define HNS3_PMU_REG_EVENT_QID_CTRL 0x28
  42. #define HNS3_PMU_REG_EVENT_QID_PARA 0x2c
  43. #define HNS3_PMU_FILTER_SUPPORT_GLOBAL BIT(0)
  44. #define HNS3_PMU_FILTER_SUPPORT_PORT BIT(1)
  45. #define HNS3_PMU_FILTER_SUPPORT_PORT_TC BIT(2)
  46. #define HNS3_PMU_FILTER_SUPPORT_FUNC BIT(3)
  47. #define HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE BIT(4)
  48. #define HNS3_PMU_FILTER_SUPPORT_FUNC_INTR BIT(5)
  49. #define HNS3_PMU_FILTER_ALL_TC 0xf
  50. #define HNS3_PMU_FILTER_ALL_QUEUE 0xffff
  51. #define HNS3_PMU_CTRL_SUBEVENT_S 4
  52. #define HNS3_PMU_CTRL_FILTER_MODE_S 24
  53. #define HNS3_PMU_GLOBAL_START BIT(0)
  54. #define HNS3_PMU_EVENT_STATUS_RESET BIT(11)
  55. #define HNS3_PMU_EVENT_EN BIT(12)
  56. #define HNS3_PMU_EVENT_OVERFLOW_RESTART BIT(15)
  57. #define HNS3_PMU_QID_PARA_FUNC_S 0
  58. #define HNS3_PMU_QID_PARA_QUEUE_S 16
  59. #define HNS3_PMU_QID_CTRL_REQ_ENABLE BIT(0)
  60. #define HNS3_PMU_QID_CTRL_DONE BIT(1)
  61. #define HNS3_PMU_QID_CTRL_MISS BIT(2)
  62. #define HNS3_PMU_INTR_MASK_OVERFLOW BIT(1)
  63. #define HNS3_PMU_MAX_HW_EVENTS 8
  64. /*
  65. * Each hardware event contains two registers (counter and ext_counter) for
  66. * bandwidth, packet rate, latency and interrupt rate. These two registers will
  67. * be triggered to run at the same when a hardware event is enabled. The meaning
  68. * of counter and ext_counter of different event type are different, their
  69. * meaning show as follow:
  70. *
  71. * +----------------+------------------+---------------+
  72. * | event type | counter | ext_counter |
  73. * +----------------+------------------+---------------+
  74. * | bandwidth | byte number | cycle number |
  75. * +----------------+------------------+---------------+
  76. * | packet rate | packet number | cycle number |
  77. * +----------------+------------------+---------------+
  78. * | latency | cycle number | packet number |
  79. * +----------------+------------------+---------------+
  80. * | interrupt rate | interrupt number | cycle number |
  81. * +----------------+------------------+---------------+
  82. *
  83. * The cycle number indicates increment of counter of hardware timer, the
  84. * frequency of hardware timer can be read from hw_clk_freq file.
  85. *
  86. * Performance of each hardware event is calculated by: counter / ext_counter.
  87. *
  88. * Since processing of data is preferred to be done in userspace, we expose
  89. * ext_counter as a separate event for userspace and use bit 16 to indicate it.
  90. * For example, event 0x00001 and 0x10001 are actually one event for hardware
  91. * because bit 0-15 are same. If the bit 16 of one event is 0 means to read
  92. * counter register, otherwise means to read ext_counter register.
  93. */
  94. /* bandwidth events */
  95. #define HNS3_PMU_EVT_BW_SSU_EGU_BYTE_NUM 0x00001
  96. #define HNS3_PMU_EVT_BW_SSU_EGU_TIME 0x10001
  97. #define HNS3_PMU_EVT_BW_SSU_RPU_BYTE_NUM 0x00002
  98. #define HNS3_PMU_EVT_BW_SSU_RPU_TIME 0x10002
  99. #define HNS3_PMU_EVT_BW_SSU_ROCE_BYTE_NUM 0x00003
  100. #define HNS3_PMU_EVT_BW_SSU_ROCE_TIME 0x10003
  101. #define HNS3_PMU_EVT_BW_ROCE_SSU_BYTE_NUM 0x00004
  102. #define HNS3_PMU_EVT_BW_ROCE_SSU_TIME 0x10004
  103. #define HNS3_PMU_EVT_BW_TPU_SSU_BYTE_NUM 0x00005
  104. #define HNS3_PMU_EVT_BW_TPU_SSU_TIME 0x10005
  105. #define HNS3_PMU_EVT_BW_RPU_RCBRX_BYTE_NUM 0x00006
  106. #define HNS3_PMU_EVT_BW_RPU_RCBRX_TIME 0x10006
  107. #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_BYTE_NUM 0x00008
  108. #define HNS3_PMU_EVT_BW_RCBTX_TXSCH_TIME 0x10008
  109. #define HNS3_PMU_EVT_BW_WR_FBD_BYTE_NUM 0x00009
  110. #define HNS3_PMU_EVT_BW_WR_FBD_TIME 0x10009
  111. #define HNS3_PMU_EVT_BW_WR_EBD_BYTE_NUM 0x0000a
  112. #define HNS3_PMU_EVT_BW_WR_EBD_TIME 0x1000a
  113. #define HNS3_PMU_EVT_BW_RD_FBD_BYTE_NUM 0x0000b
  114. #define HNS3_PMU_EVT_BW_RD_FBD_TIME 0x1000b
  115. #define HNS3_PMU_EVT_BW_RD_EBD_BYTE_NUM 0x0000c
  116. #define HNS3_PMU_EVT_BW_RD_EBD_TIME 0x1000c
  117. #define HNS3_PMU_EVT_BW_RD_PAY_M0_BYTE_NUM 0x0000d
  118. #define HNS3_PMU_EVT_BW_RD_PAY_M0_TIME 0x1000d
  119. #define HNS3_PMU_EVT_BW_RD_PAY_M1_BYTE_NUM 0x0000e
  120. #define HNS3_PMU_EVT_BW_RD_PAY_M1_TIME 0x1000e
  121. #define HNS3_PMU_EVT_BW_WR_PAY_M0_BYTE_NUM 0x0000f
  122. #define HNS3_PMU_EVT_BW_WR_PAY_M0_TIME 0x1000f
  123. #define HNS3_PMU_EVT_BW_WR_PAY_M1_BYTE_NUM 0x00010
  124. #define HNS3_PMU_EVT_BW_WR_PAY_M1_TIME 0x10010
  125. /* packet rate events */
  126. #define HNS3_PMU_EVT_PPS_IGU_SSU_PACKET_NUM 0x00100
  127. #define HNS3_PMU_EVT_PPS_IGU_SSU_TIME 0x10100
  128. #define HNS3_PMU_EVT_PPS_SSU_EGU_PACKET_NUM 0x00101
  129. #define HNS3_PMU_EVT_PPS_SSU_EGU_TIME 0x10101
  130. #define HNS3_PMU_EVT_PPS_SSU_RPU_PACKET_NUM 0x00102
  131. #define HNS3_PMU_EVT_PPS_SSU_RPU_TIME 0x10102
  132. #define HNS3_PMU_EVT_PPS_SSU_ROCE_PACKET_NUM 0x00103
  133. #define HNS3_PMU_EVT_PPS_SSU_ROCE_TIME 0x10103
  134. #define HNS3_PMU_EVT_PPS_ROCE_SSU_PACKET_NUM 0x00104
  135. #define HNS3_PMU_EVT_PPS_ROCE_SSU_TIME 0x10104
  136. #define HNS3_PMU_EVT_PPS_TPU_SSU_PACKET_NUM 0x00105
  137. #define HNS3_PMU_EVT_PPS_TPU_SSU_TIME 0x10105
  138. #define HNS3_PMU_EVT_PPS_RPU_RCBRX_PACKET_NUM 0x00106
  139. #define HNS3_PMU_EVT_PPS_RPU_RCBRX_TIME 0x10106
  140. #define HNS3_PMU_EVT_PPS_RCBTX_TPU_PACKET_NUM 0x00107
  141. #define HNS3_PMU_EVT_PPS_RCBTX_TPU_TIME 0x10107
  142. #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_PACKET_NUM 0x00108
  143. #define HNS3_PMU_EVT_PPS_RCBTX_TXSCH_TIME 0x10108
  144. #define HNS3_PMU_EVT_PPS_WR_FBD_PACKET_NUM 0x00109
  145. #define HNS3_PMU_EVT_PPS_WR_FBD_TIME 0x10109
  146. #define HNS3_PMU_EVT_PPS_WR_EBD_PACKET_NUM 0x0010a
  147. #define HNS3_PMU_EVT_PPS_WR_EBD_TIME 0x1010a
  148. #define HNS3_PMU_EVT_PPS_RD_FBD_PACKET_NUM 0x0010b
  149. #define HNS3_PMU_EVT_PPS_RD_FBD_TIME 0x1010b
  150. #define HNS3_PMU_EVT_PPS_RD_EBD_PACKET_NUM 0x0010c
  151. #define HNS3_PMU_EVT_PPS_RD_EBD_TIME 0x1010c
  152. #define HNS3_PMU_EVT_PPS_RD_PAY_M0_PACKET_NUM 0x0010d
  153. #define HNS3_PMU_EVT_PPS_RD_PAY_M0_TIME 0x1010d
  154. #define HNS3_PMU_EVT_PPS_RD_PAY_M1_PACKET_NUM 0x0010e
  155. #define HNS3_PMU_EVT_PPS_RD_PAY_M1_TIME 0x1010e
  156. #define HNS3_PMU_EVT_PPS_WR_PAY_M0_PACKET_NUM 0x0010f
  157. #define HNS3_PMU_EVT_PPS_WR_PAY_M0_TIME 0x1010f
  158. #define HNS3_PMU_EVT_PPS_WR_PAY_M1_PACKET_NUM 0x00110
  159. #define HNS3_PMU_EVT_PPS_WR_PAY_M1_TIME 0x10110
  160. #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_PACKET_NUM 0x00111
  161. #define HNS3_PMU_EVT_PPS_NICROH_TX_PRE_TIME 0x10111
  162. #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_PACKET_NUM 0x00112
  163. #define HNS3_PMU_EVT_PPS_NICROH_RX_PRE_TIME 0x10112
  164. /* latency events */
  165. #define HNS3_PMU_EVT_DLY_TX_PUSH_TIME 0x00202
  166. #define HNS3_PMU_EVT_DLY_TX_PUSH_PACKET_NUM 0x10202
  167. #define HNS3_PMU_EVT_DLY_TX_TIME 0x00204
  168. #define HNS3_PMU_EVT_DLY_TX_PACKET_NUM 0x10204
  169. #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_TIME 0x00206
  170. #define HNS3_PMU_EVT_DLY_SSU_TX_NIC_PACKET_NUM 0x10206
  171. #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_TIME 0x00207
  172. #define HNS3_PMU_EVT_DLY_SSU_TX_ROCE_PACKET_NUM 0x10207
  173. #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_TIME 0x00208
  174. #define HNS3_PMU_EVT_DLY_SSU_RX_NIC_PACKET_NUM 0x10208
  175. #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_TIME 0x00209
  176. #define HNS3_PMU_EVT_DLY_SSU_RX_ROCE_PACKET_NUM 0x10209
  177. #define HNS3_PMU_EVT_DLY_RPU_TIME 0x0020e
  178. #define HNS3_PMU_EVT_DLY_RPU_PACKET_NUM 0x1020e
  179. #define HNS3_PMU_EVT_DLY_TPU_TIME 0x0020f
  180. #define HNS3_PMU_EVT_DLY_TPU_PACKET_NUM 0x1020f
  181. #define HNS3_PMU_EVT_DLY_RPE_TIME 0x00210
  182. #define HNS3_PMU_EVT_DLY_RPE_PACKET_NUM 0x10210
  183. #define HNS3_PMU_EVT_DLY_TPE_TIME 0x00211
  184. #define HNS3_PMU_EVT_DLY_TPE_PACKET_NUM 0x10211
  185. #define HNS3_PMU_EVT_DLY_TPE_PUSH_TIME 0x00212
  186. #define HNS3_PMU_EVT_DLY_TPE_PUSH_PACKET_NUM 0x10212
  187. #define HNS3_PMU_EVT_DLY_WR_FBD_TIME 0x00213
  188. #define HNS3_PMU_EVT_DLY_WR_FBD_PACKET_NUM 0x10213
  189. #define HNS3_PMU_EVT_DLY_WR_EBD_TIME 0x00214
  190. #define HNS3_PMU_EVT_DLY_WR_EBD_PACKET_NUM 0x10214
  191. #define HNS3_PMU_EVT_DLY_RD_FBD_TIME 0x00215
  192. #define HNS3_PMU_EVT_DLY_RD_FBD_PACKET_NUM 0x10215
  193. #define HNS3_PMU_EVT_DLY_RD_EBD_TIME 0x00216
  194. #define HNS3_PMU_EVT_DLY_RD_EBD_PACKET_NUM 0x10216
  195. #define HNS3_PMU_EVT_DLY_RD_PAY_M0_TIME 0x00217
  196. #define HNS3_PMU_EVT_DLY_RD_PAY_M0_PACKET_NUM 0x10217
  197. #define HNS3_PMU_EVT_DLY_RD_PAY_M1_TIME 0x00218
  198. #define HNS3_PMU_EVT_DLY_RD_PAY_M1_PACKET_NUM 0x10218
  199. #define HNS3_PMU_EVT_DLY_WR_PAY_M0_TIME 0x00219
  200. #define HNS3_PMU_EVT_DLY_WR_PAY_M0_PACKET_NUM 0x10219
  201. #define HNS3_PMU_EVT_DLY_WR_PAY_M1_TIME 0x0021a
  202. #define HNS3_PMU_EVT_DLY_WR_PAY_M1_PACKET_NUM 0x1021a
  203. #define HNS3_PMU_EVT_DLY_MSIX_WRITE_TIME 0x0021c
  204. #define HNS3_PMU_EVT_DLY_MSIX_WRITE_PACKET_NUM 0x1021c
  205. /* interrupt rate events */
  206. #define HNS3_PMU_EVT_PPS_MSIX_NIC_INTR_NUM 0x00300
  207. #define HNS3_PMU_EVT_PPS_MSIX_NIC_TIME 0x10300
  208. /* filter mode supported by each bandwidth event */
  209. #define HNS3_PMU_FILTER_BW_SSU_EGU 0x07
  210. #define HNS3_PMU_FILTER_BW_SSU_RPU 0x1f
  211. #define HNS3_PMU_FILTER_BW_SSU_ROCE 0x0f
  212. #define HNS3_PMU_FILTER_BW_ROCE_SSU 0x0f
  213. #define HNS3_PMU_FILTER_BW_TPU_SSU 0x1f
  214. #define HNS3_PMU_FILTER_BW_RPU_RCBRX 0x11
  215. #define HNS3_PMU_FILTER_BW_RCBTX_TXSCH 0x11
  216. #define HNS3_PMU_FILTER_BW_WR_FBD 0x1b
  217. #define HNS3_PMU_FILTER_BW_WR_EBD 0x11
  218. #define HNS3_PMU_FILTER_BW_RD_FBD 0x01
  219. #define HNS3_PMU_FILTER_BW_RD_EBD 0x1b
  220. #define HNS3_PMU_FILTER_BW_RD_PAY_M0 0x01
  221. #define HNS3_PMU_FILTER_BW_RD_PAY_M1 0x01
  222. #define HNS3_PMU_FILTER_BW_WR_PAY_M0 0x01
  223. #define HNS3_PMU_FILTER_BW_WR_PAY_M1 0x01
  224. /* filter mode supported by each packet rate event */
  225. #define HNS3_PMU_FILTER_PPS_IGU_SSU 0x07
  226. #define HNS3_PMU_FILTER_PPS_SSU_EGU 0x07
  227. #define HNS3_PMU_FILTER_PPS_SSU_RPU 0x1f
  228. #define HNS3_PMU_FILTER_PPS_SSU_ROCE 0x0f
  229. #define HNS3_PMU_FILTER_PPS_ROCE_SSU 0x0f
  230. #define HNS3_PMU_FILTER_PPS_TPU_SSU 0x1f
  231. #define HNS3_PMU_FILTER_PPS_RPU_RCBRX 0x11
  232. #define HNS3_PMU_FILTER_PPS_RCBTX_TPU 0x1f
  233. #define HNS3_PMU_FILTER_PPS_RCBTX_TXSCH 0x11
  234. #define HNS3_PMU_FILTER_PPS_WR_FBD 0x1b
  235. #define HNS3_PMU_FILTER_PPS_WR_EBD 0x11
  236. #define HNS3_PMU_FILTER_PPS_RD_FBD 0x01
  237. #define HNS3_PMU_FILTER_PPS_RD_EBD 0x1b
  238. #define HNS3_PMU_FILTER_PPS_RD_PAY_M0 0x01
  239. #define HNS3_PMU_FILTER_PPS_RD_PAY_M1 0x01
  240. #define HNS3_PMU_FILTER_PPS_WR_PAY_M0 0x01
  241. #define HNS3_PMU_FILTER_PPS_WR_PAY_M1 0x01
  242. #define HNS3_PMU_FILTER_PPS_NICROH_TX_PRE 0x01
  243. #define HNS3_PMU_FILTER_PPS_NICROH_RX_PRE 0x01
  244. /* filter mode supported by each latency event */
  245. #define HNS3_PMU_FILTER_DLY_TX_PUSH 0x01
  246. #define HNS3_PMU_FILTER_DLY_TX 0x01
  247. #define HNS3_PMU_FILTER_DLY_SSU_TX_NIC 0x07
  248. #define HNS3_PMU_FILTER_DLY_SSU_TX_ROCE 0x07
  249. #define HNS3_PMU_FILTER_DLY_SSU_RX_NIC 0x07
  250. #define HNS3_PMU_FILTER_DLY_SSU_RX_ROCE 0x07
  251. #define HNS3_PMU_FILTER_DLY_RPU 0x11
  252. #define HNS3_PMU_FILTER_DLY_TPU 0x1f
  253. #define HNS3_PMU_FILTER_DLY_RPE 0x01
  254. #define HNS3_PMU_FILTER_DLY_TPE 0x0b
  255. #define HNS3_PMU_FILTER_DLY_TPE_PUSH 0x1b
  256. #define HNS3_PMU_FILTER_DLY_WR_FBD 0x1b
  257. #define HNS3_PMU_FILTER_DLY_WR_EBD 0x11
  258. #define HNS3_PMU_FILTER_DLY_RD_FBD 0x01
  259. #define HNS3_PMU_FILTER_DLY_RD_EBD 0x1b
  260. #define HNS3_PMU_FILTER_DLY_RD_PAY_M0 0x01
  261. #define HNS3_PMU_FILTER_DLY_RD_PAY_M1 0x01
  262. #define HNS3_PMU_FILTER_DLY_WR_PAY_M0 0x01
  263. #define HNS3_PMU_FILTER_DLY_WR_PAY_M1 0x01
  264. #define HNS3_PMU_FILTER_DLY_MSIX_WRITE 0x01
  265. /* filter mode supported by each interrupt rate event */
  266. #define HNS3_PMU_FILTER_INTR_MSIX_NIC 0x01
  267. enum hns3_pmu_hw_filter_mode {
  268. HNS3_PMU_HW_FILTER_GLOBAL,
  269. HNS3_PMU_HW_FILTER_PORT,
  270. HNS3_PMU_HW_FILTER_PORT_TC,
  271. HNS3_PMU_HW_FILTER_FUNC,
  272. HNS3_PMU_HW_FILTER_FUNC_QUEUE,
  273. HNS3_PMU_HW_FILTER_FUNC_INTR,
  274. };
  275. struct hns3_pmu_event_attr {
  276. u32 event;
  277. u16 filter_support;
  278. };
  279. struct hns3_pmu {
  280. struct perf_event *hw_events[HNS3_PMU_MAX_HW_EVENTS];
  281. struct hlist_node node;
  282. struct pci_dev *pdev;
  283. struct pmu pmu;
  284. void __iomem *base;
  285. int irq;
  286. int on_cpu;
  287. u32 identifier;
  288. u32 hw_clk_freq; /* hardware clock frequency of PMU */
  289. /* maximum and minimum bdf allowed by PMU */
  290. u16 bdf_min;
  291. u16 bdf_max;
  292. };
  293. #define to_hns3_pmu(p) (container_of((p), struct hns3_pmu, pmu))
  294. #define GET_PCI_DEVFN(bdf) ((bdf) & 0xff)
  295. #define FILTER_CONDITION_PORT(port) ((1 << (port)) & 0xff)
  296. #define FILTER_CONDITION_PORT_TC(port, tc) (((port) << 3) | ((tc) & 0x07))
  297. #define FILTER_CONDITION_FUNC_INTR(func, intr) (((intr) << 8) | (func))
  298. #define HNS3_PMU_FILTER_ATTR(_name, _config, _start, _end) \
  299. static inline u64 hns3_pmu_get_##_name(struct perf_event *event) \
  300. { \
  301. return FIELD_GET(GENMASK_ULL(_end, _start), \
  302. event->attr._config); \
  303. }
  304. HNS3_PMU_FILTER_ATTR(subevent, config, 0, 7);
  305. HNS3_PMU_FILTER_ATTR(event_type, config, 8, 15);
  306. HNS3_PMU_FILTER_ATTR(ext_counter_used, config, 16, 16);
  307. HNS3_PMU_FILTER_ATTR(port, config1, 0, 3);
  308. HNS3_PMU_FILTER_ATTR(tc, config1, 4, 7);
  309. HNS3_PMU_FILTER_ATTR(bdf, config1, 8, 23);
  310. HNS3_PMU_FILTER_ATTR(queue, config1, 24, 39);
  311. HNS3_PMU_FILTER_ATTR(intr, config1, 40, 51);
  312. HNS3_PMU_FILTER_ATTR(global, config1, 52, 52);
  313. #define HNS3_BW_EVT_BYTE_NUM(_name) (&(struct hns3_pmu_event_attr) {\
  314. HNS3_PMU_EVT_BW_##_name##_BYTE_NUM, \
  315. HNS3_PMU_FILTER_BW_##_name})
  316. #define HNS3_BW_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
  317. HNS3_PMU_EVT_BW_##_name##_TIME, \
  318. HNS3_PMU_FILTER_BW_##_name})
  319. #define HNS3_PPS_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\
  320. HNS3_PMU_EVT_PPS_##_name##_PACKET_NUM, \
  321. HNS3_PMU_FILTER_PPS_##_name})
  322. #define HNS3_PPS_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
  323. HNS3_PMU_EVT_PPS_##_name##_TIME, \
  324. HNS3_PMU_FILTER_PPS_##_name})
  325. #define HNS3_DLY_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
  326. HNS3_PMU_EVT_DLY_##_name##_TIME, \
  327. HNS3_PMU_FILTER_DLY_##_name})
  328. #define HNS3_DLY_EVT_PACKET_NUM(_name) (&(struct hns3_pmu_event_attr) {\
  329. HNS3_PMU_EVT_DLY_##_name##_PACKET_NUM, \
  330. HNS3_PMU_FILTER_DLY_##_name})
  331. #define HNS3_INTR_EVT_INTR_NUM(_name) (&(struct hns3_pmu_event_attr) {\
  332. HNS3_PMU_EVT_PPS_##_name##_INTR_NUM, \
  333. HNS3_PMU_FILTER_INTR_##_name})
  334. #define HNS3_INTR_EVT_TIME(_name) (&(struct hns3_pmu_event_attr) {\
  335. HNS3_PMU_EVT_PPS_##_name##_TIME, \
  336. HNS3_PMU_FILTER_INTR_##_name})
  337. static ssize_t hns3_pmu_format_show(struct device *dev,
  338. struct device_attribute *attr, char *buf)
  339. {
  340. struct dev_ext_attribute *eattr;
  341. eattr = container_of(attr, struct dev_ext_attribute, attr);
  342. return sysfs_emit(buf, "%s\n", (char *)eattr->var);
  343. }
  344. static ssize_t hns3_pmu_event_show(struct device *dev,
  345. struct device_attribute *attr, char *buf)
  346. {
  347. struct hns3_pmu_event_attr *event;
  348. struct dev_ext_attribute *eattr;
  349. eattr = container_of(attr, struct dev_ext_attribute, attr);
  350. event = eattr->var;
  351. return sysfs_emit(buf, "config=0x%x\n", event->event);
  352. }
  353. static ssize_t hns3_pmu_filter_mode_show(struct device *dev,
  354. struct device_attribute *attr,
  355. char *buf)
  356. {
  357. struct hns3_pmu_event_attr *event;
  358. struct dev_ext_attribute *eattr;
  359. int len;
  360. eattr = container_of(attr, struct dev_ext_attribute, attr);
  361. event = eattr->var;
  362. len = sysfs_emit_at(buf, 0, "filter mode supported: ");
  363. if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL)
  364. len += sysfs_emit_at(buf, len, "global ");
  365. if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT)
  366. len += sysfs_emit_at(buf, len, "port ");
  367. if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC)
  368. len += sysfs_emit_at(buf, len, "port-tc ");
  369. if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC)
  370. len += sysfs_emit_at(buf, len, "func ");
  371. if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE)
  372. len += sysfs_emit_at(buf, len, "func-queue ");
  373. if (event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR)
  374. len += sysfs_emit_at(buf, len, "func-intr ");
  375. len += sysfs_emit_at(buf, len, "\n");
  376. return len;
  377. }
  378. #define HNS3_PMU_ATTR(_name, _func, _config) \
  379. (&((struct dev_ext_attribute[]) { \
  380. { __ATTR(_name, 0444, _func, NULL), (void *)_config } \
  381. })[0].attr.attr)
  382. #define HNS3_PMU_FORMAT_ATTR(_name, _format) \
  383. HNS3_PMU_ATTR(_name, hns3_pmu_format_show, (void *)_format)
  384. #define HNS3_PMU_EVENT_ATTR(_name, _event) \
  385. HNS3_PMU_ATTR(_name, hns3_pmu_event_show, (void *)_event)
  386. #define HNS3_PMU_FLT_MODE_ATTR(_name, _event) \
  387. HNS3_PMU_ATTR(_name, hns3_pmu_filter_mode_show, (void *)_event)
  388. #define HNS3_PMU_BW_EVT_PAIR(_name, _macro) \
  389. HNS3_PMU_EVENT_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \
  390. HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))
  391. #define HNS3_PMU_PPS_EVT_PAIR(_name, _macro) \
  392. HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \
  393. HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))
  394. #define HNS3_PMU_DLY_EVT_PAIR(_name, _macro) \
  395. HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \
  396. HNS3_PMU_EVENT_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))
  397. #define HNS3_PMU_INTR_EVT_PAIR(_name, _macro) \
  398. HNS3_PMU_EVENT_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \
  399. HNS3_PMU_EVENT_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))
  400. #define HNS3_PMU_BW_FLT_MODE_PAIR(_name, _macro) \
  401. HNS3_PMU_FLT_MODE_ATTR(_name##_byte_num, HNS3_BW_EVT_BYTE_NUM(_macro)), \
  402. HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_BW_EVT_TIME(_macro))
  403. #define HNS3_PMU_PPS_FLT_MODE_PAIR(_name, _macro) \
  404. HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_PPS_EVT_PACKET_NUM(_macro)), \
  405. HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_PPS_EVT_TIME(_macro))
  406. #define HNS3_PMU_DLY_FLT_MODE_PAIR(_name, _macro) \
  407. HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_DLY_EVT_TIME(_macro)), \
  408. HNS3_PMU_FLT_MODE_ATTR(_name##_packet_num, HNS3_DLY_EVT_PACKET_NUM(_macro))
  409. #define HNS3_PMU_INTR_FLT_MODE_PAIR(_name, _macro) \
  410. HNS3_PMU_FLT_MODE_ATTR(_name##_intr_num, HNS3_INTR_EVT_INTR_NUM(_macro)), \
  411. HNS3_PMU_FLT_MODE_ATTR(_name##_time, HNS3_INTR_EVT_TIME(_macro))
  412. static u8 hns3_pmu_hw_filter_modes[] = {
  413. HNS3_PMU_HW_FILTER_GLOBAL,
  414. HNS3_PMU_HW_FILTER_PORT,
  415. HNS3_PMU_HW_FILTER_PORT_TC,
  416. HNS3_PMU_HW_FILTER_FUNC,
  417. HNS3_PMU_HW_FILTER_FUNC_QUEUE,
  418. HNS3_PMU_HW_FILTER_FUNC_INTR,
  419. };
  420. #define HNS3_PMU_SET_HW_FILTER(_hwc, _mode) \
  421. ((_hwc)->addr_filters = (void *)&hns3_pmu_hw_filter_modes[(_mode)])
  422. static ssize_t identifier_show(struct device *dev,
  423. struct device_attribute *attr, char *buf)
  424. {
  425. struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
  426. return sysfs_emit(buf, "0x%x\n", hns3_pmu->identifier);
  427. }
  428. static DEVICE_ATTR_RO(identifier);
  429. static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
  430. char *buf)
  431. {
  432. struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
  433. return sysfs_emit(buf, "%d\n", hns3_pmu->on_cpu);
  434. }
  435. static DEVICE_ATTR_RO(cpumask);
  436. static ssize_t bdf_min_show(struct device *dev, struct device_attribute *attr,
  437. char *buf)
  438. {
  439. struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
  440. u16 bdf = hns3_pmu->bdf_min;
  441. return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),
  442. PCI_SLOT(bdf), PCI_FUNC(bdf));
  443. }
  444. static DEVICE_ATTR_RO(bdf_min);
  445. static ssize_t bdf_max_show(struct device *dev, struct device_attribute *attr,
  446. char *buf)
  447. {
  448. struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
  449. u16 bdf = hns3_pmu->bdf_max;
  450. return sysfs_emit(buf, "%02x:%02x.%x\n", PCI_BUS_NUM(bdf),
  451. PCI_SLOT(bdf), PCI_FUNC(bdf));
  452. }
  453. static DEVICE_ATTR_RO(bdf_max);
  454. static ssize_t hw_clk_freq_show(struct device *dev,
  455. struct device_attribute *attr, char *buf)
  456. {
  457. struct hns3_pmu *hns3_pmu = to_hns3_pmu(dev_get_drvdata(dev));
  458. return sysfs_emit(buf, "%u\n", hns3_pmu->hw_clk_freq);
  459. }
  460. static DEVICE_ATTR_RO(hw_clk_freq);
  461. static struct attribute *hns3_pmu_events_attr[] = {
  462. /* bandwidth events */
  463. HNS3_PMU_BW_EVT_PAIR(bw_ssu_egu, SSU_EGU),
  464. HNS3_PMU_BW_EVT_PAIR(bw_ssu_rpu, SSU_RPU),
  465. HNS3_PMU_BW_EVT_PAIR(bw_ssu_roce, SSU_ROCE),
  466. HNS3_PMU_BW_EVT_PAIR(bw_roce_ssu, ROCE_SSU),
  467. HNS3_PMU_BW_EVT_PAIR(bw_tpu_ssu, TPU_SSU),
  468. HNS3_PMU_BW_EVT_PAIR(bw_rpu_rcbrx, RPU_RCBRX),
  469. HNS3_PMU_BW_EVT_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),
  470. HNS3_PMU_BW_EVT_PAIR(bw_wr_fbd, WR_FBD),
  471. HNS3_PMU_BW_EVT_PAIR(bw_wr_ebd, WR_EBD),
  472. HNS3_PMU_BW_EVT_PAIR(bw_rd_fbd, RD_FBD),
  473. HNS3_PMU_BW_EVT_PAIR(bw_rd_ebd, RD_EBD),
  474. HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m0, RD_PAY_M0),
  475. HNS3_PMU_BW_EVT_PAIR(bw_rd_pay_m1, RD_PAY_M1),
  476. HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m0, WR_PAY_M0),
  477. HNS3_PMU_BW_EVT_PAIR(bw_wr_pay_m1, WR_PAY_M1),
  478. /* packet rate events */
  479. HNS3_PMU_PPS_EVT_PAIR(pps_igu_ssu, IGU_SSU),
  480. HNS3_PMU_PPS_EVT_PAIR(pps_ssu_egu, SSU_EGU),
  481. HNS3_PMU_PPS_EVT_PAIR(pps_ssu_rpu, SSU_RPU),
  482. HNS3_PMU_PPS_EVT_PAIR(pps_ssu_roce, SSU_ROCE),
  483. HNS3_PMU_PPS_EVT_PAIR(pps_roce_ssu, ROCE_SSU),
  484. HNS3_PMU_PPS_EVT_PAIR(pps_tpu_ssu, TPU_SSU),
  485. HNS3_PMU_PPS_EVT_PAIR(pps_rpu_rcbrx, RPU_RCBRX),
  486. HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_tpu, RCBTX_TPU),
  487. HNS3_PMU_PPS_EVT_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),
  488. HNS3_PMU_PPS_EVT_PAIR(pps_wr_fbd, WR_FBD),
  489. HNS3_PMU_PPS_EVT_PAIR(pps_wr_ebd, WR_EBD),
  490. HNS3_PMU_PPS_EVT_PAIR(pps_rd_fbd, RD_FBD),
  491. HNS3_PMU_PPS_EVT_PAIR(pps_rd_ebd, RD_EBD),
  492. HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m0, RD_PAY_M0),
  493. HNS3_PMU_PPS_EVT_PAIR(pps_rd_pay_m1, RD_PAY_M1),
  494. HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m0, WR_PAY_M0),
  495. HNS3_PMU_PPS_EVT_PAIR(pps_wr_pay_m1, WR_PAY_M1),
  496. HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),
  497. HNS3_PMU_PPS_EVT_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),
  498. /* latency events */
  499. HNS3_PMU_DLY_EVT_PAIR(dly_tx_push_to_mac, TX_PUSH),
  500. HNS3_PMU_DLY_EVT_PAIR(dly_tx_normal_to_mac, TX),
  501. HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),
  502. HNS3_PMU_DLY_EVT_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),
  503. HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),
  504. HNS3_PMU_DLY_EVT_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),
  505. HNS3_PMU_DLY_EVT_PAIR(dly_rpu, RPU),
  506. HNS3_PMU_DLY_EVT_PAIR(dly_tpu, TPU),
  507. HNS3_PMU_DLY_EVT_PAIR(dly_rpe, RPE),
  508. HNS3_PMU_DLY_EVT_PAIR(dly_tpe_normal, TPE),
  509. HNS3_PMU_DLY_EVT_PAIR(dly_tpe_push, TPE_PUSH),
  510. HNS3_PMU_DLY_EVT_PAIR(dly_wr_fbd, WR_FBD),
  511. HNS3_PMU_DLY_EVT_PAIR(dly_wr_ebd, WR_EBD),
  512. HNS3_PMU_DLY_EVT_PAIR(dly_rd_fbd, RD_FBD),
  513. HNS3_PMU_DLY_EVT_PAIR(dly_rd_ebd, RD_EBD),
  514. HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m0, RD_PAY_M0),
  515. HNS3_PMU_DLY_EVT_PAIR(dly_rd_pay_m1, RD_PAY_M1),
  516. HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m0, WR_PAY_M0),
  517. HNS3_PMU_DLY_EVT_PAIR(dly_wr_pay_m1, WR_PAY_M1),
  518. HNS3_PMU_DLY_EVT_PAIR(dly_msix_write, MSIX_WRITE),
  519. /* interrupt rate events */
  520. HNS3_PMU_INTR_EVT_PAIR(pps_intr_msix_nic, MSIX_NIC),
  521. NULL
  522. };
  523. static struct attribute *hns3_pmu_filter_mode_attr[] = {
  524. /* bandwidth events */
  525. HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_egu, SSU_EGU),
  526. HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_rpu, SSU_RPU),
  527. HNS3_PMU_BW_FLT_MODE_PAIR(bw_ssu_roce, SSU_ROCE),
  528. HNS3_PMU_BW_FLT_MODE_PAIR(bw_roce_ssu, ROCE_SSU),
  529. HNS3_PMU_BW_FLT_MODE_PAIR(bw_tpu_ssu, TPU_SSU),
  530. HNS3_PMU_BW_FLT_MODE_PAIR(bw_rpu_rcbrx, RPU_RCBRX),
  531. HNS3_PMU_BW_FLT_MODE_PAIR(bw_rcbtx_txsch, RCBTX_TXSCH),
  532. HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_fbd, WR_FBD),
  533. HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_ebd, WR_EBD),
  534. HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_fbd, RD_FBD),
  535. HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_ebd, RD_EBD),
  536. HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m0, RD_PAY_M0),
  537. HNS3_PMU_BW_FLT_MODE_PAIR(bw_rd_pay_m1, RD_PAY_M1),
  538. HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m0, WR_PAY_M0),
  539. HNS3_PMU_BW_FLT_MODE_PAIR(bw_wr_pay_m1, WR_PAY_M1),
  540. /* packet rate events */
  541. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_igu_ssu, IGU_SSU),
  542. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_egu, SSU_EGU),
  543. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_rpu, SSU_RPU),
  544. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_ssu_roce, SSU_ROCE),
  545. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_roce_ssu, ROCE_SSU),
  546. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_tpu_ssu, TPU_SSU),
  547. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rpu_rcbrx, RPU_RCBRX),
  548. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_tpu, RCBTX_TPU),
  549. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rcbtx_txsch, RCBTX_TXSCH),
  550. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_fbd, WR_FBD),
  551. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_ebd, WR_EBD),
  552. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_fbd, RD_FBD),
  553. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_ebd, RD_EBD),
  554. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m0, RD_PAY_M0),
  555. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_rd_pay_m1, RD_PAY_M1),
  556. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m0, WR_PAY_M0),
  557. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_wr_pay_m1, WR_PAY_M1),
  558. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_tx_pre, NICROH_TX_PRE),
  559. HNS3_PMU_PPS_FLT_MODE_PAIR(pps_intr_nicroh_rx_pre, NICROH_RX_PRE),
  560. /* latency events */
  561. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_push_to_mac, TX_PUSH),
  562. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tx_normal_to_mac, TX),
  563. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_nic, SSU_TX_NIC),
  564. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_tx_th_roce, SSU_TX_ROCE),
  565. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_nic, SSU_RX_NIC),
  566. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_ssu_rx_th_roce, SSU_RX_ROCE),
  567. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpu, RPU),
  568. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpu, TPU),
  569. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rpe, RPE),
  570. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_normal, TPE),
  571. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_tpe_push, TPE_PUSH),
  572. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_fbd, WR_FBD),
  573. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_ebd, WR_EBD),
  574. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_fbd, RD_FBD),
  575. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_ebd, RD_EBD),
  576. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m0, RD_PAY_M0),
  577. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_rd_pay_m1, RD_PAY_M1),
  578. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m0, WR_PAY_M0),
  579. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_wr_pay_m1, WR_PAY_M1),
  580. HNS3_PMU_DLY_FLT_MODE_PAIR(dly_msix_write, MSIX_WRITE),
  581. /* interrupt rate events */
  582. HNS3_PMU_INTR_FLT_MODE_PAIR(pps_intr_msix_nic, MSIX_NIC),
  583. NULL
  584. };
  585. static struct attribute_group hns3_pmu_events_group = {
  586. .name = "events",
  587. .attrs = hns3_pmu_events_attr,
  588. };
  589. static struct attribute_group hns3_pmu_filter_mode_group = {
  590. .name = "filtermode",
  591. .attrs = hns3_pmu_filter_mode_attr,
  592. };
  593. static struct attribute *hns3_pmu_format_attr[] = {
  594. HNS3_PMU_FORMAT_ATTR(subevent, "config:0-7"),
  595. HNS3_PMU_FORMAT_ATTR(event_type, "config:8-15"),
  596. HNS3_PMU_FORMAT_ATTR(ext_counter_used, "config:16"),
  597. HNS3_PMU_FORMAT_ATTR(port, "config1:0-3"),
  598. HNS3_PMU_FORMAT_ATTR(tc, "config1:4-7"),
  599. HNS3_PMU_FORMAT_ATTR(bdf, "config1:8-23"),
  600. HNS3_PMU_FORMAT_ATTR(queue, "config1:24-39"),
  601. HNS3_PMU_FORMAT_ATTR(intr, "config1:40-51"),
  602. HNS3_PMU_FORMAT_ATTR(global, "config1:52"),
  603. NULL
  604. };
  605. static struct attribute_group hns3_pmu_format_group = {
  606. .name = "format",
  607. .attrs = hns3_pmu_format_attr,
  608. };
  609. static struct attribute *hns3_pmu_cpumask_attrs[] = {
  610. &dev_attr_cpumask.attr,
  611. NULL
  612. };
  613. static struct attribute_group hns3_pmu_cpumask_attr_group = {
  614. .attrs = hns3_pmu_cpumask_attrs,
  615. };
  616. static struct attribute *hns3_pmu_identifier_attrs[] = {
  617. &dev_attr_identifier.attr,
  618. NULL
  619. };
  620. static struct attribute_group hns3_pmu_identifier_attr_group = {
  621. .attrs = hns3_pmu_identifier_attrs,
  622. };
  623. static struct attribute *hns3_pmu_bdf_range_attrs[] = {
  624. &dev_attr_bdf_min.attr,
  625. &dev_attr_bdf_max.attr,
  626. NULL
  627. };
  628. static struct attribute_group hns3_pmu_bdf_range_attr_group = {
  629. .attrs = hns3_pmu_bdf_range_attrs,
  630. };
  631. static struct attribute *hns3_pmu_hw_clk_freq_attrs[] = {
  632. &dev_attr_hw_clk_freq.attr,
  633. NULL
  634. };
  635. static struct attribute_group hns3_pmu_hw_clk_freq_attr_group = {
  636. .attrs = hns3_pmu_hw_clk_freq_attrs,
  637. };
  638. static const struct attribute_group *hns3_pmu_attr_groups[] = {
  639. &hns3_pmu_events_group,
  640. &hns3_pmu_filter_mode_group,
  641. &hns3_pmu_format_group,
  642. &hns3_pmu_cpumask_attr_group,
  643. &hns3_pmu_identifier_attr_group,
  644. &hns3_pmu_bdf_range_attr_group,
  645. &hns3_pmu_hw_clk_freq_attr_group,
  646. NULL
  647. };
  648. static u32 hns3_pmu_get_event(struct perf_event *event)
  649. {
  650. return hns3_pmu_get_ext_counter_used(event) << 16 |
  651. hns3_pmu_get_event_type(event) << 8 |
  652. hns3_pmu_get_subevent(event);
  653. }
  654. static u32 hns3_pmu_get_real_event(struct perf_event *event)
  655. {
  656. return hns3_pmu_get_event_type(event) << 8 |
  657. hns3_pmu_get_subevent(event);
  658. }
  659. static u32 hns3_pmu_get_offset(u32 offset, u32 idx)
  660. {
  661. return offset + HNS3_PMU_REG_EVENT_OFFSET +
  662. HNS3_PMU_REG_EVENT_SIZE * idx;
  663. }
  664. static u32 hns3_pmu_readl(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)
  665. {
  666. u32 offset = hns3_pmu_get_offset(reg_offset, idx);
  667. return readl(hns3_pmu->base + offset);
  668. }
  669. static void hns3_pmu_writel(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,
  670. u32 val)
  671. {
  672. u32 offset = hns3_pmu_get_offset(reg_offset, idx);
  673. writel(val, hns3_pmu->base + offset);
  674. }
  675. static u64 hns3_pmu_readq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx)
  676. {
  677. u32 offset = hns3_pmu_get_offset(reg_offset, idx);
  678. return readq(hns3_pmu->base + offset);
  679. }
  680. static void hns3_pmu_writeq(struct hns3_pmu *hns3_pmu, u32 reg_offset, u32 idx,
  681. u64 val)
  682. {
  683. u32 offset = hns3_pmu_get_offset(reg_offset, idx);
  684. writeq(val, hns3_pmu->base + offset);
  685. }
  686. static bool hns3_pmu_cmp_event(struct perf_event *target,
  687. struct perf_event *event)
  688. {
  689. return hns3_pmu_get_real_event(target) == hns3_pmu_get_real_event(event);
  690. }
  691. static int hns3_pmu_find_related_event_idx(struct hns3_pmu *hns3_pmu,
  692. struct perf_event *event)
  693. {
  694. struct perf_event *sibling;
  695. int hw_event_used = 0;
  696. int idx;
  697. for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
  698. sibling = hns3_pmu->hw_events[idx];
  699. if (!sibling)
  700. continue;
  701. hw_event_used++;
  702. if (!hns3_pmu_cmp_event(sibling, event))
  703. continue;
  704. /* Related events is used in group */
  705. if (sibling->group_leader == event->group_leader)
  706. return idx;
  707. }
  708. /* No related event and all hardware events are used up */
  709. if (hw_event_used >= HNS3_PMU_MAX_HW_EVENTS)
  710. return -EBUSY;
  711. /* No related event and there is extra hardware events can be use */
  712. return -ENOENT;
  713. }
  714. static int hns3_pmu_get_event_idx(struct hns3_pmu *hns3_pmu)
  715. {
  716. int idx;
  717. for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
  718. if (!hns3_pmu->hw_events[idx])
  719. return idx;
  720. }
  721. return -EBUSY;
  722. }
  723. static bool hns3_pmu_valid_bdf(struct hns3_pmu *hns3_pmu, u16 bdf)
  724. {
  725. struct pci_dev *pdev;
  726. if (bdf < hns3_pmu->bdf_min || bdf > hns3_pmu->bdf_max) {
  727. pci_err(hns3_pmu->pdev, "Invalid EP device: %#x!\n", bdf);
  728. return false;
  729. }
  730. pdev = pci_get_domain_bus_and_slot(pci_domain_nr(hns3_pmu->pdev->bus),
  731. PCI_BUS_NUM(bdf),
  732. GET_PCI_DEVFN(bdf));
  733. if (!pdev) {
  734. pci_err(hns3_pmu->pdev, "Nonexistent EP device: %#x!\n", bdf);
  735. return false;
  736. }
  737. pci_dev_put(pdev);
  738. return true;
  739. }
  740. static void hns3_pmu_set_qid_para(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,
  741. u16 queue)
  742. {
  743. u32 val;
  744. val = GET_PCI_DEVFN(bdf);
  745. val |= (u32)queue << HNS3_PMU_QID_PARA_QUEUE_S;
  746. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_PARA, idx, val);
  747. }
  748. static bool hns3_pmu_qid_req_start(struct hns3_pmu *hns3_pmu, u32 idx)
  749. {
  750. bool queue_id_valid = false;
  751. u32 reg_qid_ctrl, val;
  752. int err;
  753. /* enable queue id request */
  754. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx,
  755. HNS3_PMU_QID_CTRL_REQ_ENABLE);
  756. reg_qid_ctrl = hns3_pmu_get_offset(HNS3_PMU_REG_EVENT_QID_CTRL, idx);
  757. err = readl_poll_timeout(hns3_pmu->base + reg_qid_ctrl, val,
  758. val & HNS3_PMU_QID_CTRL_DONE, 1, 1000);
  759. if (err == -ETIMEDOUT) {
  760. pci_err(hns3_pmu->pdev, "QID request timeout!\n");
  761. goto out;
  762. }
  763. queue_id_valid = !(val & HNS3_PMU_QID_CTRL_MISS);
  764. out:
  765. /* disable qid request and clear status */
  766. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_QID_CTRL, idx, 0);
  767. return queue_id_valid;
  768. }
  769. static bool hns3_pmu_valid_queue(struct hns3_pmu *hns3_pmu, u32 idx, u16 bdf,
  770. u16 queue)
  771. {
  772. hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue);
  773. return hns3_pmu_qid_req_start(hns3_pmu, idx);
  774. }
  775. static struct hns3_pmu_event_attr *hns3_pmu_get_pmu_event(u32 event)
  776. {
  777. struct hns3_pmu_event_attr *pmu_event;
  778. struct dev_ext_attribute *eattr;
  779. struct device_attribute *dattr;
  780. struct attribute *attr;
  781. u32 i;
  782. for (i = 0; i < ARRAY_SIZE(hns3_pmu_events_attr) - 1; i++) {
  783. attr = hns3_pmu_events_attr[i];
  784. dattr = container_of(attr, struct device_attribute, attr);
  785. eattr = container_of(dattr, struct dev_ext_attribute, attr);
  786. pmu_event = eattr->var;
  787. if (event == pmu_event->event)
  788. return pmu_event;
  789. }
  790. return NULL;
  791. }
  792. static int hns3_pmu_set_func_mode(struct perf_event *event,
  793. struct hns3_pmu *hns3_pmu)
  794. {
  795. struct hw_perf_event *hwc = &event->hw;
  796. u16 bdf = hns3_pmu_get_bdf(event);
  797. if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))
  798. return -ENOENT;
  799. HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC);
  800. return 0;
  801. }
  802. static int hns3_pmu_set_func_queue_mode(struct perf_event *event,
  803. struct hns3_pmu *hns3_pmu)
  804. {
  805. u16 queue_id = hns3_pmu_get_queue(event);
  806. struct hw_perf_event *hwc = &event->hw;
  807. u16 bdf = hns3_pmu_get_bdf(event);
  808. if (!hns3_pmu_valid_bdf(hns3_pmu, bdf))
  809. return -ENOENT;
  810. if (!hns3_pmu_valid_queue(hns3_pmu, hwc->idx, bdf, queue_id)) {
  811. pci_err(hns3_pmu->pdev, "Invalid queue: %u\n", queue_id);
  812. return -ENOENT;
  813. }
  814. HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_QUEUE);
  815. return 0;
  816. }
  817. static bool
  818. hns3_pmu_is_enabled_global_mode(struct perf_event *event,
  819. struct hns3_pmu_event_attr *pmu_event)
  820. {
  821. u8 global = hns3_pmu_get_global(event);
  822. if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_GLOBAL))
  823. return false;
  824. return global;
  825. }
  826. static bool hns3_pmu_is_enabled_func_mode(struct perf_event *event,
  827. struct hns3_pmu_event_attr *pmu_event)
  828. {
  829. u16 queue_id = hns3_pmu_get_queue(event);
  830. u16 bdf = hns3_pmu_get_bdf(event);
  831. if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC))
  832. return false;
  833. else if (queue_id != HNS3_PMU_FILTER_ALL_QUEUE)
  834. return false;
  835. return bdf;
  836. }
  837. static bool
  838. hns3_pmu_is_enabled_func_queue_mode(struct perf_event *event,
  839. struct hns3_pmu_event_attr *pmu_event)
  840. {
  841. u16 queue_id = hns3_pmu_get_queue(event);
  842. u16 bdf = hns3_pmu_get_bdf(event);
  843. if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_QUEUE))
  844. return false;
  845. else if (queue_id == HNS3_PMU_FILTER_ALL_QUEUE)
  846. return false;
  847. return bdf;
  848. }
  849. static bool hns3_pmu_is_enabled_port_mode(struct perf_event *event,
  850. struct hns3_pmu_event_attr *pmu_event)
  851. {
  852. u8 tc_id = hns3_pmu_get_tc(event);
  853. if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT))
  854. return false;
  855. return tc_id == HNS3_PMU_FILTER_ALL_TC;
  856. }
  857. static bool
  858. hns3_pmu_is_enabled_port_tc_mode(struct perf_event *event,
  859. struct hns3_pmu_event_attr *pmu_event)
  860. {
  861. u8 tc_id = hns3_pmu_get_tc(event);
  862. if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_PORT_TC))
  863. return false;
  864. return tc_id != HNS3_PMU_FILTER_ALL_TC;
  865. }
  866. static bool
  867. hns3_pmu_is_enabled_func_intr_mode(struct perf_event *event,
  868. struct hns3_pmu *hns3_pmu,
  869. struct hns3_pmu_event_attr *pmu_event)
  870. {
  871. u16 bdf = hns3_pmu_get_bdf(event);
  872. if (!(pmu_event->filter_support & HNS3_PMU_FILTER_SUPPORT_FUNC_INTR))
  873. return false;
  874. return hns3_pmu_valid_bdf(hns3_pmu, bdf);
  875. }
  876. static int hns3_pmu_select_filter_mode(struct perf_event *event,
  877. struct hns3_pmu *hns3_pmu)
  878. {
  879. u32 event_id = hns3_pmu_get_event(event);
  880. struct hw_perf_event *hwc = &event->hw;
  881. struct hns3_pmu_event_attr *pmu_event;
  882. pmu_event = hns3_pmu_get_pmu_event(event_id);
  883. if (!pmu_event) {
  884. pci_err(hns3_pmu->pdev, "Invalid pmu event\n");
  885. return -ENOENT;
  886. }
  887. if (hns3_pmu_is_enabled_global_mode(event, pmu_event)) {
  888. HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_GLOBAL);
  889. return 0;
  890. }
  891. if (hns3_pmu_is_enabled_func_mode(event, pmu_event))
  892. return hns3_pmu_set_func_mode(event, hns3_pmu);
  893. if (hns3_pmu_is_enabled_func_queue_mode(event, pmu_event))
  894. return hns3_pmu_set_func_queue_mode(event, hns3_pmu);
  895. if (hns3_pmu_is_enabled_port_mode(event, pmu_event)) {
  896. HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT);
  897. return 0;
  898. }
  899. if (hns3_pmu_is_enabled_port_tc_mode(event, pmu_event)) {
  900. HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_PORT_TC);
  901. return 0;
  902. }
  903. if (hns3_pmu_is_enabled_func_intr_mode(event, hns3_pmu, pmu_event)) {
  904. HNS3_PMU_SET_HW_FILTER(hwc, HNS3_PMU_HW_FILTER_FUNC_INTR);
  905. return 0;
  906. }
  907. return -ENOENT;
  908. }
  909. static bool hns3_pmu_validate_event_group(struct perf_event *event)
  910. {
  911. struct perf_event *sibling, *leader = event->group_leader;
  912. struct perf_event *event_group[HNS3_PMU_MAX_HW_EVENTS];
  913. int counters = 1;
  914. int num;
  915. event_group[0] = leader;
  916. if (!is_software_event(leader)) {
  917. if (leader->pmu != event->pmu)
  918. return false;
  919. if (leader != event && !hns3_pmu_cmp_event(leader, event))
  920. event_group[counters++] = event;
  921. }
  922. for_each_sibling_event(sibling, event->group_leader) {
  923. if (is_software_event(sibling))
  924. continue;
  925. if (sibling->pmu != event->pmu)
  926. return false;
  927. for (num = 0; num < counters; num++) {
  928. if (hns3_pmu_cmp_event(event_group[num], sibling))
  929. break;
  930. }
  931. if (num == counters)
  932. event_group[counters++] = sibling;
  933. }
  934. return counters <= HNS3_PMU_MAX_HW_EVENTS;
  935. }
  936. static u32 hns3_pmu_get_filter_condition(struct perf_event *event)
  937. {
  938. struct hw_perf_event *hwc = &event->hw;
  939. u16 intr_id = hns3_pmu_get_intr(event);
  940. u8 port_id = hns3_pmu_get_port(event);
  941. u16 bdf = hns3_pmu_get_bdf(event);
  942. u8 tc_id = hns3_pmu_get_tc(event);
  943. u8 filter_mode;
  944. filter_mode = *(u8 *)hwc->addr_filters;
  945. switch (filter_mode) {
  946. case HNS3_PMU_HW_FILTER_PORT:
  947. return FILTER_CONDITION_PORT(port_id);
  948. case HNS3_PMU_HW_FILTER_PORT_TC:
  949. return FILTER_CONDITION_PORT_TC(port_id, tc_id);
  950. case HNS3_PMU_HW_FILTER_FUNC:
  951. case HNS3_PMU_HW_FILTER_FUNC_QUEUE:
  952. return GET_PCI_DEVFN(bdf);
  953. case HNS3_PMU_HW_FILTER_FUNC_INTR:
  954. return FILTER_CONDITION_FUNC_INTR(GET_PCI_DEVFN(bdf), intr_id);
  955. default:
  956. break;
  957. }
  958. return 0;
  959. }
  960. static void hns3_pmu_config_filter(struct perf_event *event)
  961. {
  962. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  963. u8 event_type = hns3_pmu_get_event_type(event);
  964. u8 subevent_id = hns3_pmu_get_subevent(event);
  965. u16 queue_id = hns3_pmu_get_queue(event);
  966. struct hw_perf_event *hwc = &event->hw;
  967. u8 filter_mode = *(u8 *)hwc->addr_filters;
  968. u16 bdf = hns3_pmu_get_bdf(event);
  969. u32 idx = hwc->idx;
  970. u32 val;
  971. val = event_type;
  972. val |= subevent_id << HNS3_PMU_CTRL_SUBEVENT_S;
  973. val |= filter_mode << HNS3_PMU_CTRL_FILTER_MODE_S;
  974. val |= HNS3_PMU_EVENT_OVERFLOW_RESTART;
  975. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
  976. val = hns3_pmu_get_filter_condition(event);
  977. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_HIGH, idx, val);
  978. if (filter_mode == HNS3_PMU_HW_FILTER_FUNC_QUEUE)
  979. hns3_pmu_set_qid_para(hns3_pmu, idx, bdf, queue_id);
  980. }
  981. static void hns3_pmu_enable_counter(struct hns3_pmu *hns3_pmu,
  982. struct hw_perf_event *hwc)
  983. {
  984. u32 idx = hwc->idx;
  985. u32 val;
  986. val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
  987. val |= HNS3_PMU_EVENT_EN;
  988. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
  989. }
  990. static void hns3_pmu_disable_counter(struct hns3_pmu *hns3_pmu,
  991. struct hw_perf_event *hwc)
  992. {
  993. u32 idx = hwc->idx;
  994. u32 val;
  995. val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
  996. val &= ~HNS3_PMU_EVENT_EN;
  997. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
  998. }
  999. static void hns3_pmu_enable_intr(struct hns3_pmu *hns3_pmu,
  1000. struct hw_perf_event *hwc)
  1001. {
  1002. u32 idx = hwc->idx;
  1003. u32 val;
  1004. val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);
  1005. val &= ~HNS3_PMU_INTR_MASK_OVERFLOW;
  1006. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);
  1007. }
  1008. static void hns3_pmu_disable_intr(struct hns3_pmu *hns3_pmu,
  1009. struct hw_perf_event *hwc)
  1010. {
  1011. u32 idx = hwc->idx;
  1012. u32 val;
  1013. val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx);
  1014. val |= HNS3_PMU_INTR_MASK_OVERFLOW;
  1015. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_INTR_MASK, idx, val);
  1016. }
  1017. static void hns3_pmu_clear_intr_status(struct hns3_pmu *hns3_pmu, u32 idx)
  1018. {
  1019. u32 val;
  1020. val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
  1021. val |= HNS3_PMU_EVENT_STATUS_RESET;
  1022. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
  1023. val = hns3_pmu_readl(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx);
  1024. val &= ~HNS3_PMU_EVENT_STATUS_RESET;
  1025. hns3_pmu_writel(hns3_pmu, HNS3_PMU_REG_EVENT_CTRL_LOW, idx, val);
  1026. }
  1027. static u64 hns3_pmu_read_counter(struct perf_event *event)
  1028. {
  1029. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1030. return hns3_pmu_readq(hns3_pmu, event->hw.event_base, event->hw.idx);
  1031. }
  1032. static void hns3_pmu_write_counter(struct perf_event *event, u64 value)
  1033. {
  1034. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1035. u32 idx = event->hw.idx;
  1036. hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_COUNTER, idx, value);
  1037. hns3_pmu_writeq(hns3_pmu, HNS3_PMU_REG_EVENT_EXT_COUNTER, idx, value);
  1038. }
  1039. static void hns3_pmu_init_counter(struct perf_event *event)
  1040. {
  1041. struct hw_perf_event *hwc = &event->hw;
  1042. local64_set(&hwc->prev_count, 0);
  1043. hns3_pmu_write_counter(event, 0);
  1044. }
  1045. static int hns3_pmu_event_init(struct perf_event *event)
  1046. {
  1047. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1048. struct hw_perf_event *hwc = &event->hw;
  1049. int idx;
  1050. int ret;
  1051. if (event->attr.type != event->pmu->type)
  1052. return -ENOENT;
  1053. /* Sampling is not supported */
  1054. if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
  1055. return -EOPNOTSUPP;
  1056. event->cpu = hns3_pmu->on_cpu;
  1057. idx = hns3_pmu_get_event_idx(hns3_pmu);
  1058. if (idx < 0) {
  1059. pci_err(hns3_pmu->pdev, "Up to %u events are supported!\n",
  1060. HNS3_PMU_MAX_HW_EVENTS);
  1061. return -EBUSY;
  1062. }
  1063. hwc->idx = idx;
  1064. ret = hns3_pmu_select_filter_mode(event, hns3_pmu);
  1065. if (ret) {
  1066. pci_err(hns3_pmu->pdev, "Invalid filter, ret = %d.\n", ret);
  1067. return ret;
  1068. }
  1069. if (!hns3_pmu_validate_event_group(event)) {
  1070. pci_err(hns3_pmu->pdev, "Invalid event group.\n");
  1071. return -EINVAL;
  1072. }
  1073. if (hns3_pmu_get_ext_counter_used(event))
  1074. hwc->event_base = HNS3_PMU_REG_EVENT_EXT_COUNTER;
  1075. else
  1076. hwc->event_base = HNS3_PMU_REG_EVENT_COUNTER;
  1077. return 0;
  1078. }
  1079. static void hns3_pmu_read(struct perf_event *event)
  1080. {
  1081. struct hw_perf_event *hwc = &event->hw;
  1082. u64 new_cnt, prev_cnt, delta;
  1083. do {
  1084. prev_cnt = local64_read(&hwc->prev_count);
  1085. new_cnt = hns3_pmu_read_counter(event);
  1086. } while (local64_cmpxchg(&hwc->prev_count, prev_cnt, new_cnt) !=
  1087. prev_cnt);
  1088. delta = new_cnt - prev_cnt;
  1089. local64_add(delta, &event->count);
  1090. }
  1091. static void hns3_pmu_start(struct perf_event *event, int flags)
  1092. {
  1093. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1094. struct hw_perf_event *hwc = &event->hw;
  1095. if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
  1096. return;
  1097. WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
  1098. hwc->state = 0;
  1099. hns3_pmu_config_filter(event);
  1100. hns3_pmu_init_counter(event);
  1101. hns3_pmu_enable_intr(hns3_pmu, hwc);
  1102. hns3_pmu_enable_counter(hns3_pmu, hwc);
  1103. perf_event_update_userpage(event);
  1104. }
  1105. static void hns3_pmu_stop(struct perf_event *event, int flags)
  1106. {
  1107. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1108. struct hw_perf_event *hwc = &event->hw;
  1109. hns3_pmu_disable_counter(hns3_pmu, hwc);
  1110. hns3_pmu_disable_intr(hns3_pmu, hwc);
  1111. WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
  1112. hwc->state |= PERF_HES_STOPPED;
  1113. if (hwc->state & PERF_HES_UPTODATE)
  1114. return;
  1115. /* Read hardware counter and update the perf counter statistics */
  1116. hns3_pmu_read(event);
  1117. hwc->state |= PERF_HES_UPTODATE;
  1118. }
  1119. static int hns3_pmu_add(struct perf_event *event, int flags)
  1120. {
  1121. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1122. struct hw_perf_event *hwc = &event->hw;
  1123. int idx;
  1124. hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
  1125. /* Check all working events to find a related event. */
  1126. idx = hns3_pmu_find_related_event_idx(hns3_pmu, event);
  1127. if (idx < 0 && idx != -ENOENT)
  1128. return idx;
  1129. /* Current event shares an enabled hardware event with related event */
  1130. if (idx >= 0 && idx < HNS3_PMU_MAX_HW_EVENTS) {
  1131. hwc->idx = idx;
  1132. goto start_count;
  1133. }
  1134. idx = hns3_pmu_get_event_idx(hns3_pmu);
  1135. if (idx < 0)
  1136. return idx;
  1137. hwc->idx = idx;
  1138. hns3_pmu->hw_events[idx] = event;
  1139. start_count:
  1140. if (flags & PERF_EF_START)
  1141. hns3_pmu_start(event, PERF_EF_RELOAD);
  1142. return 0;
  1143. }
  1144. static void hns3_pmu_del(struct perf_event *event, int flags)
  1145. {
  1146. struct hns3_pmu *hns3_pmu = to_hns3_pmu(event->pmu);
  1147. struct hw_perf_event *hwc = &event->hw;
  1148. hns3_pmu_stop(event, PERF_EF_UPDATE);
  1149. hns3_pmu->hw_events[hwc->idx] = NULL;
  1150. perf_event_update_userpage(event);
  1151. }
  1152. static void hns3_pmu_enable(struct pmu *pmu)
  1153. {
  1154. struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);
  1155. u32 val;
  1156. val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
  1157. val |= HNS3_PMU_GLOBAL_START;
  1158. writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
  1159. }
  1160. static void hns3_pmu_disable(struct pmu *pmu)
  1161. {
  1162. struct hns3_pmu *hns3_pmu = to_hns3_pmu(pmu);
  1163. u32 val;
  1164. val = readl(hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
  1165. val &= ~HNS3_PMU_GLOBAL_START;
  1166. writel(val, hns3_pmu->base + HNS3_PMU_REG_GLOBAL_CTRL);
  1167. }
  1168. static int hns3_pmu_alloc_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)
  1169. {
  1170. u16 device_id;
  1171. char *name;
  1172. u32 val;
  1173. hns3_pmu->base = pcim_iomap_table(pdev)[BAR_2];
  1174. if (!hns3_pmu->base) {
  1175. pci_err(pdev, "ioremap failed\n");
  1176. return -ENOMEM;
  1177. }
  1178. hns3_pmu->hw_clk_freq = readl(hns3_pmu->base + HNS3_PMU_REG_CLOCK_FREQ);
  1179. val = readl(hns3_pmu->base + HNS3_PMU_REG_BDF);
  1180. hns3_pmu->bdf_min = val & 0xffff;
  1181. hns3_pmu->bdf_max = val >> 16;
  1182. val = readl(hns3_pmu->base + HNS3_PMU_REG_DEVICE_ID);
  1183. device_id = val & 0xffff;
  1184. name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "hns3_pmu_sicl_%u", device_id);
  1185. if (!name)
  1186. return -ENOMEM;
  1187. hns3_pmu->pdev = pdev;
  1188. hns3_pmu->on_cpu = -1;
  1189. hns3_pmu->identifier = readl(hns3_pmu->base + HNS3_PMU_REG_VERSION);
  1190. hns3_pmu->pmu = (struct pmu) {
  1191. .name = name,
  1192. .module = THIS_MODULE,
  1193. .event_init = hns3_pmu_event_init,
  1194. .pmu_enable = hns3_pmu_enable,
  1195. .pmu_disable = hns3_pmu_disable,
  1196. .add = hns3_pmu_add,
  1197. .del = hns3_pmu_del,
  1198. .start = hns3_pmu_start,
  1199. .stop = hns3_pmu_stop,
  1200. .read = hns3_pmu_read,
  1201. .task_ctx_nr = perf_invalid_context,
  1202. .attr_groups = hns3_pmu_attr_groups,
  1203. .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
  1204. };
  1205. return 0;
  1206. }
  1207. static irqreturn_t hns3_pmu_irq(int irq, void *data)
  1208. {
  1209. struct hns3_pmu *hns3_pmu = data;
  1210. u32 intr_status, idx;
  1211. for (idx = 0; idx < HNS3_PMU_MAX_HW_EVENTS; idx++) {
  1212. intr_status = hns3_pmu_readl(hns3_pmu,
  1213. HNS3_PMU_REG_EVENT_INTR_STATUS,
  1214. idx);
  1215. /*
  1216. * As each counter will restart from 0 when it is overflowed,
  1217. * extra processing is no need, just clear interrupt status.
  1218. */
  1219. if (intr_status)
  1220. hns3_pmu_clear_intr_status(hns3_pmu, idx);
  1221. }
  1222. return IRQ_HANDLED;
  1223. }
  1224. static int hns3_pmu_online_cpu(unsigned int cpu, struct hlist_node *node)
  1225. {
  1226. struct hns3_pmu *hns3_pmu;
  1227. hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);
  1228. if (!hns3_pmu)
  1229. return -ENODEV;
  1230. if (hns3_pmu->on_cpu == -1) {
  1231. hns3_pmu->on_cpu = cpu;
  1232. irq_set_affinity(hns3_pmu->irq, cpumask_of(cpu));
  1233. }
  1234. return 0;
  1235. }
  1236. static int hns3_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
  1237. {
  1238. struct hns3_pmu *hns3_pmu;
  1239. unsigned int target;
  1240. hns3_pmu = hlist_entry_safe(node, struct hns3_pmu, node);
  1241. if (!hns3_pmu)
  1242. return -ENODEV;
  1243. /* Nothing to do if this CPU doesn't own the PMU */
  1244. if (hns3_pmu->on_cpu != cpu)
  1245. return 0;
  1246. /* Choose a new CPU from all online cpus */
  1247. target = cpumask_any_but(cpu_online_mask, cpu);
  1248. if (target >= nr_cpu_ids)
  1249. return 0;
  1250. perf_pmu_migrate_context(&hns3_pmu->pmu, cpu, target);
  1251. hns3_pmu->on_cpu = target;
  1252. irq_set_affinity(hns3_pmu->irq, cpumask_of(target));
  1253. return 0;
  1254. }
  1255. static void hns3_pmu_free_irq(void *data)
  1256. {
  1257. struct pci_dev *pdev = data;
  1258. pci_free_irq_vectors(pdev);
  1259. }
  1260. static int hns3_pmu_irq_register(struct pci_dev *pdev,
  1261. struct hns3_pmu *hns3_pmu)
  1262. {
  1263. int irq, ret;
  1264. ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
  1265. if (ret < 0) {
  1266. pci_err(pdev, "failed to enable MSI vectors, ret = %d.\n", ret);
  1267. return ret;
  1268. }
  1269. ret = devm_add_action(&pdev->dev, hns3_pmu_free_irq, pdev);
  1270. if (ret) {
  1271. pci_err(pdev, "failed to add free irq action, ret = %d.\n", ret);
  1272. return ret;
  1273. }
  1274. irq = pci_irq_vector(pdev, 0);
  1275. ret = devm_request_irq(&pdev->dev, irq, hns3_pmu_irq, 0,
  1276. hns3_pmu->pmu.name, hns3_pmu);
  1277. if (ret) {
  1278. pci_err(pdev, "failed to register irq, ret = %d.\n", ret);
  1279. return ret;
  1280. }
  1281. hns3_pmu->irq = irq;
  1282. return 0;
  1283. }
  1284. static int hns3_pmu_init_pmu(struct pci_dev *pdev, struct hns3_pmu *hns3_pmu)
  1285. {
  1286. int ret;
  1287. ret = hns3_pmu_alloc_pmu(pdev, hns3_pmu);
  1288. if (ret)
  1289. return ret;
  1290. ret = hns3_pmu_irq_register(pdev, hns3_pmu);
  1291. if (ret)
  1292. return ret;
  1293. ret = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
  1294. &hns3_pmu->node);
  1295. if (ret) {
  1296. pci_err(pdev, "failed to register hotplug, ret = %d.\n", ret);
  1297. return ret;
  1298. }
  1299. ret = perf_pmu_register(&hns3_pmu->pmu, hns3_pmu->pmu.name, -1);
  1300. if (ret) {
  1301. pci_err(pdev, "failed to register perf PMU, ret = %d.\n", ret);
  1302. cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
  1303. &hns3_pmu->node);
  1304. }
  1305. return ret;
  1306. }
  1307. static void hns3_pmu_uninit_pmu(struct pci_dev *pdev)
  1308. {
  1309. struct hns3_pmu *hns3_pmu = pci_get_drvdata(pdev);
  1310. perf_pmu_unregister(&hns3_pmu->pmu);
  1311. cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
  1312. &hns3_pmu->node);
  1313. }
  1314. static int hns3_pmu_init_dev(struct pci_dev *pdev)
  1315. {
  1316. int ret;
  1317. ret = pcim_enable_device(pdev);
  1318. if (ret) {
  1319. pci_err(pdev, "failed to enable pci device, ret = %d.\n", ret);
  1320. return ret;
  1321. }
  1322. ret = pcim_iomap_regions(pdev, BIT(BAR_2), "hns3_pmu");
  1323. if (ret < 0) {
  1324. pci_err(pdev, "failed to request pci region, ret = %d.\n", ret);
  1325. return ret;
  1326. }
  1327. pci_set_master(pdev);
  1328. return 0;
  1329. }
  1330. static int hns3_pmu_probe(struct pci_dev *pdev, const struct pci_device_id *id)
  1331. {
  1332. struct hns3_pmu *hns3_pmu;
  1333. int ret;
  1334. hns3_pmu = devm_kzalloc(&pdev->dev, sizeof(*hns3_pmu), GFP_KERNEL);
  1335. if (!hns3_pmu)
  1336. return -ENOMEM;
  1337. ret = hns3_pmu_init_dev(pdev);
  1338. if (ret)
  1339. return ret;
  1340. ret = hns3_pmu_init_pmu(pdev, hns3_pmu);
  1341. if (ret) {
  1342. pci_clear_master(pdev);
  1343. return ret;
  1344. }
  1345. pci_set_drvdata(pdev, hns3_pmu);
  1346. return ret;
  1347. }
  1348. static void hns3_pmu_remove(struct pci_dev *pdev)
  1349. {
  1350. hns3_pmu_uninit_pmu(pdev);
  1351. pci_clear_master(pdev);
  1352. pci_set_drvdata(pdev, NULL);
  1353. }
  1354. static const struct pci_device_id hns3_pmu_ids[] = {
  1355. { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa22b) },
  1356. { 0, }
  1357. };
  1358. MODULE_DEVICE_TABLE(pci, hns3_pmu_ids);
  1359. static struct pci_driver hns3_pmu_driver = {
  1360. .name = "hns3_pmu",
  1361. .id_table = hns3_pmu_ids,
  1362. .probe = hns3_pmu_probe,
  1363. .remove = hns3_pmu_remove,
  1364. };
  1365. static int __init hns3_pmu_module_init(void)
  1366. {
  1367. int ret;
  1368. ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE,
  1369. "AP_PERF_ARM_HNS3_PMU_ONLINE",
  1370. hns3_pmu_online_cpu,
  1371. hns3_pmu_offline_cpu);
  1372. if (ret) {
  1373. pr_err("failed to setup HNS3 PMU hotplug, ret = %d.\n", ret);
  1374. return ret;
  1375. }
  1376. ret = pci_register_driver(&hns3_pmu_driver);
  1377. if (ret) {
  1378. pr_err("failed to register pci driver, ret = %d.\n", ret);
  1379. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);
  1380. }
  1381. return ret;
  1382. }
  1383. module_init(hns3_pmu_module_init);
  1384. static void __exit hns3_pmu_module_exit(void)
  1385. {
  1386. pci_unregister_driver(&hns3_pmu_driver);
  1387. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE);
  1388. }
  1389. module_exit(hns3_pmu_module_exit);
  1390. MODULE_DESCRIPTION("HNS3 PMU driver");
  1391. MODULE_LICENSE("GPL v2");