p4.c 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404
  1. /*
  2. * Netburst Performance Events (P4, old Xeon)
  3. *
  4. * Copyright (C) 2010 Parallels, Inc., Cyrill Gorcunov <[email protected]>
  5. * Copyright (C) 2010 Intel Corporation, Lin Ming <[email protected]>
  6. *
  7. * For licencing details see kernel-base/COPYING
  8. */
  9. #include <linux/perf_event.h>
  10. #include <asm/perf_event_p4.h>
  11. #include <asm/hardirq.h>
  12. #include <asm/apic.h>
  13. #include "../perf_event.h"
  14. #define P4_CNTR_LIMIT 3
  15. /*
  16. * array indices: 0,1 - HT threads, used with HT enabled cpu
  17. */
  18. struct p4_event_bind {
  19. unsigned int opcode; /* Event code and ESCR selector */
  20. unsigned int escr_msr[2]; /* ESCR MSR for this event */
  21. unsigned int escr_emask; /* valid ESCR EventMask bits */
  22. unsigned int shared; /* event is shared across threads */
  23. char cntr[2][P4_CNTR_LIMIT]; /* counter index (offset), -1 on absence */
  24. };
  25. struct p4_pebs_bind {
  26. unsigned int metric_pebs;
  27. unsigned int metric_vert;
  28. };
  29. /* it sets P4_PEBS_ENABLE_UOP_TAG as well */
  30. #define P4_GEN_PEBS_BIND(name, pebs, vert) \
  31. [P4_PEBS_METRIC__##name] = { \
  32. .metric_pebs = pebs | P4_PEBS_ENABLE_UOP_TAG, \
  33. .metric_vert = vert, \
  34. }
  35. /*
  36. * note we have P4_PEBS_ENABLE_UOP_TAG always set here
  37. *
  38. * it's needed for mapping P4_PEBS_CONFIG_METRIC_MASK bits of
  39. * event configuration to find out which values are to be
  40. * written into MSR_IA32_PEBS_ENABLE and MSR_P4_PEBS_MATRIX_VERT
  41. * registers
  42. */
  43. static struct p4_pebs_bind p4_pebs_bind_map[] = {
  44. P4_GEN_PEBS_BIND(1stl_cache_load_miss_retired, 0x0000001, 0x0000001),
  45. P4_GEN_PEBS_BIND(2ndl_cache_load_miss_retired, 0x0000002, 0x0000001),
  46. P4_GEN_PEBS_BIND(dtlb_load_miss_retired, 0x0000004, 0x0000001),
  47. P4_GEN_PEBS_BIND(dtlb_store_miss_retired, 0x0000004, 0x0000002),
  48. P4_GEN_PEBS_BIND(dtlb_all_miss_retired, 0x0000004, 0x0000003),
  49. P4_GEN_PEBS_BIND(tagged_mispred_branch, 0x0018000, 0x0000010),
  50. P4_GEN_PEBS_BIND(mob_load_replay_retired, 0x0000200, 0x0000001),
  51. P4_GEN_PEBS_BIND(split_load_retired, 0x0000400, 0x0000001),
  52. P4_GEN_PEBS_BIND(split_store_retired, 0x0000400, 0x0000002),
  53. };
  54. /*
  55. * Note that we don't use CCCR1 here, there is an
  56. * exception for P4_BSQ_ALLOCATION but we just have
  57. * no workaround
  58. *
  59. * consider this binding as resources which particular
  60. * event may borrow, it doesn't contain EventMask,
  61. * Tags and friends -- they are left to a caller
  62. */
  63. static struct p4_event_bind p4_event_bind_map[] = {
  64. [P4_EVENT_TC_DELIVER_MODE] = {
  65. .opcode = P4_OPCODE(P4_EVENT_TC_DELIVER_MODE),
  66. .escr_msr = { MSR_P4_TC_ESCR0, MSR_P4_TC_ESCR1 },
  67. .escr_emask =
  68. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, DD) |
  69. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, DB) |
  70. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, DI) |
  71. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, BD) |
  72. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, BB) |
  73. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, BI) |
  74. P4_ESCR_EMASK_BIT(P4_EVENT_TC_DELIVER_MODE, ID),
  75. .shared = 1,
  76. .cntr = { {4, 5, -1}, {6, 7, -1} },
  77. },
  78. [P4_EVENT_BPU_FETCH_REQUEST] = {
  79. .opcode = P4_OPCODE(P4_EVENT_BPU_FETCH_REQUEST),
  80. .escr_msr = { MSR_P4_BPU_ESCR0, MSR_P4_BPU_ESCR1 },
  81. .escr_emask =
  82. P4_ESCR_EMASK_BIT(P4_EVENT_BPU_FETCH_REQUEST, TCMISS),
  83. .cntr = { {0, -1, -1}, {2, -1, -1} },
  84. },
  85. [P4_EVENT_ITLB_REFERENCE] = {
  86. .opcode = P4_OPCODE(P4_EVENT_ITLB_REFERENCE),
  87. .escr_msr = { MSR_P4_ITLB_ESCR0, MSR_P4_ITLB_ESCR1 },
  88. .escr_emask =
  89. P4_ESCR_EMASK_BIT(P4_EVENT_ITLB_REFERENCE, HIT) |
  90. P4_ESCR_EMASK_BIT(P4_EVENT_ITLB_REFERENCE, MISS) |
  91. P4_ESCR_EMASK_BIT(P4_EVENT_ITLB_REFERENCE, HIT_UK),
  92. .cntr = { {0, -1, -1}, {2, -1, -1} },
  93. },
  94. [P4_EVENT_MEMORY_CANCEL] = {
  95. .opcode = P4_OPCODE(P4_EVENT_MEMORY_CANCEL),
  96. .escr_msr = { MSR_P4_DAC_ESCR0, MSR_P4_DAC_ESCR1 },
  97. .escr_emask =
  98. P4_ESCR_EMASK_BIT(P4_EVENT_MEMORY_CANCEL, ST_RB_FULL) |
  99. P4_ESCR_EMASK_BIT(P4_EVENT_MEMORY_CANCEL, 64K_CONF),
  100. .cntr = { {8, 9, -1}, {10, 11, -1} },
  101. },
  102. [P4_EVENT_MEMORY_COMPLETE] = {
  103. .opcode = P4_OPCODE(P4_EVENT_MEMORY_COMPLETE),
  104. .escr_msr = { MSR_P4_SAAT_ESCR0 , MSR_P4_SAAT_ESCR1 },
  105. .escr_emask =
  106. P4_ESCR_EMASK_BIT(P4_EVENT_MEMORY_COMPLETE, LSC) |
  107. P4_ESCR_EMASK_BIT(P4_EVENT_MEMORY_COMPLETE, SSC),
  108. .cntr = { {8, 9, -1}, {10, 11, -1} },
  109. },
  110. [P4_EVENT_LOAD_PORT_REPLAY] = {
  111. .opcode = P4_OPCODE(P4_EVENT_LOAD_PORT_REPLAY),
  112. .escr_msr = { MSR_P4_SAAT_ESCR0, MSR_P4_SAAT_ESCR1 },
  113. .escr_emask =
  114. P4_ESCR_EMASK_BIT(P4_EVENT_LOAD_PORT_REPLAY, SPLIT_LD),
  115. .cntr = { {8, 9, -1}, {10, 11, -1} },
  116. },
  117. [P4_EVENT_STORE_PORT_REPLAY] = {
  118. .opcode = P4_OPCODE(P4_EVENT_STORE_PORT_REPLAY),
  119. .escr_msr = { MSR_P4_SAAT_ESCR0 , MSR_P4_SAAT_ESCR1 },
  120. .escr_emask =
  121. P4_ESCR_EMASK_BIT(P4_EVENT_STORE_PORT_REPLAY, SPLIT_ST),
  122. .cntr = { {8, 9, -1}, {10, 11, -1} },
  123. },
  124. [P4_EVENT_MOB_LOAD_REPLAY] = {
  125. .opcode = P4_OPCODE(P4_EVENT_MOB_LOAD_REPLAY),
  126. .escr_msr = { MSR_P4_MOB_ESCR0, MSR_P4_MOB_ESCR1 },
  127. .escr_emask =
  128. P4_ESCR_EMASK_BIT(P4_EVENT_MOB_LOAD_REPLAY, NO_STA) |
  129. P4_ESCR_EMASK_BIT(P4_EVENT_MOB_LOAD_REPLAY, NO_STD) |
  130. P4_ESCR_EMASK_BIT(P4_EVENT_MOB_LOAD_REPLAY, PARTIAL_DATA) |
  131. P4_ESCR_EMASK_BIT(P4_EVENT_MOB_LOAD_REPLAY, UNALGN_ADDR),
  132. .cntr = { {0, -1, -1}, {2, -1, -1} },
  133. },
  134. [P4_EVENT_PAGE_WALK_TYPE] = {
  135. .opcode = P4_OPCODE(P4_EVENT_PAGE_WALK_TYPE),
  136. .escr_msr = { MSR_P4_PMH_ESCR0, MSR_P4_PMH_ESCR1 },
  137. .escr_emask =
  138. P4_ESCR_EMASK_BIT(P4_EVENT_PAGE_WALK_TYPE, DTMISS) |
  139. P4_ESCR_EMASK_BIT(P4_EVENT_PAGE_WALK_TYPE, ITMISS),
  140. .shared = 1,
  141. .cntr = { {0, -1, -1}, {2, -1, -1} },
  142. },
  143. [P4_EVENT_BSQ_CACHE_REFERENCE] = {
  144. .opcode = P4_OPCODE(P4_EVENT_BSQ_CACHE_REFERENCE),
  145. .escr_msr = { MSR_P4_BSU_ESCR0, MSR_P4_BSU_ESCR1 },
  146. .escr_emask =
  147. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITS) |
  148. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITE) |
  149. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITM) |
  150. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITS) |
  151. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITE) |
  152. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITM) |
  153. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_MISS) |
  154. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_MISS) |
  155. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, WR_2ndL_MISS),
  156. .cntr = { {0, -1, -1}, {2, -1, -1} },
  157. },
  158. [P4_EVENT_IOQ_ALLOCATION] = {
  159. .opcode = P4_OPCODE(P4_EVENT_IOQ_ALLOCATION),
  160. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  161. .escr_emask =
  162. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, DEFAULT) |
  163. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, ALL_READ) |
  164. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, ALL_WRITE) |
  165. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, MEM_UC) |
  166. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, MEM_WC) |
  167. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, MEM_WT) |
  168. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, MEM_WP) |
  169. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, MEM_WB) |
  170. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, OWN) |
  171. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, OTHER) |
  172. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ALLOCATION, PREFETCH),
  173. .cntr = { {0, -1, -1}, {2, -1, -1} },
  174. },
  175. [P4_EVENT_IOQ_ACTIVE_ENTRIES] = { /* shared ESCR */
  176. .opcode = P4_OPCODE(P4_EVENT_IOQ_ACTIVE_ENTRIES),
  177. .escr_msr = { MSR_P4_FSB_ESCR1, MSR_P4_FSB_ESCR1 },
  178. .escr_emask =
  179. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, DEFAULT) |
  180. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, ALL_READ) |
  181. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, ALL_WRITE) |
  182. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_UC) |
  183. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WC) |
  184. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WT) |
  185. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WP) |
  186. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, MEM_WB) |
  187. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, OWN) |
  188. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, OTHER) |
  189. P4_ESCR_EMASK_BIT(P4_EVENT_IOQ_ACTIVE_ENTRIES, PREFETCH),
  190. .cntr = { {2, -1, -1}, {3, -1, -1} },
  191. },
  192. [P4_EVENT_FSB_DATA_ACTIVITY] = {
  193. .opcode = P4_OPCODE(P4_EVENT_FSB_DATA_ACTIVITY),
  194. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  195. .escr_emask =
  196. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_DRV) |
  197. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_OWN) |
  198. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_OTHER) |
  199. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DBSY_DRV) |
  200. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DBSY_OWN) |
  201. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DBSY_OTHER),
  202. .shared = 1,
  203. .cntr = { {0, -1, -1}, {2, -1, -1} },
  204. },
  205. [P4_EVENT_BSQ_ALLOCATION] = { /* shared ESCR, broken CCCR1 */
  206. .opcode = P4_OPCODE(P4_EVENT_BSQ_ALLOCATION),
  207. .escr_msr = { MSR_P4_BSU_ESCR0, MSR_P4_BSU_ESCR0 },
  208. .escr_emask =
  209. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_TYPE0) |
  210. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_TYPE1) |
  211. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_LEN0) |
  212. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_LEN1) |
  213. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_IO_TYPE) |
  214. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_LOCK_TYPE) |
  215. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_CACHE_TYPE) |
  216. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_SPLIT_TYPE) |
  217. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_DEM_TYPE) |
  218. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, REQ_ORD_TYPE) |
  219. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, MEM_TYPE0) |
  220. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, MEM_TYPE1) |
  221. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ALLOCATION, MEM_TYPE2),
  222. .cntr = { {0, -1, -1}, {1, -1, -1} },
  223. },
  224. [P4_EVENT_BSQ_ACTIVE_ENTRIES] = { /* shared ESCR */
  225. .opcode = P4_OPCODE(P4_EVENT_BSQ_ACTIVE_ENTRIES),
  226. .escr_msr = { MSR_P4_BSU_ESCR1 , MSR_P4_BSU_ESCR1 },
  227. .escr_emask =
  228. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_TYPE0) |
  229. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_TYPE1) |
  230. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_LEN0) |
  231. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_LEN1) |
  232. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_IO_TYPE) |
  233. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_LOCK_TYPE) |
  234. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_CACHE_TYPE) |
  235. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_SPLIT_TYPE) |
  236. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_DEM_TYPE) |
  237. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, REQ_ORD_TYPE) |
  238. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, MEM_TYPE0) |
  239. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, MEM_TYPE1) |
  240. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_ACTIVE_ENTRIES, MEM_TYPE2),
  241. .cntr = { {2, -1, -1}, {3, -1, -1} },
  242. },
  243. [P4_EVENT_SSE_INPUT_ASSIST] = {
  244. .opcode = P4_OPCODE(P4_EVENT_SSE_INPUT_ASSIST),
  245. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  246. .escr_emask =
  247. P4_ESCR_EMASK_BIT(P4_EVENT_SSE_INPUT_ASSIST, ALL),
  248. .shared = 1,
  249. .cntr = { {8, 9, -1}, {10, 11, -1} },
  250. },
  251. [P4_EVENT_PACKED_SP_UOP] = {
  252. .opcode = P4_OPCODE(P4_EVENT_PACKED_SP_UOP),
  253. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  254. .escr_emask =
  255. P4_ESCR_EMASK_BIT(P4_EVENT_PACKED_SP_UOP, ALL),
  256. .shared = 1,
  257. .cntr = { {8, 9, -1}, {10, 11, -1} },
  258. },
  259. [P4_EVENT_PACKED_DP_UOP] = {
  260. .opcode = P4_OPCODE(P4_EVENT_PACKED_DP_UOP),
  261. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  262. .escr_emask =
  263. P4_ESCR_EMASK_BIT(P4_EVENT_PACKED_DP_UOP, ALL),
  264. .shared = 1,
  265. .cntr = { {8, 9, -1}, {10, 11, -1} },
  266. },
  267. [P4_EVENT_SCALAR_SP_UOP] = {
  268. .opcode = P4_OPCODE(P4_EVENT_SCALAR_SP_UOP),
  269. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  270. .escr_emask =
  271. P4_ESCR_EMASK_BIT(P4_EVENT_SCALAR_SP_UOP, ALL),
  272. .shared = 1,
  273. .cntr = { {8, 9, -1}, {10, 11, -1} },
  274. },
  275. [P4_EVENT_SCALAR_DP_UOP] = {
  276. .opcode = P4_OPCODE(P4_EVENT_SCALAR_DP_UOP),
  277. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  278. .escr_emask =
  279. P4_ESCR_EMASK_BIT(P4_EVENT_SCALAR_DP_UOP, ALL),
  280. .shared = 1,
  281. .cntr = { {8, 9, -1}, {10, 11, -1} },
  282. },
  283. [P4_EVENT_64BIT_MMX_UOP] = {
  284. .opcode = P4_OPCODE(P4_EVENT_64BIT_MMX_UOP),
  285. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  286. .escr_emask =
  287. P4_ESCR_EMASK_BIT(P4_EVENT_64BIT_MMX_UOP, ALL),
  288. .shared = 1,
  289. .cntr = { {8, 9, -1}, {10, 11, -1} },
  290. },
  291. [P4_EVENT_128BIT_MMX_UOP] = {
  292. .opcode = P4_OPCODE(P4_EVENT_128BIT_MMX_UOP),
  293. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  294. .escr_emask =
  295. P4_ESCR_EMASK_BIT(P4_EVENT_128BIT_MMX_UOP, ALL),
  296. .shared = 1,
  297. .cntr = { {8, 9, -1}, {10, 11, -1} },
  298. },
  299. [P4_EVENT_X87_FP_UOP] = {
  300. .opcode = P4_OPCODE(P4_EVENT_X87_FP_UOP),
  301. .escr_msr = { MSR_P4_FIRM_ESCR0, MSR_P4_FIRM_ESCR1 },
  302. .escr_emask =
  303. P4_ESCR_EMASK_BIT(P4_EVENT_X87_FP_UOP, ALL),
  304. .shared = 1,
  305. .cntr = { {8, 9, -1}, {10, 11, -1} },
  306. },
  307. [P4_EVENT_TC_MISC] = {
  308. .opcode = P4_OPCODE(P4_EVENT_TC_MISC),
  309. .escr_msr = { MSR_P4_TC_ESCR0, MSR_P4_TC_ESCR1 },
  310. .escr_emask =
  311. P4_ESCR_EMASK_BIT(P4_EVENT_TC_MISC, FLUSH),
  312. .cntr = { {4, 5, -1}, {6, 7, -1} },
  313. },
  314. [P4_EVENT_GLOBAL_POWER_EVENTS] = {
  315. .opcode = P4_OPCODE(P4_EVENT_GLOBAL_POWER_EVENTS),
  316. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  317. .escr_emask =
  318. P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING),
  319. .cntr = { {0, -1, -1}, {2, -1, -1} },
  320. },
  321. [P4_EVENT_TC_MS_XFER] = {
  322. .opcode = P4_OPCODE(P4_EVENT_TC_MS_XFER),
  323. .escr_msr = { MSR_P4_MS_ESCR0, MSR_P4_MS_ESCR1 },
  324. .escr_emask =
  325. P4_ESCR_EMASK_BIT(P4_EVENT_TC_MS_XFER, CISC),
  326. .cntr = { {4, 5, -1}, {6, 7, -1} },
  327. },
  328. [P4_EVENT_UOP_QUEUE_WRITES] = {
  329. .opcode = P4_OPCODE(P4_EVENT_UOP_QUEUE_WRITES),
  330. .escr_msr = { MSR_P4_MS_ESCR0, MSR_P4_MS_ESCR1 },
  331. .escr_emask =
  332. P4_ESCR_EMASK_BIT(P4_EVENT_UOP_QUEUE_WRITES, FROM_TC_BUILD) |
  333. P4_ESCR_EMASK_BIT(P4_EVENT_UOP_QUEUE_WRITES, FROM_TC_DELIVER) |
  334. P4_ESCR_EMASK_BIT(P4_EVENT_UOP_QUEUE_WRITES, FROM_ROM),
  335. .cntr = { {4, 5, -1}, {6, 7, -1} },
  336. },
  337. [P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE] = {
  338. .opcode = P4_OPCODE(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE),
  339. .escr_msr = { MSR_P4_TBPU_ESCR0 , MSR_P4_TBPU_ESCR0 },
  340. .escr_emask =
  341. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, CONDITIONAL) |
  342. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, CALL) |
  343. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, RETURN) |
  344. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_MISPRED_BRANCH_TYPE, INDIRECT),
  345. .cntr = { {4, 5, -1}, {6, 7, -1} },
  346. },
  347. [P4_EVENT_RETIRED_BRANCH_TYPE] = {
  348. .opcode = P4_OPCODE(P4_EVENT_RETIRED_BRANCH_TYPE),
  349. .escr_msr = { MSR_P4_TBPU_ESCR0 , MSR_P4_TBPU_ESCR1 },
  350. .escr_emask =
  351. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, CONDITIONAL) |
  352. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, CALL) |
  353. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, RETURN) |
  354. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, INDIRECT),
  355. .cntr = { {4, 5, -1}, {6, 7, -1} },
  356. },
  357. [P4_EVENT_RESOURCE_STALL] = {
  358. .opcode = P4_OPCODE(P4_EVENT_RESOURCE_STALL),
  359. .escr_msr = { MSR_P4_ALF_ESCR0, MSR_P4_ALF_ESCR1 },
  360. .escr_emask =
  361. P4_ESCR_EMASK_BIT(P4_EVENT_RESOURCE_STALL, SBFULL),
  362. .cntr = { {12, 13, 16}, {14, 15, 17} },
  363. },
  364. [P4_EVENT_WC_BUFFER] = {
  365. .opcode = P4_OPCODE(P4_EVENT_WC_BUFFER),
  366. .escr_msr = { MSR_P4_DAC_ESCR0, MSR_P4_DAC_ESCR1 },
  367. .escr_emask =
  368. P4_ESCR_EMASK_BIT(P4_EVENT_WC_BUFFER, WCB_EVICTS) |
  369. P4_ESCR_EMASK_BIT(P4_EVENT_WC_BUFFER, WCB_FULL_EVICTS),
  370. .shared = 1,
  371. .cntr = { {8, 9, -1}, {10, 11, -1} },
  372. },
  373. [P4_EVENT_B2B_CYCLES] = {
  374. .opcode = P4_OPCODE(P4_EVENT_B2B_CYCLES),
  375. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  376. .escr_emask = 0,
  377. .cntr = { {0, -1, -1}, {2, -1, -1} },
  378. },
  379. [P4_EVENT_BNR] = {
  380. .opcode = P4_OPCODE(P4_EVENT_BNR),
  381. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  382. .escr_emask = 0,
  383. .cntr = { {0, -1, -1}, {2, -1, -1} },
  384. },
  385. [P4_EVENT_SNOOP] = {
  386. .opcode = P4_OPCODE(P4_EVENT_SNOOP),
  387. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  388. .escr_emask = 0,
  389. .cntr = { {0, -1, -1}, {2, -1, -1} },
  390. },
  391. [P4_EVENT_RESPONSE] = {
  392. .opcode = P4_OPCODE(P4_EVENT_RESPONSE),
  393. .escr_msr = { MSR_P4_FSB_ESCR0, MSR_P4_FSB_ESCR1 },
  394. .escr_emask = 0,
  395. .cntr = { {0, -1, -1}, {2, -1, -1} },
  396. },
  397. [P4_EVENT_FRONT_END_EVENT] = {
  398. .opcode = P4_OPCODE(P4_EVENT_FRONT_END_EVENT),
  399. .escr_msr = { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
  400. .escr_emask =
  401. P4_ESCR_EMASK_BIT(P4_EVENT_FRONT_END_EVENT, NBOGUS) |
  402. P4_ESCR_EMASK_BIT(P4_EVENT_FRONT_END_EVENT, BOGUS),
  403. .cntr = { {12, 13, 16}, {14, 15, 17} },
  404. },
  405. [P4_EVENT_EXECUTION_EVENT] = {
  406. .opcode = P4_OPCODE(P4_EVENT_EXECUTION_EVENT),
  407. .escr_msr = { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
  408. .escr_emask =
  409. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS0) |
  410. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS1) |
  411. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS2) |
  412. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS3) |
  413. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS0) |
  414. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS1) |
  415. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS2) |
  416. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS3),
  417. .cntr = { {12, 13, 16}, {14, 15, 17} },
  418. },
  419. [P4_EVENT_REPLAY_EVENT] = {
  420. .opcode = P4_OPCODE(P4_EVENT_REPLAY_EVENT),
  421. .escr_msr = { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
  422. .escr_emask =
  423. P4_ESCR_EMASK_BIT(P4_EVENT_REPLAY_EVENT, NBOGUS) |
  424. P4_ESCR_EMASK_BIT(P4_EVENT_REPLAY_EVENT, BOGUS),
  425. .cntr = { {12, 13, 16}, {14, 15, 17} },
  426. },
  427. [P4_EVENT_INSTR_RETIRED] = {
  428. .opcode = P4_OPCODE(P4_EVENT_INSTR_RETIRED),
  429. .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
  430. .escr_emask =
  431. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, NBOGUSNTAG) |
  432. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, NBOGUSTAG) |
  433. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, BOGUSNTAG) |
  434. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, BOGUSTAG),
  435. .cntr = { {12, 13, 16}, {14, 15, 17} },
  436. },
  437. [P4_EVENT_UOPS_RETIRED] = {
  438. .opcode = P4_OPCODE(P4_EVENT_UOPS_RETIRED),
  439. .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
  440. .escr_emask =
  441. P4_ESCR_EMASK_BIT(P4_EVENT_UOPS_RETIRED, NBOGUS) |
  442. P4_ESCR_EMASK_BIT(P4_EVENT_UOPS_RETIRED, BOGUS),
  443. .cntr = { {12, 13, 16}, {14, 15, 17} },
  444. },
  445. [P4_EVENT_UOP_TYPE] = {
  446. .opcode = P4_OPCODE(P4_EVENT_UOP_TYPE),
  447. .escr_msr = { MSR_P4_RAT_ESCR0, MSR_P4_RAT_ESCR1 },
  448. .escr_emask =
  449. P4_ESCR_EMASK_BIT(P4_EVENT_UOP_TYPE, TAGLOADS) |
  450. P4_ESCR_EMASK_BIT(P4_EVENT_UOP_TYPE, TAGSTORES),
  451. .cntr = { {12, 13, 16}, {14, 15, 17} },
  452. },
  453. [P4_EVENT_BRANCH_RETIRED] = {
  454. .opcode = P4_OPCODE(P4_EVENT_BRANCH_RETIRED),
  455. .escr_msr = { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
  456. .escr_emask =
  457. P4_ESCR_EMASK_BIT(P4_EVENT_BRANCH_RETIRED, MMNP) |
  458. P4_ESCR_EMASK_BIT(P4_EVENT_BRANCH_RETIRED, MMNM) |
  459. P4_ESCR_EMASK_BIT(P4_EVENT_BRANCH_RETIRED, MMTP) |
  460. P4_ESCR_EMASK_BIT(P4_EVENT_BRANCH_RETIRED, MMTM),
  461. .cntr = { {12, 13, 16}, {14, 15, 17} },
  462. },
  463. [P4_EVENT_MISPRED_BRANCH_RETIRED] = {
  464. .opcode = P4_OPCODE(P4_EVENT_MISPRED_BRANCH_RETIRED),
  465. .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
  466. .escr_emask =
  467. P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS),
  468. .cntr = { {12, 13, 16}, {14, 15, 17} },
  469. },
  470. [P4_EVENT_X87_ASSIST] = {
  471. .opcode = P4_OPCODE(P4_EVENT_X87_ASSIST),
  472. .escr_msr = { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
  473. .escr_emask =
  474. P4_ESCR_EMASK_BIT(P4_EVENT_X87_ASSIST, FPSU) |
  475. P4_ESCR_EMASK_BIT(P4_EVENT_X87_ASSIST, FPSO) |
  476. P4_ESCR_EMASK_BIT(P4_EVENT_X87_ASSIST, POAO) |
  477. P4_ESCR_EMASK_BIT(P4_EVENT_X87_ASSIST, POAU) |
  478. P4_ESCR_EMASK_BIT(P4_EVENT_X87_ASSIST, PREA),
  479. .cntr = { {12, 13, 16}, {14, 15, 17} },
  480. },
  481. [P4_EVENT_MACHINE_CLEAR] = {
  482. .opcode = P4_OPCODE(P4_EVENT_MACHINE_CLEAR),
  483. .escr_msr = { MSR_P4_CRU_ESCR2, MSR_P4_CRU_ESCR3 },
  484. .escr_emask =
  485. P4_ESCR_EMASK_BIT(P4_EVENT_MACHINE_CLEAR, CLEAR) |
  486. P4_ESCR_EMASK_BIT(P4_EVENT_MACHINE_CLEAR, MOCLEAR) |
  487. P4_ESCR_EMASK_BIT(P4_EVENT_MACHINE_CLEAR, SMCLEAR),
  488. .cntr = { {12, 13, 16}, {14, 15, 17} },
  489. },
  490. [P4_EVENT_INSTR_COMPLETED] = {
  491. .opcode = P4_OPCODE(P4_EVENT_INSTR_COMPLETED),
  492. .escr_msr = { MSR_P4_CRU_ESCR0, MSR_P4_CRU_ESCR1 },
  493. .escr_emask =
  494. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_COMPLETED, NBOGUS) |
  495. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_COMPLETED, BOGUS),
  496. .cntr = { {12, 13, 16}, {14, 15, 17} },
  497. },
  498. };
  499. #define P4_GEN_CACHE_EVENT(event, bit, metric) \
  500. p4_config_pack_escr(P4_ESCR_EVENT(event) | \
  501. P4_ESCR_EMASK_BIT(event, bit)) | \
  502. p4_config_pack_cccr(metric | \
  503. P4_CCCR_ESEL(P4_OPCODE_ESEL(P4_OPCODE(event))))
  504. static __initconst const u64 p4_hw_cache_event_ids
  505. [PERF_COUNT_HW_CACHE_MAX]
  506. [PERF_COUNT_HW_CACHE_OP_MAX]
  507. [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  508. {
  509. [ C(L1D ) ] = {
  510. [ C(OP_READ) ] = {
  511. [ C(RESULT_ACCESS) ] = 0x0,
  512. [ C(RESULT_MISS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
  513. P4_PEBS_METRIC__1stl_cache_load_miss_retired),
  514. },
  515. },
  516. [ C(LL ) ] = {
  517. [ C(OP_READ) ] = {
  518. [ C(RESULT_ACCESS) ] = 0x0,
  519. [ C(RESULT_MISS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
  520. P4_PEBS_METRIC__2ndl_cache_load_miss_retired),
  521. },
  522. },
  523. [ C(DTLB) ] = {
  524. [ C(OP_READ) ] = {
  525. [ C(RESULT_ACCESS) ] = 0x0,
  526. [ C(RESULT_MISS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
  527. P4_PEBS_METRIC__dtlb_load_miss_retired),
  528. },
  529. [ C(OP_WRITE) ] = {
  530. [ C(RESULT_ACCESS) ] = 0x0,
  531. [ C(RESULT_MISS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_REPLAY_EVENT, NBOGUS,
  532. P4_PEBS_METRIC__dtlb_store_miss_retired),
  533. },
  534. },
  535. [ C(ITLB) ] = {
  536. [ C(OP_READ) ] = {
  537. [ C(RESULT_ACCESS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_ITLB_REFERENCE, HIT,
  538. P4_PEBS_METRIC__none),
  539. [ C(RESULT_MISS) ] = P4_GEN_CACHE_EVENT(P4_EVENT_ITLB_REFERENCE, MISS,
  540. P4_PEBS_METRIC__none),
  541. },
  542. [ C(OP_WRITE) ] = {
  543. [ C(RESULT_ACCESS) ] = -1,
  544. [ C(RESULT_MISS) ] = -1,
  545. },
  546. [ C(OP_PREFETCH) ] = {
  547. [ C(RESULT_ACCESS) ] = -1,
  548. [ C(RESULT_MISS) ] = -1,
  549. },
  550. },
  551. [ C(NODE) ] = {
  552. [ C(OP_READ) ] = {
  553. [ C(RESULT_ACCESS) ] = -1,
  554. [ C(RESULT_MISS) ] = -1,
  555. },
  556. [ C(OP_WRITE) ] = {
  557. [ C(RESULT_ACCESS) ] = -1,
  558. [ C(RESULT_MISS) ] = -1,
  559. },
  560. [ C(OP_PREFETCH) ] = {
  561. [ C(RESULT_ACCESS) ] = -1,
  562. [ C(RESULT_MISS) ] = -1,
  563. },
  564. },
  565. };
  566. /*
  567. * Because of Netburst being quite restricted in how many
  568. * identical events may run simultaneously, we introduce event aliases,
  569. * ie the different events which have the same functionality but
  570. * utilize non-intersected resources (ESCR/CCCR/counter registers).
  571. *
  572. * This allow us to relax restrictions a bit and run two or more
  573. * identical events together.
  574. *
  575. * Never set any custom internal bits such as P4_CONFIG_HT,
  576. * P4_CONFIG_ALIASABLE or bits for P4_PEBS_METRIC, they are
  577. * either up to date automatically or not applicable at all.
  578. */
  579. static struct p4_event_alias {
  580. u64 original;
  581. u64 alternative;
  582. } p4_event_aliases[] = {
  583. {
  584. /*
  585. * Non-halted cycles can be substituted with non-sleeping cycles (see
  586. * Intel SDM Vol3b for details). We need this alias to be able
  587. * to run nmi-watchdog and 'perf top' (or any other user space tool
  588. * which is interested in running PERF_COUNT_HW_CPU_CYCLES)
  589. * simultaneously.
  590. */
  591. .original =
  592. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_GLOBAL_POWER_EVENTS) |
  593. P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)),
  594. .alternative =
  595. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_EXECUTION_EVENT) |
  596. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS0)|
  597. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS1)|
  598. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS2)|
  599. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, NBOGUS3)|
  600. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS0) |
  601. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS1) |
  602. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS2) |
  603. P4_ESCR_EMASK_BIT(P4_EVENT_EXECUTION_EVENT, BOGUS3))|
  604. p4_config_pack_cccr(P4_CCCR_THRESHOLD(15) | P4_CCCR_COMPLEMENT |
  605. P4_CCCR_COMPARE),
  606. },
  607. };
  608. static u64 p4_get_alias_event(u64 config)
  609. {
  610. u64 config_match;
  611. int i;
  612. /*
  613. * Only event with special mark is allowed,
  614. * we're to be sure it didn't come as malformed
  615. * RAW event.
  616. */
  617. if (!(config & P4_CONFIG_ALIASABLE))
  618. return 0;
  619. config_match = config & P4_CONFIG_EVENT_ALIAS_MASK;
  620. for (i = 0; i < ARRAY_SIZE(p4_event_aliases); i++) {
  621. if (config_match == p4_event_aliases[i].original) {
  622. config_match = p4_event_aliases[i].alternative;
  623. break;
  624. } else if (config_match == p4_event_aliases[i].alternative) {
  625. config_match = p4_event_aliases[i].original;
  626. break;
  627. }
  628. }
  629. if (i >= ARRAY_SIZE(p4_event_aliases))
  630. return 0;
  631. return config_match | (config & P4_CONFIG_EVENT_ALIAS_IMMUTABLE_BITS);
  632. }
  633. static u64 p4_general_events[PERF_COUNT_HW_MAX] = {
  634. /* non-halted CPU clocks */
  635. [PERF_COUNT_HW_CPU_CYCLES] =
  636. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_GLOBAL_POWER_EVENTS) |
  637. P4_ESCR_EMASK_BIT(P4_EVENT_GLOBAL_POWER_EVENTS, RUNNING)) |
  638. P4_CONFIG_ALIASABLE,
  639. /*
  640. * retired instructions
  641. * in a sake of simplicity we don't use the FSB tagging
  642. */
  643. [PERF_COUNT_HW_INSTRUCTIONS] =
  644. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_INSTR_RETIRED) |
  645. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, NBOGUSNTAG) |
  646. P4_ESCR_EMASK_BIT(P4_EVENT_INSTR_RETIRED, BOGUSNTAG)),
  647. /* cache hits */
  648. [PERF_COUNT_HW_CACHE_REFERENCES] =
  649. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_BSQ_CACHE_REFERENCE) |
  650. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITS) |
  651. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITE) |
  652. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_HITM) |
  653. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITS) |
  654. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITE) |
  655. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_HITM)),
  656. /* cache misses */
  657. [PERF_COUNT_HW_CACHE_MISSES] =
  658. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_BSQ_CACHE_REFERENCE) |
  659. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_2ndL_MISS) |
  660. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, RD_3rdL_MISS) |
  661. P4_ESCR_EMASK_BIT(P4_EVENT_BSQ_CACHE_REFERENCE, WR_2ndL_MISS)),
  662. /* branch instructions retired */
  663. [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] =
  664. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_RETIRED_BRANCH_TYPE) |
  665. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, CONDITIONAL) |
  666. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, CALL) |
  667. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, RETURN) |
  668. P4_ESCR_EMASK_BIT(P4_EVENT_RETIRED_BRANCH_TYPE, INDIRECT)),
  669. /* mispredicted branches retired */
  670. [PERF_COUNT_HW_BRANCH_MISSES] =
  671. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_MISPRED_BRANCH_RETIRED) |
  672. P4_ESCR_EMASK_BIT(P4_EVENT_MISPRED_BRANCH_RETIRED, NBOGUS)),
  673. /* bus ready clocks (cpu is driving #DRDY_DRV\#DRDY_OWN): */
  674. [PERF_COUNT_HW_BUS_CYCLES] =
  675. p4_config_pack_escr(P4_ESCR_EVENT(P4_EVENT_FSB_DATA_ACTIVITY) |
  676. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_DRV) |
  677. P4_ESCR_EMASK_BIT(P4_EVENT_FSB_DATA_ACTIVITY, DRDY_OWN)) |
  678. p4_config_pack_cccr(P4_CCCR_EDGE | P4_CCCR_COMPARE),
  679. };
  680. static struct p4_event_bind *p4_config_get_bind(u64 config)
  681. {
  682. unsigned int evnt = p4_config_unpack_event(config);
  683. struct p4_event_bind *bind = NULL;
  684. if (evnt < ARRAY_SIZE(p4_event_bind_map))
  685. bind = &p4_event_bind_map[evnt];
  686. return bind;
  687. }
  688. static u64 p4_pmu_event_map(int hw_event)
  689. {
  690. struct p4_event_bind *bind;
  691. unsigned int esel;
  692. u64 config;
  693. config = p4_general_events[hw_event];
  694. bind = p4_config_get_bind(config);
  695. esel = P4_OPCODE_ESEL(bind->opcode);
  696. config |= p4_config_pack_cccr(P4_CCCR_ESEL(esel));
  697. return config;
  698. }
  699. /* check cpu model specifics */
  700. static bool p4_event_match_cpu_model(unsigned int event_idx)
  701. {
  702. /* INSTR_COMPLETED event only exist for model 3, 4, 6 (Prescott) */
  703. if (event_idx == P4_EVENT_INSTR_COMPLETED) {
  704. if (boot_cpu_data.x86_model != 3 &&
  705. boot_cpu_data.x86_model != 4 &&
  706. boot_cpu_data.x86_model != 6)
  707. return false;
  708. }
  709. /*
  710. * For info
  711. * - IQ_ESCR0, IQ_ESCR1 only for models 1 and 2
  712. */
  713. return true;
  714. }
  715. static int p4_validate_raw_event(struct perf_event *event)
  716. {
  717. unsigned int v, emask;
  718. /* User data may have out-of-bound event index */
  719. v = p4_config_unpack_event(event->attr.config);
  720. if (v >= ARRAY_SIZE(p4_event_bind_map))
  721. return -EINVAL;
  722. /* It may be unsupported: */
  723. if (!p4_event_match_cpu_model(v))
  724. return -EINVAL;
  725. /*
  726. * NOTE: P4_CCCR_THREAD_ANY has not the same meaning as
  727. * in Architectural Performance Monitoring, it means not
  728. * on _which_ logical cpu to count but rather _when_, ie it
  729. * depends on logical cpu state -- count event if one cpu active,
  730. * none, both or any, so we just allow user to pass any value
  731. * desired.
  732. *
  733. * In turn we always set Tx_OS/Tx_USR bits bound to logical
  734. * cpu without their propagation to another cpu
  735. */
  736. /*
  737. * if an event is shared across the logical threads
  738. * the user needs special permissions to be able to use it
  739. */
  740. if (p4_ht_active() && p4_event_bind_map[v].shared) {
  741. v = perf_allow_cpu(&event->attr);
  742. if (v)
  743. return v;
  744. }
  745. /* ESCR EventMask bits may be invalid */
  746. emask = p4_config_unpack_escr(event->attr.config) & P4_ESCR_EVENTMASK_MASK;
  747. if (emask & ~p4_event_bind_map[v].escr_emask)
  748. return -EINVAL;
  749. /*
  750. * it may have some invalid PEBS bits
  751. */
  752. if (p4_config_pebs_has(event->attr.config, P4_PEBS_CONFIG_ENABLE))
  753. return -EINVAL;
  754. v = p4_config_unpack_metric(event->attr.config);
  755. if (v >= ARRAY_SIZE(p4_pebs_bind_map))
  756. return -EINVAL;
  757. return 0;
  758. }
  759. static int p4_hw_config(struct perf_event *event)
  760. {
  761. int cpu = get_cpu();
  762. int rc = 0;
  763. u32 escr, cccr;
  764. /*
  765. * the reason we use cpu that early is that: if we get scheduled
  766. * first time on the same cpu -- we will not need swap thread
  767. * specific flags in config (and will save some cpu cycles)
  768. */
  769. cccr = p4_default_cccr_conf(cpu);
  770. escr = p4_default_escr_conf(cpu, event->attr.exclude_kernel,
  771. event->attr.exclude_user);
  772. event->hw.config = p4_config_pack_escr(escr) |
  773. p4_config_pack_cccr(cccr);
  774. if (p4_ht_active() && p4_ht_thread(cpu))
  775. event->hw.config = p4_set_ht_bit(event->hw.config);
  776. if (event->attr.type == PERF_TYPE_RAW) {
  777. struct p4_event_bind *bind;
  778. unsigned int esel;
  779. /*
  780. * Clear bits we reserve to be managed by kernel itself
  781. * and never allowed from a user space
  782. */
  783. event->attr.config &= P4_CONFIG_MASK;
  784. rc = p4_validate_raw_event(event);
  785. if (rc)
  786. goto out;
  787. /*
  788. * Note that for RAW events we allow user to use P4_CCCR_RESERVED
  789. * bits since we keep additional info here (for cache events and etc)
  790. */
  791. event->hw.config |= event->attr.config;
  792. bind = p4_config_get_bind(event->attr.config);
  793. if (!bind) {
  794. rc = -EINVAL;
  795. goto out;
  796. }
  797. esel = P4_OPCODE_ESEL(bind->opcode);
  798. event->hw.config |= p4_config_pack_cccr(P4_CCCR_ESEL(esel));
  799. }
  800. rc = x86_setup_perfctr(event);
  801. out:
  802. put_cpu();
  803. return rc;
  804. }
  805. static inline int p4_pmu_clear_cccr_ovf(struct hw_perf_event *hwc)
  806. {
  807. u64 v;
  808. /* an official way for overflow indication */
  809. rdmsrl(hwc->config_base, v);
  810. if (v & P4_CCCR_OVF) {
  811. wrmsrl(hwc->config_base, v & ~P4_CCCR_OVF);
  812. return 1;
  813. }
  814. /*
  815. * In some circumstances the overflow might issue an NMI but did
  816. * not set P4_CCCR_OVF bit. Because a counter holds a negative value
  817. * we simply check for high bit being set, if it's cleared it means
  818. * the counter has reached zero value and continued counting before
  819. * real NMI signal was received:
  820. */
  821. rdmsrl(hwc->event_base, v);
  822. if (!(v & ARCH_P4_UNFLAGGED_BIT))
  823. return 1;
  824. return 0;
  825. }
  826. static void p4_pmu_disable_pebs(void)
  827. {
  828. /*
  829. * FIXME
  830. *
  831. * It's still allowed that two threads setup same cache
  832. * events so we can't simply clear metrics until we knew
  833. * no one is depending on us, so we need kind of counter
  834. * for "ReplayEvent" users.
  835. *
  836. * What is more complex -- RAW events, if user (for some
  837. * reason) will pass some cache event metric with improper
  838. * event opcode -- it's fine from hardware point of view
  839. * but completely nonsense from "meaning" of such action.
  840. *
  841. * So at moment let leave metrics turned on forever -- it's
  842. * ok for now but need to be revisited!
  843. *
  844. * (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, 0);
  845. * (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, 0);
  846. */
  847. }
  848. static inline void p4_pmu_disable_event(struct perf_event *event)
  849. {
  850. struct hw_perf_event *hwc = &event->hw;
  851. /*
  852. * If event gets disabled while counter is in overflowed
  853. * state we need to clear P4_CCCR_OVF, otherwise interrupt get
  854. * asserted again and again
  855. */
  856. (void)wrmsrl_safe(hwc->config_base,
  857. p4_config_unpack_cccr(hwc->config) & ~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED);
  858. }
  859. static void p4_pmu_disable_all(void)
  860. {
  861. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  862. int idx;
  863. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  864. struct perf_event *event = cpuc->events[idx];
  865. if (!test_bit(idx, cpuc->active_mask))
  866. continue;
  867. p4_pmu_disable_event(event);
  868. }
  869. p4_pmu_disable_pebs();
  870. }
  871. /* configuration must be valid */
  872. static void p4_pmu_enable_pebs(u64 config)
  873. {
  874. struct p4_pebs_bind *bind;
  875. unsigned int idx;
  876. BUILD_BUG_ON(P4_PEBS_METRIC__max > P4_PEBS_CONFIG_METRIC_MASK);
  877. idx = p4_config_unpack_metric(config);
  878. if (idx == P4_PEBS_METRIC__none)
  879. return;
  880. bind = &p4_pebs_bind_map[idx];
  881. (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, (u64)bind->metric_pebs);
  882. (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, (u64)bind->metric_vert);
  883. }
  884. static void __p4_pmu_enable_event(struct perf_event *event)
  885. {
  886. struct hw_perf_event *hwc = &event->hw;
  887. int thread = p4_ht_config_thread(hwc->config);
  888. u64 escr_conf = p4_config_unpack_escr(p4_clear_ht_bit(hwc->config));
  889. unsigned int idx = p4_config_unpack_event(hwc->config);
  890. struct p4_event_bind *bind;
  891. u64 escr_addr, cccr;
  892. bind = &p4_event_bind_map[idx];
  893. escr_addr = bind->escr_msr[thread];
  894. /*
  895. * - we dont support cascaded counters yet
  896. * - and counter 1 is broken (erratum)
  897. */
  898. WARN_ON_ONCE(p4_is_event_cascaded(hwc->config));
  899. WARN_ON_ONCE(hwc->idx == 1);
  900. /* we need a real Event value */
  901. escr_conf &= ~P4_ESCR_EVENT_MASK;
  902. escr_conf |= P4_ESCR_EVENT(P4_OPCODE_EVNT(bind->opcode));
  903. cccr = p4_config_unpack_cccr(hwc->config);
  904. /*
  905. * it could be Cache event so we need to write metrics
  906. * into additional MSRs
  907. */
  908. p4_pmu_enable_pebs(hwc->config);
  909. (void)wrmsrl_safe(escr_addr, escr_conf);
  910. (void)wrmsrl_safe(hwc->config_base,
  911. (cccr & ~P4_CCCR_RESERVED) | P4_CCCR_ENABLE);
  912. }
  913. static DEFINE_PER_CPU(unsigned long [BITS_TO_LONGS(X86_PMC_IDX_MAX)], p4_running);
  914. static void p4_pmu_enable_event(struct perf_event *event)
  915. {
  916. int idx = event->hw.idx;
  917. __set_bit(idx, per_cpu(p4_running, smp_processor_id()));
  918. __p4_pmu_enable_event(event);
  919. }
  920. static void p4_pmu_enable_all(int added)
  921. {
  922. struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
  923. int idx;
  924. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  925. struct perf_event *event = cpuc->events[idx];
  926. if (!test_bit(idx, cpuc->active_mask))
  927. continue;
  928. __p4_pmu_enable_event(event);
  929. }
  930. }
  931. static int p4_pmu_set_period(struct perf_event *event)
  932. {
  933. struct hw_perf_event *hwc = &event->hw;
  934. s64 left = this_cpu_read(pmc_prev_left[hwc->idx]);
  935. int ret;
  936. ret = x86_perf_event_set_period(event);
  937. if (hwc->event_base) {
  938. /*
  939. * This handles erratum N15 in intel doc 249199-029,
  940. * the counter may not be updated correctly on write
  941. * so we need a second write operation to do the trick
  942. * (the official workaround didn't work)
  943. *
  944. * the former idea is taken from OProfile code
  945. */
  946. wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
  947. }
  948. return ret;
  949. }
  950. static int p4_pmu_handle_irq(struct pt_regs *regs)
  951. {
  952. struct perf_sample_data data;
  953. struct cpu_hw_events *cpuc;
  954. struct perf_event *event;
  955. struct hw_perf_event *hwc;
  956. int idx, handled = 0;
  957. u64 val;
  958. cpuc = this_cpu_ptr(&cpu_hw_events);
  959. for (idx = 0; idx < x86_pmu.num_counters; idx++) {
  960. int overflow;
  961. if (!test_bit(idx, cpuc->active_mask)) {
  962. /* catch in-flight IRQs */
  963. if (__test_and_clear_bit(idx, per_cpu(p4_running, smp_processor_id())))
  964. handled++;
  965. continue;
  966. }
  967. event = cpuc->events[idx];
  968. hwc = &event->hw;
  969. WARN_ON_ONCE(hwc->idx != idx);
  970. /* it might be unflagged overflow */
  971. overflow = p4_pmu_clear_cccr_ovf(hwc);
  972. val = x86_perf_event_update(event);
  973. if (!overflow && (val & (1ULL << (x86_pmu.cntval_bits - 1))))
  974. continue;
  975. handled += overflow;
  976. /* event overflow for sure */
  977. perf_sample_data_init(&data, 0, hwc->last_period);
  978. if (!static_call(x86_pmu_set_period)(event))
  979. continue;
  980. if (perf_event_overflow(event, &data, regs))
  981. x86_pmu_stop(event, 0);
  982. }
  983. if (handled)
  984. inc_irq_stat(apic_perf_irqs);
  985. /*
  986. * When dealing with the unmasking of the LVTPC on P4 perf hw, it has
  987. * been observed that the OVF bit flag has to be cleared first _before_
  988. * the LVTPC can be unmasked.
  989. *
  990. * The reason is the NMI line will continue to be asserted while the OVF
  991. * bit is set. This causes a second NMI to generate if the LVTPC is
  992. * unmasked before the OVF bit is cleared, leading to unknown NMI
  993. * messages.
  994. */
  995. apic_write(APIC_LVTPC, APIC_DM_NMI);
  996. return handled;
  997. }
  998. /*
  999. * swap thread specific fields according to a thread
  1000. * we are going to run on
  1001. */
  1002. static void p4_pmu_swap_config_ts(struct hw_perf_event *hwc, int cpu)
  1003. {
  1004. u32 escr, cccr;
  1005. /*
  1006. * we either lucky and continue on same cpu or no HT support
  1007. */
  1008. if (!p4_should_swap_ts(hwc->config, cpu))
  1009. return;
  1010. /*
  1011. * the event is migrated from an another logical
  1012. * cpu, so we need to swap thread specific flags
  1013. */
  1014. escr = p4_config_unpack_escr(hwc->config);
  1015. cccr = p4_config_unpack_cccr(hwc->config);
  1016. if (p4_ht_thread(cpu)) {
  1017. cccr &= ~P4_CCCR_OVF_PMI_T0;
  1018. cccr |= P4_CCCR_OVF_PMI_T1;
  1019. if (escr & P4_ESCR_T0_OS) {
  1020. escr &= ~P4_ESCR_T0_OS;
  1021. escr |= P4_ESCR_T1_OS;
  1022. }
  1023. if (escr & P4_ESCR_T0_USR) {
  1024. escr &= ~P4_ESCR_T0_USR;
  1025. escr |= P4_ESCR_T1_USR;
  1026. }
  1027. hwc->config = p4_config_pack_escr(escr);
  1028. hwc->config |= p4_config_pack_cccr(cccr);
  1029. hwc->config |= P4_CONFIG_HT;
  1030. } else {
  1031. cccr &= ~P4_CCCR_OVF_PMI_T1;
  1032. cccr |= P4_CCCR_OVF_PMI_T0;
  1033. if (escr & P4_ESCR_T1_OS) {
  1034. escr &= ~P4_ESCR_T1_OS;
  1035. escr |= P4_ESCR_T0_OS;
  1036. }
  1037. if (escr & P4_ESCR_T1_USR) {
  1038. escr &= ~P4_ESCR_T1_USR;
  1039. escr |= P4_ESCR_T0_USR;
  1040. }
  1041. hwc->config = p4_config_pack_escr(escr);
  1042. hwc->config |= p4_config_pack_cccr(cccr);
  1043. hwc->config &= ~P4_CONFIG_HT;
  1044. }
  1045. }
  1046. /*
  1047. * ESCR address hashing is tricky, ESCRs are not sequential
  1048. * in memory but all starts from MSR_P4_BSU_ESCR0 (0x03a0) and
  1049. * the metric between any ESCRs is laid in range [0xa0,0xe1]
  1050. *
  1051. * so we make ~70% filled hashtable
  1052. */
  1053. #define P4_ESCR_MSR_BASE 0x000003a0
  1054. #define P4_ESCR_MSR_MAX 0x000003e1
  1055. #define P4_ESCR_MSR_TABLE_SIZE (P4_ESCR_MSR_MAX - P4_ESCR_MSR_BASE + 1)
  1056. #define P4_ESCR_MSR_IDX(msr) (msr - P4_ESCR_MSR_BASE)
  1057. #define P4_ESCR_MSR_TABLE_ENTRY(msr) [P4_ESCR_MSR_IDX(msr)] = msr
  1058. static const unsigned int p4_escr_table[P4_ESCR_MSR_TABLE_SIZE] = {
  1059. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ALF_ESCR0),
  1060. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ALF_ESCR1),
  1061. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BPU_ESCR0),
  1062. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BPU_ESCR1),
  1063. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BSU_ESCR0),
  1064. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_BSU_ESCR1),
  1065. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR0),
  1066. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR1),
  1067. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR2),
  1068. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR3),
  1069. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR4),
  1070. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_CRU_ESCR5),
  1071. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_DAC_ESCR0),
  1072. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_DAC_ESCR1),
  1073. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FIRM_ESCR0),
  1074. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FIRM_ESCR1),
  1075. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FLAME_ESCR0),
  1076. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FLAME_ESCR1),
  1077. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FSB_ESCR0),
  1078. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_FSB_ESCR1),
  1079. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IQ_ESCR0),
  1080. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IQ_ESCR1),
  1081. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IS_ESCR0),
  1082. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IS_ESCR1),
  1083. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ITLB_ESCR0),
  1084. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_ITLB_ESCR1),
  1085. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IX_ESCR0),
  1086. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_IX_ESCR1),
  1087. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MOB_ESCR0),
  1088. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MOB_ESCR1),
  1089. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MS_ESCR0),
  1090. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_MS_ESCR1),
  1091. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_PMH_ESCR0),
  1092. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_PMH_ESCR1),
  1093. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_RAT_ESCR0),
  1094. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_RAT_ESCR1),
  1095. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SAAT_ESCR0),
  1096. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SAAT_ESCR1),
  1097. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SSU_ESCR0),
  1098. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_SSU_ESCR1),
  1099. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TBPU_ESCR0),
  1100. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TBPU_ESCR1),
  1101. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TC_ESCR0),
  1102. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_TC_ESCR1),
  1103. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_U2L_ESCR0),
  1104. P4_ESCR_MSR_TABLE_ENTRY(MSR_P4_U2L_ESCR1),
  1105. };
  1106. static int p4_get_escr_idx(unsigned int addr)
  1107. {
  1108. unsigned int idx = P4_ESCR_MSR_IDX(addr);
  1109. if (unlikely(idx >= P4_ESCR_MSR_TABLE_SIZE ||
  1110. !p4_escr_table[idx] ||
  1111. p4_escr_table[idx] != addr)) {
  1112. WARN_ONCE(1, "P4 PMU: Wrong address passed: %x\n", addr);
  1113. return -1;
  1114. }
  1115. return idx;
  1116. }
  1117. static int p4_next_cntr(int thread, unsigned long *used_mask,
  1118. struct p4_event_bind *bind)
  1119. {
  1120. int i, j;
  1121. for (i = 0; i < P4_CNTR_LIMIT; i++) {
  1122. j = bind->cntr[thread][i];
  1123. if (j != -1 && !test_bit(j, used_mask))
  1124. return j;
  1125. }
  1126. return -1;
  1127. }
  1128. static int p4_pmu_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
  1129. {
  1130. unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  1131. unsigned long escr_mask[BITS_TO_LONGS(P4_ESCR_MSR_TABLE_SIZE)];
  1132. int cpu = smp_processor_id();
  1133. struct hw_perf_event *hwc;
  1134. struct p4_event_bind *bind;
  1135. unsigned int i, thread, num;
  1136. int cntr_idx, escr_idx;
  1137. u64 config_alias;
  1138. int pass;
  1139. bitmap_zero(used_mask, X86_PMC_IDX_MAX);
  1140. bitmap_zero(escr_mask, P4_ESCR_MSR_TABLE_SIZE);
  1141. for (i = 0, num = n; i < n; i++, num--) {
  1142. hwc = &cpuc->event_list[i]->hw;
  1143. thread = p4_ht_thread(cpu);
  1144. pass = 0;
  1145. again:
  1146. /*
  1147. * It's possible to hit a circular lock
  1148. * between original and alternative events
  1149. * if both are scheduled already.
  1150. */
  1151. if (pass > 2)
  1152. goto done;
  1153. bind = p4_config_get_bind(hwc->config);
  1154. escr_idx = p4_get_escr_idx(bind->escr_msr[thread]);
  1155. if (unlikely(escr_idx == -1))
  1156. goto done;
  1157. if (hwc->idx != -1 && !p4_should_swap_ts(hwc->config, cpu)) {
  1158. cntr_idx = hwc->idx;
  1159. if (assign)
  1160. assign[i] = hwc->idx;
  1161. goto reserve;
  1162. }
  1163. cntr_idx = p4_next_cntr(thread, used_mask, bind);
  1164. if (cntr_idx == -1 || test_bit(escr_idx, escr_mask)) {
  1165. /*
  1166. * Check whether an event alias is still available.
  1167. */
  1168. config_alias = p4_get_alias_event(hwc->config);
  1169. if (!config_alias)
  1170. goto done;
  1171. hwc->config = config_alias;
  1172. pass++;
  1173. goto again;
  1174. }
  1175. /*
  1176. * Perf does test runs to see if a whole group can be assigned
  1177. * together successfully. There can be multiple rounds of this.
  1178. * Unfortunately, p4_pmu_swap_config_ts touches the hwc->config
  1179. * bits, such that the next round of group assignments will
  1180. * cause the above p4_should_swap_ts to pass instead of fail.
  1181. * This leads to counters exclusive to thread0 being used by
  1182. * thread1.
  1183. *
  1184. * Solve this with a cheap hack, reset the idx back to -1 to
  1185. * force a new lookup (p4_next_cntr) to get the right counter
  1186. * for the right thread.
  1187. *
  1188. * This probably doesn't comply with the general spirit of how
  1189. * perf wants to work, but P4 is special. :-(
  1190. */
  1191. if (p4_should_swap_ts(hwc->config, cpu))
  1192. hwc->idx = -1;
  1193. p4_pmu_swap_config_ts(hwc, cpu);
  1194. if (assign)
  1195. assign[i] = cntr_idx;
  1196. reserve:
  1197. set_bit(cntr_idx, used_mask);
  1198. set_bit(escr_idx, escr_mask);
  1199. }
  1200. done:
  1201. return num ? -EINVAL : 0;
  1202. }
  1203. PMU_FORMAT_ATTR(cccr, "config:0-31" );
  1204. PMU_FORMAT_ATTR(escr, "config:32-62");
  1205. PMU_FORMAT_ATTR(ht, "config:63" );
  1206. static struct attribute *intel_p4_formats_attr[] = {
  1207. &format_attr_cccr.attr,
  1208. &format_attr_escr.attr,
  1209. &format_attr_ht.attr,
  1210. NULL,
  1211. };
  1212. static __initconst const struct x86_pmu p4_pmu = {
  1213. .name = "Netburst P4/Xeon",
  1214. .handle_irq = p4_pmu_handle_irq,
  1215. .disable_all = p4_pmu_disable_all,
  1216. .enable_all = p4_pmu_enable_all,
  1217. .enable = p4_pmu_enable_event,
  1218. .disable = p4_pmu_disable_event,
  1219. .set_period = p4_pmu_set_period,
  1220. .eventsel = MSR_P4_BPU_CCCR0,
  1221. .perfctr = MSR_P4_BPU_PERFCTR0,
  1222. .event_map = p4_pmu_event_map,
  1223. .max_events = ARRAY_SIZE(p4_general_events),
  1224. .get_event_constraints = x86_get_event_constraints,
  1225. /*
  1226. * IF HT disabled we may need to use all
  1227. * ARCH_P4_MAX_CCCR counters simultaneously
  1228. * though leave it restricted at moment assuming
  1229. * HT is on
  1230. */
  1231. .num_counters = ARCH_P4_MAX_CCCR,
  1232. .apic = 1,
  1233. .cntval_bits = ARCH_P4_CNTRVAL_BITS,
  1234. .cntval_mask = ARCH_P4_CNTRVAL_MASK,
  1235. .max_period = (1ULL << (ARCH_P4_CNTRVAL_BITS - 1)) - 1,
  1236. .hw_config = p4_hw_config,
  1237. .schedule_events = p4_pmu_schedule_events,
  1238. .format_attrs = intel_p4_formats_attr,
  1239. };
  1240. __init int p4_pmu_init(void)
  1241. {
  1242. unsigned int low, high;
  1243. int i, reg;
  1244. /* If we get stripped -- indexing fails */
  1245. BUILD_BUG_ON(ARCH_P4_MAX_CCCR > INTEL_PMC_MAX_GENERIC);
  1246. rdmsr(MSR_IA32_MISC_ENABLE, low, high);
  1247. if (!(low & (1 << 7))) {
  1248. pr_cont("unsupported Netburst CPU model %d ",
  1249. boot_cpu_data.x86_model);
  1250. return -ENODEV;
  1251. }
  1252. memcpy(hw_cache_event_ids, p4_hw_cache_event_ids,
  1253. sizeof(hw_cache_event_ids));
  1254. pr_cont("Netburst events, ");
  1255. x86_pmu = p4_pmu;
  1256. /*
  1257. * Even though the counters are configured to interrupt a particular
  1258. * logical processor when an overflow happens, testing has shown that
  1259. * on kdump kernels (which uses a single cpu), thread1's counter
  1260. * continues to run and will report an NMI on thread0. Due to the
  1261. * overflow bug, this leads to a stream of unknown NMIs.
  1262. *
  1263. * Solve this by zero'ing out the registers to mimic a reset.
  1264. */
  1265. for (i = 0; i < x86_pmu.num_counters; i++) {
  1266. reg = x86_pmu_config_addr(i);
  1267. wrmsrl_safe(reg, 0ULL);
  1268. }
  1269. return 0;
  1270. }