arm-ccn.c 45 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright (C) 2014 ARM Limited
  5. */
  6. #include <linux/ctype.h>
  7. #include <linux/hrtimer.h>
  8. #include <linux/idr.h>
  9. #include <linux/interrupt.h>
  10. #include <linux/io.h>
  11. #include <linux/module.h>
  12. #include <linux/mod_devicetable.h>
  13. #include <linux/perf_event.h>
  14. #include <linux/platform_device.h>
  15. #include <linux/slab.h>
  16. #define CCN_NUM_XP_PORTS 2
  17. #define CCN_NUM_VCS 4
  18. #define CCN_NUM_REGIONS 256
  19. #define CCN_REGION_SIZE 0x10000
  20. #define CCN_ALL_OLY_ID 0xff00
  21. #define CCN_ALL_OLY_ID__OLY_ID__SHIFT 0
  22. #define CCN_ALL_OLY_ID__OLY_ID__MASK 0x1f
  23. #define CCN_ALL_OLY_ID__NODE_ID__SHIFT 8
  24. #define CCN_ALL_OLY_ID__NODE_ID__MASK 0x3f
  25. #define CCN_MN_ERRINT_STATUS 0x0008
  26. #define CCN_MN_ERRINT_STATUS__INTREQ__DESSERT 0x11
  27. #define CCN_MN_ERRINT_STATUS__ALL_ERRORS__ENABLE 0x02
  28. #define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLED 0x20
  29. #define CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE 0x22
  30. #define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_ENABLE 0x04
  31. #define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLED 0x40
  32. #define CCN_MN_ERRINT_STATUS__CORRECTED_ERRORS_DISABLE 0x44
  33. #define CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE 0x08
  34. #define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED 0x80
  35. #define CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE 0x88
  36. #define CCN_MN_OLY_COMP_LIST_63_0 0x01e0
  37. #define CCN_MN_ERR_SIG_VAL_63_0 0x0300
  38. #define CCN_MN_ERR_SIG_VAL_63_0__DT (1 << 1)
  39. #define CCN_DT_ACTIVE_DSM 0x0000
  40. #define CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(n) ((n) * 8)
  41. #define CCN_DT_ACTIVE_DSM__DSM_ID__MASK 0xff
  42. #define CCN_DT_CTL 0x0028
  43. #define CCN_DT_CTL__DT_EN (1 << 0)
  44. #define CCN_DT_PMEVCNT(n) (0x0100 + (n) * 0x8)
  45. #define CCN_DT_PMCCNTR 0x0140
  46. #define CCN_DT_PMCCNTRSR 0x0190
  47. #define CCN_DT_PMOVSR 0x0198
  48. #define CCN_DT_PMOVSR_CLR 0x01a0
  49. #define CCN_DT_PMOVSR_CLR__MASK 0x1f
  50. #define CCN_DT_PMCR 0x01a8
  51. #define CCN_DT_PMCR__OVFL_INTR_EN (1 << 6)
  52. #define CCN_DT_PMCR__PMU_EN (1 << 0)
  53. #define CCN_DT_PMSR 0x01b0
  54. #define CCN_DT_PMSR_REQ 0x01b8
  55. #define CCN_DT_PMSR_CLR 0x01c0
  56. #define CCN_HNF_PMU_EVENT_SEL 0x0600
  57. #define CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4)
  58. #define CCN_HNF_PMU_EVENT_SEL__ID__MASK 0xf
  59. #define CCN_XP_DT_CONFIG 0x0300
  60. #define CCN_XP_DT_CONFIG__DT_CFG__SHIFT(n) ((n) * 4)
  61. #define CCN_XP_DT_CONFIG__DT_CFG__MASK 0xf
  62. #define CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH 0x0
  63. #define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT_0_OR_1 0x1
  64. #define CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(n) (0x2 + (n))
  65. #define CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(n) (0x4 + (n))
  66. #define CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(d, n) (0x8 + (d) * 4 + (n))
  67. #define CCN_XP_DT_INTERFACE_SEL 0x0308
  68. #define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(n) (0 + (n) * 8)
  69. #define CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK 0x1
  70. #define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(n) (1 + (n) * 8)
  71. #define CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK 0x1
  72. #define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(n) (2 + (n) * 8)
  73. #define CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK 0x3
  74. #define CCN_XP_DT_CMP_VAL_L(n) (0x0310 + (n) * 0x40)
  75. #define CCN_XP_DT_CMP_VAL_H(n) (0x0318 + (n) * 0x40)
  76. #define CCN_XP_DT_CMP_MASK_L(n) (0x0320 + (n) * 0x40)
  77. #define CCN_XP_DT_CMP_MASK_H(n) (0x0328 + (n) * 0x40)
  78. #define CCN_XP_DT_CONTROL 0x0370
  79. #define CCN_XP_DT_CONTROL__DT_ENABLE (1 << 0)
  80. #define CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(n) (12 + (n) * 4)
  81. #define CCN_XP_DT_CONTROL__WP_ARM_SEL__MASK 0xf
  82. #define CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS 0xf
  83. #define CCN_XP_PMU_EVENT_SEL 0x0600
  84. #define CCN_XP_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 7)
  85. #define CCN_XP_PMU_EVENT_SEL__ID__MASK 0x3f
  86. #define CCN_SBAS_PMU_EVENT_SEL 0x0600
  87. #define CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4)
  88. #define CCN_SBAS_PMU_EVENT_SEL__ID__MASK 0xf
  89. #define CCN_RNI_PMU_EVENT_SEL 0x0600
  90. #define CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(n) ((n) * 4)
  91. #define CCN_RNI_PMU_EVENT_SEL__ID__MASK 0xf
  92. #define CCN_TYPE_MN 0x01
  93. #define CCN_TYPE_DT 0x02
  94. #define CCN_TYPE_HNF 0x04
  95. #define CCN_TYPE_HNI 0x05
  96. #define CCN_TYPE_XP 0x08
  97. #define CCN_TYPE_SBSX 0x0c
  98. #define CCN_TYPE_SBAS 0x10
  99. #define CCN_TYPE_RNI_1P 0x14
  100. #define CCN_TYPE_RNI_2P 0x15
  101. #define CCN_TYPE_RNI_3P 0x16
  102. #define CCN_TYPE_RND_1P 0x18 /* RN-D = RN-I + DVM */
  103. #define CCN_TYPE_RND_2P 0x19
  104. #define CCN_TYPE_RND_3P 0x1a
  105. #define CCN_TYPE_CYCLES 0xff /* Pseudotype */
  106. #define CCN_EVENT_WATCHPOINT 0xfe /* Pseudoevent */
  107. #define CCN_NUM_PMU_EVENTS 4
  108. #define CCN_NUM_XP_WATCHPOINTS 2 /* See DT.dbg_id.num_watchpoints */
  109. #define CCN_NUM_PMU_EVENT_COUNTERS 8 /* See DT.dbg_id.num_pmucntr */
  110. #define CCN_IDX_PMU_CYCLE_COUNTER CCN_NUM_PMU_EVENT_COUNTERS
  111. #define CCN_NUM_PREDEFINED_MASKS 4
  112. #define CCN_IDX_MASK_ANY (CCN_NUM_PMU_EVENT_COUNTERS + 0)
  113. #define CCN_IDX_MASK_EXACT (CCN_NUM_PMU_EVENT_COUNTERS + 1)
  114. #define CCN_IDX_MASK_ORDER (CCN_NUM_PMU_EVENT_COUNTERS + 2)
  115. #define CCN_IDX_MASK_OPCODE (CCN_NUM_PMU_EVENT_COUNTERS + 3)
  116. struct arm_ccn_component {
  117. void __iomem *base;
  118. u32 type;
  119. DECLARE_BITMAP(pmu_events_mask, CCN_NUM_PMU_EVENTS);
  120. union {
  121. struct {
  122. DECLARE_BITMAP(dt_cmp_mask, CCN_NUM_XP_WATCHPOINTS);
  123. } xp;
  124. };
  125. };
  126. #define pmu_to_arm_ccn(_pmu) container_of(container_of(_pmu, \
  127. struct arm_ccn_dt, pmu), struct arm_ccn, dt)
  128. struct arm_ccn_dt {
  129. int id;
  130. void __iomem *base;
  131. spinlock_t config_lock;
  132. DECLARE_BITMAP(pmu_counters_mask, CCN_NUM_PMU_EVENT_COUNTERS + 1);
  133. struct {
  134. struct arm_ccn_component *source;
  135. struct perf_event *event;
  136. } pmu_counters[CCN_NUM_PMU_EVENT_COUNTERS + 1];
  137. struct {
  138. u64 l, h;
  139. } cmp_mask[CCN_NUM_PMU_EVENT_COUNTERS + CCN_NUM_PREDEFINED_MASKS];
  140. struct hrtimer hrtimer;
  141. unsigned int cpu;
  142. struct hlist_node node;
  143. struct pmu pmu;
  144. };
  145. struct arm_ccn {
  146. struct device *dev;
  147. void __iomem *base;
  148. unsigned int irq;
  149. unsigned sbas_present:1;
  150. unsigned sbsx_present:1;
  151. int num_nodes;
  152. struct arm_ccn_component *node;
  153. int num_xps;
  154. struct arm_ccn_component *xp;
  155. struct arm_ccn_dt dt;
  156. int mn_id;
  157. };
  158. static int arm_ccn_node_to_xp(int node)
  159. {
  160. return node / CCN_NUM_XP_PORTS;
  161. }
  162. static int arm_ccn_node_to_xp_port(int node)
  163. {
  164. return node % CCN_NUM_XP_PORTS;
  165. }
  166. /*
  167. * Bit shifts and masks in these defines must be kept in sync with
  168. * arm_ccn_pmu_config_set() and CCN_FORMAT_ATTRs below!
  169. */
  170. #define CCN_CONFIG_NODE(_config) (((_config) >> 0) & 0xff)
  171. #define CCN_CONFIG_XP(_config) (((_config) >> 0) & 0xff)
  172. #define CCN_CONFIG_TYPE(_config) (((_config) >> 8) & 0xff)
  173. #define CCN_CONFIG_EVENT(_config) (((_config) >> 16) & 0xff)
  174. #define CCN_CONFIG_PORT(_config) (((_config) >> 24) & 0x3)
  175. #define CCN_CONFIG_BUS(_config) (((_config) >> 24) & 0x3)
  176. #define CCN_CONFIG_VC(_config) (((_config) >> 26) & 0x7)
  177. #define CCN_CONFIG_DIR(_config) (((_config) >> 29) & 0x1)
  178. #define CCN_CONFIG_MASK(_config) (((_config) >> 30) & 0xf)
  179. static void arm_ccn_pmu_config_set(u64 *config, u32 node_xp, u32 type, u32 port)
  180. {
  181. *config &= ~((0xff << 0) | (0xff << 8) | (0x3 << 24));
  182. *config |= (node_xp << 0) | (type << 8) | (port << 24);
  183. }
  184. static ssize_t arm_ccn_pmu_format_show(struct device *dev,
  185. struct device_attribute *attr, char *buf)
  186. {
  187. struct dev_ext_attribute *ea = container_of(attr,
  188. struct dev_ext_attribute, attr);
  189. return sysfs_emit(buf, "%s\n", (char *)ea->var);
  190. }
  191. #define CCN_FORMAT_ATTR(_name, _config) \
  192. struct dev_ext_attribute arm_ccn_pmu_format_attr_##_name = \
  193. { __ATTR(_name, S_IRUGO, arm_ccn_pmu_format_show, \
  194. NULL), _config }
  195. static CCN_FORMAT_ATTR(node, "config:0-7");
  196. static CCN_FORMAT_ATTR(xp, "config:0-7");
  197. static CCN_FORMAT_ATTR(type, "config:8-15");
  198. static CCN_FORMAT_ATTR(event, "config:16-23");
  199. static CCN_FORMAT_ATTR(port, "config:24-25");
  200. static CCN_FORMAT_ATTR(bus, "config:24-25");
  201. static CCN_FORMAT_ATTR(vc, "config:26-28");
  202. static CCN_FORMAT_ATTR(dir, "config:29-29");
  203. static CCN_FORMAT_ATTR(mask, "config:30-33");
  204. static CCN_FORMAT_ATTR(cmp_l, "config1:0-62");
  205. static CCN_FORMAT_ATTR(cmp_h, "config2:0-59");
  206. static struct attribute *arm_ccn_pmu_format_attrs[] = {
  207. &arm_ccn_pmu_format_attr_node.attr.attr,
  208. &arm_ccn_pmu_format_attr_xp.attr.attr,
  209. &arm_ccn_pmu_format_attr_type.attr.attr,
  210. &arm_ccn_pmu_format_attr_event.attr.attr,
  211. &arm_ccn_pmu_format_attr_port.attr.attr,
  212. &arm_ccn_pmu_format_attr_bus.attr.attr,
  213. &arm_ccn_pmu_format_attr_vc.attr.attr,
  214. &arm_ccn_pmu_format_attr_dir.attr.attr,
  215. &arm_ccn_pmu_format_attr_mask.attr.attr,
  216. &arm_ccn_pmu_format_attr_cmp_l.attr.attr,
  217. &arm_ccn_pmu_format_attr_cmp_h.attr.attr,
  218. NULL
  219. };
  220. static const struct attribute_group arm_ccn_pmu_format_attr_group = {
  221. .name = "format",
  222. .attrs = arm_ccn_pmu_format_attrs,
  223. };
  224. struct arm_ccn_pmu_event {
  225. struct device_attribute attr;
  226. u32 type;
  227. u32 event;
  228. int num_ports;
  229. int num_vcs;
  230. const char *def;
  231. int mask;
  232. };
  233. #define CCN_EVENT_ATTR(_name) \
  234. __ATTR(_name, S_IRUGO, arm_ccn_pmu_event_show, NULL)
  235. /*
  236. * Events defined in TRM for MN, HN-I and SBSX are actually watchpoints set on
  237. * their ports in XP they are connected to. For the sake of usability they are
  238. * explicitly defined here (and translated into a relevant watchpoint in
  239. * arm_ccn_pmu_event_init()) so the user can easily request them without deep
  240. * knowledge of the flit format.
  241. */
  242. #define CCN_EVENT_MN(_name, _def, _mask) { .attr = CCN_EVENT_ATTR(mn_##_name), \
  243. .type = CCN_TYPE_MN, .event = CCN_EVENT_WATCHPOINT, \
  244. .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, \
  245. .def = _def, .mask = _mask, }
  246. #define CCN_EVENT_HNI(_name, _def, _mask) { \
  247. .attr = CCN_EVENT_ATTR(hni_##_name), .type = CCN_TYPE_HNI, \
  248. .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \
  249. .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, }
  250. #define CCN_EVENT_SBSX(_name, _def, _mask) { \
  251. .attr = CCN_EVENT_ATTR(sbsx_##_name), .type = CCN_TYPE_SBSX, \
  252. .event = CCN_EVENT_WATCHPOINT, .num_ports = CCN_NUM_XP_PORTS, \
  253. .num_vcs = CCN_NUM_VCS, .def = _def, .mask = _mask, }
  254. #define CCN_EVENT_HNF(_name, _event) { .attr = CCN_EVENT_ATTR(hnf_##_name), \
  255. .type = CCN_TYPE_HNF, .event = _event, }
  256. #define CCN_EVENT_XP(_name, _event) { .attr = CCN_EVENT_ATTR(xp_##_name), \
  257. .type = CCN_TYPE_XP, .event = _event, \
  258. .num_ports = CCN_NUM_XP_PORTS, .num_vcs = CCN_NUM_VCS, }
  259. /*
  260. * RN-I & RN-D (RN-D = RN-I + DVM) nodes have different type ID depending
  261. * on configuration. One of them is picked to represent the whole group,
  262. * as they all share the same event types.
  263. */
  264. #define CCN_EVENT_RNI(_name, _event) { .attr = CCN_EVENT_ATTR(rni_##_name), \
  265. .type = CCN_TYPE_RNI_3P, .event = _event, }
  266. #define CCN_EVENT_SBAS(_name, _event) { .attr = CCN_EVENT_ATTR(sbas_##_name), \
  267. .type = CCN_TYPE_SBAS, .event = _event, }
  268. #define CCN_EVENT_CYCLES(_name) { .attr = CCN_EVENT_ATTR(_name), \
  269. .type = CCN_TYPE_CYCLES }
  270. static ssize_t arm_ccn_pmu_event_show(struct device *dev,
  271. struct device_attribute *attr, char *buf)
  272. {
  273. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  274. struct arm_ccn_pmu_event *event = container_of(attr,
  275. struct arm_ccn_pmu_event, attr);
  276. int res;
  277. res = sysfs_emit(buf, "type=0x%x", event->type);
  278. if (event->event)
  279. res += sysfs_emit_at(buf, res, ",event=0x%x", event->event);
  280. if (event->def)
  281. res += sysfs_emit_at(buf, res, ",%s", event->def);
  282. if (event->mask)
  283. res += sysfs_emit_at(buf, res, ",mask=0x%x", event->mask);
  284. /* Arguments required by an event */
  285. switch (event->type) {
  286. case CCN_TYPE_CYCLES:
  287. break;
  288. case CCN_TYPE_XP:
  289. res += sysfs_emit_at(buf, res, ",xp=?,vc=?");
  290. if (event->event == CCN_EVENT_WATCHPOINT)
  291. res += sysfs_emit_at(buf, res,
  292. ",port=?,dir=?,cmp_l=?,cmp_h=?,mask=?");
  293. else
  294. res += sysfs_emit_at(buf, res, ",bus=?");
  295. break;
  296. case CCN_TYPE_MN:
  297. res += sysfs_emit_at(buf, res, ",node=%d", ccn->mn_id);
  298. break;
  299. default:
  300. res += sysfs_emit_at(buf, res, ",node=?");
  301. break;
  302. }
  303. res += sysfs_emit_at(buf, res, "\n");
  304. return res;
  305. }
  306. static umode_t arm_ccn_pmu_events_is_visible(struct kobject *kobj,
  307. struct attribute *attr, int index)
  308. {
  309. struct device *dev = kobj_to_dev(kobj);
  310. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  311. struct device_attribute *dev_attr = container_of(attr,
  312. struct device_attribute, attr);
  313. struct arm_ccn_pmu_event *event = container_of(dev_attr,
  314. struct arm_ccn_pmu_event, attr);
  315. if (event->type == CCN_TYPE_SBAS && !ccn->sbas_present)
  316. return 0;
  317. if (event->type == CCN_TYPE_SBSX && !ccn->sbsx_present)
  318. return 0;
  319. return attr->mode;
  320. }
  321. static struct arm_ccn_pmu_event arm_ccn_pmu_events[] = {
  322. CCN_EVENT_MN(eobarrier, "dir=1,vc=0,cmp_h=0x1c00", CCN_IDX_MASK_OPCODE),
  323. CCN_EVENT_MN(ecbarrier, "dir=1,vc=0,cmp_h=0x1e00", CCN_IDX_MASK_OPCODE),
  324. CCN_EVENT_MN(dvmop, "dir=1,vc=0,cmp_h=0x2800", CCN_IDX_MASK_OPCODE),
  325. CCN_EVENT_HNI(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY),
  326. CCN_EVENT_HNI(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY),
  327. CCN_EVENT_HNI(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY),
  328. CCN_EVENT_HNI(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY),
  329. CCN_EVENT_HNI(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000",
  330. CCN_IDX_MASK_ORDER),
  331. CCN_EVENT_SBSX(txdatflits, "dir=1,vc=3", CCN_IDX_MASK_ANY),
  332. CCN_EVENT_SBSX(rxdatflits, "dir=0,vc=3", CCN_IDX_MASK_ANY),
  333. CCN_EVENT_SBSX(txreqflits, "dir=1,vc=0", CCN_IDX_MASK_ANY),
  334. CCN_EVENT_SBSX(rxreqflits, "dir=0,vc=0", CCN_IDX_MASK_ANY),
  335. CCN_EVENT_SBSX(rxreqflits_order, "dir=0,vc=0,cmp_h=0x8000",
  336. CCN_IDX_MASK_ORDER),
  337. CCN_EVENT_HNF(cache_miss, 0x1),
  338. CCN_EVENT_HNF(l3_sf_cache_access, 0x02),
  339. CCN_EVENT_HNF(cache_fill, 0x3),
  340. CCN_EVENT_HNF(pocq_retry, 0x4),
  341. CCN_EVENT_HNF(pocq_reqs_recvd, 0x5),
  342. CCN_EVENT_HNF(sf_hit, 0x6),
  343. CCN_EVENT_HNF(sf_evictions, 0x7),
  344. CCN_EVENT_HNF(snoops_sent, 0x8),
  345. CCN_EVENT_HNF(snoops_broadcast, 0x9),
  346. CCN_EVENT_HNF(l3_eviction, 0xa),
  347. CCN_EVENT_HNF(l3_fill_invalid_way, 0xb),
  348. CCN_EVENT_HNF(mc_retries, 0xc),
  349. CCN_EVENT_HNF(mc_reqs, 0xd),
  350. CCN_EVENT_HNF(qos_hh_retry, 0xe),
  351. CCN_EVENT_RNI(rdata_beats_p0, 0x1),
  352. CCN_EVENT_RNI(rdata_beats_p1, 0x2),
  353. CCN_EVENT_RNI(rdata_beats_p2, 0x3),
  354. CCN_EVENT_RNI(rxdat_flits, 0x4),
  355. CCN_EVENT_RNI(txdat_flits, 0x5),
  356. CCN_EVENT_RNI(txreq_flits, 0x6),
  357. CCN_EVENT_RNI(txreq_flits_retried, 0x7),
  358. CCN_EVENT_RNI(rrt_full, 0x8),
  359. CCN_EVENT_RNI(wrt_full, 0x9),
  360. CCN_EVENT_RNI(txreq_flits_replayed, 0xa),
  361. CCN_EVENT_XP(upload_starvation, 0x1),
  362. CCN_EVENT_XP(download_starvation, 0x2),
  363. CCN_EVENT_XP(respin, 0x3),
  364. CCN_EVENT_XP(valid_flit, 0x4),
  365. CCN_EVENT_XP(watchpoint, CCN_EVENT_WATCHPOINT),
  366. CCN_EVENT_SBAS(rdata_beats_p0, 0x1),
  367. CCN_EVENT_SBAS(rxdat_flits, 0x4),
  368. CCN_EVENT_SBAS(txdat_flits, 0x5),
  369. CCN_EVENT_SBAS(txreq_flits, 0x6),
  370. CCN_EVENT_SBAS(txreq_flits_retried, 0x7),
  371. CCN_EVENT_SBAS(rrt_full, 0x8),
  372. CCN_EVENT_SBAS(wrt_full, 0x9),
  373. CCN_EVENT_SBAS(txreq_flits_replayed, 0xa),
  374. CCN_EVENT_CYCLES(cycles),
  375. };
  376. /* Populated in arm_ccn_init() */
  377. static struct attribute
  378. *arm_ccn_pmu_events_attrs[ARRAY_SIZE(arm_ccn_pmu_events) + 1];
  379. static const struct attribute_group arm_ccn_pmu_events_attr_group = {
  380. .name = "events",
  381. .is_visible = arm_ccn_pmu_events_is_visible,
  382. .attrs = arm_ccn_pmu_events_attrs,
  383. };
  384. static u64 *arm_ccn_pmu_get_cmp_mask(struct arm_ccn *ccn, const char *name)
  385. {
  386. unsigned long i;
  387. if (WARN_ON(!name || !name[0] || !isxdigit(name[0]) || !name[1]))
  388. return NULL;
  389. i = isdigit(name[0]) ? name[0] - '0' : 0xa + tolower(name[0]) - 'a';
  390. switch (name[1]) {
  391. case 'l':
  392. return &ccn->dt.cmp_mask[i].l;
  393. case 'h':
  394. return &ccn->dt.cmp_mask[i].h;
  395. default:
  396. return NULL;
  397. }
  398. }
  399. static ssize_t arm_ccn_pmu_cmp_mask_show(struct device *dev,
  400. struct device_attribute *attr, char *buf)
  401. {
  402. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  403. u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name);
  404. return mask ? sysfs_emit(buf, "0x%016llx\n", *mask) : -EINVAL;
  405. }
  406. static ssize_t arm_ccn_pmu_cmp_mask_store(struct device *dev,
  407. struct device_attribute *attr, const char *buf, size_t count)
  408. {
  409. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  410. u64 *mask = arm_ccn_pmu_get_cmp_mask(ccn, attr->attr.name);
  411. int err = -EINVAL;
  412. if (mask)
  413. err = kstrtoull(buf, 0, mask);
  414. return err ? err : count;
  415. }
  416. #define CCN_CMP_MASK_ATTR(_name) \
  417. struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \
  418. __ATTR(_name, S_IRUGO | S_IWUSR, \
  419. arm_ccn_pmu_cmp_mask_show, arm_ccn_pmu_cmp_mask_store)
  420. #define CCN_CMP_MASK_ATTR_RO(_name) \
  421. struct device_attribute arm_ccn_pmu_cmp_mask_attr_##_name = \
  422. __ATTR(_name, S_IRUGO, arm_ccn_pmu_cmp_mask_show, NULL)
  423. static CCN_CMP_MASK_ATTR(0l);
  424. static CCN_CMP_MASK_ATTR(0h);
  425. static CCN_CMP_MASK_ATTR(1l);
  426. static CCN_CMP_MASK_ATTR(1h);
  427. static CCN_CMP_MASK_ATTR(2l);
  428. static CCN_CMP_MASK_ATTR(2h);
  429. static CCN_CMP_MASK_ATTR(3l);
  430. static CCN_CMP_MASK_ATTR(3h);
  431. static CCN_CMP_MASK_ATTR(4l);
  432. static CCN_CMP_MASK_ATTR(4h);
  433. static CCN_CMP_MASK_ATTR(5l);
  434. static CCN_CMP_MASK_ATTR(5h);
  435. static CCN_CMP_MASK_ATTR(6l);
  436. static CCN_CMP_MASK_ATTR(6h);
  437. static CCN_CMP_MASK_ATTR(7l);
  438. static CCN_CMP_MASK_ATTR(7h);
  439. static CCN_CMP_MASK_ATTR_RO(8l);
  440. static CCN_CMP_MASK_ATTR_RO(8h);
  441. static CCN_CMP_MASK_ATTR_RO(9l);
  442. static CCN_CMP_MASK_ATTR_RO(9h);
  443. static CCN_CMP_MASK_ATTR_RO(al);
  444. static CCN_CMP_MASK_ATTR_RO(ah);
  445. static CCN_CMP_MASK_ATTR_RO(bl);
  446. static CCN_CMP_MASK_ATTR_RO(bh);
  447. static struct attribute *arm_ccn_pmu_cmp_mask_attrs[] = {
  448. &arm_ccn_pmu_cmp_mask_attr_0l.attr, &arm_ccn_pmu_cmp_mask_attr_0h.attr,
  449. &arm_ccn_pmu_cmp_mask_attr_1l.attr, &arm_ccn_pmu_cmp_mask_attr_1h.attr,
  450. &arm_ccn_pmu_cmp_mask_attr_2l.attr, &arm_ccn_pmu_cmp_mask_attr_2h.attr,
  451. &arm_ccn_pmu_cmp_mask_attr_3l.attr, &arm_ccn_pmu_cmp_mask_attr_3h.attr,
  452. &arm_ccn_pmu_cmp_mask_attr_4l.attr, &arm_ccn_pmu_cmp_mask_attr_4h.attr,
  453. &arm_ccn_pmu_cmp_mask_attr_5l.attr, &arm_ccn_pmu_cmp_mask_attr_5h.attr,
  454. &arm_ccn_pmu_cmp_mask_attr_6l.attr, &arm_ccn_pmu_cmp_mask_attr_6h.attr,
  455. &arm_ccn_pmu_cmp_mask_attr_7l.attr, &arm_ccn_pmu_cmp_mask_attr_7h.attr,
  456. &arm_ccn_pmu_cmp_mask_attr_8l.attr, &arm_ccn_pmu_cmp_mask_attr_8h.attr,
  457. &arm_ccn_pmu_cmp_mask_attr_9l.attr, &arm_ccn_pmu_cmp_mask_attr_9h.attr,
  458. &arm_ccn_pmu_cmp_mask_attr_al.attr, &arm_ccn_pmu_cmp_mask_attr_ah.attr,
  459. &arm_ccn_pmu_cmp_mask_attr_bl.attr, &arm_ccn_pmu_cmp_mask_attr_bh.attr,
  460. NULL
  461. };
  462. static const struct attribute_group arm_ccn_pmu_cmp_mask_attr_group = {
  463. .name = "cmp_mask",
  464. .attrs = arm_ccn_pmu_cmp_mask_attrs,
  465. };
  466. static ssize_t arm_ccn_pmu_cpumask_show(struct device *dev,
  467. struct device_attribute *attr, char *buf)
  468. {
  469. struct arm_ccn *ccn = pmu_to_arm_ccn(dev_get_drvdata(dev));
  470. return cpumap_print_to_pagebuf(true, buf, cpumask_of(ccn->dt.cpu));
  471. }
  472. static struct device_attribute arm_ccn_pmu_cpumask_attr =
  473. __ATTR(cpumask, S_IRUGO, arm_ccn_pmu_cpumask_show, NULL);
  474. static struct attribute *arm_ccn_pmu_cpumask_attrs[] = {
  475. &arm_ccn_pmu_cpumask_attr.attr,
  476. NULL,
  477. };
  478. static const struct attribute_group arm_ccn_pmu_cpumask_attr_group = {
  479. .attrs = arm_ccn_pmu_cpumask_attrs,
  480. };
  481. /*
  482. * Default poll period is 10ms, which is way over the top anyway,
  483. * as in the worst case scenario (an event every cycle), with 1GHz
  484. * clocked bus, the smallest, 32 bit counter will overflow in
  485. * more than 4s.
  486. */
  487. static unsigned int arm_ccn_pmu_poll_period_us = 10000;
  488. module_param_named(pmu_poll_period_us, arm_ccn_pmu_poll_period_us, uint,
  489. S_IRUGO | S_IWUSR);
  490. static ktime_t arm_ccn_pmu_timer_period(void)
  491. {
  492. return ns_to_ktime((u64)arm_ccn_pmu_poll_period_us * 1000);
  493. }
  494. static const struct attribute_group *arm_ccn_pmu_attr_groups[] = {
  495. &arm_ccn_pmu_events_attr_group,
  496. &arm_ccn_pmu_format_attr_group,
  497. &arm_ccn_pmu_cmp_mask_attr_group,
  498. &arm_ccn_pmu_cpumask_attr_group,
  499. NULL
  500. };
  501. static int arm_ccn_pmu_alloc_bit(unsigned long *bitmap, unsigned long size)
  502. {
  503. int bit;
  504. do {
  505. bit = find_first_zero_bit(bitmap, size);
  506. if (bit >= size)
  507. return -EAGAIN;
  508. } while (test_and_set_bit(bit, bitmap));
  509. return bit;
  510. }
  511. /* All RN-I and RN-D nodes have identical PMUs */
  512. static int arm_ccn_pmu_type_eq(u32 a, u32 b)
  513. {
  514. if (a == b)
  515. return 1;
  516. switch (a) {
  517. case CCN_TYPE_RNI_1P:
  518. case CCN_TYPE_RNI_2P:
  519. case CCN_TYPE_RNI_3P:
  520. case CCN_TYPE_RND_1P:
  521. case CCN_TYPE_RND_2P:
  522. case CCN_TYPE_RND_3P:
  523. switch (b) {
  524. case CCN_TYPE_RNI_1P:
  525. case CCN_TYPE_RNI_2P:
  526. case CCN_TYPE_RNI_3P:
  527. case CCN_TYPE_RND_1P:
  528. case CCN_TYPE_RND_2P:
  529. case CCN_TYPE_RND_3P:
  530. return 1;
  531. }
  532. break;
  533. }
  534. return 0;
  535. }
  536. static int arm_ccn_pmu_event_alloc(struct perf_event *event)
  537. {
  538. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  539. struct hw_perf_event *hw = &event->hw;
  540. u32 node_xp, type, event_id;
  541. struct arm_ccn_component *source;
  542. int bit;
  543. node_xp = CCN_CONFIG_NODE(event->attr.config);
  544. type = CCN_CONFIG_TYPE(event->attr.config);
  545. event_id = CCN_CONFIG_EVENT(event->attr.config);
  546. /* Allocate the cycle counter */
  547. if (type == CCN_TYPE_CYCLES) {
  548. if (test_and_set_bit(CCN_IDX_PMU_CYCLE_COUNTER,
  549. ccn->dt.pmu_counters_mask))
  550. return -EAGAIN;
  551. hw->idx = CCN_IDX_PMU_CYCLE_COUNTER;
  552. ccn->dt.pmu_counters[CCN_IDX_PMU_CYCLE_COUNTER].event = event;
  553. return 0;
  554. }
  555. /* Allocate an event counter */
  556. hw->idx = arm_ccn_pmu_alloc_bit(ccn->dt.pmu_counters_mask,
  557. CCN_NUM_PMU_EVENT_COUNTERS);
  558. if (hw->idx < 0) {
  559. dev_dbg(ccn->dev, "No more counters available!\n");
  560. return -EAGAIN;
  561. }
  562. if (type == CCN_TYPE_XP)
  563. source = &ccn->xp[node_xp];
  564. else
  565. source = &ccn->node[node_xp];
  566. ccn->dt.pmu_counters[hw->idx].source = source;
  567. /* Allocate an event source or a watchpoint */
  568. if (type == CCN_TYPE_XP && event_id == CCN_EVENT_WATCHPOINT)
  569. bit = arm_ccn_pmu_alloc_bit(source->xp.dt_cmp_mask,
  570. CCN_NUM_XP_WATCHPOINTS);
  571. else
  572. bit = arm_ccn_pmu_alloc_bit(source->pmu_events_mask,
  573. CCN_NUM_PMU_EVENTS);
  574. if (bit < 0) {
  575. dev_dbg(ccn->dev, "No more event sources/watchpoints on node/XP %d!\n",
  576. node_xp);
  577. clear_bit(hw->idx, ccn->dt.pmu_counters_mask);
  578. return -EAGAIN;
  579. }
  580. hw->config_base = bit;
  581. ccn->dt.pmu_counters[hw->idx].event = event;
  582. return 0;
  583. }
  584. static void arm_ccn_pmu_event_release(struct perf_event *event)
  585. {
  586. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  587. struct hw_perf_event *hw = &event->hw;
  588. if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER) {
  589. clear_bit(CCN_IDX_PMU_CYCLE_COUNTER, ccn->dt.pmu_counters_mask);
  590. } else {
  591. struct arm_ccn_component *source =
  592. ccn->dt.pmu_counters[hw->idx].source;
  593. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP &&
  594. CCN_CONFIG_EVENT(event->attr.config) ==
  595. CCN_EVENT_WATCHPOINT)
  596. clear_bit(hw->config_base, source->xp.dt_cmp_mask);
  597. else
  598. clear_bit(hw->config_base, source->pmu_events_mask);
  599. clear_bit(hw->idx, ccn->dt.pmu_counters_mask);
  600. }
  601. ccn->dt.pmu_counters[hw->idx].source = NULL;
  602. ccn->dt.pmu_counters[hw->idx].event = NULL;
  603. }
  604. static int arm_ccn_pmu_event_init(struct perf_event *event)
  605. {
  606. struct arm_ccn *ccn;
  607. struct hw_perf_event *hw = &event->hw;
  608. u32 node_xp, type, event_id;
  609. int valid;
  610. int i;
  611. struct perf_event *sibling;
  612. if (event->attr.type != event->pmu->type)
  613. return -ENOENT;
  614. ccn = pmu_to_arm_ccn(event->pmu);
  615. if (hw->sample_period) {
  616. dev_dbg(ccn->dev, "Sampling not supported!\n");
  617. return -EOPNOTSUPP;
  618. }
  619. if (has_branch_stack(event)) {
  620. dev_dbg(ccn->dev, "Can't exclude execution levels!\n");
  621. return -EINVAL;
  622. }
  623. if (event->cpu < 0) {
  624. dev_dbg(ccn->dev, "Can't provide per-task data!\n");
  625. return -EOPNOTSUPP;
  626. }
  627. /*
  628. * Many perf core operations (eg. events rotation) operate on a
  629. * single CPU context. This is obvious for CPU PMUs, where one
  630. * expects the same sets of events being observed on all CPUs,
  631. * but can lead to issues for off-core PMUs, like CCN, where each
  632. * event could be theoretically assigned to a different CPU. To
  633. * mitigate this, we enforce CPU assignment to one, selected
  634. * processor (the one described in the "cpumask" attribute).
  635. */
  636. event->cpu = ccn->dt.cpu;
  637. node_xp = CCN_CONFIG_NODE(event->attr.config);
  638. type = CCN_CONFIG_TYPE(event->attr.config);
  639. event_id = CCN_CONFIG_EVENT(event->attr.config);
  640. /* Validate node/xp vs topology */
  641. switch (type) {
  642. case CCN_TYPE_MN:
  643. if (node_xp != ccn->mn_id) {
  644. dev_dbg(ccn->dev, "Invalid MN ID %d!\n", node_xp);
  645. return -EINVAL;
  646. }
  647. break;
  648. case CCN_TYPE_XP:
  649. if (node_xp >= ccn->num_xps) {
  650. dev_dbg(ccn->dev, "Invalid XP ID %d!\n", node_xp);
  651. return -EINVAL;
  652. }
  653. break;
  654. case CCN_TYPE_CYCLES:
  655. break;
  656. default:
  657. if (node_xp >= ccn->num_nodes) {
  658. dev_dbg(ccn->dev, "Invalid node ID %d!\n", node_xp);
  659. return -EINVAL;
  660. }
  661. if (!arm_ccn_pmu_type_eq(type, ccn->node[node_xp].type)) {
  662. dev_dbg(ccn->dev, "Invalid type 0x%x for node %d!\n",
  663. type, node_xp);
  664. return -EINVAL;
  665. }
  666. break;
  667. }
  668. /* Validate event ID vs available for the type */
  669. for (i = 0, valid = 0; i < ARRAY_SIZE(arm_ccn_pmu_events) && !valid;
  670. i++) {
  671. struct arm_ccn_pmu_event *e = &arm_ccn_pmu_events[i];
  672. u32 port = CCN_CONFIG_PORT(event->attr.config);
  673. u32 vc = CCN_CONFIG_VC(event->attr.config);
  674. if (!arm_ccn_pmu_type_eq(type, e->type))
  675. continue;
  676. if (event_id != e->event)
  677. continue;
  678. if (e->num_ports && port >= e->num_ports) {
  679. dev_dbg(ccn->dev, "Invalid port %d for node/XP %d!\n",
  680. port, node_xp);
  681. return -EINVAL;
  682. }
  683. if (e->num_vcs && vc >= e->num_vcs) {
  684. dev_dbg(ccn->dev, "Invalid vc %d for node/XP %d!\n",
  685. vc, node_xp);
  686. return -EINVAL;
  687. }
  688. valid = 1;
  689. }
  690. if (!valid) {
  691. dev_dbg(ccn->dev, "Invalid event 0x%x for node/XP %d!\n",
  692. event_id, node_xp);
  693. return -EINVAL;
  694. }
  695. /* Watchpoint-based event for a node is actually set on XP */
  696. if (event_id == CCN_EVENT_WATCHPOINT && type != CCN_TYPE_XP) {
  697. u32 port;
  698. type = CCN_TYPE_XP;
  699. port = arm_ccn_node_to_xp_port(node_xp);
  700. node_xp = arm_ccn_node_to_xp(node_xp);
  701. arm_ccn_pmu_config_set(&event->attr.config,
  702. node_xp, type, port);
  703. }
  704. /*
  705. * We must NOT create groups containing mixed PMUs, although software
  706. * events are acceptable (for example to create a CCN group
  707. * periodically read when a hrtimer aka cpu-clock leader triggers).
  708. */
  709. if (event->group_leader->pmu != event->pmu &&
  710. !is_software_event(event->group_leader))
  711. return -EINVAL;
  712. for_each_sibling_event(sibling, event->group_leader) {
  713. if (sibling->pmu != event->pmu &&
  714. !is_software_event(sibling))
  715. return -EINVAL;
  716. }
  717. return 0;
  718. }
  719. static u64 arm_ccn_pmu_read_counter(struct arm_ccn *ccn, int idx)
  720. {
  721. u64 res;
  722. if (idx == CCN_IDX_PMU_CYCLE_COUNTER) {
  723. #ifdef readq
  724. res = readq(ccn->dt.base + CCN_DT_PMCCNTR);
  725. #else
  726. /* 40 bit counter, can do snapshot and read in two parts */
  727. writel(0x1, ccn->dt.base + CCN_DT_PMSR_REQ);
  728. while (!(readl(ccn->dt.base + CCN_DT_PMSR) & 0x1))
  729. ;
  730. writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR);
  731. res = readl(ccn->dt.base + CCN_DT_PMCCNTRSR + 4) & 0xff;
  732. res <<= 32;
  733. res |= readl(ccn->dt.base + CCN_DT_PMCCNTRSR);
  734. #endif
  735. } else {
  736. res = readl(ccn->dt.base + CCN_DT_PMEVCNT(idx));
  737. }
  738. return res;
  739. }
  740. static void arm_ccn_pmu_event_update(struct perf_event *event)
  741. {
  742. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  743. struct hw_perf_event *hw = &event->hw;
  744. u64 prev_count, new_count, mask;
  745. do {
  746. prev_count = local64_read(&hw->prev_count);
  747. new_count = arm_ccn_pmu_read_counter(ccn, hw->idx);
  748. } while (local64_xchg(&hw->prev_count, new_count) != prev_count);
  749. mask = (1LLU << (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER ? 40 : 32)) - 1;
  750. local64_add((new_count - prev_count) & mask, &event->count);
  751. }
  752. static void arm_ccn_pmu_xp_dt_config(struct perf_event *event, int enable)
  753. {
  754. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  755. struct hw_perf_event *hw = &event->hw;
  756. struct arm_ccn_component *xp;
  757. u32 val, dt_cfg;
  758. /* Nothing to do for cycle counter */
  759. if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER)
  760. return;
  761. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP)
  762. xp = &ccn->xp[CCN_CONFIG_XP(event->attr.config)];
  763. else
  764. xp = &ccn->xp[arm_ccn_node_to_xp(
  765. CCN_CONFIG_NODE(event->attr.config))];
  766. if (enable)
  767. dt_cfg = hw->event_base;
  768. else
  769. dt_cfg = CCN_XP_DT_CONFIG__DT_CFG__PASS_THROUGH;
  770. spin_lock(&ccn->dt.config_lock);
  771. val = readl(xp->base + CCN_XP_DT_CONFIG);
  772. val &= ~(CCN_XP_DT_CONFIG__DT_CFG__MASK <<
  773. CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx));
  774. val |= dt_cfg << CCN_XP_DT_CONFIG__DT_CFG__SHIFT(hw->idx);
  775. writel(val, xp->base + CCN_XP_DT_CONFIG);
  776. spin_unlock(&ccn->dt.config_lock);
  777. }
  778. static void arm_ccn_pmu_event_start(struct perf_event *event, int flags)
  779. {
  780. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  781. struct hw_perf_event *hw = &event->hw;
  782. local64_set(&event->hw.prev_count,
  783. arm_ccn_pmu_read_counter(ccn, hw->idx));
  784. hw->state = 0;
  785. /* Set the DT bus input, engaging the counter */
  786. arm_ccn_pmu_xp_dt_config(event, 1);
  787. }
  788. static void arm_ccn_pmu_event_stop(struct perf_event *event, int flags)
  789. {
  790. struct hw_perf_event *hw = &event->hw;
  791. /* Disable counting, setting the DT bus to pass-through mode */
  792. arm_ccn_pmu_xp_dt_config(event, 0);
  793. if (flags & PERF_EF_UPDATE)
  794. arm_ccn_pmu_event_update(event);
  795. hw->state |= PERF_HES_STOPPED;
  796. }
  797. static void arm_ccn_pmu_xp_watchpoint_config(struct perf_event *event)
  798. {
  799. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  800. struct hw_perf_event *hw = &event->hw;
  801. struct arm_ccn_component *source =
  802. ccn->dt.pmu_counters[hw->idx].source;
  803. unsigned long wp = hw->config_base;
  804. u32 val;
  805. u64 cmp_l = event->attr.config1;
  806. u64 cmp_h = event->attr.config2;
  807. u64 mask_l = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].l;
  808. u64 mask_h = ccn->dt.cmp_mask[CCN_CONFIG_MASK(event->attr.config)].h;
  809. hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__WATCHPOINT(wp);
  810. /* Direction (RX/TX), device (port) & virtual channel */
  811. val = readl(source->base + CCN_XP_DT_INTERFACE_SEL);
  812. val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__MASK <<
  813. CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp));
  814. val |= CCN_CONFIG_DIR(event->attr.config) <<
  815. CCN_XP_DT_INTERFACE_SEL__DT_IO_SEL__SHIFT(wp);
  816. val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__MASK <<
  817. CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp));
  818. val |= CCN_CONFIG_PORT(event->attr.config) <<
  819. CCN_XP_DT_INTERFACE_SEL__DT_DEV_SEL__SHIFT(wp);
  820. val &= ~(CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__MASK <<
  821. CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp));
  822. val |= CCN_CONFIG_VC(event->attr.config) <<
  823. CCN_XP_DT_INTERFACE_SEL__DT_VC_SEL__SHIFT(wp);
  824. writel(val, source->base + CCN_XP_DT_INTERFACE_SEL);
  825. /* Comparison values */
  826. writel(cmp_l & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_L(wp));
  827. writel((cmp_l >> 32) & 0x7fffffff,
  828. source->base + CCN_XP_DT_CMP_VAL_L(wp) + 4);
  829. writel(cmp_h & 0xffffffff, source->base + CCN_XP_DT_CMP_VAL_H(wp));
  830. writel((cmp_h >> 32) & 0x0fffffff,
  831. source->base + CCN_XP_DT_CMP_VAL_H(wp) + 4);
  832. /* Mask */
  833. writel(mask_l & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_L(wp));
  834. writel((mask_l >> 32) & 0x7fffffff,
  835. source->base + CCN_XP_DT_CMP_MASK_L(wp) + 4);
  836. writel(mask_h & 0xffffffff, source->base + CCN_XP_DT_CMP_MASK_H(wp));
  837. writel((mask_h >> 32) & 0x0fffffff,
  838. source->base + CCN_XP_DT_CMP_MASK_H(wp) + 4);
  839. }
  840. static void arm_ccn_pmu_xp_event_config(struct perf_event *event)
  841. {
  842. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  843. struct hw_perf_event *hw = &event->hw;
  844. struct arm_ccn_component *source =
  845. ccn->dt.pmu_counters[hw->idx].source;
  846. u32 val, id;
  847. hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__XP_PMU_EVENT(hw->config_base);
  848. id = (CCN_CONFIG_VC(event->attr.config) << 4) |
  849. (CCN_CONFIG_BUS(event->attr.config) << 3) |
  850. (CCN_CONFIG_EVENT(event->attr.config) << 0);
  851. val = readl(source->base + CCN_XP_PMU_EVENT_SEL);
  852. val &= ~(CCN_XP_PMU_EVENT_SEL__ID__MASK <<
  853. CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base));
  854. val |= id << CCN_XP_PMU_EVENT_SEL__ID__SHIFT(hw->config_base);
  855. writel(val, source->base + CCN_XP_PMU_EVENT_SEL);
  856. }
  857. static void arm_ccn_pmu_node_event_config(struct perf_event *event)
  858. {
  859. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  860. struct hw_perf_event *hw = &event->hw;
  861. struct arm_ccn_component *source =
  862. ccn->dt.pmu_counters[hw->idx].source;
  863. u32 type = CCN_CONFIG_TYPE(event->attr.config);
  864. u32 val, port;
  865. port = arm_ccn_node_to_xp_port(CCN_CONFIG_NODE(event->attr.config));
  866. hw->event_base = CCN_XP_DT_CONFIG__DT_CFG__DEVICE_PMU_EVENT(port,
  867. hw->config_base);
  868. /* These *_event_sel regs should be identical, but let's make sure... */
  869. BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL != CCN_SBAS_PMU_EVENT_SEL);
  870. BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL != CCN_RNI_PMU_EVENT_SEL);
  871. BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(1) !=
  872. CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1));
  873. BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__SHIFT(1) !=
  874. CCN_RNI_PMU_EVENT_SEL__ID__SHIFT(1));
  875. BUILD_BUG_ON(CCN_HNF_PMU_EVENT_SEL__ID__MASK !=
  876. CCN_SBAS_PMU_EVENT_SEL__ID__MASK);
  877. BUILD_BUG_ON(CCN_SBAS_PMU_EVENT_SEL__ID__MASK !=
  878. CCN_RNI_PMU_EVENT_SEL__ID__MASK);
  879. if (WARN_ON(type != CCN_TYPE_HNF && type != CCN_TYPE_SBAS &&
  880. !arm_ccn_pmu_type_eq(type, CCN_TYPE_RNI_3P)))
  881. return;
  882. /* Set the event id for the pre-allocated counter */
  883. val = readl(source->base + CCN_HNF_PMU_EVENT_SEL);
  884. val &= ~(CCN_HNF_PMU_EVENT_SEL__ID__MASK <<
  885. CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base));
  886. val |= CCN_CONFIG_EVENT(event->attr.config) <<
  887. CCN_HNF_PMU_EVENT_SEL__ID__SHIFT(hw->config_base);
  888. writel(val, source->base + CCN_HNF_PMU_EVENT_SEL);
  889. }
  890. static void arm_ccn_pmu_event_config(struct perf_event *event)
  891. {
  892. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  893. struct hw_perf_event *hw = &event->hw;
  894. u32 xp, offset, val;
  895. /* Cycle counter requires no setup */
  896. if (hw->idx == CCN_IDX_PMU_CYCLE_COUNTER)
  897. return;
  898. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP)
  899. xp = CCN_CONFIG_XP(event->attr.config);
  900. else
  901. xp = arm_ccn_node_to_xp(CCN_CONFIG_NODE(event->attr.config));
  902. spin_lock(&ccn->dt.config_lock);
  903. /* Set the DT bus "distance" register */
  904. offset = (hw->idx / 4) * 4;
  905. val = readl(ccn->dt.base + CCN_DT_ACTIVE_DSM + offset);
  906. val &= ~(CCN_DT_ACTIVE_DSM__DSM_ID__MASK <<
  907. CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4));
  908. val |= xp << CCN_DT_ACTIVE_DSM__DSM_ID__SHIFT(hw->idx % 4);
  909. writel(val, ccn->dt.base + CCN_DT_ACTIVE_DSM + offset);
  910. if (CCN_CONFIG_TYPE(event->attr.config) == CCN_TYPE_XP) {
  911. if (CCN_CONFIG_EVENT(event->attr.config) ==
  912. CCN_EVENT_WATCHPOINT)
  913. arm_ccn_pmu_xp_watchpoint_config(event);
  914. else
  915. arm_ccn_pmu_xp_event_config(event);
  916. } else {
  917. arm_ccn_pmu_node_event_config(event);
  918. }
  919. spin_unlock(&ccn->dt.config_lock);
  920. }
  921. static int arm_ccn_pmu_active_counters(struct arm_ccn *ccn)
  922. {
  923. return bitmap_weight(ccn->dt.pmu_counters_mask,
  924. CCN_NUM_PMU_EVENT_COUNTERS + 1);
  925. }
  926. static int arm_ccn_pmu_event_add(struct perf_event *event, int flags)
  927. {
  928. int err;
  929. struct hw_perf_event *hw = &event->hw;
  930. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  931. err = arm_ccn_pmu_event_alloc(event);
  932. if (err)
  933. return err;
  934. /*
  935. * Pin the timer, so that the overflows are handled by the chosen
  936. * event->cpu (this is the same one as presented in "cpumask"
  937. * attribute).
  938. */
  939. if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 1)
  940. hrtimer_start(&ccn->dt.hrtimer, arm_ccn_pmu_timer_period(),
  941. HRTIMER_MODE_REL_PINNED);
  942. arm_ccn_pmu_event_config(event);
  943. hw->state = PERF_HES_STOPPED;
  944. if (flags & PERF_EF_START)
  945. arm_ccn_pmu_event_start(event, PERF_EF_UPDATE);
  946. return 0;
  947. }
  948. static void arm_ccn_pmu_event_del(struct perf_event *event, int flags)
  949. {
  950. struct arm_ccn *ccn = pmu_to_arm_ccn(event->pmu);
  951. arm_ccn_pmu_event_stop(event, PERF_EF_UPDATE);
  952. arm_ccn_pmu_event_release(event);
  953. if (!ccn->irq && arm_ccn_pmu_active_counters(ccn) == 0)
  954. hrtimer_cancel(&ccn->dt.hrtimer);
  955. }
  956. static void arm_ccn_pmu_event_read(struct perf_event *event)
  957. {
  958. arm_ccn_pmu_event_update(event);
  959. }
  960. static void arm_ccn_pmu_enable(struct pmu *pmu)
  961. {
  962. struct arm_ccn *ccn = pmu_to_arm_ccn(pmu);
  963. u32 val = readl(ccn->dt.base + CCN_DT_PMCR);
  964. val |= CCN_DT_PMCR__PMU_EN;
  965. writel(val, ccn->dt.base + CCN_DT_PMCR);
  966. }
  967. static void arm_ccn_pmu_disable(struct pmu *pmu)
  968. {
  969. struct arm_ccn *ccn = pmu_to_arm_ccn(pmu);
  970. u32 val = readl(ccn->dt.base + CCN_DT_PMCR);
  971. val &= ~CCN_DT_PMCR__PMU_EN;
  972. writel(val, ccn->dt.base + CCN_DT_PMCR);
  973. }
  974. static irqreturn_t arm_ccn_pmu_overflow_handler(struct arm_ccn_dt *dt)
  975. {
  976. u32 pmovsr = readl(dt->base + CCN_DT_PMOVSR);
  977. int idx;
  978. if (!pmovsr)
  979. return IRQ_NONE;
  980. writel(pmovsr, dt->base + CCN_DT_PMOVSR_CLR);
  981. BUILD_BUG_ON(CCN_IDX_PMU_CYCLE_COUNTER != CCN_NUM_PMU_EVENT_COUNTERS);
  982. for (idx = 0; idx < CCN_NUM_PMU_EVENT_COUNTERS + 1; idx++) {
  983. struct perf_event *event = dt->pmu_counters[idx].event;
  984. int overflowed = pmovsr & BIT(idx);
  985. WARN_ON_ONCE(overflowed && !event &&
  986. idx != CCN_IDX_PMU_CYCLE_COUNTER);
  987. if (!event || !overflowed)
  988. continue;
  989. arm_ccn_pmu_event_update(event);
  990. }
  991. return IRQ_HANDLED;
  992. }
  993. static enum hrtimer_restart arm_ccn_pmu_timer_handler(struct hrtimer *hrtimer)
  994. {
  995. struct arm_ccn_dt *dt = container_of(hrtimer, struct arm_ccn_dt,
  996. hrtimer);
  997. unsigned long flags;
  998. local_irq_save(flags);
  999. arm_ccn_pmu_overflow_handler(dt);
  1000. local_irq_restore(flags);
  1001. hrtimer_forward_now(hrtimer, arm_ccn_pmu_timer_period());
  1002. return HRTIMER_RESTART;
  1003. }
  1004. static int arm_ccn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
  1005. {
  1006. struct arm_ccn_dt *dt = hlist_entry_safe(node, struct arm_ccn_dt, node);
  1007. struct arm_ccn *ccn = container_of(dt, struct arm_ccn, dt);
  1008. unsigned int target;
  1009. if (cpu != dt->cpu)
  1010. return 0;
  1011. target = cpumask_any_but(cpu_online_mask, cpu);
  1012. if (target >= nr_cpu_ids)
  1013. return 0;
  1014. perf_pmu_migrate_context(&dt->pmu, cpu, target);
  1015. dt->cpu = target;
  1016. if (ccn->irq)
  1017. WARN_ON(irq_set_affinity(ccn->irq, cpumask_of(dt->cpu)));
  1018. return 0;
  1019. }
  1020. static DEFINE_IDA(arm_ccn_pmu_ida);
  1021. static int arm_ccn_pmu_init(struct arm_ccn *ccn)
  1022. {
  1023. int i;
  1024. char *name;
  1025. int err;
  1026. /* Initialize DT subsystem */
  1027. ccn->dt.base = ccn->base + CCN_REGION_SIZE;
  1028. spin_lock_init(&ccn->dt.config_lock);
  1029. writel(CCN_DT_PMOVSR_CLR__MASK, ccn->dt.base + CCN_DT_PMOVSR_CLR);
  1030. writel(CCN_DT_CTL__DT_EN, ccn->dt.base + CCN_DT_CTL);
  1031. writel(CCN_DT_PMCR__OVFL_INTR_EN | CCN_DT_PMCR__PMU_EN,
  1032. ccn->dt.base + CCN_DT_PMCR);
  1033. writel(0x1, ccn->dt.base + CCN_DT_PMSR_CLR);
  1034. for (i = 0; i < ccn->num_xps; i++) {
  1035. writel(0, ccn->xp[i].base + CCN_XP_DT_CONFIG);
  1036. writel((CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS <<
  1037. CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(0)) |
  1038. (CCN_XP_DT_CONTROL__WP_ARM_SEL__ALWAYS <<
  1039. CCN_XP_DT_CONTROL__WP_ARM_SEL__SHIFT(1)) |
  1040. CCN_XP_DT_CONTROL__DT_ENABLE,
  1041. ccn->xp[i].base + CCN_XP_DT_CONTROL);
  1042. }
  1043. ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].l = ~0;
  1044. ccn->dt.cmp_mask[CCN_IDX_MASK_ANY].h = ~0;
  1045. ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].l = 0;
  1046. ccn->dt.cmp_mask[CCN_IDX_MASK_EXACT].h = 0;
  1047. ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].l = ~0;
  1048. ccn->dt.cmp_mask[CCN_IDX_MASK_ORDER].h = ~(0x1 << 15);
  1049. ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].l = ~0;
  1050. ccn->dt.cmp_mask[CCN_IDX_MASK_OPCODE].h = ~(0x1f << 9);
  1051. /* Get a convenient /sys/event_source/devices/ name */
  1052. ccn->dt.id = ida_alloc(&arm_ccn_pmu_ida, GFP_KERNEL);
  1053. if (ccn->dt.id == 0) {
  1054. name = "ccn";
  1055. } else {
  1056. name = devm_kasprintf(ccn->dev, GFP_KERNEL, "ccn_%d",
  1057. ccn->dt.id);
  1058. if (!name) {
  1059. err = -ENOMEM;
  1060. goto error_choose_name;
  1061. }
  1062. }
  1063. /* Perf driver registration */
  1064. ccn->dt.pmu = (struct pmu) {
  1065. .module = THIS_MODULE,
  1066. .attr_groups = arm_ccn_pmu_attr_groups,
  1067. .task_ctx_nr = perf_invalid_context,
  1068. .event_init = arm_ccn_pmu_event_init,
  1069. .add = arm_ccn_pmu_event_add,
  1070. .del = arm_ccn_pmu_event_del,
  1071. .start = arm_ccn_pmu_event_start,
  1072. .stop = arm_ccn_pmu_event_stop,
  1073. .read = arm_ccn_pmu_event_read,
  1074. .pmu_enable = arm_ccn_pmu_enable,
  1075. .pmu_disable = arm_ccn_pmu_disable,
  1076. .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
  1077. };
  1078. /* No overflow interrupt? Have to use a timer instead. */
  1079. if (!ccn->irq) {
  1080. dev_info(ccn->dev, "No access to interrupts, using timer.\n");
  1081. hrtimer_init(&ccn->dt.hrtimer, CLOCK_MONOTONIC,
  1082. HRTIMER_MODE_REL);
  1083. ccn->dt.hrtimer.function = arm_ccn_pmu_timer_handler;
  1084. }
  1085. /* Pick one CPU which we will use to collect data from CCN... */
  1086. ccn->dt.cpu = raw_smp_processor_id();
  1087. /* Also make sure that the overflow interrupt is handled by this CPU */
  1088. if (ccn->irq) {
  1089. err = irq_set_affinity(ccn->irq, cpumask_of(ccn->dt.cpu));
  1090. if (err) {
  1091. dev_err(ccn->dev, "Failed to set interrupt affinity!\n");
  1092. goto error_set_affinity;
  1093. }
  1094. }
  1095. cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1096. &ccn->dt.node);
  1097. err = perf_pmu_register(&ccn->dt.pmu, name, -1);
  1098. if (err)
  1099. goto error_pmu_register;
  1100. return 0;
  1101. error_pmu_register:
  1102. cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1103. &ccn->dt.node);
  1104. error_set_affinity:
  1105. error_choose_name:
  1106. ida_free(&arm_ccn_pmu_ida, ccn->dt.id);
  1107. for (i = 0; i < ccn->num_xps; i++)
  1108. writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL);
  1109. writel(0, ccn->dt.base + CCN_DT_PMCR);
  1110. return err;
  1111. }
  1112. static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn)
  1113. {
  1114. int i;
  1115. cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1116. &ccn->dt.node);
  1117. for (i = 0; i < ccn->num_xps; i++)
  1118. writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL);
  1119. writel(0, ccn->dt.base + CCN_DT_PMCR);
  1120. perf_pmu_unregister(&ccn->dt.pmu);
  1121. ida_free(&arm_ccn_pmu_ida, ccn->dt.id);
  1122. }
  1123. static int arm_ccn_for_each_valid_region(struct arm_ccn *ccn,
  1124. int (*callback)(struct arm_ccn *ccn, int region,
  1125. void __iomem *base, u32 type, u32 id))
  1126. {
  1127. int region;
  1128. for (region = 0; region < CCN_NUM_REGIONS; region++) {
  1129. u32 val, type, id;
  1130. void __iomem *base;
  1131. int err;
  1132. val = readl(ccn->base + CCN_MN_OLY_COMP_LIST_63_0 +
  1133. 4 * (region / 32));
  1134. if (!(val & (1 << (region % 32))))
  1135. continue;
  1136. base = ccn->base + region * CCN_REGION_SIZE;
  1137. val = readl(base + CCN_ALL_OLY_ID);
  1138. type = (val >> CCN_ALL_OLY_ID__OLY_ID__SHIFT) &
  1139. CCN_ALL_OLY_ID__OLY_ID__MASK;
  1140. id = (val >> CCN_ALL_OLY_ID__NODE_ID__SHIFT) &
  1141. CCN_ALL_OLY_ID__NODE_ID__MASK;
  1142. err = callback(ccn, region, base, type, id);
  1143. if (err)
  1144. return err;
  1145. }
  1146. return 0;
  1147. }
  1148. static int arm_ccn_get_nodes_num(struct arm_ccn *ccn, int region,
  1149. void __iomem *base, u32 type, u32 id)
  1150. {
  1151. if (type == CCN_TYPE_XP && id >= ccn->num_xps)
  1152. ccn->num_xps = id + 1;
  1153. else if (id >= ccn->num_nodes)
  1154. ccn->num_nodes = id + 1;
  1155. return 0;
  1156. }
  1157. static int arm_ccn_init_nodes(struct arm_ccn *ccn, int region,
  1158. void __iomem *base, u32 type, u32 id)
  1159. {
  1160. struct arm_ccn_component *component;
  1161. dev_dbg(ccn->dev, "Region %d: id=%u, type=0x%02x\n", region, id, type);
  1162. switch (type) {
  1163. case CCN_TYPE_MN:
  1164. ccn->mn_id = id;
  1165. return 0;
  1166. case CCN_TYPE_DT:
  1167. return 0;
  1168. case CCN_TYPE_XP:
  1169. component = &ccn->xp[id];
  1170. break;
  1171. case CCN_TYPE_SBSX:
  1172. ccn->sbsx_present = 1;
  1173. component = &ccn->node[id];
  1174. break;
  1175. case CCN_TYPE_SBAS:
  1176. ccn->sbas_present = 1;
  1177. fallthrough;
  1178. default:
  1179. component = &ccn->node[id];
  1180. break;
  1181. }
  1182. component->base = base;
  1183. component->type = type;
  1184. return 0;
  1185. }
  1186. static irqreturn_t arm_ccn_error_handler(struct arm_ccn *ccn,
  1187. const u32 *err_sig_val)
  1188. {
  1189. /* This should be really handled by firmware... */
  1190. dev_err(ccn->dev, "Error reported in %08x%08x%08x%08x%08x%08x.\n",
  1191. err_sig_val[5], err_sig_val[4], err_sig_val[3],
  1192. err_sig_val[2], err_sig_val[1], err_sig_val[0]);
  1193. dev_err(ccn->dev, "Disabling interrupt generation for all errors.\n");
  1194. writel(CCN_MN_ERRINT_STATUS__ALL_ERRORS__DISABLE,
  1195. ccn->base + CCN_MN_ERRINT_STATUS);
  1196. return IRQ_HANDLED;
  1197. }
  1198. static irqreturn_t arm_ccn_irq_handler(int irq, void *dev_id)
  1199. {
  1200. irqreturn_t res = IRQ_NONE;
  1201. struct arm_ccn *ccn = dev_id;
  1202. u32 err_sig_val[6];
  1203. u32 err_or;
  1204. int i;
  1205. /* PMU overflow is a special case */
  1206. err_or = err_sig_val[0] = readl(ccn->base + CCN_MN_ERR_SIG_VAL_63_0);
  1207. if (err_or & CCN_MN_ERR_SIG_VAL_63_0__DT) {
  1208. err_or &= ~CCN_MN_ERR_SIG_VAL_63_0__DT;
  1209. res = arm_ccn_pmu_overflow_handler(&ccn->dt);
  1210. }
  1211. /* Have to read all err_sig_vals to clear them */
  1212. for (i = 1; i < ARRAY_SIZE(err_sig_val); i++) {
  1213. err_sig_val[i] = readl(ccn->base +
  1214. CCN_MN_ERR_SIG_VAL_63_0 + i * 4);
  1215. err_or |= err_sig_val[i];
  1216. }
  1217. if (err_or)
  1218. res |= arm_ccn_error_handler(ccn, err_sig_val);
  1219. if (res != IRQ_NONE)
  1220. writel(CCN_MN_ERRINT_STATUS__INTREQ__DESSERT,
  1221. ccn->base + CCN_MN_ERRINT_STATUS);
  1222. return res;
  1223. }
  1224. static int arm_ccn_probe(struct platform_device *pdev)
  1225. {
  1226. struct arm_ccn *ccn;
  1227. int irq;
  1228. int err;
  1229. ccn = devm_kzalloc(&pdev->dev, sizeof(*ccn), GFP_KERNEL);
  1230. if (!ccn)
  1231. return -ENOMEM;
  1232. ccn->dev = &pdev->dev;
  1233. platform_set_drvdata(pdev, ccn);
  1234. ccn->base = devm_platform_ioremap_resource(pdev, 0);
  1235. if (IS_ERR(ccn->base))
  1236. return PTR_ERR(ccn->base);
  1237. irq = platform_get_irq(pdev, 0);
  1238. if (irq < 0)
  1239. return irq;
  1240. /* Check if we can use the interrupt */
  1241. writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLE,
  1242. ccn->base + CCN_MN_ERRINT_STATUS);
  1243. if (readl(ccn->base + CCN_MN_ERRINT_STATUS) &
  1244. CCN_MN_ERRINT_STATUS__PMU_EVENTS__DISABLED) {
  1245. /* Can set 'disable' bits, so can acknowledge interrupts */
  1246. writel(CCN_MN_ERRINT_STATUS__PMU_EVENTS__ENABLE,
  1247. ccn->base + CCN_MN_ERRINT_STATUS);
  1248. err = devm_request_irq(ccn->dev, irq, arm_ccn_irq_handler,
  1249. IRQF_NOBALANCING | IRQF_NO_THREAD,
  1250. dev_name(ccn->dev), ccn);
  1251. if (err)
  1252. return err;
  1253. ccn->irq = irq;
  1254. }
  1255. /* Build topology */
  1256. err = arm_ccn_for_each_valid_region(ccn, arm_ccn_get_nodes_num);
  1257. if (err)
  1258. return err;
  1259. ccn->node = devm_kcalloc(ccn->dev, ccn->num_nodes, sizeof(*ccn->node),
  1260. GFP_KERNEL);
  1261. ccn->xp = devm_kcalloc(ccn->dev, ccn->num_xps, sizeof(*ccn->node),
  1262. GFP_KERNEL);
  1263. if (!ccn->node || !ccn->xp)
  1264. return -ENOMEM;
  1265. err = arm_ccn_for_each_valid_region(ccn, arm_ccn_init_nodes);
  1266. if (err)
  1267. return err;
  1268. return arm_ccn_pmu_init(ccn);
  1269. }
  1270. static int arm_ccn_remove(struct platform_device *pdev)
  1271. {
  1272. struct arm_ccn *ccn = platform_get_drvdata(pdev);
  1273. arm_ccn_pmu_cleanup(ccn);
  1274. return 0;
  1275. }
  1276. static const struct of_device_id arm_ccn_match[] = {
  1277. { .compatible = "arm,ccn-502", },
  1278. { .compatible = "arm,ccn-504", },
  1279. { .compatible = "arm,ccn-512", },
  1280. {},
  1281. };
  1282. MODULE_DEVICE_TABLE(of, arm_ccn_match);
  1283. static struct platform_driver arm_ccn_driver = {
  1284. .driver = {
  1285. .name = "arm-ccn",
  1286. .of_match_table = arm_ccn_match,
  1287. .suppress_bind_attrs = true,
  1288. },
  1289. .probe = arm_ccn_probe,
  1290. .remove = arm_ccn_remove,
  1291. };
  1292. static int __init arm_ccn_init(void)
  1293. {
  1294. int i, ret;
  1295. ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CCN_ONLINE,
  1296. "perf/arm/ccn:online", NULL,
  1297. arm_ccn_pmu_offline_cpu);
  1298. if (ret)
  1299. return ret;
  1300. for (i = 0; i < ARRAY_SIZE(arm_ccn_pmu_events); i++)
  1301. arm_ccn_pmu_events_attrs[i] = &arm_ccn_pmu_events[i].attr.attr;
  1302. ret = platform_driver_register(&arm_ccn_driver);
  1303. if (ret)
  1304. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CCN_ONLINE);
  1305. return ret;
  1306. }
  1307. static void __exit arm_ccn_exit(void)
  1308. {
  1309. platform_driver_unregister(&arm_ccn_driver);
  1310. cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_CCN_ONLINE);
  1311. }
  1312. module_init(arm_ccn_init);
  1313. module_exit(arm_ccn_exit);
  1314. MODULE_AUTHOR("Pawel Moll <[email protected]>");
  1315. MODULE_LICENSE("GPL v2");