perfmon.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
  3. #include <linux/sched/task.h>
  4. #include <linux/io-64-nonatomic-lo-hi.h>
  5. #include "idxd.h"
  6. #include "perfmon.h"
  7. static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
  8. char *buf);
  9. static cpumask_t perfmon_dsa_cpu_mask;
  10. static bool cpuhp_set_up;
  11. static enum cpuhp_state cpuhp_slot;
  12. /*
  13. * perf userspace reads this attribute to determine which cpus to open
  14. * counters on. It's connected to perfmon_dsa_cpu_mask, which is
  15. * maintained by the cpu hotplug handlers.
  16. */
  17. static DEVICE_ATTR_RO(cpumask);
  18. static struct attribute *perfmon_cpumask_attrs[] = {
  19. &dev_attr_cpumask.attr,
  20. NULL,
  21. };
  22. static struct attribute_group cpumask_attr_group = {
  23. .attrs = perfmon_cpumask_attrs,
  24. };
  25. /*
  26. * These attributes specify the bits in the config word that the perf
  27. * syscall uses to pass the event ids and categories to perfmon.
  28. */
  29. DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
  30. DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
  31. /*
  32. * These attributes specify the bits in the config1 word that the perf
  33. * syscall uses to pass filter data to perfmon.
  34. */
  35. DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
  36. DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
  37. DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
  38. DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
  39. DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
  40. #define PERFMON_FILTERS_START 2
  41. #define PERFMON_FILTERS_MAX 5
  42. static struct attribute *perfmon_format_attrs[] = {
  43. &format_attr_idxd_event_category.attr,
  44. &format_attr_idxd_event.attr,
  45. &format_attr_idxd_filter_wq.attr,
  46. &format_attr_idxd_filter_tc.attr,
  47. &format_attr_idxd_filter_pgsz.attr,
  48. &format_attr_idxd_filter_sz.attr,
  49. &format_attr_idxd_filter_eng.attr,
  50. NULL,
  51. };
  52. static struct attribute_group perfmon_format_attr_group = {
  53. .name = "format",
  54. .attrs = perfmon_format_attrs,
  55. };
  56. static const struct attribute_group *perfmon_attr_groups[] = {
  57. &perfmon_format_attr_group,
  58. &cpumask_attr_group,
  59. NULL,
  60. };
  61. static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
  62. char *buf)
  63. {
  64. return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask);
  65. }
  66. static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
  67. {
  68. return &idxd_pmu->pmu == event->pmu;
  69. }
  70. static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
  71. struct perf_event *leader,
  72. bool do_grp)
  73. {
  74. struct perf_event *event;
  75. int n, max_count;
  76. max_count = idxd_pmu->n_counters;
  77. n = idxd_pmu->n_events;
  78. if (n >= max_count)
  79. return -EINVAL;
  80. if (is_idxd_event(idxd_pmu, leader)) {
  81. idxd_pmu->event_list[n] = leader;
  82. idxd_pmu->event_list[n]->hw.idx = n;
  83. n++;
  84. }
  85. if (!do_grp)
  86. return n;
  87. for_each_sibling_event(event, leader) {
  88. if (!is_idxd_event(idxd_pmu, event) ||
  89. event->state <= PERF_EVENT_STATE_OFF)
  90. continue;
  91. if (n >= max_count)
  92. return -EINVAL;
  93. idxd_pmu->event_list[n] = event;
  94. idxd_pmu->event_list[n]->hw.idx = n;
  95. n++;
  96. }
  97. return n;
  98. }
  99. static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
  100. struct perf_event *event, int idx)
  101. {
  102. struct idxd_device *idxd = idxd_pmu->idxd;
  103. struct hw_perf_event *hwc = &event->hw;
  104. hwc->idx = idx;
  105. hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
  106. hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
  107. }
  108. static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
  109. struct perf_event *event)
  110. {
  111. int i;
  112. for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
  113. if (!test_and_set_bit(i, idxd_pmu->used_mask))
  114. return i;
  115. return -EINVAL;
  116. }
  117. /*
  118. * Check whether there are enough counters to satisfy that all the
  119. * events in the group can actually be scheduled at the same time.
  120. *
  121. * To do this, create a fake idxd_pmu object so the event collection
  122. * and assignment functions can be used without affecting the internal
  123. * state of the real idxd_pmu object.
  124. */
  125. static int perfmon_validate_group(struct idxd_pmu *pmu,
  126. struct perf_event *event)
  127. {
  128. struct perf_event *leader = event->group_leader;
  129. struct idxd_pmu *fake_pmu;
  130. int i, ret = 0, n, idx;
  131. fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
  132. if (!fake_pmu)
  133. return -ENOMEM;
  134. fake_pmu->pmu.name = pmu->pmu.name;
  135. fake_pmu->n_counters = pmu->n_counters;
  136. n = perfmon_collect_events(fake_pmu, leader, true);
  137. if (n < 0) {
  138. ret = n;
  139. goto out;
  140. }
  141. fake_pmu->n_events = n;
  142. n = perfmon_collect_events(fake_pmu, event, false);
  143. if (n < 0) {
  144. ret = n;
  145. goto out;
  146. }
  147. fake_pmu->n_events = n;
  148. for (i = 0; i < n; i++) {
  149. event = fake_pmu->event_list[i];
  150. idx = perfmon_assign_event(fake_pmu, event);
  151. if (idx < 0) {
  152. ret = idx;
  153. goto out;
  154. }
  155. }
  156. out:
  157. kfree(fake_pmu);
  158. return ret;
  159. }
  160. static int perfmon_pmu_event_init(struct perf_event *event)
  161. {
  162. struct idxd_device *idxd;
  163. int ret = 0;
  164. idxd = event_to_idxd(event);
  165. event->hw.idx = -1;
  166. if (event->attr.type != event->pmu->type)
  167. return -ENOENT;
  168. /* sampling not supported */
  169. if (event->attr.sample_period)
  170. return -EINVAL;
  171. if (event->cpu < 0)
  172. return -EINVAL;
  173. if (event->pmu != &idxd->idxd_pmu->pmu)
  174. return -EINVAL;
  175. event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
  176. event->cpu = idxd->idxd_pmu->cpu;
  177. event->hw.config = event->attr.config;
  178. if (event->group_leader != event)
  179. /* non-group events have themselves as leader */
  180. ret = perfmon_validate_group(idxd->idxd_pmu, event);
  181. return ret;
  182. }
  183. static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
  184. {
  185. struct hw_perf_event *hwc = &event->hw;
  186. struct idxd_device *idxd;
  187. int cntr = hwc->idx;
  188. idxd = event_to_idxd(event);
  189. return ioread64(CNTRDATA_REG(idxd, cntr));
  190. }
  191. static void perfmon_pmu_event_update(struct perf_event *event)
  192. {
  193. struct idxd_device *idxd = event_to_idxd(event);
  194. u64 prev_raw_count, new_raw_count, delta, p, n;
  195. int shift = 64 - idxd->idxd_pmu->counter_width;
  196. struct hw_perf_event *hwc = &event->hw;
  197. do {
  198. prev_raw_count = local64_read(&hwc->prev_count);
  199. new_raw_count = perfmon_pmu_read_counter(event);
  200. } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
  201. new_raw_count) != prev_raw_count);
  202. n = (new_raw_count << shift);
  203. p = (prev_raw_count << shift);
  204. delta = ((n - p) >> shift);
  205. local64_add(delta, &event->count);
  206. }
  207. void perfmon_counter_overflow(struct idxd_device *idxd)
  208. {
  209. int i, n_counters, max_loop = OVERFLOW_SIZE;
  210. struct perf_event *event;
  211. unsigned long ovfstatus;
  212. n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
  213. ovfstatus = ioread32(OVFSTATUS_REG(idxd));
  214. /*
  215. * While updating overflowed counters, other counters behind
  216. * them could overflow and be missed in a given pass.
  217. * Normally this could happen at most n_counters times, but in
  218. * theory a tiny counter width could result in continual
  219. * overflows and endless looping. max_loop provides a
  220. * failsafe in that highly unlikely case.
  221. */
  222. while (ovfstatus && max_loop--) {
  223. /* Figure out which counter(s) overflowed */
  224. for_each_set_bit(i, &ovfstatus, n_counters) {
  225. unsigned long ovfstatus_clear = 0;
  226. /* Update event->count for overflowed counter */
  227. event = idxd->idxd_pmu->event_list[i];
  228. perfmon_pmu_event_update(event);
  229. /* Writing 1 to OVFSTATUS bit clears it */
  230. set_bit(i, &ovfstatus_clear);
  231. iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
  232. }
  233. ovfstatus = ioread32(OVFSTATUS_REG(idxd));
  234. }
  235. /*
  236. * Should never happen. If so, it means a counter(s) looped
  237. * around twice while this handler was running.
  238. */
  239. WARN_ON_ONCE(ovfstatus);
  240. }
  241. static inline void perfmon_reset_config(struct idxd_device *idxd)
  242. {
  243. iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
  244. iowrite32(0, OVFSTATUS_REG(idxd));
  245. iowrite32(0, PERFFRZ_REG(idxd));
  246. }
  247. static inline void perfmon_reset_counters(struct idxd_device *idxd)
  248. {
  249. iowrite32(CNTR_RESET, PERFRST_REG(idxd));
  250. }
  251. static inline void perfmon_reset(struct idxd_device *idxd)
  252. {
  253. perfmon_reset_config(idxd);
  254. perfmon_reset_counters(idxd);
  255. }
  256. static void perfmon_pmu_event_start(struct perf_event *event, int mode)
  257. {
  258. u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
  259. u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
  260. struct hw_perf_event *hwc = &event->hw;
  261. union filter_cfg flt_cfg;
  262. union event_cfg event_cfg;
  263. struct idxd_device *idxd;
  264. int cntr;
  265. idxd = event_to_idxd(event);
  266. event->hw.idx = hwc->idx;
  267. cntr = hwc->idx;
  268. /* Obtain event category and event value from user space */
  269. event_cfg.val = event->attr.config;
  270. flt_cfg.val = event->attr.config1;
  271. event_cat = event_cfg.event_cat;
  272. event_enc = event_cfg.event_enc;
  273. /* Obtain filter configuration from user space */
  274. flt_wq = flt_cfg.wq;
  275. flt_tc = flt_cfg.tc;
  276. flt_pg_sz = flt_cfg.pg_sz;
  277. flt_xfer_sz = flt_cfg.xfer_sz;
  278. flt_eng = flt_cfg.eng;
  279. if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
  280. iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
  281. if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
  282. iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
  283. if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
  284. iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
  285. if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
  286. iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
  287. if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
  288. iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
  289. /* Read the start value */
  290. cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
  291. local64_set(&event->hw.prev_count, cntrdata);
  292. /* Set counter to event/category */
  293. cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
  294. cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
  295. /* Set interrupt on overflow and counter enable bits */
  296. cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
  297. iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
  298. }
  299. static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
  300. {
  301. struct hw_perf_event *hwc = &event->hw;
  302. struct idxd_device *idxd;
  303. int i, cntr = hwc->idx;
  304. u64 cntr_cfg;
  305. idxd = event_to_idxd(event);
  306. /* remove this event from event list */
  307. for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
  308. if (event != idxd->idxd_pmu->event_list[i])
  309. continue;
  310. for (++i; i < idxd->idxd_pmu->n_events; i++)
  311. idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
  312. --idxd->idxd_pmu->n_events;
  313. break;
  314. }
  315. cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
  316. cntr_cfg &= ~CNTRCFG_ENABLE;
  317. iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
  318. if (mode == PERF_EF_UPDATE)
  319. perfmon_pmu_event_update(event);
  320. event->hw.idx = -1;
  321. clear_bit(cntr, idxd->idxd_pmu->used_mask);
  322. }
  323. static void perfmon_pmu_event_del(struct perf_event *event, int mode)
  324. {
  325. perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
  326. }
  327. static int perfmon_pmu_event_add(struct perf_event *event, int flags)
  328. {
  329. struct idxd_device *idxd = event_to_idxd(event);
  330. struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
  331. struct hw_perf_event *hwc = &event->hw;
  332. int idx, n;
  333. n = perfmon_collect_events(idxd_pmu, event, false);
  334. if (n < 0)
  335. return n;
  336. hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
  337. if (!(flags & PERF_EF_START))
  338. hwc->state |= PERF_HES_ARCH;
  339. idx = perfmon_assign_event(idxd_pmu, event);
  340. if (idx < 0)
  341. return idx;
  342. perfmon_assign_hw_event(idxd_pmu, event, idx);
  343. if (flags & PERF_EF_START)
  344. perfmon_pmu_event_start(event, 0);
  345. idxd_pmu->n_events = n;
  346. return 0;
  347. }
  348. static void enable_perfmon_pmu(struct idxd_device *idxd)
  349. {
  350. iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
  351. }
  352. static void disable_perfmon_pmu(struct idxd_device *idxd)
  353. {
  354. iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
  355. }
  356. static void perfmon_pmu_enable(struct pmu *pmu)
  357. {
  358. struct idxd_device *idxd = pmu_to_idxd(pmu);
  359. enable_perfmon_pmu(idxd);
  360. }
  361. static void perfmon_pmu_disable(struct pmu *pmu)
  362. {
  363. struct idxd_device *idxd = pmu_to_idxd(pmu);
  364. disable_perfmon_pmu(idxd);
  365. }
  366. static void skip_filter(int i)
  367. {
  368. int j;
  369. for (j = i; j < PERFMON_FILTERS_MAX; j++)
  370. perfmon_format_attrs[PERFMON_FILTERS_START + j] =
  371. perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
  372. }
  373. static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
  374. {
  375. int i;
  376. for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
  377. if (!test_bit(i, &idxd_pmu->supported_filters))
  378. skip_filter(i);
  379. }
  380. idxd_pmu->pmu.name = idxd_pmu->name;
  381. idxd_pmu->pmu.attr_groups = perfmon_attr_groups;
  382. idxd_pmu->pmu.task_ctx_nr = perf_invalid_context;
  383. idxd_pmu->pmu.event_init = perfmon_pmu_event_init;
  384. idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable,
  385. idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable,
  386. idxd_pmu->pmu.add = perfmon_pmu_event_add;
  387. idxd_pmu->pmu.del = perfmon_pmu_event_del;
  388. idxd_pmu->pmu.start = perfmon_pmu_event_start;
  389. idxd_pmu->pmu.stop = perfmon_pmu_event_stop;
  390. idxd_pmu->pmu.read = perfmon_pmu_event_update;
  391. idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
  392. idxd_pmu->pmu.module = THIS_MODULE;
  393. }
  394. void perfmon_pmu_remove(struct idxd_device *idxd)
  395. {
  396. if (!idxd->idxd_pmu)
  397. return;
  398. cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node);
  399. perf_pmu_unregister(&idxd->idxd_pmu->pmu);
  400. kfree(idxd->idxd_pmu);
  401. idxd->idxd_pmu = NULL;
  402. }
  403. static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node)
  404. {
  405. struct idxd_pmu *idxd_pmu;
  406. idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
  407. /* select the first online CPU as the designated reader */
  408. if (cpumask_empty(&perfmon_dsa_cpu_mask)) {
  409. cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask);
  410. idxd_pmu->cpu = cpu;
  411. }
  412. return 0;
  413. }
  414. static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node)
  415. {
  416. struct idxd_pmu *idxd_pmu;
  417. unsigned int target;
  418. idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
  419. if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask))
  420. return 0;
  421. target = cpumask_any_but(cpu_online_mask, cpu);
  422. /* migrate events if there is a valid target */
  423. if (target < nr_cpu_ids)
  424. cpumask_set_cpu(target, &perfmon_dsa_cpu_mask);
  425. else
  426. target = -1;
  427. perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target);
  428. return 0;
  429. }
  430. int perfmon_pmu_init(struct idxd_device *idxd)
  431. {
  432. union idxd_perfcap perfcap;
  433. struct idxd_pmu *idxd_pmu;
  434. int rc = -ENODEV;
  435. /*
  436. * perfmon module initialization failed, nothing to do
  437. */
  438. if (!cpuhp_set_up)
  439. return -ENODEV;
  440. /*
  441. * If perfmon_offset or num_counters is 0, it means perfmon is
  442. * not supported on this hardware.
  443. */
  444. if (idxd->perfmon_offset == 0)
  445. return -ENODEV;
  446. idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
  447. if (!idxd_pmu)
  448. return -ENOMEM;
  449. idxd_pmu->idxd = idxd;
  450. idxd->idxd_pmu = idxd_pmu;
  451. if (idxd->data->type == IDXD_TYPE_DSA) {
  452. rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);
  453. if (rc < 0)
  454. goto free;
  455. } else if (idxd->data->type == IDXD_TYPE_IAX) {
  456. rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);
  457. if (rc < 0)
  458. goto free;
  459. } else {
  460. goto free;
  461. }
  462. perfmon_reset(idxd);
  463. perfcap.bits = ioread64(PERFCAP_REG(idxd));
  464. /*
  465. * If total perf counter is 0, stop further registration.
  466. * This is necessary in order to support driver running on
  467. * guest which does not have pmon support.
  468. */
  469. if (perfcap.num_perf_counter == 0)
  470. goto free;
  471. /* A counter width of 0 means it can't count */
  472. if (perfcap.counter_width == 0)
  473. goto free;
  474. /* Overflow interrupt and counter freeze support must be available */
  475. if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
  476. goto free;
  477. /* Number of event categories cannot be 0 */
  478. if (perfcap.num_event_category == 0)
  479. goto free;
  480. /*
  481. * We don't support per-counter capabilities for now.
  482. */
  483. if (perfcap.cap_per_counter)
  484. goto free;
  485. idxd_pmu->n_event_categories = perfcap.num_event_category;
  486. idxd_pmu->supported_event_categories = perfcap.global_event_category;
  487. idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
  488. /* check filter capability. If 0, then filters are not supported */
  489. idxd_pmu->supported_filters = perfcap.filter;
  490. if (perfcap.filter)
  491. idxd_pmu->n_filters = hweight8(perfcap.filter);
  492. /* Store the total number of counters categories, and counter width */
  493. idxd_pmu->n_counters = perfcap.num_perf_counter;
  494. idxd_pmu->counter_width = perfcap.counter_width;
  495. idxd_pmu_init(idxd_pmu);
  496. rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);
  497. if (rc)
  498. goto free;
  499. rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node);
  500. if (rc) {
  501. perf_pmu_unregister(&idxd->idxd_pmu->pmu);
  502. goto free;
  503. }
  504. out:
  505. return rc;
  506. free:
  507. kfree(idxd_pmu);
  508. idxd->idxd_pmu = NULL;
  509. goto out;
  510. }
  511. void __init perfmon_init(void)
  512. {
  513. int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
  514. "driver/dma/idxd/perf:online",
  515. perf_event_cpu_online,
  516. perf_event_cpu_offline);
  517. if (WARN_ON(rc < 0))
  518. return;
  519. cpuhp_slot = rc;
  520. cpuhp_set_up = true;
  521. }
  522. void __exit perfmon_exit(void)
  523. {
  524. if (cpuhp_set_up)
  525. cpuhp_remove_multi_state(cpuhp_slot);
  526. }