123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662 |
- // SPDX-License-Identifier: GPL-2.0
- /* Copyright(c) 2020 Intel Corporation. All rights rsvd. */
- #include <linux/sched/task.h>
- #include <linux/io-64-nonatomic-lo-hi.h>
- #include "idxd.h"
- #include "perfmon.h"
- static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
- char *buf);
- static cpumask_t perfmon_dsa_cpu_mask;
- static bool cpuhp_set_up;
- static enum cpuhp_state cpuhp_slot;
- /*
- * perf userspace reads this attribute to determine which cpus to open
- * counters on. It's connected to perfmon_dsa_cpu_mask, which is
- * maintained by the cpu hotplug handlers.
- */
- static DEVICE_ATTR_RO(cpumask);
- static struct attribute *perfmon_cpumask_attrs[] = {
- &dev_attr_cpumask.attr,
- NULL,
- };
- static struct attribute_group cpumask_attr_group = {
- .attrs = perfmon_cpumask_attrs,
- };
- /*
- * These attributes specify the bits in the config word that the perf
- * syscall uses to pass the event ids and categories to perfmon.
- */
- DEFINE_PERFMON_FORMAT_ATTR(event_category, "config:0-3");
- DEFINE_PERFMON_FORMAT_ATTR(event, "config:4-31");
- /*
- * These attributes specify the bits in the config1 word that the perf
- * syscall uses to pass filter data to perfmon.
- */
- DEFINE_PERFMON_FORMAT_ATTR(filter_wq, "config1:0-31");
- DEFINE_PERFMON_FORMAT_ATTR(filter_tc, "config1:32-39");
- DEFINE_PERFMON_FORMAT_ATTR(filter_pgsz, "config1:40-43");
- DEFINE_PERFMON_FORMAT_ATTR(filter_sz, "config1:44-51");
- DEFINE_PERFMON_FORMAT_ATTR(filter_eng, "config1:52-59");
- #define PERFMON_FILTERS_START 2
- #define PERFMON_FILTERS_MAX 5
- static struct attribute *perfmon_format_attrs[] = {
- &format_attr_idxd_event_category.attr,
- &format_attr_idxd_event.attr,
- &format_attr_idxd_filter_wq.attr,
- &format_attr_idxd_filter_tc.attr,
- &format_attr_idxd_filter_pgsz.attr,
- &format_attr_idxd_filter_sz.attr,
- &format_attr_idxd_filter_eng.attr,
- NULL,
- };
- static struct attribute_group perfmon_format_attr_group = {
- .name = "format",
- .attrs = perfmon_format_attrs,
- };
- static const struct attribute_group *perfmon_attr_groups[] = {
- &perfmon_format_attr_group,
- &cpumask_attr_group,
- NULL,
- };
- static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr,
- char *buf)
- {
- return cpumap_print_to_pagebuf(true, buf, &perfmon_dsa_cpu_mask);
- }
- static bool is_idxd_event(struct idxd_pmu *idxd_pmu, struct perf_event *event)
- {
- return &idxd_pmu->pmu == event->pmu;
- }
- static int perfmon_collect_events(struct idxd_pmu *idxd_pmu,
- struct perf_event *leader,
- bool do_grp)
- {
- struct perf_event *event;
- int n, max_count;
- max_count = idxd_pmu->n_counters;
- n = idxd_pmu->n_events;
- if (n >= max_count)
- return -EINVAL;
- if (is_idxd_event(idxd_pmu, leader)) {
- idxd_pmu->event_list[n] = leader;
- idxd_pmu->event_list[n]->hw.idx = n;
- n++;
- }
- if (!do_grp)
- return n;
- for_each_sibling_event(event, leader) {
- if (!is_idxd_event(idxd_pmu, event) ||
- event->state <= PERF_EVENT_STATE_OFF)
- continue;
- if (n >= max_count)
- return -EINVAL;
- idxd_pmu->event_list[n] = event;
- idxd_pmu->event_list[n]->hw.idx = n;
- n++;
- }
- return n;
- }
- static void perfmon_assign_hw_event(struct idxd_pmu *idxd_pmu,
- struct perf_event *event, int idx)
- {
- struct idxd_device *idxd = idxd_pmu->idxd;
- struct hw_perf_event *hwc = &event->hw;
- hwc->idx = idx;
- hwc->config_base = ioread64(CNTRCFG_REG(idxd, idx));
- hwc->event_base = ioread64(CNTRCFG_REG(idxd, idx));
- }
- static int perfmon_assign_event(struct idxd_pmu *idxd_pmu,
- struct perf_event *event)
- {
- int i;
- for (i = 0; i < IDXD_PMU_EVENT_MAX; i++)
- if (!test_and_set_bit(i, idxd_pmu->used_mask))
- return i;
- return -EINVAL;
- }
- /*
- * Check whether there are enough counters to satisfy that all the
- * events in the group can actually be scheduled at the same time.
- *
- * To do this, create a fake idxd_pmu object so the event collection
- * and assignment functions can be used without affecting the internal
- * state of the real idxd_pmu object.
- */
- static int perfmon_validate_group(struct idxd_pmu *pmu,
- struct perf_event *event)
- {
- struct perf_event *leader = event->group_leader;
- struct idxd_pmu *fake_pmu;
- int i, ret = 0, n, idx;
- fake_pmu = kzalloc(sizeof(*fake_pmu), GFP_KERNEL);
- if (!fake_pmu)
- return -ENOMEM;
- fake_pmu->pmu.name = pmu->pmu.name;
- fake_pmu->n_counters = pmu->n_counters;
- n = perfmon_collect_events(fake_pmu, leader, true);
- if (n < 0) {
- ret = n;
- goto out;
- }
- fake_pmu->n_events = n;
- n = perfmon_collect_events(fake_pmu, event, false);
- if (n < 0) {
- ret = n;
- goto out;
- }
- fake_pmu->n_events = n;
- for (i = 0; i < n; i++) {
- event = fake_pmu->event_list[i];
- idx = perfmon_assign_event(fake_pmu, event);
- if (idx < 0) {
- ret = idx;
- goto out;
- }
- }
- out:
- kfree(fake_pmu);
- return ret;
- }
- static int perfmon_pmu_event_init(struct perf_event *event)
- {
- struct idxd_device *idxd;
- int ret = 0;
- idxd = event_to_idxd(event);
- event->hw.idx = -1;
- if (event->attr.type != event->pmu->type)
- return -ENOENT;
- /* sampling not supported */
- if (event->attr.sample_period)
- return -EINVAL;
- if (event->cpu < 0)
- return -EINVAL;
- if (event->pmu != &idxd->idxd_pmu->pmu)
- return -EINVAL;
- event->hw.event_base = ioread64(PERFMON_TABLE_OFFSET(idxd));
- event->cpu = idxd->idxd_pmu->cpu;
- event->hw.config = event->attr.config;
- if (event->group_leader != event)
- /* non-group events have themselves as leader */
- ret = perfmon_validate_group(idxd->idxd_pmu, event);
- return ret;
- }
- static inline u64 perfmon_pmu_read_counter(struct perf_event *event)
- {
- struct hw_perf_event *hwc = &event->hw;
- struct idxd_device *idxd;
- int cntr = hwc->idx;
- idxd = event_to_idxd(event);
- return ioread64(CNTRDATA_REG(idxd, cntr));
- }
- static void perfmon_pmu_event_update(struct perf_event *event)
- {
- struct idxd_device *idxd = event_to_idxd(event);
- u64 prev_raw_count, new_raw_count, delta, p, n;
- int shift = 64 - idxd->idxd_pmu->counter_width;
- struct hw_perf_event *hwc = &event->hw;
- do {
- prev_raw_count = local64_read(&hwc->prev_count);
- new_raw_count = perfmon_pmu_read_counter(event);
- } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
- new_raw_count) != prev_raw_count);
- n = (new_raw_count << shift);
- p = (prev_raw_count << shift);
- delta = ((n - p) >> shift);
- local64_add(delta, &event->count);
- }
- void perfmon_counter_overflow(struct idxd_device *idxd)
- {
- int i, n_counters, max_loop = OVERFLOW_SIZE;
- struct perf_event *event;
- unsigned long ovfstatus;
- n_counters = min(idxd->idxd_pmu->n_counters, OVERFLOW_SIZE);
- ovfstatus = ioread32(OVFSTATUS_REG(idxd));
- /*
- * While updating overflowed counters, other counters behind
- * them could overflow and be missed in a given pass.
- * Normally this could happen at most n_counters times, but in
- * theory a tiny counter width could result in continual
- * overflows and endless looping. max_loop provides a
- * failsafe in that highly unlikely case.
- */
- while (ovfstatus && max_loop--) {
- /* Figure out which counter(s) overflowed */
- for_each_set_bit(i, &ovfstatus, n_counters) {
- unsigned long ovfstatus_clear = 0;
- /* Update event->count for overflowed counter */
- event = idxd->idxd_pmu->event_list[i];
- perfmon_pmu_event_update(event);
- /* Writing 1 to OVFSTATUS bit clears it */
- set_bit(i, &ovfstatus_clear);
- iowrite32(ovfstatus_clear, OVFSTATUS_REG(idxd));
- }
- ovfstatus = ioread32(OVFSTATUS_REG(idxd));
- }
- /*
- * Should never happen. If so, it means a counter(s) looped
- * around twice while this handler was running.
- */
- WARN_ON_ONCE(ovfstatus);
- }
- static inline void perfmon_reset_config(struct idxd_device *idxd)
- {
- iowrite32(CONFIG_RESET, PERFRST_REG(idxd));
- iowrite32(0, OVFSTATUS_REG(idxd));
- iowrite32(0, PERFFRZ_REG(idxd));
- }
- static inline void perfmon_reset_counters(struct idxd_device *idxd)
- {
- iowrite32(CNTR_RESET, PERFRST_REG(idxd));
- }
- static inline void perfmon_reset(struct idxd_device *idxd)
- {
- perfmon_reset_config(idxd);
- perfmon_reset_counters(idxd);
- }
- static void perfmon_pmu_event_start(struct perf_event *event, int mode)
- {
- u32 flt_wq, flt_tc, flt_pg_sz, flt_xfer_sz, flt_eng = 0;
- u64 cntr_cfg, cntrdata, event_enc, event_cat = 0;
- struct hw_perf_event *hwc = &event->hw;
- union filter_cfg flt_cfg;
- union event_cfg event_cfg;
- struct idxd_device *idxd;
- int cntr;
- idxd = event_to_idxd(event);
- event->hw.idx = hwc->idx;
- cntr = hwc->idx;
- /* Obtain event category and event value from user space */
- event_cfg.val = event->attr.config;
- flt_cfg.val = event->attr.config1;
- event_cat = event_cfg.event_cat;
- event_enc = event_cfg.event_enc;
- /* Obtain filter configuration from user space */
- flt_wq = flt_cfg.wq;
- flt_tc = flt_cfg.tc;
- flt_pg_sz = flt_cfg.pg_sz;
- flt_xfer_sz = flt_cfg.xfer_sz;
- flt_eng = flt_cfg.eng;
- if (flt_wq && test_bit(FLT_WQ, &idxd->idxd_pmu->supported_filters))
- iowrite32(flt_wq, FLTCFG_REG(idxd, cntr, FLT_WQ));
- if (flt_tc && test_bit(FLT_TC, &idxd->idxd_pmu->supported_filters))
- iowrite32(flt_tc, FLTCFG_REG(idxd, cntr, FLT_TC));
- if (flt_pg_sz && test_bit(FLT_PG_SZ, &idxd->idxd_pmu->supported_filters))
- iowrite32(flt_pg_sz, FLTCFG_REG(idxd, cntr, FLT_PG_SZ));
- if (flt_xfer_sz && test_bit(FLT_XFER_SZ, &idxd->idxd_pmu->supported_filters))
- iowrite32(flt_xfer_sz, FLTCFG_REG(idxd, cntr, FLT_XFER_SZ));
- if (flt_eng && test_bit(FLT_ENG, &idxd->idxd_pmu->supported_filters))
- iowrite32(flt_eng, FLTCFG_REG(idxd, cntr, FLT_ENG));
- /* Read the start value */
- cntrdata = ioread64(CNTRDATA_REG(idxd, cntr));
- local64_set(&event->hw.prev_count, cntrdata);
- /* Set counter to event/category */
- cntr_cfg = event_cat << CNTRCFG_CATEGORY_SHIFT;
- cntr_cfg |= event_enc << CNTRCFG_EVENT_SHIFT;
- /* Set interrupt on overflow and counter enable bits */
- cntr_cfg |= (CNTRCFG_IRQ_OVERFLOW | CNTRCFG_ENABLE);
- iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
- }
- static void perfmon_pmu_event_stop(struct perf_event *event, int mode)
- {
- struct hw_perf_event *hwc = &event->hw;
- struct idxd_device *idxd;
- int i, cntr = hwc->idx;
- u64 cntr_cfg;
- idxd = event_to_idxd(event);
- /* remove this event from event list */
- for (i = 0; i < idxd->idxd_pmu->n_events; i++) {
- if (event != idxd->idxd_pmu->event_list[i])
- continue;
- for (++i; i < idxd->idxd_pmu->n_events; i++)
- idxd->idxd_pmu->event_list[i - 1] = idxd->idxd_pmu->event_list[i];
- --idxd->idxd_pmu->n_events;
- break;
- }
- cntr_cfg = ioread64(CNTRCFG_REG(idxd, cntr));
- cntr_cfg &= ~CNTRCFG_ENABLE;
- iowrite64(cntr_cfg, CNTRCFG_REG(idxd, cntr));
- if (mode == PERF_EF_UPDATE)
- perfmon_pmu_event_update(event);
- event->hw.idx = -1;
- clear_bit(cntr, idxd->idxd_pmu->used_mask);
- }
- static void perfmon_pmu_event_del(struct perf_event *event, int mode)
- {
- perfmon_pmu_event_stop(event, PERF_EF_UPDATE);
- }
- static int perfmon_pmu_event_add(struct perf_event *event, int flags)
- {
- struct idxd_device *idxd = event_to_idxd(event);
- struct idxd_pmu *idxd_pmu = idxd->idxd_pmu;
- struct hw_perf_event *hwc = &event->hw;
- int idx, n;
- n = perfmon_collect_events(idxd_pmu, event, false);
- if (n < 0)
- return n;
- hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
- if (!(flags & PERF_EF_START))
- hwc->state |= PERF_HES_ARCH;
- idx = perfmon_assign_event(idxd_pmu, event);
- if (idx < 0)
- return idx;
- perfmon_assign_hw_event(idxd_pmu, event, idx);
- if (flags & PERF_EF_START)
- perfmon_pmu_event_start(event, 0);
- idxd_pmu->n_events = n;
- return 0;
- }
- static void enable_perfmon_pmu(struct idxd_device *idxd)
- {
- iowrite32(COUNTER_UNFREEZE, PERFFRZ_REG(idxd));
- }
- static void disable_perfmon_pmu(struct idxd_device *idxd)
- {
- iowrite32(COUNTER_FREEZE, PERFFRZ_REG(idxd));
- }
- static void perfmon_pmu_enable(struct pmu *pmu)
- {
- struct idxd_device *idxd = pmu_to_idxd(pmu);
- enable_perfmon_pmu(idxd);
- }
- static void perfmon_pmu_disable(struct pmu *pmu)
- {
- struct idxd_device *idxd = pmu_to_idxd(pmu);
- disable_perfmon_pmu(idxd);
- }
- static void skip_filter(int i)
- {
- int j;
- for (j = i; j < PERFMON_FILTERS_MAX; j++)
- perfmon_format_attrs[PERFMON_FILTERS_START + j] =
- perfmon_format_attrs[PERFMON_FILTERS_START + j + 1];
- }
- static void idxd_pmu_init(struct idxd_pmu *idxd_pmu)
- {
- int i;
- for (i = 0 ; i < PERFMON_FILTERS_MAX; i++) {
- if (!test_bit(i, &idxd_pmu->supported_filters))
- skip_filter(i);
- }
- idxd_pmu->pmu.name = idxd_pmu->name;
- idxd_pmu->pmu.attr_groups = perfmon_attr_groups;
- idxd_pmu->pmu.task_ctx_nr = perf_invalid_context;
- idxd_pmu->pmu.event_init = perfmon_pmu_event_init;
- idxd_pmu->pmu.pmu_enable = perfmon_pmu_enable,
- idxd_pmu->pmu.pmu_disable = perfmon_pmu_disable,
- idxd_pmu->pmu.add = perfmon_pmu_event_add;
- idxd_pmu->pmu.del = perfmon_pmu_event_del;
- idxd_pmu->pmu.start = perfmon_pmu_event_start;
- idxd_pmu->pmu.stop = perfmon_pmu_event_stop;
- idxd_pmu->pmu.read = perfmon_pmu_event_update;
- idxd_pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
- idxd_pmu->pmu.module = THIS_MODULE;
- }
- void perfmon_pmu_remove(struct idxd_device *idxd)
- {
- if (!idxd->idxd_pmu)
- return;
- cpuhp_state_remove_instance(cpuhp_slot, &idxd->idxd_pmu->cpuhp_node);
- perf_pmu_unregister(&idxd->idxd_pmu->pmu);
- kfree(idxd->idxd_pmu);
- idxd->idxd_pmu = NULL;
- }
- static int perf_event_cpu_online(unsigned int cpu, struct hlist_node *node)
- {
- struct idxd_pmu *idxd_pmu;
- idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
- /* select the first online CPU as the designated reader */
- if (cpumask_empty(&perfmon_dsa_cpu_mask)) {
- cpumask_set_cpu(cpu, &perfmon_dsa_cpu_mask);
- idxd_pmu->cpu = cpu;
- }
- return 0;
- }
- static int perf_event_cpu_offline(unsigned int cpu, struct hlist_node *node)
- {
- struct idxd_pmu *idxd_pmu;
- unsigned int target;
- idxd_pmu = hlist_entry_safe(node, typeof(*idxd_pmu), cpuhp_node);
- if (!cpumask_test_and_clear_cpu(cpu, &perfmon_dsa_cpu_mask))
- return 0;
- target = cpumask_any_but(cpu_online_mask, cpu);
- /* migrate events if there is a valid target */
- if (target < nr_cpu_ids)
- cpumask_set_cpu(target, &perfmon_dsa_cpu_mask);
- else
- target = -1;
- perf_pmu_migrate_context(&idxd_pmu->pmu, cpu, target);
- return 0;
- }
- int perfmon_pmu_init(struct idxd_device *idxd)
- {
- union idxd_perfcap perfcap;
- struct idxd_pmu *idxd_pmu;
- int rc = -ENODEV;
- /*
- * perfmon module initialization failed, nothing to do
- */
- if (!cpuhp_set_up)
- return -ENODEV;
- /*
- * If perfmon_offset or num_counters is 0, it means perfmon is
- * not supported on this hardware.
- */
- if (idxd->perfmon_offset == 0)
- return -ENODEV;
- idxd_pmu = kzalloc(sizeof(*idxd_pmu), GFP_KERNEL);
- if (!idxd_pmu)
- return -ENOMEM;
- idxd_pmu->idxd = idxd;
- idxd->idxd_pmu = idxd_pmu;
- if (idxd->data->type == IDXD_TYPE_DSA) {
- rc = sprintf(idxd_pmu->name, "dsa%d", idxd->id);
- if (rc < 0)
- goto free;
- } else if (idxd->data->type == IDXD_TYPE_IAX) {
- rc = sprintf(idxd_pmu->name, "iax%d", idxd->id);
- if (rc < 0)
- goto free;
- } else {
- goto free;
- }
- perfmon_reset(idxd);
- perfcap.bits = ioread64(PERFCAP_REG(idxd));
- /*
- * If total perf counter is 0, stop further registration.
- * This is necessary in order to support driver running on
- * guest which does not have pmon support.
- */
- if (perfcap.num_perf_counter == 0)
- goto free;
- /* A counter width of 0 means it can't count */
- if (perfcap.counter_width == 0)
- goto free;
- /* Overflow interrupt and counter freeze support must be available */
- if (!perfcap.overflow_interrupt || !perfcap.counter_freeze)
- goto free;
- /* Number of event categories cannot be 0 */
- if (perfcap.num_event_category == 0)
- goto free;
- /*
- * We don't support per-counter capabilities for now.
- */
- if (perfcap.cap_per_counter)
- goto free;
- idxd_pmu->n_event_categories = perfcap.num_event_category;
- idxd_pmu->supported_event_categories = perfcap.global_event_category;
- idxd_pmu->per_counter_caps_supported = perfcap.cap_per_counter;
- /* check filter capability. If 0, then filters are not supported */
- idxd_pmu->supported_filters = perfcap.filter;
- if (perfcap.filter)
- idxd_pmu->n_filters = hweight8(perfcap.filter);
- /* Store the total number of counters categories, and counter width */
- idxd_pmu->n_counters = perfcap.num_perf_counter;
- idxd_pmu->counter_width = perfcap.counter_width;
- idxd_pmu_init(idxd_pmu);
- rc = perf_pmu_register(&idxd_pmu->pmu, idxd_pmu->name, -1);
- if (rc)
- goto free;
- rc = cpuhp_state_add_instance(cpuhp_slot, &idxd_pmu->cpuhp_node);
- if (rc) {
- perf_pmu_unregister(&idxd->idxd_pmu->pmu);
- goto free;
- }
- out:
- return rc;
- free:
- kfree(idxd_pmu);
- idxd->idxd_pmu = NULL;
- goto out;
- }
- void __init perfmon_init(void)
- {
- int rc = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
- "driver/dma/idxd/perf:online",
- perf_event_cpu_online,
- perf_event_cpu_offline);
- if (WARN_ON(rc < 0))
- return;
- cpuhp_slot = rc;
- cpuhp_set_up = true;
- }
- void __exit perfmon_exit(void)
- {
- if (cpuhp_set_up)
- cpuhp_remove_multi_state(cpuhp_slot);
- }
|