123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779 |
- // SPDX-License-Identifier: GPL-2.0+
- // Copyright 2017 IBM Corp.
- #include <linux/sched/mm.h>
- #include <linux/mutex.h>
- #include <linux/mm.h>
- #include <linux/mm_types.h>
- #include <linux/mmu_context.h>
- #include <linux/mmu_notifier.h>
- #include <linux/irqdomain.h>
- #include <asm/copro.h>
- #include <asm/pnv-ocxl.h>
- #include <asm/xive.h>
- #include <misc/ocxl.h>
- #include "ocxl_internal.h"
- #include "trace.h"
- #define SPA_PASID_BITS 15
- #define SPA_PASID_MAX ((1 << SPA_PASID_BITS) - 1)
- #define SPA_PE_MASK SPA_PASID_MAX
- #define SPA_SPA_SIZE_LOG 22 /* Each SPA is 4 Mb */
- #define SPA_CFG_SF (1ull << (63-0))
- #define SPA_CFG_TA (1ull << (63-1))
- #define SPA_CFG_HV (1ull << (63-3))
- #define SPA_CFG_UV (1ull << (63-4))
- #define SPA_CFG_XLAT_hpt (0ull << (63-6)) /* Hashed page table (HPT) mode */
- #define SPA_CFG_XLAT_roh (2ull << (63-6)) /* Radix on HPT mode */
- #define SPA_CFG_XLAT_ror (3ull << (63-6)) /* Radix on Radix mode */
- #define SPA_CFG_PR (1ull << (63-49))
- #define SPA_CFG_TC (1ull << (63-54))
- #define SPA_CFG_DR (1ull << (63-59))
- #define SPA_XSL_TF (1ull << (63-3)) /* Translation fault */
- #define SPA_XSL_S (1ull << (63-38)) /* Store operation */
- #define SPA_PE_VALID 0x80000000
- struct ocxl_link;
- struct pe_data {
- struct mm_struct *mm;
- /* callback to trigger when a translation fault occurs */
- void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr);
- /* opaque pointer to be passed to the above callback */
- void *xsl_err_data;
- struct rcu_head rcu;
- struct ocxl_link *link;
- struct mmu_notifier mmu_notifier;
- };
- struct spa {
- struct ocxl_process_element *spa_mem;
- int spa_order;
- struct mutex spa_lock;
- struct radix_tree_root pe_tree; /* Maps PE handles to pe_data */
- char *irq_name;
- int virq;
- void __iomem *reg_dsisr;
- void __iomem *reg_dar;
- void __iomem *reg_tfc;
- void __iomem *reg_pe_handle;
- /*
- * The following field are used by the memory fault
- * interrupt handler. We can only have one interrupt at a
- * time. The NPU won't raise another interrupt until the
- * previous one has been ack'd by writing to the TFC register
- */
- struct xsl_fault {
- struct work_struct fault_work;
- u64 pe;
- u64 dsisr;
- u64 dar;
- struct pe_data pe_data;
- } xsl_fault;
- };
- /*
- * A opencapi link can be used be by several PCI functions. We have
- * one link per device slot.
- *
- * A linked list of opencapi links should suffice, as there's a
- * limited number of opencapi slots on a system and lookup is only
- * done when the device is probed
- */
- struct ocxl_link {
- struct list_head list;
- struct kref ref;
- int domain;
- int bus;
- int dev;
- void __iomem *arva; /* ATSD register virtual address */
- spinlock_t atsd_lock; /* to serialize shootdowns */
- atomic_t irq_available;
- struct spa *spa;
- void *platform_data;
- };
- static LIST_HEAD(links_list);
- static DEFINE_MUTEX(links_list_lock);
- enum xsl_response {
- CONTINUE,
- ADDRESS_ERROR,
- RESTART,
- };
- static void read_irq(struct spa *spa, u64 *dsisr, u64 *dar, u64 *pe)
- {
- u64 reg;
- *dsisr = in_be64(spa->reg_dsisr);
- *dar = in_be64(spa->reg_dar);
- reg = in_be64(spa->reg_pe_handle);
- *pe = reg & SPA_PE_MASK;
- }
- static void ack_irq(struct spa *spa, enum xsl_response r)
- {
- u64 reg = 0;
- /* continue is not supported */
- if (r == RESTART)
- reg = PPC_BIT(31);
- else if (r == ADDRESS_ERROR)
- reg = PPC_BIT(30);
- else
- WARN(1, "Invalid irq response %d\n", r);
- if (reg) {
- trace_ocxl_fault_ack(spa->spa_mem, spa->xsl_fault.pe,
- spa->xsl_fault.dsisr, spa->xsl_fault.dar, reg);
- out_be64(spa->reg_tfc, reg);
- }
- }
- static void xsl_fault_handler_bh(struct work_struct *fault_work)
- {
- vm_fault_t flt = 0;
- unsigned long access, flags, inv_flags = 0;
- enum xsl_response r;
- struct xsl_fault *fault = container_of(fault_work, struct xsl_fault,
- fault_work);
- struct spa *spa = container_of(fault, struct spa, xsl_fault);
- int rc;
- /*
- * We must release a reference on mm_users whenever exiting this
- * function (taken in the memory fault interrupt handler)
- */
- rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr,
- &flt);
- if (rc) {
- pr_debug("copro_handle_mm_fault failed: %d\n", rc);
- if (fault->pe_data.xsl_err_cb) {
- fault->pe_data.xsl_err_cb(
- fault->pe_data.xsl_err_data,
- fault->dar, fault->dsisr);
- }
- r = ADDRESS_ERROR;
- goto ack;
- }
- if (!radix_enabled()) {
- /*
- * update_mmu_cache() will not have loaded the hash
- * since current->trap is not a 0x400 or 0x300, so
- * just call hash_page_mm() here.
- */
- access = _PAGE_PRESENT | _PAGE_READ;
- if (fault->dsisr & SPA_XSL_S)
- access |= _PAGE_WRITE;
- if (get_region_id(fault->dar) != USER_REGION_ID)
- access |= _PAGE_PRIVILEGED;
- local_irq_save(flags);
- hash_page_mm(fault->pe_data.mm, fault->dar, access, 0x300,
- inv_flags);
- local_irq_restore(flags);
- }
- r = RESTART;
- ack:
- mmput(fault->pe_data.mm);
- ack_irq(spa, r);
- }
- static irqreturn_t xsl_fault_handler(int irq, void *data)
- {
- struct ocxl_link *link = (struct ocxl_link *) data;
- struct spa *spa = link->spa;
- u64 dsisr, dar, pe_handle;
- struct pe_data *pe_data;
- struct ocxl_process_element *pe;
- int pid;
- bool schedule = false;
- read_irq(spa, &dsisr, &dar, &pe_handle);
- trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1);
- WARN_ON(pe_handle > SPA_PE_MASK);
- pe = spa->spa_mem + pe_handle;
- pid = be32_to_cpu(pe->pid);
- /* We could be reading all null values here if the PE is being
- * removed while an interrupt kicks in. It's not supposed to
- * happen if the driver notified the AFU to terminate the
- * PASID, and the AFU waited for pending operations before
- * acknowledging. But even if it happens, we won't find a
- * memory context below and fail silently, so it should be ok.
- */
- if (!(dsisr & SPA_XSL_TF)) {
- WARN(1, "Invalid xsl interrupt fault register %#llx\n", dsisr);
- ack_irq(spa, ADDRESS_ERROR);
- return IRQ_HANDLED;
- }
- rcu_read_lock();
- pe_data = radix_tree_lookup(&spa->pe_tree, pe_handle);
- if (!pe_data) {
- /*
- * Could only happen if the driver didn't notify the
- * AFU about PASID termination before removing the PE,
- * or the AFU didn't wait for all memory access to
- * have completed.
- *
- * Either way, we fail early, but we shouldn't log an
- * error message, as it is a valid (if unexpected)
- * scenario
- */
- rcu_read_unlock();
- pr_debug("Unknown mm context for xsl interrupt\n");
- ack_irq(spa, ADDRESS_ERROR);
- return IRQ_HANDLED;
- }
- if (!pe_data->mm) {
- /*
- * translation fault from a kernel context - an OpenCAPI
- * device tried to access a bad kernel address
- */
- rcu_read_unlock();
- pr_warn("Unresolved OpenCAPI xsl fault in kernel context\n");
- ack_irq(spa, ADDRESS_ERROR);
- return IRQ_HANDLED;
- }
- WARN_ON(pe_data->mm->context.id != pid);
- if (mmget_not_zero(pe_data->mm)) {
- spa->xsl_fault.pe = pe_handle;
- spa->xsl_fault.dar = dar;
- spa->xsl_fault.dsisr = dsisr;
- spa->xsl_fault.pe_data = *pe_data;
- schedule = true;
- /* mm_users count released by bottom half */
- }
- rcu_read_unlock();
- if (schedule)
- schedule_work(&spa->xsl_fault.fault_work);
- else
- ack_irq(spa, ADDRESS_ERROR);
- return IRQ_HANDLED;
- }
- static void unmap_irq_registers(struct spa *spa)
- {
- pnv_ocxl_unmap_xsl_regs(spa->reg_dsisr, spa->reg_dar, spa->reg_tfc,
- spa->reg_pe_handle);
- }
- static int map_irq_registers(struct pci_dev *dev, struct spa *spa)
- {
- return pnv_ocxl_map_xsl_regs(dev, &spa->reg_dsisr, &spa->reg_dar,
- &spa->reg_tfc, &spa->reg_pe_handle);
- }
- static int setup_xsl_irq(struct pci_dev *dev, struct ocxl_link *link)
- {
- struct spa *spa = link->spa;
- int rc;
- int hwirq;
- rc = pnv_ocxl_get_xsl_irq(dev, &hwirq);
- if (rc)
- return rc;
- rc = map_irq_registers(dev, spa);
- if (rc)
- return rc;
- spa->irq_name = kasprintf(GFP_KERNEL, "ocxl-xsl-%x-%x-%x",
- link->domain, link->bus, link->dev);
- if (!spa->irq_name) {
- dev_err(&dev->dev, "Can't allocate name for xsl interrupt\n");
- rc = -ENOMEM;
- goto err_xsl;
- }
- /*
- * At some point, we'll need to look into allowing a higher
- * number of interrupts. Could we have an IRQ domain per link?
- */
- spa->virq = irq_create_mapping(NULL, hwirq);
- if (!spa->virq) {
- dev_err(&dev->dev,
- "irq_create_mapping failed for translation interrupt\n");
- rc = -EINVAL;
- goto err_name;
- }
- dev_dbg(&dev->dev, "hwirq %d mapped to virq %d\n", hwirq, spa->virq);
- rc = request_irq(spa->virq, xsl_fault_handler, 0, spa->irq_name,
- link);
- if (rc) {
- dev_err(&dev->dev,
- "request_irq failed for translation interrupt: %d\n",
- rc);
- rc = -EINVAL;
- goto err_mapping;
- }
- return 0;
- err_mapping:
- irq_dispose_mapping(spa->virq);
- err_name:
- kfree(spa->irq_name);
- err_xsl:
- unmap_irq_registers(spa);
- return rc;
- }
- static void release_xsl_irq(struct ocxl_link *link)
- {
- struct spa *spa = link->spa;
- if (spa->virq) {
- free_irq(spa->virq, link);
- irq_dispose_mapping(spa->virq);
- }
- kfree(spa->irq_name);
- unmap_irq_registers(spa);
- }
- static int alloc_spa(struct pci_dev *dev, struct ocxl_link *link)
- {
- struct spa *spa;
- spa = kzalloc(sizeof(struct spa), GFP_KERNEL);
- if (!spa)
- return -ENOMEM;
- mutex_init(&spa->spa_lock);
- INIT_RADIX_TREE(&spa->pe_tree, GFP_KERNEL);
- INIT_WORK(&spa->xsl_fault.fault_work, xsl_fault_handler_bh);
- spa->spa_order = SPA_SPA_SIZE_LOG - PAGE_SHIFT;
- spa->spa_mem = (struct ocxl_process_element *)
- __get_free_pages(GFP_KERNEL | __GFP_ZERO, spa->spa_order);
- if (!spa->spa_mem) {
- dev_err(&dev->dev, "Can't allocate Shared Process Area\n");
- kfree(spa);
- return -ENOMEM;
- }
- pr_debug("Allocated SPA for %x:%x:%x at %p\n", link->domain, link->bus,
- link->dev, spa->spa_mem);
- link->spa = spa;
- return 0;
- }
- static void free_spa(struct ocxl_link *link)
- {
- struct spa *spa = link->spa;
- pr_debug("Freeing SPA for %x:%x:%x\n", link->domain, link->bus,
- link->dev);
- if (spa && spa->spa_mem) {
- free_pages((unsigned long) spa->spa_mem, spa->spa_order);
- kfree(spa);
- link->spa = NULL;
- }
- }
- static int alloc_link(struct pci_dev *dev, int PE_mask, struct ocxl_link **out_link)
- {
- struct ocxl_link *link;
- int rc;
- link = kzalloc(sizeof(struct ocxl_link), GFP_KERNEL);
- if (!link)
- return -ENOMEM;
- kref_init(&link->ref);
- link->domain = pci_domain_nr(dev->bus);
- link->bus = dev->bus->number;
- link->dev = PCI_SLOT(dev->devfn);
- atomic_set(&link->irq_available, MAX_IRQ_PER_LINK);
- spin_lock_init(&link->atsd_lock);
- rc = alloc_spa(dev, link);
- if (rc)
- goto err_free;
- rc = setup_xsl_irq(dev, link);
- if (rc)
- goto err_spa;
- /* platform specific hook */
- rc = pnv_ocxl_spa_setup(dev, link->spa->spa_mem, PE_mask,
- &link->platform_data);
- if (rc)
- goto err_xsl_irq;
- /* if link->arva is not defeined, MMIO registers are not used to
- * generate TLB invalidate. PowerBus snooping is enabled.
- * Otherwise, PowerBus snooping is disabled. TLB Invalidates are
- * initiated using MMIO registers.
- */
- pnv_ocxl_map_lpar(dev, mfspr(SPRN_LPID), 0, &link->arva);
- *out_link = link;
- return 0;
- err_xsl_irq:
- release_xsl_irq(link);
- err_spa:
- free_spa(link);
- err_free:
- kfree(link);
- return rc;
- }
- static void free_link(struct ocxl_link *link)
- {
- release_xsl_irq(link);
- free_spa(link);
- kfree(link);
- }
- int ocxl_link_setup(struct pci_dev *dev, int PE_mask, void **link_handle)
- {
- int rc = 0;
- struct ocxl_link *link;
- mutex_lock(&links_list_lock);
- list_for_each_entry(link, &links_list, list) {
- /* The functions of a device all share the same link */
- if (link->domain == pci_domain_nr(dev->bus) &&
- link->bus == dev->bus->number &&
- link->dev == PCI_SLOT(dev->devfn)) {
- kref_get(&link->ref);
- *link_handle = link;
- goto unlock;
- }
- }
- rc = alloc_link(dev, PE_mask, &link);
- if (rc)
- goto unlock;
- list_add(&link->list, &links_list);
- *link_handle = link;
- unlock:
- mutex_unlock(&links_list_lock);
- return rc;
- }
- EXPORT_SYMBOL_GPL(ocxl_link_setup);
- static void release_xsl(struct kref *ref)
- {
- struct ocxl_link *link = container_of(ref, struct ocxl_link, ref);
- if (link->arva) {
- pnv_ocxl_unmap_lpar(link->arva);
- link->arva = NULL;
- }
- list_del(&link->list);
- /* call platform code before releasing data */
- pnv_ocxl_spa_release(link->platform_data);
- free_link(link);
- }
- void ocxl_link_release(struct pci_dev *dev, void *link_handle)
- {
- struct ocxl_link *link = (struct ocxl_link *) link_handle;
- mutex_lock(&links_list_lock);
- kref_put(&link->ref, release_xsl);
- mutex_unlock(&links_list_lock);
- }
- EXPORT_SYMBOL_GPL(ocxl_link_release);
- static void invalidate_range(struct mmu_notifier *mn,
- struct mm_struct *mm,
- unsigned long start, unsigned long end)
- {
- struct pe_data *pe_data = container_of(mn, struct pe_data, mmu_notifier);
- struct ocxl_link *link = pe_data->link;
- unsigned long addr, pid, page_size = PAGE_SIZE;
- pid = mm->context.id;
- trace_ocxl_mmu_notifier_range(start, end, pid);
- spin_lock(&link->atsd_lock);
- for (addr = start; addr < end; addr += page_size)
- pnv_ocxl_tlb_invalidate(link->arva, pid, addr, page_size);
- spin_unlock(&link->atsd_lock);
- }
- static const struct mmu_notifier_ops ocxl_mmu_notifier_ops = {
- .invalidate_range = invalidate_range,
- };
- static u64 calculate_cfg_state(bool kernel)
- {
- u64 state;
- state = SPA_CFG_DR;
- if (mfspr(SPRN_LPCR) & LPCR_TC)
- state |= SPA_CFG_TC;
- if (radix_enabled())
- state |= SPA_CFG_XLAT_ror;
- else
- state |= SPA_CFG_XLAT_hpt;
- state |= SPA_CFG_HV;
- if (kernel) {
- if (mfmsr() & MSR_SF)
- state |= SPA_CFG_SF;
- } else {
- state |= SPA_CFG_PR;
- if (!test_tsk_thread_flag(current, TIF_32BIT))
- state |= SPA_CFG_SF;
- }
- return state;
- }
- int ocxl_link_add_pe(void *link_handle, int pasid, u32 pidr, u32 tidr,
- u64 amr, u16 bdf, struct mm_struct *mm,
- void (*xsl_err_cb)(void *data, u64 addr, u64 dsisr),
- void *xsl_err_data)
- {
- struct ocxl_link *link = (struct ocxl_link *) link_handle;
- struct spa *spa = link->spa;
- struct ocxl_process_element *pe;
- int pe_handle, rc = 0;
- struct pe_data *pe_data;
- BUILD_BUG_ON(sizeof(struct ocxl_process_element) != 128);
- if (pasid > SPA_PASID_MAX)
- return -EINVAL;
- mutex_lock(&spa->spa_lock);
- pe_handle = pasid & SPA_PE_MASK;
- pe = spa->spa_mem + pe_handle;
- if (pe->software_state) {
- rc = -EBUSY;
- goto unlock;
- }
- pe_data = kmalloc(sizeof(*pe_data), GFP_KERNEL);
- if (!pe_data) {
- rc = -ENOMEM;
- goto unlock;
- }
- pe_data->mm = mm;
- pe_data->xsl_err_cb = xsl_err_cb;
- pe_data->xsl_err_data = xsl_err_data;
- pe_data->link = link;
- pe_data->mmu_notifier.ops = &ocxl_mmu_notifier_ops;
- memset(pe, 0, sizeof(struct ocxl_process_element));
- pe->config_state = cpu_to_be64(calculate_cfg_state(pidr == 0));
- pe->pasid = cpu_to_be32(pasid << (31 - 19));
- pe->bdf = cpu_to_be16(bdf);
- pe->lpid = cpu_to_be32(mfspr(SPRN_LPID));
- pe->pid = cpu_to_be32(pidr);
- pe->tid = cpu_to_be32(tidr);
- pe->amr = cpu_to_be64(amr);
- pe->software_state = cpu_to_be32(SPA_PE_VALID);
- /*
- * For user contexts, register a copro so that TLBIs are seen
- * by the nest MMU. If we have a kernel context, TLBIs are
- * already global.
- */
- if (mm) {
- mm_context_add_copro(mm);
- if (link->arva) {
- /* Use MMIO registers for the TLB Invalidate
- * operations.
- */
- trace_ocxl_init_mmu_notifier(pasid, mm->context.id);
- mmu_notifier_register(&pe_data->mmu_notifier, mm);
- }
- }
- /*
- * Barrier is to make sure PE is visible in the SPA before it
- * is used by the device. It also helps with the global TLBI
- * invalidation
- */
- mb();
- radix_tree_insert(&spa->pe_tree, pe_handle, pe_data);
- /*
- * The mm must stay valid for as long as the device uses it. We
- * lower the count when the context is removed from the SPA.
- *
- * We grab mm_count (and not mm_users), as we don't want to
- * end up in a circular dependency if a process mmaps its
- * mmio, therefore incrementing the file ref count when
- * calling mmap(), and forgets to unmap before exiting. In
- * that scenario, when the kernel handles the death of the
- * process, the file is not cleaned because unmap was not
- * called, and the mm wouldn't be freed because we would still
- * have a reference on mm_users. Incrementing mm_count solves
- * the problem.
- */
- if (mm)
- mmgrab(mm);
- trace_ocxl_context_add(current->pid, spa->spa_mem, pasid, pidr, tidr);
- unlock:
- mutex_unlock(&spa->spa_lock);
- return rc;
- }
- EXPORT_SYMBOL_GPL(ocxl_link_add_pe);
- int ocxl_link_update_pe(void *link_handle, int pasid, __u16 tid)
- {
- struct ocxl_link *link = (struct ocxl_link *) link_handle;
- struct spa *spa = link->spa;
- struct ocxl_process_element *pe;
- int pe_handle, rc;
- if (pasid > SPA_PASID_MAX)
- return -EINVAL;
- pe_handle = pasid & SPA_PE_MASK;
- pe = spa->spa_mem + pe_handle;
- mutex_lock(&spa->spa_lock);
- pe->tid = cpu_to_be32(tid);
- /*
- * The barrier makes sure the PE is updated
- * before we clear the NPU context cache below, so that the
- * old PE cannot be reloaded erroneously.
- */
- mb();
- /*
- * hook to platform code
- * On powerpc, the entry needs to be cleared from the context
- * cache of the NPU.
- */
- rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
- WARN_ON(rc);
- mutex_unlock(&spa->spa_lock);
- return rc;
- }
- int ocxl_link_remove_pe(void *link_handle, int pasid)
- {
- struct ocxl_link *link = (struct ocxl_link *) link_handle;
- struct spa *spa = link->spa;
- struct ocxl_process_element *pe;
- struct pe_data *pe_data;
- int pe_handle, rc;
- if (pasid > SPA_PASID_MAX)
- return -EINVAL;
- /*
- * About synchronization with our memory fault handler:
- *
- * Before removing the PE, the driver is supposed to have
- * notified the AFU, which should have cleaned up and make
- * sure the PASID is no longer in use, including pending
- * interrupts. However, there's no way to be sure...
- *
- * We clear the PE and remove the context from our radix
- * tree. From that point on, any new interrupt for that
- * context will fail silently, which is ok. As mentioned
- * above, that's not expected, but it could happen if the
- * driver or AFU didn't do the right thing.
- *
- * There could still be a bottom half running, but we don't
- * need to wait/flush, as it is managing a reference count on
- * the mm it reads from the radix tree.
- */
- pe_handle = pasid & SPA_PE_MASK;
- pe = spa->spa_mem + pe_handle;
- mutex_lock(&spa->spa_lock);
- if (!(be32_to_cpu(pe->software_state) & SPA_PE_VALID)) {
- rc = -EINVAL;
- goto unlock;
- }
- trace_ocxl_context_remove(current->pid, spa->spa_mem, pasid,
- be32_to_cpu(pe->pid), be32_to_cpu(pe->tid));
- memset(pe, 0, sizeof(struct ocxl_process_element));
- /*
- * The barrier makes sure the PE is removed from the SPA
- * before we clear the NPU context cache below, so that the
- * old PE cannot be reloaded erroneously.
- */
- mb();
- /*
- * hook to platform code
- * On powerpc, the entry needs to be cleared from the context
- * cache of the NPU.
- */
- rc = pnv_ocxl_spa_remove_pe_from_cache(link->platform_data, pe_handle);
- WARN_ON(rc);
- pe_data = radix_tree_delete(&spa->pe_tree, pe_handle);
- if (!pe_data) {
- WARN(1, "Couldn't find pe data when removing PE\n");
- } else {
- if (pe_data->mm) {
- if (link->arva) {
- trace_ocxl_release_mmu_notifier(pasid,
- pe_data->mm->context.id);
- mmu_notifier_unregister(&pe_data->mmu_notifier,
- pe_data->mm);
- spin_lock(&link->atsd_lock);
- pnv_ocxl_tlb_invalidate(link->arva,
- pe_data->mm->context.id,
- 0ull,
- PAGE_SIZE);
- spin_unlock(&link->atsd_lock);
- }
- mm_context_remove_copro(pe_data->mm);
- mmdrop(pe_data->mm);
- }
- kfree_rcu(pe_data, rcu);
- }
- unlock:
- mutex_unlock(&spa->spa_lock);
- return rc;
- }
- EXPORT_SYMBOL_GPL(ocxl_link_remove_pe);
- int ocxl_link_irq_alloc(void *link_handle, int *hw_irq)
- {
- struct ocxl_link *link = (struct ocxl_link *) link_handle;
- int irq;
- if (atomic_dec_if_positive(&link->irq_available) < 0)
- return -ENOSPC;
- irq = xive_native_alloc_irq();
- if (!irq) {
- atomic_inc(&link->irq_available);
- return -ENXIO;
- }
- *hw_irq = irq;
- return 0;
- }
- EXPORT_SYMBOL_GPL(ocxl_link_irq_alloc);
- void ocxl_link_free_irq(void *link_handle, int hw_irq)
- {
- struct ocxl_link *link = (struct ocxl_link *) link_handle;
- xive_native_free_irq(hw_irq);
- atomic_inc(&link->irq_available);
- }
- EXPORT_SYMBOL_GPL(ocxl_link_free_irq);
|