123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414 |
- // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
- #include <linux/irq.h>
- #include <linux/module.h>
- #include <linux/ntb.h>
- #include <linux/msi.h>
- #include <linux/pci.h>
- struct ntb_msi {
- u64 base_addr;
- u64 end_addr;
- void (*desc_changed)(void *ctx);
- u32 __iomem *peer_mws[];
- };
- /**
- * ntb_msi_init() - Initialize the MSI context
- * @ntb: NTB device context
- *
- * This function must be called before any other ntb_msi function.
- * It initializes the context for MSI operations and maps
- * the peer memory windows.
- *
- * This function reserves the last N outbound memory windows (where N
- * is the number of peers).
- *
- * Return: Zero on success, otherwise a negative error number.
- */
- int ntb_msi_init(struct ntb_dev *ntb,
- void (*desc_changed)(void *ctx))
- {
- phys_addr_t mw_phys_addr;
- resource_size_t mw_size;
- int peer_widx;
- int peers;
- int ret;
- int i;
- peers = ntb_peer_port_count(ntb);
- if (peers <= 0)
- return -EINVAL;
- ntb->msi = devm_kzalloc(&ntb->dev, struct_size(ntb->msi, peer_mws, peers),
- GFP_KERNEL);
- if (!ntb->msi)
- return -ENOMEM;
- ntb->msi->desc_changed = desc_changed;
- for (i = 0; i < peers; i++) {
- peer_widx = ntb_peer_mw_count(ntb) - 1 - i;
- ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr,
- &mw_size);
- if (ret)
- goto unroll;
- ntb->msi->peer_mws[i] = devm_ioremap(&ntb->dev, mw_phys_addr,
- mw_size);
- if (!ntb->msi->peer_mws[i]) {
- ret = -EFAULT;
- goto unroll;
- }
- }
- return 0;
- unroll:
- for (i = 0; i < peers; i++)
- if (ntb->msi->peer_mws[i])
- devm_iounmap(&ntb->dev, ntb->msi->peer_mws[i]);
- devm_kfree(&ntb->dev, ntb->msi);
- ntb->msi = NULL;
- return ret;
- }
- EXPORT_SYMBOL(ntb_msi_init);
- /**
- * ntb_msi_setup_mws() - Initialize the MSI inbound memory windows
- * @ntb: NTB device context
- *
- * This function sets up the required inbound memory windows. It should be
- * called from a work function after a link up event.
- *
- * Over the entire network, this function will reserves the last N
- * inbound memory windows for each peer (where N is the number of peers).
- *
- * ntb_msi_init() must be called before this function.
- *
- * Return: Zero on success, otherwise a negative error number.
- */
- int ntb_msi_setup_mws(struct ntb_dev *ntb)
- {
- struct msi_desc *desc;
- u64 addr;
- int peer, peer_widx;
- resource_size_t addr_align, size_align, size_max;
- resource_size_t mw_size = SZ_32K;
- resource_size_t mw_min_size = mw_size;
- int i;
- int ret;
- if (!ntb->msi)
- return -EINVAL;
- msi_lock_descs(&ntb->pdev->dev);
- desc = msi_first_desc(&ntb->pdev->dev, MSI_DESC_ASSOCIATED);
- addr = desc->msg.address_lo + ((uint64_t)desc->msg.address_hi << 32);
- msi_unlock_descs(&ntb->pdev->dev);
- for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
- peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
- if (peer_widx < 0)
- return peer_widx;
- ret = ntb_mw_get_align(ntb, peer, peer_widx, &addr_align,
- NULL, NULL);
- if (ret)
- return ret;
- addr &= ~(addr_align - 1);
- }
- for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
- peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
- if (peer_widx < 0) {
- ret = peer_widx;
- goto error_out;
- }
- ret = ntb_mw_get_align(ntb, peer, peer_widx, NULL,
- &size_align, &size_max);
- if (ret)
- goto error_out;
- mw_size = round_up(mw_size, size_align);
- mw_size = max(mw_size, size_max);
- if (mw_size < mw_min_size)
- mw_min_size = mw_size;
- ret = ntb_mw_set_trans(ntb, peer, peer_widx,
- addr, mw_size);
- if (ret)
- goto error_out;
- }
- ntb->msi->base_addr = addr;
- ntb->msi->end_addr = addr + mw_min_size;
- return 0;
- error_out:
- for (i = 0; i < peer; i++) {
- peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
- if (peer_widx < 0)
- continue;
- ntb_mw_clear_trans(ntb, i, peer_widx);
- }
- return ret;
- }
- EXPORT_SYMBOL(ntb_msi_setup_mws);
- /**
- * ntb_msi_clear_mws() - Clear all inbound memory windows
- * @ntb: NTB device context
- *
- * This function tears down the resources used by ntb_msi_setup_mws().
- */
- void ntb_msi_clear_mws(struct ntb_dev *ntb)
- {
- int peer;
- int peer_widx;
- for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
- peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
- if (peer_widx < 0)
- continue;
- ntb_mw_clear_trans(ntb, peer, peer_widx);
- }
- }
- EXPORT_SYMBOL(ntb_msi_clear_mws);
- struct ntb_msi_devres {
- struct ntb_dev *ntb;
- struct msi_desc *entry;
- struct ntb_msi_desc *msi_desc;
- };
- static int ntb_msi_set_desc(struct ntb_dev *ntb, struct msi_desc *entry,
- struct ntb_msi_desc *msi_desc)
- {
- u64 addr;
- addr = entry->msg.address_lo +
- ((uint64_t)entry->msg.address_hi << 32);
- if (addr < ntb->msi->base_addr || addr >= ntb->msi->end_addr) {
- dev_warn_once(&ntb->dev,
- "IRQ %d: MSI Address not within the memory window (%llx, [%llx %llx])\n",
- entry->irq, addr, ntb->msi->base_addr,
- ntb->msi->end_addr);
- return -EFAULT;
- }
- msi_desc->addr_offset = addr - ntb->msi->base_addr;
- msi_desc->data = entry->msg.data;
- return 0;
- }
- static void ntb_msi_write_msg(struct msi_desc *entry, void *data)
- {
- struct ntb_msi_devres *dr = data;
- WARN_ON(ntb_msi_set_desc(dr->ntb, entry, dr->msi_desc));
- if (dr->ntb->msi->desc_changed)
- dr->ntb->msi->desc_changed(dr->ntb->ctx);
- }
- static void ntbm_msi_callback_release(struct device *dev, void *res)
- {
- struct ntb_msi_devres *dr = res;
- dr->entry->write_msi_msg = NULL;
- dr->entry->write_msi_msg_data = NULL;
- }
- static int ntbm_msi_setup_callback(struct ntb_dev *ntb, struct msi_desc *entry,
- struct ntb_msi_desc *msi_desc)
- {
- struct ntb_msi_devres *dr;
- dr = devres_alloc(ntbm_msi_callback_release,
- sizeof(struct ntb_msi_devres), GFP_KERNEL);
- if (!dr)
- return -ENOMEM;
- dr->ntb = ntb;
- dr->entry = entry;
- dr->msi_desc = msi_desc;
- devres_add(&ntb->dev, dr);
- dr->entry->write_msi_msg = ntb_msi_write_msg;
- dr->entry->write_msi_msg_data = dr;
- return 0;
- }
- /**
- * ntbm_msi_request_threaded_irq() - allocate an MSI interrupt
- * @ntb: NTB device context
- * @handler: Function to be called when the IRQ occurs
- * @thread_fn: Function to be called in a threaded interrupt context. NULL
- * for clients which handle everything in @handler
- * @name: An ascii name for the claiming device, dev_name(dev) if NULL
- * @dev_id: A cookie passed back to the handler function
- * @msi_desc: MSI descriptor data which triggers the interrupt
- *
- * This function assigns an interrupt handler to an unused
- * MSI interrupt and returns the descriptor used to trigger
- * it. The descriptor can then be sent to a peer to trigger
- * the interrupt.
- *
- * The interrupt resource is managed with devres so it will
- * be automatically freed when the NTB device is torn down.
- *
- * If an IRQ allocated with this function needs to be freed
- * separately, ntbm_free_irq() must be used.
- *
- * Return: IRQ number assigned on success, otherwise a negative error number.
- */
- int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler,
- irq_handler_t thread_fn,
- const char *name, void *dev_id,
- struct ntb_msi_desc *msi_desc)
- {
- struct device *dev = &ntb->pdev->dev;
- struct msi_desc *entry;
- int ret;
- if (!ntb->msi)
- return -EINVAL;
- msi_lock_descs(dev);
- msi_for_each_desc(entry, dev, MSI_DESC_ASSOCIATED) {
- if (irq_has_action(entry->irq))
- continue;
- ret = devm_request_threaded_irq(&ntb->dev, entry->irq, handler,
- thread_fn, 0, name, dev_id);
- if (ret)
- continue;
- if (ntb_msi_set_desc(ntb, entry, msi_desc)) {
- devm_free_irq(&ntb->dev, entry->irq, dev_id);
- continue;
- }
- ret = ntbm_msi_setup_callback(ntb, entry, msi_desc);
- if (ret) {
- devm_free_irq(&ntb->dev, entry->irq, dev_id);
- goto unlock;
- }
- ret = entry->irq;
- goto unlock;
- }
- ret = -ENODEV;
- unlock:
- msi_unlock_descs(dev);
- return ret;
- }
- EXPORT_SYMBOL(ntbm_msi_request_threaded_irq);
- static int ntbm_msi_callback_match(struct device *dev, void *res, void *data)
- {
- struct ntb_dev *ntb = dev_ntb(dev);
- struct ntb_msi_devres *dr = res;
- return dr->ntb == ntb && dr->entry == data;
- }
- /**
- * ntbm_msi_free_irq() - free an interrupt
- * @ntb: NTB device context
- * @irq: Interrupt line to free
- * @dev_id: Device identity to free
- *
- * This function should be used to manually free IRQs allocated with
- * ntbm_request_[threaded_]irq().
- */
- void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq, void *dev_id)
- {
- struct msi_desc *entry = irq_get_msi_desc(irq);
- entry->write_msi_msg = NULL;
- entry->write_msi_msg_data = NULL;
- WARN_ON(devres_destroy(&ntb->dev, ntbm_msi_callback_release,
- ntbm_msi_callback_match, entry));
- devm_free_irq(&ntb->dev, irq, dev_id);
- }
- EXPORT_SYMBOL(ntbm_msi_free_irq);
- /**
- * ntb_msi_peer_trigger() - Trigger an interrupt handler on a peer
- * @ntb: NTB device context
- * @peer: Peer index
- * @desc: MSI descriptor data which triggers the interrupt
- *
- * This function triggers an interrupt on a peer. It requires
- * the descriptor structure to have been passed from that peer
- * by some other means.
- *
- * Return: Zero on success, otherwise a negative error number.
- */
- int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer,
- struct ntb_msi_desc *desc)
- {
- int idx;
- if (!ntb->msi)
- return -EINVAL;
- idx = desc->addr_offset / sizeof(*ntb->msi->peer_mws[peer]);
- iowrite32(desc->data, &ntb->msi->peer_mws[peer][idx]);
- return 0;
- }
- EXPORT_SYMBOL(ntb_msi_peer_trigger);
- /**
- * ntb_msi_peer_addr() - Get the DMA address to trigger a peer's MSI interrupt
- * @ntb: NTB device context
- * @peer: Peer index
- * @desc: MSI descriptor data which triggers the interrupt
- * @msi_addr: Physical address to trigger the interrupt
- *
- * This function allows using DMA engines to trigger an interrupt
- * (for example, trigger an interrupt to process the data after
- * sending it). To trigger the interrupt, write @desc.data to the address
- * returned in @msi_addr
- *
- * Return: Zero on success, otherwise a negative error number.
- */
- int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer,
- struct ntb_msi_desc *desc,
- phys_addr_t *msi_addr)
- {
- int peer_widx = ntb_peer_mw_count(ntb) - 1 - peer;
- phys_addr_t mw_phys_addr;
- int ret;
- ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr, NULL);
- if (ret)
- return ret;
- if (msi_addr)
- *msi_addr = mw_phys_addr + desc->addr_offset;
- return 0;
- }
- EXPORT_SYMBOL(ntb_msi_peer_addr);
|