msi.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414
  1. // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
  2. #include <linux/irq.h>
  3. #include <linux/module.h>
  4. #include <linux/ntb.h>
  5. #include <linux/msi.h>
  6. #include <linux/pci.h>
  7. struct ntb_msi {
  8. u64 base_addr;
  9. u64 end_addr;
  10. void (*desc_changed)(void *ctx);
  11. u32 __iomem *peer_mws[];
  12. };
  13. /**
  14. * ntb_msi_init() - Initialize the MSI context
  15. * @ntb: NTB device context
  16. *
  17. * This function must be called before any other ntb_msi function.
  18. * It initializes the context for MSI operations and maps
  19. * the peer memory windows.
  20. *
  21. * This function reserves the last N outbound memory windows (where N
  22. * is the number of peers).
  23. *
  24. * Return: Zero on success, otherwise a negative error number.
  25. */
  26. int ntb_msi_init(struct ntb_dev *ntb,
  27. void (*desc_changed)(void *ctx))
  28. {
  29. phys_addr_t mw_phys_addr;
  30. resource_size_t mw_size;
  31. int peer_widx;
  32. int peers;
  33. int ret;
  34. int i;
  35. peers = ntb_peer_port_count(ntb);
  36. if (peers <= 0)
  37. return -EINVAL;
  38. ntb->msi = devm_kzalloc(&ntb->dev, struct_size(ntb->msi, peer_mws, peers),
  39. GFP_KERNEL);
  40. if (!ntb->msi)
  41. return -ENOMEM;
  42. ntb->msi->desc_changed = desc_changed;
  43. for (i = 0; i < peers; i++) {
  44. peer_widx = ntb_peer_mw_count(ntb) - 1 - i;
  45. ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr,
  46. &mw_size);
  47. if (ret)
  48. goto unroll;
  49. ntb->msi->peer_mws[i] = devm_ioremap(&ntb->dev, mw_phys_addr,
  50. mw_size);
  51. if (!ntb->msi->peer_mws[i]) {
  52. ret = -EFAULT;
  53. goto unroll;
  54. }
  55. }
  56. return 0;
  57. unroll:
  58. for (i = 0; i < peers; i++)
  59. if (ntb->msi->peer_mws[i])
  60. devm_iounmap(&ntb->dev, ntb->msi->peer_mws[i]);
  61. devm_kfree(&ntb->dev, ntb->msi);
  62. ntb->msi = NULL;
  63. return ret;
  64. }
  65. EXPORT_SYMBOL(ntb_msi_init);
  66. /**
  67. * ntb_msi_setup_mws() - Initialize the MSI inbound memory windows
  68. * @ntb: NTB device context
  69. *
  70. * This function sets up the required inbound memory windows. It should be
  71. * called from a work function after a link up event.
  72. *
  73. * Over the entire network, this function will reserves the last N
  74. * inbound memory windows for each peer (where N is the number of peers).
  75. *
  76. * ntb_msi_init() must be called before this function.
  77. *
  78. * Return: Zero on success, otherwise a negative error number.
  79. */
  80. int ntb_msi_setup_mws(struct ntb_dev *ntb)
  81. {
  82. struct msi_desc *desc;
  83. u64 addr;
  84. int peer, peer_widx;
  85. resource_size_t addr_align, size_align, size_max;
  86. resource_size_t mw_size = SZ_32K;
  87. resource_size_t mw_min_size = mw_size;
  88. int i;
  89. int ret;
  90. if (!ntb->msi)
  91. return -EINVAL;
  92. msi_lock_descs(&ntb->pdev->dev);
  93. desc = msi_first_desc(&ntb->pdev->dev, MSI_DESC_ASSOCIATED);
  94. addr = desc->msg.address_lo + ((uint64_t)desc->msg.address_hi << 32);
  95. msi_unlock_descs(&ntb->pdev->dev);
  96. for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
  97. peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
  98. if (peer_widx < 0)
  99. return peer_widx;
  100. ret = ntb_mw_get_align(ntb, peer, peer_widx, &addr_align,
  101. NULL, NULL);
  102. if (ret)
  103. return ret;
  104. addr &= ~(addr_align - 1);
  105. }
  106. for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
  107. peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
  108. if (peer_widx < 0) {
  109. ret = peer_widx;
  110. goto error_out;
  111. }
  112. ret = ntb_mw_get_align(ntb, peer, peer_widx, NULL,
  113. &size_align, &size_max);
  114. if (ret)
  115. goto error_out;
  116. mw_size = round_up(mw_size, size_align);
  117. mw_size = max(mw_size, size_max);
  118. if (mw_size < mw_min_size)
  119. mw_min_size = mw_size;
  120. ret = ntb_mw_set_trans(ntb, peer, peer_widx,
  121. addr, mw_size);
  122. if (ret)
  123. goto error_out;
  124. }
  125. ntb->msi->base_addr = addr;
  126. ntb->msi->end_addr = addr + mw_min_size;
  127. return 0;
  128. error_out:
  129. for (i = 0; i < peer; i++) {
  130. peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
  131. if (peer_widx < 0)
  132. continue;
  133. ntb_mw_clear_trans(ntb, i, peer_widx);
  134. }
  135. return ret;
  136. }
  137. EXPORT_SYMBOL(ntb_msi_setup_mws);
  138. /**
  139. * ntb_msi_clear_mws() - Clear all inbound memory windows
  140. * @ntb: NTB device context
  141. *
  142. * This function tears down the resources used by ntb_msi_setup_mws().
  143. */
  144. void ntb_msi_clear_mws(struct ntb_dev *ntb)
  145. {
  146. int peer;
  147. int peer_widx;
  148. for (peer = 0; peer < ntb_peer_port_count(ntb); peer++) {
  149. peer_widx = ntb_peer_highest_mw_idx(ntb, peer);
  150. if (peer_widx < 0)
  151. continue;
  152. ntb_mw_clear_trans(ntb, peer, peer_widx);
  153. }
  154. }
  155. EXPORT_SYMBOL(ntb_msi_clear_mws);
  156. struct ntb_msi_devres {
  157. struct ntb_dev *ntb;
  158. struct msi_desc *entry;
  159. struct ntb_msi_desc *msi_desc;
  160. };
  161. static int ntb_msi_set_desc(struct ntb_dev *ntb, struct msi_desc *entry,
  162. struct ntb_msi_desc *msi_desc)
  163. {
  164. u64 addr;
  165. addr = entry->msg.address_lo +
  166. ((uint64_t)entry->msg.address_hi << 32);
  167. if (addr < ntb->msi->base_addr || addr >= ntb->msi->end_addr) {
  168. dev_warn_once(&ntb->dev,
  169. "IRQ %d: MSI Address not within the memory window (%llx, [%llx %llx])\n",
  170. entry->irq, addr, ntb->msi->base_addr,
  171. ntb->msi->end_addr);
  172. return -EFAULT;
  173. }
  174. msi_desc->addr_offset = addr - ntb->msi->base_addr;
  175. msi_desc->data = entry->msg.data;
  176. return 0;
  177. }
  178. static void ntb_msi_write_msg(struct msi_desc *entry, void *data)
  179. {
  180. struct ntb_msi_devres *dr = data;
  181. WARN_ON(ntb_msi_set_desc(dr->ntb, entry, dr->msi_desc));
  182. if (dr->ntb->msi->desc_changed)
  183. dr->ntb->msi->desc_changed(dr->ntb->ctx);
  184. }
  185. static void ntbm_msi_callback_release(struct device *dev, void *res)
  186. {
  187. struct ntb_msi_devres *dr = res;
  188. dr->entry->write_msi_msg = NULL;
  189. dr->entry->write_msi_msg_data = NULL;
  190. }
  191. static int ntbm_msi_setup_callback(struct ntb_dev *ntb, struct msi_desc *entry,
  192. struct ntb_msi_desc *msi_desc)
  193. {
  194. struct ntb_msi_devres *dr;
  195. dr = devres_alloc(ntbm_msi_callback_release,
  196. sizeof(struct ntb_msi_devres), GFP_KERNEL);
  197. if (!dr)
  198. return -ENOMEM;
  199. dr->ntb = ntb;
  200. dr->entry = entry;
  201. dr->msi_desc = msi_desc;
  202. devres_add(&ntb->dev, dr);
  203. dr->entry->write_msi_msg = ntb_msi_write_msg;
  204. dr->entry->write_msi_msg_data = dr;
  205. return 0;
  206. }
  207. /**
  208. * ntbm_msi_request_threaded_irq() - allocate an MSI interrupt
  209. * @ntb: NTB device context
  210. * @handler: Function to be called when the IRQ occurs
  211. * @thread_fn: Function to be called in a threaded interrupt context. NULL
  212. * for clients which handle everything in @handler
  213. * @name: An ascii name for the claiming device, dev_name(dev) if NULL
  214. * @dev_id: A cookie passed back to the handler function
  215. * @msi_desc: MSI descriptor data which triggers the interrupt
  216. *
  217. * This function assigns an interrupt handler to an unused
  218. * MSI interrupt and returns the descriptor used to trigger
  219. * it. The descriptor can then be sent to a peer to trigger
  220. * the interrupt.
  221. *
  222. * The interrupt resource is managed with devres so it will
  223. * be automatically freed when the NTB device is torn down.
  224. *
  225. * If an IRQ allocated with this function needs to be freed
  226. * separately, ntbm_free_irq() must be used.
  227. *
  228. * Return: IRQ number assigned on success, otherwise a negative error number.
  229. */
  230. int ntbm_msi_request_threaded_irq(struct ntb_dev *ntb, irq_handler_t handler,
  231. irq_handler_t thread_fn,
  232. const char *name, void *dev_id,
  233. struct ntb_msi_desc *msi_desc)
  234. {
  235. struct device *dev = &ntb->pdev->dev;
  236. struct msi_desc *entry;
  237. int ret;
  238. if (!ntb->msi)
  239. return -EINVAL;
  240. msi_lock_descs(dev);
  241. msi_for_each_desc(entry, dev, MSI_DESC_ASSOCIATED) {
  242. if (irq_has_action(entry->irq))
  243. continue;
  244. ret = devm_request_threaded_irq(&ntb->dev, entry->irq, handler,
  245. thread_fn, 0, name, dev_id);
  246. if (ret)
  247. continue;
  248. if (ntb_msi_set_desc(ntb, entry, msi_desc)) {
  249. devm_free_irq(&ntb->dev, entry->irq, dev_id);
  250. continue;
  251. }
  252. ret = ntbm_msi_setup_callback(ntb, entry, msi_desc);
  253. if (ret) {
  254. devm_free_irq(&ntb->dev, entry->irq, dev_id);
  255. goto unlock;
  256. }
  257. ret = entry->irq;
  258. goto unlock;
  259. }
  260. ret = -ENODEV;
  261. unlock:
  262. msi_unlock_descs(dev);
  263. return ret;
  264. }
  265. EXPORT_SYMBOL(ntbm_msi_request_threaded_irq);
  266. static int ntbm_msi_callback_match(struct device *dev, void *res, void *data)
  267. {
  268. struct ntb_dev *ntb = dev_ntb(dev);
  269. struct ntb_msi_devres *dr = res;
  270. return dr->ntb == ntb && dr->entry == data;
  271. }
  272. /**
  273. * ntbm_msi_free_irq() - free an interrupt
  274. * @ntb: NTB device context
  275. * @irq: Interrupt line to free
  276. * @dev_id: Device identity to free
  277. *
  278. * This function should be used to manually free IRQs allocated with
  279. * ntbm_request_[threaded_]irq().
  280. */
  281. void ntbm_msi_free_irq(struct ntb_dev *ntb, unsigned int irq, void *dev_id)
  282. {
  283. struct msi_desc *entry = irq_get_msi_desc(irq);
  284. entry->write_msi_msg = NULL;
  285. entry->write_msi_msg_data = NULL;
  286. WARN_ON(devres_destroy(&ntb->dev, ntbm_msi_callback_release,
  287. ntbm_msi_callback_match, entry));
  288. devm_free_irq(&ntb->dev, irq, dev_id);
  289. }
  290. EXPORT_SYMBOL(ntbm_msi_free_irq);
  291. /**
  292. * ntb_msi_peer_trigger() - Trigger an interrupt handler on a peer
  293. * @ntb: NTB device context
  294. * @peer: Peer index
  295. * @desc: MSI descriptor data which triggers the interrupt
  296. *
  297. * This function triggers an interrupt on a peer. It requires
  298. * the descriptor structure to have been passed from that peer
  299. * by some other means.
  300. *
  301. * Return: Zero on success, otherwise a negative error number.
  302. */
  303. int ntb_msi_peer_trigger(struct ntb_dev *ntb, int peer,
  304. struct ntb_msi_desc *desc)
  305. {
  306. int idx;
  307. if (!ntb->msi)
  308. return -EINVAL;
  309. idx = desc->addr_offset / sizeof(*ntb->msi->peer_mws[peer]);
  310. iowrite32(desc->data, &ntb->msi->peer_mws[peer][idx]);
  311. return 0;
  312. }
  313. EXPORT_SYMBOL(ntb_msi_peer_trigger);
  314. /**
  315. * ntb_msi_peer_addr() - Get the DMA address to trigger a peer's MSI interrupt
  316. * @ntb: NTB device context
  317. * @peer: Peer index
  318. * @desc: MSI descriptor data which triggers the interrupt
  319. * @msi_addr: Physical address to trigger the interrupt
  320. *
  321. * This function allows using DMA engines to trigger an interrupt
  322. * (for example, trigger an interrupt to process the data after
  323. * sending it). To trigger the interrupt, write @desc.data to the address
  324. * returned in @msi_addr
  325. *
  326. * Return: Zero on success, otherwise a negative error number.
  327. */
  328. int ntb_msi_peer_addr(struct ntb_dev *ntb, int peer,
  329. struct ntb_msi_desc *desc,
  330. phys_addr_t *msi_addr)
  331. {
  332. int peer_widx = ntb_peer_mw_count(ntb) - 1 - peer;
  333. phys_addr_t mw_phys_addr;
  334. int ret;
  335. ret = ntb_peer_mw_get_addr(ntb, peer_widx, &mw_phys_addr, NULL);
  336. if (ret)
  337. return ret;
  338. if (msi_addr)
  339. *msi_addr = mw_phys_addr + desc->addr_offset;
  340. return 0;
  341. }
  342. EXPORT_SYMBOL(ntb_msi_peer_addr);