irq.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright 2014 IBM Corp.
  4. */
  5. #include <linux/interrupt.h>
  6. #include <linux/irqdomain.h>
  7. #include <linux/workqueue.h>
  8. #include <linux/sched.h>
  9. #include <linux/wait.h>
  10. #include <linux/slab.h>
  11. #include <linux/pid.h>
  12. #include <asm/cputable.h>
  13. #include <misc/cxl-base.h>
  14. #include "cxl.h"
  15. #include "trace.h"
  16. static int afu_irq_range_start(void)
  17. {
  18. if (cpu_has_feature(CPU_FTR_HVMODE))
  19. return 1;
  20. return 0;
  21. }
  22. static irqreturn_t schedule_cxl_fault(struct cxl_context *ctx, u64 dsisr, u64 dar)
  23. {
  24. ctx->dsisr = dsisr;
  25. ctx->dar = dar;
  26. schedule_work(&ctx->fault_work);
  27. return IRQ_HANDLED;
  28. }
  29. irqreturn_t cxl_irq_psl9(int irq, struct cxl_context *ctx, struct cxl_irq_info *irq_info)
  30. {
  31. u64 dsisr, dar;
  32. dsisr = irq_info->dsisr;
  33. dar = irq_info->dar;
  34. trace_cxl_psl9_irq(ctx, irq, dsisr, dar);
  35. pr_devel("CXL interrupt %i for afu pe: %i DSISR: %#llx DAR: %#llx\n", irq, ctx->pe, dsisr, dar);
  36. if (dsisr & CXL_PSL9_DSISR_An_TF) {
  37. pr_devel("CXL interrupt: Scheduling translation fault handling for later (pe: %i)\n", ctx->pe);
  38. return schedule_cxl_fault(ctx, dsisr, dar);
  39. }
  40. if (dsisr & CXL_PSL9_DSISR_An_PE)
  41. return cxl_ops->handle_psl_slice_error(ctx, dsisr,
  42. irq_info->errstat);
  43. if (dsisr & CXL_PSL9_DSISR_An_AE) {
  44. pr_devel("CXL interrupt: AFU Error 0x%016llx\n", irq_info->afu_err);
  45. if (ctx->pending_afu_err) {
  46. /*
  47. * This shouldn't happen - the PSL treats these errors
  48. * as fatal and will have reset the AFU, so there's not
  49. * much point buffering multiple AFU errors.
  50. * OTOH if we DO ever see a storm of these come in it's
  51. * probably best that we log them somewhere:
  52. */
  53. dev_err_ratelimited(&ctx->afu->dev, "CXL AFU Error undelivered to pe %i: 0x%016llx\n",
  54. ctx->pe, irq_info->afu_err);
  55. } else {
  56. spin_lock(&ctx->lock);
  57. ctx->afu_err = irq_info->afu_err;
  58. ctx->pending_afu_err = 1;
  59. spin_unlock(&ctx->lock);
  60. wake_up_all(&ctx->wq);
  61. }
  62. cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_A, 0);
  63. return IRQ_HANDLED;
  64. }
  65. if (dsisr & CXL_PSL9_DSISR_An_OC)
  66. pr_devel("CXL interrupt: OS Context Warning\n");
  67. WARN(1, "Unhandled CXL PSL IRQ\n");
  68. return IRQ_HANDLED;
  69. }
  70. irqreturn_t cxl_irq_psl8(int irq, struct cxl_context *ctx, struct cxl_irq_info *irq_info)
  71. {
  72. u64 dsisr, dar;
  73. dsisr = irq_info->dsisr;
  74. dar = irq_info->dar;
  75. trace_cxl_psl_irq(ctx, irq, dsisr, dar);
  76. pr_devel("CXL interrupt %i for afu pe: %i DSISR: %#llx DAR: %#llx\n", irq, ctx->pe, dsisr, dar);
  77. if (dsisr & CXL_PSL_DSISR_An_DS) {
  78. /*
  79. * We don't inherently need to sleep to handle this, but we do
  80. * need to get a ref to the task's mm, which we can't do from
  81. * irq context without the potential for a deadlock since it
  82. * takes the task_lock. An alternate option would be to keep a
  83. * reference to the task's mm the entire time it has cxl open,
  84. * but to do that we need to solve the issue where we hold a
  85. * ref to the mm, but the mm can hold a ref to the fd after an
  86. * mmap preventing anything from being cleaned up.
  87. */
  88. pr_devel("Scheduling segment miss handling for later pe: %i\n", ctx->pe);
  89. return schedule_cxl_fault(ctx, dsisr, dar);
  90. }
  91. if (dsisr & CXL_PSL_DSISR_An_M)
  92. pr_devel("CXL interrupt: PTE not found\n");
  93. if (dsisr & CXL_PSL_DSISR_An_P)
  94. pr_devel("CXL interrupt: Storage protection violation\n");
  95. if (dsisr & CXL_PSL_DSISR_An_A)
  96. pr_devel("CXL interrupt: AFU lock access to write through or cache inhibited storage\n");
  97. if (dsisr & CXL_PSL_DSISR_An_S)
  98. pr_devel("CXL interrupt: Access was afu_wr or afu_zero\n");
  99. if (dsisr & CXL_PSL_DSISR_An_K)
  100. pr_devel("CXL interrupt: Access not permitted by virtual page class key protection\n");
  101. if (dsisr & CXL_PSL_DSISR_An_DM) {
  102. /*
  103. * In some cases we might be able to handle the fault
  104. * immediately if hash_page would succeed, but we still need
  105. * the task's mm, which as above we can't get without a lock
  106. */
  107. pr_devel("Scheduling page fault handling for later pe: %i\n", ctx->pe);
  108. return schedule_cxl_fault(ctx, dsisr, dar);
  109. }
  110. if (dsisr & CXL_PSL_DSISR_An_ST)
  111. WARN(1, "CXL interrupt: Segment Table PTE not found\n");
  112. if (dsisr & CXL_PSL_DSISR_An_UR)
  113. pr_devel("CXL interrupt: AURP PTE not found\n");
  114. if (dsisr & CXL_PSL_DSISR_An_PE)
  115. return cxl_ops->handle_psl_slice_error(ctx, dsisr,
  116. irq_info->errstat);
  117. if (dsisr & CXL_PSL_DSISR_An_AE) {
  118. pr_devel("CXL interrupt: AFU Error 0x%016llx\n", irq_info->afu_err);
  119. if (ctx->pending_afu_err) {
  120. /*
  121. * This shouldn't happen - the PSL treats these errors
  122. * as fatal and will have reset the AFU, so there's not
  123. * much point buffering multiple AFU errors.
  124. * OTOH if we DO ever see a storm of these come in it's
  125. * probably best that we log them somewhere:
  126. */
  127. dev_err_ratelimited(&ctx->afu->dev, "CXL AFU Error "
  128. "undelivered to pe %i: 0x%016llx\n",
  129. ctx->pe, irq_info->afu_err);
  130. } else {
  131. spin_lock(&ctx->lock);
  132. ctx->afu_err = irq_info->afu_err;
  133. ctx->pending_afu_err = true;
  134. spin_unlock(&ctx->lock);
  135. wake_up_all(&ctx->wq);
  136. }
  137. cxl_ops->ack_irq(ctx, CXL_PSL_TFC_An_A, 0);
  138. return IRQ_HANDLED;
  139. }
  140. if (dsisr & CXL_PSL_DSISR_An_OC)
  141. pr_devel("CXL interrupt: OS Context Warning\n");
  142. WARN(1, "Unhandled CXL PSL IRQ\n");
  143. return IRQ_HANDLED;
  144. }
  145. static irqreturn_t cxl_irq_afu(int irq, void *data)
  146. {
  147. struct cxl_context *ctx = data;
  148. irq_hw_number_t hwirq = irqd_to_hwirq(irq_get_irq_data(irq));
  149. int irq_off, afu_irq = 0;
  150. __u16 range;
  151. int r;
  152. /*
  153. * Look for the interrupt number.
  154. * On bare-metal, we know range 0 only contains the PSL
  155. * interrupt so we could start counting at range 1 and initialize
  156. * afu_irq at 1.
  157. * In a guest, range 0 also contains AFU interrupts, so it must
  158. * be counted for. Therefore we initialize afu_irq at 0 to take into
  159. * account the PSL interrupt.
  160. *
  161. * For code-readability, it just seems easier to go over all
  162. * the ranges on bare-metal and guest. The end result is the same.
  163. */
  164. for (r = 0; r < CXL_IRQ_RANGES; r++) {
  165. irq_off = hwirq - ctx->irqs.offset[r];
  166. range = ctx->irqs.range[r];
  167. if (irq_off >= 0 && irq_off < range) {
  168. afu_irq += irq_off;
  169. break;
  170. }
  171. afu_irq += range;
  172. }
  173. if (unlikely(r >= CXL_IRQ_RANGES)) {
  174. WARN(1, "Received AFU IRQ out of range for pe %i (virq %i hwirq %lx)\n",
  175. ctx->pe, irq, hwirq);
  176. return IRQ_HANDLED;
  177. }
  178. trace_cxl_afu_irq(ctx, afu_irq, irq, hwirq);
  179. pr_devel("Received AFU interrupt %i for pe: %i (virq %i hwirq %lx)\n",
  180. afu_irq, ctx->pe, irq, hwirq);
  181. if (unlikely(!ctx->irq_bitmap)) {
  182. WARN(1, "Received AFU IRQ for context with no IRQ bitmap\n");
  183. return IRQ_HANDLED;
  184. }
  185. spin_lock(&ctx->lock);
  186. set_bit(afu_irq - 1, ctx->irq_bitmap);
  187. ctx->pending_irq = true;
  188. spin_unlock(&ctx->lock);
  189. wake_up_all(&ctx->wq);
  190. return IRQ_HANDLED;
  191. }
  192. unsigned int cxl_map_irq(struct cxl *adapter, irq_hw_number_t hwirq,
  193. irq_handler_t handler, void *cookie, const char *name)
  194. {
  195. unsigned int virq;
  196. int result;
  197. /* IRQ Domain? */
  198. virq = irq_create_mapping(NULL, hwirq);
  199. if (!virq) {
  200. dev_warn(&adapter->dev, "cxl_map_irq: irq_create_mapping failed\n");
  201. return 0;
  202. }
  203. if (cxl_ops->setup_irq)
  204. cxl_ops->setup_irq(adapter, hwirq, virq);
  205. pr_devel("hwirq %#lx mapped to virq %u\n", hwirq, virq);
  206. result = request_irq(virq, handler, 0, name, cookie);
  207. if (result) {
  208. dev_warn(&adapter->dev, "cxl_map_irq: request_irq failed: %i\n", result);
  209. return 0;
  210. }
  211. return virq;
  212. }
  213. void cxl_unmap_irq(unsigned int virq, void *cookie)
  214. {
  215. free_irq(virq, cookie);
  216. }
  217. int cxl_register_one_irq(struct cxl *adapter,
  218. irq_handler_t handler,
  219. void *cookie,
  220. irq_hw_number_t *dest_hwirq,
  221. unsigned int *dest_virq,
  222. const char *name)
  223. {
  224. int hwirq, virq;
  225. if ((hwirq = cxl_ops->alloc_one_irq(adapter)) < 0)
  226. return hwirq;
  227. if (!(virq = cxl_map_irq(adapter, hwirq, handler, cookie, name)))
  228. goto err;
  229. *dest_hwirq = hwirq;
  230. *dest_virq = virq;
  231. return 0;
  232. err:
  233. cxl_ops->release_one_irq(adapter, hwirq);
  234. return -ENOMEM;
  235. }
  236. void afu_irq_name_free(struct cxl_context *ctx)
  237. {
  238. struct cxl_irq_name *irq_name, *tmp;
  239. list_for_each_entry_safe(irq_name, tmp, &ctx->irq_names, list) {
  240. kfree(irq_name->name);
  241. list_del(&irq_name->list);
  242. kfree(irq_name);
  243. }
  244. }
  245. int afu_allocate_irqs(struct cxl_context *ctx, u32 count)
  246. {
  247. int rc, r, i, j = 1;
  248. struct cxl_irq_name *irq_name;
  249. int alloc_count;
  250. /*
  251. * In native mode, range 0 is reserved for the multiplexed
  252. * PSL interrupt. It has been allocated when the AFU was initialized.
  253. *
  254. * In a guest, the PSL interrupt is not mutliplexed, but per-context,
  255. * and is the first interrupt from range 0. It still needs to be
  256. * allocated, so bump the count by one.
  257. */
  258. if (cpu_has_feature(CPU_FTR_HVMODE))
  259. alloc_count = count;
  260. else
  261. alloc_count = count + 1;
  262. if ((rc = cxl_ops->alloc_irq_ranges(&ctx->irqs, ctx->afu->adapter,
  263. alloc_count)))
  264. return rc;
  265. if (cpu_has_feature(CPU_FTR_HVMODE)) {
  266. /* Multiplexed PSL Interrupt */
  267. ctx->irqs.offset[0] = ctx->afu->native->psl_hwirq;
  268. ctx->irqs.range[0] = 1;
  269. }
  270. ctx->irq_count = count;
  271. ctx->irq_bitmap = bitmap_zalloc(count, GFP_KERNEL);
  272. if (!ctx->irq_bitmap)
  273. goto out;
  274. /*
  275. * Allocate names first. If any fail, bail out before allocating
  276. * actual hardware IRQs.
  277. */
  278. for (r = afu_irq_range_start(); r < CXL_IRQ_RANGES; r++) {
  279. for (i = 0; i < ctx->irqs.range[r]; i++) {
  280. irq_name = kmalloc(sizeof(struct cxl_irq_name),
  281. GFP_KERNEL);
  282. if (!irq_name)
  283. goto out;
  284. irq_name->name = kasprintf(GFP_KERNEL, "cxl-%s-pe%i-%i",
  285. dev_name(&ctx->afu->dev),
  286. ctx->pe, j);
  287. if (!irq_name->name) {
  288. kfree(irq_name);
  289. goto out;
  290. }
  291. /* Add to tail so next look get the correct order */
  292. list_add_tail(&irq_name->list, &ctx->irq_names);
  293. j++;
  294. }
  295. }
  296. return 0;
  297. out:
  298. cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
  299. bitmap_free(ctx->irq_bitmap);
  300. afu_irq_name_free(ctx);
  301. return -ENOMEM;
  302. }
  303. static void afu_register_hwirqs(struct cxl_context *ctx)
  304. {
  305. irq_hw_number_t hwirq;
  306. struct cxl_irq_name *irq_name;
  307. int r, i;
  308. irqreturn_t (*handler)(int irq, void *data);
  309. /* We've allocated all memory now, so let's do the irq allocations */
  310. irq_name = list_first_entry(&ctx->irq_names, struct cxl_irq_name, list);
  311. for (r = afu_irq_range_start(); r < CXL_IRQ_RANGES; r++) {
  312. hwirq = ctx->irqs.offset[r];
  313. for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
  314. if (r == 0 && i == 0)
  315. /*
  316. * The very first interrupt of range 0 is
  317. * always the PSL interrupt, but we only
  318. * need to connect a handler for guests,
  319. * because there's one PSL interrupt per
  320. * context.
  321. * On bare-metal, the PSL interrupt is
  322. * multiplexed and was setup when the AFU
  323. * was configured.
  324. */
  325. handler = cxl_ops->psl_interrupt;
  326. else
  327. handler = cxl_irq_afu;
  328. cxl_map_irq(ctx->afu->adapter, hwirq, handler, ctx,
  329. irq_name->name);
  330. irq_name = list_next_entry(irq_name, list);
  331. }
  332. }
  333. }
  334. int afu_register_irqs(struct cxl_context *ctx, u32 count)
  335. {
  336. int rc;
  337. rc = afu_allocate_irqs(ctx, count);
  338. if (rc)
  339. return rc;
  340. afu_register_hwirqs(ctx);
  341. return 0;
  342. }
  343. void afu_release_irqs(struct cxl_context *ctx, void *cookie)
  344. {
  345. irq_hw_number_t hwirq;
  346. unsigned int virq;
  347. int r, i;
  348. for (r = afu_irq_range_start(); r < CXL_IRQ_RANGES; r++) {
  349. hwirq = ctx->irqs.offset[r];
  350. for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
  351. virq = irq_find_mapping(NULL, hwirq);
  352. if (virq)
  353. cxl_unmap_irq(virq, cookie);
  354. }
  355. }
  356. afu_irq_name_free(ctx);
  357. cxl_ops->release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
  358. ctx->irq_count = 0;
  359. }
  360. void cxl_afu_decode_psl_serr(struct cxl_afu *afu, u64 serr)
  361. {
  362. dev_crit(&afu->dev,
  363. "PSL Slice error received. Check AFU for root cause.\n");
  364. dev_crit(&afu->dev, "PSL_SERR_An: 0x%016llx\n", serr);
  365. if (serr & CXL_PSL_SERR_An_afuto)
  366. dev_crit(&afu->dev, "AFU MMIO Timeout\n");
  367. if (serr & CXL_PSL_SERR_An_afudis)
  368. dev_crit(&afu->dev,
  369. "MMIO targeted Accelerator that was not enabled\n");
  370. if (serr & CXL_PSL_SERR_An_afuov)
  371. dev_crit(&afu->dev, "AFU CTAG Overflow\n");
  372. if (serr & CXL_PSL_SERR_An_badsrc)
  373. dev_crit(&afu->dev, "Bad Interrupt Source\n");
  374. if (serr & CXL_PSL_SERR_An_badctx)
  375. dev_crit(&afu->dev, "Bad Context Handle\n");
  376. if (serr & CXL_PSL_SERR_An_llcmdis)
  377. dev_crit(&afu->dev, "LLCMD to Disabled AFU\n");
  378. if (serr & CXL_PSL_SERR_An_llcmdto)
  379. dev_crit(&afu->dev, "LLCMD Timeout to AFU\n");
  380. if (serr & CXL_PSL_SERR_An_afupar)
  381. dev_crit(&afu->dev, "AFU MMIO Parity Error\n");
  382. if (serr & CXL_PSL_SERR_An_afudup)
  383. dev_crit(&afu->dev, "AFU MMIO Duplicate CTAG Error\n");
  384. if (serr & CXL_PSL_SERR_An_AE)
  385. dev_crit(&afu->dev,
  386. "AFU asserted JDONE with JERROR in AFU Directed Mode\n");
  387. }