iosapic.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * I/O SAPIC support.
  4. *
  5. * Copyright (C) 1999 Intel Corp.
  6. * Copyright (C) 1999 Asit Mallick <[email protected]>
  7. * Copyright (C) 2000-2002 J.I. Lee <[email protected]>
  8. * Copyright (C) 1999-2000, 2002-2003 Hewlett-Packard Co.
  9. * David Mosberger-Tang <[email protected]>
  10. * Copyright (C) 1999 VA Linux Systems
  11. * Copyright (C) 1999,2000 Walt Drummond <[email protected]>
  12. *
  13. * 00/04/19 D. Mosberger Rewritten to mirror more closely the x86 I/O
  14. * APIC code. In particular, we now have separate
  15. * handlers for edge and level triggered
  16. * interrupts.
  17. * 00/10/27 Asit Mallick, Goutham Rao <[email protected]> IRQ vector
  18. * allocation PCI to vector mapping, shared PCI
  19. * interrupts.
  20. * 00/10/27 D. Mosberger Document things a bit more to make them more
  21. * understandable. Clean up much of the old
  22. * IOSAPIC cruft.
  23. * 01/07/27 J.I. Lee PCI irq routing, Platform/Legacy interrupts
  24. * and fixes for ACPI S5(SoftOff) support.
  25. * 02/01/23 J.I. Lee iosapic pgm fixes for PCI irq routing from _PRT
  26. * 02/01/07 E. Focht <[email protected]> Redirectable interrupt
  27. * vectors in iosapic_set_affinity(),
  28. * initializations for /proc/irq/#/smp_affinity
  29. * 02/04/02 P. Diefenbaugh Cleaned up ACPI PCI IRQ routing.
  30. * 02/04/18 J.I. Lee bug fix in iosapic_init_pci_irq
  31. * 02/04/30 J.I. Lee bug fix in find_iosapic to fix ACPI PCI IRQ to
  32. * IOSAPIC mapping error
  33. * 02/07/29 T. Kochi Allocate interrupt vectors dynamically
  34. * 02/08/04 T. Kochi Cleaned up terminology (irq, global system
  35. * interrupt, vector, etc.)
  36. * 02/09/20 D. Mosberger Simplified by taking advantage of ACPI's
  37. * pci_irq code.
  38. * 03/02/19 B. Helgaas Make pcat_compat system-wide, not per-IOSAPIC.
  39. * Remove iosapic_address & gsi_base from
  40. * external interfaces. Rationalize
  41. * __init/__devinit attributes.
  42. * 04/12/04 Ashok Raj <[email protected]> Intel Corporation 2004
  43. * Updated to work with irq migration necessary
  44. * for CPU Hotplug
  45. */
  46. /*
  47. * Here is what the interrupt logic between a PCI device and the kernel looks
  48. * like:
  49. *
  50. * (1) A PCI device raises one of the four interrupt pins (INTA, INTB, INTC,
  51. * INTD). The device is uniquely identified by its bus-, and slot-number
  52. * (the function number does not matter here because all functions share
  53. * the same interrupt lines).
  54. *
  55. * (2) The motherboard routes the interrupt line to a pin on a IOSAPIC
  56. * controller. Multiple interrupt lines may have to share the same
  57. * IOSAPIC pin (if they're level triggered and use the same polarity).
  58. * Each interrupt line has a unique Global System Interrupt (GSI) number
  59. * which can be calculated as the sum of the controller's base GSI number
  60. * and the IOSAPIC pin number to which the line connects.
  61. *
  62. * (3) The IOSAPIC uses an internal routing table entries (RTEs) to map the
  63. * IOSAPIC pin into the IA-64 interrupt vector. This interrupt vector is then
  64. * sent to the CPU.
  65. *
  66. * (4) The kernel recognizes an interrupt as an IRQ. The IRQ interface is
  67. * used as architecture-independent interrupt handling mechanism in Linux.
  68. * As an IRQ is a number, we have to have
  69. * IA-64 interrupt vector number <-> IRQ number mapping. On smaller
  70. * systems, we use one-to-one mapping between IA-64 vector and IRQ.
  71. *
  72. * To sum up, there are three levels of mappings involved:
  73. *
  74. * PCI pin -> global system interrupt (GSI) -> IA-64 vector <-> IRQ
  75. *
  76. * Note: The term "IRQ" is loosely used everywhere in Linux kernel to
  77. * describe interrupts. Now we use "IRQ" only for Linux IRQ's. ISA IRQ
  78. * (isa_irq) is the only exception in this source code.
  79. */
  80. #include <linux/acpi.h>
  81. #include <linux/init.h>
  82. #include <linux/irq.h>
  83. #include <linux/kernel.h>
  84. #include <linux/list.h>
  85. #include <linux/pci.h>
  86. #include <linux/slab.h>
  87. #include <linux/smp.h>
  88. #include <linux/string.h>
  89. #include <linux/memblock.h>
  90. #include <asm/delay.h>
  91. #include <asm/hw_irq.h>
  92. #include <asm/io.h>
  93. #include <asm/iosapic.h>
  94. #include <asm/processor.h>
  95. #include <asm/ptrace.h>
  96. #include <asm/xtp.h>
  97. #undef DEBUG_INTERRUPT_ROUTING
  98. #ifdef DEBUG_INTERRUPT_ROUTING
  99. #define DBG(fmt...) printk(fmt)
  100. #else
  101. #define DBG(fmt...)
  102. #endif
  103. static DEFINE_SPINLOCK(iosapic_lock);
  104. /*
  105. * These tables map IA-64 vectors to the IOSAPIC pin that generates this
  106. * vector.
  107. */
  108. #define NO_REF_RTE 0
  109. static struct iosapic {
  110. char __iomem *addr; /* base address of IOSAPIC */
  111. unsigned int gsi_base; /* GSI base */
  112. unsigned short num_rte; /* # of RTEs on this IOSAPIC */
  113. int rtes_inuse; /* # of RTEs in use on this IOSAPIC */
  114. #ifdef CONFIG_NUMA
  115. unsigned short node; /* numa node association via pxm */
  116. #endif
  117. spinlock_t lock; /* lock for indirect reg access */
  118. } iosapic_lists[NR_IOSAPICS];
  119. struct iosapic_rte_info {
  120. struct list_head rte_list; /* RTEs sharing the same vector */
  121. char rte_index; /* IOSAPIC RTE index */
  122. int refcnt; /* reference counter */
  123. struct iosapic *iosapic;
  124. } ____cacheline_aligned;
  125. static struct iosapic_intr_info {
  126. struct list_head rtes; /* RTEs using this vector (empty =>
  127. * not an IOSAPIC interrupt) */
  128. int count; /* # of registered RTEs */
  129. u32 low32; /* current value of low word of
  130. * Redirection table entry */
  131. unsigned int dest; /* destination CPU physical ID */
  132. unsigned char dmode : 3; /* delivery mode (see iosapic.h) */
  133. unsigned char polarity: 1; /* interrupt polarity
  134. * (see iosapic.h) */
  135. unsigned char trigger : 1; /* trigger mode (see iosapic.h) */
  136. } iosapic_intr_info[NR_IRQS];
  137. static unsigned char pcat_compat; /* 8259 compatibility flag */
  138. static inline void
  139. iosapic_write(struct iosapic *iosapic, unsigned int reg, u32 val)
  140. {
  141. unsigned long flags;
  142. spin_lock_irqsave(&iosapic->lock, flags);
  143. __iosapic_write(iosapic->addr, reg, val);
  144. spin_unlock_irqrestore(&iosapic->lock, flags);
  145. }
  146. /*
  147. * Find an IOSAPIC associated with a GSI
  148. */
  149. static inline int
  150. find_iosapic (unsigned int gsi)
  151. {
  152. int i;
  153. for (i = 0; i < NR_IOSAPICS; i++) {
  154. if ((unsigned) (gsi - iosapic_lists[i].gsi_base) <
  155. iosapic_lists[i].num_rte)
  156. return i;
  157. }
  158. return -1;
  159. }
  160. static inline int __gsi_to_irq(unsigned int gsi)
  161. {
  162. int irq;
  163. struct iosapic_intr_info *info;
  164. struct iosapic_rte_info *rte;
  165. for (irq = 0; irq < NR_IRQS; irq++) {
  166. info = &iosapic_intr_info[irq];
  167. list_for_each_entry(rte, &info->rtes, rte_list)
  168. if (rte->iosapic->gsi_base + rte->rte_index == gsi)
  169. return irq;
  170. }
  171. return -1;
  172. }
  173. int
  174. gsi_to_irq (unsigned int gsi)
  175. {
  176. unsigned long flags;
  177. int irq;
  178. spin_lock_irqsave(&iosapic_lock, flags);
  179. irq = __gsi_to_irq(gsi);
  180. spin_unlock_irqrestore(&iosapic_lock, flags);
  181. return irq;
  182. }
  183. static struct iosapic_rte_info *find_rte(unsigned int irq, unsigned int gsi)
  184. {
  185. struct iosapic_rte_info *rte;
  186. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list)
  187. if (rte->iosapic->gsi_base + rte->rte_index == gsi)
  188. return rte;
  189. return NULL;
  190. }
  191. static void
  192. set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask)
  193. {
  194. unsigned long pol, trigger, dmode;
  195. u32 low32, high32;
  196. int rte_index;
  197. char redir;
  198. struct iosapic_rte_info *rte;
  199. ia64_vector vector = irq_to_vector(irq);
  200. DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest);
  201. rte = find_rte(irq, gsi);
  202. if (!rte)
  203. return; /* not an IOSAPIC interrupt */
  204. rte_index = rte->rte_index;
  205. pol = iosapic_intr_info[irq].polarity;
  206. trigger = iosapic_intr_info[irq].trigger;
  207. dmode = iosapic_intr_info[irq].dmode;
  208. redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0;
  209. #ifdef CONFIG_SMP
  210. set_irq_affinity_info(irq, (int)(dest & 0xffff), redir);
  211. #endif
  212. low32 = ((pol << IOSAPIC_POLARITY_SHIFT) |
  213. (trigger << IOSAPIC_TRIGGER_SHIFT) |
  214. (dmode << IOSAPIC_DELIVERY_SHIFT) |
  215. ((mask ? 1 : 0) << IOSAPIC_MASK_SHIFT) |
  216. vector);
  217. /* dest contains both id and eid */
  218. high32 = (dest << IOSAPIC_DEST_SHIFT);
  219. iosapic_write(rte->iosapic, IOSAPIC_RTE_HIGH(rte_index), high32);
  220. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  221. iosapic_intr_info[irq].low32 = low32;
  222. iosapic_intr_info[irq].dest = dest;
  223. }
  224. static void
  225. iosapic_nop (struct irq_data *data)
  226. {
  227. /* do nothing... */
  228. }
  229. #ifdef CONFIG_KEXEC
  230. void
  231. kexec_disable_iosapic(void)
  232. {
  233. struct iosapic_intr_info *info;
  234. struct iosapic_rte_info *rte;
  235. ia64_vector vec;
  236. int irq;
  237. for (irq = 0; irq < NR_IRQS; irq++) {
  238. info = &iosapic_intr_info[irq];
  239. vec = irq_to_vector(irq);
  240. list_for_each_entry(rte, &info->rtes,
  241. rte_list) {
  242. iosapic_write(rte->iosapic,
  243. IOSAPIC_RTE_LOW(rte->rte_index),
  244. IOSAPIC_MASK|vec);
  245. iosapic_eoi(rte->iosapic->addr, vec);
  246. }
  247. }
  248. }
  249. #endif
  250. static void
  251. mask_irq (struct irq_data *data)
  252. {
  253. unsigned int irq = data->irq;
  254. u32 low32;
  255. int rte_index;
  256. struct iosapic_rte_info *rte;
  257. if (!iosapic_intr_info[irq].count)
  258. return; /* not an IOSAPIC interrupt! */
  259. /* set only the mask bit */
  260. low32 = iosapic_intr_info[irq].low32 |= IOSAPIC_MASK;
  261. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
  262. rte_index = rte->rte_index;
  263. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  264. }
  265. }
  266. static void
  267. unmask_irq (struct irq_data *data)
  268. {
  269. unsigned int irq = data->irq;
  270. u32 low32;
  271. int rte_index;
  272. struct iosapic_rte_info *rte;
  273. if (!iosapic_intr_info[irq].count)
  274. return; /* not an IOSAPIC interrupt! */
  275. low32 = iosapic_intr_info[irq].low32 &= ~IOSAPIC_MASK;
  276. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
  277. rte_index = rte->rte_index;
  278. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  279. }
  280. }
  281. static int
  282. iosapic_set_affinity(struct irq_data *data, const struct cpumask *mask,
  283. bool force)
  284. {
  285. #ifdef CONFIG_SMP
  286. unsigned int irq = data->irq;
  287. u32 high32, low32;
  288. int cpu, dest, rte_index;
  289. int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0;
  290. struct iosapic_rte_info *rte;
  291. struct iosapic *iosapic;
  292. irq &= (~IA64_IRQ_REDIRECTED);
  293. cpu = cpumask_first_and(cpu_online_mask, mask);
  294. if (cpu >= nr_cpu_ids)
  295. return -1;
  296. if (irq_prepare_move(irq, cpu))
  297. return -1;
  298. dest = cpu_physical_id(cpu);
  299. if (!iosapic_intr_info[irq].count)
  300. return -1; /* not an IOSAPIC interrupt */
  301. set_irq_affinity_info(irq, dest, redir);
  302. /* dest contains both id and eid */
  303. high32 = dest << IOSAPIC_DEST_SHIFT;
  304. low32 = iosapic_intr_info[irq].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT);
  305. if (redir)
  306. /* change delivery mode to lowest priority */
  307. low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
  308. else
  309. /* change delivery mode to fixed */
  310. low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT);
  311. low32 &= IOSAPIC_VECTOR_MASK;
  312. low32 |= irq_to_vector(irq);
  313. iosapic_intr_info[irq].low32 = low32;
  314. iosapic_intr_info[irq].dest = dest;
  315. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
  316. iosapic = rte->iosapic;
  317. rte_index = rte->rte_index;
  318. iosapic_write(iosapic, IOSAPIC_RTE_HIGH(rte_index), high32);
  319. iosapic_write(iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
  320. }
  321. #endif
  322. return 0;
  323. }
  324. /*
  325. * Handlers for level-triggered interrupts.
  326. */
  327. static unsigned int
  328. iosapic_startup_level_irq (struct irq_data *data)
  329. {
  330. unmask_irq(data);
  331. return 0;
  332. }
  333. static void
  334. iosapic_unmask_level_irq (struct irq_data *data)
  335. {
  336. unsigned int irq = data->irq;
  337. ia64_vector vec = irq_to_vector(irq);
  338. struct iosapic_rte_info *rte;
  339. int do_unmask_irq = 0;
  340. irq_complete_move(irq);
  341. if (unlikely(irqd_is_setaffinity_pending(data))) {
  342. do_unmask_irq = 1;
  343. mask_irq(data);
  344. } else
  345. unmask_irq(data);
  346. list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list)
  347. iosapic_eoi(rte->iosapic->addr, vec);
  348. if (unlikely(do_unmask_irq)) {
  349. irq_move_masked_irq(data);
  350. unmask_irq(data);
  351. }
  352. }
  353. #define iosapic_shutdown_level_irq mask_irq
  354. #define iosapic_enable_level_irq unmask_irq
  355. #define iosapic_disable_level_irq mask_irq
  356. #define iosapic_ack_level_irq iosapic_nop
  357. static struct irq_chip irq_type_iosapic_level = {
  358. .name = "IO-SAPIC-level",
  359. .irq_startup = iosapic_startup_level_irq,
  360. .irq_shutdown = iosapic_shutdown_level_irq,
  361. .irq_enable = iosapic_enable_level_irq,
  362. .irq_disable = iosapic_disable_level_irq,
  363. .irq_ack = iosapic_ack_level_irq,
  364. .irq_mask = mask_irq,
  365. .irq_unmask = iosapic_unmask_level_irq,
  366. .irq_set_affinity = iosapic_set_affinity
  367. };
  368. /*
  369. * Handlers for edge-triggered interrupts.
  370. */
  371. static unsigned int
  372. iosapic_startup_edge_irq (struct irq_data *data)
  373. {
  374. unmask_irq(data);
  375. /*
  376. * IOSAPIC simply drops interrupts pended while the
  377. * corresponding pin was masked, so we can't know if an
  378. * interrupt is pending already. Let's hope not...
  379. */
  380. return 0;
  381. }
  382. static void
  383. iosapic_ack_edge_irq (struct irq_data *data)
  384. {
  385. irq_complete_move(data->irq);
  386. irq_move_irq(data);
  387. }
  388. #define iosapic_enable_edge_irq unmask_irq
  389. #define iosapic_disable_edge_irq iosapic_nop
  390. static struct irq_chip irq_type_iosapic_edge = {
  391. .name = "IO-SAPIC-edge",
  392. .irq_startup = iosapic_startup_edge_irq,
  393. .irq_shutdown = iosapic_disable_edge_irq,
  394. .irq_enable = iosapic_enable_edge_irq,
  395. .irq_disable = iosapic_disable_edge_irq,
  396. .irq_ack = iosapic_ack_edge_irq,
  397. .irq_mask = mask_irq,
  398. .irq_unmask = unmask_irq,
  399. .irq_set_affinity = iosapic_set_affinity
  400. };
  401. static unsigned int
  402. iosapic_version (char __iomem *addr)
  403. {
  404. /*
  405. * IOSAPIC Version Register return 32 bit structure like:
  406. * {
  407. * unsigned int version : 8;
  408. * unsigned int reserved1 : 8;
  409. * unsigned int max_redir : 8;
  410. * unsigned int reserved2 : 8;
  411. * }
  412. */
  413. return __iosapic_read(addr, IOSAPIC_VERSION);
  414. }
  415. static int iosapic_find_sharable_irq(unsigned long trigger, unsigned long pol)
  416. {
  417. int i, irq = -ENOSPC, min_count = -1;
  418. struct iosapic_intr_info *info;
  419. /*
  420. * shared vectors for edge-triggered interrupts are not
  421. * supported yet
  422. */
  423. if (trigger == IOSAPIC_EDGE)
  424. return -EINVAL;
  425. for (i = 0; i < NR_IRQS; i++) {
  426. info = &iosapic_intr_info[i];
  427. if (info->trigger == trigger && info->polarity == pol &&
  428. (info->dmode == IOSAPIC_FIXED ||
  429. info->dmode == IOSAPIC_LOWEST_PRIORITY) &&
  430. can_request_irq(i, IRQF_SHARED)) {
  431. if (min_count == -1 || info->count < min_count) {
  432. irq = i;
  433. min_count = info->count;
  434. }
  435. }
  436. }
  437. return irq;
  438. }
  439. /*
  440. * if the given vector is already owned by other,
  441. * assign a new vector for the other and make the vector available
  442. */
  443. static void __init
  444. iosapic_reassign_vector (int irq)
  445. {
  446. int new_irq;
  447. if (iosapic_intr_info[irq].count) {
  448. new_irq = create_irq();
  449. if (new_irq < 0)
  450. panic("%s: out of interrupt vectors!\n", __func__);
  451. printk(KERN_INFO "Reassigning vector %d to %d\n",
  452. irq_to_vector(irq), irq_to_vector(new_irq));
  453. memcpy(&iosapic_intr_info[new_irq], &iosapic_intr_info[irq],
  454. sizeof(struct iosapic_intr_info));
  455. INIT_LIST_HEAD(&iosapic_intr_info[new_irq].rtes);
  456. list_move(iosapic_intr_info[irq].rtes.next,
  457. &iosapic_intr_info[new_irq].rtes);
  458. memset(&iosapic_intr_info[irq], 0,
  459. sizeof(struct iosapic_intr_info));
  460. iosapic_intr_info[irq].low32 = IOSAPIC_MASK;
  461. INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes);
  462. }
  463. }
  464. static inline int irq_is_shared (int irq)
  465. {
  466. return (iosapic_intr_info[irq].count > 1);
  467. }
  468. struct irq_chip*
  469. ia64_native_iosapic_get_irq_chip(unsigned long trigger)
  470. {
  471. if (trigger == IOSAPIC_EDGE)
  472. return &irq_type_iosapic_edge;
  473. else
  474. return &irq_type_iosapic_level;
  475. }
  476. static int
  477. register_intr (unsigned int gsi, int irq, unsigned char delivery,
  478. unsigned long polarity, unsigned long trigger)
  479. {
  480. struct irq_chip *chip, *irq_type;
  481. int index;
  482. struct iosapic_rte_info *rte;
  483. index = find_iosapic(gsi);
  484. if (index < 0) {
  485. printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n",
  486. __func__, gsi);
  487. return -ENODEV;
  488. }
  489. rte = find_rte(irq, gsi);
  490. if (!rte) {
  491. rte = kzalloc(sizeof (*rte), GFP_ATOMIC);
  492. if (!rte) {
  493. printk(KERN_WARNING "%s: cannot allocate memory\n",
  494. __func__);
  495. return -ENOMEM;
  496. }
  497. rte->iosapic = &iosapic_lists[index];
  498. rte->rte_index = gsi - rte->iosapic->gsi_base;
  499. rte->refcnt++;
  500. list_add_tail(&rte->rte_list, &iosapic_intr_info[irq].rtes);
  501. iosapic_intr_info[irq].count++;
  502. iosapic_lists[index].rtes_inuse++;
  503. }
  504. else if (rte->refcnt == NO_REF_RTE) {
  505. struct iosapic_intr_info *info = &iosapic_intr_info[irq];
  506. if (info->count > 0 &&
  507. (info->trigger != trigger || info->polarity != polarity)){
  508. printk (KERN_WARNING
  509. "%s: cannot override the interrupt\n",
  510. __func__);
  511. return -EINVAL;
  512. }
  513. rte->refcnt++;
  514. iosapic_intr_info[irq].count++;
  515. iosapic_lists[index].rtes_inuse++;
  516. }
  517. iosapic_intr_info[irq].polarity = polarity;
  518. iosapic_intr_info[irq].dmode = delivery;
  519. iosapic_intr_info[irq].trigger = trigger;
  520. irq_type = iosapic_get_irq_chip(trigger);
  521. chip = irq_get_chip(irq);
  522. if (irq_type != NULL && chip != irq_type) {
  523. if (chip != &no_irq_chip)
  524. printk(KERN_WARNING
  525. "%s: changing vector %d from %s to %s\n",
  526. __func__, irq_to_vector(irq),
  527. chip->name, irq_type->name);
  528. chip = irq_type;
  529. }
  530. irq_set_chip_handler_name_locked(irq_get_irq_data(irq), chip,
  531. trigger == IOSAPIC_EDGE ? handle_edge_irq : handle_level_irq,
  532. NULL);
  533. return 0;
  534. }
  535. static unsigned int
  536. get_target_cpu (unsigned int gsi, int irq)
  537. {
  538. #ifdef CONFIG_SMP
  539. static int cpu = -1;
  540. extern int cpe_vector;
  541. cpumask_t domain = irq_to_domain(irq);
  542. /*
  543. * In case of vector shared by multiple RTEs, all RTEs that
  544. * share the vector need to use the same destination CPU.
  545. */
  546. if (iosapic_intr_info[irq].count)
  547. return iosapic_intr_info[irq].dest;
  548. /*
  549. * If the platform supports redirection via XTP, let it
  550. * distribute interrupts.
  551. */
  552. if (smp_int_redirect & SMP_IRQ_REDIRECTION)
  553. return cpu_physical_id(smp_processor_id());
  554. /*
  555. * Some interrupts (ACPI SCI, for instance) are registered
  556. * before the BSP is marked as online.
  557. */
  558. if (!cpu_online(smp_processor_id()))
  559. return cpu_physical_id(smp_processor_id());
  560. if (cpe_vector > 0 && irq_to_vector(irq) == IA64_CPEP_VECTOR)
  561. return get_cpei_target_cpu();
  562. #ifdef CONFIG_NUMA
  563. {
  564. int num_cpus, cpu_index, iosapic_index, numa_cpu, i = 0;
  565. const struct cpumask *cpu_mask;
  566. iosapic_index = find_iosapic(gsi);
  567. if (iosapic_index < 0 ||
  568. iosapic_lists[iosapic_index].node == MAX_NUMNODES)
  569. goto skip_numa_setup;
  570. cpu_mask = cpumask_of_node(iosapic_lists[iosapic_index].node);
  571. num_cpus = 0;
  572. for_each_cpu_and(numa_cpu, cpu_mask, &domain) {
  573. if (cpu_online(numa_cpu))
  574. num_cpus++;
  575. }
  576. if (!num_cpus)
  577. goto skip_numa_setup;
  578. /* Use irq assignment to distribute across cpus in node */
  579. cpu_index = irq % num_cpus;
  580. for_each_cpu_and(numa_cpu, cpu_mask, &domain)
  581. if (cpu_online(numa_cpu) && i++ >= cpu_index)
  582. break;
  583. if (numa_cpu < nr_cpu_ids)
  584. return cpu_physical_id(numa_cpu);
  585. }
  586. skip_numa_setup:
  587. #endif
  588. /*
  589. * Otherwise, round-robin interrupt vectors across all the
  590. * processors. (It'd be nice if we could be smarter in the
  591. * case of NUMA.)
  592. */
  593. do {
  594. if (++cpu >= nr_cpu_ids)
  595. cpu = 0;
  596. } while (!cpu_online(cpu) || !cpumask_test_cpu(cpu, &domain));
  597. return cpu_physical_id(cpu);
  598. #else /* CONFIG_SMP */
  599. return cpu_physical_id(smp_processor_id());
  600. #endif
  601. }
  602. static inline unsigned char choose_dmode(void)
  603. {
  604. #ifdef CONFIG_SMP
  605. if (smp_int_redirect & SMP_IRQ_REDIRECTION)
  606. return IOSAPIC_LOWEST_PRIORITY;
  607. #endif
  608. return IOSAPIC_FIXED;
  609. }
  610. /*
  611. * ACPI can describe IOSAPIC interrupts via static tables and namespace
  612. * methods. This provides an interface to register those interrupts and
  613. * program the IOSAPIC RTE.
  614. */
  615. int
  616. iosapic_register_intr (unsigned int gsi,
  617. unsigned long polarity, unsigned long trigger)
  618. {
  619. int irq, mask = 1, err;
  620. unsigned int dest;
  621. unsigned long flags;
  622. struct iosapic_rte_info *rte;
  623. u32 low32;
  624. unsigned char dmode;
  625. struct irq_desc *desc;
  626. /*
  627. * If this GSI has already been registered (i.e., it's a
  628. * shared interrupt, or we lost a race to register it),
  629. * don't touch the RTE.
  630. */
  631. spin_lock_irqsave(&iosapic_lock, flags);
  632. irq = __gsi_to_irq(gsi);
  633. if (irq > 0) {
  634. rte = find_rte(irq, gsi);
  635. if(iosapic_intr_info[irq].count == 0) {
  636. assign_irq_vector(irq);
  637. irq_init_desc(irq);
  638. } else if (rte->refcnt != NO_REF_RTE) {
  639. rte->refcnt++;
  640. goto unlock_iosapic_lock;
  641. }
  642. } else
  643. irq = create_irq();
  644. /* If vector is running out, we try to find a sharable vector */
  645. if (irq < 0) {
  646. irq = iosapic_find_sharable_irq(trigger, polarity);
  647. if (irq < 0)
  648. goto unlock_iosapic_lock;
  649. }
  650. desc = irq_to_desc(irq);
  651. raw_spin_lock(&desc->lock);
  652. dest = get_target_cpu(gsi, irq);
  653. dmode = choose_dmode();
  654. err = register_intr(gsi, irq, dmode, polarity, trigger);
  655. if (err < 0) {
  656. raw_spin_unlock(&desc->lock);
  657. irq = err;
  658. goto unlock_iosapic_lock;
  659. }
  660. /*
  661. * If the vector is shared and already unmasked for other
  662. * interrupt sources, don't mask it.
  663. */
  664. low32 = iosapic_intr_info[irq].low32;
  665. if (irq_is_shared(irq) && !(low32 & IOSAPIC_MASK))
  666. mask = 0;
  667. set_rte(gsi, irq, dest, mask);
  668. printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n",
  669. gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  670. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  671. cpu_logical_id(dest), dest, irq_to_vector(irq));
  672. raw_spin_unlock(&desc->lock);
  673. unlock_iosapic_lock:
  674. spin_unlock_irqrestore(&iosapic_lock, flags);
  675. return irq;
  676. }
  677. void
  678. iosapic_unregister_intr (unsigned int gsi)
  679. {
  680. unsigned long flags;
  681. int irq, index;
  682. u32 low32;
  683. unsigned long trigger, polarity;
  684. unsigned int dest;
  685. struct iosapic_rte_info *rte;
  686. /*
  687. * If the irq associated with the gsi is not found,
  688. * iosapic_unregister_intr() is unbalanced. We need to check
  689. * this again after getting locks.
  690. */
  691. irq = gsi_to_irq(gsi);
  692. if (irq < 0) {
  693. printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
  694. gsi);
  695. WARN_ON(1);
  696. return;
  697. }
  698. spin_lock_irqsave(&iosapic_lock, flags);
  699. if ((rte = find_rte(irq, gsi)) == NULL) {
  700. printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
  701. gsi);
  702. WARN_ON(1);
  703. goto out;
  704. }
  705. if (--rte->refcnt > 0)
  706. goto out;
  707. rte->refcnt = NO_REF_RTE;
  708. /* Mask the interrupt */
  709. low32 = iosapic_intr_info[irq].low32 | IOSAPIC_MASK;
  710. iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), low32);
  711. iosapic_intr_info[irq].count--;
  712. index = find_iosapic(gsi);
  713. iosapic_lists[index].rtes_inuse--;
  714. WARN_ON(iosapic_lists[index].rtes_inuse < 0);
  715. trigger = iosapic_intr_info[irq].trigger;
  716. polarity = iosapic_intr_info[irq].polarity;
  717. dest = iosapic_intr_info[irq].dest;
  718. printk(KERN_INFO
  719. "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n",
  720. gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  721. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  722. cpu_logical_id(dest), dest, irq_to_vector(irq));
  723. if (iosapic_intr_info[irq].count == 0) {
  724. #ifdef CONFIG_SMP
  725. /* Clear affinity */
  726. irq_data_update_affinity(irq_get_irq_data(irq), cpu_all_mask);
  727. #endif
  728. /* Clear the interrupt information */
  729. iosapic_intr_info[irq].dest = 0;
  730. iosapic_intr_info[irq].dmode = 0;
  731. iosapic_intr_info[irq].polarity = 0;
  732. iosapic_intr_info[irq].trigger = 0;
  733. iosapic_intr_info[irq].low32 |= IOSAPIC_MASK;
  734. /* Destroy and reserve IRQ */
  735. destroy_and_reserve_irq(irq);
  736. }
  737. out:
  738. spin_unlock_irqrestore(&iosapic_lock, flags);
  739. }
  740. /*
  741. * ACPI calls this when it finds an entry for a platform interrupt.
  742. */
  743. int __init
  744. iosapic_register_platform_intr (u32 int_type, unsigned int gsi,
  745. int iosapic_vector, u16 eid, u16 id,
  746. unsigned long polarity, unsigned long trigger)
  747. {
  748. static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"};
  749. unsigned char delivery;
  750. int irq, vector, mask = 0;
  751. unsigned int dest = ((id << 8) | eid) & 0xffff;
  752. switch (int_type) {
  753. case ACPI_INTERRUPT_PMI:
  754. irq = vector = iosapic_vector;
  755. bind_irq_vector(irq, vector, CPU_MASK_ALL);
  756. /*
  757. * since PMI vector is alloc'd by FW(ACPI) not by kernel,
  758. * we need to make sure the vector is available
  759. */
  760. iosapic_reassign_vector(irq);
  761. delivery = IOSAPIC_PMI;
  762. break;
  763. case ACPI_INTERRUPT_INIT:
  764. irq = create_irq();
  765. if (irq < 0)
  766. panic("%s: out of interrupt vectors!\n", __func__);
  767. vector = irq_to_vector(irq);
  768. delivery = IOSAPIC_INIT;
  769. break;
  770. case ACPI_INTERRUPT_CPEI:
  771. irq = vector = IA64_CPE_VECTOR;
  772. BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL));
  773. delivery = IOSAPIC_FIXED;
  774. mask = 1;
  775. break;
  776. default:
  777. printk(KERN_ERR "%s: invalid int type 0x%x\n", __func__,
  778. int_type);
  779. return -1;
  780. }
  781. register_intr(gsi, irq, delivery, polarity, trigger);
  782. printk(KERN_INFO
  783. "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)"
  784. " vector %d\n",
  785. int_type < ARRAY_SIZE(name) ? name[int_type] : "unknown",
  786. int_type, gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
  787. (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
  788. cpu_logical_id(dest), dest, vector);
  789. set_rte(gsi, irq, dest, mask);
  790. return vector;
  791. }
  792. /*
  793. * ACPI calls this when it finds an entry for a legacy ISA IRQ override.
  794. */
  795. void iosapic_override_isa_irq(unsigned int isa_irq, unsigned int gsi,
  796. unsigned long polarity, unsigned long trigger)
  797. {
  798. int vector, irq;
  799. unsigned int dest = cpu_physical_id(smp_processor_id());
  800. unsigned char dmode;
  801. irq = vector = isa_irq_to_vector(isa_irq);
  802. BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL));
  803. dmode = choose_dmode();
  804. register_intr(gsi, irq, dmode, polarity, trigger);
  805. DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n",
  806. isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level",
  807. polarity == IOSAPIC_POL_HIGH ? "high" : "low",
  808. cpu_logical_id(dest), dest, vector);
  809. set_rte(gsi, irq, dest, 1);
  810. }
  811. void __init
  812. ia64_native_iosapic_pcat_compat_init(void)
  813. {
  814. if (pcat_compat) {
  815. /*
  816. * Disable the compatibility mode interrupts (8259 style),
  817. * needs IN/OUT support enabled.
  818. */
  819. printk(KERN_INFO
  820. "%s: Disabling PC-AT compatible 8259 interrupts\n",
  821. __func__);
  822. outb(0xff, 0xA1);
  823. outb(0xff, 0x21);
  824. }
  825. }
  826. void __init
  827. iosapic_system_init (int system_pcat_compat)
  828. {
  829. int irq;
  830. for (irq = 0; irq < NR_IRQS; ++irq) {
  831. iosapic_intr_info[irq].low32 = IOSAPIC_MASK;
  832. /* mark as unused */
  833. INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes);
  834. iosapic_intr_info[irq].count = 0;
  835. }
  836. pcat_compat = system_pcat_compat;
  837. if (pcat_compat)
  838. iosapic_pcat_compat_init();
  839. }
  840. static inline int
  841. iosapic_alloc (void)
  842. {
  843. int index;
  844. for (index = 0; index < NR_IOSAPICS; index++)
  845. if (!iosapic_lists[index].addr)
  846. return index;
  847. printk(KERN_WARNING "%s: failed to allocate iosapic\n", __func__);
  848. return -1;
  849. }
  850. static inline void
  851. iosapic_free (int index)
  852. {
  853. memset(&iosapic_lists[index], 0, sizeof(iosapic_lists[0]));
  854. }
  855. static inline int
  856. iosapic_check_gsi_range (unsigned int gsi_base, unsigned int ver)
  857. {
  858. int index;
  859. unsigned int gsi_end, base, end;
  860. /* check gsi range */
  861. gsi_end = gsi_base + ((ver >> 16) & 0xff);
  862. for (index = 0; index < NR_IOSAPICS; index++) {
  863. if (!iosapic_lists[index].addr)
  864. continue;
  865. base = iosapic_lists[index].gsi_base;
  866. end = base + iosapic_lists[index].num_rte - 1;
  867. if (gsi_end < base || end < gsi_base)
  868. continue; /* OK */
  869. return -EBUSY;
  870. }
  871. return 0;
  872. }
  873. static int
  874. iosapic_delete_rte(unsigned int irq, unsigned int gsi)
  875. {
  876. struct iosapic_rte_info *rte, *temp;
  877. list_for_each_entry_safe(rte, temp, &iosapic_intr_info[irq].rtes,
  878. rte_list) {
  879. if (rte->iosapic->gsi_base + rte->rte_index == gsi) {
  880. if (rte->refcnt)
  881. return -EBUSY;
  882. list_del(&rte->rte_list);
  883. kfree(rte);
  884. return 0;
  885. }
  886. }
  887. return -EINVAL;
  888. }
  889. int iosapic_init(unsigned long phys_addr, unsigned int gsi_base)
  890. {
  891. int num_rte, err, index;
  892. unsigned int isa_irq, ver;
  893. char __iomem *addr;
  894. unsigned long flags;
  895. spin_lock_irqsave(&iosapic_lock, flags);
  896. index = find_iosapic(gsi_base);
  897. if (index >= 0) {
  898. spin_unlock_irqrestore(&iosapic_lock, flags);
  899. return -EBUSY;
  900. }
  901. addr = ioremap(phys_addr, 0);
  902. if (addr == NULL) {
  903. spin_unlock_irqrestore(&iosapic_lock, flags);
  904. return -ENOMEM;
  905. }
  906. ver = iosapic_version(addr);
  907. if ((err = iosapic_check_gsi_range(gsi_base, ver))) {
  908. iounmap(addr);
  909. spin_unlock_irqrestore(&iosapic_lock, flags);
  910. return err;
  911. }
  912. /*
  913. * The MAX_REDIR register holds the highest input pin number
  914. * (starting from 0). We add 1 so that we can use it for
  915. * number of pins (= RTEs)
  916. */
  917. num_rte = ((ver >> 16) & 0xff) + 1;
  918. index = iosapic_alloc();
  919. iosapic_lists[index].addr = addr;
  920. iosapic_lists[index].gsi_base = gsi_base;
  921. iosapic_lists[index].num_rte = num_rte;
  922. #ifdef CONFIG_NUMA
  923. iosapic_lists[index].node = MAX_NUMNODES;
  924. #endif
  925. spin_lock_init(&iosapic_lists[index].lock);
  926. spin_unlock_irqrestore(&iosapic_lock, flags);
  927. if ((gsi_base == 0) && pcat_compat) {
  928. /*
  929. * Map the legacy ISA devices into the IOSAPIC data. Some of
  930. * these may get reprogrammed later on with data from the ACPI
  931. * Interrupt Source Override table.
  932. */
  933. for (isa_irq = 0; isa_irq < 16; ++isa_irq)
  934. iosapic_override_isa_irq(isa_irq, isa_irq,
  935. IOSAPIC_POL_HIGH,
  936. IOSAPIC_EDGE);
  937. }
  938. return 0;
  939. }
  940. int iosapic_remove(unsigned int gsi_base)
  941. {
  942. int i, irq, index, err = 0;
  943. unsigned long flags;
  944. spin_lock_irqsave(&iosapic_lock, flags);
  945. index = find_iosapic(gsi_base);
  946. if (index < 0) {
  947. printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n",
  948. __func__, gsi_base);
  949. goto out;
  950. }
  951. if (iosapic_lists[index].rtes_inuse) {
  952. err = -EBUSY;
  953. printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n",
  954. __func__, gsi_base);
  955. goto out;
  956. }
  957. for (i = gsi_base; i < gsi_base + iosapic_lists[index].num_rte; i++) {
  958. irq = __gsi_to_irq(i);
  959. if (irq < 0)
  960. continue;
  961. err = iosapic_delete_rte(irq, i);
  962. if (err)
  963. goto out;
  964. }
  965. iounmap(iosapic_lists[index].addr);
  966. iosapic_free(index);
  967. out:
  968. spin_unlock_irqrestore(&iosapic_lock, flags);
  969. return err;
  970. }
  971. #ifdef CONFIG_NUMA
  972. void map_iosapic_to_node(unsigned int gsi_base, int node)
  973. {
  974. int index;
  975. index = find_iosapic(gsi_base);
  976. if (index < 0) {
  977. printk(KERN_WARNING "%s: No IOSAPIC for GSI %u\n",
  978. __func__, gsi_base);
  979. return;
  980. }
  981. iosapic_lists[index].node = node;
  982. return;
  983. }
  984. #endif