irq_32.c 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
  4. *
  5. * This file contains the lowest level x86-specific interrupt
  6. * entry, irq-stacks and irq statistics code. All the remaining
  7. * irq logic is done by the generic kernel/irq/ code and
  8. * by the x86-specific irq controller code. (e.g. i8259.c and
  9. * io_apic.c.)
  10. */
  11. #include <linux/seq_file.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/irq.h>
  14. #include <linux/kernel_stat.h>
  15. #include <linux/notifier.h>
  16. #include <linux/cpu.h>
  17. #include <linux/delay.h>
  18. #include <linux/uaccess.h>
  19. #include <linux/percpu.h>
  20. #include <linux/mm.h>
  21. #include <asm/apic.h>
  22. #include <asm/nospec-branch.h>
  23. #include <asm/softirq_stack.h>
  24. #ifdef CONFIG_DEBUG_STACKOVERFLOW
  25. int sysctl_panic_on_stackoverflow __read_mostly;
  26. /* Debugging check for stack overflow: is there less than 1KB free? */
  27. static int check_stack_overflow(void)
  28. {
  29. long sp;
  30. __asm__ __volatile__("andl %%esp,%0" :
  31. "=r" (sp) : "0" (THREAD_SIZE - 1));
  32. return sp < (sizeof(struct thread_info) + STACK_WARN);
  33. }
  34. static void print_stack_overflow(void)
  35. {
  36. printk(KERN_WARNING "low stack detected by irq handler\n");
  37. dump_stack();
  38. if (sysctl_panic_on_stackoverflow)
  39. panic("low stack detected by irq handler - check messages\n");
  40. }
  41. #else
  42. static inline int check_stack_overflow(void) { return 0; }
  43. static inline void print_stack_overflow(void) { }
  44. #endif
  45. DEFINE_PER_CPU(struct irq_stack *, hardirq_stack_ptr);
  46. DEFINE_PER_CPU(struct irq_stack *, softirq_stack_ptr);
  47. static void call_on_stack(void *func, void *stack)
  48. {
  49. asm volatile("xchgl %%ebx,%%esp \n"
  50. CALL_NOSPEC
  51. "movl %%ebx,%%esp \n"
  52. : "=b" (stack)
  53. : "0" (stack),
  54. [thunk_target] "D"(func)
  55. : "memory", "cc", "edx", "ecx", "eax");
  56. }
  57. static inline void *current_stack(void)
  58. {
  59. return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
  60. }
  61. static inline int execute_on_irq_stack(int overflow, struct irq_desc *desc)
  62. {
  63. struct irq_stack *curstk, *irqstk;
  64. u32 *isp, *prev_esp, arg1;
  65. curstk = (struct irq_stack *) current_stack();
  66. irqstk = __this_cpu_read(hardirq_stack_ptr);
  67. /*
  68. * this is where we switch to the IRQ stack. However, if we are
  69. * already using the IRQ stack (because we interrupted a hardirq
  70. * handler) we can't do that and just have to keep using the
  71. * current stack (which is the irq stack already after all)
  72. */
  73. if (unlikely(curstk == irqstk))
  74. return 0;
  75. isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
  76. /* Save the next esp at the bottom of the stack */
  77. prev_esp = (u32 *)irqstk;
  78. *prev_esp = current_stack_pointer;
  79. if (unlikely(overflow))
  80. call_on_stack(print_stack_overflow, isp);
  81. asm volatile("xchgl %%ebx,%%esp \n"
  82. CALL_NOSPEC
  83. "movl %%ebx,%%esp \n"
  84. : "=a" (arg1), "=b" (isp)
  85. : "0" (desc), "1" (isp),
  86. [thunk_target] "D" (desc->handle_irq)
  87. : "memory", "cc", "ecx");
  88. return 1;
  89. }
  90. /*
  91. * Allocate per-cpu stacks for hardirq and softirq processing
  92. */
  93. int irq_init_percpu_irqstack(unsigned int cpu)
  94. {
  95. int node = cpu_to_node(cpu);
  96. struct page *ph, *ps;
  97. if (per_cpu(hardirq_stack_ptr, cpu))
  98. return 0;
  99. ph = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER);
  100. if (!ph)
  101. return -ENOMEM;
  102. ps = alloc_pages_node(node, THREADINFO_GFP, THREAD_SIZE_ORDER);
  103. if (!ps) {
  104. __free_pages(ph, THREAD_SIZE_ORDER);
  105. return -ENOMEM;
  106. }
  107. per_cpu(hardirq_stack_ptr, cpu) = page_address(ph);
  108. per_cpu(softirq_stack_ptr, cpu) = page_address(ps);
  109. return 0;
  110. }
  111. #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
  112. void do_softirq_own_stack(void)
  113. {
  114. struct irq_stack *irqstk;
  115. u32 *isp, *prev_esp;
  116. irqstk = __this_cpu_read(softirq_stack_ptr);
  117. /* build the stack frame on the softirq stack */
  118. isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
  119. /* Push the previous esp onto the stack */
  120. prev_esp = (u32 *)irqstk;
  121. *prev_esp = current_stack_pointer;
  122. call_on_stack(__do_softirq, isp);
  123. }
  124. #endif
  125. void __handle_irq(struct irq_desc *desc, struct pt_regs *regs)
  126. {
  127. int overflow = check_stack_overflow();
  128. if (user_mode(regs) || !execute_on_irq_stack(overflow, desc)) {
  129. if (unlikely(overflow))
  130. print_stack_overflow();
  131. generic_handle_irq_desc(desc);
  132. }
  133. }