kasan_init.c 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * This file contains kasan initialization code for ARM.
  4. *
  5. * Copyright (c) 2018 Samsung Electronics Co., Ltd.
  6. * Author: Andrey Ryabinin <[email protected]>
  7. * Author: Linus Walleij <[email protected]>
  8. */
  9. #define pr_fmt(fmt) "kasan: " fmt
  10. #include <linux/kasan.h>
  11. #include <linux/kernel.h>
  12. #include <linux/memblock.h>
  13. #include <linux/sched/task.h>
  14. #include <linux/start_kernel.h>
  15. #include <linux/pgtable.h>
  16. #include <asm/cputype.h>
  17. #include <asm/highmem.h>
  18. #include <asm/mach/map.h>
  19. #include <asm/memory.h>
  20. #include <asm/page.h>
  21. #include <asm/pgalloc.h>
  22. #include <asm/procinfo.h>
  23. #include <asm/proc-fns.h>
  24. #include "mm.h"
  25. static pgd_t tmp_pgd_table[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
  26. pmd_t tmp_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
  27. static __init void *kasan_alloc_block(size_t size)
  28. {
  29. return memblock_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
  30. MEMBLOCK_ALLOC_NOLEAKTRACE, NUMA_NO_NODE);
  31. }
  32. static void __init kasan_pte_populate(pmd_t *pmdp, unsigned long addr,
  33. unsigned long end, bool early)
  34. {
  35. unsigned long next;
  36. pte_t *ptep = pte_offset_kernel(pmdp, addr);
  37. do {
  38. pte_t entry;
  39. void *p;
  40. next = addr + PAGE_SIZE;
  41. if (!early) {
  42. if (!pte_none(READ_ONCE(*ptep)))
  43. continue;
  44. p = kasan_alloc_block(PAGE_SIZE);
  45. if (!p) {
  46. panic("%s failed to allocate shadow page for address 0x%lx\n",
  47. __func__, addr);
  48. return;
  49. }
  50. memset(p, KASAN_SHADOW_INIT, PAGE_SIZE);
  51. entry = pfn_pte(virt_to_pfn(p),
  52. __pgprot(pgprot_val(PAGE_KERNEL)));
  53. } else if (pte_none(READ_ONCE(*ptep))) {
  54. /*
  55. * The early shadow memory is mapping all KASan
  56. * operations to one and the same page in memory,
  57. * "kasan_early_shadow_page" so that the instrumentation
  58. * will work on a scratch area until we can set up the
  59. * proper KASan shadow memory.
  60. */
  61. entry = pfn_pte(virt_to_pfn(kasan_early_shadow_page),
  62. __pgprot(_L_PTE_DEFAULT | L_PTE_DIRTY | L_PTE_XN));
  63. } else {
  64. /*
  65. * Early shadow mappings are PMD_SIZE aligned, so if the
  66. * first entry is already set, they must all be set.
  67. */
  68. return;
  69. }
  70. set_pte_at(&init_mm, addr, ptep, entry);
  71. } while (ptep++, addr = next, addr != end);
  72. }
  73. /*
  74. * The pmd (page middle directory) is only used on LPAE
  75. */
  76. static void __init kasan_pmd_populate(pud_t *pudp, unsigned long addr,
  77. unsigned long end, bool early)
  78. {
  79. unsigned long next;
  80. pmd_t *pmdp = pmd_offset(pudp, addr);
  81. do {
  82. if (pmd_none(*pmdp)) {
  83. /*
  84. * We attempt to allocate a shadow block for the PMDs
  85. * used by the PTEs for this address if it isn't already
  86. * allocated.
  87. */
  88. void *p = early ? kasan_early_shadow_pte :
  89. kasan_alloc_block(PAGE_SIZE);
  90. if (!p) {
  91. panic("%s failed to allocate shadow block for address 0x%lx\n",
  92. __func__, addr);
  93. return;
  94. }
  95. pmd_populate_kernel(&init_mm, pmdp, p);
  96. flush_pmd_entry(pmdp);
  97. }
  98. next = pmd_addr_end(addr, end);
  99. kasan_pte_populate(pmdp, addr, next, early);
  100. } while (pmdp++, addr = next, addr != end);
  101. }
  102. static void __init kasan_pgd_populate(unsigned long addr, unsigned long end,
  103. bool early)
  104. {
  105. unsigned long next;
  106. pgd_t *pgdp;
  107. p4d_t *p4dp;
  108. pud_t *pudp;
  109. pgdp = pgd_offset_k(addr);
  110. do {
  111. /*
  112. * Allocate and populate the shadow block of p4d folded into
  113. * pud folded into pmd if it doesn't already exist
  114. */
  115. if (!early && pgd_none(*pgdp)) {
  116. void *p = kasan_alloc_block(PAGE_SIZE);
  117. if (!p) {
  118. panic("%s failed to allocate shadow block for address 0x%lx\n",
  119. __func__, addr);
  120. return;
  121. }
  122. pgd_populate(&init_mm, pgdp, p);
  123. }
  124. next = pgd_addr_end(addr, end);
  125. /*
  126. * We just immediately jump over the p4d and pud page
  127. * directories since we believe ARM32 will never gain four
  128. * nor five level page tables.
  129. */
  130. p4dp = p4d_offset(pgdp, addr);
  131. pudp = pud_offset(p4dp, addr);
  132. kasan_pmd_populate(pudp, addr, next, early);
  133. } while (pgdp++, addr = next, addr != end);
  134. }
  135. extern struct proc_info_list *lookup_processor_type(unsigned int);
  136. void __init kasan_early_init(void)
  137. {
  138. struct proc_info_list *list;
  139. /*
  140. * locate processor in the list of supported processor
  141. * types. The linker builds this table for us from the
  142. * entries in arch/arm/mm/proc-*.S
  143. */
  144. list = lookup_processor_type(read_cpuid_id());
  145. if (list) {
  146. #ifdef MULTI_CPU
  147. processor = *list->proc;
  148. #endif
  149. }
  150. BUILD_BUG_ON((KASAN_SHADOW_END - (1UL << 29)) != KASAN_SHADOW_OFFSET);
  151. /*
  152. * We walk the page table and set all of the shadow memory to point
  153. * to the scratch page.
  154. */
  155. kasan_pgd_populate(KASAN_SHADOW_START, KASAN_SHADOW_END, true);
  156. }
  157. static void __init clear_pgds(unsigned long start,
  158. unsigned long end)
  159. {
  160. for (; start && start < end; start += PMD_SIZE)
  161. pmd_clear(pmd_off_k(start));
  162. }
  163. static int __init create_mapping(void *start, void *end)
  164. {
  165. void *shadow_start, *shadow_end;
  166. shadow_start = kasan_mem_to_shadow(start);
  167. shadow_end = kasan_mem_to_shadow(end);
  168. pr_info("Mapping kernel virtual memory block: %px-%px at shadow: %px-%px\n",
  169. start, end, shadow_start, shadow_end);
  170. kasan_pgd_populate((unsigned long)shadow_start & PAGE_MASK,
  171. PAGE_ALIGN((unsigned long)shadow_end), false);
  172. return 0;
  173. }
  174. void __init kasan_init(void)
  175. {
  176. phys_addr_t pa_start, pa_end;
  177. u64 i;
  178. /*
  179. * We are going to perform proper setup of shadow memory.
  180. *
  181. * At first we should unmap early shadow (clear_pgds() call bellow).
  182. * However, instrumented code can't execute without shadow memory.
  183. *
  184. * To keep the early shadow memory MMU tables around while setting up
  185. * the proper shadow memory, we copy swapper_pg_dir (the initial page
  186. * table) to tmp_pgd_table and use that to keep the early shadow memory
  187. * mapped until the full shadow setup is finished. Then we swap back
  188. * to the proper swapper_pg_dir.
  189. */
  190. memcpy(tmp_pgd_table, swapper_pg_dir, sizeof(tmp_pgd_table));
  191. #ifdef CONFIG_ARM_LPAE
  192. /* We need to be in the same PGD or this won't work */
  193. BUILD_BUG_ON(pgd_index(KASAN_SHADOW_START) !=
  194. pgd_index(KASAN_SHADOW_END));
  195. memcpy(tmp_pmd_table,
  196. (void*)pgd_page_vaddr(*pgd_offset_k(KASAN_SHADOW_START)),
  197. sizeof(tmp_pmd_table));
  198. set_pgd(&tmp_pgd_table[pgd_index(KASAN_SHADOW_START)],
  199. __pgd(__pa(tmp_pmd_table) | PMD_TYPE_TABLE | L_PGD_SWAPPER));
  200. #endif
  201. cpu_switch_mm(tmp_pgd_table, &init_mm);
  202. local_flush_tlb_all();
  203. clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
  204. if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
  205. kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_START),
  206. kasan_mem_to_shadow((void *)VMALLOC_END));
  207. kasan_populate_early_shadow(kasan_mem_to_shadow((void *)VMALLOC_END),
  208. kasan_mem_to_shadow((void *)-1UL) + 1);
  209. for_each_mem_range(i, &pa_start, &pa_end) {
  210. void *start = __va(pa_start);
  211. void *end = __va(pa_end);
  212. /* Do not attempt to shadow highmem */
  213. if (pa_start >= arm_lowmem_limit) {
  214. pr_info("Skip highmem block at %pa-%pa\n", &pa_start, &pa_end);
  215. continue;
  216. }
  217. if (pa_end > arm_lowmem_limit) {
  218. pr_info("Truncating shadow for memory block at %pa-%pa to lowmem region at %pa\n",
  219. &pa_start, &pa_end, &arm_lowmem_limit);
  220. end = __va(arm_lowmem_limit);
  221. }
  222. if (start >= end) {
  223. pr_info("Skipping invalid memory block %pa-%pa (virtual %p-%p)\n",
  224. &pa_start, &pa_end, start, end);
  225. continue;
  226. }
  227. create_mapping(start, end);
  228. }
  229. /*
  230. * 1. The module global variables are in MODULES_VADDR ~ MODULES_END,
  231. * so we need to map this area if CONFIG_KASAN_VMALLOC=n. With
  232. * VMALLOC support KASAN will manage this region dynamically,
  233. * refer to kasan_populate_vmalloc() and ARM's implementation of
  234. * module_alloc().
  235. * 2. PKMAP_BASE ~ PKMAP_BASE+PMD_SIZE's shadow and MODULES_VADDR
  236. * ~ MODULES_END's shadow is in the same PMD_SIZE, so we can't
  237. * use kasan_populate_zero_shadow.
  238. */
  239. if (!IS_ENABLED(CONFIG_KASAN_VMALLOC) && IS_ENABLED(CONFIG_MODULES))
  240. create_mapping((void *)MODULES_VADDR, (void *)(MODULES_END));
  241. create_mapping((void *)PKMAP_BASE, (void *)(PKMAP_BASE + PMD_SIZE));
  242. /*
  243. * KAsan may reuse the contents of kasan_early_shadow_pte directly, so
  244. * we should make sure that it maps the zero page read-only.
  245. */
  246. for (i = 0; i < PTRS_PER_PTE; i++)
  247. set_pte_at(&init_mm, KASAN_SHADOW_START + i*PAGE_SIZE,
  248. &kasan_early_shadow_pte[i],
  249. pfn_pte(virt_to_pfn(kasan_early_shadow_page),
  250. __pgprot(pgprot_val(PAGE_KERNEL)
  251. | L_PTE_RDONLY)));
  252. cpu_switch_mm(swapper_pg_dir, &init_mm);
  253. local_flush_tlb_all();
  254. memset(kasan_early_shadow_page, 0, PAGE_SIZE);
  255. pr_info("Kernel address sanitizer initialized\n");
  256. init_task.kasan_depth = 0;
  257. }