kexec_core.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kexec.c - kexec system call core code.
  4. * Copyright (C) 2002-2004 Eric Biederman <[email protected]>
  5. */
  6. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  7. #include <linux/capability.h>
  8. #include <linux/mm.h>
  9. #include <linux/file.h>
  10. #include <linux/slab.h>
  11. #include <linux/fs.h>
  12. #include <linux/kexec.h>
  13. #include <linux/mutex.h>
  14. #include <linux/list.h>
  15. #include <linux/highmem.h>
  16. #include <linux/syscalls.h>
  17. #include <linux/reboot.h>
  18. #include <linux/ioport.h>
  19. #include <linux/hardirq.h>
  20. #include <linux/elf.h>
  21. #include <linux/elfcore.h>
  22. #include <linux/utsname.h>
  23. #include <linux/numa.h>
  24. #include <linux/suspend.h>
  25. #include <linux/device.h>
  26. #include <linux/freezer.h>
  27. #include <linux/panic_notifier.h>
  28. #include <linux/pm.h>
  29. #include <linux/cpu.h>
  30. #include <linux/uaccess.h>
  31. #include <linux/io.h>
  32. #include <linux/console.h>
  33. #include <linux/vmalloc.h>
  34. #include <linux/swap.h>
  35. #include <linux/syscore_ops.h>
  36. #include <linux/compiler.h>
  37. #include <linux/hugetlb.h>
  38. #include <linux/objtool.h>
  39. #include <linux/kmsg_dump.h>
  40. #include <asm/page.h>
  41. #include <asm/sections.h>
  42. #include <crypto/hash.h>
  43. #include "kexec_internal.h"
  44. atomic_t __kexec_lock = ATOMIC_INIT(0);
  45. /* Per cpu memory for storing cpu states in case of system crash. */
  46. note_buf_t __percpu *crash_notes;
  47. /* Flag to indicate we are going to kexec a new kernel */
  48. bool kexec_in_progress = false;
  49. /* Location of the reserved area for the crash kernel */
  50. struct resource crashk_res = {
  51. .name = "Crash kernel",
  52. .start = 0,
  53. .end = 0,
  54. .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
  55. .desc = IORES_DESC_CRASH_KERNEL
  56. };
  57. struct resource crashk_low_res = {
  58. .name = "Crash kernel",
  59. .start = 0,
  60. .end = 0,
  61. .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM,
  62. .desc = IORES_DESC_CRASH_KERNEL
  63. };
  64. int kexec_should_crash(struct task_struct *p)
  65. {
  66. /*
  67. * If crash_kexec_post_notifiers is enabled, don't run
  68. * crash_kexec() here yet, which must be run after panic
  69. * notifiers in panic().
  70. */
  71. if (crash_kexec_post_notifiers)
  72. return 0;
  73. /*
  74. * There are 4 panic() calls in make_task_dead() path, each of which
  75. * corresponds to each of these 4 conditions.
  76. */
  77. if (in_interrupt() || !p->pid || is_global_init(p) || panic_on_oops)
  78. return 1;
  79. return 0;
  80. }
  81. int kexec_crash_loaded(void)
  82. {
  83. return !!kexec_crash_image;
  84. }
  85. EXPORT_SYMBOL_GPL(kexec_crash_loaded);
  86. /*
  87. * When kexec transitions to the new kernel there is a one-to-one
  88. * mapping between physical and virtual addresses. On processors
  89. * where you can disable the MMU this is trivial, and easy. For
  90. * others it is still a simple predictable page table to setup.
  91. *
  92. * In that environment kexec copies the new kernel to its final
  93. * resting place. This means I can only support memory whose
  94. * physical address can fit in an unsigned long. In particular
  95. * addresses where (pfn << PAGE_SHIFT) > ULONG_MAX cannot be handled.
  96. * If the assembly stub has more restrictive requirements
  97. * KEXEC_SOURCE_MEMORY_LIMIT and KEXEC_DEST_MEMORY_LIMIT can be
  98. * defined more restrictively in <asm/kexec.h>.
  99. *
  100. * The code for the transition from the current kernel to the
  101. * new kernel is placed in the control_code_buffer, whose size
  102. * is given by KEXEC_CONTROL_PAGE_SIZE. In the best case only a single
  103. * page of memory is necessary, but some architectures require more.
  104. * Because this memory must be identity mapped in the transition from
  105. * virtual to physical addresses it must live in the range
  106. * 0 - TASK_SIZE, as only the user space mappings are arbitrarily
  107. * modifiable.
  108. *
  109. * The assembly stub in the control code buffer is passed a linked list
  110. * of descriptor pages detailing the source pages of the new kernel,
  111. * and the destination addresses of those source pages. As this data
  112. * structure is not used in the context of the current OS, it must
  113. * be self-contained.
  114. *
  115. * The code has been made to work with highmem pages and will use a
  116. * destination page in its final resting place (if it happens
  117. * to allocate it). The end product of this is that most of the
  118. * physical address space, and most of RAM can be used.
  119. *
  120. * Future directions include:
  121. * - allocating a page table with the control code buffer identity
  122. * mapped, to simplify machine_kexec and make kexec_on_panic more
  123. * reliable.
  124. */
  125. /*
  126. * KIMAGE_NO_DEST is an impossible destination address..., for
  127. * allocating pages whose destination address we do not care about.
  128. */
  129. #define KIMAGE_NO_DEST (-1UL)
  130. #define PAGE_COUNT(x) (((x) + PAGE_SIZE - 1) >> PAGE_SHIFT)
  131. static struct page *kimage_alloc_page(struct kimage *image,
  132. gfp_t gfp_mask,
  133. unsigned long dest);
  134. int sanity_check_segment_list(struct kimage *image)
  135. {
  136. int i;
  137. unsigned long nr_segments = image->nr_segments;
  138. unsigned long total_pages = 0;
  139. unsigned long nr_pages = totalram_pages();
  140. /*
  141. * Verify we have good destination addresses. The caller is
  142. * responsible for making certain we don't attempt to load
  143. * the new image into invalid or reserved areas of RAM. This
  144. * just verifies it is an address we can use.
  145. *
  146. * Since the kernel does everything in page size chunks ensure
  147. * the destination addresses are page aligned. Too many
  148. * special cases crop of when we don't do this. The most
  149. * insidious is getting overlapping destination addresses
  150. * simply because addresses are changed to page size
  151. * granularity.
  152. */
  153. for (i = 0; i < nr_segments; i++) {
  154. unsigned long mstart, mend;
  155. mstart = image->segment[i].mem;
  156. mend = mstart + image->segment[i].memsz;
  157. if (mstart > mend)
  158. return -EADDRNOTAVAIL;
  159. if ((mstart & ~PAGE_MASK) || (mend & ~PAGE_MASK))
  160. return -EADDRNOTAVAIL;
  161. if (mend >= KEXEC_DESTINATION_MEMORY_LIMIT)
  162. return -EADDRNOTAVAIL;
  163. }
  164. /* Verify our destination addresses do not overlap.
  165. * If we alloed overlapping destination addresses
  166. * through very weird things can happen with no
  167. * easy explanation as one segment stops on another.
  168. */
  169. for (i = 0; i < nr_segments; i++) {
  170. unsigned long mstart, mend;
  171. unsigned long j;
  172. mstart = image->segment[i].mem;
  173. mend = mstart + image->segment[i].memsz;
  174. for (j = 0; j < i; j++) {
  175. unsigned long pstart, pend;
  176. pstart = image->segment[j].mem;
  177. pend = pstart + image->segment[j].memsz;
  178. /* Do the segments overlap ? */
  179. if ((mend > pstart) && (mstart < pend))
  180. return -EINVAL;
  181. }
  182. }
  183. /* Ensure our buffer sizes are strictly less than
  184. * our memory sizes. This should always be the case,
  185. * and it is easier to check up front than to be surprised
  186. * later on.
  187. */
  188. for (i = 0; i < nr_segments; i++) {
  189. if (image->segment[i].bufsz > image->segment[i].memsz)
  190. return -EINVAL;
  191. }
  192. /*
  193. * Verify that no more than half of memory will be consumed. If the
  194. * request from userspace is too large, a large amount of time will be
  195. * wasted allocating pages, which can cause a soft lockup.
  196. */
  197. for (i = 0; i < nr_segments; i++) {
  198. if (PAGE_COUNT(image->segment[i].memsz) > nr_pages / 2)
  199. return -EINVAL;
  200. total_pages += PAGE_COUNT(image->segment[i].memsz);
  201. }
  202. if (total_pages > nr_pages / 2)
  203. return -EINVAL;
  204. /*
  205. * Verify we have good destination addresses. Normally
  206. * the caller is responsible for making certain we don't
  207. * attempt to load the new image into invalid or reserved
  208. * areas of RAM. But crash kernels are preloaded into a
  209. * reserved area of ram. We must ensure the addresses
  210. * are in the reserved area otherwise preloading the
  211. * kernel could corrupt things.
  212. */
  213. if (image->type == KEXEC_TYPE_CRASH) {
  214. for (i = 0; i < nr_segments; i++) {
  215. unsigned long mstart, mend;
  216. mstart = image->segment[i].mem;
  217. mend = mstart + image->segment[i].memsz - 1;
  218. /* Ensure we are within the crash kernel limits */
  219. if ((mstart < phys_to_boot_phys(crashk_res.start)) ||
  220. (mend > phys_to_boot_phys(crashk_res.end)))
  221. return -EADDRNOTAVAIL;
  222. }
  223. }
  224. return 0;
  225. }
  226. struct kimage *do_kimage_alloc_init(void)
  227. {
  228. struct kimage *image;
  229. /* Allocate a controlling structure */
  230. image = kzalloc(sizeof(*image), GFP_KERNEL);
  231. if (!image)
  232. return NULL;
  233. image->head = 0;
  234. image->entry = &image->head;
  235. image->last_entry = &image->head;
  236. image->control_page = ~0; /* By default this does not apply */
  237. image->type = KEXEC_TYPE_DEFAULT;
  238. /* Initialize the list of control pages */
  239. INIT_LIST_HEAD(&image->control_pages);
  240. /* Initialize the list of destination pages */
  241. INIT_LIST_HEAD(&image->dest_pages);
  242. /* Initialize the list of unusable pages */
  243. INIT_LIST_HEAD(&image->unusable_pages);
  244. return image;
  245. }
  246. int kimage_is_destination_range(struct kimage *image,
  247. unsigned long start,
  248. unsigned long end)
  249. {
  250. unsigned long i;
  251. for (i = 0; i < image->nr_segments; i++) {
  252. unsigned long mstart, mend;
  253. mstart = image->segment[i].mem;
  254. mend = mstart + image->segment[i].memsz;
  255. if ((end > mstart) && (start < mend))
  256. return 1;
  257. }
  258. return 0;
  259. }
  260. static struct page *kimage_alloc_pages(gfp_t gfp_mask, unsigned int order)
  261. {
  262. struct page *pages;
  263. if (fatal_signal_pending(current))
  264. return NULL;
  265. pages = alloc_pages(gfp_mask & ~__GFP_ZERO, order);
  266. if (pages) {
  267. unsigned int count, i;
  268. pages->mapping = NULL;
  269. set_page_private(pages, order);
  270. count = 1 << order;
  271. for (i = 0; i < count; i++)
  272. SetPageReserved(pages + i);
  273. arch_kexec_post_alloc_pages(page_address(pages), count,
  274. gfp_mask);
  275. if (gfp_mask & __GFP_ZERO)
  276. for (i = 0; i < count; i++)
  277. clear_highpage(pages + i);
  278. }
  279. return pages;
  280. }
  281. static void kimage_free_pages(struct page *page)
  282. {
  283. unsigned int order, count, i;
  284. order = page_private(page);
  285. count = 1 << order;
  286. arch_kexec_pre_free_pages(page_address(page), count);
  287. for (i = 0; i < count; i++)
  288. ClearPageReserved(page + i);
  289. __free_pages(page, order);
  290. }
  291. void kimage_free_page_list(struct list_head *list)
  292. {
  293. struct page *page, *next;
  294. list_for_each_entry_safe(page, next, list, lru) {
  295. list_del(&page->lru);
  296. kimage_free_pages(page);
  297. }
  298. }
  299. static struct page *kimage_alloc_normal_control_pages(struct kimage *image,
  300. unsigned int order)
  301. {
  302. /* Control pages are special, they are the intermediaries
  303. * that are needed while we copy the rest of the pages
  304. * to their final resting place. As such they must
  305. * not conflict with either the destination addresses
  306. * or memory the kernel is already using.
  307. *
  308. * The only case where we really need more than one of
  309. * these are for architectures where we cannot disable
  310. * the MMU and must instead generate an identity mapped
  311. * page table for all of the memory.
  312. *
  313. * At worst this runs in O(N) of the image size.
  314. */
  315. struct list_head extra_pages;
  316. struct page *pages;
  317. unsigned int count;
  318. count = 1 << order;
  319. INIT_LIST_HEAD(&extra_pages);
  320. /* Loop while I can allocate a page and the page allocated
  321. * is a destination page.
  322. */
  323. do {
  324. unsigned long pfn, epfn, addr, eaddr;
  325. pages = kimage_alloc_pages(KEXEC_CONTROL_MEMORY_GFP, order);
  326. if (!pages)
  327. break;
  328. pfn = page_to_boot_pfn(pages);
  329. epfn = pfn + count;
  330. addr = pfn << PAGE_SHIFT;
  331. eaddr = epfn << PAGE_SHIFT;
  332. if ((epfn >= (KEXEC_CONTROL_MEMORY_LIMIT >> PAGE_SHIFT)) ||
  333. kimage_is_destination_range(image, addr, eaddr)) {
  334. list_add(&pages->lru, &extra_pages);
  335. pages = NULL;
  336. }
  337. } while (!pages);
  338. if (pages) {
  339. /* Remember the allocated page... */
  340. list_add(&pages->lru, &image->control_pages);
  341. /* Because the page is already in it's destination
  342. * location we will never allocate another page at
  343. * that address. Therefore kimage_alloc_pages
  344. * will not return it (again) and we don't need
  345. * to give it an entry in image->segment[].
  346. */
  347. }
  348. /* Deal with the destination pages I have inadvertently allocated.
  349. *
  350. * Ideally I would convert multi-page allocations into single
  351. * page allocations, and add everything to image->dest_pages.
  352. *
  353. * For now it is simpler to just free the pages.
  354. */
  355. kimage_free_page_list(&extra_pages);
  356. return pages;
  357. }
  358. static struct page *kimage_alloc_crash_control_pages(struct kimage *image,
  359. unsigned int order)
  360. {
  361. /* Control pages are special, they are the intermediaries
  362. * that are needed while we copy the rest of the pages
  363. * to their final resting place. As such they must
  364. * not conflict with either the destination addresses
  365. * or memory the kernel is already using.
  366. *
  367. * Control pages are also the only pags we must allocate
  368. * when loading a crash kernel. All of the other pages
  369. * are specified by the segments and we just memcpy
  370. * into them directly.
  371. *
  372. * The only case where we really need more than one of
  373. * these are for architectures where we cannot disable
  374. * the MMU and must instead generate an identity mapped
  375. * page table for all of the memory.
  376. *
  377. * Given the low demand this implements a very simple
  378. * allocator that finds the first hole of the appropriate
  379. * size in the reserved memory region, and allocates all
  380. * of the memory up to and including the hole.
  381. */
  382. unsigned long hole_start, hole_end, size;
  383. struct page *pages;
  384. pages = NULL;
  385. size = (1 << order) << PAGE_SHIFT;
  386. hole_start = (image->control_page + (size - 1)) & ~(size - 1);
  387. hole_end = hole_start + size - 1;
  388. while (hole_end <= crashk_res.end) {
  389. unsigned long i;
  390. cond_resched();
  391. if (hole_end > KEXEC_CRASH_CONTROL_MEMORY_LIMIT)
  392. break;
  393. /* See if I overlap any of the segments */
  394. for (i = 0; i < image->nr_segments; i++) {
  395. unsigned long mstart, mend;
  396. mstart = image->segment[i].mem;
  397. mend = mstart + image->segment[i].memsz - 1;
  398. if ((hole_end >= mstart) && (hole_start <= mend)) {
  399. /* Advance the hole to the end of the segment */
  400. hole_start = (mend + (size - 1)) & ~(size - 1);
  401. hole_end = hole_start + size - 1;
  402. break;
  403. }
  404. }
  405. /* If I don't overlap any segments I have found my hole! */
  406. if (i == image->nr_segments) {
  407. pages = pfn_to_page(hole_start >> PAGE_SHIFT);
  408. image->control_page = hole_end;
  409. break;
  410. }
  411. }
  412. /* Ensure that these pages are decrypted if SME is enabled. */
  413. if (pages)
  414. arch_kexec_post_alloc_pages(page_address(pages), 1 << order, 0);
  415. return pages;
  416. }
  417. struct page *kimage_alloc_control_pages(struct kimage *image,
  418. unsigned int order)
  419. {
  420. struct page *pages = NULL;
  421. switch (image->type) {
  422. case KEXEC_TYPE_DEFAULT:
  423. pages = kimage_alloc_normal_control_pages(image, order);
  424. break;
  425. case KEXEC_TYPE_CRASH:
  426. pages = kimage_alloc_crash_control_pages(image, order);
  427. break;
  428. }
  429. return pages;
  430. }
  431. int kimage_crash_copy_vmcoreinfo(struct kimage *image)
  432. {
  433. struct page *vmcoreinfo_page;
  434. void *safecopy;
  435. if (image->type != KEXEC_TYPE_CRASH)
  436. return 0;
  437. /*
  438. * For kdump, allocate one vmcoreinfo safe copy from the
  439. * crash memory. as we have arch_kexec_protect_crashkres()
  440. * after kexec syscall, we naturally protect it from write
  441. * (even read) access under kernel direct mapping. But on
  442. * the other hand, we still need to operate it when crash
  443. * happens to generate vmcoreinfo note, hereby we rely on
  444. * vmap for this purpose.
  445. */
  446. vmcoreinfo_page = kimage_alloc_control_pages(image, 0);
  447. if (!vmcoreinfo_page) {
  448. pr_warn("Could not allocate vmcoreinfo buffer\n");
  449. return -ENOMEM;
  450. }
  451. safecopy = vmap(&vmcoreinfo_page, 1, VM_MAP, PAGE_KERNEL);
  452. if (!safecopy) {
  453. pr_warn("Could not vmap vmcoreinfo buffer\n");
  454. return -ENOMEM;
  455. }
  456. image->vmcoreinfo_data_copy = safecopy;
  457. crash_update_vmcoreinfo_safecopy(safecopy);
  458. return 0;
  459. }
  460. static int kimage_add_entry(struct kimage *image, kimage_entry_t entry)
  461. {
  462. if (*image->entry != 0)
  463. image->entry++;
  464. if (image->entry == image->last_entry) {
  465. kimage_entry_t *ind_page;
  466. struct page *page;
  467. page = kimage_alloc_page(image, GFP_KERNEL, KIMAGE_NO_DEST);
  468. if (!page)
  469. return -ENOMEM;
  470. ind_page = page_address(page);
  471. *image->entry = virt_to_boot_phys(ind_page) | IND_INDIRECTION;
  472. image->entry = ind_page;
  473. image->last_entry = ind_page +
  474. ((PAGE_SIZE/sizeof(kimage_entry_t)) - 1);
  475. }
  476. *image->entry = entry;
  477. image->entry++;
  478. *image->entry = 0;
  479. return 0;
  480. }
  481. static int kimage_set_destination(struct kimage *image,
  482. unsigned long destination)
  483. {
  484. int result;
  485. destination &= PAGE_MASK;
  486. result = kimage_add_entry(image, destination | IND_DESTINATION);
  487. return result;
  488. }
  489. static int kimage_add_page(struct kimage *image, unsigned long page)
  490. {
  491. int result;
  492. page &= PAGE_MASK;
  493. result = kimage_add_entry(image, page | IND_SOURCE);
  494. return result;
  495. }
  496. static void kimage_free_extra_pages(struct kimage *image)
  497. {
  498. /* Walk through and free any extra destination pages I may have */
  499. kimage_free_page_list(&image->dest_pages);
  500. /* Walk through and free any unusable pages I have cached */
  501. kimage_free_page_list(&image->unusable_pages);
  502. }
  503. void kimage_terminate(struct kimage *image)
  504. {
  505. if (*image->entry != 0)
  506. image->entry++;
  507. *image->entry = IND_DONE;
  508. }
  509. #define for_each_kimage_entry(image, ptr, entry) \
  510. for (ptr = &image->head; (entry = *ptr) && !(entry & IND_DONE); \
  511. ptr = (entry & IND_INDIRECTION) ? \
  512. boot_phys_to_virt((entry & PAGE_MASK)) : ptr + 1)
  513. static void kimage_free_entry(kimage_entry_t entry)
  514. {
  515. struct page *page;
  516. page = boot_pfn_to_page(entry >> PAGE_SHIFT);
  517. kimage_free_pages(page);
  518. }
  519. void kimage_free(struct kimage *image)
  520. {
  521. kimage_entry_t *ptr, entry;
  522. kimage_entry_t ind = 0;
  523. if (!image)
  524. return;
  525. if (image->vmcoreinfo_data_copy) {
  526. crash_update_vmcoreinfo_safecopy(NULL);
  527. vunmap(image->vmcoreinfo_data_copy);
  528. }
  529. kimage_free_extra_pages(image);
  530. for_each_kimage_entry(image, ptr, entry) {
  531. if (entry & IND_INDIRECTION) {
  532. /* Free the previous indirection page */
  533. if (ind & IND_INDIRECTION)
  534. kimage_free_entry(ind);
  535. /* Save this indirection page until we are
  536. * done with it.
  537. */
  538. ind = entry;
  539. } else if (entry & IND_SOURCE)
  540. kimage_free_entry(entry);
  541. }
  542. /* Free the final indirection page */
  543. if (ind & IND_INDIRECTION)
  544. kimage_free_entry(ind);
  545. /* Handle any machine specific cleanup */
  546. machine_kexec_cleanup(image);
  547. /* Free the kexec control pages... */
  548. kimage_free_page_list(&image->control_pages);
  549. /*
  550. * Free up any temporary buffers allocated. This might hit if
  551. * error occurred much later after buffer allocation.
  552. */
  553. if (image->file_mode)
  554. kimage_file_post_load_cleanup(image);
  555. kfree(image);
  556. }
  557. static kimage_entry_t *kimage_dst_used(struct kimage *image,
  558. unsigned long page)
  559. {
  560. kimage_entry_t *ptr, entry;
  561. unsigned long destination = 0;
  562. for_each_kimage_entry(image, ptr, entry) {
  563. if (entry & IND_DESTINATION)
  564. destination = entry & PAGE_MASK;
  565. else if (entry & IND_SOURCE) {
  566. if (page == destination)
  567. return ptr;
  568. destination += PAGE_SIZE;
  569. }
  570. }
  571. return NULL;
  572. }
  573. static struct page *kimage_alloc_page(struct kimage *image,
  574. gfp_t gfp_mask,
  575. unsigned long destination)
  576. {
  577. /*
  578. * Here we implement safeguards to ensure that a source page
  579. * is not copied to its destination page before the data on
  580. * the destination page is no longer useful.
  581. *
  582. * To do this we maintain the invariant that a source page is
  583. * either its own destination page, or it is not a
  584. * destination page at all.
  585. *
  586. * That is slightly stronger than required, but the proof
  587. * that no problems will not occur is trivial, and the
  588. * implementation is simply to verify.
  589. *
  590. * When allocating all pages normally this algorithm will run
  591. * in O(N) time, but in the worst case it will run in O(N^2)
  592. * time. If the runtime is a problem the data structures can
  593. * be fixed.
  594. */
  595. struct page *page;
  596. unsigned long addr;
  597. /*
  598. * Walk through the list of destination pages, and see if I
  599. * have a match.
  600. */
  601. list_for_each_entry(page, &image->dest_pages, lru) {
  602. addr = page_to_boot_pfn(page) << PAGE_SHIFT;
  603. if (addr == destination) {
  604. list_del(&page->lru);
  605. return page;
  606. }
  607. }
  608. page = NULL;
  609. while (1) {
  610. kimage_entry_t *old;
  611. /* Allocate a page, if we run out of memory give up */
  612. page = kimage_alloc_pages(gfp_mask, 0);
  613. if (!page)
  614. return NULL;
  615. /* If the page cannot be used file it away */
  616. if (page_to_boot_pfn(page) >
  617. (KEXEC_SOURCE_MEMORY_LIMIT >> PAGE_SHIFT)) {
  618. list_add(&page->lru, &image->unusable_pages);
  619. continue;
  620. }
  621. addr = page_to_boot_pfn(page) << PAGE_SHIFT;
  622. /* If it is the destination page we want use it */
  623. if (addr == destination)
  624. break;
  625. /* If the page is not a destination page use it */
  626. if (!kimage_is_destination_range(image, addr,
  627. addr + PAGE_SIZE))
  628. break;
  629. /*
  630. * I know that the page is someones destination page.
  631. * See if there is already a source page for this
  632. * destination page. And if so swap the source pages.
  633. */
  634. old = kimage_dst_used(image, addr);
  635. if (old) {
  636. /* If so move it */
  637. unsigned long old_addr;
  638. struct page *old_page;
  639. old_addr = *old & PAGE_MASK;
  640. old_page = boot_pfn_to_page(old_addr >> PAGE_SHIFT);
  641. copy_highpage(page, old_page);
  642. *old = addr | (*old & ~PAGE_MASK);
  643. /* The old page I have found cannot be a
  644. * destination page, so return it if it's
  645. * gfp_flags honor the ones passed in.
  646. */
  647. if (!(gfp_mask & __GFP_HIGHMEM) &&
  648. PageHighMem(old_page)) {
  649. kimage_free_pages(old_page);
  650. continue;
  651. }
  652. page = old_page;
  653. break;
  654. }
  655. /* Place the page on the destination list, to be used later */
  656. list_add(&page->lru, &image->dest_pages);
  657. }
  658. return page;
  659. }
  660. static int kimage_load_normal_segment(struct kimage *image,
  661. struct kexec_segment *segment)
  662. {
  663. unsigned long maddr;
  664. size_t ubytes, mbytes;
  665. int result;
  666. unsigned char __user *buf = NULL;
  667. unsigned char *kbuf = NULL;
  668. if (image->file_mode)
  669. kbuf = segment->kbuf;
  670. else
  671. buf = segment->buf;
  672. ubytes = segment->bufsz;
  673. mbytes = segment->memsz;
  674. maddr = segment->mem;
  675. result = kimage_set_destination(image, maddr);
  676. if (result < 0)
  677. goto out;
  678. while (mbytes) {
  679. struct page *page;
  680. char *ptr;
  681. size_t uchunk, mchunk;
  682. page = kimage_alloc_page(image, GFP_HIGHUSER, maddr);
  683. if (!page) {
  684. result = -ENOMEM;
  685. goto out;
  686. }
  687. result = kimage_add_page(image, page_to_boot_pfn(page)
  688. << PAGE_SHIFT);
  689. if (result < 0)
  690. goto out;
  691. ptr = kmap_local_page(page);
  692. /* Start with a clear page */
  693. clear_page(ptr);
  694. ptr += maddr & ~PAGE_MASK;
  695. mchunk = min_t(size_t, mbytes,
  696. PAGE_SIZE - (maddr & ~PAGE_MASK));
  697. uchunk = min(ubytes, mchunk);
  698. /* For file based kexec, source pages are in kernel memory */
  699. if (image->file_mode)
  700. memcpy(ptr, kbuf, uchunk);
  701. else
  702. result = copy_from_user(ptr, buf, uchunk);
  703. kunmap_local(ptr);
  704. if (result) {
  705. result = -EFAULT;
  706. goto out;
  707. }
  708. ubytes -= uchunk;
  709. maddr += mchunk;
  710. if (image->file_mode)
  711. kbuf += mchunk;
  712. else
  713. buf += mchunk;
  714. mbytes -= mchunk;
  715. cond_resched();
  716. }
  717. out:
  718. return result;
  719. }
  720. static int kimage_load_crash_segment(struct kimage *image,
  721. struct kexec_segment *segment)
  722. {
  723. /* For crash dumps kernels we simply copy the data from
  724. * user space to it's destination.
  725. * We do things a page at a time for the sake of kmap.
  726. */
  727. unsigned long maddr;
  728. size_t ubytes, mbytes;
  729. int result;
  730. unsigned char __user *buf = NULL;
  731. unsigned char *kbuf = NULL;
  732. result = 0;
  733. if (image->file_mode)
  734. kbuf = segment->kbuf;
  735. else
  736. buf = segment->buf;
  737. ubytes = segment->bufsz;
  738. mbytes = segment->memsz;
  739. maddr = segment->mem;
  740. while (mbytes) {
  741. struct page *page;
  742. char *ptr;
  743. size_t uchunk, mchunk;
  744. page = boot_pfn_to_page(maddr >> PAGE_SHIFT);
  745. if (!page) {
  746. result = -ENOMEM;
  747. goto out;
  748. }
  749. arch_kexec_post_alloc_pages(page_address(page), 1, 0);
  750. ptr = kmap_local_page(page);
  751. ptr += maddr & ~PAGE_MASK;
  752. mchunk = min_t(size_t, mbytes,
  753. PAGE_SIZE - (maddr & ~PAGE_MASK));
  754. uchunk = min(ubytes, mchunk);
  755. if (mchunk > uchunk) {
  756. /* Zero the trailing part of the page */
  757. memset(ptr + uchunk, 0, mchunk - uchunk);
  758. }
  759. /* For file based kexec, source pages are in kernel memory */
  760. if (image->file_mode)
  761. memcpy(ptr, kbuf, uchunk);
  762. else
  763. result = copy_from_user(ptr, buf, uchunk);
  764. kexec_flush_icache_page(page);
  765. kunmap_local(ptr);
  766. arch_kexec_pre_free_pages(page_address(page), 1);
  767. if (result) {
  768. result = -EFAULT;
  769. goto out;
  770. }
  771. ubytes -= uchunk;
  772. maddr += mchunk;
  773. if (image->file_mode)
  774. kbuf += mchunk;
  775. else
  776. buf += mchunk;
  777. mbytes -= mchunk;
  778. cond_resched();
  779. }
  780. out:
  781. return result;
  782. }
  783. int kimage_load_segment(struct kimage *image,
  784. struct kexec_segment *segment)
  785. {
  786. int result = -ENOMEM;
  787. switch (image->type) {
  788. case KEXEC_TYPE_DEFAULT:
  789. result = kimage_load_normal_segment(image, segment);
  790. break;
  791. case KEXEC_TYPE_CRASH:
  792. result = kimage_load_crash_segment(image, segment);
  793. break;
  794. }
  795. return result;
  796. }
  797. struct kimage *kexec_image;
  798. struct kimage *kexec_crash_image;
  799. int kexec_load_disabled;
  800. #ifdef CONFIG_SYSCTL
  801. static struct ctl_table kexec_core_sysctls[] = {
  802. {
  803. .procname = "kexec_load_disabled",
  804. .data = &kexec_load_disabled,
  805. .maxlen = sizeof(int),
  806. .mode = 0644,
  807. /* only handle a transition from default "0" to "1" */
  808. .proc_handler = proc_dointvec_minmax,
  809. .extra1 = SYSCTL_ONE,
  810. .extra2 = SYSCTL_ONE,
  811. },
  812. { }
  813. };
  814. static int __init kexec_core_sysctl_init(void)
  815. {
  816. register_sysctl_init("kernel", kexec_core_sysctls);
  817. return 0;
  818. }
  819. late_initcall(kexec_core_sysctl_init);
  820. #endif
  821. /*
  822. * No panic_cpu check version of crash_kexec(). This function is called
  823. * only when panic_cpu holds the current CPU number; this is the only CPU
  824. * which processes crash_kexec routines.
  825. */
  826. void __noclone __crash_kexec(struct pt_regs *regs)
  827. {
  828. /* Take the kexec_lock here to prevent sys_kexec_load
  829. * running on one cpu from replacing the crash kernel
  830. * we are using after a panic on a different cpu.
  831. *
  832. * If the crash kernel was not located in a fixed area
  833. * of memory the xchg(&kexec_crash_image) would be
  834. * sufficient. But since I reuse the memory...
  835. */
  836. if (kexec_trylock()) {
  837. if (kexec_crash_image) {
  838. struct pt_regs fixed_regs;
  839. crash_setup_regs(&fixed_regs, regs);
  840. crash_save_vmcoreinfo();
  841. machine_crash_shutdown(&fixed_regs);
  842. machine_kexec(kexec_crash_image);
  843. }
  844. kexec_unlock();
  845. }
  846. }
  847. STACK_FRAME_NON_STANDARD(__crash_kexec);
  848. void crash_kexec(struct pt_regs *regs)
  849. {
  850. int old_cpu, this_cpu;
  851. /*
  852. * Only one CPU is allowed to execute the crash_kexec() code as with
  853. * panic(). Otherwise parallel calls of panic() and crash_kexec()
  854. * may stop each other. To exclude them, we use panic_cpu here too.
  855. */
  856. this_cpu = raw_smp_processor_id();
  857. old_cpu = atomic_cmpxchg(&panic_cpu, PANIC_CPU_INVALID, this_cpu);
  858. if (old_cpu == PANIC_CPU_INVALID) {
  859. /* This is the 1st CPU which comes here, so go ahead. */
  860. __crash_kexec(regs);
  861. /*
  862. * Reset panic_cpu to allow another panic()/crash_kexec()
  863. * call.
  864. */
  865. atomic_set(&panic_cpu, PANIC_CPU_INVALID);
  866. }
  867. }
  868. ssize_t crash_get_memory_size(void)
  869. {
  870. ssize_t size = 0;
  871. if (!kexec_trylock())
  872. return -EBUSY;
  873. if (crashk_res.end != crashk_res.start)
  874. size = resource_size(&crashk_res);
  875. kexec_unlock();
  876. return size;
  877. }
  878. int crash_shrink_memory(unsigned long new_size)
  879. {
  880. int ret = 0;
  881. unsigned long start, end;
  882. unsigned long old_size;
  883. struct resource *ram_res;
  884. if (!kexec_trylock())
  885. return -EBUSY;
  886. if (kexec_crash_image) {
  887. ret = -ENOENT;
  888. goto unlock;
  889. }
  890. start = crashk_res.start;
  891. end = crashk_res.end;
  892. old_size = (end == 0) ? 0 : end - start + 1;
  893. new_size = roundup(new_size, KEXEC_CRASH_MEM_ALIGN);
  894. if (new_size >= old_size) {
  895. ret = (new_size == old_size) ? 0 : -EINVAL;
  896. goto unlock;
  897. }
  898. ram_res = kzalloc(sizeof(*ram_res), GFP_KERNEL);
  899. if (!ram_res) {
  900. ret = -ENOMEM;
  901. goto unlock;
  902. }
  903. end = start + new_size;
  904. crash_free_reserved_phys_range(end, crashk_res.end);
  905. if ((start == end) && (crashk_res.parent != NULL))
  906. release_resource(&crashk_res);
  907. ram_res->start = end;
  908. ram_res->end = crashk_res.end;
  909. ram_res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM;
  910. ram_res->name = "System RAM";
  911. crashk_res.end = end - 1;
  912. insert_resource(&iomem_resource, ram_res);
  913. unlock:
  914. kexec_unlock();
  915. return ret;
  916. }
  917. void crash_save_cpu(struct pt_regs *regs, int cpu)
  918. {
  919. struct elf_prstatus prstatus;
  920. u32 *buf;
  921. if ((cpu < 0) || (cpu >= nr_cpu_ids))
  922. return;
  923. /* Using ELF notes here is opportunistic.
  924. * I need a well defined structure format
  925. * for the data I pass, and I need tags
  926. * on the data to indicate what information I have
  927. * squirrelled away. ELF notes happen to provide
  928. * all of that, so there is no need to invent something new.
  929. */
  930. buf = (u32 *)per_cpu_ptr(crash_notes, cpu);
  931. if (!buf)
  932. return;
  933. memset(&prstatus, 0, sizeof(prstatus));
  934. prstatus.common.pr_pid = current->pid;
  935. elf_core_copy_regs(&prstatus.pr_reg, regs);
  936. buf = append_elf_note(buf, KEXEC_CORE_NOTE_NAME, NT_PRSTATUS,
  937. &prstatus, sizeof(prstatus));
  938. final_note(buf);
  939. }
  940. static int __init crash_notes_memory_init(void)
  941. {
  942. /* Allocate memory for saving cpu registers. */
  943. size_t size, align;
  944. /*
  945. * crash_notes could be allocated across 2 vmalloc pages when percpu
  946. * is vmalloc based . vmalloc doesn't guarantee 2 continuous vmalloc
  947. * pages are also on 2 continuous physical pages. In this case the
  948. * 2nd part of crash_notes in 2nd page could be lost since only the
  949. * starting address and size of crash_notes are exported through sysfs.
  950. * Here round up the size of crash_notes to the nearest power of two
  951. * and pass it to __alloc_percpu as align value. This can make sure
  952. * crash_notes is allocated inside one physical page.
  953. */
  954. size = sizeof(note_buf_t);
  955. align = min(roundup_pow_of_two(sizeof(note_buf_t)), PAGE_SIZE);
  956. /*
  957. * Break compile if size is bigger than PAGE_SIZE since crash_notes
  958. * definitely will be in 2 pages with that.
  959. */
  960. BUILD_BUG_ON(size > PAGE_SIZE);
  961. crash_notes = __alloc_percpu(size, align);
  962. if (!crash_notes) {
  963. pr_warn("Memory allocation for saving cpu register states failed\n");
  964. return -ENOMEM;
  965. }
  966. return 0;
  967. }
  968. subsys_initcall(crash_notes_memory_init);
  969. /*
  970. * Move into place and start executing a preloaded standalone
  971. * executable. If nothing was preloaded return an error.
  972. */
  973. int kernel_kexec(void)
  974. {
  975. int error = 0;
  976. if (!kexec_trylock())
  977. return -EBUSY;
  978. if (!kexec_image) {
  979. error = -EINVAL;
  980. goto Unlock;
  981. }
  982. #ifdef CONFIG_KEXEC_JUMP
  983. if (kexec_image->preserve_context) {
  984. pm_prepare_console();
  985. error = freeze_processes();
  986. if (error) {
  987. error = -EBUSY;
  988. goto Restore_console;
  989. }
  990. suspend_console();
  991. error = dpm_suspend_start(PMSG_FREEZE);
  992. if (error)
  993. goto Resume_console;
  994. /* At this point, dpm_suspend_start() has been called,
  995. * but *not* dpm_suspend_end(). We *must* call
  996. * dpm_suspend_end() now. Otherwise, drivers for
  997. * some devices (e.g. interrupt controllers) become
  998. * desynchronized with the actual state of the
  999. * hardware at resume time, and evil weirdness ensues.
  1000. */
  1001. error = dpm_suspend_end(PMSG_FREEZE);
  1002. if (error)
  1003. goto Resume_devices;
  1004. error = suspend_disable_secondary_cpus();
  1005. if (error)
  1006. goto Enable_cpus;
  1007. local_irq_disable();
  1008. error = syscore_suspend();
  1009. if (error)
  1010. goto Enable_irqs;
  1011. } else
  1012. #endif
  1013. {
  1014. kexec_in_progress = true;
  1015. kernel_restart_prepare("kexec reboot");
  1016. migrate_to_reboot_cpu();
  1017. /*
  1018. * migrate_to_reboot_cpu() disables CPU hotplug assuming that
  1019. * no further code needs to use CPU hotplug (which is true in
  1020. * the reboot case). However, the kexec path depends on using
  1021. * CPU hotplug again; so re-enable it here.
  1022. */
  1023. cpu_hotplug_enable();
  1024. pr_notice("Starting new kernel\n");
  1025. machine_shutdown();
  1026. }
  1027. kmsg_dump(KMSG_DUMP_SHUTDOWN);
  1028. machine_kexec(kexec_image);
  1029. #ifdef CONFIG_KEXEC_JUMP
  1030. if (kexec_image->preserve_context) {
  1031. syscore_resume();
  1032. Enable_irqs:
  1033. local_irq_enable();
  1034. Enable_cpus:
  1035. suspend_enable_secondary_cpus();
  1036. dpm_resume_start(PMSG_RESTORE);
  1037. Resume_devices:
  1038. dpm_resume_end(PMSG_RESTORE);
  1039. Resume_console:
  1040. resume_console();
  1041. thaw_processes();
  1042. Restore_console:
  1043. pm_restore_console();
  1044. }
  1045. #endif
  1046. Unlock:
  1047. kexec_unlock();
  1048. return error;
  1049. }