util.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/mm.h>
  3. #include <linux/slab.h>
  4. #include <linux/string.h>
  5. #include <linux/compiler.h>
  6. #include <linux/export.h>
  7. #include <linux/err.h>
  8. #include <linux/sched.h>
  9. #include <linux/sched/mm.h>
  10. #include <linux/sched/signal.h>
  11. #include <linux/sched/task_stack.h>
  12. #include <linux/security.h>
  13. #include <linux/swap.h>
  14. #include <linux/swapops.h>
  15. #include <linux/mman.h>
  16. #include <linux/hugetlb.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/userfaultfd_k.h>
  19. #include <linux/elf.h>
  20. #include <linux/elf-randomize.h>
  21. #include <linux/personality.h>
  22. #include <linux/random.h>
  23. #include <linux/processor.h>
  24. #include <linux/sizes.h>
  25. #include <linux/compat.h>
  26. #include <linux/uaccess.h>
  27. #include <trace/hooks/mm.h>
  28. #include "internal.h"
  29. #include "swap.h"
  30. #ifndef __GENKSYMS__
  31. #include <trace/hooks/syscall_check.h>
  32. #include <trace/hooks/mm.h>
  33. #endif
  34. /**
  35. * kfree_const - conditionally free memory
  36. * @x: pointer to the memory
  37. *
  38. * Function calls kfree only if @x is not in .rodata section.
  39. */
  40. void kfree_const(const void *x)
  41. {
  42. if (!is_kernel_rodata((unsigned long)x))
  43. kfree(x);
  44. }
  45. EXPORT_SYMBOL(kfree_const);
  46. /**
  47. * kstrdup - allocate space for and copy an existing string
  48. * @s: the string to duplicate
  49. * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  50. *
  51. * Return: newly allocated copy of @s or %NULL in case of error
  52. */
  53. char *kstrdup(const char *s, gfp_t gfp)
  54. {
  55. size_t len;
  56. char *buf;
  57. if (!s)
  58. return NULL;
  59. len = strlen(s) + 1;
  60. buf = kmalloc_track_caller(len, gfp);
  61. if (buf)
  62. memcpy(buf, s, len);
  63. return buf;
  64. }
  65. EXPORT_SYMBOL(kstrdup);
  66. /**
  67. * kstrdup_const - conditionally duplicate an existing const string
  68. * @s: the string to duplicate
  69. * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  70. *
  71. * Note: Strings allocated by kstrdup_const should be freed by kfree_const and
  72. * must not be passed to krealloc().
  73. *
  74. * Return: source string if it is in .rodata section otherwise
  75. * fallback to kstrdup.
  76. */
  77. const char *kstrdup_const(const char *s, gfp_t gfp)
  78. {
  79. if (is_kernel_rodata((unsigned long)s))
  80. return s;
  81. return kstrdup(s, gfp);
  82. }
  83. EXPORT_SYMBOL(kstrdup_const);
  84. /**
  85. * kstrndup - allocate space for and copy an existing string
  86. * @s: the string to duplicate
  87. * @max: read at most @max chars from @s
  88. * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  89. *
  90. * Note: Use kmemdup_nul() instead if the size is known exactly.
  91. *
  92. * Return: newly allocated copy of @s or %NULL in case of error
  93. */
  94. char *kstrndup(const char *s, size_t max, gfp_t gfp)
  95. {
  96. size_t len;
  97. char *buf;
  98. if (!s)
  99. return NULL;
  100. len = strnlen(s, max);
  101. buf = kmalloc_track_caller(len+1, gfp);
  102. if (buf) {
  103. memcpy(buf, s, len);
  104. buf[len] = '\0';
  105. }
  106. return buf;
  107. }
  108. EXPORT_SYMBOL(kstrndup);
  109. /**
  110. * kmemdup - duplicate region of memory
  111. *
  112. * @src: memory region to duplicate
  113. * @len: memory region length
  114. * @gfp: GFP mask to use
  115. *
  116. * Return: newly allocated copy of @src or %NULL in case of error
  117. */
  118. void *kmemdup(const void *src, size_t len, gfp_t gfp)
  119. {
  120. void *p;
  121. p = kmalloc_track_caller(len, gfp);
  122. if (p)
  123. memcpy(p, src, len);
  124. return p;
  125. }
  126. EXPORT_SYMBOL(kmemdup);
  127. /**
  128. * kmemdup_nul - Create a NUL-terminated string from unterminated data
  129. * @s: The data to stringify
  130. * @len: The size of the data
  131. * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  132. *
  133. * Return: newly allocated copy of @s with NUL-termination or %NULL in
  134. * case of error
  135. */
  136. char *kmemdup_nul(const char *s, size_t len, gfp_t gfp)
  137. {
  138. char *buf;
  139. if (!s)
  140. return NULL;
  141. buf = kmalloc_track_caller(len + 1, gfp);
  142. if (buf) {
  143. memcpy(buf, s, len);
  144. buf[len] = '\0';
  145. }
  146. return buf;
  147. }
  148. EXPORT_SYMBOL(kmemdup_nul);
  149. /**
  150. * memdup_user - duplicate memory region from user space
  151. *
  152. * @src: source address in user space
  153. * @len: number of bytes to copy
  154. *
  155. * Return: an ERR_PTR() on failure. Result is physically
  156. * contiguous, to be freed by kfree().
  157. */
  158. void *memdup_user(const void __user *src, size_t len)
  159. {
  160. void *p;
  161. p = kmalloc_track_caller(len, GFP_USER | __GFP_NOWARN);
  162. if (!p)
  163. return ERR_PTR(-ENOMEM);
  164. if (copy_from_user(p, src, len)) {
  165. kfree(p);
  166. return ERR_PTR(-EFAULT);
  167. }
  168. return p;
  169. }
  170. EXPORT_SYMBOL(memdup_user);
  171. /**
  172. * vmemdup_user - duplicate memory region from user space
  173. *
  174. * @src: source address in user space
  175. * @len: number of bytes to copy
  176. *
  177. * Return: an ERR_PTR() on failure. Result may be not
  178. * physically contiguous. Use kvfree() to free.
  179. */
  180. void *vmemdup_user(const void __user *src, size_t len)
  181. {
  182. void *p;
  183. p = kvmalloc(len, GFP_USER);
  184. if (!p)
  185. return ERR_PTR(-ENOMEM);
  186. if (copy_from_user(p, src, len)) {
  187. kvfree(p);
  188. return ERR_PTR(-EFAULT);
  189. }
  190. return p;
  191. }
  192. EXPORT_SYMBOL(vmemdup_user);
  193. /**
  194. * strndup_user - duplicate an existing string from user space
  195. * @s: The string to duplicate
  196. * @n: Maximum number of bytes to copy, including the trailing NUL.
  197. *
  198. * Return: newly allocated copy of @s or an ERR_PTR() in case of error
  199. */
  200. char *strndup_user(const char __user *s, long n)
  201. {
  202. char *p;
  203. long length;
  204. length = strnlen_user(s, n);
  205. if (!length)
  206. return ERR_PTR(-EFAULT);
  207. if (length > n)
  208. return ERR_PTR(-EINVAL);
  209. p = memdup_user(s, length);
  210. if (IS_ERR(p))
  211. return p;
  212. p[length - 1] = '\0';
  213. return p;
  214. }
  215. EXPORT_SYMBOL(strndup_user);
  216. /**
  217. * memdup_user_nul - duplicate memory region from user space and NUL-terminate
  218. *
  219. * @src: source address in user space
  220. * @len: number of bytes to copy
  221. *
  222. * Return: an ERR_PTR() on failure.
  223. */
  224. void *memdup_user_nul(const void __user *src, size_t len)
  225. {
  226. char *p;
  227. /*
  228. * Always use GFP_KERNEL, since copy_from_user() can sleep and
  229. * cause pagefault, which makes it pointless to use GFP_NOFS
  230. * or GFP_ATOMIC.
  231. */
  232. p = kmalloc_track_caller(len + 1, GFP_KERNEL);
  233. if (!p)
  234. return ERR_PTR(-ENOMEM);
  235. if (copy_from_user(p, src, len)) {
  236. kfree(p);
  237. return ERR_PTR(-EFAULT);
  238. }
  239. p[len] = '\0';
  240. return p;
  241. }
  242. EXPORT_SYMBOL(memdup_user_nul);
  243. /* Check if the vma is being used as a stack by this task */
  244. int vma_is_stack_for_current(struct vm_area_struct *vma)
  245. {
  246. struct task_struct * __maybe_unused t = current;
  247. return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
  248. }
  249. /*
  250. * Change backing file, only valid to use during initial VMA setup.
  251. */
  252. void vma_set_file(struct vm_area_struct *vma, struct file *file)
  253. {
  254. /* Changing an anonymous vma with this is illegal */
  255. get_file(file);
  256. swap(vma->vm_file, file);
  257. fput(file);
  258. }
  259. EXPORT_SYMBOL(vma_set_file);
  260. #ifndef STACK_RND_MASK
  261. #define STACK_RND_MASK (0x7ff >> (PAGE_SHIFT - 12)) /* 8MB of VA */
  262. #endif
  263. unsigned long randomize_stack_top(unsigned long stack_top)
  264. {
  265. unsigned long random_variable = 0;
  266. if (current->flags & PF_RANDOMIZE) {
  267. random_variable = get_random_long();
  268. random_variable &= STACK_RND_MASK;
  269. random_variable <<= PAGE_SHIFT;
  270. }
  271. #ifdef CONFIG_STACK_GROWSUP
  272. return PAGE_ALIGN(stack_top) + random_variable;
  273. #else
  274. return PAGE_ALIGN(stack_top) - random_variable;
  275. #endif
  276. }
  277. /**
  278. * randomize_page - Generate a random, page aligned address
  279. * @start: The smallest acceptable address the caller will take.
  280. * @range: The size of the area, starting at @start, within which the
  281. * random address must fall.
  282. *
  283. * If @start + @range would overflow, @range is capped.
  284. *
  285. * NOTE: Historical use of randomize_range, which this replaces, presumed that
  286. * @start was already page aligned. We now align it regardless.
  287. *
  288. * Return: A page aligned address within [start, start + range). On error,
  289. * @start is returned.
  290. */
  291. unsigned long randomize_page(unsigned long start, unsigned long range)
  292. {
  293. if (!PAGE_ALIGNED(start)) {
  294. range -= PAGE_ALIGN(start) - start;
  295. start = PAGE_ALIGN(start);
  296. }
  297. if (start > ULONG_MAX - range)
  298. range = ULONG_MAX - start;
  299. range >>= PAGE_SHIFT;
  300. if (range == 0)
  301. return start;
  302. return start + (get_random_long() % range << PAGE_SHIFT);
  303. }
  304. #ifdef CONFIG_ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT
  305. unsigned long __weak arch_randomize_brk(struct mm_struct *mm)
  306. {
  307. /* Is the current task 32bit ? */
  308. if (!IS_ENABLED(CONFIG_64BIT) || is_compat_task())
  309. return randomize_page(mm->brk, SZ_32M);
  310. return randomize_page(mm->brk, SZ_1G);
  311. }
  312. unsigned long arch_mmap_rnd(void)
  313. {
  314. unsigned long rnd;
  315. #ifdef CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS
  316. if (is_compat_task())
  317. rnd = get_random_long() & ((1UL << mmap_rnd_compat_bits) - 1);
  318. else
  319. #endif /* CONFIG_HAVE_ARCH_MMAP_RND_COMPAT_BITS */
  320. rnd = get_random_long() & ((1UL << mmap_rnd_bits) - 1);
  321. return rnd << PAGE_SHIFT;
  322. }
  323. static int mmap_is_legacy(struct rlimit *rlim_stack)
  324. {
  325. if (current->personality & ADDR_COMPAT_LAYOUT)
  326. return 1;
  327. if (rlim_stack->rlim_cur == RLIM_INFINITY)
  328. return 1;
  329. return sysctl_legacy_va_layout;
  330. }
  331. /*
  332. * Leave enough space between the mmap area and the stack to honour ulimit in
  333. * the face of randomisation.
  334. */
  335. #define MIN_GAP (SZ_128M)
  336. #define MAX_GAP (STACK_TOP / 6 * 5)
  337. static unsigned long mmap_base(unsigned long rnd, struct rlimit *rlim_stack)
  338. {
  339. unsigned long gap = rlim_stack->rlim_cur;
  340. unsigned long pad = stack_guard_gap;
  341. /* Account for stack randomization if necessary */
  342. if (current->flags & PF_RANDOMIZE)
  343. pad += (STACK_RND_MASK << PAGE_SHIFT);
  344. /* Values close to RLIM_INFINITY can overflow. */
  345. if (gap + pad > gap)
  346. gap += pad;
  347. if (gap < MIN_GAP)
  348. gap = MIN_GAP;
  349. else if (gap > MAX_GAP)
  350. gap = MAX_GAP;
  351. return PAGE_ALIGN(STACK_TOP - gap - rnd);
  352. }
  353. void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
  354. {
  355. unsigned long random_factor = 0UL;
  356. if (current->flags & PF_RANDOMIZE)
  357. random_factor = arch_mmap_rnd();
  358. if (mmap_is_legacy(rlim_stack)) {
  359. mm->mmap_base = TASK_UNMAPPED_BASE + random_factor;
  360. mm->get_unmapped_area = arch_get_unmapped_area;
  361. } else {
  362. mm->mmap_base = mmap_base(random_factor, rlim_stack);
  363. mm->get_unmapped_area = arch_get_unmapped_area_topdown;
  364. }
  365. }
  366. #elif defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
  367. void arch_pick_mmap_layout(struct mm_struct *mm, struct rlimit *rlim_stack)
  368. {
  369. mm->mmap_base = TASK_UNMAPPED_BASE;
  370. mm->get_unmapped_area = arch_get_unmapped_area;
  371. }
  372. #endif
  373. /**
  374. * __account_locked_vm - account locked pages to an mm's locked_vm
  375. * @mm: mm to account against
  376. * @pages: number of pages to account
  377. * @inc: %true if @pages should be considered positive, %false if not
  378. * @task: task used to check RLIMIT_MEMLOCK
  379. * @bypass_rlim: %true if checking RLIMIT_MEMLOCK should be skipped
  380. *
  381. * Assumes @task and @mm are valid (i.e. at least one reference on each), and
  382. * that mmap_lock is held as writer.
  383. *
  384. * Return:
  385. * * 0 on success
  386. * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
  387. */
  388. int __account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc,
  389. struct task_struct *task, bool bypass_rlim)
  390. {
  391. unsigned long locked_vm, limit;
  392. int ret = 0;
  393. mmap_assert_write_locked(mm);
  394. locked_vm = mm->locked_vm;
  395. if (inc) {
  396. if (!bypass_rlim) {
  397. limit = task_rlimit(task, RLIMIT_MEMLOCK) >> PAGE_SHIFT;
  398. if (locked_vm + pages > limit)
  399. ret = -ENOMEM;
  400. }
  401. if (!ret)
  402. mm->locked_vm = locked_vm + pages;
  403. } else {
  404. WARN_ON_ONCE(pages > locked_vm);
  405. mm->locked_vm = locked_vm - pages;
  406. }
  407. pr_debug("%s: [%d] caller %ps %c%lu %lu/%lu%s\n", __func__, task->pid,
  408. (void *)_RET_IP_, (inc) ? '+' : '-', pages << PAGE_SHIFT,
  409. locked_vm << PAGE_SHIFT, task_rlimit(task, RLIMIT_MEMLOCK),
  410. ret ? " - exceeded" : "");
  411. return ret;
  412. }
  413. EXPORT_SYMBOL_GPL(__account_locked_vm);
  414. /**
  415. * account_locked_vm - account locked pages to an mm's locked_vm
  416. * @mm: mm to account against, may be NULL
  417. * @pages: number of pages to account
  418. * @inc: %true if @pages should be considered positive, %false if not
  419. *
  420. * Assumes a non-NULL @mm is valid (i.e. at least one reference on it).
  421. *
  422. * Return:
  423. * * 0 on success, or if mm is NULL
  424. * * -ENOMEM if RLIMIT_MEMLOCK would be exceeded.
  425. */
  426. int account_locked_vm(struct mm_struct *mm, unsigned long pages, bool inc)
  427. {
  428. int ret;
  429. if (pages == 0 || !mm)
  430. return 0;
  431. mmap_write_lock(mm);
  432. ret = __account_locked_vm(mm, pages, inc, current,
  433. capable(CAP_IPC_LOCK));
  434. mmap_write_unlock(mm);
  435. return ret;
  436. }
  437. EXPORT_SYMBOL_GPL(account_locked_vm);
  438. unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
  439. unsigned long len, unsigned long prot,
  440. unsigned long flag, unsigned long pgoff)
  441. {
  442. unsigned long ret;
  443. struct mm_struct *mm = current->mm;
  444. unsigned long populate;
  445. LIST_HEAD(uf);
  446. ret = security_mmap_file(file, prot, flag);
  447. if (!ret) {
  448. if (mmap_write_lock_killable(mm))
  449. return -EINTR;
  450. ret = do_mmap(file, addr, len, prot, flag, pgoff, &populate,
  451. &uf);
  452. mmap_write_unlock(mm);
  453. userfaultfd_unmap_complete(mm, &uf);
  454. if (populate)
  455. mm_populate(ret, populate);
  456. }
  457. trace_android_vh_check_mmap_file(file, prot, flag, ret);
  458. return ret;
  459. }
  460. unsigned long vm_mmap(struct file *file, unsigned long addr,
  461. unsigned long len, unsigned long prot,
  462. unsigned long flag, unsigned long offset)
  463. {
  464. if (unlikely(offset + PAGE_ALIGN(len) < offset))
  465. return -EINVAL;
  466. if (unlikely(offset_in_page(offset)))
  467. return -EINVAL;
  468. return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
  469. }
  470. EXPORT_SYMBOL(vm_mmap);
  471. /**
  472. * kvmalloc_node - attempt to allocate physically contiguous memory, but upon
  473. * failure, fall back to non-contiguous (vmalloc) allocation.
  474. * @size: size of the request.
  475. * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
  476. * @node: numa node to allocate from
  477. *
  478. * Uses kmalloc to get the memory but if the allocation fails then falls back
  479. * to the vmalloc allocator. Use kvfree for freeing the memory.
  480. *
  481. * GFP_NOWAIT and GFP_ATOMIC are not supported, neither is the __GFP_NORETRY modifier.
  482. * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
  483. * preferable to the vmalloc fallback, due to visible performance drawbacks.
  484. *
  485. * Return: pointer to the allocated memory of %NULL in case of failure
  486. */
  487. void *kvmalloc_node(size_t size, gfp_t flags, int node)
  488. {
  489. gfp_t kmalloc_flags = flags;
  490. void *ret;
  491. bool use_vmalloc = false;
  492. trace_android_vh_kvmalloc_node_use_vmalloc(size, &kmalloc_flags, &use_vmalloc);
  493. if (use_vmalloc)
  494. goto use_vmalloc_node;
  495. /*
  496. * We want to attempt a large physically contiguous block first because
  497. * it is less likely to fragment multiple larger blocks and therefore
  498. * contribute to a long term fragmentation less than vmalloc fallback.
  499. * However make sure that larger requests are not too disruptive - no
  500. * OOM killer and no allocation failure warnings as we have a fallback.
  501. */
  502. if (size > PAGE_SIZE) {
  503. kmalloc_flags |= __GFP_NOWARN;
  504. if (!(kmalloc_flags & __GFP_RETRY_MAYFAIL))
  505. kmalloc_flags |= __GFP_NORETRY;
  506. /* nofail semantic is implemented by the vmalloc fallback */
  507. kmalloc_flags &= ~__GFP_NOFAIL;
  508. }
  509. trace_android_vh_adjust_kvmalloc_flags(get_order(size), &kmalloc_flags);
  510. ret = kmalloc_node(size, kmalloc_flags, node);
  511. /*
  512. * It doesn't really make sense to fallback to vmalloc for sub page
  513. * requests
  514. */
  515. if (ret || size <= PAGE_SIZE)
  516. return ret;
  517. /* non-sleeping allocations are not supported by vmalloc */
  518. if (!gfpflags_allow_blocking(flags))
  519. return NULL;
  520. /* Don't even allow crazy sizes */
  521. if (unlikely(size > INT_MAX)) {
  522. WARN_ON_ONCE(!(flags & __GFP_NOWARN));
  523. return NULL;
  524. }
  525. /*
  526. * kvmalloc() can always use VM_ALLOW_HUGE_VMAP,
  527. * since the callers already cannot assume anything
  528. * about the resulting pointer, and cannot play
  529. * protection games.
  530. */
  531. use_vmalloc_node:
  532. return __vmalloc_node_range(size, 1, VMALLOC_START, VMALLOC_END,
  533. flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP,
  534. node, __builtin_return_address(0));
  535. }
  536. EXPORT_SYMBOL(kvmalloc_node);
  537. /**
  538. * kvfree() - Free memory.
  539. * @addr: Pointer to allocated memory.
  540. *
  541. * kvfree frees memory allocated by any of vmalloc(), kmalloc() or kvmalloc().
  542. * It is slightly more efficient to use kfree() or vfree() if you are certain
  543. * that you know which one to use.
  544. *
  545. * Context: Either preemptible task context or not-NMI interrupt.
  546. */
  547. void kvfree(const void *addr)
  548. {
  549. if (is_vmalloc_addr(addr))
  550. vfree(addr);
  551. else
  552. kfree(addr);
  553. }
  554. EXPORT_SYMBOL(kvfree);
  555. /**
  556. * kvfree_sensitive - Free a data object containing sensitive information.
  557. * @addr: address of the data object to be freed.
  558. * @len: length of the data object.
  559. *
  560. * Use the special memzero_explicit() function to clear the content of a
  561. * kvmalloc'ed object containing sensitive data to make sure that the
  562. * compiler won't optimize out the data clearing.
  563. */
  564. void kvfree_sensitive(const void *addr, size_t len)
  565. {
  566. if (likely(!ZERO_OR_NULL_PTR(addr))) {
  567. memzero_explicit((void *)addr, len);
  568. kvfree(addr);
  569. }
  570. }
  571. EXPORT_SYMBOL(kvfree_sensitive);
  572. void *kvrealloc(const void *p, size_t oldsize, size_t newsize, gfp_t flags)
  573. {
  574. void *newp;
  575. if (oldsize >= newsize)
  576. return (void *)p;
  577. newp = kvmalloc(newsize, flags);
  578. if (!newp)
  579. return NULL;
  580. memcpy(newp, p, oldsize);
  581. kvfree(p);
  582. return newp;
  583. }
  584. EXPORT_SYMBOL(kvrealloc);
  585. /**
  586. * __vmalloc_array - allocate memory for a virtually contiguous array.
  587. * @n: number of elements.
  588. * @size: element size.
  589. * @flags: the type of memory to allocate (see kmalloc).
  590. */
  591. void *__vmalloc_array(size_t n, size_t size, gfp_t flags)
  592. {
  593. size_t bytes;
  594. if (unlikely(check_mul_overflow(n, size, &bytes)))
  595. return NULL;
  596. return __vmalloc(bytes, flags);
  597. }
  598. EXPORT_SYMBOL(__vmalloc_array);
  599. /**
  600. * vmalloc_array - allocate memory for a virtually contiguous array.
  601. * @n: number of elements.
  602. * @size: element size.
  603. */
  604. void *vmalloc_array(size_t n, size_t size)
  605. {
  606. return __vmalloc_array(n, size, GFP_KERNEL);
  607. }
  608. EXPORT_SYMBOL(vmalloc_array);
  609. /**
  610. * __vcalloc - allocate and zero memory for a virtually contiguous array.
  611. * @n: number of elements.
  612. * @size: element size.
  613. * @flags: the type of memory to allocate (see kmalloc).
  614. */
  615. void *__vcalloc(size_t n, size_t size, gfp_t flags)
  616. {
  617. return __vmalloc_array(n, size, flags | __GFP_ZERO);
  618. }
  619. EXPORT_SYMBOL(__vcalloc);
  620. /**
  621. * vcalloc - allocate and zero memory for a virtually contiguous array.
  622. * @n: number of elements.
  623. * @size: element size.
  624. */
  625. void *vcalloc(size_t n, size_t size)
  626. {
  627. return __vmalloc_array(n, size, GFP_KERNEL | __GFP_ZERO);
  628. }
  629. EXPORT_SYMBOL(vcalloc);
  630. /* Neutral page->mapping pointer to address_space or anon_vma or other */
  631. void *page_rmapping(struct page *page)
  632. {
  633. return folio_raw_mapping(page_folio(page));
  634. }
  635. /**
  636. * folio_mapped - Is this folio mapped into userspace?
  637. * @folio: The folio.
  638. *
  639. * Return: True if any page in this folio is referenced by user page tables.
  640. */
  641. bool folio_mapped(struct folio *folio)
  642. {
  643. long i, nr;
  644. if (!folio_test_large(folio))
  645. return atomic_read(&folio->_mapcount) >= 0;
  646. if (atomic_read(folio_mapcount_ptr(folio)) >= 0)
  647. return true;
  648. if (folio_test_hugetlb(folio))
  649. return false;
  650. nr = folio_nr_pages(folio);
  651. for (i = 0; i < nr; i++) {
  652. if (atomic_read(&folio_page(folio, i)->_mapcount) >= 0)
  653. return true;
  654. }
  655. return false;
  656. }
  657. EXPORT_SYMBOL(folio_mapped);
  658. struct anon_vma *folio_anon_vma(struct folio *folio)
  659. {
  660. unsigned long mapping = (unsigned long)folio->mapping;
  661. if ((mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
  662. return NULL;
  663. return (void *)(mapping - PAGE_MAPPING_ANON);
  664. }
  665. /**
  666. * folio_mapping - Find the mapping where this folio is stored.
  667. * @folio: The folio.
  668. *
  669. * For folios which are in the page cache, return the mapping that this
  670. * page belongs to. Folios in the swap cache return the swap mapping
  671. * this page is stored in (which is different from the mapping for the
  672. * swap file or swap device where the data is stored).
  673. *
  674. * You can call this for folios which aren't in the swap cache or page
  675. * cache and it will return NULL.
  676. */
  677. struct address_space *folio_mapping(struct folio *folio)
  678. {
  679. struct address_space *mapping;
  680. /* This happens if someone calls flush_dcache_page on slab page */
  681. if (unlikely(folio_test_slab(folio)))
  682. return NULL;
  683. if (unlikely(folio_test_swapcache(folio)))
  684. return swap_address_space(folio_swap_entry(folio));
  685. mapping = folio->mapping;
  686. if ((unsigned long)mapping & PAGE_MAPPING_FLAGS)
  687. return NULL;
  688. return mapping;
  689. }
  690. EXPORT_SYMBOL(folio_mapping);
  691. /* Slow path of page_mapcount() for compound pages */
  692. int __page_mapcount(struct page *page)
  693. {
  694. int ret;
  695. ret = atomic_read(&page->_mapcount) + 1;
  696. /*
  697. * For file THP page->_mapcount contains total number of mapping
  698. * of the page: no need to look into compound_mapcount.
  699. */
  700. if (!PageAnon(page) && !PageHuge(page))
  701. return ret;
  702. page = compound_head(page);
  703. ret += atomic_read(compound_mapcount_ptr(page)) + 1;
  704. if (PageDoubleMap(page))
  705. ret--;
  706. return ret;
  707. }
  708. EXPORT_SYMBOL_GPL(__page_mapcount);
  709. /**
  710. * folio_mapcount() - Calculate the number of mappings of this folio.
  711. * @folio: The folio.
  712. *
  713. * A large folio tracks both how many times the entire folio is mapped,
  714. * and how many times each individual page in the folio is mapped.
  715. * This function calculates the total number of times the folio is
  716. * mapped.
  717. *
  718. * Return: The number of times this folio is mapped.
  719. */
  720. int folio_mapcount(struct folio *folio)
  721. {
  722. int i, compound, nr, ret;
  723. if (likely(!folio_test_large(folio)))
  724. return atomic_read(&folio->_mapcount) + 1;
  725. compound = folio_entire_mapcount(folio);
  726. if (folio_test_hugetlb(folio))
  727. return compound;
  728. ret = compound;
  729. nr = folio_nr_pages(folio);
  730. for (i = 0; i < nr; i++)
  731. ret += atomic_read(&folio_page(folio, i)->_mapcount) + 1;
  732. /* File pages has compound_mapcount included in _mapcount */
  733. if (!folio_test_anon(folio))
  734. return ret - compound * nr;
  735. if (folio_test_double_map(folio))
  736. ret -= nr;
  737. return ret;
  738. }
  739. /**
  740. * folio_copy - Copy the contents of one folio to another.
  741. * @dst: Folio to copy to.
  742. * @src: Folio to copy from.
  743. *
  744. * The bytes in the folio represented by @src are copied to @dst.
  745. * Assumes the caller has validated that @dst is at least as large as @src.
  746. * Can be called in atomic context for order-0 folios, but if the folio is
  747. * larger, it may sleep.
  748. */
  749. void folio_copy(struct folio *dst, struct folio *src)
  750. {
  751. long i = 0;
  752. long nr = folio_nr_pages(src);
  753. for (;;) {
  754. copy_highpage(folio_page(dst, i), folio_page(src, i));
  755. if (++i == nr)
  756. break;
  757. cond_resched();
  758. }
  759. }
  760. int sysctl_overcommit_memory __read_mostly = OVERCOMMIT_GUESS;
  761. int sysctl_overcommit_ratio __read_mostly = 50;
  762. unsigned long sysctl_overcommit_kbytes __read_mostly;
  763. int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
  764. unsigned long sysctl_user_reserve_kbytes __read_mostly = 1UL << 17; /* 128MB */
  765. unsigned long sysctl_admin_reserve_kbytes __read_mostly = 1UL << 13; /* 8MB */
  766. int overcommit_ratio_handler(struct ctl_table *table, int write, void *buffer,
  767. size_t *lenp, loff_t *ppos)
  768. {
  769. int ret;
  770. ret = proc_dointvec(table, write, buffer, lenp, ppos);
  771. if (ret == 0 && write)
  772. sysctl_overcommit_kbytes = 0;
  773. return ret;
  774. }
  775. static void sync_overcommit_as(struct work_struct *dummy)
  776. {
  777. percpu_counter_sync(&vm_committed_as);
  778. }
  779. int overcommit_policy_handler(struct ctl_table *table, int write, void *buffer,
  780. size_t *lenp, loff_t *ppos)
  781. {
  782. struct ctl_table t;
  783. int new_policy = -1;
  784. int ret;
  785. /*
  786. * The deviation of sync_overcommit_as could be big with loose policy
  787. * like OVERCOMMIT_ALWAYS/OVERCOMMIT_GUESS. When changing policy to
  788. * strict OVERCOMMIT_NEVER, we need to reduce the deviation to comply
  789. * with the strict "NEVER", and to avoid possible race condition (even
  790. * though user usually won't too frequently do the switching to policy
  791. * OVERCOMMIT_NEVER), the switch is done in the following order:
  792. * 1. changing the batch
  793. * 2. sync percpu count on each CPU
  794. * 3. switch the policy
  795. */
  796. if (write) {
  797. t = *table;
  798. t.data = &new_policy;
  799. ret = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
  800. if (ret || new_policy == -1)
  801. return ret;
  802. mm_compute_batch(new_policy);
  803. if (new_policy == OVERCOMMIT_NEVER)
  804. schedule_on_each_cpu(sync_overcommit_as);
  805. sysctl_overcommit_memory = new_policy;
  806. } else {
  807. ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  808. }
  809. return ret;
  810. }
  811. int overcommit_kbytes_handler(struct ctl_table *table, int write, void *buffer,
  812. size_t *lenp, loff_t *ppos)
  813. {
  814. int ret;
  815. ret = proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
  816. if (ret == 0 && write)
  817. sysctl_overcommit_ratio = 0;
  818. return ret;
  819. }
  820. /*
  821. * Committed memory limit enforced when OVERCOMMIT_NEVER policy is used
  822. */
  823. unsigned long vm_commit_limit(void)
  824. {
  825. unsigned long allowed;
  826. if (sysctl_overcommit_kbytes)
  827. allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10);
  828. else
  829. allowed = ((totalram_pages() - hugetlb_total_pages())
  830. * sysctl_overcommit_ratio / 100);
  831. allowed += total_swap_pages;
  832. return allowed;
  833. }
  834. /*
  835. * Make sure vm_committed_as in one cacheline and not cacheline shared with
  836. * other variables. It can be updated by several CPUs frequently.
  837. */
  838. struct percpu_counter vm_committed_as ____cacheline_aligned_in_smp;
  839. /*
  840. * The global memory commitment made in the system can be a metric
  841. * that can be used to drive ballooning decisions when Linux is hosted
  842. * as a guest. On Hyper-V, the host implements a policy engine for dynamically
  843. * balancing memory across competing virtual machines that are hosted.
  844. * Several metrics drive this policy engine including the guest reported
  845. * memory commitment.
  846. *
  847. * The time cost of this is very low for small platforms, and for big
  848. * platform like a 2S/36C/72T Skylake server, in worst case where
  849. * vm_committed_as's spinlock is under severe contention, the time cost
  850. * could be about 30~40 microseconds.
  851. */
  852. unsigned long vm_memory_committed(void)
  853. {
  854. return percpu_counter_sum_positive(&vm_committed_as);
  855. }
  856. EXPORT_SYMBOL_GPL(vm_memory_committed);
  857. /*
  858. * Check that a process has enough memory to allocate a new virtual
  859. * mapping. 0 means there is enough memory for the allocation to
  860. * succeed and -ENOMEM implies there is not.
  861. *
  862. * We currently support three overcommit policies, which are set via the
  863. * vm.overcommit_memory sysctl. See Documentation/mm/overcommit-accounting.rst
  864. *
  865. * Strict overcommit modes added 2002 Feb 26 by Alan Cox.
  866. * Additional code 2002 Jul 20 by Robert Love.
  867. *
  868. * cap_sys_admin is 1 if the process has admin privileges, 0 otherwise.
  869. *
  870. * Note this is a helper function intended to be used by LSMs which
  871. * wish to use this logic.
  872. */
  873. int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
  874. {
  875. long allowed;
  876. vm_acct_memory(pages);
  877. /*
  878. * Sometimes we want to use more memory than we have
  879. */
  880. if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
  881. return 0;
  882. if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
  883. if (pages > totalram_pages() + total_swap_pages)
  884. goto error;
  885. return 0;
  886. }
  887. allowed = vm_commit_limit();
  888. /*
  889. * Reserve some for root
  890. */
  891. if (!cap_sys_admin)
  892. allowed -= sysctl_admin_reserve_kbytes >> (PAGE_SHIFT - 10);
  893. /*
  894. * Don't let a single process grow so big a user can't recover
  895. */
  896. if (mm) {
  897. long reserve = sysctl_user_reserve_kbytes >> (PAGE_SHIFT - 10);
  898. allowed -= min_t(long, mm->total_vm / 32, reserve);
  899. }
  900. if (percpu_counter_read_positive(&vm_committed_as) < allowed)
  901. return 0;
  902. error:
  903. pr_warn_ratelimited("%s: pid: %d, comm: %s, no enough memory for the allocation\n",
  904. __func__, current->pid, current->comm);
  905. vm_unacct_memory(pages);
  906. return -ENOMEM;
  907. }
  908. /**
  909. * get_cmdline() - copy the cmdline value to a buffer.
  910. * @task: the task whose cmdline value to copy.
  911. * @buffer: the buffer to copy to.
  912. * @buflen: the length of the buffer. Larger cmdline values are truncated
  913. * to this length.
  914. *
  915. * Return: the size of the cmdline field copied. Note that the copy does
  916. * not guarantee an ending NULL byte.
  917. */
  918. int get_cmdline(struct task_struct *task, char *buffer, int buflen)
  919. {
  920. int res = 0;
  921. unsigned int len;
  922. struct mm_struct *mm = get_task_mm(task);
  923. unsigned long arg_start, arg_end, env_start, env_end;
  924. if (!mm)
  925. goto out;
  926. if (!mm->arg_end)
  927. goto out_mm; /* Shh! No looking before we're done */
  928. spin_lock(&mm->arg_lock);
  929. arg_start = mm->arg_start;
  930. arg_end = mm->arg_end;
  931. env_start = mm->env_start;
  932. env_end = mm->env_end;
  933. spin_unlock(&mm->arg_lock);
  934. len = arg_end - arg_start;
  935. if (len > buflen)
  936. len = buflen;
  937. res = access_process_vm(task, arg_start, buffer, len, FOLL_FORCE);
  938. /*
  939. * If the nul at the end of args has been overwritten, then
  940. * assume application is using setproctitle(3).
  941. */
  942. if (res > 0 && buffer[res-1] != '\0' && len < buflen) {
  943. len = strnlen(buffer, res);
  944. if (len < res) {
  945. res = len;
  946. } else {
  947. len = env_end - env_start;
  948. if (len > buflen - res)
  949. len = buflen - res;
  950. res += access_process_vm(task, env_start,
  951. buffer+res, len,
  952. FOLL_FORCE);
  953. res = strnlen(buffer, res);
  954. }
  955. }
  956. out_mm:
  957. mmput(mm);
  958. out:
  959. return res;
  960. }
  961. int __weak memcmp_pages(struct page *page1, struct page *page2)
  962. {
  963. char *addr1, *addr2;
  964. int ret;
  965. addr1 = kmap_atomic(page1);
  966. addr2 = kmap_atomic(page2);
  967. ret = memcmp(addr1, addr2, PAGE_SIZE);
  968. kunmap_atomic(addr2);
  969. kunmap_atomic(addr1);
  970. return ret;
  971. }
  972. #ifdef CONFIG_PRINTK
  973. /**
  974. * mem_dump_obj - Print available provenance information
  975. * @object: object for which to find provenance information.
  976. *
  977. * This function uses pr_cont(), so that the caller is expected to have
  978. * printed out whatever preamble is appropriate. The provenance information
  979. * depends on the type of object and on how much debugging is enabled.
  980. * For example, for a slab-cache object, the slab name is printed, and,
  981. * if available, the return address and stack trace from the allocation
  982. * and last free path of that object.
  983. */
  984. void mem_dump_obj(void *object)
  985. {
  986. const char *type;
  987. if (kmem_valid_obj(object)) {
  988. kmem_dump_obj(object);
  989. return;
  990. }
  991. if (vmalloc_dump_obj(object))
  992. return;
  993. if (is_vmalloc_addr(object))
  994. type = "vmalloc memory";
  995. else if (virt_addr_valid(object))
  996. type = "non-slab/vmalloc memory";
  997. else if (object == NULL)
  998. type = "NULL pointer";
  999. else if (object == ZERO_SIZE_PTR)
  1000. type = "zero-size pointer";
  1001. else
  1002. type = "non-paged memory";
  1003. pr_cont(" %s\n", type);
  1004. }
  1005. EXPORT_SYMBOL_GPL(mem_dump_obj);
  1006. #endif
  1007. /*
  1008. * A driver might set a page logically offline -- PageOffline() -- and
  1009. * turn the page inaccessible in the hypervisor; after that, access to page
  1010. * content can be fatal.
  1011. *
  1012. * Some special PFN walkers -- i.e., /proc/kcore -- read content of random
  1013. * pages after checking PageOffline(); however, these PFN walkers can race
  1014. * with drivers that set PageOffline().
  1015. *
  1016. * page_offline_freeze()/page_offline_thaw() allows for a subsystem to
  1017. * synchronize with such drivers, achieving that a page cannot be set
  1018. * PageOffline() while frozen.
  1019. *
  1020. * page_offline_begin()/page_offline_end() is used by drivers that care about
  1021. * such races when setting a page PageOffline().
  1022. */
  1023. static DECLARE_RWSEM(page_offline_rwsem);
  1024. void page_offline_freeze(void)
  1025. {
  1026. down_read(&page_offline_rwsem);
  1027. }
  1028. void page_offline_thaw(void)
  1029. {
  1030. up_read(&page_offline_rwsem);
  1031. }
  1032. void page_offline_begin(void)
  1033. {
  1034. down_write(&page_offline_rwsem);
  1035. }
  1036. EXPORT_SYMBOL(page_offline_begin);
  1037. void page_offline_end(void)
  1038. {
  1039. up_write(&page_offline_rwsem);
  1040. }
  1041. EXPORT_SYMBOL(page_offline_end);
  1042. #ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_FOLIO
  1043. void flush_dcache_folio(struct folio *folio)
  1044. {
  1045. long i, nr = folio_nr_pages(folio);
  1046. for (i = 0; i < nr; i++)
  1047. flush_dcache_page(folio_page(folio, i));
  1048. }
  1049. EXPORT_SYMBOL(flush_dcache_folio);
  1050. #endif