hugetlbpage.c 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * IBM System z Huge TLB Page Support for Kernel.
  4. *
  5. * Copyright IBM Corp. 2007,2020
  6. * Author(s): Gerald Schaefer <[email protected]>
  7. */
  8. #define KMSG_COMPONENT "hugetlb"
  9. #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10. #include <asm/pgalloc.h>
  11. #include <linux/mm.h>
  12. #include <linux/hugetlb.h>
  13. #include <linux/mman.h>
  14. #include <linux/sched/mm.h>
  15. #include <linux/security.h>
  16. /*
  17. * If the bit selected by single-bit bitmask "a" is set within "x", move
  18. * it to the position indicated by single-bit bitmask "b".
  19. */
  20. #define move_set_bit(x, a, b) (((x) & (a)) >> ilog2(a) << ilog2(b))
  21. static inline unsigned long __pte_to_rste(pte_t pte)
  22. {
  23. unsigned long rste;
  24. /*
  25. * Convert encoding pte bits pmd / pud bits
  26. * lIR.uswrdy.p dy..R...I...wr
  27. * empty 010.000000.0 -> 00..0...1...00
  28. * prot-none, clean, old 111.000000.1 -> 00..1...1...00
  29. * prot-none, clean, young 111.000001.1 -> 01..1...1...00
  30. * prot-none, dirty, old 111.000010.1 -> 10..1...1...00
  31. * prot-none, dirty, young 111.000011.1 -> 11..1...1...00
  32. * read-only, clean, old 111.000100.1 -> 00..1...1...01
  33. * read-only, clean, young 101.000101.1 -> 01..1...0...01
  34. * read-only, dirty, old 111.000110.1 -> 10..1...1...01
  35. * read-only, dirty, young 101.000111.1 -> 11..1...0...01
  36. * read-write, clean, old 111.001100.1 -> 00..1...1...11
  37. * read-write, clean, young 101.001101.1 -> 01..1...0...11
  38. * read-write, dirty, old 110.001110.1 -> 10..0...1...11
  39. * read-write, dirty, young 100.001111.1 -> 11..0...0...11
  40. * HW-bits: R read-only, I invalid
  41. * SW-bits: p present, y young, d dirty, r read, w write, s special,
  42. * u unused, l large
  43. */
  44. if (pte_present(pte)) {
  45. rste = pte_val(pte) & PAGE_MASK;
  46. rste |= move_set_bit(pte_val(pte), _PAGE_READ,
  47. _SEGMENT_ENTRY_READ);
  48. rste |= move_set_bit(pte_val(pte), _PAGE_WRITE,
  49. _SEGMENT_ENTRY_WRITE);
  50. rste |= move_set_bit(pte_val(pte), _PAGE_INVALID,
  51. _SEGMENT_ENTRY_INVALID);
  52. rste |= move_set_bit(pte_val(pte), _PAGE_PROTECT,
  53. _SEGMENT_ENTRY_PROTECT);
  54. rste |= move_set_bit(pte_val(pte), _PAGE_DIRTY,
  55. _SEGMENT_ENTRY_DIRTY);
  56. rste |= move_set_bit(pte_val(pte), _PAGE_YOUNG,
  57. _SEGMENT_ENTRY_YOUNG);
  58. #ifdef CONFIG_MEM_SOFT_DIRTY
  59. rste |= move_set_bit(pte_val(pte), _PAGE_SOFT_DIRTY,
  60. _SEGMENT_ENTRY_SOFT_DIRTY);
  61. #endif
  62. rste |= move_set_bit(pte_val(pte), _PAGE_NOEXEC,
  63. _SEGMENT_ENTRY_NOEXEC);
  64. } else
  65. rste = _SEGMENT_ENTRY_EMPTY;
  66. return rste;
  67. }
  68. static inline pte_t __rste_to_pte(unsigned long rste)
  69. {
  70. unsigned long pteval;
  71. int present;
  72. if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
  73. present = pud_present(__pud(rste));
  74. else
  75. present = pmd_present(__pmd(rste));
  76. /*
  77. * Convert encoding pmd / pud bits pte bits
  78. * dy..R...I...wr lIR.uswrdy.p
  79. * empty 00..0...1...00 -> 010.000000.0
  80. * prot-none, clean, old 00..1...1...00 -> 111.000000.1
  81. * prot-none, clean, young 01..1...1...00 -> 111.000001.1
  82. * prot-none, dirty, old 10..1...1...00 -> 111.000010.1
  83. * prot-none, dirty, young 11..1...1...00 -> 111.000011.1
  84. * read-only, clean, old 00..1...1...01 -> 111.000100.1
  85. * read-only, clean, young 01..1...0...01 -> 101.000101.1
  86. * read-only, dirty, old 10..1...1...01 -> 111.000110.1
  87. * read-only, dirty, young 11..1...0...01 -> 101.000111.1
  88. * read-write, clean, old 00..1...1...11 -> 111.001100.1
  89. * read-write, clean, young 01..1...0...11 -> 101.001101.1
  90. * read-write, dirty, old 10..0...1...11 -> 110.001110.1
  91. * read-write, dirty, young 11..0...0...11 -> 100.001111.1
  92. * HW-bits: R read-only, I invalid
  93. * SW-bits: p present, y young, d dirty, r read, w write, s special,
  94. * u unused, l large
  95. */
  96. if (present) {
  97. pteval = rste & _SEGMENT_ENTRY_ORIGIN_LARGE;
  98. pteval |= _PAGE_LARGE | _PAGE_PRESENT;
  99. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_READ, _PAGE_READ);
  100. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_WRITE, _PAGE_WRITE);
  101. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_INVALID, _PAGE_INVALID);
  102. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_PROTECT, _PAGE_PROTECT);
  103. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_DIRTY, _PAGE_DIRTY);
  104. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_YOUNG, _PAGE_YOUNG);
  105. #ifdef CONFIG_MEM_SOFT_DIRTY
  106. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_SOFT_DIRTY, _PAGE_SOFT_DIRTY);
  107. #endif
  108. pteval |= move_set_bit(rste, _SEGMENT_ENTRY_NOEXEC, _PAGE_NOEXEC);
  109. } else
  110. pteval = _PAGE_INVALID;
  111. return __pte(pteval);
  112. }
  113. static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
  114. {
  115. struct page *page;
  116. unsigned long size, paddr;
  117. if (!mm_uses_skeys(mm) ||
  118. rste & _SEGMENT_ENTRY_INVALID)
  119. return;
  120. if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
  121. page = pud_page(__pud(rste));
  122. size = PUD_SIZE;
  123. paddr = rste & PUD_MASK;
  124. } else {
  125. page = pmd_page(__pmd(rste));
  126. size = PMD_SIZE;
  127. paddr = rste & PMD_MASK;
  128. }
  129. if (!test_and_set_bit(PG_arch_1, &page->flags))
  130. __storage_key_init_range(paddr, paddr + size - 1);
  131. }
  132. void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  133. pte_t *ptep, pte_t pte)
  134. {
  135. unsigned long rste;
  136. rste = __pte_to_rste(pte);
  137. if (!MACHINE_HAS_NX)
  138. rste &= ~_SEGMENT_ENTRY_NOEXEC;
  139. /* Set correct table type for 2G hugepages */
  140. if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
  141. if (likely(pte_present(pte)))
  142. rste |= _REGION3_ENTRY_LARGE;
  143. rste |= _REGION_ENTRY_TYPE_R3;
  144. } else if (likely(pte_present(pte)))
  145. rste |= _SEGMENT_ENTRY_LARGE;
  146. clear_huge_pte_skeys(mm, rste);
  147. set_pte(ptep, __pte(rste));
  148. }
  149. pte_t huge_ptep_get(pte_t *ptep)
  150. {
  151. return __rste_to_pte(pte_val(*ptep));
  152. }
  153. pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
  154. unsigned long addr, pte_t *ptep)
  155. {
  156. pte_t pte = huge_ptep_get(ptep);
  157. pmd_t *pmdp = (pmd_t *) ptep;
  158. pud_t *pudp = (pud_t *) ptep;
  159. if ((pte_val(*ptep) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
  160. pudp_xchg_direct(mm, addr, pudp, __pud(_REGION3_ENTRY_EMPTY));
  161. else
  162. pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_EMPTY));
  163. return pte;
  164. }
  165. pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
  166. unsigned long addr, unsigned long sz)
  167. {
  168. pgd_t *pgdp;
  169. p4d_t *p4dp;
  170. pud_t *pudp;
  171. pmd_t *pmdp = NULL;
  172. pgdp = pgd_offset(mm, addr);
  173. p4dp = p4d_alloc(mm, pgdp, addr);
  174. if (p4dp) {
  175. pudp = pud_alloc(mm, p4dp, addr);
  176. if (pudp) {
  177. if (sz == PUD_SIZE)
  178. return (pte_t *) pudp;
  179. else if (sz == PMD_SIZE)
  180. pmdp = pmd_alloc(mm, pudp, addr);
  181. }
  182. }
  183. return (pte_t *) pmdp;
  184. }
  185. pte_t *huge_pte_offset(struct mm_struct *mm,
  186. unsigned long addr, unsigned long sz)
  187. {
  188. pgd_t *pgdp;
  189. p4d_t *p4dp;
  190. pud_t *pudp;
  191. pmd_t *pmdp = NULL;
  192. pgdp = pgd_offset(mm, addr);
  193. if (pgd_present(*pgdp)) {
  194. p4dp = p4d_offset(pgdp, addr);
  195. if (p4d_present(*p4dp)) {
  196. pudp = pud_offset(p4dp, addr);
  197. if (pud_present(*pudp)) {
  198. if (pud_large(*pudp))
  199. return (pte_t *) pudp;
  200. pmdp = pmd_offset(pudp, addr);
  201. }
  202. }
  203. }
  204. return (pte_t *) pmdp;
  205. }
  206. int pmd_huge(pmd_t pmd)
  207. {
  208. return pmd_large(pmd);
  209. }
  210. int pud_huge(pud_t pud)
  211. {
  212. return pud_large(pud);
  213. }
  214. bool __init arch_hugetlb_valid_size(unsigned long size)
  215. {
  216. if (MACHINE_HAS_EDAT1 && size == PMD_SIZE)
  217. return true;
  218. else if (MACHINE_HAS_EDAT2 && size == PUD_SIZE)
  219. return true;
  220. else
  221. return false;
  222. }
  223. static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
  224. unsigned long addr, unsigned long len,
  225. unsigned long pgoff, unsigned long flags)
  226. {
  227. struct hstate *h = hstate_file(file);
  228. struct vm_unmapped_area_info info;
  229. info.flags = 0;
  230. info.length = len;
  231. info.low_limit = current->mm->mmap_base;
  232. info.high_limit = TASK_SIZE;
  233. info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  234. info.align_offset = 0;
  235. return vm_unmapped_area(&info);
  236. }
  237. static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
  238. unsigned long addr0, unsigned long len,
  239. unsigned long pgoff, unsigned long flags)
  240. {
  241. struct hstate *h = hstate_file(file);
  242. struct vm_unmapped_area_info info;
  243. unsigned long addr;
  244. info.flags = VM_UNMAPPED_AREA_TOPDOWN;
  245. info.length = len;
  246. info.low_limit = max(PAGE_SIZE, mmap_min_addr);
  247. info.high_limit = current->mm->mmap_base;
  248. info.align_mask = PAGE_MASK & ~huge_page_mask(h);
  249. info.align_offset = 0;
  250. addr = vm_unmapped_area(&info);
  251. /*
  252. * A failed mmap() very likely causes application failure,
  253. * so fall back to the bottom-up function here. This scenario
  254. * can happen with large stack limits and large mmap()
  255. * allocations.
  256. */
  257. if (addr & ~PAGE_MASK) {
  258. VM_BUG_ON(addr != -ENOMEM);
  259. info.flags = 0;
  260. info.low_limit = TASK_UNMAPPED_BASE;
  261. info.high_limit = TASK_SIZE;
  262. addr = vm_unmapped_area(&info);
  263. }
  264. return addr;
  265. }
  266. unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
  267. unsigned long len, unsigned long pgoff, unsigned long flags)
  268. {
  269. struct hstate *h = hstate_file(file);
  270. struct mm_struct *mm = current->mm;
  271. struct vm_area_struct *vma;
  272. if (len & ~huge_page_mask(h))
  273. return -EINVAL;
  274. if (len > TASK_SIZE - mmap_min_addr)
  275. return -ENOMEM;
  276. if (flags & MAP_FIXED) {
  277. if (prepare_hugepage_range(file, addr, len))
  278. return -EINVAL;
  279. goto check_asce_limit;
  280. }
  281. if (addr) {
  282. addr = ALIGN(addr, huge_page_size(h));
  283. vma = find_vma(mm, addr);
  284. if (TASK_SIZE - len >= addr && addr >= mmap_min_addr &&
  285. (!vma || addr + len <= vm_start_gap(vma)))
  286. goto check_asce_limit;
  287. }
  288. if (mm->get_unmapped_area == arch_get_unmapped_area)
  289. addr = hugetlb_get_unmapped_area_bottomup(file, addr, len,
  290. pgoff, flags);
  291. else
  292. addr = hugetlb_get_unmapped_area_topdown(file, addr, len,
  293. pgoff, flags);
  294. if (offset_in_page(addr))
  295. return addr;
  296. check_asce_limit:
  297. return check_asce_limit(mm, addr, len);
  298. }