memblock.c 61 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Procedures for maintaining information about logical memory blocks.
  4. *
  5. * Peter Bergner, IBM Corp. June 2001.
  6. * Copyright (C) 2001 Peter Bergner.
  7. */
  8. #include <linux/kernel.h>
  9. #include <linux/slab.h>
  10. #include <linux/init.h>
  11. #include <linux/bitops.h>
  12. #include <linux/poison.h>
  13. #include <linux/pfn.h>
  14. #include <linux/debugfs.h>
  15. #include <linux/kmemleak.h>
  16. #include <linux/seq_file.h>
  17. #include <linux/memblock.h>
  18. #include <asm/sections.h>
  19. #include <linux/io.h>
  20. #include "internal.h"
  21. #define INIT_MEMBLOCK_REGIONS 128
  22. #define INIT_PHYSMEM_REGIONS 4
  23. #ifndef INIT_MEMBLOCK_RESERVED_REGIONS
  24. # define INIT_MEMBLOCK_RESERVED_REGIONS INIT_MEMBLOCK_REGIONS
  25. #endif
  26. #ifndef INIT_MEMBLOCK_MEMORY_REGIONS
  27. #define INIT_MEMBLOCK_MEMORY_REGIONS INIT_MEMBLOCK_REGIONS
  28. #endif
  29. /**
  30. * DOC: memblock overview
  31. *
  32. * Memblock is a method of managing memory regions during the early
  33. * boot period when the usual kernel memory allocators are not up and
  34. * running.
  35. *
  36. * Memblock views the system memory as collections of contiguous
  37. * regions. There are several types of these collections:
  38. *
  39. * * ``memory`` - describes the physical memory available to the
  40. * kernel; this may differ from the actual physical memory installed
  41. * in the system, for instance when the memory is restricted with
  42. * ``mem=`` command line parameter
  43. * * ``reserved`` - describes the regions that were allocated
  44. * * ``physmem`` - describes the actual physical memory available during
  45. * boot regardless of the possible restrictions and memory hot(un)plug;
  46. * the ``physmem`` type is only available on some architectures.
  47. *
  48. * Each region is represented by struct memblock_region that
  49. * defines the region extents, its attributes and NUMA node id on NUMA
  50. * systems. Every memory type is described by the struct memblock_type
  51. * which contains an array of memory regions along with
  52. * the allocator metadata. The "memory" and "reserved" types are nicely
  53. * wrapped with struct memblock. This structure is statically
  54. * initialized at build time. The region arrays are initially sized to
  55. * %INIT_MEMBLOCK_MEMORY_REGIONS for "memory" and
  56. * %INIT_MEMBLOCK_RESERVED_REGIONS for "reserved". The region array
  57. * for "physmem" is initially sized to %INIT_PHYSMEM_REGIONS.
  58. * The memblock_allow_resize() enables automatic resizing of the region
  59. * arrays during addition of new regions. This feature should be used
  60. * with care so that memory allocated for the region array will not
  61. * overlap with areas that should be reserved, for example initrd.
  62. *
  63. * The early architecture setup should tell memblock what the physical
  64. * memory layout is by using memblock_add() or memblock_add_node()
  65. * functions. The first function does not assign the region to a NUMA
  66. * node and it is appropriate for UMA systems. Yet, it is possible to
  67. * use it on NUMA systems as well and assign the region to a NUMA node
  68. * later in the setup process using memblock_set_node(). The
  69. * memblock_add_node() performs such an assignment directly.
  70. *
  71. * Once memblock is setup the memory can be allocated using one of the
  72. * API variants:
  73. *
  74. * * memblock_phys_alloc*() - these functions return the **physical**
  75. * address of the allocated memory
  76. * * memblock_alloc*() - these functions return the **virtual** address
  77. * of the allocated memory.
  78. *
  79. * Note, that both API variants use implicit assumptions about allowed
  80. * memory ranges and the fallback methods. Consult the documentation
  81. * of memblock_alloc_internal() and memblock_alloc_range_nid()
  82. * functions for more elaborate description.
  83. *
  84. * As the system boot progresses, the architecture specific mem_init()
  85. * function frees all the memory to the buddy page allocator.
  86. *
  87. * Unless an architecture enables %CONFIG_ARCH_KEEP_MEMBLOCK, the
  88. * memblock data structures (except "physmem") will be discarded after the
  89. * system initialization completes.
  90. */
  91. #ifndef CONFIG_NUMA
  92. struct pglist_data __refdata contig_page_data;
  93. EXPORT_SYMBOL(contig_page_data);
  94. #endif
  95. unsigned long max_low_pfn;
  96. unsigned long min_low_pfn;
  97. unsigned long max_pfn;
  98. unsigned long long max_possible_pfn;
  99. static struct memblock_region memblock_memory_init_regions[INIT_MEMBLOCK_MEMORY_REGIONS] __initdata_memblock;
  100. static struct memblock_region memblock_reserved_init_regions[INIT_MEMBLOCK_RESERVED_REGIONS] __initdata_memblock;
  101. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  102. static struct memblock_region memblock_physmem_init_regions[INIT_PHYSMEM_REGIONS];
  103. #endif
  104. struct memblock memblock __initdata_memblock = {
  105. .memory.regions = memblock_memory_init_regions,
  106. .memory.cnt = 1, /* empty dummy entry */
  107. .memory.max = INIT_MEMBLOCK_MEMORY_REGIONS,
  108. .memory.name = "memory",
  109. .reserved.regions = memblock_reserved_init_regions,
  110. .reserved.cnt = 1, /* empty dummy entry */
  111. .reserved.max = INIT_MEMBLOCK_RESERVED_REGIONS,
  112. .reserved.name = "reserved",
  113. .bottom_up = false,
  114. .current_limit = MEMBLOCK_ALLOC_ANYWHERE,
  115. };
  116. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  117. struct memblock_type physmem = {
  118. .regions = memblock_physmem_init_regions,
  119. .cnt = 1, /* empty dummy entry */
  120. .max = INIT_PHYSMEM_REGIONS,
  121. .name = "physmem",
  122. };
  123. #endif
  124. /*
  125. * keep a pointer to &memblock.memory in the text section to use it in
  126. * __next_mem_range() and its helpers.
  127. * For architectures that do not keep memblock data after init, this
  128. * pointer will be reset to NULL at memblock_discard()
  129. */
  130. static __refdata struct memblock_type *memblock_memory = &memblock.memory;
  131. #define for_each_memblock_type(i, memblock_type, rgn) \
  132. for (i = 0, rgn = &memblock_type->regions[0]; \
  133. i < memblock_type->cnt; \
  134. i++, rgn = &memblock_type->regions[i])
  135. #define memblock_dbg(fmt, ...) \
  136. do { \
  137. if (memblock_debug) \
  138. pr_info(fmt, ##__VA_ARGS__); \
  139. } while (0)
  140. static int memblock_debug __initdata_memblock;
  141. static bool system_has_some_mirror __initdata_memblock = false;
  142. static int memblock_can_resize __initdata_memblock;
  143. static int memblock_memory_in_slab __initdata_memblock = 0;
  144. static int memblock_reserved_in_slab __initdata_memblock = 0;
  145. static enum memblock_flags __init_memblock choose_memblock_flags(void)
  146. {
  147. return system_has_some_mirror ? MEMBLOCK_MIRROR : MEMBLOCK_NONE;
  148. }
  149. /* adjust *@size so that (@base + *@size) doesn't overflow, return new size */
  150. static inline phys_addr_t memblock_cap_size(phys_addr_t base, phys_addr_t *size)
  151. {
  152. return *size = min(*size, PHYS_ADDR_MAX - base);
  153. }
  154. /*
  155. * Address comparison utilities
  156. */
  157. static unsigned long __init_memblock memblock_addrs_overlap(phys_addr_t base1, phys_addr_t size1,
  158. phys_addr_t base2, phys_addr_t size2)
  159. {
  160. return ((base1 < (base2 + size2)) && (base2 < (base1 + size1)));
  161. }
  162. bool __init_memblock memblock_overlaps_region(struct memblock_type *type,
  163. phys_addr_t base, phys_addr_t size)
  164. {
  165. unsigned long i;
  166. memblock_cap_size(base, &size);
  167. for (i = 0; i < type->cnt; i++)
  168. if (memblock_addrs_overlap(base, size, type->regions[i].base,
  169. type->regions[i].size))
  170. break;
  171. return i < type->cnt;
  172. }
  173. /**
  174. * __memblock_find_range_bottom_up - find free area utility in bottom-up
  175. * @start: start of candidate range
  176. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  177. * %MEMBLOCK_ALLOC_ACCESSIBLE
  178. * @size: size of free area to find
  179. * @align: alignment of free area to find
  180. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  181. * @flags: pick from blocks based on memory attributes
  182. *
  183. * Utility called from memblock_find_in_range_node(), find free area bottom-up.
  184. *
  185. * Return:
  186. * Found address on success, 0 on failure.
  187. */
  188. static phys_addr_t __init_memblock
  189. __memblock_find_range_bottom_up(phys_addr_t start, phys_addr_t end,
  190. phys_addr_t size, phys_addr_t align, int nid,
  191. enum memblock_flags flags)
  192. {
  193. phys_addr_t this_start, this_end, cand;
  194. u64 i;
  195. for_each_free_mem_range(i, nid, flags, &this_start, &this_end, NULL) {
  196. this_start = clamp(this_start, start, end);
  197. this_end = clamp(this_end, start, end);
  198. cand = round_up(this_start, align);
  199. if (cand < this_end && this_end - cand >= size)
  200. return cand;
  201. }
  202. return 0;
  203. }
  204. /**
  205. * __memblock_find_range_top_down - find free area utility, in top-down
  206. * @start: start of candidate range
  207. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  208. * %MEMBLOCK_ALLOC_ACCESSIBLE
  209. * @size: size of free area to find
  210. * @align: alignment of free area to find
  211. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  212. * @flags: pick from blocks based on memory attributes
  213. *
  214. * Utility called from memblock_find_in_range_node(), find free area top-down.
  215. *
  216. * Return:
  217. * Found address on success, 0 on failure.
  218. */
  219. static phys_addr_t __init_memblock
  220. __memblock_find_range_top_down(phys_addr_t start, phys_addr_t end,
  221. phys_addr_t size, phys_addr_t align, int nid,
  222. enum memblock_flags flags)
  223. {
  224. phys_addr_t this_start, this_end, cand;
  225. u64 i;
  226. for_each_free_mem_range_reverse(i, nid, flags, &this_start, &this_end,
  227. NULL) {
  228. this_start = clamp(this_start, start, end);
  229. this_end = clamp(this_end, start, end);
  230. if (this_end < size)
  231. continue;
  232. cand = round_down(this_end - size, align);
  233. if (cand >= this_start)
  234. return cand;
  235. }
  236. return 0;
  237. }
  238. /**
  239. * memblock_find_in_range_node - find free area in given range and node
  240. * @size: size of free area to find
  241. * @align: alignment of free area to find
  242. * @start: start of candidate range
  243. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  244. * %MEMBLOCK_ALLOC_ACCESSIBLE
  245. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  246. * @flags: pick from blocks based on memory attributes
  247. *
  248. * Find @size free area aligned to @align in the specified range and node.
  249. *
  250. * Return:
  251. * Found address on success, 0 on failure.
  252. */
  253. static phys_addr_t __init_memblock memblock_find_in_range_node(phys_addr_t size,
  254. phys_addr_t align, phys_addr_t start,
  255. phys_addr_t end, int nid,
  256. enum memblock_flags flags)
  257. {
  258. /* pump up @end */
  259. if (end == MEMBLOCK_ALLOC_ACCESSIBLE ||
  260. end == MEMBLOCK_ALLOC_NOLEAKTRACE)
  261. end = memblock.current_limit;
  262. /* avoid allocating the first page */
  263. start = max_t(phys_addr_t, start, PAGE_SIZE);
  264. end = max(start, end);
  265. if (memblock_bottom_up())
  266. return __memblock_find_range_bottom_up(start, end, size, align,
  267. nid, flags);
  268. else
  269. return __memblock_find_range_top_down(start, end, size, align,
  270. nid, flags);
  271. }
  272. /**
  273. * memblock_find_in_range - find free area in given range
  274. * @start: start of candidate range
  275. * @end: end of candidate range, can be %MEMBLOCK_ALLOC_ANYWHERE or
  276. * %MEMBLOCK_ALLOC_ACCESSIBLE
  277. * @size: size of free area to find
  278. * @align: alignment of free area to find
  279. *
  280. * Find @size free area aligned to @align in the specified range.
  281. *
  282. * Return:
  283. * Found address on success, 0 on failure.
  284. */
  285. static phys_addr_t __init_memblock memblock_find_in_range(phys_addr_t start,
  286. phys_addr_t end, phys_addr_t size,
  287. phys_addr_t align)
  288. {
  289. phys_addr_t ret;
  290. enum memblock_flags flags = choose_memblock_flags();
  291. again:
  292. ret = memblock_find_in_range_node(size, align, start, end,
  293. NUMA_NO_NODE, flags);
  294. if (!ret && (flags & MEMBLOCK_MIRROR)) {
  295. pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n",
  296. &size);
  297. flags &= ~MEMBLOCK_MIRROR;
  298. goto again;
  299. }
  300. return ret;
  301. }
  302. static void __init_memblock memblock_remove_region(struct memblock_type *type, unsigned long r)
  303. {
  304. type->total_size -= type->regions[r].size;
  305. memmove(&type->regions[r], &type->regions[r + 1],
  306. (type->cnt - (r + 1)) * sizeof(type->regions[r]));
  307. type->cnt--;
  308. /* Special case for empty arrays */
  309. if (type->cnt == 0) {
  310. WARN_ON(type->total_size != 0);
  311. type->cnt = 1;
  312. type->regions[0].base = 0;
  313. type->regions[0].size = 0;
  314. type->regions[0].flags = 0;
  315. memblock_set_region_node(&type->regions[0], MAX_NUMNODES);
  316. }
  317. }
  318. #ifndef CONFIG_ARCH_KEEP_MEMBLOCK
  319. /**
  320. * memblock_discard - discard memory and reserved arrays if they were allocated
  321. */
  322. void __init memblock_discard(void)
  323. {
  324. phys_addr_t addr, size;
  325. if (memblock.reserved.regions != memblock_reserved_init_regions) {
  326. addr = __pa(memblock.reserved.regions);
  327. size = PAGE_ALIGN(sizeof(struct memblock_region) *
  328. memblock.reserved.max);
  329. if (memblock_reserved_in_slab)
  330. kfree(memblock.reserved.regions);
  331. else
  332. memblock_free_late(addr, size);
  333. }
  334. if (memblock.memory.regions != memblock_memory_init_regions) {
  335. addr = __pa(memblock.memory.regions);
  336. size = PAGE_ALIGN(sizeof(struct memblock_region) *
  337. memblock.memory.max);
  338. if (memblock_memory_in_slab)
  339. kfree(memblock.memory.regions);
  340. else
  341. memblock_free_late(addr, size);
  342. }
  343. memblock_memory = NULL;
  344. }
  345. #endif
  346. /**
  347. * memblock_double_array - double the size of the memblock regions array
  348. * @type: memblock type of the regions array being doubled
  349. * @new_area_start: starting address of memory range to avoid overlap with
  350. * @new_area_size: size of memory range to avoid overlap with
  351. *
  352. * Double the size of the @type regions array. If memblock is being used to
  353. * allocate memory for a new reserved regions array and there is a previously
  354. * allocated memory range [@new_area_start, @new_area_start + @new_area_size]
  355. * waiting to be reserved, ensure the memory used by the new array does
  356. * not overlap.
  357. *
  358. * Return:
  359. * 0 on success, -1 on failure.
  360. */
  361. static int __init_memblock memblock_double_array(struct memblock_type *type,
  362. phys_addr_t new_area_start,
  363. phys_addr_t new_area_size)
  364. {
  365. struct memblock_region *new_array, *old_array;
  366. phys_addr_t old_alloc_size, new_alloc_size;
  367. phys_addr_t old_size, new_size, addr, new_end;
  368. int use_slab = slab_is_available();
  369. int *in_slab;
  370. /* We don't allow resizing until we know about the reserved regions
  371. * of memory that aren't suitable for allocation
  372. */
  373. if (!memblock_can_resize)
  374. return -1;
  375. /* Calculate new doubled size */
  376. old_size = type->max * sizeof(struct memblock_region);
  377. new_size = old_size << 1;
  378. /*
  379. * We need to allocated new one align to PAGE_SIZE,
  380. * so we can free them completely later.
  381. */
  382. old_alloc_size = PAGE_ALIGN(old_size);
  383. new_alloc_size = PAGE_ALIGN(new_size);
  384. /* Retrieve the slab flag */
  385. if (type == &memblock.memory)
  386. in_slab = &memblock_memory_in_slab;
  387. else
  388. in_slab = &memblock_reserved_in_slab;
  389. /* Try to find some space for it */
  390. if (use_slab) {
  391. new_array = kmalloc(new_size, GFP_KERNEL);
  392. addr = new_array ? __pa(new_array) : 0;
  393. } else {
  394. /* only exclude range when trying to double reserved.regions */
  395. if (type != &memblock.reserved)
  396. new_area_start = new_area_size = 0;
  397. addr = memblock_find_in_range(new_area_start + new_area_size,
  398. memblock.current_limit,
  399. new_alloc_size, PAGE_SIZE);
  400. if (!addr && new_area_size)
  401. addr = memblock_find_in_range(0,
  402. min(new_area_start, memblock.current_limit),
  403. new_alloc_size, PAGE_SIZE);
  404. new_array = addr ? __va(addr) : NULL;
  405. }
  406. if (!addr) {
  407. pr_err("memblock: Failed to double %s array from %ld to %ld entries !\n",
  408. type->name, type->max, type->max * 2);
  409. return -1;
  410. }
  411. new_end = addr + new_size - 1;
  412. memblock_dbg("memblock: %s is doubled to %ld at [%pa-%pa]",
  413. type->name, type->max * 2, &addr, &new_end);
  414. /*
  415. * Found space, we now need to move the array over before we add the
  416. * reserved region since it may be our reserved array itself that is
  417. * full.
  418. */
  419. memcpy(new_array, type->regions, old_size);
  420. memset(new_array + type->max, 0, old_size);
  421. old_array = type->regions;
  422. type->regions = new_array;
  423. type->max <<= 1;
  424. /* Free old array. We needn't free it if the array is the static one */
  425. if (*in_slab)
  426. kfree(old_array);
  427. else if (old_array != memblock_memory_init_regions &&
  428. old_array != memblock_reserved_init_regions)
  429. memblock_free(old_array, old_alloc_size);
  430. /*
  431. * Reserve the new array if that comes from the memblock. Otherwise, we
  432. * needn't do it
  433. */
  434. if (!use_slab)
  435. BUG_ON(memblock_reserve(addr, new_alloc_size));
  436. /* Update slab flag */
  437. *in_slab = use_slab;
  438. return 0;
  439. }
  440. /**
  441. * memblock_merge_regions - merge neighboring compatible regions
  442. * @type: memblock type to scan
  443. *
  444. * Scan @type and merge neighboring compatible regions.
  445. */
  446. static void __init_memblock memblock_merge_regions(struct memblock_type *type)
  447. {
  448. int i = 0;
  449. /* cnt never goes below 1 */
  450. while (i < type->cnt - 1) {
  451. struct memblock_region *this = &type->regions[i];
  452. struct memblock_region *next = &type->regions[i + 1];
  453. if (this->base + this->size != next->base ||
  454. memblock_get_region_node(this) !=
  455. memblock_get_region_node(next) ||
  456. this->flags != next->flags) {
  457. BUG_ON(this->base + this->size > next->base);
  458. i++;
  459. continue;
  460. }
  461. this->size += next->size;
  462. /* move forward from next + 1, index of which is i + 2 */
  463. memmove(next, next + 1, (type->cnt - (i + 2)) * sizeof(*next));
  464. type->cnt--;
  465. }
  466. }
  467. /**
  468. * memblock_insert_region - insert new memblock region
  469. * @type: memblock type to insert into
  470. * @idx: index for the insertion point
  471. * @base: base address of the new region
  472. * @size: size of the new region
  473. * @nid: node id of the new region
  474. * @flags: flags of the new region
  475. *
  476. * Insert new memblock region [@base, @base + @size) into @type at @idx.
  477. * @type must already have extra room to accommodate the new region.
  478. */
  479. static void __init_memblock memblock_insert_region(struct memblock_type *type,
  480. int idx, phys_addr_t base,
  481. phys_addr_t size,
  482. int nid,
  483. enum memblock_flags flags)
  484. {
  485. struct memblock_region *rgn = &type->regions[idx];
  486. BUG_ON(type->cnt >= type->max);
  487. memmove(rgn + 1, rgn, (type->cnt - idx) * sizeof(*rgn));
  488. rgn->base = base;
  489. rgn->size = size;
  490. rgn->flags = flags;
  491. memblock_set_region_node(rgn, nid);
  492. type->cnt++;
  493. type->total_size += size;
  494. }
  495. /**
  496. * memblock_add_range - add new memblock region
  497. * @type: memblock type to add new region into
  498. * @base: base address of the new region
  499. * @size: size of the new region
  500. * @nid: nid of the new region
  501. * @flags: flags of the new region
  502. *
  503. * Add new memblock region [@base, @base + @size) into @type. The new region
  504. * is allowed to overlap with existing ones - overlaps don't affect already
  505. * existing regions. @type is guaranteed to be minimal (all neighbouring
  506. * compatible regions are merged) after the addition.
  507. *
  508. * Return:
  509. * 0 on success, -errno on failure.
  510. */
  511. static int __init_memblock memblock_add_range(struct memblock_type *type,
  512. phys_addr_t base, phys_addr_t size,
  513. int nid, enum memblock_flags flags)
  514. {
  515. bool insert = false;
  516. phys_addr_t obase = base;
  517. phys_addr_t end = base + memblock_cap_size(base, &size);
  518. int idx, nr_new;
  519. struct memblock_region *rgn;
  520. if (!size)
  521. return 0;
  522. /* special case for empty array */
  523. if (type->regions[0].size == 0) {
  524. WARN_ON(type->cnt != 1 || type->total_size);
  525. type->regions[0].base = base;
  526. type->regions[0].size = size;
  527. type->regions[0].flags = flags;
  528. memblock_set_region_node(&type->regions[0], nid);
  529. type->total_size = size;
  530. return 0;
  531. }
  532. /*
  533. * The worst case is when new range overlaps all existing regions,
  534. * then we'll need type->cnt + 1 empty regions in @type. So if
  535. * type->cnt * 2 + 1 is less than type->max, we know
  536. * that there is enough empty regions in @type, and we can insert
  537. * regions directly.
  538. */
  539. if (type->cnt * 2 + 1 < type->max)
  540. insert = true;
  541. repeat:
  542. /*
  543. * The following is executed twice. Once with %false @insert and
  544. * then with %true. The first counts the number of regions needed
  545. * to accommodate the new area. The second actually inserts them.
  546. */
  547. base = obase;
  548. nr_new = 0;
  549. for_each_memblock_type(idx, type, rgn) {
  550. phys_addr_t rbase = rgn->base;
  551. phys_addr_t rend = rbase + rgn->size;
  552. if (rbase >= end)
  553. break;
  554. if (rend <= base)
  555. continue;
  556. /*
  557. * @rgn overlaps. If it separates the lower part of new
  558. * area, insert that portion.
  559. */
  560. if (rbase > base) {
  561. #ifdef CONFIG_NUMA
  562. WARN_ON(nid != memblock_get_region_node(rgn));
  563. #endif
  564. WARN_ON(flags != rgn->flags);
  565. nr_new++;
  566. if (insert)
  567. memblock_insert_region(type, idx++, base,
  568. rbase - base, nid,
  569. flags);
  570. }
  571. /* area below @rend is dealt with, forget about it */
  572. base = min(rend, end);
  573. }
  574. /* insert the remaining portion */
  575. if (base < end) {
  576. nr_new++;
  577. if (insert)
  578. memblock_insert_region(type, idx, base, end - base,
  579. nid, flags);
  580. }
  581. if (!nr_new)
  582. return 0;
  583. /*
  584. * If this was the first round, resize array and repeat for actual
  585. * insertions; otherwise, merge and return.
  586. */
  587. if (!insert) {
  588. while (type->cnt + nr_new > type->max)
  589. if (memblock_double_array(type, obase, size) < 0)
  590. return -ENOMEM;
  591. insert = true;
  592. goto repeat;
  593. } else {
  594. memblock_merge_regions(type);
  595. return 0;
  596. }
  597. }
  598. /**
  599. * memblock_add_node - add new memblock region within a NUMA node
  600. * @base: base address of the new region
  601. * @size: size of the new region
  602. * @nid: nid of the new region
  603. * @flags: flags of the new region
  604. *
  605. * Add new memblock region [@base, @base + @size) to the "memory"
  606. * type. See memblock_add_range() description for mode details
  607. *
  608. * Return:
  609. * 0 on success, -errno on failure.
  610. */
  611. int __init_memblock memblock_add_node(phys_addr_t base, phys_addr_t size,
  612. int nid, enum memblock_flags flags)
  613. {
  614. phys_addr_t end = base + size - 1;
  615. memblock_dbg("%s: [%pa-%pa] nid=%d flags=%x %pS\n", __func__,
  616. &base, &end, nid, flags, (void *)_RET_IP_);
  617. return memblock_add_range(&memblock.memory, base, size, nid, flags);
  618. }
  619. /**
  620. * memblock_add - add new memblock region
  621. * @base: base address of the new region
  622. * @size: size of the new region
  623. *
  624. * Add new memblock region [@base, @base + @size) to the "memory"
  625. * type. See memblock_add_range() description for mode details
  626. *
  627. * Return:
  628. * 0 on success, -errno on failure.
  629. */
  630. int __init_memblock memblock_add(phys_addr_t base, phys_addr_t size)
  631. {
  632. phys_addr_t end = base + size - 1;
  633. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  634. &base, &end, (void *)_RET_IP_);
  635. return memblock_add_range(&memblock.memory, base, size, MAX_NUMNODES, 0);
  636. }
  637. /**
  638. * memblock_isolate_range - isolate given range into disjoint memblocks
  639. * @type: memblock type to isolate range for
  640. * @base: base of range to isolate
  641. * @size: size of range to isolate
  642. * @start_rgn: out parameter for the start of isolated region
  643. * @end_rgn: out parameter for the end of isolated region
  644. *
  645. * Walk @type and ensure that regions don't cross the boundaries defined by
  646. * [@base, @base + @size). Crossing regions are split at the boundaries,
  647. * which may create at most two more regions. The index of the first
  648. * region inside the range is returned in *@start_rgn and end in *@end_rgn.
  649. *
  650. * Return:
  651. * 0 on success, -errno on failure.
  652. */
  653. static int __init_memblock memblock_isolate_range(struct memblock_type *type,
  654. phys_addr_t base, phys_addr_t size,
  655. int *start_rgn, int *end_rgn)
  656. {
  657. phys_addr_t end = base + memblock_cap_size(base, &size);
  658. int idx;
  659. struct memblock_region *rgn;
  660. *start_rgn = *end_rgn = 0;
  661. if (!size)
  662. return 0;
  663. /* we'll create at most two more regions */
  664. while (type->cnt + 2 > type->max)
  665. if (memblock_double_array(type, base, size) < 0)
  666. return -ENOMEM;
  667. for_each_memblock_type(idx, type, rgn) {
  668. phys_addr_t rbase = rgn->base;
  669. phys_addr_t rend = rbase + rgn->size;
  670. if (rbase >= end)
  671. break;
  672. if (rend <= base)
  673. continue;
  674. if (rbase < base) {
  675. /*
  676. * @rgn intersects from below. Split and continue
  677. * to process the next region - the new top half.
  678. */
  679. rgn->base = base;
  680. rgn->size -= base - rbase;
  681. type->total_size -= base - rbase;
  682. memblock_insert_region(type, idx, rbase, base - rbase,
  683. memblock_get_region_node(rgn),
  684. rgn->flags);
  685. } else if (rend > end) {
  686. /*
  687. * @rgn intersects from above. Split and redo the
  688. * current region - the new bottom half.
  689. */
  690. rgn->base = end;
  691. rgn->size -= end - rbase;
  692. type->total_size -= end - rbase;
  693. memblock_insert_region(type, idx--, rbase, end - rbase,
  694. memblock_get_region_node(rgn),
  695. rgn->flags);
  696. } else {
  697. /* @rgn is fully contained, record it */
  698. if (!*end_rgn)
  699. *start_rgn = idx;
  700. *end_rgn = idx + 1;
  701. }
  702. }
  703. return 0;
  704. }
  705. static int __init_memblock memblock_remove_range(struct memblock_type *type,
  706. phys_addr_t base, phys_addr_t size)
  707. {
  708. int start_rgn, end_rgn;
  709. int i, ret;
  710. ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
  711. if (ret)
  712. return ret;
  713. for (i = end_rgn - 1; i >= start_rgn; i--)
  714. memblock_remove_region(type, i);
  715. return 0;
  716. }
  717. int __init_memblock memblock_remove(phys_addr_t base, phys_addr_t size)
  718. {
  719. phys_addr_t end = base + size - 1;
  720. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  721. &base, &end, (void *)_RET_IP_);
  722. return memblock_remove_range(&memblock.memory, base, size);
  723. }
  724. /**
  725. * memblock_free - free boot memory allocation
  726. * @ptr: starting address of the boot memory allocation
  727. * @size: size of the boot memory block in bytes
  728. *
  729. * Free boot memory block previously allocated by memblock_alloc_xx() API.
  730. * The freeing memory will not be released to the buddy allocator.
  731. */
  732. void __init_memblock memblock_free(void *ptr, size_t size)
  733. {
  734. if (ptr)
  735. memblock_phys_free(__pa(ptr), size);
  736. }
  737. /**
  738. * memblock_phys_free - free boot memory block
  739. * @base: phys starting address of the boot memory block
  740. * @size: size of the boot memory block in bytes
  741. *
  742. * Free boot memory block previously allocated by memblock_alloc_xx() API.
  743. * The freeing memory will not be released to the buddy allocator.
  744. */
  745. int __init_memblock memblock_phys_free(phys_addr_t base, phys_addr_t size)
  746. {
  747. phys_addr_t end = base + size - 1;
  748. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  749. &base, &end, (void *)_RET_IP_);
  750. kmemleak_free_part_phys(base, size);
  751. return memblock_remove_range(&memblock.reserved, base, size);
  752. }
  753. #ifdef CONFIG_ARCH_KEEP_MEMBLOCK
  754. EXPORT_SYMBOL_GPL(memblock_free);
  755. #endif
  756. int __init_memblock memblock_reserve(phys_addr_t base, phys_addr_t size)
  757. {
  758. phys_addr_t end = base + size - 1;
  759. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  760. &base, &end, (void *)_RET_IP_);
  761. return memblock_add_range(&memblock.reserved, base, size, MAX_NUMNODES, 0);
  762. }
  763. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  764. int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
  765. {
  766. phys_addr_t end = base + size - 1;
  767. memblock_dbg("%s: [%pa-%pa] %pS\n", __func__,
  768. &base, &end, (void *)_RET_IP_);
  769. return memblock_add_range(&physmem, base, size, MAX_NUMNODES, 0);
  770. }
  771. #endif
  772. /**
  773. * memblock_setclr_flag - set or clear flag for a memory region
  774. * @base: base address of the region
  775. * @size: size of the region
  776. * @set: set or clear the flag
  777. * @flag: the flag to update
  778. *
  779. * This function isolates region [@base, @base + @size), and sets/clears flag
  780. *
  781. * Return: 0 on success, -errno on failure.
  782. */
  783. static int __init_memblock memblock_setclr_flag(phys_addr_t base,
  784. phys_addr_t size, int set, int flag)
  785. {
  786. struct memblock_type *type = &memblock.memory;
  787. int i, ret, start_rgn, end_rgn;
  788. ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
  789. if (ret)
  790. return ret;
  791. for (i = start_rgn; i < end_rgn; i++) {
  792. struct memblock_region *r = &type->regions[i];
  793. if (set)
  794. r->flags |= flag;
  795. else
  796. r->flags &= ~flag;
  797. }
  798. memblock_merge_regions(type);
  799. return 0;
  800. }
  801. /**
  802. * memblock_mark_hotplug - Mark hotpluggable memory with flag MEMBLOCK_HOTPLUG.
  803. * @base: the base phys addr of the region
  804. * @size: the size of the region
  805. *
  806. * Return: 0 on success, -errno on failure.
  807. */
  808. int __init_memblock memblock_mark_hotplug(phys_addr_t base, phys_addr_t size)
  809. {
  810. return memblock_setclr_flag(base, size, 1, MEMBLOCK_HOTPLUG);
  811. }
  812. /**
  813. * memblock_clear_hotplug - Clear flag MEMBLOCK_HOTPLUG for a specified region.
  814. * @base: the base phys addr of the region
  815. * @size: the size of the region
  816. *
  817. * Return: 0 on success, -errno on failure.
  818. */
  819. int __init_memblock memblock_clear_hotplug(phys_addr_t base, phys_addr_t size)
  820. {
  821. return memblock_setclr_flag(base, size, 0, MEMBLOCK_HOTPLUG);
  822. }
  823. /**
  824. * memblock_mark_mirror - Mark mirrored memory with flag MEMBLOCK_MIRROR.
  825. * @base: the base phys addr of the region
  826. * @size: the size of the region
  827. *
  828. * Return: 0 on success, -errno on failure.
  829. */
  830. int __init_memblock memblock_mark_mirror(phys_addr_t base, phys_addr_t size)
  831. {
  832. if (!mirrored_kernelcore)
  833. return 0;
  834. system_has_some_mirror = true;
  835. return memblock_setclr_flag(base, size, 1, MEMBLOCK_MIRROR);
  836. }
  837. /**
  838. * memblock_mark_nomap - Mark a memory region with flag MEMBLOCK_NOMAP.
  839. * @base: the base phys addr of the region
  840. * @size: the size of the region
  841. *
  842. * The memory regions marked with %MEMBLOCK_NOMAP will not be added to the
  843. * direct mapping of the physical memory. These regions will still be
  844. * covered by the memory map. The struct page representing NOMAP memory
  845. * frames in the memory map will be PageReserved()
  846. *
  847. * Note: if the memory being marked %MEMBLOCK_NOMAP was allocated from
  848. * memblock, the caller must inform kmemleak to ignore that memory
  849. *
  850. * Return: 0 on success, -errno on failure.
  851. */
  852. int __init_memblock memblock_mark_nomap(phys_addr_t base, phys_addr_t size)
  853. {
  854. return memblock_setclr_flag(base, size, 1, MEMBLOCK_NOMAP);
  855. }
  856. /**
  857. * memblock_clear_nomap - Clear flag MEMBLOCK_NOMAP for a specified region.
  858. * @base: the base phys addr of the region
  859. * @size: the size of the region
  860. *
  861. * Return: 0 on success, -errno on failure.
  862. */
  863. int __init_memblock memblock_clear_nomap(phys_addr_t base, phys_addr_t size)
  864. {
  865. return memblock_setclr_flag(base, size, 0, MEMBLOCK_NOMAP);
  866. }
  867. static bool should_skip_region(struct memblock_type *type,
  868. struct memblock_region *m,
  869. int nid, int flags)
  870. {
  871. int m_nid = memblock_get_region_node(m);
  872. /* we never skip regions when iterating memblock.reserved or physmem */
  873. if (type != memblock_memory)
  874. return false;
  875. /* only memory regions are associated with nodes, check it */
  876. if (nid != NUMA_NO_NODE && nid != m_nid)
  877. return true;
  878. /* skip hotpluggable memory regions if needed */
  879. if (movable_node_is_enabled() && memblock_is_hotpluggable(m) &&
  880. !(flags & MEMBLOCK_HOTPLUG))
  881. return true;
  882. /* if we want mirror memory skip non-mirror memory regions */
  883. if ((flags & MEMBLOCK_MIRROR) && !memblock_is_mirror(m))
  884. return true;
  885. /* skip nomap memory unless we were asked for it explicitly */
  886. if (!(flags & MEMBLOCK_NOMAP) && memblock_is_nomap(m))
  887. return true;
  888. /* skip driver-managed memory unless we were asked for it explicitly */
  889. if (!(flags & MEMBLOCK_DRIVER_MANAGED) && memblock_is_driver_managed(m))
  890. return true;
  891. return false;
  892. }
  893. /**
  894. * __next_mem_range - next function for for_each_free_mem_range() etc.
  895. * @idx: pointer to u64 loop variable
  896. * @nid: node selector, %NUMA_NO_NODE for all nodes
  897. * @flags: pick from blocks based on memory attributes
  898. * @type_a: pointer to memblock_type from where the range is taken
  899. * @type_b: pointer to memblock_type which excludes memory from being taken
  900. * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
  901. * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
  902. * @out_nid: ptr to int for nid of the range, can be %NULL
  903. *
  904. * Find the first area from *@idx which matches @nid, fill the out
  905. * parameters, and update *@idx for the next iteration. The lower 32bit of
  906. * *@idx contains index into type_a and the upper 32bit indexes the
  907. * areas before each region in type_b. For example, if type_b regions
  908. * look like the following,
  909. *
  910. * 0:[0-16), 1:[32-48), 2:[128-130)
  911. *
  912. * The upper 32bit indexes the following regions.
  913. *
  914. * 0:[0-0), 1:[16-32), 2:[48-128), 3:[130-MAX)
  915. *
  916. * As both region arrays are sorted, the function advances the two indices
  917. * in lockstep and returns each intersection.
  918. */
  919. void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags,
  920. struct memblock_type *type_a,
  921. struct memblock_type *type_b, phys_addr_t *out_start,
  922. phys_addr_t *out_end, int *out_nid)
  923. {
  924. int idx_a = *idx & 0xffffffff;
  925. int idx_b = *idx >> 32;
  926. if (WARN_ONCE(nid == MAX_NUMNODES,
  927. "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
  928. nid = NUMA_NO_NODE;
  929. for (; idx_a < type_a->cnt; idx_a++) {
  930. struct memblock_region *m = &type_a->regions[idx_a];
  931. phys_addr_t m_start = m->base;
  932. phys_addr_t m_end = m->base + m->size;
  933. int m_nid = memblock_get_region_node(m);
  934. if (should_skip_region(type_a, m, nid, flags))
  935. continue;
  936. if (!type_b) {
  937. if (out_start)
  938. *out_start = m_start;
  939. if (out_end)
  940. *out_end = m_end;
  941. if (out_nid)
  942. *out_nid = m_nid;
  943. idx_a++;
  944. *idx = (u32)idx_a | (u64)idx_b << 32;
  945. return;
  946. }
  947. /* scan areas before each reservation */
  948. for (; idx_b < type_b->cnt + 1; idx_b++) {
  949. struct memblock_region *r;
  950. phys_addr_t r_start;
  951. phys_addr_t r_end;
  952. r = &type_b->regions[idx_b];
  953. r_start = idx_b ? r[-1].base + r[-1].size : 0;
  954. r_end = idx_b < type_b->cnt ?
  955. r->base : PHYS_ADDR_MAX;
  956. /*
  957. * if idx_b advanced past idx_a,
  958. * break out to advance idx_a
  959. */
  960. if (r_start >= m_end)
  961. break;
  962. /* if the two regions intersect, we're done */
  963. if (m_start < r_end) {
  964. if (out_start)
  965. *out_start =
  966. max(m_start, r_start);
  967. if (out_end)
  968. *out_end = min(m_end, r_end);
  969. if (out_nid)
  970. *out_nid = m_nid;
  971. /*
  972. * The region which ends first is
  973. * advanced for the next iteration.
  974. */
  975. if (m_end <= r_end)
  976. idx_a++;
  977. else
  978. idx_b++;
  979. *idx = (u32)idx_a | (u64)idx_b << 32;
  980. return;
  981. }
  982. }
  983. }
  984. /* signal end of iteration */
  985. *idx = ULLONG_MAX;
  986. }
  987. /**
  988. * __next_mem_range_rev - generic next function for for_each_*_range_rev()
  989. *
  990. * @idx: pointer to u64 loop variable
  991. * @nid: node selector, %NUMA_NO_NODE for all nodes
  992. * @flags: pick from blocks based on memory attributes
  993. * @type_a: pointer to memblock_type from where the range is taken
  994. * @type_b: pointer to memblock_type which excludes memory from being taken
  995. * @out_start: ptr to phys_addr_t for start address of the range, can be %NULL
  996. * @out_end: ptr to phys_addr_t for end address of the range, can be %NULL
  997. * @out_nid: ptr to int for nid of the range, can be %NULL
  998. *
  999. * Finds the next range from type_a which is not marked as unsuitable
  1000. * in type_b.
  1001. *
  1002. * Reverse of __next_mem_range().
  1003. */
  1004. void __init_memblock __next_mem_range_rev(u64 *idx, int nid,
  1005. enum memblock_flags flags,
  1006. struct memblock_type *type_a,
  1007. struct memblock_type *type_b,
  1008. phys_addr_t *out_start,
  1009. phys_addr_t *out_end, int *out_nid)
  1010. {
  1011. int idx_a = *idx & 0xffffffff;
  1012. int idx_b = *idx >> 32;
  1013. if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
  1014. nid = NUMA_NO_NODE;
  1015. if (*idx == (u64)ULLONG_MAX) {
  1016. idx_a = type_a->cnt - 1;
  1017. if (type_b != NULL)
  1018. idx_b = type_b->cnt;
  1019. else
  1020. idx_b = 0;
  1021. }
  1022. for (; idx_a >= 0; idx_a--) {
  1023. struct memblock_region *m = &type_a->regions[idx_a];
  1024. phys_addr_t m_start = m->base;
  1025. phys_addr_t m_end = m->base + m->size;
  1026. int m_nid = memblock_get_region_node(m);
  1027. if (should_skip_region(type_a, m, nid, flags))
  1028. continue;
  1029. if (!type_b) {
  1030. if (out_start)
  1031. *out_start = m_start;
  1032. if (out_end)
  1033. *out_end = m_end;
  1034. if (out_nid)
  1035. *out_nid = m_nid;
  1036. idx_a--;
  1037. *idx = (u32)idx_a | (u64)idx_b << 32;
  1038. return;
  1039. }
  1040. /* scan areas before each reservation */
  1041. for (; idx_b >= 0; idx_b--) {
  1042. struct memblock_region *r;
  1043. phys_addr_t r_start;
  1044. phys_addr_t r_end;
  1045. r = &type_b->regions[idx_b];
  1046. r_start = idx_b ? r[-1].base + r[-1].size : 0;
  1047. r_end = idx_b < type_b->cnt ?
  1048. r->base : PHYS_ADDR_MAX;
  1049. /*
  1050. * if idx_b advanced past idx_a,
  1051. * break out to advance idx_a
  1052. */
  1053. if (r_end <= m_start)
  1054. break;
  1055. /* if the two regions intersect, we're done */
  1056. if (m_end > r_start) {
  1057. if (out_start)
  1058. *out_start = max(m_start, r_start);
  1059. if (out_end)
  1060. *out_end = min(m_end, r_end);
  1061. if (out_nid)
  1062. *out_nid = m_nid;
  1063. if (m_start >= r_start)
  1064. idx_a--;
  1065. else
  1066. idx_b--;
  1067. *idx = (u32)idx_a | (u64)idx_b << 32;
  1068. return;
  1069. }
  1070. }
  1071. }
  1072. /* signal end of iteration */
  1073. *idx = ULLONG_MAX;
  1074. }
  1075. /*
  1076. * Common iterator interface used to define for_each_mem_pfn_range().
  1077. */
  1078. void __init_memblock __next_mem_pfn_range(int *idx, int nid,
  1079. unsigned long *out_start_pfn,
  1080. unsigned long *out_end_pfn, int *out_nid)
  1081. {
  1082. struct memblock_type *type = &memblock.memory;
  1083. struct memblock_region *r;
  1084. int r_nid;
  1085. while (++*idx < type->cnt) {
  1086. r = &type->regions[*idx];
  1087. r_nid = memblock_get_region_node(r);
  1088. if (PFN_UP(r->base) >= PFN_DOWN(r->base + r->size))
  1089. continue;
  1090. if (nid == MAX_NUMNODES || nid == r_nid)
  1091. break;
  1092. }
  1093. if (*idx >= type->cnt) {
  1094. *idx = -1;
  1095. return;
  1096. }
  1097. if (out_start_pfn)
  1098. *out_start_pfn = PFN_UP(r->base);
  1099. if (out_end_pfn)
  1100. *out_end_pfn = PFN_DOWN(r->base + r->size);
  1101. if (out_nid)
  1102. *out_nid = r_nid;
  1103. }
  1104. /**
  1105. * memblock_set_node - set node ID on memblock regions
  1106. * @base: base of area to set node ID for
  1107. * @size: size of area to set node ID for
  1108. * @type: memblock type to set node ID for
  1109. * @nid: node ID to set
  1110. *
  1111. * Set the nid of memblock @type regions in [@base, @base + @size) to @nid.
  1112. * Regions which cross the area boundaries are split as necessary.
  1113. *
  1114. * Return:
  1115. * 0 on success, -errno on failure.
  1116. */
  1117. int __init_memblock memblock_set_node(phys_addr_t base, phys_addr_t size,
  1118. struct memblock_type *type, int nid)
  1119. {
  1120. #ifdef CONFIG_NUMA
  1121. int start_rgn, end_rgn;
  1122. int i, ret;
  1123. ret = memblock_isolate_range(type, base, size, &start_rgn, &end_rgn);
  1124. if (ret)
  1125. return ret;
  1126. for (i = start_rgn; i < end_rgn; i++)
  1127. memblock_set_region_node(&type->regions[i], nid);
  1128. memblock_merge_regions(type);
  1129. #endif
  1130. return 0;
  1131. }
  1132. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1133. /**
  1134. * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone()
  1135. *
  1136. * @idx: pointer to u64 loop variable
  1137. * @zone: zone in which all of the memory blocks reside
  1138. * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL
  1139. * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL
  1140. *
  1141. * This function is meant to be a zone/pfn specific wrapper for the
  1142. * for_each_mem_range type iterators. Specifically they are used in the
  1143. * deferred memory init routines and as such we were duplicating much of
  1144. * this logic throughout the code. So instead of having it in multiple
  1145. * locations it seemed like it would make more sense to centralize this to
  1146. * one new iterator that does everything they need.
  1147. */
  1148. void __init_memblock
  1149. __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone,
  1150. unsigned long *out_spfn, unsigned long *out_epfn)
  1151. {
  1152. int zone_nid = zone_to_nid(zone);
  1153. phys_addr_t spa, epa;
  1154. __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
  1155. &memblock.memory, &memblock.reserved,
  1156. &spa, &epa, NULL);
  1157. while (*idx != U64_MAX) {
  1158. unsigned long epfn = PFN_DOWN(epa);
  1159. unsigned long spfn = PFN_UP(spa);
  1160. /*
  1161. * Verify the end is at least past the start of the zone and
  1162. * that we have at least one PFN to initialize.
  1163. */
  1164. if (zone->zone_start_pfn < epfn && spfn < epfn) {
  1165. /* if we went too far just stop searching */
  1166. if (zone_end_pfn(zone) <= spfn) {
  1167. *idx = U64_MAX;
  1168. break;
  1169. }
  1170. if (out_spfn)
  1171. *out_spfn = max(zone->zone_start_pfn, spfn);
  1172. if (out_epfn)
  1173. *out_epfn = min(zone_end_pfn(zone), epfn);
  1174. return;
  1175. }
  1176. __next_mem_range(idx, zone_nid, MEMBLOCK_NONE,
  1177. &memblock.memory, &memblock.reserved,
  1178. &spa, &epa, NULL);
  1179. }
  1180. /* signal end of iteration */
  1181. if (out_spfn)
  1182. *out_spfn = ULONG_MAX;
  1183. if (out_epfn)
  1184. *out_epfn = 0;
  1185. }
  1186. #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
  1187. /**
  1188. * memblock_alloc_range_nid - allocate boot memory block
  1189. * @size: size of memory block to be allocated in bytes
  1190. * @align: alignment of the region and block's size
  1191. * @start: the lower bound of the memory region to allocate (phys address)
  1192. * @end: the upper bound of the memory region to allocate (phys address)
  1193. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1194. * @exact_nid: control the allocation fall back to other nodes
  1195. *
  1196. * The allocation is performed from memory region limited by
  1197. * memblock.current_limit if @end == %MEMBLOCK_ALLOC_ACCESSIBLE.
  1198. *
  1199. * If the specified node can not hold the requested memory and @exact_nid
  1200. * is false, the allocation falls back to any node in the system.
  1201. *
  1202. * For systems with memory mirroring, the allocation is attempted first
  1203. * from the regions with mirroring enabled and then retried from any
  1204. * memory region.
  1205. *
  1206. * In addition, function using kmemleak_alloc_phys for allocated boot
  1207. * memory block, it is never reported as leaks.
  1208. *
  1209. * Return:
  1210. * Physical address of allocated memory block on success, %0 on failure.
  1211. */
  1212. phys_addr_t __init memblock_alloc_range_nid(phys_addr_t size,
  1213. phys_addr_t align, phys_addr_t start,
  1214. phys_addr_t end, int nid,
  1215. bool exact_nid)
  1216. {
  1217. enum memblock_flags flags = choose_memblock_flags();
  1218. phys_addr_t found;
  1219. if (WARN_ONCE(nid == MAX_NUMNODES, "Usage of MAX_NUMNODES is deprecated. Use NUMA_NO_NODE instead\n"))
  1220. nid = NUMA_NO_NODE;
  1221. if (!align) {
  1222. /* Can't use WARNs this early in boot on powerpc */
  1223. dump_stack();
  1224. align = SMP_CACHE_BYTES;
  1225. }
  1226. again:
  1227. found = memblock_find_in_range_node(size, align, start, end, nid,
  1228. flags);
  1229. if (found && !memblock_reserve(found, size))
  1230. goto done;
  1231. if (nid != NUMA_NO_NODE && !exact_nid) {
  1232. found = memblock_find_in_range_node(size, align, start,
  1233. end, NUMA_NO_NODE,
  1234. flags);
  1235. if (found && !memblock_reserve(found, size))
  1236. goto done;
  1237. }
  1238. if (flags & MEMBLOCK_MIRROR) {
  1239. flags &= ~MEMBLOCK_MIRROR;
  1240. pr_warn_ratelimited("Could not allocate %pap bytes of mirrored memory\n",
  1241. &size);
  1242. goto again;
  1243. }
  1244. return 0;
  1245. done:
  1246. /*
  1247. * Skip kmemleak for those places like kasan_init() and
  1248. * early_pgtable_alloc() due to high volume.
  1249. */
  1250. if (end != MEMBLOCK_ALLOC_NOLEAKTRACE)
  1251. /*
  1252. * Memblock allocated blocks are never reported as
  1253. * leaks. This is because many of these blocks are
  1254. * only referred via the physical address which is
  1255. * not looked up by kmemleak.
  1256. */
  1257. kmemleak_alloc_phys(found, size, 0);
  1258. return found;
  1259. }
  1260. /**
  1261. * memblock_phys_alloc_range - allocate a memory block inside specified range
  1262. * @size: size of memory block to be allocated in bytes
  1263. * @align: alignment of the region and block's size
  1264. * @start: the lower bound of the memory region to allocate (physical address)
  1265. * @end: the upper bound of the memory region to allocate (physical address)
  1266. *
  1267. * Allocate @size bytes in the between @start and @end.
  1268. *
  1269. * Return: physical address of the allocated memory block on success,
  1270. * %0 on failure.
  1271. */
  1272. phys_addr_t __init memblock_phys_alloc_range(phys_addr_t size,
  1273. phys_addr_t align,
  1274. phys_addr_t start,
  1275. phys_addr_t end)
  1276. {
  1277. memblock_dbg("%s: %llu bytes align=0x%llx from=%pa max_addr=%pa %pS\n",
  1278. __func__, (u64)size, (u64)align, &start, &end,
  1279. (void *)_RET_IP_);
  1280. return memblock_alloc_range_nid(size, align, start, end, NUMA_NO_NODE,
  1281. false);
  1282. }
  1283. /**
  1284. * memblock_phys_alloc_try_nid - allocate a memory block from specified NUMA node
  1285. * @size: size of memory block to be allocated in bytes
  1286. * @align: alignment of the region and block's size
  1287. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1288. *
  1289. * Allocates memory block from the specified NUMA node. If the node
  1290. * has no available memory, attempts to allocated from any node in the
  1291. * system.
  1292. *
  1293. * Return: physical address of the allocated memory block on success,
  1294. * %0 on failure.
  1295. */
  1296. phys_addr_t __init memblock_phys_alloc_try_nid(phys_addr_t size, phys_addr_t align, int nid)
  1297. {
  1298. return memblock_alloc_range_nid(size, align, 0,
  1299. MEMBLOCK_ALLOC_ACCESSIBLE, nid, false);
  1300. }
  1301. /**
  1302. * memblock_alloc_internal - allocate boot memory block
  1303. * @size: size of memory block to be allocated in bytes
  1304. * @align: alignment of the region and block's size
  1305. * @min_addr: the lower bound of the memory region to allocate (phys address)
  1306. * @max_addr: the upper bound of the memory region to allocate (phys address)
  1307. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1308. * @exact_nid: control the allocation fall back to other nodes
  1309. *
  1310. * Allocates memory block using memblock_alloc_range_nid() and
  1311. * converts the returned physical address to virtual.
  1312. *
  1313. * The @min_addr limit is dropped if it can not be satisfied and the allocation
  1314. * will fall back to memory below @min_addr. Other constraints, such
  1315. * as node and mirrored memory will be handled again in
  1316. * memblock_alloc_range_nid().
  1317. *
  1318. * Return:
  1319. * Virtual address of allocated memory block on success, NULL on failure.
  1320. */
  1321. static void * __init memblock_alloc_internal(
  1322. phys_addr_t size, phys_addr_t align,
  1323. phys_addr_t min_addr, phys_addr_t max_addr,
  1324. int nid, bool exact_nid)
  1325. {
  1326. phys_addr_t alloc;
  1327. /*
  1328. * Detect any accidental use of these APIs after slab is ready, as at
  1329. * this moment memblock may be deinitialized already and its
  1330. * internal data may be destroyed (after execution of memblock_free_all)
  1331. */
  1332. if (WARN_ON_ONCE(slab_is_available()))
  1333. return kzalloc_node(size, GFP_NOWAIT, nid);
  1334. if (max_addr > memblock.current_limit)
  1335. max_addr = memblock.current_limit;
  1336. alloc = memblock_alloc_range_nid(size, align, min_addr, max_addr, nid,
  1337. exact_nid);
  1338. /* retry allocation without lower limit */
  1339. if (!alloc && min_addr)
  1340. alloc = memblock_alloc_range_nid(size, align, 0, max_addr, nid,
  1341. exact_nid);
  1342. if (!alloc)
  1343. return NULL;
  1344. return phys_to_virt(alloc);
  1345. }
  1346. /**
  1347. * memblock_alloc_exact_nid_raw - allocate boot memory block on the exact node
  1348. * without zeroing memory
  1349. * @size: size of memory block to be allocated in bytes
  1350. * @align: alignment of the region and block's size
  1351. * @min_addr: the lower bound of the memory region from where the allocation
  1352. * is preferred (phys address)
  1353. * @max_addr: the upper bound of the memory region from where the allocation
  1354. * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
  1355. * allocate only from memory limited by memblock.current_limit value
  1356. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1357. *
  1358. * Public function, provides additional debug information (including caller
  1359. * info), if enabled. Does not zero allocated memory.
  1360. *
  1361. * Return:
  1362. * Virtual address of allocated memory block on success, NULL on failure.
  1363. */
  1364. void * __init memblock_alloc_exact_nid_raw(
  1365. phys_addr_t size, phys_addr_t align,
  1366. phys_addr_t min_addr, phys_addr_t max_addr,
  1367. int nid)
  1368. {
  1369. memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
  1370. __func__, (u64)size, (u64)align, nid, &min_addr,
  1371. &max_addr, (void *)_RET_IP_);
  1372. return memblock_alloc_internal(size, align, min_addr, max_addr, nid,
  1373. true);
  1374. }
  1375. /**
  1376. * memblock_alloc_try_nid_raw - allocate boot memory block without zeroing
  1377. * memory and without panicking
  1378. * @size: size of memory block to be allocated in bytes
  1379. * @align: alignment of the region and block's size
  1380. * @min_addr: the lower bound of the memory region from where the allocation
  1381. * is preferred (phys address)
  1382. * @max_addr: the upper bound of the memory region from where the allocation
  1383. * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
  1384. * allocate only from memory limited by memblock.current_limit value
  1385. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1386. *
  1387. * Public function, provides additional debug information (including caller
  1388. * info), if enabled. Does not zero allocated memory, does not panic if request
  1389. * cannot be satisfied.
  1390. *
  1391. * Return:
  1392. * Virtual address of allocated memory block on success, NULL on failure.
  1393. */
  1394. void * __init memblock_alloc_try_nid_raw(
  1395. phys_addr_t size, phys_addr_t align,
  1396. phys_addr_t min_addr, phys_addr_t max_addr,
  1397. int nid)
  1398. {
  1399. memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
  1400. __func__, (u64)size, (u64)align, nid, &min_addr,
  1401. &max_addr, (void *)_RET_IP_);
  1402. return memblock_alloc_internal(size, align, min_addr, max_addr, nid,
  1403. false);
  1404. }
  1405. /**
  1406. * memblock_alloc_try_nid - allocate boot memory block
  1407. * @size: size of memory block to be allocated in bytes
  1408. * @align: alignment of the region and block's size
  1409. * @min_addr: the lower bound of the memory region from where the allocation
  1410. * is preferred (phys address)
  1411. * @max_addr: the upper bound of the memory region from where the allocation
  1412. * is preferred (phys address), or %MEMBLOCK_ALLOC_ACCESSIBLE to
  1413. * allocate only from memory limited by memblock.current_limit value
  1414. * @nid: nid of the free area to find, %NUMA_NO_NODE for any node
  1415. *
  1416. * Public function, provides additional debug information (including caller
  1417. * info), if enabled. This function zeroes the allocated memory.
  1418. *
  1419. * Return:
  1420. * Virtual address of allocated memory block on success, NULL on failure.
  1421. */
  1422. void * __init memblock_alloc_try_nid(
  1423. phys_addr_t size, phys_addr_t align,
  1424. phys_addr_t min_addr, phys_addr_t max_addr,
  1425. int nid)
  1426. {
  1427. void *ptr;
  1428. memblock_dbg("%s: %llu bytes align=0x%llx nid=%d from=%pa max_addr=%pa %pS\n",
  1429. __func__, (u64)size, (u64)align, nid, &min_addr,
  1430. &max_addr, (void *)_RET_IP_);
  1431. ptr = memblock_alloc_internal(size, align,
  1432. min_addr, max_addr, nid, false);
  1433. if (ptr)
  1434. memset(ptr, 0, size);
  1435. return ptr;
  1436. }
  1437. /**
  1438. * memblock_free_late - free pages directly to buddy allocator
  1439. * @base: phys starting address of the boot memory block
  1440. * @size: size of the boot memory block in bytes
  1441. *
  1442. * This is only useful when the memblock allocator has already been torn
  1443. * down, but we are still initializing the system. Pages are released directly
  1444. * to the buddy allocator.
  1445. */
  1446. void __init memblock_free_late(phys_addr_t base, phys_addr_t size)
  1447. {
  1448. phys_addr_t cursor, end;
  1449. end = base + size - 1;
  1450. memblock_dbg("%s: [%pa-%pa] %pS\n",
  1451. __func__, &base, &end, (void *)_RET_IP_);
  1452. kmemleak_free_part_phys(base, size);
  1453. cursor = PFN_UP(base);
  1454. end = PFN_DOWN(base + size);
  1455. for (; cursor < end; cursor++) {
  1456. memblock_free_pages(pfn_to_page(cursor), cursor, 0);
  1457. totalram_pages_inc();
  1458. }
  1459. }
  1460. /*
  1461. * Remaining API functions
  1462. */
  1463. phys_addr_t __init_memblock memblock_phys_mem_size(void)
  1464. {
  1465. return memblock.memory.total_size;
  1466. }
  1467. phys_addr_t __init_memblock memblock_reserved_size(void)
  1468. {
  1469. return memblock.reserved.total_size;
  1470. }
  1471. /* lowest address */
  1472. phys_addr_t __init_memblock memblock_start_of_DRAM(void)
  1473. {
  1474. return memblock.memory.regions[0].base;
  1475. }
  1476. phys_addr_t __init_memblock memblock_end_of_DRAM(void)
  1477. {
  1478. int idx = memblock.memory.cnt - 1;
  1479. return (memblock.memory.regions[idx].base + memblock.memory.regions[idx].size);
  1480. }
  1481. EXPORT_SYMBOL_GPL(memblock_end_of_DRAM);
  1482. static phys_addr_t __init_memblock __find_max_addr(phys_addr_t limit)
  1483. {
  1484. phys_addr_t max_addr = PHYS_ADDR_MAX;
  1485. struct memblock_region *r;
  1486. /*
  1487. * translate the memory @limit size into the max address within one of
  1488. * the memory memblock regions, if the @limit exceeds the total size
  1489. * of those regions, max_addr will keep original value PHYS_ADDR_MAX
  1490. */
  1491. for_each_mem_region(r) {
  1492. if (limit <= r->size) {
  1493. max_addr = r->base + limit;
  1494. break;
  1495. }
  1496. limit -= r->size;
  1497. }
  1498. return max_addr;
  1499. }
  1500. void __init memblock_enforce_memory_limit(phys_addr_t limit)
  1501. {
  1502. phys_addr_t max_addr;
  1503. if (!limit)
  1504. return;
  1505. max_addr = __find_max_addr(limit);
  1506. /* @limit exceeds the total size of the memory, do nothing */
  1507. if (max_addr == PHYS_ADDR_MAX)
  1508. return;
  1509. /* truncate both memory and reserved regions */
  1510. memblock_remove_range(&memblock.memory, max_addr,
  1511. PHYS_ADDR_MAX);
  1512. memblock_remove_range(&memblock.reserved, max_addr,
  1513. PHYS_ADDR_MAX);
  1514. }
  1515. void __init memblock_cap_memory_range(phys_addr_t base, phys_addr_t size)
  1516. {
  1517. int start_rgn, end_rgn;
  1518. int i, ret;
  1519. if (!size)
  1520. return;
  1521. if (!memblock_memory->total_size) {
  1522. pr_warn("%s: No memory registered yet\n", __func__);
  1523. return;
  1524. }
  1525. ret = memblock_isolate_range(&memblock.memory, base, size,
  1526. &start_rgn, &end_rgn);
  1527. if (ret)
  1528. return;
  1529. /* remove all the MAP regions */
  1530. for (i = memblock.memory.cnt - 1; i >= end_rgn; i--)
  1531. if (!memblock_is_nomap(&memblock.memory.regions[i]))
  1532. memblock_remove_region(&memblock.memory, i);
  1533. for (i = start_rgn - 1; i >= 0; i--)
  1534. if (!memblock_is_nomap(&memblock.memory.regions[i]))
  1535. memblock_remove_region(&memblock.memory, i);
  1536. /* truncate the reserved regions */
  1537. memblock_remove_range(&memblock.reserved, 0, base);
  1538. memblock_remove_range(&memblock.reserved,
  1539. base + size, PHYS_ADDR_MAX);
  1540. }
  1541. void __init memblock_mem_limit_remove_map(phys_addr_t limit)
  1542. {
  1543. phys_addr_t max_addr;
  1544. if (!limit)
  1545. return;
  1546. max_addr = __find_max_addr(limit);
  1547. /* @limit exceeds the total size of the memory, do nothing */
  1548. if (max_addr == PHYS_ADDR_MAX)
  1549. return;
  1550. memblock_cap_memory_range(0, max_addr);
  1551. }
  1552. static int __init_memblock memblock_search(struct memblock_type *type, phys_addr_t addr)
  1553. {
  1554. unsigned int left = 0, right = type->cnt;
  1555. do {
  1556. unsigned int mid = (right + left) / 2;
  1557. if (addr < type->regions[mid].base)
  1558. right = mid;
  1559. else if (addr >= (type->regions[mid].base +
  1560. type->regions[mid].size))
  1561. left = mid + 1;
  1562. else
  1563. return mid;
  1564. } while (left < right);
  1565. return -1;
  1566. }
  1567. bool __init_memblock memblock_is_reserved(phys_addr_t addr)
  1568. {
  1569. return memblock_search(&memblock.reserved, addr) != -1;
  1570. }
  1571. bool __init_memblock memblock_is_memory(phys_addr_t addr)
  1572. {
  1573. return memblock_search(&memblock.memory, addr) != -1;
  1574. }
  1575. bool __init_memblock memblock_is_map_memory(phys_addr_t addr)
  1576. {
  1577. int i = memblock_search(&memblock.memory, addr);
  1578. if (i == -1)
  1579. return false;
  1580. return !memblock_is_nomap(&memblock.memory.regions[i]);
  1581. }
  1582. int __init_memblock memblock_search_pfn_nid(unsigned long pfn,
  1583. unsigned long *start_pfn, unsigned long *end_pfn)
  1584. {
  1585. struct memblock_type *type = &memblock.memory;
  1586. int mid = memblock_search(type, PFN_PHYS(pfn));
  1587. if (mid == -1)
  1588. return -1;
  1589. *start_pfn = PFN_DOWN(type->regions[mid].base);
  1590. *end_pfn = PFN_DOWN(type->regions[mid].base + type->regions[mid].size);
  1591. return memblock_get_region_node(&type->regions[mid]);
  1592. }
  1593. /**
  1594. * memblock_is_region_memory - check if a region is a subset of memory
  1595. * @base: base of region to check
  1596. * @size: size of region to check
  1597. *
  1598. * Check if the region [@base, @base + @size) is a subset of a memory block.
  1599. *
  1600. * Return:
  1601. * 0 if false, non-zero if true
  1602. */
  1603. bool __init_memblock memblock_is_region_memory(phys_addr_t base, phys_addr_t size)
  1604. {
  1605. int idx = memblock_search(&memblock.memory, base);
  1606. phys_addr_t end = base + memblock_cap_size(base, &size);
  1607. if (idx == -1)
  1608. return false;
  1609. return (memblock.memory.regions[idx].base +
  1610. memblock.memory.regions[idx].size) >= end;
  1611. }
  1612. /**
  1613. * memblock_is_region_reserved - check if a region intersects reserved memory
  1614. * @base: base of region to check
  1615. * @size: size of region to check
  1616. *
  1617. * Check if the region [@base, @base + @size) intersects a reserved
  1618. * memory block.
  1619. *
  1620. * Return:
  1621. * True if they intersect, false if not.
  1622. */
  1623. bool __init_memblock memblock_is_region_reserved(phys_addr_t base, phys_addr_t size)
  1624. {
  1625. return memblock_overlaps_region(&memblock.reserved, base, size);
  1626. }
  1627. void __init_memblock memblock_trim_memory(phys_addr_t align)
  1628. {
  1629. phys_addr_t start, end, orig_start, orig_end;
  1630. struct memblock_region *r;
  1631. for_each_mem_region(r) {
  1632. orig_start = r->base;
  1633. orig_end = r->base + r->size;
  1634. start = round_up(orig_start, align);
  1635. end = round_down(orig_end, align);
  1636. if (start == orig_start && end == orig_end)
  1637. continue;
  1638. if (start < end) {
  1639. r->base = start;
  1640. r->size = end - start;
  1641. } else {
  1642. memblock_remove_region(&memblock.memory,
  1643. r - memblock.memory.regions);
  1644. r--;
  1645. }
  1646. }
  1647. }
  1648. void __init_memblock memblock_set_current_limit(phys_addr_t limit)
  1649. {
  1650. memblock.current_limit = limit;
  1651. }
  1652. phys_addr_t __init_memblock memblock_get_current_limit(void)
  1653. {
  1654. return memblock.current_limit;
  1655. }
  1656. static void __init_memblock memblock_dump(struct memblock_type *type)
  1657. {
  1658. phys_addr_t base, end, size;
  1659. enum memblock_flags flags;
  1660. int idx;
  1661. struct memblock_region *rgn;
  1662. pr_info(" %s.cnt = 0x%lx\n", type->name, type->cnt);
  1663. for_each_memblock_type(idx, type, rgn) {
  1664. char nid_buf[32] = "";
  1665. base = rgn->base;
  1666. size = rgn->size;
  1667. end = base + size - 1;
  1668. flags = rgn->flags;
  1669. #ifdef CONFIG_NUMA
  1670. if (memblock_get_region_node(rgn) != MAX_NUMNODES)
  1671. snprintf(nid_buf, sizeof(nid_buf), " on node %d",
  1672. memblock_get_region_node(rgn));
  1673. #endif
  1674. pr_info(" %s[%#x]\t[%pa-%pa], %pa bytes%s flags: %#x\n",
  1675. type->name, idx, &base, &end, &size, nid_buf, flags);
  1676. }
  1677. }
  1678. static void __init_memblock __memblock_dump_all(void)
  1679. {
  1680. pr_info("MEMBLOCK configuration:\n");
  1681. pr_info(" memory size = %pa reserved size = %pa\n",
  1682. &memblock.memory.total_size,
  1683. &memblock.reserved.total_size);
  1684. memblock_dump(&memblock.memory);
  1685. memblock_dump(&memblock.reserved);
  1686. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  1687. memblock_dump(&physmem);
  1688. #endif
  1689. }
  1690. void __init_memblock memblock_dump_all(void)
  1691. {
  1692. if (memblock_debug)
  1693. __memblock_dump_all();
  1694. }
  1695. void __init memblock_allow_resize(void)
  1696. {
  1697. memblock_can_resize = 1;
  1698. }
  1699. static int __init early_memblock(char *p)
  1700. {
  1701. if (p && strstr(p, "debug"))
  1702. memblock_debug = 1;
  1703. return 0;
  1704. }
  1705. early_param("memblock", early_memblock);
  1706. static void __init free_memmap(unsigned long start_pfn, unsigned long end_pfn)
  1707. {
  1708. struct page *start_pg, *end_pg;
  1709. phys_addr_t pg, pgend;
  1710. /*
  1711. * Convert start_pfn/end_pfn to a struct page pointer.
  1712. */
  1713. start_pg = pfn_to_page(start_pfn - 1) + 1;
  1714. end_pg = pfn_to_page(end_pfn - 1) + 1;
  1715. /*
  1716. * Convert to physical addresses, and round start upwards and end
  1717. * downwards.
  1718. */
  1719. pg = PAGE_ALIGN(__pa(start_pg));
  1720. pgend = __pa(end_pg) & PAGE_MASK;
  1721. /*
  1722. * If there are free pages between these, free the section of the
  1723. * memmap array.
  1724. */
  1725. if (pg < pgend)
  1726. memblock_phys_free(pg, pgend - pg);
  1727. }
  1728. /*
  1729. * The mem_map array can get very big. Free the unused area of the memory map.
  1730. */
  1731. static void __init free_unused_memmap(void)
  1732. {
  1733. unsigned long start, end, prev_end = 0;
  1734. int i;
  1735. if (!IS_ENABLED(CONFIG_HAVE_ARCH_PFN_VALID) ||
  1736. IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP))
  1737. return;
  1738. /*
  1739. * This relies on each bank being in address order.
  1740. * The banks are sorted previously in bootmem_init().
  1741. */
  1742. for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, NULL) {
  1743. #ifdef CONFIG_SPARSEMEM
  1744. /*
  1745. * Take care not to free memmap entries that don't exist
  1746. * due to SPARSEMEM sections which aren't present.
  1747. */
  1748. start = min(start, ALIGN(prev_end, PAGES_PER_SECTION));
  1749. #endif
  1750. /*
  1751. * Align down here since many operations in VM subsystem
  1752. * presume that there are no holes in the memory map inside
  1753. * a pageblock
  1754. */
  1755. start = pageblock_start_pfn(start);
  1756. /*
  1757. * If we had a previous bank, and there is a space
  1758. * between the current bank and the previous, free it.
  1759. */
  1760. if (prev_end && prev_end < start)
  1761. free_memmap(prev_end, start);
  1762. /*
  1763. * Align up here since many operations in VM subsystem
  1764. * presume that there are no holes in the memory map inside
  1765. * a pageblock
  1766. */
  1767. prev_end = pageblock_align(end);
  1768. }
  1769. #ifdef CONFIG_SPARSEMEM
  1770. if (!IS_ALIGNED(prev_end, PAGES_PER_SECTION)) {
  1771. prev_end = pageblock_align(end);
  1772. free_memmap(prev_end, ALIGN(prev_end, PAGES_PER_SECTION));
  1773. }
  1774. #endif
  1775. }
  1776. static void __init __free_pages_memory(unsigned long start, unsigned long end)
  1777. {
  1778. int order;
  1779. while (start < end) {
  1780. order = min(MAX_ORDER - 1UL, __ffs(start));
  1781. while (start + (1UL << order) > end)
  1782. order--;
  1783. memblock_free_pages(pfn_to_page(start), start, order);
  1784. start += (1UL << order);
  1785. }
  1786. }
  1787. static unsigned long __init __free_memory_core(phys_addr_t start,
  1788. phys_addr_t end)
  1789. {
  1790. unsigned long start_pfn = PFN_UP(start);
  1791. unsigned long end_pfn = min_t(unsigned long,
  1792. PFN_DOWN(end), max_low_pfn);
  1793. if (start_pfn >= end_pfn)
  1794. return 0;
  1795. __free_pages_memory(start_pfn, end_pfn);
  1796. return end_pfn - start_pfn;
  1797. }
  1798. static void __init memmap_init_reserved_pages(void)
  1799. {
  1800. struct memblock_region *region;
  1801. phys_addr_t start, end;
  1802. u64 i;
  1803. /* initialize struct pages for the reserved regions */
  1804. for_each_reserved_mem_range(i, &start, &end)
  1805. reserve_bootmem_region(start, end);
  1806. /* and also treat struct pages for the NOMAP regions as PageReserved */
  1807. for_each_mem_region(region) {
  1808. if (memblock_is_nomap(region)) {
  1809. start = region->base;
  1810. end = start + region->size;
  1811. reserve_bootmem_region(start, end);
  1812. }
  1813. }
  1814. }
  1815. static unsigned long __init free_low_memory_core_early(void)
  1816. {
  1817. unsigned long count = 0;
  1818. phys_addr_t start, end;
  1819. u64 i;
  1820. memblock_clear_hotplug(0, -1);
  1821. memmap_init_reserved_pages();
  1822. /*
  1823. * We need to use NUMA_NO_NODE instead of NODE_DATA(0)->node_id
  1824. * because in some case like Node0 doesn't have RAM installed
  1825. * low ram will be on Node1
  1826. */
  1827. for_each_free_mem_range(i, NUMA_NO_NODE, MEMBLOCK_NONE, &start, &end,
  1828. NULL)
  1829. count += __free_memory_core(start, end);
  1830. return count;
  1831. }
  1832. static int reset_managed_pages_done __initdata;
  1833. void reset_node_managed_pages(pg_data_t *pgdat)
  1834. {
  1835. struct zone *z;
  1836. for (z = pgdat->node_zones; z < pgdat->node_zones + MAX_NR_ZONES; z++)
  1837. atomic_long_set(&z->managed_pages, 0);
  1838. }
  1839. void __init reset_all_zones_managed_pages(void)
  1840. {
  1841. struct pglist_data *pgdat;
  1842. if (reset_managed_pages_done)
  1843. return;
  1844. for_each_online_pgdat(pgdat)
  1845. reset_node_managed_pages(pgdat);
  1846. reset_managed_pages_done = 1;
  1847. }
  1848. /**
  1849. * memblock_free_all - release free pages to the buddy allocator
  1850. */
  1851. void __init memblock_free_all(void)
  1852. {
  1853. unsigned long pages;
  1854. free_unused_memmap();
  1855. reset_all_zones_managed_pages();
  1856. pages = free_low_memory_core_early();
  1857. totalram_pages_add(pages);
  1858. }
  1859. #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_ARCH_KEEP_MEMBLOCK)
  1860. static int memblock_debug_show(struct seq_file *m, void *private)
  1861. {
  1862. struct memblock_type *type = m->private;
  1863. struct memblock_region *reg;
  1864. int i;
  1865. phys_addr_t end;
  1866. for (i = 0; i < type->cnt; i++) {
  1867. reg = &type->regions[i];
  1868. end = reg->base + reg->size - 1;
  1869. seq_printf(m, "%4d: ", i);
  1870. seq_printf(m, "%pa..%pa\n", &reg->base, &end);
  1871. }
  1872. return 0;
  1873. }
  1874. DEFINE_SHOW_ATTRIBUTE(memblock_debug);
  1875. static int __init memblock_init_debugfs(void)
  1876. {
  1877. struct dentry *root = debugfs_create_dir("memblock", NULL);
  1878. debugfs_create_file("memory", 0444, root,
  1879. &memblock.memory, &memblock_debug_fops);
  1880. debugfs_create_file("reserved", 0444, root,
  1881. &memblock.reserved, &memblock_debug_fops);
  1882. #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP
  1883. debugfs_create_file("physmem", 0444, root, &physmem,
  1884. &memblock_debug_fops);
  1885. #endif
  1886. return 0;
  1887. }
  1888. __initcall(memblock_init_debugfs);
  1889. #endif /* CONFIG_DEBUG_FS */