page_alloc.c 276 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807880888098810881188128813881488158816881788188819882088218822882388248825882688278828882988308831883288338834883588368837883888398840884188428843884488458846884788488849885088518852885388548855885688578858885988608861886288638864886588668867886888698870887188728873887488758876887788788879888088818882888388848885888688878888888988908891889288938894889588968897889888998900890189028903890489058906890789088909891089118912891389148915891689178918891989208921892289238924892589268927892889298930893189328933893489358936893789388939894089418942894389448945894689478948894989508951895289538954895589568957895889598960896189628963896489658966896789688969897089718972897389748975897689778978897989808981898289838984898589868987898889898990899189928993899489958996899789988999900090019002900390049005900690079008900990109011901290139014901590169017901890199020902190229023902490259026902790289029903090319032903390349035903690379038903990409041904290439044904590469047904890499050905190529053905490559056905790589059906090619062906390649065906690679068906990709071907290739074907590769077907890799080908190829083908490859086908790889089909090919092909390949095909690979098909991009101910291039104910591069107910891099110911191129113911491159116911791189119912091219122912391249125912691279128912991309131913291339134913591369137913891399140914191429143914491459146914791489149915091519152915391549155915691579158915991609161916291639164916591669167916891699170917191729173917491759176917791789179918091819182918391849185918691879188918991909191919291939194919591969197919891999200920192029203920492059206920792089209921092119212921392149215921692179218921992209221922292239224922592269227922892299230923192329233923492359236923792389239924092419242924392449245924692479248924992509251925292539254925592569257925892599260926192629263926492659266926792689269927092719272927392749275927692779278927992809281928292839284928592869287928892899290929192929293929492959296929792989299930093019302930393049305930693079308930993109311931293139314931593169317931893199320932193229323932493259326932793289329933093319332933393349335933693379338933993409341934293439344934593469347934893499350935193529353935493559356935793589359936093619362936393649365936693679368936993709371937293739374937593769377937893799380938193829383938493859386938793889389939093919392939393949395939693979398939994009401940294039404940594069407940894099410941194129413941494159416941794189419942094219422942394249425942694279428942994309431943294339434943594369437943894399440944194429443944494459446944794489449945094519452945394549455945694579458945994609461946294639464946594669467946894699470947194729473947494759476947794789479948094819482948394849485948694879488948994909491949294939494949594969497949894999500950195029503950495059506950795089509951095119512951395149515951695179518951995209521952295239524952595269527952895299530953195329533953495359536953795389539954095419542954395449545954695479548954995509551955295539554955595569557955895599560956195629563956495659566956795689569957095719572957395749575957695779578957995809581958295839584958595869587958895899590959195929593959495959596959795989599960096019602960396049605960696079608960996109611961296139614961596169617961896199620962196229623962496259626962796289629963096319632963396349635963696379638963996409641964296439644964596469647964896499650965196529653965496559656965796589659966096619662966396649665966696679668966996709671967296739674967596769677967896799680968196829683968496859686968796889689969096919692969396949695969696979698969997009701970297039704970597069707970897099710971197129713971497159716971797189719972097219722972397249725972697279728972997309731973297339734973597369737973897399740974197429743974497459746974797489749975097519752975397549755975697579758975997609761976297639764976597669767976897699770977197729773977497759776977797789779978097819782978397849785978697879788978997909791979297939794979597969797979897999800980198029803980498059806980798089809981098119812981398149815981698179818981998209821982298239824982598269827982898299830983198329833983498359836983798389839984098419842984398449845984698479848984998509851985298539854985598569857985898599860986198629863986498659866986798689869987098719872987398749875987698779878987998809881988298839884988598869887988898899890989198929893989498959896989798989899990099019902990399049905990699079908990999109911991299139914991599169917991899199920992199229923992499259926992799289929993099319932
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/mm/page_alloc.c
  4. *
  5. * Manages the free list, the system allocates free pages here.
  6. * Note that kmalloc() lives in slab.c
  7. *
  8. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  9. * Swap reorganised 29.12.95, Stephen Tweedie
  10. * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
  11. * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
  12. * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
  13. * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
  14. * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
  15. * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
  16. */
  17. #include <linux/stddef.h>
  18. #include <linux/mm.h>
  19. #include <linux/highmem.h>
  20. #include <linux/swap.h>
  21. #include <linux/swapops.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/pagemap.h>
  24. #include <linux/jiffies.h>
  25. #include <linux/memblock.h>
  26. #include <linux/compiler.h>
  27. #include <linux/kernel.h>
  28. #include <linux/kasan.h>
  29. #include <linux/kmsan.h>
  30. #include <linux/module.h>
  31. #include <linux/suspend.h>
  32. #include <linux/pagevec.h>
  33. #include <linux/blkdev.h>
  34. #include <linux/slab.h>
  35. #include <linux/ratelimit.h>
  36. #include <linux/oom.h>
  37. #include <linux/topology.h>
  38. #include <linux/sysctl.h>
  39. #include <linux/cpu.h>
  40. #include <linux/cpuset.h>
  41. #include <linux/memory_hotplug.h>
  42. #include <linux/nodemask.h>
  43. #include <linux/vmalloc.h>
  44. #include <linux/vmstat.h>
  45. #include <linux/mempolicy.h>
  46. #include <linux/memremap.h>
  47. #include <linux/stop_machine.h>
  48. #include <linux/random.h>
  49. #include <linux/sort.h>
  50. #include <linux/pfn.h>
  51. #include <linux/backing-dev.h>
  52. #include <linux/fault-inject.h>
  53. #include <linux/page-isolation.h>
  54. #include <linux/debugobjects.h>
  55. #include <linux/kmemleak.h>
  56. #include <linux/compaction.h>
  57. #include <trace/events/kmem.h>
  58. #include <trace/events/oom.h>
  59. #include <linux/prefetch.h>
  60. #include <linux/mm_inline.h>
  61. #include <linux/mmu_notifier.h>
  62. #include <linux/migrate.h>
  63. #include <linux/hugetlb.h>
  64. #include <linux/sched/rt.h>
  65. #include <linux/sched/mm.h>
  66. #include <linux/page_owner.h>
  67. #include <linux/page_table_check.h>
  68. #include <linux/kthread.h>
  69. #include <linux/memcontrol.h>
  70. #include <linux/ftrace.h>
  71. #include <linux/lockdep.h>
  72. #include <linux/nmi.h>
  73. #include <linux/psi.h>
  74. #include <linux/padata.h>
  75. #include <linux/khugepaged.h>
  76. #include <linux/buffer_head.h>
  77. #include <linux/delayacct.h>
  78. #include <trace/hooks/mm.h>
  79. #include <trace/hooks/vmscan.h>
  80. #include <asm/sections.h>
  81. #include <asm/tlbflush.h>
  82. #include <asm/div64.h>
  83. #include "internal.h"
  84. #include "shuffle.h"
  85. #include "page_reporting.h"
  86. #include "swap.h"
  87. #undef CREATE_TRACE_POINTS
  88. #include <trace/hooks/mm.h>
  89. /* Free Page Internal flags: for internal, non-pcp variants of free_pages(). */
  90. typedef int __bitwise fpi_t;
  91. /* No special request */
  92. #define FPI_NONE ((__force fpi_t)0)
  93. /*
  94. * Skip free page reporting notification for the (possibly merged) page.
  95. * This does not hinder free page reporting from grabbing the page,
  96. * reporting it and marking it "reported" - it only skips notifying
  97. * the free page reporting infrastructure about a newly freed page. For
  98. * example, used when temporarily pulling a page from a freelist and
  99. * putting it back unmodified.
  100. */
  101. #define FPI_SKIP_REPORT_NOTIFY ((__force fpi_t)BIT(0))
  102. /*
  103. * Place the (possibly merged) page to the tail of the freelist. Will ignore
  104. * page shuffling (relevant code - e.g., memory onlining - is expected to
  105. * shuffle the whole zone).
  106. *
  107. * Note: No code should rely on this flag for correctness - it's purely
  108. * to allow for optimizations when handing back either fresh pages
  109. * (memory onlining) or untouched pages (page isolation, free page
  110. * reporting).
  111. */
  112. #define FPI_TO_TAIL ((__force fpi_t)BIT(1))
  113. /*
  114. * Don't poison memory with KASAN (only for the tag-based modes).
  115. * During boot, all non-reserved memblock memory is exposed to page_alloc.
  116. * Poisoning all that memory lengthens boot time, especially on systems with
  117. * large amount of RAM. This flag is used to skip that poisoning.
  118. * This is only done for the tag-based KASAN modes, as those are able to
  119. * detect memory corruptions with the memory tags assigned by default.
  120. * All memory allocated normally after boot gets poisoned as usual.
  121. */
  122. #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2))
  123. /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
  124. static DEFINE_MUTEX(pcp_batch_high_lock);
  125. #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
  126. #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
  127. /*
  128. * On SMP, spin_trylock is sufficient protection.
  129. * On PREEMPT_RT, spin_trylock is equivalent on both SMP and UP.
  130. */
  131. #define pcp_trylock_prepare(flags) do { } while (0)
  132. #define pcp_trylock_finish(flag) do { } while (0)
  133. #else
  134. /* UP spin_trylock always succeeds so disable IRQs to prevent re-entrancy. */
  135. #define pcp_trylock_prepare(flags) local_irq_save(flags)
  136. #define pcp_trylock_finish(flags) local_irq_restore(flags)
  137. #endif
  138. /*
  139. * Locking a pcp requires a PCP lookup followed by a spinlock. To avoid
  140. * a migration causing the wrong PCP to be locked and remote memory being
  141. * potentially allocated, pin the task to the CPU for the lookup+lock.
  142. * preempt_disable is used on !RT because it is faster than migrate_disable.
  143. * migrate_disable is used on RT because otherwise RT spinlock usage is
  144. * interfered with and a high priority task cannot preempt the allocator.
  145. */
  146. #ifndef CONFIG_PREEMPT_RT
  147. #define pcpu_task_pin() preempt_disable()
  148. #define pcpu_task_unpin() preempt_enable()
  149. #else
  150. #define pcpu_task_pin() migrate_disable()
  151. #define pcpu_task_unpin() migrate_enable()
  152. #endif
  153. /*
  154. * Generic helper to lookup and a per-cpu variable with an embedded spinlock.
  155. * Return value should be used with equivalent unlock helper.
  156. */
  157. #define pcpu_spin_lock(type, member, ptr) \
  158. ({ \
  159. type *_ret; \
  160. pcpu_task_pin(); \
  161. _ret = this_cpu_ptr(ptr); \
  162. spin_lock(&_ret->member); \
  163. _ret; \
  164. })
  165. #define pcpu_spin_trylock(type, member, ptr) \
  166. ({ \
  167. type *_ret; \
  168. pcpu_task_pin(); \
  169. _ret = this_cpu_ptr(ptr); \
  170. if (!spin_trylock(&_ret->member)) { \
  171. pcpu_task_unpin(); \
  172. _ret = NULL; \
  173. } \
  174. _ret; \
  175. })
  176. #define pcpu_spin_unlock(member, ptr) \
  177. ({ \
  178. spin_unlock(&ptr->member); \
  179. pcpu_task_unpin(); \
  180. })
  181. /* struct per_cpu_pages specific helpers. */
  182. #define pcp_spin_lock(ptr) \
  183. pcpu_spin_lock(struct per_cpu_pages, lock, ptr)
  184. #define pcp_spin_trylock(ptr) \
  185. pcpu_spin_trylock(struct per_cpu_pages, lock, ptr)
  186. #define pcp_spin_unlock(ptr) \
  187. pcpu_spin_unlock(lock, ptr)
  188. #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
  189. DEFINE_PER_CPU(int, numa_node);
  190. EXPORT_PER_CPU_SYMBOL(numa_node);
  191. #endif
  192. DEFINE_STATIC_KEY_TRUE(vm_numa_stat_key);
  193. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  194. /*
  195. * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
  196. * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
  197. * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
  198. * defined in <linux/topology.h>.
  199. */
  200. DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
  201. EXPORT_PER_CPU_SYMBOL(_numa_mem_);
  202. #endif
  203. static DEFINE_MUTEX(pcpu_drain_mutex);
  204. #ifdef CONFIG_GCC_PLUGIN_LATENT_ENTROPY
  205. volatile unsigned long latent_entropy __latent_entropy;
  206. EXPORT_SYMBOL(latent_entropy);
  207. #endif
  208. /*
  209. * Array of node states.
  210. */
  211. nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
  212. [N_POSSIBLE] = NODE_MASK_ALL,
  213. [N_ONLINE] = { { [0] = 1UL } },
  214. #ifndef CONFIG_NUMA
  215. [N_NORMAL_MEMORY] = { { [0] = 1UL } },
  216. #ifdef CONFIG_HIGHMEM
  217. [N_HIGH_MEMORY] = { { [0] = 1UL } },
  218. #endif
  219. [N_MEMORY] = { { [0] = 1UL } },
  220. [N_CPU] = { { [0] = 1UL } },
  221. #endif /* NUMA */
  222. };
  223. EXPORT_SYMBOL(node_states);
  224. atomic_long_t _totalram_pages __read_mostly;
  225. EXPORT_SYMBOL(_totalram_pages);
  226. unsigned long totalreserve_pages __read_mostly;
  227. unsigned long totalcma_pages __read_mostly;
  228. int percpu_pagelist_high_fraction;
  229. gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
  230. DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_ALLOC_DEFAULT_ON, init_on_alloc);
  231. EXPORT_SYMBOL(init_on_alloc);
  232. DEFINE_STATIC_KEY_MAYBE(CONFIG_INIT_ON_FREE_DEFAULT_ON, init_on_free);
  233. EXPORT_SYMBOL(init_on_free);
  234. static bool _init_on_alloc_enabled_early __read_mostly
  235. = IS_ENABLED(CONFIG_INIT_ON_ALLOC_DEFAULT_ON);
  236. static int __init early_init_on_alloc(char *buf)
  237. {
  238. return kstrtobool(buf, &_init_on_alloc_enabled_early);
  239. }
  240. early_param("init_on_alloc", early_init_on_alloc);
  241. static bool _init_on_free_enabled_early __read_mostly
  242. = IS_ENABLED(CONFIG_INIT_ON_FREE_DEFAULT_ON);
  243. static int __init early_init_on_free(char *buf)
  244. {
  245. return kstrtobool(buf, &_init_on_free_enabled_early);
  246. }
  247. early_param("init_on_free", early_init_on_free);
  248. /*
  249. * A cached value of the page's pageblock's migratetype, used when the page is
  250. * put on a pcplist. Used to avoid the pageblock migratetype lookup when
  251. * freeing from pcplists in most cases, at the cost of possibly becoming stale.
  252. * Also the migratetype set in the page does not necessarily match the pcplist
  253. * index, e.g. page might have MIGRATE_CMA set but be on a pcplist with any
  254. * other index - this ensures that it will be put on the correct CMA freelist.
  255. */
  256. static inline int get_pcppage_migratetype(struct page *page)
  257. {
  258. return page->index;
  259. }
  260. static inline void set_pcppage_migratetype(struct page *page, int migratetype)
  261. {
  262. page->index = migratetype;
  263. }
  264. #ifdef CONFIG_PM_SLEEP
  265. /*
  266. * The following functions are used by the suspend/hibernate code to temporarily
  267. * change gfp_allowed_mask in order to avoid using I/O during memory allocations
  268. * while devices are suspended. To avoid races with the suspend/hibernate code,
  269. * they should always be called with system_transition_mutex held
  270. * (gfp_allowed_mask also should only be modified with system_transition_mutex
  271. * held, unless the suspend/hibernate code is guaranteed not to run in parallel
  272. * with that modification).
  273. */
  274. static gfp_t saved_gfp_mask;
  275. void pm_restore_gfp_mask(void)
  276. {
  277. WARN_ON(!mutex_is_locked(&system_transition_mutex));
  278. if (saved_gfp_mask) {
  279. gfp_allowed_mask = saved_gfp_mask;
  280. saved_gfp_mask = 0;
  281. }
  282. }
  283. void pm_restrict_gfp_mask(void)
  284. {
  285. WARN_ON(!mutex_is_locked(&system_transition_mutex));
  286. WARN_ON(saved_gfp_mask);
  287. saved_gfp_mask = gfp_allowed_mask;
  288. gfp_allowed_mask &= ~(__GFP_IO | __GFP_FS);
  289. }
  290. bool pm_suspended_storage(void)
  291. {
  292. if ((gfp_allowed_mask & (__GFP_IO | __GFP_FS)) == (__GFP_IO | __GFP_FS))
  293. return false;
  294. return true;
  295. }
  296. #endif /* CONFIG_PM_SLEEP */
  297. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  298. unsigned int pageblock_order __read_mostly;
  299. #endif
  300. static void __free_pages_ok(struct page *page, unsigned int order,
  301. fpi_t fpi_flags);
  302. /*
  303. * results with 256, 32 in the lowmem_reserve sysctl:
  304. * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
  305. * 1G machine -> (16M dma, 784M normal, 224M high)
  306. * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
  307. * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
  308. * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA
  309. *
  310. * TBD: should special case ZONE_DMA32 machines here - in those we normally
  311. * don't need any ZONE_NORMAL reservation
  312. */
  313. int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES] = {
  314. #ifdef CONFIG_ZONE_DMA
  315. [ZONE_DMA] = 256,
  316. #endif
  317. #ifdef CONFIG_ZONE_DMA32
  318. [ZONE_DMA32] = 256,
  319. #endif
  320. [ZONE_NORMAL] = 32,
  321. #ifdef CONFIG_HIGHMEM
  322. [ZONE_HIGHMEM] = 0,
  323. #endif
  324. [ZONE_MOVABLE] = 0,
  325. };
  326. static char * const zone_names[MAX_NR_ZONES] = {
  327. #ifdef CONFIG_ZONE_DMA
  328. "DMA",
  329. #endif
  330. #ifdef CONFIG_ZONE_DMA32
  331. "DMA32",
  332. #endif
  333. "Normal",
  334. #ifdef CONFIG_HIGHMEM
  335. "HighMem",
  336. #endif
  337. "Movable",
  338. #ifdef CONFIG_ZONE_DEVICE
  339. "Device",
  340. #endif
  341. };
  342. const char * const migratetype_names[MIGRATE_TYPES] = {
  343. "Unmovable",
  344. "Movable",
  345. "Reclaimable",
  346. #ifdef CONFIG_CMA
  347. "CMA",
  348. #endif
  349. "HighAtomic",
  350. #ifdef CONFIG_MEMORY_ISOLATION
  351. "Isolate",
  352. #endif
  353. };
  354. compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
  355. [NULL_COMPOUND_DTOR] = NULL,
  356. [COMPOUND_PAGE_DTOR] = free_compound_page,
  357. #ifdef CONFIG_HUGETLB_PAGE
  358. [HUGETLB_PAGE_DTOR] = free_huge_page,
  359. #endif
  360. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  361. [TRANSHUGE_PAGE_DTOR] = free_transhuge_page,
  362. #endif
  363. };
  364. int min_free_kbytes = 1024;
  365. int user_min_free_kbytes = -1;
  366. #ifdef CONFIG_ARCH_QTI_VM
  367. int watermark_boost_factor __read_mostly = 0;
  368. #else
  369. int watermark_boost_factor __read_mostly = 15000;
  370. #endif
  371. int watermark_scale_factor = 10;
  372. static unsigned long nr_kernel_pages __initdata;
  373. static unsigned long nr_all_pages __initdata;
  374. static unsigned long dma_reserve __initdata;
  375. static unsigned long arch_zone_lowest_possible_pfn[MAX_NR_ZONES] __initdata;
  376. static unsigned long arch_zone_highest_possible_pfn[MAX_NR_ZONES] __initdata;
  377. static unsigned long required_kernelcore __initdata;
  378. static unsigned long required_kernelcore_percent __initdata;
  379. static unsigned long required_movablecore __initdata;
  380. static unsigned long required_movablecore_percent __initdata;
  381. static unsigned long zone_movable_pfn[MAX_NUMNODES] __initdata;
  382. bool mirrored_kernelcore __initdata_memblock;
  383. /* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
  384. int movable_zone;
  385. EXPORT_SYMBOL(movable_zone);
  386. #if MAX_NUMNODES > 1
  387. unsigned int nr_node_ids __read_mostly = MAX_NUMNODES;
  388. unsigned int nr_online_nodes __read_mostly = 1;
  389. EXPORT_SYMBOL(nr_node_ids);
  390. EXPORT_SYMBOL(nr_online_nodes);
  391. #endif
  392. int page_group_by_mobility_disabled __read_mostly;
  393. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  394. /*
  395. * During boot we initialize deferred pages on-demand, as needed, but once
  396. * page_alloc_init_late() has finished, the deferred pages are all initialized,
  397. * and we can permanently disable that path.
  398. */
  399. static DEFINE_STATIC_KEY_TRUE(deferred_pages);
  400. static inline bool deferred_pages_enabled(void)
  401. {
  402. return static_branch_unlikely(&deferred_pages);
  403. }
  404. /* Returns true if the struct page for the pfn is uninitialised */
  405. static inline bool __meminit early_page_uninitialised(unsigned long pfn)
  406. {
  407. int nid = early_pfn_to_nid(pfn);
  408. if (node_online(nid) && pfn >= NODE_DATA(nid)->first_deferred_pfn)
  409. return true;
  410. return false;
  411. }
  412. /*
  413. * Returns true when the remaining initialisation should be deferred until
  414. * later in the boot cycle when it can be parallelised.
  415. */
  416. static bool __meminit
  417. defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
  418. {
  419. static unsigned long prev_end_pfn, nr_initialised;
  420. if (early_page_ext_enabled())
  421. return false;
  422. /*
  423. * prev_end_pfn static that contains the end of previous zone
  424. * No need to protect because called very early in boot before smp_init.
  425. */
  426. if (prev_end_pfn != end_pfn) {
  427. prev_end_pfn = end_pfn;
  428. nr_initialised = 0;
  429. }
  430. /* Always populate low zones for address-constrained allocations */
  431. if (end_pfn < pgdat_end_pfn(NODE_DATA(nid)))
  432. return false;
  433. if (NODE_DATA(nid)->first_deferred_pfn != ULONG_MAX)
  434. return true;
  435. /*
  436. * We start only with one section of pages, more pages are added as
  437. * needed until the rest of deferred pages are initialized.
  438. */
  439. nr_initialised++;
  440. if ((nr_initialised > PAGES_PER_SECTION) &&
  441. (pfn & (PAGES_PER_SECTION - 1)) == 0) {
  442. NODE_DATA(nid)->first_deferred_pfn = pfn;
  443. return true;
  444. }
  445. return false;
  446. }
  447. #else
  448. static inline bool deferred_pages_enabled(void)
  449. {
  450. return false;
  451. }
  452. static inline bool early_page_uninitialised(unsigned long pfn)
  453. {
  454. return false;
  455. }
  456. static inline bool defer_init(int nid, unsigned long pfn, unsigned long end_pfn)
  457. {
  458. return false;
  459. }
  460. #endif
  461. /* Return a pointer to the bitmap storing bits affecting a block of pages */
  462. static inline unsigned long *get_pageblock_bitmap(const struct page *page,
  463. unsigned long pfn)
  464. {
  465. #ifdef CONFIG_SPARSEMEM
  466. return section_to_usemap(__pfn_to_section(pfn));
  467. #else
  468. return page_zone(page)->pageblock_flags;
  469. #endif /* CONFIG_SPARSEMEM */
  470. }
  471. static inline int pfn_to_bitidx(const struct page *page, unsigned long pfn)
  472. {
  473. #ifdef CONFIG_SPARSEMEM
  474. pfn &= (PAGES_PER_SECTION-1);
  475. #else
  476. pfn = pfn - pageblock_start_pfn(page_zone(page)->zone_start_pfn);
  477. #endif /* CONFIG_SPARSEMEM */
  478. return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
  479. }
  480. static __always_inline
  481. unsigned long __get_pfnblock_flags_mask(const struct page *page,
  482. unsigned long pfn,
  483. unsigned long mask)
  484. {
  485. unsigned long *bitmap;
  486. unsigned long bitidx, word_bitidx;
  487. unsigned long word;
  488. bitmap = get_pageblock_bitmap(page, pfn);
  489. bitidx = pfn_to_bitidx(page, pfn);
  490. word_bitidx = bitidx / BITS_PER_LONG;
  491. bitidx &= (BITS_PER_LONG-1);
  492. /*
  493. * This races, without locks, with set_pfnblock_flags_mask(). Ensure
  494. * a consistent read of the memory array, so that results, even though
  495. * racy, are not corrupted.
  496. */
  497. word = READ_ONCE(bitmap[word_bitidx]);
  498. return (word >> bitidx) & mask;
  499. }
  500. /**
  501. * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages
  502. * @page: The page within the block of interest
  503. * @pfn: The target page frame number
  504. * @mask: mask of bits that the caller is interested in
  505. *
  506. * Return: pageblock_bits flags
  507. */
  508. unsigned long get_pfnblock_flags_mask(const struct page *page,
  509. unsigned long pfn, unsigned long mask)
  510. {
  511. return __get_pfnblock_flags_mask(page, pfn, mask);
  512. }
  513. EXPORT_SYMBOL_GPL(get_pfnblock_flags_mask);
  514. int isolate_anon_lru_page(struct page *page)
  515. {
  516. int ret;
  517. if (!PageLRU(page) || !PageAnon(page))
  518. return -EINVAL;
  519. if (!get_page_unless_zero(page))
  520. return -EINVAL;
  521. ret = isolate_lru_page(page);
  522. put_page(page);
  523. return ret;
  524. }
  525. EXPORT_SYMBOL_GPL(isolate_anon_lru_page);
  526. static __always_inline int get_pfnblock_migratetype(const struct page *page,
  527. unsigned long pfn)
  528. {
  529. return __get_pfnblock_flags_mask(page, pfn, MIGRATETYPE_MASK);
  530. }
  531. /**
  532. * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
  533. * @page: The page within the block of interest
  534. * @flags: The flags to set
  535. * @pfn: The target page frame number
  536. * @mask: mask of bits that the caller is interested in
  537. */
  538. void set_pfnblock_flags_mask(struct page *page, unsigned long flags,
  539. unsigned long pfn,
  540. unsigned long mask)
  541. {
  542. unsigned long *bitmap;
  543. unsigned long bitidx, word_bitidx;
  544. unsigned long word;
  545. BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
  546. BUILD_BUG_ON(MIGRATE_TYPES > (1 << PB_migratetype_bits));
  547. bitmap = get_pageblock_bitmap(page, pfn);
  548. bitidx = pfn_to_bitidx(page, pfn);
  549. word_bitidx = bitidx / BITS_PER_LONG;
  550. bitidx &= (BITS_PER_LONG-1);
  551. VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
  552. mask <<= bitidx;
  553. flags <<= bitidx;
  554. word = READ_ONCE(bitmap[word_bitidx]);
  555. do {
  556. } while (!try_cmpxchg(&bitmap[word_bitidx], &word, (word & ~mask) | flags));
  557. }
  558. void set_pageblock_migratetype(struct page *page, int migratetype)
  559. {
  560. if (unlikely(page_group_by_mobility_disabled &&
  561. migratetype < MIGRATE_PCPTYPES))
  562. migratetype = MIGRATE_UNMOVABLE;
  563. set_pfnblock_flags_mask(page, (unsigned long)migratetype,
  564. page_to_pfn(page), MIGRATETYPE_MASK);
  565. }
  566. #ifdef CONFIG_DEBUG_VM
  567. static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
  568. {
  569. int ret = 0;
  570. unsigned seq;
  571. unsigned long pfn = page_to_pfn(page);
  572. unsigned long sp, start_pfn;
  573. do {
  574. seq = zone_span_seqbegin(zone);
  575. start_pfn = zone->zone_start_pfn;
  576. sp = zone->spanned_pages;
  577. if (!zone_spans_pfn(zone, pfn))
  578. ret = 1;
  579. } while (zone_span_seqretry(zone, seq));
  580. if (ret)
  581. pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n",
  582. pfn, zone_to_nid(zone), zone->name,
  583. start_pfn, start_pfn + sp);
  584. return ret;
  585. }
  586. static int page_is_consistent(struct zone *zone, struct page *page)
  587. {
  588. if (zone != page_zone(page))
  589. return 0;
  590. return 1;
  591. }
  592. /*
  593. * Temporary debugging check for pages not lying within a given zone.
  594. */
  595. static int __maybe_unused bad_range(struct zone *zone, struct page *page)
  596. {
  597. if (page_outside_zone_boundaries(zone, page))
  598. return 1;
  599. if (!page_is_consistent(zone, page))
  600. return 1;
  601. return 0;
  602. }
  603. #else
  604. static inline int __maybe_unused bad_range(struct zone *zone, struct page *page)
  605. {
  606. return 0;
  607. }
  608. #endif
  609. static void bad_page(struct page *page, const char *reason)
  610. {
  611. static unsigned long resume;
  612. static unsigned long nr_shown;
  613. static unsigned long nr_unshown;
  614. /*
  615. * Allow a burst of 60 reports, then keep quiet for that minute;
  616. * or allow a steady drip of one report per second.
  617. */
  618. if (nr_shown == 60) {
  619. if (time_before(jiffies, resume)) {
  620. nr_unshown++;
  621. goto out;
  622. }
  623. if (nr_unshown) {
  624. pr_alert(
  625. "BUG: Bad page state: %lu messages suppressed\n",
  626. nr_unshown);
  627. nr_unshown = 0;
  628. }
  629. nr_shown = 0;
  630. }
  631. if (nr_shown++ == 0)
  632. resume = jiffies + 60 * HZ;
  633. pr_alert("BUG: Bad page state in process %s pfn:%05lx\n",
  634. current->comm, page_to_pfn(page));
  635. dump_page(page, reason);
  636. print_modules();
  637. dump_stack();
  638. out:
  639. /* Leave bad fields for debug, except PageBuddy could make trouble */
  640. page_mapcount_reset(page); /* remove PageBuddy */
  641. add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
  642. }
  643. static inline unsigned int order_to_pindex(int migratetype, int order)
  644. {
  645. int base = order;
  646. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  647. if (order > PAGE_ALLOC_COSTLY_ORDER) {
  648. VM_BUG_ON(order != pageblock_order);
  649. return NR_LOWORDER_PCP_LISTS;
  650. }
  651. #else
  652. VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
  653. #endif
  654. return (MIGRATE_PCPTYPES * base) + migratetype;
  655. }
  656. static inline int pindex_to_order(unsigned int pindex)
  657. {
  658. int order = pindex / MIGRATE_PCPTYPES;
  659. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  660. if (pindex == NR_LOWORDER_PCP_LISTS)
  661. order = pageblock_order;
  662. #else
  663. VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
  664. #endif
  665. return order;
  666. }
  667. static inline bool pcp_allowed_order(unsigned int order)
  668. {
  669. if (order <= PAGE_ALLOC_COSTLY_ORDER)
  670. return true;
  671. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  672. if (order == pageblock_order)
  673. return true;
  674. #endif
  675. return false;
  676. }
  677. static inline void free_the_page(struct page *page, unsigned int order)
  678. {
  679. if (pcp_allowed_order(order)) /* Via pcp? */
  680. free_unref_page(page, order);
  681. else
  682. __free_pages_ok(page, order, FPI_NONE);
  683. }
  684. /*
  685. * Higher-order pages are called "compound pages". They are structured thusly:
  686. *
  687. * The first PAGE_SIZE page is called the "head page" and have PG_head set.
  688. *
  689. * The remaining PAGE_SIZE pages are called "tail pages". PageTail() is encoded
  690. * in bit 0 of page->compound_head. The rest of bits is pointer to head page.
  691. *
  692. * The first tail page's ->compound_dtor holds the offset in array of compound
  693. * page destructors. See compound_page_dtors.
  694. *
  695. * The first tail page's ->compound_order holds the order of allocation.
  696. * This usage means that zero-order pages may not be compound.
  697. */
  698. void free_compound_page(struct page *page)
  699. {
  700. mem_cgroup_uncharge(page_folio(page));
  701. free_the_page(page, compound_order(page));
  702. }
  703. static void prep_compound_head(struct page *page, unsigned int order)
  704. {
  705. set_compound_page_dtor(page, COMPOUND_PAGE_DTOR);
  706. set_compound_order(page, order);
  707. atomic_set(compound_mapcount_ptr(page), -1);
  708. atomic_set(compound_pincount_ptr(page), 0);
  709. }
  710. static void prep_compound_tail(struct page *head, int tail_idx)
  711. {
  712. struct page *p = head + tail_idx;
  713. p->mapping = TAIL_MAPPING;
  714. set_compound_head(p, head);
  715. set_page_private(p, 0);
  716. }
  717. void prep_compound_page(struct page *page, unsigned int order)
  718. {
  719. int i;
  720. int nr_pages = 1 << order;
  721. __SetPageHead(page);
  722. for (i = 1; i < nr_pages; i++)
  723. prep_compound_tail(page, i);
  724. prep_compound_head(page, order);
  725. }
  726. void destroy_large_folio(struct folio *folio)
  727. {
  728. enum compound_dtor_id dtor = folio_page(folio, 1)->compound_dtor;
  729. VM_BUG_ON_FOLIO(dtor >= NR_COMPOUND_DTORS, folio);
  730. compound_page_dtors[dtor](&folio->page);
  731. }
  732. #ifdef CONFIG_DEBUG_PAGEALLOC
  733. unsigned int _debug_guardpage_minorder;
  734. bool _debug_pagealloc_enabled_early __read_mostly
  735. = IS_ENABLED(CONFIG_DEBUG_PAGEALLOC_ENABLE_DEFAULT);
  736. EXPORT_SYMBOL(_debug_pagealloc_enabled_early);
  737. DEFINE_STATIC_KEY_FALSE(_debug_pagealloc_enabled);
  738. EXPORT_SYMBOL(_debug_pagealloc_enabled);
  739. DEFINE_STATIC_KEY_FALSE(_debug_guardpage_enabled);
  740. static int __init early_debug_pagealloc(char *buf)
  741. {
  742. return kstrtobool(buf, &_debug_pagealloc_enabled_early);
  743. }
  744. early_param("debug_pagealloc", early_debug_pagealloc);
  745. static int __init debug_guardpage_minorder_setup(char *buf)
  746. {
  747. unsigned long res;
  748. if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
  749. pr_err("Bad debug_guardpage_minorder value\n");
  750. return 0;
  751. }
  752. _debug_guardpage_minorder = res;
  753. pr_info("Setting debug_guardpage_minorder to %lu\n", res);
  754. return 0;
  755. }
  756. early_param("debug_guardpage_minorder", debug_guardpage_minorder_setup);
  757. static inline bool set_page_guard(struct zone *zone, struct page *page,
  758. unsigned int order, int migratetype)
  759. {
  760. if (!debug_guardpage_enabled())
  761. return false;
  762. if (order >= debug_guardpage_minorder())
  763. return false;
  764. __SetPageGuard(page);
  765. INIT_LIST_HEAD(&page->buddy_list);
  766. set_page_private(page, order);
  767. /* Guard pages are not available for any usage */
  768. if (!is_migrate_isolate(migratetype))
  769. __mod_zone_freepage_state(zone, -(1 << order), migratetype);
  770. return true;
  771. }
  772. static inline void clear_page_guard(struct zone *zone, struct page *page,
  773. unsigned int order, int migratetype)
  774. {
  775. if (!debug_guardpage_enabled())
  776. return;
  777. __ClearPageGuard(page);
  778. set_page_private(page, 0);
  779. if (!is_migrate_isolate(migratetype))
  780. __mod_zone_freepage_state(zone, (1 << order), migratetype);
  781. }
  782. #else
  783. static inline bool set_page_guard(struct zone *zone, struct page *page,
  784. unsigned int order, int migratetype) { return false; }
  785. static inline void clear_page_guard(struct zone *zone, struct page *page,
  786. unsigned int order, int migratetype) {}
  787. #endif
  788. /*
  789. * Enable static keys related to various memory debugging and hardening options.
  790. * Some override others, and depend on early params that are evaluated in the
  791. * order of appearance. So we need to first gather the full picture of what was
  792. * enabled, and then make decisions.
  793. */
  794. void __init init_mem_debugging_and_hardening(void)
  795. {
  796. bool page_poisoning_requested = false;
  797. #ifdef CONFIG_PAGE_POISONING
  798. /*
  799. * Page poisoning is debug page alloc for some arches. If
  800. * either of those options are enabled, enable poisoning.
  801. */
  802. if (page_poisoning_enabled() ||
  803. (!IS_ENABLED(CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC) &&
  804. debug_pagealloc_enabled())) {
  805. static_branch_enable(&_page_poisoning_enabled);
  806. page_poisoning_requested = true;
  807. }
  808. #endif
  809. if ((_init_on_alloc_enabled_early || _init_on_free_enabled_early) &&
  810. page_poisoning_requested) {
  811. pr_info("mem auto-init: CONFIG_PAGE_POISONING is on, "
  812. "will take precedence over init_on_alloc and init_on_free\n");
  813. _init_on_alloc_enabled_early = false;
  814. _init_on_free_enabled_early = false;
  815. }
  816. if (_init_on_alloc_enabled_early)
  817. static_branch_enable(&init_on_alloc);
  818. else
  819. static_branch_disable(&init_on_alloc);
  820. if (_init_on_free_enabled_early)
  821. static_branch_enable(&init_on_free);
  822. else
  823. static_branch_disable(&init_on_free);
  824. if (IS_ENABLED(CONFIG_KMSAN) &&
  825. (_init_on_alloc_enabled_early || _init_on_free_enabled_early))
  826. pr_info("mem auto-init: please make sure init_on_alloc and init_on_free are disabled when running KMSAN\n");
  827. #ifdef CONFIG_DEBUG_PAGEALLOC
  828. if (!debug_pagealloc_enabled())
  829. return;
  830. static_branch_enable(&_debug_pagealloc_enabled);
  831. if (!debug_guardpage_minorder())
  832. return;
  833. static_branch_enable(&_debug_guardpage_enabled);
  834. #endif
  835. }
  836. static inline void set_buddy_order(struct page *page, unsigned int order)
  837. {
  838. set_page_private(page, order);
  839. __SetPageBuddy(page);
  840. }
  841. #ifdef CONFIG_COMPACTION
  842. static inline struct capture_control *task_capc(struct zone *zone)
  843. {
  844. struct capture_control *capc = current->capture_control;
  845. return unlikely(capc) &&
  846. !(current->flags & PF_KTHREAD) &&
  847. !capc->page &&
  848. capc->cc->zone == zone ? capc : NULL;
  849. }
  850. static inline bool
  851. compaction_capture(struct capture_control *capc, struct page *page,
  852. int order, int migratetype)
  853. {
  854. if (!capc || order != capc->cc->order)
  855. return false;
  856. /* Do not accidentally pollute CMA or isolated regions*/
  857. if (is_migrate_cma(migratetype) ||
  858. is_migrate_isolate(migratetype))
  859. return false;
  860. /*
  861. * Do not let lower order allocations pollute a movable pageblock.
  862. * This might let an unmovable request use a reclaimable pageblock
  863. * and vice-versa but no more than normal fallback logic which can
  864. * have trouble finding a high-order free page.
  865. */
  866. if (order < pageblock_order && migratetype == MIGRATE_MOVABLE)
  867. return false;
  868. capc->page = page;
  869. return true;
  870. }
  871. #else
  872. static inline struct capture_control *task_capc(struct zone *zone)
  873. {
  874. return NULL;
  875. }
  876. static inline bool
  877. compaction_capture(struct capture_control *capc, struct page *page,
  878. int order, int migratetype)
  879. {
  880. return false;
  881. }
  882. #endif /* CONFIG_COMPACTION */
  883. /* Used for pages not on another list */
  884. static inline void add_to_free_list(struct page *page, struct zone *zone,
  885. unsigned int order, int migratetype)
  886. {
  887. struct free_area *area = &zone->free_area[order];
  888. list_add(&page->buddy_list, &area->free_list[migratetype]);
  889. area->nr_free++;
  890. }
  891. /* Used for pages not on another list */
  892. static inline void add_to_free_list_tail(struct page *page, struct zone *zone,
  893. unsigned int order, int migratetype)
  894. {
  895. struct free_area *area = &zone->free_area[order];
  896. list_add_tail(&page->buddy_list, &area->free_list[migratetype]);
  897. area->nr_free++;
  898. }
  899. /*
  900. * Used for pages which are on another list. Move the pages to the tail
  901. * of the list - so the moved pages won't immediately be considered for
  902. * allocation again (e.g., optimization for memory onlining).
  903. */
  904. static inline void move_to_free_list(struct page *page, struct zone *zone,
  905. unsigned int order, int migratetype)
  906. {
  907. struct free_area *area = &zone->free_area[order];
  908. list_move_tail(&page->buddy_list, &area->free_list[migratetype]);
  909. }
  910. static inline void del_page_from_free_list(struct page *page, struct zone *zone,
  911. unsigned int order)
  912. {
  913. /* clear reported state and update reported page count */
  914. if (page_reported(page))
  915. __ClearPageReported(page);
  916. list_del(&page->buddy_list);
  917. __ClearPageBuddy(page);
  918. set_page_private(page, 0);
  919. zone->free_area[order].nr_free--;
  920. }
  921. /*
  922. * If this is not the largest possible page, check if the buddy
  923. * of the next-highest order is free. If it is, it's possible
  924. * that pages are being freed that will coalesce soon. In case,
  925. * that is happening, add the free page to the tail of the list
  926. * so it's less likely to be used soon and more likely to be merged
  927. * as a higher order page
  928. */
  929. static inline bool
  930. buddy_merge_likely(unsigned long pfn, unsigned long buddy_pfn,
  931. struct page *page, unsigned int order)
  932. {
  933. unsigned long higher_page_pfn;
  934. struct page *higher_page;
  935. if (order >= MAX_ORDER - 2)
  936. return false;
  937. higher_page_pfn = buddy_pfn & pfn;
  938. higher_page = page + (higher_page_pfn - pfn);
  939. return find_buddy_page_pfn(higher_page, higher_page_pfn, order + 1,
  940. NULL) != NULL;
  941. }
  942. /*
  943. * Freeing function for a buddy system allocator.
  944. *
  945. * The concept of a buddy system is to maintain direct-mapped table
  946. * (containing bit values) for memory blocks of various "orders".
  947. * The bottom level table contains the map for the smallest allocatable
  948. * units of memory (here, pages), and each level above it describes
  949. * pairs of units from the levels below, hence, "buddies".
  950. * At a high level, all that happens here is marking the table entry
  951. * at the bottom level available, and propagating the changes upward
  952. * as necessary, plus some accounting needed to play nicely with other
  953. * parts of the VM system.
  954. * At each level, we keep a list of pages, which are heads of continuous
  955. * free pages of length of (1 << order) and marked with PageBuddy.
  956. * Page's order is recorded in page_private(page) field.
  957. * So when we are allocating or freeing one, we can derive the state of the
  958. * other. That is, if we allocate a small block, and both were
  959. * free, the remainder of the region must be split into blocks.
  960. * If a block is freed, and its buddy is also free, then this
  961. * triggers coalescing into a block of larger size.
  962. *
  963. * -- nyc
  964. */
  965. static inline void __free_one_page(struct page *page,
  966. unsigned long pfn,
  967. struct zone *zone, unsigned int order,
  968. int migratetype, fpi_t fpi_flags)
  969. {
  970. struct capture_control *capc = task_capc(zone);
  971. unsigned long buddy_pfn = 0;
  972. unsigned long combined_pfn;
  973. struct page *buddy;
  974. bool to_tail;
  975. bool bypass = false;
  976. trace_android_vh_free_one_page_bypass(page, zone, order,
  977. migratetype, (int)fpi_flags, &bypass);
  978. if (bypass)
  979. return;
  980. VM_BUG_ON(!zone_is_initialized(zone));
  981. VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page);
  982. VM_BUG_ON(migratetype == -1);
  983. if (likely(!is_migrate_isolate(migratetype)))
  984. __mod_zone_freepage_state(zone, 1 << order, migratetype);
  985. VM_BUG_ON_PAGE(pfn & ((1 << order) - 1), page);
  986. VM_BUG_ON_PAGE(bad_range(zone, page), page);
  987. while (order < MAX_ORDER - 1) {
  988. if (compaction_capture(capc, page, order, migratetype)) {
  989. __mod_zone_freepage_state(zone, -(1 << order),
  990. migratetype);
  991. return;
  992. }
  993. buddy = find_buddy_page_pfn(page, pfn, order, &buddy_pfn);
  994. if (!buddy)
  995. goto done_merging;
  996. if (unlikely(order >= pageblock_order)) {
  997. /*
  998. * We want to prevent merge between freepages on pageblock
  999. * without fallbacks and normal pageblock. Without this,
  1000. * pageblock isolation could cause incorrect freepage or CMA
  1001. * accounting or HIGHATOMIC accounting.
  1002. */
  1003. int buddy_mt = get_pageblock_migratetype(buddy);
  1004. if (migratetype != buddy_mt
  1005. && (!migratetype_is_mergeable(migratetype) ||
  1006. !migratetype_is_mergeable(buddy_mt)))
  1007. goto done_merging;
  1008. }
  1009. /*
  1010. * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
  1011. * merge with it and move up one order.
  1012. */
  1013. if (page_is_guard(buddy))
  1014. clear_page_guard(zone, buddy, order, migratetype);
  1015. else
  1016. del_page_from_free_list(buddy, zone, order);
  1017. combined_pfn = buddy_pfn & pfn;
  1018. page = page + (combined_pfn - pfn);
  1019. pfn = combined_pfn;
  1020. order++;
  1021. }
  1022. done_merging:
  1023. set_buddy_order(page, order);
  1024. if (fpi_flags & FPI_TO_TAIL)
  1025. to_tail = true;
  1026. else if (is_shuffle_order(order))
  1027. to_tail = shuffle_pick_tail();
  1028. else
  1029. to_tail = buddy_merge_likely(pfn, buddy_pfn, page, order);
  1030. if (to_tail)
  1031. add_to_free_list_tail(page, zone, order, migratetype);
  1032. else
  1033. add_to_free_list(page, zone, order, migratetype);
  1034. /* Notify page reporting subsystem of freed page */
  1035. if (!(fpi_flags & FPI_SKIP_REPORT_NOTIFY))
  1036. page_reporting_notify_free(order);
  1037. }
  1038. /**
  1039. * split_free_page() -- split a free page at split_pfn_offset
  1040. * @free_page: the original free page
  1041. * @order: the order of the page
  1042. * @split_pfn_offset: split offset within the page
  1043. *
  1044. * Return -ENOENT if the free page is changed, otherwise 0
  1045. *
  1046. * It is used when the free page crosses two pageblocks with different migratetypes
  1047. * at split_pfn_offset within the page. The split free page will be put into
  1048. * separate migratetype lists afterwards. Otherwise, the function achieves
  1049. * nothing.
  1050. */
  1051. int split_free_page(struct page *free_page,
  1052. unsigned int order, unsigned long split_pfn_offset)
  1053. {
  1054. struct zone *zone = page_zone(free_page);
  1055. unsigned long free_page_pfn = page_to_pfn(free_page);
  1056. unsigned long pfn;
  1057. unsigned long flags;
  1058. int free_page_order;
  1059. int mt;
  1060. int ret = 0;
  1061. if (split_pfn_offset == 0)
  1062. return ret;
  1063. spin_lock_irqsave(&zone->lock, flags);
  1064. if (!PageBuddy(free_page) || buddy_order(free_page) != order) {
  1065. ret = -ENOENT;
  1066. goto out;
  1067. }
  1068. mt = get_pageblock_migratetype(free_page);
  1069. if (likely(!is_migrate_isolate(mt)))
  1070. __mod_zone_freepage_state(zone, -(1UL << order), mt);
  1071. del_page_from_free_list(free_page, zone, order);
  1072. for (pfn = free_page_pfn;
  1073. pfn < free_page_pfn + (1UL << order);) {
  1074. int mt = get_pfnblock_migratetype(pfn_to_page(pfn), pfn);
  1075. free_page_order = min_t(unsigned int,
  1076. pfn ? __ffs(pfn) : order,
  1077. __fls(split_pfn_offset));
  1078. __free_one_page(pfn_to_page(pfn), pfn, zone, free_page_order,
  1079. mt, FPI_NONE);
  1080. pfn += 1UL << free_page_order;
  1081. split_pfn_offset -= (1UL << free_page_order);
  1082. /* we have done the first part, now switch to second part */
  1083. if (split_pfn_offset == 0)
  1084. split_pfn_offset = (1UL << order) - (pfn - free_page_pfn);
  1085. }
  1086. out:
  1087. spin_unlock_irqrestore(&zone->lock, flags);
  1088. return ret;
  1089. }
  1090. /*
  1091. * A bad page could be due to a number of fields. Instead of multiple branches,
  1092. * try and check multiple fields with one check. The caller must do a detailed
  1093. * check if necessary.
  1094. */
  1095. static inline bool page_expected_state(struct page *page,
  1096. unsigned long check_flags)
  1097. {
  1098. if (unlikely(atomic_read(&page->_mapcount) != -1))
  1099. return false;
  1100. if (unlikely((unsigned long)page->mapping |
  1101. page_ref_count(page) |
  1102. #ifdef CONFIG_MEMCG
  1103. page->memcg_data |
  1104. #endif
  1105. (page->flags & check_flags)))
  1106. return false;
  1107. return true;
  1108. }
  1109. static const char *page_bad_reason(struct page *page, unsigned long flags)
  1110. {
  1111. const char *bad_reason = NULL;
  1112. if (unlikely(atomic_read(&page->_mapcount) != -1))
  1113. bad_reason = "nonzero mapcount";
  1114. if (unlikely(page->mapping != NULL))
  1115. bad_reason = "non-NULL mapping";
  1116. if (unlikely(page_ref_count(page) != 0))
  1117. bad_reason = "nonzero _refcount";
  1118. if (unlikely(page->flags & flags)) {
  1119. if (flags == PAGE_FLAGS_CHECK_AT_PREP)
  1120. bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag(s) set";
  1121. else
  1122. bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set";
  1123. }
  1124. #ifdef CONFIG_MEMCG
  1125. if (unlikely(page->memcg_data))
  1126. bad_reason = "page still charged to cgroup";
  1127. #endif
  1128. return bad_reason;
  1129. }
  1130. static void free_page_is_bad_report(struct page *page)
  1131. {
  1132. bad_page(page,
  1133. page_bad_reason(page, PAGE_FLAGS_CHECK_AT_FREE));
  1134. }
  1135. static inline bool free_page_is_bad(struct page *page)
  1136. {
  1137. if (likely(page_expected_state(page, PAGE_FLAGS_CHECK_AT_FREE)))
  1138. return false;
  1139. /* Something has gone sideways, find it */
  1140. free_page_is_bad_report(page);
  1141. return true;
  1142. }
  1143. static int free_tail_pages_check(struct page *head_page, struct page *page)
  1144. {
  1145. int ret = 1;
  1146. /*
  1147. * We rely page->lru.next never has bit 0 set, unless the page
  1148. * is PageTail(). Let's make sure that's true even for poisoned ->lru.
  1149. */
  1150. BUILD_BUG_ON((unsigned long)LIST_POISON1 & 1);
  1151. if (!IS_ENABLED(CONFIG_DEBUG_VM)) {
  1152. ret = 0;
  1153. goto out;
  1154. }
  1155. switch (page - head_page) {
  1156. case 1:
  1157. /* the first tail page: ->mapping may be compound_mapcount() */
  1158. if (unlikely(compound_mapcount(page))) {
  1159. bad_page(page, "nonzero compound_mapcount");
  1160. goto out;
  1161. }
  1162. break;
  1163. case 2:
  1164. /*
  1165. * the second tail page: ->mapping is
  1166. * deferred_list.next -- ignore value.
  1167. */
  1168. break;
  1169. default:
  1170. if (page->mapping != TAIL_MAPPING) {
  1171. bad_page(page, "corrupted mapping in tail page");
  1172. goto out;
  1173. }
  1174. break;
  1175. }
  1176. if (unlikely(!PageTail(page))) {
  1177. bad_page(page, "PageTail not set");
  1178. goto out;
  1179. }
  1180. if (unlikely(compound_head(page) != head_page)) {
  1181. bad_page(page, "compound_head not consistent");
  1182. goto out;
  1183. }
  1184. ret = 0;
  1185. out:
  1186. page->mapping = NULL;
  1187. clear_compound_head(page);
  1188. return ret;
  1189. }
  1190. /*
  1191. * Skip KASAN memory poisoning when either:
  1192. *
  1193. * 1. Deferred memory initialization has not yet completed,
  1194. * see the explanation below.
  1195. * 2. Skipping poisoning is requested via FPI_SKIP_KASAN_POISON,
  1196. * see the comment next to it.
  1197. * 3. Skipping poisoning is requested via __GFP_SKIP_KASAN_POISON,
  1198. * see the comment next to it.
  1199. * 4. The allocation is excluded from being checked due to sampling,
  1200. * see the call to kasan_unpoison_pages.
  1201. *
  1202. * Poisoning pages during deferred memory init will greatly lengthen the
  1203. * process and cause problem in large memory systems as the deferred pages
  1204. * initialization is done with interrupt disabled.
  1205. *
  1206. * Assuming that there will be no reference to those newly initialized
  1207. * pages before they are ever allocated, this should have no effect on
  1208. * KASAN memory tracking as the poison will be properly inserted at page
  1209. * allocation time. The only corner case is when pages are allocated by
  1210. * on-demand allocation and then freed again before the deferred pages
  1211. * initialization is done, but this is not likely to happen.
  1212. */
  1213. static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
  1214. {
  1215. return deferred_pages_enabled() ||
  1216. (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
  1217. (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
  1218. PageSkipKASanPoison(page);
  1219. }
  1220. static void kernel_init_pages(struct page *page, int numpages)
  1221. {
  1222. int i;
  1223. /* s390's use of memset() could override KASAN redzones. */
  1224. kasan_disable_current();
  1225. for (i = 0; i < numpages; i++)
  1226. clear_highpage_kasan_tagged(page + i);
  1227. kasan_enable_current();
  1228. }
  1229. static __always_inline bool free_pages_prepare(struct page *page,
  1230. unsigned int order, bool check_free, fpi_t fpi_flags)
  1231. {
  1232. int bad = 0;
  1233. bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags);
  1234. bool init = want_init_on_free();
  1235. VM_BUG_ON_PAGE(PageTail(page), page);
  1236. trace_mm_page_free(page, order);
  1237. kmsan_free_page(page, order);
  1238. if (unlikely(PageHWPoison(page)) && !order) {
  1239. /*
  1240. * Do not let hwpoison pages hit pcplists/buddy
  1241. * Untie memcg state and reset page's owner
  1242. */
  1243. if (memcg_kmem_enabled() && PageMemcgKmem(page))
  1244. __memcg_kmem_uncharge_page(page, order);
  1245. reset_page_owner(page, order);
  1246. free_page_pinner(page, order);
  1247. page_table_check_free(page, order);
  1248. return false;
  1249. }
  1250. /*
  1251. * Check tail pages before head page information is cleared to
  1252. * avoid checking PageCompound for order-0 pages.
  1253. */
  1254. if (unlikely(order)) {
  1255. bool compound = PageCompound(page);
  1256. int i;
  1257. VM_BUG_ON_PAGE(compound && compound_order(page) != order, page);
  1258. if (compound) {
  1259. ClearPageDoubleMap(page);
  1260. ClearPageHasHWPoisoned(page);
  1261. }
  1262. for (i = 1; i < (1 << order); i++) {
  1263. if (compound)
  1264. bad += free_tail_pages_check(page, page + i);
  1265. if (unlikely(free_page_is_bad(page + i))) {
  1266. bad++;
  1267. continue;
  1268. }
  1269. (page + i)->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  1270. }
  1271. }
  1272. if (PageMappingFlags(page))
  1273. page->mapping = NULL;
  1274. if (memcg_kmem_enabled() && PageMemcgKmem(page))
  1275. __memcg_kmem_uncharge_page(page, order);
  1276. if (check_free && free_page_is_bad(page))
  1277. bad++;
  1278. if (bad)
  1279. return false;
  1280. page_cpupid_reset_last(page);
  1281. page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
  1282. reset_page_owner(page, order);
  1283. free_page_pinner(page, order);
  1284. page_table_check_free(page, order);
  1285. if (!PageHighMem(page)) {
  1286. debug_check_no_locks_freed(page_address(page),
  1287. PAGE_SIZE << order);
  1288. debug_check_no_obj_freed(page_address(page),
  1289. PAGE_SIZE << order);
  1290. }
  1291. kernel_poison_pages(page, 1 << order);
  1292. /*
  1293. * As memory initialization might be integrated into KASAN,
  1294. * KASAN poisoning and memory initialization code must be
  1295. * kept together to avoid discrepancies in behavior.
  1296. *
  1297. * With hardware tag-based KASAN, memory tags must be set before the
  1298. * page becomes unavailable via debug_pagealloc or arch_free_page.
  1299. */
  1300. if (!skip_kasan_poison) {
  1301. kasan_poison_pages(page, order, init);
  1302. /* Memory is already initialized if KASAN did it internally. */
  1303. if (kasan_has_integrated_init())
  1304. init = false;
  1305. }
  1306. if (init)
  1307. kernel_init_pages(page, 1 << order);
  1308. /*
  1309. * arch_free_page() can make the page's contents inaccessible. s390
  1310. * does this. So nothing which can access the page's contents should
  1311. * happen after this.
  1312. */
  1313. arch_free_page(page, order);
  1314. debug_pagealloc_unmap_pages(page, 1 << order);
  1315. return true;
  1316. }
  1317. #ifdef CONFIG_DEBUG_VM
  1318. /*
  1319. * With DEBUG_VM enabled, order-0 pages are checked immediately when being freed
  1320. * to pcp lists. With debug_pagealloc also enabled, they are also rechecked when
  1321. * moved from pcp lists to free lists.
  1322. */
  1323. static bool free_pcp_prepare(struct page *page, unsigned int order)
  1324. {
  1325. return free_pages_prepare(page, order, true, FPI_NONE);
  1326. }
  1327. /* return true if this page has an inappropriate state */
  1328. static bool bulkfree_pcp_prepare(struct page *page)
  1329. {
  1330. if (debug_pagealloc_enabled_static())
  1331. return free_page_is_bad(page);
  1332. else
  1333. return false;
  1334. }
  1335. #else
  1336. /*
  1337. * With DEBUG_VM disabled, order-0 pages being freed are checked only when
  1338. * moving from pcp lists to free list in order to reduce overhead. With
  1339. * debug_pagealloc enabled, they are checked also immediately when being freed
  1340. * to the pcp lists.
  1341. */
  1342. static bool free_pcp_prepare(struct page *page, unsigned int order)
  1343. {
  1344. if (debug_pagealloc_enabled_static())
  1345. return free_pages_prepare(page, order, true, FPI_NONE);
  1346. else
  1347. return free_pages_prepare(page, order, false, FPI_NONE);
  1348. }
  1349. static bool bulkfree_pcp_prepare(struct page *page)
  1350. {
  1351. return free_page_is_bad(page);
  1352. }
  1353. #endif /* CONFIG_DEBUG_VM */
  1354. /*
  1355. * Frees a number of pages from the PCP lists
  1356. * Assumes all pages on list are in same zone.
  1357. * count is the number of pages to free.
  1358. */
  1359. static void free_pcppages_bulk(struct zone *zone, int count,
  1360. struct per_cpu_pages *pcp,
  1361. int pindex)
  1362. {
  1363. unsigned long flags;
  1364. int min_pindex = 0;
  1365. int max_pindex = NR_PCP_LISTS - 1;
  1366. unsigned int order;
  1367. bool isolated_pageblocks;
  1368. struct page *page;
  1369. /*
  1370. * Ensure proper count is passed which otherwise would stuck in the
  1371. * below while (list_empty(list)) loop.
  1372. */
  1373. count = min(pcp->count, count);
  1374. /* Ensure requested pindex is drained first. */
  1375. pindex = pindex - 1;
  1376. spin_lock_irqsave(&zone->lock, flags);
  1377. isolated_pageblocks = has_isolate_pageblock(zone);
  1378. while (count > 0) {
  1379. struct list_head *list;
  1380. int nr_pages;
  1381. /* Remove pages from lists in a round-robin fashion. */
  1382. do {
  1383. if (++pindex > max_pindex)
  1384. pindex = min_pindex;
  1385. list = &pcp->lists[pindex];
  1386. if (!list_empty(list))
  1387. break;
  1388. if (pindex == max_pindex)
  1389. max_pindex--;
  1390. if (pindex == min_pindex)
  1391. min_pindex++;
  1392. } while (1);
  1393. order = pindex_to_order(pindex);
  1394. nr_pages = 1 << order;
  1395. do {
  1396. int mt;
  1397. page = list_last_entry(list, struct page, pcp_list);
  1398. mt = get_pcppage_migratetype(page);
  1399. /* must delete to avoid corrupting pcp list */
  1400. list_del(&page->pcp_list);
  1401. count -= nr_pages;
  1402. pcp->count -= nr_pages;
  1403. if (bulkfree_pcp_prepare(page))
  1404. continue;
  1405. /* MIGRATE_ISOLATE page should not go to pcplists */
  1406. VM_BUG_ON_PAGE(is_migrate_isolate(mt), page);
  1407. /* Pageblock could have been isolated meanwhile */
  1408. if (unlikely(isolated_pageblocks))
  1409. mt = get_pageblock_migratetype(page);
  1410. __free_one_page(page, page_to_pfn(page), zone, order, mt, FPI_NONE);
  1411. trace_mm_page_pcpu_drain(page, order, mt);
  1412. } while (count > 0 && !list_empty(list));
  1413. }
  1414. spin_unlock_irqrestore(&zone->lock, flags);
  1415. }
  1416. static void free_one_page(struct zone *zone,
  1417. struct page *page, unsigned long pfn,
  1418. unsigned int order,
  1419. int migratetype, fpi_t fpi_flags)
  1420. {
  1421. unsigned long flags;
  1422. spin_lock_irqsave(&zone->lock, flags);
  1423. if (unlikely(has_isolate_pageblock(zone) ||
  1424. is_migrate_isolate(migratetype))) {
  1425. migratetype = get_pfnblock_migratetype(page, pfn);
  1426. }
  1427. __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
  1428. spin_unlock_irqrestore(&zone->lock, flags);
  1429. }
  1430. static void __meminit __init_single_page(struct page *page, unsigned long pfn,
  1431. unsigned long zone, int nid)
  1432. {
  1433. mm_zero_struct_page(page);
  1434. set_page_links(page, zone, nid, pfn);
  1435. init_page_count(page);
  1436. page_mapcount_reset(page);
  1437. page_cpupid_reset_last(page);
  1438. page_kasan_tag_reset(page);
  1439. INIT_LIST_HEAD(&page->lru);
  1440. #ifdef WANT_PAGE_VIRTUAL
  1441. /* The shift won't overflow because ZONE_NORMAL is below 4G. */
  1442. if (!is_highmem_idx(zone))
  1443. set_page_address(page, __va(pfn << PAGE_SHIFT));
  1444. #endif
  1445. }
  1446. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1447. static void __meminit init_reserved_page(unsigned long pfn)
  1448. {
  1449. pg_data_t *pgdat;
  1450. int nid, zid;
  1451. if (!early_page_uninitialised(pfn))
  1452. return;
  1453. nid = early_pfn_to_nid(pfn);
  1454. pgdat = NODE_DATA(nid);
  1455. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  1456. struct zone *zone = &pgdat->node_zones[zid];
  1457. if (zone_spans_pfn(zone, pfn))
  1458. break;
  1459. }
  1460. __init_single_page(pfn_to_page(pfn), pfn, zid, nid);
  1461. }
  1462. #else
  1463. static inline void init_reserved_page(unsigned long pfn)
  1464. {
  1465. }
  1466. #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
  1467. /*
  1468. * Initialised pages do not have PageReserved set. This function is
  1469. * called for each range allocated by the bootmem allocator and
  1470. * marks the pages PageReserved. The remaining valid pages are later
  1471. * sent to the buddy page allocator.
  1472. */
  1473. void __meminit reserve_bootmem_region(phys_addr_t start, phys_addr_t end)
  1474. {
  1475. unsigned long start_pfn = PFN_DOWN(start);
  1476. unsigned long end_pfn = PFN_UP(end);
  1477. for (; start_pfn < end_pfn; start_pfn++) {
  1478. if (pfn_valid(start_pfn)) {
  1479. struct page *page = pfn_to_page(start_pfn);
  1480. init_reserved_page(start_pfn);
  1481. /* Avoid false-positive PageTail() */
  1482. INIT_LIST_HEAD(&page->lru);
  1483. /*
  1484. * no need for atomic set_bit because the struct
  1485. * page is not visible yet so nobody should
  1486. * access it yet.
  1487. */
  1488. __SetPageReserved(page);
  1489. }
  1490. }
  1491. }
  1492. static void __free_pages_ok(struct page *page, unsigned int order,
  1493. fpi_t fpi_flags)
  1494. {
  1495. unsigned long flags;
  1496. int migratetype;
  1497. unsigned long pfn = page_to_pfn(page);
  1498. struct zone *zone = page_zone(page);
  1499. bool skip_free_unref_page = false;
  1500. if (!free_pages_prepare(page, order, true, fpi_flags))
  1501. return;
  1502. migratetype = get_pfnblock_migratetype(page, pfn);
  1503. trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
  1504. if (skip_free_unref_page)
  1505. return;
  1506. spin_lock_irqsave(&zone->lock, flags);
  1507. if (unlikely(has_isolate_pageblock(zone) ||
  1508. is_migrate_isolate(migratetype))) {
  1509. migratetype = get_pfnblock_migratetype(page, pfn);
  1510. }
  1511. __free_one_page(page, pfn, zone, order, migratetype, fpi_flags);
  1512. spin_unlock_irqrestore(&zone->lock, flags);
  1513. __count_vm_events(PGFREE, 1 << order);
  1514. }
  1515. void __free_pages_core(struct page *page, unsigned int order)
  1516. {
  1517. unsigned int nr_pages = 1 << order;
  1518. struct page *p = page;
  1519. unsigned int loop;
  1520. /*
  1521. * When initializing the memmap, __init_single_page() sets the refcount
  1522. * of all pages to 1 ("allocated"/"not free"). We have to set the
  1523. * refcount of all involved pages to 0.
  1524. */
  1525. prefetchw(p);
  1526. for (loop = 0; loop < (nr_pages - 1); loop++, p++) {
  1527. prefetchw(p + 1);
  1528. __ClearPageReserved(p);
  1529. set_page_count(p, 0);
  1530. }
  1531. __ClearPageReserved(p);
  1532. set_page_count(p, 0);
  1533. atomic_long_add(nr_pages, &page_zone(page)->managed_pages);
  1534. /*
  1535. * Bypass PCP and place fresh pages right to the tail, primarily
  1536. * relevant for memory onlining.
  1537. */
  1538. __free_pages_ok(page, order, FPI_TO_TAIL | FPI_SKIP_KASAN_POISON);
  1539. }
  1540. #ifdef CONFIG_NUMA
  1541. /*
  1542. * During memory init memblocks map pfns to nids. The search is expensive and
  1543. * this caches recent lookups. The implementation of __early_pfn_to_nid
  1544. * treats start/end as pfns.
  1545. */
  1546. struct mminit_pfnnid_cache {
  1547. unsigned long last_start;
  1548. unsigned long last_end;
  1549. int last_nid;
  1550. };
  1551. static struct mminit_pfnnid_cache early_pfnnid_cache __meminitdata;
  1552. /*
  1553. * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
  1554. */
  1555. static int __meminit __early_pfn_to_nid(unsigned long pfn,
  1556. struct mminit_pfnnid_cache *state)
  1557. {
  1558. unsigned long start_pfn, end_pfn;
  1559. int nid;
  1560. if (state->last_start <= pfn && pfn < state->last_end)
  1561. return state->last_nid;
  1562. nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn);
  1563. if (nid != NUMA_NO_NODE) {
  1564. state->last_start = start_pfn;
  1565. state->last_end = end_pfn;
  1566. state->last_nid = nid;
  1567. }
  1568. return nid;
  1569. }
  1570. int __meminit early_pfn_to_nid(unsigned long pfn)
  1571. {
  1572. static DEFINE_SPINLOCK(early_pfn_lock);
  1573. int nid;
  1574. spin_lock(&early_pfn_lock);
  1575. nid = __early_pfn_to_nid(pfn, &early_pfnnid_cache);
  1576. if (nid < 0)
  1577. nid = first_online_node;
  1578. spin_unlock(&early_pfn_lock);
  1579. return nid;
  1580. }
  1581. #endif /* CONFIG_NUMA */
  1582. void __init memblock_free_pages(struct page *page, unsigned long pfn,
  1583. unsigned int order)
  1584. {
  1585. if (early_page_uninitialised(pfn))
  1586. return;
  1587. if (!kmsan_memblock_free_pages(page, order)) {
  1588. /* KMSAN will take care of these pages. */
  1589. return;
  1590. }
  1591. __free_pages_core(page, order);
  1592. }
  1593. /*
  1594. * Check that the whole (or subset of) a pageblock given by the interval of
  1595. * [start_pfn, end_pfn) is valid and within the same zone, before scanning it
  1596. * with the migration of free compaction scanner.
  1597. *
  1598. * Return struct page pointer of start_pfn, or NULL if checks were not passed.
  1599. *
  1600. * It's possible on some configurations to have a setup like node0 node1 node0
  1601. * i.e. it's possible that all pages within a zones range of pages do not
  1602. * belong to a single zone. We assume that a border between node0 and node1
  1603. * can occur within a single pageblock, but not a node0 node1 node0
  1604. * interleaving within a single pageblock. It is therefore sufficient to check
  1605. * the first and last page of a pageblock and avoid checking each individual
  1606. * page in a pageblock.
  1607. */
  1608. struct page *__pageblock_pfn_to_page(unsigned long start_pfn,
  1609. unsigned long end_pfn, struct zone *zone)
  1610. {
  1611. struct page *start_page;
  1612. struct page *end_page;
  1613. /* end_pfn is one past the range we are checking */
  1614. end_pfn--;
  1615. if (!pfn_valid(start_pfn) || !pfn_valid(end_pfn))
  1616. return NULL;
  1617. start_page = pfn_to_online_page(start_pfn);
  1618. if (!start_page)
  1619. return NULL;
  1620. if (page_zone(start_page) != zone)
  1621. return NULL;
  1622. end_page = pfn_to_page(end_pfn);
  1623. /* This gives a shorter code than deriving page_zone(end_page) */
  1624. if (page_zone_id(start_page) != page_zone_id(end_page))
  1625. return NULL;
  1626. return start_page;
  1627. }
  1628. void set_zone_contiguous(struct zone *zone)
  1629. {
  1630. unsigned long block_start_pfn = zone->zone_start_pfn;
  1631. unsigned long block_end_pfn;
  1632. block_end_pfn = pageblock_end_pfn(block_start_pfn);
  1633. for (; block_start_pfn < zone_end_pfn(zone);
  1634. block_start_pfn = block_end_pfn,
  1635. block_end_pfn += pageblock_nr_pages) {
  1636. block_end_pfn = min(block_end_pfn, zone_end_pfn(zone));
  1637. if (!__pageblock_pfn_to_page(block_start_pfn,
  1638. block_end_pfn, zone))
  1639. return;
  1640. cond_resched();
  1641. }
  1642. /* We confirm that there is no hole */
  1643. zone->contiguous = true;
  1644. }
  1645. void clear_zone_contiguous(struct zone *zone)
  1646. {
  1647. zone->contiguous = false;
  1648. }
  1649. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1650. static void __init deferred_free_range(unsigned long pfn,
  1651. unsigned long nr_pages)
  1652. {
  1653. struct page *page;
  1654. unsigned long i;
  1655. if (!nr_pages)
  1656. return;
  1657. page = pfn_to_page(pfn);
  1658. /* Free a large naturally-aligned chunk if possible */
  1659. if (nr_pages == pageblock_nr_pages && pageblock_aligned(pfn)) {
  1660. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1661. __free_pages_core(page, pageblock_order);
  1662. return;
  1663. }
  1664. for (i = 0; i < nr_pages; i++, page++, pfn++) {
  1665. if (pageblock_aligned(pfn))
  1666. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  1667. __free_pages_core(page, 0);
  1668. }
  1669. }
  1670. /* Completion tracking for deferred_init_memmap() threads */
  1671. static atomic_t pgdat_init_n_undone __initdata;
  1672. static __initdata DECLARE_COMPLETION(pgdat_init_all_done_comp);
  1673. static inline void __init pgdat_init_report_one_done(void)
  1674. {
  1675. if (atomic_dec_and_test(&pgdat_init_n_undone))
  1676. complete(&pgdat_init_all_done_comp);
  1677. }
  1678. /*
  1679. * Returns true if page needs to be initialized or freed to buddy allocator.
  1680. *
  1681. * We check if a current large page is valid by only checking the validity
  1682. * of the head pfn.
  1683. */
  1684. static inline bool __init deferred_pfn_valid(unsigned long pfn)
  1685. {
  1686. if (pageblock_aligned(pfn) && !pfn_valid(pfn))
  1687. return false;
  1688. return true;
  1689. }
  1690. /*
  1691. * Free pages to buddy allocator. Try to free aligned pages in
  1692. * pageblock_nr_pages sizes.
  1693. */
  1694. static void __init deferred_free_pages(unsigned long pfn,
  1695. unsigned long end_pfn)
  1696. {
  1697. unsigned long nr_free = 0;
  1698. for (; pfn < end_pfn; pfn++) {
  1699. if (!deferred_pfn_valid(pfn)) {
  1700. deferred_free_range(pfn - nr_free, nr_free);
  1701. nr_free = 0;
  1702. } else if (pageblock_aligned(pfn)) {
  1703. deferred_free_range(pfn - nr_free, nr_free);
  1704. nr_free = 1;
  1705. } else {
  1706. nr_free++;
  1707. }
  1708. }
  1709. /* Free the last block of pages to allocator */
  1710. deferred_free_range(pfn - nr_free, nr_free);
  1711. }
  1712. /*
  1713. * Initialize struct pages. We minimize pfn page lookups and scheduler checks
  1714. * by performing it only once every pageblock_nr_pages.
  1715. * Return number of pages initialized.
  1716. */
  1717. static unsigned long __init deferred_init_pages(struct zone *zone,
  1718. unsigned long pfn,
  1719. unsigned long end_pfn)
  1720. {
  1721. int nid = zone_to_nid(zone);
  1722. unsigned long nr_pages = 0;
  1723. int zid = zone_idx(zone);
  1724. struct page *page = NULL;
  1725. for (; pfn < end_pfn; pfn++) {
  1726. if (!deferred_pfn_valid(pfn)) {
  1727. page = NULL;
  1728. continue;
  1729. } else if (!page || pageblock_aligned(pfn)) {
  1730. page = pfn_to_page(pfn);
  1731. } else {
  1732. page++;
  1733. }
  1734. __init_single_page(page, pfn, zid, nid);
  1735. nr_pages++;
  1736. }
  1737. return (nr_pages);
  1738. }
  1739. /*
  1740. * This function is meant to pre-load the iterator for the zone init.
  1741. * Specifically it walks through the ranges until we are caught up to the
  1742. * first_init_pfn value and exits there. If we never encounter the value we
  1743. * return false indicating there are no valid ranges left.
  1744. */
  1745. static bool __init
  1746. deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone,
  1747. unsigned long *spfn, unsigned long *epfn,
  1748. unsigned long first_init_pfn)
  1749. {
  1750. u64 j;
  1751. /*
  1752. * Start out by walking through the ranges in this zone that have
  1753. * already been initialized. We don't need to do anything with them
  1754. * so we just need to flush them out of the system.
  1755. */
  1756. for_each_free_mem_pfn_range_in_zone(j, zone, spfn, epfn) {
  1757. if (*epfn <= first_init_pfn)
  1758. continue;
  1759. if (*spfn < first_init_pfn)
  1760. *spfn = first_init_pfn;
  1761. *i = j;
  1762. return true;
  1763. }
  1764. return false;
  1765. }
  1766. /*
  1767. * Initialize and free pages. We do it in two loops: first we initialize
  1768. * struct page, then free to buddy allocator, because while we are
  1769. * freeing pages we can access pages that are ahead (computing buddy
  1770. * page in __free_one_page()).
  1771. *
  1772. * In order to try and keep some memory in the cache we have the loop
  1773. * broken along max page order boundaries. This way we will not cause
  1774. * any issues with the buddy page computation.
  1775. */
  1776. static unsigned long __init
  1777. deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn,
  1778. unsigned long *end_pfn)
  1779. {
  1780. unsigned long mo_pfn = ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES);
  1781. unsigned long spfn = *start_pfn, epfn = *end_pfn;
  1782. unsigned long nr_pages = 0;
  1783. u64 j = *i;
  1784. /* First we loop through and initialize the page values */
  1785. for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) {
  1786. unsigned long t;
  1787. if (mo_pfn <= *start_pfn)
  1788. break;
  1789. t = min(mo_pfn, *end_pfn);
  1790. nr_pages += deferred_init_pages(zone, *start_pfn, t);
  1791. if (mo_pfn < *end_pfn) {
  1792. *start_pfn = mo_pfn;
  1793. break;
  1794. }
  1795. }
  1796. /* Reset values and now loop through freeing pages as needed */
  1797. swap(j, *i);
  1798. for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) {
  1799. unsigned long t;
  1800. if (mo_pfn <= spfn)
  1801. break;
  1802. t = min(mo_pfn, epfn);
  1803. deferred_free_pages(spfn, t);
  1804. if (mo_pfn <= epfn)
  1805. break;
  1806. }
  1807. return nr_pages;
  1808. }
  1809. static void __init
  1810. deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
  1811. void *arg)
  1812. {
  1813. unsigned long spfn, epfn;
  1814. struct zone *zone = arg;
  1815. u64 i;
  1816. deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn);
  1817. /*
  1818. * Initialize and free pages in MAX_ORDER sized increments so that we
  1819. * can avoid introducing any issues with the buddy allocator.
  1820. */
  1821. while (spfn < end_pfn) {
  1822. deferred_init_maxorder(&i, zone, &spfn, &epfn);
  1823. cond_resched();
  1824. }
  1825. }
  1826. /* An arch may override for more concurrency. */
  1827. __weak int __init
  1828. deferred_page_init_max_threads(const struct cpumask *node_cpumask)
  1829. {
  1830. return 1;
  1831. }
  1832. /* Initialise remaining memory on a node */
  1833. static int __init deferred_init_memmap(void *data)
  1834. {
  1835. pg_data_t *pgdat = data;
  1836. const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
  1837. unsigned long spfn = 0, epfn = 0;
  1838. unsigned long first_init_pfn, flags;
  1839. unsigned long start = jiffies;
  1840. struct zone *zone;
  1841. int zid, max_threads;
  1842. u64 i;
  1843. /* Bind memory initialisation thread to a local node if possible */
  1844. if (!cpumask_empty(cpumask))
  1845. set_cpus_allowed_ptr(current, cpumask);
  1846. pgdat_resize_lock(pgdat, &flags);
  1847. first_init_pfn = pgdat->first_deferred_pfn;
  1848. if (first_init_pfn == ULONG_MAX) {
  1849. pgdat_resize_unlock(pgdat, &flags);
  1850. pgdat_init_report_one_done();
  1851. return 0;
  1852. }
  1853. /* Sanity check boundaries */
  1854. BUG_ON(pgdat->first_deferred_pfn < pgdat->node_start_pfn);
  1855. BUG_ON(pgdat->first_deferred_pfn > pgdat_end_pfn(pgdat));
  1856. pgdat->first_deferred_pfn = ULONG_MAX;
  1857. /*
  1858. * Once we unlock here, the zone cannot be grown anymore, thus if an
  1859. * interrupt thread must allocate this early in boot, zone must be
  1860. * pre-grown prior to start of deferred page initialization.
  1861. */
  1862. pgdat_resize_unlock(pgdat, &flags);
  1863. /* Only the highest zone is deferred so find it */
  1864. for (zid = 0; zid < MAX_NR_ZONES; zid++) {
  1865. zone = pgdat->node_zones + zid;
  1866. if (first_init_pfn < zone_end_pfn(zone))
  1867. break;
  1868. }
  1869. /* If the zone is empty somebody else may have cleared out the zone */
  1870. if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
  1871. first_init_pfn))
  1872. goto zone_empty;
  1873. max_threads = deferred_page_init_max_threads(cpumask);
  1874. while (spfn < epfn) {
  1875. unsigned long epfn_align = ALIGN(epfn, PAGES_PER_SECTION);
  1876. struct padata_mt_job job = {
  1877. .thread_fn = deferred_init_memmap_chunk,
  1878. .fn_arg = zone,
  1879. .start = spfn,
  1880. .size = epfn_align - spfn,
  1881. .align = PAGES_PER_SECTION,
  1882. .min_chunk = PAGES_PER_SECTION,
  1883. .max_threads = max_threads,
  1884. };
  1885. padata_do_multithreaded(&job);
  1886. deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
  1887. epfn_align);
  1888. }
  1889. zone_empty:
  1890. /* Sanity check that the next zone really is unpopulated */
  1891. WARN_ON(++zid < MAX_NR_ZONES && populated_zone(++zone));
  1892. pr_info("node %d deferred pages initialised in %ums\n",
  1893. pgdat->node_id, jiffies_to_msecs(jiffies - start));
  1894. pgdat_init_report_one_done();
  1895. return 0;
  1896. }
  1897. /*
  1898. * If this zone has deferred pages, try to grow it by initializing enough
  1899. * deferred pages to satisfy the allocation specified by order, rounded up to
  1900. * the nearest PAGES_PER_SECTION boundary. So we're adding memory in increments
  1901. * of SECTION_SIZE bytes by initializing struct pages in increments of
  1902. * PAGES_PER_SECTION * sizeof(struct page) bytes.
  1903. *
  1904. * Return true when zone was grown, otherwise return false. We return true even
  1905. * when we grow less than requested, to let the caller decide if there are
  1906. * enough pages to satisfy the allocation.
  1907. *
  1908. * Note: We use noinline because this function is needed only during boot, and
  1909. * it is called from a __ref function _deferred_grow_zone. This way we are
  1910. * making sure that it is not inlined into permanent text section.
  1911. */
  1912. static noinline bool __init
  1913. deferred_grow_zone(struct zone *zone, unsigned int order)
  1914. {
  1915. unsigned long nr_pages_needed = ALIGN(1 << order, PAGES_PER_SECTION);
  1916. pg_data_t *pgdat = zone->zone_pgdat;
  1917. unsigned long first_deferred_pfn = pgdat->first_deferred_pfn;
  1918. unsigned long spfn, epfn, flags;
  1919. unsigned long nr_pages = 0;
  1920. u64 i;
  1921. /* Only the last zone may have deferred pages */
  1922. if (zone_end_pfn(zone) != pgdat_end_pfn(pgdat))
  1923. return false;
  1924. pgdat_resize_lock(pgdat, &flags);
  1925. /*
  1926. * If someone grew this zone while we were waiting for spinlock, return
  1927. * true, as there might be enough pages already.
  1928. */
  1929. if (first_deferred_pfn != pgdat->first_deferred_pfn) {
  1930. pgdat_resize_unlock(pgdat, &flags);
  1931. return true;
  1932. }
  1933. /* If the zone is empty somebody else may have cleared out the zone */
  1934. if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn,
  1935. first_deferred_pfn)) {
  1936. pgdat->first_deferred_pfn = ULONG_MAX;
  1937. pgdat_resize_unlock(pgdat, &flags);
  1938. /* Retry only once. */
  1939. return first_deferred_pfn != ULONG_MAX;
  1940. }
  1941. /*
  1942. * Initialize and free pages in MAX_ORDER sized increments so
  1943. * that we can avoid introducing any issues with the buddy
  1944. * allocator.
  1945. */
  1946. while (spfn < epfn) {
  1947. /* update our first deferred PFN for this section */
  1948. first_deferred_pfn = spfn;
  1949. nr_pages += deferred_init_maxorder(&i, zone, &spfn, &epfn);
  1950. touch_nmi_watchdog();
  1951. /* We should only stop along section boundaries */
  1952. if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION)
  1953. continue;
  1954. /* If our quota has been met we can stop here */
  1955. if (nr_pages >= nr_pages_needed)
  1956. break;
  1957. }
  1958. pgdat->first_deferred_pfn = spfn;
  1959. pgdat_resize_unlock(pgdat, &flags);
  1960. return nr_pages > 0;
  1961. }
  1962. /*
  1963. * deferred_grow_zone() is __init, but it is called from
  1964. * get_page_from_freelist() during early boot until deferred_pages permanently
  1965. * disables this call. This is why we have refdata wrapper to avoid warning,
  1966. * and to ensure that the function body gets unloaded.
  1967. */
  1968. static bool __ref
  1969. _deferred_grow_zone(struct zone *zone, unsigned int order)
  1970. {
  1971. return deferred_grow_zone(zone, order);
  1972. }
  1973. #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */
  1974. void __init page_alloc_init_late(void)
  1975. {
  1976. struct zone *zone;
  1977. int nid;
  1978. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  1979. /* There will be num_node_state(N_MEMORY) threads */
  1980. atomic_set(&pgdat_init_n_undone, num_node_state(N_MEMORY));
  1981. for_each_node_state(nid, N_MEMORY) {
  1982. kthread_run(deferred_init_memmap, NODE_DATA(nid), "pgdatinit%d", nid);
  1983. }
  1984. /* Block until all are initialised */
  1985. wait_for_completion(&pgdat_init_all_done_comp);
  1986. /*
  1987. * We initialized the rest of the deferred pages. Permanently disable
  1988. * on-demand struct page initialization.
  1989. */
  1990. static_branch_disable(&deferred_pages);
  1991. /* Reinit limits that are based on free pages after the kernel is up */
  1992. files_maxfiles_init();
  1993. #endif
  1994. buffer_init();
  1995. /* Discard memblock private memory */
  1996. memblock_discard();
  1997. for_each_node_state(nid, N_MEMORY)
  1998. shuffle_free_memory(NODE_DATA(nid));
  1999. for_each_populated_zone(zone)
  2000. set_zone_contiguous(zone);
  2001. }
  2002. #ifdef CONFIG_CMA
  2003. /* Free whole pageblock and set its migration type to MIGRATE_CMA. */
  2004. void __init init_cma_reserved_pageblock(struct page *page)
  2005. {
  2006. unsigned i = pageblock_nr_pages;
  2007. struct page *p = page;
  2008. do {
  2009. __ClearPageReserved(p);
  2010. set_page_count(p, 0);
  2011. } while (++p, --i);
  2012. set_pageblock_migratetype(page, MIGRATE_CMA);
  2013. set_page_refcounted(page);
  2014. __free_pages(page, pageblock_order);
  2015. adjust_managed_page_count(page, pageblock_nr_pages);
  2016. page_zone(page)->cma_pages += pageblock_nr_pages;
  2017. }
  2018. #endif
  2019. /*
  2020. * The order of subdivision here is critical for the IO subsystem.
  2021. * Please do not alter this order without good reasons and regression
  2022. * testing. Specifically, as large blocks of memory are subdivided,
  2023. * the order in which smaller blocks are delivered depends on the order
  2024. * they're subdivided in this function. This is the primary factor
  2025. * influencing the order in which pages are delivered to the IO
  2026. * subsystem according to empirical testing, and this is also justified
  2027. * by considering the behavior of a buddy system containing a single
  2028. * large block of memory acted on by a series of small allocations.
  2029. * This behavior is a critical factor in sglist merging's success.
  2030. *
  2031. * -- nyc
  2032. */
  2033. static inline void expand(struct zone *zone, struct page *page,
  2034. int low, int high, int migratetype)
  2035. {
  2036. unsigned long size = 1 << high;
  2037. while (high > low) {
  2038. high--;
  2039. size >>= 1;
  2040. VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]);
  2041. /*
  2042. * Mark as guard pages (or page), that will allow to
  2043. * merge back to allocator when buddy will be freed.
  2044. * Corresponding page table entries will not be touched,
  2045. * pages will stay not present in virtual address space
  2046. */
  2047. if (set_page_guard(zone, &page[size], high, migratetype))
  2048. continue;
  2049. add_to_free_list(&page[size], zone, high, migratetype);
  2050. set_buddy_order(&page[size], high);
  2051. }
  2052. }
  2053. static void check_new_page_bad(struct page *page)
  2054. {
  2055. if (unlikely(page->flags & __PG_HWPOISON)) {
  2056. /* Don't complain about hwpoisoned pages */
  2057. page_mapcount_reset(page); /* remove PageBuddy */
  2058. return;
  2059. }
  2060. bad_page(page,
  2061. page_bad_reason(page, PAGE_FLAGS_CHECK_AT_PREP));
  2062. }
  2063. /*
  2064. * This page is about to be returned from the page allocator
  2065. */
  2066. static inline int check_new_page(struct page *page)
  2067. {
  2068. if (likely(page_expected_state(page,
  2069. PAGE_FLAGS_CHECK_AT_PREP|__PG_HWPOISON)))
  2070. return 0;
  2071. check_new_page_bad(page);
  2072. return 1;
  2073. }
  2074. static bool check_new_pages(struct page *page, unsigned int order)
  2075. {
  2076. int i;
  2077. for (i = 0; i < (1 << order); i++) {
  2078. struct page *p = page + i;
  2079. if (unlikely(check_new_page(p)))
  2080. return true;
  2081. }
  2082. return false;
  2083. }
  2084. #ifdef CONFIG_DEBUG_VM
  2085. /*
  2086. * With DEBUG_VM enabled, order-0 pages are checked for expected state when
  2087. * being allocated from pcp lists. With debug_pagealloc also enabled, they are
  2088. * also checked when pcp lists are refilled from the free lists.
  2089. */
  2090. static inline bool check_pcp_refill(struct page *page, unsigned int order)
  2091. {
  2092. if (debug_pagealloc_enabled_static())
  2093. return check_new_pages(page, order);
  2094. else
  2095. return false;
  2096. }
  2097. static inline bool check_new_pcp(struct page *page, unsigned int order)
  2098. {
  2099. return check_new_pages(page, order);
  2100. }
  2101. #else
  2102. /*
  2103. * With DEBUG_VM disabled, free order-0 pages are checked for expected state
  2104. * when pcp lists are being refilled from the free lists. With debug_pagealloc
  2105. * enabled, they are also checked when being allocated from the pcp lists.
  2106. */
  2107. static inline bool check_pcp_refill(struct page *page, unsigned int order)
  2108. {
  2109. return check_new_pages(page, order);
  2110. }
  2111. static inline bool check_new_pcp(struct page *page, unsigned int order)
  2112. {
  2113. if (debug_pagealloc_enabled_static())
  2114. return check_new_pages(page, order);
  2115. else
  2116. return false;
  2117. }
  2118. #endif /* CONFIG_DEBUG_VM */
  2119. static inline bool should_skip_kasan_unpoison(gfp_t flags)
  2120. {
  2121. /* Don't skip if a software KASAN mode is enabled. */
  2122. if (IS_ENABLED(CONFIG_KASAN_GENERIC) ||
  2123. IS_ENABLED(CONFIG_KASAN_SW_TAGS))
  2124. return false;
  2125. /* Skip, if hardware tag-based KASAN is not enabled. */
  2126. if (!kasan_hw_tags_enabled())
  2127. return true;
  2128. /*
  2129. * With hardware tag-based KASAN enabled, skip if this has been
  2130. * requested via __GFP_SKIP_KASAN_UNPOISON.
  2131. */
  2132. return flags & __GFP_SKIP_KASAN_UNPOISON;
  2133. }
  2134. static inline bool should_skip_init(gfp_t flags)
  2135. {
  2136. /* Don't skip, if hardware tag-based KASAN is not enabled. */
  2137. if (!kasan_hw_tags_enabled())
  2138. return false;
  2139. /* For hardware tag-based KASAN, skip if requested. */
  2140. return (flags & __GFP_SKIP_ZERO);
  2141. }
  2142. inline void post_alloc_hook(struct page *page, unsigned int order,
  2143. gfp_t gfp_flags)
  2144. {
  2145. bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
  2146. !should_skip_init(gfp_flags);
  2147. bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS);
  2148. bool reset_tags = true;
  2149. int i;
  2150. set_page_private(page, 0);
  2151. set_page_refcounted(page);
  2152. arch_alloc_page(page, order);
  2153. debug_pagealloc_map_pages(page, 1 << order);
  2154. /*
  2155. * Page unpoisoning must happen before memory initialization.
  2156. * Otherwise, the poison pattern will be overwritten for __GFP_ZERO
  2157. * allocations and the page unpoisoning code will complain.
  2158. */
  2159. kernel_unpoison_pages(page, 1 << order);
  2160. /*
  2161. * As memory initialization might be integrated into KASAN,
  2162. * KASAN unpoisoning and memory initializion code must be
  2163. * kept together to avoid discrepancies in behavior.
  2164. */
  2165. /*
  2166. * If memory tags should be zeroed
  2167. * (which happens only when memory should be initialized as well).
  2168. */
  2169. if (zero_tags) {
  2170. /* Initialize both memory and memory tags. */
  2171. for (i = 0; i != 1 << order; ++i)
  2172. tag_clear_highpage(page + i);
  2173. /* Take note that memory was initialized by the loop above. */
  2174. init = false;
  2175. }
  2176. if (!should_skip_kasan_unpoison(gfp_flags)) {
  2177. /* Try unpoisoning (or setting tags) and initializing memory. */
  2178. if (kasan_unpoison_pages(page, order, init)) {
  2179. /* Take note that memory was initialized by KASAN. */
  2180. if (kasan_has_integrated_init())
  2181. init = false;
  2182. /* Take note that memory tags were set by KASAN. */
  2183. reset_tags = false;
  2184. } else {
  2185. /*
  2186. * KASAN decided to exclude this allocation from being
  2187. * (un)poisoned due to sampling. Make KASAN skip
  2188. * poisoning when the allocation is freed.
  2189. */
  2190. SetPageSkipKASanPoison(page);
  2191. }
  2192. }
  2193. /*
  2194. * If memory tags have not been set by KASAN, reset the page tags to
  2195. * ensure page_address() dereferencing does not fault.
  2196. */
  2197. if (reset_tags) {
  2198. for (i = 0; i != 1 << order; ++i)
  2199. page_kasan_tag_reset(page + i);
  2200. }
  2201. /* If memory is still not initialized, initialize it now. */
  2202. if (init)
  2203. kernel_init_pages(page, 1 << order);
  2204. /* Propagate __GFP_SKIP_KASAN_POISON to page flags. */
  2205. if (kasan_hw_tags_enabled() && (gfp_flags & __GFP_SKIP_KASAN_POISON))
  2206. SetPageSkipKASanPoison(page);
  2207. set_page_owner(page, order, gfp_flags);
  2208. page_table_check_alloc(page, order);
  2209. }
  2210. static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
  2211. unsigned int alloc_flags)
  2212. {
  2213. post_alloc_hook(page, order, gfp_flags);
  2214. if (order && (gfp_flags & __GFP_COMP))
  2215. prep_compound_page(page, order);
  2216. /*
  2217. * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to
  2218. * allocate the page. The expectation is that the caller is taking
  2219. * steps that will free more memory. The caller should avoid the page
  2220. * being used for !PFMEMALLOC purposes.
  2221. */
  2222. if (alloc_flags & ALLOC_NO_WATERMARKS)
  2223. set_page_pfmemalloc(page);
  2224. else
  2225. clear_page_pfmemalloc(page);
  2226. trace_android_vh_test_clear_look_around_ref(page);
  2227. }
  2228. /*
  2229. * Go through the free lists for the given migratetype and remove
  2230. * the smallest available page from the freelists
  2231. */
  2232. static __always_inline
  2233. struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
  2234. int migratetype)
  2235. {
  2236. unsigned int current_order;
  2237. struct free_area *area;
  2238. struct page *page;
  2239. /* Find a page of the appropriate size in the preferred list */
  2240. for (current_order = order; current_order < MAX_ORDER; ++current_order) {
  2241. area = &(zone->free_area[current_order]);
  2242. page = get_page_from_free_area(area, migratetype);
  2243. if (!page)
  2244. continue;
  2245. del_page_from_free_list(page, zone, current_order);
  2246. expand(zone, page, order, current_order, migratetype);
  2247. set_pcppage_migratetype(page, migratetype);
  2248. trace_mm_page_alloc_zone_locked(page, order, migratetype,
  2249. pcp_allowed_order(order) &&
  2250. migratetype < MIGRATE_PCPTYPES);
  2251. return page;
  2252. }
  2253. return NULL;
  2254. }
  2255. /*
  2256. * This array describes the order lists are fallen back to when
  2257. * the free lists for the desirable migrate type are depleted
  2258. *
  2259. * The other migratetypes do not have fallbacks.
  2260. */
  2261. static int fallbacks[MIGRATE_TYPES][3] = {
  2262. [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
  2263. [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_TYPES },
  2264. [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_TYPES },
  2265. };
  2266. #ifdef CONFIG_CMA
  2267. static __always_inline struct page *__rmqueue_cma_fallback(struct zone *zone,
  2268. unsigned int order)
  2269. {
  2270. return __rmqueue_smallest(zone, order, MIGRATE_CMA);
  2271. }
  2272. #else
  2273. static inline struct page *__rmqueue_cma_fallback(struct zone *zone,
  2274. unsigned int order) { return NULL; }
  2275. #endif
  2276. /*
  2277. * Move the free pages in a range to the freelist tail of the requested type.
  2278. * Note that start_page and end_pages are not aligned on a pageblock
  2279. * boundary. If alignment is required, use move_freepages_block()
  2280. */
  2281. static int move_freepages(struct zone *zone,
  2282. unsigned long start_pfn, unsigned long end_pfn,
  2283. int migratetype, int *num_movable)
  2284. {
  2285. struct page *page;
  2286. unsigned long pfn;
  2287. unsigned int order;
  2288. int pages_moved = 0;
  2289. for (pfn = start_pfn; pfn <= end_pfn;) {
  2290. page = pfn_to_page(pfn);
  2291. if (!PageBuddy(page)) {
  2292. /*
  2293. * We assume that pages that could be isolated for
  2294. * migration are movable. But we don't actually try
  2295. * isolating, as that would be expensive.
  2296. */
  2297. if (num_movable &&
  2298. (PageLRU(page) || __PageMovable(page)))
  2299. (*num_movable)++;
  2300. pfn++;
  2301. continue;
  2302. }
  2303. /* Make sure we are not inadvertently changing nodes */
  2304. VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page);
  2305. VM_BUG_ON_PAGE(page_zone(page) != zone, page);
  2306. order = buddy_order(page);
  2307. move_to_free_list(page, zone, order, migratetype);
  2308. pfn += 1 << order;
  2309. pages_moved += 1 << order;
  2310. }
  2311. return pages_moved;
  2312. }
  2313. int move_freepages_block(struct zone *zone, struct page *page,
  2314. int migratetype, int *num_movable)
  2315. {
  2316. unsigned long start_pfn, end_pfn, pfn;
  2317. if (num_movable)
  2318. *num_movable = 0;
  2319. pfn = page_to_pfn(page);
  2320. start_pfn = pageblock_start_pfn(pfn);
  2321. end_pfn = pageblock_end_pfn(pfn) - 1;
  2322. /* Do not cross zone boundaries */
  2323. if (!zone_spans_pfn(zone, start_pfn))
  2324. start_pfn = pfn;
  2325. if (!zone_spans_pfn(zone, end_pfn))
  2326. return 0;
  2327. return move_freepages(zone, start_pfn, end_pfn, migratetype,
  2328. num_movable);
  2329. }
  2330. static void change_pageblock_range(struct page *pageblock_page,
  2331. int start_order, int migratetype)
  2332. {
  2333. int nr_pageblocks = 1 << (start_order - pageblock_order);
  2334. while (nr_pageblocks--) {
  2335. set_pageblock_migratetype(pageblock_page, migratetype);
  2336. pageblock_page += pageblock_nr_pages;
  2337. }
  2338. }
  2339. /*
  2340. * When we are falling back to another migratetype during allocation, try to
  2341. * steal extra free pages from the same pageblocks to satisfy further
  2342. * allocations, instead of polluting multiple pageblocks.
  2343. *
  2344. * If we are stealing a relatively large buddy page, it is likely there will
  2345. * be more free pages in the pageblock, so try to steal them all. For
  2346. * reclaimable and unmovable allocations, we steal regardless of page size,
  2347. * as fragmentation caused by those allocations polluting movable pageblocks
  2348. * is worse than movable allocations stealing from unmovable and reclaimable
  2349. * pageblocks.
  2350. */
  2351. static bool can_steal_fallback(unsigned int order, int start_mt)
  2352. {
  2353. /*
  2354. * Leaving this order check is intended, although there is
  2355. * relaxed order check in next check. The reason is that
  2356. * we can actually steal whole pageblock if this condition met,
  2357. * but, below check doesn't guarantee it and that is just heuristic
  2358. * so could be changed anytime.
  2359. */
  2360. if (order >= pageblock_order)
  2361. return true;
  2362. if (order >= pageblock_order / 2 ||
  2363. start_mt == MIGRATE_RECLAIMABLE ||
  2364. start_mt == MIGRATE_UNMOVABLE ||
  2365. page_group_by_mobility_disabled)
  2366. return true;
  2367. return false;
  2368. }
  2369. static inline bool boost_watermark(struct zone *zone)
  2370. {
  2371. unsigned long max_boost;
  2372. if (!watermark_boost_factor)
  2373. return false;
  2374. /*
  2375. * Don't bother in zones that are unlikely to produce results.
  2376. * On small machines, including kdump capture kernels running
  2377. * in a small area, boosting the watermark can cause an out of
  2378. * memory situation immediately.
  2379. */
  2380. if ((pageblock_nr_pages * 4) > zone_managed_pages(zone))
  2381. return false;
  2382. max_boost = mult_frac(zone->_watermark[WMARK_HIGH],
  2383. watermark_boost_factor, 10000);
  2384. /*
  2385. * high watermark may be uninitialised if fragmentation occurs
  2386. * very early in boot so do not boost. We do not fall
  2387. * through and boost by pageblock_nr_pages as failing
  2388. * allocations that early means that reclaim is not going
  2389. * to help and it may even be impossible to reclaim the
  2390. * boosted watermark resulting in a hang.
  2391. */
  2392. if (!max_boost)
  2393. return false;
  2394. max_boost = max(pageblock_nr_pages, max_boost);
  2395. zone->watermark_boost = min(zone->watermark_boost + pageblock_nr_pages,
  2396. max_boost);
  2397. return true;
  2398. }
  2399. /*
  2400. * This function implements actual steal behaviour. If order is large enough,
  2401. * we can steal whole pageblock. If not, we first move freepages in this
  2402. * pageblock to our migratetype and determine how many already-allocated pages
  2403. * are there in the pageblock with a compatible migratetype. If at least half
  2404. * of pages are free or compatible, we can change migratetype of the pageblock
  2405. * itself, so pages freed in the future will be put on the correct free list.
  2406. */
  2407. static void steal_suitable_fallback(struct zone *zone, struct page *page,
  2408. unsigned int alloc_flags, int start_type, bool whole_block)
  2409. {
  2410. unsigned int current_order = buddy_order(page);
  2411. int free_pages, movable_pages, alike_pages;
  2412. int old_block_type;
  2413. old_block_type = get_pageblock_migratetype(page);
  2414. /*
  2415. * This can happen due to races and we want to prevent broken
  2416. * highatomic accounting.
  2417. */
  2418. if (is_migrate_highatomic(old_block_type))
  2419. goto single_page;
  2420. /* Take ownership for orders >= pageblock_order */
  2421. if (current_order >= pageblock_order) {
  2422. change_pageblock_range(page, current_order, start_type);
  2423. goto single_page;
  2424. }
  2425. /*
  2426. * Boost watermarks to increase reclaim pressure to reduce the
  2427. * likelihood of future fallbacks. Wake kswapd now as the node
  2428. * may be balanced overall and kswapd will not wake naturally.
  2429. */
  2430. if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
  2431. set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
  2432. /* We are not allowed to try stealing from the whole block */
  2433. if (!whole_block)
  2434. goto single_page;
  2435. free_pages = move_freepages_block(zone, page, start_type,
  2436. &movable_pages);
  2437. /*
  2438. * Determine how many pages are compatible with our allocation.
  2439. * For movable allocation, it's the number of movable pages which
  2440. * we just obtained. For other types it's a bit more tricky.
  2441. */
  2442. if (start_type == MIGRATE_MOVABLE) {
  2443. alike_pages = movable_pages;
  2444. } else {
  2445. /*
  2446. * If we are falling back a RECLAIMABLE or UNMOVABLE allocation
  2447. * to MOVABLE pageblock, consider all non-movable pages as
  2448. * compatible. If it's UNMOVABLE falling back to RECLAIMABLE or
  2449. * vice versa, be conservative since we can't distinguish the
  2450. * exact migratetype of non-movable pages.
  2451. */
  2452. if (old_block_type == MIGRATE_MOVABLE)
  2453. alike_pages = pageblock_nr_pages
  2454. - (free_pages + movable_pages);
  2455. else
  2456. alike_pages = 0;
  2457. }
  2458. /* moving whole block can fail due to zone boundary conditions */
  2459. if (!free_pages)
  2460. goto single_page;
  2461. /*
  2462. * If a sufficient number of pages in the block are either free or of
  2463. * comparable migratability as our allocation, claim the whole block.
  2464. */
  2465. if (free_pages + alike_pages >= (1 << (pageblock_order-1)) ||
  2466. page_group_by_mobility_disabled)
  2467. set_pageblock_migratetype(page, start_type);
  2468. return;
  2469. single_page:
  2470. move_to_free_list(page, zone, current_order, start_type);
  2471. }
  2472. /*
  2473. * Check whether there is a suitable fallback freepage with requested order.
  2474. * If only_stealable is true, this function returns fallback_mt only if
  2475. * we can steal other freepages all together. This would help to reduce
  2476. * fragmentation due to mixed migratetype pages in one pageblock.
  2477. */
  2478. int find_suitable_fallback(struct free_area *area, unsigned int order,
  2479. int migratetype, bool only_stealable, bool *can_steal)
  2480. {
  2481. int i;
  2482. int fallback_mt;
  2483. if (area->nr_free == 0)
  2484. return -1;
  2485. *can_steal = false;
  2486. for (i = 0;; i++) {
  2487. fallback_mt = fallbacks[migratetype][i];
  2488. if (fallback_mt == MIGRATE_TYPES)
  2489. break;
  2490. if (free_area_empty(area, fallback_mt))
  2491. continue;
  2492. if (can_steal_fallback(order, migratetype))
  2493. *can_steal = true;
  2494. if (!only_stealable)
  2495. return fallback_mt;
  2496. if (*can_steal)
  2497. return fallback_mt;
  2498. }
  2499. return -1;
  2500. }
  2501. /*
  2502. * Reserve a pageblock for exclusive use of high-order atomic allocations if
  2503. * there are no empty page blocks that contain a page with a suitable order
  2504. */
  2505. static void reserve_highatomic_pageblock(struct page *page, struct zone *zone,
  2506. unsigned int alloc_order)
  2507. {
  2508. int mt;
  2509. unsigned long max_managed, flags;
  2510. #ifdef CONFIG_ARCH_QTI_VM
  2511. /*
  2512. * The number reserved as: minimum is 1 pageblock, maximum is
  2513. * roughly 1% of a zone. But if 1% of a zone falls below a
  2514. * pageblock size, then don't reserve any pageblocks.
  2515. * Check is race-prone but harmless.
  2516. */
  2517. if ((zone_managed_pages(zone) / 100) < pageblock_nr_pages)
  2518. return;
  2519. max_managed = ALIGN((zone_managed_pages(zone) / 100), pageblock_nr_pages);
  2520. #else
  2521. /*
  2522. * Limit the number reserved to 1 pageblock or roughly 1% of a zone.
  2523. * Check is race-prone but harmless.
  2524. */
  2525. max_managed = (zone_managed_pages(zone) / 100) + pageblock_nr_pages;
  2526. #endif
  2527. if (zone->nr_reserved_highatomic >= max_managed)
  2528. return;
  2529. spin_lock_irqsave(&zone->lock, flags);
  2530. /* Recheck the nr_reserved_highatomic limit under the lock */
  2531. if (zone->nr_reserved_highatomic >= max_managed)
  2532. goto out_unlock;
  2533. /* Yoink! */
  2534. mt = get_pageblock_migratetype(page);
  2535. /* Only reserve normal pageblocks (i.e., they can merge with others) */
  2536. if (migratetype_is_mergeable(mt)) {
  2537. zone->nr_reserved_highatomic += pageblock_nr_pages;
  2538. set_pageblock_migratetype(page, MIGRATE_HIGHATOMIC);
  2539. move_freepages_block(zone, page, MIGRATE_HIGHATOMIC, NULL);
  2540. }
  2541. out_unlock:
  2542. spin_unlock_irqrestore(&zone->lock, flags);
  2543. }
  2544. /*
  2545. * Used when an allocation is about to fail under memory pressure. This
  2546. * potentially hurts the reliability of high-order allocations when under
  2547. * intense memory pressure but failed atomic allocations should be easier
  2548. * to recover from than an OOM.
  2549. *
  2550. * If @force is true, try to unreserve a pageblock even though highatomic
  2551. * pageblock is exhausted.
  2552. */
  2553. static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
  2554. bool force)
  2555. {
  2556. struct zonelist *zonelist = ac->zonelist;
  2557. unsigned long flags;
  2558. struct zoneref *z;
  2559. struct zone *zone;
  2560. struct page *page;
  2561. int order;
  2562. bool ret;
  2563. bool skip_unreserve_highatomic = false;
  2564. for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->highest_zoneidx,
  2565. ac->nodemask) {
  2566. /*
  2567. * Preserve at least one pageblock unless memory pressure
  2568. * is really high.
  2569. */
  2570. if (!force && zone->nr_reserved_highatomic <=
  2571. pageblock_nr_pages)
  2572. continue;
  2573. trace_android_vh_unreserve_highatomic_bypass(force, zone,
  2574. &skip_unreserve_highatomic);
  2575. if (skip_unreserve_highatomic)
  2576. continue;
  2577. spin_lock_irqsave(&zone->lock, flags);
  2578. for (order = 0; order < MAX_ORDER; order++) {
  2579. struct free_area *area = &(zone->free_area[order]);
  2580. page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
  2581. if (!page)
  2582. continue;
  2583. /*
  2584. * In page freeing path, migratetype change is racy so
  2585. * we can counter several free pages in a pageblock
  2586. * in this loop although we changed the pageblock type
  2587. * from highatomic to ac->migratetype. So we should
  2588. * adjust the count once.
  2589. */
  2590. if (is_migrate_highatomic_page(page)) {
  2591. /*
  2592. * It should never happen but changes to
  2593. * locking could inadvertently allow a per-cpu
  2594. * drain to add pages to MIGRATE_HIGHATOMIC
  2595. * while unreserving so be safe and watch for
  2596. * underflows.
  2597. */
  2598. zone->nr_reserved_highatomic -= min(
  2599. pageblock_nr_pages,
  2600. zone->nr_reserved_highatomic);
  2601. }
  2602. /*
  2603. * Convert to ac->migratetype and avoid the normal
  2604. * pageblock stealing heuristics. Minimally, the caller
  2605. * is doing the work and needs the pages. More
  2606. * importantly, if the block was always converted to
  2607. * MIGRATE_UNMOVABLE or another type then the number
  2608. * of pageblocks that cannot be completely freed
  2609. * may increase.
  2610. */
  2611. set_pageblock_migratetype(page, ac->migratetype);
  2612. ret = move_freepages_block(zone, page, ac->migratetype,
  2613. NULL);
  2614. if (ret) {
  2615. spin_unlock_irqrestore(&zone->lock, flags);
  2616. return ret;
  2617. }
  2618. }
  2619. spin_unlock_irqrestore(&zone->lock, flags);
  2620. }
  2621. return false;
  2622. }
  2623. /*
  2624. * Try finding a free buddy page on the fallback list and put it on the free
  2625. * list of requested migratetype, possibly along with other pages from the same
  2626. * block, depending on fragmentation avoidance heuristics. Returns true if
  2627. * fallback was found so that __rmqueue_smallest() can grab it.
  2628. *
  2629. * The use of signed ints for order and current_order is a deliberate
  2630. * deviation from the rest of this file, to make the for loop
  2631. * condition simpler.
  2632. */
  2633. static __always_inline bool
  2634. __rmqueue_fallback(struct zone *zone, int order, int start_migratetype,
  2635. unsigned int alloc_flags)
  2636. {
  2637. struct free_area *area;
  2638. int current_order;
  2639. int min_order = order;
  2640. struct page *page;
  2641. int fallback_mt;
  2642. bool can_steal;
  2643. /*
  2644. * Do not steal pages from freelists belonging to other pageblocks
  2645. * i.e. orders < pageblock_order. If there are no local zones free,
  2646. * the zonelists will be reiterated without ALLOC_NOFRAGMENT.
  2647. */
  2648. if (order < pageblock_order && alloc_flags & ALLOC_NOFRAGMENT)
  2649. min_order = pageblock_order;
  2650. /*
  2651. * Find the largest available free page in the other list. This roughly
  2652. * approximates finding the pageblock with the most free pages, which
  2653. * would be too costly to do exactly.
  2654. */
  2655. for (current_order = MAX_ORDER - 1; current_order >= min_order;
  2656. --current_order) {
  2657. area = &(zone->free_area[current_order]);
  2658. fallback_mt = find_suitable_fallback(area, current_order,
  2659. start_migratetype, false, &can_steal);
  2660. if (fallback_mt == -1)
  2661. continue;
  2662. /*
  2663. * We cannot steal all free pages from the pageblock and the
  2664. * requested migratetype is movable. In that case it's better to
  2665. * steal and split the smallest available page instead of the
  2666. * largest available page, because even if the next movable
  2667. * allocation falls back into a different pageblock than this
  2668. * one, it won't cause permanent fragmentation.
  2669. */
  2670. if (!can_steal && start_migratetype == MIGRATE_MOVABLE
  2671. && current_order > order)
  2672. goto find_smallest;
  2673. goto do_steal;
  2674. }
  2675. return false;
  2676. find_smallest:
  2677. for (current_order = order; current_order < MAX_ORDER;
  2678. current_order++) {
  2679. area = &(zone->free_area[current_order]);
  2680. fallback_mt = find_suitable_fallback(area, current_order,
  2681. start_migratetype, false, &can_steal);
  2682. if (fallback_mt != -1)
  2683. break;
  2684. }
  2685. /*
  2686. * This should not happen - we already found a suitable fallback
  2687. * when looking for the largest page.
  2688. */
  2689. VM_BUG_ON(current_order == MAX_ORDER);
  2690. do_steal:
  2691. page = get_page_from_free_area(area, fallback_mt);
  2692. steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
  2693. can_steal);
  2694. trace_mm_page_alloc_extfrag(page, order, current_order,
  2695. start_migratetype, fallback_mt);
  2696. return true;
  2697. }
  2698. /*
  2699. * Do the hard work of removing an element from the buddy allocator.
  2700. * Call me with the zone->lock already held.
  2701. */
  2702. static __always_inline struct page *
  2703. __rmqueue(struct zone *zone, unsigned int order, int migratetype,
  2704. unsigned int alloc_flags)
  2705. {
  2706. struct page *page = NULL;
  2707. trace_android_vh_rmqueue_smallest_bypass(&page, zone, order, migratetype);
  2708. if (page)
  2709. return page;
  2710. retry:
  2711. page = __rmqueue_smallest(zone, order, migratetype);
  2712. if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype,
  2713. alloc_flags))
  2714. goto retry;
  2715. return page;
  2716. }
  2717. #ifdef CONFIG_CMA
  2718. static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
  2719. int migratetype,
  2720. unsigned int alloc_flags)
  2721. {
  2722. struct page *page = __rmqueue_cma_fallback(zone, order);
  2723. if (page)
  2724. trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA,
  2725. pcp_allowed_order(order) &&
  2726. migratetype < MIGRATE_PCPTYPES);
  2727. return page;
  2728. }
  2729. #else
  2730. static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
  2731. int migratetype,
  2732. unsigned int alloc_flags)
  2733. {
  2734. return NULL;
  2735. }
  2736. #endif
  2737. /*
  2738. * Obtain a specified number of elements from the buddy allocator, all under
  2739. * a single hold of the lock, for efficiency. Add them to the supplied list.
  2740. * Returns the number of new pages which were placed at *list.
  2741. */
  2742. static int rmqueue_bulk(struct zone *zone, unsigned int order,
  2743. unsigned long count, struct list_head *list,
  2744. int migratetype, unsigned int alloc_flags)
  2745. {
  2746. unsigned long flags;
  2747. int i, allocated = 0;
  2748. spin_lock_irqsave(&zone->lock, flags);
  2749. for (i = 0; i < count; ++i) {
  2750. struct page *page;
  2751. if (is_migrate_cma(migratetype))
  2752. page = __rmqueue_cma(zone, order, migratetype,
  2753. alloc_flags);
  2754. else
  2755. page = __rmqueue(zone, order, migratetype, alloc_flags);
  2756. if (unlikely(page == NULL))
  2757. break;
  2758. if (unlikely(check_pcp_refill(page, order)))
  2759. continue;
  2760. /*
  2761. * Split buddy pages returned by expand() are received here in
  2762. * physical page order. The page is added to the tail of
  2763. * caller's list. From the callers perspective, the linked list
  2764. * is ordered by page number under some conditions. This is
  2765. * useful for IO devices that can forward direction from the
  2766. * head, thus also in the physical page order. This is useful
  2767. * for IO devices that can merge IO requests if the physical
  2768. * pages are ordered properly.
  2769. */
  2770. list_add_tail(&page->pcp_list, list);
  2771. allocated++;
  2772. if (is_migrate_cma(get_pcppage_migratetype(page)))
  2773. __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
  2774. -(1 << order));
  2775. }
  2776. /*
  2777. * i pages were removed from the buddy list even if some leak due
  2778. * to check_pcp_refill failing so adjust NR_FREE_PAGES based
  2779. * on i. Do not confuse with 'allocated' which is the number of
  2780. * pages added to the pcp list.
  2781. */
  2782. __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
  2783. spin_unlock_irqrestore(&zone->lock, flags);
  2784. return allocated;
  2785. }
  2786. /*
  2787. * Return the pcp list that corresponds to the migrate type if that list isn't
  2788. * empty.
  2789. * If the list is empty return NULL.
  2790. */
  2791. static struct list_head *get_populated_pcp_list(struct zone *zone,
  2792. unsigned int order, struct per_cpu_pages *pcp,
  2793. int migratetype, unsigned int alloc_flags)
  2794. {
  2795. struct list_head *list = &pcp->lists[order_to_pindex(migratetype, order)];
  2796. if (list_empty(list)) {
  2797. int batch = READ_ONCE(pcp->batch);
  2798. int alloced;
  2799. trace_android_vh_rmqueue_bulk_bypass(order, pcp, migratetype, list);
  2800. if (!list_empty(list))
  2801. return list;
  2802. /*
  2803. * Scale batch relative to order if batch implies
  2804. * free pages can be stored on the PCP. Batch can
  2805. * be 1 for small zones or for boot pagesets which
  2806. * should never store free pages as the pages may
  2807. * belong to arbitrary zones.
  2808. */
  2809. if (batch > 1)
  2810. batch = max(batch >> order, 2);
  2811. alloced = rmqueue_bulk(zone, order, pcp->batch, list, migratetype, alloc_flags);
  2812. pcp->count += alloced << order;
  2813. if (list_empty(list))
  2814. list = NULL;
  2815. }
  2816. return list;
  2817. }
  2818. #ifdef CONFIG_NUMA
  2819. /*
  2820. * Called from the vmstat counter updater to drain pagesets of this
  2821. * currently executing processor on remote nodes after they have
  2822. * expired.
  2823. */
  2824. void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
  2825. {
  2826. int to_drain, batch;
  2827. batch = READ_ONCE(pcp->batch);
  2828. to_drain = min(pcp->count, batch);
  2829. if (to_drain > 0) {
  2830. spin_lock(&pcp->lock);
  2831. free_pcppages_bulk(zone, to_drain, pcp, 0);
  2832. spin_unlock(&pcp->lock);
  2833. }
  2834. }
  2835. #endif
  2836. /*
  2837. * Drain pcplists of the indicated processor and zone.
  2838. */
  2839. static void drain_pages_zone(unsigned int cpu, struct zone *zone)
  2840. {
  2841. struct per_cpu_pages *pcp;
  2842. pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
  2843. if (pcp->count) {
  2844. spin_lock(&pcp->lock);
  2845. free_pcppages_bulk(zone, pcp->count, pcp, 0);
  2846. spin_unlock(&pcp->lock);
  2847. }
  2848. }
  2849. /*
  2850. * Drain pcplists of all zones on the indicated processor.
  2851. */
  2852. static void drain_pages(unsigned int cpu)
  2853. {
  2854. struct zone *zone;
  2855. for_each_populated_zone(zone) {
  2856. drain_pages_zone(cpu, zone);
  2857. }
  2858. }
  2859. /*
  2860. * Spill all of this CPU's per-cpu pages back into the buddy allocator.
  2861. */
  2862. void drain_local_pages(struct zone *zone)
  2863. {
  2864. int cpu = smp_processor_id();
  2865. if (zone)
  2866. drain_pages_zone(cpu, zone);
  2867. else
  2868. drain_pages(cpu);
  2869. }
  2870. /*
  2871. * The implementation of drain_all_pages(), exposing an extra parameter to
  2872. * drain on all cpus.
  2873. *
  2874. * drain_all_pages() is optimized to only execute on cpus where pcplists are
  2875. * not empty. The check for non-emptiness can however race with a free to
  2876. * pcplist that has not yet increased the pcp->count from 0 to 1. Callers
  2877. * that need the guarantee that every CPU has drained can disable the
  2878. * optimizing racy check.
  2879. */
  2880. static void __drain_all_pages(struct zone *zone, bool force_all_cpus)
  2881. {
  2882. int cpu;
  2883. /*
  2884. * Allocate in the BSS so we won't require allocation in
  2885. * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
  2886. */
  2887. static cpumask_t cpus_with_pcps;
  2888. /*
  2889. * Do not drain if one is already in progress unless it's specific to
  2890. * a zone. Such callers are primarily CMA and memory hotplug and need
  2891. * the drain to be complete when the call returns.
  2892. */
  2893. if (unlikely(!mutex_trylock(&pcpu_drain_mutex))) {
  2894. if (!zone)
  2895. return;
  2896. mutex_lock(&pcpu_drain_mutex);
  2897. }
  2898. /*
  2899. * We don't care about racing with CPU hotplug event
  2900. * as offline notification will cause the notified
  2901. * cpu to drain that CPU pcps and on_each_cpu_mask
  2902. * disables preemption as part of its processing
  2903. */
  2904. for_each_online_cpu(cpu) {
  2905. struct per_cpu_pages *pcp;
  2906. struct zone *z;
  2907. bool has_pcps = false;
  2908. if (force_all_cpus) {
  2909. /*
  2910. * The pcp.count check is racy, some callers need a
  2911. * guarantee that no cpu is missed.
  2912. */
  2913. has_pcps = true;
  2914. } else if (zone) {
  2915. pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
  2916. if (pcp->count)
  2917. has_pcps = true;
  2918. } else {
  2919. for_each_populated_zone(z) {
  2920. pcp = per_cpu_ptr(z->per_cpu_pageset, cpu);
  2921. if (pcp->count) {
  2922. has_pcps = true;
  2923. break;
  2924. }
  2925. }
  2926. }
  2927. if (has_pcps)
  2928. cpumask_set_cpu(cpu, &cpus_with_pcps);
  2929. else
  2930. cpumask_clear_cpu(cpu, &cpus_with_pcps);
  2931. }
  2932. for_each_cpu(cpu, &cpus_with_pcps) {
  2933. if (zone)
  2934. drain_pages_zone(cpu, zone);
  2935. else
  2936. drain_pages(cpu);
  2937. }
  2938. mutex_unlock(&pcpu_drain_mutex);
  2939. }
  2940. /*
  2941. * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
  2942. *
  2943. * When zone parameter is non-NULL, spill just the single zone's pages.
  2944. */
  2945. void drain_all_pages(struct zone *zone)
  2946. {
  2947. __drain_all_pages(zone, false);
  2948. }
  2949. #ifdef CONFIG_HIBERNATION
  2950. /*
  2951. * Touch the watchdog for every WD_PAGE_COUNT pages.
  2952. */
  2953. #define WD_PAGE_COUNT (128*1024)
  2954. void mark_free_pages(struct zone *zone)
  2955. {
  2956. unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
  2957. unsigned long flags;
  2958. unsigned int order, t;
  2959. struct page *page;
  2960. if (zone_is_empty(zone))
  2961. return;
  2962. spin_lock_irqsave(&zone->lock, flags);
  2963. max_zone_pfn = zone_end_pfn(zone);
  2964. for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
  2965. if (pfn_valid(pfn)) {
  2966. page = pfn_to_page(pfn);
  2967. if (!--page_count) {
  2968. touch_nmi_watchdog();
  2969. page_count = WD_PAGE_COUNT;
  2970. }
  2971. if (page_zone(page) != zone)
  2972. continue;
  2973. if (!swsusp_page_is_forbidden(page))
  2974. swsusp_unset_page_free(page);
  2975. }
  2976. for_each_migratetype_order(order, t) {
  2977. list_for_each_entry(page,
  2978. &zone->free_area[order].free_list[t], buddy_list) {
  2979. unsigned long i;
  2980. pfn = page_to_pfn(page);
  2981. for (i = 0; i < (1UL << order); i++) {
  2982. if (!--page_count) {
  2983. touch_nmi_watchdog();
  2984. page_count = WD_PAGE_COUNT;
  2985. }
  2986. swsusp_set_page_free(pfn_to_page(pfn + i));
  2987. }
  2988. }
  2989. }
  2990. spin_unlock_irqrestore(&zone->lock, flags);
  2991. }
  2992. #endif /* CONFIG_PM */
  2993. static bool free_unref_page_prepare(struct page *page, unsigned long pfn,
  2994. unsigned int order)
  2995. {
  2996. int migratetype;
  2997. if (!free_pcp_prepare(page, order))
  2998. return false;
  2999. migratetype = get_pfnblock_migratetype(page, pfn);
  3000. set_pcppage_migratetype(page, migratetype);
  3001. return true;
  3002. }
  3003. static int nr_pcp_free(struct per_cpu_pages *pcp, int high, int batch,
  3004. bool free_high)
  3005. {
  3006. int min_nr_free, max_nr_free;
  3007. /* Free everything if batch freeing high-order pages. */
  3008. if (unlikely(free_high))
  3009. return pcp->count;
  3010. /* Check for PCP disabled or boot pageset */
  3011. if (unlikely(high < batch))
  3012. return 1;
  3013. /* Leave at least pcp->batch pages on the list */
  3014. min_nr_free = batch;
  3015. max_nr_free = high - batch;
  3016. /*
  3017. * Double the number of pages freed each time there is subsequent
  3018. * freeing of pages without any allocation.
  3019. */
  3020. batch <<= pcp->free_factor;
  3021. if (batch < max_nr_free)
  3022. pcp->free_factor++;
  3023. batch = clamp(batch, min_nr_free, max_nr_free);
  3024. return batch;
  3025. }
  3026. static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone,
  3027. bool free_high)
  3028. {
  3029. int high = READ_ONCE(pcp->high);
  3030. if (unlikely(!high || free_high))
  3031. return 0;
  3032. if (!test_bit(ZONE_RECLAIM_ACTIVE, &zone->flags))
  3033. return high;
  3034. /*
  3035. * If reclaim is active, limit the number of pages that can be
  3036. * stored on pcp lists
  3037. */
  3038. return min(READ_ONCE(pcp->batch) << 2, high);
  3039. }
  3040. static void free_unref_page_commit(struct zone *zone, struct per_cpu_pages *pcp,
  3041. struct page *page, int migratetype,
  3042. unsigned int order)
  3043. {
  3044. int high;
  3045. int pindex;
  3046. bool free_high;
  3047. __count_vm_events(PGFREE, 1 << order);
  3048. pindex = order_to_pindex(migratetype, order);
  3049. list_add(&page->pcp_list, &pcp->lists[pindex]);
  3050. pcp->count += 1 << order;
  3051. /*
  3052. * As high-order pages other than THP's stored on PCP can contribute
  3053. * to fragmentation, limit the number stored when PCP is heavily
  3054. * freeing without allocation. The remainder after bulk freeing
  3055. * stops will be drained from vmstat refresh context.
  3056. */
  3057. free_high = (pcp->free_factor && order && order <= PAGE_ALLOC_COSTLY_ORDER);
  3058. high = nr_pcp_high(pcp, zone, free_high);
  3059. if (pcp->count >= high) {
  3060. int batch = READ_ONCE(pcp->batch);
  3061. free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch, free_high), pcp, pindex);
  3062. }
  3063. }
  3064. /*
  3065. * Free a pcp page
  3066. */
  3067. void free_unref_page(struct page *page, unsigned int order)
  3068. {
  3069. unsigned long __maybe_unused UP_flags;
  3070. struct per_cpu_pages *pcp;
  3071. struct zone *zone;
  3072. unsigned long pfn = page_to_pfn(page);
  3073. int migratetype, pcpmigratetype;
  3074. bool skip_free_unref_page = false;
  3075. if (!free_unref_page_prepare(page, pfn, order))
  3076. return;
  3077. migratetype = get_pcppage_migratetype(page);
  3078. trace_android_vh_free_unref_page_bypass(page, order, migratetype, &skip_free_unref_page);
  3079. if (skip_free_unref_page)
  3080. return;
  3081. /*
  3082. * We only track unmovable, reclaimable, movable, and CMA on pcp lists.
  3083. * Place ISOLATE pages on the isolated list because they are being
  3084. * offlined but treat HIGHATOMIC and CMA as movable pages so we can
  3085. * get those areas back if necessary. Otherwise, we may have to free
  3086. * excessively into the page allocator
  3087. */
  3088. migratetype = pcpmigratetype = get_pcppage_migratetype(page);
  3089. if (unlikely(migratetype > MIGRATE_RECLAIMABLE)) {
  3090. if (unlikely(is_migrate_isolate(migratetype))) {
  3091. free_one_page(page_zone(page), page, pfn, order, migratetype, FPI_NONE);
  3092. return;
  3093. }
  3094. if (pcpmigratetype == MIGRATE_HIGHATOMIC)
  3095. pcpmigratetype = MIGRATE_MOVABLE;
  3096. }
  3097. zone = page_zone(page);
  3098. pcp_trylock_prepare(UP_flags);
  3099. pcp = pcp_spin_trylock(zone->per_cpu_pageset);
  3100. if (pcp) {
  3101. free_unref_page_commit(zone, pcp, page, pcpmigratetype, order);
  3102. pcp_spin_unlock(pcp);
  3103. } else {
  3104. free_one_page(zone, page, pfn, order, migratetype, FPI_NONE);
  3105. }
  3106. pcp_trylock_finish(UP_flags);
  3107. }
  3108. /*
  3109. * Free a list of 0-order pages
  3110. */
  3111. void free_unref_page_list(struct list_head *list)
  3112. {
  3113. unsigned long __maybe_unused UP_flags;
  3114. struct page *page, *next;
  3115. struct per_cpu_pages *pcp = NULL;
  3116. struct zone *locked_zone = NULL;
  3117. int batch_count = 0;
  3118. int migratetype;
  3119. /* Prepare pages for freeing */
  3120. list_for_each_entry_safe(page, next, list, lru) {
  3121. unsigned long pfn = page_to_pfn(page);
  3122. if (!free_unref_page_prepare(page, pfn, 0)) {
  3123. list_del(&page->lru);
  3124. continue;
  3125. }
  3126. /*
  3127. * Free isolated pages directly to the allocator, see
  3128. * comment in free_unref_page.
  3129. */
  3130. migratetype = get_pcppage_migratetype(page);
  3131. if (unlikely(is_migrate_isolate(migratetype))) {
  3132. list_del(&page->lru);
  3133. free_one_page(page_zone(page), page, pfn, 0, migratetype, FPI_NONE);
  3134. continue;
  3135. }
  3136. }
  3137. list_for_each_entry_safe(page, next, list, lru) {
  3138. struct zone *zone = page_zone(page);
  3139. list_del(&page->lru);
  3140. migratetype = get_pcppage_migratetype(page);
  3141. /* Different zone, different pcp lock. */
  3142. if (zone != locked_zone) {
  3143. if (pcp) {
  3144. pcp_spin_unlock(pcp);
  3145. pcp_trylock_finish(UP_flags);
  3146. }
  3147. /*
  3148. * trylock is necessary as pages may be getting freed
  3149. * from IRQ or SoftIRQ context after an IO completion.
  3150. */
  3151. pcp_trylock_prepare(UP_flags);
  3152. pcp = pcp_spin_trylock(zone->per_cpu_pageset);
  3153. if (unlikely(!pcp)) {
  3154. pcp_trylock_finish(UP_flags);
  3155. free_one_page(zone, page, page_to_pfn(page),
  3156. 0, migratetype, FPI_NONE);
  3157. locked_zone = NULL;
  3158. continue;
  3159. }
  3160. locked_zone = zone;
  3161. batch_count = 0;
  3162. }
  3163. /*
  3164. * Non-isolated types over MIGRATE_PCPTYPES get added
  3165. * to the MIGRATE_MOVABLE pcp list.
  3166. */
  3167. if (unlikely(migratetype >= MIGRATE_PCPTYPES))
  3168. migratetype = MIGRATE_MOVABLE;
  3169. trace_mm_page_free_batched(page);
  3170. free_unref_page_commit(zone, pcp, page, migratetype, 0);
  3171. /*
  3172. * Guard against excessive lock hold times when freeing
  3173. * a large list of pages. Lock will be reacquired if
  3174. * necessary on the next iteration.
  3175. */
  3176. if (++batch_count == SWAP_CLUSTER_MAX) {
  3177. pcp_spin_unlock(pcp);
  3178. pcp_trylock_finish(UP_flags);
  3179. batch_count = 0;
  3180. pcp = NULL;
  3181. locked_zone = NULL;
  3182. }
  3183. }
  3184. if (pcp) {
  3185. pcp_spin_unlock(pcp);
  3186. pcp_trylock_finish(UP_flags);
  3187. }
  3188. }
  3189. /*
  3190. * split_page takes a non-compound higher-order page, and splits it into
  3191. * n (1<<order) sub-pages: page[0..n]
  3192. * Each sub-page must be freed individually.
  3193. *
  3194. * Note: this is probably too low level an operation for use in drivers.
  3195. * Please consult with lkml before using this in your driver.
  3196. */
  3197. void split_page(struct page *page, unsigned int order)
  3198. {
  3199. int i;
  3200. VM_BUG_ON_PAGE(PageCompound(page), page);
  3201. VM_BUG_ON_PAGE(!page_count(page), page);
  3202. for (i = 1; i < (1 << order); i++)
  3203. set_page_refcounted(page + i);
  3204. split_page_owner(page, 1 << order);
  3205. split_page_memcg(page, 1 << order);
  3206. }
  3207. EXPORT_SYMBOL_GPL(split_page);
  3208. int __isolate_free_page(struct page *page, unsigned int order)
  3209. {
  3210. struct zone *zone = page_zone(page);
  3211. int mt = get_pageblock_migratetype(page);
  3212. if (!is_migrate_isolate(mt)) {
  3213. unsigned long watermark;
  3214. /*
  3215. * Obey watermarks as if the page was being allocated. We can
  3216. * emulate a high-order watermark check with a raised order-0
  3217. * watermark, because we already know our high-order page
  3218. * exists.
  3219. */
  3220. watermark = zone->_watermark[WMARK_MIN] + (1UL << order);
  3221. if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
  3222. return 0;
  3223. __mod_zone_freepage_state(zone, -(1UL << order), mt);
  3224. }
  3225. del_page_from_free_list(page, zone, order);
  3226. /*
  3227. * Set the pageblock if the isolated page is at least half of a
  3228. * pageblock
  3229. */
  3230. if (order >= pageblock_order - 1) {
  3231. struct page *endpage = page + (1 << order) - 1;
  3232. for (; page < endpage; page += pageblock_nr_pages) {
  3233. int mt = get_pageblock_migratetype(page);
  3234. /*
  3235. * Only change normal pageblocks (i.e., they can merge
  3236. * with others)
  3237. */
  3238. if (migratetype_is_mergeable(mt))
  3239. set_pageblock_migratetype(page,
  3240. MIGRATE_MOVABLE);
  3241. }
  3242. }
  3243. return 1UL << order;
  3244. }
  3245. /**
  3246. * __putback_isolated_page - Return a now-isolated page back where we got it
  3247. * @page: Page that was isolated
  3248. * @order: Order of the isolated page
  3249. * @mt: The page's pageblock's migratetype
  3250. *
  3251. * This function is meant to return a page pulled from the free lists via
  3252. * __isolate_free_page back to the free lists they were pulled from.
  3253. */
  3254. void __putback_isolated_page(struct page *page, unsigned int order, int mt)
  3255. {
  3256. struct zone *zone = page_zone(page);
  3257. /* zone lock should be held when this function is called */
  3258. lockdep_assert_held(&zone->lock);
  3259. /* Return isolated page to tail of freelist. */
  3260. __free_one_page(page, page_to_pfn(page), zone, order, mt,
  3261. FPI_SKIP_REPORT_NOTIFY | FPI_TO_TAIL);
  3262. }
  3263. /*
  3264. * Update NUMA hit/miss statistics
  3265. */
  3266. static inline void zone_statistics(struct zone *preferred_zone, struct zone *z,
  3267. long nr_account)
  3268. {
  3269. #ifdef CONFIG_NUMA
  3270. enum numa_stat_item local_stat = NUMA_LOCAL;
  3271. /* skip numa counters update if numa stats is disabled */
  3272. if (!static_branch_likely(&vm_numa_stat_key))
  3273. return;
  3274. if (zone_to_nid(z) != numa_node_id())
  3275. local_stat = NUMA_OTHER;
  3276. if (zone_to_nid(z) == zone_to_nid(preferred_zone))
  3277. __count_numa_events(z, NUMA_HIT, nr_account);
  3278. else {
  3279. __count_numa_events(z, NUMA_MISS, nr_account);
  3280. __count_numa_events(preferred_zone, NUMA_FOREIGN, nr_account);
  3281. }
  3282. __count_numa_events(z, local_stat, nr_account);
  3283. #endif
  3284. }
  3285. static __always_inline
  3286. struct page *rmqueue_buddy(struct zone *preferred_zone, struct zone *zone,
  3287. unsigned int order, unsigned int alloc_flags,
  3288. int migratetype)
  3289. {
  3290. struct page *page;
  3291. unsigned long flags;
  3292. do {
  3293. page = NULL;
  3294. spin_lock_irqsave(&zone->lock, flags);
  3295. /*
  3296. * order-0 request can reach here when the pcplist is skipped
  3297. * due to non-CMA allocation context. HIGHATOMIC area is
  3298. * reserved for high-order atomic allocation, so order-0
  3299. * request should skip it.
  3300. */
  3301. if (order > 0 && alloc_flags & ALLOC_HARDER)
  3302. page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC);
  3303. if (!page) {
  3304. if (alloc_flags & ALLOC_CMA && migratetype == MIGRATE_MOVABLE)
  3305. page = __rmqueue_cma(zone, order, migratetype,
  3306. alloc_flags);
  3307. else
  3308. page = __rmqueue(zone, order, migratetype,
  3309. alloc_flags);
  3310. if (!page) {
  3311. spin_unlock_irqrestore(&zone->lock, flags);
  3312. return NULL;
  3313. }
  3314. }
  3315. __mod_zone_freepage_state(zone, -(1 << order),
  3316. get_pcppage_migratetype(page));
  3317. spin_unlock_irqrestore(&zone->lock, flags);
  3318. } while (check_new_pages(page, order));
  3319. __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
  3320. zone_statistics(preferred_zone, zone, 1);
  3321. return page;
  3322. }
  3323. /* Remove page from the per-cpu list, caller must protect the list */
  3324. static inline
  3325. struct page *__rmqueue_pcplist(struct zone *zone, unsigned int order,
  3326. int migratetype,
  3327. unsigned int alloc_flags,
  3328. struct per_cpu_pages *pcp)
  3329. {
  3330. struct page *page = NULL;
  3331. struct list_head *list = NULL;
  3332. do {
  3333. /* First try to get CMA pages */
  3334. if (migratetype == MIGRATE_MOVABLE && alloc_flags & ALLOC_CMA)
  3335. list = get_populated_pcp_list(zone, order, pcp, get_cma_migrate_type(),
  3336. alloc_flags);
  3337. if (list == NULL) {
  3338. /*
  3339. * Either CMA is not suitable or there are no
  3340. * free CMA pages.
  3341. */
  3342. list = get_populated_pcp_list(zone, order, pcp, migratetype, alloc_flags);
  3343. if (unlikely(list == NULL) || unlikely(list_empty(list)))
  3344. return NULL;
  3345. }
  3346. page = list_first_entry(list, struct page, pcp_list);
  3347. list_del(&page->pcp_list);
  3348. pcp->count -= 1 << order;
  3349. } while (check_new_pcp(page, order));
  3350. return page;
  3351. }
  3352. /* Lock and remove page from the per-cpu list */
  3353. static struct page *rmqueue_pcplist(struct zone *preferred_zone,
  3354. struct zone *zone, unsigned int order,
  3355. int migratetype, unsigned int alloc_flags)
  3356. {
  3357. struct per_cpu_pages *pcp;
  3358. struct page *page;
  3359. unsigned long __maybe_unused UP_flags;
  3360. /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */
  3361. pcp_trylock_prepare(UP_flags);
  3362. pcp = pcp_spin_trylock(zone->per_cpu_pageset);
  3363. if (!pcp) {
  3364. pcp_trylock_finish(UP_flags);
  3365. return NULL;
  3366. }
  3367. /*
  3368. * On allocation, reduce the number of pages that are batch freed.
  3369. * See nr_pcp_free() where free_factor is increased for subsequent
  3370. * frees.
  3371. */
  3372. pcp->free_factor >>= 1;
  3373. page = __rmqueue_pcplist(zone, order, migratetype, alloc_flags, pcp);
  3374. pcp_spin_unlock(pcp);
  3375. pcp_trylock_finish(UP_flags);
  3376. if (page) {
  3377. __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order);
  3378. zone_statistics(preferred_zone, zone, 1);
  3379. }
  3380. return page;
  3381. }
  3382. /*
  3383. * Allocate a page from the given zone.
  3384. * Use pcplists for THP or "cheap" high-order allocations.
  3385. */
  3386. /*
  3387. * Do not instrument rmqueue() with KMSAN. This function may call
  3388. * __msan_poison_alloca() through a call to set_pfnblock_flags_mask().
  3389. * If __msan_poison_alloca() attempts to allocate pages for the stack depot, it
  3390. * may call rmqueue() again, which will result in a deadlock.
  3391. */
  3392. __no_sanitize_memory
  3393. static inline
  3394. struct page *rmqueue(struct zone *preferred_zone,
  3395. struct zone *zone, unsigned int order,
  3396. gfp_t gfp_flags, unsigned int alloc_flags,
  3397. int migratetype)
  3398. {
  3399. struct page *page;
  3400. /*
  3401. * We most definitely don't want callers attempting to
  3402. * allocate greater than order-1 page units with __GFP_NOFAIL.
  3403. */
  3404. WARN_ON_ONCE((gfp_flags & __GFP_NOFAIL) && (order > 1));
  3405. if (likely(pcp_allowed_order(order))) {
  3406. page = rmqueue_pcplist(preferred_zone, zone, order,
  3407. migratetype, alloc_flags);
  3408. if (likely(page))
  3409. goto out;
  3410. }
  3411. page = rmqueue_buddy(preferred_zone, zone, order, alloc_flags,
  3412. migratetype);
  3413. trace_android_vh_rmqueue(preferred_zone, zone, order,
  3414. gfp_flags, alloc_flags, migratetype);
  3415. out:
  3416. /* Separate test+clear to avoid unnecessary atomics */
  3417. if (unlikely(test_bit(ZONE_BOOSTED_WATERMARK, &zone->flags))) {
  3418. clear_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
  3419. wakeup_kswapd(zone, 0, 0, zone_idx(zone));
  3420. }
  3421. VM_BUG_ON_PAGE(page && bad_range(zone, page), page);
  3422. return page;
  3423. }
  3424. #ifdef CONFIG_FAIL_PAGE_ALLOC
  3425. static struct {
  3426. struct fault_attr attr;
  3427. bool ignore_gfp_highmem;
  3428. bool ignore_gfp_reclaim;
  3429. u32 min_order;
  3430. } fail_page_alloc = {
  3431. .attr = FAULT_ATTR_INITIALIZER,
  3432. .ignore_gfp_reclaim = true,
  3433. .ignore_gfp_highmem = true,
  3434. .min_order = 1,
  3435. };
  3436. static int __init setup_fail_page_alloc(char *str)
  3437. {
  3438. return setup_fault_attr(&fail_page_alloc.attr, str);
  3439. }
  3440. __setup("fail_page_alloc=", setup_fail_page_alloc);
  3441. static bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  3442. {
  3443. int flags = 0;
  3444. if (order < fail_page_alloc.min_order)
  3445. return false;
  3446. if (gfp_mask & __GFP_NOFAIL)
  3447. return false;
  3448. if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
  3449. return false;
  3450. if (fail_page_alloc.ignore_gfp_reclaim &&
  3451. (gfp_mask & __GFP_DIRECT_RECLAIM))
  3452. return false;
  3453. /* See comment in __should_failslab() */
  3454. if (gfp_mask & __GFP_NOWARN)
  3455. flags |= FAULT_NOWARN;
  3456. return should_fail_ex(&fail_page_alloc.attr, 1 << order, flags);
  3457. }
  3458. #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
  3459. static int __init fail_page_alloc_debugfs(void)
  3460. {
  3461. umode_t mode = S_IFREG | 0600;
  3462. struct dentry *dir;
  3463. dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
  3464. &fail_page_alloc.attr);
  3465. debugfs_create_bool("ignore-gfp-wait", mode, dir,
  3466. &fail_page_alloc.ignore_gfp_reclaim);
  3467. debugfs_create_bool("ignore-gfp-highmem", mode, dir,
  3468. &fail_page_alloc.ignore_gfp_highmem);
  3469. debugfs_create_u32("min-order", mode, dir, &fail_page_alloc.min_order);
  3470. return 0;
  3471. }
  3472. late_initcall(fail_page_alloc_debugfs);
  3473. #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
  3474. #else /* CONFIG_FAIL_PAGE_ALLOC */
  3475. static inline bool __should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  3476. {
  3477. return false;
  3478. }
  3479. #endif /* CONFIG_FAIL_PAGE_ALLOC */
  3480. noinline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
  3481. {
  3482. return __should_fail_alloc_page(gfp_mask, order);
  3483. }
  3484. ALLOW_ERROR_INJECTION(should_fail_alloc_page, TRUE);
  3485. static inline long __zone_watermark_unusable_free(struct zone *z,
  3486. unsigned int order, unsigned int alloc_flags)
  3487. {
  3488. const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
  3489. long unusable_free = (1 << order) - 1;
  3490. /*
  3491. * If the caller does not have rights to ALLOC_HARDER then subtract
  3492. * the high-atomic reserves. This will over-estimate the size of the
  3493. * atomic reserve but it avoids a search.
  3494. */
  3495. if (likely(!alloc_harder))
  3496. unusable_free += z->nr_reserved_highatomic;
  3497. #ifdef CONFIG_CMA
  3498. /* If allocation can't use CMA areas don't use free CMA pages */
  3499. if (!(alloc_flags & ALLOC_CMA))
  3500. unusable_free += zone_page_state(z, NR_FREE_CMA_PAGES);
  3501. #endif
  3502. return unusable_free;
  3503. }
  3504. /*
  3505. * Return true if free base pages are above 'mark'. For high-order checks it
  3506. * will return true of the order-0 watermark is reached and there is at least
  3507. * one free page of a suitable size. Checking now avoids taking the zone lock
  3508. * to check in the allocation paths if no pages are free.
  3509. */
  3510. bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
  3511. int highest_zoneidx, unsigned int alloc_flags,
  3512. long free_pages)
  3513. {
  3514. long min = mark;
  3515. int o;
  3516. const bool alloc_harder = (alloc_flags & (ALLOC_HARDER|ALLOC_OOM));
  3517. /* free_pages may go negative - that's OK */
  3518. free_pages -= __zone_watermark_unusable_free(z, order, alloc_flags);
  3519. if (alloc_flags & ALLOC_HIGH)
  3520. min -= min / 2;
  3521. if (unlikely(alloc_harder)) {
  3522. /*
  3523. * OOM victims can try even harder than normal ALLOC_HARDER
  3524. * users on the grounds that it's definitely going to be in
  3525. * the exit path shortly and free memory. Any allocation it
  3526. * makes during the free path will be small and short-lived.
  3527. */
  3528. if (alloc_flags & ALLOC_OOM)
  3529. min -= min / 2;
  3530. else
  3531. min -= min / 4;
  3532. }
  3533. /*
  3534. * Check watermarks for an order-0 allocation request. If these
  3535. * are not met, then a high-order request also cannot go ahead
  3536. * even if a suitable page happened to be free.
  3537. */
  3538. if (free_pages <= min + z->lowmem_reserve[highest_zoneidx])
  3539. return false;
  3540. /* If this is an order-0 request then the watermark is fine */
  3541. if (!order)
  3542. return true;
  3543. /* For a high-order request, check at least one suitable page is free */
  3544. for (o = order; o < MAX_ORDER; o++) {
  3545. struct free_area *area = &z->free_area[o];
  3546. int mt;
  3547. if (!area->nr_free)
  3548. continue;
  3549. for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
  3550. #ifdef CONFIG_CMA
  3551. /*
  3552. * Note that this check is needed only
  3553. * when MIGRATE_CMA < MIGRATE_PCPTYPES.
  3554. */
  3555. if (mt == MIGRATE_CMA)
  3556. continue;
  3557. #endif
  3558. if (!free_area_empty(area, mt))
  3559. return true;
  3560. }
  3561. #ifdef CONFIG_CMA
  3562. if ((alloc_flags & ALLOC_CMA) &&
  3563. !free_area_empty(area, MIGRATE_CMA)) {
  3564. return true;
  3565. }
  3566. #endif
  3567. if (alloc_harder && !free_area_empty(area, MIGRATE_HIGHATOMIC))
  3568. return true;
  3569. }
  3570. return false;
  3571. }
  3572. bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
  3573. int highest_zoneidx, unsigned int alloc_flags)
  3574. {
  3575. return __zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
  3576. zone_page_state(z, NR_FREE_PAGES));
  3577. }
  3578. static inline bool zone_watermark_fast(struct zone *z, unsigned int order,
  3579. unsigned long mark, int highest_zoneidx,
  3580. unsigned int alloc_flags, gfp_t gfp_mask)
  3581. {
  3582. long free_pages;
  3583. free_pages = zone_page_state(z, NR_FREE_PAGES);
  3584. /*
  3585. * Fast check for order-0 only. If this fails then the reserves
  3586. * need to be calculated.
  3587. */
  3588. if (!order) {
  3589. long usable_free;
  3590. long reserved;
  3591. usable_free = free_pages;
  3592. reserved = __zone_watermark_unusable_free(z, 0, alloc_flags);
  3593. /* reserved may over estimate high-atomic reserves. */
  3594. usable_free -= min(usable_free, reserved);
  3595. if (usable_free > mark + z->lowmem_reserve[highest_zoneidx])
  3596. return true;
  3597. }
  3598. if (__zone_watermark_ok(z, order, mark, highest_zoneidx, alloc_flags,
  3599. free_pages))
  3600. return true;
  3601. /*
  3602. * Ignore watermark boosting for GFP_ATOMIC order-0 allocations
  3603. * when checking the min watermark. The min watermark is the
  3604. * point where boosting is ignored so that kswapd is woken up
  3605. * when below the low watermark.
  3606. */
  3607. if (unlikely(!order && (gfp_mask & __GFP_ATOMIC) && z->watermark_boost
  3608. && ((alloc_flags & ALLOC_WMARK_MASK) == WMARK_MIN))) {
  3609. mark = z->_watermark[WMARK_MIN];
  3610. return __zone_watermark_ok(z, order, mark, highest_zoneidx,
  3611. alloc_flags, free_pages);
  3612. }
  3613. return false;
  3614. }
  3615. bool zone_watermark_ok_safe(struct zone *z, unsigned int order,
  3616. unsigned long mark, int highest_zoneidx)
  3617. {
  3618. long free_pages = zone_page_state(z, NR_FREE_PAGES);
  3619. if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
  3620. free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
  3621. return __zone_watermark_ok(z, order, mark, highest_zoneidx, 0,
  3622. free_pages);
  3623. }
  3624. #ifdef CONFIG_NUMA
  3625. int __read_mostly node_reclaim_distance = RECLAIM_DISTANCE;
  3626. static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
  3627. {
  3628. return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) <=
  3629. node_reclaim_distance;
  3630. }
  3631. #else /* CONFIG_NUMA */
  3632. static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone)
  3633. {
  3634. return true;
  3635. }
  3636. #endif /* CONFIG_NUMA */
  3637. /*
  3638. * The restriction on ZONE_DMA32 as being a suitable zone to use to avoid
  3639. * fragmentation is subtle. If the preferred zone was HIGHMEM then
  3640. * premature use of a lower zone may cause lowmem pressure problems that
  3641. * are worse than fragmentation. If the next zone is ZONE_DMA then it is
  3642. * probably too small. It only makes sense to spread allocations to avoid
  3643. * fragmentation between the Normal and DMA32 zones.
  3644. */
  3645. static inline unsigned int
  3646. alloc_flags_nofragment(struct zone *zone, gfp_t gfp_mask)
  3647. {
  3648. unsigned int alloc_flags;
  3649. /*
  3650. * __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
  3651. * to save a branch.
  3652. */
  3653. alloc_flags = (__force int) (gfp_mask & __GFP_KSWAPD_RECLAIM);
  3654. #ifdef CONFIG_ZONE_DMA32
  3655. if (!zone)
  3656. return alloc_flags;
  3657. if (zone_idx(zone) != ZONE_NORMAL)
  3658. return alloc_flags;
  3659. /*
  3660. * If ZONE_DMA32 exists, assume it is the one after ZONE_NORMAL and
  3661. * the pointer is within zone->zone_pgdat->node_zones[]. Also assume
  3662. * on UMA that if Normal is populated then so is DMA32.
  3663. */
  3664. BUILD_BUG_ON(ZONE_NORMAL - ZONE_DMA32 != 1);
  3665. if (nr_online_nodes > 1 && !populated_zone(--zone))
  3666. return alloc_flags;
  3667. alloc_flags |= ALLOC_NOFRAGMENT;
  3668. #endif /* CONFIG_ZONE_DMA32 */
  3669. return alloc_flags;
  3670. }
  3671. /* Must be called after current_gfp_context() which can change gfp_mask */
  3672. static inline unsigned int gfp_to_alloc_flags_cma(gfp_t gfp_mask,
  3673. unsigned int alloc_flags)
  3674. {
  3675. #ifdef CONFIG_CMA
  3676. if (gfp_migratetype(gfp_mask) == MIGRATE_MOVABLE && gfp_mask & __GFP_CMA)
  3677. alloc_flags |= ALLOC_CMA;
  3678. #endif
  3679. return alloc_flags;
  3680. }
  3681. /*
  3682. * get_page_from_freelist goes through the zonelist trying to allocate
  3683. * a page.
  3684. */
  3685. static struct page *
  3686. get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags,
  3687. const struct alloc_context *ac)
  3688. {
  3689. struct zoneref *z;
  3690. struct zone *zone;
  3691. struct pglist_data *last_pgdat = NULL;
  3692. bool last_pgdat_dirty_ok = false;
  3693. bool no_fallback;
  3694. retry:
  3695. /*
  3696. * Scan zonelist, looking for a zone with enough free.
  3697. * See also __cpuset_node_allowed() comment in kernel/cgroup/cpuset.c.
  3698. */
  3699. no_fallback = alloc_flags & ALLOC_NOFRAGMENT;
  3700. z = ac->preferred_zoneref;
  3701. for_next_zone_zonelist_nodemask(zone, z, ac->highest_zoneidx,
  3702. ac->nodemask) {
  3703. struct page *page;
  3704. unsigned long mark;
  3705. if (cpusets_enabled() &&
  3706. (alloc_flags & ALLOC_CPUSET) &&
  3707. !__cpuset_zone_allowed(zone, gfp_mask))
  3708. continue;
  3709. /*
  3710. * When allocating a page cache page for writing, we
  3711. * want to get it from a node that is within its dirty
  3712. * limit, such that no single node holds more than its
  3713. * proportional share of globally allowed dirty pages.
  3714. * The dirty limits take into account the node's
  3715. * lowmem reserves and high watermark so that kswapd
  3716. * should be able to balance it without having to
  3717. * write pages from its LRU list.
  3718. *
  3719. * XXX: For now, allow allocations to potentially
  3720. * exceed the per-node dirty limit in the slowpath
  3721. * (spread_dirty_pages unset) before going into reclaim,
  3722. * which is important when on a NUMA setup the allowed
  3723. * nodes are together not big enough to reach the
  3724. * global limit. The proper fix for these situations
  3725. * will require awareness of nodes in the
  3726. * dirty-throttling and the flusher threads.
  3727. */
  3728. if (ac->spread_dirty_pages) {
  3729. if (last_pgdat != zone->zone_pgdat) {
  3730. last_pgdat = zone->zone_pgdat;
  3731. last_pgdat_dirty_ok = node_dirty_ok(zone->zone_pgdat);
  3732. }
  3733. if (!last_pgdat_dirty_ok)
  3734. continue;
  3735. }
  3736. if (no_fallback && nr_online_nodes > 1 &&
  3737. zone != ac->preferred_zoneref->zone) {
  3738. int local_nid;
  3739. /*
  3740. * If moving to a remote node, retry but allow
  3741. * fragmenting fallbacks. Locality is more important
  3742. * than fragmentation avoidance.
  3743. */
  3744. local_nid = zone_to_nid(ac->preferred_zoneref->zone);
  3745. if (zone_to_nid(zone) != local_nid) {
  3746. alloc_flags &= ~ALLOC_NOFRAGMENT;
  3747. goto retry;
  3748. }
  3749. }
  3750. mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK);
  3751. trace_android_vh_get_page_wmark(alloc_flags, &mark);
  3752. if (!zone_watermark_fast(zone, order, mark,
  3753. ac->highest_zoneidx, alloc_flags,
  3754. gfp_mask)) {
  3755. int ret;
  3756. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  3757. /*
  3758. * Watermark failed for this zone, but see if we can
  3759. * grow this zone if it contains deferred pages.
  3760. */
  3761. if (static_branch_unlikely(&deferred_pages)) {
  3762. if (_deferred_grow_zone(zone, order))
  3763. goto try_this_zone;
  3764. }
  3765. #endif
  3766. /* Checked here to keep the fast path fast */
  3767. BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
  3768. if (alloc_flags & ALLOC_NO_WATERMARKS)
  3769. goto try_this_zone;
  3770. if (!node_reclaim_enabled() ||
  3771. !zone_allows_reclaim(ac->preferred_zoneref->zone, zone))
  3772. continue;
  3773. ret = node_reclaim(zone->zone_pgdat, gfp_mask, order);
  3774. switch (ret) {
  3775. case NODE_RECLAIM_NOSCAN:
  3776. /* did not scan */
  3777. continue;
  3778. case NODE_RECLAIM_FULL:
  3779. /* scanned but unreclaimable */
  3780. continue;
  3781. default:
  3782. /* did we reclaim enough */
  3783. if (zone_watermark_ok(zone, order, mark,
  3784. ac->highest_zoneidx, alloc_flags))
  3785. goto try_this_zone;
  3786. continue;
  3787. }
  3788. }
  3789. try_this_zone:
  3790. page = rmqueue(ac->preferred_zoneref->zone, zone, order,
  3791. gfp_mask, alloc_flags, ac->migratetype);
  3792. if (page) {
  3793. prep_new_page(page, order, gfp_mask, alloc_flags);
  3794. /*
  3795. * If this is a high-order atomic allocation then check
  3796. * if the pageblock should be reserved for the future
  3797. */
  3798. if (unlikely(order && (alloc_flags & ALLOC_HARDER)))
  3799. reserve_highatomic_pageblock(page, zone, order);
  3800. return page;
  3801. } else {
  3802. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  3803. /* Try again if zone has deferred pages */
  3804. if (static_branch_unlikely(&deferred_pages)) {
  3805. if (_deferred_grow_zone(zone, order))
  3806. goto try_this_zone;
  3807. }
  3808. #endif
  3809. }
  3810. }
  3811. /*
  3812. * It's possible on a UMA machine to get through all zones that are
  3813. * fragmented. If avoiding fragmentation, reset and try again.
  3814. */
  3815. if (no_fallback) {
  3816. alloc_flags &= ~ALLOC_NOFRAGMENT;
  3817. goto retry;
  3818. }
  3819. return NULL;
  3820. }
  3821. static void warn_alloc_show_mem(gfp_t gfp_mask, nodemask_t *nodemask)
  3822. {
  3823. unsigned int filter = SHOW_MEM_FILTER_NODES;
  3824. /*
  3825. * This documents exceptions given to allocations in certain
  3826. * contexts that are allowed to allocate outside current's set
  3827. * of allowed nodes.
  3828. */
  3829. if (!(gfp_mask & __GFP_NOMEMALLOC))
  3830. if (tsk_is_oom_victim(current) ||
  3831. (current->flags & (PF_MEMALLOC | PF_EXITING)))
  3832. filter &= ~SHOW_MEM_FILTER_NODES;
  3833. if (!in_task() || !(gfp_mask & __GFP_DIRECT_RECLAIM))
  3834. filter &= ~SHOW_MEM_FILTER_NODES;
  3835. __show_mem(filter, nodemask, gfp_zone(gfp_mask));
  3836. }
  3837. void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...)
  3838. {
  3839. struct va_format vaf;
  3840. va_list args;
  3841. static DEFINE_RATELIMIT_STATE(nopage_rs, 10*HZ, 1);
  3842. if ((gfp_mask & __GFP_NOWARN) ||
  3843. !__ratelimit(&nopage_rs) ||
  3844. ((gfp_mask & __GFP_DMA) && !has_managed_dma()))
  3845. return;
  3846. va_start(args, fmt);
  3847. vaf.fmt = fmt;
  3848. vaf.va = &args;
  3849. pr_warn("%s: %pV, mode:%#x(%pGg), nodemask=%*pbl",
  3850. current->comm, &vaf, gfp_mask, &gfp_mask,
  3851. nodemask_pr_args(nodemask));
  3852. va_end(args);
  3853. cpuset_print_current_mems_allowed();
  3854. pr_cont("\n");
  3855. dump_stack();
  3856. warn_alloc_show_mem(gfp_mask, nodemask);
  3857. }
  3858. static inline struct page *
  3859. __alloc_pages_cpuset_fallback(gfp_t gfp_mask, unsigned int order,
  3860. unsigned int alloc_flags,
  3861. const struct alloc_context *ac)
  3862. {
  3863. struct page *page;
  3864. page = get_page_from_freelist(gfp_mask, order,
  3865. alloc_flags|ALLOC_CPUSET, ac);
  3866. /*
  3867. * fallback to ignore cpuset restriction if our nodes
  3868. * are depleted
  3869. */
  3870. if (!page)
  3871. page = get_page_from_freelist(gfp_mask, order,
  3872. alloc_flags, ac);
  3873. return page;
  3874. }
  3875. static inline struct page *
  3876. __alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
  3877. const struct alloc_context *ac, unsigned long *did_some_progress)
  3878. {
  3879. struct oom_control oc = {
  3880. .zonelist = ac->zonelist,
  3881. .nodemask = ac->nodemask,
  3882. .memcg = NULL,
  3883. .gfp_mask = gfp_mask,
  3884. .order = order,
  3885. };
  3886. struct page *page;
  3887. *did_some_progress = 0;
  3888. /*
  3889. * Acquire the oom lock. If that fails, somebody else is
  3890. * making progress for us.
  3891. */
  3892. if (!mutex_trylock(&oom_lock)) {
  3893. *did_some_progress = 1;
  3894. schedule_timeout_uninterruptible(1);
  3895. trace_android_vh_mm_alloc_pages_may_oom_exit(&oc, *did_some_progress);
  3896. return NULL;
  3897. }
  3898. /*
  3899. * Go through the zonelist yet one more time, keep very high watermark
  3900. * here, this is only to catch a parallel oom killing, we must fail if
  3901. * we're still under heavy pressure. But make sure that this reclaim
  3902. * attempt shall not depend on __GFP_DIRECT_RECLAIM && !__GFP_NORETRY
  3903. * allocation which will never fail due to oom_lock already held.
  3904. */
  3905. page = get_page_from_freelist((gfp_mask | __GFP_HARDWALL) &
  3906. ~__GFP_DIRECT_RECLAIM, order,
  3907. ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac);
  3908. if (page)
  3909. goto out;
  3910. /* Coredumps can quickly deplete all memory reserves */
  3911. if (current->flags & PF_DUMPCORE)
  3912. goto out;
  3913. /* The OOM killer will not help higher order allocs */
  3914. if (order > PAGE_ALLOC_COSTLY_ORDER)
  3915. goto out;
  3916. /*
  3917. * We have already exhausted all our reclaim opportunities without any
  3918. * success so it is time to admit defeat. We will skip the OOM killer
  3919. * because it is very likely that the caller has a more reasonable
  3920. * fallback than shooting a random task.
  3921. *
  3922. * The OOM killer may not free memory on a specific node.
  3923. */
  3924. if (gfp_mask & (__GFP_RETRY_MAYFAIL | __GFP_THISNODE))
  3925. goto out;
  3926. /* The OOM killer does not needlessly kill tasks for lowmem */
  3927. if (ac->highest_zoneidx < ZONE_NORMAL)
  3928. goto out;
  3929. if (pm_suspended_storage())
  3930. goto out;
  3931. /*
  3932. * XXX: GFP_NOFS allocations should rather fail than rely on
  3933. * other request to make a forward progress.
  3934. * We are in an unfortunate situation where out_of_memory cannot
  3935. * do much for this context but let's try it to at least get
  3936. * access to memory reserved if the current task is killed (see
  3937. * out_of_memory). Once filesystems are ready to handle allocation
  3938. * failures more gracefully we should just bail out here.
  3939. */
  3940. /* Exhausted what can be done so it's blame time */
  3941. if (out_of_memory(&oc) ||
  3942. WARN_ON_ONCE_GFP(gfp_mask & __GFP_NOFAIL, gfp_mask)) {
  3943. *did_some_progress = 1;
  3944. /*
  3945. * Help non-failing allocations by giving them access to memory
  3946. * reserves
  3947. */
  3948. if (gfp_mask & __GFP_NOFAIL)
  3949. page = __alloc_pages_cpuset_fallback(gfp_mask, order,
  3950. ALLOC_NO_WATERMARKS, ac);
  3951. }
  3952. out:
  3953. mutex_unlock(&oom_lock);
  3954. trace_android_vh_mm_alloc_pages_may_oom_exit(&oc, *did_some_progress);
  3955. return page;
  3956. }
  3957. /*
  3958. * Maximum number of compaction retries with a progress before OOM
  3959. * killer is consider as the only way to move forward.
  3960. */
  3961. #define MAX_COMPACT_RETRIES 16
  3962. #ifdef CONFIG_COMPACTION
  3963. /* Try memory compaction for high-order allocations before reclaim */
  3964. static struct page *
  3965. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  3966. unsigned int alloc_flags, const struct alloc_context *ac,
  3967. enum compact_priority prio, enum compact_result *compact_result)
  3968. {
  3969. struct page *page = NULL;
  3970. unsigned long pflags;
  3971. unsigned int noreclaim_flag;
  3972. if (!order)
  3973. return NULL;
  3974. psi_memstall_enter(&pflags);
  3975. delayacct_compact_start();
  3976. noreclaim_flag = memalloc_noreclaim_save();
  3977. *compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac,
  3978. prio, &page);
  3979. memalloc_noreclaim_restore(noreclaim_flag);
  3980. psi_memstall_leave(&pflags);
  3981. delayacct_compact_end();
  3982. if (*compact_result == COMPACT_SKIPPED)
  3983. return NULL;
  3984. /*
  3985. * At least in one zone compaction wasn't deferred or skipped, so let's
  3986. * count a compaction stall
  3987. */
  3988. count_vm_event(COMPACTSTALL);
  3989. /* Prep a captured page if available */
  3990. if (page)
  3991. prep_new_page(page, order, gfp_mask, alloc_flags);
  3992. /* Try get a page from the freelist if available */
  3993. if (!page)
  3994. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  3995. if (page) {
  3996. struct zone *zone = page_zone(page);
  3997. zone->compact_blockskip_flush = false;
  3998. compaction_defer_reset(zone, order, true);
  3999. count_vm_event(COMPACTSUCCESS);
  4000. return page;
  4001. }
  4002. /*
  4003. * It's bad if compaction run occurs and fails. The most likely reason
  4004. * is that pages exist, but not enough to satisfy watermarks.
  4005. */
  4006. count_vm_event(COMPACTFAIL);
  4007. cond_resched();
  4008. return NULL;
  4009. }
  4010. static inline bool
  4011. should_compact_retry(struct alloc_context *ac, int order, int alloc_flags,
  4012. enum compact_result compact_result,
  4013. enum compact_priority *compact_priority,
  4014. int *compaction_retries)
  4015. {
  4016. int max_retries = MAX_COMPACT_RETRIES;
  4017. int min_priority;
  4018. bool ret = false;
  4019. int retries = *compaction_retries;
  4020. enum compact_priority priority = *compact_priority;
  4021. if (!order)
  4022. return false;
  4023. if (fatal_signal_pending(current))
  4024. return false;
  4025. if (compaction_made_progress(compact_result))
  4026. (*compaction_retries)++;
  4027. /*
  4028. * compaction considers all the zone as desperately out of memory
  4029. * so it doesn't really make much sense to retry except when the
  4030. * failure could be caused by insufficient priority
  4031. */
  4032. if (compaction_failed(compact_result))
  4033. goto check_priority;
  4034. /*
  4035. * compaction was skipped because there are not enough order-0 pages
  4036. * to work with, so we retry only if it looks like reclaim can help.
  4037. */
  4038. if (compaction_needs_reclaim(compact_result)) {
  4039. ret = compaction_zonelist_suitable(ac, order, alloc_flags);
  4040. goto out;
  4041. }
  4042. /*
  4043. * make sure the compaction wasn't deferred or didn't bail out early
  4044. * due to locks contention before we declare that we should give up.
  4045. * But the next retry should use a higher priority if allowed, so
  4046. * we don't just keep bailing out endlessly.
  4047. */
  4048. if (compaction_withdrawn(compact_result)) {
  4049. goto check_priority;
  4050. }
  4051. /*
  4052. * !costly requests are much more important than __GFP_RETRY_MAYFAIL
  4053. * costly ones because they are de facto nofail and invoke OOM
  4054. * killer to move on while costly can fail and users are ready
  4055. * to cope with that. 1/4 retries is rather arbitrary but we
  4056. * would need much more detailed feedback from compaction to
  4057. * make a better decision.
  4058. */
  4059. if (order > PAGE_ALLOC_COSTLY_ORDER)
  4060. max_retries /= 4;
  4061. if (*compaction_retries <= max_retries) {
  4062. ret = true;
  4063. goto out;
  4064. }
  4065. /*
  4066. * Make sure there are attempts at the highest priority if we exhausted
  4067. * all retries or failed at the lower priorities.
  4068. */
  4069. check_priority:
  4070. min_priority = (order > PAGE_ALLOC_COSTLY_ORDER) ?
  4071. MIN_COMPACT_COSTLY_PRIORITY : MIN_COMPACT_PRIORITY;
  4072. if (*compact_priority > min_priority) {
  4073. (*compact_priority)--;
  4074. *compaction_retries = 0;
  4075. ret = true;
  4076. }
  4077. out:
  4078. trace_compact_retry(order, priority, compact_result, retries, max_retries, ret);
  4079. return ret;
  4080. }
  4081. #else
  4082. static inline struct page *
  4083. __alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
  4084. unsigned int alloc_flags, const struct alloc_context *ac,
  4085. enum compact_priority prio, enum compact_result *compact_result)
  4086. {
  4087. *compact_result = COMPACT_SKIPPED;
  4088. return NULL;
  4089. }
  4090. static inline bool
  4091. should_compact_retry(struct alloc_context *ac, unsigned int order, int alloc_flags,
  4092. enum compact_result compact_result,
  4093. enum compact_priority *compact_priority,
  4094. int *compaction_retries)
  4095. {
  4096. struct zone *zone;
  4097. struct zoneref *z;
  4098. if (!order || order > PAGE_ALLOC_COSTLY_ORDER)
  4099. return false;
  4100. /*
  4101. * There are setups with compaction disabled which would prefer to loop
  4102. * inside the allocator rather than hit the oom killer prematurely.
  4103. * Let's give them a good hope and keep retrying while the order-0
  4104. * watermarks are OK.
  4105. */
  4106. for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
  4107. ac->highest_zoneidx, ac->nodemask) {
  4108. if (zone_watermark_ok(zone, 0, min_wmark_pages(zone),
  4109. ac->highest_zoneidx, alloc_flags))
  4110. return true;
  4111. }
  4112. return false;
  4113. }
  4114. #endif /* CONFIG_COMPACTION */
  4115. #ifdef CONFIG_LOCKDEP
  4116. static struct lockdep_map __fs_reclaim_map =
  4117. STATIC_LOCKDEP_MAP_INIT("fs_reclaim", &__fs_reclaim_map);
  4118. static bool __need_reclaim(gfp_t gfp_mask)
  4119. {
  4120. /* no reclaim without waiting on it */
  4121. if (!(gfp_mask & __GFP_DIRECT_RECLAIM))
  4122. return false;
  4123. /* this guy won't enter reclaim */
  4124. if (current->flags & PF_MEMALLOC)
  4125. return false;
  4126. if (gfp_mask & __GFP_NOLOCKDEP)
  4127. return false;
  4128. return true;
  4129. }
  4130. void __fs_reclaim_acquire(unsigned long ip)
  4131. {
  4132. lock_acquire_exclusive(&__fs_reclaim_map, 0, 0, NULL, ip);
  4133. }
  4134. void __fs_reclaim_release(unsigned long ip)
  4135. {
  4136. lock_release(&__fs_reclaim_map, ip);
  4137. }
  4138. void fs_reclaim_acquire(gfp_t gfp_mask)
  4139. {
  4140. gfp_mask = current_gfp_context(gfp_mask);
  4141. if (__need_reclaim(gfp_mask)) {
  4142. if (gfp_mask & __GFP_FS)
  4143. __fs_reclaim_acquire(_RET_IP_);
  4144. #ifdef CONFIG_MMU_NOTIFIER
  4145. lock_map_acquire(&__mmu_notifier_invalidate_range_start_map);
  4146. lock_map_release(&__mmu_notifier_invalidate_range_start_map);
  4147. #endif
  4148. }
  4149. }
  4150. EXPORT_SYMBOL_GPL(fs_reclaim_acquire);
  4151. void fs_reclaim_release(gfp_t gfp_mask)
  4152. {
  4153. gfp_mask = current_gfp_context(gfp_mask);
  4154. if (__need_reclaim(gfp_mask)) {
  4155. if (gfp_mask & __GFP_FS)
  4156. __fs_reclaim_release(_RET_IP_);
  4157. }
  4158. }
  4159. EXPORT_SYMBOL_GPL(fs_reclaim_release);
  4160. #endif
  4161. /*
  4162. * Zonelists may change due to hotplug during allocation. Detect when zonelists
  4163. * have been rebuilt so allocation retries. Reader side does not lock and
  4164. * retries the allocation if zonelist changes. Writer side is protected by the
  4165. * embedded spin_lock.
  4166. */
  4167. static DEFINE_SEQLOCK(zonelist_update_seq);
  4168. static unsigned int zonelist_iter_begin(void)
  4169. {
  4170. if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
  4171. return read_seqbegin(&zonelist_update_seq);
  4172. return 0;
  4173. }
  4174. static unsigned int check_retry_zonelist(unsigned int seq)
  4175. {
  4176. if (IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
  4177. return read_seqretry(&zonelist_update_seq, seq);
  4178. return seq;
  4179. }
  4180. /* Perform direct synchronous page reclaim */
  4181. static unsigned long
  4182. __perform_reclaim(gfp_t gfp_mask, unsigned int order,
  4183. const struct alloc_context *ac)
  4184. {
  4185. unsigned int noreclaim_flag;
  4186. unsigned long progress;
  4187. cond_resched();
  4188. /* We now go into synchronous reclaim */
  4189. cpuset_memory_pressure_bump();
  4190. fs_reclaim_acquire(gfp_mask);
  4191. noreclaim_flag = memalloc_noreclaim_save();
  4192. progress = try_to_free_pages(ac->zonelist, order, gfp_mask,
  4193. ac->nodemask);
  4194. memalloc_noreclaim_restore(noreclaim_flag);
  4195. fs_reclaim_release(gfp_mask);
  4196. cond_resched();
  4197. return progress;
  4198. }
  4199. /* The really slow allocator path where we enter direct reclaim */
  4200. static inline struct page *
  4201. __alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
  4202. unsigned int alloc_flags, const struct alloc_context *ac,
  4203. unsigned long *did_some_progress)
  4204. {
  4205. int retry_times = 0;
  4206. struct page *page = NULL;
  4207. unsigned long pflags;
  4208. bool drained = false;
  4209. trace_android_vh_mm_alloc_pages_direct_reclaim_enter(order);
  4210. psi_memstall_enter(&pflags);
  4211. *did_some_progress = __perform_reclaim(gfp_mask, order, ac);
  4212. if (unlikely(!(*did_some_progress)))
  4213. goto out;
  4214. retry:
  4215. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  4216. /*
  4217. * If an allocation failed after direct reclaim, it could be because
  4218. * pages are pinned on the per-cpu lists or in high alloc reserves.
  4219. * Shrink them and try again
  4220. */
  4221. if (!page && !drained) {
  4222. unreserve_highatomic_pageblock(ac, false);
  4223. drain_all_pages(NULL);
  4224. drained = true;
  4225. ++retry_times;
  4226. goto retry;
  4227. }
  4228. out:
  4229. psi_memstall_leave(&pflags);
  4230. trace_android_vh_mm_alloc_pages_direct_reclaim_exit(*did_some_progress, retry_times);
  4231. return page;
  4232. }
  4233. static void wake_all_kswapds(unsigned int order, gfp_t gfp_mask,
  4234. const struct alloc_context *ac)
  4235. {
  4236. struct zoneref *z;
  4237. struct zone *zone;
  4238. pg_data_t *last_pgdat = NULL;
  4239. enum zone_type highest_zoneidx = ac->highest_zoneidx;
  4240. for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, highest_zoneidx,
  4241. ac->nodemask) {
  4242. if (!managed_zone(zone))
  4243. continue;
  4244. if (last_pgdat != zone->zone_pgdat) {
  4245. wakeup_kswapd(zone, gfp_mask, order, highest_zoneidx);
  4246. last_pgdat = zone->zone_pgdat;
  4247. }
  4248. }
  4249. }
  4250. static inline unsigned int
  4251. gfp_to_alloc_flags(gfp_t gfp_mask)
  4252. {
  4253. unsigned int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
  4254. /*
  4255. * __GFP_HIGH is assumed to be the same as ALLOC_HIGH
  4256. * and __GFP_KSWAPD_RECLAIM is assumed to be the same as ALLOC_KSWAPD
  4257. * to save two branches.
  4258. */
  4259. BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
  4260. BUILD_BUG_ON(__GFP_KSWAPD_RECLAIM != (__force gfp_t) ALLOC_KSWAPD);
  4261. /*
  4262. * The caller may dip into page reserves a bit more if the caller
  4263. * cannot run direct reclaim, or if the caller has realtime scheduling
  4264. * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
  4265. * set both ALLOC_HARDER (__GFP_ATOMIC) and ALLOC_HIGH (__GFP_HIGH).
  4266. */
  4267. alloc_flags |= (__force int)
  4268. (gfp_mask & (__GFP_HIGH | __GFP_KSWAPD_RECLAIM));
  4269. if (gfp_mask & __GFP_ATOMIC) {
  4270. /*
  4271. * Not worth trying to allocate harder for __GFP_NOMEMALLOC even
  4272. * if it can't schedule.
  4273. */
  4274. if (!(gfp_mask & __GFP_NOMEMALLOC))
  4275. alloc_flags |= ALLOC_HARDER;
  4276. /*
  4277. * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
  4278. * comment for __cpuset_node_allowed().
  4279. */
  4280. alloc_flags &= ~ALLOC_CPUSET;
  4281. } else if (unlikely(rt_task(current)) && in_task())
  4282. alloc_flags |= ALLOC_HARDER;
  4283. alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, alloc_flags);
  4284. return alloc_flags;
  4285. }
  4286. static bool oom_reserves_allowed(struct task_struct *tsk)
  4287. {
  4288. if (!tsk_is_oom_victim(tsk))
  4289. return false;
  4290. /*
  4291. * !MMU doesn't have oom reaper so give access to memory reserves
  4292. * only to the thread with TIF_MEMDIE set
  4293. */
  4294. if (!IS_ENABLED(CONFIG_MMU) && !test_thread_flag(TIF_MEMDIE))
  4295. return false;
  4296. return true;
  4297. }
  4298. /*
  4299. * Distinguish requests which really need access to full memory
  4300. * reserves from oom victims which can live with a portion of it
  4301. */
  4302. static inline int __gfp_pfmemalloc_flags(gfp_t gfp_mask)
  4303. {
  4304. if (unlikely(gfp_mask & __GFP_NOMEMALLOC))
  4305. return 0;
  4306. if (gfp_mask & __GFP_MEMALLOC)
  4307. return ALLOC_NO_WATERMARKS;
  4308. if (in_serving_softirq() && (current->flags & PF_MEMALLOC))
  4309. return ALLOC_NO_WATERMARKS;
  4310. if (!in_interrupt()) {
  4311. if (current->flags & PF_MEMALLOC)
  4312. return ALLOC_NO_WATERMARKS;
  4313. else if (oom_reserves_allowed(current))
  4314. return ALLOC_OOM;
  4315. }
  4316. return 0;
  4317. }
  4318. bool gfp_pfmemalloc_allowed(gfp_t gfp_mask)
  4319. {
  4320. return !!__gfp_pfmemalloc_flags(gfp_mask);
  4321. }
  4322. /*
  4323. * Checks whether it makes sense to retry the reclaim to make a forward progress
  4324. * for the given allocation request.
  4325. *
  4326. * We give up when we either have tried MAX_RECLAIM_RETRIES in a row
  4327. * without success, or when we couldn't even meet the watermark if we
  4328. * reclaimed all remaining pages on the LRU lists.
  4329. *
  4330. * Returns true if a retry is viable or false to enter the oom path.
  4331. */
  4332. static inline bool
  4333. should_reclaim_retry(gfp_t gfp_mask, unsigned order,
  4334. struct alloc_context *ac, int alloc_flags,
  4335. bool did_some_progress, int *no_progress_loops)
  4336. {
  4337. struct zone *zone;
  4338. struct zoneref *z;
  4339. bool ret = false;
  4340. /*
  4341. * Costly allocations might have made a progress but this doesn't mean
  4342. * their order will become available due to high fragmentation so
  4343. * always increment the no progress counter for them
  4344. */
  4345. if (did_some_progress && order <= PAGE_ALLOC_COSTLY_ORDER)
  4346. *no_progress_loops = 0;
  4347. else
  4348. (*no_progress_loops)++;
  4349. #ifdef CONFIG_ARCH_QTI_VM
  4350. if (*no_progress_loops > MAX_RECLAIM_RETRIES)
  4351. goto out;
  4352. #else
  4353. /*
  4354. * Make sure we converge to OOM if we cannot make any progress
  4355. * several times in the row.
  4356. */
  4357. if (*no_progress_loops > MAX_RECLAIM_RETRIES) {
  4358. /* Before OOM, exhaust highatomic_reserve */
  4359. return unreserve_highatomic_pageblock(ac, true);
  4360. }
  4361. #endif
  4362. /*
  4363. * Keep reclaiming pages while there is a chance this will lead
  4364. * somewhere. If none of the target zones can satisfy our allocation
  4365. * request even if all reclaimable pages are considered then we are
  4366. * screwed and have to go OOM.
  4367. */
  4368. for_each_zone_zonelist_nodemask(zone, z, ac->zonelist,
  4369. ac->highest_zoneidx, ac->nodemask) {
  4370. unsigned long available;
  4371. unsigned long reclaimable;
  4372. unsigned long min_wmark = min_wmark_pages(zone);
  4373. bool wmark;
  4374. available = reclaimable = zone_reclaimable_pages(zone);
  4375. available += zone_page_state_snapshot(zone, NR_FREE_PAGES);
  4376. /*
  4377. * Would the allocation succeed if we reclaimed all
  4378. * reclaimable pages?
  4379. */
  4380. wmark = __zone_watermark_ok(zone, order, min_wmark,
  4381. ac->highest_zoneidx, alloc_flags, available);
  4382. trace_reclaim_retry_zone(z, order, reclaimable,
  4383. available, min_wmark, *no_progress_loops, wmark);
  4384. if (wmark) {
  4385. ret = true;
  4386. break;
  4387. }
  4388. }
  4389. /*
  4390. * Memory allocation/reclaim might be called from a WQ context and the
  4391. * current implementation of the WQ concurrency control doesn't
  4392. * recognize that a particular WQ is congested if the worker thread is
  4393. * looping without ever sleeping. Therefore we have to do a short sleep
  4394. * here rather than calling cond_resched().
  4395. */
  4396. if (current->flags & PF_WQ_WORKER)
  4397. schedule_timeout_uninterruptible(1);
  4398. else
  4399. cond_resched();
  4400. #ifdef CONFIG_ARCH_QTI_VM
  4401. out:
  4402. /* Before OOM, exhaust highatomic_reserve */
  4403. if (!ret)
  4404. return unreserve_highatomic_pageblock(ac, true);
  4405. #endif
  4406. return ret;
  4407. }
  4408. static inline bool
  4409. check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
  4410. {
  4411. /*
  4412. * It's possible that cpuset's mems_allowed and the nodemask from
  4413. * mempolicy don't intersect. This should be normally dealt with by
  4414. * policy_nodemask(), but it's possible to race with cpuset update in
  4415. * such a way the check therein was true, and then it became false
  4416. * before we got our cpuset_mems_cookie here.
  4417. * This assumes that for all allocations, ac->nodemask can come only
  4418. * from MPOL_BIND mempolicy (whose documented semantics is to be ignored
  4419. * when it does not intersect with the cpuset restrictions) or the
  4420. * caller can deal with a violated nodemask.
  4421. */
  4422. if (cpusets_enabled() && ac->nodemask &&
  4423. !cpuset_nodemask_valid_mems_allowed(ac->nodemask)) {
  4424. ac->nodemask = NULL;
  4425. return true;
  4426. }
  4427. /*
  4428. * When updating a task's mems_allowed or mempolicy nodemask, it is
  4429. * possible to race with parallel threads in such a way that our
  4430. * allocation can fail while the mask is being updated. If we are about
  4431. * to fail, check if the cpuset changed during allocation and if so,
  4432. * retry.
  4433. */
  4434. if (read_mems_allowed_retry(cpuset_mems_cookie))
  4435. return true;
  4436. return false;
  4437. }
  4438. static inline struct page *
  4439. __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
  4440. struct alloc_context *ac)
  4441. {
  4442. bool can_direct_reclaim = gfp_mask & __GFP_DIRECT_RECLAIM;
  4443. const bool costly_order = order > PAGE_ALLOC_COSTLY_ORDER;
  4444. struct page *page = NULL;
  4445. unsigned int alloc_flags;
  4446. unsigned long did_some_progress;
  4447. enum compact_priority compact_priority;
  4448. enum compact_result compact_result;
  4449. int compaction_retries;
  4450. int no_progress_loops;
  4451. unsigned int cpuset_mems_cookie;
  4452. unsigned int zonelist_iter_cookie;
  4453. int reserve_flags;
  4454. unsigned long alloc_start = jiffies;
  4455. bool should_alloc_retry = false;
  4456. /*
  4457. * We also sanity check to catch abuse of atomic reserves being used by
  4458. * callers that are not in atomic context.
  4459. */
  4460. if (WARN_ON_ONCE((gfp_mask & (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)) ==
  4461. (__GFP_ATOMIC|__GFP_DIRECT_RECLAIM)))
  4462. gfp_mask &= ~__GFP_ATOMIC;
  4463. restart:
  4464. compaction_retries = 0;
  4465. no_progress_loops = 0;
  4466. compact_priority = DEF_COMPACT_PRIORITY;
  4467. cpuset_mems_cookie = read_mems_allowed_begin();
  4468. zonelist_iter_cookie = zonelist_iter_begin();
  4469. /*
  4470. * The fast path uses conservative alloc_flags to succeed only until
  4471. * kswapd needs to be woken up, and to avoid the cost of setting up
  4472. * alloc_flags precisely. So we do that now.
  4473. */
  4474. alloc_flags = gfp_to_alloc_flags(gfp_mask);
  4475. /*
  4476. * We need to recalculate the starting point for the zonelist iterator
  4477. * because we might have used different nodemask in the fast path, or
  4478. * there was a cpuset modification and we are retrying - otherwise we
  4479. * could end up iterating over non-eligible zones endlessly.
  4480. */
  4481. ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
  4482. ac->highest_zoneidx, ac->nodemask);
  4483. if (!ac->preferred_zoneref->zone)
  4484. goto nopage;
  4485. /*
  4486. * Check for insane configurations where the cpuset doesn't contain
  4487. * any suitable zone to satisfy the request - e.g. non-movable
  4488. * GFP_HIGHUSER allocations from MOVABLE nodes only.
  4489. */
  4490. if (cpusets_insane_config() && (gfp_mask & __GFP_HARDWALL)) {
  4491. struct zoneref *z = first_zones_zonelist(ac->zonelist,
  4492. ac->highest_zoneidx,
  4493. &cpuset_current_mems_allowed);
  4494. if (!z->zone)
  4495. goto nopage;
  4496. }
  4497. if (alloc_flags & ALLOC_KSWAPD)
  4498. wake_all_kswapds(order, gfp_mask, ac);
  4499. /*
  4500. * The adjusted alloc_flags might result in immediate success, so try
  4501. * that first
  4502. */
  4503. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  4504. if (page)
  4505. goto got_pg;
  4506. /*
  4507. * For costly allocations, try direct compaction first, as it's likely
  4508. * that we have enough base pages and don't need to reclaim. For non-
  4509. * movable high-order allocations, do that as well, as compaction will
  4510. * try prevent permanent fragmentation by migrating from blocks of the
  4511. * same migratetype.
  4512. * Don't try this for allocations that are allowed to ignore
  4513. * watermarks, as the ALLOC_NO_WATERMARKS attempt didn't yet happen.
  4514. */
  4515. if (can_direct_reclaim &&
  4516. (costly_order ||
  4517. (order > 0 && ac->migratetype != MIGRATE_MOVABLE))
  4518. && !gfp_pfmemalloc_allowed(gfp_mask)) {
  4519. page = __alloc_pages_direct_compact(gfp_mask, order,
  4520. alloc_flags, ac,
  4521. INIT_COMPACT_PRIORITY,
  4522. &compact_result);
  4523. if (page)
  4524. goto got_pg;
  4525. /*
  4526. * Checks for costly allocations with __GFP_NORETRY, which
  4527. * includes some THP page fault allocations
  4528. */
  4529. if (costly_order && (gfp_mask & __GFP_NORETRY)) {
  4530. /*
  4531. * If allocating entire pageblock(s) and compaction
  4532. * failed because all zones are below low watermarks
  4533. * or is prohibited because it recently failed at this
  4534. * order, fail immediately unless the allocator has
  4535. * requested compaction and reclaim retry.
  4536. *
  4537. * Reclaim is
  4538. * - potentially very expensive because zones are far
  4539. * below their low watermarks or this is part of very
  4540. * bursty high order allocations,
  4541. * - not guaranteed to help because isolate_freepages()
  4542. * may not iterate over freed pages as part of its
  4543. * linear scan, and
  4544. * - unlikely to make entire pageblocks free on its
  4545. * own.
  4546. */
  4547. if (compact_result == COMPACT_SKIPPED ||
  4548. compact_result == COMPACT_DEFERRED)
  4549. goto nopage;
  4550. /*
  4551. * Looks like reclaim/compaction is worth trying, but
  4552. * sync compaction could be very expensive, so keep
  4553. * using async compaction.
  4554. */
  4555. compact_priority = INIT_COMPACT_PRIORITY;
  4556. }
  4557. }
  4558. retry:
  4559. /* Ensure kswapd doesn't accidentally go to sleep as long as we loop */
  4560. if (alloc_flags & ALLOC_KSWAPD)
  4561. wake_all_kswapds(order, gfp_mask, ac);
  4562. reserve_flags = __gfp_pfmemalloc_flags(gfp_mask);
  4563. if (reserve_flags)
  4564. alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, reserve_flags) |
  4565. (alloc_flags & ALLOC_KSWAPD);
  4566. /*
  4567. * Reset the nodemask and zonelist iterators if memory policies can be
  4568. * ignored. These allocations are high priority and system rather than
  4569. * user oriented.
  4570. */
  4571. if (!(alloc_flags & ALLOC_CPUSET) || reserve_flags) {
  4572. ac->nodemask = NULL;
  4573. ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
  4574. ac->highest_zoneidx, ac->nodemask);
  4575. }
  4576. /* Attempt with potentially adjusted zonelist and alloc_flags */
  4577. page = get_page_from_freelist(gfp_mask, order, alloc_flags, ac);
  4578. if (page)
  4579. goto got_pg;
  4580. /* Caller is not willing to reclaim, we can't balance anything */
  4581. if (!can_direct_reclaim)
  4582. goto nopage;
  4583. /* Avoid recursion of direct reclaim */
  4584. if (current->flags & PF_MEMALLOC)
  4585. goto nopage;
  4586. trace_android_vh_alloc_pages_reclaim_bypass(gfp_mask, order,
  4587. alloc_flags, ac->migratetype, &page);
  4588. if (page)
  4589. goto got_pg;
  4590. trace_android_vh_should_alloc_pages_retry(gfp_mask, order, &alloc_flags,
  4591. ac->migratetype, ac->preferred_zoneref->zone, &page, &should_alloc_retry);
  4592. if (should_alloc_retry)
  4593. goto retry;
  4594. /* Try direct reclaim and then allocating */
  4595. page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac,
  4596. &did_some_progress);
  4597. if (page)
  4598. goto got_pg;
  4599. /* Try direct compaction and then allocating */
  4600. page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac,
  4601. compact_priority, &compact_result);
  4602. if (page)
  4603. goto got_pg;
  4604. /* Do not loop if specifically requested */
  4605. if (gfp_mask & __GFP_NORETRY)
  4606. goto nopage;
  4607. /*
  4608. * Do not retry costly high order allocations unless they are
  4609. * __GFP_RETRY_MAYFAIL
  4610. */
  4611. if (costly_order && !(gfp_mask & __GFP_RETRY_MAYFAIL))
  4612. goto nopage;
  4613. if (should_reclaim_retry(gfp_mask, order, ac, alloc_flags,
  4614. did_some_progress > 0, &no_progress_loops))
  4615. goto retry;
  4616. /*
  4617. * It doesn't make any sense to retry for the compaction if the order-0
  4618. * reclaim is not able to make any progress because the current
  4619. * implementation of the compaction depends on the sufficient amount
  4620. * of free memory (see __compaction_suitable)
  4621. */
  4622. if (did_some_progress > 0 &&
  4623. should_compact_retry(ac, order, alloc_flags,
  4624. compact_result, &compact_priority,
  4625. &compaction_retries))
  4626. goto retry;
  4627. /*
  4628. * Deal with possible cpuset update races or zonelist updates to avoid
  4629. * a unnecessary OOM kill.
  4630. */
  4631. if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
  4632. check_retry_zonelist(zonelist_iter_cookie))
  4633. goto restart;
  4634. /* Reclaim has failed us, start killing things */
  4635. page = __alloc_pages_may_oom(gfp_mask, order, ac, &did_some_progress);
  4636. if (page)
  4637. goto got_pg;
  4638. /* Avoid allocations with no watermarks from looping endlessly */
  4639. if (tsk_is_oom_victim(current) &&
  4640. (alloc_flags & ALLOC_OOM ||
  4641. (gfp_mask & __GFP_NOMEMALLOC)))
  4642. goto nopage;
  4643. /* Retry as long as the OOM killer is making progress */
  4644. if (did_some_progress) {
  4645. no_progress_loops = 0;
  4646. goto retry;
  4647. }
  4648. nopage:
  4649. /*
  4650. * Deal with possible cpuset update races or zonelist updates to avoid
  4651. * a unnecessary OOM kill.
  4652. */
  4653. if (check_retry_cpuset(cpuset_mems_cookie, ac) ||
  4654. check_retry_zonelist(zonelist_iter_cookie))
  4655. goto restart;
  4656. /*
  4657. * Make sure that __GFP_NOFAIL request doesn't leak out and make sure
  4658. * we always retry
  4659. */
  4660. if (gfp_mask & __GFP_NOFAIL) {
  4661. /*
  4662. * All existing users of the __GFP_NOFAIL are blockable, so warn
  4663. * of any new users that actually require GFP_NOWAIT
  4664. */
  4665. if (WARN_ON_ONCE_GFP(!can_direct_reclaim, gfp_mask))
  4666. goto fail;
  4667. /*
  4668. * PF_MEMALLOC request from this context is rather bizarre
  4669. * because we cannot reclaim anything and only can loop waiting
  4670. * for somebody to do a work for us
  4671. */
  4672. WARN_ON_ONCE_GFP(current->flags & PF_MEMALLOC, gfp_mask);
  4673. /*
  4674. * non failing costly orders are a hard requirement which we
  4675. * are not prepared for much so let's warn about these users
  4676. * so that we can identify them and convert them to something
  4677. * else.
  4678. */
  4679. WARN_ON_ONCE_GFP(costly_order, gfp_mask);
  4680. /*
  4681. * Help non-failing allocations by giving them access to memory
  4682. * reserves but do not use ALLOC_NO_WATERMARKS because this
  4683. * could deplete whole memory reserves which would just make
  4684. * the situation worse
  4685. */
  4686. page = __alloc_pages_cpuset_fallback(gfp_mask, order, ALLOC_HARDER, ac);
  4687. if (page)
  4688. goto got_pg;
  4689. cond_resched();
  4690. goto retry;
  4691. }
  4692. fail:
  4693. trace_android_vh_alloc_pages_failure_bypass(gfp_mask, order,
  4694. alloc_flags, ac->migratetype, &page);
  4695. if (page)
  4696. goto got_pg;
  4697. warn_alloc(gfp_mask, ac->nodemask,
  4698. "page allocation failure: order:%u", order);
  4699. got_pg:
  4700. trace_android_vh_alloc_pages_slowpath(gfp_mask, order, alloc_start);
  4701. return page;
  4702. }
  4703. static inline bool prepare_alloc_pages(gfp_t gfp_mask, unsigned int order,
  4704. int preferred_nid, nodemask_t *nodemask,
  4705. struct alloc_context *ac, gfp_t *alloc_gfp,
  4706. unsigned int *alloc_flags)
  4707. {
  4708. ac->highest_zoneidx = gfp_zone(gfp_mask);
  4709. ac->zonelist = node_zonelist(preferred_nid, gfp_mask);
  4710. ac->nodemask = nodemask;
  4711. ac->migratetype = gfp_migratetype(gfp_mask);
  4712. if (cpusets_enabled()) {
  4713. *alloc_gfp |= __GFP_HARDWALL;
  4714. /*
  4715. * When we are in the interrupt context, it is irrelevant
  4716. * to the current task context. It means that any node ok.
  4717. */
  4718. if (in_task() && !ac->nodemask)
  4719. ac->nodemask = &cpuset_current_mems_allowed;
  4720. else
  4721. *alloc_flags |= ALLOC_CPUSET;
  4722. }
  4723. might_alloc(gfp_mask);
  4724. if (should_fail_alloc_page(gfp_mask, order))
  4725. return false;
  4726. *alloc_flags = gfp_to_alloc_flags_cma(gfp_mask, *alloc_flags);
  4727. /* Dirty zone balancing only done in the fast path */
  4728. ac->spread_dirty_pages = (gfp_mask & __GFP_WRITE);
  4729. /*
  4730. * The preferred zone is used for statistics but crucially it is
  4731. * also used as the starting point for the zonelist iterator. It
  4732. * may get reset for allocations that ignore memory policies.
  4733. */
  4734. ac->preferred_zoneref = first_zones_zonelist(ac->zonelist,
  4735. ac->highest_zoneidx, ac->nodemask);
  4736. return true;
  4737. }
  4738. /*
  4739. * __alloc_pages_bulk - Allocate a number of order-0 pages to a list or array
  4740. * @gfp: GFP flags for the allocation
  4741. * @preferred_nid: The preferred NUMA node ID to allocate from
  4742. * @nodemask: Set of nodes to allocate from, may be NULL
  4743. * @nr_pages: The number of pages desired on the list or array
  4744. * @page_list: Optional list to store the allocated pages
  4745. * @page_array: Optional array to store the pages
  4746. *
  4747. * This is a batched version of the page allocator that attempts to
  4748. * allocate nr_pages quickly. Pages are added to page_list if page_list
  4749. * is not NULL, otherwise it is assumed that the page_array is valid.
  4750. *
  4751. * For lists, nr_pages is the number of pages that should be allocated.
  4752. *
  4753. * For arrays, only NULL elements are populated with pages and nr_pages
  4754. * is the maximum number of pages that will be stored in the array.
  4755. *
  4756. * Returns the number of pages on the list or array.
  4757. */
  4758. unsigned long __alloc_pages_bulk(gfp_t gfp, int preferred_nid,
  4759. nodemask_t *nodemask, int nr_pages,
  4760. struct list_head *page_list,
  4761. struct page **page_array)
  4762. {
  4763. struct page *page;
  4764. unsigned long __maybe_unused UP_flags;
  4765. struct zone *zone;
  4766. struct zoneref *z;
  4767. struct per_cpu_pages *pcp;
  4768. struct alloc_context ac;
  4769. gfp_t alloc_gfp;
  4770. unsigned int alloc_flags = ALLOC_WMARK_LOW;
  4771. int nr_populated = 0, nr_account = 0;
  4772. /*
  4773. * Skip populated array elements to determine if any pages need
  4774. * to be allocated before disabling IRQs.
  4775. */
  4776. while (page_array && nr_populated < nr_pages && page_array[nr_populated])
  4777. nr_populated++;
  4778. /* No pages requested? */
  4779. if (unlikely(nr_pages <= 0))
  4780. goto out;
  4781. /* Already populated array? */
  4782. if (unlikely(page_array && nr_pages - nr_populated == 0))
  4783. goto out;
  4784. /* Bulk allocator does not support memcg accounting. */
  4785. if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT))
  4786. goto failed;
  4787. /* Use the single page allocator for one page. */
  4788. if (nr_pages - nr_populated == 1)
  4789. goto failed;
  4790. #ifdef CONFIG_PAGE_OWNER
  4791. /*
  4792. * PAGE_OWNER may recurse into the allocator to allocate space to
  4793. * save the stack with pagesets.lock held. Releasing/reacquiring
  4794. * removes much of the performance benefit of bulk allocation so
  4795. * force the caller to allocate one page at a time as it'll have
  4796. * similar performance to added complexity to the bulk allocator.
  4797. */
  4798. if (static_branch_unlikely(&page_owner_inited))
  4799. goto failed;
  4800. #endif
  4801. /* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
  4802. gfp &= gfp_allowed_mask;
  4803. alloc_gfp = gfp;
  4804. if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags))
  4805. goto out;
  4806. gfp = alloc_gfp;
  4807. /* Find an allowed local zone that meets the low watermark. */
  4808. for_each_zone_zonelist_nodemask(zone, z, ac.zonelist, ac.highest_zoneidx, ac.nodemask) {
  4809. unsigned long mark;
  4810. if (cpusets_enabled() && (alloc_flags & ALLOC_CPUSET) &&
  4811. !__cpuset_zone_allowed(zone, gfp)) {
  4812. continue;
  4813. }
  4814. if (nr_online_nodes > 1 && zone != ac.preferred_zoneref->zone &&
  4815. zone_to_nid(zone) != zone_to_nid(ac.preferred_zoneref->zone)) {
  4816. goto failed;
  4817. }
  4818. mark = wmark_pages(zone, alloc_flags & ALLOC_WMARK_MASK) + nr_pages;
  4819. if (zone_watermark_fast(zone, 0, mark,
  4820. zonelist_zone_idx(ac.preferred_zoneref),
  4821. alloc_flags, gfp)) {
  4822. break;
  4823. }
  4824. }
  4825. /*
  4826. * If there are no allowed local zones that meets the watermarks then
  4827. * try to allocate a single page and reclaim if necessary.
  4828. */
  4829. if (unlikely(!zone))
  4830. goto failed;
  4831. /* spin_trylock may fail due to a parallel drain or IRQ reentrancy. */
  4832. pcp_trylock_prepare(UP_flags);
  4833. pcp = pcp_spin_trylock(zone->per_cpu_pageset);
  4834. if (!pcp)
  4835. goto failed_irq;
  4836. /* Attempt the batch allocation */
  4837. while (nr_populated < nr_pages) {
  4838. /* Skip existing pages */
  4839. if (page_array && page_array[nr_populated]) {
  4840. nr_populated++;
  4841. continue;
  4842. }
  4843. page = __rmqueue_pcplist(zone, 0, ac.migratetype, alloc_flags,
  4844. pcp);
  4845. if (unlikely(!page)) {
  4846. /* Try and allocate at least one page */
  4847. if (!nr_account) {
  4848. pcp_spin_unlock(pcp);
  4849. goto failed_irq;
  4850. }
  4851. break;
  4852. }
  4853. nr_account++;
  4854. prep_new_page(page, 0, gfp, 0);
  4855. if (page_list)
  4856. list_add(&page->lru, page_list);
  4857. else
  4858. page_array[nr_populated] = page;
  4859. nr_populated++;
  4860. }
  4861. pcp_spin_unlock(pcp);
  4862. pcp_trylock_finish(UP_flags);
  4863. __count_zid_vm_events(PGALLOC, zone_idx(zone), nr_account);
  4864. zone_statistics(ac.preferred_zoneref->zone, zone, nr_account);
  4865. out:
  4866. return nr_populated;
  4867. failed_irq:
  4868. pcp_trylock_finish(UP_flags);
  4869. failed:
  4870. page = __alloc_pages(gfp, 0, preferred_nid, nodemask);
  4871. if (page) {
  4872. if (page_list)
  4873. list_add(&page->lru, page_list);
  4874. else
  4875. page_array[nr_populated] = page;
  4876. nr_populated++;
  4877. }
  4878. goto out;
  4879. }
  4880. EXPORT_SYMBOL_GPL(__alloc_pages_bulk);
  4881. /*
  4882. * This is the 'heart' of the zoned buddy allocator.
  4883. */
  4884. struct page *__alloc_pages(gfp_t gfp, unsigned int order, int preferred_nid,
  4885. nodemask_t *nodemask)
  4886. {
  4887. struct page *page;
  4888. unsigned int alloc_flags = ALLOC_WMARK_LOW;
  4889. gfp_t alloc_gfp; /* The gfp_t that was actually used for allocation */
  4890. struct alloc_context ac = { };
  4891. /*
  4892. * There are several places where we assume that the order value is sane
  4893. * so bail out early if the request is out of bound.
  4894. */
  4895. if (WARN_ON_ONCE_GFP(order >= MAX_ORDER, gfp))
  4896. return NULL;
  4897. gfp &= gfp_allowed_mask;
  4898. /*
  4899. * Apply scoped allocation constraints. This is mainly about GFP_NOFS
  4900. * resp. GFP_NOIO which has to be inherited for all allocation requests
  4901. * from a particular context which has been marked by
  4902. * memalloc_no{fs,io}_{save,restore}. And PF_MEMALLOC_PIN which ensures
  4903. * movable zones are not used during allocation.
  4904. */
  4905. gfp = current_gfp_context(gfp);
  4906. alloc_gfp = gfp;
  4907. if (!prepare_alloc_pages(gfp, order, preferred_nid, nodemask, &ac,
  4908. &alloc_gfp, &alloc_flags))
  4909. return NULL;
  4910. /*
  4911. * Forbid the first pass from falling back to types that fragment
  4912. * memory until all local zones are considered.
  4913. */
  4914. alloc_flags |= alloc_flags_nofragment(ac.preferred_zoneref->zone, gfp);
  4915. /* First allocation attempt */
  4916. page = get_page_from_freelist(alloc_gfp, order, alloc_flags, &ac);
  4917. if (likely(page))
  4918. goto out;
  4919. alloc_gfp = gfp;
  4920. ac.spread_dirty_pages = false;
  4921. /*
  4922. * Restore the original nodemask if it was potentially replaced with
  4923. * &cpuset_current_mems_allowed to optimize the fast-path attempt.
  4924. */
  4925. ac.nodemask = nodemask;
  4926. page = __alloc_pages_slowpath(alloc_gfp, order, &ac);
  4927. out:
  4928. if (memcg_kmem_enabled() && (gfp & __GFP_ACCOUNT) && page &&
  4929. unlikely(__memcg_kmem_charge_page(page, gfp, order) != 0)) {
  4930. __free_pages(page, order);
  4931. page = NULL;
  4932. }
  4933. trace_mm_page_alloc(page, order, alloc_gfp, ac.migratetype);
  4934. kmsan_alloc_page(page, order, alloc_gfp);
  4935. return page;
  4936. }
  4937. EXPORT_SYMBOL(__alloc_pages);
  4938. struct folio *__folio_alloc(gfp_t gfp, unsigned int order, int preferred_nid,
  4939. nodemask_t *nodemask)
  4940. {
  4941. struct page *page = __alloc_pages(gfp | __GFP_COMP, order,
  4942. preferred_nid, nodemask);
  4943. if (page && order > 1)
  4944. prep_transhuge_page(page);
  4945. return (struct folio *)page;
  4946. }
  4947. EXPORT_SYMBOL(__folio_alloc);
  4948. /*
  4949. * Common helper functions. Never use with __GFP_HIGHMEM because the returned
  4950. * address cannot represent highmem pages. Use alloc_pages and then kmap if
  4951. * you need to access high mem.
  4952. */
  4953. unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
  4954. {
  4955. struct page *page;
  4956. page = alloc_pages(gfp_mask & ~__GFP_HIGHMEM, order);
  4957. if (!page)
  4958. return 0;
  4959. return (unsigned long) page_address(page);
  4960. }
  4961. EXPORT_SYMBOL(__get_free_pages);
  4962. unsigned long get_zeroed_page(gfp_t gfp_mask)
  4963. {
  4964. return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
  4965. }
  4966. EXPORT_SYMBOL(get_zeroed_page);
  4967. /**
  4968. * __free_pages - Free pages allocated with alloc_pages().
  4969. * @page: The page pointer returned from alloc_pages().
  4970. * @order: The order of the allocation.
  4971. *
  4972. * This function can free multi-page allocations that are not compound
  4973. * pages. It does not check that the @order passed in matches that of
  4974. * the allocation, so it is easy to leak memory. Freeing more memory
  4975. * than was allocated will probably emit a warning.
  4976. *
  4977. * If the last reference to this page is speculative, it will be released
  4978. * by put_page() which only frees the first page of a non-compound
  4979. * allocation. To prevent the remaining pages from being leaked, we free
  4980. * the subsequent pages here. If you want to use the page's reference
  4981. * count to decide when to free the allocation, you should allocate a
  4982. * compound page, and use put_page() instead of __free_pages().
  4983. *
  4984. * Context: May be called in interrupt context or while holding a normal
  4985. * spinlock, but not in NMI context or while holding a raw spinlock.
  4986. */
  4987. void __free_pages(struct page *page, unsigned int order)
  4988. {
  4989. /* get PageHead before we drop reference */
  4990. int head = PageHead(page);
  4991. if (put_page_testzero(page))
  4992. free_the_page(page, order);
  4993. else if (!head)
  4994. while (order-- > 0)
  4995. free_the_page(page + (1 << order), order);
  4996. }
  4997. EXPORT_SYMBOL(__free_pages);
  4998. void free_pages(unsigned long addr, unsigned int order)
  4999. {
  5000. if (addr != 0) {
  5001. VM_BUG_ON(!virt_addr_valid((void *)addr));
  5002. __free_pages(virt_to_page((void *)addr), order);
  5003. }
  5004. }
  5005. EXPORT_SYMBOL(free_pages);
  5006. /*
  5007. * Page Fragment:
  5008. * An arbitrary-length arbitrary-offset area of memory which resides
  5009. * within a 0 or higher order page. Multiple fragments within that page
  5010. * are individually refcounted, in the page's reference counter.
  5011. *
  5012. * The page_frag functions below provide a simple allocation framework for
  5013. * page fragments. This is used by the network stack and network device
  5014. * drivers to provide a backing region of memory for use as either an
  5015. * sk_buff->head, or to be used in the "frags" portion of skb_shared_info.
  5016. */
  5017. static struct page *__page_frag_cache_refill(struct page_frag_cache *nc,
  5018. gfp_t gfp_mask)
  5019. {
  5020. struct page *page = NULL;
  5021. gfp_t gfp = gfp_mask;
  5022. #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
  5023. gfp_mask |= __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY |
  5024. __GFP_NOMEMALLOC;
  5025. page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
  5026. PAGE_FRAG_CACHE_MAX_ORDER);
  5027. nc->size = page ? PAGE_FRAG_CACHE_MAX_SIZE : PAGE_SIZE;
  5028. #endif
  5029. if (unlikely(!page))
  5030. page = alloc_pages_node(NUMA_NO_NODE, gfp, 0);
  5031. nc->va = page ? page_address(page) : NULL;
  5032. return page;
  5033. }
  5034. void __page_frag_cache_drain(struct page *page, unsigned int count)
  5035. {
  5036. VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
  5037. if (page_ref_sub_and_test(page, count))
  5038. free_the_page(page, compound_order(page));
  5039. }
  5040. EXPORT_SYMBOL(__page_frag_cache_drain);
  5041. void *page_frag_alloc_align(struct page_frag_cache *nc,
  5042. unsigned int fragsz, gfp_t gfp_mask,
  5043. unsigned int align_mask)
  5044. {
  5045. unsigned int size = PAGE_SIZE;
  5046. struct page *page;
  5047. int offset;
  5048. if (unlikely(!nc->va)) {
  5049. refill:
  5050. page = __page_frag_cache_refill(nc, gfp_mask);
  5051. if (!page)
  5052. return NULL;
  5053. #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
  5054. /* if size can vary use size else just use PAGE_SIZE */
  5055. size = nc->size;
  5056. #endif
  5057. /* Even if we own the page, we do not use atomic_set().
  5058. * This would break get_page_unless_zero() users.
  5059. */
  5060. page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE);
  5061. /* reset page count bias and offset to start of new frag */
  5062. nc->pfmemalloc = page_is_pfmemalloc(page);
  5063. nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
  5064. nc->offset = size;
  5065. }
  5066. offset = nc->offset - fragsz;
  5067. if (unlikely(offset < 0)) {
  5068. page = virt_to_page(nc->va);
  5069. if (!page_ref_sub_and_test(page, nc->pagecnt_bias))
  5070. goto refill;
  5071. if (unlikely(nc->pfmemalloc)) {
  5072. free_the_page(page, compound_order(page));
  5073. goto refill;
  5074. }
  5075. #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE)
  5076. /* if size can vary use size else just use PAGE_SIZE */
  5077. size = nc->size;
  5078. #endif
  5079. /* OK, page count is 0, we can safely set it */
  5080. set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1);
  5081. /* reset page count bias and offset to start of new frag */
  5082. nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1;
  5083. offset = size - fragsz;
  5084. if (unlikely(offset < 0)) {
  5085. /*
  5086. * The caller is trying to allocate a fragment
  5087. * with fragsz > PAGE_SIZE but the cache isn't big
  5088. * enough to satisfy the request, this may
  5089. * happen in low memory conditions.
  5090. * We don't release the cache page because
  5091. * it could make memory pressure worse
  5092. * so we simply return NULL here.
  5093. */
  5094. return NULL;
  5095. }
  5096. }
  5097. nc->pagecnt_bias--;
  5098. offset &= align_mask;
  5099. nc->offset = offset;
  5100. return nc->va + offset;
  5101. }
  5102. EXPORT_SYMBOL(page_frag_alloc_align);
  5103. /*
  5104. * Frees a page fragment allocated out of either a compound or order 0 page.
  5105. */
  5106. void page_frag_free(void *addr)
  5107. {
  5108. struct page *page = virt_to_head_page(addr);
  5109. if (unlikely(put_page_testzero(page)))
  5110. free_the_page(page, compound_order(page));
  5111. }
  5112. EXPORT_SYMBOL(page_frag_free);
  5113. static void *make_alloc_exact(unsigned long addr, unsigned int order,
  5114. size_t size)
  5115. {
  5116. if (addr) {
  5117. unsigned long nr = DIV_ROUND_UP(size, PAGE_SIZE);
  5118. struct page *page = virt_to_page((void *)addr);
  5119. struct page *last = page + nr;
  5120. split_page_owner(page, 1 << order);
  5121. split_page_memcg(page, 1 << order);
  5122. while (page < --last)
  5123. set_page_refcounted(last);
  5124. last = page + (1UL << order);
  5125. for (page += nr; page < last; page++)
  5126. __free_pages_ok(page, 0, FPI_TO_TAIL);
  5127. }
  5128. return (void *)addr;
  5129. }
  5130. /**
  5131. * alloc_pages_exact - allocate an exact number physically-contiguous pages.
  5132. * @size: the number of bytes to allocate
  5133. * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
  5134. *
  5135. * This function is similar to alloc_pages(), except that it allocates the
  5136. * minimum number of pages to satisfy the request. alloc_pages() can only
  5137. * allocate memory in power-of-two pages.
  5138. *
  5139. * This function is also limited by MAX_ORDER.
  5140. *
  5141. * Memory allocated by this function must be released by free_pages_exact().
  5142. *
  5143. * Return: pointer to the allocated area or %NULL in case of error.
  5144. */
  5145. void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
  5146. {
  5147. unsigned int order = get_order(size);
  5148. unsigned long addr;
  5149. if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM)))
  5150. gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM);
  5151. addr = __get_free_pages(gfp_mask, order);
  5152. return make_alloc_exact(addr, order, size);
  5153. }
  5154. EXPORT_SYMBOL(alloc_pages_exact);
  5155. /**
  5156. * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
  5157. * pages on a node.
  5158. * @nid: the preferred node ID where memory should be allocated
  5159. * @size: the number of bytes to allocate
  5160. * @gfp_mask: GFP flags for the allocation, must not contain __GFP_COMP
  5161. *
  5162. * Like alloc_pages_exact(), but try to allocate on node nid first before falling
  5163. * back.
  5164. *
  5165. * Return: pointer to the allocated area or %NULL in case of error.
  5166. */
  5167. void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
  5168. {
  5169. unsigned int order = get_order(size);
  5170. struct page *p;
  5171. if (WARN_ON_ONCE(gfp_mask & (__GFP_COMP | __GFP_HIGHMEM)))
  5172. gfp_mask &= ~(__GFP_COMP | __GFP_HIGHMEM);
  5173. p = alloc_pages_node(nid, gfp_mask, order);
  5174. if (!p)
  5175. return NULL;
  5176. return make_alloc_exact((unsigned long)page_address(p), order, size);
  5177. }
  5178. /**
  5179. * free_pages_exact - release memory allocated via alloc_pages_exact()
  5180. * @virt: the value returned by alloc_pages_exact.
  5181. * @size: size of allocation, same value as passed to alloc_pages_exact().
  5182. *
  5183. * Release the memory allocated by a previous call to alloc_pages_exact.
  5184. */
  5185. void free_pages_exact(void *virt, size_t size)
  5186. {
  5187. unsigned long addr = (unsigned long)virt;
  5188. unsigned long end = addr + PAGE_ALIGN(size);
  5189. while (addr < end) {
  5190. free_page(addr);
  5191. addr += PAGE_SIZE;
  5192. }
  5193. }
  5194. EXPORT_SYMBOL(free_pages_exact);
  5195. /**
  5196. * nr_free_zone_pages - count number of pages beyond high watermark
  5197. * @offset: The zone index of the highest zone
  5198. *
  5199. * nr_free_zone_pages() counts the number of pages which are beyond the
  5200. * high watermark within all zones at or below a given zone index. For each
  5201. * zone, the number of pages is calculated as:
  5202. *
  5203. * nr_free_zone_pages = managed_pages - high_pages
  5204. *
  5205. * Return: number of pages beyond high watermark.
  5206. */
  5207. static unsigned long nr_free_zone_pages(int offset)
  5208. {
  5209. struct zoneref *z;
  5210. struct zone *zone;
  5211. /* Just pick one node, since fallback list is circular */
  5212. unsigned long sum = 0;
  5213. struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
  5214. for_each_zone_zonelist(zone, z, zonelist, offset) {
  5215. unsigned long size = zone_managed_pages(zone);
  5216. unsigned long high = high_wmark_pages(zone);
  5217. if (size > high)
  5218. sum += size - high;
  5219. }
  5220. return sum;
  5221. }
  5222. /**
  5223. * nr_free_buffer_pages - count number of pages beyond high watermark
  5224. *
  5225. * nr_free_buffer_pages() counts the number of pages which are beyond the high
  5226. * watermark within ZONE_DMA and ZONE_NORMAL.
  5227. *
  5228. * Return: number of pages beyond high watermark within ZONE_DMA and
  5229. * ZONE_NORMAL.
  5230. */
  5231. unsigned long nr_free_buffer_pages(void)
  5232. {
  5233. return nr_free_zone_pages(gfp_zone(GFP_USER));
  5234. }
  5235. EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
  5236. static inline void show_node(struct zone *zone)
  5237. {
  5238. if (IS_ENABLED(CONFIG_NUMA))
  5239. printk("Node %d ", zone_to_nid(zone));
  5240. }
  5241. long si_mem_available(void)
  5242. {
  5243. long available;
  5244. unsigned long pagecache;
  5245. unsigned long wmark_low = 0;
  5246. unsigned long pages[NR_LRU_LISTS];
  5247. unsigned long reclaimable;
  5248. struct zone *zone;
  5249. int lru;
  5250. for (lru = LRU_BASE; lru < NR_LRU_LISTS; lru++)
  5251. pages[lru] = global_node_page_state(NR_LRU_BASE + lru);
  5252. for_each_zone(zone)
  5253. wmark_low += low_wmark_pages(zone);
  5254. /*
  5255. * Estimate the amount of memory available for userspace allocations,
  5256. * without causing swapping or OOM.
  5257. */
  5258. available = global_zone_page_state(NR_FREE_PAGES) - totalreserve_pages;
  5259. /*
  5260. * Not all the page cache can be freed, otherwise the system will
  5261. * start swapping or thrashing. Assume at least half of the page
  5262. * cache, or the low watermark worth of cache, needs to stay.
  5263. */
  5264. pagecache = pages[LRU_ACTIVE_FILE] + pages[LRU_INACTIVE_FILE];
  5265. pagecache -= min(pagecache / 2, wmark_low);
  5266. available += pagecache;
  5267. /*
  5268. * Part of the reclaimable slab and other kernel memory consists of
  5269. * items that are in use, and cannot be freed. Cap this estimate at the
  5270. * low watermark.
  5271. */
  5272. reclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B) +
  5273. global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE);
  5274. available += reclaimable - min(reclaimable / 2, wmark_low);
  5275. trace_android_vh_si_mem_available_adjust(&available);
  5276. if (available < 0)
  5277. available = 0;
  5278. return available;
  5279. }
  5280. EXPORT_SYMBOL_GPL(si_mem_available);
  5281. void si_meminfo(struct sysinfo *val)
  5282. {
  5283. val->totalram = totalram_pages();
  5284. val->sharedram = global_node_page_state(NR_SHMEM);
  5285. val->freeram = global_zone_page_state(NR_FREE_PAGES);
  5286. val->bufferram = nr_blockdev_pages();
  5287. val->totalhigh = totalhigh_pages();
  5288. val->freehigh = nr_free_highpages();
  5289. val->mem_unit = PAGE_SIZE;
  5290. trace_android_vh_si_meminfo_adjust(&val->totalram, &val->freeram);
  5291. }
  5292. EXPORT_SYMBOL(si_meminfo);
  5293. #ifdef CONFIG_NUMA
  5294. void si_meminfo_node(struct sysinfo *val, int nid)
  5295. {
  5296. int zone_type; /* needs to be signed */
  5297. unsigned long managed_pages = 0;
  5298. unsigned long managed_highpages = 0;
  5299. unsigned long free_highpages = 0;
  5300. pg_data_t *pgdat = NODE_DATA(nid);
  5301. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++)
  5302. managed_pages += zone_managed_pages(&pgdat->node_zones[zone_type]);
  5303. val->totalram = managed_pages;
  5304. val->sharedram = node_page_state(pgdat, NR_SHMEM);
  5305. val->freeram = sum_zone_node_page_state(nid, NR_FREE_PAGES);
  5306. #ifdef CONFIG_HIGHMEM
  5307. for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
  5308. struct zone *zone = &pgdat->node_zones[zone_type];
  5309. if (is_highmem(zone)) {
  5310. managed_highpages += zone_managed_pages(zone);
  5311. free_highpages += zone_page_state(zone, NR_FREE_PAGES);
  5312. }
  5313. }
  5314. val->totalhigh = managed_highpages;
  5315. val->freehigh = free_highpages;
  5316. #else
  5317. val->totalhigh = managed_highpages;
  5318. val->freehigh = free_highpages;
  5319. #endif
  5320. val->mem_unit = PAGE_SIZE;
  5321. }
  5322. #endif
  5323. /*
  5324. * Determine whether the node should be displayed or not, depending on whether
  5325. * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
  5326. */
  5327. static bool show_mem_node_skip(unsigned int flags, int nid, nodemask_t *nodemask)
  5328. {
  5329. if (!(flags & SHOW_MEM_FILTER_NODES))
  5330. return false;
  5331. /*
  5332. * no node mask - aka implicit memory numa policy. Do not bother with
  5333. * the synchronization - read_mems_allowed_begin - because we do not
  5334. * have to be precise here.
  5335. */
  5336. if (!nodemask)
  5337. nodemask = &cpuset_current_mems_allowed;
  5338. return !node_isset(nid, *nodemask);
  5339. }
  5340. #define K(x) ((x) << (PAGE_SHIFT-10))
  5341. static void show_migration_types(unsigned char type)
  5342. {
  5343. static const char types[MIGRATE_TYPES] = {
  5344. [MIGRATE_UNMOVABLE] = 'U',
  5345. [MIGRATE_MOVABLE] = 'M',
  5346. [MIGRATE_RECLAIMABLE] = 'E',
  5347. [MIGRATE_HIGHATOMIC] = 'H',
  5348. #ifdef CONFIG_CMA
  5349. [MIGRATE_CMA] = 'C',
  5350. #endif
  5351. #ifdef CONFIG_MEMORY_ISOLATION
  5352. [MIGRATE_ISOLATE] = 'I',
  5353. #endif
  5354. };
  5355. char tmp[MIGRATE_TYPES + 1];
  5356. char *p = tmp;
  5357. int i;
  5358. for (i = 0; i < MIGRATE_TYPES; i++) {
  5359. if (type & (1 << i))
  5360. *p++ = types[i];
  5361. }
  5362. *p = '\0';
  5363. printk(KERN_CONT "(%s) ", tmp);
  5364. }
  5365. static bool node_has_managed_zones(pg_data_t *pgdat, int max_zone_idx)
  5366. {
  5367. int zone_idx;
  5368. for (zone_idx = 0; zone_idx <= max_zone_idx; zone_idx++)
  5369. if (zone_managed_pages(pgdat->node_zones + zone_idx))
  5370. return true;
  5371. return false;
  5372. }
  5373. /*
  5374. * Show free area list (used inside shift_scroll-lock stuff)
  5375. * We also calculate the percentage fragmentation. We do this by counting the
  5376. * memory on each free list with the exception of the first item on the list.
  5377. *
  5378. * Bits in @filter:
  5379. * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's
  5380. * cpuset.
  5381. */
  5382. void __show_free_areas(unsigned int filter, nodemask_t *nodemask, int max_zone_idx)
  5383. {
  5384. unsigned long free_pcp = 0;
  5385. int cpu, nid;
  5386. struct zone *zone;
  5387. pg_data_t *pgdat;
  5388. for_each_populated_zone(zone) {
  5389. if (zone_idx(zone) > max_zone_idx)
  5390. continue;
  5391. if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
  5392. continue;
  5393. for_each_online_cpu(cpu)
  5394. free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count;
  5395. }
  5396. printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
  5397. " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
  5398. " unevictable:%lu dirty:%lu writeback:%lu\n"
  5399. " slab_reclaimable:%lu slab_unreclaimable:%lu\n"
  5400. " mapped:%lu shmem:%lu pagetables:%lu\n"
  5401. " sec_pagetables:%lu bounce:%lu\n"
  5402. " kernel_misc_reclaimable:%lu\n"
  5403. " free:%lu free_pcp:%lu free_cma:%lu\n",
  5404. global_node_page_state(NR_ACTIVE_ANON),
  5405. global_node_page_state(NR_INACTIVE_ANON),
  5406. global_node_page_state(NR_ISOLATED_ANON),
  5407. global_node_page_state(NR_ACTIVE_FILE),
  5408. global_node_page_state(NR_INACTIVE_FILE),
  5409. global_node_page_state(NR_ISOLATED_FILE),
  5410. global_node_page_state(NR_UNEVICTABLE),
  5411. global_node_page_state(NR_FILE_DIRTY),
  5412. global_node_page_state(NR_WRITEBACK),
  5413. global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B),
  5414. global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B),
  5415. global_node_page_state(NR_FILE_MAPPED),
  5416. global_node_page_state(NR_SHMEM),
  5417. global_node_page_state(NR_PAGETABLE),
  5418. global_node_page_state(NR_SECONDARY_PAGETABLE),
  5419. global_zone_page_state(NR_BOUNCE),
  5420. global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE),
  5421. global_zone_page_state(NR_FREE_PAGES),
  5422. free_pcp,
  5423. global_zone_page_state(NR_FREE_CMA_PAGES));
  5424. for_each_online_pgdat(pgdat) {
  5425. if (show_mem_node_skip(filter, pgdat->node_id, nodemask))
  5426. continue;
  5427. if (!node_has_managed_zones(pgdat, max_zone_idx))
  5428. continue;
  5429. printk("Node %d"
  5430. " active_anon:%lukB"
  5431. " inactive_anon:%lukB"
  5432. " active_file:%lukB"
  5433. " inactive_file:%lukB"
  5434. " unevictable:%lukB"
  5435. " isolated(anon):%lukB"
  5436. " isolated(file):%lukB"
  5437. " mapped:%lukB"
  5438. " dirty:%lukB"
  5439. " writeback:%lukB"
  5440. " shmem:%lukB"
  5441. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  5442. " shmem_thp: %lukB"
  5443. " shmem_pmdmapped: %lukB"
  5444. " anon_thp: %lukB"
  5445. #endif
  5446. " writeback_tmp:%lukB"
  5447. " kernel_stack:%lukB"
  5448. #ifdef CONFIG_SHADOW_CALL_STACK
  5449. " shadow_call_stack:%lukB"
  5450. #endif
  5451. " pagetables:%lukB"
  5452. " sec_pagetables:%lukB"
  5453. " all_unreclaimable? %s"
  5454. "\n",
  5455. pgdat->node_id,
  5456. K(node_page_state(pgdat, NR_ACTIVE_ANON)),
  5457. K(node_page_state(pgdat, NR_INACTIVE_ANON)),
  5458. K(node_page_state(pgdat, NR_ACTIVE_FILE)),
  5459. K(node_page_state(pgdat, NR_INACTIVE_FILE)),
  5460. K(node_page_state(pgdat, NR_UNEVICTABLE)),
  5461. K(node_page_state(pgdat, NR_ISOLATED_ANON)),
  5462. K(node_page_state(pgdat, NR_ISOLATED_FILE)),
  5463. K(node_page_state(pgdat, NR_FILE_MAPPED)),
  5464. K(node_page_state(pgdat, NR_FILE_DIRTY)),
  5465. K(node_page_state(pgdat, NR_WRITEBACK)),
  5466. K(node_page_state(pgdat, NR_SHMEM)),
  5467. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  5468. K(node_page_state(pgdat, NR_SHMEM_THPS)),
  5469. K(node_page_state(pgdat, NR_SHMEM_PMDMAPPED)),
  5470. K(node_page_state(pgdat, NR_ANON_THPS)),
  5471. #endif
  5472. K(node_page_state(pgdat, NR_WRITEBACK_TEMP)),
  5473. node_page_state(pgdat, NR_KERNEL_STACK_KB),
  5474. #ifdef CONFIG_SHADOW_CALL_STACK
  5475. node_page_state(pgdat, NR_KERNEL_SCS_KB),
  5476. #endif
  5477. K(node_page_state(pgdat, NR_PAGETABLE)),
  5478. K(node_page_state(pgdat, NR_SECONDARY_PAGETABLE)),
  5479. pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ?
  5480. "yes" : "no");
  5481. }
  5482. for_each_populated_zone(zone) {
  5483. int i;
  5484. if (zone_idx(zone) > max_zone_idx)
  5485. continue;
  5486. if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
  5487. continue;
  5488. free_pcp = 0;
  5489. for_each_online_cpu(cpu)
  5490. free_pcp += per_cpu_ptr(zone->per_cpu_pageset, cpu)->count;
  5491. show_node(zone);
  5492. printk(KERN_CONT
  5493. "%s"
  5494. " free:%lukB"
  5495. " boost:%lukB"
  5496. " min:%lukB"
  5497. " low:%lukB"
  5498. " high:%lukB"
  5499. " reserved_highatomic:%luKB"
  5500. " active_anon:%lukB"
  5501. " inactive_anon:%lukB"
  5502. " active_file:%lukB"
  5503. " inactive_file:%lukB"
  5504. " unevictable:%lukB"
  5505. " writepending:%lukB"
  5506. " present:%lukB"
  5507. " managed:%lukB"
  5508. " mlocked:%lukB"
  5509. " bounce:%lukB"
  5510. " free_pcp:%lukB"
  5511. " local_pcp:%ukB"
  5512. " free_cma:%lukB"
  5513. "\n",
  5514. zone->name,
  5515. K(zone_page_state(zone, NR_FREE_PAGES)),
  5516. K(zone->watermark_boost),
  5517. K(min_wmark_pages(zone)),
  5518. K(low_wmark_pages(zone)),
  5519. K(high_wmark_pages(zone)),
  5520. K(zone->nr_reserved_highatomic),
  5521. K(zone_page_state(zone, NR_ZONE_ACTIVE_ANON)),
  5522. K(zone_page_state(zone, NR_ZONE_INACTIVE_ANON)),
  5523. K(zone_page_state(zone, NR_ZONE_ACTIVE_FILE)),
  5524. K(zone_page_state(zone, NR_ZONE_INACTIVE_FILE)),
  5525. K(zone_page_state(zone, NR_ZONE_UNEVICTABLE)),
  5526. K(zone_page_state(zone, NR_ZONE_WRITE_PENDING)),
  5527. K(zone->present_pages),
  5528. K(zone_managed_pages(zone)),
  5529. K(zone_page_state(zone, NR_MLOCK)),
  5530. K(zone_page_state(zone, NR_BOUNCE)),
  5531. K(free_pcp),
  5532. K(this_cpu_read(zone->per_cpu_pageset->count)),
  5533. K(zone_page_state(zone, NR_FREE_CMA_PAGES)));
  5534. printk("lowmem_reserve[]:");
  5535. for (i = 0; i < MAX_NR_ZONES; i++)
  5536. printk(KERN_CONT " %ld", zone->lowmem_reserve[i]);
  5537. printk(KERN_CONT "\n");
  5538. }
  5539. for_each_populated_zone(zone) {
  5540. unsigned int order;
  5541. unsigned long nr[MAX_ORDER], flags, total = 0;
  5542. unsigned char types[MAX_ORDER];
  5543. if (zone_idx(zone) > max_zone_idx)
  5544. continue;
  5545. if (show_mem_node_skip(filter, zone_to_nid(zone), nodemask))
  5546. continue;
  5547. show_node(zone);
  5548. printk(KERN_CONT "%s: ", zone->name);
  5549. spin_lock_irqsave(&zone->lock, flags);
  5550. for (order = 0; order < MAX_ORDER; order++) {
  5551. struct free_area *area = &zone->free_area[order];
  5552. int type;
  5553. nr[order] = area->nr_free;
  5554. total += nr[order] << order;
  5555. types[order] = 0;
  5556. for (type = 0; type < MIGRATE_TYPES; type++) {
  5557. if (!free_area_empty(area, type))
  5558. types[order] |= 1 << type;
  5559. }
  5560. }
  5561. spin_unlock_irqrestore(&zone->lock, flags);
  5562. for (order = 0; order < MAX_ORDER; order++) {
  5563. printk(KERN_CONT "%lu*%lukB ",
  5564. nr[order], K(1UL) << order);
  5565. if (nr[order])
  5566. show_migration_types(types[order]);
  5567. }
  5568. printk(KERN_CONT "= %lukB\n", K(total));
  5569. }
  5570. for_each_online_node(nid) {
  5571. if (show_mem_node_skip(filter, nid, nodemask))
  5572. continue;
  5573. hugetlb_show_meminfo_node(nid);
  5574. }
  5575. printk("%ld total pagecache pages\n", global_node_page_state(NR_FILE_PAGES));
  5576. show_swap_cache_info();
  5577. }
  5578. static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
  5579. {
  5580. zoneref->zone = zone;
  5581. zoneref->zone_idx = zone_idx(zone);
  5582. }
  5583. /*
  5584. * Builds allocation fallback zone lists.
  5585. *
  5586. * Add all populated zones of a node to the zonelist.
  5587. */
  5588. static int build_zonerefs_node(pg_data_t *pgdat, struct zoneref *zonerefs)
  5589. {
  5590. struct zone *zone;
  5591. enum zone_type zone_type = MAX_NR_ZONES;
  5592. int nr_zones = 0;
  5593. do {
  5594. zone_type--;
  5595. zone = pgdat->node_zones + zone_type;
  5596. if (populated_zone(zone)) {
  5597. zoneref_set_zone(zone, &zonerefs[nr_zones++]);
  5598. check_highest_zone(zone_type);
  5599. }
  5600. } while (zone_type);
  5601. return nr_zones;
  5602. }
  5603. #ifdef CONFIG_NUMA
  5604. static int __parse_numa_zonelist_order(char *s)
  5605. {
  5606. /*
  5607. * We used to support different zonelists modes but they turned
  5608. * out to be just not useful. Let's keep the warning in place
  5609. * if somebody still use the cmd line parameter so that we do
  5610. * not fail it silently
  5611. */
  5612. if (!(*s == 'd' || *s == 'D' || *s == 'n' || *s == 'N')) {
  5613. pr_warn("Ignoring unsupported numa_zonelist_order value: %s\n", s);
  5614. return -EINVAL;
  5615. }
  5616. return 0;
  5617. }
  5618. char numa_zonelist_order[] = "Node";
  5619. /*
  5620. * sysctl handler for numa_zonelist_order
  5621. */
  5622. int numa_zonelist_order_handler(struct ctl_table *table, int write,
  5623. void *buffer, size_t *length, loff_t *ppos)
  5624. {
  5625. if (write)
  5626. return __parse_numa_zonelist_order(buffer);
  5627. return proc_dostring(table, write, buffer, length, ppos);
  5628. }
  5629. static int node_load[MAX_NUMNODES];
  5630. /**
  5631. * find_next_best_node - find the next node that should appear in a given node's fallback list
  5632. * @node: node whose fallback list we're appending
  5633. * @used_node_mask: nodemask_t of already used nodes
  5634. *
  5635. * We use a number of factors to determine which is the next node that should
  5636. * appear on a given node's fallback list. The node should not have appeared
  5637. * already in @node's fallback list, and it should be the next closest node
  5638. * according to the distance array (which contains arbitrary distance values
  5639. * from each node to each node in the system), and should also prefer nodes
  5640. * with no CPUs, since presumably they'll have very little allocation pressure
  5641. * on them otherwise.
  5642. *
  5643. * Return: node id of the found node or %NUMA_NO_NODE if no node is found.
  5644. */
  5645. int find_next_best_node(int node, nodemask_t *used_node_mask)
  5646. {
  5647. int n, val;
  5648. int min_val = INT_MAX;
  5649. int best_node = NUMA_NO_NODE;
  5650. /* Use the local node if we haven't already */
  5651. if (!node_isset(node, *used_node_mask)) {
  5652. node_set(node, *used_node_mask);
  5653. return node;
  5654. }
  5655. for_each_node_state(n, N_MEMORY) {
  5656. /* Don't want a node to appear more than once */
  5657. if (node_isset(n, *used_node_mask))
  5658. continue;
  5659. /* Use the distance array to find the distance */
  5660. val = node_distance(node, n);
  5661. /* Penalize nodes under us ("prefer the next node") */
  5662. val += (n < node);
  5663. /* Give preference to headless and unused nodes */
  5664. if (!cpumask_empty(cpumask_of_node(n)))
  5665. val += PENALTY_FOR_NODE_WITH_CPUS;
  5666. /* Slight preference for less loaded node */
  5667. val *= MAX_NUMNODES;
  5668. val += node_load[n];
  5669. if (val < min_val) {
  5670. min_val = val;
  5671. best_node = n;
  5672. }
  5673. }
  5674. if (best_node >= 0)
  5675. node_set(best_node, *used_node_mask);
  5676. return best_node;
  5677. }
  5678. /*
  5679. * Build zonelists ordered by node and zones within node.
  5680. * This results in maximum locality--normal zone overflows into local
  5681. * DMA zone, if any--but risks exhausting DMA zone.
  5682. */
  5683. static void build_zonelists_in_node_order(pg_data_t *pgdat, int *node_order,
  5684. unsigned nr_nodes)
  5685. {
  5686. struct zoneref *zonerefs;
  5687. int i;
  5688. zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
  5689. for (i = 0; i < nr_nodes; i++) {
  5690. int nr_zones;
  5691. pg_data_t *node = NODE_DATA(node_order[i]);
  5692. nr_zones = build_zonerefs_node(node, zonerefs);
  5693. zonerefs += nr_zones;
  5694. }
  5695. zonerefs->zone = NULL;
  5696. zonerefs->zone_idx = 0;
  5697. }
  5698. /*
  5699. * Build gfp_thisnode zonelists
  5700. */
  5701. static void build_thisnode_zonelists(pg_data_t *pgdat)
  5702. {
  5703. struct zoneref *zonerefs;
  5704. int nr_zones;
  5705. zonerefs = pgdat->node_zonelists[ZONELIST_NOFALLBACK]._zonerefs;
  5706. nr_zones = build_zonerefs_node(pgdat, zonerefs);
  5707. zonerefs += nr_zones;
  5708. zonerefs->zone = NULL;
  5709. zonerefs->zone_idx = 0;
  5710. }
  5711. /*
  5712. * Build zonelists ordered by zone and nodes within zones.
  5713. * This results in conserving DMA zone[s] until all Normal memory is
  5714. * exhausted, but results in overflowing to remote node while memory
  5715. * may still exist in local DMA zone.
  5716. */
  5717. static void build_zonelists(pg_data_t *pgdat)
  5718. {
  5719. static int node_order[MAX_NUMNODES];
  5720. int node, nr_nodes = 0;
  5721. nodemask_t used_mask = NODE_MASK_NONE;
  5722. int local_node, prev_node;
  5723. /* NUMA-aware ordering of nodes */
  5724. local_node = pgdat->node_id;
  5725. prev_node = local_node;
  5726. memset(node_order, 0, sizeof(node_order));
  5727. while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
  5728. /*
  5729. * We don't want to pressure a particular node.
  5730. * So adding penalty to the first node in same
  5731. * distance group to make it round-robin.
  5732. */
  5733. if (node_distance(local_node, node) !=
  5734. node_distance(local_node, prev_node))
  5735. node_load[node] += 1;
  5736. node_order[nr_nodes++] = node;
  5737. prev_node = node;
  5738. }
  5739. build_zonelists_in_node_order(pgdat, node_order, nr_nodes);
  5740. build_thisnode_zonelists(pgdat);
  5741. pr_info("Fallback order for Node %d: ", local_node);
  5742. for (node = 0; node < nr_nodes; node++)
  5743. pr_cont("%d ", node_order[node]);
  5744. pr_cont("\n");
  5745. }
  5746. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  5747. /*
  5748. * Return node id of node used for "local" allocations.
  5749. * I.e., first node id of first zone in arg node's generic zonelist.
  5750. * Used for initializing percpu 'numa_mem', which is used primarily
  5751. * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
  5752. */
  5753. int local_memory_node(int node)
  5754. {
  5755. struct zoneref *z;
  5756. z = first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
  5757. gfp_zone(GFP_KERNEL),
  5758. NULL);
  5759. return zone_to_nid(z->zone);
  5760. }
  5761. #endif
  5762. static void setup_min_unmapped_ratio(void);
  5763. static void setup_min_slab_ratio(void);
  5764. #else /* CONFIG_NUMA */
  5765. static void build_zonelists(pg_data_t *pgdat)
  5766. {
  5767. int node, local_node;
  5768. struct zoneref *zonerefs;
  5769. int nr_zones;
  5770. local_node = pgdat->node_id;
  5771. zonerefs = pgdat->node_zonelists[ZONELIST_FALLBACK]._zonerefs;
  5772. nr_zones = build_zonerefs_node(pgdat, zonerefs);
  5773. zonerefs += nr_zones;
  5774. /*
  5775. * Now we build the zonelist so that it contains the zones
  5776. * of all the other nodes.
  5777. * We don't want to pressure a particular node, so when
  5778. * building the zones for node N, we make sure that the
  5779. * zones coming right after the local ones are those from
  5780. * node N+1 (modulo N)
  5781. */
  5782. for (node = local_node + 1; node < MAX_NUMNODES; node++) {
  5783. if (!node_online(node))
  5784. continue;
  5785. nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
  5786. zonerefs += nr_zones;
  5787. }
  5788. for (node = 0; node < local_node; node++) {
  5789. if (!node_online(node))
  5790. continue;
  5791. nr_zones = build_zonerefs_node(NODE_DATA(node), zonerefs);
  5792. zonerefs += nr_zones;
  5793. }
  5794. zonerefs->zone = NULL;
  5795. zonerefs->zone_idx = 0;
  5796. }
  5797. #endif /* CONFIG_NUMA */
  5798. /*
  5799. * Boot pageset table. One per cpu which is going to be used for all
  5800. * zones and all nodes. The parameters will be set in such a way
  5801. * that an item put on a list will immediately be handed over to
  5802. * the buddy list. This is safe since pageset manipulation is done
  5803. * with interrupts disabled.
  5804. *
  5805. * The boot_pagesets must be kept even after bootup is complete for
  5806. * unused processors and/or zones. They do play a role for bootstrapping
  5807. * hotplugged processors.
  5808. *
  5809. * zoneinfo_show() and maybe other functions do
  5810. * not check if the processor is online before following the pageset pointer.
  5811. * Other parts of the kernel may not check if the zone is available.
  5812. */
  5813. static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats);
  5814. /* These effectively disable the pcplists in the boot pageset completely */
  5815. #define BOOT_PAGESET_HIGH 0
  5816. #define BOOT_PAGESET_BATCH 1
  5817. static DEFINE_PER_CPU(struct per_cpu_pages, boot_pageset);
  5818. static DEFINE_PER_CPU(struct per_cpu_zonestat, boot_zonestats);
  5819. static DEFINE_PER_CPU(struct per_cpu_nodestat, boot_nodestats);
  5820. static void __build_all_zonelists(void *data)
  5821. {
  5822. int nid;
  5823. int __maybe_unused cpu;
  5824. pg_data_t *self = data;
  5825. unsigned long flags;
  5826. /*
  5827. * The zonelist_update_seq must be acquired with irqsave because the
  5828. * reader can be invoked from IRQ with GFP_ATOMIC.
  5829. */
  5830. write_seqlock_irqsave(&zonelist_update_seq, flags);
  5831. /*
  5832. * Also disable synchronous printk() to prevent any printk() from
  5833. * trying to hold port->lock, for
  5834. * tty_insert_flip_string_and_push_buffer() on other CPU might be
  5835. * calling kmalloc(GFP_ATOMIC | __GFP_NOWARN) with port->lock held.
  5836. */
  5837. printk_deferred_enter();
  5838. #ifdef CONFIG_NUMA
  5839. memset(node_load, 0, sizeof(node_load));
  5840. #endif
  5841. /*
  5842. * This node is hotadded and no memory is yet present. So just
  5843. * building zonelists is fine - no need to touch other nodes.
  5844. */
  5845. if (self && !node_online(self->node_id)) {
  5846. build_zonelists(self);
  5847. } else {
  5848. /*
  5849. * All possible nodes have pgdat preallocated
  5850. * in free_area_init
  5851. */
  5852. for_each_node(nid) {
  5853. pg_data_t *pgdat = NODE_DATA(nid);
  5854. build_zonelists(pgdat);
  5855. }
  5856. #ifdef CONFIG_HAVE_MEMORYLESS_NODES
  5857. /*
  5858. * We now know the "local memory node" for each node--
  5859. * i.e., the node of the first zone in the generic zonelist.
  5860. * Set up numa_mem percpu variable for on-line cpus. During
  5861. * boot, only the boot cpu should be on-line; we'll init the
  5862. * secondary cpus' numa_mem as they come on-line. During
  5863. * node/memory hotplug, we'll fixup all on-line cpus.
  5864. */
  5865. for_each_online_cpu(cpu)
  5866. set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
  5867. #endif
  5868. }
  5869. printk_deferred_exit();
  5870. write_sequnlock_irqrestore(&zonelist_update_seq, flags);
  5871. }
  5872. static noinline void __init
  5873. build_all_zonelists_init(void)
  5874. {
  5875. int cpu;
  5876. __build_all_zonelists(NULL);
  5877. /*
  5878. * Initialize the boot_pagesets that are going to be used
  5879. * for bootstrapping processors. The real pagesets for
  5880. * each zone will be allocated later when the per cpu
  5881. * allocator is available.
  5882. *
  5883. * boot_pagesets are used also for bootstrapping offline
  5884. * cpus if the system is already booted because the pagesets
  5885. * are needed to initialize allocators on a specific cpu too.
  5886. * F.e. the percpu allocator needs the page allocator which
  5887. * needs the percpu allocator in order to allocate its pagesets
  5888. * (a chicken-egg dilemma).
  5889. */
  5890. for_each_possible_cpu(cpu)
  5891. per_cpu_pages_init(&per_cpu(boot_pageset, cpu), &per_cpu(boot_zonestats, cpu));
  5892. mminit_verify_zonelist();
  5893. cpuset_init_current_mems_allowed();
  5894. }
  5895. /*
  5896. * unless system_state == SYSTEM_BOOTING.
  5897. *
  5898. * __ref due to call of __init annotated helper build_all_zonelists_init
  5899. * [protected by SYSTEM_BOOTING].
  5900. */
  5901. void __ref build_all_zonelists(pg_data_t *pgdat)
  5902. {
  5903. unsigned long vm_total_pages;
  5904. if (system_state == SYSTEM_BOOTING) {
  5905. build_all_zonelists_init();
  5906. } else {
  5907. __build_all_zonelists(pgdat);
  5908. /* cpuset refresh routine should be here */
  5909. }
  5910. /* Get the number of free pages beyond high watermark in all zones. */
  5911. vm_total_pages = nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
  5912. /*
  5913. * Disable grouping by mobility if the number of pages in the
  5914. * system is too low to allow the mechanism to work. It would be
  5915. * more accurate, but expensive to check per-zone. This check is
  5916. * made on memory-hotadd so a system can start with mobility
  5917. * disabled and enable it later
  5918. */
  5919. if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
  5920. page_group_by_mobility_disabled = 1;
  5921. else
  5922. page_group_by_mobility_disabled = 0;
  5923. pr_info("Built %u zonelists, mobility grouping %s. Total pages: %ld\n",
  5924. nr_online_nodes,
  5925. page_group_by_mobility_disabled ? "off" : "on",
  5926. vm_total_pages);
  5927. #ifdef CONFIG_NUMA
  5928. pr_info("Policy zone: %s\n", zone_names[policy_zone]);
  5929. #endif
  5930. }
  5931. /* If zone is ZONE_MOVABLE but memory is mirrored, it is an overlapped init */
  5932. static bool __meminit
  5933. overlap_memmap_init(unsigned long zone, unsigned long *pfn)
  5934. {
  5935. static struct memblock_region *r;
  5936. if (mirrored_kernelcore && zone == ZONE_MOVABLE) {
  5937. if (!r || *pfn >= memblock_region_memory_end_pfn(r)) {
  5938. for_each_mem_region(r) {
  5939. if (*pfn < memblock_region_memory_end_pfn(r))
  5940. break;
  5941. }
  5942. }
  5943. if (*pfn >= memblock_region_memory_base_pfn(r) &&
  5944. memblock_is_mirror(r)) {
  5945. *pfn = memblock_region_memory_end_pfn(r);
  5946. return true;
  5947. }
  5948. }
  5949. return false;
  5950. }
  5951. /*
  5952. * Initially all pages are reserved - free ones are freed
  5953. * up by memblock_free_all() once the early boot process is
  5954. * done. Non-atomic initialization, single-pass.
  5955. *
  5956. * All aligned pageblocks are initialized to the specified migratetype
  5957. * (usually MIGRATE_MOVABLE). Besides setting the migratetype, no related
  5958. * zone stats (e.g., nr_isolate_pageblock) are touched.
  5959. */
  5960. void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone,
  5961. unsigned long start_pfn, unsigned long zone_end_pfn,
  5962. enum meminit_context context,
  5963. struct vmem_altmap *altmap, int migratetype)
  5964. {
  5965. unsigned long pfn, end_pfn = start_pfn + size;
  5966. struct page *page;
  5967. if (highest_memmap_pfn < end_pfn - 1)
  5968. highest_memmap_pfn = end_pfn - 1;
  5969. #ifdef CONFIG_ZONE_DEVICE
  5970. /*
  5971. * Honor reservation requested by the driver for this ZONE_DEVICE
  5972. * memory. We limit the total number of pages to initialize to just
  5973. * those that might contain the memory mapping. We will defer the
  5974. * ZONE_DEVICE page initialization until after we have released
  5975. * the hotplug lock.
  5976. */
  5977. if (zone == ZONE_DEVICE) {
  5978. if (!altmap)
  5979. return;
  5980. if (start_pfn == altmap->base_pfn)
  5981. start_pfn += altmap->reserve;
  5982. end_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
  5983. }
  5984. #endif
  5985. for (pfn = start_pfn; pfn < end_pfn; ) {
  5986. /*
  5987. * There can be holes in boot-time mem_map[]s handed to this
  5988. * function. They do not exist on hotplugged memory.
  5989. */
  5990. if (context == MEMINIT_EARLY) {
  5991. if (overlap_memmap_init(zone, &pfn))
  5992. continue;
  5993. if (defer_init(nid, pfn, zone_end_pfn))
  5994. break;
  5995. }
  5996. page = pfn_to_page(pfn);
  5997. __init_single_page(page, pfn, zone, nid);
  5998. if (context == MEMINIT_HOTPLUG)
  5999. __SetPageReserved(page);
  6000. /*
  6001. * Usually, we want to mark the pageblock MIGRATE_MOVABLE,
  6002. * such that unmovable allocations won't be scattered all
  6003. * over the place during system boot.
  6004. */
  6005. if (pageblock_aligned(pfn)) {
  6006. set_pageblock_migratetype(page, migratetype);
  6007. cond_resched();
  6008. }
  6009. pfn++;
  6010. }
  6011. }
  6012. #ifdef CONFIG_ZONE_DEVICE
  6013. static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
  6014. unsigned long zone_idx, int nid,
  6015. struct dev_pagemap *pgmap)
  6016. {
  6017. __init_single_page(page, pfn, zone_idx, nid);
  6018. /*
  6019. * Mark page reserved as it will need to wait for onlining
  6020. * phase for it to be fully associated with a zone.
  6021. *
  6022. * We can use the non-atomic __set_bit operation for setting
  6023. * the flag as we are still initializing the pages.
  6024. */
  6025. __SetPageReserved(page);
  6026. /*
  6027. * ZONE_DEVICE pages union ->lru with a ->pgmap back pointer
  6028. * and zone_device_data. It is a bug if a ZONE_DEVICE page is
  6029. * ever freed or placed on a driver-private list.
  6030. */
  6031. page->pgmap = pgmap;
  6032. page->zone_device_data = NULL;
  6033. /*
  6034. * Mark the block movable so that blocks are reserved for
  6035. * movable at startup. This will force kernel allocations
  6036. * to reserve their blocks rather than leaking throughout
  6037. * the address space during boot when many long-lived
  6038. * kernel allocations are made.
  6039. *
  6040. * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
  6041. * because this is done early in section_activate()
  6042. */
  6043. if (pageblock_aligned(pfn)) {
  6044. set_pageblock_migratetype(page, MIGRATE_MOVABLE);
  6045. cond_resched();
  6046. }
  6047. /*
  6048. * ZONE_DEVICE pages are released directly to the driver page allocator
  6049. * which will set the page count to 1 when allocating the page.
  6050. */
  6051. if (pgmap->type == MEMORY_DEVICE_PRIVATE ||
  6052. pgmap->type == MEMORY_DEVICE_COHERENT)
  6053. set_page_count(page, 0);
  6054. }
  6055. /*
  6056. * With compound page geometry and when struct pages are stored in ram most
  6057. * tail pages are reused. Consequently, the amount of unique struct pages to
  6058. * initialize is a lot smaller that the total amount of struct pages being
  6059. * mapped. This is a paired / mild layering violation with explicit knowledge
  6060. * of how the sparse_vmemmap internals handle compound pages in the lack
  6061. * of an altmap. See vmemmap_populate_compound_pages().
  6062. */
  6063. static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,
  6064. unsigned long nr_pages)
  6065. {
  6066. return is_power_of_2(sizeof(struct page)) &&
  6067. !altmap ? 2 * (PAGE_SIZE / sizeof(struct page)) : nr_pages;
  6068. }
  6069. static void __ref memmap_init_compound(struct page *head,
  6070. unsigned long head_pfn,
  6071. unsigned long zone_idx, int nid,
  6072. struct dev_pagemap *pgmap,
  6073. unsigned long nr_pages)
  6074. {
  6075. unsigned long pfn, end_pfn = head_pfn + nr_pages;
  6076. unsigned int order = pgmap->vmemmap_shift;
  6077. __SetPageHead(head);
  6078. for (pfn = head_pfn + 1; pfn < end_pfn; pfn++) {
  6079. struct page *page = pfn_to_page(pfn);
  6080. __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
  6081. prep_compound_tail(head, pfn - head_pfn);
  6082. set_page_count(page, 0);
  6083. /*
  6084. * The first tail page stores compound_mapcount_ptr() and
  6085. * compound_order() and the second tail page stores
  6086. * compound_pincount_ptr(). Call prep_compound_head() after
  6087. * the first and second tail pages have been initialized to
  6088. * not have the data overwritten.
  6089. */
  6090. if (pfn == head_pfn + 2)
  6091. prep_compound_head(head, order);
  6092. }
  6093. }
  6094. void __ref memmap_init_zone_device(struct zone *zone,
  6095. unsigned long start_pfn,
  6096. unsigned long nr_pages,
  6097. struct dev_pagemap *pgmap)
  6098. {
  6099. unsigned long pfn, end_pfn = start_pfn + nr_pages;
  6100. struct pglist_data *pgdat = zone->zone_pgdat;
  6101. struct vmem_altmap *altmap = pgmap_altmap(pgmap);
  6102. unsigned int pfns_per_compound = pgmap_vmemmap_nr(pgmap);
  6103. unsigned long zone_idx = zone_idx(zone);
  6104. unsigned long start = jiffies;
  6105. int nid = pgdat->node_id;
  6106. if (WARN_ON_ONCE(!pgmap || zone_idx != ZONE_DEVICE))
  6107. return;
  6108. /*
  6109. * The call to memmap_init should have already taken care
  6110. * of the pages reserved for the memmap, so we can just jump to
  6111. * the end of that region and start processing the device pages.
  6112. */
  6113. if (altmap) {
  6114. start_pfn = altmap->base_pfn + vmem_altmap_offset(altmap);
  6115. nr_pages = end_pfn - start_pfn;
  6116. }
  6117. for (pfn = start_pfn; pfn < end_pfn; pfn += pfns_per_compound) {
  6118. struct page *page = pfn_to_page(pfn);
  6119. __init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
  6120. if (pfns_per_compound == 1)
  6121. continue;
  6122. memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
  6123. compound_nr_pages(altmap, pfns_per_compound));
  6124. }
  6125. pr_info("%s initialised %lu pages in %ums\n", __func__,
  6126. nr_pages, jiffies_to_msecs(jiffies - start));
  6127. }
  6128. #endif
  6129. static void __meminit zone_init_free_lists(struct zone *zone)
  6130. {
  6131. unsigned int order, t;
  6132. for_each_migratetype_order(order, t) {
  6133. INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
  6134. zone->free_area[order].nr_free = 0;
  6135. }
  6136. }
  6137. /*
  6138. * Only struct pages that correspond to ranges defined by memblock.memory
  6139. * are zeroed and initialized by going through __init_single_page() during
  6140. * memmap_init_zone_range().
  6141. *
  6142. * But, there could be struct pages that correspond to holes in
  6143. * memblock.memory. This can happen because of the following reasons:
  6144. * - physical memory bank size is not necessarily the exact multiple of the
  6145. * arbitrary section size
  6146. * - early reserved memory may not be listed in memblock.memory
  6147. * - memory layouts defined with memmap= kernel parameter may not align
  6148. * nicely with memmap sections
  6149. *
  6150. * Explicitly initialize those struct pages so that:
  6151. * - PG_Reserved is set
  6152. * - zone and node links point to zone and node that span the page if the
  6153. * hole is in the middle of a zone
  6154. * - zone and node links point to adjacent zone/node if the hole falls on
  6155. * the zone boundary; the pages in such holes will be prepended to the
  6156. * zone/node above the hole except for the trailing pages in the last
  6157. * section that will be appended to the zone/node below.
  6158. */
  6159. static void __init init_unavailable_range(unsigned long spfn,
  6160. unsigned long epfn,
  6161. int zone, int node)
  6162. {
  6163. unsigned long pfn;
  6164. u64 pgcnt = 0;
  6165. for (pfn = spfn; pfn < epfn; pfn++) {
  6166. if (!pfn_valid(pageblock_start_pfn(pfn))) {
  6167. pfn = pageblock_end_pfn(pfn) - 1;
  6168. continue;
  6169. }
  6170. __init_single_page(pfn_to_page(pfn), pfn, zone, node);
  6171. __SetPageReserved(pfn_to_page(pfn));
  6172. pgcnt++;
  6173. }
  6174. if (pgcnt)
  6175. pr_info("On node %d, zone %s: %lld pages in unavailable ranges",
  6176. node, zone_names[zone], pgcnt);
  6177. }
  6178. static void __init memmap_init_zone_range(struct zone *zone,
  6179. unsigned long start_pfn,
  6180. unsigned long end_pfn,
  6181. unsigned long *hole_pfn)
  6182. {
  6183. unsigned long zone_start_pfn = zone->zone_start_pfn;
  6184. unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
  6185. int nid = zone_to_nid(zone), zone_id = zone_idx(zone);
  6186. start_pfn = clamp(start_pfn, zone_start_pfn, zone_end_pfn);
  6187. end_pfn = clamp(end_pfn, zone_start_pfn, zone_end_pfn);
  6188. if (start_pfn >= end_pfn)
  6189. return;
  6190. memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
  6191. zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE);
  6192. if (*hole_pfn < start_pfn)
  6193. init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
  6194. *hole_pfn = end_pfn;
  6195. }
  6196. static void __init memmap_init(void)
  6197. {
  6198. unsigned long start_pfn, end_pfn;
  6199. unsigned long hole_pfn = 0;
  6200. int i, j, zone_id = 0, nid;
  6201. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  6202. struct pglist_data *node = NODE_DATA(nid);
  6203. for (j = 0; j < MAX_NR_ZONES; j++) {
  6204. struct zone *zone = node->node_zones + j;
  6205. if (!populated_zone(zone))
  6206. continue;
  6207. memmap_init_zone_range(zone, start_pfn, end_pfn,
  6208. &hole_pfn);
  6209. zone_id = j;
  6210. }
  6211. }
  6212. #ifdef CONFIG_SPARSEMEM
  6213. /*
  6214. * Initialize the memory map for hole in the range [memory_end,
  6215. * section_end].
  6216. * Append the pages in this hole to the highest zone in the last
  6217. * node.
  6218. * The call to init_unavailable_range() is outside the ifdef to
  6219. * silence the compiler warining about zone_id set but not used;
  6220. * for FLATMEM it is a nop anyway
  6221. */
  6222. end_pfn = round_up(end_pfn, PAGES_PER_SECTION);
  6223. if (hole_pfn < end_pfn)
  6224. #endif
  6225. init_unavailable_range(hole_pfn, end_pfn, zone_id, nid);
  6226. }
  6227. void __init *memmap_alloc(phys_addr_t size, phys_addr_t align,
  6228. phys_addr_t min_addr, int nid, bool exact_nid)
  6229. {
  6230. void *ptr;
  6231. if (exact_nid)
  6232. ptr = memblock_alloc_exact_nid_raw(size, align, min_addr,
  6233. MEMBLOCK_ALLOC_ACCESSIBLE,
  6234. nid);
  6235. else
  6236. ptr = memblock_alloc_try_nid_raw(size, align, min_addr,
  6237. MEMBLOCK_ALLOC_ACCESSIBLE,
  6238. nid);
  6239. if (ptr && size > 0)
  6240. page_init_poison(ptr, size);
  6241. return ptr;
  6242. }
  6243. static int zone_batchsize(struct zone *zone)
  6244. {
  6245. #ifdef CONFIG_MMU
  6246. int batch;
  6247. /*
  6248. * The number of pages to batch allocate is either ~0.1%
  6249. * of the zone or 1MB, whichever is smaller. The batch
  6250. * size is striking a balance between allocation latency
  6251. * and zone lock contention.
  6252. */
  6253. batch = min(zone_managed_pages(zone) >> 10, SZ_1M / PAGE_SIZE);
  6254. batch /= 4; /* We effectively *= 4 below */
  6255. if (batch < 1)
  6256. batch = 1;
  6257. /*
  6258. * Clamp the batch to a 2^n - 1 value. Having a power
  6259. * of 2 value was found to be more likely to have
  6260. * suboptimal cache aliasing properties in some cases.
  6261. *
  6262. * For example if 2 tasks are alternately allocating
  6263. * batches of pages, one task can end up with a lot
  6264. * of pages of one half of the possible page colors
  6265. * and the other with pages of the other colors.
  6266. */
  6267. batch = rounddown_pow_of_two(batch + batch/2) - 1;
  6268. return batch;
  6269. #else
  6270. /* The deferral and batching of frees should be suppressed under NOMMU
  6271. * conditions.
  6272. *
  6273. * The problem is that NOMMU needs to be able to allocate large chunks
  6274. * of contiguous memory as there's no hardware page translation to
  6275. * assemble apparent contiguous memory from discontiguous pages.
  6276. *
  6277. * Queueing large contiguous runs of pages for batching, however,
  6278. * causes the pages to actually be freed in smaller chunks. As there
  6279. * can be a significant delay between the individual batches being
  6280. * recycled, this leads to the once large chunks of space being
  6281. * fragmented and becoming unavailable for high-order allocations.
  6282. */
  6283. return 0;
  6284. #endif
  6285. }
  6286. static int zone_highsize(struct zone *zone, int batch, int cpu_online)
  6287. {
  6288. #ifdef CONFIG_MMU
  6289. int high;
  6290. int nr_split_cpus;
  6291. unsigned long total_pages;
  6292. if (!percpu_pagelist_high_fraction) {
  6293. /*
  6294. * By default, the high value of the pcp is based on the zone
  6295. * low watermark so that if they are full then background
  6296. * reclaim will not be started prematurely.
  6297. */
  6298. total_pages = low_wmark_pages(zone);
  6299. } else {
  6300. /*
  6301. * If percpu_pagelist_high_fraction is configured, the high
  6302. * value is based on a fraction of the managed pages in the
  6303. * zone.
  6304. */
  6305. total_pages = zone_managed_pages(zone) / percpu_pagelist_high_fraction;
  6306. }
  6307. /*
  6308. * Split the high value across all online CPUs local to the zone. Note
  6309. * that early in boot that CPUs may not be online yet and that during
  6310. * CPU hotplug that the cpumask is not yet updated when a CPU is being
  6311. * onlined. For memory nodes that have no CPUs, split pcp->high across
  6312. * all online CPUs to mitigate the risk that reclaim is triggered
  6313. * prematurely due to pages stored on pcp lists.
  6314. */
  6315. nr_split_cpus = cpumask_weight(cpumask_of_node(zone_to_nid(zone))) + cpu_online;
  6316. if (!nr_split_cpus)
  6317. nr_split_cpus = num_online_cpus();
  6318. high = total_pages / nr_split_cpus;
  6319. /*
  6320. * Ensure high is at least batch*4. The multiple is based on the
  6321. * historical relationship between high and batch.
  6322. */
  6323. high = max(high, batch << 2);
  6324. return high;
  6325. #else
  6326. return 0;
  6327. #endif
  6328. }
  6329. /*
  6330. * pcp->high and pcp->batch values are related and generally batch is lower
  6331. * than high. They are also related to pcp->count such that count is lower
  6332. * than high, and as soon as it reaches high, the pcplist is flushed.
  6333. *
  6334. * However, guaranteeing these relations at all times would require e.g. write
  6335. * barriers here but also careful usage of read barriers at the read side, and
  6336. * thus be prone to error and bad for performance. Thus the update only prevents
  6337. * store tearing. Any new users of pcp->batch and pcp->high should ensure they
  6338. * can cope with those fields changing asynchronously, and fully trust only the
  6339. * pcp->count field on the local CPU with interrupts disabled.
  6340. *
  6341. * mutex_is_locked(&pcp_batch_high_lock) required when calling this function
  6342. * outside of boot time (or some other assurance that no concurrent updaters
  6343. * exist).
  6344. */
  6345. static void pageset_update(struct per_cpu_pages *pcp, unsigned long high,
  6346. unsigned long batch)
  6347. {
  6348. WRITE_ONCE(pcp->batch, batch);
  6349. WRITE_ONCE(pcp->high, high);
  6350. }
  6351. static void per_cpu_pages_init(struct per_cpu_pages *pcp, struct per_cpu_zonestat *pzstats)
  6352. {
  6353. int pindex;
  6354. memset(pcp, 0, sizeof(*pcp));
  6355. memset(pzstats, 0, sizeof(*pzstats));
  6356. spin_lock_init(&pcp->lock);
  6357. for (pindex = 0; pindex < NR_PCP_LISTS; pindex++)
  6358. INIT_LIST_HEAD(&pcp->lists[pindex]);
  6359. /*
  6360. * Set batch and high values safe for a boot pageset. A true percpu
  6361. * pageset's initialization will update them subsequently. Here we don't
  6362. * need to be as careful as pageset_update() as nobody can access the
  6363. * pageset yet.
  6364. */
  6365. pcp->high = BOOT_PAGESET_HIGH;
  6366. pcp->batch = BOOT_PAGESET_BATCH;
  6367. pcp->free_factor = 0;
  6368. }
  6369. static void __zone_set_pageset_high_and_batch(struct zone *zone, unsigned long high,
  6370. unsigned long batch)
  6371. {
  6372. struct per_cpu_pages *pcp;
  6373. int cpu;
  6374. for_each_possible_cpu(cpu) {
  6375. pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
  6376. pageset_update(pcp, high, batch);
  6377. }
  6378. }
  6379. /*
  6380. * Calculate and set new high and batch values for all per-cpu pagesets of a
  6381. * zone based on the zone's size.
  6382. */
  6383. static void zone_set_pageset_high_and_batch(struct zone *zone, int cpu_online)
  6384. {
  6385. int new_high, new_batch;
  6386. new_batch = max(1, zone_batchsize(zone));
  6387. new_high = zone_highsize(zone, new_batch, cpu_online);
  6388. if (zone->pageset_high == new_high &&
  6389. zone->pageset_batch == new_batch)
  6390. return;
  6391. zone->pageset_high = new_high;
  6392. zone->pageset_batch = new_batch;
  6393. __zone_set_pageset_high_and_batch(zone, new_high, new_batch);
  6394. }
  6395. void __meminit setup_zone_pageset(struct zone *zone)
  6396. {
  6397. int cpu;
  6398. /* Size may be 0 on !SMP && !NUMA */
  6399. if (sizeof(struct per_cpu_zonestat) > 0)
  6400. zone->per_cpu_zonestats = alloc_percpu(struct per_cpu_zonestat);
  6401. zone->per_cpu_pageset = alloc_percpu(struct per_cpu_pages);
  6402. for_each_possible_cpu(cpu) {
  6403. struct per_cpu_pages *pcp;
  6404. struct per_cpu_zonestat *pzstats;
  6405. pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
  6406. pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
  6407. per_cpu_pages_init(pcp, pzstats);
  6408. }
  6409. zone_set_pageset_high_and_batch(zone, 0);
  6410. }
  6411. /*
  6412. * The zone indicated has a new number of managed_pages; batch sizes and percpu
  6413. * page high values need to be recalculated.
  6414. */
  6415. static void zone_pcp_update(struct zone *zone, int cpu_online)
  6416. {
  6417. mutex_lock(&pcp_batch_high_lock);
  6418. zone_set_pageset_high_and_batch(zone, cpu_online);
  6419. mutex_unlock(&pcp_batch_high_lock);
  6420. }
  6421. /*
  6422. * Allocate per cpu pagesets and initialize them.
  6423. * Before this call only boot pagesets were available.
  6424. */
  6425. void __init setup_per_cpu_pageset(void)
  6426. {
  6427. struct pglist_data *pgdat;
  6428. struct zone *zone;
  6429. int __maybe_unused cpu;
  6430. for_each_populated_zone(zone)
  6431. setup_zone_pageset(zone);
  6432. #ifdef CONFIG_NUMA
  6433. /*
  6434. * Unpopulated zones continue using the boot pagesets.
  6435. * The numa stats for these pagesets need to be reset.
  6436. * Otherwise, they will end up skewing the stats of
  6437. * the nodes these zones are associated with.
  6438. */
  6439. for_each_possible_cpu(cpu) {
  6440. struct per_cpu_zonestat *pzstats = &per_cpu(boot_zonestats, cpu);
  6441. memset(pzstats->vm_numa_event, 0,
  6442. sizeof(pzstats->vm_numa_event));
  6443. }
  6444. #endif
  6445. for_each_online_pgdat(pgdat)
  6446. pgdat->per_cpu_nodestats =
  6447. alloc_percpu(struct per_cpu_nodestat);
  6448. }
  6449. static __meminit void zone_pcp_init(struct zone *zone)
  6450. {
  6451. /*
  6452. * per cpu subsystem is not up at this point. The following code
  6453. * relies on the ability of the linker to provide the
  6454. * offset of a (static) per cpu variable into the per cpu area.
  6455. */
  6456. zone->per_cpu_pageset = &boot_pageset;
  6457. zone->per_cpu_zonestats = &boot_zonestats;
  6458. zone->pageset_high = BOOT_PAGESET_HIGH;
  6459. zone->pageset_batch = BOOT_PAGESET_BATCH;
  6460. if (populated_zone(zone))
  6461. pr_debug(" %s zone: %lu pages, LIFO batch:%u\n", zone->name,
  6462. zone->present_pages, zone_batchsize(zone));
  6463. }
  6464. void __meminit init_currently_empty_zone(struct zone *zone,
  6465. unsigned long zone_start_pfn,
  6466. unsigned long size)
  6467. {
  6468. struct pglist_data *pgdat = zone->zone_pgdat;
  6469. int zone_idx = zone_idx(zone) + 1;
  6470. if (zone_idx > pgdat->nr_zones)
  6471. pgdat->nr_zones = zone_idx;
  6472. zone->zone_start_pfn = zone_start_pfn;
  6473. mminit_dprintk(MMINIT_TRACE, "memmap_init",
  6474. "Initialising map node %d zone %lu pfns %lu -> %lu\n",
  6475. pgdat->node_id,
  6476. (unsigned long)zone_idx(zone),
  6477. zone_start_pfn, (zone_start_pfn + size));
  6478. zone_init_free_lists(zone);
  6479. zone->initialized = 1;
  6480. }
  6481. /**
  6482. * get_pfn_range_for_nid - Return the start and end page frames for a node
  6483. * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
  6484. * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
  6485. * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
  6486. *
  6487. * It returns the start and end page frame of a node based on information
  6488. * provided by memblock_set_node(). If called for a node
  6489. * with no available memory, a warning is printed and the start and end
  6490. * PFNs will be 0.
  6491. */
  6492. void __init get_pfn_range_for_nid(unsigned int nid,
  6493. unsigned long *start_pfn, unsigned long *end_pfn)
  6494. {
  6495. unsigned long this_start_pfn, this_end_pfn;
  6496. int i;
  6497. *start_pfn = -1UL;
  6498. *end_pfn = 0;
  6499. for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
  6500. *start_pfn = min(*start_pfn, this_start_pfn);
  6501. *end_pfn = max(*end_pfn, this_end_pfn);
  6502. }
  6503. if (*start_pfn == -1UL)
  6504. *start_pfn = 0;
  6505. }
  6506. /*
  6507. * This finds a zone that can be used for ZONE_MOVABLE pages. The
  6508. * assumption is made that zones within a node are ordered in monotonic
  6509. * increasing memory addresses so that the "highest" populated zone is used
  6510. */
  6511. static void __init find_usable_zone_for_movable(void)
  6512. {
  6513. int zone_index;
  6514. for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
  6515. if (zone_index == ZONE_MOVABLE)
  6516. continue;
  6517. if (arch_zone_highest_possible_pfn[zone_index] >
  6518. arch_zone_lowest_possible_pfn[zone_index])
  6519. break;
  6520. }
  6521. VM_BUG_ON(zone_index == -1);
  6522. movable_zone = zone_index;
  6523. }
  6524. /*
  6525. * The zone ranges provided by the architecture do not include ZONE_MOVABLE
  6526. * because it is sized independent of architecture. Unlike the other zones,
  6527. * the starting point for ZONE_MOVABLE is not fixed. It may be different
  6528. * in each node depending on the size of each node and how evenly kernelcore
  6529. * is distributed. This helper function adjusts the zone ranges
  6530. * provided by the architecture for a given node by using the end of the
  6531. * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
  6532. * zones within a node are in order of monotonic increases memory addresses
  6533. */
  6534. static void __init adjust_zone_range_for_zone_movable(int nid,
  6535. unsigned long zone_type,
  6536. unsigned long node_start_pfn,
  6537. unsigned long node_end_pfn,
  6538. unsigned long *zone_start_pfn,
  6539. unsigned long *zone_end_pfn)
  6540. {
  6541. /* Only adjust if ZONE_MOVABLE is on this node */
  6542. if (zone_movable_pfn[nid]) {
  6543. /* Size ZONE_MOVABLE */
  6544. if (zone_type == ZONE_MOVABLE) {
  6545. *zone_start_pfn = zone_movable_pfn[nid];
  6546. *zone_end_pfn = min(node_end_pfn,
  6547. arch_zone_highest_possible_pfn[movable_zone]);
  6548. /* Adjust for ZONE_MOVABLE starting within this range */
  6549. } else if (!mirrored_kernelcore &&
  6550. *zone_start_pfn < zone_movable_pfn[nid] &&
  6551. *zone_end_pfn > zone_movable_pfn[nid]) {
  6552. *zone_end_pfn = zone_movable_pfn[nid];
  6553. /* Check if this whole range is within ZONE_MOVABLE */
  6554. } else if (*zone_start_pfn >= zone_movable_pfn[nid])
  6555. *zone_start_pfn = *zone_end_pfn;
  6556. }
  6557. }
  6558. /*
  6559. * Return the number of pages a zone spans in a node, including holes
  6560. * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
  6561. */
  6562. static unsigned long __init zone_spanned_pages_in_node(int nid,
  6563. unsigned long zone_type,
  6564. unsigned long node_start_pfn,
  6565. unsigned long node_end_pfn,
  6566. unsigned long *zone_start_pfn,
  6567. unsigned long *zone_end_pfn)
  6568. {
  6569. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  6570. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  6571. /* When hotadd a new node from cpu_up(), the node should be empty */
  6572. if (!node_start_pfn && !node_end_pfn)
  6573. return 0;
  6574. /* Get the start and end of the zone */
  6575. *zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  6576. *zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  6577. adjust_zone_range_for_zone_movable(nid, zone_type,
  6578. node_start_pfn, node_end_pfn,
  6579. zone_start_pfn, zone_end_pfn);
  6580. /* Check that this node has pages within the zone's required range */
  6581. if (*zone_end_pfn < node_start_pfn || *zone_start_pfn > node_end_pfn)
  6582. return 0;
  6583. /* Move the zone boundaries inside the node if necessary */
  6584. *zone_end_pfn = min(*zone_end_pfn, node_end_pfn);
  6585. *zone_start_pfn = max(*zone_start_pfn, node_start_pfn);
  6586. /* Return the spanned pages */
  6587. return *zone_end_pfn - *zone_start_pfn;
  6588. }
  6589. /*
  6590. * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
  6591. * then all holes in the requested range will be accounted for.
  6592. */
  6593. unsigned long __init __absent_pages_in_range(int nid,
  6594. unsigned long range_start_pfn,
  6595. unsigned long range_end_pfn)
  6596. {
  6597. unsigned long nr_absent = range_end_pfn - range_start_pfn;
  6598. unsigned long start_pfn, end_pfn;
  6599. int i;
  6600. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  6601. start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
  6602. end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
  6603. nr_absent -= end_pfn - start_pfn;
  6604. }
  6605. return nr_absent;
  6606. }
  6607. /**
  6608. * absent_pages_in_range - Return number of page frames in holes within a range
  6609. * @start_pfn: The start PFN to start searching for holes
  6610. * @end_pfn: The end PFN to stop searching for holes
  6611. *
  6612. * Return: the number of pages frames in memory holes within a range.
  6613. */
  6614. unsigned long __init absent_pages_in_range(unsigned long start_pfn,
  6615. unsigned long end_pfn)
  6616. {
  6617. return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
  6618. }
  6619. /* Return the number of page frames in holes in a zone on a node */
  6620. static unsigned long __init zone_absent_pages_in_node(int nid,
  6621. unsigned long zone_type,
  6622. unsigned long node_start_pfn,
  6623. unsigned long node_end_pfn)
  6624. {
  6625. unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
  6626. unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
  6627. unsigned long zone_start_pfn, zone_end_pfn;
  6628. unsigned long nr_absent;
  6629. /* When hotadd a new node from cpu_up(), the node should be empty */
  6630. if (!node_start_pfn && !node_end_pfn)
  6631. return 0;
  6632. zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
  6633. zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
  6634. adjust_zone_range_for_zone_movable(nid, zone_type,
  6635. node_start_pfn, node_end_pfn,
  6636. &zone_start_pfn, &zone_end_pfn);
  6637. nr_absent = __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
  6638. /*
  6639. * ZONE_MOVABLE handling.
  6640. * Treat pages to be ZONE_MOVABLE in ZONE_NORMAL as absent pages
  6641. * and vice versa.
  6642. */
  6643. if (mirrored_kernelcore && zone_movable_pfn[nid]) {
  6644. unsigned long start_pfn, end_pfn;
  6645. struct memblock_region *r;
  6646. for_each_mem_region(r) {
  6647. start_pfn = clamp(memblock_region_memory_base_pfn(r),
  6648. zone_start_pfn, zone_end_pfn);
  6649. end_pfn = clamp(memblock_region_memory_end_pfn(r),
  6650. zone_start_pfn, zone_end_pfn);
  6651. if (zone_type == ZONE_MOVABLE &&
  6652. memblock_is_mirror(r))
  6653. nr_absent += end_pfn - start_pfn;
  6654. if (zone_type == ZONE_NORMAL &&
  6655. !memblock_is_mirror(r))
  6656. nr_absent += end_pfn - start_pfn;
  6657. }
  6658. }
  6659. return nr_absent;
  6660. }
  6661. static void __init calculate_node_totalpages(struct pglist_data *pgdat,
  6662. unsigned long node_start_pfn,
  6663. unsigned long node_end_pfn)
  6664. {
  6665. unsigned long realtotalpages = 0, totalpages = 0;
  6666. enum zone_type i;
  6667. for (i = 0; i < MAX_NR_ZONES; i++) {
  6668. struct zone *zone = pgdat->node_zones + i;
  6669. unsigned long zone_start_pfn, zone_end_pfn;
  6670. unsigned long spanned, absent;
  6671. unsigned long size, real_size;
  6672. spanned = zone_spanned_pages_in_node(pgdat->node_id, i,
  6673. node_start_pfn,
  6674. node_end_pfn,
  6675. &zone_start_pfn,
  6676. &zone_end_pfn);
  6677. absent = zone_absent_pages_in_node(pgdat->node_id, i,
  6678. node_start_pfn,
  6679. node_end_pfn);
  6680. size = spanned;
  6681. real_size = size - absent;
  6682. if (size)
  6683. zone->zone_start_pfn = zone_start_pfn;
  6684. else
  6685. zone->zone_start_pfn = 0;
  6686. zone->spanned_pages = size;
  6687. zone->present_pages = real_size;
  6688. #if defined(CONFIG_MEMORY_HOTPLUG)
  6689. zone->present_early_pages = real_size;
  6690. #endif
  6691. totalpages += size;
  6692. realtotalpages += real_size;
  6693. }
  6694. pgdat->node_spanned_pages = totalpages;
  6695. pgdat->node_present_pages = realtotalpages;
  6696. pr_debug("On node %d totalpages: %lu\n", pgdat->node_id, realtotalpages);
  6697. }
  6698. #ifndef CONFIG_SPARSEMEM
  6699. /*
  6700. * Calculate the size of the zone->blockflags rounded to an unsigned long
  6701. * Start by making sure zonesize is a multiple of pageblock_order by rounding
  6702. * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
  6703. * round what is now in bits to nearest long in bits, then return it in
  6704. * bytes.
  6705. */
  6706. static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize)
  6707. {
  6708. unsigned long usemapsize;
  6709. zonesize += zone_start_pfn & (pageblock_nr_pages-1);
  6710. usemapsize = roundup(zonesize, pageblock_nr_pages);
  6711. usemapsize = usemapsize >> pageblock_order;
  6712. usemapsize *= NR_PAGEBLOCK_BITS;
  6713. usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
  6714. return usemapsize / 8;
  6715. }
  6716. static void __ref setup_usemap(struct zone *zone)
  6717. {
  6718. unsigned long usemapsize = usemap_size(zone->zone_start_pfn,
  6719. zone->spanned_pages);
  6720. zone->pageblock_flags = NULL;
  6721. if (usemapsize) {
  6722. zone->pageblock_flags =
  6723. memblock_alloc_node(usemapsize, SMP_CACHE_BYTES,
  6724. zone_to_nid(zone));
  6725. if (!zone->pageblock_flags)
  6726. panic("Failed to allocate %ld bytes for zone %s pageblock flags on node %d\n",
  6727. usemapsize, zone->name, zone_to_nid(zone));
  6728. }
  6729. }
  6730. #else
  6731. static inline void setup_usemap(struct zone *zone) {}
  6732. #endif /* CONFIG_SPARSEMEM */
  6733. #ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
  6734. /* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
  6735. void __init set_pageblock_order(void)
  6736. {
  6737. unsigned int order = MAX_ORDER - 1;
  6738. /* Check that pageblock_nr_pages has not already been setup */
  6739. if (pageblock_order)
  6740. return;
  6741. /* Don't let pageblocks exceed the maximum allocation granularity. */
  6742. if (HPAGE_SHIFT > PAGE_SHIFT && HUGETLB_PAGE_ORDER < order)
  6743. order = HUGETLB_PAGE_ORDER;
  6744. /*
  6745. * Assume the largest contiguous order of interest is a huge page.
  6746. * This value may be variable depending on boot parameters on IA64 and
  6747. * powerpc.
  6748. */
  6749. pageblock_order = order;
  6750. }
  6751. #else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  6752. /*
  6753. * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
  6754. * is unused as pageblock_order is set at compile-time. See
  6755. * include/linux/pageblock-flags.h for the values of pageblock_order based on
  6756. * the kernel config
  6757. */
  6758. void __init set_pageblock_order(void)
  6759. {
  6760. }
  6761. #endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
  6762. static unsigned long __init calc_memmap_size(unsigned long spanned_pages,
  6763. unsigned long present_pages)
  6764. {
  6765. unsigned long pages = spanned_pages;
  6766. /*
  6767. * Provide a more accurate estimation if there are holes within
  6768. * the zone and SPARSEMEM is in use. If there are holes within the
  6769. * zone, each populated memory region may cost us one or two extra
  6770. * memmap pages due to alignment because memmap pages for each
  6771. * populated regions may not be naturally aligned on page boundary.
  6772. * So the (present_pages >> 4) heuristic is a tradeoff for that.
  6773. */
  6774. if (spanned_pages > present_pages + (present_pages >> 4) &&
  6775. IS_ENABLED(CONFIG_SPARSEMEM))
  6776. pages = present_pages;
  6777. return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT;
  6778. }
  6779. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  6780. static void pgdat_init_split_queue(struct pglist_data *pgdat)
  6781. {
  6782. struct deferred_split *ds_queue = &pgdat->deferred_split_queue;
  6783. spin_lock_init(&ds_queue->split_queue_lock);
  6784. INIT_LIST_HEAD(&ds_queue->split_queue);
  6785. ds_queue->split_queue_len = 0;
  6786. }
  6787. #else
  6788. static void pgdat_init_split_queue(struct pglist_data *pgdat) {}
  6789. #endif
  6790. #ifdef CONFIG_COMPACTION
  6791. static void pgdat_init_kcompactd(struct pglist_data *pgdat)
  6792. {
  6793. init_waitqueue_head(&pgdat->kcompactd_wait);
  6794. }
  6795. #else
  6796. static void pgdat_init_kcompactd(struct pglist_data *pgdat) {}
  6797. #endif
  6798. static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
  6799. {
  6800. int i;
  6801. pgdat_resize_init(pgdat);
  6802. pgdat_kswapd_lock_init(pgdat);
  6803. pgdat_init_split_queue(pgdat);
  6804. pgdat_init_kcompactd(pgdat);
  6805. init_waitqueue_head(&pgdat->kswapd_wait);
  6806. init_waitqueue_head(&pgdat->pfmemalloc_wait);
  6807. for (i = 0; i < NR_VMSCAN_THROTTLE; i++)
  6808. init_waitqueue_head(&pgdat->reclaim_wait[i]);
  6809. pgdat_page_ext_init(pgdat);
  6810. lruvec_init(&pgdat->__lruvec);
  6811. }
  6812. static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
  6813. unsigned long remaining_pages)
  6814. {
  6815. atomic_long_set(&zone->managed_pages, remaining_pages);
  6816. zone_set_nid(zone, nid);
  6817. zone->name = zone_names[idx];
  6818. zone->zone_pgdat = NODE_DATA(nid);
  6819. spin_lock_init(&zone->lock);
  6820. zone_seqlock_init(zone);
  6821. zone_pcp_init(zone);
  6822. }
  6823. /*
  6824. * Set up the zone data structures
  6825. * - init pgdat internals
  6826. * - init all zones belonging to this node
  6827. *
  6828. * NOTE: this function is only called during memory hotplug
  6829. */
  6830. #ifdef CONFIG_MEMORY_HOTPLUG
  6831. void __ref free_area_init_core_hotplug(struct pglist_data *pgdat)
  6832. {
  6833. int nid = pgdat->node_id;
  6834. enum zone_type z;
  6835. int cpu;
  6836. pgdat_init_internals(pgdat);
  6837. if (pgdat->per_cpu_nodestats == &boot_nodestats)
  6838. pgdat->per_cpu_nodestats = alloc_percpu(struct per_cpu_nodestat);
  6839. /*
  6840. * Reset the nr_zones, order and highest_zoneidx before reuse.
  6841. * Note that kswapd will init kswapd_highest_zoneidx properly
  6842. * when it starts in the near future.
  6843. */
  6844. pgdat->nr_zones = 0;
  6845. pgdat->kswapd_order = 0;
  6846. pgdat->kswapd_highest_zoneidx = 0;
  6847. pgdat->node_start_pfn = 0;
  6848. for_each_online_cpu(cpu) {
  6849. struct per_cpu_nodestat *p;
  6850. p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
  6851. memset(p, 0, sizeof(*p));
  6852. }
  6853. for (z = 0; z < MAX_NR_ZONES; z++)
  6854. zone_init_internals(&pgdat->node_zones[z], z, nid, 0);
  6855. }
  6856. #endif
  6857. /*
  6858. * Set up the zone data structures:
  6859. * - mark all pages reserved
  6860. * - mark all memory queues empty
  6861. * - clear the memory bitmaps
  6862. *
  6863. * NOTE: pgdat should get zeroed by caller.
  6864. * NOTE: this function is only called during early init.
  6865. */
  6866. static void __init free_area_init_core(struct pglist_data *pgdat)
  6867. {
  6868. enum zone_type j;
  6869. int nid = pgdat->node_id;
  6870. pgdat_init_internals(pgdat);
  6871. pgdat->per_cpu_nodestats = &boot_nodestats;
  6872. for (j = 0; j < MAX_NR_ZONES; j++) {
  6873. struct zone *zone = pgdat->node_zones + j;
  6874. unsigned long size, freesize, memmap_pages;
  6875. size = zone->spanned_pages;
  6876. freesize = zone->present_pages;
  6877. /*
  6878. * Adjust freesize so that it accounts for how much memory
  6879. * is used by this zone for memmap. This affects the watermark
  6880. * and per-cpu initialisations
  6881. */
  6882. memmap_pages = calc_memmap_size(size, freesize);
  6883. if (!is_highmem_idx(j)) {
  6884. if (freesize >= memmap_pages) {
  6885. freesize -= memmap_pages;
  6886. if (memmap_pages)
  6887. pr_debug(" %s zone: %lu pages used for memmap\n",
  6888. zone_names[j], memmap_pages);
  6889. } else
  6890. pr_warn(" %s zone: %lu memmap pages exceeds freesize %lu\n",
  6891. zone_names[j], memmap_pages, freesize);
  6892. }
  6893. /* Account for reserved pages */
  6894. if (j == 0 && freesize > dma_reserve) {
  6895. freesize -= dma_reserve;
  6896. pr_debug(" %s zone: %lu pages reserved\n", zone_names[0], dma_reserve);
  6897. }
  6898. if (!is_highmem_idx(j))
  6899. nr_kernel_pages += freesize;
  6900. /* Charge for highmem memmap if there are enough kernel pages */
  6901. else if (nr_kernel_pages > memmap_pages * 2)
  6902. nr_kernel_pages -= memmap_pages;
  6903. nr_all_pages += freesize;
  6904. /*
  6905. * Set an approximate value for lowmem here, it will be adjusted
  6906. * when the bootmem allocator frees pages into the buddy system.
  6907. * And all highmem pages will be managed by the buddy system.
  6908. */
  6909. zone_init_internals(zone, j, nid, freesize);
  6910. if (!size)
  6911. continue;
  6912. set_pageblock_order();
  6913. setup_usemap(zone);
  6914. init_currently_empty_zone(zone, zone->zone_start_pfn, size);
  6915. }
  6916. }
  6917. #ifdef CONFIG_FLATMEM
  6918. static void __init alloc_node_mem_map(struct pglist_data *pgdat)
  6919. {
  6920. unsigned long __maybe_unused start = 0;
  6921. unsigned long __maybe_unused offset = 0;
  6922. /* Skip empty nodes */
  6923. if (!pgdat->node_spanned_pages)
  6924. return;
  6925. start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
  6926. offset = pgdat->node_start_pfn - start;
  6927. /* ia64 gets its own node_mem_map, before this, without bootmem */
  6928. if (!pgdat->node_mem_map) {
  6929. unsigned long size, end;
  6930. struct page *map;
  6931. /*
  6932. * The zone's endpoints aren't required to be MAX_ORDER
  6933. * aligned but the node_mem_map endpoints must be in order
  6934. * for the buddy allocator to function correctly.
  6935. */
  6936. end = pgdat_end_pfn(pgdat);
  6937. end = ALIGN(end, MAX_ORDER_NR_PAGES);
  6938. size = (end - start) * sizeof(struct page);
  6939. map = memmap_alloc(size, SMP_CACHE_BYTES, MEMBLOCK_LOW_LIMIT,
  6940. pgdat->node_id, false);
  6941. if (!map)
  6942. panic("Failed to allocate %ld bytes for node %d memory map\n",
  6943. size, pgdat->node_id);
  6944. pgdat->node_mem_map = map + offset;
  6945. }
  6946. pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
  6947. __func__, pgdat->node_id, (unsigned long)pgdat,
  6948. (unsigned long)pgdat->node_mem_map);
  6949. #ifndef CONFIG_NUMA
  6950. /*
  6951. * With no DISCONTIG, the global mem_map is just set as node 0's
  6952. */
  6953. if (pgdat == NODE_DATA(0)) {
  6954. mem_map = NODE_DATA(0)->node_mem_map;
  6955. if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
  6956. mem_map -= offset;
  6957. }
  6958. #endif
  6959. }
  6960. #else
  6961. static inline void alloc_node_mem_map(struct pglist_data *pgdat) { }
  6962. #endif /* CONFIG_FLATMEM */
  6963. #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
  6964. static inline void pgdat_set_deferred_range(pg_data_t *pgdat)
  6965. {
  6966. pgdat->first_deferred_pfn = ULONG_MAX;
  6967. }
  6968. #else
  6969. static inline void pgdat_set_deferred_range(pg_data_t *pgdat) {}
  6970. #endif
  6971. static void __init free_area_init_node(int nid)
  6972. {
  6973. pg_data_t *pgdat = NODE_DATA(nid);
  6974. unsigned long start_pfn = 0;
  6975. unsigned long end_pfn = 0;
  6976. /* pg_data_t should be reset to zero when it's allocated */
  6977. WARN_ON(pgdat->nr_zones || pgdat->kswapd_highest_zoneidx);
  6978. get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
  6979. pgdat->node_id = nid;
  6980. pgdat->node_start_pfn = start_pfn;
  6981. pgdat->per_cpu_nodestats = NULL;
  6982. if (start_pfn != end_pfn) {
  6983. pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid,
  6984. (u64)start_pfn << PAGE_SHIFT,
  6985. end_pfn ? ((u64)end_pfn << PAGE_SHIFT) - 1 : 0);
  6986. } else {
  6987. pr_info("Initmem setup node %d as memoryless\n", nid);
  6988. }
  6989. calculate_node_totalpages(pgdat, start_pfn, end_pfn);
  6990. alloc_node_mem_map(pgdat);
  6991. pgdat_set_deferred_range(pgdat);
  6992. free_area_init_core(pgdat);
  6993. lru_gen_init_pgdat(pgdat);
  6994. }
  6995. static void __init free_area_init_memoryless_node(int nid)
  6996. {
  6997. free_area_init_node(nid);
  6998. }
  6999. #if MAX_NUMNODES > 1
  7000. /*
  7001. * Figure out the number of possible node ids.
  7002. */
  7003. void __init setup_nr_node_ids(void)
  7004. {
  7005. unsigned int highest;
  7006. highest = find_last_bit(node_possible_map.bits, MAX_NUMNODES);
  7007. nr_node_ids = highest + 1;
  7008. }
  7009. #endif
  7010. /**
  7011. * node_map_pfn_alignment - determine the maximum internode alignment
  7012. *
  7013. * This function should be called after node map is populated and sorted.
  7014. * It calculates the maximum power of two alignment which can distinguish
  7015. * all the nodes.
  7016. *
  7017. * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
  7018. * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
  7019. * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
  7020. * shifted, 1GiB is enough and this function will indicate so.
  7021. *
  7022. * This is used to test whether pfn -> nid mapping of the chosen memory
  7023. * model has fine enough granularity to avoid incorrect mapping for the
  7024. * populated node map.
  7025. *
  7026. * Return: the determined alignment in pfn's. 0 if there is no alignment
  7027. * requirement (single node).
  7028. */
  7029. unsigned long __init node_map_pfn_alignment(void)
  7030. {
  7031. unsigned long accl_mask = 0, last_end = 0;
  7032. unsigned long start, end, mask;
  7033. int last_nid = NUMA_NO_NODE;
  7034. int i, nid;
  7035. for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
  7036. if (!start || last_nid < 0 || last_nid == nid) {
  7037. last_nid = nid;
  7038. last_end = end;
  7039. continue;
  7040. }
  7041. /*
  7042. * Start with a mask granular enough to pin-point to the
  7043. * start pfn and tick off bits one-by-one until it becomes
  7044. * too coarse to separate the current node from the last.
  7045. */
  7046. mask = ~((1 << __ffs(start)) - 1);
  7047. while (mask && last_end <= (start & (mask << 1)))
  7048. mask <<= 1;
  7049. /* accumulate all internode masks */
  7050. accl_mask |= mask;
  7051. }
  7052. /* convert mask to number of pages */
  7053. return ~accl_mask + 1;
  7054. }
  7055. /*
  7056. * early_calculate_totalpages()
  7057. * Sum pages in active regions for movable zone.
  7058. * Populate N_MEMORY for calculating usable_nodes.
  7059. */
  7060. static unsigned long __init early_calculate_totalpages(void)
  7061. {
  7062. unsigned long totalpages = 0;
  7063. unsigned long start_pfn, end_pfn;
  7064. int i, nid;
  7065. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  7066. unsigned long pages = end_pfn - start_pfn;
  7067. totalpages += pages;
  7068. if (pages)
  7069. node_set_state(nid, N_MEMORY);
  7070. }
  7071. return totalpages;
  7072. }
  7073. /*
  7074. * Find the PFN the Movable zone begins in each node. Kernel memory
  7075. * is spread evenly between nodes as long as the nodes have enough
  7076. * memory. When they don't, some nodes will have more kernelcore than
  7077. * others
  7078. */
  7079. static void __init find_zone_movable_pfns_for_nodes(void)
  7080. {
  7081. int i, nid;
  7082. unsigned long usable_startpfn;
  7083. unsigned long kernelcore_node, kernelcore_remaining;
  7084. /* save the state before borrow the nodemask */
  7085. nodemask_t saved_node_state = node_states[N_MEMORY];
  7086. unsigned long totalpages = early_calculate_totalpages();
  7087. int usable_nodes = nodes_weight(node_states[N_MEMORY]);
  7088. struct memblock_region *r;
  7089. /* Need to find movable_zone earlier when movable_node is specified. */
  7090. find_usable_zone_for_movable();
  7091. /*
  7092. * If movable_node is specified, ignore kernelcore and movablecore
  7093. * options.
  7094. */
  7095. if (movable_node_is_enabled()) {
  7096. for_each_mem_region(r) {
  7097. if (!memblock_is_hotpluggable(r))
  7098. continue;
  7099. nid = memblock_get_region_node(r);
  7100. usable_startpfn = PFN_DOWN(r->base);
  7101. zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
  7102. min(usable_startpfn, zone_movable_pfn[nid]) :
  7103. usable_startpfn;
  7104. }
  7105. goto out2;
  7106. }
  7107. /*
  7108. * If kernelcore=mirror is specified, ignore movablecore option
  7109. */
  7110. if (mirrored_kernelcore) {
  7111. bool mem_below_4gb_not_mirrored = false;
  7112. for_each_mem_region(r) {
  7113. if (memblock_is_mirror(r))
  7114. continue;
  7115. nid = memblock_get_region_node(r);
  7116. usable_startpfn = memblock_region_memory_base_pfn(r);
  7117. if (usable_startpfn < PHYS_PFN(SZ_4G)) {
  7118. mem_below_4gb_not_mirrored = true;
  7119. continue;
  7120. }
  7121. zone_movable_pfn[nid] = zone_movable_pfn[nid] ?
  7122. min(usable_startpfn, zone_movable_pfn[nid]) :
  7123. usable_startpfn;
  7124. }
  7125. if (mem_below_4gb_not_mirrored)
  7126. pr_warn("This configuration results in unmirrored kernel memory.\n");
  7127. goto out2;
  7128. }
  7129. /*
  7130. * If kernelcore=nn% or movablecore=nn% was specified, calculate the
  7131. * amount of necessary memory.
  7132. */
  7133. if (required_kernelcore_percent)
  7134. required_kernelcore = (totalpages * 100 * required_kernelcore_percent) /
  7135. 10000UL;
  7136. if (required_movablecore_percent)
  7137. required_movablecore = (totalpages * 100 * required_movablecore_percent) /
  7138. 10000UL;
  7139. /*
  7140. * If movablecore= was specified, calculate what size of
  7141. * kernelcore that corresponds so that memory usable for
  7142. * any allocation type is evenly spread. If both kernelcore
  7143. * and movablecore are specified, then the value of kernelcore
  7144. * will be used for required_kernelcore if it's greater than
  7145. * what movablecore would have allowed.
  7146. */
  7147. if (required_movablecore) {
  7148. unsigned long corepages;
  7149. /*
  7150. * Round-up so that ZONE_MOVABLE is at least as large as what
  7151. * was requested by the user
  7152. */
  7153. required_movablecore =
  7154. roundup(required_movablecore, MAX_ORDER_NR_PAGES);
  7155. required_movablecore = min(totalpages, required_movablecore);
  7156. corepages = totalpages - required_movablecore;
  7157. required_kernelcore = max(required_kernelcore, corepages);
  7158. }
  7159. /*
  7160. * If kernelcore was not specified or kernelcore size is larger
  7161. * than totalpages, there is no ZONE_MOVABLE.
  7162. */
  7163. if (!required_kernelcore || required_kernelcore >= totalpages)
  7164. goto out;
  7165. /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
  7166. usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
  7167. restart:
  7168. /* Spread kernelcore memory as evenly as possible throughout nodes */
  7169. kernelcore_node = required_kernelcore / usable_nodes;
  7170. for_each_node_state(nid, N_MEMORY) {
  7171. unsigned long start_pfn, end_pfn;
  7172. /*
  7173. * Recalculate kernelcore_node if the division per node
  7174. * now exceeds what is necessary to satisfy the requested
  7175. * amount of memory for the kernel
  7176. */
  7177. if (required_kernelcore < kernelcore_node)
  7178. kernelcore_node = required_kernelcore / usable_nodes;
  7179. /*
  7180. * As the map is walked, we track how much memory is usable
  7181. * by the kernel using kernelcore_remaining. When it is
  7182. * 0, the rest of the node is usable by ZONE_MOVABLE
  7183. */
  7184. kernelcore_remaining = kernelcore_node;
  7185. /* Go through each range of PFNs within this node */
  7186. for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
  7187. unsigned long size_pages;
  7188. start_pfn = max(start_pfn, zone_movable_pfn[nid]);
  7189. if (start_pfn >= end_pfn)
  7190. continue;
  7191. /* Account for what is only usable for kernelcore */
  7192. if (start_pfn < usable_startpfn) {
  7193. unsigned long kernel_pages;
  7194. kernel_pages = min(end_pfn, usable_startpfn)
  7195. - start_pfn;
  7196. kernelcore_remaining -= min(kernel_pages,
  7197. kernelcore_remaining);
  7198. required_kernelcore -= min(kernel_pages,
  7199. required_kernelcore);
  7200. /* Continue if range is now fully accounted */
  7201. if (end_pfn <= usable_startpfn) {
  7202. /*
  7203. * Push zone_movable_pfn to the end so
  7204. * that if we have to rebalance
  7205. * kernelcore across nodes, we will
  7206. * not double account here
  7207. */
  7208. zone_movable_pfn[nid] = end_pfn;
  7209. continue;
  7210. }
  7211. start_pfn = usable_startpfn;
  7212. }
  7213. /*
  7214. * The usable PFN range for ZONE_MOVABLE is from
  7215. * start_pfn->end_pfn. Calculate size_pages as the
  7216. * number of pages used as kernelcore
  7217. */
  7218. size_pages = end_pfn - start_pfn;
  7219. if (size_pages > kernelcore_remaining)
  7220. size_pages = kernelcore_remaining;
  7221. zone_movable_pfn[nid] = start_pfn + size_pages;
  7222. /*
  7223. * Some kernelcore has been met, update counts and
  7224. * break if the kernelcore for this node has been
  7225. * satisfied
  7226. */
  7227. required_kernelcore -= min(required_kernelcore,
  7228. size_pages);
  7229. kernelcore_remaining -= size_pages;
  7230. if (!kernelcore_remaining)
  7231. break;
  7232. }
  7233. }
  7234. /*
  7235. * If there is still required_kernelcore, we do another pass with one
  7236. * less node in the count. This will push zone_movable_pfn[nid] further
  7237. * along on the nodes that still have memory until kernelcore is
  7238. * satisfied
  7239. */
  7240. usable_nodes--;
  7241. if (usable_nodes && required_kernelcore > usable_nodes)
  7242. goto restart;
  7243. out2:
  7244. /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
  7245. for (nid = 0; nid < MAX_NUMNODES; nid++) {
  7246. unsigned long start_pfn, end_pfn;
  7247. zone_movable_pfn[nid] =
  7248. roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
  7249. get_pfn_range_for_nid(nid, &start_pfn, &end_pfn);
  7250. if (zone_movable_pfn[nid] >= end_pfn)
  7251. zone_movable_pfn[nid] = 0;
  7252. }
  7253. out:
  7254. /* restore the node_state */
  7255. node_states[N_MEMORY] = saved_node_state;
  7256. }
  7257. /* Any regular or high memory on that node ? */
  7258. static void check_for_memory(pg_data_t *pgdat, int nid)
  7259. {
  7260. enum zone_type zone_type;
  7261. for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) {
  7262. struct zone *zone = &pgdat->node_zones[zone_type];
  7263. if (populated_zone(zone)) {
  7264. if (IS_ENABLED(CONFIG_HIGHMEM))
  7265. node_set_state(nid, N_HIGH_MEMORY);
  7266. if (zone_type <= ZONE_NORMAL)
  7267. node_set_state(nid, N_NORMAL_MEMORY);
  7268. break;
  7269. }
  7270. }
  7271. }
  7272. /*
  7273. * Some architectures, e.g. ARC may have ZONE_HIGHMEM below ZONE_NORMAL. For
  7274. * such cases we allow max_zone_pfn sorted in the descending order
  7275. */
  7276. bool __weak arch_has_descending_max_zone_pfns(void)
  7277. {
  7278. return false;
  7279. }
  7280. /**
  7281. * free_area_init - Initialise all pg_data_t and zone data
  7282. * @max_zone_pfn: an array of max PFNs for each zone
  7283. *
  7284. * This will call free_area_init_node() for each active node in the system.
  7285. * Using the page ranges provided by memblock_set_node(), the size of each
  7286. * zone in each node and their holes is calculated. If the maximum PFN
  7287. * between two adjacent zones match, it is assumed that the zone is empty.
  7288. * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
  7289. * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
  7290. * starts where the previous one ended. For example, ZONE_DMA32 starts
  7291. * at arch_max_dma_pfn.
  7292. */
  7293. void __init free_area_init(unsigned long *max_zone_pfn)
  7294. {
  7295. unsigned long start_pfn, end_pfn;
  7296. int i, nid, zone;
  7297. bool descending;
  7298. /* Record where the zone boundaries are */
  7299. memset(arch_zone_lowest_possible_pfn, 0,
  7300. sizeof(arch_zone_lowest_possible_pfn));
  7301. memset(arch_zone_highest_possible_pfn, 0,
  7302. sizeof(arch_zone_highest_possible_pfn));
  7303. start_pfn = PHYS_PFN(memblock_start_of_DRAM());
  7304. descending = arch_has_descending_max_zone_pfns();
  7305. for (i = 0; i < MAX_NR_ZONES; i++) {
  7306. if (descending)
  7307. zone = MAX_NR_ZONES - i - 1;
  7308. else
  7309. zone = i;
  7310. if (zone == ZONE_MOVABLE)
  7311. continue;
  7312. end_pfn = max(max_zone_pfn[zone], start_pfn);
  7313. arch_zone_lowest_possible_pfn[zone] = start_pfn;
  7314. arch_zone_highest_possible_pfn[zone] = end_pfn;
  7315. start_pfn = end_pfn;
  7316. }
  7317. /* Find the PFNs that ZONE_MOVABLE begins at in each node */
  7318. memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
  7319. find_zone_movable_pfns_for_nodes();
  7320. /* Print out the zone ranges */
  7321. pr_info("Zone ranges:\n");
  7322. for (i = 0; i < MAX_NR_ZONES; i++) {
  7323. if (i == ZONE_MOVABLE)
  7324. continue;
  7325. pr_info(" %-8s ", zone_names[i]);
  7326. if (arch_zone_lowest_possible_pfn[i] ==
  7327. arch_zone_highest_possible_pfn[i])
  7328. pr_cont("empty\n");
  7329. else
  7330. pr_cont("[mem %#018Lx-%#018Lx]\n",
  7331. (u64)arch_zone_lowest_possible_pfn[i]
  7332. << PAGE_SHIFT,
  7333. ((u64)arch_zone_highest_possible_pfn[i]
  7334. << PAGE_SHIFT) - 1);
  7335. }
  7336. /* Print out the PFNs ZONE_MOVABLE begins at in each node */
  7337. pr_info("Movable zone start for each node\n");
  7338. for (i = 0; i < MAX_NUMNODES; i++) {
  7339. if (zone_movable_pfn[i])
  7340. pr_info(" Node %d: %#018Lx\n", i,
  7341. (u64)zone_movable_pfn[i] << PAGE_SHIFT);
  7342. }
  7343. /*
  7344. * Print out the early node map, and initialize the
  7345. * subsection-map relative to active online memory ranges to
  7346. * enable future "sub-section" extensions of the memory map.
  7347. */
  7348. pr_info("Early memory node ranges\n");
  7349. for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
  7350. pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid,
  7351. (u64)start_pfn << PAGE_SHIFT,
  7352. ((u64)end_pfn << PAGE_SHIFT) - 1);
  7353. subsection_map_init(start_pfn, end_pfn - start_pfn);
  7354. }
  7355. /* Initialise every node */
  7356. mminit_verify_pageflags_layout();
  7357. setup_nr_node_ids();
  7358. for_each_node(nid) {
  7359. pg_data_t *pgdat;
  7360. if (!node_online(nid)) {
  7361. pr_info("Initializing node %d as memoryless\n", nid);
  7362. /* Allocator not initialized yet */
  7363. pgdat = arch_alloc_nodedata(nid);
  7364. if (!pgdat) {
  7365. pr_err("Cannot allocate %zuB for node %d.\n",
  7366. sizeof(*pgdat), nid);
  7367. continue;
  7368. }
  7369. arch_refresh_nodedata(nid, pgdat);
  7370. free_area_init_memoryless_node(nid);
  7371. /*
  7372. * We do not want to confuse userspace by sysfs
  7373. * files/directories for node without any memory
  7374. * attached to it, so this node is not marked as
  7375. * N_MEMORY and not marked online so that no sysfs
  7376. * hierarchy will be created via register_one_node for
  7377. * it. The pgdat will get fully initialized by
  7378. * hotadd_init_pgdat() when memory is hotplugged into
  7379. * this node.
  7380. */
  7381. continue;
  7382. }
  7383. pgdat = NODE_DATA(nid);
  7384. free_area_init_node(nid);
  7385. /* Any memory on that node */
  7386. if (pgdat->node_present_pages)
  7387. node_set_state(nid, N_MEMORY);
  7388. check_for_memory(pgdat, nid);
  7389. }
  7390. memmap_init();
  7391. }
  7392. static int __init cmdline_parse_core(char *p, unsigned long *core,
  7393. unsigned long *percent)
  7394. {
  7395. unsigned long long coremem;
  7396. char *endptr;
  7397. if (!p)
  7398. return -EINVAL;
  7399. /* Value may be a percentage of total memory, otherwise bytes */
  7400. coremem = simple_strtoull(p, &endptr, 0);
  7401. if (*endptr == '%') {
  7402. /* Paranoid check for percent values greater than 100 */
  7403. WARN_ON(coremem > 100);
  7404. *percent = coremem;
  7405. } else {
  7406. coremem = memparse(p, &p);
  7407. /* Paranoid check that UL is enough for the coremem value */
  7408. WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
  7409. *core = coremem >> PAGE_SHIFT;
  7410. *percent = 0UL;
  7411. }
  7412. return 0;
  7413. }
  7414. /*
  7415. * kernelcore=size sets the amount of memory for use for allocations that
  7416. * cannot be reclaimed or migrated.
  7417. */
  7418. static int __init cmdline_parse_kernelcore(char *p)
  7419. {
  7420. /* parse kernelcore=mirror */
  7421. if (parse_option_str(p, "mirror")) {
  7422. mirrored_kernelcore = true;
  7423. return 0;
  7424. }
  7425. return cmdline_parse_core(p, &required_kernelcore,
  7426. &required_kernelcore_percent);
  7427. }
  7428. /*
  7429. * movablecore=size sets the amount of memory for use for allocations that
  7430. * can be reclaimed or migrated.
  7431. */
  7432. static int __init cmdline_parse_movablecore(char *p)
  7433. {
  7434. return cmdline_parse_core(p, &required_movablecore,
  7435. &required_movablecore_percent);
  7436. }
  7437. early_param("kernelcore", cmdline_parse_kernelcore);
  7438. early_param("movablecore", cmdline_parse_movablecore);
  7439. void adjust_managed_page_count(struct page *page, long count)
  7440. {
  7441. atomic_long_add(count, &page_zone(page)->managed_pages);
  7442. totalram_pages_add(count);
  7443. #ifdef CONFIG_HIGHMEM
  7444. if (PageHighMem(page))
  7445. totalhigh_pages_add(count);
  7446. #endif
  7447. }
  7448. EXPORT_SYMBOL(adjust_managed_page_count);
  7449. unsigned long free_reserved_area(void *start, void *end, int poison, const char *s)
  7450. {
  7451. void *pos;
  7452. unsigned long pages = 0;
  7453. start = (void *)PAGE_ALIGN((unsigned long)start);
  7454. end = (void *)((unsigned long)end & PAGE_MASK);
  7455. for (pos = start; pos < end; pos += PAGE_SIZE, pages++) {
  7456. struct page *page = virt_to_page(pos);
  7457. void *direct_map_addr;
  7458. /*
  7459. * 'direct_map_addr' might be different from 'pos'
  7460. * because some architectures' virt_to_page()
  7461. * work with aliases. Getting the direct map
  7462. * address ensures that we get a _writeable_
  7463. * alias for the memset().
  7464. */
  7465. direct_map_addr = page_address(page);
  7466. /*
  7467. * Perform a kasan-unchecked memset() since this memory
  7468. * has not been initialized.
  7469. */
  7470. direct_map_addr = kasan_reset_tag(direct_map_addr);
  7471. if ((unsigned int)poison <= 0xFF)
  7472. memset(direct_map_addr, poison, PAGE_SIZE);
  7473. free_reserved_page(page);
  7474. }
  7475. if (pages && s)
  7476. pr_info("Freeing %s memory: %ldK\n", s, K(pages));
  7477. return pages;
  7478. }
  7479. void __init mem_init_print_info(void)
  7480. {
  7481. unsigned long physpages, codesize, datasize, rosize, bss_size;
  7482. unsigned long init_code_size, init_data_size;
  7483. physpages = get_num_physpages();
  7484. codesize = _etext - _stext;
  7485. datasize = _edata - _sdata;
  7486. rosize = __end_rodata - __start_rodata;
  7487. bss_size = __bss_stop - __bss_start;
  7488. init_data_size = __init_end - __init_begin;
  7489. init_code_size = _einittext - _sinittext;
  7490. /*
  7491. * Detect special cases and adjust section sizes accordingly:
  7492. * 1) .init.* may be embedded into .data sections
  7493. * 2) .init.text.* may be out of [__init_begin, __init_end],
  7494. * please refer to arch/tile/kernel/vmlinux.lds.S.
  7495. * 3) .rodata.* may be embedded into .text or .data sections.
  7496. */
  7497. #define adj_init_size(start, end, size, pos, adj) \
  7498. do { \
  7499. if (&start[0] <= &pos[0] && &pos[0] < &end[0] && size > adj) \
  7500. size -= adj; \
  7501. } while (0)
  7502. adj_init_size(__init_begin, __init_end, init_data_size,
  7503. _sinittext, init_code_size);
  7504. adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size);
  7505. adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size);
  7506. adj_init_size(_stext, _etext, codesize, __start_rodata, rosize);
  7507. adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize);
  7508. #undef adj_init_size
  7509. pr_info("Memory: %luK/%luK available (%luK kernel code, %luK rwdata, %luK rodata, %luK init, %luK bss, %luK reserved, %luK cma-reserved"
  7510. #ifdef CONFIG_HIGHMEM
  7511. ", %luK highmem"
  7512. #endif
  7513. ")\n",
  7514. K(nr_free_pages()), K(physpages),
  7515. codesize / SZ_1K, datasize / SZ_1K, rosize / SZ_1K,
  7516. (init_data_size + init_code_size) / SZ_1K, bss_size / SZ_1K,
  7517. K(physpages - totalram_pages() - totalcma_pages),
  7518. K(totalcma_pages)
  7519. #ifdef CONFIG_HIGHMEM
  7520. , K(totalhigh_pages())
  7521. #endif
  7522. );
  7523. }
  7524. /**
  7525. * set_dma_reserve - set the specified number of pages reserved in the first zone
  7526. * @new_dma_reserve: The number of pages to mark reserved
  7527. *
  7528. * The per-cpu batchsize and zone watermarks are determined by managed_pages.
  7529. * In the DMA zone, a significant percentage may be consumed by kernel image
  7530. * and other unfreeable allocations which can skew the watermarks badly. This
  7531. * function may optionally be used to account for unfreeable pages in the
  7532. * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
  7533. * smaller per-cpu batchsize.
  7534. */
  7535. void __init set_dma_reserve(unsigned long new_dma_reserve)
  7536. {
  7537. dma_reserve = new_dma_reserve;
  7538. }
  7539. static int page_alloc_cpu_dead(unsigned int cpu)
  7540. {
  7541. struct zone *zone;
  7542. lru_add_drain_cpu(cpu);
  7543. mlock_page_drain_remote(cpu);
  7544. drain_pages(cpu);
  7545. /*
  7546. * Spill the event counters of the dead processor
  7547. * into the current processors event counters.
  7548. * This artificially elevates the count of the current
  7549. * processor.
  7550. */
  7551. vm_events_fold_cpu(cpu);
  7552. /*
  7553. * Zero the differential counters of the dead processor
  7554. * so that the vm statistics are consistent.
  7555. *
  7556. * This is only okay since the processor is dead and cannot
  7557. * race with what we are doing.
  7558. */
  7559. cpu_vm_stats_fold(cpu);
  7560. for_each_populated_zone(zone)
  7561. zone_pcp_update(zone, 0);
  7562. return 0;
  7563. }
  7564. static int page_alloc_cpu_online(unsigned int cpu)
  7565. {
  7566. struct zone *zone;
  7567. for_each_populated_zone(zone)
  7568. zone_pcp_update(zone, 1);
  7569. return 0;
  7570. }
  7571. #ifdef CONFIG_NUMA
  7572. int hashdist = HASHDIST_DEFAULT;
  7573. static int __init set_hashdist(char *str)
  7574. {
  7575. if (!str)
  7576. return 0;
  7577. hashdist = simple_strtoul(str, &str, 0);
  7578. return 1;
  7579. }
  7580. __setup("hashdist=", set_hashdist);
  7581. #endif
  7582. void __init page_alloc_init(void)
  7583. {
  7584. int ret;
  7585. #ifdef CONFIG_NUMA
  7586. if (num_node_state(N_MEMORY) == 1)
  7587. hashdist = 0;
  7588. #endif
  7589. ret = cpuhp_setup_state_nocalls(CPUHP_PAGE_ALLOC,
  7590. "mm/page_alloc:pcp",
  7591. page_alloc_cpu_online,
  7592. page_alloc_cpu_dead);
  7593. WARN_ON(ret < 0);
  7594. }
  7595. /*
  7596. * calculate_totalreserve_pages - called when sysctl_lowmem_reserve_ratio
  7597. * or min_free_kbytes changes.
  7598. */
  7599. static void calculate_totalreserve_pages(void)
  7600. {
  7601. struct pglist_data *pgdat;
  7602. unsigned long reserve_pages = 0;
  7603. enum zone_type i, j;
  7604. for_each_online_pgdat(pgdat) {
  7605. pgdat->totalreserve_pages = 0;
  7606. for (i = 0; i < MAX_NR_ZONES; i++) {
  7607. struct zone *zone = pgdat->node_zones + i;
  7608. long max = 0;
  7609. unsigned long managed_pages = zone_managed_pages(zone);
  7610. /* Find valid and maximum lowmem_reserve in the zone */
  7611. for (j = i; j < MAX_NR_ZONES; j++) {
  7612. if (zone->lowmem_reserve[j] > max)
  7613. max = zone->lowmem_reserve[j];
  7614. }
  7615. /* we treat the high watermark as reserved pages. */
  7616. max += high_wmark_pages(zone);
  7617. if (max > managed_pages)
  7618. max = managed_pages;
  7619. pgdat->totalreserve_pages += max;
  7620. reserve_pages += max;
  7621. }
  7622. }
  7623. totalreserve_pages = reserve_pages;
  7624. }
  7625. /*
  7626. * setup_per_zone_lowmem_reserve - called whenever
  7627. * sysctl_lowmem_reserve_ratio changes. Ensures that each zone
  7628. * has a correct pages reserved value, so an adequate number of
  7629. * pages are left in the zone after a successful __alloc_pages().
  7630. */
  7631. static void setup_per_zone_lowmem_reserve(void)
  7632. {
  7633. struct pglist_data *pgdat;
  7634. enum zone_type i, j;
  7635. for_each_online_pgdat(pgdat) {
  7636. for (i = 0; i < MAX_NR_ZONES - 1; i++) {
  7637. struct zone *zone = &pgdat->node_zones[i];
  7638. int ratio = sysctl_lowmem_reserve_ratio[i];
  7639. bool clear = !ratio || !zone_managed_pages(zone);
  7640. unsigned long managed_pages = 0;
  7641. for (j = i + 1; j < MAX_NR_ZONES; j++) {
  7642. struct zone *upper_zone = &pgdat->node_zones[j];
  7643. managed_pages += zone_managed_pages(upper_zone);
  7644. if (clear)
  7645. zone->lowmem_reserve[j] = 0;
  7646. else
  7647. zone->lowmem_reserve[j] = managed_pages / ratio;
  7648. }
  7649. }
  7650. }
  7651. /* update totalreserve_pages */
  7652. calculate_totalreserve_pages();
  7653. }
  7654. static void __setup_per_zone_wmarks(void)
  7655. {
  7656. unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
  7657. unsigned long lowmem_pages = 0;
  7658. struct zone *zone;
  7659. unsigned long flags;
  7660. /* Calculate total number of !ZONE_HIGHMEM pages */
  7661. for_each_zone(zone) {
  7662. if (!is_highmem(zone))
  7663. lowmem_pages += zone_managed_pages(zone);
  7664. }
  7665. for_each_zone(zone) {
  7666. u64 tmp;
  7667. spin_lock_irqsave(&zone->lock, flags);
  7668. tmp = (u64)pages_min * zone_managed_pages(zone);
  7669. do_div(tmp, lowmem_pages);
  7670. if (is_highmem(zone)) {
  7671. /*
  7672. * __GFP_HIGH and PF_MEMALLOC allocations usually don't
  7673. * need highmem pages, so cap pages_min to a small
  7674. * value here.
  7675. *
  7676. * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
  7677. * deltas control async page reclaim, and so should
  7678. * not be capped for highmem.
  7679. */
  7680. unsigned long min_pages;
  7681. min_pages = zone_managed_pages(zone) / 1024;
  7682. min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL);
  7683. zone->_watermark[WMARK_MIN] = min_pages;
  7684. } else {
  7685. /*
  7686. * If it's a lowmem zone, reserve a number of pages
  7687. * proportionate to the zone's size.
  7688. */
  7689. zone->_watermark[WMARK_MIN] = tmp;
  7690. }
  7691. /*
  7692. * Set the kswapd watermarks distance according to the
  7693. * scale factor in proportion to available memory, but
  7694. * ensure a minimum size on small systems.
  7695. */
  7696. tmp = max_t(u64, tmp >> 2,
  7697. mult_frac(zone_managed_pages(zone),
  7698. watermark_scale_factor, 10000));
  7699. zone->watermark_boost = 0;
  7700. zone->_watermark[WMARK_LOW] = min_wmark_pages(zone) + tmp;
  7701. zone->_watermark[WMARK_HIGH] = low_wmark_pages(zone) + tmp;
  7702. zone->_watermark[WMARK_PROMO] = high_wmark_pages(zone) + tmp;
  7703. spin_unlock_irqrestore(&zone->lock, flags);
  7704. }
  7705. /* update totalreserve_pages */
  7706. calculate_totalreserve_pages();
  7707. }
  7708. /**
  7709. * setup_per_zone_wmarks - called when min_free_kbytes changes
  7710. * or when memory is hot-{added|removed}
  7711. *
  7712. * Ensures that the watermark[min,low,high] values for each zone are set
  7713. * correctly with respect to min_free_kbytes.
  7714. */
  7715. void setup_per_zone_wmarks(void)
  7716. {
  7717. struct zone *zone;
  7718. static DEFINE_SPINLOCK(lock);
  7719. spin_lock(&lock);
  7720. __setup_per_zone_wmarks();
  7721. spin_unlock(&lock);
  7722. /*
  7723. * The watermark size have changed so update the pcpu batch
  7724. * and high limits or the limits may be inappropriate.
  7725. */
  7726. for_each_zone(zone)
  7727. zone_pcp_update(zone, 0);
  7728. }
  7729. /*
  7730. * Initialise min_free_kbytes.
  7731. *
  7732. * For small machines we want it small (128k min). For large machines
  7733. * we want it large (256MB max). But it is not linear, because network
  7734. * bandwidth does not increase linearly with machine size. We use
  7735. *
  7736. * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
  7737. * min_free_kbytes = sqrt(lowmem_kbytes * 16)
  7738. *
  7739. * which yields
  7740. *
  7741. * 16MB: 512k
  7742. * 32MB: 724k
  7743. * 64MB: 1024k
  7744. * 128MB: 1448k
  7745. * 256MB: 2048k
  7746. * 512MB: 2896k
  7747. * 1024MB: 4096k
  7748. * 2048MB: 5792k
  7749. * 4096MB: 8192k
  7750. * 8192MB: 11584k
  7751. * 16384MB: 16384k
  7752. */
  7753. void calculate_min_free_kbytes(void)
  7754. {
  7755. unsigned long lowmem_kbytes;
  7756. int new_min_free_kbytes;
  7757. lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
  7758. new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
  7759. if (new_min_free_kbytes > user_min_free_kbytes)
  7760. min_free_kbytes = clamp(new_min_free_kbytes, 128, 262144);
  7761. else
  7762. pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n",
  7763. new_min_free_kbytes, user_min_free_kbytes);
  7764. }
  7765. int __meminit init_per_zone_wmark_min(void)
  7766. {
  7767. calculate_min_free_kbytes();
  7768. setup_per_zone_wmarks();
  7769. refresh_zone_stat_thresholds();
  7770. setup_per_zone_lowmem_reserve();
  7771. #ifdef CONFIG_NUMA
  7772. setup_min_unmapped_ratio();
  7773. setup_min_slab_ratio();
  7774. #endif
  7775. khugepaged_min_free_kbytes_update();
  7776. return 0;
  7777. }
  7778. postcore_initcall(init_per_zone_wmark_min)
  7779. /*
  7780. * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
  7781. * that we can call two helper functions whenever min_free_kbytes
  7782. * changes.
  7783. */
  7784. int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write,
  7785. void *buffer, size_t *length, loff_t *ppos)
  7786. {
  7787. int rc;
  7788. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7789. if (rc)
  7790. return rc;
  7791. if (write) {
  7792. user_min_free_kbytes = min_free_kbytes;
  7793. setup_per_zone_wmarks();
  7794. }
  7795. return 0;
  7796. }
  7797. int watermark_scale_factor_sysctl_handler(struct ctl_table *table, int write,
  7798. void *buffer, size_t *length, loff_t *ppos)
  7799. {
  7800. int rc;
  7801. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7802. if (rc)
  7803. return rc;
  7804. if (write)
  7805. setup_per_zone_wmarks();
  7806. return 0;
  7807. }
  7808. #ifdef CONFIG_NUMA
  7809. static void setup_min_unmapped_ratio(void)
  7810. {
  7811. pg_data_t *pgdat;
  7812. struct zone *zone;
  7813. for_each_online_pgdat(pgdat)
  7814. pgdat->min_unmapped_pages = 0;
  7815. for_each_zone(zone)
  7816. zone->zone_pgdat->min_unmapped_pages += (zone_managed_pages(zone) *
  7817. sysctl_min_unmapped_ratio) / 100;
  7818. }
  7819. int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write,
  7820. void *buffer, size_t *length, loff_t *ppos)
  7821. {
  7822. int rc;
  7823. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7824. if (rc)
  7825. return rc;
  7826. setup_min_unmapped_ratio();
  7827. return 0;
  7828. }
  7829. static void setup_min_slab_ratio(void)
  7830. {
  7831. pg_data_t *pgdat;
  7832. struct zone *zone;
  7833. for_each_online_pgdat(pgdat)
  7834. pgdat->min_slab_pages = 0;
  7835. for_each_zone(zone)
  7836. zone->zone_pgdat->min_slab_pages += (zone_managed_pages(zone) *
  7837. sysctl_min_slab_ratio) / 100;
  7838. }
  7839. int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write,
  7840. void *buffer, size_t *length, loff_t *ppos)
  7841. {
  7842. int rc;
  7843. rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7844. if (rc)
  7845. return rc;
  7846. setup_min_slab_ratio();
  7847. return 0;
  7848. }
  7849. #endif
  7850. /*
  7851. * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
  7852. * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
  7853. * whenever sysctl_lowmem_reserve_ratio changes.
  7854. *
  7855. * The reserve ratio obviously has absolutely no relation with the
  7856. * minimum watermarks. The lowmem reserve ratio can only make sense
  7857. * if in function of the boot time zone sizes.
  7858. */
  7859. int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write,
  7860. void *buffer, size_t *length, loff_t *ppos)
  7861. {
  7862. int i;
  7863. proc_dointvec_minmax(table, write, buffer, length, ppos);
  7864. for (i = 0; i < MAX_NR_ZONES; i++) {
  7865. if (sysctl_lowmem_reserve_ratio[i] < 1)
  7866. sysctl_lowmem_reserve_ratio[i] = 0;
  7867. }
  7868. setup_per_zone_lowmem_reserve();
  7869. return 0;
  7870. }
  7871. /*
  7872. * percpu_pagelist_high_fraction - changes the pcp->high for each zone on each
  7873. * cpu. It is the fraction of total pages in each zone that a hot per cpu
  7874. * pagelist can have before it gets flushed back to buddy allocator.
  7875. */
  7876. int percpu_pagelist_high_fraction_sysctl_handler(struct ctl_table *table,
  7877. int write, void *buffer, size_t *length, loff_t *ppos)
  7878. {
  7879. struct zone *zone;
  7880. int old_percpu_pagelist_high_fraction;
  7881. int ret;
  7882. mutex_lock(&pcp_batch_high_lock);
  7883. old_percpu_pagelist_high_fraction = percpu_pagelist_high_fraction;
  7884. ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
  7885. if (!write || ret < 0)
  7886. goto out;
  7887. /* Sanity checking to avoid pcp imbalance */
  7888. if (percpu_pagelist_high_fraction &&
  7889. percpu_pagelist_high_fraction < MIN_PERCPU_PAGELIST_HIGH_FRACTION) {
  7890. percpu_pagelist_high_fraction = old_percpu_pagelist_high_fraction;
  7891. ret = -EINVAL;
  7892. goto out;
  7893. }
  7894. /* No change? */
  7895. if (percpu_pagelist_high_fraction == old_percpu_pagelist_high_fraction)
  7896. goto out;
  7897. for_each_populated_zone(zone)
  7898. zone_set_pageset_high_and_batch(zone, 0);
  7899. out:
  7900. mutex_unlock(&pcp_batch_high_lock);
  7901. return ret;
  7902. }
  7903. #ifndef __HAVE_ARCH_RESERVED_KERNEL_PAGES
  7904. /*
  7905. * Returns the number of pages that arch has reserved but
  7906. * is not known to alloc_large_system_hash().
  7907. */
  7908. static unsigned long __init arch_reserved_kernel_pages(void)
  7909. {
  7910. return 0;
  7911. }
  7912. #endif
  7913. /*
  7914. * Adaptive scale is meant to reduce sizes of hash tables on large memory
  7915. * machines. As memory size is increased the scale is also increased but at
  7916. * slower pace. Starting from ADAPT_SCALE_BASE (64G), every time memory
  7917. * quadruples the scale is increased by one, which means the size of hash table
  7918. * only doubles, instead of quadrupling as well.
  7919. * Because 32-bit systems cannot have large physical memory, where this scaling
  7920. * makes sense, it is disabled on such platforms.
  7921. */
  7922. #if __BITS_PER_LONG > 32
  7923. #define ADAPT_SCALE_BASE (64ul << 30)
  7924. #define ADAPT_SCALE_SHIFT 2
  7925. #define ADAPT_SCALE_NPAGES (ADAPT_SCALE_BASE >> PAGE_SHIFT)
  7926. #endif
  7927. /*
  7928. * allocate a large system hash table from bootmem
  7929. * - it is assumed that the hash table must contain an exact power-of-2
  7930. * quantity of entries
  7931. * - limit is the number of hash buckets, not the total allocation size
  7932. */
  7933. void *__init alloc_large_system_hash(const char *tablename,
  7934. unsigned long bucketsize,
  7935. unsigned long numentries,
  7936. int scale,
  7937. int flags,
  7938. unsigned int *_hash_shift,
  7939. unsigned int *_hash_mask,
  7940. unsigned long low_limit,
  7941. unsigned long high_limit)
  7942. {
  7943. unsigned long long max = high_limit;
  7944. unsigned long log2qty, size;
  7945. void *table;
  7946. gfp_t gfp_flags;
  7947. bool virt;
  7948. bool huge;
  7949. /* allow the kernel cmdline to have a say */
  7950. if (!numentries) {
  7951. /* round applicable memory size up to nearest megabyte */
  7952. numentries = nr_kernel_pages;
  7953. numentries -= arch_reserved_kernel_pages();
  7954. /* It isn't necessary when PAGE_SIZE >= 1MB */
  7955. if (PAGE_SIZE < SZ_1M)
  7956. numentries = round_up(numentries, SZ_1M / PAGE_SIZE);
  7957. #if __BITS_PER_LONG > 32
  7958. if (!high_limit) {
  7959. unsigned long adapt;
  7960. for (adapt = ADAPT_SCALE_NPAGES; adapt < numentries;
  7961. adapt <<= ADAPT_SCALE_SHIFT)
  7962. scale++;
  7963. }
  7964. #endif
  7965. /* limit to 1 bucket per 2^scale bytes of low memory */
  7966. if (scale > PAGE_SHIFT)
  7967. numentries >>= (scale - PAGE_SHIFT);
  7968. else
  7969. numentries <<= (PAGE_SHIFT - scale);
  7970. /* Make sure we've got at least a 0-order allocation.. */
  7971. if (unlikely(flags & HASH_SMALL)) {
  7972. /* Makes no sense without HASH_EARLY */
  7973. WARN_ON(!(flags & HASH_EARLY));
  7974. if (!(numentries >> *_hash_shift)) {
  7975. numentries = 1UL << *_hash_shift;
  7976. BUG_ON(!numentries);
  7977. }
  7978. } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
  7979. numentries = PAGE_SIZE / bucketsize;
  7980. }
  7981. numentries = roundup_pow_of_two(numentries);
  7982. /* limit allocation size to 1/16 total memory by default */
  7983. if (max == 0) {
  7984. max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
  7985. do_div(max, bucketsize);
  7986. }
  7987. max = min(max, 0x80000000ULL);
  7988. if (numentries < low_limit)
  7989. numentries = low_limit;
  7990. if (numentries > max)
  7991. numentries = max;
  7992. log2qty = ilog2(numentries);
  7993. gfp_flags = (flags & HASH_ZERO) ? GFP_ATOMIC | __GFP_ZERO : GFP_ATOMIC;
  7994. do {
  7995. virt = false;
  7996. size = bucketsize << log2qty;
  7997. if (flags & HASH_EARLY) {
  7998. if (flags & HASH_ZERO)
  7999. table = memblock_alloc(size, SMP_CACHE_BYTES);
  8000. else
  8001. table = memblock_alloc_raw(size,
  8002. SMP_CACHE_BYTES);
  8003. } else if (get_order(size) >= MAX_ORDER || hashdist) {
  8004. table = vmalloc_huge(size, gfp_flags);
  8005. virt = true;
  8006. if (table)
  8007. huge = is_vm_area_hugepages(table);
  8008. } else {
  8009. /*
  8010. * If bucketsize is not a power-of-two, we may free
  8011. * some pages at the end of hash table which
  8012. * alloc_pages_exact() automatically does
  8013. */
  8014. table = alloc_pages_exact(size, gfp_flags);
  8015. kmemleak_alloc(table, size, 1, gfp_flags);
  8016. }
  8017. } while (!table && size > PAGE_SIZE && --log2qty);
  8018. if (!table)
  8019. panic("Failed to allocate %s hash table\n", tablename);
  8020. pr_info("%s hash table entries: %ld (order: %d, %lu bytes, %s)\n",
  8021. tablename, 1UL << log2qty, ilog2(size) - PAGE_SHIFT, size,
  8022. virt ? (huge ? "vmalloc hugepage" : "vmalloc") : "linear");
  8023. if (_hash_shift)
  8024. *_hash_shift = log2qty;
  8025. if (_hash_mask)
  8026. *_hash_mask = (1 << log2qty) - 1;
  8027. return table;
  8028. }
  8029. #ifdef CONFIG_CONTIG_ALLOC
  8030. #if defined(CONFIG_DYNAMIC_DEBUG) || \
  8031. (defined(CONFIG_DYNAMIC_DEBUG_CORE) && defined(DYNAMIC_DEBUG_MODULE))
  8032. /* Usage: See admin-guide/dynamic-debug-howto.rst */
  8033. static void alloc_contig_dump_pages(struct list_head *page_list)
  8034. {
  8035. DEFINE_DYNAMIC_DEBUG_METADATA(descriptor, "migrate failure");
  8036. if (DYNAMIC_DEBUG_BRANCH(descriptor)) {
  8037. struct page *page;
  8038. dump_stack();
  8039. list_for_each_entry(page, page_list, lru)
  8040. dump_page(page, "migration failure");
  8041. }
  8042. }
  8043. #else
  8044. static inline void alloc_contig_dump_pages(struct list_head *page_list)
  8045. {
  8046. }
  8047. #endif
  8048. /*
  8049. * [start, end) must belong to a single zone.
  8050. * @migratetype: using migratetype to filter the type of migration in
  8051. * trace_mm_alloc_contig_migrate_range_info.
  8052. */
  8053. int __alloc_contig_migrate_range(struct compact_control *cc,
  8054. unsigned long start, unsigned long end,
  8055. int migratetype)
  8056. {
  8057. /* This function is based on compact_zone() from compaction.c. */
  8058. unsigned int nr_reclaimed;
  8059. unsigned long pfn = start;
  8060. unsigned int tries = 0;
  8061. unsigned int max_tries = 5;
  8062. int ret = 0;
  8063. struct migration_target_control mtc = {
  8064. .nid = zone_to_nid(cc->zone),
  8065. .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_RETRY_MAYFAIL,
  8066. };
  8067. struct page *page;
  8068. unsigned long total_mapped = 0;
  8069. unsigned long total_migrated = 0;
  8070. unsigned long total_reclaimed = 0;
  8071. if (cc->gfp_mask & __GFP_NORETRY)
  8072. max_tries = 1;
  8073. lru_cache_disable();
  8074. while (pfn < end || !list_empty(&cc->migratepages)) {
  8075. if (fatal_signal_pending(current)) {
  8076. ret = -EINTR;
  8077. break;
  8078. }
  8079. if (list_empty(&cc->migratepages)) {
  8080. cc->nr_migratepages = 0;
  8081. ret = isolate_migratepages_range(cc, pfn, end);
  8082. if (ret && ret != -EAGAIN)
  8083. break;
  8084. pfn = cc->migrate_pfn;
  8085. tries = 0;
  8086. } else if (++tries == max_tries) {
  8087. ret = -EBUSY;
  8088. break;
  8089. }
  8090. nr_reclaimed = reclaim_clean_pages_from_list(cc->zone,
  8091. &cc->migratepages);
  8092. cc->nr_migratepages -= nr_reclaimed;
  8093. if (trace_mm_alloc_contig_migrate_range_info_enabled()) {
  8094. total_reclaimed += nr_reclaimed;
  8095. list_for_each_entry(page, &cc->migratepages, lru)
  8096. total_mapped += page_mapcount(page);
  8097. }
  8098. ret = migrate_pages(&cc->migratepages, alloc_migration_target,
  8099. NULL, (unsigned long)&mtc, cc->mode, MR_CONTIG_RANGE, NULL);
  8100. if (trace_mm_alloc_contig_migrate_range_info_enabled() && !ret)
  8101. total_migrated += cc->nr_migratepages;
  8102. /*
  8103. * On -ENOMEM, migrate_pages() bails out right away. It is pointless
  8104. * to retry again over this error, so do the same here.
  8105. */
  8106. if (ret == -ENOMEM)
  8107. break;
  8108. }
  8109. lru_cache_enable();
  8110. if (ret < 0) {
  8111. if (!(cc->gfp_mask & __GFP_NOWARN) && ret == -EBUSY) {
  8112. struct page *page;
  8113. alloc_contig_dump_pages(&cc->migratepages);
  8114. list_for_each_entry(page, &cc->migratepages, lru) {
  8115. /* The page will be freed by putback_movable_pages soon */
  8116. if (page_count(page) == 1)
  8117. continue;
  8118. page_pinner_failure_detect(page);
  8119. }
  8120. }
  8121. putback_movable_pages(&cc->migratepages);
  8122. }
  8123. trace_mm_alloc_contig_migrate_range_info(start, end, migratetype,
  8124. total_migrated,
  8125. total_reclaimed,
  8126. total_mapped);
  8127. return (ret < 0) ? ret : 0;
  8128. }
  8129. /**
  8130. * alloc_contig_range() -- tries to allocate given range of pages
  8131. * @start: start PFN to allocate
  8132. * @end: one-past-the-last PFN to allocate
  8133. * @migratetype: migratetype of the underlying pageblocks (either
  8134. * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
  8135. * in range must have the same migratetype and it must
  8136. * be either of the two.
  8137. * @gfp_mask: GFP mask to use during compaction
  8138. *
  8139. * The PFN range does not have to be pageblock aligned. The PFN range must
  8140. * belong to a single zone.
  8141. *
  8142. * The first thing this routine does is attempt to MIGRATE_ISOLATE all
  8143. * pageblocks in the range. Once isolated, the pageblocks should not
  8144. * be modified by others.
  8145. *
  8146. * Return: zero on success or negative error code. On success all
  8147. * pages which PFN is in [start, end) are allocated for the caller and
  8148. * need to be freed with free_contig_range().
  8149. */
  8150. int alloc_contig_range(unsigned long start, unsigned long end,
  8151. unsigned migratetype, gfp_t gfp_mask)
  8152. {
  8153. unsigned long outer_start, outer_end;
  8154. int order;
  8155. int ret = 0;
  8156. struct compact_control cc = {
  8157. .nr_migratepages = 0,
  8158. .order = -1,
  8159. .zone = page_zone(pfn_to_page(start)),
  8160. /*
  8161. * Use MIGRATE_ASYNC for __GFP_NORETRY requests as it never
  8162. * blocks.
  8163. */
  8164. .mode = gfp_mask & __GFP_NORETRY ? MIGRATE_ASYNC : MIGRATE_SYNC,
  8165. .ignore_skip_hint = true,
  8166. .no_set_skip_hint = true,
  8167. .gfp_mask = current_gfp_context(gfp_mask),
  8168. .alloc_contig = true,
  8169. };
  8170. INIT_LIST_HEAD(&cc.migratepages);
  8171. /*
  8172. * What we do here is we mark all pageblocks in range as
  8173. * MIGRATE_ISOLATE. Because pageblock and max order pages may
  8174. * have different sizes, and due to the way page allocator
  8175. * work, start_isolate_page_range() has special handlings for this.
  8176. *
  8177. * Once the pageblocks are marked as MIGRATE_ISOLATE, we
  8178. * migrate the pages from an unaligned range (ie. pages that
  8179. * we are interested in). This will put all the pages in
  8180. * range back to page allocator as MIGRATE_ISOLATE.
  8181. *
  8182. * When this is done, we take the pages in range from page
  8183. * allocator removing them from the buddy system. This way
  8184. * page allocator will never consider using them.
  8185. *
  8186. * This lets us mark the pageblocks back as
  8187. * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
  8188. * aligned range but not in the unaligned, original range are
  8189. * put back to page allocator so that buddy can use them.
  8190. */
  8191. ret = start_isolate_page_range(start, end, migratetype, 0, gfp_mask);
  8192. if (ret)
  8193. goto done;
  8194. drain_all_pages(cc.zone);
  8195. /*
  8196. * In case of -EBUSY, we'd like to know which page causes problem.
  8197. * So, just fall through. test_pages_isolated() has a tracepoint
  8198. * which will report the busy page.
  8199. *
  8200. * It is possible that busy pages could become available before
  8201. * the call to test_pages_isolated, and the range will actually be
  8202. * allocated. So, if we fall through be sure to clear ret so that
  8203. * -EBUSY is not accidentally used or returned to caller.
  8204. */
  8205. ret = __alloc_contig_migrate_range(&cc, start, end, migratetype);
  8206. if (ret && (ret != -EBUSY || (gfp_mask & __GFP_NORETRY)))
  8207. goto done;
  8208. ret = 0;
  8209. /*
  8210. * Pages from [start, end) are within a pageblock_nr_pages
  8211. * aligned blocks that are marked as MIGRATE_ISOLATE. What's
  8212. * more, all pages in [start, end) are free in page allocator.
  8213. * What we are going to do is to allocate all pages from
  8214. * [start, end) (that is remove them from page allocator).
  8215. *
  8216. * The only problem is that pages at the beginning and at the
  8217. * end of interesting range may be not aligned with pages that
  8218. * page allocator holds, ie. they can be part of higher order
  8219. * pages. Because of this, we reserve the bigger range and
  8220. * once this is done free the pages we are not interested in.
  8221. *
  8222. * We don't have to hold zone->lock here because the pages are
  8223. * isolated thus they won't get removed from buddy.
  8224. */
  8225. order = 0;
  8226. outer_start = start;
  8227. while (!PageBuddy(pfn_to_page(outer_start))) {
  8228. if (++order >= MAX_ORDER) {
  8229. outer_start = start;
  8230. break;
  8231. }
  8232. outer_start &= ~0UL << order;
  8233. }
  8234. if (outer_start != start) {
  8235. order = buddy_order(pfn_to_page(outer_start));
  8236. /*
  8237. * outer_start page could be small order buddy page and
  8238. * it doesn't include start page. Adjust outer_start
  8239. * in this case to report failed page properly
  8240. * on tracepoint in test_pages_isolated()
  8241. */
  8242. if (outer_start + (1UL << order) <= start)
  8243. outer_start = start;
  8244. }
  8245. /* Make sure the range is really isolated. */
  8246. if (test_pages_isolated(outer_start, end, 0)) {
  8247. ret = -EBUSY;
  8248. goto done;
  8249. }
  8250. /* Grab isolated pages from freelists. */
  8251. outer_end = isolate_freepages_range(&cc, outer_start, end);
  8252. if (!outer_end) {
  8253. ret = -EBUSY;
  8254. goto done;
  8255. }
  8256. /* Free head and tail (if any) */
  8257. if (start != outer_start)
  8258. free_contig_range(outer_start, start - outer_start);
  8259. if (end != outer_end)
  8260. free_contig_range(end, outer_end - end);
  8261. done:
  8262. undo_isolate_page_range(start, end, migratetype);
  8263. return ret;
  8264. }
  8265. EXPORT_SYMBOL(alloc_contig_range);
  8266. static int __alloc_contig_pages(unsigned long start_pfn,
  8267. unsigned long nr_pages, gfp_t gfp_mask)
  8268. {
  8269. unsigned long end_pfn = start_pfn + nr_pages;
  8270. return alloc_contig_range(start_pfn, end_pfn, MIGRATE_MOVABLE,
  8271. gfp_mask);
  8272. }
  8273. static bool pfn_range_valid_contig(struct zone *z, unsigned long start_pfn,
  8274. unsigned long nr_pages)
  8275. {
  8276. unsigned long i, end_pfn = start_pfn + nr_pages;
  8277. struct page *page;
  8278. for (i = start_pfn; i < end_pfn; i++) {
  8279. page = pfn_to_online_page(i);
  8280. if (!page)
  8281. return false;
  8282. if (page_zone(page) != z)
  8283. return false;
  8284. if (PageReserved(page))
  8285. return false;
  8286. if (PageHuge(page))
  8287. return false;
  8288. }
  8289. return true;
  8290. }
  8291. static bool zone_spans_last_pfn(const struct zone *zone,
  8292. unsigned long start_pfn, unsigned long nr_pages)
  8293. {
  8294. unsigned long last_pfn = start_pfn + nr_pages - 1;
  8295. return zone_spans_pfn(zone, last_pfn);
  8296. }
  8297. /**
  8298. * alloc_contig_pages() -- tries to find and allocate contiguous range of pages
  8299. * @nr_pages: Number of contiguous pages to allocate
  8300. * @gfp_mask: GFP mask to limit search and used during compaction
  8301. * @nid: Target node
  8302. * @nodemask: Mask for other possible nodes
  8303. *
  8304. * This routine is a wrapper around alloc_contig_range(). It scans over zones
  8305. * on an applicable zonelist to find a contiguous pfn range which can then be
  8306. * tried for allocation with alloc_contig_range(). This routine is intended
  8307. * for allocation requests which can not be fulfilled with the buddy allocator.
  8308. *
  8309. * The allocated memory is always aligned to a page boundary. If nr_pages is a
  8310. * power of two, then allocated range is also guaranteed to be aligned to same
  8311. * nr_pages (e.g. 1GB request would be aligned to 1GB).
  8312. *
  8313. * Allocated pages can be freed with free_contig_range() or by manually calling
  8314. * __free_page() on each allocated page.
  8315. *
  8316. * Return: pointer to contiguous pages on success, or NULL if not successful.
  8317. */
  8318. struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
  8319. int nid, nodemask_t *nodemask)
  8320. {
  8321. unsigned long ret, pfn, flags;
  8322. struct zonelist *zonelist;
  8323. struct zone *zone;
  8324. struct zoneref *z;
  8325. zonelist = node_zonelist(nid, gfp_mask);
  8326. for_each_zone_zonelist_nodemask(zone, z, zonelist,
  8327. gfp_zone(gfp_mask), nodemask) {
  8328. spin_lock_irqsave(&zone->lock, flags);
  8329. pfn = ALIGN(zone->zone_start_pfn, nr_pages);
  8330. while (zone_spans_last_pfn(zone, pfn, nr_pages)) {
  8331. if (pfn_range_valid_contig(zone, pfn, nr_pages)) {
  8332. /*
  8333. * We release the zone lock here because
  8334. * alloc_contig_range() will also lock the zone
  8335. * at some point. If there's an allocation
  8336. * spinning on this lock, it may win the race
  8337. * and cause alloc_contig_range() to fail...
  8338. */
  8339. spin_unlock_irqrestore(&zone->lock, flags);
  8340. ret = __alloc_contig_pages(pfn, nr_pages,
  8341. gfp_mask);
  8342. if (!ret)
  8343. return pfn_to_page(pfn);
  8344. spin_lock_irqsave(&zone->lock, flags);
  8345. }
  8346. pfn += nr_pages;
  8347. }
  8348. spin_unlock_irqrestore(&zone->lock, flags);
  8349. }
  8350. return NULL;
  8351. }
  8352. #endif /* CONFIG_CONTIG_ALLOC */
  8353. void free_contig_range(unsigned long pfn, unsigned long nr_pages)
  8354. {
  8355. unsigned long count = 0;
  8356. for (; nr_pages--; pfn++) {
  8357. struct page *page = pfn_to_page(pfn);
  8358. count += page_count(page) != 1;
  8359. __free_page(page);
  8360. }
  8361. WARN(count != 0, "%lu pages are still in use!\n", count);
  8362. }
  8363. EXPORT_SYMBOL(free_contig_range);
  8364. /*
  8365. * Effectively disable pcplists for the zone by setting the high limit to 0
  8366. * and draining all cpus. A concurrent page freeing on another CPU that's about
  8367. * to put the page on pcplist will either finish before the drain and the page
  8368. * will be drained, or observe the new high limit and skip the pcplist.
  8369. *
  8370. * Must be paired with a call to zone_pcp_enable().
  8371. */
  8372. void zone_pcp_disable(struct zone *zone)
  8373. {
  8374. mutex_lock(&pcp_batch_high_lock);
  8375. __zone_set_pageset_high_and_batch(zone, 0, 1);
  8376. __drain_all_pages(zone, true);
  8377. }
  8378. void zone_pcp_enable(struct zone *zone)
  8379. {
  8380. __zone_set_pageset_high_and_batch(zone, zone->pageset_high, zone->pageset_batch);
  8381. mutex_unlock(&pcp_batch_high_lock);
  8382. }
  8383. void zone_pcp_reset(struct zone *zone)
  8384. {
  8385. int cpu;
  8386. struct per_cpu_zonestat *pzstats;
  8387. if (zone->per_cpu_pageset != &boot_pageset) {
  8388. for_each_online_cpu(cpu) {
  8389. pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
  8390. drain_zonestat(zone, pzstats);
  8391. }
  8392. free_percpu(zone->per_cpu_pageset);
  8393. zone->per_cpu_pageset = &boot_pageset;
  8394. if (zone->per_cpu_zonestats != &boot_zonestats) {
  8395. free_percpu(zone->per_cpu_zonestats);
  8396. zone->per_cpu_zonestats = &boot_zonestats;
  8397. }
  8398. }
  8399. }
  8400. #ifdef CONFIG_MEMORY_HOTREMOVE
  8401. /*
  8402. * All pages in the range must be in a single zone, must not contain holes,
  8403. * must span full sections, and must be isolated before calling this function.
  8404. */
  8405. void __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
  8406. {
  8407. unsigned long pfn = start_pfn;
  8408. struct page *page;
  8409. struct zone *zone;
  8410. unsigned int order;
  8411. unsigned long flags;
  8412. offline_mem_sections(pfn, end_pfn);
  8413. zone = page_zone(pfn_to_page(pfn));
  8414. spin_lock_irqsave(&zone->lock, flags);
  8415. while (pfn < end_pfn) {
  8416. page = pfn_to_page(pfn);
  8417. /*
  8418. * The HWPoisoned page may be not in buddy system, and
  8419. * page_count() is not 0.
  8420. */
  8421. if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
  8422. pfn++;
  8423. continue;
  8424. }
  8425. /*
  8426. * At this point all remaining PageOffline() pages have a
  8427. * reference count of 0 and can simply be skipped.
  8428. */
  8429. if (PageOffline(page)) {
  8430. BUG_ON(page_count(page));
  8431. BUG_ON(PageBuddy(page));
  8432. pfn++;
  8433. continue;
  8434. }
  8435. BUG_ON(page_count(page));
  8436. BUG_ON(!PageBuddy(page));
  8437. order = buddy_order(page);
  8438. del_page_from_free_list(page, zone, order);
  8439. pfn += (1 << order);
  8440. }
  8441. spin_unlock_irqrestore(&zone->lock, flags);
  8442. }
  8443. #endif
  8444. /*
  8445. * This function returns a stable result only if called under zone lock.
  8446. */
  8447. bool is_free_buddy_page(struct page *page)
  8448. {
  8449. unsigned long pfn = page_to_pfn(page);
  8450. unsigned int order;
  8451. for (order = 0; order < MAX_ORDER; order++) {
  8452. struct page *page_head = page - (pfn & ((1 << order) - 1));
  8453. if (PageBuddy(page_head) &&
  8454. buddy_order_unsafe(page_head) >= order)
  8455. break;
  8456. }
  8457. return order < MAX_ORDER;
  8458. }
  8459. EXPORT_SYMBOL(is_free_buddy_page);
  8460. #ifdef CONFIG_MEMORY_FAILURE
  8461. /*
  8462. * Break down a higher-order page in sub-pages, and keep our target out of
  8463. * buddy allocator.
  8464. */
  8465. static void break_down_buddy_pages(struct zone *zone, struct page *page,
  8466. struct page *target, int low, int high,
  8467. int migratetype)
  8468. {
  8469. unsigned long size = 1 << high;
  8470. struct page *current_buddy, *next_page;
  8471. while (high > low) {
  8472. high--;
  8473. size >>= 1;
  8474. if (target >= &page[size]) {
  8475. next_page = page + size;
  8476. current_buddy = page;
  8477. } else {
  8478. next_page = page;
  8479. current_buddy = page + size;
  8480. }
  8481. page = next_page;
  8482. if (set_page_guard(zone, current_buddy, high, migratetype))
  8483. continue;
  8484. if (current_buddy != target) {
  8485. add_to_free_list(current_buddy, zone, high, migratetype);
  8486. set_buddy_order(current_buddy, high);
  8487. }
  8488. }
  8489. }
  8490. /*
  8491. * Take a page that will be marked as poisoned off the buddy allocator.
  8492. */
  8493. bool take_page_off_buddy(struct page *page)
  8494. {
  8495. struct zone *zone = page_zone(page);
  8496. unsigned long pfn = page_to_pfn(page);
  8497. unsigned long flags;
  8498. unsigned int order;
  8499. bool ret = false;
  8500. spin_lock_irqsave(&zone->lock, flags);
  8501. for (order = 0; order < MAX_ORDER; order++) {
  8502. struct page *page_head = page - (pfn & ((1 << order) - 1));
  8503. int page_order = buddy_order(page_head);
  8504. if (PageBuddy(page_head) && page_order >= order) {
  8505. unsigned long pfn_head = page_to_pfn(page_head);
  8506. int migratetype = get_pfnblock_migratetype(page_head,
  8507. pfn_head);
  8508. del_page_from_free_list(page_head, zone, page_order);
  8509. break_down_buddy_pages(zone, page_head, page, 0,
  8510. page_order, migratetype);
  8511. SetPageHWPoisonTakenOff(page);
  8512. if (!is_migrate_isolate(migratetype))
  8513. __mod_zone_freepage_state(zone, -1, migratetype);
  8514. ret = true;
  8515. break;
  8516. }
  8517. if (page_count(page_head) > 0)
  8518. break;
  8519. }
  8520. spin_unlock_irqrestore(&zone->lock, flags);
  8521. return ret;
  8522. }
  8523. /*
  8524. * Cancel takeoff done by take_page_off_buddy().
  8525. */
  8526. bool put_page_back_buddy(struct page *page)
  8527. {
  8528. struct zone *zone = page_zone(page);
  8529. unsigned long pfn = page_to_pfn(page);
  8530. unsigned long flags;
  8531. int migratetype = get_pfnblock_migratetype(page, pfn);
  8532. bool ret = false;
  8533. spin_lock_irqsave(&zone->lock, flags);
  8534. if (put_page_testzero(page)) {
  8535. ClearPageHWPoisonTakenOff(page);
  8536. __free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);
  8537. if (TestClearPageHWPoison(page)) {
  8538. ret = true;
  8539. }
  8540. }
  8541. spin_unlock_irqrestore(&zone->lock, flags);
  8542. return ret;
  8543. }
  8544. #endif
  8545. #ifdef CONFIG_ZONE_DMA
  8546. bool has_managed_dma(void)
  8547. {
  8548. struct pglist_data *pgdat;
  8549. for_each_online_pgdat(pgdat) {
  8550. struct zone *zone = &pgdat->node_zones[ZONE_DMA];
  8551. if (managed_zone(zone))
  8552. return true;
  8553. }
  8554. return false;
  8555. }
  8556. #endif /* CONFIG_ZONE_DMA */