verifier.c 451 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505105061050710508105091051010511105121051310514105151051610517105181051910520105211052210523105241052510526105271052810529105301053110532105331053410535105361053710538105391054010541105421054310544105451054610547105481054910550105511055210553105541055510556105571055810559105601056110562105631056410565105661056710568105691057010571105721057310574105751057610577105781057910580105811058210583105841058510586105871058810589105901059110592105931059410595105961059710598105991060010601106021060310604106051060610607106081060910610106111061210613106141061510616106171061810619106201062110622106231062410625106261062710628106291063010631106321063310634106351063610637106381063910640106411064210643106441064510646106471064810649106501065110652106531065410655106561065710658106591066010661106621066310664106651066610667106681066910670106711067210673106741067510676106771067810679106801068110682106831068410685106861068710688106891069010691106921069310694106951069610697106981069910700107011070210703107041070510706107071070810709107101071110712107131071410715107161071710718107191072010721107221072310724107251072610727107281072910730107311073210733107341073510736107371073810739107401074110742107431074410745107461074710748107491075010751107521075310754107551075610757107581075910760107611076210763107641076510766107671076810769107701077110772107731077410775107761077710778107791078010781107821078310784107851078610787107881078910790107911079210793107941079510796107971079810799108001080110802108031080410805108061080710808108091081010811108121081310814108151081610817108181081910820108211082210823108241082510826108271082810829108301083110832108331083410835108361083710838108391084010841108421084310844108451084610847108481084910850108511085210853108541085510856108571085810859108601086110862108631086410865108661086710868108691087010871108721087310874108751087610877108781087910880108811088210883108841088510886108871088810889108901089110892108931089410895108961089710898108991090010901109021090310904109051090610907109081090910910109111091210913109141091510916109171091810919109201092110922109231092410925109261092710928109291093010931109321093310934109351093610937109381093910940109411094210943109441094510946109471094810949109501095110952109531095410955109561095710958109591096010961109621096310964109651096610967109681096910970109711097210973109741097510976109771097810979109801098110982109831098410985109861098710988109891099010991109921099310994109951099610997109981099911000110011100211003110041100511006110071100811009110101101111012110131101411015110161101711018110191102011021110221102311024110251102611027110281102911030110311103211033110341103511036110371103811039110401104111042110431104411045110461104711048110491105011051110521105311054110551105611057110581105911060110611106211063110641106511066110671106811069110701107111072110731107411075110761107711078110791108011081110821108311084110851108611087110881108911090110911109211093110941109511096110971109811099111001110111102111031110411105111061110711108111091111011111111121111311114111151111611117111181111911120111211112211123111241112511126111271112811129111301113111132111331113411135111361113711138111391114011141111421114311144111451114611147111481114911150111511115211153111541115511156111571115811159111601116111162111631116411165111661116711168111691117011171111721117311174111751117611177111781117911180111811118211183111841118511186111871118811189111901119111192111931119411195111961119711198111991120011201112021120311204112051120611207112081120911210112111121211213112141121511216112171121811219112201122111222112231122411225112261122711228112291123011231112321123311234112351123611237112381123911240112411124211243112441124511246112471124811249112501125111252112531125411255112561125711258112591126011261112621126311264112651126611267112681126911270112711127211273112741127511276112771127811279112801128111282112831128411285112861128711288112891129011291112921129311294112951129611297112981129911300113011130211303113041130511306113071130811309113101131111312113131131411315113161131711318113191132011321113221132311324113251132611327113281132911330113311133211333113341133511336113371133811339113401134111342113431134411345113461134711348113491135011351113521135311354113551135611357113581135911360113611136211363113641136511366113671136811369113701137111372113731137411375113761137711378113791138011381113821138311384113851138611387113881138911390113911139211393113941139511396113971139811399114001140111402114031140411405114061140711408114091141011411114121141311414114151141611417114181141911420114211142211423114241142511426114271142811429114301143111432114331143411435114361143711438114391144011441114421144311444114451144611447114481144911450114511145211453114541145511456114571145811459114601146111462114631146411465114661146711468114691147011471114721147311474114751147611477114781147911480114811148211483114841148511486114871148811489114901149111492114931149411495114961149711498114991150011501115021150311504115051150611507115081150911510115111151211513115141151511516115171151811519115201152111522115231152411525115261152711528115291153011531115321153311534115351153611537115381153911540115411154211543115441154511546115471154811549115501155111552115531155411555115561155711558115591156011561115621156311564115651156611567115681156911570115711157211573115741157511576115771157811579115801158111582115831158411585115861158711588115891159011591115921159311594115951159611597115981159911600116011160211603116041160511606116071160811609116101161111612116131161411615116161161711618116191162011621116221162311624116251162611627116281162911630116311163211633116341163511636116371163811639116401164111642116431164411645116461164711648116491165011651116521165311654116551165611657116581165911660116611166211663116641166511666116671166811669116701167111672116731167411675116761167711678116791168011681116821168311684116851168611687116881168911690116911169211693116941169511696116971169811699117001170111702117031170411705117061170711708117091171011711117121171311714117151171611717117181171911720117211172211723117241172511726117271172811729117301173111732117331173411735117361173711738117391174011741117421174311744117451174611747117481174911750117511175211753117541175511756117571175811759117601176111762117631176411765117661176711768117691177011771117721177311774117751177611777117781177911780117811178211783117841178511786117871178811789117901179111792117931179411795117961179711798117991180011801118021180311804118051180611807118081180911810118111181211813118141181511816118171181811819118201182111822118231182411825118261182711828118291183011831118321183311834118351183611837118381183911840118411184211843118441184511846118471184811849118501185111852118531185411855118561185711858118591186011861118621186311864118651186611867118681186911870118711187211873118741187511876118771187811879118801188111882118831188411885118861188711888118891189011891118921189311894118951189611897118981189911900119011190211903119041190511906119071190811909119101191111912119131191411915119161191711918119191192011921119221192311924119251192611927119281192911930119311193211933119341193511936119371193811939119401194111942119431194411945119461194711948119491195011951119521195311954119551195611957119581195911960119611196211963119641196511966119671196811969119701197111972119731197411975119761197711978119791198011981119821198311984119851198611987119881198911990119911199211993119941199511996119971199811999120001200112002120031200412005120061200712008120091201012011120121201312014120151201612017120181201912020120211202212023120241202512026120271202812029120301203112032120331203412035120361203712038120391204012041120421204312044120451204612047120481204912050120511205212053120541205512056120571205812059120601206112062120631206412065120661206712068120691207012071120721207312074120751207612077120781207912080120811208212083120841208512086120871208812089120901209112092120931209412095120961209712098120991210012101121021210312104121051210612107121081210912110121111211212113121141211512116121171211812119121201212112122121231212412125121261212712128121291213012131121321213312134121351213612137121381213912140121411214212143121441214512146121471214812149121501215112152121531215412155121561215712158121591216012161121621216312164121651216612167121681216912170121711217212173121741217512176121771217812179121801218112182121831218412185121861218712188121891219012191121921219312194121951219612197121981219912200122011220212203122041220512206122071220812209122101221112212122131221412215122161221712218122191222012221122221222312224122251222612227122281222912230122311223212233122341223512236122371223812239122401224112242122431224412245122461224712248122491225012251122521225312254122551225612257122581225912260122611226212263122641226512266122671226812269122701227112272122731227412275122761227712278122791228012281122821228312284122851228612287122881228912290122911229212293122941229512296122971229812299123001230112302123031230412305123061230712308123091231012311123121231312314123151231612317123181231912320123211232212323123241232512326123271232812329123301233112332123331233412335123361233712338123391234012341123421234312344123451234612347123481234912350123511235212353123541235512356123571235812359123601236112362123631236412365123661236712368123691237012371123721237312374123751237612377123781237912380123811238212383123841238512386123871238812389123901239112392123931239412395123961239712398123991240012401124021240312404124051240612407124081240912410124111241212413124141241512416124171241812419124201242112422124231242412425124261242712428124291243012431124321243312434124351243612437124381243912440124411244212443124441244512446124471244812449124501245112452124531245412455124561245712458124591246012461124621246312464124651246612467124681246912470124711247212473124741247512476124771247812479124801248112482124831248412485124861248712488124891249012491124921249312494124951249612497124981249912500125011250212503125041250512506125071250812509125101251112512125131251412515125161251712518125191252012521125221252312524125251252612527125281252912530125311253212533125341253512536125371253812539125401254112542125431254412545125461254712548125491255012551125521255312554125551255612557125581255912560125611256212563125641256512566125671256812569125701257112572125731257412575125761257712578125791258012581125821258312584125851258612587125881258912590125911259212593125941259512596125971259812599126001260112602126031260412605126061260712608126091261012611126121261312614126151261612617126181261912620126211262212623126241262512626126271262812629126301263112632126331263412635126361263712638126391264012641126421264312644126451264612647126481264912650126511265212653126541265512656126571265812659126601266112662126631266412665126661266712668126691267012671126721267312674126751267612677126781267912680126811268212683126841268512686126871268812689126901269112692126931269412695126961269712698126991270012701127021270312704127051270612707127081270912710127111271212713127141271512716127171271812719127201272112722127231272412725127261272712728127291273012731127321273312734127351273612737127381273912740127411274212743127441274512746127471274812749127501275112752127531275412755127561275712758127591276012761127621276312764127651276612767127681276912770127711277212773127741277512776127771277812779127801278112782127831278412785127861278712788127891279012791127921279312794127951279612797127981279912800128011280212803128041280512806128071280812809128101281112812128131281412815128161281712818128191282012821128221282312824128251282612827128281282912830128311283212833128341283512836128371283812839128401284112842128431284412845128461284712848128491285012851128521285312854128551285612857128581285912860128611286212863128641286512866128671286812869128701287112872128731287412875128761287712878128791288012881128821288312884128851288612887128881288912890128911289212893128941289512896128971289812899129001290112902129031290412905129061290712908129091291012911129121291312914129151291612917129181291912920129211292212923129241292512926129271292812929129301293112932129331293412935129361293712938129391294012941129421294312944129451294612947129481294912950129511295212953129541295512956129571295812959129601296112962129631296412965129661296712968129691297012971129721297312974129751297612977129781297912980129811298212983129841298512986129871298812989129901299112992129931299412995129961299712998129991300013001130021300313004130051300613007130081300913010130111301213013130141301513016130171301813019130201302113022130231302413025130261302713028130291303013031130321303313034130351303613037130381303913040130411304213043130441304513046130471304813049130501305113052130531305413055130561305713058130591306013061130621306313064130651306613067130681306913070130711307213073130741307513076130771307813079130801308113082130831308413085130861308713088130891309013091130921309313094130951309613097130981309913100131011310213103131041310513106131071310813109131101311113112131131311413115131161311713118131191312013121131221312313124131251312613127131281312913130131311313213133131341313513136131371313813139131401314113142131431314413145131461314713148131491315013151131521315313154131551315613157131581315913160131611316213163131641316513166131671316813169131701317113172131731317413175131761317713178131791318013181131821318313184131851318613187131881318913190131911319213193131941319513196131971319813199132001320113202132031320413205132061320713208132091321013211132121321313214132151321613217132181321913220132211322213223132241322513226132271322813229132301323113232132331323413235132361323713238132391324013241132421324313244132451324613247132481324913250132511325213253132541325513256132571325813259132601326113262132631326413265132661326713268132691327013271132721327313274132751327613277132781327913280132811328213283132841328513286132871328813289132901329113292132931329413295132961329713298132991330013301133021330313304133051330613307133081330913310133111331213313133141331513316133171331813319133201332113322133231332413325133261332713328133291333013331133321333313334133351333613337133381333913340133411334213343133441334513346133471334813349133501335113352133531335413355133561335713358133591336013361133621336313364133651336613367133681336913370133711337213373133741337513376133771337813379133801338113382133831338413385133861338713388133891339013391133921339313394133951339613397133981339913400134011340213403134041340513406134071340813409134101341113412134131341413415134161341713418134191342013421134221342313424134251342613427134281342913430134311343213433134341343513436134371343813439134401344113442134431344413445134461344713448134491345013451134521345313454134551345613457134581345913460134611346213463134641346513466134671346813469134701347113472134731347413475134761347713478134791348013481134821348313484134851348613487134881348913490134911349213493134941349513496134971349813499135001350113502135031350413505135061350713508135091351013511135121351313514135151351613517135181351913520135211352213523135241352513526135271352813529135301353113532135331353413535135361353713538135391354013541135421354313544135451354613547135481354913550135511355213553135541355513556135571355813559135601356113562135631356413565135661356713568135691357013571135721357313574135751357613577135781357913580135811358213583135841358513586135871358813589135901359113592135931359413595135961359713598135991360013601136021360313604136051360613607136081360913610136111361213613136141361513616136171361813619136201362113622136231362413625136261362713628136291363013631136321363313634136351363613637136381363913640136411364213643136441364513646136471364813649136501365113652136531365413655136561365713658136591366013661136621366313664136651366613667136681366913670136711367213673136741367513676136771367813679136801368113682136831368413685136861368713688136891369013691136921369313694136951369613697136981369913700137011370213703137041370513706137071370813709137101371113712137131371413715137161371713718137191372013721137221372313724137251372613727137281372913730137311373213733137341373513736137371373813739137401374113742137431374413745137461374713748137491375013751137521375313754137551375613757137581375913760137611376213763137641376513766137671376813769137701377113772137731377413775137761377713778137791378013781137821378313784137851378613787137881378913790137911379213793137941379513796137971379813799138001380113802138031380413805138061380713808138091381013811138121381313814138151381613817138181381913820138211382213823138241382513826138271382813829138301383113832138331383413835138361383713838138391384013841138421384313844138451384613847138481384913850138511385213853138541385513856138571385813859138601386113862138631386413865138661386713868138691387013871138721387313874138751387613877138781387913880138811388213883138841388513886138871388813889138901389113892138931389413895138961389713898138991390013901139021390313904139051390613907139081390913910139111391213913139141391513916139171391813919139201392113922139231392413925139261392713928139291393013931139321393313934139351393613937139381393913940139411394213943139441394513946139471394813949139501395113952139531395413955139561395713958139591396013961139621396313964139651396613967139681396913970139711397213973139741397513976139771397813979139801398113982139831398413985139861398713988139891399013991139921399313994139951399613997139981399914000140011400214003140041400514006140071400814009140101401114012140131401414015140161401714018140191402014021140221402314024140251402614027140281402914030140311403214033140341403514036140371403814039140401404114042140431404414045140461404714048140491405014051140521405314054140551405614057140581405914060140611406214063140641406514066140671406814069140701407114072140731407414075140761407714078140791408014081140821408314084140851408614087140881408914090140911409214093140941409514096140971409814099141001410114102141031410414105141061410714108141091411014111141121411314114141151411614117141181411914120141211412214123141241412514126141271412814129141301413114132141331413414135141361413714138141391414014141141421414314144141451414614147141481414914150141511415214153141541415514156141571415814159141601416114162141631416414165141661416714168141691417014171141721417314174141751417614177141781417914180141811418214183141841418514186141871418814189141901419114192141931419414195141961419714198141991420014201142021420314204142051420614207142081420914210142111421214213142141421514216142171421814219142201422114222142231422414225142261422714228142291423014231142321423314234142351423614237142381423914240142411424214243142441424514246142471424814249142501425114252142531425414255142561425714258142591426014261142621426314264142651426614267142681426914270142711427214273142741427514276142771427814279142801428114282142831428414285142861428714288142891429014291142921429314294142951429614297142981429914300143011430214303143041430514306143071430814309143101431114312143131431414315143161431714318143191432014321143221432314324143251432614327143281432914330143311433214333143341433514336143371433814339143401434114342143431434414345143461434714348143491435014351143521435314354143551435614357143581435914360143611436214363143641436514366143671436814369143701437114372143731437414375143761437714378143791438014381143821438314384143851438614387143881438914390143911439214393143941439514396143971439814399144001440114402144031440414405144061440714408144091441014411144121441314414144151441614417144181441914420144211442214423144241442514426144271442814429144301443114432144331443414435144361443714438144391444014441144421444314444144451444614447144481444914450144511445214453144541445514456144571445814459144601446114462144631446414465144661446714468144691447014471144721447314474144751447614477144781447914480144811448214483144841448514486144871448814489144901449114492144931449414495144961449714498144991450014501145021450314504145051450614507145081450914510145111451214513145141451514516145171451814519145201452114522145231452414525145261452714528145291453014531145321453314534145351453614537145381453914540145411454214543145441454514546145471454814549145501455114552145531455414555145561455714558145591456014561145621456314564145651456614567145681456914570145711457214573145741457514576145771457814579145801458114582145831458414585145861458714588145891459014591145921459314594145951459614597145981459914600146011460214603146041460514606146071460814609146101461114612146131461414615146161461714618146191462014621146221462314624146251462614627146281462914630146311463214633146341463514636146371463814639146401464114642146431464414645146461464714648146491465014651146521465314654146551465614657146581465914660146611466214663146641466514666146671466814669146701467114672146731467414675146761467714678146791468014681146821468314684146851468614687146881468914690146911469214693146941469514696146971469814699147001470114702147031470414705147061470714708147091471014711147121471314714147151471614717147181471914720147211472214723147241472514726147271472814729147301473114732147331473414735147361473714738147391474014741147421474314744147451474614747147481474914750147511475214753147541475514756147571475814759147601476114762147631476414765147661476714768147691477014771147721477314774147751477614777147781477914780147811478214783147841478514786147871478814789147901479114792147931479414795147961479714798147991480014801148021480314804148051480614807148081480914810148111481214813148141481514816148171481814819148201482114822148231482414825148261482714828148291483014831148321483314834148351483614837148381483914840148411484214843148441484514846148471484814849148501485114852148531485414855148561485714858148591486014861148621486314864148651486614867148681486914870148711487214873148741487514876148771487814879148801488114882148831488414885148861488714888148891489014891148921489314894148951489614897148981489914900149011490214903149041490514906149071490814909149101491114912149131491414915149161491714918149191492014921149221492314924149251492614927149281492914930149311493214933149341493514936149371493814939149401494114942149431494414945149461494714948149491495014951149521495314954149551495614957149581495914960149611496214963149641496514966149671496814969149701497114972149731497414975149761497714978149791498014981149821498314984149851498614987149881498914990149911499214993149941499514996149971499814999150001500115002150031500415005150061500715008150091501015011150121501315014150151501615017150181501915020150211502215023150241502515026150271502815029150301503115032150331503415035150361503715038150391504015041150421504315044150451504615047150481504915050150511505215053150541505515056150571505815059150601506115062150631506415065150661506715068150691507015071150721507315074150751507615077150781507915080150811508215083150841508515086150871508815089150901509115092150931509415095150961509715098150991510015101151021510315104151051510615107151081510915110151111511215113151141511515116151171511815119151201512115122151231512415125151261512715128151291513015131151321513315134151351513615137151381513915140151411514215143151441514515146151471514815149151501515115152151531515415155151561515715158151591516015161151621516315164151651516615167151681516915170151711517215173151741517515176151771517815179151801518115182151831518415185151861518715188151891519015191151921519315194151951519615197151981519915200152011520215203152041520515206152071520815209152101521115212152131521415215152161521715218152191522015221152221522315224152251522615227152281522915230152311523215233152341523515236152371523815239152401524115242152431524415245152461524715248152491525015251152521525315254152551525615257152581525915260152611526215263152641526515266152671526815269152701527115272152731527415275152761527715278152791528015281152821528315284152851528615287152881528915290152911529215293152941529515296152971529815299153001530115302153031530415305153061530715308153091531015311153121531315314153151531615317153181531915320153211532215323153241532515326153271532815329153301533115332153331533415335153361533715338153391534015341153421534315344153451534615347153481534915350153511535215353153541535515356153571535815359153601536115362153631536415365153661536715368153691537015371153721537315374153751537615377153781537915380153811538215383153841538515386153871538815389153901539115392153931539415395153961539715398153991540015401154021540315404154051540615407154081540915410154111541215413154141541515416154171541815419154201542115422154231542415425154261542715428154291543015431154321543315434154351543615437154381543915440154411544215443154441544515446154471544815449154501545115452154531545415455154561545715458154591546015461154621546315464154651546615467154681546915470154711547215473154741547515476154771547815479154801548115482154831548415485154861548715488154891549015491154921549315494154951549615497154981549915500155011550215503155041550515506155071550815509155101551115512155131551415515155161551715518155191552015521155221552315524155251552615527155281552915530155311553215533155341553515536155371553815539155401554115542155431554415545155461554715548155491555015551155521555315554155551555615557155581555915560155611556215563155641556515566155671556815569155701557115572155731557415575155761557715578155791558015581155821558315584155851558615587155881558915590155911559215593155941559515596155971559815599156001560115602156031560415605156061560715608156091561015611156121561315614156151561615617156181561915620156211562215623156241562515626156271562815629156301563115632156331563415635156361563715638156391564015641156421564315644156451564615647156481564915650156511565215653156541565515656156571565815659156601566115662156631566415665156661566715668156691567015671156721567315674156751567615677156781567915680156811568215683156841568515686156871568815689156901569115692156931569415695156961569715698
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
  3. * Copyright (c) 2016 Facebook
  4. * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
  5. */
  6. #include <uapi/linux/btf.h>
  7. #include <linux/bpf-cgroup.h>
  8. #include <linux/kernel.h>
  9. #include <linux/types.h>
  10. #include <linux/slab.h>
  11. #include <linux/bpf.h>
  12. #include <linux/btf.h>
  13. #include <linux/bpf_verifier.h>
  14. #include <linux/filter.h>
  15. #include <net/netlink.h>
  16. #include <linux/file.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/stringify.h>
  19. #include <linux/bsearch.h>
  20. #include <linux/sort.h>
  21. #include <linux/perf_event.h>
  22. #include <linux/ctype.h>
  23. #include <linux/error-injection.h>
  24. #include <linux/bpf_lsm.h>
  25. #include <linux/btf_ids.h>
  26. #include <linux/poison.h>
  27. #include "disasm.h"
  28. static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
  29. #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \
  30. [_id] = & _name ## _verifier_ops,
  31. #define BPF_MAP_TYPE(_id, _ops)
  32. #define BPF_LINK_TYPE(_id, _name)
  33. #include <linux/bpf_types.h>
  34. #undef BPF_PROG_TYPE
  35. #undef BPF_MAP_TYPE
  36. #undef BPF_LINK_TYPE
  37. };
  38. /* bpf_check() is a static code analyzer that walks eBPF program
  39. * instruction by instruction and updates register/stack state.
  40. * All paths of conditional branches are analyzed until 'bpf_exit' insn.
  41. *
  42. * The first pass is depth-first-search to check that the program is a DAG.
  43. * It rejects the following programs:
  44. * - larger than BPF_MAXINSNS insns
  45. * - if loop is present (detected via back-edge)
  46. * - unreachable insns exist (shouldn't be a forest. program = one function)
  47. * - out of bounds or malformed jumps
  48. * The second pass is all possible path descent from the 1st insn.
  49. * Since it's analyzing all paths through the program, the length of the
  50. * analysis is limited to 64k insn, which may be hit even if total number of
  51. * insn is less then 4K, but there are too many branches that change stack/regs.
  52. * Number of 'branches to be analyzed' is limited to 1k
  53. *
  54. * On entry to each instruction, each register has a type, and the instruction
  55. * changes the types of the registers depending on instruction semantics.
  56. * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
  57. * copied to R1.
  58. *
  59. * All registers are 64-bit.
  60. * R0 - return register
  61. * R1-R5 argument passing registers
  62. * R6-R9 callee saved registers
  63. * R10 - frame pointer read-only
  64. *
  65. * At the start of BPF program the register R1 contains a pointer to bpf_context
  66. * and has type PTR_TO_CTX.
  67. *
  68. * Verifier tracks arithmetic operations on pointers in case:
  69. * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
  70. * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
  71. * 1st insn copies R10 (which has FRAME_PTR) type into R1
  72. * and 2nd arithmetic instruction is pattern matched to recognize
  73. * that it wants to construct a pointer to some element within stack.
  74. * So after 2nd insn, the register R1 has type PTR_TO_STACK
  75. * (and -20 constant is saved for further stack bounds checking).
  76. * Meaning that this reg is a pointer to stack plus known immediate constant.
  77. *
  78. * Most of the time the registers have SCALAR_VALUE type, which
  79. * means the register has some value, but it's not a valid pointer.
  80. * (like pointer plus pointer becomes SCALAR_VALUE type)
  81. *
  82. * When verifier sees load or store instructions the type of base register
  83. * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
  84. * four pointer types recognized by check_mem_access() function.
  85. *
  86. * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
  87. * and the range of [ptr, ptr + map's value_size) is accessible.
  88. *
  89. * registers used to pass values to function calls are checked against
  90. * function argument constraints.
  91. *
  92. * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
  93. * It means that the register type passed to this function must be
  94. * PTR_TO_STACK and it will be used inside the function as
  95. * 'pointer to map element key'
  96. *
  97. * For example the argument constraints for bpf_map_lookup_elem():
  98. * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
  99. * .arg1_type = ARG_CONST_MAP_PTR,
  100. * .arg2_type = ARG_PTR_TO_MAP_KEY,
  101. *
  102. * ret_type says that this function returns 'pointer to map elem value or null'
  103. * function expects 1st argument to be a const pointer to 'struct bpf_map' and
  104. * 2nd argument should be a pointer to stack, which will be used inside
  105. * the helper function as a pointer to map element key.
  106. *
  107. * On the kernel side the helper function looks like:
  108. * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
  109. * {
  110. * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
  111. * void *key = (void *) (unsigned long) r2;
  112. * void *value;
  113. *
  114. * here kernel can access 'key' and 'map' pointers safely, knowing that
  115. * [key, key + map->key_size) bytes are valid and were initialized on
  116. * the stack of eBPF program.
  117. * }
  118. *
  119. * Corresponding eBPF program may look like:
  120. * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR
  121. * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
  122. * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP
  123. * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
  124. * here verifier looks at prototype of map_lookup_elem() and sees:
  125. * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
  126. * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
  127. *
  128. * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
  129. * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
  130. * and were initialized prior to this call.
  131. * If it's ok, then verifier allows this BPF_CALL insn and looks at
  132. * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
  133. * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
  134. * returns either pointer to map value or NULL.
  135. *
  136. * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
  137. * insn, the register holding that pointer in the true branch changes state to
  138. * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
  139. * branch. See check_cond_jmp_op().
  140. *
  141. * After the call R0 is set to return type of the function and registers R1-R5
  142. * are set to NOT_INIT to indicate that they are no longer readable.
  143. *
  144. * The following reference types represent a potential reference to a kernel
  145. * resource which, after first being allocated, must be checked and freed by
  146. * the BPF program:
  147. * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
  148. *
  149. * When the verifier sees a helper call return a reference type, it allocates a
  150. * pointer id for the reference and stores it in the current function state.
  151. * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
  152. * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
  153. * passes through a NULL-check conditional. For the branch wherein the state is
  154. * changed to CONST_IMM, the verifier releases the reference.
  155. *
  156. * For each helper function that allocates a reference, such as
  157. * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
  158. * bpf_sk_release(). When a reference type passes into the release function,
  159. * the verifier also releases the reference. If any unchecked or unreleased
  160. * reference remains at the end of the program, the verifier rejects it.
  161. */
  162. /* verifier_state + insn_idx are pushed to stack when branch is encountered */
  163. struct bpf_verifier_stack_elem {
  164. /* verifer state is 'st'
  165. * before processing instruction 'insn_idx'
  166. * and after processing instruction 'prev_insn_idx'
  167. */
  168. struct bpf_verifier_state st;
  169. int insn_idx;
  170. int prev_insn_idx;
  171. struct bpf_verifier_stack_elem *next;
  172. /* length of verifier log at the time this state was pushed on stack */
  173. u32 log_pos;
  174. };
  175. #define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192
  176. #define BPF_COMPLEXITY_LIMIT_STATES 64
  177. #define BPF_MAP_KEY_POISON (1ULL << 63)
  178. #define BPF_MAP_KEY_SEEN (1ULL << 62)
  179. #define BPF_MAP_PTR_UNPRIV 1UL
  180. #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \
  181. POISON_POINTER_DELTA))
  182. #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))
  183. static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx);
  184. static int release_reference(struct bpf_verifier_env *env, int ref_obj_id);
  185. static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
  186. {
  187. return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON;
  188. }
  189. static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
  190. {
  191. return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV;
  192. }
  193. static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
  194. const struct bpf_map *map, bool unpriv)
  195. {
  196. BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
  197. unpriv |= bpf_map_ptr_unpriv(aux);
  198. aux->map_ptr_state = (unsigned long)map |
  199. (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
  200. }
  201. static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
  202. {
  203. return aux->map_key_state & BPF_MAP_KEY_POISON;
  204. }
  205. static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
  206. {
  207. return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
  208. }
  209. static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
  210. {
  211. return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
  212. }
  213. static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
  214. {
  215. bool poisoned = bpf_map_key_poisoned(aux);
  216. aux->map_key_state = state | BPF_MAP_KEY_SEEN |
  217. (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
  218. }
  219. static bool bpf_pseudo_call(const struct bpf_insn *insn)
  220. {
  221. return insn->code == (BPF_JMP | BPF_CALL) &&
  222. insn->src_reg == BPF_PSEUDO_CALL;
  223. }
  224. static bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn)
  225. {
  226. return insn->code == (BPF_JMP | BPF_CALL) &&
  227. insn->src_reg == BPF_PSEUDO_KFUNC_CALL;
  228. }
  229. struct bpf_call_arg_meta {
  230. struct bpf_map *map_ptr;
  231. bool raw_mode;
  232. bool pkt_access;
  233. u8 release_regno;
  234. int regno;
  235. int access_size;
  236. int mem_size;
  237. u64 msize_max_value;
  238. int ref_obj_id;
  239. int map_uid;
  240. int func_id;
  241. struct btf *btf;
  242. u32 btf_id;
  243. struct btf *ret_btf;
  244. u32 ret_btf_id;
  245. u32 subprogno;
  246. struct bpf_map_value_off_desc *kptr_off_desc;
  247. u8 uninit_dynptr_regno;
  248. };
  249. struct btf *btf_vmlinux;
  250. static DEFINE_MUTEX(bpf_verifier_lock);
  251. static const struct bpf_line_info *
  252. find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
  253. {
  254. const struct bpf_line_info *linfo;
  255. const struct bpf_prog *prog;
  256. u32 i, nr_linfo;
  257. prog = env->prog;
  258. nr_linfo = prog->aux->nr_linfo;
  259. if (!nr_linfo || insn_off >= prog->len)
  260. return NULL;
  261. linfo = prog->aux->linfo;
  262. for (i = 1; i < nr_linfo; i++)
  263. if (insn_off < linfo[i].insn_off)
  264. break;
  265. return &linfo[i - 1];
  266. }
  267. void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
  268. va_list args)
  269. {
  270. unsigned int n;
  271. n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
  272. WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
  273. "verifier log line truncated - local buffer too short\n");
  274. if (log->level == BPF_LOG_KERNEL) {
  275. bool newline = n > 0 && log->kbuf[n - 1] == '\n';
  276. pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");
  277. return;
  278. }
  279. n = min(log->len_total - log->len_used - 1, n);
  280. log->kbuf[n] = '\0';
  281. if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
  282. log->len_used += n;
  283. else
  284. log->ubuf = NULL;
  285. }
  286. static void bpf_vlog_reset(struct bpf_verifier_log *log, u32 new_pos)
  287. {
  288. char zero = 0;
  289. if (!bpf_verifier_log_needed(log))
  290. return;
  291. log->len_used = new_pos;
  292. if (put_user(zero, log->ubuf + new_pos))
  293. log->ubuf = NULL;
  294. }
  295. /* log_level controls verbosity level of eBPF verifier.
  296. * bpf_verifier_log_write() is used to dump the verification trace to the log,
  297. * so the user can figure out what's wrong with the program
  298. */
  299. __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
  300. const char *fmt, ...)
  301. {
  302. va_list args;
  303. if (!bpf_verifier_log_needed(&env->log))
  304. return;
  305. va_start(args, fmt);
  306. bpf_verifier_vlog(&env->log, fmt, args);
  307. va_end(args);
  308. }
  309. EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
  310. __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
  311. {
  312. struct bpf_verifier_env *env = private_data;
  313. va_list args;
  314. if (!bpf_verifier_log_needed(&env->log))
  315. return;
  316. va_start(args, fmt);
  317. bpf_verifier_vlog(&env->log, fmt, args);
  318. va_end(args);
  319. }
  320. __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
  321. const char *fmt, ...)
  322. {
  323. va_list args;
  324. if (!bpf_verifier_log_needed(log))
  325. return;
  326. va_start(args, fmt);
  327. bpf_verifier_vlog(log, fmt, args);
  328. va_end(args);
  329. }
  330. EXPORT_SYMBOL_GPL(bpf_log);
  331. static const char *ltrim(const char *s)
  332. {
  333. while (isspace(*s))
  334. s++;
  335. return s;
  336. }
  337. __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
  338. u32 insn_off,
  339. const char *prefix_fmt, ...)
  340. {
  341. const struct bpf_line_info *linfo;
  342. if (!bpf_verifier_log_needed(&env->log))
  343. return;
  344. linfo = find_linfo(env, insn_off);
  345. if (!linfo || linfo == env->prev_linfo)
  346. return;
  347. if (prefix_fmt) {
  348. va_list args;
  349. va_start(args, prefix_fmt);
  350. bpf_verifier_vlog(&env->log, prefix_fmt, args);
  351. va_end(args);
  352. }
  353. verbose(env, "%s\n",
  354. ltrim(btf_name_by_offset(env->prog->aux->btf,
  355. linfo->line_off)));
  356. env->prev_linfo = linfo;
  357. }
  358. static void verbose_invalid_scalar(struct bpf_verifier_env *env,
  359. struct bpf_reg_state *reg,
  360. struct tnum *range, const char *ctx,
  361. const char *reg_name)
  362. {
  363. char tn_buf[48];
  364. verbose(env, "At %s the register %s ", ctx, reg_name);
  365. if (!tnum_is_unknown(reg->var_off)) {
  366. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  367. verbose(env, "has value %s", tn_buf);
  368. } else {
  369. verbose(env, "has unknown scalar value");
  370. }
  371. tnum_strn(tn_buf, sizeof(tn_buf), *range);
  372. verbose(env, " should have been in %s\n", tn_buf);
  373. }
  374. static bool type_is_pkt_pointer(enum bpf_reg_type type)
  375. {
  376. type = base_type(type);
  377. return type == PTR_TO_PACKET ||
  378. type == PTR_TO_PACKET_META;
  379. }
  380. static bool type_is_sk_pointer(enum bpf_reg_type type)
  381. {
  382. return type == PTR_TO_SOCKET ||
  383. type == PTR_TO_SOCK_COMMON ||
  384. type == PTR_TO_TCP_SOCK ||
  385. type == PTR_TO_XDP_SOCK;
  386. }
  387. static bool reg_type_not_null(enum bpf_reg_type type)
  388. {
  389. return type == PTR_TO_SOCKET ||
  390. type == PTR_TO_TCP_SOCK ||
  391. type == PTR_TO_MAP_VALUE ||
  392. type == PTR_TO_MAP_KEY ||
  393. type == PTR_TO_SOCK_COMMON;
  394. }
  395. static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
  396. {
  397. return reg->type == PTR_TO_MAP_VALUE &&
  398. map_value_has_spin_lock(reg->map_ptr);
  399. }
  400. static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type)
  401. {
  402. type = base_type(type);
  403. return type == PTR_TO_SOCKET || type == PTR_TO_TCP_SOCK ||
  404. type == PTR_TO_MEM || type == PTR_TO_BTF_ID;
  405. }
  406. static bool type_is_rdonly_mem(u32 type)
  407. {
  408. return type & MEM_RDONLY;
  409. }
  410. static bool type_may_be_null(u32 type)
  411. {
  412. return type & PTR_MAYBE_NULL;
  413. }
  414. static bool is_acquire_function(enum bpf_func_id func_id,
  415. const struct bpf_map *map)
  416. {
  417. enum bpf_map_type map_type = map ? map->map_type : BPF_MAP_TYPE_UNSPEC;
  418. if (func_id == BPF_FUNC_sk_lookup_tcp ||
  419. func_id == BPF_FUNC_sk_lookup_udp ||
  420. func_id == BPF_FUNC_skc_lookup_tcp ||
  421. func_id == BPF_FUNC_ringbuf_reserve ||
  422. func_id == BPF_FUNC_kptr_xchg)
  423. return true;
  424. if (func_id == BPF_FUNC_map_lookup_elem &&
  425. (map_type == BPF_MAP_TYPE_SOCKMAP ||
  426. map_type == BPF_MAP_TYPE_SOCKHASH))
  427. return true;
  428. return false;
  429. }
  430. static bool is_ptr_cast_function(enum bpf_func_id func_id)
  431. {
  432. return func_id == BPF_FUNC_tcp_sock ||
  433. func_id == BPF_FUNC_sk_fullsock ||
  434. func_id == BPF_FUNC_skc_to_tcp_sock ||
  435. func_id == BPF_FUNC_skc_to_tcp6_sock ||
  436. func_id == BPF_FUNC_skc_to_udp6_sock ||
  437. func_id == BPF_FUNC_skc_to_mptcp_sock ||
  438. func_id == BPF_FUNC_skc_to_tcp_timewait_sock ||
  439. func_id == BPF_FUNC_skc_to_tcp_request_sock;
  440. }
  441. static bool is_dynptr_ref_function(enum bpf_func_id func_id)
  442. {
  443. return func_id == BPF_FUNC_dynptr_data;
  444. }
  445. static bool is_callback_calling_function(enum bpf_func_id func_id)
  446. {
  447. return func_id == BPF_FUNC_for_each_map_elem ||
  448. func_id == BPF_FUNC_timer_set_callback ||
  449. func_id == BPF_FUNC_find_vma ||
  450. func_id == BPF_FUNC_loop ||
  451. func_id == BPF_FUNC_user_ringbuf_drain;
  452. }
  453. static bool helper_multiple_ref_obj_use(enum bpf_func_id func_id,
  454. const struct bpf_map *map)
  455. {
  456. int ref_obj_uses = 0;
  457. if (is_ptr_cast_function(func_id))
  458. ref_obj_uses++;
  459. if (is_acquire_function(func_id, map))
  460. ref_obj_uses++;
  461. if (is_dynptr_ref_function(func_id))
  462. ref_obj_uses++;
  463. return ref_obj_uses > 1;
  464. }
  465. static bool is_cmpxchg_insn(const struct bpf_insn *insn)
  466. {
  467. return BPF_CLASS(insn->code) == BPF_STX &&
  468. BPF_MODE(insn->code) == BPF_ATOMIC &&
  469. insn->imm == BPF_CMPXCHG;
  470. }
  471. /* string representation of 'enum bpf_reg_type'
  472. *
  473. * Note that reg_type_str() can not appear more than once in a single verbose()
  474. * statement.
  475. */
  476. static const char *reg_type_str(struct bpf_verifier_env *env,
  477. enum bpf_reg_type type)
  478. {
  479. char postfix[16] = {0}, prefix[32] = {0};
  480. static const char * const str[] = {
  481. [NOT_INIT] = "?",
  482. [SCALAR_VALUE] = "scalar",
  483. [PTR_TO_CTX] = "ctx",
  484. [CONST_PTR_TO_MAP] = "map_ptr",
  485. [PTR_TO_MAP_VALUE] = "map_value",
  486. [PTR_TO_STACK] = "fp",
  487. [PTR_TO_PACKET] = "pkt",
  488. [PTR_TO_PACKET_META] = "pkt_meta",
  489. [PTR_TO_PACKET_END] = "pkt_end",
  490. [PTR_TO_FLOW_KEYS] = "flow_keys",
  491. [PTR_TO_SOCKET] = "sock",
  492. [PTR_TO_SOCK_COMMON] = "sock_common",
  493. [PTR_TO_TCP_SOCK] = "tcp_sock",
  494. [PTR_TO_TP_BUFFER] = "tp_buffer",
  495. [PTR_TO_XDP_SOCK] = "xdp_sock",
  496. [PTR_TO_BTF_ID] = "ptr_",
  497. [PTR_TO_MEM] = "mem",
  498. [PTR_TO_BUF] = "buf",
  499. [PTR_TO_FUNC] = "func",
  500. [PTR_TO_MAP_KEY] = "map_key",
  501. [PTR_TO_DYNPTR] = "dynptr_ptr",
  502. };
  503. if (type & PTR_MAYBE_NULL) {
  504. if (base_type(type) == PTR_TO_BTF_ID)
  505. strncpy(postfix, "or_null_", 16);
  506. else
  507. strncpy(postfix, "_or_null", 16);
  508. }
  509. if (type & MEM_RDONLY)
  510. strncpy(prefix, "rdonly_", 32);
  511. if (type & MEM_ALLOC)
  512. strncpy(prefix, "alloc_", 32);
  513. if (type & MEM_USER)
  514. strncpy(prefix, "user_", 32);
  515. if (type & MEM_PERCPU)
  516. strncpy(prefix, "percpu_", 32);
  517. if (type & PTR_UNTRUSTED)
  518. strncpy(prefix, "untrusted_", 32);
  519. snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s",
  520. prefix, str[base_type(type)], postfix);
  521. return env->type_str_buf;
  522. }
  523. static char slot_type_char[] = {
  524. [STACK_INVALID] = '?',
  525. [STACK_SPILL] = 'r',
  526. [STACK_MISC] = 'm',
  527. [STACK_ZERO] = '0',
  528. [STACK_DYNPTR] = 'd',
  529. };
  530. static void print_liveness(struct bpf_verifier_env *env,
  531. enum bpf_reg_liveness live)
  532. {
  533. if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
  534. verbose(env, "_");
  535. if (live & REG_LIVE_READ)
  536. verbose(env, "r");
  537. if (live & REG_LIVE_WRITTEN)
  538. verbose(env, "w");
  539. if (live & REG_LIVE_DONE)
  540. verbose(env, "D");
  541. }
  542. static int get_spi(s32 off)
  543. {
  544. return (-off - 1) / BPF_REG_SIZE;
  545. }
  546. static bool is_spi_bounds_valid(struct bpf_func_state *state, int spi, int nr_slots)
  547. {
  548. int allocated_slots = state->allocated_stack / BPF_REG_SIZE;
  549. /* We need to check that slots between [spi - nr_slots + 1, spi] are
  550. * within [0, allocated_stack).
  551. *
  552. * Please note that the spi grows downwards. For example, a dynptr
  553. * takes the size of two stack slots; the first slot will be at
  554. * spi and the second slot will be at spi - 1.
  555. */
  556. return spi - nr_slots + 1 >= 0 && spi < allocated_slots;
  557. }
  558. static struct bpf_func_state *func(struct bpf_verifier_env *env,
  559. const struct bpf_reg_state *reg)
  560. {
  561. struct bpf_verifier_state *cur = env->cur_state;
  562. return cur->frame[reg->frameno];
  563. }
  564. static const char *kernel_type_name(const struct btf* btf, u32 id)
  565. {
  566. return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
  567. }
  568. static void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno)
  569. {
  570. env->scratched_regs |= 1U << regno;
  571. }
  572. static void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi)
  573. {
  574. env->scratched_stack_slots |= 1ULL << spi;
  575. }
  576. static bool reg_scratched(const struct bpf_verifier_env *env, u32 regno)
  577. {
  578. return (env->scratched_regs >> regno) & 1;
  579. }
  580. static bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno)
  581. {
  582. return (env->scratched_stack_slots >> regno) & 1;
  583. }
  584. static bool verifier_state_scratched(const struct bpf_verifier_env *env)
  585. {
  586. return env->scratched_regs || env->scratched_stack_slots;
  587. }
  588. static void mark_verifier_state_clean(struct bpf_verifier_env *env)
  589. {
  590. env->scratched_regs = 0U;
  591. env->scratched_stack_slots = 0ULL;
  592. }
  593. /* Used for printing the entire verifier state. */
  594. static void mark_verifier_state_scratched(struct bpf_verifier_env *env)
  595. {
  596. env->scratched_regs = ~0U;
  597. env->scratched_stack_slots = ~0ULL;
  598. }
  599. static enum bpf_dynptr_type arg_to_dynptr_type(enum bpf_arg_type arg_type)
  600. {
  601. switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
  602. case DYNPTR_TYPE_LOCAL:
  603. return BPF_DYNPTR_TYPE_LOCAL;
  604. case DYNPTR_TYPE_RINGBUF:
  605. return BPF_DYNPTR_TYPE_RINGBUF;
  606. default:
  607. return BPF_DYNPTR_TYPE_INVALID;
  608. }
  609. }
  610. static bool dynptr_type_refcounted(enum bpf_dynptr_type type)
  611. {
  612. return type == BPF_DYNPTR_TYPE_RINGBUF;
  613. }
  614. static int mark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
  615. enum bpf_arg_type arg_type, int insn_idx)
  616. {
  617. struct bpf_func_state *state = func(env, reg);
  618. enum bpf_dynptr_type type;
  619. int spi, i, id;
  620. spi = get_spi(reg->off);
  621. if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS))
  622. return -EINVAL;
  623. for (i = 0; i < BPF_REG_SIZE; i++) {
  624. state->stack[spi].slot_type[i] = STACK_DYNPTR;
  625. state->stack[spi - 1].slot_type[i] = STACK_DYNPTR;
  626. }
  627. type = arg_to_dynptr_type(arg_type);
  628. if (type == BPF_DYNPTR_TYPE_INVALID)
  629. return -EINVAL;
  630. state->stack[spi].spilled_ptr.dynptr.first_slot = true;
  631. state->stack[spi].spilled_ptr.dynptr.type = type;
  632. state->stack[spi - 1].spilled_ptr.dynptr.type = type;
  633. if (dynptr_type_refcounted(type)) {
  634. /* The id is used to track proper releasing */
  635. id = acquire_reference_state(env, insn_idx);
  636. if (id < 0)
  637. return id;
  638. state->stack[spi].spilled_ptr.id = id;
  639. state->stack[spi - 1].spilled_ptr.id = id;
  640. }
  641. return 0;
  642. }
  643. static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  644. {
  645. struct bpf_func_state *state = func(env, reg);
  646. int spi, i;
  647. spi = get_spi(reg->off);
  648. if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS))
  649. return -EINVAL;
  650. for (i = 0; i < BPF_REG_SIZE; i++) {
  651. state->stack[spi].slot_type[i] = STACK_INVALID;
  652. state->stack[spi - 1].slot_type[i] = STACK_INVALID;
  653. }
  654. /* Invalidate any slices associated with this dynptr */
  655. if (dynptr_type_refcounted(state->stack[spi].spilled_ptr.dynptr.type)) {
  656. release_reference(env, state->stack[spi].spilled_ptr.id);
  657. state->stack[spi].spilled_ptr.id = 0;
  658. state->stack[spi - 1].spilled_ptr.id = 0;
  659. }
  660. state->stack[spi].spilled_ptr.dynptr.first_slot = false;
  661. state->stack[spi].spilled_ptr.dynptr.type = 0;
  662. state->stack[spi - 1].spilled_ptr.dynptr.type = 0;
  663. return 0;
  664. }
  665. static bool is_dynptr_reg_valid_uninit(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  666. {
  667. struct bpf_func_state *state = func(env, reg);
  668. int spi = get_spi(reg->off);
  669. int i;
  670. if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS))
  671. return true;
  672. for (i = 0; i < BPF_REG_SIZE; i++) {
  673. if (state->stack[spi].slot_type[i] == STACK_DYNPTR ||
  674. state->stack[spi - 1].slot_type[i] == STACK_DYNPTR)
  675. return false;
  676. }
  677. return true;
  678. }
  679. bool is_dynptr_reg_valid_init(struct bpf_verifier_env *env,
  680. struct bpf_reg_state *reg)
  681. {
  682. struct bpf_func_state *state = func(env, reg);
  683. int spi = get_spi(reg->off);
  684. int i;
  685. if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) ||
  686. !state->stack[spi].spilled_ptr.dynptr.first_slot)
  687. return false;
  688. for (i = 0; i < BPF_REG_SIZE; i++) {
  689. if (state->stack[spi].slot_type[i] != STACK_DYNPTR ||
  690. state->stack[spi - 1].slot_type[i] != STACK_DYNPTR)
  691. return false;
  692. }
  693. return true;
  694. }
  695. bool is_dynptr_type_expected(struct bpf_verifier_env *env,
  696. struct bpf_reg_state *reg,
  697. enum bpf_arg_type arg_type)
  698. {
  699. struct bpf_func_state *state = func(env, reg);
  700. enum bpf_dynptr_type dynptr_type;
  701. int spi = get_spi(reg->off);
  702. /* ARG_PTR_TO_DYNPTR takes any type of dynptr */
  703. if (arg_type == ARG_PTR_TO_DYNPTR)
  704. return true;
  705. dynptr_type = arg_to_dynptr_type(arg_type);
  706. return state->stack[spi].spilled_ptr.dynptr.type == dynptr_type;
  707. }
  708. /* The reg state of a pointer or a bounded scalar was saved when
  709. * it was spilled to the stack.
  710. */
  711. static bool is_spilled_reg(const struct bpf_stack_state *stack)
  712. {
  713. return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL;
  714. }
  715. static void scrub_spilled_slot(u8 *stype)
  716. {
  717. if (*stype != STACK_INVALID)
  718. *stype = STACK_MISC;
  719. }
  720. static void print_verifier_state(struct bpf_verifier_env *env,
  721. const struct bpf_func_state *state,
  722. bool print_all)
  723. {
  724. const struct bpf_reg_state *reg;
  725. enum bpf_reg_type t;
  726. int i;
  727. if (state->frameno)
  728. verbose(env, " frame%d:", state->frameno);
  729. for (i = 0; i < MAX_BPF_REG; i++) {
  730. reg = &state->regs[i];
  731. t = reg->type;
  732. if (t == NOT_INIT)
  733. continue;
  734. if (!print_all && !reg_scratched(env, i))
  735. continue;
  736. verbose(env, " R%d", i);
  737. print_liveness(env, reg->live);
  738. verbose(env, "=");
  739. if (t == SCALAR_VALUE && reg->precise)
  740. verbose(env, "P");
  741. if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
  742. tnum_is_const(reg->var_off)) {
  743. /* reg->off should be 0 for SCALAR_VALUE */
  744. verbose(env, "%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t));
  745. verbose(env, "%lld", reg->var_off.value + reg->off);
  746. } else {
  747. const char *sep = "";
  748. verbose(env, "%s", reg_type_str(env, t));
  749. if (base_type(t) == PTR_TO_BTF_ID)
  750. verbose(env, "%s", kernel_type_name(reg->btf, reg->btf_id));
  751. verbose(env, "(");
  752. /*
  753. * _a stands for append, was shortened to avoid multiline statements below.
  754. * This macro is used to output a comma separated list of attributes.
  755. */
  756. #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, __VA_ARGS__); sep = ","; })
  757. if (reg->id)
  758. verbose_a("id=%d", reg->id);
  759. if (reg_type_may_be_refcounted_or_null(t) && reg->ref_obj_id)
  760. verbose_a("ref_obj_id=%d", reg->ref_obj_id);
  761. if (t != SCALAR_VALUE)
  762. verbose_a("off=%d", reg->off);
  763. if (type_is_pkt_pointer(t))
  764. verbose_a("r=%d", reg->range);
  765. else if (base_type(t) == CONST_PTR_TO_MAP ||
  766. base_type(t) == PTR_TO_MAP_KEY ||
  767. base_type(t) == PTR_TO_MAP_VALUE)
  768. verbose_a("ks=%d,vs=%d",
  769. reg->map_ptr->key_size,
  770. reg->map_ptr->value_size);
  771. if (tnum_is_const(reg->var_off)) {
  772. /* Typically an immediate SCALAR_VALUE, but
  773. * could be a pointer whose offset is too big
  774. * for reg->off
  775. */
  776. verbose_a("imm=%llx", reg->var_off.value);
  777. } else {
  778. if (reg->smin_value != reg->umin_value &&
  779. reg->smin_value != S64_MIN)
  780. verbose_a("smin=%lld", (long long)reg->smin_value);
  781. if (reg->smax_value != reg->umax_value &&
  782. reg->smax_value != S64_MAX)
  783. verbose_a("smax=%lld", (long long)reg->smax_value);
  784. if (reg->umin_value != 0)
  785. verbose_a("umin=%llu", (unsigned long long)reg->umin_value);
  786. if (reg->umax_value != U64_MAX)
  787. verbose_a("umax=%llu", (unsigned long long)reg->umax_value);
  788. if (!tnum_is_unknown(reg->var_off)) {
  789. char tn_buf[48];
  790. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  791. verbose_a("var_off=%s", tn_buf);
  792. }
  793. if (reg->s32_min_value != reg->smin_value &&
  794. reg->s32_min_value != S32_MIN)
  795. verbose_a("s32_min=%d", (int)(reg->s32_min_value));
  796. if (reg->s32_max_value != reg->smax_value &&
  797. reg->s32_max_value != S32_MAX)
  798. verbose_a("s32_max=%d", (int)(reg->s32_max_value));
  799. if (reg->u32_min_value != reg->umin_value &&
  800. reg->u32_min_value != U32_MIN)
  801. verbose_a("u32_min=%d", (int)(reg->u32_min_value));
  802. if (reg->u32_max_value != reg->umax_value &&
  803. reg->u32_max_value != U32_MAX)
  804. verbose_a("u32_max=%d", (int)(reg->u32_max_value));
  805. }
  806. #undef verbose_a
  807. verbose(env, ")");
  808. }
  809. }
  810. for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
  811. char types_buf[BPF_REG_SIZE + 1];
  812. bool valid = false;
  813. int j;
  814. for (j = 0; j < BPF_REG_SIZE; j++) {
  815. if (state->stack[i].slot_type[j] != STACK_INVALID)
  816. valid = true;
  817. types_buf[j] = slot_type_char[
  818. state->stack[i].slot_type[j]];
  819. }
  820. types_buf[BPF_REG_SIZE] = 0;
  821. if (!valid)
  822. continue;
  823. if (!print_all && !stack_slot_scratched(env, i))
  824. continue;
  825. verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
  826. print_liveness(env, state->stack[i].spilled_ptr.live);
  827. if (is_spilled_reg(&state->stack[i])) {
  828. reg = &state->stack[i].spilled_ptr;
  829. t = reg->type;
  830. verbose(env, "=%s", t == SCALAR_VALUE ? "" : reg_type_str(env, t));
  831. if (t == SCALAR_VALUE && reg->precise)
  832. verbose(env, "P");
  833. if (t == SCALAR_VALUE && tnum_is_const(reg->var_off))
  834. verbose(env, "%lld", reg->var_off.value + reg->off);
  835. } else {
  836. verbose(env, "=%s", types_buf);
  837. }
  838. }
  839. if (state->acquired_refs && state->refs[0].id) {
  840. verbose(env, " refs=%d", state->refs[0].id);
  841. for (i = 1; i < state->acquired_refs; i++)
  842. if (state->refs[i].id)
  843. verbose(env, ",%d", state->refs[i].id);
  844. }
  845. if (state->in_callback_fn)
  846. verbose(env, " cb");
  847. if (state->in_async_callback_fn)
  848. verbose(env, " async_cb");
  849. verbose(env, "\n");
  850. if (!print_all)
  851. mark_verifier_state_clean(env);
  852. }
  853. static inline u32 vlog_alignment(u32 pos)
  854. {
  855. return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
  856. BPF_LOG_MIN_ALIGNMENT) - pos - 1;
  857. }
  858. static void print_insn_state(struct bpf_verifier_env *env,
  859. const struct bpf_func_state *state)
  860. {
  861. if (env->prev_log_len && env->prev_log_len == env->log.len_used) {
  862. /* remove new line character */
  863. bpf_vlog_reset(&env->log, env->prev_log_len - 1);
  864. verbose(env, "%*c;", vlog_alignment(env->prev_insn_print_len), ' ');
  865. } else {
  866. verbose(env, "%d:", env->insn_idx);
  867. }
  868. print_verifier_state(env, state, false);
  869. }
  870. /* copy array src of length n * size bytes to dst. dst is reallocated if it's too
  871. * small to hold src. This is different from krealloc since we don't want to preserve
  872. * the contents of dst.
  873. *
  874. * Leaves dst untouched if src is NULL or length is zero. Returns NULL if memory could
  875. * not be allocated.
  876. */
  877. static void *copy_array(void *dst, const void *src, size_t n, size_t size, gfp_t flags)
  878. {
  879. size_t alloc_bytes;
  880. void *orig = dst;
  881. size_t bytes;
  882. if (ZERO_OR_NULL_PTR(src))
  883. goto out;
  884. if (unlikely(check_mul_overflow(n, size, &bytes)))
  885. return NULL;
  886. alloc_bytes = max(ksize(orig), kmalloc_size_roundup(bytes));
  887. dst = krealloc(orig, alloc_bytes, flags);
  888. if (!dst) {
  889. kfree(orig);
  890. return NULL;
  891. }
  892. memcpy(dst, src, bytes);
  893. out:
  894. return dst ? dst : ZERO_SIZE_PTR;
  895. }
  896. /* resize an array from old_n items to new_n items. the array is reallocated if it's too
  897. * small to hold new_n items. new items are zeroed out if the array grows.
  898. *
  899. * Contrary to krealloc_array, does not free arr if new_n is zero.
  900. */
  901. static void *realloc_array(void *arr, size_t old_n, size_t new_n, size_t size)
  902. {
  903. size_t alloc_size;
  904. void *new_arr;
  905. if (!new_n || old_n == new_n)
  906. goto out;
  907. alloc_size = kmalloc_size_roundup(size_mul(new_n, size));
  908. new_arr = krealloc(arr, alloc_size, GFP_KERNEL);
  909. if (!new_arr) {
  910. kfree(arr);
  911. return NULL;
  912. }
  913. arr = new_arr;
  914. if (new_n > old_n)
  915. memset(arr + old_n * size, 0, (new_n - old_n) * size);
  916. out:
  917. return arr ? arr : ZERO_SIZE_PTR;
  918. }
  919. static int copy_reference_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
  920. {
  921. dst->refs = copy_array(dst->refs, src->refs, src->acquired_refs,
  922. sizeof(struct bpf_reference_state), GFP_KERNEL);
  923. if (!dst->refs)
  924. return -ENOMEM;
  925. dst->acquired_refs = src->acquired_refs;
  926. return 0;
  927. }
  928. static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_state *src)
  929. {
  930. size_t n = src->allocated_stack / BPF_REG_SIZE;
  931. dst->stack = copy_array(dst->stack, src->stack, n, sizeof(struct bpf_stack_state),
  932. GFP_KERNEL);
  933. if (!dst->stack)
  934. return -ENOMEM;
  935. dst->allocated_stack = src->allocated_stack;
  936. return 0;
  937. }
  938. static int resize_reference_state(struct bpf_func_state *state, size_t n)
  939. {
  940. state->refs = realloc_array(state->refs, state->acquired_refs, n,
  941. sizeof(struct bpf_reference_state));
  942. if (!state->refs)
  943. return -ENOMEM;
  944. state->acquired_refs = n;
  945. return 0;
  946. }
  947. static int grow_stack_state(struct bpf_func_state *state, int size)
  948. {
  949. size_t old_n = state->allocated_stack / BPF_REG_SIZE, n = size / BPF_REG_SIZE;
  950. if (old_n >= n)
  951. return 0;
  952. state->stack = realloc_array(state->stack, old_n, n, sizeof(struct bpf_stack_state));
  953. if (!state->stack)
  954. return -ENOMEM;
  955. state->allocated_stack = size;
  956. return 0;
  957. }
  958. /* Acquire a pointer id from the env and update the state->refs to include
  959. * this new pointer reference.
  960. * On success, returns a valid pointer id to associate with the register
  961. * On failure, returns a negative errno.
  962. */
  963. static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
  964. {
  965. struct bpf_func_state *state = cur_func(env);
  966. int new_ofs = state->acquired_refs;
  967. int id, err;
  968. err = resize_reference_state(state, state->acquired_refs + 1);
  969. if (err)
  970. return err;
  971. id = ++env->id_gen;
  972. state->refs[new_ofs].id = id;
  973. state->refs[new_ofs].insn_idx = insn_idx;
  974. state->refs[new_ofs].callback_ref = state->in_callback_fn ? state->frameno : 0;
  975. return id;
  976. }
  977. /* release function corresponding to acquire_reference_state(). Idempotent. */
  978. static int release_reference_state(struct bpf_func_state *state, int ptr_id)
  979. {
  980. int i, last_idx;
  981. last_idx = state->acquired_refs - 1;
  982. for (i = 0; i < state->acquired_refs; i++) {
  983. if (state->refs[i].id == ptr_id) {
  984. /* Cannot release caller references in callbacks */
  985. if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
  986. return -EINVAL;
  987. if (last_idx && i != last_idx)
  988. memcpy(&state->refs[i], &state->refs[last_idx],
  989. sizeof(*state->refs));
  990. memset(&state->refs[last_idx], 0, sizeof(*state->refs));
  991. state->acquired_refs--;
  992. return 0;
  993. }
  994. }
  995. return -EINVAL;
  996. }
  997. static void free_func_state(struct bpf_func_state *state)
  998. {
  999. if (!state)
  1000. return;
  1001. kfree(state->refs);
  1002. kfree(state->stack);
  1003. kfree(state);
  1004. }
  1005. static void clear_jmp_history(struct bpf_verifier_state *state)
  1006. {
  1007. kfree(state->jmp_history);
  1008. state->jmp_history = NULL;
  1009. state->jmp_history_cnt = 0;
  1010. }
  1011. static void free_verifier_state(struct bpf_verifier_state *state,
  1012. bool free_self)
  1013. {
  1014. int i;
  1015. for (i = 0; i <= state->curframe; i++) {
  1016. free_func_state(state->frame[i]);
  1017. state->frame[i] = NULL;
  1018. }
  1019. clear_jmp_history(state);
  1020. if (free_self)
  1021. kfree(state);
  1022. }
  1023. /* copy verifier state from src to dst growing dst stack space
  1024. * when necessary to accommodate larger src stack
  1025. */
  1026. static int copy_func_state(struct bpf_func_state *dst,
  1027. const struct bpf_func_state *src)
  1028. {
  1029. int err;
  1030. memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
  1031. err = copy_reference_state(dst, src);
  1032. if (err)
  1033. return err;
  1034. return copy_stack_state(dst, src);
  1035. }
  1036. static int copy_verifier_state(struct bpf_verifier_state *dst_state,
  1037. const struct bpf_verifier_state *src)
  1038. {
  1039. struct bpf_func_state *dst;
  1040. int i, err;
  1041. dst_state->jmp_history = copy_array(dst_state->jmp_history, src->jmp_history,
  1042. src->jmp_history_cnt, sizeof(struct bpf_idx_pair),
  1043. GFP_USER);
  1044. if (!dst_state->jmp_history)
  1045. return -ENOMEM;
  1046. dst_state->jmp_history_cnt = src->jmp_history_cnt;
  1047. /* if dst has more stack frames then src frame, free them */
  1048. for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
  1049. free_func_state(dst_state->frame[i]);
  1050. dst_state->frame[i] = NULL;
  1051. }
  1052. dst_state->speculative = src->speculative;
  1053. dst_state->curframe = src->curframe;
  1054. dst_state->active_spin_lock = src->active_spin_lock;
  1055. dst_state->branches = src->branches;
  1056. dst_state->parent = src->parent;
  1057. dst_state->first_insn_idx = src->first_insn_idx;
  1058. dst_state->last_insn_idx = src->last_insn_idx;
  1059. for (i = 0; i <= src->curframe; i++) {
  1060. dst = dst_state->frame[i];
  1061. if (!dst) {
  1062. dst = kzalloc(sizeof(*dst), GFP_KERNEL);
  1063. if (!dst)
  1064. return -ENOMEM;
  1065. dst_state->frame[i] = dst;
  1066. }
  1067. err = copy_func_state(dst, src->frame[i]);
  1068. if (err)
  1069. return err;
  1070. }
  1071. return 0;
  1072. }
  1073. static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
  1074. {
  1075. while (st) {
  1076. u32 br = --st->branches;
  1077. /* WARN_ON(br > 1) technically makes sense here,
  1078. * but see comment in push_stack(), hence:
  1079. */
  1080. WARN_ONCE((int)br < 0,
  1081. "BUG update_branch_counts:branches_to_explore=%d\n",
  1082. br);
  1083. if (br)
  1084. break;
  1085. st = st->parent;
  1086. }
  1087. }
  1088. static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
  1089. int *insn_idx, bool pop_log)
  1090. {
  1091. struct bpf_verifier_state *cur = env->cur_state;
  1092. struct bpf_verifier_stack_elem *elem, *head = env->head;
  1093. int err;
  1094. if (env->head == NULL)
  1095. return -ENOENT;
  1096. if (cur) {
  1097. err = copy_verifier_state(cur, &head->st);
  1098. if (err)
  1099. return err;
  1100. }
  1101. if (pop_log)
  1102. bpf_vlog_reset(&env->log, head->log_pos);
  1103. if (insn_idx)
  1104. *insn_idx = head->insn_idx;
  1105. if (prev_insn_idx)
  1106. *prev_insn_idx = head->prev_insn_idx;
  1107. elem = head->next;
  1108. free_verifier_state(&head->st, false);
  1109. kfree(head);
  1110. env->head = elem;
  1111. env->stack_size--;
  1112. return 0;
  1113. }
  1114. static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
  1115. int insn_idx, int prev_insn_idx,
  1116. bool speculative)
  1117. {
  1118. struct bpf_verifier_state *cur = env->cur_state;
  1119. struct bpf_verifier_stack_elem *elem;
  1120. int err;
  1121. elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
  1122. if (!elem)
  1123. goto err;
  1124. elem->insn_idx = insn_idx;
  1125. elem->prev_insn_idx = prev_insn_idx;
  1126. elem->next = env->head;
  1127. elem->log_pos = env->log.len_used;
  1128. env->head = elem;
  1129. env->stack_size++;
  1130. err = copy_verifier_state(&elem->st, cur);
  1131. if (err)
  1132. goto err;
  1133. elem->st.speculative |= speculative;
  1134. if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
  1135. verbose(env, "The sequence of %d jumps is too complex.\n",
  1136. env->stack_size);
  1137. goto err;
  1138. }
  1139. if (elem->st.parent) {
  1140. ++elem->st.parent->branches;
  1141. /* WARN_ON(branches > 2) technically makes sense here,
  1142. * but
  1143. * 1. speculative states will bump 'branches' for non-branch
  1144. * instructions
  1145. * 2. is_state_visited() heuristics may decide not to create
  1146. * a new state for a sequence of branches and all such current
  1147. * and cloned states will be pointing to a single parent state
  1148. * which might have large 'branches' count.
  1149. */
  1150. }
  1151. return &elem->st;
  1152. err:
  1153. free_verifier_state(env->cur_state, true);
  1154. env->cur_state = NULL;
  1155. /* pop all elements and return */
  1156. while (!pop_stack(env, NULL, NULL, false));
  1157. return NULL;
  1158. }
  1159. #define CALLER_SAVED_REGS 6
  1160. static const int caller_saved[CALLER_SAVED_REGS] = {
  1161. BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
  1162. };
  1163. static void __mark_reg_not_init(const struct bpf_verifier_env *env,
  1164. struct bpf_reg_state *reg);
  1165. /* This helper doesn't clear reg->id */
  1166. static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
  1167. {
  1168. reg->var_off = tnum_const(imm);
  1169. reg->smin_value = (s64)imm;
  1170. reg->smax_value = (s64)imm;
  1171. reg->umin_value = imm;
  1172. reg->umax_value = imm;
  1173. reg->s32_min_value = (s32)imm;
  1174. reg->s32_max_value = (s32)imm;
  1175. reg->u32_min_value = (u32)imm;
  1176. reg->u32_max_value = (u32)imm;
  1177. }
  1178. /* Mark the unknown part of a register (variable offset or scalar value) as
  1179. * known to have the value @imm.
  1180. */
  1181. static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
  1182. {
  1183. /* Clear id, off, and union(map_ptr, range) */
  1184. memset(((u8 *)reg) + sizeof(reg->type), 0,
  1185. offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
  1186. ___mark_reg_known(reg, imm);
  1187. }
  1188. static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
  1189. {
  1190. reg->var_off = tnum_const_subreg(reg->var_off, imm);
  1191. reg->s32_min_value = (s32)imm;
  1192. reg->s32_max_value = (s32)imm;
  1193. reg->u32_min_value = (u32)imm;
  1194. reg->u32_max_value = (u32)imm;
  1195. }
  1196. /* Mark the 'variable offset' part of a register as zero. This should be
  1197. * used only on registers holding a pointer type.
  1198. */
  1199. static void __mark_reg_known_zero(struct bpf_reg_state *reg)
  1200. {
  1201. __mark_reg_known(reg, 0);
  1202. }
  1203. static void __mark_reg_const_zero(struct bpf_reg_state *reg)
  1204. {
  1205. __mark_reg_known(reg, 0);
  1206. reg->type = SCALAR_VALUE;
  1207. }
  1208. static void mark_reg_known_zero(struct bpf_verifier_env *env,
  1209. struct bpf_reg_state *regs, u32 regno)
  1210. {
  1211. if (WARN_ON(regno >= MAX_BPF_REG)) {
  1212. verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
  1213. /* Something bad happened, let's kill all regs */
  1214. for (regno = 0; regno < MAX_BPF_REG; regno++)
  1215. __mark_reg_not_init(env, regs + regno);
  1216. return;
  1217. }
  1218. __mark_reg_known_zero(regs + regno);
  1219. }
  1220. static void mark_ptr_not_null_reg(struct bpf_reg_state *reg)
  1221. {
  1222. if (base_type(reg->type) == PTR_TO_MAP_VALUE) {
  1223. const struct bpf_map *map = reg->map_ptr;
  1224. if (map->inner_map_meta) {
  1225. reg->type = CONST_PTR_TO_MAP;
  1226. reg->map_ptr = map->inner_map_meta;
  1227. /* transfer reg's id which is unique for every map_lookup_elem
  1228. * as UID of the inner map.
  1229. */
  1230. if (map_value_has_timer(map->inner_map_meta))
  1231. reg->map_uid = reg->id;
  1232. } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
  1233. reg->type = PTR_TO_XDP_SOCK;
  1234. } else if (map->map_type == BPF_MAP_TYPE_SOCKMAP ||
  1235. map->map_type == BPF_MAP_TYPE_SOCKHASH) {
  1236. reg->type = PTR_TO_SOCKET;
  1237. } else {
  1238. reg->type = PTR_TO_MAP_VALUE;
  1239. }
  1240. return;
  1241. }
  1242. reg->type &= ~PTR_MAYBE_NULL;
  1243. }
  1244. static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
  1245. {
  1246. return type_is_pkt_pointer(reg->type);
  1247. }
  1248. static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
  1249. {
  1250. return reg_is_pkt_pointer(reg) ||
  1251. reg->type == PTR_TO_PACKET_END;
  1252. }
  1253. /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
  1254. static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
  1255. enum bpf_reg_type which)
  1256. {
  1257. /* The register can already have a range from prior markings.
  1258. * This is fine as long as it hasn't been advanced from its
  1259. * origin.
  1260. */
  1261. return reg->type == which &&
  1262. reg->id == 0 &&
  1263. reg->off == 0 &&
  1264. tnum_equals_const(reg->var_off, 0);
  1265. }
  1266. /* Reset the min/max bounds of a register */
  1267. static void __mark_reg_unbounded(struct bpf_reg_state *reg)
  1268. {
  1269. reg->smin_value = S64_MIN;
  1270. reg->smax_value = S64_MAX;
  1271. reg->umin_value = 0;
  1272. reg->umax_value = U64_MAX;
  1273. reg->s32_min_value = S32_MIN;
  1274. reg->s32_max_value = S32_MAX;
  1275. reg->u32_min_value = 0;
  1276. reg->u32_max_value = U32_MAX;
  1277. }
  1278. static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
  1279. {
  1280. reg->smin_value = S64_MIN;
  1281. reg->smax_value = S64_MAX;
  1282. reg->umin_value = 0;
  1283. reg->umax_value = U64_MAX;
  1284. }
  1285. static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
  1286. {
  1287. reg->s32_min_value = S32_MIN;
  1288. reg->s32_max_value = S32_MAX;
  1289. reg->u32_min_value = 0;
  1290. reg->u32_max_value = U32_MAX;
  1291. }
  1292. static void __update_reg32_bounds(struct bpf_reg_state *reg)
  1293. {
  1294. struct tnum var32_off = tnum_subreg(reg->var_off);
  1295. /* min signed is max(sign bit) | min(other bits) */
  1296. reg->s32_min_value = max_t(s32, reg->s32_min_value,
  1297. var32_off.value | (var32_off.mask & S32_MIN));
  1298. /* max signed is min(sign bit) | max(other bits) */
  1299. reg->s32_max_value = min_t(s32, reg->s32_max_value,
  1300. var32_off.value | (var32_off.mask & S32_MAX));
  1301. reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
  1302. reg->u32_max_value = min(reg->u32_max_value,
  1303. (u32)(var32_off.value | var32_off.mask));
  1304. }
  1305. static void __update_reg64_bounds(struct bpf_reg_state *reg)
  1306. {
  1307. /* min signed is max(sign bit) | min(other bits) */
  1308. reg->smin_value = max_t(s64, reg->smin_value,
  1309. reg->var_off.value | (reg->var_off.mask & S64_MIN));
  1310. /* max signed is min(sign bit) | max(other bits) */
  1311. reg->smax_value = min_t(s64, reg->smax_value,
  1312. reg->var_off.value | (reg->var_off.mask & S64_MAX));
  1313. reg->umin_value = max(reg->umin_value, reg->var_off.value);
  1314. reg->umax_value = min(reg->umax_value,
  1315. reg->var_off.value | reg->var_off.mask);
  1316. }
  1317. static void __update_reg_bounds(struct bpf_reg_state *reg)
  1318. {
  1319. __update_reg32_bounds(reg);
  1320. __update_reg64_bounds(reg);
  1321. }
  1322. /* Uses signed min/max values to inform unsigned, and vice-versa */
  1323. static void __reg32_deduce_bounds(struct bpf_reg_state *reg)
  1324. {
  1325. /* Learn sign from signed bounds.
  1326. * If we cannot cross the sign boundary, then signed and unsigned bounds
  1327. * are the same, so combine. This works even in the negative case, e.g.
  1328. * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
  1329. */
  1330. if (reg->s32_min_value >= 0 || reg->s32_max_value < 0) {
  1331. reg->s32_min_value = reg->u32_min_value =
  1332. max_t(u32, reg->s32_min_value, reg->u32_min_value);
  1333. reg->s32_max_value = reg->u32_max_value =
  1334. min_t(u32, reg->s32_max_value, reg->u32_max_value);
  1335. return;
  1336. }
  1337. /* Learn sign from unsigned bounds. Signed bounds cross the sign
  1338. * boundary, so we must be careful.
  1339. */
  1340. if ((s32)reg->u32_max_value >= 0) {
  1341. /* Positive. We can't learn anything from the smin, but smax
  1342. * is positive, hence safe.
  1343. */
  1344. reg->s32_min_value = reg->u32_min_value;
  1345. reg->s32_max_value = reg->u32_max_value =
  1346. min_t(u32, reg->s32_max_value, reg->u32_max_value);
  1347. } else if ((s32)reg->u32_min_value < 0) {
  1348. /* Negative. We can't learn anything from the smax, but smin
  1349. * is negative, hence safe.
  1350. */
  1351. reg->s32_min_value = reg->u32_min_value =
  1352. max_t(u32, reg->s32_min_value, reg->u32_min_value);
  1353. reg->s32_max_value = reg->u32_max_value;
  1354. }
  1355. }
  1356. static void __reg64_deduce_bounds(struct bpf_reg_state *reg)
  1357. {
  1358. /* Learn sign from signed bounds.
  1359. * If we cannot cross the sign boundary, then signed and unsigned bounds
  1360. * are the same, so combine. This works even in the negative case, e.g.
  1361. * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
  1362. */
  1363. if (reg->smin_value >= 0 || reg->smax_value < 0) {
  1364. reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
  1365. reg->umin_value);
  1366. reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
  1367. reg->umax_value);
  1368. return;
  1369. }
  1370. /* Learn sign from unsigned bounds. Signed bounds cross the sign
  1371. * boundary, so we must be careful.
  1372. */
  1373. if ((s64)reg->umax_value >= 0) {
  1374. /* Positive. We can't learn anything from the smin, but smax
  1375. * is positive, hence safe.
  1376. */
  1377. reg->smin_value = reg->umin_value;
  1378. reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
  1379. reg->umax_value);
  1380. } else if ((s64)reg->umin_value < 0) {
  1381. /* Negative. We can't learn anything from the smax, but smin
  1382. * is negative, hence safe.
  1383. */
  1384. reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
  1385. reg->umin_value);
  1386. reg->smax_value = reg->umax_value;
  1387. }
  1388. }
  1389. static void __reg_deduce_bounds(struct bpf_reg_state *reg)
  1390. {
  1391. __reg32_deduce_bounds(reg);
  1392. __reg64_deduce_bounds(reg);
  1393. }
  1394. /* Attempts to improve var_off based on unsigned min/max information */
  1395. static void __reg_bound_offset(struct bpf_reg_state *reg)
  1396. {
  1397. struct tnum var64_off = tnum_intersect(reg->var_off,
  1398. tnum_range(reg->umin_value,
  1399. reg->umax_value));
  1400. struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
  1401. tnum_range(reg->u32_min_value,
  1402. reg->u32_max_value));
  1403. reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
  1404. }
  1405. static void reg_bounds_sync(struct bpf_reg_state *reg)
  1406. {
  1407. /* We might have learned new bounds from the var_off. */
  1408. __update_reg_bounds(reg);
  1409. /* We might have learned something about the sign bit. */
  1410. __reg_deduce_bounds(reg);
  1411. /* We might have learned some bits from the bounds. */
  1412. __reg_bound_offset(reg);
  1413. /* Intersecting with the old var_off might have improved our bounds
  1414. * slightly, e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
  1415. * then new var_off is (0; 0x7f...fc) which improves our umax.
  1416. */
  1417. __update_reg_bounds(reg);
  1418. }
  1419. static bool __reg32_bound_s64(s32 a)
  1420. {
  1421. return a >= 0 && a <= S32_MAX;
  1422. }
  1423. static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
  1424. {
  1425. reg->umin_value = reg->u32_min_value;
  1426. reg->umax_value = reg->u32_max_value;
  1427. /* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
  1428. * be positive otherwise set to worse case bounds and refine later
  1429. * from tnum.
  1430. */
  1431. if (__reg32_bound_s64(reg->s32_min_value) &&
  1432. __reg32_bound_s64(reg->s32_max_value)) {
  1433. reg->smin_value = reg->s32_min_value;
  1434. reg->smax_value = reg->s32_max_value;
  1435. } else {
  1436. reg->smin_value = 0;
  1437. reg->smax_value = U32_MAX;
  1438. }
  1439. }
  1440. static void __reg_combine_32_into_64(struct bpf_reg_state *reg)
  1441. {
  1442. /* special case when 64-bit register has upper 32-bit register
  1443. * zeroed. Typically happens after zext or <<32, >>32 sequence
  1444. * allowing us to use 32-bit bounds directly,
  1445. */
  1446. if (tnum_equals_const(tnum_clear_subreg(reg->var_off), 0)) {
  1447. __reg_assign_32_into_64(reg);
  1448. } else {
  1449. /* Otherwise the best we can do is push lower 32bit known and
  1450. * unknown bits into register (var_off set from jmp logic)
  1451. * then learn as much as possible from the 64-bit tnum
  1452. * known and unknown bits. The previous smin/smax bounds are
  1453. * invalid here because of jmp32 compare so mark them unknown
  1454. * so they do not impact tnum bounds calculation.
  1455. */
  1456. __mark_reg64_unbounded(reg);
  1457. }
  1458. reg_bounds_sync(reg);
  1459. }
  1460. static bool __reg64_bound_s32(s64 a)
  1461. {
  1462. return a >= S32_MIN && a <= S32_MAX;
  1463. }
  1464. static bool __reg64_bound_u32(u64 a)
  1465. {
  1466. return a >= U32_MIN && a <= U32_MAX;
  1467. }
  1468. static void __reg_combine_64_into_32(struct bpf_reg_state *reg)
  1469. {
  1470. __mark_reg32_unbounded(reg);
  1471. if (__reg64_bound_s32(reg->smin_value) && __reg64_bound_s32(reg->smax_value)) {
  1472. reg->s32_min_value = (s32)reg->smin_value;
  1473. reg->s32_max_value = (s32)reg->smax_value;
  1474. }
  1475. if (__reg64_bound_u32(reg->umin_value) && __reg64_bound_u32(reg->umax_value)) {
  1476. reg->u32_min_value = (u32)reg->umin_value;
  1477. reg->u32_max_value = (u32)reg->umax_value;
  1478. }
  1479. reg_bounds_sync(reg);
  1480. }
  1481. /* Mark a register as having a completely unknown (scalar) value. */
  1482. static void __mark_reg_unknown(const struct bpf_verifier_env *env,
  1483. struct bpf_reg_state *reg)
  1484. {
  1485. /*
  1486. * Clear type, id, off, and union(map_ptr, range) and
  1487. * padding between 'type' and union
  1488. */
  1489. memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
  1490. reg->type = SCALAR_VALUE;
  1491. reg->var_off = tnum_unknown;
  1492. reg->frameno = 0;
  1493. reg->precise = !env->bpf_capable;
  1494. __mark_reg_unbounded(reg);
  1495. }
  1496. static void mark_reg_unknown(struct bpf_verifier_env *env,
  1497. struct bpf_reg_state *regs, u32 regno)
  1498. {
  1499. if (WARN_ON(regno >= MAX_BPF_REG)) {
  1500. verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
  1501. /* Something bad happened, let's kill all regs except FP */
  1502. for (regno = 0; regno < BPF_REG_FP; regno++)
  1503. __mark_reg_not_init(env, regs + regno);
  1504. return;
  1505. }
  1506. __mark_reg_unknown(env, regs + regno);
  1507. }
  1508. static void __mark_reg_not_init(const struct bpf_verifier_env *env,
  1509. struct bpf_reg_state *reg)
  1510. {
  1511. __mark_reg_unknown(env, reg);
  1512. reg->type = NOT_INIT;
  1513. }
  1514. static void mark_reg_not_init(struct bpf_verifier_env *env,
  1515. struct bpf_reg_state *regs, u32 regno)
  1516. {
  1517. if (WARN_ON(regno >= MAX_BPF_REG)) {
  1518. verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
  1519. /* Something bad happened, let's kill all regs except FP */
  1520. for (regno = 0; regno < BPF_REG_FP; regno++)
  1521. __mark_reg_not_init(env, regs + regno);
  1522. return;
  1523. }
  1524. __mark_reg_not_init(env, regs + regno);
  1525. }
  1526. static void mark_btf_ld_reg(struct bpf_verifier_env *env,
  1527. struct bpf_reg_state *regs, u32 regno,
  1528. enum bpf_reg_type reg_type,
  1529. struct btf *btf, u32 btf_id,
  1530. enum bpf_type_flag flag)
  1531. {
  1532. if (reg_type == SCALAR_VALUE) {
  1533. mark_reg_unknown(env, regs, regno);
  1534. return;
  1535. }
  1536. mark_reg_known_zero(env, regs, regno);
  1537. regs[regno].type = PTR_TO_BTF_ID | flag;
  1538. regs[regno].btf = btf;
  1539. regs[regno].btf_id = btf_id;
  1540. }
  1541. #define DEF_NOT_SUBREG (0)
  1542. static void init_reg_state(struct bpf_verifier_env *env,
  1543. struct bpf_func_state *state)
  1544. {
  1545. struct bpf_reg_state *regs = state->regs;
  1546. int i;
  1547. for (i = 0; i < MAX_BPF_REG; i++) {
  1548. mark_reg_not_init(env, regs, i);
  1549. regs[i].live = REG_LIVE_NONE;
  1550. regs[i].parent = NULL;
  1551. regs[i].subreg_def = DEF_NOT_SUBREG;
  1552. }
  1553. /* frame pointer */
  1554. regs[BPF_REG_FP].type = PTR_TO_STACK;
  1555. mark_reg_known_zero(env, regs, BPF_REG_FP);
  1556. regs[BPF_REG_FP].frameno = state->frameno;
  1557. }
  1558. #define BPF_MAIN_FUNC (-1)
  1559. static void init_func_state(struct bpf_verifier_env *env,
  1560. struct bpf_func_state *state,
  1561. int callsite, int frameno, int subprogno)
  1562. {
  1563. state->callsite = callsite;
  1564. state->frameno = frameno;
  1565. state->subprogno = subprogno;
  1566. state->callback_ret_range = tnum_range(0, 0);
  1567. init_reg_state(env, state);
  1568. mark_verifier_state_scratched(env);
  1569. }
  1570. /* Similar to push_stack(), but for async callbacks */
  1571. static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env,
  1572. int insn_idx, int prev_insn_idx,
  1573. int subprog)
  1574. {
  1575. struct bpf_verifier_stack_elem *elem;
  1576. struct bpf_func_state *frame;
  1577. elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
  1578. if (!elem)
  1579. goto err;
  1580. elem->insn_idx = insn_idx;
  1581. elem->prev_insn_idx = prev_insn_idx;
  1582. elem->next = env->head;
  1583. elem->log_pos = env->log.len_used;
  1584. env->head = elem;
  1585. env->stack_size++;
  1586. if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
  1587. verbose(env,
  1588. "The sequence of %d jumps is too complex for async cb.\n",
  1589. env->stack_size);
  1590. goto err;
  1591. }
  1592. /* Unlike push_stack() do not copy_verifier_state().
  1593. * The caller state doesn't matter.
  1594. * This is async callback. It starts in a fresh stack.
  1595. * Initialize it similar to do_check_common().
  1596. */
  1597. elem->st.branches = 1;
  1598. frame = kzalloc(sizeof(*frame), GFP_KERNEL);
  1599. if (!frame)
  1600. goto err;
  1601. init_func_state(env, frame,
  1602. BPF_MAIN_FUNC /* callsite */,
  1603. 0 /* frameno within this callchain */,
  1604. subprog /* subprog number within this prog */);
  1605. elem->st.frame[0] = frame;
  1606. return &elem->st;
  1607. err:
  1608. free_verifier_state(env->cur_state, true);
  1609. env->cur_state = NULL;
  1610. /* pop all elements and return */
  1611. while (!pop_stack(env, NULL, NULL, false));
  1612. return NULL;
  1613. }
  1614. enum reg_arg_type {
  1615. SRC_OP, /* register is used as source operand */
  1616. DST_OP, /* register is used as destination operand */
  1617. DST_OP_NO_MARK /* same as above, check only, don't mark */
  1618. };
  1619. static int cmp_subprogs(const void *a, const void *b)
  1620. {
  1621. return ((struct bpf_subprog_info *)a)->start -
  1622. ((struct bpf_subprog_info *)b)->start;
  1623. }
  1624. static int find_subprog(struct bpf_verifier_env *env, int off)
  1625. {
  1626. struct bpf_subprog_info *p;
  1627. p = bsearch(&off, env->subprog_info, env->subprog_cnt,
  1628. sizeof(env->subprog_info[0]), cmp_subprogs);
  1629. if (!p)
  1630. return -ENOENT;
  1631. return p - env->subprog_info;
  1632. }
  1633. static int add_subprog(struct bpf_verifier_env *env, int off)
  1634. {
  1635. int insn_cnt = env->prog->len;
  1636. int ret;
  1637. if (off >= insn_cnt || off < 0) {
  1638. verbose(env, "call to invalid destination\n");
  1639. return -EINVAL;
  1640. }
  1641. ret = find_subprog(env, off);
  1642. if (ret >= 0)
  1643. return ret;
  1644. if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
  1645. verbose(env, "too many subprograms\n");
  1646. return -E2BIG;
  1647. }
  1648. /* determine subprog starts. The end is one before the next starts */
  1649. env->subprog_info[env->subprog_cnt++].start = off;
  1650. sort(env->subprog_info, env->subprog_cnt,
  1651. sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
  1652. return env->subprog_cnt - 1;
  1653. }
  1654. #define MAX_KFUNC_DESCS 256
  1655. #define MAX_KFUNC_BTFS 256
  1656. struct bpf_kfunc_desc {
  1657. struct btf_func_model func_model;
  1658. u32 func_id;
  1659. s32 imm;
  1660. u16 offset;
  1661. };
  1662. struct bpf_kfunc_btf {
  1663. struct btf *btf;
  1664. struct module *module;
  1665. u16 offset;
  1666. };
  1667. struct bpf_kfunc_desc_tab {
  1668. struct bpf_kfunc_desc descs[MAX_KFUNC_DESCS];
  1669. u32 nr_descs;
  1670. };
  1671. struct bpf_kfunc_btf_tab {
  1672. struct bpf_kfunc_btf descs[MAX_KFUNC_BTFS];
  1673. u32 nr_descs;
  1674. };
  1675. static int kfunc_desc_cmp_by_id_off(const void *a, const void *b)
  1676. {
  1677. const struct bpf_kfunc_desc *d0 = a;
  1678. const struct bpf_kfunc_desc *d1 = b;
  1679. /* func_id is not greater than BTF_MAX_TYPE */
  1680. return d0->func_id - d1->func_id ?: d0->offset - d1->offset;
  1681. }
  1682. static int kfunc_btf_cmp_by_off(const void *a, const void *b)
  1683. {
  1684. const struct bpf_kfunc_btf *d0 = a;
  1685. const struct bpf_kfunc_btf *d1 = b;
  1686. return d0->offset - d1->offset;
  1687. }
  1688. static const struct bpf_kfunc_desc *
  1689. find_kfunc_desc(const struct bpf_prog *prog, u32 func_id, u16 offset)
  1690. {
  1691. struct bpf_kfunc_desc desc = {
  1692. .func_id = func_id,
  1693. .offset = offset,
  1694. };
  1695. struct bpf_kfunc_desc_tab *tab;
  1696. tab = prog->aux->kfunc_tab;
  1697. return bsearch(&desc, tab->descs, tab->nr_descs,
  1698. sizeof(tab->descs[0]), kfunc_desc_cmp_by_id_off);
  1699. }
  1700. static struct btf *__find_kfunc_desc_btf(struct bpf_verifier_env *env,
  1701. s16 offset)
  1702. {
  1703. struct bpf_kfunc_btf kf_btf = { .offset = offset };
  1704. struct bpf_kfunc_btf_tab *tab;
  1705. struct bpf_kfunc_btf *b;
  1706. struct module *mod;
  1707. struct btf *btf;
  1708. int btf_fd;
  1709. tab = env->prog->aux->kfunc_btf_tab;
  1710. b = bsearch(&kf_btf, tab->descs, tab->nr_descs,
  1711. sizeof(tab->descs[0]), kfunc_btf_cmp_by_off);
  1712. if (!b) {
  1713. if (tab->nr_descs == MAX_KFUNC_BTFS) {
  1714. verbose(env, "too many different module BTFs\n");
  1715. return ERR_PTR(-E2BIG);
  1716. }
  1717. if (bpfptr_is_null(env->fd_array)) {
  1718. verbose(env, "kfunc offset > 0 without fd_array is invalid\n");
  1719. return ERR_PTR(-EPROTO);
  1720. }
  1721. if (copy_from_bpfptr_offset(&btf_fd, env->fd_array,
  1722. offset * sizeof(btf_fd),
  1723. sizeof(btf_fd)))
  1724. return ERR_PTR(-EFAULT);
  1725. btf = btf_get_by_fd(btf_fd);
  1726. if (IS_ERR(btf)) {
  1727. verbose(env, "invalid module BTF fd specified\n");
  1728. return btf;
  1729. }
  1730. if (!btf_is_module(btf)) {
  1731. verbose(env, "BTF fd for kfunc is not a module BTF\n");
  1732. btf_put(btf);
  1733. return ERR_PTR(-EINVAL);
  1734. }
  1735. mod = btf_try_get_module(btf);
  1736. if (!mod) {
  1737. btf_put(btf);
  1738. return ERR_PTR(-ENXIO);
  1739. }
  1740. b = &tab->descs[tab->nr_descs++];
  1741. b->btf = btf;
  1742. b->module = mod;
  1743. b->offset = offset;
  1744. sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
  1745. kfunc_btf_cmp_by_off, NULL);
  1746. }
  1747. return b->btf;
  1748. }
  1749. void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab)
  1750. {
  1751. if (!tab)
  1752. return;
  1753. while (tab->nr_descs--) {
  1754. module_put(tab->descs[tab->nr_descs].module);
  1755. btf_put(tab->descs[tab->nr_descs].btf);
  1756. }
  1757. kfree(tab);
  1758. }
  1759. static struct btf *find_kfunc_desc_btf(struct bpf_verifier_env *env, s16 offset)
  1760. {
  1761. if (offset) {
  1762. if (offset < 0) {
  1763. /* In the future, this can be allowed to increase limit
  1764. * of fd index into fd_array, interpreted as u16.
  1765. */
  1766. verbose(env, "negative offset disallowed for kernel module function call\n");
  1767. return ERR_PTR(-EINVAL);
  1768. }
  1769. return __find_kfunc_desc_btf(env, offset);
  1770. }
  1771. return btf_vmlinux ?: ERR_PTR(-ENOENT);
  1772. }
  1773. static int add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, s16 offset)
  1774. {
  1775. const struct btf_type *func, *func_proto;
  1776. struct bpf_kfunc_btf_tab *btf_tab;
  1777. struct bpf_kfunc_desc_tab *tab;
  1778. struct bpf_prog_aux *prog_aux;
  1779. struct bpf_kfunc_desc *desc;
  1780. const char *func_name;
  1781. struct btf *desc_btf;
  1782. unsigned long call_imm;
  1783. unsigned long addr;
  1784. int err;
  1785. prog_aux = env->prog->aux;
  1786. tab = prog_aux->kfunc_tab;
  1787. btf_tab = prog_aux->kfunc_btf_tab;
  1788. if (!tab) {
  1789. if (!btf_vmlinux) {
  1790. verbose(env, "calling kernel function is not supported without CONFIG_DEBUG_INFO_BTF\n");
  1791. return -ENOTSUPP;
  1792. }
  1793. if (!env->prog->jit_requested) {
  1794. verbose(env, "JIT is required for calling kernel function\n");
  1795. return -ENOTSUPP;
  1796. }
  1797. if (!bpf_jit_supports_kfunc_call()) {
  1798. verbose(env, "JIT does not support calling kernel function\n");
  1799. return -ENOTSUPP;
  1800. }
  1801. if (!env->prog->gpl_compatible) {
  1802. verbose(env, "cannot call kernel function from non-GPL compatible program\n");
  1803. return -EINVAL;
  1804. }
  1805. tab = kzalloc(sizeof(*tab), GFP_KERNEL);
  1806. if (!tab)
  1807. return -ENOMEM;
  1808. prog_aux->kfunc_tab = tab;
  1809. }
  1810. /* func_id == 0 is always invalid, but instead of returning an error, be
  1811. * conservative and wait until the code elimination pass before returning
  1812. * error, so that invalid calls that get pruned out can be in BPF programs
  1813. * loaded from userspace. It is also required that offset be untouched
  1814. * for such calls.
  1815. */
  1816. if (!func_id && !offset)
  1817. return 0;
  1818. if (!btf_tab && offset) {
  1819. btf_tab = kzalloc(sizeof(*btf_tab), GFP_KERNEL);
  1820. if (!btf_tab)
  1821. return -ENOMEM;
  1822. prog_aux->kfunc_btf_tab = btf_tab;
  1823. }
  1824. desc_btf = find_kfunc_desc_btf(env, offset);
  1825. if (IS_ERR(desc_btf)) {
  1826. verbose(env, "failed to find BTF for kernel function\n");
  1827. return PTR_ERR(desc_btf);
  1828. }
  1829. if (find_kfunc_desc(env->prog, func_id, offset))
  1830. return 0;
  1831. if (tab->nr_descs == MAX_KFUNC_DESCS) {
  1832. verbose(env, "too many different kernel function calls\n");
  1833. return -E2BIG;
  1834. }
  1835. func = btf_type_by_id(desc_btf, func_id);
  1836. if (!func || !btf_type_is_func(func)) {
  1837. verbose(env, "kernel btf_id %u is not a function\n",
  1838. func_id);
  1839. return -EINVAL;
  1840. }
  1841. func_proto = btf_type_by_id(desc_btf, func->type);
  1842. if (!func_proto || !btf_type_is_func_proto(func_proto)) {
  1843. verbose(env, "kernel function btf_id %u does not have a valid func_proto\n",
  1844. func_id);
  1845. return -EINVAL;
  1846. }
  1847. func_name = btf_name_by_offset(desc_btf, func->name_off);
  1848. addr = kallsyms_lookup_name(func_name);
  1849. if (!addr) {
  1850. verbose(env, "cannot find address for kernel function %s\n",
  1851. func_name);
  1852. return -EINVAL;
  1853. }
  1854. call_imm = BPF_CALL_IMM(addr);
  1855. /* Check whether or not the relative offset overflows desc->imm */
  1856. if ((unsigned long)(s32)call_imm != call_imm) {
  1857. verbose(env, "address of kernel function %s is out of range\n",
  1858. func_name);
  1859. return -EINVAL;
  1860. }
  1861. desc = &tab->descs[tab->nr_descs++];
  1862. desc->func_id = func_id;
  1863. desc->imm = call_imm;
  1864. desc->offset = offset;
  1865. err = btf_distill_func_proto(&env->log, desc_btf,
  1866. func_proto, func_name,
  1867. &desc->func_model);
  1868. if (!err)
  1869. sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
  1870. kfunc_desc_cmp_by_id_off, NULL);
  1871. return err;
  1872. }
  1873. static int kfunc_desc_cmp_by_imm(const void *a, const void *b)
  1874. {
  1875. const struct bpf_kfunc_desc *d0 = a;
  1876. const struct bpf_kfunc_desc *d1 = b;
  1877. if (d0->imm > d1->imm)
  1878. return 1;
  1879. else if (d0->imm < d1->imm)
  1880. return -1;
  1881. return 0;
  1882. }
  1883. static void sort_kfunc_descs_by_imm(struct bpf_prog *prog)
  1884. {
  1885. struct bpf_kfunc_desc_tab *tab;
  1886. tab = prog->aux->kfunc_tab;
  1887. if (!tab)
  1888. return;
  1889. sort(tab->descs, tab->nr_descs, sizeof(tab->descs[0]),
  1890. kfunc_desc_cmp_by_imm, NULL);
  1891. }
  1892. bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog)
  1893. {
  1894. return !!prog->aux->kfunc_tab;
  1895. }
  1896. const struct btf_func_model *
  1897. bpf_jit_find_kfunc_model(const struct bpf_prog *prog,
  1898. const struct bpf_insn *insn)
  1899. {
  1900. const struct bpf_kfunc_desc desc = {
  1901. .imm = insn->imm,
  1902. };
  1903. const struct bpf_kfunc_desc *res;
  1904. struct bpf_kfunc_desc_tab *tab;
  1905. tab = prog->aux->kfunc_tab;
  1906. res = bsearch(&desc, tab->descs, tab->nr_descs,
  1907. sizeof(tab->descs[0]), kfunc_desc_cmp_by_imm);
  1908. return res ? &res->func_model : NULL;
  1909. }
  1910. static int add_subprog_and_kfunc(struct bpf_verifier_env *env)
  1911. {
  1912. struct bpf_subprog_info *subprog = env->subprog_info;
  1913. struct bpf_insn *insn = env->prog->insnsi;
  1914. int i, ret, insn_cnt = env->prog->len;
  1915. /* Add entry function. */
  1916. ret = add_subprog(env, 0);
  1917. if (ret)
  1918. return ret;
  1919. for (i = 0; i < insn_cnt; i++, insn++) {
  1920. if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn) &&
  1921. !bpf_pseudo_kfunc_call(insn))
  1922. continue;
  1923. if (!env->bpf_capable) {
  1924. verbose(env, "loading/calling other bpf or kernel functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
  1925. return -EPERM;
  1926. }
  1927. if (bpf_pseudo_func(insn) || bpf_pseudo_call(insn))
  1928. ret = add_subprog(env, i + insn->imm + 1);
  1929. else
  1930. ret = add_kfunc_call(env, insn->imm, insn->off);
  1931. if (ret < 0)
  1932. return ret;
  1933. }
  1934. /* Add a fake 'exit' subprog which could simplify subprog iteration
  1935. * logic. 'subprog_cnt' should not be increased.
  1936. */
  1937. subprog[env->subprog_cnt].start = insn_cnt;
  1938. if (env->log.level & BPF_LOG_LEVEL2)
  1939. for (i = 0; i < env->subprog_cnt; i++)
  1940. verbose(env, "func#%d @%d\n", i, subprog[i].start);
  1941. return 0;
  1942. }
  1943. static int check_subprogs(struct bpf_verifier_env *env)
  1944. {
  1945. int i, subprog_start, subprog_end, off, cur_subprog = 0;
  1946. struct bpf_subprog_info *subprog = env->subprog_info;
  1947. struct bpf_insn *insn = env->prog->insnsi;
  1948. int insn_cnt = env->prog->len;
  1949. /* now check that all jumps are within the same subprog */
  1950. subprog_start = subprog[cur_subprog].start;
  1951. subprog_end = subprog[cur_subprog + 1].start;
  1952. for (i = 0; i < insn_cnt; i++) {
  1953. u8 code = insn[i].code;
  1954. if (code == (BPF_JMP | BPF_CALL) &&
  1955. insn[i].imm == BPF_FUNC_tail_call &&
  1956. insn[i].src_reg != BPF_PSEUDO_CALL)
  1957. subprog[cur_subprog].has_tail_call = true;
  1958. if (BPF_CLASS(code) == BPF_LD &&
  1959. (BPF_MODE(code) == BPF_ABS || BPF_MODE(code) == BPF_IND))
  1960. subprog[cur_subprog].has_ld_abs = true;
  1961. if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
  1962. goto next;
  1963. if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
  1964. goto next;
  1965. off = i + insn[i].off + 1;
  1966. if (off < subprog_start || off >= subprog_end) {
  1967. verbose(env, "jump out of range from insn %d to %d\n", i, off);
  1968. return -EINVAL;
  1969. }
  1970. next:
  1971. if (i == subprog_end - 1) {
  1972. /* to avoid fall-through from one subprog into another
  1973. * the last insn of the subprog should be either exit
  1974. * or unconditional jump back
  1975. */
  1976. if (code != (BPF_JMP | BPF_EXIT) &&
  1977. code != (BPF_JMP | BPF_JA)) {
  1978. verbose(env, "last insn is not an exit or jmp\n");
  1979. return -EINVAL;
  1980. }
  1981. subprog_start = subprog_end;
  1982. cur_subprog++;
  1983. if (cur_subprog < env->subprog_cnt)
  1984. subprog_end = subprog[cur_subprog + 1].start;
  1985. }
  1986. }
  1987. return 0;
  1988. }
  1989. /* Parentage chain of this register (or stack slot) should take care of all
  1990. * issues like callee-saved registers, stack slot allocation time, etc.
  1991. */
  1992. static int mark_reg_read(struct bpf_verifier_env *env,
  1993. const struct bpf_reg_state *state,
  1994. struct bpf_reg_state *parent, u8 flag)
  1995. {
  1996. bool writes = parent == state->parent; /* Observe write marks */
  1997. int cnt = 0;
  1998. while (parent) {
  1999. /* if read wasn't screened by an earlier write ... */
  2000. if (writes && state->live & REG_LIVE_WRITTEN)
  2001. break;
  2002. if (parent->live & REG_LIVE_DONE) {
  2003. verbose(env, "verifier BUG type %s var_off %lld off %d\n",
  2004. reg_type_str(env, parent->type),
  2005. parent->var_off.value, parent->off);
  2006. return -EFAULT;
  2007. }
  2008. /* The first condition is more likely to be true than the
  2009. * second, checked it first.
  2010. */
  2011. if ((parent->live & REG_LIVE_READ) == flag ||
  2012. parent->live & REG_LIVE_READ64)
  2013. /* The parentage chain never changes and
  2014. * this parent was already marked as LIVE_READ.
  2015. * There is no need to keep walking the chain again and
  2016. * keep re-marking all parents as LIVE_READ.
  2017. * This case happens when the same register is read
  2018. * multiple times without writes into it in-between.
  2019. * Also, if parent has the stronger REG_LIVE_READ64 set,
  2020. * then no need to set the weak REG_LIVE_READ32.
  2021. */
  2022. break;
  2023. /* ... then we depend on parent's value */
  2024. parent->live |= flag;
  2025. /* REG_LIVE_READ64 overrides REG_LIVE_READ32. */
  2026. if (flag == REG_LIVE_READ64)
  2027. parent->live &= ~REG_LIVE_READ32;
  2028. state = parent;
  2029. parent = state->parent;
  2030. writes = true;
  2031. cnt++;
  2032. }
  2033. if (env->longest_mark_read_walk < cnt)
  2034. env->longest_mark_read_walk = cnt;
  2035. return 0;
  2036. }
  2037. /* This function is supposed to be used by the following 32-bit optimization
  2038. * code only. It returns TRUE if the source or destination register operates
  2039. * on 64-bit, otherwise return FALSE.
  2040. */
  2041. static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn,
  2042. u32 regno, struct bpf_reg_state *reg, enum reg_arg_type t)
  2043. {
  2044. u8 code, class, op;
  2045. code = insn->code;
  2046. class = BPF_CLASS(code);
  2047. op = BPF_OP(code);
  2048. if (class == BPF_JMP) {
  2049. /* BPF_EXIT for "main" will reach here. Return TRUE
  2050. * conservatively.
  2051. */
  2052. if (op == BPF_EXIT)
  2053. return true;
  2054. if (op == BPF_CALL) {
  2055. /* BPF to BPF call will reach here because of marking
  2056. * caller saved clobber with DST_OP_NO_MARK for which we
  2057. * don't care the register def because they are anyway
  2058. * marked as NOT_INIT already.
  2059. */
  2060. if (insn->src_reg == BPF_PSEUDO_CALL)
  2061. return false;
  2062. /* Helper call will reach here because of arg type
  2063. * check, conservatively return TRUE.
  2064. */
  2065. if (t == SRC_OP)
  2066. return true;
  2067. return false;
  2068. }
  2069. }
  2070. if (class == BPF_ALU64 || class == BPF_JMP ||
  2071. /* BPF_END always use BPF_ALU class. */
  2072. (class == BPF_ALU && op == BPF_END && insn->imm == 64))
  2073. return true;
  2074. if (class == BPF_ALU || class == BPF_JMP32)
  2075. return false;
  2076. if (class == BPF_LDX) {
  2077. if (t != SRC_OP)
  2078. return BPF_SIZE(code) == BPF_DW;
  2079. /* LDX source must be ptr. */
  2080. return true;
  2081. }
  2082. if (class == BPF_STX) {
  2083. /* BPF_STX (including atomic variants) has multiple source
  2084. * operands, one of which is a ptr. Check whether the caller is
  2085. * asking about it.
  2086. */
  2087. if (t == SRC_OP && reg->type != SCALAR_VALUE)
  2088. return true;
  2089. return BPF_SIZE(code) == BPF_DW;
  2090. }
  2091. if (class == BPF_LD) {
  2092. u8 mode = BPF_MODE(code);
  2093. /* LD_IMM64 */
  2094. if (mode == BPF_IMM)
  2095. return true;
  2096. /* Both LD_IND and LD_ABS return 32-bit data. */
  2097. if (t != SRC_OP)
  2098. return false;
  2099. /* Implicit ctx ptr. */
  2100. if (regno == BPF_REG_6)
  2101. return true;
  2102. /* Explicit source could be any width. */
  2103. return true;
  2104. }
  2105. if (class == BPF_ST)
  2106. /* The only source register for BPF_ST is a ptr. */
  2107. return true;
  2108. /* Conservatively return true at default. */
  2109. return true;
  2110. }
  2111. /* Return the regno defined by the insn, or -1. */
  2112. static int insn_def_regno(const struct bpf_insn *insn)
  2113. {
  2114. switch (BPF_CLASS(insn->code)) {
  2115. case BPF_JMP:
  2116. case BPF_JMP32:
  2117. case BPF_ST:
  2118. return -1;
  2119. case BPF_STX:
  2120. if (BPF_MODE(insn->code) == BPF_ATOMIC &&
  2121. (insn->imm & BPF_FETCH)) {
  2122. if (insn->imm == BPF_CMPXCHG)
  2123. return BPF_REG_0;
  2124. else
  2125. return insn->src_reg;
  2126. } else {
  2127. return -1;
  2128. }
  2129. default:
  2130. return insn->dst_reg;
  2131. }
  2132. }
  2133. /* Return TRUE if INSN has defined any 32-bit value explicitly. */
  2134. static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
  2135. {
  2136. int dst_reg = insn_def_regno(insn);
  2137. if (dst_reg == -1)
  2138. return false;
  2139. return !is_reg64(env, insn, dst_reg, NULL, DST_OP);
  2140. }
  2141. static void mark_insn_zext(struct bpf_verifier_env *env,
  2142. struct bpf_reg_state *reg)
  2143. {
  2144. s32 def_idx = reg->subreg_def;
  2145. if (def_idx == DEF_NOT_SUBREG)
  2146. return;
  2147. env->insn_aux_data[def_idx - 1].zext_dst = true;
  2148. /* The dst will be zero extended, so won't be sub-register anymore. */
  2149. reg->subreg_def = DEF_NOT_SUBREG;
  2150. }
  2151. static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
  2152. enum reg_arg_type t)
  2153. {
  2154. struct bpf_verifier_state *vstate = env->cur_state;
  2155. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  2156. struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
  2157. struct bpf_reg_state *reg, *regs = state->regs;
  2158. bool rw64;
  2159. if (regno >= MAX_BPF_REG) {
  2160. verbose(env, "R%d is invalid\n", regno);
  2161. return -EINVAL;
  2162. }
  2163. mark_reg_scratched(env, regno);
  2164. reg = &regs[regno];
  2165. rw64 = is_reg64(env, insn, regno, reg, t);
  2166. if (t == SRC_OP) {
  2167. /* check whether register used as source operand can be read */
  2168. if (reg->type == NOT_INIT) {
  2169. verbose(env, "R%d !read_ok\n", regno);
  2170. return -EACCES;
  2171. }
  2172. /* We don't need to worry about FP liveness because it's read-only */
  2173. if (regno == BPF_REG_FP)
  2174. return 0;
  2175. if (rw64)
  2176. mark_insn_zext(env, reg);
  2177. return mark_reg_read(env, reg, reg->parent,
  2178. rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
  2179. } else {
  2180. /* check whether register used as dest operand can be written to */
  2181. if (regno == BPF_REG_FP) {
  2182. verbose(env, "frame pointer is read only\n");
  2183. return -EACCES;
  2184. }
  2185. reg->live |= REG_LIVE_WRITTEN;
  2186. reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
  2187. if (t == DST_OP)
  2188. mark_reg_unknown(env, regs, regno);
  2189. }
  2190. return 0;
  2191. }
  2192. /* for any branch, call, exit record the history of jmps in the given state */
  2193. static int push_jmp_history(struct bpf_verifier_env *env,
  2194. struct bpf_verifier_state *cur)
  2195. {
  2196. u32 cnt = cur->jmp_history_cnt;
  2197. struct bpf_idx_pair *p;
  2198. size_t alloc_size;
  2199. cnt++;
  2200. alloc_size = kmalloc_size_roundup(size_mul(cnt, sizeof(*p)));
  2201. p = krealloc(cur->jmp_history, alloc_size, GFP_USER);
  2202. if (!p)
  2203. return -ENOMEM;
  2204. p[cnt - 1].idx = env->insn_idx;
  2205. p[cnt - 1].prev_idx = env->prev_insn_idx;
  2206. cur->jmp_history = p;
  2207. cur->jmp_history_cnt = cnt;
  2208. return 0;
  2209. }
  2210. /* Backtrack one insn at a time. If idx is not at the top of recorded
  2211. * history then previous instruction came from straight line execution.
  2212. */
  2213. static int get_prev_insn_idx(struct bpf_verifier_state *st, int i,
  2214. u32 *history)
  2215. {
  2216. u32 cnt = *history;
  2217. if (cnt && st->jmp_history[cnt - 1].idx == i) {
  2218. i = st->jmp_history[cnt - 1].prev_idx;
  2219. (*history)--;
  2220. } else {
  2221. i--;
  2222. }
  2223. return i;
  2224. }
  2225. static const char *disasm_kfunc_name(void *data, const struct bpf_insn *insn)
  2226. {
  2227. const struct btf_type *func;
  2228. struct btf *desc_btf;
  2229. if (insn->src_reg != BPF_PSEUDO_KFUNC_CALL)
  2230. return NULL;
  2231. desc_btf = find_kfunc_desc_btf(data, insn->off);
  2232. if (IS_ERR(desc_btf))
  2233. return "<error>";
  2234. func = btf_type_by_id(desc_btf, insn->imm);
  2235. return btf_name_by_offset(desc_btf, func->name_off);
  2236. }
  2237. /* For given verifier state backtrack_insn() is called from the last insn to
  2238. * the first insn. Its purpose is to compute a bitmask of registers and
  2239. * stack slots that needs precision in the parent verifier state.
  2240. */
  2241. static int backtrack_insn(struct bpf_verifier_env *env, int idx,
  2242. u32 *reg_mask, u64 *stack_mask)
  2243. {
  2244. const struct bpf_insn_cbs cbs = {
  2245. .cb_call = disasm_kfunc_name,
  2246. .cb_print = verbose,
  2247. .private_data = env,
  2248. };
  2249. struct bpf_insn *insn = env->prog->insnsi + idx;
  2250. u8 class = BPF_CLASS(insn->code);
  2251. u8 opcode = BPF_OP(insn->code);
  2252. u8 mode = BPF_MODE(insn->code);
  2253. u32 dreg = 1u << insn->dst_reg;
  2254. u32 sreg = 1u << insn->src_reg;
  2255. u32 spi;
  2256. if (insn->code == 0)
  2257. return 0;
  2258. if (env->log.level & BPF_LOG_LEVEL2) {
  2259. verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
  2260. verbose(env, "%d: ", idx);
  2261. print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
  2262. }
  2263. if (class == BPF_ALU || class == BPF_ALU64) {
  2264. if (!(*reg_mask & dreg))
  2265. return 0;
  2266. if (opcode == BPF_END || opcode == BPF_NEG) {
  2267. /* sreg is reserved and unused
  2268. * dreg still need precision before this insn
  2269. */
  2270. return 0;
  2271. } else if (opcode == BPF_MOV) {
  2272. if (BPF_SRC(insn->code) == BPF_X) {
  2273. /* dreg = sreg
  2274. * dreg needs precision after this insn
  2275. * sreg needs precision before this insn
  2276. */
  2277. *reg_mask &= ~dreg;
  2278. *reg_mask |= sreg;
  2279. } else {
  2280. /* dreg = K
  2281. * dreg needs precision after this insn.
  2282. * Corresponding register is already marked
  2283. * as precise=true in this verifier state.
  2284. * No further markings in parent are necessary
  2285. */
  2286. *reg_mask &= ~dreg;
  2287. }
  2288. } else {
  2289. if (BPF_SRC(insn->code) == BPF_X) {
  2290. /* dreg += sreg
  2291. * both dreg and sreg need precision
  2292. * before this insn
  2293. */
  2294. *reg_mask |= sreg;
  2295. } /* else dreg += K
  2296. * dreg still needs precision before this insn
  2297. */
  2298. }
  2299. } else if (class == BPF_LDX) {
  2300. if (!(*reg_mask & dreg))
  2301. return 0;
  2302. *reg_mask &= ~dreg;
  2303. /* scalars can only be spilled into stack w/o losing precision.
  2304. * Load from any other memory can be zero extended.
  2305. * The desire to keep that precision is already indicated
  2306. * by 'precise' mark in corresponding register of this state.
  2307. * No further tracking necessary.
  2308. */
  2309. if (insn->src_reg != BPF_REG_FP)
  2310. return 0;
  2311. /* dreg = *(u64 *)[fp - off] was a fill from the stack.
  2312. * that [fp - off] slot contains scalar that needs to be
  2313. * tracked with precision
  2314. */
  2315. spi = (-insn->off - 1) / BPF_REG_SIZE;
  2316. if (spi >= 64) {
  2317. verbose(env, "BUG spi %d\n", spi);
  2318. WARN_ONCE(1, "verifier backtracking bug");
  2319. return -EFAULT;
  2320. }
  2321. *stack_mask |= 1ull << spi;
  2322. } else if (class == BPF_STX || class == BPF_ST) {
  2323. if (*reg_mask & dreg)
  2324. /* stx & st shouldn't be using _scalar_ dst_reg
  2325. * to access memory. It means backtracking
  2326. * encountered a case of pointer subtraction.
  2327. */
  2328. return -ENOTSUPP;
  2329. /* scalars can only be spilled into stack */
  2330. if (insn->dst_reg != BPF_REG_FP)
  2331. return 0;
  2332. spi = (-insn->off - 1) / BPF_REG_SIZE;
  2333. if (spi >= 64) {
  2334. verbose(env, "BUG spi %d\n", spi);
  2335. WARN_ONCE(1, "verifier backtracking bug");
  2336. return -EFAULT;
  2337. }
  2338. if (!(*stack_mask & (1ull << spi)))
  2339. return 0;
  2340. *stack_mask &= ~(1ull << spi);
  2341. if (class == BPF_STX)
  2342. *reg_mask |= sreg;
  2343. } else if (class == BPF_JMP || class == BPF_JMP32) {
  2344. if (opcode == BPF_CALL) {
  2345. if (insn->src_reg == BPF_PSEUDO_CALL)
  2346. return -ENOTSUPP;
  2347. /* kfunc with imm==0 is invalid and fixup_kfunc_call will
  2348. * catch this error later. Make backtracking conservative
  2349. * with ENOTSUPP.
  2350. */
  2351. if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL && insn->imm == 0)
  2352. return -ENOTSUPP;
  2353. /* BPF helpers that invoke callback subprogs are
  2354. * equivalent to BPF_PSEUDO_CALL above
  2355. */
  2356. if (insn->src_reg == 0 && is_callback_calling_function(insn->imm))
  2357. return -ENOTSUPP;
  2358. /* regular helper call sets R0 */
  2359. *reg_mask &= ~1;
  2360. if (*reg_mask & 0x3f) {
  2361. /* if backtracing was looking for registers R1-R5
  2362. * they should have been found already.
  2363. */
  2364. verbose(env, "BUG regs %x\n", *reg_mask);
  2365. WARN_ONCE(1, "verifier backtracking bug");
  2366. return -EFAULT;
  2367. }
  2368. } else if (opcode == BPF_EXIT) {
  2369. return -ENOTSUPP;
  2370. } else if (BPF_SRC(insn->code) == BPF_X) {
  2371. if (!(*reg_mask & (dreg | sreg)))
  2372. return 0;
  2373. /* dreg <cond> sreg
  2374. * Both dreg and sreg need precision before
  2375. * this insn. If only sreg was marked precise
  2376. * before it would be equally necessary to
  2377. * propagate it to dreg.
  2378. */
  2379. *reg_mask |= (sreg | dreg);
  2380. /* else dreg <cond> K
  2381. * Only dreg still needs precision before
  2382. * this insn, so for the K-based conditional
  2383. * there is nothing new to be marked.
  2384. */
  2385. }
  2386. } else if (class == BPF_LD) {
  2387. if (!(*reg_mask & dreg))
  2388. return 0;
  2389. *reg_mask &= ~dreg;
  2390. /* It's ld_imm64 or ld_abs or ld_ind.
  2391. * For ld_imm64 no further tracking of precision
  2392. * into parent is necessary
  2393. */
  2394. if (mode == BPF_IND || mode == BPF_ABS)
  2395. /* to be analyzed */
  2396. return -ENOTSUPP;
  2397. }
  2398. return 0;
  2399. }
  2400. /* the scalar precision tracking algorithm:
  2401. * . at the start all registers have precise=false.
  2402. * . scalar ranges are tracked as normal through alu and jmp insns.
  2403. * . once precise value of the scalar register is used in:
  2404. * . ptr + scalar alu
  2405. * . if (scalar cond K|scalar)
  2406. * . helper_call(.., scalar, ...) where ARG_CONST is expected
  2407. * backtrack through the verifier states and mark all registers and
  2408. * stack slots with spilled constants that these scalar regisers
  2409. * should be precise.
  2410. * . during state pruning two registers (or spilled stack slots)
  2411. * are equivalent if both are not precise.
  2412. *
  2413. * Note the verifier cannot simply walk register parentage chain,
  2414. * since many different registers and stack slots could have been
  2415. * used to compute single precise scalar.
  2416. *
  2417. * The approach of starting with precise=true for all registers and then
  2418. * backtrack to mark a register as not precise when the verifier detects
  2419. * that program doesn't care about specific value (e.g., when helper
  2420. * takes register as ARG_ANYTHING parameter) is not safe.
  2421. *
  2422. * It's ok to walk single parentage chain of the verifier states.
  2423. * It's possible that this backtracking will go all the way till 1st insn.
  2424. * All other branches will be explored for needing precision later.
  2425. *
  2426. * The backtracking needs to deal with cases like:
  2427. * R8=map_value(id=0,off=0,ks=4,vs=1952,imm=0) R9_w=map_value(id=0,off=40,ks=4,vs=1952,imm=0)
  2428. * r9 -= r8
  2429. * r5 = r9
  2430. * if r5 > 0x79f goto pc+7
  2431. * R5_w=inv(id=0,umax_value=1951,var_off=(0x0; 0x7ff))
  2432. * r5 += 1
  2433. * ...
  2434. * call bpf_perf_event_output#25
  2435. * where .arg5_type = ARG_CONST_SIZE_OR_ZERO
  2436. *
  2437. * and this case:
  2438. * r6 = 1
  2439. * call foo // uses callee's r6 inside to compute r0
  2440. * r0 += r6
  2441. * if r0 == 0 goto
  2442. *
  2443. * to track above reg_mask/stack_mask needs to be independent for each frame.
  2444. *
  2445. * Also if parent's curframe > frame where backtracking started,
  2446. * the verifier need to mark registers in both frames, otherwise callees
  2447. * may incorrectly prune callers. This is similar to
  2448. * commit 7640ead93924 ("bpf: verifier: make sure callees don't prune with caller differences")
  2449. *
  2450. * For now backtracking falls back into conservative marking.
  2451. */
  2452. static void mark_all_scalars_precise(struct bpf_verifier_env *env,
  2453. struct bpf_verifier_state *st)
  2454. {
  2455. struct bpf_func_state *func;
  2456. struct bpf_reg_state *reg;
  2457. int i, j;
  2458. /* big hammer: mark all scalars precise in this path.
  2459. * pop_stack may still get !precise scalars.
  2460. * We also skip current state and go straight to first parent state,
  2461. * because precision markings in current non-checkpointed state are
  2462. * not needed. See why in the comment in __mark_chain_precision below.
  2463. */
  2464. for (st = st->parent; st; st = st->parent) {
  2465. for (i = 0; i <= st->curframe; i++) {
  2466. func = st->frame[i];
  2467. for (j = 0; j < BPF_REG_FP; j++) {
  2468. reg = &func->regs[j];
  2469. if (reg->type != SCALAR_VALUE)
  2470. continue;
  2471. reg->precise = true;
  2472. }
  2473. for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
  2474. if (!is_spilled_reg(&func->stack[j]))
  2475. continue;
  2476. reg = &func->stack[j].spilled_ptr;
  2477. if (reg->type != SCALAR_VALUE)
  2478. continue;
  2479. reg->precise = true;
  2480. }
  2481. }
  2482. }
  2483. }
  2484. static void mark_all_scalars_imprecise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
  2485. {
  2486. struct bpf_func_state *func;
  2487. struct bpf_reg_state *reg;
  2488. int i, j;
  2489. for (i = 0; i <= st->curframe; i++) {
  2490. func = st->frame[i];
  2491. for (j = 0; j < BPF_REG_FP; j++) {
  2492. reg = &func->regs[j];
  2493. if (reg->type != SCALAR_VALUE)
  2494. continue;
  2495. reg->precise = false;
  2496. }
  2497. for (j = 0; j < func->allocated_stack / BPF_REG_SIZE; j++) {
  2498. if (!is_spilled_reg(&func->stack[j]))
  2499. continue;
  2500. reg = &func->stack[j].spilled_ptr;
  2501. if (reg->type != SCALAR_VALUE)
  2502. continue;
  2503. reg->precise = false;
  2504. }
  2505. }
  2506. }
  2507. /*
  2508. * __mark_chain_precision() backtracks BPF program instruction sequence and
  2509. * chain of verifier states making sure that register *regno* (if regno >= 0)
  2510. * and/or stack slot *spi* (if spi >= 0) are marked as precisely tracked
  2511. * SCALARS, as well as any other registers and slots that contribute to
  2512. * a tracked state of given registers/stack slots, depending on specific BPF
  2513. * assembly instructions (see backtrack_insns() for exact instruction handling
  2514. * logic). This backtracking relies on recorded jmp_history and is able to
  2515. * traverse entire chain of parent states. This process ends only when all the
  2516. * necessary registers/slots and their transitive dependencies are marked as
  2517. * precise.
  2518. *
  2519. * One important and subtle aspect is that precise marks *do not matter* in
  2520. * the currently verified state (current state). It is important to understand
  2521. * why this is the case.
  2522. *
  2523. * First, note that current state is the state that is not yet "checkpointed",
  2524. * i.e., it is not yet put into env->explored_states, and it has no children
  2525. * states as well. It's ephemeral, and can end up either a) being discarded if
  2526. * compatible explored state is found at some point or BPF_EXIT instruction is
  2527. * reached or b) checkpointed and put into env->explored_states, branching out
  2528. * into one or more children states.
  2529. *
  2530. * In the former case, precise markings in current state are completely
  2531. * ignored by state comparison code (see regsafe() for details). Only
  2532. * checkpointed ("old") state precise markings are important, and if old
  2533. * state's register/slot is precise, regsafe() assumes current state's
  2534. * register/slot as precise and checks value ranges exactly and precisely. If
  2535. * states turn out to be compatible, current state's necessary precise
  2536. * markings and any required parent states' precise markings are enforced
  2537. * after the fact with propagate_precision() logic, after the fact. But it's
  2538. * important to realize that in this case, even after marking current state
  2539. * registers/slots as precise, we immediately discard current state. So what
  2540. * actually matters is any of the precise markings propagated into current
  2541. * state's parent states, which are always checkpointed (due to b) case above).
  2542. * As such, for scenario a) it doesn't matter if current state has precise
  2543. * markings set or not.
  2544. *
  2545. * Now, for the scenario b), checkpointing and forking into child(ren)
  2546. * state(s). Note that before current state gets to checkpointing step, any
  2547. * processed instruction always assumes precise SCALAR register/slot
  2548. * knowledge: if precise value or range is useful to prune jump branch, BPF
  2549. * verifier takes this opportunity enthusiastically. Similarly, when
  2550. * register's value is used to calculate offset or memory address, exact
  2551. * knowledge of SCALAR range is assumed, checked, and enforced. So, similar to
  2552. * what we mentioned above about state comparison ignoring precise markings
  2553. * during state comparison, BPF verifier ignores and also assumes precise
  2554. * markings *at will* during instruction verification process. But as verifier
  2555. * assumes precision, it also propagates any precision dependencies across
  2556. * parent states, which are not yet finalized, so can be further restricted
  2557. * based on new knowledge gained from restrictions enforced by their children
  2558. * states. This is so that once those parent states are finalized, i.e., when
  2559. * they have no more active children state, state comparison logic in
  2560. * is_state_visited() would enforce strict and precise SCALAR ranges, if
  2561. * required for correctness.
  2562. *
  2563. * To build a bit more intuition, note also that once a state is checkpointed,
  2564. * the path we took to get to that state is not important. This is crucial
  2565. * property for state pruning. When state is checkpointed and finalized at
  2566. * some instruction index, it can be correctly and safely used to "short
  2567. * circuit" any *compatible* state that reaches exactly the same instruction
  2568. * index. I.e., if we jumped to that instruction from a completely different
  2569. * code path than original finalized state was derived from, it doesn't
  2570. * matter, current state can be discarded because from that instruction
  2571. * forward having a compatible state will ensure we will safely reach the
  2572. * exit. States describe preconditions for further exploration, but completely
  2573. * forget the history of how we got here.
  2574. *
  2575. * This also means that even if we needed precise SCALAR range to get to
  2576. * finalized state, but from that point forward *that same* SCALAR register is
  2577. * never used in a precise context (i.e., it's precise value is not needed for
  2578. * correctness), it's correct and safe to mark such register as "imprecise"
  2579. * (i.e., precise marking set to false). This is what we rely on when we do
  2580. * not set precise marking in current state. If no child state requires
  2581. * precision for any given SCALAR register, it's safe to dictate that it can
  2582. * be imprecise. If any child state does require this register to be precise,
  2583. * we'll mark it precise later retroactively during precise markings
  2584. * propagation from child state to parent states.
  2585. *
  2586. * Skipping precise marking setting in current state is a mild version of
  2587. * relying on the above observation. But we can utilize this property even
  2588. * more aggressively by proactively forgetting any precise marking in the
  2589. * current state (which we inherited from the parent state), right before we
  2590. * checkpoint it and branch off into new child state. This is done by
  2591. * mark_all_scalars_imprecise() to hopefully get more permissive and generic
  2592. * finalized states which help in short circuiting more future states.
  2593. */
  2594. static int __mark_chain_precision(struct bpf_verifier_env *env, int frame, int regno,
  2595. int spi)
  2596. {
  2597. struct bpf_verifier_state *st = env->cur_state;
  2598. int first_idx = st->first_insn_idx;
  2599. int last_idx = env->insn_idx;
  2600. struct bpf_func_state *func;
  2601. struct bpf_reg_state *reg;
  2602. u32 reg_mask = regno >= 0 ? 1u << regno : 0;
  2603. u64 stack_mask = spi >= 0 ? 1ull << spi : 0;
  2604. bool skip_first = true;
  2605. bool new_marks = false;
  2606. int i, err;
  2607. if (!env->bpf_capable)
  2608. return 0;
  2609. /* Do sanity checks against current state of register and/or stack
  2610. * slot, but don't set precise flag in current state, as precision
  2611. * tracking in the current state is unnecessary.
  2612. */
  2613. func = st->frame[frame];
  2614. if (regno >= 0) {
  2615. reg = &func->regs[regno];
  2616. if (reg->type != SCALAR_VALUE) {
  2617. WARN_ONCE(1, "backtracing misuse");
  2618. return -EFAULT;
  2619. }
  2620. new_marks = true;
  2621. }
  2622. while (spi >= 0) {
  2623. if (!is_spilled_reg(&func->stack[spi])) {
  2624. stack_mask = 0;
  2625. break;
  2626. }
  2627. reg = &func->stack[spi].spilled_ptr;
  2628. if (reg->type != SCALAR_VALUE) {
  2629. stack_mask = 0;
  2630. break;
  2631. }
  2632. new_marks = true;
  2633. break;
  2634. }
  2635. if (!new_marks)
  2636. return 0;
  2637. if (!reg_mask && !stack_mask)
  2638. return 0;
  2639. for (;;) {
  2640. DECLARE_BITMAP(mask, 64);
  2641. u32 history = st->jmp_history_cnt;
  2642. if (env->log.level & BPF_LOG_LEVEL2)
  2643. verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
  2644. if (last_idx < 0) {
  2645. /* we are at the entry into subprog, which
  2646. * is expected for global funcs, but only if
  2647. * requested precise registers are R1-R5
  2648. * (which are global func's input arguments)
  2649. */
  2650. if (st->curframe == 0 &&
  2651. st->frame[0]->subprogno > 0 &&
  2652. st->frame[0]->callsite == BPF_MAIN_FUNC &&
  2653. stack_mask == 0 && (reg_mask & ~0x3e) == 0) {
  2654. bitmap_from_u64(mask, reg_mask);
  2655. for_each_set_bit(i, mask, 32) {
  2656. reg = &st->frame[0]->regs[i];
  2657. if (reg->type != SCALAR_VALUE) {
  2658. reg_mask &= ~(1u << i);
  2659. continue;
  2660. }
  2661. reg->precise = true;
  2662. }
  2663. return 0;
  2664. }
  2665. verbose(env, "BUG backtracing func entry subprog %d reg_mask %x stack_mask %llx\n",
  2666. st->frame[0]->subprogno, reg_mask, stack_mask);
  2667. WARN_ONCE(1, "verifier backtracking bug");
  2668. return -EFAULT;
  2669. }
  2670. for (i = last_idx;;) {
  2671. if (skip_first) {
  2672. err = 0;
  2673. skip_first = false;
  2674. } else {
  2675. err = backtrack_insn(env, i, &reg_mask, &stack_mask);
  2676. }
  2677. if (err == -ENOTSUPP) {
  2678. mark_all_scalars_precise(env, st);
  2679. return 0;
  2680. } else if (err) {
  2681. return err;
  2682. }
  2683. if (!reg_mask && !stack_mask)
  2684. /* Found assignment(s) into tracked register in this state.
  2685. * Since this state is already marked, just return.
  2686. * Nothing to be tracked further in the parent state.
  2687. */
  2688. return 0;
  2689. if (i == first_idx)
  2690. break;
  2691. i = get_prev_insn_idx(st, i, &history);
  2692. if (i >= env->prog->len) {
  2693. /* This can happen if backtracking reached insn 0
  2694. * and there are still reg_mask or stack_mask
  2695. * to backtrack.
  2696. * It means the backtracking missed the spot where
  2697. * particular register was initialized with a constant.
  2698. */
  2699. verbose(env, "BUG backtracking idx %d\n", i);
  2700. WARN_ONCE(1, "verifier backtracking bug");
  2701. return -EFAULT;
  2702. }
  2703. }
  2704. st = st->parent;
  2705. if (!st)
  2706. break;
  2707. new_marks = false;
  2708. func = st->frame[frame];
  2709. bitmap_from_u64(mask, reg_mask);
  2710. for_each_set_bit(i, mask, 32) {
  2711. reg = &func->regs[i];
  2712. if (reg->type != SCALAR_VALUE) {
  2713. reg_mask &= ~(1u << i);
  2714. continue;
  2715. }
  2716. if (!reg->precise)
  2717. new_marks = true;
  2718. reg->precise = true;
  2719. }
  2720. bitmap_from_u64(mask, stack_mask);
  2721. for_each_set_bit(i, mask, 64) {
  2722. if (i >= func->allocated_stack / BPF_REG_SIZE) {
  2723. /* the sequence of instructions:
  2724. * 2: (bf) r3 = r10
  2725. * 3: (7b) *(u64 *)(r3 -8) = r0
  2726. * 4: (79) r4 = *(u64 *)(r10 -8)
  2727. * doesn't contain jmps. It's backtracked
  2728. * as a single block.
  2729. * During backtracking insn 3 is not recognized as
  2730. * stack access, so at the end of backtracking
  2731. * stack slot fp-8 is still marked in stack_mask.
  2732. * However the parent state may not have accessed
  2733. * fp-8 and it's "unallocated" stack space.
  2734. * In such case fallback to conservative.
  2735. */
  2736. mark_all_scalars_precise(env, st);
  2737. return 0;
  2738. }
  2739. if (!is_spilled_reg(&func->stack[i])) {
  2740. stack_mask &= ~(1ull << i);
  2741. continue;
  2742. }
  2743. reg = &func->stack[i].spilled_ptr;
  2744. if (reg->type != SCALAR_VALUE) {
  2745. stack_mask &= ~(1ull << i);
  2746. continue;
  2747. }
  2748. if (!reg->precise)
  2749. new_marks = true;
  2750. reg->precise = true;
  2751. }
  2752. if (env->log.level & BPF_LOG_LEVEL2) {
  2753. verbose(env, "parent %s regs=%x stack=%llx marks:",
  2754. new_marks ? "didn't have" : "already had",
  2755. reg_mask, stack_mask);
  2756. print_verifier_state(env, func, true);
  2757. }
  2758. if (!reg_mask && !stack_mask)
  2759. break;
  2760. if (!new_marks)
  2761. break;
  2762. last_idx = st->last_insn_idx;
  2763. first_idx = st->first_insn_idx;
  2764. }
  2765. return 0;
  2766. }
  2767. int mark_chain_precision(struct bpf_verifier_env *env, int regno)
  2768. {
  2769. return __mark_chain_precision(env, env->cur_state->curframe, regno, -1);
  2770. }
  2771. static int mark_chain_precision_frame(struct bpf_verifier_env *env, int frame, int regno)
  2772. {
  2773. return __mark_chain_precision(env, frame, regno, -1);
  2774. }
  2775. static int mark_chain_precision_stack_frame(struct bpf_verifier_env *env, int frame, int spi)
  2776. {
  2777. return __mark_chain_precision(env, frame, -1, spi);
  2778. }
  2779. static bool is_spillable_regtype(enum bpf_reg_type type)
  2780. {
  2781. switch (base_type(type)) {
  2782. case PTR_TO_MAP_VALUE:
  2783. case PTR_TO_STACK:
  2784. case PTR_TO_CTX:
  2785. case PTR_TO_PACKET:
  2786. case PTR_TO_PACKET_META:
  2787. case PTR_TO_PACKET_END:
  2788. case PTR_TO_FLOW_KEYS:
  2789. case CONST_PTR_TO_MAP:
  2790. case PTR_TO_SOCKET:
  2791. case PTR_TO_SOCK_COMMON:
  2792. case PTR_TO_TCP_SOCK:
  2793. case PTR_TO_XDP_SOCK:
  2794. case PTR_TO_BTF_ID:
  2795. case PTR_TO_BUF:
  2796. case PTR_TO_MEM:
  2797. case PTR_TO_FUNC:
  2798. case PTR_TO_MAP_KEY:
  2799. return true;
  2800. default:
  2801. return false;
  2802. }
  2803. }
  2804. /* Does this register contain a constant zero? */
  2805. static bool register_is_null(struct bpf_reg_state *reg)
  2806. {
  2807. return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
  2808. }
  2809. static bool register_is_const(struct bpf_reg_state *reg)
  2810. {
  2811. return reg->type == SCALAR_VALUE && tnum_is_const(reg->var_off);
  2812. }
  2813. static bool __is_scalar_unbounded(struct bpf_reg_state *reg)
  2814. {
  2815. return tnum_is_unknown(reg->var_off) &&
  2816. reg->smin_value == S64_MIN && reg->smax_value == S64_MAX &&
  2817. reg->umin_value == 0 && reg->umax_value == U64_MAX &&
  2818. reg->s32_min_value == S32_MIN && reg->s32_max_value == S32_MAX &&
  2819. reg->u32_min_value == 0 && reg->u32_max_value == U32_MAX;
  2820. }
  2821. static bool register_is_bounded(struct bpf_reg_state *reg)
  2822. {
  2823. return reg->type == SCALAR_VALUE && !__is_scalar_unbounded(reg);
  2824. }
  2825. static bool __is_pointer_value(bool allow_ptr_leaks,
  2826. const struct bpf_reg_state *reg)
  2827. {
  2828. if (allow_ptr_leaks)
  2829. return false;
  2830. return reg->type != SCALAR_VALUE;
  2831. }
  2832. /* Copy src state preserving dst->parent and dst->live fields */
  2833. static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src)
  2834. {
  2835. struct bpf_reg_state *parent = dst->parent;
  2836. enum bpf_reg_liveness live = dst->live;
  2837. *dst = *src;
  2838. dst->parent = parent;
  2839. dst->live = live;
  2840. }
  2841. static void save_register_state(struct bpf_func_state *state,
  2842. int spi, struct bpf_reg_state *reg,
  2843. int size)
  2844. {
  2845. int i;
  2846. copy_register_state(&state->stack[spi].spilled_ptr, reg);
  2847. if (size == BPF_REG_SIZE)
  2848. state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
  2849. for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
  2850. state->stack[spi].slot_type[i - 1] = STACK_SPILL;
  2851. /* size < 8 bytes spill */
  2852. for (; i; i--)
  2853. scrub_spilled_slot(&state->stack[spi].slot_type[i - 1]);
  2854. }
  2855. static bool is_bpf_st_mem(struct bpf_insn *insn)
  2856. {
  2857. return BPF_CLASS(insn->code) == BPF_ST && BPF_MODE(insn->code) == BPF_MEM;
  2858. }
  2859. /* check_stack_{read,write}_fixed_off functions track spill/fill of registers,
  2860. * stack boundary and alignment are checked in check_mem_access()
  2861. */
  2862. static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
  2863. /* stack frame we're writing to */
  2864. struct bpf_func_state *state,
  2865. int off, int size, int value_regno,
  2866. int insn_idx)
  2867. {
  2868. struct bpf_func_state *cur; /* state of the current function */
  2869. int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
  2870. struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
  2871. struct bpf_reg_state *reg = NULL;
  2872. u32 dst_reg = insn->dst_reg;
  2873. err = grow_stack_state(state, round_up(slot + 1, BPF_REG_SIZE));
  2874. if (err)
  2875. return err;
  2876. /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
  2877. * so it's aligned access and [off, off + size) are within stack limits
  2878. */
  2879. if (!env->allow_ptr_leaks &&
  2880. state->stack[spi].slot_type[0] == STACK_SPILL &&
  2881. size != BPF_REG_SIZE) {
  2882. verbose(env, "attempt to corrupt spilled pointer on stack\n");
  2883. return -EACCES;
  2884. }
  2885. cur = env->cur_state->frame[env->cur_state->curframe];
  2886. if (value_regno >= 0)
  2887. reg = &cur->regs[value_regno];
  2888. if (!env->bypass_spec_v4) {
  2889. bool sanitize = reg && is_spillable_regtype(reg->type);
  2890. for (i = 0; i < size; i++) {
  2891. u8 type = state->stack[spi].slot_type[i];
  2892. if (type != STACK_MISC && type != STACK_ZERO) {
  2893. sanitize = true;
  2894. break;
  2895. }
  2896. }
  2897. if (sanitize)
  2898. env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
  2899. }
  2900. mark_stack_slot_scratched(env, spi);
  2901. if (reg && !(off % BPF_REG_SIZE) && register_is_bounded(reg) &&
  2902. !register_is_null(reg) && env->bpf_capable) {
  2903. if (dst_reg != BPF_REG_FP) {
  2904. /* The backtracking logic can only recognize explicit
  2905. * stack slot address like [fp - 8]. Other spill of
  2906. * scalar via different register has to be conservative.
  2907. * Backtrack from here and mark all registers as precise
  2908. * that contributed into 'reg' being a constant.
  2909. */
  2910. err = mark_chain_precision(env, value_regno);
  2911. if (err)
  2912. return err;
  2913. }
  2914. save_register_state(state, spi, reg, size);
  2915. /* Break the relation on a narrowing spill. */
  2916. if (fls64(reg->umax_value) > BITS_PER_BYTE * size)
  2917. state->stack[spi].spilled_ptr.id = 0;
  2918. } else if (!reg && !(off % BPF_REG_SIZE) && is_bpf_st_mem(insn) &&
  2919. insn->imm != 0 && env->bpf_capable) {
  2920. struct bpf_reg_state fake_reg = {};
  2921. __mark_reg_known(&fake_reg, insn->imm);
  2922. fake_reg.type = SCALAR_VALUE;
  2923. save_register_state(state, spi, &fake_reg, size);
  2924. } else if (reg && is_spillable_regtype(reg->type)) {
  2925. /* register containing pointer is being spilled into stack */
  2926. if (size != BPF_REG_SIZE) {
  2927. verbose_linfo(env, insn_idx, "; ");
  2928. verbose(env, "invalid size of register spill\n");
  2929. return -EACCES;
  2930. }
  2931. if (state != cur && reg->type == PTR_TO_STACK) {
  2932. verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
  2933. return -EINVAL;
  2934. }
  2935. save_register_state(state, spi, reg, size);
  2936. } else {
  2937. u8 type = STACK_MISC;
  2938. /* regular write of data into stack destroys any spilled ptr */
  2939. state->stack[spi].spilled_ptr.type = NOT_INIT;
  2940. /* Mark slots as STACK_MISC if they belonged to spilled ptr. */
  2941. if (is_spilled_reg(&state->stack[spi]))
  2942. for (i = 0; i < BPF_REG_SIZE; i++)
  2943. scrub_spilled_slot(&state->stack[spi].slot_type[i]);
  2944. /* only mark the slot as written if all 8 bytes were written
  2945. * otherwise read propagation may incorrectly stop too soon
  2946. * when stack slots are partially written.
  2947. * This heuristic means that read propagation will be
  2948. * conservative, since it will add reg_live_read marks
  2949. * to stack slots all the way to first state when programs
  2950. * writes+reads less than 8 bytes
  2951. */
  2952. if (size == BPF_REG_SIZE)
  2953. state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
  2954. /* when we zero initialize stack slots mark them as such */
  2955. if ((reg && register_is_null(reg)) ||
  2956. (!reg && is_bpf_st_mem(insn) && insn->imm == 0)) {
  2957. /* backtracking doesn't work for STACK_ZERO yet. */
  2958. err = mark_chain_precision(env, value_regno);
  2959. if (err)
  2960. return err;
  2961. type = STACK_ZERO;
  2962. }
  2963. /* Mark slots affected by this stack write. */
  2964. for (i = 0; i < size; i++)
  2965. state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
  2966. type;
  2967. }
  2968. return 0;
  2969. }
  2970. /* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
  2971. * known to contain a variable offset.
  2972. * This function checks whether the write is permitted and conservatively
  2973. * tracks the effects of the write, considering that each stack slot in the
  2974. * dynamic range is potentially written to.
  2975. *
  2976. * 'off' includes 'regno->off'.
  2977. * 'value_regno' can be -1, meaning that an unknown value is being written to
  2978. * the stack.
  2979. *
  2980. * Spilled pointers in range are not marked as written because we don't know
  2981. * what's going to be actually written. This means that read propagation for
  2982. * future reads cannot be terminated by this write.
  2983. *
  2984. * For privileged programs, uninitialized stack slots are considered
  2985. * initialized by this write (even though we don't know exactly what offsets
  2986. * are going to be written to). The idea is that we don't want the verifier to
  2987. * reject future reads that access slots written to through variable offsets.
  2988. */
  2989. static int check_stack_write_var_off(struct bpf_verifier_env *env,
  2990. /* func where register points to */
  2991. struct bpf_func_state *state,
  2992. int ptr_regno, int off, int size,
  2993. int value_regno, int insn_idx)
  2994. {
  2995. struct bpf_func_state *cur; /* state of the current function */
  2996. int min_off, max_off;
  2997. int i, err;
  2998. struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
  2999. bool writing_zero = false;
  3000. /* set if the fact that we're writing a zero is used to let any
  3001. * stack slots remain STACK_ZERO
  3002. */
  3003. bool zero_used = false;
  3004. cur = env->cur_state->frame[env->cur_state->curframe];
  3005. ptr_reg = &cur->regs[ptr_regno];
  3006. min_off = ptr_reg->smin_value + off;
  3007. max_off = ptr_reg->smax_value + off + size;
  3008. if (value_regno >= 0)
  3009. value_reg = &cur->regs[value_regno];
  3010. if (value_reg && register_is_null(value_reg))
  3011. writing_zero = true;
  3012. err = grow_stack_state(state, round_up(-min_off, BPF_REG_SIZE));
  3013. if (err)
  3014. return err;
  3015. /* Variable offset writes destroy any spilled pointers in range. */
  3016. for (i = min_off; i < max_off; i++) {
  3017. u8 new_type, *stype;
  3018. int slot, spi;
  3019. slot = -i - 1;
  3020. spi = slot / BPF_REG_SIZE;
  3021. stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
  3022. mark_stack_slot_scratched(env, spi);
  3023. if (!env->allow_ptr_leaks && *stype != STACK_MISC && *stype != STACK_ZERO) {
  3024. /* Reject the write if range we may write to has not
  3025. * been initialized beforehand. If we didn't reject
  3026. * here, the ptr status would be erased below (even
  3027. * though not all slots are actually overwritten),
  3028. * possibly opening the door to leaks.
  3029. *
  3030. * We do however catch STACK_INVALID case below, and
  3031. * only allow reading possibly uninitialized memory
  3032. * later for CAP_PERFMON, as the write may not happen to
  3033. * that slot.
  3034. */
  3035. verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d",
  3036. insn_idx, i);
  3037. return -EINVAL;
  3038. }
  3039. /* Erase all spilled pointers. */
  3040. state->stack[spi].spilled_ptr.type = NOT_INIT;
  3041. /* Update the slot type. */
  3042. new_type = STACK_MISC;
  3043. if (writing_zero && *stype == STACK_ZERO) {
  3044. new_type = STACK_ZERO;
  3045. zero_used = true;
  3046. }
  3047. /* If the slot is STACK_INVALID, we check whether it's OK to
  3048. * pretend that it will be initialized by this write. The slot
  3049. * might not actually be written to, and so if we mark it as
  3050. * initialized future reads might leak uninitialized memory.
  3051. * For privileged programs, we will accept such reads to slots
  3052. * that may or may not be written because, if we're reject
  3053. * them, the error would be too confusing.
  3054. */
  3055. if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
  3056. verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d",
  3057. insn_idx, i);
  3058. return -EINVAL;
  3059. }
  3060. *stype = new_type;
  3061. }
  3062. if (zero_used) {
  3063. /* backtracking doesn't work for STACK_ZERO yet. */
  3064. err = mark_chain_precision(env, value_regno);
  3065. if (err)
  3066. return err;
  3067. }
  3068. return 0;
  3069. }
  3070. /* When register 'dst_regno' is assigned some values from stack[min_off,
  3071. * max_off), we set the register's type according to the types of the
  3072. * respective stack slots. If all the stack values are known to be zeros, then
  3073. * so is the destination reg. Otherwise, the register is considered to be
  3074. * SCALAR. This function does not deal with register filling; the caller must
  3075. * ensure that all spilled registers in the stack range have been marked as
  3076. * read.
  3077. */
  3078. static void mark_reg_stack_read(struct bpf_verifier_env *env,
  3079. /* func where src register points to */
  3080. struct bpf_func_state *ptr_state,
  3081. int min_off, int max_off, int dst_regno)
  3082. {
  3083. struct bpf_verifier_state *vstate = env->cur_state;
  3084. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  3085. int i, slot, spi;
  3086. u8 *stype;
  3087. int zeros = 0;
  3088. for (i = min_off; i < max_off; i++) {
  3089. slot = -i - 1;
  3090. spi = slot / BPF_REG_SIZE;
  3091. stype = ptr_state->stack[spi].slot_type;
  3092. if (stype[slot % BPF_REG_SIZE] != STACK_ZERO)
  3093. break;
  3094. zeros++;
  3095. }
  3096. if (zeros == max_off - min_off) {
  3097. /* any access_size read into register is zero extended,
  3098. * so the whole register == const_zero
  3099. */
  3100. __mark_reg_const_zero(&state->regs[dst_regno]);
  3101. /* backtracking doesn't support STACK_ZERO yet,
  3102. * so mark it precise here, so that later
  3103. * backtracking can stop here.
  3104. * Backtracking may not need this if this register
  3105. * doesn't participate in pointer adjustment.
  3106. * Forward propagation of precise flag is not
  3107. * necessary either. This mark is only to stop
  3108. * backtracking. Any register that contributed
  3109. * to const 0 was marked precise before spill.
  3110. */
  3111. state->regs[dst_regno].precise = true;
  3112. } else {
  3113. /* have read misc data from the stack */
  3114. mark_reg_unknown(env, state->regs, dst_regno);
  3115. }
  3116. state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
  3117. }
  3118. /* Read the stack at 'off' and put the results into the register indicated by
  3119. * 'dst_regno'. It handles reg filling if the addressed stack slot is a
  3120. * spilled reg.
  3121. *
  3122. * 'dst_regno' can be -1, meaning that the read value is not going to a
  3123. * register.
  3124. *
  3125. * The access is assumed to be within the current stack bounds.
  3126. */
  3127. static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
  3128. /* func where src register points to */
  3129. struct bpf_func_state *reg_state,
  3130. int off, int size, int dst_regno)
  3131. {
  3132. struct bpf_verifier_state *vstate = env->cur_state;
  3133. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  3134. int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
  3135. struct bpf_reg_state *reg;
  3136. u8 *stype, type;
  3137. stype = reg_state->stack[spi].slot_type;
  3138. reg = &reg_state->stack[spi].spilled_ptr;
  3139. if (is_spilled_reg(&reg_state->stack[spi])) {
  3140. u8 spill_size = 1;
  3141. for (i = BPF_REG_SIZE - 1; i > 0 && stype[i - 1] == STACK_SPILL; i--)
  3142. spill_size++;
  3143. if (size != BPF_REG_SIZE || spill_size != BPF_REG_SIZE) {
  3144. if (reg->type != SCALAR_VALUE) {
  3145. verbose_linfo(env, env->insn_idx, "; ");
  3146. verbose(env, "invalid size of register fill\n");
  3147. return -EACCES;
  3148. }
  3149. mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
  3150. if (dst_regno < 0)
  3151. return 0;
  3152. if (!(off % BPF_REG_SIZE) && size == spill_size) {
  3153. /* The earlier check_reg_arg() has decided the
  3154. * subreg_def for this insn. Save it first.
  3155. */
  3156. s32 subreg_def = state->regs[dst_regno].subreg_def;
  3157. copy_register_state(&state->regs[dst_regno], reg);
  3158. state->regs[dst_regno].subreg_def = subreg_def;
  3159. } else {
  3160. for (i = 0; i < size; i++) {
  3161. type = stype[(slot - i) % BPF_REG_SIZE];
  3162. if (type == STACK_SPILL)
  3163. continue;
  3164. if (type == STACK_MISC)
  3165. continue;
  3166. verbose(env, "invalid read from stack off %d+%d size %d\n",
  3167. off, i, size);
  3168. return -EACCES;
  3169. }
  3170. mark_reg_unknown(env, state->regs, dst_regno);
  3171. }
  3172. state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
  3173. return 0;
  3174. }
  3175. if (dst_regno >= 0) {
  3176. /* restore register state from stack */
  3177. copy_register_state(&state->regs[dst_regno], reg);
  3178. /* mark reg as written since spilled pointer state likely
  3179. * has its liveness marks cleared by is_state_visited()
  3180. * which resets stack/reg liveness for state transitions
  3181. */
  3182. state->regs[dst_regno].live |= REG_LIVE_WRITTEN;
  3183. } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
  3184. /* If dst_regno==-1, the caller is asking us whether
  3185. * it is acceptable to use this value as a SCALAR_VALUE
  3186. * (e.g. for XADD).
  3187. * We must not allow unprivileged callers to do that
  3188. * with spilled pointers.
  3189. */
  3190. verbose(env, "leaking pointer from stack off %d\n",
  3191. off);
  3192. return -EACCES;
  3193. }
  3194. mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
  3195. } else {
  3196. for (i = 0; i < size; i++) {
  3197. type = stype[(slot - i) % BPF_REG_SIZE];
  3198. if (type == STACK_MISC)
  3199. continue;
  3200. if (type == STACK_ZERO)
  3201. continue;
  3202. verbose(env, "invalid read from stack off %d+%d size %d\n",
  3203. off, i, size);
  3204. return -EACCES;
  3205. }
  3206. mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
  3207. if (dst_regno >= 0)
  3208. mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
  3209. }
  3210. return 0;
  3211. }
  3212. enum bpf_access_src {
  3213. ACCESS_DIRECT = 1, /* the access is performed by an instruction */
  3214. ACCESS_HELPER = 2, /* the access is performed by a helper */
  3215. };
  3216. static int check_stack_range_initialized(struct bpf_verifier_env *env,
  3217. int regno, int off, int access_size,
  3218. bool zero_size_allowed,
  3219. enum bpf_access_src type,
  3220. struct bpf_call_arg_meta *meta);
  3221. static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
  3222. {
  3223. return cur_regs(env) + regno;
  3224. }
  3225. /* Read the stack at 'ptr_regno + off' and put the result into the register
  3226. * 'dst_regno'.
  3227. * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
  3228. * but not its variable offset.
  3229. * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
  3230. *
  3231. * As opposed to check_stack_read_fixed_off, this function doesn't deal with
  3232. * filling registers (i.e. reads of spilled register cannot be detected when
  3233. * the offset is not fixed). We conservatively mark 'dst_regno' as containing
  3234. * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
  3235. * offset; for a fixed offset check_stack_read_fixed_off should be used
  3236. * instead.
  3237. */
  3238. static int check_stack_read_var_off(struct bpf_verifier_env *env,
  3239. int ptr_regno, int off, int size, int dst_regno)
  3240. {
  3241. /* The state of the source register. */
  3242. struct bpf_reg_state *reg = reg_state(env, ptr_regno);
  3243. struct bpf_func_state *ptr_state = func(env, reg);
  3244. int err;
  3245. int min_off, max_off;
  3246. /* Note that we pass a NULL meta, so raw access will not be permitted.
  3247. */
  3248. err = check_stack_range_initialized(env, ptr_regno, off, size,
  3249. false, ACCESS_DIRECT, NULL);
  3250. if (err)
  3251. return err;
  3252. min_off = reg->smin_value + off;
  3253. max_off = reg->smax_value + off;
  3254. mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
  3255. return 0;
  3256. }
  3257. /* check_stack_read dispatches to check_stack_read_fixed_off or
  3258. * check_stack_read_var_off.
  3259. *
  3260. * The caller must ensure that the offset falls within the allocated stack
  3261. * bounds.
  3262. *
  3263. * 'dst_regno' is a register which will receive the value from the stack. It
  3264. * can be -1, meaning that the read value is not going to a register.
  3265. */
  3266. static int check_stack_read(struct bpf_verifier_env *env,
  3267. int ptr_regno, int off, int size,
  3268. int dst_regno)
  3269. {
  3270. struct bpf_reg_state *reg = reg_state(env, ptr_regno);
  3271. struct bpf_func_state *state = func(env, reg);
  3272. int err;
  3273. /* Some accesses are only permitted with a static offset. */
  3274. bool var_off = !tnum_is_const(reg->var_off);
  3275. /* The offset is required to be static when reads don't go to a
  3276. * register, in order to not leak pointers (see
  3277. * check_stack_read_fixed_off).
  3278. */
  3279. if (dst_regno < 0 && var_off) {
  3280. char tn_buf[48];
  3281. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  3282. verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
  3283. tn_buf, off, size);
  3284. return -EACCES;
  3285. }
  3286. /* Variable offset is prohibited for unprivileged mode for simplicity
  3287. * since it requires corresponding support in Spectre masking for stack
  3288. * ALU. See also retrieve_ptr_limit(). The check in
  3289. * check_stack_access_for_ptr_arithmetic() called by
  3290. * adjust_ptr_min_max_vals() prevents users from creating stack pointers
  3291. * with variable offsets, therefore no check is required here. Further,
  3292. * just checking it here would be insufficient as speculative stack
  3293. * writes could still lead to unsafe speculative behaviour.
  3294. */
  3295. if (!var_off) {
  3296. off += reg->var_off.value;
  3297. err = check_stack_read_fixed_off(env, state, off, size,
  3298. dst_regno);
  3299. } else {
  3300. /* Variable offset stack reads need more conservative handling
  3301. * than fixed offset ones. Note that dst_regno >= 0 on this
  3302. * branch.
  3303. */
  3304. err = check_stack_read_var_off(env, ptr_regno, off, size,
  3305. dst_regno);
  3306. }
  3307. return err;
  3308. }
  3309. /* check_stack_write dispatches to check_stack_write_fixed_off or
  3310. * check_stack_write_var_off.
  3311. *
  3312. * 'ptr_regno' is the register used as a pointer into the stack.
  3313. * 'off' includes 'ptr_regno->off', but not its variable offset (if any).
  3314. * 'value_regno' is the register whose value we're writing to the stack. It can
  3315. * be -1, meaning that we're not writing from a register.
  3316. *
  3317. * The caller must ensure that the offset falls within the maximum stack size.
  3318. */
  3319. static int check_stack_write(struct bpf_verifier_env *env,
  3320. int ptr_regno, int off, int size,
  3321. int value_regno, int insn_idx)
  3322. {
  3323. struct bpf_reg_state *reg = reg_state(env, ptr_regno);
  3324. struct bpf_func_state *state = func(env, reg);
  3325. int err;
  3326. if (tnum_is_const(reg->var_off)) {
  3327. off += reg->var_off.value;
  3328. err = check_stack_write_fixed_off(env, state, off, size,
  3329. value_regno, insn_idx);
  3330. } else {
  3331. /* Variable offset stack reads need more conservative handling
  3332. * than fixed offset ones.
  3333. */
  3334. err = check_stack_write_var_off(env, state,
  3335. ptr_regno, off, size,
  3336. value_regno, insn_idx);
  3337. }
  3338. return err;
  3339. }
  3340. static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
  3341. int off, int size, enum bpf_access_type type)
  3342. {
  3343. struct bpf_reg_state *regs = cur_regs(env);
  3344. struct bpf_map *map = regs[regno].map_ptr;
  3345. u32 cap = bpf_map_flags_to_cap(map);
  3346. if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
  3347. verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n",
  3348. map->value_size, off, size);
  3349. return -EACCES;
  3350. }
  3351. if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
  3352. verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n",
  3353. map->value_size, off, size);
  3354. return -EACCES;
  3355. }
  3356. return 0;
  3357. }
  3358. /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
  3359. static int __check_mem_access(struct bpf_verifier_env *env, int regno,
  3360. int off, int size, u32 mem_size,
  3361. bool zero_size_allowed)
  3362. {
  3363. bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
  3364. struct bpf_reg_state *reg;
  3365. if (off >= 0 && size_ok && (u64)off + size <= mem_size)
  3366. return 0;
  3367. reg = &cur_regs(env)[regno];
  3368. switch (reg->type) {
  3369. case PTR_TO_MAP_KEY:
  3370. verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
  3371. mem_size, off, size);
  3372. break;
  3373. case PTR_TO_MAP_VALUE:
  3374. verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
  3375. mem_size, off, size);
  3376. break;
  3377. case PTR_TO_PACKET:
  3378. case PTR_TO_PACKET_META:
  3379. case PTR_TO_PACKET_END:
  3380. verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
  3381. off, size, regno, reg->id, off, mem_size);
  3382. break;
  3383. case PTR_TO_MEM:
  3384. default:
  3385. verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n",
  3386. mem_size, off, size);
  3387. }
  3388. return -EACCES;
  3389. }
  3390. /* check read/write into a memory region with possible variable offset */
  3391. static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
  3392. int off, int size, u32 mem_size,
  3393. bool zero_size_allowed)
  3394. {
  3395. struct bpf_verifier_state *vstate = env->cur_state;
  3396. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  3397. struct bpf_reg_state *reg = &state->regs[regno];
  3398. int err;
  3399. /* We may have adjusted the register pointing to memory region, so we
  3400. * need to try adding each of min_value and max_value to off
  3401. * to make sure our theoretical access will be safe.
  3402. *
  3403. * The minimum value is only important with signed
  3404. * comparisons where we can't assume the floor of a
  3405. * value is 0. If we are using signed variables for our
  3406. * index'es we need to make sure that whatever we use
  3407. * will have a set floor within our range.
  3408. */
  3409. if (reg->smin_value < 0 &&
  3410. (reg->smin_value == S64_MIN ||
  3411. (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
  3412. reg->smin_value + off < 0)) {
  3413. verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
  3414. regno);
  3415. return -EACCES;
  3416. }
  3417. err = __check_mem_access(env, regno, reg->smin_value + off, size,
  3418. mem_size, zero_size_allowed);
  3419. if (err) {
  3420. verbose(env, "R%d min value is outside of the allowed memory range\n",
  3421. regno);
  3422. return err;
  3423. }
  3424. /* If we haven't set a max value then we need to bail since we can't be
  3425. * sure we won't do bad things.
  3426. * If reg->umax_value + off could overflow, treat that as unbounded too.
  3427. */
  3428. if (reg->umax_value >= BPF_MAX_VAR_OFF) {
  3429. verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
  3430. regno);
  3431. return -EACCES;
  3432. }
  3433. err = __check_mem_access(env, regno, reg->umax_value + off, size,
  3434. mem_size, zero_size_allowed);
  3435. if (err) {
  3436. verbose(env, "R%d max value is outside of the allowed memory range\n",
  3437. regno);
  3438. return err;
  3439. }
  3440. return 0;
  3441. }
  3442. static int __check_ptr_off_reg(struct bpf_verifier_env *env,
  3443. const struct bpf_reg_state *reg, int regno,
  3444. bool fixed_off_ok)
  3445. {
  3446. /* Access to this pointer-typed register or passing it to a helper
  3447. * is only allowed in its original, unmodified form.
  3448. */
  3449. if (reg->off < 0) {
  3450. verbose(env, "negative offset %s ptr R%d off=%d disallowed\n",
  3451. reg_type_str(env, reg->type), regno, reg->off);
  3452. return -EACCES;
  3453. }
  3454. if (!fixed_off_ok && reg->off) {
  3455. verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n",
  3456. reg_type_str(env, reg->type), regno, reg->off);
  3457. return -EACCES;
  3458. }
  3459. if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
  3460. char tn_buf[48];
  3461. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  3462. verbose(env, "variable %s access var_off=%s disallowed\n",
  3463. reg_type_str(env, reg->type), tn_buf);
  3464. return -EACCES;
  3465. }
  3466. return 0;
  3467. }
  3468. int check_ptr_off_reg(struct bpf_verifier_env *env,
  3469. const struct bpf_reg_state *reg, int regno)
  3470. {
  3471. return __check_ptr_off_reg(env, reg, regno, false);
  3472. }
  3473. static int map_kptr_match_type(struct bpf_verifier_env *env,
  3474. struct bpf_map_value_off_desc *off_desc,
  3475. struct bpf_reg_state *reg, u32 regno)
  3476. {
  3477. const char *targ_name = kernel_type_name(off_desc->kptr.btf, off_desc->kptr.btf_id);
  3478. int perm_flags = PTR_MAYBE_NULL;
  3479. const char *reg_name = "";
  3480. /* Only unreferenced case accepts untrusted pointers */
  3481. if (off_desc->type == BPF_KPTR_UNREF)
  3482. perm_flags |= PTR_UNTRUSTED;
  3483. if (base_type(reg->type) != PTR_TO_BTF_ID || (type_flag(reg->type) & ~perm_flags))
  3484. goto bad_type;
  3485. if (!btf_is_kernel(reg->btf)) {
  3486. verbose(env, "R%d must point to kernel BTF\n", regno);
  3487. return -EINVAL;
  3488. }
  3489. /* We need to verify reg->type and reg->btf, before accessing reg->btf */
  3490. reg_name = kernel_type_name(reg->btf, reg->btf_id);
  3491. /* For ref_ptr case, release function check should ensure we get one
  3492. * referenced PTR_TO_BTF_ID, and that its fixed offset is 0. For the
  3493. * normal store of unreferenced kptr, we must ensure var_off is zero.
  3494. * Since ref_ptr cannot be accessed directly by BPF insns, checks for
  3495. * reg->off and reg->ref_obj_id are not needed here.
  3496. */
  3497. if (__check_ptr_off_reg(env, reg, regno, true))
  3498. return -EACCES;
  3499. /* A full type match is needed, as BTF can be vmlinux or module BTF, and
  3500. * we also need to take into account the reg->off.
  3501. *
  3502. * We want to support cases like:
  3503. *
  3504. * struct foo {
  3505. * struct bar br;
  3506. * struct baz bz;
  3507. * };
  3508. *
  3509. * struct foo *v;
  3510. * v = func(); // PTR_TO_BTF_ID
  3511. * val->foo = v; // reg->off is zero, btf and btf_id match type
  3512. * val->bar = &v->br; // reg->off is still zero, but we need to retry with
  3513. * // first member type of struct after comparison fails
  3514. * val->baz = &v->bz; // reg->off is non-zero, so struct needs to be walked
  3515. * // to match type
  3516. *
  3517. * In the kptr_ref case, check_func_arg_reg_off already ensures reg->off
  3518. * is zero. We must also ensure that btf_struct_ids_match does not walk
  3519. * the struct to match type against first member of struct, i.e. reject
  3520. * second case from above. Hence, when type is BPF_KPTR_REF, we set
  3521. * strict mode to true for type match.
  3522. */
  3523. if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
  3524. off_desc->kptr.btf, off_desc->kptr.btf_id,
  3525. off_desc->type == BPF_KPTR_REF))
  3526. goto bad_type;
  3527. return 0;
  3528. bad_type:
  3529. verbose(env, "invalid kptr access, R%d type=%s%s ", regno,
  3530. reg_type_str(env, reg->type), reg_name);
  3531. verbose(env, "expected=%s%s", reg_type_str(env, PTR_TO_BTF_ID), targ_name);
  3532. if (off_desc->type == BPF_KPTR_UNREF)
  3533. verbose(env, " or %s%s\n", reg_type_str(env, PTR_TO_BTF_ID | PTR_UNTRUSTED),
  3534. targ_name);
  3535. else
  3536. verbose(env, "\n");
  3537. return -EINVAL;
  3538. }
  3539. static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
  3540. int value_regno, int insn_idx,
  3541. struct bpf_map_value_off_desc *off_desc)
  3542. {
  3543. struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
  3544. int class = BPF_CLASS(insn->code);
  3545. struct bpf_reg_state *val_reg;
  3546. /* Things we already checked for in check_map_access and caller:
  3547. * - Reject cases where variable offset may touch kptr
  3548. * - size of access (must be BPF_DW)
  3549. * - tnum_is_const(reg->var_off)
  3550. * - off_desc->offset == off + reg->var_off.value
  3551. */
  3552. /* Only BPF_[LDX,STX,ST] | BPF_MEM | BPF_DW is supported */
  3553. if (BPF_MODE(insn->code) != BPF_MEM) {
  3554. verbose(env, "kptr in map can only be accessed using BPF_MEM instruction mode\n");
  3555. return -EACCES;
  3556. }
  3557. /* We only allow loading referenced kptr, since it will be marked as
  3558. * untrusted, similar to unreferenced kptr.
  3559. */
  3560. if (class != BPF_LDX && off_desc->type == BPF_KPTR_REF) {
  3561. verbose(env, "store to referenced kptr disallowed\n");
  3562. return -EACCES;
  3563. }
  3564. if (class == BPF_LDX) {
  3565. val_reg = reg_state(env, value_regno);
  3566. /* We can simply mark the value_regno receiving the pointer
  3567. * value from map as PTR_TO_BTF_ID, with the correct type.
  3568. */
  3569. mark_btf_ld_reg(env, cur_regs(env), value_regno, PTR_TO_BTF_ID, off_desc->kptr.btf,
  3570. off_desc->kptr.btf_id, PTR_MAYBE_NULL | PTR_UNTRUSTED);
  3571. /* For mark_ptr_or_null_reg */
  3572. val_reg->id = ++env->id_gen;
  3573. } else if (class == BPF_STX) {
  3574. val_reg = reg_state(env, value_regno);
  3575. if (!register_is_null(val_reg) &&
  3576. map_kptr_match_type(env, off_desc, val_reg, value_regno))
  3577. return -EACCES;
  3578. } else if (class == BPF_ST) {
  3579. if (insn->imm) {
  3580. verbose(env, "BPF_ST imm must be 0 when storing to kptr at off=%u\n",
  3581. off_desc->offset);
  3582. return -EACCES;
  3583. }
  3584. } else {
  3585. verbose(env, "kptr in map can only be accessed using BPF_LDX/BPF_STX/BPF_ST\n");
  3586. return -EACCES;
  3587. }
  3588. return 0;
  3589. }
  3590. /* check read/write into a map element with possible variable offset */
  3591. static int check_map_access(struct bpf_verifier_env *env, u32 regno,
  3592. int off, int size, bool zero_size_allowed,
  3593. enum bpf_access_src src)
  3594. {
  3595. struct bpf_verifier_state *vstate = env->cur_state;
  3596. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  3597. struct bpf_reg_state *reg = &state->regs[regno];
  3598. struct bpf_map *map = reg->map_ptr;
  3599. int err;
  3600. err = check_mem_region_access(env, regno, off, size, map->value_size,
  3601. zero_size_allowed);
  3602. if (err)
  3603. return err;
  3604. if (map_value_has_spin_lock(map)) {
  3605. u32 lock = map->spin_lock_off;
  3606. /* if any part of struct bpf_spin_lock can be touched by
  3607. * load/store reject this program.
  3608. * To check that [x1, x2) overlaps with [y1, y2)
  3609. * it is sufficient to check x1 < y2 && y1 < x2.
  3610. */
  3611. if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
  3612. lock < reg->umax_value + off + size) {
  3613. verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
  3614. return -EACCES;
  3615. }
  3616. }
  3617. if (map_value_has_timer(map)) {
  3618. u32 t = map->timer_off;
  3619. if (reg->smin_value + off < t + sizeof(struct bpf_timer) &&
  3620. t < reg->umax_value + off + size) {
  3621. verbose(env, "bpf_timer cannot be accessed directly by load/store\n");
  3622. return -EACCES;
  3623. }
  3624. }
  3625. if (map_value_has_kptrs(map)) {
  3626. struct bpf_map_value_off *tab = map->kptr_off_tab;
  3627. int i;
  3628. for (i = 0; i < tab->nr_off; i++) {
  3629. u32 p = tab->off[i].offset;
  3630. if (reg->smin_value + off < p + sizeof(u64) &&
  3631. p < reg->umax_value + off + size) {
  3632. if (src != ACCESS_DIRECT) {
  3633. verbose(env, "kptr cannot be accessed indirectly by helper\n");
  3634. return -EACCES;
  3635. }
  3636. if (!tnum_is_const(reg->var_off)) {
  3637. verbose(env, "kptr access cannot have variable offset\n");
  3638. return -EACCES;
  3639. }
  3640. if (p != off + reg->var_off.value) {
  3641. verbose(env, "kptr access misaligned expected=%u off=%llu\n",
  3642. p, off + reg->var_off.value);
  3643. return -EACCES;
  3644. }
  3645. if (size != bpf_size_to_bytes(BPF_DW)) {
  3646. verbose(env, "kptr access size must be BPF_DW\n");
  3647. return -EACCES;
  3648. }
  3649. break;
  3650. }
  3651. }
  3652. }
  3653. return err;
  3654. }
  3655. #define MAX_PACKET_OFF 0xffff
  3656. static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
  3657. const struct bpf_call_arg_meta *meta,
  3658. enum bpf_access_type t)
  3659. {
  3660. enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
  3661. switch (prog_type) {
  3662. /* Program types only with direct read access go here! */
  3663. case BPF_PROG_TYPE_LWT_IN:
  3664. case BPF_PROG_TYPE_LWT_OUT:
  3665. case BPF_PROG_TYPE_LWT_SEG6LOCAL:
  3666. case BPF_PROG_TYPE_SK_REUSEPORT:
  3667. case BPF_PROG_TYPE_FLOW_DISSECTOR:
  3668. case BPF_PROG_TYPE_CGROUP_SKB:
  3669. if (t == BPF_WRITE)
  3670. return false;
  3671. fallthrough;
  3672. /* Program types with direct read + write access go here! */
  3673. case BPF_PROG_TYPE_SCHED_CLS:
  3674. case BPF_PROG_TYPE_SCHED_ACT:
  3675. case BPF_PROG_TYPE_XDP:
  3676. case BPF_PROG_TYPE_LWT_XMIT:
  3677. case BPF_PROG_TYPE_SK_SKB:
  3678. case BPF_PROG_TYPE_SK_MSG:
  3679. if (meta)
  3680. return meta->pkt_access;
  3681. env->seen_direct_write = true;
  3682. return true;
  3683. case BPF_PROG_TYPE_CGROUP_SOCKOPT:
  3684. if (t == BPF_WRITE)
  3685. env->seen_direct_write = true;
  3686. return true;
  3687. default:
  3688. return false;
  3689. }
  3690. }
  3691. static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
  3692. int size, bool zero_size_allowed)
  3693. {
  3694. struct bpf_reg_state *regs = cur_regs(env);
  3695. struct bpf_reg_state *reg = &regs[regno];
  3696. int err;
  3697. /* We may have added a variable offset to the packet pointer; but any
  3698. * reg->range we have comes after that. We are only checking the fixed
  3699. * offset.
  3700. */
  3701. /* We don't allow negative numbers, because we aren't tracking enough
  3702. * detail to prove they're safe.
  3703. */
  3704. if (reg->smin_value < 0) {
  3705. verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
  3706. regno);
  3707. return -EACCES;
  3708. }
  3709. err = reg->range < 0 ? -EINVAL :
  3710. __check_mem_access(env, regno, off, size, reg->range,
  3711. zero_size_allowed);
  3712. if (err) {
  3713. verbose(env, "R%d offset is outside of the packet\n", regno);
  3714. return err;
  3715. }
  3716. /* __check_mem_access has made sure "off + size - 1" is within u16.
  3717. * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
  3718. * otherwise find_good_pkt_pointers would have refused to set range info
  3719. * that __check_mem_access would have rejected this pkt access.
  3720. * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
  3721. */
  3722. env->prog->aux->max_pkt_offset =
  3723. max_t(u32, env->prog->aux->max_pkt_offset,
  3724. off + reg->umax_value + size - 1);
  3725. return err;
  3726. }
  3727. /* check access to 'struct bpf_context' fields. Supports fixed offsets only */
  3728. static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
  3729. enum bpf_access_type t, enum bpf_reg_type *reg_type,
  3730. struct btf **btf, u32 *btf_id)
  3731. {
  3732. struct bpf_insn_access_aux info = {
  3733. .reg_type = *reg_type,
  3734. .log = &env->log,
  3735. };
  3736. if (env->ops->is_valid_access &&
  3737. env->ops->is_valid_access(off, size, t, env->prog, &info)) {
  3738. /* A non zero info.ctx_field_size indicates that this field is a
  3739. * candidate for later verifier transformation to load the whole
  3740. * field and then apply a mask when accessed with a narrower
  3741. * access than actual ctx access size. A zero info.ctx_field_size
  3742. * will only allow for whole field access and rejects any other
  3743. * type of narrower access.
  3744. */
  3745. *reg_type = info.reg_type;
  3746. if (base_type(*reg_type) == PTR_TO_BTF_ID) {
  3747. *btf = info.btf;
  3748. *btf_id = info.btf_id;
  3749. } else {
  3750. env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
  3751. }
  3752. /* remember the offset of last byte accessed in ctx */
  3753. if (env->prog->aux->max_ctx_offset < off + size)
  3754. env->prog->aux->max_ctx_offset = off + size;
  3755. return 0;
  3756. }
  3757. verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
  3758. return -EACCES;
  3759. }
  3760. static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
  3761. int size)
  3762. {
  3763. if (size < 0 || off < 0 ||
  3764. (u64)off + size > sizeof(struct bpf_flow_keys)) {
  3765. verbose(env, "invalid access to flow keys off=%d size=%d\n",
  3766. off, size);
  3767. return -EACCES;
  3768. }
  3769. return 0;
  3770. }
  3771. static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
  3772. u32 regno, int off, int size,
  3773. enum bpf_access_type t)
  3774. {
  3775. struct bpf_reg_state *regs = cur_regs(env);
  3776. struct bpf_reg_state *reg = &regs[regno];
  3777. struct bpf_insn_access_aux info = {};
  3778. bool valid;
  3779. if (reg->smin_value < 0) {
  3780. verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
  3781. regno);
  3782. return -EACCES;
  3783. }
  3784. switch (reg->type) {
  3785. case PTR_TO_SOCK_COMMON:
  3786. valid = bpf_sock_common_is_valid_access(off, size, t, &info);
  3787. break;
  3788. case PTR_TO_SOCKET:
  3789. valid = bpf_sock_is_valid_access(off, size, t, &info);
  3790. break;
  3791. case PTR_TO_TCP_SOCK:
  3792. valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
  3793. break;
  3794. case PTR_TO_XDP_SOCK:
  3795. valid = bpf_xdp_sock_is_valid_access(off, size, t, &info);
  3796. break;
  3797. default:
  3798. valid = false;
  3799. }
  3800. if (valid) {
  3801. env->insn_aux_data[insn_idx].ctx_field_size =
  3802. info.ctx_field_size;
  3803. return 0;
  3804. }
  3805. verbose(env, "R%d invalid %s access off=%d size=%d\n",
  3806. regno, reg_type_str(env, reg->type), off, size);
  3807. return -EACCES;
  3808. }
  3809. static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
  3810. {
  3811. return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
  3812. }
  3813. static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
  3814. {
  3815. const struct bpf_reg_state *reg = reg_state(env, regno);
  3816. return reg->type == PTR_TO_CTX;
  3817. }
  3818. static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
  3819. {
  3820. const struct bpf_reg_state *reg = reg_state(env, regno);
  3821. return type_is_sk_pointer(reg->type);
  3822. }
  3823. static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
  3824. {
  3825. const struct bpf_reg_state *reg = reg_state(env, regno);
  3826. return type_is_pkt_pointer(reg->type);
  3827. }
  3828. static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
  3829. {
  3830. const struct bpf_reg_state *reg = reg_state(env, regno);
  3831. /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
  3832. return reg->type == PTR_TO_FLOW_KEYS;
  3833. }
  3834. static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
  3835. const struct bpf_reg_state *reg,
  3836. int off, int size, bool strict)
  3837. {
  3838. struct tnum reg_off;
  3839. int ip_align;
  3840. /* Byte size accesses are always allowed. */
  3841. if (!strict || size == 1)
  3842. return 0;
  3843. /* For platforms that do not have a Kconfig enabling
  3844. * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
  3845. * NET_IP_ALIGN is universally set to '2'. And on platforms
  3846. * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
  3847. * to this code only in strict mode where we want to emulate
  3848. * the NET_IP_ALIGN==2 checking. Therefore use an
  3849. * unconditional IP align value of '2'.
  3850. */
  3851. ip_align = 2;
  3852. reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
  3853. if (!tnum_is_aligned(reg_off, size)) {
  3854. char tn_buf[48];
  3855. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  3856. verbose(env,
  3857. "misaligned packet access off %d+%s+%d+%d size %d\n",
  3858. ip_align, tn_buf, reg->off, off, size);
  3859. return -EACCES;
  3860. }
  3861. return 0;
  3862. }
  3863. static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
  3864. const struct bpf_reg_state *reg,
  3865. const char *pointer_desc,
  3866. int off, int size, bool strict)
  3867. {
  3868. struct tnum reg_off;
  3869. /* Byte size accesses are always allowed. */
  3870. if (!strict || size == 1)
  3871. return 0;
  3872. reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
  3873. if (!tnum_is_aligned(reg_off, size)) {
  3874. char tn_buf[48];
  3875. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  3876. verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
  3877. pointer_desc, tn_buf, reg->off, off, size);
  3878. return -EACCES;
  3879. }
  3880. return 0;
  3881. }
  3882. static int check_ptr_alignment(struct bpf_verifier_env *env,
  3883. const struct bpf_reg_state *reg, int off,
  3884. int size, bool strict_alignment_once)
  3885. {
  3886. bool strict = env->strict_alignment || strict_alignment_once;
  3887. const char *pointer_desc = "";
  3888. switch (reg->type) {
  3889. case PTR_TO_PACKET:
  3890. case PTR_TO_PACKET_META:
  3891. /* Special case, because of NET_IP_ALIGN. Given metadata sits
  3892. * right in front, treat it the very same way.
  3893. */
  3894. return check_pkt_ptr_alignment(env, reg, off, size, strict);
  3895. case PTR_TO_FLOW_KEYS:
  3896. pointer_desc = "flow keys ";
  3897. break;
  3898. case PTR_TO_MAP_KEY:
  3899. pointer_desc = "key ";
  3900. break;
  3901. case PTR_TO_MAP_VALUE:
  3902. pointer_desc = "value ";
  3903. break;
  3904. case PTR_TO_CTX:
  3905. pointer_desc = "context ";
  3906. break;
  3907. case PTR_TO_STACK:
  3908. pointer_desc = "stack ";
  3909. /* The stack spill tracking logic in check_stack_write_fixed_off()
  3910. * and check_stack_read_fixed_off() relies on stack accesses being
  3911. * aligned.
  3912. */
  3913. strict = true;
  3914. break;
  3915. case PTR_TO_SOCKET:
  3916. pointer_desc = "sock ";
  3917. break;
  3918. case PTR_TO_SOCK_COMMON:
  3919. pointer_desc = "sock_common ";
  3920. break;
  3921. case PTR_TO_TCP_SOCK:
  3922. pointer_desc = "tcp_sock ";
  3923. break;
  3924. case PTR_TO_XDP_SOCK:
  3925. pointer_desc = "xdp_sock ";
  3926. break;
  3927. default:
  3928. break;
  3929. }
  3930. return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
  3931. strict);
  3932. }
  3933. static int update_stack_depth(struct bpf_verifier_env *env,
  3934. const struct bpf_func_state *func,
  3935. int off)
  3936. {
  3937. u16 stack = env->subprog_info[func->subprogno].stack_depth;
  3938. if (stack >= -off)
  3939. return 0;
  3940. /* update known max for given subprogram */
  3941. env->subprog_info[func->subprogno].stack_depth = -off;
  3942. return 0;
  3943. }
  3944. /* starting from main bpf function walk all instructions of the function
  3945. * and recursively walk all callees that given function can call.
  3946. * Ignore jump and exit insns.
  3947. * Since recursion is prevented by check_cfg() this algorithm
  3948. * only needs a local stack of MAX_CALL_FRAMES to remember callsites
  3949. */
  3950. static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx)
  3951. {
  3952. struct bpf_subprog_info *subprog = env->subprog_info;
  3953. struct bpf_insn *insn = env->prog->insnsi;
  3954. int depth = 0, frame = 0, i, subprog_end;
  3955. bool tail_call_reachable = false;
  3956. int ret_insn[MAX_CALL_FRAMES];
  3957. int ret_prog[MAX_CALL_FRAMES];
  3958. int j;
  3959. i = subprog[idx].start;
  3960. process_func:
  3961. /* protect against potential stack overflow that might happen when
  3962. * bpf2bpf calls get combined with tailcalls. Limit the caller's stack
  3963. * depth for such case down to 256 so that the worst case scenario
  3964. * would result in 8k stack size (32 which is tailcall limit * 256 =
  3965. * 8k).
  3966. *
  3967. * To get the idea what might happen, see an example:
  3968. * func1 -> sub rsp, 128
  3969. * subfunc1 -> sub rsp, 256
  3970. * tailcall1 -> add rsp, 256
  3971. * func2 -> sub rsp, 192 (total stack size = 128 + 192 = 320)
  3972. * subfunc2 -> sub rsp, 64
  3973. * subfunc22 -> sub rsp, 128
  3974. * tailcall2 -> add rsp, 128
  3975. * func3 -> sub rsp, 32 (total stack size 128 + 192 + 64 + 32 = 416)
  3976. *
  3977. * tailcall will unwind the current stack frame but it will not get rid
  3978. * of caller's stack as shown on the example above.
  3979. */
  3980. if (idx && subprog[idx].has_tail_call && depth >= 256) {
  3981. verbose(env,
  3982. "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
  3983. depth);
  3984. return -EACCES;
  3985. }
  3986. /* round up to 32-bytes, since this is granularity
  3987. * of interpreter stack size
  3988. */
  3989. depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
  3990. if (depth > MAX_BPF_STACK) {
  3991. verbose(env, "combined stack size of %d calls is %d. Too large\n",
  3992. frame + 1, depth);
  3993. return -EACCES;
  3994. }
  3995. continue_func:
  3996. subprog_end = subprog[idx + 1].start;
  3997. for (; i < subprog_end; i++) {
  3998. int next_insn, sidx;
  3999. if (!bpf_pseudo_call(insn + i) && !bpf_pseudo_func(insn + i))
  4000. continue;
  4001. /* remember insn and function to return to */
  4002. ret_insn[frame] = i + 1;
  4003. ret_prog[frame] = idx;
  4004. /* find the callee */
  4005. next_insn = i + insn[i].imm + 1;
  4006. sidx = find_subprog(env, next_insn);
  4007. if (sidx < 0) {
  4008. WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
  4009. next_insn);
  4010. return -EFAULT;
  4011. }
  4012. if (subprog[sidx].is_async_cb) {
  4013. if (subprog[sidx].has_tail_call) {
  4014. verbose(env, "verifier bug. subprog has tail_call and async cb\n");
  4015. return -EFAULT;
  4016. }
  4017. /* async callbacks don't increase bpf prog stack size unless called directly */
  4018. if (!bpf_pseudo_call(insn + i))
  4019. continue;
  4020. }
  4021. i = next_insn;
  4022. idx = sidx;
  4023. if (subprog[idx].has_tail_call)
  4024. tail_call_reachable = true;
  4025. frame++;
  4026. if (frame >= MAX_CALL_FRAMES) {
  4027. verbose(env, "the call stack of %d frames is too deep !\n",
  4028. frame);
  4029. return -E2BIG;
  4030. }
  4031. goto process_func;
  4032. }
  4033. /* if tail call got detected across bpf2bpf calls then mark each of the
  4034. * currently present subprog frames as tail call reachable subprogs;
  4035. * this info will be utilized by JIT so that we will be preserving the
  4036. * tail call counter throughout bpf2bpf calls combined with tailcalls
  4037. */
  4038. if (tail_call_reachable)
  4039. for (j = 0; j < frame; j++)
  4040. subprog[ret_prog[j]].tail_call_reachable = true;
  4041. if (subprog[0].tail_call_reachable)
  4042. env->prog->aux->tail_call_reachable = true;
  4043. /* end of for() loop means the last insn of the 'subprog'
  4044. * was reached. Doesn't matter whether it was JA or EXIT
  4045. */
  4046. if (frame == 0)
  4047. return 0;
  4048. depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
  4049. frame--;
  4050. i = ret_insn[frame];
  4051. idx = ret_prog[frame];
  4052. goto continue_func;
  4053. }
  4054. static int check_max_stack_depth(struct bpf_verifier_env *env)
  4055. {
  4056. struct bpf_subprog_info *si = env->subprog_info;
  4057. int ret;
  4058. for (int i = 0; i < env->subprog_cnt; i++) {
  4059. if (!i || si[i].is_async_cb) {
  4060. ret = check_max_stack_depth_subprog(env, i);
  4061. if (ret < 0)
  4062. return ret;
  4063. }
  4064. continue;
  4065. }
  4066. return 0;
  4067. }
  4068. #ifndef CONFIG_BPF_JIT_ALWAYS_ON
  4069. static int get_callee_stack_depth(struct bpf_verifier_env *env,
  4070. const struct bpf_insn *insn, int idx)
  4071. {
  4072. int start = idx + insn->imm + 1, subprog;
  4073. subprog = find_subprog(env, start);
  4074. if (subprog < 0) {
  4075. WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
  4076. start);
  4077. return -EFAULT;
  4078. }
  4079. return env->subprog_info[subprog].stack_depth;
  4080. }
  4081. #endif
  4082. static int __check_buffer_access(struct bpf_verifier_env *env,
  4083. const char *buf_info,
  4084. const struct bpf_reg_state *reg,
  4085. int regno, int off, int size)
  4086. {
  4087. if (off < 0) {
  4088. verbose(env,
  4089. "R%d invalid %s buffer access: off=%d, size=%d\n",
  4090. regno, buf_info, off, size);
  4091. return -EACCES;
  4092. }
  4093. if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
  4094. char tn_buf[48];
  4095. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  4096. verbose(env,
  4097. "R%d invalid variable buffer offset: off=%d, var_off=%s\n",
  4098. regno, off, tn_buf);
  4099. return -EACCES;
  4100. }
  4101. return 0;
  4102. }
  4103. static int check_tp_buffer_access(struct bpf_verifier_env *env,
  4104. const struct bpf_reg_state *reg,
  4105. int regno, int off, int size)
  4106. {
  4107. int err;
  4108. err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
  4109. if (err)
  4110. return err;
  4111. if (off + size > env->prog->aux->max_tp_access)
  4112. env->prog->aux->max_tp_access = off + size;
  4113. return 0;
  4114. }
  4115. static int check_buffer_access(struct bpf_verifier_env *env,
  4116. const struct bpf_reg_state *reg,
  4117. int regno, int off, int size,
  4118. bool zero_size_allowed,
  4119. u32 *max_access)
  4120. {
  4121. const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
  4122. int err;
  4123. err = __check_buffer_access(env, buf_info, reg, regno, off, size);
  4124. if (err)
  4125. return err;
  4126. if (off + size > *max_access)
  4127. *max_access = off + size;
  4128. return 0;
  4129. }
  4130. /* BPF architecture zero extends alu32 ops into 64-bit registesr */
  4131. static void zext_32_to_64(struct bpf_reg_state *reg)
  4132. {
  4133. reg->var_off = tnum_subreg(reg->var_off);
  4134. __reg_assign_32_into_64(reg);
  4135. }
  4136. /* truncate register to smaller size (in bytes)
  4137. * must be called with size < BPF_REG_SIZE
  4138. */
  4139. static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
  4140. {
  4141. u64 mask;
  4142. /* clear high bits in bit representation */
  4143. reg->var_off = tnum_cast(reg->var_off, size);
  4144. /* fix arithmetic bounds */
  4145. mask = ((u64)1 << (size * 8)) - 1;
  4146. if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
  4147. reg->umin_value &= mask;
  4148. reg->umax_value &= mask;
  4149. } else {
  4150. reg->umin_value = 0;
  4151. reg->umax_value = mask;
  4152. }
  4153. reg->smin_value = reg->umin_value;
  4154. reg->smax_value = reg->umax_value;
  4155. /* If size is smaller than 32bit register the 32bit register
  4156. * values are also truncated so we push 64-bit bounds into
  4157. * 32-bit bounds. Above were truncated < 32-bits already.
  4158. */
  4159. if (size >= 4)
  4160. return;
  4161. __reg_combine_64_into_32(reg);
  4162. }
  4163. static bool bpf_map_is_rdonly(const struct bpf_map *map)
  4164. {
  4165. /* A map is considered read-only if the following condition are true:
  4166. *
  4167. * 1) BPF program side cannot change any of the map content. The
  4168. * BPF_F_RDONLY_PROG flag is throughout the lifetime of a map
  4169. * and was set at map creation time.
  4170. * 2) The map value(s) have been initialized from user space by a
  4171. * loader and then "frozen", such that no new map update/delete
  4172. * operations from syscall side are possible for the rest of
  4173. * the map's lifetime from that point onwards.
  4174. * 3) Any parallel/pending map update/delete operations from syscall
  4175. * side have been completed. Only after that point, it's safe to
  4176. * assume that map value(s) are immutable.
  4177. */
  4178. return (map->map_flags & BPF_F_RDONLY_PROG) &&
  4179. READ_ONCE(map->frozen) &&
  4180. !bpf_map_write_active(map);
  4181. }
  4182. static int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val)
  4183. {
  4184. void *ptr;
  4185. u64 addr;
  4186. int err;
  4187. err = map->ops->map_direct_value_addr(map, &addr, off);
  4188. if (err)
  4189. return err;
  4190. ptr = (void *)(long)addr + off;
  4191. switch (size) {
  4192. case sizeof(u8):
  4193. *val = (u64)*(u8 *)ptr;
  4194. break;
  4195. case sizeof(u16):
  4196. *val = (u64)*(u16 *)ptr;
  4197. break;
  4198. case sizeof(u32):
  4199. *val = (u64)*(u32 *)ptr;
  4200. break;
  4201. case sizeof(u64):
  4202. *val = *(u64 *)ptr;
  4203. break;
  4204. default:
  4205. return -EINVAL;
  4206. }
  4207. return 0;
  4208. }
  4209. static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
  4210. struct bpf_reg_state *regs,
  4211. int regno, int off, int size,
  4212. enum bpf_access_type atype,
  4213. int value_regno)
  4214. {
  4215. struct bpf_reg_state *reg = regs + regno;
  4216. const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
  4217. const char *tname = btf_name_by_offset(reg->btf, t->name_off);
  4218. enum bpf_type_flag flag = 0;
  4219. u32 btf_id;
  4220. int ret;
  4221. if (off < 0) {
  4222. verbose(env,
  4223. "R%d is ptr_%s invalid negative access: off=%d\n",
  4224. regno, tname, off);
  4225. return -EACCES;
  4226. }
  4227. if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
  4228. char tn_buf[48];
  4229. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  4230. verbose(env,
  4231. "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
  4232. regno, tname, off, tn_buf);
  4233. return -EACCES;
  4234. }
  4235. if (reg->type & MEM_USER) {
  4236. verbose(env,
  4237. "R%d is ptr_%s access user memory: off=%d\n",
  4238. regno, tname, off);
  4239. return -EACCES;
  4240. }
  4241. if (reg->type & MEM_PERCPU) {
  4242. verbose(env,
  4243. "R%d is ptr_%s access percpu memory: off=%d\n",
  4244. regno, tname, off);
  4245. return -EACCES;
  4246. }
  4247. if (env->ops->btf_struct_access) {
  4248. ret = env->ops->btf_struct_access(&env->log, reg->btf, t,
  4249. off, size, atype, &btf_id, &flag);
  4250. } else {
  4251. if (atype != BPF_READ) {
  4252. verbose(env, "only read is supported\n");
  4253. return -EACCES;
  4254. }
  4255. ret = btf_struct_access(&env->log, reg->btf, t, off, size,
  4256. atype, &btf_id, &flag);
  4257. }
  4258. if (ret < 0)
  4259. return ret;
  4260. /* If this is an untrusted pointer, all pointers formed by walking it
  4261. * also inherit the untrusted flag.
  4262. */
  4263. if (type_flag(reg->type) & PTR_UNTRUSTED)
  4264. flag |= PTR_UNTRUSTED;
  4265. if (atype == BPF_READ && value_regno >= 0)
  4266. mark_btf_ld_reg(env, regs, value_regno, ret, reg->btf, btf_id, flag);
  4267. return 0;
  4268. }
  4269. static int check_ptr_to_map_access(struct bpf_verifier_env *env,
  4270. struct bpf_reg_state *regs,
  4271. int regno, int off, int size,
  4272. enum bpf_access_type atype,
  4273. int value_regno)
  4274. {
  4275. struct bpf_reg_state *reg = regs + regno;
  4276. struct bpf_map *map = reg->map_ptr;
  4277. enum bpf_type_flag flag = 0;
  4278. const struct btf_type *t;
  4279. const char *tname;
  4280. u32 btf_id;
  4281. int ret;
  4282. if (!btf_vmlinux) {
  4283. verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
  4284. return -ENOTSUPP;
  4285. }
  4286. if (!map->ops->map_btf_id || !*map->ops->map_btf_id) {
  4287. verbose(env, "map_ptr access not supported for map type %d\n",
  4288. map->map_type);
  4289. return -ENOTSUPP;
  4290. }
  4291. t = btf_type_by_id(btf_vmlinux, *map->ops->map_btf_id);
  4292. tname = btf_name_by_offset(btf_vmlinux, t->name_off);
  4293. if (!env->allow_ptr_to_map_access) {
  4294. verbose(env,
  4295. "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n",
  4296. tname);
  4297. return -EPERM;
  4298. }
  4299. if (off < 0) {
  4300. verbose(env, "R%d is %s invalid negative access: off=%d\n",
  4301. regno, tname, off);
  4302. return -EACCES;
  4303. }
  4304. if (atype != BPF_READ) {
  4305. verbose(env, "only read from %s is supported\n", tname);
  4306. return -EACCES;
  4307. }
  4308. ret = btf_struct_access(&env->log, btf_vmlinux, t, off, size, atype, &btf_id, &flag);
  4309. if (ret < 0)
  4310. return ret;
  4311. if (value_regno >= 0)
  4312. mark_btf_ld_reg(env, regs, value_regno, ret, btf_vmlinux, btf_id, flag);
  4313. return 0;
  4314. }
  4315. /* Check that the stack access at the given offset is within bounds. The
  4316. * maximum valid offset is -1.
  4317. *
  4318. * The minimum valid offset is -MAX_BPF_STACK for writes, and
  4319. * -state->allocated_stack for reads.
  4320. */
  4321. static int check_stack_slot_within_bounds(int off,
  4322. struct bpf_func_state *state,
  4323. enum bpf_access_type t)
  4324. {
  4325. int min_valid_off;
  4326. if (t == BPF_WRITE)
  4327. min_valid_off = -MAX_BPF_STACK;
  4328. else
  4329. min_valid_off = -state->allocated_stack;
  4330. if (off < min_valid_off || off > -1)
  4331. return -EACCES;
  4332. return 0;
  4333. }
  4334. /* Check that the stack access at 'regno + off' falls within the maximum stack
  4335. * bounds.
  4336. *
  4337. * 'off' includes `regno->offset`, but not its dynamic part (if any).
  4338. */
  4339. static int check_stack_access_within_bounds(
  4340. struct bpf_verifier_env *env,
  4341. int regno, int off, int access_size,
  4342. enum bpf_access_src src, enum bpf_access_type type)
  4343. {
  4344. struct bpf_reg_state *regs = cur_regs(env);
  4345. struct bpf_reg_state *reg = regs + regno;
  4346. struct bpf_func_state *state = func(env, reg);
  4347. int min_off, max_off;
  4348. int err;
  4349. char *err_extra;
  4350. if (src == ACCESS_HELPER)
  4351. /* We don't know if helpers are reading or writing (or both). */
  4352. err_extra = " indirect access to";
  4353. else if (type == BPF_READ)
  4354. err_extra = " read from";
  4355. else
  4356. err_extra = " write to";
  4357. if (tnum_is_const(reg->var_off)) {
  4358. min_off = reg->var_off.value + off;
  4359. if (access_size > 0)
  4360. max_off = min_off + access_size - 1;
  4361. else
  4362. max_off = min_off;
  4363. } else {
  4364. if (reg->smax_value >= BPF_MAX_VAR_OFF ||
  4365. reg->smin_value <= -BPF_MAX_VAR_OFF) {
  4366. verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
  4367. err_extra, regno);
  4368. return -EACCES;
  4369. }
  4370. min_off = reg->smin_value + off;
  4371. if (access_size > 0)
  4372. max_off = reg->smax_value + off + access_size - 1;
  4373. else
  4374. max_off = min_off;
  4375. }
  4376. err = check_stack_slot_within_bounds(min_off, state, type);
  4377. if (!err)
  4378. err = check_stack_slot_within_bounds(max_off, state, type);
  4379. if (err) {
  4380. if (tnum_is_const(reg->var_off)) {
  4381. verbose(env, "invalid%s stack R%d off=%d size=%d\n",
  4382. err_extra, regno, off, access_size);
  4383. } else {
  4384. char tn_buf[48];
  4385. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  4386. verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n",
  4387. err_extra, regno, tn_buf, access_size);
  4388. }
  4389. }
  4390. return err;
  4391. }
  4392. /* check whether memory at (regno + off) is accessible for t = (read | write)
  4393. * if t==write, value_regno is a register which value is stored into memory
  4394. * if t==read, value_regno is a register which will receive the value from memory
  4395. * if t==write && value_regno==-1, some unknown value is stored into memory
  4396. * if t==read && value_regno==-1, don't care what we read from memory
  4397. */
  4398. static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
  4399. int off, int bpf_size, enum bpf_access_type t,
  4400. int value_regno, bool strict_alignment_once)
  4401. {
  4402. struct bpf_reg_state *regs = cur_regs(env);
  4403. struct bpf_reg_state *reg = regs + regno;
  4404. struct bpf_func_state *state;
  4405. int size, err = 0;
  4406. size = bpf_size_to_bytes(bpf_size);
  4407. if (size < 0)
  4408. return size;
  4409. /* alignment checks will add in reg->off themselves */
  4410. err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
  4411. if (err)
  4412. return err;
  4413. /* for access checks, reg->off is just part of off */
  4414. off += reg->off;
  4415. if (reg->type == PTR_TO_MAP_KEY) {
  4416. if (t == BPF_WRITE) {
  4417. verbose(env, "write to change key R%d not allowed\n", regno);
  4418. return -EACCES;
  4419. }
  4420. err = check_mem_region_access(env, regno, off, size,
  4421. reg->map_ptr->key_size, false);
  4422. if (err)
  4423. return err;
  4424. if (value_regno >= 0)
  4425. mark_reg_unknown(env, regs, value_regno);
  4426. } else if (reg->type == PTR_TO_MAP_VALUE) {
  4427. struct bpf_map_value_off_desc *kptr_off_desc = NULL;
  4428. if (t == BPF_WRITE && value_regno >= 0 &&
  4429. is_pointer_value(env, value_regno)) {
  4430. verbose(env, "R%d leaks addr into map\n", value_regno);
  4431. return -EACCES;
  4432. }
  4433. err = check_map_access_type(env, regno, off, size, t);
  4434. if (err)
  4435. return err;
  4436. err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);
  4437. if (err)
  4438. return err;
  4439. if (tnum_is_const(reg->var_off))
  4440. kptr_off_desc = bpf_map_kptr_off_contains(reg->map_ptr,
  4441. off + reg->var_off.value);
  4442. if (kptr_off_desc) {
  4443. err = check_map_kptr_access(env, regno, value_regno, insn_idx,
  4444. kptr_off_desc);
  4445. } else if (t == BPF_READ && value_regno >= 0) {
  4446. struct bpf_map *map = reg->map_ptr;
  4447. /* if map is read-only, track its contents as scalars */
  4448. if (tnum_is_const(reg->var_off) &&
  4449. bpf_map_is_rdonly(map) &&
  4450. map->ops->map_direct_value_addr) {
  4451. int map_off = off + reg->var_off.value;
  4452. u64 val = 0;
  4453. err = bpf_map_direct_read(map, map_off, size,
  4454. &val);
  4455. if (err)
  4456. return err;
  4457. regs[value_regno].type = SCALAR_VALUE;
  4458. __mark_reg_known(&regs[value_regno], val);
  4459. } else {
  4460. mark_reg_unknown(env, regs, value_regno);
  4461. }
  4462. }
  4463. } else if (base_type(reg->type) == PTR_TO_MEM) {
  4464. bool rdonly_mem = type_is_rdonly_mem(reg->type);
  4465. if (type_may_be_null(reg->type)) {
  4466. verbose(env, "R%d invalid mem access '%s'\n", regno,
  4467. reg_type_str(env, reg->type));
  4468. return -EACCES;
  4469. }
  4470. if (t == BPF_WRITE && rdonly_mem) {
  4471. verbose(env, "R%d cannot write into %s\n",
  4472. regno, reg_type_str(env, reg->type));
  4473. return -EACCES;
  4474. }
  4475. if (t == BPF_WRITE && value_regno >= 0 &&
  4476. is_pointer_value(env, value_regno)) {
  4477. verbose(env, "R%d leaks addr into mem\n", value_regno);
  4478. return -EACCES;
  4479. }
  4480. err = check_mem_region_access(env, regno, off, size,
  4481. reg->mem_size, false);
  4482. if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
  4483. mark_reg_unknown(env, regs, value_regno);
  4484. } else if (reg->type == PTR_TO_CTX) {
  4485. enum bpf_reg_type reg_type = SCALAR_VALUE;
  4486. struct btf *btf = NULL;
  4487. u32 btf_id = 0;
  4488. if (t == BPF_WRITE && value_regno >= 0 &&
  4489. is_pointer_value(env, value_regno)) {
  4490. verbose(env, "R%d leaks addr into ctx\n", value_regno);
  4491. return -EACCES;
  4492. }
  4493. err = check_ptr_off_reg(env, reg, regno);
  4494. if (err < 0)
  4495. return err;
  4496. err = check_ctx_access(env, insn_idx, off, size, t, &reg_type, &btf,
  4497. &btf_id);
  4498. if (err)
  4499. verbose_linfo(env, insn_idx, "; ");
  4500. if (!err && t == BPF_READ && value_regno >= 0) {
  4501. /* ctx access returns either a scalar, or a
  4502. * PTR_TO_PACKET[_META,_END]. In the latter
  4503. * case, we know the offset is zero.
  4504. */
  4505. if (reg_type == SCALAR_VALUE) {
  4506. mark_reg_unknown(env, regs, value_regno);
  4507. } else {
  4508. mark_reg_known_zero(env, regs,
  4509. value_regno);
  4510. if (type_may_be_null(reg_type))
  4511. regs[value_regno].id = ++env->id_gen;
  4512. /* A load of ctx field could have different
  4513. * actual load size with the one encoded in the
  4514. * insn. When the dst is PTR, it is for sure not
  4515. * a sub-register.
  4516. */
  4517. regs[value_regno].subreg_def = DEF_NOT_SUBREG;
  4518. if (base_type(reg_type) == PTR_TO_BTF_ID) {
  4519. regs[value_regno].btf = btf;
  4520. regs[value_regno].btf_id = btf_id;
  4521. }
  4522. }
  4523. regs[value_regno].type = reg_type;
  4524. }
  4525. } else if (reg->type == PTR_TO_STACK) {
  4526. /* Basic bounds checks. */
  4527. err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
  4528. if (err)
  4529. return err;
  4530. state = func(env, reg);
  4531. err = update_stack_depth(env, state, off);
  4532. if (err)
  4533. return err;
  4534. if (t == BPF_READ)
  4535. err = check_stack_read(env, regno, off, size,
  4536. value_regno);
  4537. else
  4538. err = check_stack_write(env, regno, off, size,
  4539. value_regno, insn_idx);
  4540. } else if (reg_is_pkt_pointer(reg)) {
  4541. if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
  4542. verbose(env, "cannot write into packet\n");
  4543. return -EACCES;
  4544. }
  4545. if (t == BPF_WRITE && value_regno >= 0 &&
  4546. is_pointer_value(env, value_regno)) {
  4547. verbose(env, "R%d leaks addr into packet\n",
  4548. value_regno);
  4549. return -EACCES;
  4550. }
  4551. err = check_packet_access(env, regno, off, size, false);
  4552. if (!err && t == BPF_READ && value_regno >= 0)
  4553. mark_reg_unknown(env, regs, value_regno);
  4554. } else if (reg->type == PTR_TO_FLOW_KEYS) {
  4555. if (t == BPF_WRITE && value_regno >= 0 &&
  4556. is_pointer_value(env, value_regno)) {
  4557. verbose(env, "R%d leaks addr into flow keys\n",
  4558. value_regno);
  4559. return -EACCES;
  4560. }
  4561. err = check_flow_keys_access(env, off, size);
  4562. if (!err && t == BPF_READ && value_regno >= 0)
  4563. mark_reg_unknown(env, regs, value_regno);
  4564. } else if (type_is_sk_pointer(reg->type)) {
  4565. if (t == BPF_WRITE) {
  4566. verbose(env, "R%d cannot write into %s\n",
  4567. regno, reg_type_str(env, reg->type));
  4568. return -EACCES;
  4569. }
  4570. err = check_sock_access(env, insn_idx, regno, off, size, t);
  4571. if (!err && value_regno >= 0)
  4572. mark_reg_unknown(env, regs, value_regno);
  4573. } else if (reg->type == PTR_TO_TP_BUFFER) {
  4574. err = check_tp_buffer_access(env, reg, regno, off, size);
  4575. if (!err && t == BPF_READ && value_regno >= 0)
  4576. mark_reg_unknown(env, regs, value_regno);
  4577. } else if (base_type(reg->type) == PTR_TO_BTF_ID &&
  4578. !type_may_be_null(reg->type)) {
  4579. err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
  4580. value_regno);
  4581. } else if (reg->type == CONST_PTR_TO_MAP) {
  4582. err = check_ptr_to_map_access(env, regs, regno, off, size, t,
  4583. value_regno);
  4584. } else if (base_type(reg->type) == PTR_TO_BUF) {
  4585. bool rdonly_mem = type_is_rdonly_mem(reg->type);
  4586. u32 *max_access;
  4587. if (rdonly_mem) {
  4588. if (t == BPF_WRITE) {
  4589. verbose(env, "R%d cannot write into %s\n",
  4590. regno, reg_type_str(env, reg->type));
  4591. return -EACCES;
  4592. }
  4593. max_access = &env->prog->aux->max_rdonly_access;
  4594. } else {
  4595. max_access = &env->prog->aux->max_rdwr_access;
  4596. }
  4597. err = check_buffer_access(env, reg, regno, off, size, false,
  4598. max_access);
  4599. if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
  4600. mark_reg_unknown(env, regs, value_regno);
  4601. } else {
  4602. verbose(env, "R%d invalid mem access '%s'\n", regno,
  4603. reg_type_str(env, reg->type));
  4604. return -EACCES;
  4605. }
  4606. if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
  4607. regs[value_regno].type == SCALAR_VALUE) {
  4608. /* b/h/w load zero-extends, mark upper bits as known 0 */
  4609. coerce_reg_to_size(&regs[value_regno], size);
  4610. }
  4611. return err;
  4612. }
  4613. static int check_atomic(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
  4614. {
  4615. int load_reg;
  4616. int err;
  4617. switch (insn->imm) {
  4618. case BPF_ADD:
  4619. case BPF_ADD | BPF_FETCH:
  4620. case BPF_AND:
  4621. case BPF_AND | BPF_FETCH:
  4622. case BPF_OR:
  4623. case BPF_OR | BPF_FETCH:
  4624. case BPF_XOR:
  4625. case BPF_XOR | BPF_FETCH:
  4626. case BPF_XCHG:
  4627. case BPF_CMPXCHG:
  4628. break;
  4629. default:
  4630. verbose(env, "BPF_ATOMIC uses invalid atomic opcode %02x\n", insn->imm);
  4631. return -EINVAL;
  4632. }
  4633. if (BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) {
  4634. verbose(env, "invalid atomic operand size\n");
  4635. return -EINVAL;
  4636. }
  4637. /* check src1 operand */
  4638. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  4639. if (err)
  4640. return err;
  4641. /* check src2 operand */
  4642. err = check_reg_arg(env, insn->dst_reg, SRC_OP);
  4643. if (err)
  4644. return err;
  4645. if (insn->imm == BPF_CMPXCHG) {
  4646. /* Check comparison of R0 with memory location */
  4647. const u32 aux_reg = BPF_REG_0;
  4648. err = check_reg_arg(env, aux_reg, SRC_OP);
  4649. if (err)
  4650. return err;
  4651. if (is_pointer_value(env, aux_reg)) {
  4652. verbose(env, "R%d leaks addr into mem\n", aux_reg);
  4653. return -EACCES;
  4654. }
  4655. }
  4656. if (is_pointer_value(env, insn->src_reg)) {
  4657. verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
  4658. return -EACCES;
  4659. }
  4660. if (is_ctx_reg(env, insn->dst_reg) ||
  4661. is_pkt_reg(env, insn->dst_reg) ||
  4662. is_flow_key_reg(env, insn->dst_reg) ||
  4663. is_sk_reg(env, insn->dst_reg)) {
  4664. verbose(env, "BPF_ATOMIC stores into R%d %s is not allowed\n",
  4665. insn->dst_reg,
  4666. reg_type_str(env, reg_state(env, insn->dst_reg)->type));
  4667. return -EACCES;
  4668. }
  4669. if (insn->imm & BPF_FETCH) {
  4670. if (insn->imm == BPF_CMPXCHG)
  4671. load_reg = BPF_REG_0;
  4672. else
  4673. load_reg = insn->src_reg;
  4674. /* check and record load of old value */
  4675. err = check_reg_arg(env, load_reg, DST_OP);
  4676. if (err)
  4677. return err;
  4678. } else {
  4679. /* This instruction accesses a memory location but doesn't
  4680. * actually load it into a register.
  4681. */
  4682. load_reg = -1;
  4683. }
  4684. /* Check whether we can read the memory, with second call for fetch
  4685. * case to simulate the register fill.
  4686. */
  4687. err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
  4688. BPF_SIZE(insn->code), BPF_READ, -1, true);
  4689. if (!err && load_reg >= 0)
  4690. err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
  4691. BPF_SIZE(insn->code), BPF_READ, load_reg,
  4692. true);
  4693. if (err)
  4694. return err;
  4695. /* Check whether we can write into the same memory. */
  4696. err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
  4697. BPF_SIZE(insn->code), BPF_WRITE, -1, true);
  4698. if (err)
  4699. return err;
  4700. return 0;
  4701. }
  4702. /* When register 'regno' is used to read the stack (either directly or through
  4703. * a helper function) make sure that it's within stack boundary and, depending
  4704. * on the access type, that all elements of the stack are initialized.
  4705. *
  4706. * 'off' includes 'regno->off', but not its dynamic part (if any).
  4707. *
  4708. * All registers that have been spilled on the stack in the slots within the
  4709. * read offsets are marked as read.
  4710. */
  4711. static int check_stack_range_initialized(
  4712. struct bpf_verifier_env *env, int regno, int off,
  4713. int access_size, bool zero_size_allowed,
  4714. enum bpf_access_src type, struct bpf_call_arg_meta *meta)
  4715. {
  4716. struct bpf_reg_state *reg = reg_state(env, regno);
  4717. struct bpf_func_state *state = func(env, reg);
  4718. int err, min_off, max_off, i, j, slot, spi;
  4719. char *err_extra = type == ACCESS_HELPER ? " indirect" : "";
  4720. enum bpf_access_type bounds_check_type;
  4721. /* Some accesses can write anything into the stack, others are
  4722. * read-only.
  4723. */
  4724. bool clobber = false;
  4725. if (access_size == 0 && !zero_size_allowed) {
  4726. verbose(env, "invalid zero-sized read\n");
  4727. return -EACCES;
  4728. }
  4729. if (type == ACCESS_HELPER) {
  4730. /* The bounds checks for writes are more permissive than for
  4731. * reads. However, if raw_mode is not set, we'll do extra
  4732. * checks below.
  4733. */
  4734. bounds_check_type = BPF_WRITE;
  4735. clobber = true;
  4736. } else {
  4737. bounds_check_type = BPF_READ;
  4738. }
  4739. err = check_stack_access_within_bounds(env, regno, off, access_size,
  4740. type, bounds_check_type);
  4741. if (err)
  4742. return err;
  4743. if (tnum_is_const(reg->var_off)) {
  4744. min_off = max_off = reg->var_off.value + off;
  4745. } else {
  4746. /* Variable offset is prohibited for unprivileged mode for
  4747. * simplicity since it requires corresponding support in
  4748. * Spectre masking for stack ALU.
  4749. * See also retrieve_ptr_limit().
  4750. */
  4751. if (!env->bypass_spec_v1) {
  4752. char tn_buf[48];
  4753. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  4754. verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n",
  4755. regno, err_extra, tn_buf);
  4756. return -EACCES;
  4757. }
  4758. /* Only initialized buffer on stack is allowed to be accessed
  4759. * with variable offset. With uninitialized buffer it's hard to
  4760. * guarantee that whole memory is marked as initialized on
  4761. * helper return since specific bounds are unknown what may
  4762. * cause uninitialized stack leaking.
  4763. */
  4764. if (meta && meta->raw_mode)
  4765. meta = NULL;
  4766. min_off = reg->smin_value + off;
  4767. max_off = reg->smax_value + off;
  4768. }
  4769. if (meta && meta->raw_mode) {
  4770. meta->access_size = access_size;
  4771. meta->regno = regno;
  4772. return 0;
  4773. }
  4774. for (i = min_off; i < max_off + access_size; i++) {
  4775. u8 *stype;
  4776. slot = -i - 1;
  4777. spi = slot / BPF_REG_SIZE;
  4778. if (state->allocated_stack <= slot)
  4779. goto err;
  4780. stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
  4781. if (*stype == STACK_MISC)
  4782. goto mark;
  4783. if (*stype == STACK_ZERO) {
  4784. if (clobber) {
  4785. /* helper can write anything into the stack */
  4786. *stype = STACK_MISC;
  4787. }
  4788. goto mark;
  4789. }
  4790. if (is_spilled_reg(&state->stack[spi]) &&
  4791. (state->stack[spi].spilled_ptr.type == SCALAR_VALUE ||
  4792. env->allow_ptr_leaks)) {
  4793. if (clobber) {
  4794. __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
  4795. for (j = 0; j < BPF_REG_SIZE; j++)
  4796. scrub_spilled_slot(&state->stack[spi].slot_type[j]);
  4797. }
  4798. goto mark;
  4799. }
  4800. err:
  4801. if (tnum_is_const(reg->var_off)) {
  4802. verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n",
  4803. err_extra, regno, min_off, i - min_off, access_size);
  4804. } else {
  4805. char tn_buf[48];
  4806. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  4807. verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n",
  4808. err_extra, regno, tn_buf, i - min_off, access_size);
  4809. }
  4810. return -EACCES;
  4811. mark:
  4812. /* reading any byte out of 8-byte 'spill_slot' will cause
  4813. * the whole slot to be marked as 'read'
  4814. */
  4815. mark_reg_read(env, &state->stack[spi].spilled_ptr,
  4816. state->stack[spi].spilled_ptr.parent,
  4817. REG_LIVE_READ64);
  4818. /* We do not set REG_LIVE_WRITTEN for stack slot, as we can not
  4819. * be sure that whether stack slot is written to or not. Hence,
  4820. * we must still conservatively propagate reads upwards even if
  4821. * helper may write to the entire memory range.
  4822. */
  4823. }
  4824. return update_stack_depth(env, state, min_off);
  4825. }
  4826. static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
  4827. int access_size, bool zero_size_allowed,
  4828. struct bpf_call_arg_meta *meta)
  4829. {
  4830. struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
  4831. u32 *max_access;
  4832. switch (base_type(reg->type)) {
  4833. case PTR_TO_PACKET:
  4834. case PTR_TO_PACKET_META:
  4835. return check_packet_access(env, regno, reg->off, access_size,
  4836. zero_size_allowed);
  4837. case PTR_TO_MAP_KEY:
  4838. if (meta && meta->raw_mode) {
  4839. verbose(env, "R%d cannot write into %s\n", regno,
  4840. reg_type_str(env, reg->type));
  4841. return -EACCES;
  4842. }
  4843. return check_mem_region_access(env, regno, reg->off, access_size,
  4844. reg->map_ptr->key_size, false);
  4845. case PTR_TO_MAP_VALUE:
  4846. if (check_map_access_type(env, regno, reg->off, access_size,
  4847. meta && meta->raw_mode ? BPF_WRITE :
  4848. BPF_READ))
  4849. return -EACCES;
  4850. return check_map_access(env, regno, reg->off, access_size,
  4851. zero_size_allowed, ACCESS_HELPER);
  4852. case PTR_TO_MEM:
  4853. if (type_is_rdonly_mem(reg->type)) {
  4854. if (meta && meta->raw_mode) {
  4855. verbose(env, "R%d cannot write into %s\n", regno,
  4856. reg_type_str(env, reg->type));
  4857. return -EACCES;
  4858. }
  4859. }
  4860. return check_mem_region_access(env, regno, reg->off,
  4861. access_size, reg->mem_size,
  4862. zero_size_allowed);
  4863. case PTR_TO_BUF:
  4864. if (type_is_rdonly_mem(reg->type)) {
  4865. if (meta && meta->raw_mode) {
  4866. verbose(env, "R%d cannot write into %s\n", regno,
  4867. reg_type_str(env, reg->type));
  4868. return -EACCES;
  4869. }
  4870. max_access = &env->prog->aux->max_rdonly_access;
  4871. } else {
  4872. max_access = &env->prog->aux->max_rdwr_access;
  4873. }
  4874. return check_buffer_access(env, reg, regno, reg->off,
  4875. access_size, zero_size_allowed,
  4876. max_access);
  4877. case PTR_TO_STACK:
  4878. return check_stack_range_initialized(
  4879. env,
  4880. regno, reg->off, access_size,
  4881. zero_size_allowed, ACCESS_HELPER, meta);
  4882. case PTR_TO_CTX:
  4883. /* in case the function doesn't know how to access the context,
  4884. * (because we are in a program of type SYSCALL for example), we
  4885. * can not statically check its size.
  4886. * Dynamically check it now.
  4887. */
  4888. if (!env->ops->convert_ctx_access) {
  4889. enum bpf_access_type atype = meta && meta->raw_mode ? BPF_WRITE : BPF_READ;
  4890. int offset = access_size - 1;
  4891. /* Allow zero-byte read from PTR_TO_CTX */
  4892. if (access_size == 0)
  4893. return zero_size_allowed ? 0 : -EACCES;
  4894. return check_mem_access(env, env->insn_idx, regno, offset, BPF_B,
  4895. atype, -1, false);
  4896. }
  4897. fallthrough;
  4898. default: /* scalar_value or invalid ptr */
  4899. /* Allow zero-byte read from NULL, regardless of pointer type */
  4900. if (zero_size_allowed && access_size == 0 &&
  4901. register_is_null(reg))
  4902. return 0;
  4903. verbose(env, "R%d type=%s ", regno,
  4904. reg_type_str(env, reg->type));
  4905. verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
  4906. return -EACCES;
  4907. }
  4908. }
  4909. static int check_mem_size_reg(struct bpf_verifier_env *env,
  4910. struct bpf_reg_state *reg, u32 regno,
  4911. bool zero_size_allowed,
  4912. struct bpf_call_arg_meta *meta)
  4913. {
  4914. int err;
  4915. /* This is used to refine r0 return value bounds for helpers
  4916. * that enforce this value as an upper bound on return values.
  4917. * See do_refine_retval_range() for helpers that can refine
  4918. * the return value. C type of helper is u32 so we pull register
  4919. * bound from umax_value however, if negative verifier errors
  4920. * out. Only upper bounds can be learned because retval is an
  4921. * int type and negative retvals are allowed.
  4922. */
  4923. meta->msize_max_value = reg->umax_value;
  4924. /* The register is SCALAR_VALUE; the access check
  4925. * happens using its boundaries.
  4926. */
  4927. if (!tnum_is_const(reg->var_off))
  4928. /* For unprivileged variable accesses, disable raw
  4929. * mode so that the program is required to
  4930. * initialize all the memory that the helper could
  4931. * just partially fill up.
  4932. */
  4933. meta = NULL;
  4934. if (reg->smin_value < 0) {
  4935. verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
  4936. regno);
  4937. return -EACCES;
  4938. }
  4939. if (reg->umin_value == 0) {
  4940. err = check_helper_mem_access(env, regno - 1, 0,
  4941. zero_size_allowed,
  4942. meta);
  4943. if (err)
  4944. return err;
  4945. }
  4946. if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
  4947. verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
  4948. regno);
  4949. return -EACCES;
  4950. }
  4951. err = check_helper_mem_access(env, regno - 1,
  4952. reg->umax_value,
  4953. zero_size_allowed, meta);
  4954. if (!err)
  4955. err = mark_chain_precision(env, regno);
  4956. return err;
  4957. }
  4958. int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
  4959. u32 regno, u32 mem_size)
  4960. {
  4961. bool may_be_null = type_may_be_null(reg->type);
  4962. struct bpf_reg_state saved_reg;
  4963. struct bpf_call_arg_meta meta;
  4964. int err;
  4965. if (register_is_null(reg))
  4966. return 0;
  4967. memset(&meta, 0, sizeof(meta));
  4968. /* Assuming that the register contains a value check if the memory
  4969. * access is safe. Temporarily save and restore the register's state as
  4970. * the conversion shouldn't be visible to a caller.
  4971. */
  4972. if (may_be_null) {
  4973. saved_reg = *reg;
  4974. mark_ptr_not_null_reg(reg);
  4975. }
  4976. err = check_helper_mem_access(env, regno, mem_size, true, &meta);
  4977. /* Check access for BPF_WRITE */
  4978. meta.raw_mode = true;
  4979. err = err ?: check_helper_mem_access(env, regno, mem_size, true, &meta);
  4980. if (may_be_null)
  4981. *reg = saved_reg;
  4982. return err;
  4983. }
  4984. int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
  4985. u32 regno)
  4986. {
  4987. struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];
  4988. bool may_be_null = type_may_be_null(mem_reg->type);
  4989. struct bpf_reg_state saved_reg;
  4990. struct bpf_call_arg_meta meta;
  4991. int err;
  4992. WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5);
  4993. memset(&meta, 0, sizeof(meta));
  4994. if (may_be_null) {
  4995. saved_reg = *mem_reg;
  4996. mark_ptr_not_null_reg(mem_reg);
  4997. }
  4998. err = check_mem_size_reg(env, reg, regno, true, &meta);
  4999. /* Check access for BPF_WRITE */
  5000. meta.raw_mode = true;
  5001. err = err ?: check_mem_size_reg(env, reg, regno, true, &meta);
  5002. if (may_be_null)
  5003. *mem_reg = saved_reg;
  5004. return err;
  5005. }
  5006. /* Implementation details:
  5007. * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
  5008. * Two bpf_map_lookups (even with the same key) will have different reg->id.
  5009. * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after
  5010. * value_or_null->value transition, since the verifier only cares about
  5011. * the range of access to valid map value pointer and doesn't care about actual
  5012. * address of the map element.
  5013. * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
  5014. * reg->id > 0 after value_or_null->value transition. By doing so
  5015. * two bpf_map_lookups will be considered two different pointers that
  5016. * point to different bpf_spin_locks.
  5017. * The verifier allows taking only one bpf_spin_lock at a time to avoid
  5018. * dead-locks.
  5019. * Since only one bpf_spin_lock is allowed the checks are simpler than
  5020. * reg_is_refcounted() logic. The verifier needs to remember only
  5021. * one spin_lock instead of array of acquired_refs.
  5022. * cur_state->active_spin_lock remembers which map value element got locked
  5023. * and clears it after bpf_spin_unlock.
  5024. */
  5025. static int process_spin_lock(struct bpf_verifier_env *env, int regno,
  5026. bool is_lock)
  5027. {
  5028. struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
  5029. struct bpf_verifier_state *cur = env->cur_state;
  5030. bool is_const = tnum_is_const(reg->var_off);
  5031. struct bpf_map *map = reg->map_ptr;
  5032. u64 val = reg->var_off.value;
  5033. if (!is_const) {
  5034. verbose(env,
  5035. "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
  5036. regno);
  5037. return -EINVAL;
  5038. }
  5039. if (!map->btf) {
  5040. verbose(env,
  5041. "map '%s' has to have BTF in order to use bpf_spin_lock\n",
  5042. map->name);
  5043. return -EINVAL;
  5044. }
  5045. if (!map_value_has_spin_lock(map)) {
  5046. if (map->spin_lock_off == -E2BIG)
  5047. verbose(env,
  5048. "map '%s' has more than one 'struct bpf_spin_lock'\n",
  5049. map->name);
  5050. else if (map->spin_lock_off == -ENOENT)
  5051. verbose(env,
  5052. "map '%s' doesn't have 'struct bpf_spin_lock'\n",
  5053. map->name);
  5054. else
  5055. verbose(env,
  5056. "map '%s' is not a struct type or bpf_spin_lock is mangled\n",
  5057. map->name);
  5058. return -EINVAL;
  5059. }
  5060. if (map->spin_lock_off != val + reg->off) {
  5061. verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
  5062. val + reg->off);
  5063. return -EINVAL;
  5064. }
  5065. if (is_lock) {
  5066. if (cur->active_spin_lock) {
  5067. verbose(env,
  5068. "Locking two bpf_spin_locks are not allowed\n");
  5069. return -EINVAL;
  5070. }
  5071. cur->active_spin_lock = reg->id;
  5072. } else {
  5073. if (!cur->active_spin_lock) {
  5074. verbose(env, "bpf_spin_unlock without taking a lock\n");
  5075. return -EINVAL;
  5076. }
  5077. if (cur->active_spin_lock != reg->id) {
  5078. verbose(env, "bpf_spin_unlock of different lock\n");
  5079. return -EINVAL;
  5080. }
  5081. cur->active_spin_lock = 0;
  5082. }
  5083. return 0;
  5084. }
  5085. static int process_timer_func(struct bpf_verifier_env *env, int regno,
  5086. struct bpf_call_arg_meta *meta)
  5087. {
  5088. struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
  5089. bool is_const = tnum_is_const(reg->var_off);
  5090. struct bpf_map *map = reg->map_ptr;
  5091. u64 val = reg->var_off.value;
  5092. if (!is_const) {
  5093. verbose(env,
  5094. "R%d doesn't have constant offset. bpf_timer has to be at the constant offset\n",
  5095. regno);
  5096. return -EINVAL;
  5097. }
  5098. if (!map->btf) {
  5099. verbose(env, "map '%s' has to have BTF in order to use bpf_timer\n",
  5100. map->name);
  5101. return -EINVAL;
  5102. }
  5103. if (!map_value_has_timer(map)) {
  5104. if (map->timer_off == -E2BIG)
  5105. verbose(env,
  5106. "map '%s' has more than one 'struct bpf_timer'\n",
  5107. map->name);
  5108. else if (map->timer_off == -ENOENT)
  5109. verbose(env,
  5110. "map '%s' doesn't have 'struct bpf_timer'\n",
  5111. map->name);
  5112. else
  5113. verbose(env,
  5114. "map '%s' is not a struct type or bpf_timer is mangled\n",
  5115. map->name);
  5116. return -EINVAL;
  5117. }
  5118. if (map->timer_off != val + reg->off) {
  5119. verbose(env, "off %lld doesn't point to 'struct bpf_timer' that is at %d\n",
  5120. val + reg->off, map->timer_off);
  5121. return -EINVAL;
  5122. }
  5123. if (meta->map_ptr) {
  5124. verbose(env, "verifier bug. Two map pointers in a timer helper\n");
  5125. return -EFAULT;
  5126. }
  5127. meta->map_uid = reg->map_uid;
  5128. meta->map_ptr = map;
  5129. return 0;
  5130. }
  5131. static int process_kptr_func(struct bpf_verifier_env *env, int regno,
  5132. struct bpf_call_arg_meta *meta)
  5133. {
  5134. struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
  5135. struct bpf_map_value_off_desc *off_desc;
  5136. struct bpf_map *map_ptr = reg->map_ptr;
  5137. u32 kptr_off;
  5138. int ret;
  5139. if (!tnum_is_const(reg->var_off)) {
  5140. verbose(env,
  5141. "R%d doesn't have constant offset. kptr has to be at the constant offset\n",
  5142. regno);
  5143. return -EINVAL;
  5144. }
  5145. if (!map_ptr->btf) {
  5146. verbose(env, "map '%s' has to have BTF in order to use bpf_kptr_xchg\n",
  5147. map_ptr->name);
  5148. return -EINVAL;
  5149. }
  5150. if (!map_value_has_kptrs(map_ptr)) {
  5151. ret = PTR_ERR_OR_ZERO(map_ptr->kptr_off_tab);
  5152. if (ret == -E2BIG)
  5153. verbose(env, "map '%s' has more than %d kptr\n", map_ptr->name,
  5154. BPF_MAP_VALUE_OFF_MAX);
  5155. else if (ret == -EEXIST)
  5156. verbose(env, "map '%s' has repeating kptr BTF tags\n", map_ptr->name);
  5157. else
  5158. verbose(env, "map '%s' has no valid kptr\n", map_ptr->name);
  5159. return -EINVAL;
  5160. }
  5161. meta->map_ptr = map_ptr;
  5162. kptr_off = reg->off + reg->var_off.value;
  5163. off_desc = bpf_map_kptr_off_contains(map_ptr, kptr_off);
  5164. if (!off_desc) {
  5165. verbose(env, "off=%d doesn't point to kptr\n", kptr_off);
  5166. return -EACCES;
  5167. }
  5168. if (off_desc->type != BPF_KPTR_REF) {
  5169. verbose(env, "off=%d kptr isn't referenced kptr\n", kptr_off);
  5170. return -EACCES;
  5171. }
  5172. meta->kptr_off_desc = off_desc;
  5173. return 0;
  5174. }
  5175. static bool arg_type_is_mem_size(enum bpf_arg_type type)
  5176. {
  5177. return type == ARG_CONST_SIZE ||
  5178. type == ARG_CONST_SIZE_OR_ZERO;
  5179. }
  5180. static bool arg_type_is_release(enum bpf_arg_type type)
  5181. {
  5182. return type & OBJ_RELEASE;
  5183. }
  5184. static bool arg_type_is_dynptr(enum bpf_arg_type type)
  5185. {
  5186. return base_type(type) == ARG_PTR_TO_DYNPTR;
  5187. }
  5188. static int int_ptr_type_to_size(enum bpf_arg_type type)
  5189. {
  5190. if (type == ARG_PTR_TO_INT)
  5191. return sizeof(u32);
  5192. else if (type == ARG_PTR_TO_LONG)
  5193. return sizeof(u64);
  5194. return -EINVAL;
  5195. }
  5196. static int resolve_map_arg_type(struct bpf_verifier_env *env,
  5197. const struct bpf_call_arg_meta *meta,
  5198. enum bpf_arg_type *arg_type)
  5199. {
  5200. if (!meta->map_ptr) {
  5201. /* kernel subsystem misconfigured verifier */
  5202. verbose(env, "invalid map_ptr to access map->type\n");
  5203. return -EACCES;
  5204. }
  5205. switch (meta->map_ptr->map_type) {
  5206. case BPF_MAP_TYPE_SOCKMAP:
  5207. case BPF_MAP_TYPE_SOCKHASH:
  5208. if (*arg_type == ARG_PTR_TO_MAP_VALUE) {
  5209. *arg_type = ARG_PTR_TO_BTF_ID_SOCK_COMMON;
  5210. } else {
  5211. verbose(env, "invalid arg_type for sockmap/sockhash\n");
  5212. return -EINVAL;
  5213. }
  5214. break;
  5215. case BPF_MAP_TYPE_BLOOM_FILTER:
  5216. if (meta->func_id == BPF_FUNC_map_peek_elem)
  5217. *arg_type = ARG_PTR_TO_MAP_VALUE;
  5218. break;
  5219. default:
  5220. break;
  5221. }
  5222. return 0;
  5223. }
  5224. struct bpf_reg_types {
  5225. const enum bpf_reg_type types[10];
  5226. u32 *btf_id;
  5227. };
  5228. static const struct bpf_reg_types map_key_value_types = {
  5229. .types = {
  5230. PTR_TO_STACK,
  5231. PTR_TO_PACKET,
  5232. PTR_TO_PACKET_META,
  5233. PTR_TO_MAP_KEY,
  5234. PTR_TO_MAP_VALUE,
  5235. },
  5236. };
  5237. static const struct bpf_reg_types sock_types = {
  5238. .types = {
  5239. PTR_TO_SOCK_COMMON,
  5240. PTR_TO_SOCKET,
  5241. PTR_TO_TCP_SOCK,
  5242. PTR_TO_XDP_SOCK,
  5243. },
  5244. };
  5245. #ifdef CONFIG_NET
  5246. static const struct bpf_reg_types btf_id_sock_common_types = {
  5247. .types = {
  5248. PTR_TO_SOCK_COMMON,
  5249. PTR_TO_SOCKET,
  5250. PTR_TO_TCP_SOCK,
  5251. PTR_TO_XDP_SOCK,
  5252. PTR_TO_BTF_ID,
  5253. },
  5254. .btf_id = &btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON],
  5255. };
  5256. #endif
  5257. static const struct bpf_reg_types mem_types = {
  5258. .types = {
  5259. PTR_TO_STACK,
  5260. PTR_TO_PACKET,
  5261. PTR_TO_PACKET_META,
  5262. PTR_TO_MAP_KEY,
  5263. PTR_TO_MAP_VALUE,
  5264. PTR_TO_MEM,
  5265. PTR_TO_MEM | MEM_ALLOC,
  5266. PTR_TO_BUF,
  5267. },
  5268. };
  5269. static const struct bpf_reg_types int_ptr_types = {
  5270. .types = {
  5271. PTR_TO_STACK,
  5272. PTR_TO_PACKET,
  5273. PTR_TO_PACKET_META,
  5274. PTR_TO_MAP_KEY,
  5275. PTR_TO_MAP_VALUE,
  5276. },
  5277. };
  5278. static const struct bpf_reg_types fullsock_types = { .types = { PTR_TO_SOCKET } };
  5279. static const struct bpf_reg_types scalar_types = { .types = { SCALAR_VALUE } };
  5280. static const struct bpf_reg_types context_types = { .types = { PTR_TO_CTX } };
  5281. static const struct bpf_reg_types alloc_mem_types = { .types = { PTR_TO_MEM | MEM_ALLOC } };
  5282. static const struct bpf_reg_types const_map_ptr_types = { .types = { CONST_PTR_TO_MAP } };
  5283. static const struct bpf_reg_types btf_ptr_types = { .types = { PTR_TO_BTF_ID } };
  5284. static const struct bpf_reg_types spin_lock_types = { .types = { PTR_TO_MAP_VALUE } };
  5285. static const struct bpf_reg_types percpu_btf_ptr_types = { .types = { PTR_TO_BTF_ID | MEM_PERCPU } };
  5286. static const struct bpf_reg_types func_ptr_types = { .types = { PTR_TO_FUNC } };
  5287. static const struct bpf_reg_types stack_ptr_types = { .types = { PTR_TO_STACK } };
  5288. static const struct bpf_reg_types const_str_ptr_types = { .types = { PTR_TO_MAP_VALUE } };
  5289. static const struct bpf_reg_types timer_types = { .types = { PTR_TO_MAP_VALUE } };
  5290. static const struct bpf_reg_types kptr_types = { .types = { PTR_TO_MAP_VALUE } };
  5291. static const struct bpf_reg_types dynptr_types = {
  5292. .types = {
  5293. PTR_TO_STACK,
  5294. PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL,
  5295. }
  5296. };
  5297. static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
  5298. [ARG_PTR_TO_MAP_KEY] = &map_key_value_types,
  5299. [ARG_PTR_TO_MAP_VALUE] = &map_key_value_types,
  5300. [ARG_CONST_SIZE] = &scalar_types,
  5301. [ARG_CONST_SIZE_OR_ZERO] = &scalar_types,
  5302. [ARG_CONST_ALLOC_SIZE_OR_ZERO] = &scalar_types,
  5303. [ARG_CONST_MAP_PTR] = &const_map_ptr_types,
  5304. [ARG_PTR_TO_CTX] = &context_types,
  5305. [ARG_PTR_TO_SOCK_COMMON] = &sock_types,
  5306. #ifdef CONFIG_NET
  5307. [ARG_PTR_TO_BTF_ID_SOCK_COMMON] = &btf_id_sock_common_types,
  5308. #endif
  5309. [ARG_PTR_TO_SOCKET] = &fullsock_types,
  5310. [ARG_PTR_TO_BTF_ID] = &btf_ptr_types,
  5311. [ARG_PTR_TO_SPIN_LOCK] = &spin_lock_types,
  5312. [ARG_PTR_TO_MEM] = &mem_types,
  5313. [ARG_PTR_TO_ALLOC_MEM] = &alloc_mem_types,
  5314. [ARG_PTR_TO_INT] = &int_ptr_types,
  5315. [ARG_PTR_TO_LONG] = &int_ptr_types,
  5316. [ARG_PTR_TO_PERCPU_BTF_ID] = &percpu_btf_ptr_types,
  5317. [ARG_PTR_TO_FUNC] = &func_ptr_types,
  5318. [ARG_PTR_TO_STACK] = &stack_ptr_types,
  5319. [ARG_PTR_TO_CONST_STR] = &const_str_ptr_types,
  5320. [ARG_PTR_TO_TIMER] = &timer_types,
  5321. [ARG_PTR_TO_KPTR] = &kptr_types,
  5322. [ARG_PTR_TO_DYNPTR] = &dynptr_types,
  5323. };
  5324. static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
  5325. enum bpf_arg_type arg_type,
  5326. const u32 *arg_btf_id,
  5327. struct bpf_call_arg_meta *meta)
  5328. {
  5329. struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
  5330. enum bpf_reg_type expected, type = reg->type;
  5331. const struct bpf_reg_types *compatible;
  5332. int i, j;
  5333. compatible = compatible_reg_types[base_type(arg_type)];
  5334. if (!compatible) {
  5335. verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
  5336. return -EFAULT;
  5337. }
  5338. /* ARG_PTR_TO_MEM + RDONLY is compatible with PTR_TO_MEM and PTR_TO_MEM + RDONLY,
  5339. * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM and NOT with PTR_TO_MEM + RDONLY
  5340. *
  5341. * Same for MAYBE_NULL:
  5342. *
  5343. * ARG_PTR_TO_MEM + MAYBE_NULL is compatible with PTR_TO_MEM and PTR_TO_MEM + MAYBE_NULL,
  5344. * but ARG_PTR_TO_MEM is compatible only with PTR_TO_MEM but NOT with PTR_TO_MEM + MAYBE_NULL
  5345. *
  5346. * Therefore we fold these flags depending on the arg_type before comparison.
  5347. */
  5348. if (arg_type & MEM_RDONLY)
  5349. type &= ~MEM_RDONLY;
  5350. if (arg_type & PTR_MAYBE_NULL)
  5351. type &= ~PTR_MAYBE_NULL;
  5352. for (i = 0; i < ARRAY_SIZE(compatible->types); i++) {
  5353. expected = compatible->types[i];
  5354. if (expected == NOT_INIT)
  5355. break;
  5356. if (type == expected)
  5357. goto found;
  5358. }
  5359. verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
  5360. for (j = 0; j + 1 < i; j++)
  5361. verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
  5362. verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
  5363. return -EACCES;
  5364. found:
  5365. if (reg->type == PTR_TO_BTF_ID) {
  5366. /* For bpf_sk_release, it needs to match against first member
  5367. * 'struct sock_common', hence make an exception for it. This
  5368. * allows bpf_sk_release to work for multiple socket types.
  5369. */
  5370. bool strict_type_match = arg_type_is_release(arg_type) &&
  5371. meta->func_id != BPF_FUNC_sk_release;
  5372. if (!arg_btf_id) {
  5373. if (!compatible->btf_id) {
  5374. verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
  5375. return -EFAULT;
  5376. }
  5377. arg_btf_id = compatible->btf_id;
  5378. }
  5379. if (meta->func_id == BPF_FUNC_kptr_xchg) {
  5380. if (map_kptr_match_type(env, meta->kptr_off_desc, reg, regno))
  5381. return -EACCES;
  5382. } else {
  5383. if (arg_btf_id == BPF_PTR_POISON) {
  5384. verbose(env, "verifier internal error:");
  5385. verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
  5386. regno);
  5387. return -EACCES;
  5388. }
  5389. if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id, reg->off,
  5390. btf_vmlinux, *arg_btf_id,
  5391. strict_type_match)) {
  5392. verbose(env, "R%d is of type %s but %s is expected\n",
  5393. regno, kernel_type_name(reg->btf, reg->btf_id),
  5394. kernel_type_name(btf_vmlinux, *arg_btf_id));
  5395. return -EACCES;
  5396. }
  5397. }
  5398. }
  5399. return 0;
  5400. }
  5401. int check_func_arg_reg_off(struct bpf_verifier_env *env,
  5402. const struct bpf_reg_state *reg, int regno,
  5403. enum bpf_arg_type arg_type)
  5404. {
  5405. enum bpf_reg_type type = reg->type;
  5406. bool fixed_off_ok = false;
  5407. switch ((u32)type) {
  5408. /* Pointer types where reg offset is explicitly allowed: */
  5409. case PTR_TO_STACK:
  5410. if (arg_type_is_dynptr(arg_type) && reg->off % BPF_REG_SIZE) {
  5411. verbose(env, "cannot pass in dynptr at an offset\n");
  5412. return -EINVAL;
  5413. }
  5414. fallthrough;
  5415. case PTR_TO_PACKET:
  5416. case PTR_TO_PACKET_META:
  5417. case PTR_TO_MAP_KEY:
  5418. case PTR_TO_MAP_VALUE:
  5419. case PTR_TO_MEM:
  5420. case PTR_TO_MEM | MEM_RDONLY:
  5421. case PTR_TO_MEM | MEM_ALLOC:
  5422. case PTR_TO_BUF:
  5423. case PTR_TO_BUF | MEM_RDONLY:
  5424. case SCALAR_VALUE:
  5425. /* Some of the argument types nevertheless require a
  5426. * zero register offset.
  5427. */
  5428. if (base_type(arg_type) != ARG_PTR_TO_ALLOC_MEM)
  5429. return 0;
  5430. break;
  5431. /* All the rest must be rejected, except PTR_TO_BTF_ID which allows
  5432. * fixed offset.
  5433. */
  5434. case PTR_TO_BTF_ID:
  5435. /* When referenced PTR_TO_BTF_ID is passed to release function,
  5436. * it's fixed offset must be 0. In the other cases, fixed offset
  5437. * can be non-zero.
  5438. */
  5439. if (arg_type_is_release(arg_type) && reg->off) {
  5440. verbose(env, "R%d must have zero offset when passed to release func\n",
  5441. regno);
  5442. return -EINVAL;
  5443. }
  5444. /* For arg is release pointer, fixed_off_ok must be false, but
  5445. * we already checked and rejected reg->off != 0 above, so set
  5446. * to true to allow fixed offset for all other cases.
  5447. */
  5448. fixed_off_ok = true;
  5449. break;
  5450. default:
  5451. break;
  5452. }
  5453. return __check_ptr_off_reg(env, reg, regno, fixed_off_ok);
  5454. }
  5455. static u32 stack_slot_get_id(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
  5456. {
  5457. struct bpf_func_state *state = func(env, reg);
  5458. int spi = get_spi(reg->off);
  5459. return state->stack[spi].spilled_ptr.id;
  5460. }
  5461. static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
  5462. struct bpf_call_arg_meta *meta,
  5463. const struct bpf_func_proto *fn)
  5464. {
  5465. u32 regno = BPF_REG_1 + arg;
  5466. struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
  5467. enum bpf_arg_type arg_type = fn->arg_type[arg];
  5468. enum bpf_reg_type type = reg->type;
  5469. u32 *arg_btf_id = NULL;
  5470. int err = 0;
  5471. if (arg_type == ARG_DONTCARE)
  5472. return 0;
  5473. err = check_reg_arg(env, regno, SRC_OP);
  5474. if (err)
  5475. return err;
  5476. if (arg_type == ARG_ANYTHING) {
  5477. if (is_pointer_value(env, regno)) {
  5478. verbose(env, "R%d leaks addr into helper function\n",
  5479. regno);
  5480. return -EACCES;
  5481. }
  5482. return 0;
  5483. }
  5484. if (type_is_pkt_pointer(type) &&
  5485. !may_access_direct_pkt_data(env, meta, BPF_READ)) {
  5486. verbose(env, "helper access to the packet is not allowed\n");
  5487. return -EACCES;
  5488. }
  5489. if (base_type(arg_type) == ARG_PTR_TO_MAP_VALUE) {
  5490. err = resolve_map_arg_type(env, meta, &arg_type);
  5491. if (err)
  5492. return err;
  5493. }
  5494. if (register_is_null(reg) && type_may_be_null(arg_type))
  5495. /* A NULL register has a SCALAR_VALUE type, so skip
  5496. * type checking.
  5497. */
  5498. goto skip_type_check;
  5499. /* arg_btf_id and arg_size are in a union. */
  5500. if (base_type(arg_type) == ARG_PTR_TO_BTF_ID)
  5501. arg_btf_id = fn->arg_btf_id[arg];
  5502. err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
  5503. if (err)
  5504. return err;
  5505. err = check_func_arg_reg_off(env, reg, regno, arg_type);
  5506. if (err)
  5507. return err;
  5508. skip_type_check:
  5509. if (arg_type_is_release(arg_type)) {
  5510. if (arg_type_is_dynptr(arg_type)) {
  5511. struct bpf_func_state *state = func(env, reg);
  5512. int spi = get_spi(reg->off);
  5513. if (!is_spi_bounds_valid(state, spi, BPF_DYNPTR_NR_SLOTS) ||
  5514. !state->stack[spi].spilled_ptr.id) {
  5515. verbose(env, "arg %d is an unacquired reference\n", regno);
  5516. return -EINVAL;
  5517. }
  5518. } else if (!reg->ref_obj_id && !register_is_null(reg)) {
  5519. verbose(env, "R%d must be referenced when passed to release function\n",
  5520. regno);
  5521. return -EINVAL;
  5522. }
  5523. if (meta->release_regno) {
  5524. verbose(env, "verifier internal error: more than one release argument\n");
  5525. return -EFAULT;
  5526. }
  5527. meta->release_regno = regno;
  5528. }
  5529. if (reg->ref_obj_id) {
  5530. if (meta->ref_obj_id) {
  5531. verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
  5532. regno, reg->ref_obj_id,
  5533. meta->ref_obj_id);
  5534. return -EFAULT;
  5535. }
  5536. meta->ref_obj_id = reg->ref_obj_id;
  5537. }
  5538. switch (base_type(arg_type)) {
  5539. case ARG_CONST_MAP_PTR:
  5540. /* bpf_map_xxx(map_ptr) call: remember that map_ptr */
  5541. if (meta->map_ptr) {
  5542. /* Use map_uid (which is unique id of inner map) to reject:
  5543. * inner_map1 = bpf_map_lookup_elem(outer_map, key1)
  5544. * inner_map2 = bpf_map_lookup_elem(outer_map, key2)
  5545. * if (inner_map1 && inner_map2) {
  5546. * timer = bpf_map_lookup_elem(inner_map1);
  5547. * if (timer)
  5548. * // mismatch would have been allowed
  5549. * bpf_timer_init(timer, inner_map2);
  5550. * }
  5551. *
  5552. * Comparing map_ptr is enough to distinguish normal and outer maps.
  5553. */
  5554. if (meta->map_ptr != reg->map_ptr ||
  5555. meta->map_uid != reg->map_uid) {
  5556. verbose(env,
  5557. "timer pointer in R1 map_uid=%d doesn't match map pointer in R2 map_uid=%d\n",
  5558. meta->map_uid, reg->map_uid);
  5559. return -EINVAL;
  5560. }
  5561. }
  5562. meta->map_ptr = reg->map_ptr;
  5563. meta->map_uid = reg->map_uid;
  5564. break;
  5565. case ARG_PTR_TO_MAP_KEY:
  5566. /* bpf_map_xxx(..., map_ptr, ..., key) call:
  5567. * check that [key, key + map->key_size) are within
  5568. * stack limits and initialized
  5569. */
  5570. if (!meta->map_ptr) {
  5571. /* in function declaration map_ptr must come before
  5572. * map_key, so that it's verified and known before
  5573. * we have to check map_key here. Otherwise it means
  5574. * that kernel subsystem misconfigured verifier
  5575. */
  5576. verbose(env, "invalid map_ptr to access map->key\n");
  5577. return -EACCES;
  5578. }
  5579. err = check_helper_mem_access(env, regno,
  5580. meta->map_ptr->key_size, false,
  5581. NULL);
  5582. break;
  5583. case ARG_PTR_TO_MAP_VALUE:
  5584. if (type_may_be_null(arg_type) && register_is_null(reg))
  5585. return 0;
  5586. /* bpf_map_xxx(..., map_ptr, ..., value) call:
  5587. * check [value, value + map->value_size) validity
  5588. */
  5589. if (!meta->map_ptr) {
  5590. /* kernel subsystem misconfigured verifier */
  5591. verbose(env, "invalid map_ptr to access map->value\n");
  5592. return -EACCES;
  5593. }
  5594. meta->raw_mode = arg_type & MEM_UNINIT;
  5595. err = check_helper_mem_access(env, regno,
  5596. meta->map_ptr->value_size, false,
  5597. meta);
  5598. break;
  5599. case ARG_PTR_TO_PERCPU_BTF_ID:
  5600. if (!reg->btf_id) {
  5601. verbose(env, "Helper has invalid btf_id in R%d\n", regno);
  5602. return -EACCES;
  5603. }
  5604. meta->ret_btf = reg->btf;
  5605. meta->ret_btf_id = reg->btf_id;
  5606. break;
  5607. case ARG_PTR_TO_SPIN_LOCK:
  5608. if (meta->func_id == BPF_FUNC_spin_lock) {
  5609. if (process_spin_lock(env, regno, true))
  5610. return -EACCES;
  5611. } else if (meta->func_id == BPF_FUNC_spin_unlock) {
  5612. if (process_spin_lock(env, regno, false))
  5613. return -EACCES;
  5614. } else {
  5615. verbose(env, "verifier internal error\n");
  5616. return -EFAULT;
  5617. }
  5618. break;
  5619. case ARG_PTR_TO_TIMER:
  5620. if (process_timer_func(env, regno, meta))
  5621. return -EACCES;
  5622. break;
  5623. case ARG_PTR_TO_FUNC:
  5624. meta->subprogno = reg->subprogno;
  5625. break;
  5626. case ARG_PTR_TO_MEM:
  5627. /* The access to this pointer is only checked when we hit the
  5628. * next is_mem_size argument below.
  5629. */
  5630. meta->raw_mode = arg_type & MEM_UNINIT;
  5631. if (arg_type & MEM_FIXED_SIZE) {
  5632. err = check_helper_mem_access(env, regno,
  5633. fn->arg_size[arg], false,
  5634. meta);
  5635. }
  5636. break;
  5637. case ARG_CONST_SIZE:
  5638. err = check_mem_size_reg(env, reg, regno, false, meta);
  5639. break;
  5640. case ARG_CONST_SIZE_OR_ZERO:
  5641. err = check_mem_size_reg(env, reg, regno, true, meta);
  5642. break;
  5643. case ARG_PTR_TO_DYNPTR:
  5644. /* We only need to check for initialized / uninitialized helper
  5645. * dynptr args if the dynptr is not PTR_TO_DYNPTR, as the
  5646. * assumption is that if it is, that a helper function
  5647. * initialized the dynptr on behalf of the BPF program.
  5648. */
  5649. if (base_type(reg->type) == PTR_TO_DYNPTR)
  5650. break;
  5651. if (arg_type & MEM_UNINIT) {
  5652. if (!is_dynptr_reg_valid_uninit(env, reg)) {
  5653. verbose(env, "Dynptr has to be an uninitialized dynptr\n");
  5654. return -EINVAL;
  5655. }
  5656. /* We only support one dynptr being uninitialized at the moment,
  5657. * which is sufficient for the helper functions we have right now.
  5658. */
  5659. if (meta->uninit_dynptr_regno) {
  5660. verbose(env, "verifier internal error: multiple uninitialized dynptr args\n");
  5661. return -EFAULT;
  5662. }
  5663. meta->uninit_dynptr_regno = regno;
  5664. } else if (!is_dynptr_reg_valid_init(env, reg)) {
  5665. verbose(env,
  5666. "Expected an initialized dynptr as arg #%d\n",
  5667. arg + 1);
  5668. return -EINVAL;
  5669. } else if (!is_dynptr_type_expected(env, reg, arg_type)) {
  5670. const char *err_extra = "";
  5671. switch (arg_type & DYNPTR_TYPE_FLAG_MASK) {
  5672. case DYNPTR_TYPE_LOCAL:
  5673. err_extra = "local";
  5674. break;
  5675. case DYNPTR_TYPE_RINGBUF:
  5676. err_extra = "ringbuf";
  5677. break;
  5678. default:
  5679. err_extra = "<unknown>";
  5680. break;
  5681. }
  5682. verbose(env,
  5683. "Expected a dynptr of type %s as arg #%d\n",
  5684. err_extra, arg + 1);
  5685. return -EINVAL;
  5686. }
  5687. break;
  5688. case ARG_CONST_ALLOC_SIZE_OR_ZERO:
  5689. if (!tnum_is_const(reg->var_off)) {
  5690. verbose(env, "R%d is not a known constant'\n",
  5691. regno);
  5692. return -EACCES;
  5693. }
  5694. meta->mem_size = reg->var_off.value;
  5695. err = mark_chain_precision(env, regno);
  5696. if (err)
  5697. return err;
  5698. break;
  5699. case ARG_PTR_TO_INT:
  5700. case ARG_PTR_TO_LONG:
  5701. {
  5702. int size = int_ptr_type_to_size(arg_type);
  5703. err = check_helper_mem_access(env, regno, size, false, meta);
  5704. if (err)
  5705. return err;
  5706. err = check_ptr_alignment(env, reg, 0, size, true);
  5707. break;
  5708. }
  5709. case ARG_PTR_TO_CONST_STR:
  5710. {
  5711. struct bpf_map *map = reg->map_ptr;
  5712. int map_off;
  5713. u64 map_addr;
  5714. char *str_ptr;
  5715. if (!bpf_map_is_rdonly(map)) {
  5716. verbose(env, "R%d does not point to a readonly map'\n", regno);
  5717. return -EACCES;
  5718. }
  5719. if (!tnum_is_const(reg->var_off)) {
  5720. verbose(env, "R%d is not a constant address'\n", regno);
  5721. return -EACCES;
  5722. }
  5723. if (!map->ops->map_direct_value_addr) {
  5724. verbose(env, "no direct value access support for this map type\n");
  5725. return -EACCES;
  5726. }
  5727. err = check_map_access(env, regno, reg->off,
  5728. map->value_size - reg->off, false,
  5729. ACCESS_HELPER);
  5730. if (err)
  5731. return err;
  5732. map_off = reg->off + reg->var_off.value;
  5733. err = map->ops->map_direct_value_addr(map, &map_addr, map_off);
  5734. if (err) {
  5735. verbose(env, "direct value access on string failed\n");
  5736. return err;
  5737. }
  5738. str_ptr = (char *)(long)(map_addr);
  5739. if (!strnchr(str_ptr + map_off, map->value_size - map_off, 0)) {
  5740. verbose(env, "string is not zero-terminated\n");
  5741. return -EINVAL;
  5742. }
  5743. break;
  5744. }
  5745. case ARG_PTR_TO_KPTR:
  5746. if (process_kptr_func(env, regno, meta))
  5747. return -EACCES;
  5748. break;
  5749. }
  5750. return err;
  5751. }
  5752. static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
  5753. {
  5754. enum bpf_attach_type eatype = env->prog->expected_attach_type;
  5755. enum bpf_prog_type type = resolve_prog_type(env->prog);
  5756. if (func_id != BPF_FUNC_map_update_elem)
  5757. return false;
  5758. /* It's not possible to get access to a locked struct sock in these
  5759. * contexts, so updating is safe.
  5760. */
  5761. switch (type) {
  5762. case BPF_PROG_TYPE_TRACING:
  5763. if (eatype == BPF_TRACE_ITER)
  5764. return true;
  5765. break;
  5766. case BPF_PROG_TYPE_SOCKET_FILTER:
  5767. case BPF_PROG_TYPE_SCHED_CLS:
  5768. case BPF_PROG_TYPE_SCHED_ACT:
  5769. case BPF_PROG_TYPE_XDP:
  5770. case BPF_PROG_TYPE_SK_REUSEPORT:
  5771. case BPF_PROG_TYPE_FLOW_DISSECTOR:
  5772. case BPF_PROG_TYPE_SK_LOOKUP:
  5773. return true;
  5774. default:
  5775. break;
  5776. }
  5777. verbose(env, "cannot update sockmap in this context\n");
  5778. return false;
  5779. }
  5780. static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
  5781. {
  5782. return env->prog->jit_requested &&
  5783. bpf_jit_supports_subprog_tailcalls();
  5784. }
  5785. static int check_map_func_compatibility(struct bpf_verifier_env *env,
  5786. struct bpf_map *map, int func_id)
  5787. {
  5788. if (!map)
  5789. return 0;
  5790. /* We need a two way check, first is from map perspective ... */
  5791. switch (map->map_type) {
  5792. case BPF_MAP_TYPE_PROG_ARRAY:
  5793. if (func_id != BPF_FUNC_tail_call)
  5794. goto error;
  5795. break;
  5796. case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
  5797. if (func_id != BPF_FUNC_perf_event_read &&
  5798. func_id != BPF_FUNC_perf_event_output &&
  5799. func_id != BPF_FUNC_skb_output &&
  5800. func_id != BPF_FUNC_perf_event_read_value &&
  5801. func_id != BPF_FUNC_xdp_output)
  5802. goto error;
  5803. break;
  5804. case BPF_MAP_TYPE_RINGBUF:
  5805. if (func_id != BPF_FUNC_ringbuf_output &&
  5806. func_id != BPF_FUNC_ringbuf_reserve &&
  5807. func_id != BPF_FUNC_ringbuf_query &&
  5808. func_id != BPF_FUNC_ringbuf_reserve_dynptr &&
  5809. func_id != BPF_FUNC_ringbuf_submit_dynptr &&
  5810. func_id != BPF_FUNC_ringbuf_discard_dynptr)
  5811. goto error;
  5812. break;
  5813. case BPF_MAP_TYPE_USER_RINGBUF:
  5814. if (func_id != BPF_FUNC_user_ringbuf_drain)
  5815. goto error;
  5816. break;
  5817. case BPF_MAP_TYPE_STACK_TRACE:
  5818. if (func_id != BPF_FUNC_get_stackid)
  5819. goto error;
  5820. break;
  5821. case BPF_MAP_TYPE_CGROUP_ARRAY:
  5822. if (func_id != BPF_FUNC_skb_under_cgroup &&
  5823. func_id != BPF_FUNC_current_task_under_cgroup)
  5824. goto error;
  5825. break;
  5826. case BPF_MAP_TYPE_CGROUP_STORAGE:
  5827. case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
  5828. if (func_id != BPF_FUNC_get_local_storage)
  5829. goto error;
  5830. break;
  5831. case BPF_MAP_TYPE_DEVMAP:
  5832. case BPF_MAP_TYPE_DEVMAP_HASH:
  5833. if (func_id != BPF_FUNC_redirect_map &&
  5834. func_id != BPF_FUNC_map_lookup_elem)
  5835. goto error;
  5836. break;
  5837. /* Restrict bpf side of cpumap and xskmap, open when use-cases
  5838. * appear.
  5839. */
  5840. case BPF_MAP_TYPE_CPUMAP:
  5841. if (func_id != BPF_FUNC_redirect_map)
  5842. goto error;
  5843. break;
  5844. case BPF_MAP_TYPE_XSKMAP:
  5845. if (func_id != BPF_FUNC_redirect_map &&
  5846. func_id != BPF_FUNC_map_lookup_elem)
  5847. goto error;
  5848. break;
  5849. case BPF_MAP_TYPE_ARRAY_OF_MAPS:
  5850. case BPF_MAP_TYPE_HASH_OF_MAPS:
  5851. if (func_id != BPF_FUNC_map_lookup_elem)
  5852. goto error;
  5853. break;
  5854. case BPF_MAP_TYPE_SOCKMAP:
  5855. if (func_id != BPF_FUNC_sk_redirect_map &&
  5856. func_id != BPF_FUNC_sock_map_update &&
  5857. func_id != BPF_FUNC_map_delete_elem &&
  5858. func_id != BPF_FUNC_msg_redirect_map &&
  5859. func_id != BPF_FUNC_sk_select_reuseport &&
  5860. func_id != BPF_FUNC_map_lookup_elem &&
  5861. !may_update_sockmap(env, func_id))
  5862. goto error;
  5863. break;
  5864. case BPF_MAP_TYPE_SOCKHASH:
  5865. if (func_id != BPF_FUNC_sk_redirect_hash &&
  5866. func_id != BPF_FUNC_sock_hash_update &&
  5867. func_id != BPF_FUNC_map_delete_elem &&
  5868. func_id != BPF_FUNC_msg_redirect_hash &&
  5869. func_id != BPF_FUNC_sk_select_reuseport &&
  5870. func_id != BPF_FUNC_map_lookup_elem &&
  5871. !may_update_sockmap(env, func_id))
  5872. goto error;
  5873. break;
  5874. case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
  5875. if (func_id != BPF_FUNC_sk_select_reuseport)
  5876. goto error;
  5877. break;
  5878. case BPF_MAP_TYPE_QUEUE:
  5879. case BPF_MAP_TYPE_STACK:
  5880. if (func_id != BPF_FUNC_map_peek_elem &&
  5881. func_id != BPF_FUNC_map_pop_elem &&
  5882. func_id != BPF_FUNC_map_push_elem)
  5883. goto error;
  5884. break;
  5885. case BPF_MAP_TYPE_SK_STORAGE:
  5886. if (func_id != BPF_FUNC_sk_storage_get &&
  5887. func_id != BPF_FUNC_sk_storage_delete)
  5888. goto error;
  5889. break;
  5890. case BPF_MAP_TYPE_INODE_STORAGE:
  5891. if (func_id != BPF_FUNC_inode_storage_get &&
  5892. func_id != BPF_FUNC_inode_storage_delete)
  5893. goto error;
  5894. break;
  5895. case BPF_MAP_TYPE_TASK_STORAGE:
  5896. if (func_id != BPF_FUNC_task_storage_get &&
  5897. func_id != BPF_FUNC_task_storage_delete)
  5898. goto error;
  5899. break;
  5900. case BPF_MAP_TYPE_BLOOM_FILTER:
  5901. if (func_id != BPF_FUNC_map_peek_elem &&
  5902. func_id != BPF_FUNC_map_push_elem)
  5903. goto error;
  5904. break;
  5905. default:
  5906. break;
  5907. }
  5908. /* ... and second from the function itself. */
  5909. switch (func_id) {
  5910. case BPF_FUNC_tail_call:
  5911. if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
  5912. goto error;
  5913. if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
  5914. verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
  5915. return -EINVAL;
  5916. }
  5917. break;
  5918. case BPF_FUNC_perf_event_read:
  5919. case BPF_FUNC_perf_event_output:
  5920. case BPF_FUNC_perf_event_read_value:
  5921. case BPF_FUNC_skb_output:
  5922. case BPF_FUNC_xdp_output:
  5923. if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
  5924. goto error;
  5925. break;
  5926. case BPF_FUNC_ringbuf_output:
  5927. case BPF_FUNC_ringbuf_reserve:
  5928. case BPF_FUNC_ringbuf_query:
  5929. case BPF_FUNC_ringbuf_reserve_dynptr:
  5930. case BPF_FUNC_ringbuf_submit_dynptr:
  5931. case BPF_FUNC_ringbuf_discard_dynptr:
  5932. if (map->map_type != BPF_MAP_TYPE_RINGBUF)
  5933. goto error;
  5934. break;
  5935. case BPF_FUNC_user_ringbuf_drain:
  5936. if (map->map_type != BPF_MAP_TYPE_USER_RINGBUF)
  5937. goto error;
  5938. break;
  5939. case BPF_FUNC_get_stackid:
  5940. if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
  5941. goto error;
  5942. break;
  5943. case BPF_FUNC_current_task_under_cgroup:
  5944. case BPF_FUNC_skb_under_cgroup:
  5945. if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
  5946. goto error;
  5947. break;
  5948. case BPF_FUNC_redirect_map:
  5949. if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
  5950. map->map_type != BPF_MAP_TYPE_DEVMAP_HASH &&
  5951. map->map_type != BPF_MAP_TYPE_CPUMAP &&
  5952. map->map_type != BPF_MAP_TYPE_XSKMAP)
  5953. goto error;
  5954. break;
  5955. case BPF_FUNC_sk_redirect_map:
  5956. case BPF_FUNC_msg_redirect_map:
  5957. case BPF_FUNC_sock_map_update:
  5958. if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
  5959. goto error;
  5960. break;
  5961. case BPF_FUNC_sk_redirect_hash:
  5962. case BPF_FUNC_msg_redirect_hash:
  5963. case BPF_FUNC_sock_hash_update:
  5964. if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
  5965. goto error;
  5966. break;
  5967. case BPF_FUNC_get_local_storage:
  5968. if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
  5969. map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
  5970. goto error;
  5971. break;
  5972. case BPF_FUNC_sk_select_reuseport:
  5973. if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY &&
  5974. map->map_type != BPF_MAP_TYPE_SOCKMAP &&
  5975. map->map_type != BPF_MAP_TYPE_SOCKHASH)
  5976. goto error;
  5977. break;
  5978. case BPF_FUNC_map_pop_elem:
  5979. if (map->map_type != BPF_MAP_TYPE_QUEUE &&
  5980. map->map_type != BPF_MAP_TYPE_STACK)
  5981. goto error;
  5982. break;
  5983. case BPF_FUNC_map_peek_elem:
  5984. case BPF_FUNC_map_push_elem:
  5985. if (map->map_type != BPF_MAP_TYPE_QUEUE &&
  5986. map->map_type != BPF_MAP_TYPE_STACK &&
  5987. map->map_type != BPF_MAP_TYPE_BLOOM_FILTER)
  5988. goto error;
  5989. break;
  5990. case BPF_FUNC_map_lookup_percpu_elem:
  5991. if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY &&
  5992. map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
  5993. map->map_type != BPF_MAP_TYPE_LRU_PERCPU_HASH)
  5994. goto error;
  5995. break;
  5996. case BPF_FUNC_sk_storage_get:
  5997. case BPF_FUNC_sk_storage_delete:
  5998. if (map->map_type != BPF_MAP_TYPE_SK_STORAGE)
  5999. goto error;
  6000. break;
  6001. case BPF_FUNC_inode_storage_get:
  6002. case BPF_FUNC_inode_storage_delete:
  6003. if (map->map_type != BPF_MAP_TYPE_INODE_STORAGE)
  6004. goto error;
  6005. break;
  6006. case BPF_FUNC_task_storage_get:
  6007. case BPF_FUNC_task_storage_delete:
  6008. if (map->map_type != BPF_MAP_TYPE_TASK_STORAGE)
  6009. goto error;
  6010. break;
  6011. default:
  6012. break;
  6013. }
  6014. return 0;
  6015. error:
  6016. verbose(env, "cannot pass map_type %d into func %s#%d\n",
  6017. map->map_type, func_id_name(func_id), func_id);
  6018. return -EINVAL;
  6019. }
  6020. static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
  6021. {
  6022. int count = 0;
  6023. if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
  6024. count++;
  6025. if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
  6026. count++;
  6027. if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
  6028. count++;
  6029. if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
  6030. count++;
  6031. if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
  6032. count++;
  6033. /* We only support one arg being in raw mode at the moment,
  6034. * which is sufficient for the helper functions we have
  6035. * right now.
  6036. */
  6037. return count <= 1;
  6038. }
  6039. static bool check_args_pair_invalid(const struct bpf_func_proto *fn, int arg)
  6040. {
  6041. bool is_fixed = fn->arg_type[arg] & MEM_FIXED_SIZE;
  6042. bool has_size = fn->arg_size[arg] != 0;
  6043. bool is_next_size = false;
  6044. if (arg + 1 < ARRAY_SIZE(fn->arg_type))
  6045. is_next_size = arg_type_is_mem_size(fn->arg_type[arg + 1]);
  6046. if (base_type(fn->arg_type[arg]) != ARG_PTR_TO_MEM)
  6047. return is_next_size;
  6048. return has_size == is_next_size || is_next_size == is_fixed;
  6049. }
  6050. static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
  6051. {
  6052. /* bpf_xxx(..., buf, len) call will access 'len'
  6053. * bytes from memory 'buf'. Both arg types need
  6054. * to be paired, so make sure there's no buggy
  6055. * helper function specification.
  6056. */
  6057. if (arg_type_is_mem_size(fn->arg1_type) ||
  6058. check_args_pair_invalid(fn, 0) ||
  6059. check_args_pair_invalid(fn, 1) ||
  6060. check_args_pair_invalid(fn, 2) ||
  6061. check_args_pair_invalid(fn, 3) ||
  6062. check_args_pair_invalid(fn, 4))
  6063. return false;
  6064. return true;
  6065. }
  6066. static bool check_btf_id_ok(const struct bpf_func_proto *fn)
  6067. {
  6068. int i;
  6069. for (i = 0; i < ARRAY_SIZE(fn->arg_type); i++) {
  6070. if (base_type(fn->arg_type[i]) == ARG_PTR_TO_BTF_ID && !fn->arg_btf_id[i])
  6071. return false;
  6072. if (base_type(fn->arg_type[i]) != ARG_PTR_TO_BTF_ID && fn->arg_btf_id[i] &&
  6073. /* arg_btf_id and arg_size are in a union. */
  6074. (base_type(fn->arg_type[i]) != ARG_PTR_TO_MEM ||
  6075. !(fn->arg_type[i] & MEM_FIXED_SIZE)))
  6076. return false;
  6077. }
  6078. return true;
  6079. }
  6080. static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
  6081. {
  6082. return check_raw_mode_ok(fn) &&
  6083. check_arg_pair_ok(fn) &&
  6084. check_btf_id_ok(fn) ? 0 : -EINVAL;
  6085. }
  6086. /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
  6087. * are now invalid, so turn them into unknown SCALAR_VALUE.
  6088. */
  6089. static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
  6090. {
  6091. struct bpf_func_state *state;
  6092. struct bpf_reg_state *reg;
  6093. bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
  6094. if (reg_is_pkt_pointer_any(reg))
  6095. __mark_reg_unknown(env, reg);
  6096. }));
  6097. }
  6098. enum {
  6099. AT_PKT_END = -1,
  6100. BEYOND_PKT_END = -2,
  6101. };
  6102. static void mark_pkt_end(struct bpf_verifier_state *vstate, int regn, bool range_open)
  6103. {
  6104. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  6105. struct bpf_reg_state *reg = &state->regs[regn];
  6106. if (reg->type != PTR_TO_PACKET)
  6107. /* PTR_TO_PACKET_META is not supported yet */
  6108. return;
  6109. /* The 'reg' is pkt > pkt_end or pkt >= pkt_end.
  6110. * How far beyond pkt_end it goes is unknown.
  6111. * if (!range_open) it's the case of pkt >= pkt_end
  6112. * if (range_open) it's the case of pkt > pkt_end
  6113. * hence this pointer is at least 1 byte bigger than pkt_end
  6114. */
  6115. if (range_open)
  6116. reg->range = BEYOND_PKT_END;
  6117. else
  6118. reg->range = AT_PKT_END;
  6119. }
  6120. /* The pointer with the specified id has released its reference to kernel
  6121. * resources. Identify all copies of the same pointer and clear the reference.
  6122. */
  6123. static int release_reference(struct bpf_verifier_env *env,
  6124. int ref_obj_id)
  6125. {
  6126. struct bpf_func_state *state;
  6127. struct bpf_reg_state *reg;
  6128. int err;
  6129. err = release_reference_state(cur_func(env), ref_obj_id);
  6130. if (err)
  6131. return err;
  6132. bpf_for_each_reg_in_vstate(env->cur_state, state, reg, ({
  6133. if (reg->ref_obj_id == ref_obj_id) {
  6134. if (!env->allow_ptr_leaks)
  6135. __mark_reg_not_init(env, reg);
  6136. else
  6137. __mark_reg_unknown(env, reg);
  6138. }
  6139. }));
  6140. return 0;
  6141. }
  6142. static void clear_caller_saved_regs(struct bpf_verifier_env *env,
  6143. struct bpf_reg_state *regs)
  6144. {
  6145. int i;
  6146. /* after the call registers r0 - r5 were scratched */
  6147. for (i = 0; i < CALLER_SAVED_REGS; i++) {
  6148. mark_reg_not_init(env, regs, caller_saved[i]);
  6149. check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
  6150. }
  6151. }
  6152. typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
  6153. struct bpf_func_state *caller,
  6154. struct bpf_func_state *callee,
  6155. int insn_idx);
  6156. static int set_callee_state(struct bpf_verifier_env *env,
  6157. struct bpf_func_state *caller,
  6158. struct bpf_func_state *callee, int insn_idx);
  6159. static int __check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
  6160. int *insn_idx, int subprog,
  6161. set_callee_state_fn set_callee_state_cb)
  6162. {
  6163. struct bpf_verifier_state *state = env->cur_state;
  6164. struct bpf_func_info_aux *func_info_aux;
  6165. struct bpf_func_state *caller, *callee;
  6166. int err;
  6167. bool is_global = false;
  6168. if (state->curframe + 1 >= MAX_CALL_FRAMES) {
  6169. verbose(env, "the call stack of %d frames is too deep\n",
  6170. state->curframe + 2);
  6171. return -E2BIG;
  6172. }
  6173. caller = state->frame[state->curframe];
  6174. if (state->frame[state->curframe + 1]) {
  6175. verbose(env, "verifier bug. Frame %d already allocated\n",
  6176. state->curframe + 1);
  6177. return -EFAULT;
  6178. }
  6179. func_info_aux = env->prog->aux->func_info_aux;
  6180. if (func_info_aux)
  6181. is_global = func_info_aux[subprog].linkage == BTF_FUNC_GLOBAL;
  6182. err = btf_check_subprog_call(env, subprog, caller->regs);
  6183. if (err == -EFAULT)
  6184. return err;
  6185. if (is_global) {
  6186. if (err) {
  6187. verbose(env, "Caller passes invalid args into func#%d\n",
  6188. subprog);
  6189. return err;
  6190. } else {
  6191. if (env->log.level & BPF_LOG_LEVEL)
  6192. verbose(env,
  6193. "Func#%d is global and valid. Skipping.\n",
  6194. subprog);
  6195. clear_caller_saved_regs(env, caller->regs);
  6196. /* All global functions return a 64-bit SCALAR_VALUE */
  6197. mark_reg_unknown(env, caller->regs, BPF_REG_0);
  6198. caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
  6199. /* continue with next insn after call */
  6200. return 0;
  6201. }
  6202. }
  6203. /* set_callee_state is used for direct subprog calls, but we are
  6204. * interested in validating only BPF helpers that can call subprogs as
  6205. * callbacks
  6206. */
  6207. if (set_callee_state_cb != set_callee_state && !is_callback_calling_function(insn->imm)) {
  6208. verbose(env, "verifier bug: helper %s#%d is not marked as callback-calling\n",
  6209. func_id_name(insn->imm), insn->imm);
  6210. return -EFAULT;
  6211. }
  6212. if (insn->code == (BPF_JMP | BPF_CALL) &&
  6213. insn->src_reg == 0 &&
  6214. insn->imm == BPF_FUNC_timer_set_callback) {
  6215. struct bpf_verifier_state *async_cb;
  6216. /* there is no real recursion here. timer callbacks are async */
  6217. env->subprog_info[subprog].is_async_cb = true;
  6218. async_cb = push_async_cb(env, env->subprog_info[subprog].start,
  6219. *insn_idx, subprog);
  6220. if (!async_cb)
  6221. return -EFAULT;
  6222. callee = async_cb->frame[0];
  6223. callee->async_entry_cnt = caller->async_entry_cnt + 1;
  6224. /* Convert bpf_timer_set_callback() args into timer callback args */
  6225. err = set_callee_state_cb(env, caller, callee, *insn_idx);
  6226. if (err)
  6227. return err;
  6228. clear_caller_saved_regs(env, caller->regs);
  6229. mark_reg_unknown(env, caller->regs, BPF_REG_0);
  6230. caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
  6231. /* continue with next insn after call */
  6232. return 0;
  6233. }
  6234. callee = kzalloc(sizeof(*callee), GFP_KERNEL);
  6235. if (!callee)
  6236. return -ENOMEM;
  6237. state->frame[state->curframe + 1] = callee;
  6238. /* callee cannot access r0, r6 - r9 for reading and has to write
  6239. * into its own stack before reading from it.
  6240. * callee can read/write into caller's stack
  6241. */
  6242. init_func_state(env, callee,
  6243. /* remember the callsite, it will be used by bpf_exit */
  6244. *insn_idx /* callsite */,
  6245. state->curframe + 1 /* frameno within this callchain */,
  6246. subprog /* subprog number within this prog */);
  6247. /* Transfer references to the callee */
  6248. err = copy_reference_state(callee, caller);
  6249. if (err)
  6250. goto err_out;
  6251. err = set_callee_state_cb(env, caller, callee, *insn_idx);
  6252. if (err)
  6253. goto err_out;
  6254. clear_caller_saved_regs(env, caller->regs);
  6255. /* only increment it after check_reg_arg() finished */
  6256. state->curframe++;
  6257. /* and go analyze first insn of the callee */
  6258. *insn_idx = env->subprog_info[subprog].start - 1;
  6259. if (env->log.level & BPF_LOG_LEVEL) {
  6260. verbose(env, "caller:\n");
  6261. print_verifier_state(env, caller, true);
  6262. verbose(env, "callee:\n");
  6263. print_verifier_state(env, callee, true);
  6264. }
  6265. return 0;
  6266. err_out:
  6267. free_func_state(callee);
  6268. state->frame[state->curframe + 1] = NULL;
  6269. return err;
  6270. }
  6271. int map_set_for_each_callback_args(struct bpf_verifier_env *env,
  6272. struct bpf_func_state *caller,
  6273. struct bpf_func_state *callee)
  6274. {
  6275. /* bpf_for_each_map_elem(struct bpf_map *map, void *callback_fn,
  6276. * void *callback_ctx, u64 flags);
  6277. * callback_fn(struct bpf_map *map, void *key, void *value,
  6278. * void *callback_ctx);
  6279. */
  6280. callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
  6281. callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
  6282. __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
  6283. callee->regs[BPF_REG_2].map_ptr = caller->regs[BPF_REG_1].map_ptr;
  6284. callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
  6285. __mark_reg_known_zero(&callee->regs[BPF_REG_3]);
  6286. callee->regs[BPF_REG_3].map_ptr = caller->regs[BPF_REG_1].map_ptr;
  6287. /* pointer to stack or null */
  6288. callee->regs[BPF_REG_4] = caller->regs[BPF_REG_3];
  6289. /* unused */
  6290. __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
  6291. return 0;
  6292. }
  6293. static int set_callee_state(struct bpf_verifier_env *env,
  6294. struct bpf_func_state *caller,
  6295. struct bpf_func_state *callee, int insn_idx)
  6296. {
  6297. int i;
  6298. /* copy r1 - r5 args that callee can access. The copy includes parent
  6299. * pointers, which connects us up to the liveness chain
  6300. */
  6301. for (i = BPF_REG_1; i <= BPF_REG_5; i++)
  6302. callee->regs[i] = caller->regs[i];
  6303. return 0;
  6304. }
  6305. static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
  6306. int *insn_idx)
  6307. {
  6308. int subprog, target_insn;
  6309. target_insn = *insn_idx + insn->imm + 1;
  6310. subprog = find_subprog(env, target_insn);
  6311. if (subprog < 0) {
  6312. verbose(env, "verifier bug. No program starts at insn %d\n",
  6313. target_insn);
  6314. return -EFAULT;
  6315. }
  6316. return __check_func_call(env, insn, insn_idx, subprog, set_callee_state);
  6317. }
  6318. static int set_map_elem_callback_state(struct bpf_verifier_env *env,
  6319. struct bpf_func_state *caller,
  6320. struct bpf_func_state *callee,
  6321. int insn_idx)
  6322. {
  6323. struct bpf_insn_aux_data *insn_aux = &env->insn_aux_data[insn_idx];
  6324. struct bpf_map *map;
  6325. int err;
  6326. if (bpf_map_ptr_poisoned(insn_aux)) {
  6327. verbose(env, "tail_call abusing map_ptr\n");
  6328. return -EINVAL;
  6329. }
  6330. map = BPF_MAP_PTR(insn_aux->map_ptr_state);
  6331. if (!map->ops->map_set_for_each_callback_args ||
  6332. !map->ops->map_for_each_callback) {
  6333. verbose(env, "callback function not allowed for map\n");
  6334. return -ENOTSUPP;
  6335. }
  6336. err = map->ops->map_set_for_each_callback_args(env, caller, callee);
  6337. if (err)
  6338. return err;
  6339. callee->in_callback_fn = true;
  6340. callee->callback_ret_range = tnum_range(0, 1);
  6341. return 0;
  6342. }
  6343. static int set_loop_callback_state(struct bpf_verifier_env *env,
  6344. struct bpf_func_state *caller,
  6345. struct bpf_func_state *callee,
  6346. int insn_idx)
  6347. {
  6348. /* bpf_loop(u32 nr_loops, void *callback_fn, void *callback_ctx,
  6349. * u64 flags);
  6350. * callback_fn(u32 index, void *callback_ctx);
  6351. */
  6352. callee->regs[BPF_REG_1].type = SCALAR_VALUE;
  6353. callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
  6354. /* unused */
  6355. __mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
  6356. __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
  6357. __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
  6358. callee->in_callback_fn = true;
  6359. callee->callback_ret_range = tnum_range(0, 1);
  6360. return 0;
  6361. }
  6362. static int set_timer_callback_state(struct bpf_verifier_env *env,
  6363. struct bpf_func_state *caller,
  6364. struct bpf_func_state *callee,
  6365. int insn_idx)
  6366. {
  6367. struct bpf_map *map_ptr = caller->regs[BPF_REG_1].map_ptr;
  6368. /* bpf_timer_set_callback(struct bpf_timer *timer, void *callback_fn);
  6369. * callback_fn(struct bpf_map *map, void *key, void *value);
  6370. */
  6371. callee->regs[BPF_REG_1].type = CONST_PTR_TO_MAP;
  6372. __mark_reg_known_zero(&callee->regs[BPF_REG_1]);
  6373. callee->regs[BPF_REG_1].map_ptr = map_ptr;
  6374. callee->regs[BPF_REG_2].type = PTR_TO_MAP_KEY;
  6375. __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
  6376. callee->regs[BPF_REG_2].map_ptr = map_ptr;
  6377. callee->regs[BPF_REG_3].type = PTR_TO_MAP_VALUE;
  6378. __mark_reg_known_zero(&callee->regs[BPF_REG_3]);
  6379. callee->regs[BPF_REG_3].map_ptr = map_ptr;
  6380. /* unused */
  6381. __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
  6382. __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
  6383. callee->in_async_callback_fn = true;
  6384. callee->callback_ret_range = tnum_range(0, 1);
  6385. return 0;
  6386. }
  6387. static int set_find_vma_callback_state(struct bpf_verifier_env *env,
  6388. struct bpf_func_state *caller,
  6389. struct bpf_func_state *callee,
  6390. int insn_idx)
  6391. {
  6392. /* bpf_find_vma(struct task_struct *task, u64 addr,
  6393. * void *callback_fn, void *callback_ctx, u64 flags)
  6394. * (callback_fn)(struct task_struct *task,
  6395. * struct vm_area_struct *vma, void *callback_ctx);
  6396. */
  6397. callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1];
  6398. callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID;
  6399. __mark_reg_known_zero(&callee->regs[BPF_REG_2]);
  6400. callee->regs[BPF_REG_2].btf = btf_vmlinux;
  6401. callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA],
  6402. /* pointer to stack or null */
  6403. callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4];
  6404. /* unused */
  6405. __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
  6406. __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
  6407. callee->in_callback_fn = true;
  6408. callee->callback_ret_range = tnum_range(0, 1);
  6409. return 0;
  6410. }
  6411. static int set_user_ringbuf_callback_state(struct bpf_verifier_env *env,
  6412. struct bpf_func_state *caller,
  6413. struct bpf_func_state *callee,
  6414. int insn_idx)
  6415. {
  6416. /* bpf_user_ringbuf_drain(struct bpf_map *map, void *callback_fn, void
  6417. * callback_ctx, u64 flags);
  6418. * callback_fn(struct bpf_dynptr_t* dynptr, void *callback_ctx);
  6419. */
  6420. __mark_reg_not_init(env, &callee->regs[BPF_REG_0]);
  6421. callee->regs[BPF_REG_1].type = PTR_TO_DYNPTR | DYNPTR_TYPE_LOCAL;
  6422. __mark_reg_known_zero(&callee->regs[BPF_REG_1]);
  6423. callee->regs[BPF_REG_2] = caller->regs[BPF_REG_3];
  6424. /* unused */
  6425. __mark_reg_not_init(env, &callee->regs[BPF_REG_3]);
  6426. __mark_reg_not_init(env, &callee->regs[BPF_REG_4]);
  6427. __mark_reg_not_init(env, &callee->regs[BPF_REG_5]);
  6428. callee->in_callback_fn = true;
  6429. callee->callback_ret_range = tnum_range(0, 1);
  6430. return 0;
  6431. }
  6432. static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
  6433. {
  6434. struct bpf_verifier_state *state = env->cur_state;
  6435. struct bpf_func_state *caller, *callee;
  6436. struct bpf_reg_state *r0;
  6437. int err;
  6438. callee = state->frame[state->curframe];
  6439. r0 = &callee->regs[BPF_REG_0];
  6440. if (r0->type == PTR_TO_STACK) {
  6441. /* technically it's ok to return caller's stack pointer
  6442. * (or caller's caller's pointer) back to the caller,
  6443. * since these pointers are valid. Only current stack
  6444. * pointer will be invalid as soon as function exits,
  6445. * but let's be conservative
  6446. */
  6447. verbose(env, "cannot return stack pointer to the caller\n");
  6448. return -EINVAL;
  6449. }
  6450. caller = state->frame[state->curframe - 1];
  6451. if (callee->in_callback_fn) {
  6452. /* enforce R0 return value range [0, 1]. */
  6453. struct tnum range = callee->callback_ret_range;
  6454. if (r0->type != SCALAR_VALUE) {
  6455. verbose(env, "R0 not a scalar value\n");
  6456. return -EACCES;
  6457. }
  6458. if (!tnum_in(range, r0->var_off)) {
  6459. verbose_invalid_scalar(env, r0, &range, "callback return", "R0");
  6460. return -EINVAL;
  6461. }
  6462. } else {
  6463. /* return to the caller whatever r0 had in the callee */
  6464. caller->regs[BPF_REG_0] = *r0;
  6465. }
  6466. /* callback_fn frame should have released its own additions to parent's
  6467. * reference state at this point, or check_reference_leak would
  6468. * complain, hence it must be the same as the caller. There is no need
  6469. * to copy it back.
  6470. */
  6471. if (!callee->in_callback_fn) {
  6472. /* Transfer references to the caller */
  6473. err = copy_reference_state(caller, callee);
  6474. if (err)
  6475. return err;
  6476. }
  6477. *insn_idx = callee->callsite + 1;
  6478. if (env->log.level & BPF_LOG_LEVEL) {
  6479. verbose(env, "returning from callee:\n");
  6480. print_verifier_state(env, callee, true);
  6481. verbose(env, "to caller at %d:\n", *insn_idx);
  6482. print_verifier_state(env, caller, true);
  6483. }
  6484. /* clear everything in the callee */
  6485. free_func_state(callee);
  6486. state->frame[state->curframe--] = NULL;
  6487. return 0;
  6488. }
  6489. static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
  6490. int func_id,
  6491. struct bpf_call_arg_meta *meta)
  6492. {
  6493. struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];
  6494. if (ret_type != RET_INTEGER ||
  6495. (func_id != BPF_FUNC_get_stack &&
  6496. func_id != BPF_FUNC_get_task_stack &&
  6497. func_id != BPF_FUNC_probe_read_str &&
  6498. func_id != BPF_FUNC_probe_read_kernel_str &&
  6499. func_id != BPF_FUNC_probe_read_user_str))
  6500. return;
  6501. ret_reg->smax_value = meta->msize_max_value;
  6502. ret_reg->s32_max_value = meta->msize_max_value;
  6503. ret_reg->smin_value = -MAX_ERRNO;
  6504. ret_reg->s32_min_value = -MAX_ERRNO;
  6505. reg_bounds_sync(ret_reg);
  6506. }
  6507. static int
  6508. record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
  6509. int func_id, int insn_idx)
  6510. {
  6511. struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
  6512. struct bpf_map *map = meta->map_ptr;
  6513. if (func_id != BPF_FUNC_tail_call &&
  6514. func_id != BPF_FUNC_map_lookup_elem &&
  6515. func_id != BPF_FUNC_map_update_elem &&
  6516. func_id != BPF_FUNC_map_delete_elem &&
  6517. func_id != BPF_FUNC_map_push_elem &&
  6518. func_id != BPF_FUNC_map_pop_elem &&
  6519. func_id != BPF_FUNC_map_peek_elem &&
  6520. func_id != BPF_FUNC_for_each_map_elem &&
  6521. func_id != BPF_FUNC_redirect_map &&
  6522. func_id != BPF_FUNC_map_lookup_percpu_elem)
  6523. return 0;
  6524. if (map == NULL) {
  6525. verbose(env, "kernel subsystem misconfigured verifier\n");
  6526. return -EINVAL;
  6527. }
  6528. /* In case of read-only, some additional restrictions
  6529. * need to be applied in order to prevent altering the
  6530. * state of the map from program side.
  6531. */
  6532. if ((map->map_flags & BPF_F_RDONLY_PROG) &&
  6533. (func_id == BPF_FUNC_map_delete_elem ||
  6534. func_id == BPF_FUNC_map_update_elem ||
  6535. func_id == BPF_FUNC_map_push_elem ||
  6536. func_id == BPF_FUNC_map_pop_elem)) {
  6537. verbose(env, "write into map forbidden\n");
  6538. return -EACCES;
  6539. }
  6540. if (!BPF_MAP_PTR(aux->map_ptr_state))
  6541. bpf_map_ptr_store(aux, meta->map_ptr,
  6542. !meta->map_ptr->bypass_spec_v1);
  6543. else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr)
  6544. bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
  6545. !meta->map_ptr->bypass_spec_v1);
  6546. return 0;
  6547. }
  6548. static int
  6549. record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
  6550. int func_id, int insn_idx)
  6551. {
  6552. struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
  6553. struct bpf_reg_state *regs = cur_regs(env), *reg;
  6554. struct bpf_map *map = meta->map_ptr;
  6555. u64 val, max;
  6556. int err;
  6557. if (func_id != BPF_FUNC_tail_call)
  6558. return 0;
  6559. if (!map || map->map_type != BPF_MAP_TYPE_PROG_ARRAY) {
  6560. verbose(env, "kernel subsystem misconfigured verifier\n");
  6561. return -EINVAL;
  6562. }
  6563. reg = &regs[BPF_REG_3];
  6564. val = reg->var_off.value;
  6565. max = map->max_entries;
  6566. if (!(register_is_const(reg) && val < max)) {
  6567. bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
  6568. return 0;
  6569. }
  6570. err = mark_chain_precision(env, BPF_REG_3);
  6571. if (err)
  6572. return err;
  6573. if (bpf_map_key_unseen(aux))
  6574. bpf_map_key_store(aux, val);
  6575. else if (!bpf_map_key_poisoned(aux) &&
  6576. bpf_map_key_immediate(aux) != val)
  6577. bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
  6578. return 0;
  6579. }
  6580. static int check_reference_leak(struct bpf_verifier_env *env)
  6581. {
  6582. struct bpf_func_state *state = cur_func(env);
  6583. bool refs_lingering = false;
  6584. int i;
  6585. if (state->frameno && !state->in_callback_fn)
  6586. return 0;
  6587. for (i = 0; i < state->acquired_refs; i++) {
  6588. if (state->in_callback_fn && state->refs[i].callback_ref != state->frameno)
  6589. continue;
  6590. verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
  6591. state->refs[i].id, state->refs[i].insn_idx);
  6592. refs_lingering = true;
  6593. }
  6594. return refs_lingering ? -EINVAL : 0;
  6595. }
  6596. static int check_bpf_snprintf_call(struct bpf_verifier_env *env,
  6597. struct bpf_reg_state *regs)
  6598. {
  6599. struct bpf_reg_state *fmt_reg = &regs[BPF_REG_3];
  6600. struct bpf_reg_state *data_len_reg = &regs[BPF_REG_5];
  6601. struct bpf_map *fmt_map = fmt_reg->map_ptr;
  6602. int err, fmt_map_off, num_args;
  6603. u64 fmt_addr;
  6604. char *fmt;
  6605. /* data must be an array of u64 */
  6606. if (data_len_reg->var_off.value % 8)
  6607. return -EINVAL;
  6608. num_args = data_len_reg->var_off.value / 8;
  6609. /* fmt being ARG_PTR_TO_CONST_STR guarantees that var_off is const
  6610. * and map_direct_value_addr is set.
  6611. */
  6612. fmt_map_off = fmt_reg->off + fmt_reg->var_off.value;
  6613. err = fmt_map->ops->map_direct_value_addr(fmt_map, &fmt_addr,
  6614. fmt_map_off);
  6615. if (err) {
  6616. verbose(env, "verifier bug\n");
  6617. return -EFAULT;
  6618. }
  6619. fmt = (char *)(long)fmt_addr + fmt_map_off;
  6620. /* We are also guaranteed that fmt+fmt_map_off is NULL terminated, we
  6621. * can focus on validating the format specifiers.
  6622. */
  6623. err = bpf_bprintf_prepare(fmt, UINT_MAX, NULL, NULL, num_args);
  6624. if (err < 0)
  6625. verbose(env, "Invalid format string\n");
  6626. return err;
  6627. }
  6628. static int check_get_func_ip(struct bpf_verifier_env *env)
  6629. {
  6630. enum bpf_prog_type type = resolve_prog_type(env->prog);
  6631. int func_id = BPF_FUNC_get_func_ip;
  6632. if (type == BPF_PROG_TYPE_TRACING) {
  6633. if (!bpf_prog_has_trampoline(env->prog)) {
  6634. verbose(env, "func %s#%d supported only for fentry/fexit/fmod_ret programs\n",
  6635. func_id_name(func_id), func_id);
  6636. return -ENOTSUPP;
  6637. }
  6638. return 0;
  6639. } else if (type == BPF_PROG_TYPE_KPROBE) {
  6640. return 0;
  6641. }
  6642. verbose(env, "func %s#%d not supported for program type %d\n",
  6643. func_id_name(func_id), func_id, type);
  6644. return -ENOTSUPP;
  6645. }
  6646. static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
  6647. {
  6648. return &env->insn_aux_data[env->insn_idx];
  6649. }
  6650. static bool loop_flag_is_zero(struct bpf_verifier_env *env)
  6651. {
  6652. struct bpf_reg_state *regs = cur_regs(env);
  6653. struct bpf_reg_state *reg = &regs[BPF_REG_4];
  6654. bool reg_is_null = register_is_null(reg);
  6655. if (reg_is_null)
  6656. mark_chain_precision(env, BPF_REG_4);
  6657. return reg_is_null;
  6658. }
  6659. static void update_loop_inline_state(struct bpf_verifier_env *env, u32 subprogno)
  6660. {
  6661. struct bpf_loop_inline_state *state = &cur_aux(env)->loop_inline_state;
  6662. if (!state->initialized) {
  6663. state->initialized = 1;
  6664. state->fit_for_inline = loop_flag_is_zero(env);
  6665. state->callback_subprogno = subprogno;
  6666. return;
  6667. }
  6668. if (!state->fit_for_inline)
  6669. return;
  6670. state->fit_for_inline = (loop_flag_is_zero(env) &&
  6671. state->callback_subprogno == subprogno);
  6672. }
  6673. static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
  6674. int *insn_idx_p)
  6675. {
  6676. enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
  6677. const struct bpf_func_proto *fn = NULL;
  6678. enum bpf_return_type ret_type;
  6679. enum bpf_type_flag ret_flag;
  6680. struct bpf_reg_state *regs;
  6681. struct bpf_call_arg_meta meta;
  6682. int insn_idx = *insn_idx_p;
  6683. bool changes_data;
  6684. int i, err, func_id;
  6685. /* find function prototype */
  6686. func_id = insn->imm;
  6687. if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
  6688. verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
  6689. func_id);
  6690. return -EINVAL;
  6691. }
  6692. if (env->ops->get_func_proto)
  6693. fn = env->ops->get_func_proto(func_id, env->prog);
  6694. if (!fn) {
  6695. verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
  6696. func_id);
  6697. return -EINVAL;
  6698. }
  6699. /* eBPF programs must be GPL compatible to use GPL-ed functions */
  6700. if (!env->prog->gpl_compatible && fn->gpl_only) {
  6701. verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
  6702. return -EINVAL;
  6703. }
  6704. if (fn->allowed && !fn->allowed(env->prog)) {
  6705. verbose(env, "helper call is not allowed in probe\n");
  6706. return -EINVAL;
  6707. }
  6708. /* With LD_ABS/IND some JITs save/restore skb from r1. */
  6709. changes_data = bpf_helper_changes_pkt_data(fn->func);
  6710. if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
  6711. verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
  6712. func_id_name(func_id), func_id);
  6713. return -EINVAL;
  6714. }
  6715. memset(&meta, 0, sizeof(meta));
  6716. meta.pkt_access = fn->pkt_access;
  6717. err = check_func_proto(fn, func_id);
  6718. if (err) {
  6719. verbose(env, "kernel subsystem misconfigured func %s#%d\n",
  6720. func_id_name(func_id), func_id);
  6721. return err;
  6722. }
  6723. meta.func_id = func_id;
  6724. /* check args */
  6725. for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
  6726. err = check_func_arg(env, i, &meta, fn);
  6727. if (err)
  6728. return err;
  6729. }
  6730. err = record_func_map(env, &meta, func_id, insn_idx);
  6731. if (err)
  6732. return err;
  6733. err = record_func_key(env, &meta, func_id, insn_idx);
  6734. if (err)
  6735. return err;
  6736. /* Mark slots with STACK_MISC in case of raw mode, stack offset
  6737. * is inferred from register state.
  6738. */
  6739. for (i = 0; i < meta.access_size; i++) {
  6740. err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
  6741. BPF_WRITE, -1, false);
  6742. if (err)
  6743. return err;
  6744. }
  6745. regs = cur_regs(env);
  6746. if (meta.uninit_dynptr_regno) {
  6747. /* we write BPF_DW bits (8 bytes) at a time */
  6748. for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
  6749. err = check_mem_access(env, insn_idx, meta.uninit_dynptr_regno,
  6750. i, BPF_DW, BPF_WRITE, -1, false);
  6751. if (err)
  6752. return err;
  6753. }
  6754. err = mark_stack_slots_dynptr(env, &regs[meta.uninit_dynptr_regno],
  6755. fn->arg_type[meta.uninit_dynptr_regno - BPF_REG_1],
  6756. insn_idx);
  6757. if (err)
  6758. return err;
  6759. }
  6760. if (meta.release_regno) {
  6761. err = -EINVAL;
  6762. if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1]))
  6763. err = unmark_stack_slots_dynptr(env, &regs[meta.release_regno]);
  6764. else if (meta.ref_obj_id)
  6765. err = release_reference(env, meta.ref_obj_id);
  6766. /* meta.ref_obj_id can only be 0 if register that is meant to be
  6767. * released is NULL, which must be > R0.
  6768. */
  6769. else if (register_is_null(&regs[meta.release_regno]))
  6770. err = 0;
  6771. if (err) {
  6772. verbose(env, "func %s#%d reference has not been acquired before\n",
  6773. func_id_name(func_id), func_id);
  6774. return err;
  6775. }
  6776. }
  6777. switch (func_id) {
  6778. case BPF_FUNC_tail_call:
  6779. err = check_reference_leak(env);
  6780. if (err) {
  6781. verbose(env, "tail_call would lead to reference leak\n");
  6782. return err;
  6783. }
  6784. break;
  6785. case BPF_FUNC_get_local_storage:
  6786. /* check that flags argument in get_local_storage(map, flags) is 0,
  6787. * this is required because get_local_storage() can't return an error.
  6788. */
  6789. if (!register_is_null(&regs[BPF_REG_2])) {
  6790. verbose(env, "get_local_storage() doesn't support non-zero flags\n");
  6791. return -EINVAL;
  6792. }
  6793. break;
  6794. case BPF_FUNC_for_each_map_elem:
  6795. err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
  6796. set_map_elem_callback_state);
  6797. break;
  6798. case BPF_FUNC_timer_set_callback:
  6799. err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
  6800. set_timer_callback_state);
  6801. break;
  6802. case BPF_FUNC_find_vma:
  6803. err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
  6804. set_find_vma_callback_state);
  6805. break;
  6806. case BPF_FUNC_snprintf:
  6807. err = check_bpf_snprintf_call(env, regs);
  6808. break;
  6809. case BPF_FUNC_loop:
  6810. update_loop_inline_state(env, meta.subprogno);
  6811. err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
  6812. set_loop_callback_state);
  6813. break;
  6814. case BPF_FUNC_dynptr_from_mem:
  6815. if (regs[BPF_REG_1].type != PTR_TO_MAP_VALUE) {
  6816. verbose(env, "Unsupported reg type %s for bpf_dynptr_from_mem data\n",
  6817. reg_type_str(env, regs[BPF_REG_1].type));
  6818. return -EACCES;
  6819. }
  6820. break;
  6821. case BPF_FUNC_set_retval:
  6822. if (prog_type == BPF_PROG_TYPE_LSM &&
  6823. env->prog->expected_attach_type == BPF_LSM_CGROUP) {
  6824. if (!env->prog->aux->attach_func_proto->type) {
  6825. /* Make sure programs that attach to void
  6826. * hooks don't try to modify return value.
  6827. */
  6828. verbose(env, "BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
  6829. return -EINVAL;
  6830. }
  6831. }
  6832. break;
  6833. case BPF_FUNC_dynptr_data:
  6834. for (i = 0; i < MAX_BPF_FUNC_REG_ARGS; i++) {
  6835. if (arg_type_is_dynptr(fn->arg_type[i])) {
  6836. struct bpf_reg_state *reg = &regs[BPF_REG_1 + i];
  6837. if (meta.ref_obj_id) {
  6838. verbose(env, "verifier internal error: meta.ref_obj_id already set\n");
  6839. return -EFAULT;
  6840. }
  6841. if (base_type(reg->type) != PTR_TO_DYNPTR)
  6842. /* Find the id of the dynptr we're
  6843. * tracking the reference of
  6844. */
  6845. meta.ref_obj_id = stack_slot_get_id(env, reg);
  6846. break;
  6847. }
  6848. }
  6849. if (i == MAX_BPF_FUNC_REG_ARGS) {
  6850. verbose(env, "verifier internal error: no dynptr in bpf_dynptr_data()\n");
  6851. return -EFAULT;
  6852. }
  6853. break;
  6854. case BPF_FUNC_user_ringbuf_drain:
  6855. err = __check_func_call(env, insn, insn_idx_p, meta.subprogno,
  6856. set_user_ringbuf_callback_state);
  6857. break;
  6858. }
  6859. if (err)
  6860. return err;
  6861. /* reset caller saved regs */
  6862. for (i = 0; i < CALLER_SAVED_REGS; i++) {
  6863. mark_reg_not_init(env, regs, caller_saved[i]);
  6864. check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
  6865. }
  6866. /* helper call returns 64-bit value. */
  6867. regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
  6868. /* update return register (already marked as written above) */
  6869. ret_type = fn->ret_type;
  6870. ret_flag = type_flag(ret_type);
  6871. switch (base_type(ret_type)) {
  6872. case RET_INTEGER:
  6873. /* sets type to SCALAR_VALUE */
  6874. mark_reg_unknown(env, regs, BPF_REG_0);
  6875. break;
  6876. case RET_VOID:
  6877. regs[BPF_REG_0].type = NOT_INIT;
  6878. break;
  6879. case RET_PTR_TO_MAP_VALUE:
  6880. /* There is no offset yet applied, variable or fixed */
  6881. mark_reg_known_zero(env, regs, BPF_REG_0);
  6882. /* remember map_ptr, so that check_map_access()
  6883. * can check 'value_size' boundary of memory access
  6884. * to map element returned from bpf_map_lookup_elem()
  6885. */
  6886. if (meta.map_ptr == NULL) {
  6887. verbose(env,
  6888. "kernel subsystem misconfigured verifier\n");
  6889. return -EINVAL;
  6890. }
  6891. regs[BPF_REG_0].map_ptr = meta.map_ptr;
  6892. regs[BPF_REG_0].map_uid = meta.map_uid;
  6893. regs[BPF_REG_0].type = PTR_TO_MAP_VALUE | ret_flag;
  6894. if (!type_may_be_null(ret_type) &&
  6895. map_value_has_spin_lock(meta.map_ptr)) {
  6896. regs[BPF_REG_0].id = ++env->id_gen;
  6897. }
  6898. break;
  6899. case RET_PTR_TO_SOCKET:
  6900. mark_reg_known_zero(env, regs, BPF_REG_0);
  6901. regs[BPF_REG_0].type = PTR_TO_SOCKET | ret_flag;
  6902. break;
  6903. case RET_PTR_TO_SOCK_COMMON:
  6904. mark_reg_known_zero(env, regs, BPF_REG_0);
  6905. regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON | ret_flag;
  6906. break;
  6907. case RET_PTR_TO_TCP_SOCK:
  6908. mark_reg_known_zero(env, regs, BPF_REG_0);
  6909. regs[BPF_REG_0].type = PTR_TO_TCP_SOCK | ret_flag;
  6910. break;
  6911. case RET_PTR_TO_ALLOC_MEM:
  6912. mark_reg_known_zero(env, regs, BPF_REG_0);
  6913. regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
  6914. regs[BPF_REG_0].mem_size = meta.mem_size;
  6915. break;
  6916. case RET_PTR_TO_MEM_OR_BTF_ID:
  6917. {
  6918. const struct btf_type *t;
  6919. mark_reg_known_zero(env, regs, BPF_REG_0);
  6920. t = btf_type_skip_modifiers(meta.ret_btf, meta.ret_btf_id, NULL);
  6921. if (!btf_type_is_struct(t)) {
  6922. u32 tsize;
  6923. const struct btf_type *ret;
  6924. const char *tname;
  6925. /* resolve the type size of ksym. */
  6926. ret = btf_resolve_size(meta.ret_btf, t, &tsize);
  6927. if (IS_ERR(ret)) {
  6928. tname = btf_name_by_offset(meta.ret_btf, t->name_off);
  6929. verbose(env, "unable to resolve the size of type '%s': %ld\n",
  6930. tname, PTR_ERR(ret));
  6931. return -EINVAL;
  6932. }
  6933. regs[BPF_REG_0].type = PTR_TO_MEM | ret_flag;
  6934. regs[BPF_REG_0].mem_size = tsize;
  6935. } else {
  6936. /* MEM_RDONLY may be carried from ret_flag, but it
  6937. * doesn't apply on PTR_TO_BTF_ID. Fold it, otherwise
  6938. * it will confuse the check of PTR_TO_BTF_ID in
  6939. * check_mem_access().
  6940. */
  6941. ret_flag &= ~MEM_RDONLY;
  6942. regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
  6943. regs[BPF_REG_0].btf = meta.ret_btf;
  6944. regs[BPF_REG_0].btf_id = meta.ret_btf_id;
  6945. }
  6946. break;
  6947. }
  6948. case RET_PTR_TO_BTF_ID:
  6949. {
  6950. struct btf *ret_btf;
  6951. int ret_btf_id;
  6952. mark_reg_known_zero(env, regs, BPF_REG_0);
  6953. regs[BPF_REG_0].type = PTR_TO_BTF_ID | ret_flag;
  6954. if (func_id == BPF_FUNC_kptr_xchg) {
  6955. ret_btf = meta.kptr_off_desc->kptr.btf;
  6956. ret_btf_id = meta.kptr_off_desc->kptr.btf_id;
  6957. } else {
  6958. if (fn->ret_btf_id == BPF_PTR_POISON) {
  6959. verbose(env, "verifier internal error:");
  6960. verbose(env, "func %s has non-overwritten BPF_PTR_POISON return type\n",
  6961. func_id_name(func_id));
  6962. return -EINVAL;
  6963. }
  6964. ret_btf = btf_vmlinux;
  6965. ret_btf_id = *fn->ret_btf_id;
  6966. }
  6967. if (ret_btf_id == 0) {
  6968. verbose(env, "invalid return type %u of func %s#%d\n",
  6969. base_type(ret_type), func_id_name(func_id),
  6970. func_id);
  6971. return -EINVAL;
  6972. }
  6973. regs[BPF_REG_0].btf = ret_btf;
  6974. regs[BPF_REG_0].btf_id = ret_btf_id;
  6975. break;
  6976. }
  6977. default:
  6978. verbose(env, "unknown return type %u of func %s#%d\n",
  6979. base_type(ret_type), func_id_name(func_id), func_id);
  6980. return -EINVAL;
  6981. }
  6982. if (type_may_be_null(regs[BPF_REG_0].type))
  6983. regs[BPF_REG_0].id = ++env->id_gen;
  6984. if (helper_multiple_ref_obj_use(func_id, meta.map_ptr)) {
  6985. verbose(env, "verifier internal error: func %s#%d sets ref_obj_id more than once\n",
  6986. func_id_name(func_id), func_id);
  6987. return -EFAULT;
  6988. }
  6989. if (is_ptr_cast_function(func_id) || is_dynptr_ref_function(func_id)) {
  6990. /* For release_reference() */
  6991. regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
  6992. } else if (is_acquire_function(func_id, meta.map_ptr)) {
  6993. int id = acquire_reference_state(env, insn_idx);
  6994. if (id < 0)
  6995. return id;
  6996. /* For mark_ptr_or_null_reg() */
  6997. regs[BPF_REG_0].id = id;
  6998. /* For release_reference() */
  6999. regs[BPF_REG_0].ref_obj_id = id;
  7000. }
  7001. do_refine_retval_range(regs, fn->ret_type, func_id, &meta);
  7002. err = check_map_func_compatibility(env, meta.map_ptr, func_id);
  7003. if (err)
  7004. return err;
  7005. if ((func_id == BPF_FUNC_get_stack ||
  7006. func_id == BPF_FUNC_get_task_stack) &&
  7007. !env->prog->has_callchain_buf) {
  7008. const char *err_str;
  7009. #ifdef CONFIG_PERF_EVENTS
  7010. err = get_callchain_buffers(sysctl_perf_event_max_stack);
  7011. err_str = "cannot get callchain buffer for func %s#%d\n";
  7012. #else
  7013. err = -ENOTSUPP;
  7014. err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
  7015. #endif
  7016. if (err) {
  7017. verbose(env, err_str, func_id_name(func_id), func_id);
  7018. return err;
  7019. }
  7020. env->prog->has_callchain_buf = true;
  7021. }
  7022. if (func_id == BPF_FUNC_get_stackid || func_id == BPF_FUNC_get_stack)
  7023. env->prog->call_get_stack = true;
  7024. if (func_id == BPF_FUNC_get_func_ip) {
  7025. if (check_get_func_ip(env))
  7026. return -ENOTSUPP;
  7027. env->prog->call_get_func_ip = true;
  7028. }
  7029. if (changes_data)
  7030. clear_all_pkt_pointers(env);
  7031. return 0;
  7032. }
  7033. /* mark_btf_func_reg_size() is used when the reg size is determined by
  7034. * the BTF func_proto's return value size and argument.
  7035. */
  7036. static void mark_btf_func_reg_size(struct bpf_verifier_env *env, u32 regno,
  7037. size_t reg_size)
  7038. {
  7039. struct bpf_reg_state *reg = &cur_regs(env)[regno];
  7040. if (regno == BPF_REG_0) {
  7041. /* Function return value */
  7042. reg->live |= REG_LIVE_WRITTEN;
  7043. reg->subreg_def = reg_size == sizeof(u64) ?
  7044. DEF_NOT_SUBREG : env->insn_idx + 1;
  7045. } else {
  7046. /* Function argument */
  7047. if (reg_size == sizeof(u64)) {
  7048. mark_insn_zext(env, reg);
  7049. mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
  7050. } else {
  7051. mark_reg_read(env, reg, reg->parent, REG_LIVE_READ32);
  7052. }
  7053. }
  7054. }
  7055. static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
  7056. int *insn_idx_p)
  7057. {
  7058. const struct btf_type *t, *func, *func_proto, *ptr_type;
  7059. struct bpf_reg_state *regs = cur_regs(env);
  7060. struct bpf_kfunc_arg_meta meta = { 0 };
  7061. const char *func_name, *ptr_type_name;
  7062. u32 i, nargs, func_id, ptr_type_id;
  7063. int err, insn_idx = *insn_idx_p;
  7064. const struct btf_param *args;
  7065. struct btf *desc_btf;
  7066. u32 *kfunc_flags;
  7067. bool acq;
  7068. /* skip for now, but return error when we find this in fixup_kfunc_call */
  7069. if (!insn->imm)
  7070. return 0;
  7071. desc_btf = find_kfunc_desc_btf(env, insn->off);
  7072. if (IS_ERR(desc_btf))
  7073. return PTR_ERR(desc_btf);
  7074. func_id = insn->imm;
  7075. func = btf_type_by_id(desc_btf, func_id);
  7076. func_name = btf_name_by_offset(desc_btf, func->name_off);
  7077. func_proto = btf_type_by_id(desc_btf, func->type);
  7078. kfunc_flags = btf_kfunc_id_set_contains(desc_btf, resolve_prog_type(env->prog), func_id);
  7079. if (!kfunc_flags) {
  7080. verbose(env, "calling kernel function %s is not allowed\n",
  7081. func_name);
  7082. return -EACCES;
  7083. }
  7084. if (*kfunc_flags & KF_DESTRUCTIVE && !capable(CAP_SYS_BOOT)) {
  7085. verbose(env, "destructive kfunc calls require CAP_SYS_BOOT capabilities\n");
  7086. return -EACCES;
  7087. }
  7088. acq = *kfunc_flags & KF_ACQUIRE;
  7089. meta.flags = *kfunc_flags;
  7090. /* Check the arguments */
  7091. err = btf_check_kfunc_arg_match(env, desc_btf, func_id, regs, &meta);
  7092. if (err < 0)
  7093. return err;
  7094. /* In case of release function, we get register number of refcounted
  7095. * PTR_TO_BTF_ID back from btf_check_kfunc_arg_match, do the release now
  7096. */
  7097. if (err) {
  7098. err = release_reference(env, regs[err].ref_obj_id);
  7099. if (err) {
  7100. verbose(env, "kfunc %s#%d reference has not been acquired before\n",
  7101. func_name, func_id);
  7102. return err;
  7103. }
  7104. }
  7105. for (i = 0; i < CALLER_SAVED_REGS; i++)
  7106. mark_reg_not_init(env, regs, caller_saved[i]);
  7107. /* Check return type */
  7108. t = btf_type_skip_modifiers(desc_btf, func_proto->type, NULL);
  7109. if (acq && !btf_type_is_struct_ptr(desc_btf, t)) {
  7110. verbose(env, "acquire kernel function does not return PTR_TO_BTF_ID\n");
  7111. return -EINVAL;
  7112. }
  7113. if (btf_type_is_scalar(t)) {
  7114. mark_reg_unknown(env, regs, BPF_REG_0);
  7115. mark_btf_func_reg_size(env, BPF_REG_0, t->size);
  7116. } else if (btf_type_is_ptr(t)) {
  7117. ptr_type = btf_type_skip_modifiers(desc_btf, t->type,
  7118. &ptr_type_id);
  7119. if (!btf_type_is_struct(ptr_type)) {
  7120. if (!meta.r0_size) {
  7121. ptr_type_name = btf_name_by_offset(desc_btf,
  7122. ptr_type->name_off);
  7123. verbose(env,
  7124. "kernel function %s returns pointer type %s %s is not supported\n",
  7125. func_name,
  7126. btf_type_str(ptr_type),
  7127. ptr_type_name);
  7128. return -EINVAL;
  7129. }
  7130. mark_reg_known_zero(env, regs, BPF_REG_0);
  7131. regs[BPF_REG_0].type = PTR_TO_MEM;
  7132. regs[BPF_REG_0].mem_size = meta.r0_size;
  7133. if (meta.r0_rdonly)
  7134. regs[BPF_REG_0].type |= MEM_RDONLY;
  7135. /* Ensures we don't access the memory after a release_reference() */
  7136. if (meta.ref_obj_id)
  7137. regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
  7138. } else {
  7139. mark_reg_known_zero(env, regs, BPF_REG_0);
  7140. regs[BPF_REG_0].btf = desc_btf;
  7141. regs[BPF_REG_0].type = PTR_TO_BTF_ID;
  7142. regs[BPF_REG_0].btf_id = ptr_type_id;
  7143. }
  7144. if (*kfunc_flags & KF_RET_NULL) {
  7145. regs[BPF_REG_0].type |= PTR_MAYBE_NULL;
  7146. /* For mark_ptr_or_null_reg, see 93c230e3f5bd6 */
  7147. regs[BPF_REG_0].id = ++env->id_gen;
  7148. }
  7149. mark_btf_func_reg_size(env, BPF_REG_0, sizeof(void *));
  7150. if (acq) {
  7151. int id = acquire_reference_state(env, insn_idx);
  7152. if (id < 0)
  7153. return id;
  7154. regs[BPF_REG_0].id = id;
  7155. regs[BPF_REG_0].ref_obj_id = id;
  7156. }
  7157. } /* else { add_kfunc_call() ensures it is btf_type_is_void(t) } */
  7158. nargs = btf_type_vlen(func_proto);
  7159. args = (const struct btf_param *)(func_proto + 1);
  7160. for (i = 0; i < nargs; i++) {
  7161. u32 regno = i + 1;
  7162. t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
  7163. if (btf_type_is_ptr(t))
  7164. mark_btf_func_reg_size(env, regno, sizeof(void *));
  7165. else
  7166. /* scalar. ensured by btf_check_kfunc_arg_match() */
  7167. mark_btf_func_reg_size(env, regno, t->size);
  7168. }
  7169. return 0;
  7170. }
  7171. static bool signed_add_overflows(s64 a, s64 b)
  7172. {
  7173. /* Do the add in u64, where overflow is well-defined */
  7174. s64 res = (s64)((u64)a + (u64)b);
  7175. if (b < 0)
  7176. return res > a;
  7177. return res < a;
  7178. }
  7179. static bool signed_add32_overflows(s32 a, s32 b)
  7180. {
  7181. /* Do the add in u32, where overflow is well-defined */
  7182. s32 res = (s32)((u32)a + (u32)b);
  7183. if (b < 0)
  7184. return res > a;
  7185. return res < a;
  7186. }
  7187. static bool signed_sub_overflows(s64 a, s64 b)
  7188. {
  7189. /* Do the sub in u64, where overflow is well-defined */
  7190. s64 res = (s64)((u64)a - (u64)b);
  7191. if (b < 0)
  7192. return res < a;
  7193. return res > a;
  7194. }
  7195. static bool signed_sub32_overflows(s32 a, s32 b)
  7196. {
  7197. /* Do the sub in u32, where overflow is well-defined */
  7198. s32 res = (s32)((u32)a - (u32)b);
  7199. if (b < 0)
  7200. return res < a;
  7201. return res > a;
  7202. }
  7203. static bool check_reg_sane_offset(struct bpf_verifier_env *env,
  7204. const struct bpf_reg_state *reg,
  7205. enum bpf_reg_type type)
  7206. {
  7207. bool known = tnum_is_const(reg->var_off);
  7208. s64 val = reg->var_off.value;
  7209. s64 smin = reg->smin_value;
  7210. if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
  7211. verbose(env, "math between %s pointer and %lld is not allowed\n",
  7212. reg_type_str(env, type), val);
  7213. return false;
  7214. }
  7215. if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
  7216. verbose(env, "%s pointer offset %d is not allowed\n",
  7217. reg_type_str(env, type), reg->off);
  7218. return false;
  7219. }
  7220. if (smin == S64_MIN) {
  7221. verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
  7222. reg_type_str(env, type));
  7223. return false;
  7224. }
  7225. if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
  7226. verbose(env, "value %lld makes %s pointer be out of bounds\n",
  7227. smin, reg_type_str(env, type));
  7228. return false;
  7229. }
  7230. return true;
  7231. }
  7232. enum {
  7233. REASON_BOUNDS = -1,
  7234. REASON_TYPE = -2,
  7235. REASON_PATHS = -3,
  7236. REASON_LIMIT = -4,
  7237. REASON_STACK = -5,
  7238. };
  7239. static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
  7240. u32 *alu_limit, bool mask_to_left)
  7241. {
  7242. u32 max = 0, ptr_limit = 0;
  7243. switch (ptr_reg->type) {
  7244. case PTR_TO_STACK:
  7245. /* Offset 0 is out-of-bounds, but acceptable start for the
  7246. * left direction, see BPF_REG_FP. Also, unknown scalar
  7247. * offset where we would need to deal with min/max bounds is
  7248. * currently prohibited for unprivileged.
  7249. */
  7250. max = MAX_BPF_STACK + mask_to_left;
  7251. ptr_limit = -(ptr_reg->var_off.value + ptr_reg->off);
  7252. break;
  7253. case PTR_TO_MAP_VALUE:
  7254. max = ptr_reg->map_ptr->value_size;
  7255. ptr_limit = (mask_to_left ?
  7256. ptr_reg->smin_value :
  7257. ptr_reg->umax_value) + ptr_reg->off;
  7258. break;
  7259. default:
  7260. return REASON_TYPE;
  7261. }
  7262. if (ptr_limit >= max)
  7263. return REASON_LIMIT;
  7264. *alu_limit = ptr_limit;
  7265. return 0;
  7266. }
  7267. static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
  7268. const struct bpf_insn *insn)
  7269. {
  7270. return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
  7271. }
  7272. static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
  7273. u32 alu_state, u32 alu_limit)
  7274. {
  7275. /* If we arrived here from different branches with different
  7276. * state or limits to sanitize, then this won't work.
  7277. */
  7278. if (aux->alu_state &&
  7279. (aux->alu_state != alu_state ||
  7280. aux->alu_limit != alu_limit))
  7281. return REASON_PATHS;
  7282. /* Corresponding fixup done in do_misc_fixups(). */
  7283. aux->alu_state = alu_state;
  7284. aux->alu_limit = alu_limit;
  7285. return 0;
  7286. }
  7287. static int sanitize_val_alu(struct bpf_verifier_env *env,
  7288. struct bpf_insn *insn)
  7289. {
  7290. struct bpf_insn_aux_data *aux = cur_aux(env);
  7291. if (can_skip_alu_sanitation(env, insn))
  7292. return 0;
  7293. return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
  7294. }
  7295. static bool sanitize_needed(u8 opcode)
  7296. {
  7297. return opcode == BPF_ADD || opcode == BPF_SUB;
  7298. }
  7299. struct bpf_sanitize_info {
  7300. struct bpf_insn_aux_data aux;
  7301. bool mask_to_left;
  7302. };
  7303. static struct bpf_verifier_state *
  7304. sanitize_speculative_path(struct bpf_verifier_env *env,
  7305. const struct bpf_insn *insn,
  7306. u32 next_idx, u32 curr_idx)
  7307. {
  7308. struct bpf_verifier_state *branch;
  7309. struct bpf_reg_state *regs;
  7310. branch = push_stack(env, next_idx, curr_idx, true);
  7311. if (branch && insn) {
  7312. regs = branch->frame[branch->curframe]->regs;
  7313. if (BPF_SRC(insn->code) == BPF_K) {
  7314. mark_reg_unknown(env, regs, insn->dst_reg);
  7315. } else if (BPF_SRC(insn->code) == BPF_X) {
  7316. mark_reg_unknown(env, regs, insn->dst_reg);
  7317. mark_reg_unknown(env, regs, insn->src_reg);
  7318. }
  7319. }
  7320. return branch;
  7321. }
  7322. static int sanitize_ptr_alu(struct bpf_verifier_env *env,
  7323. struct bpf_insn *insn,
  7324. const struct bpf_reg_state *ptr_reg,
  7325. const struct bpf_reg_state *off_reg,
  7326. struct bpf_reg_state *dst_reg,
  7327. struct bpf_sanitize_info *info,
  7328. const bool commit_window)
  7329. {
  7330. struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
  7331. struct bpf_verifier_state *vstate = env->cur_state;
  7332. bool off_is_imm = tnum_is_const(off_reg->var_off);
  7333. bool off_is_neg = off_reg->smin_value < 0;
  7334. bool ptr_is_dst_reg = ptr_reg == dst_reg;
  7335. u8 opcode = BPF_OP(insn->code);
  7336. u32 alu_state, alu_limit;
  7337. struct bpf_reg_state tmp;
  7338. bool ret;
  7339. int err;
  7340. if (can_skip_alu_sanitation(env, insn))
  7341. return 0;
  7342. /* We already marked aux for masking from non-speculative
  7343. * paths, thus we got here in the first place. We only care
  7344. * to explore bad access from here.
  7345. */
  7346. if (vstate->speculative)
  7347. goto do_sim;
  7348. if (!commit_window) {
  7349. if (!tnum_is_const(off_reg->var_off) &&
  7350. (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
  7351. return REASON_BOUNDS;
  7352. info->mask_to_left = (opcode == BPF_ADD && off_is_neg) ||
  7353. (opcode == BPF_SUB && !off_is_neg);
  7354. }
  7355. err = retrieve_ptr_limit(ptr_reg, &alu_limit, info->mask_to_left);
  7356. if (err < 0)
  7357. return err;
  7358. if (commit_window) {
  7359. /* In commit phase we narrow the masking window based on
  7360. * the observed pointer move after the simulated operation.
  7361. */
  7362. alu_state = info->aux.alu_state;
  7363. alu_limit = abs(info->aux.alu_limit - alu_limit);
  7364. } else {
  7365. alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
  7366. alu_state |= off_is_imm ? BPF_ALU_IMMEDIATE : 0;
  7367. alu_state |= ptr_is_dst_reg ?
  7368. BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;
  7369. /* Limit pruning on unknown scalars to enable deep search for
  7370. * potential masking differences from other program paths.
  7371. */
  7372. if (!off_is_imm)
  7373. env->explore_alu_limits = true;
  7374. }
  7375. err = update_alu_sanitation_state(aux, alu_state, alu_limit);
  7376. if (err < 0)
  7377. return err;
  7378. do_sim:
  7379. /* If we're in commit phase, we're done here given we already
  7380. * pushed the truncated dst_reg into the speculative verification
  7381. * stack.
  7382. *
  7383. * Also, when register is a known constant, we rewrite register-based
  7384. * operation to immediate-based, and thus do not need masking (and as
  7385. * a consequence, do not need to simulate the zero-truncation either).
  7386. */
  7387. if (commit_window || off_is_imm)
  7388. return 0;
  7389. /* Simulate and find potential out-of-bounds access under
  7390. * speculative execution from truncation as a result of
  7391. * masking when off was not within expected range. If off
  7392. * sits in dst, then we temporarily need to move ptr there
  7393. * to simulate dst (== 0) +/-= ptr. Needed, for example,
  7394. * for cases where we use K-based arithmetic in one direction
  7395. * and truncated reg-based in the other in order to explore
  7396. * bad access.
  7397. */
  7398. if (!ptr_is_dst_reg) {
  7399. tmp = *dst_reg;
  7400. copy_register_state(dst_reg, ptr_reg);
  7401. }
  7402. ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1,
  7403. env->insn_idx);
  7404. if (!ptr_is_dst_reg && ret)
  7405. *dst_reg = tmp;
  7406. return !ret ? REASON_STACK : 0;
  7407. }
  7408. static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
  7409. {
  7410. struct bpf_verifier_state *vstate = env->cur_state;
  7411. /* If we simulate paths under speculation, we don't update the
  7412. * insn as 'seen' such that when we verify unreachable paths in
  7413. * the non-speculative domain, sanitize_dead_code() can still
  7414. * rewrite/sanitize them.
  7415. */
  7416. if (!vstate->speculative)
  7417. env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
  7418. }
  7419. static int sanitize_err(struct bpf_verifier_env *env,
  7420. const struct bpf_insn *insn, int reason,
  7421. const struct bpf_reg_state *off_reg,
  7422. const struct bpf_reg_state *dst_reg)
  7423. {
  7424. static const char *err = "pointer arithmetic with it prohibited for !root";
  7425. const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub";
  7426. u32 dst = insn->dst_reg, src = insn->src_reg;
  7427. switch (reason) {
  7428. case REASON_BOUNDS:
  7429. verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n",
  7430. off_reg == dst_reg ? dst : src, err);
  7431. break;
  7432. case REASON_TYPE:
  7433. verbose(env, "R%d has pointer with unsupported alu operation, %s\n",
  7434. off_reg == dst_reg ? src : dst, err);
  7435. break;
  7436. case REASON_PATHS:
  7437. verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n",
  7438. dst, op, err);
  7439. break;
  7440. case REASON_LIMIT:
  7441. verbose(env, "R%d tried to %s beyond pointer bounds, %s\n",
  7442. dst, op, err);
  7443. break;
  7444. case REASON_STACK:
  7445. verbose(env, "R%d could not be pushed for speculative verification, %s\n",
  7446. dst, err);
  7447. break;
  7448. default:
  7449. verbose(env, "verifier internal error: unknown reason (%d)\n",
  7450. reason);
  7451. break;
  7452. }
  7453. return -EACCES;
  7454. }
  7455. /* check that stack access falls within stack limits and that 'reg' doesn't
  7456. * have a variable offset.
  7457. *
  7458. * Variable offset is prohibited for unprivileged mode for simplicity since it
  7459. * requires corresponding support in Spectre masking for stack ALU. See also
  7460. * retrieve_ptr_limit().
  7461. *
  7462. *
  7463. * 'off' includes 'reg->off'.
  7464. */
  7465. static int check_stack_access_for_ptr_arithmetic(
  7466. struct bpf_verifier_env *env,
  7467. int regno,
  7468. const struct bpf_reg_state *reg,
  7469. int off)
  7470. {
  7471. if (!tnum_is_const(reg->var_off)) {
  7472. char tn_buf[48];
  7473. tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
  7474. verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n",
  7475. regno, tn_buf, off);
  7476. return -EACCES;
  7477. }
  7478. if (off >= 0 || off < -MAX_BPF_STACK) {
  7479. verbose(env, "R%d stack pointer arithmetic goes out of range, "
  7480. "prohibited for !root; off=%d\n", regno, off);
  7481. return -EACCES;
  7482. }
  7483. return 0;
  7484. }
  7485. static int sanitize_check_bounds(struct bpf_verifier_env *env,
  7486. const struct bpf_insn *insn,
  7487. const struct bpf_reg_state *dst_reg)
  7488. {
  7489. u32 dst = insn->dst_reg;
  7490. /* For unprivileged we require that resulting offset must be in bounds
  7491. * in order to be able to sanitize access later on.
  7492. */
  7493. if (env->bypass_spec_v1)
  7494. return 0;
  7495. switch (dst_reg->type) {
  7496. case PTR_TO_STACK:
  7497. if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg,
  7498. dst_reg->off + dst_reg->var_off.value))
  7499. return -EACCES;
  7500. break;
  7501. case PTR_TO_MAP_VALUE:
  7502. if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) {
  7503. verbose(env, "R%d pointer arithmetic of map value goes out of range, "
  7504. "prohibited for !root\n", dst);
  7505. return -EACCES;
  7506. }
  7507. break;
  7508. default:
  7509. break;
  7510. }
  7511. return 0;
  7512. }
  7513. /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
  7514. * Caller should also handle BPF_MOV case separately.
  7515. * If we return -EACCES, caller may want to try again treating pointer as a
  7516. * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks.
  7517. */
  7518. static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
  7519. struct bpf_insn *insn,
  7520. const struct bpf_reg_state *ptr_reg,
  7521. const struct bpf_reg_state *off_reg)
  7522. {
  7523. struct bpf_verifier_state *vstate = env->cur_state;
  7524. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  7525. struct bpf_reg_state *regs = state->regs, *dst_reg;
  7526. bool known = tnum_is_const(off_reg->var_off);
  7527. s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
  7528. smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
  7529. u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
  7530. umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
  7531. struct bpf_sanitize_info info = {};
  7532. u8 opcode = BPF_OP(insn->code);
  7533. u32 dst = insn->dst_reg;
  7534. int ret;
  7535. dst_reg = &regs[dst];
  7536. if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
  7537. smin_val > smax_val || umin_val > umax_val) {
  7538. /* Taint dst register if offset had invalid bounds derived from
  7539. * e.g. dead branches.
  7540. */
  7541. __mark_reg_unknown(env, dst_reg);
  7542. return 0;
  7543. }
  7544. if (BPF_CLASS(insn->code) != BPF_ALU64) {
  7545. /* 32-bit ALU ops on pointers produce (meaningless) scalars */
  7546. if (opcode == BPF_SUB && env->allow_ptr_leaks) {
  7547. __mark_reg_unknown(env, dst_reg);
  7548. return 0;
  7549. }
  7550. verbose(env,
  7551. "R%d 32-bit pointer arithmetic prohibited\n",
  7552. dst);
  7553. return -EACCES;
  7554. }
  7555. if (ptr_reg->type & PTR_MAYBE_NULL) {
  7556. verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
  7557. dst, reg_type_str(env, ptr_reg->type));
  7558. return -EACCES;
  7559. }
  7560. switch (base_type(ptr_reg->type)) {
  7561. case CONST_PTR_TO_MAP:
  7562. /* smin_val represents the known value */
  7563. if (known && smin_val == 0 && opcode == BPF_ADD)
  7564. break;
  7565. fallthrough;
  7566. case PTR_TO_PACKET_END:
  7567. case PTR_TO_SOCKET:
  7568. case PTR_TO_SOCK_COMMON:
  7569. case PTR_TO_TCP_SOCK:
  7570. case PTR_TO_XDP_SOCK:
  7571. verbose(env, "R%d pointer arithmetic on %s prohibited\n",
  7572. dst, reg_type_str(env, ptr_reg->type));
  7573. return -EACCES;
  7574. default:
  7575. break;
  7576. }
  7577. /* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
  7578. * The id may be overwritten later if we create a new variable offset.
  7579. */
  7580. dst_reg->type = ptr_reg->type;
  7581. dst_reg->id = ptr_reg->id;
  7582. if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
  7583. !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
  7584. return -EINVAL;
  7585. /* pointer types do not carry 32-bit bounds at the moment. */
  7586. __mark_reg32_unbounded(dst_reg);
  7587. if (sanitize_needed(opcode)) {
  7588. ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg,
  7589. &info, false);
  7590. if (ret < 0)
  7591. return sanitize_err(env, insn, ret, off_reg, dst_reg);
  7592. }
  7593. switch (opcode) {
  7594. case BPF_ADD:
  7595. /* We can take a fixed offset as long as it doesn't overflow
  7596. * the s32 'off' field
  7597. */
  7598. if (known && (ptr_reg->off + smin_val ==
  7599. (s64)(s32)(ptr_reg->off + smin_val))) {
  7600. /* pointer += K. Accumulate it into fixed offset */
  7601. dst_reg->smin_value = smin_ptr;
  7602. dst_reg->smax_value = smax_ptr;
  7603. dst_reg->umin_value = umin_ptr;
  7604. dst_reg->umax_value = umax_ptr;
  7605. dst_reg->var_off = ptr_reg->var_off;
  7606. dst_reg->off = ptr_reg->off + smin_val;
  7607. dst_reg->raw = ptr_reg->raw;
  7608. break;
  7609. }
  7610. /* A new variable offset is created. Note that off_reg->off
  7611. * == 0, since it's a scalar.
  7612. * dst_reg gets the pointer type and since some positive
  7613. * integer value was added to the pointer, give it a new 'id'
  7614. * if it's a PTR_TO_PACKET.
  7615. * this creates a new 'base' pointer, off_reg (variable) gets
  7616. * added into the variable offset, and we copy the fixed offset
  7617. * from ptr_reg.
  7618. */
  7619. if (signed_add_overflows(smin_ptr, smin_val) ||
  7620. signed_add_overflows(smax_ptr, smax_val)) {
  7621. dst_reg->smin_value = S64_MIN;
  7622. dst_reg->smax_value = S64_MAX;
  7623. } else {
  7624. dst_reg->smin_value = smin_ptr + smin_val;
  7625. dst_reg->smax_value = smax_ptr + smax_val;
  7626. }
  7627. if (umin_ptr + umin_val < umin_ptr ||
  7628. umax_ptr + umax_val < umax_ptr) {
  7629. dst_reg->umin_value = 0;
  7630. dst_reg->umax_value = U64_MAX;
  7631. } else {
  7632. dst_reg->umin_value = umin_ptr + umin_val;
  7633. dst_reg->umax_value = umax_ptr + umax_val;
  7634. }
  7635. dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
  7636. dst_reg->off = ptr_reg->off;
  7637. dst_reg->raw = ptr_reg->raw;
  7638. if (reg_is_pkt_pointer(ptr_reg)) {
  7639. dst_reg->id = ++env->id_gen;
  7640. /* something was added to pkt_ptr, set range to zero */
  7641. memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
  7642. }
  7643. break;
  7644. case BPF_SUB:
  7645. if (dst_reg == off_reg) {
  7646. /* scalar -= pointer. Creates an unknown scalar */
  7647. verbose(env, "R%d tried to subtract pointer from scalar\n",
  7648. dst);
  7649. return -EACCES;
  7650. }
  7651. /* We don't allow subtraction from FP, because (according to
  7652. * test_verifier.c test "invalid fp arithmetic", JITs might not
  7653. * be able to deal with it.
  7654. */
  7655. if (ptr_reg->type == PTR_TO_STACK) {
  7656. verbose(env, "R%d subtraction from stack pointer prohibited\n",
  7657. dst);
  7658. return -EACCES;
  7659. }
  7660. if (known && (ptr_reg->off - smin_val ==
  7661. (s64)(s32)(ptr_reg->off - smin_val))) {
  7662. /* pointer -= K. Subtract it from fixed offset */
  7663. dst_reg->smin_value = smin_ptr;
  7664. dst_reg->smax_value = smax_ptr;
  7665. dst_reg->umin_value = umin_ptr;
  7666. dst_reg->umax_value = umax_ptr;
  7667. dst_reg->var_off = ptr_reg->var_off;
  7668. dst_reg->id = ptr_reg->id;
  7669. dst_reg->off = ptr_reg->off - smin_val;
  7670. dst_reg->raw = ptr_reg->raw;
  7671. break;
  7672. }
  7673. /* A new variable offset is created. If the subtrahend is known
  7674. * nonnegative, then any reg->range we had before is still good.
  7675. */
  7676. if (signed_sub_overflows(smin_ptr, smax_val) ||
  7677. signed_sub_overflows(smax_ptr, smin_val)) {
  7678. /* Overflow possible, we know nothing */
  7679. dst_reg->smin_value = S64_MIN;
  7680. dst_reg->smax_value = S64_MAX;
  7681. } else {
  7682. dst_reg->smin_value = smin_ptr - smax_val;
  7683. dst_reg->smax_value = smax_ptr - smin_val;
  7684. }
  7685. if (umin_ptr < umax_val) {
  7686. /* Overflow possible, we know nothing */
  7687. dst_reg->umin_value = 0;
  7688. dst_reg->umax_value = U64_MAX;
  7689. } else {
  7690. /* Cannot overflow (as long as bounds are consistent) */
  7691. dst_reg->umin_value = umin_ptr - umax_val;
  7692. dst_reg->umax_value = umax_ptr - umin_val;
  7693. }
  7694. dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
  7695. dst_reg->off = ptr_reg->off;
  7696. dst_reg->raw = ptr_reg->raw;
  7697. if (reg_is_pkt_pointer(ptr_reg)) {
  7698. dst_reg->id = ++env->id_gen;
  7699. /* something was added to pkt_ptr, set range to zero */
  7700. if (smin_val < 0)
  7701. memset(&dst_reg->raw, 0, sizeof(dst_reg->raw));
  7702. }
  7703. break;
  7704. case BPF_AND:
  7705. case BPF_OR:
  7706. case BPF_XOR:
  7707. /* bitwise ops on pointers are troublesome, prohibit. */
  7708. verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
  7709. dst, bpf_alu_string[opcode >> 4]);
  7710. return -EACCES;
  7711. default:
  7712. /* other operators (e.g. MUL,LSH) produce non-pointer results */
  7713. verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
  7714. dst, bpf_alu_string[opcode >> 4]);
  7715. return -EACCES;
  7716. }
  7717. if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
  7718. return -EINVAL;
  7719. reg_bounds_sync(dst_reg);
  7720. if (sanitize_check_bounds(env, insn, dst_reg) < 0)
  7721. return -EACCES;
  7722. if (sanitize_needed(opcode)) {
  7723. ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg,
  7724. &info, true);
  7725. if (ret < 0)
  7726. return sanitize_err(env, insn, ret, off_reg, dst_reg);
  7727. }
  7728. return 0;
  7729. }
  7730. static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
  7731. struct bpf_reg_state *src_reg)
  7732. {
  7733. s32 smin_val = src_reg->s32_min_value;
  7734. s32 smax_val = src_reg->s32_max_value;
  7735. u32 umin_val = src_reg->u32_min_value;
  7736. u32 umax_val = src_reg->u32_max_value;
  7737. if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) ||
  7738. signed_add32_overflows(dst_reg->s32_max_value, smax_val)) {
  7739. dst_reg->s32_min_value = S32_MIN;
  7740. dst_reg->s32_max_value = S32_MAX;
  7741. } else {
  7742. dst_reg->s32_min_value += smin_val;
  7743. dst_reg->s32_max_value += smax_val;
  7744. }
  7745. if (dst_reg->u32_min_value + umin_val < umin_val ||
  7746. dst_reg->u32_max_value + umax_val < umax_val) {
  7747. dst_reg->u32_min_value = 0;
  7748. dst_reg->u32_max_value = U32_MAX;
  7749. } else {
  7750. dst_reg->u32_min_value += umin_val;
  7751. dst_reg->u32_max_value += umax_val;
  7752. }
  7753. }
  7754. static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
  7755. struct bpf_reg_state *src_reg)
  7756. {
  7757. s64 smin_val = src_reg->smin_value;
  7758. s64 smax_val = src_reg->smax_value;
  7759. u64 umin_val = src_reg->umin_value;
  7760. u64 umax_val = src_reg->umax_value;
  7761. if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
  7762. signed_add_overflows(dst_reg->smax_value, smax_val)) {
  7763. dst_reg->smin_value = S64_MIN;
  7764. dst_reg->smax_value = S64_MAX;
  7765. } else {
  7766. dst_reg->smin_value += smin_val;
  7767. dst_reg->smax_value += smax_val;
  7768. }
  7769. if (dst_reg->umin_value + umin_val < umin_val ||
  7770. dst_reg->umax_value + umax_val < umax_val) {
  7771. dst_reg->umin_value = 0;
  7772. dst_reg->umax_value = U64_MAX;
  7773. } else {
  7774. dst_reg->umin_value += umin_val;
  7775. dst_reg->umax_value += umax_val;
  7776. }
  7777. }
  7778. static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
  7779. struct bpf_reg_state *src_reg)
  7780. {
  7781. s32 smin_val = src_reg->s32_min_value;
  7782. s32 smax_val = src_reg->s32_max_value;
  7783. u32 umin_val = src_reg->u32_min_value;
  7784. u32 umax_val = src_reg->u32_max_value;
  7785. if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) ||
  7786. signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) {
  7787. /* Overflow possible, we know nothing */
  7788. dst_reg->s32_min_value = S32_MIN;
  7789. dst_reg->s32_max_value = S32_MAX;
  7790. } else {
  7791. dst_reg->s32_min_value -= smax_val;
  7792. dst_reg->s32_max_value -= smin_val;
  7793. }
  7794. if (dst_reg->u32_min_value < umax_val) {
  7795. /* Overflow possible, we know nothing */
  7796. dst_reg->u32_min_value = 0;
  7797. dst_reg->u32_max_value = U32_MAX;
  7798. } else {
  7799. /* Cannot overflow (as long as bounds are consistent) */
  7800. dst_reg->u32_min_value -= umax_val;
  7801. dst_reg->u32_max_value -= umin_val;
  7802. }
  7803. }
  7804. static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
  7805. struct bpf_reg_state *src_reg)
  7806. {
  7807. s64 smin_val = src_reg->smin_value;
  7808. s64 smax_val = src_reg->smax_value;
  7809. u64 umin_val = src_reg->umin_value;
  7810. u64 umax_val = src_reg->umax_value;
  7811. if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
  7812. signed_sub_overflows(dst_reg->smax_value, smin_val)) {
  7813. /* Overflow possible, we know nothing */
  7814. dst_reg->smin_value = S64_MIN;
  7815. dst_reg->smax_value = S64_MAX;
  7816. } else {
  7817. dst_reg->smin_value -= smax_val;
  7818. dst_reg->smax_value -= smin_val;
  7819. }
  7820. if (dst_reg->umin_value < umax_val) {
  7821. /* Overflow possible, we know nothing */
  7822. dst_reg->umin_value = 0;
  7823. dst_reg->umax_value = U64_MAX;
  7824. } else {
  7825. /* Cannot overflow (as long as bounds are consistent) */
  7826. dst_reg->umin_value -= umax_val;
  7827. dst_reg->umax_value -= umin_val;
  7828. }
  7829. }
  7830. static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
  7831. struct bpf_reg_state *src_reg)
  7832. {
  7833. s32 smin_val = src_reg->s32_min_value;
  7834. u32 umin_val = src_reg->u32_min_value;
  7835. u32 umax_val = src_reg->u32_max_value;
  7836. if (smin_val < 0 || dst_reg->s32_min_value < 0) {
  7837. /* Ain't nobody got time to multiply that sign */
  7838. __mark_reg32_unbounded(dst_reg);
  7839. return;
  7840. }
  7841. /* Both values are positive, so we can work with unsigned and
  7842. * copy the result to signed (unless it exceeds S32_MAX).
  7843. */
  7844. if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) {
  7845. /* Potential overflow, we know nothing */
  7846. __mark_reg32_unbounded(dst_reg);
  7847. return;
  7848. }
  7849. dst_reg->u32_min_value *= umin_val;
  7850. dst_reg->u32_max_value *= umax_val;
  7851. if (dst_reg->u32_max_value > S32_MAX) {
  7852. /* Overflow possible, we know nothing */
  7853. dst_reg->s32_min_value = S32_MIN;
  7854. dst_reg->s32_max_value = S32_MAX;
  7855. } else {
  7856. dst_reg->s32_min_value = dst_reg->u32_min_value;
  7857. dst_reg->s32_max_value = dst_reg->u32_max_value;
  7858. }
  7859. }
  7860. static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
  7861. struct bpf_reg_state *src_reg)
  7862. {
  7863. s64 smin_val = src_reg->smin_value;
  7864. u64 umin_val = src_reg->umin_value;
  7865. u64 umax_val = src_reg->umax_value;
  7866. if (smin_val < 0 || dst_reg->smin_value < 0) {
  7867. /* Ain't nobody got time to multiply that sign */
  7868. __mark_reg64_unbounded(dst_reg);
  7869. return;
  7870. }
  7871. /* Both values are positive, so we can work with unsigned and
  7872. * copy the result to signed (unless it exceeds S64_MAX).
  7873. */
  7874. if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
  7875. /* Potential overflow, we know nothing */
  7876. __mark_reg64_unbounded(dst_reg);
  7877. return;
  7878. }
  7879. dst_reg->umin_value *= umin_val;
  7880. dst_reg->umax_value *= umax_val;
  7881. if (dst_reg->umax_value > S64_MAX) {
  7882. /* Overflow possible, we know nothing */
  7883. dst_reg->smin_value = S64_MIN;
  7884. dst_reg->smax_value = S64_MAX;
  7885. } else {
  7886. dst_reg->smin_value = dst_reg->umin_value;
  7887. dst_reg->smax_value = dst_reg->umax_value;
  7888. }
  7889. }
  7890. static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
  7891. struct bpf_reg_state *src_reg)
  7892. {
  7893. bool src_known = tnum_subreg_is_const(src_reg->var_off);
  7894. bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
  7895. struct tnum var32_off = tnum_subreg(dst_reg->var_off);
  7896. s32 smin_val = src_reg->s32_min_value;
  7897. u32 umax_val = src_reg->u32_max_value;
  7898. if (src_known && dst_known) {
  7899. __mark_reg32_known(dst_reg, var32_off.value);
  7900. return;
  7901. }
  7902. /* We get our minimum from the var_off, since that's inherently
  7903. * bitwise. Our maximum is the minimum of the operands' maxima.
  7904. */
  7905. dst_reg->u32_min_value = var32_off.value;
  7906. dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
  7907. if (dst_reg->s32_min_value < 0 || smin_val < 0) {
  7908. /* Lose signed bounds when ANDing negative numbers,
  7909. * ain't nobody got time for that.
  7910. */
  7911. dst_reg->s32_min_value = S32_MIN;
  7912. dst_reg->s32_max_value = S32_MAX;
  7913. } else {
  7914. /* ANDing two positives gives a positive, so safe to
  7915. * cast result into s64.
  7916. */
  7917. dst_reg->s32_min_value = dst_reg->u32_min_value;
  7918. dst_reg->s32_max_value = dst_reg->u32_max_value;
  7919. }
  7920. }
  7921. static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
  7922. struct bpf_reg_state *src_reg)
  7923. {
  7924. bool src_known = tnum_is_const(src_reg->var_off);
  7925. bool dst_known = tnum_is_const(dst_reg->var_off);
  7926. s64 smin_val = src_reg->smin_value;
  7927. u64 umax_val = src_reg->umax_value;
  7928. if (src_known && dst_known) {
  7929. __mark_reg_known(dst_reg, dst_reg->var_off.value);
  7930. return;
  7931. }
  7932. /* We get our minimum from the var_off, since that's inherently
  7933. * bitwise. Our maximum is the minimum of the operands' maxima.
  7934. */
  7935. dst_reg->umin_value = dst_reg->var_off.value;
  7936. dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
  7937. if (dst_reg->smin_value < 0 || smin_val < 0) {
  7938. /* Lose signed bounds when ANDing negative numbers,
  7939. * ain't nobody got time for that.
  7940. */
  7941. dst_reg->smin_value = S64_MIN;
  7942. dst_reg->smax_value = S64_MAX;
  7943. } else {
  7944. /* ANDing two positives gives a positive, so safe to
  7945. * cast result into s64.
  7946. */
  7947. dst_reg->smin_value = dst_reg->umin_value;
  7948. dst_reg->smax_value = dst_reg->umax_value;
  7949. }
  7950. /* We may learn something more from the var_off */
  7951. __update_reg_bounds(dst_reg);
  7952. }
  7953. static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
  7954. struct bpf_reg_state *src_reg)
  7955. {
  7956. bool src_known = tnum_subreg_is_const(src_reg->var_off);
  7957. bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
  7958. struct tnum var32_off = tnum_subreg(dst_reg->var_off);
  7959. s32 smin_val = src_reg->s32_min_value;
  7960. u32 umin_val = src_reg->u32_min_value;
  7961. if (src_known && dst_known) {
  7962. __mark_reg32_known(dst_reg, var32_off.value);
  7963. return;
  7964. }
  7965. /* We get our maximum from the var_off, and our minimum is the
  7966. * maximum of the operands' minima
  7967. */
  7968. dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
  7969. dst_reg->u32_max_value = var32_off.value | var32_off.mask;
  7970. if (dst_reg->s32_min_value < 0 || smin_val < 0) {
  7971. /* Lose signed bounds when ORing negative numbers,
  7972. * ain't nobody got time for that.
  7973. */
  7974. dst_reg->s32_min_value = S32_MIN;
  7975. dst_reg->s32_max_value = S32_MAX;
  7976. } else {
  7977. /* ORing two positives gives a positive, so safe to
  7978. * cast result into s64.
  7979. */
  7980. dst_reg->s32_min_value = dst_reg->u32_min_value;
  7981. dst_reg->s32_max_value = dst_reg->u32_max_value;
  7982. }
  7983. }
  7984. static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
  7985. struct bpf_reg_state *src_reg)
  7986. {
  7987. bool src_known = tnum_is_const(src_reg->var_off);
  7988. bool dst_known = tnum_is_const(dst_reg->var_off);
  7989. s64 smin_val = src_reg->smin_value;
  7990. u64 umin_val = src_reg->umin_value;
  7991. if (src_known && dst_known) {
  7992. __mark_reg_known(dst_reg, dst_reg->var_off.value);
  7993. return;
  7994. }
  7995. /* We get our maximum from the var_off, and our minimum is the
  7996. * maximum of the operands' minima
  7997. */
  7998. dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
  7999. dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
  8000. if (dst_reg->smin_value < 0 || smin_val < 0) {
  8001. /* Lose signed bounds when ORing negative numbers,
  8002. * ain't nobody got time for that.
  8003. */
  8004. dst_reg->smin_value = S64_MIN;
  8005. dst_reg->smax_value = S64_MAX;
  8006. } else {
  8007. /* ORing two positives gives a positive, so safe to
  8008. * cast result into s64.
  8009. */
  8010. dst_reg->smin_value = dst_reg->umin_value;
  8011. dst_reg->smax_value = dst_reg->umax_value;
  8012. }
  8013. /* We may learn something more from the var_off */
  8014. __update_reg_bounds(dst_reg);
  8015. }
  8016. static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
  8017. struct bpf_reg_state *src_reg)
  8018. {
  8019. bool src_known = tnum_subreg_is_const(src_reg->var_off);
  8020. bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
  8021. struct tnum var32_off = tnum_subreg(dst_reg->var_off);
  8022. s32 smin_val = src_reg->s32_min_value;
  8023. if (src_known && dst_known) {
  8024. __mark_reg32_known(dst_reg, var32_off.value);
  8025. return;
  8026. }
  8027. /* We get both minimum and maximum from the var32_off. */
  8028. dst_reg->u32_min_value = var32_off.value;
  8029. dst_reg->u32_max_value = var32_off.value | var32_off.mask;
  8030. if (dst_reg->s32_min_value >= 0 && smin_val >= 0) {
  8031. /* XORing two positive sign numbers gives a positive,
  8032. * so safe to cast u32 result into s32.
  8033. */
  8034. dst_reg->s32_min_value = dst_reg->u32_min_value;
  8035. dst_reg->s32_max_value = dst_reg->u32_max_value;
  8036. } else {
  8037. dst_reg->s32_min_value = S32_MIN;
  8038. dst_reg->s32_max_value = S32_MAX;
  8039. }
  8040. }
  8041. static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
  8042. struct bpf_reg_state *src_reg)
  8043. {
  8044. bool src_known = tnum_is_const(src_reg->var_off);
  8045. bool dst_known = tnum_is_const(dst_reg->var_off);
  8046. s64 smin_val = src_reg->smin_value;
  8047. if (src_known && dst_known) {
  8048. /* dst_reg->var_off.value has been updated earlier */
  8049. __mark_reg_known(dst_reg, dst_reg->var_off.value);
  8050. return;
  8051. }
  8052. /* We get both minimum and maximum from the var_off. */
  8053. dst_reg->umin_value = dst_reg->var_off.value;
  8054. dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
  8055. if (dst_reg->smin_value >= 0 && smin_val >= 0) {
  8056. /* XORing two positive sign numbers gives a positive,
  8057. * so safe to cast u64 result into s64.
  8058. */
  8059. dst_reg->smin_value = dst_reg->umin_value;
  8060. dst_reg->smax_value = dst_reg->umax_value;
  8061. } else {
  8062. dst_reg->smin_value = S64_MIN;
  8063. dst_reg->smax_value = S64_MAX;
  8064. }
  8065. __update_reg_bounds(dst_reg);
  8066. }
  8067. static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
  8068. u64 umin_val, u64 umax_val)
  8069. {
  8070. /* We lose all sign bit information (except what we can pick
  8071. * up from var_off)
  8072. */
  8073. dst_reg->s32_min_value = S32_MIN;
  8074. dst_reg->s32_max_value = S32_MAX;
  8075. /* If we might shift our top bit out, then we know nothing */
  8076. if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
  8077. dst_reg->u32_min_value = 0;
  8078. dst_reg->u32_max_value = U32_MAX;
  8079. } else {
  8080. dst_reg->u32_min_value <<= umin_val;
  8081. dst_reg->u32_max_value <<= umax_val;
  8082. }
  8083. }
  8084. static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
  8085. struct bpf_reg_state *src_reg)
  8086. {
  8087. u32 umax_val = src_reg->u32_max_value;
  8088. u32 umin_val = src_reg->u32_min_value;
  8089. /* u32 alu operation will zext upper bits */
  8090. struct tnum subreg = tnum_subreg(dst_reg->var_off);
  8091. __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
  8092. dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val));
  8093. /* Not required but being careful mark reg64 bounds as unknown so
  8094. * that we are forced to pick them up from tnum and zext later and
  8095. * if some path skips this step we are still safe.
  8096. */
  8097. __mark_reg64_unbounded(dst_reg);
  8098. __update_reg32_bounds(dst_reg);
  8099. }
  8100. static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
  8101. u64 umin_val, u64 umax_val)
  8102. {
  8103. /* Special case <<32 because it is a common compiler pattern to sign
  8104. * extend subreg by doing <<32 s>>32. In this case if 32bit bounds are
  8105. * positive we know this shift will also be positive so we can track
  8106. * bounds correctly. Otherwise we lose all sign bit information except
  8107. * what we can pick up from var_off. Perhaps we can generalize this
  8108. * later to shifts of any length.
  8109. */
  8110. if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0)
  8111. dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
  8112. else
  8113. dst_reg->smax_value = S64_MAX;
  8114. if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0)
  8115. dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
  8116. else
  8117. dst_reg->smin_value = S64_MIN;
  8118. /* If we might shift our top bit out, then we know nothing */
  8119. if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
  8120. dst_reg->umin_value = 0;
  8121. dst_reg->umax_value = U64_MAX;
  8122. } else {
  8123. dst_reg->umin_value <<= umin_val;
  8124. dst_reg->umax_value <<= umax_val;
  8125. }
  8126. }
  8127. static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
  8128. struct bpf_reg_state *src_reg)
  8129. {
  8130. u64 umax_val = src_reg->umax_value;
  8131. u64 umin_val = src_reg->umin_value;
  8132. /* scalar64 calc uses 32bit unshifted bounds so must be called first */
  8133. __scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
  8134. __scalar32_min_max_lsh(dst_reg, umin_val, umax_val);
  8135. dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
  8136. /* We may learn something more from the var_off */
  8137. __update_reg_bounds(dst_reg);
  8138. }
  8139. static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
  8140. struct bpf_reg_state *src_reg)
  8141. {
  8142. struct tnum subreg = tnum_subreg(dst_reg->var_off);
  8143. u32 umax_val = src_reg->u32_max_value;
  8144. u32 umin_val = src_reg->u32_min_value;
  8145. /* BPF_RSH is an unsigned shift. If the value in dst_reg might
  8146. * be negative, then either:
  8147. * 1) src_reg might be zero, so the sign bit of the result is
  8148. * unknown, so we lose our signed bounds
  8149. * 2) it's known negative, thus the unsigned bounds capture the
  8150. * signed bounds
  8151. * 3) the signed bounds cross zero, so they tell us nothing
  8152. * about the result
  8153. * If the value in dst_reg is known nonnegative, then again the
  8154. * unsigned bounds capture the signed bounds.
  8155. * Thus, in all cases it suffices to blow away our signed bounds
  8156. * and rely on inferring new ones from the unsigned bounds and
  8157. * var_off of the result.
  8158. */
  8159. dst_reg->s32_min_value = S32_MIN;
  8160. dst_reg->s32_max_value = S32_MAX;
  8161. dst_reg->var_off = tnum_rshift(subreg, umin_val);
  8162. dst_reg->u32_min_value >>= umax_val;
  8163. dst_reg->u32_max_value >>= umin_val;
  8164. __mark_reg64_unbounded(dst_reg);
  8165. __update_reg32_bounds(dst_reg);
  8166. }
  8167. static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
  8168. struct bpf_reg_state *src_reg)
  8169. {
  8170. u64 umax_val = src_reg->umax_value;
  8171. u64 umin_val = src_reg->umin_value;
  8172. /* BPF_RSH is an unsigned shift. If the value in dst_reg might
  8173. * be negative, then either:
  8174. * 1) src_reg might be zero, so the sign bit of the result is
  8175. * unknown, so we lose our signed bounds
  8176. * 2) it's known negative, thus the unsigned bounds capture the
  8177. * signed bounds
  8178. * 3) the signed bounds cross zero, so they tell us nothing
  8179. * about the result
  8180. * If the value in dst_reg is known nonnegative, then again the
  8181. * unsigned bounds capture the signed bounds.
  8182. * Thus, in all cases it suffices to blow away our signed bounds
  8183. * and rely on inferring new ones from the unsigned bounds and
  8184. * var_off of the result.
  8185. */
  8186. dst_reg->smin_value = S64_MIN;
  8187. dst_reg->smax_value = S64_MAX;
  8188. dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
  8189. dst_reg->umin_value >>= umax_val;
  8190. dst_reg->umax_value >>= umin_val;
  8191. /* Its not easy to operate on alu32 bounds here because it depends
  8192. * on bits being shifted in. Take easy way out and mark unbounded
  8193. * so we can recalculate later from tnum.
  8194. */
  8195. __mark_reg32_unbounded(dst_reg);
  8196. __update_reg_bounds(dst_reg);
  8197. }
  8198. static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
  8199. struct bpf_reg_state *src_reg)
  8200. {
  8201. u64 umin_val = src_reg->u32_min_value;
  8202. /* Upon reaching here, src_known is true and
  8203. * umax_val is equal to umin_val.
  8204. */
  8205. dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
  8206. dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
  8207. dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
  8208. /* blow away the dst_reg umin_value/umax_value and rely on
  8209. * dst_reg var_off to refine the result.
  8210. */
  8211. dst_reg->u32_min_value = 0;
  8212. dst_reg->u32_max_value = U32_MAX;
  8213. __mark_reg64_unbounded(dst_reg);
  8214. __update_reg32_bounds(dst_reg);
  8215. }
  8216. static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
  8217. struct bpf_reg_state *src_reg)
  8218. {
  8219. u64 umin_val = src_reg->umin_value;
  8220. /* Upon reaching here, src_known is true and umax_val is equal
  8221. * to umin_val.
  8222. */
  8223. dst_reg->smin_value >>= umin_val;
  8224. dst_reg->smax_value >>= umin_val;
  8225. dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
  8226. /* blow away the dst_reg umin_value/umax_value and rely on
  8227. * dst_reg var_off to refine the result.
  8228. */
  8229. dst_reg->umin_value = 0;
  8230. dst_reg->umax_value = U64_MAX;
  8231. /* Its not easy to operate on alu32 bounds here because it depends
  8232. * on bits being shifted in from upper 32-bits. Take easy way out
  8233. * and mark unbounded so we can recalculate later from tnum.
  8234. */
  8235. __mark_reg32_unbounded(dst_reg);
  8236. __update_reg_bounds(dst_reg);
  8237. }
  8238. /* WARNING: This function does calculations on 64-bit values, but the actual
  8239. * execution may occur on 32-bit values. Therefore, things like bitshifts
  8240. * need extra checks in the 32-bit case.
  8241. */
  8242. static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
  8243. struct bpf_insn *insn,
  8244. struct bpf_reg_state *dst_reg,
  8245. struct bpf_reg_state src_reg)
  8246. {
  8247. struct bpf_reg_state *regs = cur_regs(env);
  8248. u8 opcode = BPF_OP(insn->code);
  8249. bool src_known;
  8250. s64 smin_val, smax_val;
  8251. u64 umin_val, umax_val;
  8252. s32 s32_min_val, s32_max_val;
  8253. u32 u32_min_val, u32_max_val;
  8254. u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
  8255. bool alu32 = (BPF_CLASS(insn->code) != BPF_ALU64);
  8256. int ret;
  8257. smin_val = src_reg.smin_value;
  8258. smax_val = src_reg.smax_value;
  8259. umin_val = src_reg.umin_value;
  8260. umax_val = src_reg.umax_value;
  8261. s32_min_val = src_reg.s32_min_value;
  8262. s32_max_val = src_reg.s32_max_value;
  8263. u32_min_val = src_reg.u32_min_value;
  8264. u32_max_val = src_reg.u32_max_value;
  8265. if (alu32) {
  8266. src_known = tnum_subreg_is_const(src_reg.var_off);
  8267. if ((src_known &&
  8268. (s32_min_val != s32_max_val || u32_min_val != u32_max_val)) ||
  8269. s32_min_val > s32_max_val || u32_min_val > u32_max_val) {
  8270. /* Taint dst register if offset had invalid bounds
  8271. * derived from e.g. dead branches.
  8272. */
  8273. __mark_reg_unknown(env, dst_reg);
  8274. return 0;
  8275. }
  8276. } else {
  8277. src_known = tnum_is_const(src_reg.var_off);
  8278. if ((src_known &&
  8279. (smin_val != smax_val || umin_val != umax_val)) ||
  8280. smin_val > smax_val || umin_val > umax_val) {
  8281. /* Taint dst register if offset had invalid bounds
  8282. * derived from e.g. dead branches.
  8283. */
  8284. __mark_reg_unknown(env, dst_reg);
  8285. return 0;
  8286. }
  8287. }
  8288. if (!src_known &&
  8289. opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
  8290. __mark_reg_unknown(env, dst_reg);
  8291. return 0;
  8292. }
  8293. if (sanitize_needed(opcode)) {
  8294. ret = sanitize_val_alu(env, insn);
  8295. if (ret < 0)
  8296. return sanitize_err(env, insn, ret, NULL, NULL);
  8297. }
  8298. /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops.
  8299. * There are two classes of instructions: The first class we track both
  8300. * alu32 and alu64 sign/unsigned bounds independently this provides the
  8301. * greatest amount of precision when alu operations are mixed with jmp32
  8302. * operations. These operations are BPF_ADD, BPF_SUB, BPF_MUL, BPF_ADD,
  8303. * and BPF_OR. This is possible because these ops have fairly easy to
  8304. * understand and calculate behavior in both 32-bit and 64-bit alu ops.
  8305. * See alu32 verifier tests for examples. The second class of
  8306. * operations, BPF_LSH, BPF_RSH, and BPF_ARSH, however are not so easy
  8307. * with regards to tracking sign/unsigned bounds because the bits may
  8308. * cross subreg boundaries in the alu64 case. When this happens we mark
  8309. * the reg unbounded in the subreg bound space and use the resulting
  8310. * tnum to calculate an approximation of the sign/unsigned bounds.
  8311. */
  8312. switch (opcode) {
  8313. case BPF_ADD:
  8314. scalar32_min_max_add(dst_reg, &src_reg);
  8315. scalar_min_max_add(dst_reg, &src_reg);
  8316. dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
  8317. break;
  8318. case BPF_SUB:
  8319. scalar32_min_max_sub(dst_reg, &src_reg);
  8320. scalar_min_max_sub(dst_reg, &src_reg);
  8321. dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
  8322. break;
  8323. case BPF_MUL:
  8324. dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
  8325. scalar32_min_max_mul(dst_reg, &src_reg);
  8326. scalar_min_max_mul(dst_reg, &src_reg);
  8327. break;
  8328. case BPF_AND:
  8329. dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
  8330. scalar32_min_max_and(dst_reg, &src_reg);
  8331. scalar_min_max_and(dst_reg, &src_reg);
  8332. break;
  8333. case BPF_OR:
  8334. dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
  8335. scalar32_min_max_or(dst_reg, &src_reg);
  8336. scalar_min_max_or(dst_reg, &src_reg);
  8337. break;
  8338. case BPF_XOR:
  8339. dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off);
  8340. scalar32_min_max_xor(dst_reg, &src_reg);
  8341. scalar_min_max_xor(dst_reg, &src_reg);
  8342. break;
  8343. case BPF_LSH:
  8344. if (umax_val >= insn_bitness) {
  8345. /* Shifts greater than 31 or 63 are undefined.
  8346. * This includes shifts by a negative number.
  8347. */
  8348. mark_reg_unknown(env, regs, insn->dst_reg);
  8349. break;
  8350. }
  8351. if (alu32)
  8352. scalar32_min_max_lsh(dst_reg, &src_reg);
  8353. else
  8354. scalar_min_max_lsh(dst_reg, &src_reg);
  8355. break;
  8356. case BPF_RSH:
  8357. if (umax_val >= insn_bitness) {
  8358. /* Shifts greater than 31 or 63 are undefined.
  8359. * This includes shifts by a negative number.
  8360. */
  8361. mark_reg_unknown(env, regs, insn->dst_reg);
  8362. break;
  8363. }
  8364. if (alu32)
  8365. scalar32_min_max_rsh(dst_reg, &src_reg);
  8366. else
  8367. scalar_min_max_rsh(dst_reg, &src_reg);
  8368. break;
  8369. case BPF_ARSH:
  8370. if (umax_val >= insn_bitness) {
  8371. /* Shifts greater than 31 or 63 are undefined.
  8372. * This includes shifts by a negative number.
  8373. */
  8374. mark_reg_unknown(env, regs, insn->dst_reg);
  8375. break;
  8376. }
  8377. if (alu32)
  8378. scalar32_min_max_arsh(dst_reg, &src_reg);
  8379. else
  8380. scalar_min_max_arsh(dst_reg, &src_reg);
  8381. break;
  8382. default:
  8383. mark_reg_unknown(env, regs, insn->dst_reg);
  8384. break;
  8385. }
  8386. /* ALU32 ops are zero extended into 64bit register */
  8387. if (alu32)
  8388. zext_32_to_64(dst_reg);
  8389. reg_bounds_sync(dst_reg);
  8390. return 0;
  8391. }
  8392. /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
  8393. * and var_off.
  8394. */
  8395. static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
  8396. struct bpf_insn *insn)
  8397. {
  8398. struct bpf_verifier_state *vstate = env->cur_state;
  8399. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  8400. struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
  8401. struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
  8402. u8 opcode = BPF_OP(insn->code);
  8403. int err;
  8404. dst_reg = &regs[insn->dst_reg];
  8405. src_reg = NULL;
  8406. if (dst_reg->type != SCALAR_VALUE)
  8407. ptr_reg = dst_reg;
  8408. else
  8409. /* Make sure ID is cleared otherwise dst_reg min/max could be
  8410. * incorrectly propagated into other registers by find_equal_scalars()
  8411. */
  8412. dst_reg->id = 0;
  8413. if (BPF_SRC(insn->code) == BPF_X) {
  8414. src_reg = &regs[insn->src_reg];
  8415. if (src_reg->type != SCALAR_VALUE) {
  8416. if (dst_reg->type != SCALAR_VALUE) {
  8417. /* Combining two pointers by any ALU op yields
  8418. * an arbitrary scalar. Disallow all math except
  8419. * pointer subtraction
  8420. */
  8421. if (opcode == BPF_SUB && env->allow_ptr_leaks) {
  8422. mark_reg_unknown(env, regs, insn->dst_reg);
  8423. return 0;
  8424. }
  8425. verbose(env, "R%d pointer %s pointer prohibited\n",
  8426. insn->dst_reg,
  8427. bpf_alu_string[opcode >> 4]);
  8428. return -EACCES;
  8429. } else {
  8430. /* scalar += pointer
  8431. * This is legal, but we have to reverse our
  8432. * src/dest handling in computing the range
  8433. */
  8434. err = mark_chain_precision(env, insn->dst_reg);
  8435. if (err)
  8436. return err;
  8437. return adjust_ptr_min_max_vals(env, insn,
  8438. src_reg, dst_reg);
  8439. }
  8440. } else if (ptr_reg) {
  8441. /* pointer += scalar */
  8442. err = mark_chain_precision(env, insn->src_reg);
  8443. if (err)
  8444. return err;
  8445. return adjust_ptr_min_max_vals(env, insn,
  8446. dst_reg, src_reg);
  8447. } else if (dst_reg->precise) {
  8448. /* if dst_reg is precise, src_reg should be precise as well */
  8449. err = mark_chain_precision(env, insn->src_reg);
  8450. if (err)
  8451. return err;
  8452. }
  8453. } else {
  8454. /* Pretend the src is a reg with a known value, since we only
  8455. * need to be able to read from this state.
  8456. */
  8457. off_reg.type = SCALAR_VALUE;
  8458. __mark_reg_known(&off_reg, insn->imm);
  8459. src_reg = &off_reg;
  8460. if (ptr_reg) /* pointer += K */
  8461. return adjust_ptr_min_max_vals(env, insn,
  8462. ptr_reg, src_reg);
  8463. }
  8464. /* Got here implies adding two SCALAR_VALUEs */
  8465. if (WARN_ON_ONCE(ptr_reg)) {
  8466. print_verifier_state(env, state, true);
  8467. verbose(env, "verifier internal error: unexpected ptr_reg\n");
  8468. return -EINVAL;
  8469. }
  8470. if (WARN_ON(!src_reg)) {
  8471. print_verifier_state(env, state, true);
  8472. verbose(env, "verifier internal error: no src_reg\n");
  8473. return -EINVAL;
  8474. }
  8475. return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
  8476. }
  8477. /* check validity of 32-bit and 64-bit arithmetic operations */
  8478. static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
  8479. {
  8480. struct bpf_reg_state *regs = cur_regs(env);
  8481. u8 opcode = BPF_OP(insn->code);
  8482. int err;
  8483. if (opcode == BPF_END || opcode == BPF_NEG) {
  8484. if (opcode == BPF_NEG) {
  8485. if (BPF_SRC(insn->code) != BPF_K ||
  8486. insn->src_reg != BPF_REG_0 ||
  8487. insn->off != 0 || insn->imm != 0) {
  8488. verbose(env, "BPF_NEG uses reserved fields\n");
  8489. return -EINVAL;
  8490. }
  8491. } else {
  8492. if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
  8493. (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
  8494. BPF_CLASS(insn->code) == BPF_ALU64) {
  8495. verbose(env, "BPF_END uses reserved fields\n");
  8496. return -EINVAL;
  8497. }
  8498. }
  8499. /* check src operand */
  8500. err = check_reg_arg(env, insn->dst_reg, SRC_OP);
  8501. if (err)
  8502. return err;
  8503. if (is_pointer_value(env, insn->dst_reg)) {
  8504. verbose(env, "R%d pointer arithmetic prohibited\n",
  8505. insn->dst_reg);
  8506. return -EACCES;
  8507. }
  8508. /* check dest operand */
  8509. err = check_reg_arg(env, insn->dst_reg, DST_OP);
  8510. if (err)
  8511. return err;
  8512. } else if (opcode == BPF_MOV) {
  8513. if (BPF_SRC(insn->code) == BPF_X) {
  8514. if (insn->imm != 0 || insn->off != 0) {
  8515. verbose(env, "BPF_MOV uses reserved fields\n");
  8516. return -EINVAL;
  8517. }
  8518. /* check src operand */
  8519. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  8520. if (err)
  8521. return err;
  8522. } else {
  8523. if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
  8524. verbose(env, "BPF_MOV uses reserved fields\n");
  8525. return -EINVAL;
  8526. }
  8527. }
  8528. /* check dest operand, mark as required later */
  8529. err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
  8530. if (err)
  8531. return err;
  8532. if (BPF_SRC(insn->code) == BPF_X) {
  8533. struct bpf_reg_state *src_reg = regs + insn->src_reg;
  8534. struct bpf_reg_state *dst_reg = regs + insn->dst_reg;
  8535. if (BPF_CLASS(insn->code) == BPF_ALU64) {
  8536. /* case: R1 = R2
  8537. * copy register state to dest reg
  8538. */
  8539. if (src_reg->type == SCALAR_VALUE && !src_reg->id)
  8540. /* Assign src and dst registers the same ID
  8541. * that will be used by find_equal_scalars()
  8542. * to propagate min/max range.
  8543. */
  8544. src_reg->id = ++env->id_gen;
  8545. copy_register_state(dst_reg, src_reg);
  8546. dst_reg->live |= REG_LIVE_WRITTEN;
  8547. dst_reg->subreg_def = DEF_NOT_SUBREG;
  8548. } else {
  8549. /* R1 = (u32) R2 */
  8550. if (is_pointer_value(env, insn->src_reg)) {
  8551. verbose(env,
  8552. "R%d partial copy of pointer\n",
  8553. insn->src_reg);
  8554. return -EACCES;
  8555. } else if (src_reg->type == SCALAR_VALUE) {
  8556. copy_register_state(dst_reg, src_reg);
  8557. /* Make sure ID is cleared otherwise
  8558. * dst_reg min/max could be incorrectly
  8559. * propagated into src_reg by find_equal_scalars()
  8560. */
  8561. dst_reg->id = 0;
  8562. dst_reg->live |= REG_LIVE_WRITTEN;
  8563. dst_reg->subreg_def = env->insn_idx + 1;
  8564. } else {
  8565. mark_reg_unknown(env, regs,
  8566. insn->dst_reg);
  8567. }
  8568. zext_32_to_64(dst_reg);
  8569. reg_bounds_sync(dst_reg);
  8570. }
  8571. } else {
  8572. /* case: R = imm
  8573. * remember the value we stored into this reg
  8574. */
  8575. /* clear any state __mark_reg_known doesn't set */
  8576. mark_reg_unknown(env, regs, insn->dst_reg);
  8577. regs[insn->dst_reg].type = SCALAR_VALUE;
  8578. if (BPF_CLASS(insn->code) == BPF_ALU64) {
  8579. __mark_reg_known(regs + insn->dst_reg,
  8580. insn->imm);
  8581. } else {
  8582. __mark_reg_known(regs + insn->dst_reg,
  8583. (u32)insn->imm);
  8584. }
  8585. }
  8586. } else if (opcode > BPF_END) {
  8587. verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
  8588. return -EINVAL;
  8589. } else { /* all other ALU ops: and, sub, xor, add, ... */
  8590. if (BPF_SRC(insn->code) == BPF_X) {
  8591. if (insn->imm != 0 || insn->off != 0) {
  8592. verbose(env, "BPF_ALU uses reserved fields\n");
  8593. return -EINVAL;
  8594. }
  8595. /* check src1 operand */
  8596. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  8597. if (err)
  8598. return err;
  8599. } else {
  8600. if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
  8601. verbose(env, "BPF_ALU uses reserved fields\n");
  8602. return -EINVAL;
  8603. }
  8604. }
  8605. /* check src2 operand */
  8606. err = check_reg_arg(env, insn->dst_reg, SRC_OP);
  8607. if (err)
  8608. return err;
  8609. if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
  8610. BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
  8611. verbose(env, "div by zero\n");
  8612. return -EINVAL;
  8613. }
  8614. if ((opcode == BPF_LSH || opcode == BPF_RSH ||
  8615. opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
  8616. int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;
  8617. if (insn->imm < 0 || insn->imm >= size) {
  8618. verbose(env, "invalid shift %d\n", insn->imm);
  8619. return -EINVAL;
  8620. }
  8621. }
  8622. /* check dest operand */
  8623. err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
  8624. if (err)
  8625. return err;
  8626. return adjust_reg_min_max_vals(env, insn);
  8627. }
  8628. return 0;
  8629. }
  8630. static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
  8631. struct bpf_reg_state *dst_reg,
  8632. enum bpf_reg_type type,
  8633. bool range_right_open)
  8634. {
  8635. struct bpf_func_state *state;
  8636. struct bpf_reg_state *reg;
  8637. int new_range;
  8638. if (dst_reg->off < 0 ||
  8639. (dst_reg->off == 0 && range_right_open))
  8640. /* This doesn't give us any range */
  8641. return;
  8642. if (dst_reg->umax_value > MAX_PACKET_OFF ||
  8643. dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
  8644. /* Risk of overflow. For instance, ptr + (1<<63) may be less
  8645. * than pkt_end, but that's because it's also less than pkt.
  8646. */
  8647. return;
  8648. new_range = dst_reg->off;
  8649. if (range_right_open)
  8650. new_range++;
  8651. /* Examples for register markings:
  8652. *
  8653. * pkt_data in dst register:
  8654. *
  8655. * r2 = r3;
  8656. * r2 += 8;
  8657. * if (r2 > pkt_end) goto <handle exception>
  8658. * <access okay>
  8659. *
  8660. * r2 = r3;
  8661. * r2 += 8;
  8662. * if (r2 < pkt_end) goto <access okay>
  8663. * <handle exception>
  8664. *
  8665. * Where:
  8666. * r2 == dst_reg, pkt_end == src_reg
  8667. * r2=pkt(id=n,off=8,r=0)
  8668. * r3=pkt(id=n,off=0,r=0)
  8669. *
  8670. * pkt_data in src register:
  8671. *
  8672. * r2 = r3;
  8673. * r2 += 8;
  8674. * if (pkt_end >= r2) goto <access okay>
  8675. * <handle exception>
  8676. *
  8677. * r2 = r3;
  8678. * r2 += 8;
  8679. * if (pkt_end <= r2) goto <handle exception>
  8680. * <access okay>
  8681. *
  8682. * Where:
  8683. * pkt_end == dst_reg, r2 == src_reg
  8684. * r2=pkt(id=n,off=8,r=0)
  8685. * r3=pkt(id=n,off=0,r=0)
  8686. *
  8687. * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
  8688. * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
  8689. * and [r3, r3 + 8-1) respectively is safe to access depending on
  8690. * the check.
  8691. */
  8692. /* If our ids match, then we must have the same max_value. And we
  8693. * don't care about the other reg's fixed offset, since if it's too big
  8694. * the range won't allow anything.
  8695. * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
  8696. */
  8697. bpf_for_each_reg_in_vstate(vstate, state, reg, ({
  8698. if (reg->type == type && reg->id == dst_reg->id)
  8699. /* keep the maximum range already checked */
  8700. reg->range = max(reg->range, new_range);
  8701. }));
  8702. }
  8703. static int is_branch32_taken(struct bpf_reg_state *reg, u32 val, u8 opcode)
  8704. {
  8705. struct tnum subreg = tnum_subreg(reg->var_off);
  8706. s32 sval = (s32)val;
  8707. switch (opcode) {
  8708. case BPF_JEQ:
  8709. if (tnum_is_const(subreg))
  8710. return !!tnum_equals_const(subreg, val);
  8711. break;
  8712. case BPF_JNE:
  8713. if (tnum_is_const(subreg))
  8714. return !tnum_equals_const(subreg, val);
  8715. break;
  8716. case BPF_JSET:
  8717. if ((~subreg.mask & subreg.value) & val)
  8718. return 1;
  8719. if (!((subreg.mask | subreg.value) & val))
  8720. return 0;
  8721. break;
  8722. case BPF_JGT:
  8723. if (reg->u32_min_value > val)
  8724. return 1;
  8725. else if (reg->u32_max_value <= val)
  8726. return 0;
  8727. break;
  8728. case BPF_JSGT:
  8729. if (reg->s32_min_value > sval)
  8730. return 1;
  8731. else if (reg->s32_max_value <= sval)
  8732. return 0;
  8733. break;
  8734. case BPF_JLT:
  8735. if (reg->u32_max_value < val)
  8736. return 1;
  8737. else if (reg->u32_min_value >= val)
  8738. return 0;
  8739. break;
  8740. case BPF_JSLT:
  8741. if (reg->s32_max_value < sval)
  8742. return 1;
  8743. else if (reg->s32_min_value >= sval)
  8744. return 0;
  8745. break;
  8746. case BPF_JGE:
  8747. if (reg->u32_min_value >= val)
  8748. return 1;
  8749. else if (reg->u32_max_value < val)
  8750. return 0;
  8751. break;
  8752. case BPF_JSGE:
  8753. if (reg->s32_min_value >= sval)
  8754. return 1;
  8755. else if (reg->s32_max_value < sval)
  8756. return 0;
  8757. break;
  8758. case BPF_JLE:
  8759. if (reg->u32_max_value <= val)
  8760. return 1;
  8761. else if (reg->u32_min_value > val)
  8762. return 0;
  8763. break;
  8764. case BPF_JSLE:
  8765. if (reg->s32_max_value <= sval)
  8766. return 1;
  8767. else if (reg->s32_min_value > sval)
  8768. return 0;
  8769. break;
  8770. }
  8771. return -1;
  8772. }
  8773. static int is_branch64_taken(struct bpf_reg_state *reg, u64 val, u8 opcode)
  8774. {
  8775. s64 sval = (s64)val;
  8776. switch (opcode) {
  8777. case BPF_JEQ:
  8778. if (tnum_is_const(reg->var_off))
  8779. return !!tnum_equals_const(reg->var_off, val);
  8780. break;
  8781. case BPF_JNE:
  8782. if (tnum_is_const(reg->var_off))
  8783. return !tnum_equals_const(reg->var_off, val);
  8784. break;
  8785. case BPF_JSET:
  8786. if ((~reg->var_off.mask & reg->var_off.value) & val)
  8787. return 1;
  8788. if (!((reg->var_off.mask | reg->var_off.value) & val))
  8789. return 0;
  8790. break;
  8791. case BPF_JGT:
  8792. if (reg->umin_value > val)
  8793. return 1;
  8794. else if (reg->umax_value <= val)
  8795. return 0;
  8796. break;
  8797. case BPF_JSGT:
  8798. if (reg->smin_value > sval)
  8799. return 1;
  8800. else if (reg->smax_value <= sval)
  8801. return 0;
  8802. break;
  8803. case BPF_JLT:
  8804. if (reg->umax_value < val)
  8805. return 1;
  8806. else if (reg->umin_value >= val)
  8807. return 0;
  8808. break;
  8809. case BPF_JSLT:
  8810. if (reg->smax_value < sval)
  8811. return 1;
  8812. else if (reg->smin_value >= sval)
  8813. return 0;
  8814. break;
  8815. case BPF_JGE:
  8816. if (reg->umin_value >= val)
  8817. return 1;
  8818. else if (reg->umax_value < val)
  8819. return 0;
  8820. break;
  8821. case BPF_JSGE:
  8822. if (reg->smin_value >= sval)
  8823. return 1;
  8824. else if (reg->smax_value < sval)
  8825. return 0;
  8826. break;
  8827. case BPF_JLE:
  8828. if (reg->umax_value <= val)
  8829. return 1;
  8830. else if (reg->umin_value > val)
  8831. return 0;
  8832. break;
  8833. case BPF_JSLE:
  8834. if (reg->smax_value <= sval)
  8835. return 1;
  8836. else if (reg->smin_value > sval)
  8837. return 0;
  8838. break;
  8839. }
  8840. return -1;
  8841. }
  8842. /* compute branch direction of the expression "if (reg opcode val) goto target;"
  8843. * and return:
  8844. * 1 - branch will be taken and "goto target" will be executed
  8845. * 0 - branch will not be taken and fall-through to next insn
  8846. * -1 - unknown. Example: "if (reg < 5)" is unknown when register value
  8847. * range [0,10]
  8848. */
  8849. static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
  8850. bool is_jmp32)
  8851. {
  8852. if (__is_pointer_value(false, reg)) {
  8853. if (!reg_type_not_null(reg->type))
  8854. return -1;
  8855. /* If pointer is valid tests against zero will fail so we can
  8856. * use this to direct branch taken.
  8857. */
  8858. if (val != 0)
  8859. return -1;
  8860. switch (opcode) {
  8861. case BPF_JEQ:
  8862. return 0;
  8863. case BPF_JNE:
  8864. return 1;
  8865. default:
  8866. return -1;
  8867. }
  8868. }
  8869. if (is_jmp32)
  8870. return is_branch32_taken(reg, val, opcode);
  8871. return is_branch64_taken(reg, val, opcode);
  8872. }
  8873. static int flip_opcode(u32 opcode)
  8874. {
  8875. /* How can we transform "a <op> b" into "b <op> a"? */
  8876. static const u8 opcode_flip[16] = {
  8877. /* these stay the same */
  8878. [BPF_JEQ >> 4] = BPF_JEQ,
  8879. [BPF_JNE >> 4] = BPF_JNE,
  8880. [BPF_JSET >> 4] = BPF_JSET,
  8881. /* these swap "lesser" and "greater" (L and G in the opcodes) */
  8882. [BPF_JGE >> 4] = BPF_JLE,
  8883. [BPF_JGT >> 4] = BPF_JLT,
  8884. [BPF_JLE >> 4] = BPF_JGE,
  8885. [BPF_JLT >> 4] = BPF_JGT,
  8886. [BPF_JSGE >> 4] = BPF_JSLE,
  8887. [BPF_JSGT >> 4] = BPF_JSLT,
  8888. [BPF_JSLE >> 4] = BPF_JSGE,
  8889. [BPF_JSLT >> 4] = BPF_JSGT
  8890. };
  8891. return opcode_flip[opcode >> 4];
  8892. }
  8893. static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg,
  8894. struct bpf_reg_state *src_reg,
  8895. u8 opcode)
  8896. {
  8897. struct bpf_reg_state *pkt;
  8898. if (src_reg->type == PTR_TO_PACKET_END) {
  8899. pkt = dst_reg;
  8900. } else if (dst_reg->type == PTR_TO_PACKET_END) {
  8901. pkt = src_reg;
  8902. opcode = flip_opcode(opcode);
  8903. } else {
  8904. return -1;
  8905. }
  8906. if (pkt->range >= 0)
  8907. return -1;
  8908. switch (opcode) {
  8909. case BPF_JLE:
  8910. /* pkt <= pkt_end */
  8911. fallthrough;
  8912. case BPF_JGT:
  8913. /* pkt > pkt_end */
  8914. if (pkt->range == BEYOND_PKT_END)
  8915. /* pkt has at last one extra byte beyond pkt_end */
  8916. return opcode == BPF_JGT;
  8917. break;
  8918. case BPF_JLT:
  8919. /* pkt < pkt_end */
  8920. fallthrough;
  8921. case BPF_JGE:
  8922. /* pkt >= pkt_end */
  8923. if (pkt->range == BEYOND_PKT_END || pkt->range == AT_PKT_END)
  8924. return opcode == BPF_JGE;
  8925. break;
  8926. }
  8927. return -1;
  8928. }
  8929. /* Adjusts the register min/max values in the case that the dst_reg is the
  8930. * variable register that we are working on, and src_reg is a constant or we're
  8931. * simply doing a BPF_K check.
  8932. * In JEQ/JNE cases we also adjust the var_off values.
  8933. */
  8934. static void reg_set_min_max(struct bpf_reg_state *true_reg,
  8935. struct bpf_reg_state *false_reg,
  8936. u64 val, u32 val32,
  8937. u8 opcode, bool is_jmp32)
  8938. {
  8939. struct tnum false_32off = tnum_subreg(false_reg->var_off);
  8940. struct tnum false_64off = false_reg->var_off;
  8941. struct tnum true_32off = tnum_subreg(true_reg->var_off);
  8942. struct tnum true_64off = true_reg->var_off;
  8943. s64 sval = (s64)val;
  8944. s32 sval32 = (s32)val32;
  8945. /* If the dst_reg is a pointer, we can't learn anything about its
  8946. * variable offset from the compare (unless src_reg were a pointer into
  8947. * the same object, but we don't bother with that.
  8948. * Since false_reg and true_reg have the same type by construction, we
  8949. * only need to check one of them for pointerness.
  8950. */
  8951. if (__is_pointer_value(false, false_reg))
  8952. return;
  8953. switch (opcode) {
  8954. /* JEQ/JNE comparison doesn't change the register equivalence.
  8955. *
  8956. * r1 = r2;
  8957. * if (r1 == 42) goto label;
  8958. * ...
  8959. * label: // here both r1 and r2 are known to be 42.
  8960. *
  8961. * Hence when marking register as known preserve it's ID.
  8962. */
  8963. case BPF_JEQ:
  8964. if (is_jmp32) {
  8965. __mark_reg32_known(true_reg, val32);
  8966. true_32off = tnum_subreg(true_reg->var_off);
  8967. } else {
  8968. ___mark_reg_known(true_reg, val);
  8969. true_64off = true_reg->var_off;
  8970. }
  8971. break;
  8972. case BPF_JNE:
  8973. if (is_jmp32) {
  8974. __mark_reg32_known(false_reg, val32);
  8975. false_32off = tnum_subreg(false_reg->var_off);
  8976. } else {
  8977. ___mark_reg_known(false_reg, val);
  8978. false_64off = false_reg->var_off;
  8979. }
  8980. break;
  8981. case BPF_JSET:
  8982. if (is_jmp32) {
  8983. false_32off = tnum_and(false_32off, tnum_const(~val32));
  8984. if (is_power_of_2(val32))
  8985. true_32off = tnum_or(true_32off,
  8986. tnum_const(val32));
  8987. } else {
  8988. false_64off = tnum_and(false_64off, tnum_const(~val));
  8989. if (is_power_of_2(val))
  8990. true_64off = tnum_or(true_64off,
  8991. tnum_const(val));
  8992. }
  8993. break;
  8994. case BPF_JGE:
  8995. case BPF_JGT:
  8996. {
  8997. if (is_jmp32) {
  8998. u32 false_umax = opcode == BPF_JGT ? val32 : val32 - 1;
  8999. u32 true_umin = opcode == BPF_JGT ? val32 + 1 : val32;
  9000. false_reg->u32_max_value = min(false_reg->u32_max_value,
  9001. false_umax);
  9002. true_reg->u32_min_value = max(true_reg->u32_min_value,
  9003. true_umin);
  9004. } else {
  9005. u64 false_umax = opcode == BPF_JGT ? val : val - 1;
  9006. u64 true_umin = opcode == BPF_JGT ? val + 1 : val;
  9007. false_reg->umax_value = min(false_reg->umax_value, false_umax);
  9008. true_reg->umin_value = max(true_reg->umin_value, true_umin);
  9009. }
  9010. break;
  9011. }
  9012. case BPF_JSGE:
  9013. case BPF_JSGT:
  9014. {
  9015. if (is_jmp32) {
  9016. s32 false_smax = opcode == BPF_JSGT ? sval32 : sval32 - 1;
  9017. s32 true_smin = opcode == BPF_JSGT ? sval32 + 1 : sval32;
  9018. false_reg->s32_max_value = min(false_reg->s32_max_value, false_smax);
  9019. true_reg->s32_min_value = max(true_reg->s32_min_value, true_smin);
  9020. } else {
  9021. s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1;
  9022. s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;
  9023. false_reg->smax_value = min(false_reg->smax_value, false_smax);
  9024. true_reg->smin_value = max(true_reg->smin_value, true_smin);
  9025. }
  9026. break;
  9027. }
  9028. case BPF_JLE:
  9029. case BPF_JLT:
  9030. {
  9031. if (is_jmp32) {
  9032. u32 false_umin = opcode == BPF_JLT ? val32 : val32 + 1;
  9033. u32 true_umax = opcode == BPF_JLT ? val32 - 1 : val32;
  9034. false_reg->u32_min_value = max(false_reg->u32_min_value,
  9035. false_umin);
  9036. true_reg->u32_max_value = min(true_reg->u32_max_value,
  9037. true_umax);
  9038. } else {
  9039. u64 false_umin = opcode == BPF_JLT ? val : val + 1;
  9040. u64 true_umax = opcode == BPF_JLT ? val - 1 : val;
  9041. false_reg->umin_value = max(false_reg->umin_value, false_umin);
  9042. true_reg->umax_value = min(true_reg->umax_value, true_umax);
  9043. }
  9044. break;
  9045. }
  9046. case BPF_JSLE:
  9047. case BPF_JSLT:
  9048. {
  9049. if (is_jmp32) {
  9050. s32 false_smin = opcode == BPF_JSLT ? sval32 : sval32 + 1;
  9051. s32 true_smax = opcode == BPF_JSLT ? sval32 - 1 : sval32;
  9052. false_reg->s32_min_value = max(false_reg->s32_min_value, false_smin);
  9053. true_reg->s32_max_value = min(true_reg->s32_max_value, true_smax);
  9054. } else {
  9055. s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1;
  9056. s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;
  9057. false_reg->smin_value = max(false_reg->smin_value, false_smin);
  9058. true_reg->smax_value = min(true_reg->smax_value, true_smax);
  9059. }
  9060. break;
  9061. }
  9062. default:
  9063. return;
  9064. }
  9065. if (is_jmp32) {
  9066. false_reg->var_off = tnum_or(tnum_clear_subreg(false_64off),
  9067. tnum_subreg(false_32off));
  9068. true_reg->var_off = tnum_or(tnum_clear_subreg(true_64off),
  9069. tnum_subreg(true_32off));
  9070. __reg_combine_32_into_64(false_reg);
  9071. __reg_combine_32_into_64(true_reg);
  9072. } else {
  9073. false_reg->var_off = false_64off;
  9074. true_reg->var_off = true_64off;
  9075. __reg_combine_64_into_32(false_reg);
  9076. __reg_combine_64_into_32(true_reg);
  9077. }
  9078. }
  9079. /* Same as above, but for the case that dst_reg holds a constant and src_reg is
  9080. * the variable reg.
  9081. */
  9082. static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
  9083. struct bpf_reg_state *false_reg,
  9084. u64 val, u32 val32,
  9085. u8 opcode, bool is_jmp32)
  9086. {
  9087. opcode = flip_opcode(opcode);
  9088. /* This uses zero as "not present in table"; luckily the zero opcode,
  9089. * BPF_JA, can't get here.
  9090. */
  9091. if (opcode)
  9092. reg_set_min_max(true_reg, false_reg, val, val32, opcode, is_jmp32);
  9093. }
  9094. /* Regs are known to be equal, so intersect their min/max/var_off */
  9095. static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
  9096. struct bpf_reg_state *dst_reg)
  9097. {
  9098. src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
  9099. dst_reg->umin_value);
  9100. src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
  9101. dst_reg->umax_value);
  9102. src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
  9103. dst_reg->smin_value);
  9104. src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
  9105. dst_reg->smax_value);
  9106. src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
  9107. dst_reg->var_off);
  9108. reg_bounds_sync(src_reg);
  9109. reg_bounds_sync(dst_reg);
  9110. }
  9111. static void reg_combine_min_max(struct bpf_reg_state *true_src,
  9112. struct bpf_reg_state *true_dst,
  9113. struct bpf_reg_state *false_src,
  9114. struct bpf_reg_state *false_dst,
  9115. u8 opcode)
  9116. {
  9117. switch (opcode) {
  9118. case BPF_JEQ:
  9119. __reg_combine_min_max(true_src, true_dst);
  9120. break;
  9121. case BPF_JNE:
  9122. __reg_combine_min_max(false_src, false_dst);
  9123. break;
  9124. }
  9125. }
  9126. static void mark_ptr_or_null_reg(struct bpf_func_state *state,
  9127. struct bpf_reg_state *reg, u32 id,
  9128. bool is_null)
  9129. {
  9130. if (type_may_be_null(reg->type) && reg->id == id &&
  9131. !WARN_ON_ONCE(!reg->id)) {
  9132. if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
  9133. !tnum_equals_const(reg->var_off, 0) ||
  9134. reg->off)) {
  9135. /* Old offset (both fixed and variable parts) should
  9136. * have been known-zero, because we don't allow pointer
  9137. * arithmetic on pointers that might be NULL. If we
  9138. * see this happening, don't convert the register.
  9139. */
  9140. return;
  9141. }
  9142. if (is_null) {
  9143. reg->type = SCALAR_VALUE;
  9144. /* We don't need id and ref_obj_id from this point
  9145. * onwards anymore, thus we should better reset it,
  9146. * so that state pruning has chances to take effect.
  9147. */
  9148. reg->id = 0;
  9149. reg->ref_obj_id = 0;
  9150. return;
  9151. }
  9152. mark_ptr_not_null_reg(reg);
  9153. if (!reg_may_point_to_spin_lock(reg)) {
  9154. /* For not-NULL ptr, reg->ref_obj_id will be reset
  9155. * in release_reference().
  9156. *
  9157. * reg->id is still used by spin_lock ptr. Other
  9158. * than spin_lock ptr type, reg->id can be reset.
  9159. */
  9160. reg->id = 0;
  9161. }
  9162. }
  9163. }
  9164. /* The logic is similar to find_good_pkt_pointers(), both could eventually
  9165. * be folded together at some point.
  9166. */
  9167. static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
  9168. bool is_null)
  9169. {
  9170. struct bpf_func_state *state = vstate->frame[vstate->curframe];
  9171. struct bpf_reg_state *regs = state->regs, *reg;
  9172. u32 ref_obj_id = regs[regno].ref_obj_id;
  9173. u32 id = regs[regno].id;
  9174. if (ref_obj_id && ref_obj_id == id && is_null)
  9175. /* regs[regno] is in the " == NULL" branch.
  9176. * No one could have freed the reference state before
  9177. * doing the NULL check.
  9178. */
  9179. WARN_ON_ONCE(release_reference_state(state, id));
  9180. bpf_for_each_reg_in_vstate(vstate, state, reg, ({
  9181. mark_ptr_or_null_reg(state, reg, id, is_null);
  9182. }));
  9183. }
  9184. static bool try_match_pkt_pointers(const struct bpf_insn *insn,
  9185. struct bpf_reg_state *dst_reg,
  9186. struct bpf_reg_state *src_reg,
  9187. struct bpf_verifier_state *this_branch,
  9188. struct bpf_verifier_state *other_branch)
  9189. {
  9190. if (BPF_SRC(insn->code) != BPF_X)
  9191. return false;
  9192. /* Pointers are always 64-bit. */
  9193. if (BPF_CLASS(insn->code) == BPF_JMP32)
  9194. return false;
  9195. switch (BPF_OP(insn->code)) {
  9196. case BPF_JGT:
  9197. if ((dst_reg->type == PTR_TO_PACKET &&
  9198. src_reg->type == PTR_TO_PACKET_END) ||
  9199. (dst_reg->type == PTR_TO_PACKET_META &&
  9200. reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
  9201. /* pkt_data' > pkt_end, pkt_meta' > pkt_data */
  9202. find_good_pkt_pointers(this_branch, dst_reg,
  9203. dst_reg->type, false);
  9204. mark_pkt_end(other_branch, insn->dst_reg, true);
  9205. } else if ((dst_reg->type == PTR_TO_PACKET_END &&
  9206. src_reg->type == PTR_TO_PACKET) ||
  9207. (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
  9208. src_reg->type == PTR_TO_PACKET_META)) {
  9209. /* pkt_end > pkt_data', pkt_data > pkt_meta' */
  9210. find_good_pkt_pointers(other_branch, src_reg,
  9211. src_reg->type, true);
  9212. mark_pkt_end(this_branch, insn->src_reg, false);
  9213. } else {
  9214. return false;
  9215. }
  9216. break;
  9217. case BPF_JLT:
  9218. if ((dst_reg->type == PTR_TO_PACKET &&
  9219. src_reg->type == PTR_TO_PACKET_END) ||
  9220. (dst_reg->type == PTR_TO_PACKET_META &&
  9221. reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
  9222. /* pkt_data' < pkt_end, pkt_meta' < pkt_data */
  9223. find_good_pkt_pointers(other_branch, dst_reg,
  9224. dst_reg->type, true);
  9225. mark_pkt_end(this_branch, insn->dst_reg, false);
  9226. } else if ((dst_reg->type == PTR_TO_PACKET_END &&
  9227. src_reg->type == PTR_TO_PACKET) ||
  9228. (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
  9229. src_reg->type == PTR_TO_PACKET_META)) {
  9230. /* pkt_end < pkt_data', pkt_data > pkt_meta' */
  9231. find_good_pkt_pointers(this_branch, src_reg,
  9232. src_reg->type, false);
  9233. mark_pkt_end(other_branch, insn->src_reg, true);
  9234. } else {
  9235. return false;
  9236. }
  9237. break;
  9238. case BPF_JGE:
  9239. if ((dst_reg->type == PTR_TO_PACKET &&
  9240. src_reg->type == PTR_TO_PACKET_END) ||
  9241. (dst_reg->type == PTR_TO_PACKET_META &&
  9242. reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
  9243. /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
  9244. find_good_pkt_pointers(this_branch, dst_reg,
  9245. dst_reg->type, true);
  9246. mark_pkt_end(other_branch, insn->dst_reg, false);
  9247. } else if ((dst_reg->type == PTR_TO_PACKET_END &&
  9248. src_reg->type == PTR_TO_PACKET) ||
  9249. (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
  9250. src_reg->type == PTR_TO_PACKET_META)) {
  9251. /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
  9252. find_good_pkt_pointers(other_branch, src_reg,
  9253. src_reg->type, false);
  9254. mark_pkt_end(this_branch, insn->src_reg, true);
  9255. } else {
  9256. return false;
  9257. }
  9258. break;
  9259. case BPF_JLE:
  9260. if ((dst_reg->type == PTR_TO_PACKET &&
  9261. src_reg->type == PTR_TO_PACKET_END) ||
  9262. (dst_reg->type == PTR_TO_PACKET_META &&
  9263. reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
  9264. /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
  9265. find_good_pkt_pointers(other_branch, dst_reg,
  9266. dst_reg->type, false);
  9267. mark_pkt_end(this_branch, insn->dst_reg, true);
  9268. } else if ((dst_reg->type == PTR_TO_PACKET_END &&
  9269. src_reg->type == PTR_TO_PACKET) ||
  9270. (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
  9271. src_reg->type == PTR_TO_PACKET_META)) {
  9272. /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
  9273. find_good_pkt_pointers(this_branch, src_reg,
  9274. src_reg->type, true);
  9275. mark_pkt_end(other_branch, insn->src_reg, false);
  9276. } else {
  9277. return false;
  9278. }
  9279. break;
  9280. default:
  9281. return false;
  9282. }
  9283. return true;
  9284. }
  9285. static void find_equal_scalars(struct bpf_verifier_state *vstate,
  9286. struct bpf_reg_state *known_reg)
  9287. {
  9288. struct bpf_func_state *state;
  9289. struct bpf_reg_state *reg;
  9290. bpf_for_each_reg_in_vstate(vstate, state, reg, ({
  9291. if (reg->type == SCALAR_VALUE && reg->id == known_reg->id)
  9292. copy_register_state(reg, known_reg);
  9293. }));
  9294. }
  9295. static int check_cond_jmp_op(struct bpf_verifier_env *env,
  9296. struct bpf_insn *insn, int *insn_idx)
  9297. {
  9298. struct bpf_verifier_state *this_branch = env->cur_state;
  9299. struct bpf_verifier_state *other_branch;
  9300. struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
  9301. struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL;
  9302. u8 opcode = BPF_OP(insn->code);
  9303. bool is_jmp32;
  9304. int pred = -1;
  9305. int err;
  9306. /* Only conditional jumps are expected to reach here. */
  9307. if (opcode == BPF_JA || opcode > BPF_JSLE) {
  9308. verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
  9309. return -EINVAL;
  9310. }
  9311. /* check src2 operand */
  9312. err = check_reg_arg(env, insn->dst_reg, SRC_OP);
  9313. if (err)
  9314. return err;
  9315. dst_reg = &regs[insn->dst_reg];
  9316. if (BPF_SRC(insn->code) == BPF_X) {
  9317. if (insn->imm != 0) {
  9318. verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
  9319. return -EINVAL;
  9320. }
  9321. /* check src1 operand */
  9322. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  9323. if (err)
  9324. return err;
  9325. src_reg = &regs[insn->src_reg];
  9326. if (!(reg_is_pkt_pointer_any(dst_reg) && reg_is_pkt_pointer_any(src_reg)) &&
  9327. is_pointer_value(env, insn->src_reg)) {
  9328. verbose(env, "R%d pointer comparison prohibited\n",
  9329. insn->src_reg);
  9330. return -EACCES;
  9331. }
  9332. } else {
  9333. if (insn->src_reg != BPF_REG_0) {
  9334. verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
  9335. return -EINVAL;
  9336. }
  9337. }
  9338. is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
  9339. if (BPF_SRC(insn->code) == BPF_K) {
  9340. pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32);
  9341. } else if (src_reg->type == SCALAR_VALUE &&
  9342. is_jmp32 && tnum_is_const(tnum_subreg(src_reg->var_off))) {
  9343. pred = is_branch_taken(dst_reg,
  9344. tnum_subreg(src_reg->var_off).value,
  9345. opcode,
  9346. is_jmp32);
  9347. } else if (src_reg->type == SCALAR_VALUE &&
  9348. !is_jmp32 && tnum_is_const(src_reg->var_off)) {
  9349. pred = is_branch_taken(dst_reg,
  9350. src_reg->var_off.value,
  9351. opcode,
  9352. is_jmp32);
  9353. } else if (reg_is_pkt_pointer_any(dst_reg) &&
  9354. reg_is_pkt_pointer_any(src_reg) &&
  9355. !is_jmp32) {
  9356. pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode);
  9357. }
  9358. if (pred >= 0) {
  9359. /* If we get here with a dst_reg pointer type it is because
  9360. * above is_branch_taken() special cased the 0 comparison.
  9361. */
  9362. if (!__is_pointer_value(false, dst_reg))
  9363. err = mark_chain_precision(env, insn->dst_reg);
  9364. if (BPF_SRC(insn->code) == BPF_X && !err &&
  9365. !__is_pointer_value(false, src_reg))
  9366. err = mark_chain_precision(env, insn->src_reg);
  9367. if (err)
  9368. return err;
  9369. }
  9370. if (pred == 1) {
  9371. /* Only follow the goto, ignore fall-through. If needed, push
  9372. * the fall-through branch for simulation under speculative
  9373. * execution.
  9374. */
  9375. if (!env->bypass_spec_v1 &&
  9376. !sanitize_speculative_path(env, insn, *insn_idx + 1,
  9377. *insn_idx))
  9378. return -EFAULT;
  9379. if (env->log.level & BPF_LOG_LEVEL)
  9380. print_insn_state(env, this_branch->frame[this_branch->curframe]);
  9381. *insn_idx += insn->off;
  9382. return 0;
  9383. } else if (pred == 0) {
  9384. /* Only follow the fall-through branch, since that's where the
  9385. * program will go. If needed, push the goto branch for
  9386. * simulation under speculative execution.
  9387. */
  9388. if (!env->bypass_spec_v1 &&
  9389. !sanitize_speculative_path(env, insn,
  9390. *insn_idx + insn->off + 1,
  9391. *insn_idx))
  9392. return -EFAULT;
  9393. if (env->log.level & BPF_LOG_LEVEL)
  9394. print_insn_state(env, this_branch->frame[this_branch->curframe]);
  9395. return 0;
  9396. }
  9397. other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
  9398. false);
  9399. if (!other_branch)
  9400. return -EFAULT;
  9401. other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
  9402. /* detect if we are comparing against a constant value so we can adjust
  9403. * our min/max values for our dst register.
  9404. * this is only legit if both are scalars (or pointers to the same
  9405. * object, I suppose, but we don't support that right now), because
  9406. * otherwise the different base pointers mean the offsets aren't
  9407. * comparable.
  9408. */
  9409. if (BPF_SRC(insn->code) == BPF_X) {
  9410. struct bpf_reg_state *src_reg = &regs[insn->src_reg];
  9411. if (dst_reg->type == SCALAR_VALUE &&
  9412. src_reg->type == SCALAR_VALUE) {
  9413. if (tnum_is_const(src_reg->var_off) ||
  9414. (is_jmp32 &&
  9415. tnum_is_const(tnum_subreg(src_reg->var_off))))
  9416. reg_set_min_max(&other_branch_regs[insn->dst_reg],
  9417. dst_reg,
  9418. src_reg->var_off.value,
  9419. tnum_subreg(src_reg->var_off).value,
  9420. opcode, is_jmp32);
  9421. else if (tnum_is_const(dst_reg->var_off) ||
  9422. (is_jmp32 &&
  9423. tnum_is_const(tnum_subreg(dst_reg->var_off))))
  9424. reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
  9425. src_reg,
  9426. dst_reg->var_off.value,
  9427. tnum_subreg(dst_reg->var_off).value,
  9428. opcode, is_jmp32);
  9429. else if (!is_jmp32 &&
  9430. (opcode == BPF_JEQ || opcode == BPF_JNE))
  9431. /* Comparing for equality, we can combine knowledge */
  9432. reg_combine_min_max(&other_branch_regs[insn->src_reg],
  9433. &other_branch_regs[insn->dst_reg],
  9434. src_reg, dst_reg, opcode);
  9435. if (src_reg->id &&
  9436. !WARN_ON_ONCE(src_reg->id != other_branch_regs[insn->src_reg].id)) {
  9437. find_equal_scalars(this_branch, src_reg);
  9438. find_equal_scalars(other_branch, &other_branch_regs[insn->src_reg]);
  9439. }
  9440. }
  9441. } else if (dst_reg->type == SCALAR_VALUE) {
  9442. reg_set_min_max(&other_branch_regs[insn->dst_reg],
  9443. dst_reg, insn->imm, (u32)insn->imm,
  9444. opcode, is_jmp32);
  9445. }
  9446. if (dst_reg->type == SCALAR_VALUE && dst_reg->id &&
  9447. !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) {
  9448. find_equal_scalars(this_branch, dst_reg);
  9449. find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]);
  9450. }
  9451. /* detect if R == 0 where R is returned from bpf_map_lookup_elem().
  9452. * NOTE: these optimizations below are related with pointer comparison
  9453. * which will never be JMP32.
  9454. */
  9455. if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
  9456. insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
  9457. type_may_be_null(dst_reg->type)) {
  9458. /* Mark all identical registers in each branch as either
  9459. * safe or unknown depending R == 0 or R != 0 conditional.
  9460. */
  9461. mark_ptr_or_null_regs(this_branch, insn->dst_reg,
  9462. opcode == BPF_JNE);
  9463. mark_ptr_or_null_regs(other_branch, insn->dst_reg,
  9464. opcode == BPF_JEQ);
  9465. } else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
  9466. this_branch, other_branch) &&
  9467. is_pointer_value(env, insn->dst_reg)) {
  9468. verbose(env, "R%d pointer comparison prohibited\n",
  9469. insn->dst_reg);
  9470. return -EACCES;
  9471. }
  9472. if (env->log.level & BPF_LOG_LEVEL)
  9473. print_insn_state(env, this_branch->frame[this_branch->curframe]);
  9474. return 0;
  9475. }
  9476. /* verify BPF_LD_IMM64 instruction */
  9477. static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
  9478. {
  9479. struct bpf_insn_aux_data *aux = cur_aux(env);
  9480. struct bpf_reg_state *regs = cur_regs(env);
  9481. struct bpf_reg_state *dst_reg;
  9482. struct bpf_map *map;
  9483. int err;
  9484. if (BPF_SIZE(insn->code) != BPF_DW) {
  9485. verbose(env, "invalid BPF_LD_IMM insn\n");
  9486. return -EINVAL;
  9487. }
  9488. if (insn->off != 0) {
  9489. verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
  9490. return -EINVAL;
  9491. }
  9492. err = check_reg_arg(env, insn->dst_reg, DST_OP);
  9493. if (err)
  9494. return err;
  9495. dst_reg = &regs[insn->dst_reg];
  9496. if (insn->src_reg == 0) {
  9497. u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;
  9498. dst_reg->type = SCALAR_VALUE;
  9499. __mark_reg_known(&regs[insn->dst_reg], imm);
  9500. return 0;
  9501. }
  9502. /* All special src_reg cases are listed below. From this point onwards
  9503. * we either succeed and assign a corresponding dst_reg->type after
  9504. * zeroing the offset, or fail and reject the program.
  9505. */
  9506. mark_reg_known_zero(env, regs, insn->dst_reg);
  9507. if (insn->src_reg == BPF_PSEUDO_BTF_ID) {
  9508. dst_reg->type = aux->btf_var.reg_type;
  9509. switch (base_type(dst_reg->type)) {
  9510. case PTR_TO_MEM:
  9511. dst_reg->mem_size = aux->btf_var.mem_size;
  9512. break;
  9513. case PTR_TO_BTF_ID:
  9514. dst_reg->btf = aux->btf_var.btf;
  9515. dst_reg->btf_id = aux->btf_var.btf_id;
  9516. break;
  9517. default:
  9518. verbose(env, "bpf verifier is misconfigured\n");
  9519. return -EFAULT;
  9520. }
  9521. return 0;
  9522. }
  9523. if (insn->src_reg == BPF_PSEUDO_FUNC) {
  9524. struct bpf_prog_aux *aux = env->prog->aux;
  9525. u32 subprogno = find_subprog(env,
  9526. env->insn_idx + insn->imm + 1);
  9527. if (!aux->func_info) {
  9528. verbose(env, "missing btf func_info\n");
  9529. return -EINVAL;
  9530. }
  9531. if (aux->func_info_aux[subprogno].linkage != BTF_FUNC_STATIC) {
  9532. verbose(env, "callback function not static\n");
  9533. return -EINVAL;
  9534. }
  9535. dst_reg->type = PTR_TO_FUNC;
  9536. dst_reg->subprogno = subprogno;
  9537. return 0;
  9538. }
  9539. map = env->used_maps[aux->map_index];
  9540. dst_reg->map_ptr = map;
  9541. if (insn->src_reg == BPF_PSEUDO_MAP_VALUE ||
  9542. insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE) {
  9543. dst_reg->type = PTR_TO_MAP_VALUE;
  9544. dst_reg->off = aux->map_off;
  9545. if (map_value_has_spin_lock(map))
  9546. dst_reg->id = ++env->id_gen;
  9547. } else if (insn->src_reg == BPF_PSEUDO_MAP_FD ||
  9548. insn->src_reg == BPF_PSEUDO_MAP_IDX) {
  9549. dst_reg->type = CONST_PTR_TO_MAP;
  9550. } else {
  9551. verbose(env, "bpf verifier is misconfigured\n");
  9552. return -EINVAL;
  9553. }
  9554. return 0;
  9555. }
  9556. static bool may_access_skb(enum bpf_prog_type type)
  9557. {
  9558. switch (type) {
  9559. case BPF_PROG_TYPE_SOCKET_FILTER:
  9560. case BPF_PROG_TYPE_SCHED_CLS:
  9561. case BPF_PROG_TYPE_SCHED_ACT:
  9562. return true;
  9563. default:
  9564. return false;
  9565. }
  9566. }
  9567. /* verify safety of LD_ABS|LD_IND instructions:
  9568. * - they can only appear in the programs where ctx == skb
  9569. * - since they are wrappers of function calls, they scratch R1-R5 registers,
  9570. * preserve R6-R9, and store return value into R0
  9571. *
  9572. * Implicit input:
  9573. * ctx == skb == R6 == CTX
  9574. *
  9575. * Explicit input:
  9576. * SRC == any register
  9577. * IMM == 32-bit immediate
  9578. *
  9579. * Output:
  9580. * R0 - 8/16/32-bit skb data converted to cpu endianness
  9581. */
  9582. static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
  9583. {
  9584. struct bpf_reg_state *regs = cur_regs(env);
  9585. static const int ctx_reg = BPF_REG_6;
  9586. u8 mode = BPF_MODE(insn->code);
  9587. int i, err;
  9588. if (!may_access_skb(resolve_prog_type(env->prog))) {
  9589. verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
  9590. return -EINVAL;
  9591. }
  9592. if (!env->ops->gen_ld_abs) {
  9593. verbose(env, "bpf verifier is misconfigured\n");
  9594. return -EINVAL;
  9595. }
  9596. if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
  9597. BPF_SIZE(insn->code) == BPF_DW ||
  9598. (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
  9599. verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
  9600. return -EINVAL;
  9601. }
  9602. /* check whether implicit source operand (register R6) is readable */
  9603. err = check_reg_arg(env, ctx_reg, SRC_OP);
  9604. if (err)
  9605. return err;
  9606. /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
  9607. * gen_ld_abs() may terminate the program at runtime, leading to
  9608. * reference leak.
  9609. */
  9610. err = check_reference_leak(env);
  9611. if (err) {
  9612. verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
  9613. return err;
  9614. }
  9615. if (env->cur_state->active_spin_lock) {
  9616. verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
  9617. return -EINVAL;
  9618. }
  9619. if (regs[ctx_reg].type != PTR_TO_CTX) {
  9620. verbose(env,
  9621. "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
  9622. return -EINVAL;
  9623. }
  9624. if (mode == BPF_IND) {
  9625. /* check explicit source operand */
  9626. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  9627. if (err)
  9628. return err;
  9629. }
  9630. err = check_ptr_off_reg(env, &regs[ctx_reg], ctx_reg);
  9631. if (err < 0)
  9632. return err;
  9633. /* reset caller saved regs to unreadable */
  9634. for (i = 0; i < CALLER_SAVED_REGS; i++) {
  9635. mark_reg_not_init(env, regs, caller_saved[i]);
  9636. check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
  9637. }
  9638. /* mark destination R0 register as readable, since it contains
  9639. * the value fetched from the packet.
  9640. * Already marked as written above.
  9641. */
  9642. mark_reg_unknown(env, regs, BPF_REG_0);
  9643. /* ld_abs load up to 32-bit skb data. */
  9644. regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
  9645. return 0;
  9646. }
  9647. static int check_return_code(struct bpf_verifier_env *env)
  9648. {
  9649. struct tnum enforce_attach_type_range = tnum_unknown;
  9650. const struct bpf_prog *prog = env->prog;
  9651. struct bpf_reg_state *reg;
  9652. struct tnum range = tnum_range(0, 1), const_0 = tnum_const(0);
  9653. enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
  9654. int err;
  9655. struct bpf_func_state *frame = env->cur_state->frame[0];
  9656. const bool is_subprog = frame->subprogno;
  9657. /* LSM and struct_ops func-ptr's return type could be "void" */
  9658. if (!is_subprog) {
  9659. switch (prog_type) {
  9660. case BPF_PROG_TYPE_LSM:
  9661. if (prog->expected_attach_type == BPF_LSM_CGROUP)
  9662. /* See below, can be 0 or 0-1 depending on hook. */
  9663. break;
  9664. fallthrough;
  9665. case BPF_PROG_TYPE_STRUCT_OPS:
  9666. if (!prog->aux->attach_func_proto->type)
  9667. return 0;
  9668. break;
  9669. default:
  9670. break;
  9671. }
  9672. }
  9673. /* eBPF calling convention is such that R0 is used
  9674. * to return the value from eBPF program.
  9675. * Make sure that it's readable at this time
  9676. * of bpf_exit, which means that program wrote
  9677. * something into it earlier
  9678. */
  9679. err = check_reg_arg(env, BPF_REG_0, SRC_OP);
  9680. if (err)
  9681. return err;
  9682. if (is_pointer_value(env, BPF_REG_0)) {
  9683. verbose(env, "R0 leaks addr as return value\n");
  9684. return -EACCES;
  9685. }
  9686. reg = cur_regs(env) + BPF_REG_0;
  9687. if (frame->in_async_callback_fn) {
  9688. /* enforce return zero from async callbacks like timer */
  9689. if (reg->type != SCALAR_VALUE) {
  9690. verbose(env, "In async callback the register R0 is not a known value (%s)\n",
  9691. reg_type_str(env, reg->type));
  9692. return -EINVAL;
  9693. }
  9694. if (!tnum_in(const_0, reg->var_off)) {
  9695. verbose_invalid_scalar(env, reg, &const_0, "async callback", "R0");
  9696. return -EINVAL;
  9697. }
  9698. return 0;
  9699. }
  9700. if (is_subprog) {
  9701. if (reg->type != SCALAR_VALUE) {
  9702. verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
  9703. reg_type_str(env, reg->type));
  9704. return -EINVAL;
  9705. }
  9706. return 0;
  9707. }
  9708. switch (prog_type) {
  9709. case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
  9710. if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
  9711. env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
  9712. env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
  9713. env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
  9714. env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
  9715. env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME)
  9716. range = tnum_range(1, 1);
  9717. if (env->prog->expected_attach_type == BPF_CGROUP_INET4_BIND ||
  9718. env->prog->expected_attach_type == BPF_CGROUP_INET6_BIND)
  9719. range = tnum_range(0, 3);
  9720. break;
  9721. case BPF_PROG_TYPE_CGROUP_SKB:
  9722. if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
  9723. range = tnum_range(0, 3);
  9724. enforce_attach_type_range = tnum_range(2, 3);
  9725. }
  9726. break;
  9727. case BPF_PROG_TYPE_CGROUP_SOCK:
  9728. case BPF_PROG_TYPE_SOCK_OPS:
  9729. case BPF_PROG_TYPE_CGROUP_DEVICE:
  9730. case BPF_PROG_TYPE_CGROUP_SYSCTL:
  9731. case BPF_PROG_TYPE_CGROUP_SOCKOPT:
  9732. break;
  9733. case BPF_PROG_TYPE_RAW_TRACEPOINT:
  9734. if (!env->prog->aux->attach_btf_id)
  9735. return 0;
  9736. range = tnum_const(0);
  9737. break;
  9738. case BPF_PROG_TYPE_TRACING:
  9739. switch (env->prog->expected_attach_type) {
  9740. case BPF_TRACE_FENTRY:
  9741. case BPF_TRACE_FEXIT:
  9742. range = tnum_const(0);
  9743. break;
  9744. case BPF_TRACE_RAW_TP:
  9745. case BPF_MODIFY_RETURN:
  9746. return 0;
  9747. case BPF_TRACE_ITER:
  9748. break;
  9749. default:
  9750. return -ENOTSUPP;
  9751. }
  9752. break;
  9753. case BPF_PROG_TYPE_SK_LOOKUP:
  9754. range = tnum_range(SK_DROP, SK_PASS);
  9755. break;
  9756. case BPF_PROG_TYPE_LSM:
  9757. if (env->prog->expected_attach_type != BPF_LSM_CGROUP) {
  9758. /* Regular BPF_PROG_TYPE_LSM programs can return
  9759. * any value.
  9760. */
  9761. return 0;
  9762. }
  9763. if (!env->prog->aux->attach_func_proto->type) {
  9764. /* Make sure programs that attach to void
  9765. * hooks don't try to modify return value.
  9766. */
  9767. range = tnum_range(1, 1);
  9768. }
  9769. break;
  9770. case BPF_PROG_TYPE_EXT:
  9771. /* freplace program can return anything as its return value
  9772. * depends on the to-be-replaced kernel func or bpf program.
  9773. */
  9774. default:
  9775. return 0;
  9776. }
  9777. if (reg->type != SCALAR_VALUE) {
  9778. verbose(env, "At program exit the register R0 is not a known value (%s)\n",
  9779. reg_type_str(env, reg->type));
  9780. return -EINVAL;
  9781. }
  9782. if (!tnum_in(range, reg->var_off)) {
  9783. verbose_invalid_scalar(env, reg, &range, "program exit", "R0");
  9784. if (prog->expected_attach_type == BPF_LSM_CGROUP &&
  9785. prog_type == BPF_PROG_TYPE_LSM &&
  9786. !prog->aux->attach_func_proto->type)
  9787. verbose(env, "Note, BPF_LSM_CGROUP that attach to void LSM hooks can't modify return value!\n");
  9788. return -EINVAL;
  9789. }
  9790. if (!tnum_is_unknown(enforce_attach_type_range) &&
  9791. tnum_in(enforce_attach_type_range, reg->var_off))
  9792. env->prog->enforce_expected_attach_type = 1;
  9793. return 0;
  9794. }
  9795. /* non-recursive DFS pseudo code
  9796. * 1 procedure DFS-iterative(G,v):
  9797. * 2 label v as discovered
  9798. * 3 let S be a stack
  9799. * 4 S.push(v)
  9800. * 5 while S is not empty
  9801. * 6 t <- S.pop()
  9802. * 7 if t is what we're looking for:
  9803. * 8 return t
  9804. * 9 for all edges e in G.adjacentEdges(t) do
  9805. * 10 if edge e is already labelled
  9806. * 11 continue with the next edge
  9807. * 12 w <- G.adjacentVertex(t,e)
  9808. * 13 if vertex w is not discovered and not explored
  9809. * 14 label e as tree-edge
  9810. * 15 label w as discovered
  9811. * 16 S.push(w)
  9812. * 17 continue at 5
  9813. * 18 else if vertex w is discovered
  9814. * 19 label e as back-edge
  9815. * 20 else
  9816. * 21 // vertex w is explored
  9817. * 22 label e as forward- or cross-edge
  9818. * 23 label t as explored
  9819. * 24 S.pop()
  9820. *
  9821. * convention:
  9822. * 0x10 - discovered
  9823. * 0x11 - discovered and fall-through edge labelled
  9824. * 0x12 - discovered and fall-through and branch edges labelled
  9825. * 0x20 - explored
  9826. */
  9827. enum {
  9828. DISCOVERED = 0x10,
  9829. EXPLORED = 0x20,
  9830. FALLTHROUGH = 1,
  9831. BRANCH = 2,
  9832. };
  9833. static u32 state_htab_size(struct bpf_verifier_env *env)
  9834. {
  9835. return env->prog->len;
  9836. }
  9837. static struct bpf_verifier_state_list **explored_state(
  9838. struct bpf_verifier_env *env,
  9839. int idx)
  9840. {
  9841. struct bpf_verifier_state *cur = env->cur_state;
  9842. struct bpf_func_state *state = cur->frame[cur->curframe];
  9843. return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
  9844. }
  9845. static void init_explored_state(struct bpf_verifier_env *env, int idx)
  9846. {
  9847. env->insn_aux_data[idx].prune_point = true;
  9848. }
  9849. enum {
  9850. DONE_EXPLORING = 0,
  9851. KEEP_EXPLORING = 1,
  9852. };
  9853. /* t, w, e - match pseudo-code above:
  9854. * t - index of current instruction
  9855. * w - next instruction
  9856. * e - edge
  9857. */
  9858. static int push_insn(int t, int w, int e, struct bpf_verifier_env *env,
  9859. bool loop_ok)
  9860. {
  9861. int *insn_stack = env->cfg.insn_stack;
  9862. int *insn_state = env->cfg.insn_state;
  9863. if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
  9864. return DONE_EXPLORING;
  9865. if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
  9866. return DONE_EXPLORING;
  9867. if (w < 0 || w >= env->prog->len) {
  9868. verbose_linfo(env, t, "%d: ", t);
  9869. verbose(env, "jump out of range from insn %d to %d\n", t, w);
  9870. return -EINVAL;
  9871. }
  9872. if (e == BRANCH)
  9873. /* mark branch target for state pruning */
  9874. init_explored_state(env, w);
  9875. if (insn_state[w] == 0) {
  9876. /* tree-edge */
  9877. insn_state[t] = DISCOVERED | e;
  9878. insn_state[w] = DISCOVERED;
  9879. if (env->cfg.cur_stack >= env->prog->len)
  9880. return -E2BIG;
  9881. insn_stack[env->cfg.cur_stack++] = w;
  9882. return KEEP_EXPLORING;
  9883. } else if ((insn_state[w] & 0xF0) == DISCOVERED) {
  9884. if (loop_ok && env->bpf_capable)
  9885. return DONE_EXPLORING;
  9886. verbose_linfo(env, t, "%d: ", t);
  9887. verbose_linfo(env, w, "%d: ", w);
  9888. verbose(env, "back-edge from insn %d to %d\n", t, w);
  9889. return -EINVAL;
  9890. } else if (insn_state[w] == EXPLORED) {
  9891. /* forward- or cross-edge */
  9892. insn_state[t] = DISCOVERED | e;
  9893. } else {
  9894. verbose(env, "insn state internal bug\n");
  9895. return -EFAULT;
  9896. }
  9897. return DONE_EXPLORING;
  9898. }
  9899. static int visit_func_call_insn(int t, int insn_cnt,
  9900. struct bpf_insn *insns,
  9901. struct bpf_verifier_env *env,
  9902. bool visit_callee)
  9903. {
  9904. int ret;
  9905. ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
  9906. if (ret)
  9907. return ret;
  9908. if (t + 1 < insn_cnt)
  9909. init_explored_state(env, t + 1);
  9910. if (visit_callee) {
  9911. init_explored_state(env, t);
  9912. ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env,
  9913. /* It's ok to allow recursion from CFG point of
  9914. * view. __check_func_call() will do the actual
  9915. * check.
  9916. */
  9917. bpf_pseudo_func(insns + t));
  9918. }
  9919. return ret;
  9920. }
  9921. /* Visits the instruction at index t and returns one of the following:
  9922. * < 0 - an error occurred
  9923. * DONE_EXPLORING - the instruction was fully explored
  9924. * KEEP_EXPLORING - there is still work to be done before it is fully explored
  9925. */
  9926. static int visit_insn(int t, int insn_cnt, struct bpf_verifier_env *env)
  9927. {
  9928. struct bpf_insn *insns = env->prog->insnsi;
  9929. int ret;
  9930. if (bpf_pseudo_func(insns + t))
  9931. return visit_func_call_insn(t, insn_cnt, insns, env, true);
  9932. /* All non-branch instructions have a single fall-through edge. */
  9933. if (BPF_CLASS(insns[t].code) != BPF_JMP &&
  9934. BPF_CLASS(insns[t].code) != BPF_JMP32)
  9935. return push_insn(t, t + 1, FALLTHROUGH, env, false);
  9936. switch (BPF_OP(insns[t].code)) {
  9937. case BPF_EXIT:
  9938. return DONE_EXPLORING;
  9939. case BPF_CALL:
  9940. if (insns[t].imm == BPF_FUNC_timer_set_callback)
  9941. /* Mark this call insn to trigger is_state_visited() check
  9942. * before call itself is processed by __check_func_call().
  9943. * Otherwise new async state will be pushed for further
  9944. * exploration.
  9945. */
  9946. init_explored_state(env, t);
  9947. return visit_func_call_insn(t, insn_cnt, insns, env,
  9948. insns[t].src_reg == BPF_PSEUDO_CALL);
  9949. case BPF_JA:
  9950. if (BPF_SRC(insns[t].code) != BPF_K)
  9951. return -EINVAL;
  9952. /* unconditional jump with single edge */
  9953. ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env,
  9954. true);
  9955. if (ret)
  9956. return ret;
  9957. /* unconditional jmp is not a good pruning point,
  9958. * but it's marked, since backtracking needs
  9959. * to record jmp history in is_state_visited().
  9960. */
  9961. init_explored_state(env, t + insns[t].off + 1);
  9962. /* tell verifier to check for equivalent states
  9963. * after every call and jump
  9964. */
  9965. if (t + 1 < insn_cnt)
  9966. init_explored_state(env, t + 1);
  9967. return ret;
  9968. default:
  9969. /* conditional jump with two edges */
  9970. init_explored_state(env, t);
  9971. ret = push_insn(t, t + 1, FALLTHROUGH, env, true);
  9972. if (ret)
  9973. return ret;
  9974. return push_insn(t, t + insns[t].off + 1, BRANCH, env, true);
  9975. }
  9976. }
  9977. /* non-recursive depth-first-search to detect loops in BPF program
  9978. * loop == back-edge in directed graph
  9979. */
  9980. static int check_cfg(struct bpf_verifier_env *env)
  9981. {
  9982. int insn_cnt = env->prog->len;
  9983. int *insn_stack, *insn_state;
  9984. int ret = 0;
  9985. int i;
  9986. insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
  9987. if (!insn_state)
  9988. return -ENOMEM;
  9989. insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
  9990. if (!insn_stack) {
  9991. kvfree(insn_state);
  9992. return -ENOMEM;
  9993. }
  9994. insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
  9995. insn_stack[0] = 0; /* 0 is the first instruction */
  9996. env->cfg.cur_stack = 1;
  9997. while (env->cfg.cur_stack > 0) {
  9998. int t = insn_stack[env->cfg.cur_stack - 1];
  9999. ret = visit_insn(t, insn_cnt, env);
  10000. switch (ret) {
  10001. case DONE_EXPLORING:
  10002. insn_state[t] = EXPLORED;
  10003. env->cfg.cur_stack--;
  10004. break;
  10005. case KEEP_EXPLORING:
  10006. break;
  10007. default:
  10008. if (ret > 0) {
  10009. verbose(env, "visit_insn internal bug\n");
  10010. ret = -EFAULT;
  10011. }
  10012. goto err_free;
  10013. }
  10014. }
  10015. if (env->cfg.cur_stack < 0) {
  10016. verbose(env, "pop stack internal bug\n");
  10017. ret = -EFAULT;
  10018. goto err_free;
  10019. }
  10020. for (i = 0; i < insn_cnt; i++) {
  10021. if (insn_state[i] != EXPLORED) {
  10022. verbose(env, "unreachable insn %d\n", i);
  10023. ret = -EINVAL;
  10024. goto err_free;
  10025. }
  10026. }
  10027. ret = 0; /* cfg looks good */
  10028. err_free:
  10029. kvfree(insn_state);
  10030. kvfree(insn_stack);
  10031. env->cfg.insn_state = env->cfg.insn_stack = NULL;
  10032. return ret;
  10033. }
  10034. static int check_abnormal_return(struct bpf_verifier_env *env)
  10035. {
  10036. int i;
  10037. for (i = 1; i < env->subprog_cnt; i++) {
  10038. if (env->subprog_info[i].has_ld_abs) {
  10039. verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
  10040. return -EINVAL;
  10041. }
  10042. if (env->subprog_info[i].has_tail_call) {
  10043. verbose(env, "tail_call is not allowed in subprogs without BTF\n");
  10044. return -EINVAL;
  10045. }
  10046. }
  10047. return 0;
  10048. }
  10049. /* The minimum supported BTF func info size */
  10050. #define MIN_BPF_FUNCINFO_SIZE 8
  10051. #define MAX_FUNCINFO_REC_SIZE 252
  10052. static int check_btf_func(struct bpf_verifier_env *env,
  10053. const union bpf_attr *attr,
  10054. bpfptr_t uattr)
  10055. {
  10056. const struct btf_type *type, *func_proto, *ret_type;
  10057. u32 i, nfuncs, urec_size, min_size;
  10058. u32 krec_size = sizeof(struct bpf_func_info);
  10059. struct bpf_func_info *krecord;
  10060. struct bpf_func_info_aux *info_aux = NULL;
  10061. struct bpf_prog *prog;
  10062. const struct btf *btf;
  10063. bpfptr_t urecord;
  10064. u32 prev_offset = 0;
  10065. bool scalar_return;
  10066. int ret = -ENOMEM;
  10067. nfuncs = attr->func_info_cnt;
  10068. if (!nfuncs) {
  10069. if (check_abnormal_return(env))
  10070. return -EINVAL;
  10071. return 0;
  10072. }
  10073. if (nfuncs != env->subprog_cnt) {
  10074. verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
  10075. return -EINVAL;
  10076. }
  10077. urec_size = attr->func_info_rec_size;
  10078. if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
  10079. urec_size > MAX_FUNCINFO_REC_SIZE ||
  10080. urec_size % sizeof(u32)) {
  10081. verbose(env, "invalid func info rec size %u\n", urec_size);
  10082. return -EINVAL;
  10083. }
  10084. prog = env->prog;
  10085. btf = prog->aux->btf;
  10086. urecord = make_bpfptr(attr->func_info, uattr.is_kernel);
  10087. min_size = min_t(u32, krec_size, urec_size);
  10088. krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
  10089. if (!krecord)
  10090. return -ENOMEM;
  10091. info_aux = kcalloc(nfuncs, sizeof(*info_aux), GFP_KERNEL | __GFP_NOWARN);
  10092. if (!info_aux)
  10093. goto err_free;
  10094. for (i = 0; i < nfuncs; i++) {
  10095. ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
  10096. if (ret) {
  10097. if (ret == -E2BIG) {
  10098. verbose(env, "nonzero tailing record in func info");
  10099. /* set the size kernel expects so loader can zero
  10100. * out the rest of the record.
  10101. */
  10102. if (copy_to_bpfptr_offset(uattr,
  10103. offsetof(union bpf_attr, func_info_rec_size),
  10104. &min_size, sizeof(min_size)))
  10105. ret = -EFAULT;
  10106. }
  10107. goto err_free;
  10108. }
  10109. if (copy_from_bpfptr(&krecord[i], urecord, min_size)) {
  10110. ret = -EFAULT;
  10111. goto err_free;
  10112. }
  10113. /* check insn_off */
  10114. ret = -EINVAL;
  10115. if (i == 0) {
  10116. if (krecord[i].insn_off) {
  10117. verbose(env,
  10118. "nonzero insn_off %u for the first func info record",
  10119. krecord[i].insn_off);
  10120. goto err_free;
  10121. }
  10122. } else if (krecord[i].insn_off <= prev_offset) {
  10123. verbose(env,
  10124. "same or smaller insn offset (%u) than previous func info record (%u)",
  10125. krecord[i].insn_off, prev_offset);
  10126. goto err_free;
  10127. }
  10128. if (env->subprog_info[i].start != krecord[i].insn_off) {
  10129. verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
  10130. goto err_free;
  10131. }
  10132. /* check type_id */
  10133. type = btf_type_by_id(btf, krecord[i].type_id);
  10134. if (!type || !btf_type_is_func(type)) {
  10135. verbose(env, "invalid type id %d in func info",
  10136. krecord[i].type_id);
  10137. goto err_free;
  10138. }
  10139. info_aux[i].linkage = BTF_INFO_VLEN(type->info);
  10140. func_proto = btf_type_by_id(btf, type->type);
  10141. if (unlikely(!func_proto || !btf_type_is_func_proto(func_proto)))
  10142. /* btf_func_check() already verified it during BTF load */
  10143. goto err_free;
  10144. ret_type = btf_type_skip_modifiers(btf, func_proto->type, NULL);
  10145. scalar_return =
  10146. btf_type_is_small_int(ret_type) || btf_is_any_enum(ret_type);
  10147. if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
  10148. verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
  10149. goto err_free;
  10150. }
  10151. if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
  10152. verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
  10153. goto err_free;
  10154. }
  10155. prev_offset = krecord[i].insn_off;
  10156. bpfptr_add(&urecord, urec_size);
  10157. }
  10158. prog->aux->func_info = krecord;
  10159. prog->aux->func_info_cnt = nfuncs;
  10160. prog->aux->func_info_aux = info_aux;
  10161. return 0;
  10162. err_free:
  10163. kvfree(krecord);
  10164. kfree(info_aux);
  10165. return ret;
  10166. }
  10167. static void adjust_btf_func(struct bpf_verifier_env *env)
  10168. {
  10169. struct bpf_prog_aux *aux = env->prog->aux;
  10170. int i;
  10171. if (!aux->func_info)
  10172. return;
  10173. for (i = 0; i < env->subprog_cnt; i++)
  10174. aux->func_info[i].insn_off = env->subprog_info[i].start;
  10175. }
  10176. #define MIN_BPF_LINEINFO_SIZE offsetofend(struct bpf_line_info, line_col)
  10177. #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE
  10178. static int check_btf_line(struct bpf_verifier_env *env,
  10179. const union bpf_attr *attr,
  10180. bpfptr_t uattr)
  10181. {
  10182. u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
  10183. struct bpf_subprog_info *sub;
  10184. struct bpf_line_info *linfo;
  10185. struct bpf_prog *prog;
  10186. const struct btf *btf;
  10187. bpfptr_t ulinfo;
  10188. int err;
  10189. nr_linfo = attr->line_info_cnt;
  10190. if (!nr_linfo)
  10191. return 0;
  10192. if (nr_linfo > INT_MAX / sizeof(struct bpf_line_info))
  10193. return -EINVAL;
  10194. rec_size = attr->line_info_rec_size;
  10195. if (rec_size < MIN_BPF_LINEINFO_SIZE ||
  10196. rec_size > MAX_LINEINFO_REC_SIZE ||
  10197. rec_size & (sizeof(u32) - 1))
  10198. return -EINVAL;
  10199. /* Need to zero it in case the userspace may
  10200. * pass in a smaller bpf_line_info object.
  10201. */
  10202. linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
  10203. GFP_KERNEL | __GFP_NOWARN);
  10204. if (!linfo)
  10205. return -ENOMEM;
  10206. prog = env->prog;
  10207. btf = prog->aux->btf;
  10208. s = 0;
  10209. sub = env->subprog_info;
  10210. ulinfo = make_bpfptr(attr->line_info, uattr.is_kernel);
  10211. expected_size = sizeof(struct bpf_line_info);
  10212. ncopy = min_t(u32, expected_size, rec_size);
  10213. for (i = 0; i < nr_linfo; i++) {
  10214. err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
  10215. if (err) {
  10216. if (err == -E2BIG) {
  10217. verbose(env, "nonzero tailing record in line_info");
  10218. if (copy_to_bpfptr_offset(uattr,
  10219. offsetof(union bpf_attr, line_info_rec_size),
  10220. &expected_size, sizeof(expected_size)))
  10221. err = -EFAULT;
  10222. }
  10223. goto err_free;
  10224. }
  10225. if (copy_from_bpfptr(&linfo[i], ulinfo, ncopy)) {
  10226. err = -EFAULT;
  10227. goto err_free;
  10228. }
  10229. /*
  10230. * Check insn_off to ensure
  10231. * 1) strictly increasing AND
  10232. * 2) bounded by prog->len
  10233. *
  10234. * The linfo[0].insn_off == 0 check logically falls into
  10235. * the later "missing bpf_line_info for func..." case
  10236. * because the first linfo[0].insn_off must be the
  10237. * first sub also and the first sub must have
  10238. * subprog_info[0].start == 0.
  10239. */
  10240. if ((i && linfo[i].insn_off <= prev_offset) ||
  10241. linfo[i].insn_off >= prog->len) {
  10242. verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
  10243. i, linfo[i].insn_off, prev_offset,
  10244. prog->len);
  10245. err = -EINVAL;
  10246. goto err_free;
  10247. }
  10248. if (!prog->insnsi[linfo[i].insn_off].code) {
  10249. verbose(env,
  10250. "Invalid insn code at line_info[%u].insn_off\n",
  10251. i);
  10252. err = -EINVAL;
  10253. goto err_free;
  10254. }
  10255. if (!btf_name_by_offset(btf, linfo[i].line_off) ||
  10256. !btf_name_by_offset(btf, linfo[i].file_name_off)) {
  10257. verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
  10258. err = -EINVAL;
  10259. goto err_free;
  10260. }
  10261. if (s != env->subprog_cnt) {
  10262. if (linfo[i].insn_off == sub[s].start) {
  10263. sub[s].linfo_idx = i;
  10264. s++;
  10265. } else if (sub[s].start < linfo[i].insn_off) {
  10266. verbose(env, "missing bpf_line_info for func#%u\n", s);
  10267. err = -EINVAL;
  10268. goto err_free;
  10269. }
  10270. }
  10271. prev_offset = linfo[i].insn_off;
  10272. bpfptr_add(&ulinfo, rec_size);
  10273. }
  10274. if (s != env->subprog_cnt) {
  10275. verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
  10276. env->subprog_cnt - s, s);
  10277. err = -EINVAL;
  10278. goto err_free;
  10279. }
  10280. prog->aux->linfo = linfo;
  10281. prog->aux->nr_linfo = nr_linfo;
  10282. return 0;
  10283. err_free:
  10284. kvfree(linfo);
  10285. return err;
  10286. }
  10287. #define MIN_CORE_RELO_SIZE sizeof(struct bpf_core_relo)
  10288. #define MAX_CORE_RELO_SIZE MAX_FUNCINFO_REC_SIZE
  10289. static int check_core_relo(struct bpf_verifier_env *env,
  10290. const union bpf_attr *attr,
  10291. bpfptr_t uattr)
  10292. {
  10293. u32 i, nr_core_relo, ncopy, expected_size, rec_size;
  10294. struct bpf_core_relo core_relo = {};
  10295. struct bpf_prog *prog = env->prog;
  10296. const struct btf *btf = prog->aux->btf;
  10297. struct bpf_core_ctx ctx = {
  10298. .log = &env->log,
  10299. .btf = btf,
  10300. };
  10301. bpfptr_t u_core_relo;
  10302. int err;
  10303. nr_core_relo = attr->core_relo_cnt;
  10304. if (!nr_core_relo)
  10305. return 0;
  10306. if (nr_core_relo > INT_MAX / sizeof(struct bpf_core_relo))
  10307. return -EINVAL;
  10308. rec_size = attr->core_relo_rec_size;
  10309. if (rec_size < MIN_CORE_RELO_SIZE ||
  10310. rec_size > MAX_CORE_RELO_SIZE ||
  10311. rec_size % sizeof(u32))
  10312. return -EINVAL;
  10313. u_core_relo = make_bpfptr(attr->core_relos, uattr.is_kernel);
  10314. expected_size = sizeof(struct bpf_core_relo);
  10315. ncopy = min_t(u32, expected_size, rec_size);
  10316. /* Unlike func_info and line_info, copy and apply each CO-RE
  10317. * relocation record one at a time.
  10318. */
  10319. for (i = 0; i < nr_core_relo; i++) {
  10320. /* future proofing when sizeof(bpf_core_relo) changes */
  10321. err = bpf_check_uarg_tail_zero(u_core_relo, expected_size, rec_size);
  10322. if (err) {
  10323. if (err == -E2BIG) {
  10324. verbose(env, "nonzero tailing record in core_relo");
  10325. if (copy_to_bpfptr_offset(uattr,
  10326. offsetof(union bpf_attr, core_relo_rec_size),
  10327. &expected_size, sizeof(expected_size)))
  10328. err = -EFAULT;
  10329. }
  10330. break;
  10331. }
  10332. if (copy_from_bpfptr(&core_relo, u_core_relo, ncopy)) {
  10333. err = -EFAULT;
  10334. break;
  10335. }
  10336. if (core_relo.insn_off % 8 || core_relo.insn_off / 8 >= prog->len) {
  10337. verbose(env, "Invalid core_relo[%u].insn_off:%u prog->len:%u\n",
  10338. i, core_relo.insn_off, prog->len);
  10339. err = -EINVAL;
  10340. break;
  10341. }
  10342. err = bpf_core_apply(&ctx, &core_relo, i,
  10343. &prog->insnsi[core_relo.insn_off / 8]);
  10344. if (err)
  10345. break;
  10346. bpfptr_add(&u_core_relo, rec_size);
  10347. }
  10348. return err;
  10349. }
  10350. static int check_btf_info(struct bpf_verifier_env *env,
  10351. const union bpf_attr *attr,
  10352. bpfptr_t uattr)
  10353. {
  10354. struct btf *btf;
  10355. int err;
  10356. if (!attr->func_info_cnt && !attr->line_info_cnt) {
  10357. if (check_abnormal_return(env))
  10358. return -EINVAL;
  10359. return 0;
  10360. }
  10361. btf = btf_get_by_fd(attr->prog_btf_fd);
  10362. if (IS_ERR(btf))
  10363. return PTR_ERR(btf);
  10364. if (btf_is_kernel(btf)) {
  10365. btf_put(btf);
  10366. return -EACCES;
  10367. }
  10368. env->prog->aux->btf = btf;
  10369. err = check_btf_func(env, attr, uattr);
  10370. if (err)
  10371. return err;
  10372. err = check_btf_line(env, attr, uattr);
  10373. if (err)
  10374. return err;
  10375. err = check_core_relo(env, attr, uattr);
  10376. if (err)
  10377. return err;
  10378. return 0;
  10379. }
  10380. /* check %cur's range satisfies %old's */
  10381. static bool range_within(struct bpf_reg_state *old,
  10382. struct bpf_reg_state *cur)
  10383. {
  10384. return old->umin_value <= cur->umin_value &&
  10385. old->umax_value >= cur->umax_value &&
  10386. old->smin_value <= cur->smin_value &&
  10387. old->smax_value >= cur->smax_value &&
  10388. old->u32_min_value <= cur->u32_min_value &&
  10389. old->u32_max_value >= cur->u32_max_value &&
  10390. old->s32_min_value <= cur->s32_min_value &&
  10391. old->s32_max_value >= cur->s32_max_value;
  10392. }
  10393. /* If in the old state two registers had the same id, then they need to have
  10394. * the same id in the new state as well. But that id could be different from
  10395. * the old state, so we need to track the mapping from old to new ids.
  10396. * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
  10397. * regs with old id 5 must also have new id 9 for the new state to be safe. But
  10398. * regs with a different old id could still have new id 9, we don't care about
  10399. * that.
  10400. * So we look through our idmap to see if this old id has been seen before. If
  10401. * so, we require the new id to match; otherwise, we add the id pair to the map.
  10402. */
  10403. static bool check_ids(u32 old_id, u32 cur_id, struct bpf_id_pair *idmap)
  10404. {
  10405. unsigned int i;
  10406. for (i = 0; i < BPF_ID_MAP_SIZE; i++) {
  10407. if (!idmap[i].old) {
  10408. /* Reached an empty slot; haven't seen this id before */
  10409. idmap[i].old = old_id;
  10410. idmap[i].cur = cur_id;
  10411. return true;
  10412. }
  10413. if (idmap[i].old == old_id)
  10414. return idmap[i].cur == cur_id;
  10415. }
  10416. /* We ran out of idmap slots, which should be impossible */
  10417. WARN_ON_ONCE(1);
  10418. return false;
  10419. }
  10420. static void clean_func_state(struct bpf_verifier_env *env,
  10421. struct bpf_func_state *st)
  10422. {
  10423. enum bpf_reg_liveness live;
  10424. int i, j;
  10425. for (i = 0; i < BPF_REG_FP; i++) {
  10426. live = st->regs[i].live;
  10427. /* liveness must not touch this register anymore */
  10428. st->regs[i].live |= REG_LIVE_DONE;
  10429. if (!(live & REG_LIVE_READ))
  10430. /* since the register is unused, clear its state
  10431. * to make further comparison simpler
  10432. */
  10433. __mark_reg_not_init(env, &st->regs[i]);
  10434. }
  10435. for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
  10436. live = st->stack[i].spilled_ptr.live;
  10437. /* liveness must not touch this stack slot anymore */
  10438. st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
  10439. if (!(live & REG_LIVE_READ)) {
  10440. __mark_reg_not_init(env, &st->stack[i].spilled_ptr);
  10441. for (j = 0; j < BPF_REG_SIZE; j++)
  10442. st->stack[i].slot_type[j] = STACK_INVALID;
  10443. }
  10444. }
  10445. }
  10446. static void clean_verifier_state(struct bpf_verifier_env *env,
  10447. struct bpf_verifier_state *st)
  10448. {
  10449. int i;
  10450. if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
  10451. /* all regs in this state in all frames were already marked */
  10452. return;
  10453. for (i = 0; i <= st->curframe; i++)
  10454. clean_func_state(env, st->frame[i]);
  10455. }
  10456. /* the parentage chains form a tree.
  10457. * the verifier states are added to state lists at given insn and
  10458. * pushed into state stack for future exploration.
  10459. * when the verifier reaches bpf_exit insn some of the verifer states
  10460. * stored in the state lists have their final liveness state already,
  10461. * but a lot of states will get revised from liveness point of view when
  10462. * the verifier explores other branches.
  10463. * Example:
  10464. * 1: r0 = 1
  10465. * 2: if r1 == 100 goto pc+1
  10466. * 3: r0 = 2
  10467. * 4: exit
  10468. * when the verifier reaches exit insn the register r0 in the state list of
  10469. * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
  10470. * of insn 2 and goes exploring further. At the insn 4 it will walk the
  10471. * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
  10472. *
  10473. * Since the verifier pushes the branch states as it sees them while exploring
  10474. * the program the condition of walking the branch instruction for the second
  10475. * time means that all states below this branch were already explored and
  10476. * their final liveness marks are already propagated.
  10477. * Hence when the verifier completes the search of state list in is_state_visited()
  10478. * we can call this clean_live_states() function to mark all liveness states
  10479. * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
  10480. * will not be used.
  10481. * This function also clears the registers and stack for states that !READ
  10482. * to simplify state merging.
  10483. *
  10484. * Important note here that walking the same branch instruction in the callee
  10485. * doesn't meant that the states are DONE. The verifier has to compare
  10486. * the callsites
  10487. */
  10488. static void clean_live_states(struct bpf_verifier_env *env, int insn,
  10489. struct bpf_verifier_state *cur)
  10490. {
  10491. struct bpf_verifier_state_list *sl;
  10492. int i;
  10493. sl = *explored_state(env, insn);
  10494. while (sl) {
  10495. if (sl->state.branches)
  10496. goto next;
  10497. if (sl->state.insn_idx != insn ||
  10498. sl->state.curframe != cur->curframe)
  10499. goto next;
  10500. for (i = 0; i <= cur->curframe; i++)
  10501. if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
  10502. goto next;
  10503. clean_verifier_state(env, &sl->state);
  10504. next:
  10505. sl = sl->next;
  10506. }
  10507. }
  10508. /* Returns true if (rold safe implies rcur safe) */
  10509. static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold,
  10510. struct bpf_reg_state *rcur, struct bpf_id_pair *idmap)
  10511. {
  10512. bool equal;
  10513. if (!(rold->live & REG_LIVE_READ))
  10514. /* explored state didn't use this */
  10515. return true;
  10516. equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
  10517. if (rold->type == PTR_TO_STACK)
  10518. /* two stack pointers are equal only if they're pointing to
  10519. * the same stack frame, since fp-8 in foo != fp-8 in bar
  10520. */
  10521. return equal && rold->frameno == rcur->frameno;
  10522. if (equal)
  10523. return true;
  10524. if (rold->type == NOT_INIT)
  10525. /* explored state can't have used this */
  10526. return true;
  10527. if (rcur->type == NOT_INIT)
  10528. return false;
  10529. switch (base_type(rold->type)) {
  10530. case SCALAR_VALUE:
  10531. if (env->explore_alu_limits)
  10532. return false;
  10533. if (rcur->type == SCALAR_VALUE) {
  10534. if (!rold->precise)
  10535. return true;
  10536. /* new val must satisfy old val knowledge */
  10537. return range_within(rold, rcur) &&
  10538. tnum_in(rold->var_off, rcur->var_off);
  10539. } else {
  10540. /* We're trying to use a pointer in place of a scalar.
  10541. * Even if the scalar was unbounded, this could lead to
  10542. * pointer leaks because scalars are allowed to leak
  10543. * while pointers are not. We could make this safe in
  10544. * special cases if root is calling us, but it's
  10545. * probably not worth the hassle.
  10546. */
  10547. return false;
  10548. }
  10549. case PTR_TO_MAP_KEY:
  10550. case PTR_TO_MAP_VALUE:
  10551. /* a PTR_TO_MAP_VALUE could be safe to use as a
  10552. * PTR_TO_MAP_VALUE_OR_NULL into the same map.
  10553. * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
  10554. * checked, doing so could have affected others with the same
  10555. * id, and we can't check for that because we lost the id when
  10556. * we converted to a PTR_TO_MAP_VALUE.
  10557. */
  10558. if (type_may_be_null(rold->type)) {
  10559. if (!type_may_be_null(rcur->type))
  10560. return false;
  10561. if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
  10562. return false;
  10563. /* Check our ids match any regs they're supposed to */
  10564. return check_ids(rold->id, rcur->id, idmap);
  10565. }
  10566. /* If the new min/max/var_off satisfy the old ones and
  10567. * everything else matches, we are OK.
  10568. * 'id' is not compared, since it's only used for maps with
  10569. * bpf_spin_lock inside map element and in such cases if
  10570. * the rest of the prog is valid for one map element then
  10571. * it's valid for all map elements regardless of the key
  10572. * used in bpf_map_lookup()
  10573. */
  10574. return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
  10575. range_within(rold, rcur) &&
  10576. tnum_in(rold->var_off, rcur->var_off);
  10577. case PTR_TO_PACKET_META:
  10578. case PTR_TO_PACKET:
  10579. if (rcur->type != rold->type)
  10580. return false;
  10581. /* We must have at least as much range as the old ptr
  10582. * did, so that any accesses which were safe before are
  10583. * still safe. This is true even if old range < old off,
  10584. * since someone could have accessed through (ptr - k), or
  10585. * even done ptr -= k in a register, to get a safe access.
  10586. */
  10587. if (rold->range > rcur->range)
  10588. return false;
  10589. /* If the offsets don't match, we can't trust our alignment;
  10590. * nor can we be sure that we won't fall out of range.
  10591. */
  10592. if (rold->off != rcur->off)
  10593. return false;
  10594. /* id relations must be preserved */
  10595. if (rold->id && !check_ids(rold->id, rcur->id, idmap))
  10596. return false;
  10597. /* new val must satisfy old val knowledge */
  10598. return range_within(rold, rcur) &&
  10599. tnum_in(rold->var_off, rcur->var_off);
  10600. case PTR_TO_CTX:
  10601. case CONST_PTR_TO_MAP:
  10602. case PTR_TO_PACKET_END:
  10603. case PTR_TO_FLOW_KEYS:
  10604. case PTR_TO_SOCKET:
  10605. case PTR_TO_SOCK_COMMON:
  10606. case PTR_TO_TCP_SOCK:
  10607. case PTR_TO_XDP_SOCK:
  10608. /* Only valid matches are exact, which memcmp() above
  10609. * would have accepted
  10610. */
  10611. default:
  10612. /* Don't know what's going on, just say it's not safe */
  10613. return false;
  10614. }
  10615. /* Shouldn't get here; if we do, say it's not safe */
  10616. WARN_ON_ONCE(1);
  10617. return false;
  10618. }
  10619. static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
  10620. struct bpf_func_state *cur, struct bpf_id_pair *idmap)
  10621. {
  10622. int i, spi;
  10623. /* walk slots of the explored stack and ignore any additional
  10624. * slots in the current stack, since explored(safe) state
  10625. * didn't use them
  10626. */
  10627. for (i = 0; i < old->allocated_stack; i++) {
  10628. spi = i / BPF_REG_SIZE;
  10629. if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
  10630. i += BPF_REG_SIZE - 1;
  10631. /* explored state didn't use this */
  10632. continue;
  10633. }
  10634. if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
  10635. continue;
  10636. /* explored stack has more populated slots than current stack
  10637. * and these slots were used
  10638. */
  10639. if (i >= cur->allocated_stack)
  10640. return false;
  10641. /* if old state was safe with misc data in the stack
  10642. * it will be safe with zero-initialized stack.
  10643. * The opposite is not true
  10644. */
  10645. if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
  10646. cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
  10647. continue;
  10648. if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
  10649. cur->stack[spi].slot_type[i % BPF_REG_SIZE])
  10650. /* Ex: old explored (safe) state has STACK_SPILL in
  10651. * this stack slot, but current has STACK_MISC ->
  10652. * this verifier states are not equivalent,
  10653. * return false to continue verification of this path
  10654. */
  10655. return false;
  10656. if (i % BPF_REG_SIZE != BPF_REG_SIZE - 1)
  10657. continue;
  10658. if (!is_spilled_reg(&old->stack[spi]))
  10659. continue;
  10660. if (!regsafe(env, &old->stack[spi].spilled_ptr,
  10661. &cur->stack[spi].spilled_ptr, idmap))
  10662. /* when explored and current stack slot are both storing
  10663. * spilled registers, check that stored pointers types
  10664. * are the same as well.
  10665. * Ex: explored safe path could have stored
  10666. * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
  10667. * but current path has stored:
  10668. * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
  10669. * such verifier states are not equivalent.
  10670. * return false to continue verification of this path
  10671. */
  10672. return false;
  10673. }
  10674. return true;
  10675. }
  10676. static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
  10677. {
  10678. if (old->acquired_refs != cur->acquired_refs)
  10679. return false;
  10680. return !memcmp(old->refs, cur->refs,
  10681. sizeof(*old->refs) * old->acquired_refs);
  10682. }
  10683. /* compare two verifier states
  10684. *
  10685. * all states stored in state_list are known to be valid, since
  10686. * verifier reached 'bpf_exit' instruction through them
  10687. *
  10688. * this function is called when verifier exploring different branches of
  10689. * execution popped from the state stack. If it sees an old state that has
  10690. * more strict register state and more strict stack state then this execution
  10691. * branch doesn't need to be explored further, since verifier already
  10692. * concluded that more strict state leads to valid finish.
  10693. *
  10694. * Therefore two states are equivalent if register state is more conservative
  10695. * and explored stack state is more conservative than the current one.
  10696. * Example:
  10697. * explored current
  10698. * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
  10699. * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
  10700. *
  10701. * In other words if current stack state (one being explored) has more
  10702. * valid slots than old one that already passed validation, it means
  10703. * the verifier can stop exploring and conclude that current state is valid too
  10704. *
  10705. * Similarly with registers. If explored state has register type as invalid
  10706. * whereas register type in current state is meaningful, it means that
  10707. * the current state will reach 'bpf_exit' instruction safely
  10708. */
  10709. static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old,
  10710. struct bpf_func_state *cur)
  10711. {
  10712. int i;
  10713. memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));
  10714. for (i = 0; i < MAX_BPF_REG; i++)
  10715. if (!regsafe(env, &old->regs[i], &cur->regs[i],
  10716. env->idmap_scratch))
  10717. return false;
  10718. if (!stacksafe(env, old, cur, env->idmap_scratch))
  10719. return false;
  10720. if (!refsafe(old, cur))
  10721. return false;
  10722. return true;
  10723. }
  10724. static bool states_equal(struct bpf_verifier_env *env,
  10725. struct bpf_verifier_state *old,
  10726. struct bpf_verifier_state *cur)
  10727. {
  10728. int i;
  10729. if (old->curframe != cur->curframe)
  10730. return false;
  10731. /* Verification state from speculative execution simulation
  10732. * must never prune a non-speculative execution one.
  10733. */
  10734. if (old->speculative && !cur->speculative)
  10735. return false;
  10736. if (old->active_spin_lock != cur->active_spin_lock)
  10737. return false;
  10738. /* for states to be equal callsites have to be the same
  10739. * and all frame states need to be equivalent
  10740. */
  10741. for (i = 0; i <= old->curframe; i++) {
  10742. if (old->frame[i]->callsite != cur->frame[i]->callsite)
  10743. return false;
  10744. if (!func_states_equal(env, old->frame[i], cur->frame[i]))
  10745. return false;
  10746. }
  10747. return true;
  10748. }
  10749. /* Return 0 if no propagation happened. Return negative error code if error
  10750. * happened. Otherwise, return the propagated bit.
  10751. */
  10752. static int propagate_liveness_reg(struct bpf_verifier_env *env,
  10753. struct bpf_reg_state *reg,
  10754. struct bpf_reg_state *parent_reg)
  10755. {
  10756. u8 parent_flag = parent_reg->live & REG_LIVE_READ;
  10757. u8 flag = reg->live & REG_LIVE_READ;
  10758. int err;
  10759. /* When comes here, read flags of PARENT_REG or REG could be any of
  10760. * REG_LIVE_READ64, REG_LIVE_READ32, REG_LIVE_NONE. There is no need
  10761. * of propagation if PARENT_REG has strongest REG_LIVE_READ64.
  10762. */
  10763. if (parent_flag == REG_LIVE_READ64 ||
  10764. /* Or if there is no read flag from REG. */
  10765. !flag ||
  10766. /* Or if the read flag from REG is the same as PARENT_REG. */
  10767. parent_flag == flag)
  10768. return 0;
  10769. err = mark_reg_read(env, reg, parent_reg, flag);
  10770. if (err)
  10771. return err;
  10772. return flag;
  10773. }
  10774. /* A write screens off any subsequent reads; but write marks come from the
  10775. * straight-line code between a state and its parent. When we arrive at an
  10776. * equivalent state (jump target or such) we didn't arrive by the straight-line
  10777. * code, so read marks in the state must propagate to the parent regardless
  10778. * of the state's write marks. That's what 'parent == state->parent' comparison
  10779. * in mark_reg_read() is for.
  10780. */
  10781. static int propagate_liveness(struct bpf_verifier_env *env,
  10782. const struct bpf_verifier_state *vstate,
  10783. struct bpf_verifier_state *vparent)
  10784. {
  10785. struct bpf_reg_state *state_reg, *parent_reg;
  10786. struct bpf_func_state *state, *parent;
  10787. int i, frame, err = 0;
  10788. if (vparent->curframe != vstate->curframe) {
  10789. WARN(1, "propagate_live: parent frame %d current frame %d\n",
  10790. vparent->curframe, vstate->curframe);
  10791. return -EFAULT;
  10792. }
  10793. /* Propagate read liveness of registers... */
  10794. BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
  10795. for (frame = 0; frame <= vstate->curframe; frame++) {
  10796. parent = vparent->frame[frame];
  10797. state = vstate->frame[frame];
  10798. parent_reg = parent->regs;
  10799. state_reg = state->regs;
  10800. /* We don't need to worry about FP liveness, it's read-only */
  10801. for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) {
  10802. err = propagate_liveness_reg(env, &state_reg[i],
  10803. &parent_reg[i]);
  10804. if (err < 0)
  10805. return err;
  10806. if (err == REG_LIVE_READ64)
  10807. mark_insn_zext(env, &parent_reg[i]);
  10808. }
  10809. /* Propagate stack slots. */
  10810. for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
  10811. i < parent->allocated_stack / BPF_REG_SIZE; i++) {
  10812. parent_reg = &parent->stack[i].spilled_ptr;
  10813. state_reg = &state->stack[i].spilled_ptr;
  10814. err = propagate_liveness_reg(env, state_reg,
  10815. parent_reg);
  10816. if (err < 0)
  10817. return err;
  10818. }
  10819. }
  10820. return 0;
  10821. }
  10822. /* find precise scalars in the previous equivalent state and
  10823. * propagate them into the current state
  10824. */
  10825. static int propagate_precision(struct bpf_verifier_env *env,
  10826. const struct bpf_verifier_state *old)
  10827. {
  10828. struct bpf_reg_state *state_reg;
  10829. struct bpf_func_state *state;
  10830. int i, err = 0, fr;
  10831. for (fr = old->curframe; fr >= 0; fr--) {
  10832. state = old->frame[fr];
  10833. state_reg = state->regs;
  10834. for (i = 0; i < BPF_REG_FP; i++, state_reg++) {
  10835. if (state_reg->type != SCALAR_VALUE ||
  10836. !state_reg->precise ||
  10837. !(state_reg->live & REG_LIVE_READ))
  10838. continue;
  10839. if (env->log.level & BPF_LOG_LEVEL2)
  10840. verbose(env, "frame %d: propagating r%d\n", fr, i);
  10841. err = mark_chain_precision_frame(env, fr, i);
  10842. if (err < 0)
  10843. return err;
  10844. }
  10845. for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
  10846. if (!is_spilled_reg(&state->stack[i]))
  10847. continue;
  10848. state_reg = &state->stack[i].spilled_ptr;
  10849. if (state_reg->type != SCALAR_VALUE ||
  10850. !state_reg->precise ||
  10851. !(state_reg->live & REG_LIVE_READ))
  10852. continue;
  10853. if (env->log.level & BPF_LOG_LEVEL2)
  10854. verbose(env, "frame %d: propagating fp%d\n",
  10855. fr, (-i - 1) * BPF_REG_SIZE);
  10856. err = mark_chain_precision_stack_frame(env, fr, i);
  10857. if (err < 0)
  10858. return err;
  10859. }
  10860. }
  10861. return 0;
  10862. }
  10863. static bool states_maybe_looping(struct bpf_verifier_state *old,
  10864. struct bpf_verifier_state *cur)
  10865. {
  10866. struct bpf_func_state *fold, *fcur;
  10867. int i, fr = cur->curframe;
  10868. if (old->curframe != fr)
  10869. return false;
  10870. fold = old->frame[fr];
  10871. fcur = cur->frame[fr];
  10872. for (i = 0; i < MAX_BPF_REG; i++)
  10873. if (memcmp(&fold->regs[i], &fcur->regs[i],
  10874. offsetof(struct bpf_reg_state, parent)))
  10875. return false;
  10876. return true;
  10877. }
  10878. static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
  10879. {
  10880. struct bpf_verifier_state_list *new_sl;
  10881. struct bpf_verifier_state_list *sl, **pprev;
  10882. struct bpf_verifier_state *cur = env->cur_state, *new;
  10883. int i, j, err, states_cnt = 0;
  10884. bool add_new_state = env->test_state_freq ? true : false;
  10885. cur->last_insn_idx = env->prev_insn_idx;
  10886. if (!env->insn_aux_data[insn_idx].prune_point)
  10887. /* this 'insn_idx' instruction wasn't marked, so we will not
  10888. * be doing state search here
  10889. */
  10890. return 0;
  10891. /* bpf progs typically have pruning point every 4 instructions
  10892. * http://vger.kernel.org/bpfconf2019.html#session-1
  10893. * Do not add new state for future pruning if the verifier hasn't seen
  10894. * at least 2 jumps and at least 8 instructions.
  10895. * This heuristics helps decrease 'total_states' and 'peak_states' metric.
  10896. * In tests that amounts to up to 50% reduction into total verifier
  10897. * memory consumption and 20% verifier time speedup.
  10898. */
  10899. if (env->jmps_processed - env->prev_jmps_processed >= 2 &&
  10900. env->insn_processed - env->prev_insn_processed >= 8)
  10901. add_new_state = true;
  10902. pprev = explored_state(env, insn_idx);
  10903. sl = *pprev;
  10904. clean_live_states(env, insn_idx, cur);
  10905. while (sl) {
  10906. states_cnt++;
  10907. if (sl->state.insn_idx != insn_idx)
  10908. goto next;
  10909. if (sl->state.branches) {
  10910. struct bpf_func_state *frame = sl->state.frame[sl->state.curframe];
  10911. if (frame->in_async_callback_fn &&
  10912. frame->async_entry_cnt != cur->frame[cur->curframe]->async_entry_cnt) {
  10913. /* Different async_entry_cnt means that the verifier is
  10914. * processing another entry into async callback.
  10915. * Seeing the same state is not an indication of infinite
  10916. * loop or infinite recursion.
  10917. * But finding the same state doesn't mean that it's safe
  10918. * to stop processing the current state. The previous state
  10919. * hasn't yet reached bpf_exit, since state.branches > 0.
  10920. * Checking in_async_callback_fn alone is not enough either.
  10921. * Since the verifier still needs to catch infinite loops
  10922. * inside async callbacks.
  10923. */
  10924. } else if (states_maybe_looping(&sl->state, cur) &&
  10925. states_equal(env, &sl->state, cur)) {
  10926. verbose_linfo(env, insn_idx, "; ");
  10927. verbose(env, "infinite loop detected at insn %d\n", insn_idx);
  10928. return -EINVAL;
  10929. }
  10930. /* if the verifier is processing a loop, avoid adding new state
  10931. * too often, since different loop iterations have distinct
  10932. * states and may not help future pruning.
  10933. * This threshold shouldn't be too low to make sure that
  10934. * a loop with large bound will be rejected quickly.
  10935. * The most abusive loop will be:
  10936. * r1 += 1
  10937. * if r1 < 1000000 goto pc-2
  10938. * 1M insn_procssed limit / 100 == 10k peak states.
  10939. * This threshold shouldn't be too high either, since states
  10940. * at the end of the loop are likely to be useful in pruning.
  10941. */
  10942. if (env->jmps_processed - env->prev_jmps_processed < 20 &&
  10943. env->insn_processed - env->prev_insn_processed < 100)
  10944. add_new_state = false;
  10945. goto miss;
  10946. }
  10947. if (states_equal(env, &sl->state, cur)) {
  10948. sl->hit_cnt++;
  10949. /* reached equivalent register/stack state,
  10950. * prune the search.
  10951. * Registers read by the continuation are read by us.
  10952. * If we have any write marks in env->cur_state, they
  10953. * will prevent corresponding reads in the continuation
  10954. * from reaching our parent (an explored_state). Our
  10955. * own state will get the read marks recorded, but
  10956. * they'll be immediately forgotten as we're pruning
  10957. * this state and will pop a new one.
  10958. */
  10959. err = propagate_liveness(env, &sl->state, cur);
  10960. /* if previous state reached the exit with precision and
  10961. * current state is equivalent to it (except precsion marks)
  10962. * the precision needs to be propagated back in
  10963. * the current state.
  10964. */
  10965. err = err ? : push_jmp_history(env, cur);
  10966. err = err ? : propagate_precision(env, &sl->state);
  10967. if (err)
  10968. return err;
  10969. return 1;
  10970. }
  10971. miss:
  10972. /* when new state is not going to be added do not increase miss count.
  10973. * Otherwise several loop iterations will remove the state
  10974. * recorded earlier. The goal of these heuristics is to have
  10975. * states from some iterations of the loop (some in the beginning
  10976. * and some at the end) to help pruning.
  10977. */
  10978. if (add_new_state)
  10979. sl->miss_cnt++;
  10980. /* heuristic to determine whether this state is beneficial
  10981. * to keep checking from state equivalence point of view.
  10982. * Higher numbers increase max_states_per_insn and verification time,
  10983. * but do not meaningfully decrease insn_processed.
  10984. */
  10985. if (sl->miss_cnt > sl->hit_cnt * 3 + 3) {
  10986. /* the state is unlikely to be useful. Remove it to
  10987. * speed up verification
  10988. */
  10989. *pprev = sl->next;
  10990. if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) {
  10991. u32 br = sl->state.branches;
  10992. WARN_ONCE(br,
  10993. "BUG live_done but branches_to_explore %d\n",
  10994. br);
  10995. free_verifier_state(&sl->state, false);
  10996. kfree(sl);
  10997. env->peak_states--;
  10998. } else {
  10999. /* cannot free this state, since parentage chain may
  11000. * walk it later. Add it for free_list instead to
  11001. * be freed at the end of verification
  11002. */
  11003. sl->next = env->free_list;
  11004. env->free_list = sl;
  11005. }
  11006. sl = *pprev;
  11007. continue;
  11008. }
  11009. next:
  11010. pprev = &sl->next;
  11011. sl = *pprev;
  11012. }
  11013. if (env->max_states_per_insn < states_cnt)
  11014. env->max_states_per_insn = states_cnt;
  11015. if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
  11016. return push_jmp_history(env, cur);
  11017. if (!add_new_state)
  11018. return push_jmp_history(env, cur);
  11019. /* There were no equivalent states, remember the current one.
  11020. * Technically the current state is not proven to be safe yet,
  11021. * but it will either reach outer most bpf_exit (which means it's safe)
  11022. * or it will be rejected. When there are no loops the verifier won't be
  11023. * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
  11024. * again on the way to bpf_exit.
  11025. * When looping the sl->state.branches will be > 0 and this state
  11026. * will not be considered for equivalence until branches == 0.
  11027. */
  11028. new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
  11029. if (!new_sl)
  11030. return -ENOMEM;
  11031. env->total_states++;
  11032. env->peak_states++;
  11033. env->prev_jmps_processed = env->jmps_processed;
  11034. env->prev_insn_processed = env->insn_processed;
  11035. /* forget precise markings we inherited, see __mark_chain_precision */
  11036. if (env->bpf_capable)
  11037. mark_all_scalars_imprecise(env, cur);
  11038. /* add new state to the head of linked list */
  11039. new = &new_sl->state;
  11040. err = copy_verifier_state(new, cur);
  11041. if (err) {
  11042. free_verifier_state(new, false);
  11043. kfree(new_sl);
  11044. return err;
  11045. }
  11046. new->insn_idx = insn_idx;
  11047. WARN_ONCE(new->branches != 1,
  11048. "BUG is_state_visited:branches_to_explore=%d insn %d\n", new->branches, insn_idx);
  11049. cur->parent = new;
  11050. cur->first_insn_idx = insn_idx;
  11051. clear_jmp_history(cur);
  11052. new_sl->next = *explored_state(env, insn_idx);
  11053. *explored_state(env, insn_idx) = new_sl;
  11054. /* connect new state to parentage chain. Current frame needs all
  11055. * registers connected. Only r6 - r9 of the callers are alive (pushed
  11056. * to the stack implicitly by JITs) so in callers' frames connect just
  11057. * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
  11058. * the state of the call instruction (with WRITTEN set), and r0 comes
  11059. * from callee with its full parentage chain, anyway.
  11060. */
  11061. /* clear write marks in current state: the writes we did are not writes
  11062. * our child did, so they don't screen off its reads from us.
  11063. * (There are no read marks in current state, because reads always mark
  11064. * their parent and current state never has children yet. Only
  11065. * explored_states can get read marks.)
  11066. */
  11067. for (j = 0; j <= cur->curframe; j++) {
  11068. for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
  11069. cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
  11070. for (i = 0; i < BPF_REG_FP; i++)
  11071. cur->frame[j]->regs[i].live = REG_LIVE_NONE;
  11072. }
  11073. /* all stack frames are accessible from callee, clear them all */
  11074. for (j = 0; j <= cur->curframe; j++) {
  11075. struct bpf_func_state *frame = cur->frame[j];
  11076. struct bpf_func_state *newframe = new->frame[j];
  11077. for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
  11078. frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
  11079. frame->stack[i].spilled_ptr.parent =
  11080. &newframe->stack[i].spilled_ptr;
  11081. }
  11082. }
  11083. return 0;
  11084. }
  11085. /* Return true if it's OK to have the same insn return a different type. */
  11086. static bool reg_type_mismatch_ok(enum bpf_reg_type type)
  11087. {
  11088. switch (base_type(type)) {
  11089. case PTR_TO_CTX:
  11090. case PTR_TO_SOCKET:
  11091. case PTR_TO_SOCK_COMMON:
  11092. case PTR_TO_TCP_SOCK:
  11093. case PTR_TO_XDP_SOCK:
  11094. case PTR_TO_BTF_ID:
  11095. return false;
  11096. default:
  11097. return true;
  11098. }
  11099. }
  11100. /* If an instruction was previously used with particular pointer types, then we
  11101. * need to be careful to avoid cases such as the below, where it may be ok
  11102. * for one branch accessing the pointer, but not ok for the other branch:
  11103. *
  11104. * R1 = sock_ptr
  11105. * goto X;
  11106. * ...
  11107. * R1 = some_other_valid_ptr;
  11108. * goto X;
  11109. * ...
  11110. * R2 = *(u32 *)(R1 + 0);
  11111. */
  11112. static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
  11113. {
  11114. return src != prev && (!reg_type_mismatch_ok(src) ||
  11115. !reg_type_mismatch_ok(prev));
  11116. }
  11117. static int do_check(struct bpf_verifier_env *env)
  11118. {
  11119. bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
  11120. struct bpf_verifier_state *state = env->cur_state;
  11121. struct bpf_insn *insns = env->prog->insnsi;
  11122. struct bpf_reg_state *regs;
  11123. int insn_cnt = env->prog->len;
  11124. bool do_print_state = false;
  11125. int prev_insn_idx = -1;
  11126. for (;;) {
  11127. struct bpf_insn *insn;
  11128. u8 class;
  11129. int err;
  11130. env->prev_insn_idx = prev_insn_idx;
  11131. if (env->insn_idx >= insn_cnt) {
  11132. verbose(env, "invalid insn idx %d insn_cnt %d\n",
  11133. env->insn_idx, insn_cnt);
  11134. return -EFAULT;
  11135. }
  11136. insn = &insns[env->insn_idx];
  11137. class = BPF_CLASS(insn->code);
  11138. if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
  11139. verbose(env,
  11140. "BPF program is too large. Processed %d insn\n",
  11141. env->insn_processed);
  11142. return -E2BIG;
  11143. }
  11144. err = is_state_visited(env, env->insn_idx);
  11145. if (err < 0)
  11146. return err;
  11147. if (err == 1) {
  11148. /* found equivalent state, can prune the search */
  11149. if (env->log.level & BPF_LOG_LEVEL) {
  11150. if (do_print_state)
  11151. verbose(env, "\nfrom %d to %d%s: safe\n",
  11152. env->prev_insn_idx, env->insn_idx,
  11153. env->cur_state->speculative ?
  11154. " (speculative execution)" : "");
  11155. else
  11156. verbose(env, "%d: safe\n", env->insn_idx);
  11157. }
  11158. goto process_bpf_exit;
  11159. }
  11160. if (signal_pending(current))
  11161. return -EAGAIN;
  11162. if (need_resched())
  11163. cond_resched();
  11164. if (env->log.level & BPF_LOG_LEVEL2 && do_print_state) {
  11165. verbose(env, "\nfrom %d to %d%s:",
  11166. env->prev_insn_idx, env->insn_idx,
  11167. env->cur_state->speculative ?
  11168. " (speculative execution)" : "");
  11169. print_verifier_state(env, state->frame[state->curframe], true);
  11170. do_print_state = false;
  11171. }
  11172. if (env->log.level & BPF_LOG_LEVEL) {
  11173. const struct bpf_insn_cbs cbs = {
  11174. .cb_call = disasm_kfunc_name,
  11175. .cb_print = verbose,
  11176. .private_data = env,
  11177. };
  11178. if (verifier_state_scratched(env))
  11179. print_insn_state(env, state->frame[state->curframe]);
  11180. verbose_linfo(env, env->insn_idx, "; ");
  11181. env->prev_log_len = env->log.len_used;
  11182. verbose(env, "%d: ", env->insn_idx);
  11183. print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
  11184. env->prev_insn_print_len = env->log.len_used - env->prev_log_len;
  11185. env->prev_log_len = env->log.len_used;
  11186. }
  11187. if (bpf_prog_is_dev_bound(env->prog->aux)) {
  11188. err = bpf_prog_offload_verify_insn(env, env->insn_idx,
  11189. env->prev_insn_idx);
  11190. if (err)
  11191. return err;
  11192. }
  11193. regs = cur_regs(env);
  11194. sanitize_mark_insn_seen(env);
  11195. prev_insn_idx = env->insn_idx;
  11196. if (class == BPF_ALU || class == BPF_ALU64) {
  11197. err = check_alu_op(env, insn);
  11198. if (err)
  11199. return err;
  11200. } else if (class == BPF_LDX) {
  11201. enum bpf_reg_type *prev_src_type, src_reg_type;
  11202. /* check for reserved fields is already done */
  11203. /* check src operand */
  11204. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  11205. if (err)
  11206. return err;
  11207. err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
  11208. if (err)
  11209. return err;
  11210. src_reg_type = regs[insn->src_reg].type;
  11211. /* check that memory (src_reg + off) is readable,
  11212. * the state of dst_reg will be updated by this func
  11213. */
  11214. err = check_mem_access(env, env->insn_idx, insn->src_reg,
  11215. insn->off, BPF_SIZE(insn->code),
  11216. BPF_READ, insn->dst_reg, false);
  11217. if (err)
  11218. return err;
  11219. prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
  11220. if (*prev_src_type == NOT_INIT) {
  11221. /* saw a valid insn
  11222. * dst_reg = *(u32 *)(src_reg + off)
  11223. * save type to validate intersecting paths
  11224. */
  11225. *prev_src_type = src_reg_type;
  11226. } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
  11227. /* ABuser program is trying to use the same insn
  11228. * dst_reg = *(u32*) (src_reg + off)
  11229. * with different pointer types:
  11230. * src_reg == ctx in one branch and
  11231. * src_reg == stack|map in some other branch.
  11232. * Reject it.
  11233. */
  11234. verbose(env, "same insn cannot be used with different pointers\n");
  11235. return -EINVAL;
  11236. }
  11237. } else if (class == BPF_STX) {
  11238. enum bpf_reg_type *prev_dst_type, dst_reg_type;
  11239. if (BPF_MODE(insn->code) == BPF_ATOMIC) {
  11240. err = check_atomic(env, env->insn_idx, insn);
  11241. if (err)
  11242. return err;
  11243. env->insn_idx++;
  11244. continue;
  11245. }
  11246. if (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0) {
  11247. verbose(env, "BPF_STX uses reserved fields\n");
  11248. return -EINVAL;
  11249. }
  11250. /* check src1 operand */
  11251. err = check_reg_arg(env, insn->src_reg, SRC_OP);
  11252. if (err)
  11253. return err;
  11254. /* check src2 operand */
  11255. err = check_reg_arg(env, insn->dst_reg, SRC_OP);
  11256. if (err)
  11257. return err;
  11258. dst_reg_type = regs[insn->dst_reg].type;
  11259. /* check that memory (dst_reg + off) is writeable */
  11260. err = check_mem_access(env, env->insn_idx, insn->dst_reg,
  11261. insn->off, BPF_SIZE(insn->code),
  11262. BPF_WRITE, insn->src_reg, false);
  11263. if (err)
  11264. return err;
  11265. prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
  11266. if (*prev_dst_type == NOT_INIT) {
  11267. *prev_dst_type = dst_reg_type;
  11268. } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
  11269. verbose(env, "same insn cannot be used with different pointers\n");
  11270. return -EINVAL;
  11271. }
  11272. } else if (class == BPF_ST) {
  11273. if (BPF_MODE(insn->code) != BPF_MEM ||
  11274. insn->src_reg != BPF_REG_0) {
  11275. verbose(env, "BPF_ST uses reserved fields\n");
  11276. return -EINVAL;
  11277. }
  11278. /* check src operand */
  11279. err = check_reg_arg(env, insn->dst_reg, SRC_OP);
  11280. if (err)
  11281. return err;
  11282. if (is_ctx_reg(env, insn->dst_reg)) {
  11283. verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
  11284. insn->dst_reg,
  11285. reg_type_str(env, reg_state(env, insn->dst_reg)->type));
  11286. return -EACCES;
  11287. }
  11288. /* check that memory (dst_reg + off) is writeable */
  11289. err = check_mem_access(env, env->insn_idx, insn->dst_reg,
  11290. insn->off, BPF_SIZE(insn->code),
  11291. BPF_WRITE, -1, false);
  11292. if (err)
  11293. return err;
  11294. } else if (class == BPF_JMP || class == BPF_JMP32) {
  11295. u8 opcode = BPF_OP(insn->code);
  11296. env->jmps_processed++;
  11297. if (opcode == BPF_CALL) {
  11298. if (BPF_SRC(insn->code) != BPF_K ||
  11299. (insn->src_reg != BPF_PSEUDO_KFUNC_CALL
  11300. && insn->off != 0) ||
  11301. (insn->src_reg != BPF_REG_0 &&
  11302. insn->src_reg != BPF_PSEUDO_CALL &&
  11303. insn->src_reg != BPF_PSEUDO_KFUNC_CALL) ||
  11304. insn->dst_reg != BPF_REG_0 ||
  11305. class == BPF_JMP32) {
  11306. verbose(env, "BPF_CALL uses reserved fields\n");
  11307. return -EINVAL;
  11308. }
  11309. if (env->cur_state->active_spin_lock &&
  11310. (insn->src_reg == BPF_PSEUDO_CALL ||
  11311. insn->imm != BPF_FUNC_spin_unlock)) {
  11312. verbose(env, "function calls are not allowed while holding a lock\n");
  11313. return -EINVAL;
  11314. }
  11315. if (insn->src_reg == BPF_PSEUDO_CALL)
  11316. err = check_func_call(env, insn, &env->insn_idx);
  11317. else if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
  11318. err = check_kfunc_call(env, insn, &env->insn_idx);
  11319. else
  11320. err = check_helper_call(env, insn, &env->insn_idx);
  11321. if (err)
  11322. return err;
  11323. } else if (opcode == BPF_JA) {
  11324. if (BPF_SRC(insn->code) != BPF_K ||
  11325. insn->imm != 0 ||
  11326. insn->src_reg != BPF_REG_0 ||
  11327. insn->dst_reg != BPF_REG_0 ||
  11328. class == BPF_JMP32) {
  11329. verbose(env, "BPF_JA uses reserved fields\n");
  11330. return -EINVAL;
  11331. }
  11332. env->insn_idx += insn->off + 1;
  11333. continue;
  11334. } else if (opcode == BPF_EXIT) {
  11335. if (BPF_SRC(insn->code) != BPF_K ||
  11336. insn->imm != 0 ||
  11337. insn->src_reg != BPF_REG_0 ||
  11338. insn->dst_reg != BPF_REG_0 ||
  11339. class == BPF_JMP32) {
  11340. verbose(env, "BPF_EXIT uses reserved fields\n");
  11341. return -EINVAL;
  11342. }
  11343. if (env->cur_state->active_spin_lock) {
  11344. verbose(env, "bpf_spin_unlock is missing\n");
  11345. return -EINVAL;
  11346. }
  11347. /* We must do check_reference_leak here before
  11348. * prepare_func_exit to handle the case when
  11349. * state->curframe > 0, it may be a callback
  11350. * function, for which reference_state must
  11351. * match caller reference state when it exits.
  11352. */
  11353. err = check_reference_leak(env);
  11354. if (err)
  11355. return err;
  11356. if (state->curframe) {
  11357. /* exit from nested function */
  11358. err = prepare_func_exit(env, &env->insn_idx);
  11359. if (err)
  11360. return err;
  11361. do_print_state = true;
  11362. continue;
  11363. }
  11364. err = check_return_code(env);
  11365. if (err)
  11366. return err;
  11367. process_bpf_exit:
  11368. mark_verifier_state_scratched(env);
  11369. update_branch_counts(env, env->cur_state);
  11370. err = pop_stack(env, &prev_insn_idx,
  11371. &env->insn_idx, pop_log);
  11372. if (err < 0) {
  11373. if (err != -ENOENT)
  11374. return err;
  11375. break;
  11376. } else {
  11377. do_print_state = true;
  11378. continue;
  11379. }
  11380. } else {
  11381. err = check_cond_jmp_op(env, insn, &env->insn_idx);
  11382. if (err)
  11383. return err;
  11384. }
  11385. } else if (class == BPF_LD) {
  11386. u8 mode = BPF_MODE(insn->code);
  11387. if (mode == BPF_ABS || mode == BPF_IND) {
  11388. err = check_ld_abs(env, insn);
  11389. if (err)
  11390. return err;
  11391. } else if (mode == BPF_IMM) {
  11392. err = check_ld_imm(env, insn);
  11393. if (err)
  11394. return err;
  11395. env->insn_idx++;
  11396. sanitize_mark_insn_seen(env);
  11397. } else {
  11398. verbose(env, "invalid BPF_LD mode\n");
  11399. return -EINVAL;
  11400. }
  11401. } else {
  11402. verbose(env, "unknown insn class %d\n", class);
  11403. return -EINVAL;
  11404. }
  11405. env->insn_idx++;
  11406. }
  11407. return 0;
  11408. }
  11409. static int find_btf_percpu_datasec(struct btf *btf)
  11410. {
  11411. const struct btf_type *t;
  11412. const char *tname;
  11413. int i, n;
  11414. /*
  11415. * Both vmlinux and module each have their own ".data..percpu"
  11416. * DATASECs in BTF. So for module's case, we need to skip vmlinux BTF
  11417. * types to look at only module's own BTF types.
  11418. */
  11419. n = btf_nr_types(btf);
  11420. if (btf_is_module(btf))
  11421. i = btf_nr_types(btf_vmlinux);
  11422. else
  11423. i = 1;
  11424. for(; i < n; i++) {
  11425. t = btf_type_by_id(btf, i);
  11426. if (BTF_INFO_KIND(t->info) != BTF_KIND_DATASEC)
  11427. continue;
  11428. tname = btf_name_by_offset(btf, t->name_off);
  11429. if (!strcmp(tname, ".data..percpu"))
  11430. return i;
  11431. }
  11432. return -ENOENT;
  11433. }
  11434. /* replace pseudo btf_id with kernel symbol address */
  11435. static int check_pseudo_btf_id(struct bpf_verifier_env *env,
  11436. struct bpf_insn *insn,
  11437. struct bpf_insn_aux_data *aux)
  11438. {
  11439. const struct btf_var_secinfo *vsi;
  11440. const struct btf_type *datasec;
  11441. struct btf_mod_pair *btf_mod;
  11442. const struct btf_type *t;
  11443. const char *sym_name;
  11444. bool percpu = false;
  11445. u32 type, id = insn->imm;
  11446. struct btf *btf;
  11447. s32 datasec_id;
  11448. u64 addr;
  11449. int i, btf_fd, err;
  11450. btf_fd = insn[1].imm;
  11451. if (btf_fd) {
  11452. btf = btf_get_by_fd(btf_fd);
  11453. if (IS_ERR(btf)) {
  11454. verbose(env, "invalid module BTF object FD specified.\n");
  11455. return -EINVAL;
  11456. }
  11457. } else {
  11458. if (!btf_vmlinux) {
  11459. verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
  11460. return -EINVAL;
  11461. }
  11462. btf = btf_vmlinux;
  11463. btf_get(btf);
  11464. }
  11465. t = btf_type_by_id(btf, id);
  11466. if (!t) {
  11467. verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
  11468. err = -ENOENT;
  11469. goto err_put;
  11470. }
  11471. if (!btf_type_is_var(t)) {
  11472. verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id);
  11473. err = -EINVAL;
  11474. goto err_put;
  11475. }
  11476. sym_name = btf_name_by_offset(btf, t->name_off);
  11477. addr = kallsyms_lookup_name(sym_name);
  11478. if (!addr) {
  11479. verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n",
  11480. sym_name);
  11481. err = -ENOENT;
  11482. goto err_put;
  11483. }
  11484. datasec_id = find_btf_percpu_datasec(btf);
  11485. if (datasec_id > 0) {
  11486. datasec = btf_type_by_id(btf, datasec_id);
  11487. for_each_vsi(i, datasec, vsi) {
  11488. if (vsi->type == id) {
  11489. percpu = true;
  11490. break;
  11491. }
  11492. }
  11493. }
  11494. insn[0].imm = (u32)addr;
  11495. insn[1].imm = addr >> 32;
  11496. type = t->type;
  11497. t = btf_type_skip_modifiers(btf, type, NULL);
  11498. if (percpu) {
  11499. aux->btf_var.reg_type = PTR_TO_BTF_ID | MEM_PERCPU;
  11500. aux->btf_var.btf = btf;
  11501. aux->btf_var.btf_id = type;
  11502. } else if (!btf_type_is_struct(t)) {
  11503. const struct btf_type *ret;
  11504. const char *tname;
  11505. u32 tsize;
  11506. /* resolve the type size of ksym. */
  11507. ret = btf_resolve_size(btf, t, &tsize);
  11508. if (IS_ERR(ret)) {
  11509. tname = btf_name_by_offset(btf, t->name_off);
  11510. verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n",
  11511. tname, PTR_ERR(ret));
  11512. err = -EINVAL;
  11513. goto err_put;
  11514. }
  11515. aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
  11516. aux->btf_var.mem_size = tsize;
  11517. } else {
  11518. aux->btf_var.reg_type = PTR_TO_BTF_ID;
  11519. aux->btf_var.btf = btf;
  11520. aux->btf_var.btf_id = type;
  11521. }
  11522. /* check whether we recorded this BTF (and maybe module) already */
  11523. for (i = 0; i < env->used_btf_cnt; i++) {
  11524. if (env->used_btfs[i].btf == btf) {
  11525. btf_put(btf);
  11526. return 0;
  11527. }
  11528. }
  11529. if (env->used_btf_cnt >= MAX_USED_BTFS) {
  11530. err = -E2BIG;
  11531. goto err_put;
  11532. }
  11533. btf_mod = &env->used_btfs[env->used_btf_cnt];
  11534. btf_mod->btf = btf;
  11535. btf_mod->module = NULL;
  11536. /* if we reference variables from kernel module, bump its refcount */
  11537. if (btf_is_module(btf)) {
  11538. btf_mod->module = btf_try_get_module(btf);
  11539. if (!btf_mod->module) {
  11540. err = -ENXIO;
  11541. goto err_put;
  11542. }
  11543. }
  11544. env->used_btf_cnt++;
  11545. return 0;
  11546. err_put:
  11547. btf_put(btf);
  11548. return err;
  11549. }
  11550. static bool is_tracing_prog_type(enum bpf_prog_type type)
  11551. {
  11552. switch (type) {
  11553. case BPF_PROG_TYPE_KPROBE:
  11554. case BPF_PROG_TYPE_TRACEPOINT:
  11555. case BPF_PROG_TYPE_PERF_EVENT:
  11556. case BPF_PROG_TYPE_RAW_TRACEPOINT:
  11557. case BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE:
  11558. return true;
  11559. default:
  11560. return false;
  11561. }
  11562. }
  11563. static int check_map_prog_compatibility(struct bpf_verifier_env *env,
  11564. struct bpf_map *map,
  11565. struct bpf_prog *prog)
  11566. {
  11567. enum bpf_prog_type prog_type = resolve_prog_type(prog);
  11568. if (map_value_has_spin_lock(map)) {
  11569. if (prog_type == BPF_PROG_TYPE_SOCKET_FILTER) {
  11570. verbose(env, "socket filter progs cannot use bpf_spin_lock yet\n");
  11571. return -EINVAL;
  11572. }
  11573. if (is_tracing_prog_type(prog_type)) {
  11574. verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
  11575. return -EINVAL;
  11576. }
  11577. if (prog->aux->sleepable) {
  11578. verbose(env, "sleepable progs cannot use bpf_spin_lock yet\n");
  11579. return -EINVAL;
  11580. }
  11581. }
  11582. if (map_value_has_timer(map)) {
  11583. if (is_tracing_prog_type(prog_type)) {
  11584. verbose(env, "tracing progs cannot use bpf_timer yet\n");
  11585. return -EINVAL;
  11586. }
  11587. }
  11588. if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
  11589. !bpf_offload_prog_map_match(prog, map)) {
  11590. verbose(env, "offload device mismatch between prog and map\n");
  11591. return -EINVAL;
  11592. }
  11593. if (map->map_type == BPF_MAP_TYPE_STRUCT_OPS) {
  11594. verbose(env, "bpf_struct_ops map cannot be used in prog\n");
  11595. return -EINVAL;
  11596. }
  11597. if (prog->aux->sleepable)
  11598. switch (map->map_type) {
  11599. case BPF_MAP_TYPE_HASH:
  11600. case BPF_MAP_TYPE_LRU_HASH:
  11601. case BPF_MAP_TYPE_ARRAY:
  11602. case BPF_MAP_TYPE_PERCPU_HASH:
  11603. case BPF_MAP_TYPE_PERCPU_ARRAY:
  11604. case BPF_MAP_TYPE_LRU_PERCPU_HASH:
  11605. case BPF_MAP_TYPE_ARRAY_OF_MAPS:
  11606. case BPF_MAP_TYPE_HASH_OF_MAPS:
  11607. case BPF_MAP_TYPE_RINGBUF:
  11608. case BPF_MAP_TYPE_USER_RINGBUF:
  11609. case BPF_MAP_TYPE_INODE_STORAGE:
  11610. case BPF_MAP_TYPE_SK_STORAGE:
  11611. case BPF_MAP_TYPE_TASK_STORAGE:
  11612. break;
  11613. default:
  11614. verbose(env,
  11615. "Sleepable programs can only use array, hash, and ringbuf maps\n");
  11616. return -EINVAL;
  11617. }
  11618. return 0;
  11619. }
  11620. static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
  11621. {
  11622. return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
  11623. map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
  11624. }
  11625. /* find and rewrite pseudo imm in ld_imm64 instructions:
  11626. *
  11627. * 1. if it accesses map FD, replace it with actual map pointer.
  11628. * 2. if it accesses btf_id of a VAR, replace it with pointer to the var.
  11629. *
  11630. * NOTE: btf_vmlinux is required for converting pseudo btf_id.
  11631. */
  11632. static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
  11633. {
  11634. struct bpf_insn *insn = env->prog->insnsi;
  11635. int insn_cnt = env->prog->len;
  11636. int i, j, err;
  11637. err = bpf_prog_calc_tag(env->prog);
  11638. if (err)
  11639. return err;
  11640. for (i = 0; i < insn_cnt; i++, insn++) {
  11641. if (BPF_CLASS(insn->code) == BPF_LDX &&
  11642. (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
  11643. verbose(env, "BPF_LDX uses reserved fields\n");
  11644. return -EINVAL;
  11645. }
  11646. if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
  11647. struct bpf_insn_aux_data *aux;
  11648. struct bpf_map *map;
  11649. struct fd f;
  11650. u64 addr;
  11651. u32 fd;
  11652. if (i == insn_cnt - 1 || insn[1].code != 0 ||
  11653. insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
  11654. insn[1].off != 0) {
  11655. verbose(env, "invalid bpf_ld_imm64 insn\n");
  11656. return -EINVAL;
  11657. }
  11658. if (insn[0].src_reg == 0)
  11659. /* valid generic load 64-bit imm */
  11660. goto next_insn;
  11661. if (insn[0].src_reg == BPF_PSEUDO_BTF_ID) {
  11662. aux = &env->insn_aux_data[i];
  11663. err = check_pseudo_btf_id(env, insn, aux);
  11664. if (err)
  11665. return err;
  11666. goto next_insn;
  11667. }
  11668. if (insn[0].src_reg == BPF_PSEUDO_FUNC) {
  11669. aux = &env->insn_aux_data[i];
  11670. aux->ptr_type = PTR_TO_FUNC;
  11671. goto next_insn;
  11672. }
  11673. /* In final convert_pseudo_ld_imm64() step, this is
  11674. * converted into regular 64-bit imm load insn.
  11675. */
  11676. switch (insn[0].src_reg) {
  11677. case BPF_PSEUDO_MAP_VALUE:
  11678. case BPF_PSEUDO_MAP_IDX_VALUE:
  11679. break;
  11680. case BPF_PSEUDO_MAP_FD:
  11681. case BPF_PSEUDO_MAP_IDX:
  11682. if (insn[1].imm == 0)
  11683. break;
  11684. fallthrough;
  11685. default:
  11686. verbose(env, "unrecognized bpf_ld_imm64 insn\n");
  11687. return -EINVAL;
  11688. }
  11689. switch (insn[0].src_reg) {
  11690. case BPF_PSEUDO_MAP_IDX_VALUE:
  11691. case BPF_PSEUDO_MAP_IDX:
  11692. if (bpfptr_is_null(env->fd_array)) {
  11693. verbose(env, "fd_idx without fd_array is invalid\n");
  11694. return -EPROTO;
  11695. }
  11696. if (copy_from_bpfptr_offset(&fd, env->fd_array,
  11697. insn[0].imm * sizeof(fd),
  11698. sizeof(fd)))
  11699. return -EFAULT;
  11700. break;
  11701. default:
  11702. fd = insn[0].imm;
  11703. break;
  11704. }
  11705. f = fdget(fd);
  11706. map = __bpf_map_get(f);
  11707. if (IS_ERR(map)) {
  11708. verbose(env, "fd %d is not pointing to valid bpf_map\n",
  11709. insn[0].imm);
  11710. return PTR_ERR(map);
  11711. }
  11712. err = check_map_prog_compatibility(env, map, env->prog);
  11713. if (err) {
  11714. fdput(f);
  11715. return err;
  11716. }
  11717. aux = &env->insn_aux_data[i];
  11718. if (insn[0].src_reg == BPF_PSEUDO_MAP_FD ||
  11719. insn[0].src_reg == BPF_PSEUDO_MAP_IDX) {
  11720. addr = (unsigned long)map;
  11721. } else {
  11722. u32 off = insn[1].imm;
  11723. if (off >= BPF_MAX_VAR_OFF) {
  11724. verbose(env, "direct value offset of %u is not allowed\n", off);
  11725. fdput(f);
  11726. return -EINVAL;
  11727. }
  11728. if (!map->ops->map_direct_value_addr) {
  11729. verbose(env, "no direct value access support for this map type\n");
  11730. fdput(f);
  11731. return -EINVAL;
  11732. }
  11733. err = map->ops->map_direct_value_addr(map, &addr, off);
  11734. if (err) {
  11735. verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n",
  11736. map->value_size, off);
  11737. fdput(f);
  11738. return err;
  11739. }
  11740. aux->map_off = off;
  11741. addr += off;
  11742. }
  11743. insn[0].imm = (u32)addr;
  11744. insn[1].imm = addr >> 32;
  11745. /* check whether we recorded this map already */
  11746. for (j = 0; j < env->used_map_cnt; j++) {
  11747. if (env->used_maps[j] == map) {
  11748. aux->map_index = j;
  11749. fdput(f);
  11750. goto next_insn;
  11751. }
  11752. }
  11753. if (env->used_map_cnt >= MAX_USED_MAPS) {
  11754. fdput(f);
  11755. return -E2BIG;
  11756. }
  11757. /* hold the map. If the program is rejected by verifier,
  11758. * the map will be released by release_maps() or it
  11759. * will be used by the valid program until it's unloaded
  11760. * and all maps are released in free_used_maps()
  11761. */
  11762. bpf_map_inc(map);
  11763. aux->map_index = env->used_map_cnt;
  11764. env->used_maps[env->used_map_cnt++] = map;
  11765. if (bpf_map_is_cgroup_storage(map) &&
  11766. bpf_cgroup_storage_assign(env->prog->aux, map)) {
  11767. verbose(env, "only one cgroup storage of each type is allowed\n");
  11768. fdput(f);
  11769. return -EBUSY;
  11770. }
  11771. fdput(f);
  11772. next_insn:
  11773. insn++;
  11774. i++;
  11775. continue;
  11776. }
  11777. /* Basic sanity check before we invest more work here. */
  11778. if (!bpf_opcode_in_insntable(insn->code)) {
  11779. verbose(env, "unknown opcode %02x\n", insn->code);
  11780. return -EINVAL;
  11781. }
  11782. }
  11783. /* now all pseudo BPF_LD_IMM64 instructions load valid
  11784. * 'struct bpf_map *' into a register instead of user map_fd.
  11785. * These pointers will be used later by verifier to validate map access.
  11786. */
  11787. return 0;
  11788. }
  11789. /* drop refcnt of maps used by the rejected program */
  11790. static void release_maps(struct bpf_verifier_env *env)
  11791. {
  11792. __bpf_free_used_maps(env->prog->aux, env->used_maps,
  11793. env->used_map_cnt);
  11794. }
  11795. /* drop refcnt of maps used by the rejected program */
  11796. static void release_btfs(struct bpf_verifier_env *env)
  11797. {
  11798. __bpf_free_used_btfs(env->prog->aux, env->used_btfs,
  11799. env->used_btf_cnt);
  11800. }
  11801. /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
  11802. static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
  11803. {
  11804. struct bpf_insn *insn = env->prog->insnsi;
  11805. int insn_cnt = env->prog->len;
  11806. int i;
  11807. for (i = 0; i < insn_cnt; i++, insn++) {
  11808. if (insn->code != (BPF_LD | BPF_IMM | BPF_DW))
  11809. continue;
  11810. if (insn->src_reg == BPF_PSEUDO_FUNC)
  11811. continue;
  11812. insn->src_reg = 0;
  11813. }
  11814. }
  11815. /* single env->prog->insni[off] instruction was replaced with the range
  11816. * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying
  11817. * [0, off) and [off, end) to new locations, so the patched range stays zero
  11818. */
  11819. static void adjust_insn_aux_data(struct bpf_verifier_env *env,
  11820. struct bpf_insn_aux_data *new_data,
  11821. struct bpf_prog *new_prog, u32 off, u32 cnt)
  11822. {
  11823. struct bpf_insn_aux_data *old_data = env->insn_aux_data;
  11824. struct bpf_insn *insn = new_prog->insnsi;
  11825. u32 old_seen = old_data[off].seen;
  11826. u32 prog_len;
  11827. int i;
  11828. /* aux info at OFF always needs adjustment, no matter fast path
  11829. * (cnt == 1) is taken or not. There is no guarantee INSN at OFF is the
  11830. * original insn at old prog.
  11831. */
  11832. old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
  11833. if (cnt == 1)
  11834. return;
  11835. prog_len = new_prog->len;
  11836. memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
  11837. memcpy(new_data + off + cnt - 1, old_data + off,
  11838. sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
  11839. for (i = off; i < off + cnt - 1; i++) {
  11840. /* Expand insni[off]'s seen count to the patched range. */
  11841. new_data[i].seen = old_seen;
  11842. new_data[i].zext_dst = insn_has_def32(env, insn + i);
  11843. }
  11844. env->insn_aux_data = new_data;
  11845. vfree(old_data);
  11846. }
  11847. static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
  11848. {
  11849. int i;
  11850. if (len == 1)
  11851. return;
  11852. /* NOTE: fake 'exit' subprog should be updated as well. */
  11853. for (i = 0; i <= env->subprog_cnt; i++) {
  11854. if (env->subprog_info[i].start <= off)
  11855. continue;
  11856. env->subprog_info[i].start += len - 1;
  11857. }
  11858. }
  11859. static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
  11860. {
  11861. struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
  11862. int i, sz = prog->aux->size_poke_tab;
  11863. struct bpf_jit_poke_descriptor *desc;
  11864. for (i = 0; i < sz; i++) {
  11865. desc = &tab[i];
  11866. if (desc->insn_idx <= off)
  11867. continue;
  11868. desc->insn_idx += len - 1;
  11869. }
  11870. }
  11871. static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
  11872. const struct bpf_insn *patch, u32 len)
  11873. {
  11874. struct bpf_prog *new_prog;
  11875. struct bpf_insn_aux_data *new_data = NULL;
  11876. if (len > 1) {
  11877. new_data = vzalloc(array_size(env->prog->len + len - 1,
  11878. sizeof(struct bpf_insn_aux_data)));
  11879. if (!new_data)
  11880. return NULL;
  11881. }
  11882. new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
  11883. if (IS_ERR(new_prog)) {
  11884. if (PTR_ERR(new_prog) == -ERANGE)
  11885. verbose(env,
  11886. "insn %d cannot be patched due to 16-bit range\n",
  11887. env->insn_aux_data[off].orig_idx);
  11888. vfree(new_data);
  11889. return NULL;
  11890. }
  11891. adjust_insn_aux_data(env, new_data, new_prog, off, len);
  11892. adjust_subprog_starts(env, off, len);
  11893. adjust_poke_descs(new_prog, off, len);
  11894. return new_prog;
  11895. }
  11896. static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
  11897. u32 off, u32 cnt)
  11898. {
  11899. int i, j;
  11900. /* find first prog starting at or after off (first to remove) */
  11901. for (i = 0; i < env->subprog_cnt; i++)
  11902. if (env->subprog_info[i].start >= off)
  11903. break;
  11904. /* find first prog starting at or after off + cnt (first to stay) */
  11905. for (j = i; j < env->subprog_cnt; j++)
  11906. if (env->subprog_info[j].start >= off + cnt)
  11907. break;
  11908. /* if j doesn't start exactly at off + cnt, we are just removing
  11909. * the front of previous prog
  11910. */
  11911. if (env->subprog_info[j].start != off + cnt)
  11912. j--;
  11913. if (j > i) {
  11914. struct bpf_prog_aux *aux = env->prog->aux;
  11915. int move;
  11916. /* move fake 'exit' subprog as well */
  11917. move = env->subprog_cnt + 1 - j;
  11918. memmove(env->subprog_info + i,
  11919. env->subprog_info + j,
  11920. sizeof(*env->subprog_info) * move);
  11921. env->subprog_cnt -= j - i;
  11922. /* remove func_info */
  11923. if (aux->func_info) {
  11924. move = aux->func_info_cnt - j;
  11925. memmove(aux->func_info + i,
  11926. aux->func_info + j,
  11927. sizeof(*aux->func_info) * move);
  11928. aux->func_info_cnt -= j - i;
  11929. /* func_info->insn_off is set after all code rewrites,
  11930. * in adjust_btf_func() - no need to adjust
  11931. */
  11932. }
  11933. } else {
  11934. /* convert i from "first prog to remove" to "first to adjust" */
  11935. if (env->subprog_info[i].start == off)
  11936. i++;
  11937. }
  11938. /* update fake 'exit' subprog as well */
  11939. for (; i <= env->subprog_cnt; i++)
  11940. env->subprog_info[i].start -= cnt;
  11941. return 0;
  11942. }
  11943. static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
  11944. u32 cnt)
  11945. {
  11946. struct bpf_prog *prog = env->prog;
  11947. u32 i, l_off, l_cnt, nr_linfo;
  11948. struct bpf_line_info *linfo;
  11949. nr_linfo = prog->aux->nr_linfo;
  11950. if (!nr_linfo)
  11951. return 0;
  11952. linfo = prog->aux->linfo;
  11953. /* find first line info to remove, count lines to be removed */
  11954. for (i = 0; i < nr_linfo; i++)
  11955. if (linfo[i].insn_off >= off)
  11956. break;
  11957. l_off = i;
  11958. l_cnt = 0;
  11959. for (; i < nr_linfo; i++)
  11960. if (linfo[i].insn_off < off + cnt)
  11961. l_cnt++;
  11962. else
  11963. break;
  11964. /* First live insn doesn't match first live linfo, it needs to "inherit"
  11965. * last removed linfo. prog is already modified, so prog->len == off
  11966. * means no live instructions after (tail of the program was removed).
  11967. */
  11968. if (prog->len != off && l_cnt &&
  11969. (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
  11970. l_cnt--;
  11971. linfo[--i].insn_off = off + cnt;
  11972. }
  11973. /* remove the line info which refer to the removed instructions */
  11974. if (l_cnt) {
  11975. memmove(linfo + l_off, linfo + i,
  11976. sizeof(*linfo) * (nr_linfo - i));
  11977. prog->aux->nr_linfo -= l_cnt;
  11978. nr_linfo = prog->aux->nr_linfo;
  11979. }
  11980. /* pull all linfo[i].insn_off >= off + cnt in by cnt */
  11981. for (i = l_off; i < nr_linfo; i++)
  11982. linfo[i].insn_off -= cnt;
  11983. /* fix up all subprogs (incl. 'exit') which start >= off */
  11984. for (i = 0; i <= env->subprog_cnt; i++)
  11985. if (env->subprog_info[i].linfo_idx > l_off) {
  11986. /* program may have started in the removed region but
  11987. * may not be fully removed
  11988. */
  11989. if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
  11990. env->subprog_info[i].linfo_idx -= l_cnt;
  11991. else
  11992. env->subprog_info[i].linfo_idx = l_off;
  11993. }
  11994. return 0;
  11995. }
  11996. static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
  11997. {
  11998. struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
  11999. unsigned int orig_prog_len = env->prog->len;
  12000. int err;
  12001. if (bpf_prog_is_dev_bound(env->prog->aux))
  12002. bpf_prog_offload_remove_insns(env, off, cnt);
  12003. err = bpf_remove_insns(env->prog, off, cnt);
  12004. if (err)
  12005. return err;
  12006. err = adjust_subprog_starts_after_remove(env, off, cnt);
  12007. if (err)
  12008. return err;
  12009. err = bpf_adj_linfo_after_remove(env, off, cnt);
  12010. if (err)
  12011. return err;
  12012. memmove(aux_data + off, aux_data + off + cnt,
  12013. sizeof(*aux_data) * (orig_prog_len - off - cnt));
  12014. return 0;
  12015. }
  12016. /* The verifier does more data flow analysis than llvm and will not
  12017. * explore branches that are dead at run time. Malicious programs can
  12018. * have dead code too. Therefore replace all dead at-run-time code
  12019. * with 'ja -1'.
  12020. *
  12021. * Just nops are not optimal, e.g. if they would sit at the end of the
  12022. * program and through another bug we would manage to jump there, then
  12023. * we'd execute beyond program memory otherwise. Returning exception
  12024. * code also wouldn't work since we can have subprogs where the dead
  12025. * code could be located.
  12026. */
  12027. static void sanitize_dead_code(struct bpf_verifier_env *env)
  12028. {
  12029. struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
  12030. struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
  12031. struct bpf_insn *insn = env->prog->insnsi;
  12032. const int insn_cnt = env->prog->len;
  12033. int i;
  12034. for (i = 0; i < insn_cnt; i++) {
  12035. if (aux_data[i].seen)
  12036. continue;
  12037. memcpy(insn + i, &trap, sizeof(trap));
  12038. aux_data[i].zext_dst = false;
  12039. }
  12040. }
  12041. static bool insn_is_cond_jump(u8 code)
  12042. {
  12043. u8 op;
  12044. if (BPF_CLASS(code) == BPF_JMP32)
  12045. return true;
  12046. if (BPF_CLASS(code) != BPF_JMP)
  12047. return false;
  12048. op = BPF_OP(code);
  12049. return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
  12050. }
  12051. static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
  12052. {
  12053. struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
  12054. struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
  12055. struct bpf_insn *insn = env->prog->insnsi;
  12056. const int insn_cnt = env->prog->len;
  12057. int i;
  12058. for (i = 0; i < insn_cnt; i++, insn++) {
  12059. if (!insn_is_cond_jump(insn->code))
  12060. continue;
  12061. if (!aux_data[i + 1].seen)
  12062. ja.off = insn->off;
  12063. else if (!aux_data[i + 1 + insn->off].seen)
  12064. ja.off = 0;
  12065. else
  12066. continue;
  12067. if (bpf_prog_is_dev_bound(env->prog->aux))
  12068. bpf_prog_offload_replace_insn(env, i, &ja);
  12069. memcpy(insn, &ja, sizeof(ja));
  12070. }
  12071. }
  12072. static int opt_remove_dead_code(struct bpf_verifier_env *env)
  12073. {
  12074. struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
  12075. int insn_cnt = env->prog->len;
  12076. int i, err;
  12077. for (i = 0; i < insn_cnt; i++) {
  12078. int j;
  12079. j = 0;
  12080. while (i + j < insn_cnt && !aux_data[i + j].seen)
  12081. j++;
  12082. if (!j)
  12083. continue;
  12084. err = verifier_remove_insns(env, i, j);
  12085. if (err)
  12086. return err;
  12087. insn_cnt = env->prog->len;
  12088. }
  12089. return 0;
  12090. }
  12091. static int opt_remove_nops(struct bpf_verifier_env *env)
  12092. {
  12093. const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
  12094. struct bpf_insn *insn = env->prog->insnsi;
  12095. int insn_cnt = env->prog->len;
  12096. int i, err;
  12097. for (i = 0; i < insn_cnt; i++) {
  12098. if (memcmp(&insn[i], &ja, sizeof(ja)))
  12099. continue;
  12100. err = verifier_remove_insns(env, i, 1);
  12101. if (err)
  12102. return err;
  12103. insn_cnt--;
  12104. i--;
  12105. }
  12106. return 0;
  12107. }
  12108. static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env,
  12109. const union bpf_attr *attr)
  12110. {
  12111. struct bpf_insn *patch, zext_patch[2], rnd_hi32_patch[4];
  12112. struct bpf_insn_aux_data *aux = env->insn_aux_data;
  12113. int i, patch_len, delta = 0, len = env->prog->len;
  12114. struct bpf_insn *insns = env->prog->insnsi;
  12115. struct bpf_prog *new_prog;
  12116. bool rnd_hi32;
  12117. rnd_hi32 = attr->prog_flags & BPF_F_TEST_RND_HI32;
  12118. zext_patch[1] = BPF_ZEXT_REG(0);
  12119. rnd_hi32_patch[1] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, 0);
  12120. rnd_hi32_patch[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32);
  12121. rnd_hi32_patch[3] = BPF_ALU64_REG(BPF_OR, 0, BPF_REG_AX);
  12122. for (i = 0; i < len; i++) {
  12123. int adj_idx = i + delta;
  12124. struct bpf_insn insn;
  12125. int load_reg;
  12126. insn = insns[adj_idx];
  12127. load_reg = insn_def_regno(&insn);
  12128. if (!aux[adj_idx].zext_dst) {
  12129. u8 code, class;
  12130. u32 imm_rnd;
  12131. if (!rnd_hi32)
  12132. continue;
  12133. code = insn.code;
  12134. class = BPF_CLASS(code);
  12135. if (load_reg == -1)
  12136. continue;
  12137. /* NOTE: arg "reg" (the fourth one) is only used for
  12138. * BPF_STX + SRC_OP, so it is safe to pass NULL
  12139. * here.
  12140. */
  12141. if (is_reg64(env, &insn, load_reg, NULL, DST_OP)) {
  12142. if (class == BPF_LD &&
  12143. BPF_MODE(code) == BPF_IMM)
  12144. i++;
  12145. continue;
  12146. }
  12147. /* ctx load could be transformed into wider load. */
  12148. if (class == BPF_LDX &&
  12149. aux[adj_idx].ptr_type == PTR_TO_CTX)
  12150. continue;
  12151. imm_rnd = get_random_u32();
  12152. rnd_hi32_patch[0] = insn;
  12153. rnd_hi32_patch[1].imm = imm_rnd;
  12154. rnd_hi32_patch[3].dst_reg = load_reg;
  12155. patch = rnd_hi32_patch;
  12156. patch_len = 4;
  12157. goto apply_patch_buffer;
  12158. }
  12159. /* Add in an zero-extend instruction if a) the JIT has requested
  12160. * it or b) it's a CMPXCHG.
  12161. *
  12162. * The latter is because: BPF_CMPXCHG always loads a value into
  12163. * R0, therefore always zero-extends. However some archs'
  12164. * equivalent instruction only does this load when the
  12165. * comparison is successful. This detail of CMPXCHG is
  12166. * orthogonal to the general zero-extension behaviour of the
  12167. * CPU, so it's treated independently of bpf_jit_needs_zext.
  12168. */
  12169. if (!bpf_jit_needs_zext() && !is_cmpxchg_insn(&insn))
  12170. continue;
  12171. /* Zero-extension is done by the caller. */
  12172. if (bpf_pseudo_kfunc_call(&insn))
  12173. continue;
  12174. if (WARN_ON(load_reg == -1)) {
  12175. verbose(env, "verifier bug. zext_dst is set, but no reg is defined\n");
  12176. return -EFAULT;
  12177. }
  12178. zext_patch[0] = insn;
  12179. zext_patch[1].dst_reg = load_reg;
  12180. zext_patch[1].src_reg = load_reg;
  12181. patch = zext_patch;
  12182. patch_len = 2;
  12183. apply_patch_buffer:
  12184. new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
  12185. if (!new_prog)
  12186. return -ENOMEM;
  12187. env->prog = new_prog;
  12188. insns = new_prog->insnsi;
  12189. aux = env->insn_aux_data;
  12190. delta += patch_len - 1;
  12191. }
  12192. return 0;
  12193. }
  12194. /* convert load instructions that access fields of a context type into a
  12195. * sequence of instructions that access fields of the underlying structure:
  12196. * struct __sk_buff -> struct sk_buff
  12197. * struct bpf_sock_ops -> struct sock
  12198. */
  12199. static int convert_ctx_accesses(struct bpf_verifier_env *env)
  12200. {
  12201. const struct bpf_verifier_ops *ops = env->ops;
  12202. int i, cnt, size, ctx_field_size, delta = 0;
  12203. const int insn_cnt = env->prog->len;
  12204. struct bpf_insn insn_buf[16], *insn;
  12205. u32 target_size, size_default, off;
  12206. struct bpf_prog *new_prog;
  12207. enum bpf_access_type type;
  12208. bool is_narrower_load;
  12209. if (ops->gen_prologue || env->seen_direct_write) {
  12210. if (!ops->gen_prologue) {
  12211. verbose(env, "bpf verifier is misconfigured\n");
  12212. return -EINVAL;
  12213. }
  12214. cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
  12215. env->prog);
  12216. if (cnt >= ARRAY_SIZE(insn_buf)) {
  12217. verbose(env, "bpf verifier is misconfigured\n");
  12218. return -EINVAL;
  12219. } else if (cnt) {
  12220. new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
  12221. if (!new_prog)
  12222. return -ENOMEM;
  12223. env->prog = new_prog;
  12224. delta += cnt - 1;
  12225. }
  12226. }
  12227. if (bpf_prog_is_dev_bound(env->prog->aux))
  12228. return 0;
  12229. insn = env->prog->insnsi + delta;
  12230. for (i = 0; i < insn_cnt; i++, insn++) {
  12231. bpf_convert_ctx_access_t convert_ctx_access;
  12232. bool ctx_access;
  12233. if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
  12234. insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
  12235. insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
  12236. insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) {
  12237. type = BPF_READ;
  12238. ctx_access = true;
  12239. } else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
  12240. insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
  12241. insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
  12242. insn->code == (BPF_STX | BPF_MEM | BPF_DW) ||
  12243. insn->code == (BPF_ST | BPF_MEM | BPF_B) ||
  12244. insn->code == (BPF_ST | BPF_MEM | BPF_H) ||
  12245. insn->code == (BPF_ST | BPF_MEM | BPF_W) ||
  12246. insn->code == (BPF_ST | BPF_MEM | BPF_DW)) {
  12247. type = BPF_WRITE;
  12248. ctx_access = BPF_CLASS(insn->code) == BPF_STX;
  12249. } else {
  12250. continue;
  12251. }
  12252. if (type == BPF_WRITE &&
  12253. env->insn_aux_data[i + delta].sanitize_stack_spill) {
  12254. struct bpf_insn patch[] = {
  12255. *insn,
  12256. BPF_ST_NOSPEC(),
  12257. };
  12258. cnt = ARRAY_SIZE(patch);
  12259. new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
  12260. if (!new_prog)
  12261. return -ENOMEM;
  12262. delta += cnt - 1;
  12263. env->prog = new_prog;
  12264. insn = new_prog->insnsi + i + delta;
  12265. continue;
  12266. }
  12267. if (!ctx_access)
  12268. continue;
  12269. switch ((int)env->insn_aux_data[i + delta].ptr_type) {
  12270. case PTR_TO_CTX:
  12271. if (!ops->convert_ctx_access)
  12272. continue;
  12273. convert_ctx_access = ops->convert_ctx_access;
  12274. break;
  12275. case PTR_TO_SOCKET:
  12276. case PTR_TO_SOCK_COMMON:
  12277. convert_ctx_access = bpf_sock_convert_ctx_access;
  12278. break;
  12279. case PTR_TO_TCP_SOCK:
  12280. convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
  12281. break;
  12282. case PTR_TO_XDP_SOCK:
  12283. convert_ctx_access = bpf_xdp_sock_convert_ctx_access;
  12284. break;
  12285. case PTR_TO_BTF_ID:
  12286. case PTR_TO_BTF_ID | PTR_UNTRUSTED:
  12287. if (type == BPF_READ) {
  12288. insn->code = BPF_LDX | BPF_PROBE_MEM |
  12289. BPF_SIZE((insn)->code);
  12290. env->prog->aux->num_exentries++;
  12291. }
  12292. continue;
  12293. default:
  12294. continue;
  12295. }
  12296. ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
  12297. size = BPF_LDST_BYTES(insn);
  12298. /* If the read access is a narrower load of the field,
  12299. * convert to a 4/8-byte load, to minimum program type specific
  12300. * convert_ctx_access changes. If conversion is successful,
  12301. * we will apply proper mask to the result.
  12302. */
  12303. is_narrower_load = size < ctx_field_size;
  12304. size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
  12305. off = insn->off;
  12306. if (is_narrower_load) {
  12307. u8 size_code;
  12308. if (type == BPF_WRITE) {
  12309. verbose(env, "bpf verifier narrow ctx access misconfigured\n");
  12310. return -EINVAL;
  12311. }
  12312. size_code = BPF_H;
  12313. if (ctx_field_size == 4)
  12314. size_code = BPF_W;
  12315. else if (ctx_field_size == 8)
  12316. size_code = BPF_DW;
  12317. insn->off = off & ~(size_default - 1);
  12318. insn->code = BPF_LDX | BPF_MEM | size_code;
  12319. }
  12320. target_size = 0;
  12321. cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
  12322. &target_size);
  12323. if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
  12324. (ctx_field_size && !target_size)) {
  12325. verbose(env, "bpf verifier is misconfigured\n");
  12326. return -EINVAL;
  12327. }
  12328. if (is_narrower_load && size < target_size) {
  12329. u8 shift = bpf_ctx_narrow_access_offset(
  12330. off, size, size_default) * 8;
  12331. if (shift && cnt + 1 >= ARRAY_SIZE(insn_buf)) {
  12332. verbose(env, "bpf verifier narrow ctx load misconfigured\n");
  12333. return -EINVAL;
  12334. }
  12335. if (ctx_field_size <= 4) {
  12336. if (shift)
  12337. insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
  12338. insn->dst_reg,
  12339. shift);
  12340. insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
  12341. (1 << size * 8) - 1);
  12342. } else {
  12343. if (shift)
  12344. insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
  12345. insn->dst_reg,
  12346. shift);
  12347. insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
  12348. (1ULL << size * 8) - 1);
  12349. }
  12350. }
  12351. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  12352. if (!new_prog)
  12353. return -ENOMEM;
  12354. delta += cnt - 1;
  12355. /* keep walking new program and skip insns we just inserted */
  12356. env->prog = new_prog;
  12357. insn = new_prog->insnsi + i + delta;
  12358. }
  12359. return 0;
  12360. }
  12361. static int jit_subprogs(struct bpf_verifier_env *env)
  12362. {
  12363. struct bpf_prog *prog = env->prog, **func, *tmp;
  12364. int i, j, subprog_start, subprog_end = 0, len, subprog;
  12365. struct bpf_map *map_ptr;
  12366. struct bpf_insn *insn;
  12367. void *old_bpf_func;
  12368. int err, num_exentries;
  12369. if (env->subprog_cnt <= 1)
  12370. return 0;
  12371. for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
  12372. if (!bpf_pseudo_func(insn) && !bpf_pseudo_call(insn))
  12373. continue;
  12374. /* Upon error here we cannot fall back to interpreter but
  12375. * need a hard reject of the program. Thus -EFAULT is
  12376. * propagated in any case.
  12377. */
  12378. subprog = find_subprog(env, i + insn->imm + 1);
  12379. if (subprog < 0) {
  12380. WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
  12381. i + insn->imm + 1);
  12382. return -EFAULT;
  12383. }
  12384. /* temporarily remember subprog id inside insn instead of
  12385. * aux_data, since next loop will split up all insns into funcs
  12386. */
  12387. insn->off = subprog;
  12388. /* remember original imm in case JIT fails and fallback
  12389. * to interpreter will be needed
  12390. */
  12391. env->insn_aux_data[i].call_imm = insn->imm;
  12392. /* point imm to __bpf_call_base+1 from JITs point of view */
  12393. insn->imm = 1;
  12394. if (bpf_pseudo_func(insn))
  12395. /* jit (e.g. x86_64) may emit fewer instructions
  12396. * if it learns a u32 imm is the same as a u64 imm.
  12397. * Force a non zero here.
  12398. */
  12399. insn[1].imm = 1;
  12400. }
  12401. err = bpf_prog_alloc_jited_linfo(prog);
  12402. if (err)
  12403. goto out_undo_insn;
  12404. err = -ENOMEM;
  12405. func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
  12406. if (!func)
  12407. goto out_undo_insn;
  12408. for (i = 0; i < env->subprog_cnt; i++) {
  12409. subprog_start = subprog_end;
  12410. subprog_end = env->subprog_info[i + 1].start;
  12411. len = subprog_end - subprog_start;
  12412. /* bpf_prog_run() doesn't call subprogs directly,
  12413. * hence main prog stats include the runtime of subprogs.
  12414. * subprogs don't have IDs and not reachable via prog_get_next_id
  12415. * func[i]->stats will never be accessed and stays NULL
  12416. */
  12417. func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
  12418. if (!func[i])
  12419. goto out_free;
  12420. memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
  12421. len * sizeof(struct bpf_insn));
  12422. func[i]->type = prog->type;
  12423. func[i]->len = len;
  12424. if (bpf_prog_calc_tag(func[i]))
  12425. goto out_free;
  12426. func[i]->is_func = 1;
  12427. func[i]->aux->func_idx = i;
  12428. /* Below members will be freed only at prog->aux */
  12429. func[i]->aux->btf = prog->aux->btf;
  12430. func[i]->aux->func_info = prog->aux->func_info;
  12431. func[i]->aux->func_info_cnt = prog->aux->func_info_cnt;
  12432. func[i]->aux->poke_tab = prog->aux->poke_tab;
  12433. func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
  12434. for (j = 0; j < prog->aux->size_poke_tab; j++) {
  12435. struct bpf_jit_poke_descriptor *poke;
  12436. poke = &prog->aux->poke_tab[j];
  12437. if (poke->insn_idx < subprog_end &&
  12438. poke->insn_idx >= subprog_start)
  12439. poke->aux = func[i]->aux;
  12440. }
  12441. func[i]->aux->name[0] = 'F';
  12442. func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
  12443. func[i]->jit_requested = 1;
  12444. func[i]->blinding_requested = prog->blinding_requested;
  12445. func[i]->aux->kfunc_tab = prog->aux->kfunc_tab;
  12446. func[i]->aux->kfunc_btf_tab = prog->aux->kfunc_btf_tab;
  12447. func[i]->aux->linfo = prog->aux->linfo;
  12448. func[i]->aux->nr_linfo = prog->aux->nr_linfo;
  12449. func[i]->aux->jited_linfo = prog->aux->jited_linfo;
  12450. func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
  12451. num_exentries = 0;
  12452. insn = func[i]->insnsi;
  12453. for (j = 0; j < func[i]->len; j++, insn++) {
  12454. if (BPF_CLASS(insn->code) == BPF_LDX &&
  12455. BPF_MODE(insn->code) == BPF_PROBE_MEM)
  12456. num_exentries++;
  12457. }
  12458. func[i]->aux->num_exentries = num_exentries;
  12459. func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
  12460. func[i] = bpf_int_jit_compile(func[i]);
  12461. if (!func[i]->jited) {
  12462. err = -ENOTSUPP;
  12463. goto out_free;
  12464. }
  12465. cond_resched();
  12466. }
  12467. /* at this point all bpf functions were successfully JITed
  12468. * now populate all bpf_calls with correct addresses and
  12469. * run last pass of JIT
  12470. */
  12471. for (i = 0; i < env->subprog_cnt; i++) {
  12472. insn = func[i]->insnsi;
  12473. for (j = 0; j < func[i]->len; j++, insn++) {
  12474. if (bpf_pseudo_func(insn)) {
  12475. subprog = insn->off;
  12476. insn[0].imm = (u32)(long)func[subprog]->bpf_func;
  12477. insn[1].imm = ((u64)(long)func[subprog]->bpf_func) >> 32;
  12478. continue;
  12479. }
  12480. if (!bpf_pseudo_call(insn))
  12481. continue;
  12482. subprog = insn->off;
  12483. insn->imm = BPF_CALL_IMM(func[subprog]->bpf_func);
  12484. }
  12485. /* we use the aux data to keep a list of the start addresses
  12486. * of the JITed images for each function in the program
  12487. *
  12488. * for some architectures, such as powerpc64, the imm field
  12489. * might not be large enough to hold the offset of the start
  12490. * address of the callee's JITed image from __bpf_call_base
  12491. *
  12492. * in such cases, we can lookup the start address of a callee
  12493. * by using its subprog id, available from the off field of
  12494. * the call instruction, as an index for this list
  12495. */
  12496. func[i]->aux->func = func;
  12497. func[i]->aux->func_cnt = env->subprog_cnt;
  12498. }
  12499. for (i = 0; i < env->subprog_cnt; i++) {
  12500. old_bpf_func = func[i]->bpf_func;
  12501. tmp = bpf_int_jit_compile(func[i]);
  12502. if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
  12503. verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
  12504. err = -ENOTSUPP;
  12505. goto out_free;
  12506. }
  12507. cond_resched();
  12508. }
  12509. /* finally lock prog and jit images for all functions and
  12510. * populate kallsysm. Begin at the first subprogram, since
  12511. * bpf_prog_load will add the kallsyms for the main program.
  12512. */
  12513. for (i = 1; i < env->subprog_cnt; i++) {
  12514. bpf_prog_lock_ro(func[i]);
  12515. bpf_prog_kallsyms_add(func[i]);
  12516. }
  12517. /* Last step: make now unused interpreter insns from main
  12518. * prog consistent for later dump requests, so they can
  12519. * later look the same as if they were interpreted only.
  12520. */
  12521. for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
  12522. if (bpf_pseudo_func(insn)) {
  12523. insn[0].imm = env->insn_aux_data[i].call_imm;
  12524. insn[1].imm = insn->off;
  12525. insn->off = 0;
  12526. continue;
  12527. }
  12528. if (!bpf_pseudo_call(insn))
  12529. continue;
  12530. insn->off = env->insn_aux_data[i].call_imm;
  12531. subprog = find_subprog(env, i + insn->off + 1);
  12532. insn->imm = subprog;
  12533. }
  12534. prog->jited = 1;
  12535. prog->bpf_func = func[0]->bpf_func;
  12536. prog->jited_len = func[0]->jited_len;
  12537. prog->aux->extable = func[0]->aux->extable;
  12538. prog->aux->num_exentries = func[0]->aux->num_exentries;
  12539. prog->aux->func = func;
  12540. prog->aux->func_cnt = env->subprog_cnt;
  12541. bpf_prog_jit_attempt_done(prog);
  12542. return 0;
  12543. out_free:
  12544. /* We failed JIT'ing, so at this point we need to unregister poke
  12545. * descriptors from subprogs, so that kernel is not attempting to
  12546. * patch it anymore as we're freeing the subprog JIT memory.
  12547. */
  12548. for (i = 0; i < prog->aux->size_poke_tab; i++) {
  12549. map_ptr = prog->aux->poke_tab[i].tail_call.map;
  12550. map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
  12551. }
  12552. /* At this point we're guaranteed that poke descriptors are not
  12553. * live anymore. We can just unlink its descriptor table as it's
  12554. * released with the main prog.
  12555. */
  12556. for (i = 0; i < env->subprog_cnt; i++) {
  12557. if (!func[i])
  12558. continue;
  12559. func[i]->aux->poke_tab = NULL;
  12560. bpf_jit_free(func[i]);
  12561. }
  12562. kfree(func);
  12563. out_undo_insn:
  12564. /* cleanup main prog to be interpreted */
  12565. prog->jit_requested = 0;
  12566. prog->blinding_requested = 0;
  12567. for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
  12568. if (!bpf_pseudo_call(insn))
  12569. continue;
  12570. insn->off = 0;
  12571. insn->imm = env->insn_aux_data[i].call_imm;
  12572. }
  12573. bpf_prog_jit_attempt_done(prog);
  12574. return err;
  12575. }
  12576. static int fixup_call_args(struct bpf_verifier_env *env)
  12577. {
  12578. #ifndef CONFIG_BPF_JIT_ALWAYS_ON
  12579. struct bpf_prog *prog = env->prog;
  12580. struct bpf_insn *insn = prog->insnsi;
  12581. bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
  12582. int i, depth;
  12583. #endif
  12584. int err = 0;
  12585. if (env->prog->jit_requested &&
  12586. !bpf_prog_is_dev_bound(env->prog->aux)) {
  12587. err = jit_subprogs(env);
  12588. if (err == 0)
  12589. return 0;
  12590. if (err == -EFAULT)
  12591. return err;
  12592. }
  12593. #ifndef CONFIG_BPF_JIT_ALWAYS_ON
  12594. if (has_kfunc_call) {
  12595. verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
  12596. return -EINVAL;
  12597. }
  12598. if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
  12599. /* When JIT fails the progs with bpf2bpf calls and tail_calls
  12600. * have to be rejected, since interpreter doesn't support them yet.
  12601. */
  12602. verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
  12603. return -EINVAL;
  12604. }
  12605. for (i = 0; i < prog->len; i++, insn++) {
  12606. if (bpf_pseudo_func(insn)) {
  12607. /* When JIT fails the progs with callback calls
  12608. * have to be rejected, since interpreter doesn't support them yet.
  12609. */
  12610. verbose(env, "callbacks are not allowed in non-JITed programs\n");
  12611. return -EINVAL;
  12612. }
  12613. if (!bpf_pseudo_call(insn))
  12614. continue;
  12615. depth = get_callee_stack_depth(env, insn, i);
  12616. if (depth < 0)
  12617. return depth;
  12618. bpf_patch_call_args(insn, depth);
  12619. }
  12620. err = 0;
  12621. #endif
  12622. return err;
  12623. }
  12624. static int fixup_kfunc_call(struct bpf_verifier_env *env,
  12625. struct bpf_insn *insn)
  12626. {
  12627. const struct bpf_kfunc_desc *desc;
  12628. if (!insn->imm) {
  12629. verbose(env, "invalid kernel function call not eliminated in verifier pass\n");
  12630. return -EINVAL;
  12631. }
  12632. /* insn->imm has the btf func_id. Replace it with
  12633. * an address (relative to __bpf_base_call).
  12634. */
  12635. desc = find_kfunc_desc(env->prog, insn->imm, insn->off);
  12636. if (!desc) {
  12637. verbose(env, "verifier internal error: kernel function descriptor not found for func_id %u\n",
  12638. insn->imm);
  12639. return -EFAULT;
  12640. }
  12641. insn->imm = desc->imm;
  12642. return 0;
  12643. }
  12644. /* Do various post-verification rewrites in a single program pass.
  12645. * These rewrites simplify JIT and interpreter implementations.
  12646. */
  12647. static int do_misc_fixups(struct bpf_verifier_env *env)
  12648. {
  12649. struct bpf_prog *prog = env->prog;
  12650. enum bpf_attach_type eatype = prog->expected_attach_type;
  12651. enum bpf_prog_type prog_type = resolve_prog_type(prog);
  12652. struct bpf_insn *insn = prog->insnsi;
  12653. const struct bpf_func_proto *fn;
  12654. const int insn_cnt = prog->len;
  12655. const struct bpf_map_ops *ops;
  12656. struct bpf_insn_aux_data *aux;
  12657. struct bpf_insn insn_buf[16];
  12658. struct bpf_prog *new_prog;
  12659. struct bpf_map *map_ptr;
  12660. int i, ret, cnt, delta = 0;
  12661. for (i = 0; i < insn_cnt; i++, insn++) {
  12662. /* Make divide-by-zero exceptions impossible. */
  12663. if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
  12664. insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
  12665. insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
  12666. insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
  12667. bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
  12668. bool isdiv = BPF_OP(insn->code) == BPF_DIV;
  12669. struct bpf_insn *patchlet;
  12670. struct bpf_insn chk_and_div[] = {
  12671. /* [R,W]x div 0 -> 0 */
  12672. BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
  12673. BPF_JNE | BPF_K, insn->src_reg,
  12674. 0, 2, 0),
  12675. BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
  12676. BPF_JMP_IMM(BPF_JA, 0, 0, 1),
  12677. *insn,
  12678. };
  12679. struct bpf_insn chk_and_mod[] = {
  12680. /* [R,W]x mod 0 -> [R,W]x */
  12681. BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
  12682. BPF_JEQ | BPF_K, insn->src_reg,
  12683. 0, 1 + (is64 ? 0 : 1), 0),
  12684. *insn,
  12685. BPF_JMP_IMM(BPF_JA, 0, 0, 1),
  12686. BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
  12687. };
  12688. patchlet = isdiv ? chk_and_div : chk_and_mod;
  12689. cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
  12690. ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
  12691. new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
  12692. if (!new_prog)
  12693. return -ENOMEM;
  12694. delta += cnt - 1;
  12695. env->prog = prog = new_prog;
  12696. insn = new_prog->insnsi + i + delta;
  12697. continue;
  12698. }
  12699. /* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
  12700. if (BPF_CLASS(insn->code) == BPF_LD &&
  12701. (BPF_MODE(insn->code) == BPF_ABS ||
  12702. BPF_MODE(insn->code) == BPF_IND)) {
  12703. cnt = env->ops->gen_ld_abs(insn, insn_buf);
  12704. if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
  12705. verbose(env, "bpf verifier is misconfigured\n");
  12706. return -EINVAL;
  12707. }
  12708. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  12709. if (!new_prog)
  12710. return -ENOMEM;
  12711. delta += cnt - 1;
  12712. env->prog = prog = new_prog;
  12713. insn = new_prog->insnsi + i + delta;
  12714. continue;
  12715. }
  12716. /* Rewrite pointer arithmetic to mitigate speculation attacks. */
  12717. if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
  12718. insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
  12719. const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
  12720. const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
  12721. struct bpf_insn *patch = &insn_buf[0];
  12722. bool issrc, isneg, isimm;
  12723. u32 off_reg;
  12724. aux = &env->insn_aux_data[i + delta];
  12725. if (!aux->alu_state ||
  12726. aux->alu_state == BPF_ALU_NON_POINTER)
  12727. continue;
  12728. isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
  12729. issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
  12730. BPF_ALU_SANITIZE_SRC;
  12731. isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
  12732. off_reg = issrc ? insn->src_reg : insn->dst_reg;
  12733. if (isimm) {
  12734. *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
  12735. } else {
  12736. if (isneg)
  12737. *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
  12738. *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
  12739. *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
  12740. *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
  12741. *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
  12742. *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
  12743. *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
  12744. }
  12745. if (!issrc)
  12746. *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
  12747. insn->src_reg = BPF_REG_AX;
  12748. if (isneg)
  12749. insn->code = insn->code == code_add ?
  12750. code_sub : code_add;
  12751. *patch++ = *insn;
  12752. if (issrc && isneg && !isimm)
  12753. *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
  12754. cnt = patch - insn_buf;
  12755. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  12756. if (!new_prog)
  12757. return -ENOMEM;
  12758. delta += cnt - 1;
  12759. env->prog = prog = new_prog;
  12760. insn = new_prog->insnsi + i + delta;
  12761. continue;
  12762. }
  12763. if (insn->code != (BPF_JMP | BPF_CALL))
  12764. continue;
  12765. if (insn->src_reg == BPF_PSEUDO_CALL)
  12766. continue;
  12767. if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
  12768. ret = fixup_kfunc_call(env, insn);
  12769. if (ret)
  12770. return ret;
  12771. continue;
  12772. }
  12773. if (insn->imm == BPF_FUNC_get_route_realm)
  12774. prog->dst_needed = 1;
  12775. if (insn->imm == BPF_FUNC_get_prandom_u32)
  12776. bpf_user_rnd_init_once();
  12777. if (insn->imm == BPF_FUNC_override_return)
  12778. prog->kprobe_override = 1;
  12779. if (insn->imm == BPF_FUNC_tail_call) {
  12780. /* If we tail call into other programs, we
  12781. * cannot make any assumptions since they can
  12782. * be replaced dynamically during runtime in
  12783. * the program array.
  12784. */
  12785. prog->cb_access = 1;
  12786. if (!allow_tail_call_in_subprogs(env))
  12787. prog->aux->stack_depth = MAX_BPF_STACK;
  12788. prog->aux->max_pkt_offset = MAX_PACKET_OFF;
  12789. /* mark bpf_tail_call as different opcode to avoid
  12790. * conditional branch in the interpreter for every normal
  12791. * call and to prevent accidental JITing by JIT compiler
  12792. * that doesn't support bpf_tail_call yet
  12793. */
  12794. insn->imm = 0;
  12795. insn->code = BPF_JMP | BPF_TAIL_CALL;
  12796. aux = &env->insn_aux_data[i + delta];
  12797. if (env->bpf_capable && !prog->blinding_requested &&
  12798. prog->jit_requested &&
  12799. !bpf_map_key_poisoned(aux) &&
  12800. !bpf_map_ptr_poisoned(aux) &&
  12801. !bpf_map_ptr_unpriv(aux)) {
  12802. struct bpf_jit_poke_descriptor desc = {
  12803. .reason = BPF_POKE_REASON_TAIL_CALL,
  12804. .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
  12805. .tail_call.key = bpf_map_key_immediate(aux),
  12806. .insn_idx = i + delta,
  12807. };
  12808. ret = bpf_jit_add_poke_descriptor(prog, &desc);
  12809. if (ret < 0) {
  12810. verbose(env, "adding tail call poke descriptor failed\n");
  12811. return ret;
  12812. }
  12813. insn->imm = ret + 1;
  12814. continue;
  12815. }
  12816. if (!bpf_map_ptr_unpriv(aux))
  12817. continue;
  12818. /* instead of changing every JIT dealing with tail_call
  12819. * emit two extra insns:
  12820. * if (index >= max_entries) goto out;
  12821. * index &= array->index_mask;
  12822. * to avoid out-of-bounds cpu speculation
  12823. */
  12824. if (bpf_map_ptr_poisoned(aux)) {
  12825. verbose(env, "tail_call abusing map_ptr\n");
  12826. return -EINVAL;
  12827. }
  12828. map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
  12829. insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
  12830. map_ptr->max_entries, 2);
  12831. insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
  12832. container_of(map_ptr,
  12833. struct bpf_array,
  12834. map)->index_mask);
  12835. insn_buf[2] = *insn;
  12836. cnt = 3;
  12837. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  12838. if (!new_prog)
  12839. return -ENOMEM;
  12840. delta += cnt - 1;
  12841. env->prog = prog = new_prog;
  12842. insn = new_prog->insnsi + i + delta;
  12843. continue;
  12844. }
  12845. if (insn->imm == BPF_FUNC_timer_set_callback) {
  12846. /* The verifier will process callback_fn as many times as necessary
  12847. * with different maps and the register states prepared by
  12848. * set_timer_callback_state will be accurate.
  12849. *
  12850. * The following use case is valid:
  12851. * map1 is shared by prog1, prog2, prog3.
  12852. * prog1 calls bpf_timer_init for some map1 elements
  12853. * prog2 calls bpf_timer_set_callback for some map1 elements.
  12854. * Those that were not bpf_timer_init-ed will return -EINVAL.
  12855. * prog3 calls bpf_timer_start for some map1 elements.
  12856. * Those that were not both bpf_timer_init-ed and
  12857. * bpf_timer_set_callback-ed will return -EINVAL.
  12858. */
  12859. struct bpf_insn ld_addrs[2] = {
  12860. BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
  12861. };
  12862. insn_buf[0] = ld_addrs[0];
  12863. insn_buf[1] = ld_addrs[1];
  12864. insn_buf[2] = *insn;
  12865. cnt = 3;
  12866. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  12867. if (!new_prog)
  12868. return -ENOMEM;
  12869. delta += cnt - 1;
  12870. env->prog = prog = new_prog;
  12871. insn = new_prog->insnsi + i + delta;
  12872. goto patch_call_imm;
  12873. }
  12874. if (insn->imm == BPF_FUNC_task_storage_get ||
  12875. insn->imm == BPF_FUNC_sk_storage_get ||
  12876. insn->imm == BPF_FUNC_inode_storage_get) {
  12877. if (env->prog->aux->sleepable)
  12878. insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
  12879. else
  12880. insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
  12881. insn_buf[1] = *insn;
  12882. cnt = 2;
  12883. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  12884. if (!new_prog)
  12885. return -ENOMEM;
  12886. delta += cnt - 1;
  12887. env->prog = prog = new_prog;
  12888. insn = new_prog->insnsi + i + delta;
  12889. goto patch_call_imm;
  12890. }
  12891. /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
  12892. * and other inlining handlers are currently limited to 64 bit
  12893. * only.
  12894. */
  12895. if (prog->jit_requested && BITS_PER_LONG == 64 &&
  12896. (insn->imm == BPF_FUNC_map_lookup_elem ||
  12897. insn->imm == BPF_FUNC_map_update_elem ||
  12898. insn->imm == BPF_FUNC_map_delete_elem ||
  12899. insn->imm == BPF_FUNC_map_push_elem ||
  12900. insn->imm == BPF_FUNC_map_pop_elem ||
  12901. insn->imm == BPF_FUNC_map_peek_elem ||
  12902. insn->imm == BPF_FUNC_redirect_map ||
  12903. insn->imm == BPF_FUNC_for_each_map_elem ||
  12904. insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
  12905. aux = &env->insn_aux_data[i + delta];
  12906. if (bpf_map_ptr_poisoned(aux))
  12907. goto patch_call_imm;
  12908. map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
  12909. ops = map_ptr->ops;
  12910. if (insn->imm == BPF_FUNC_map_lookup_elem &&
  12911. ops->map_gen_lookup) {
  12912. cnt = ops->map_gen_lookup(map_ptr, insn_buf);
  12913. if (cnt == -EOPNOTSUPP)
  12914. goto patch_map_ops_generic;
  12915. if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
  12916. verbose(env, "bpf verifier is misconfigured\n");
  12917. return -EINVAL;
  12918. }
  12919. new_prog = bpf_patch_insn_data(env, i + delta,
  12920. insn_buf, cnt);
  12921. if (!new_prog)
  12922. return -ENOMEM;
  12923. delta += cnt - 1;
  12924. env->prog = prog = new_prog;
  12925. insn = new_prog->insnsi + i + delta;
  12926. continue;
  12927. }
  12928. BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
  12929. (void *(*)(struct bpf_map *map, void *key))NULL));
  12930. BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
  12931. (int (*)(struct bpf_map *map, void *key))NULL));
  12932. BUILD_BUG_ON(!__same_type(ops->map_update_elem,
  12933. (int (*)(struct bpf_map *map, void *key, void *value,
  12934. u64 flags))NULL));
  12935. BUILD_BUG_ON(!__same_type(ops->map_push_elem,
  12936. (int (*)(struct bpf_map *map, void *value,
  12937. u64 flags))NULL));
  12938. BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
  12939. (int (*)(struct bpf_map *map, void *value))NULL));
  12940. BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
  12941. (int (*)(struct bpf_map *map, void *value))NULL));
  12942. BUILD_BUG_ON(!__same_type(ops->map_redirect,
  12943. (int (*)(struct bpf_map *map, u32 ifindex, u64 flags))NULL));
  12944. BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
  12945. (int (*)(struct bpf_map *map,
  12946. bpf_callback_t callback_fn,
  12947. void *callback_ctx,
  12948. u64 flags))NULL));
  12949. BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem,
  12950. (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));
  12951. patch_map_ops_generic:
  12952. switch (insn->imm) {
  12953. case BPF_FUNC_map_lookup_elem:
  12954. insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
  12955. continue;
  12956. case BPF_FUNC_map_update_elem:
  12957. insn->imm = BPF_CALL_IMM(ops->map_update_elem);
  12958. continue;
  12959. case BPF_FUNC_map_delete_elem:
  12960. insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
  12961. continue;
  12962. case BPF_FUNC_map_push_elem:
  12963. insn->imm = BPF_CALL_IMM(ops->map_push_elem);
  12964. continue;
  12965. case BPF_FUNC_map_pop_elem:
  12966. insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
  12967. continue;
  12968. case BPF_FUNC_map_peek_elem:
  12969. insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
  12970. continue;
  12971. case BPF_FUNC_redirect_map:
  12972. insn->imm = BPF_CALL_IMM(ops->map_redirect);
  12973. continue;
  12974. case BPF_FUNC_for_each_map_elem:
  12975. insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
  12976. continue;
  12977. case BPF_FUNC_map_lookup_percpu_elem:
  12978. insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);
  12979. continue;
  12980. }
  12981. goto patch_call_imm;
  12982. }
  12983. /* Implement bpf_jiffies64 inline. */
  12984. if (prog->jit_requested && BITS_PER_LONG == 64 &&
  12985. insn->imm == BPF_FUNC_jiffies64) {
  12986. struct bpf_insn ld_jiffies_addr[2] = {
  12987. BPF_LD_IMM64(BPF_REG_0,
  12988. (unsigned long)&jiffies),
  12989. };
  12990. insn_buf[0] = ld_jiffies_addr[0];
  12991. insn_buf[1] = ld_jiffies_addr[1];
  12992. insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
  12993. BPF_REG_0, 0);
  12994. cnt = 3;
  12995. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
  12996. cnt);
  12997. if (!new_prog)
  12998. return -ENOMEM;
  12999. delta += cnt - 1;
  13000. env->prog = prog = new_prog;
  13001. insn = new_prog->insnsi + i + delta;
  13002. continue;
  13003. }
  13004. /* Implement bpf_get_func_arg inline. */
  13005. if (prog_type == BPF_PROG_TYPE_TRACING &&
  13006. insn->imm == BPF_FUNC_get_func_arg) {
  13007. /* Load nr_args from ctx - 8 */
  13008. insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
  13009. insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
  13010. insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
  13011. insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
  13012. insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
  13013. insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
  13014. insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
  13015. insn_buf[7] = BPF_JMP_A(1);
  13016. insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
  13017. cnt = 9;
  13018. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  13019. if (!new_prog)
  13020. return -ENOMEM;
  13021. delta += cnt - 1;
  13022. env->prog = prog = new_prog;
  13023. insn = new_prog->insnsi + i + delta;
  13024. continue;
  13025. }
  13026. /* Implement bpf_get_func_ret inline. */
  13027. if (prog_type == BPF_PROG_TYPE_TRACING &&
  13028. insn->imm == BPF_FUNC_get_func_ret) {
  13029. if (eatype == BPF_TRACE_FEXIT ||
  13030. eatype == BPF_MODIFY_RETURN) {
  13031. /* Load nr_args from ctx - 8 */
  13032. insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
  13033. insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
  13034. insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
  13035. insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
  13036. insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
  13037. insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
  13038. cnt = 6;
  13039. } else {
  13040. insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
  13041. cnt = 1;
  13042. }
  13043. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
  13044. if (!new_prog)
  13045. return -ENOMEM;
  13046. delta += cnt - 1;
  13047. env->prog = prog = new_prog;
  13048. insn = new_prog->insnsi + i + delta;
  13049. continue;
  13050. }
  13051. /* Implement get_func_arg_cnt inline. */
  13052. if (prog_type == BPF_PROG_TYPE_TRACING &&
  13053. insn->imm == BPF_FUNC_get_func_arg_cnt) {
  13054. /* Load nr_args from ctx - 8 */
  13055. insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
  13056. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
  13057. if (!new_prog)
  13058. return -ENOMEM;
  13059. env->prog = prog = new_prog;
  13060. insn = new_prog->insnsi + i + delta;
  13061. continue;
  13062. }
  13063. /* Implement bpf_get_func_ip inline. */
  13064. if (prog_type == BPF_PROG_TYPE_TRACING &&
  13065. insn->imm == BPF_FUNC_get_func_ip) {
  13066. /* Load IP address from ctx - 16 */
  13067. insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
  13068. new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
  13069. if (!new_prog)
  13070. return -ENOMEM;
  13071. env->prog = prog = new_prog;
  13072. insn = new_prog->insnsi + i + delta;
  13073. continue;
  13074. }
  13075. patch_call_imm:
  13076. fn = env->ops->get_func_proto(insn->imm, env->prog);
  13077. /* all functions that have prototype and verifier allowed
  13078. * programs to call them, must be real in-kernel functions
  13079. */
  13080. if (!fn->func) {
  13081. verbose(env,
  13082. "kernel subsystem misconfigured func %s#%d\n",
  13083. func_id_name(insn->imm), insn->imm);
  13084. return -EFAULT;
  13085. }
  13086. insn->imm = fn->func - __bpf_call_base;
  13087. }
  13088. /* Since poke tab is now finalized, publish aux to tracker. */
  13089. for (i = 0; i < prog->aux->size_poke_tab; i++) {
  13090. map_ptr = prog->aux->poke_tab[i].tail_call.map;
  13091. if (!map_ptr->ops->map_poke_track ||
  13092. !map_ptr->ops->map_poke_untrack ||
  13093. !map_ptr->ops->map_poke_run) {
  13094. verbose(env, "bpf verifier is misconfigured\n");
  13095. return -EINVAL;
  13096. }
  13097. ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
  13098. if (ret < 0) {
  13099. verbose(env, "tracking tail call prog failed\n");
  13100. return ret;
  13101. }
  13102. }
  13103. sort_kfunc_descs_by_imm(env->prog);
  13104. return 0;
  13105. }
  13106. static struct bpf_prog *inline_bpf_loop(struct bpf_verifier_env *env,
  13107. int position,
  13108. s32 stack_base,
  13109. u32 callback_subprogno,
  13110. u32 *cnt)
  13111. {
  13112. s32 r6_offset = stack_base + 0 * BPF_REG_SIZE;
  13113. s32 r7_offset = stack_base + 1 * BPF_REG_SIZE;
  13114. s32 r8_offset = stack_base + 2 * BPF_REG_SIZE;
  13115. int reg_loop_max = BPF_REG_6;
  13116. int reg_loop_cnt = BPF_REG_7;
  13117. int reg_loop_ctx = BPF_REG_8;
  13118. struct bpf_prog *new_prog;
  13119. u32 callback_start;
  13120. u32 call_insn_offset;
  13121. s32 callback_offset;
  13122. /* This represents an inlined version of bpf_iter.c:bpf_loop,
  13123. * be careful to modify this code in sync.
  13124. */
  13125. struct bpf_insn insn_buf[] = {
  13126. /* Return error and jump to the end of the patch if
  13127. * expected number of iterations is too big.
  13128. */
  13129. BPF_JMP_IMM(BPF_JLE, BPF_REG_1, BPF_MAX_LOOPS, 2),
  13130. BPF_MOV32_IMM(BPF_REG_0, -E2BIG),
  13131. BPF_JMP_IMM(BPF_JA, 0, 0, 16),
  13132. /* spill R6, R7, R8 to use these as loop vars */
  13133. BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_6, r6_offset),
  13134. BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_7, r7_offset),
  13135. BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_8, r8_offset),
  13136. /* initialize loop vars */
  13137. BPF_MOV64_REG(reg_loop_max, BPF_REG_1),
  13138. BPF_MOV32_IMM(reg_loop_cnt, 0),
  13139. BPF_MOV64_REG(reg_loop_ctx, BPF_REG_3),
  13140. /* loop header,
  13141. * if reg_loop_cnt >= reg_loop_max skip the loop body
  13142. */
  13143. BPF_JMP_REG(BPF_JGE, reg_loop_cnt, reg_loop_max, 5),
  13144. /* callback call,
  13145. * correct callback offset would be set after patching
  13146. */
  13147. BPF_MOV64_REG(BPF_REG_1, reg_loop_cnt),
  13148. BPF_MOV64_REG(BPF_REG_2, reg_loop_ctx),
  13149. BPF_CALL_REL(0),
  13150. /* increment loop counter */
  13151. BPF_ALU64_IMM(BPF_ADD, reg_loop_cnt, 1),
  13152. /* jump to loop header if callback returned 0 */
  13153. BPF_JMP_IMM(BPF_JEQ, BPF_REG_0, 0, -6),
  13154. /* return value of bpf_loop,
  13155. * set R0 to the number of iterations
  13156. */
  13157. BPF_MOV64_REG(BPF_REG_0, reg_loop_cnt),
  13158. /* restore original values of R6, R7, R8 */
  13159. BPF_LDX_MEM(BPF_DW, BPF_REG_6, BPF_REG_10, r6_offset),
  13160. BPF_LDX_MEM(BPF_DW, BPF_REG_7, BPF_REG_10, r7_offset),
  13161. BPF_LDX_MEM(BPF_DW, BPF_REG_8, BPF_REG_10, r8_offset),
  13162. };
  13163. *cnt = ARRAY_SIZE(insn_buf);
  13164. new_prog = bpf_patch_insn_data(env, position, insn_buf, *cnt);
  13165. if (!new_prog)
  13166. return new_prog;
  13167. /* callback start is known only after patching */
  13168. callback_start = env->subprog_info[callback_subprogno].start;
  13169. /* Note: insn_buf[12] is an offset of BPF_CALL_REL instruction */
  13170. call_insn_offset = position + 12;
  13171. callback_offset = callback_start - call_insn_offset - 1;
  13172. new_prog->insnsi[call_insn_offset].imm = callback_offset;
  13173. return new_prog;
  13174. }
  13175. static bool is_bpf_loop_call(struct bpf_insn *insn)
  13176. {
  13177. return insn->code == (BPF_JMP | BPF_CALL) &&
  13178. insn->src_reg == 0 &&
  13179. insn->imm == BPF_FUNC_loop;
  13180. }
  13181. /* For all sub-programs in the program (including main) check
  13182. * insn_aux_data to see if there are bpf_loop calls that require
  13183. * inlining. If such calls are found the calls are replaced with a
  13184. * sequence of instructions produced by `inline_bpf_loop` function and
  13185. * subprog stack_depth is increased by the size of 3 registers.
  13186. * This stack space is used to spill values of the R6, R7, R8. These
  13187. * registers are used to store the loop bound, counter and context
  13188. * variables.
  13189. */
  13190. static int optimize_bpf_loop(struct bpf_verifier_env *env)
  13191. {
  13192. struct bpf_subprog_info *subprogs = env->subprog_info;
  13193. int i, cur_subprog = 0, cnt, delta = 0;
  13194. struct bpf_insn *insn = env->prog->insnsi;
  13195. int insn_cnt = env->prog->len;
  13196. u16 stack_depth = subprogs[cur_subprog].stack_depth;
  13197. u16 stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
  13198. u16 stack_depth_extra = 0;
  13199. for (i = 0; i < insn_cnt; i++, insn++) {
  13200. struct bpf_loop_inline_state *inline_state =
  13201. &env->insn_aux_data[i + delta].loop_inline_state;
  13202. if (is_bpf_loop_call(insn) && inline_state->fit_for_inline) {
  13203. struct bpf_prog *new_prog;
  13204. stack_depth_extra = BPF_REG_SIZE * 3 + stack_depth_roundup;
  13205. new_prog = inline_bpf_loop(env,
  13206. i + delta,
  13207. -(stack_depth + stack_depth_extra),
  13208. inline_state->callback_subprogno,
  13209. &cnt);
  13210. if (!new_prog)
  13211. return -ENOMEM;
  13212. delta += cnt - 1;
  13213. env->prog = new_prog;
  13214. insn = new_prog->insnsi + i + delta;
  13215. }
  13216. if (subprogs[cur_subprog + 1].start == i + delta + 1) {
  13217. subprogs[cur_subprog].stack_depth += stack_depth_extra;
  13218. cur_subprog++;
  13219. stack_depth = subprogs[cur_subprog].stack_depth;
  13220. stack_depth_roundup = round_up(stack_depth, 8) - stack_depth;
  13221. stack_depth_extra = 0;
  13222. }
  13223. }
  13224. env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
  13225. return 0;
  13226. }
  13227. static void free_states(struct bpf_verifier_env *env)
  13228. {
  13229. struct bpf_verifier_state_list *sl, *sln;
  13230. int i;
  13231. sl = env->free_list;
  13232. while (sl) {
  13233. sln = sl->next;
  13234. free_verifier_state(&sl->state, false);
  13235. kfree(sl);
  13236. sl = sln;
  13237. }
  13238. env->free_list = NULL;
  13239. if (!env->explored_states)
  13240. return;
  13241. for (i = 0; i < state_htab_size(env); i++) {
  13242. sl = env->explored_states[i];
  13243. while (sl) {
  13244. sln = sl->next;
  13245. free_verifier_state(&sl->state, false);
  13246. kfree(sl);
  13247. sl = sln;
  13248. }
  13249. env->explored_states[i] = NULL;
  13250. }
  13251. }
  13252. static int do_check_common(struct bpf_verifier_env *env, int subprog)
  13253. {
  13254. bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
  13255. struct bpf_verifier_state *state;
  13256. struct bpf_reg_state *regs;
  13257. int ret, i;
  13258. env->prev_linfo = NULL;
  13259. env->pass_cnt++;
  13260. state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
  13261. if (!state)
  13262. return -ENOMEM;
  13263. state->curframe = 0;
  13264. state->speculative = false;
  13265. state->branches = 1;
  13266. state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
  13267. if (!state->frame[0]) {
  13268. kfree(state);
  13269. return -ENOMEM;
  13270. }
  13271. env->cur_state = state;
  13272. init_func_state(env, state->frame[0],
  13273. BPF_MAIN_FUNC /* callsite */,
  13274. 0 /* frameno */,
  13275. subprog);
  13276. state->first_insn_idx = env->subprog_info[subprog].start;
  13277. state->last_insn_idx = -1;
  13278. regs = state->frame[state->curframe]->regs;
  13279. if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
  13280. ret = btf_prepare_func_args(env, subprog, regs);
  13281. if (ret)
  13282. goto out;
  13283. for (i = BPF_REG_1; i <= BPF_REG_5; i++) {
  13284. if (regs[i].type == PTR_TO_CTX)
  13285. mark_reg_known_zero(env, regs, i);
  13286. else if (regs[i].type == SCALAR_VALUE)
  13287. mark_reg_unknown(env, regs, i);
  13288. else if (base_type(regs[i].type) == PTR_TO_MEM) {
  13289. const u32 mem_size = regs[i].mem_size;
  13290. mark_reg_known_zero(env, regs, i);
  13291. regs[i].mem_size = mem_size;
  13292. regs[i].id = ++env->id_gen;
  13293. }
  13294. }
  13295. } else {
  13296. /* 1st arg to a function */
  13297. regs[BPF_REG_1].type = PTR_TO_CTX;
  13298. mark_reg_known_zero(env, regs, BPF_REG_1);
  13299. ret = btf_check_subprog_arg_match(env, subprog, regs);
  13300. if (ret == -EFAULT)
  13301. /* unlikely verifier bug. abort.
  13302. * ret == 0 and ret < 0 are sadly acceptable for
  13303. * main() function due to backward compatibility.
  13304. * Like socket filter program may be written as:
  13305. * int bpf_prog(struct pt_regs *ctx)
  13306. * and never dereference that ctx in the program.
  13307. * 'struct pt_regs' is a type mismatch for socket
  13308. * filter that should be using 'struct __sk_buff'.
  13309. */
  13310. goto out;
  13311. }
  13312. ret = do_check(env);
  13313. out:
  13314. /* check for NULL is necessary, since cur_state can be freed inside
  13315. * do_check() under memory pressure.
  13316. */
  13317. if (env->cur_state) {
  13318. free_verifier_state(env->cur_state, true);
  13319. env->cur_state = NULL;
  13320. }
  13321. while (!pop_stack(env, NULL, NULL, false));
  13322. if (!ret && pop_log)
  13323. bpf_vlog_reset(&env->log, 0);
  13324. free_states(env);
  13325. return ret;
  13326. }
  13327. /* Verify all global functions in a BPF program one by one based on their BTF.
  13328. * All global functions must pass verification. Otherwise the whole program is rejected.
  13329. * Consider:
  13330. * int bar(int);
  13331. * int foo(int f)
  13332. * {
  13333. * return bar(f);
  13334. * }
  13335. * int bar(int b)
  13336. * {
  13337. * ...
  13338. * }
  13339. * foo() will be verified first for R1=any_scalar_value. During verification it
  13340. * will be assumed that bar() already verified successfully and call to bar()
  13341. * from foo() will be checked for type match only. Later bar() will be verified
  13342. * independently to check that it's safe for R1=any_scalar_value.
  13343. */
  13344. static int do_check_subprogs(struct bpf_verifier_env *env)
  13345. {
  13346. struct bpf_prog_aux *aux = env->prog->aux;
  13347. int i, ret;
  13348. if (!aux->func_info)
  13349. return 0;
  13350. for (i = 1; i < env->subprog_cnt; i++) {
  13351. if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL)
  13352. continue;
  13353. env->insn_idx = env->subprog_info[i].start;
  13354. WARN_ON_ONCE(env->insn_idx == 0);
  13355. ret = do_check_common(env, i);
  13356. if (ret) {
  13357. return ret;
  13358. } else if (env->log.level & BPF_LOG_LEVEL) {
  13359. verbose(env,
  13360. "Func#%d is safe for any args that match its prototype\n",
  13361. i);
  13362. }
  13363. }
  13364. return 0;
  13365. }
  13366. static int do_check_main(struct bpf_verifier_env *env)
  13367. {
  13368. int ret;
  13369. env->insn_idx = 0;
  13370. ret = do_check_common(env, 0);
  13371. if (!ret)
  13372. env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
  13373. return ret;
  13374. }
  13375. static void print_verification_stats(struct bpf_verifier_env *env)
  13376. {
  13377. int i;
  13378. if (env->log.level & BPF_LOG_STATS) {
  13379. verbose(env, "verification time %lld usec\n",
  13380. div_u64(env->verification_time, 1000));
  13381. verbose(env, "stack depth ");
  13382. for (i = 0; i < env->subprog_cnt; i++) {
  13383. u32 depth = env->subprog_info[i].stack_depth;
  13384. verbose(env, "%d", depth);
  13385. if (i + 1 < env->subprog_cnt)
  13386. verbose(env, "+");
  13387. }
  13388. verbose(env, "\n");
  13389. }
  13390. verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
  13391. "total_states %d peak_states %d mark_read %d\n",
  13392. env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS,
  13393. env->max_states_per_insn, env->total_states,
  13394. env->peak_states, env->longest_mark_read_walk);
  13395. }
  13396. static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
  13397. {
  13398. const struct btf_type *t, *func_proto;
  13399. const struct bpf_struct_ops *st_ops;
  13400. const struct btf_member *member;
  13401. struct bpf_prog *prog = env->prog;
  13402. u32 btf_id, member_idx;
  13403. const char *mname;
  13404. if (!prog->gpl_compatible) {
  13405. verbose(env, "struct ops programs must have a GPL compatible license\n");
  13406. return -EINVAL;
  13407. }
  13408. btf_id = prog->aux->attach_btf_id;
  13409. st_ops = bpf_struct_ops_find(btf_id);
  13410. if (!st_ops) {
  13411. verbose(env, "attach_btf_id %u is not a supported struct\n",
  13412. btf_id);
  13413. return -ENOTSUPP;
  13414. }
  13415. t = st_ops->type;
  13416. member_idx = prog->expected_attach_type;
  13417. if (member_idx >= btf_type_vlen(t)) {
  13418. verbose(env, "attach to invalid member idx %u of struct %s\n",
  13419. member_idx, st_ops->name);
  13420. return -EINVAL;
  13421. }
  13422. member = &btf_type_member(t)[member_idx];
  13423. mname = btf_name_by_offset(btf_vmlinux, member->name_off);
  13424. func_proto = btf_type_resolve_func_ptr(btf_vmlinux, member->type,
  13425. NULL);
  13426. if (!func_proto) {
  13427. verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n",
  13428. mname, member_idx, st_ops->name);
  13429. return -EINVAL;
  13430. }
  13431. if (st_ops->check_member) {
  13432. int err = st_ops->check_member(t, member);
  13433. if (err) {
  13434. verbose(env, "attach to unsupported member %s of struct %s\n",
  13435. mname, st_ops->name);
  13436. return err;
  13437. }
  13438. }
  13439. prog->aux->attach_func_proto = func_proto;
  13440. prog->aux->attach_func_name = mname;
  13441. env->ops = st_ops->verifier_ops;
  13442. return 0;
  13443. }
  13444. #define SECURITY_PREFIX "security_"
  13445. static int check_attach_modify_return(unsigned long addr, const char *func_name)
  13446. {
  13447. if (within_error_injection_list(addr) ||
  13448. !strncmp(SECURITY_PREFIX, func_name, sizeof(SECURITY_PREFIX) - 1))
  13449. return 0;
  13450. return -EINVAL;
  13451. }
  13452. /* list of non-sleepable functions that are otherwise on
  13453. * ALLOW_ERROR_INJECTION list
  13454. */
  13455. BTF_SET_START(btf_non_sleepable_error_inject)
  13456. /* Three functions below can be called from sleepable and non-sleepable context.
  13457. * Assume non-sleepable from bpf safety point of view.
  13458. */
  13459. BTF_ID(func, __filemap_add_folio)
  13460. BTF_ID(func, should_fail_alloc_page)
  13461. BTF_ID(func, should_failslab)
  13462. BTF_SET_END(btf_non_sleepable_error_inject)
  13463. static int check_non_sleepable_error_inject(u32 btf_id)
  13464. {
  13465. return btf_id_set_contains(&btf_non_sleepable_error_inject, btf_id);
  13466. }
  13467. int bpf_check_attach_target(struct bpf_verifier_log *log,
  13468. const struct bpf_prog *prog,
  13469. const struct bpf_prog *tgt_prog,
  13470. u32 btf_id,
  13471. struct bpf_attach_target_info *tgt_info)
  13472. {
  13473. bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
  13474. const char prefix[] = "btf_trace_";
  13475. int ret = 0, subprog = -1, i;
  13476. const struct btf_type *t;
  13477. bool conservative = true;
  13478. const char *tname;
  13479. struct btf *btf;
  13480. long addr = 0;
  13481. if (!btf_id) {
  13482. bpf_log(log, "Tracing programs must provide btf_id\n");
  13483. return -EINVAL;
  13484. }
  13485. btf = tgt_prog ? tgt_prog->aux->btf : prog->aux->attach_btf;
  13486. if (!btf) {
  13487. bpf_log(log,
  13488. "FENTRY/FEXIT program can only be attached to another program annotated with BTF\n");
  13489. return -EINVAL;
  13490. }
  13491. t = btf_type_by_id(btf, btf_id);
  13492. if (!t) {
  13493. bpf_log(log, "attach_btf_id %u is invalid\n", btf_id);
  13494. return -EINVAL;
  13495. }
  13496. tname = btf_name_by_offset(btf, t->name_off);
  13497. if (!tname) {
  13498. bpf_log(log, "attach_btf_id %u doesn't have a name\n", btf_id);
  13499. return -EINVAL;
  13500. }
  13501. if (tgt_prog) {
  13502. struct bpf_prog_aux *aux = tgt_prog->aux;
  13503. for (i = 0; i < aux->func_info_cnt; i++)
  13504. if (aux->func_info[i].type_id == btf_id) {
  13505. subprog = i;
  13506. break;
  13507. }
  13508. if (subprog == -1) {
  13509. bpf_log(log, "Subprog %s doesn't exist\n", tname);
  13510. return -EINVAL;
  13511. }
  13512. conservative = aux->func_info_aux[subprog].unreliable;
  13513. if (prog_extension) {
  13514. if (conservative) {
  13515. bpf_log(log,
  13516. "Cannot replace static functions\n");
  13517. return -EINVAL;
  13518. }
  13519. if (!prog->jit_requested) {
  13520. bpf_log(log,
  13521. "Extension programs should be JITed\n");
  13522. return -EINVAL;
  13523. }
  13524. }
  13525. if (!tgt_prog->jited) {
  13526. bpf_log(log, "Can attach to only JITed progs\n");
  13527. return -EINVAL;
  13528. }
  13529. if (tgt_prog->type == prog->type) {
  13530. /* Cannot fentry/fexit another fentry/fexit program.
  13531. * Cannot attach program extension to another extension.
  13532. * It's ok to attach fentry/fexit to extension program.
  13533. */
  13534. bpf_log(log, "Cannot recursively attach\n");
  13535. return -EINVAL;
  13536. }
  13537. if (tgt_prog->type == BPF_PROG_TYPE_TRACING &&
  13538. prog_extension &&
  13539. (tgt_prog->expected_attach_type == BPF_TRACE_FENTRY ||
  13540. tgt_prog->expected_attach_type == BPF_TRACE_FEXIT)) {
  13541. /* Program extensions can extend all program types
  13542. * except fentry/fexit. The reason is the following.
  13543. * The fentry/fexit programs are used for performance
  13544. * analysis, stats and can be attached to any program
  13545. * type except themselves. When extension program is
  13546. * replacing XDP function it is necessary to allow
  13547. * performance analysis of all functions. Both original
  13548. * XDP program and its program extension. Hence
  13549. * attaching fentry/fexit to BPF_PROG_TYPE_EXT is
  13550. * allowed. If extending of fentry/fexit was allowed it
  13551. * would be possible to create long call chain
  13552. * fentry->extension->fentry->extension beyond
  13553. * reasonable stack size. Hence extending fentry is not
  13554. * allowed.
  13555. */
  13556. bpf_log(log, "Cannot extend fentry/fexit\n");
  13557. return -EINVAL;
  13558. }
  13559. } else {
  13560. if (prog_extension) {
  13561. bpf_log(log, "Cannot replace kernel functions\n");
  13562. return -EINVAL;
  13563. }
  13564. }
  13565. switch (prog->expected_attach_type) {
  13566. case BPF_TRACE_RAW_TP:
  13567. if (tgt_prog) {
  13568. bpf_log(log,
  13569. "Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
  13570. return -EINVAL;
  13571. }
  13572. if (!btf_type_is_typedef(t)) {
  13573. bpf_log(log, "attach_btf_id %u is not a typedef\n",
  13574. btf_id);
  13575. return -EINVAL;
  13576. }
  13577. if (strncmp(prefix, tname, sizeof(prefix) - 1)) {
  13578. bpf_log(log, "attach_btf_id %u points to wrong type name %s\n",
  13579. btf_id, tname);
  13580. return -EINVAL;
  13581. }
  13582. tname += sizeof(prefix) - 1;
  13583. t = btf_type_by_id(btf, t->type);
  13584. if (!btf_type_is_ptr(t))
  13585. /* should never happen in valid vmlinux build */
  13586. return -EINVAL;
  13587. t = btf_type_by_id(btf, t->type);
  13588. if (!btf_type_is_func_proto(t))
  13589. /* should never happen in valid vmlinux build */
  13590. return -EINVAL;
  13591. break;
  13592. case BPF_TRACE_ITER:
  13593. if (!btf_type_is_func(t)) {
  13594. bpf_log(log, "attach_btf_id %u is not a function\n",
  13595. btf_id);
  13596. return -EINVAL;
  13597. }
  13598. t = btf_type_by_id(btf, t->type);
  13599. if (!btf_type_is_func_proto(t))
  13600. return -EINVAL;
  13601. ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
  13602. if (ret)
  13603. return ret;
  13604. break;
  13605. default:
  13606. if (!prog_extension)
  13607. return -EINVAL;
  13608. fallthrough;
  13609. case BPF_MODIFY_RETURN:
  13610. case BPF_LSM_MAC:
  13611. case BPF_LSM_CGROUP:
  13612. case BPF_TRACE_FENTRY:
  13613. case BPF_TRACE_FEXIT:
  13614. if (!btf_type_is_func(t)) {
  13615. bpf_log(log, "attach_btf_id %u is not a function\n",
  13616. btf_id);
  13617. return -EINVAL;
  13618. }
  13619. if (prog_extension &&
  13620. btf_check_type_match(log, prog, btf, t))
  13621. return -EINVAL;
  13622. t = btf_type_by_id(btf, t->type);
  13623. if (!btf_type_is_func_proto(t))
  13624. return -EINVAL;
  13625. if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
  13626. (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
  13627. prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type))
  13628. return -EINVAL;
  13629. if (tgt_prog && conservative)
  13630. t = NULL;
  13631. ret = btf_distill_func_proto(log, btf, t, tname, &tgt_info->fmodel);
  13632. if (ret < 0)
  13633. return ret;
  13634. if (tgt_prog) {
  13635. if (subprog == 0)
  13636. addr = (long) tgt_prog->bpf_func;
  13637. else
  13638. addr = (long) tgt_prog->aux->func[subprog]->bpf_func;
  13639. } else {
  13640. addr = kallsyms_lookup_name(tname);
  13641. if (!addr) {
  13642. bpf_log(log,
  13643. "The address of function %s cannot be found\n",
  13644. tname);
  13645. return -ENOENT;
  13646. }
  13647. }
  13648. if (prog->aux->sleepable) {
  13649. ret = -EINVAL;
  13650. switch (prog->type) {
  13651. case BPF_PROG_TYPE_TRACING:
  13652. /* fentry/fexit/fmod_ret progs can be sleepable only if they are
  13653. * attached to ALLOW_ERROR_INJECTION and are not in denylist.
  13654. */
  13655. if (!check_non_sleepable_error_inject(btf_id) &&
  13656. within_error_injection_list(addr))
  13657. ret = 0;
  13658. break;
  13659. case BPF_PROG_TYPE_LSM:
  13660. /* LSM progs check that they are attached to bpf_lsm_*() funcs.
  13661. * Only some of them are sleepable.
  13662. */
  13663. if (bpf_lsm_is_sleepable_hook(btf_id))
  13664. ret = 0;
  13665. break;
  13666. default:
  13667. break;
  13668. }
  13669. if (ret) {
  13670. bpf_log(log, "%s is not sleepable\n", tname);
  13671. return ret;
  13672. }
  13673. } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
  13674. if (tgt_prog) {
  13675. bpf_log(log, "can't modify return codes of BPF programs\n");
  13676. return -EINVAL;
  13677. }
  13678. ret = check_attach_modify_return(addr, tname);
  13679. if (ret) {
  13680. bpf_log(log, "%s() is not modifiable\n", tname);
  13681. return ret;
  13682. }
  13683. }
  13684. break;
  13685. }
  13686. tgt_info->tgt_addr = addr;
  13687. tgt_info->tgt_name = tname;
  13688. tgt_info->tgt_type = t;
  13689. return 0;
  13690. }
  13691. BTF_SET_START(btf_id_deny)
  13692. BTF_ID_UNUSED
  13693. #ifdef CONFIG_SMP
  13694. BTF_ID(func, migrate_disable)
  13695. BTF_ID(func, migrate_enable)
  13696. #endif
  13697. #if !defined CONFIG_PREEMPT_RCU && !defined CONFIG_TINY_RCU
  13698. BTF_ID(func, rcu_read_unlock_strict)
  13699. #endif
  13700. #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_TRACE_PREEMPT_TOGGLE)
  13701. BTF_ID(func, preempt_count_add)
  13702. BTF_ID(func, preempt_count_sub)
  13703. #endif
  13704. BTF_SET_END(btf_id_deny)
  13705. static int check_attach_btf_id(struct bpf_verifier_env *env)
  13706. {
  13707. struct bpf_prog *prog = env->prog;
  13708. struct bpf_prog *tgt_prog = prog->aux->dst_prog;
  13709. struct bpf_attach_target_info tgt_info = {};
  13710. u32 btf_id = prog->aux->attach_btf_id;
  13711. struct bpf_trampoline *tr;
  13712. int ret;
  13713. u64 key;
  13714. if (prog->type == BPF_PROG_TYPE_SYSCALL) {
  13715. if (prog->aux->sleepable)
  13716. /* attach_btf_id checked to be zero already */
  13717. return 0;
  13718. verbose(env, "Syscall programs can only be sleepable\n");
  13719. return -EINVAL;
  13720. }
  13721. if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING &&
  13722. prog->type != BPF_PROG_TYPE_LSM && prog->type != BPF_PROG_TYPE_KPROBE) {
  13723. verbose(env, "Only fentry/fexit/fmod_ret, lsm, and kprobe/uprobe programs can be sleepable\n");
  13724. return -EINVAL;
  13725. }
  13726. if (prog->type == BPF_PROG_TYPE_STRUCT_OPS)
  13727. return check_struct_ops_btf_id(env);
  13728. if (prog->type != BPF_PROG_TYPE_TRACING &&
  13729. prog->type != BPF_PROG_TYPE_LSM &&
  13730. prog->type != BPF_PROG_TYPE_EXT)
  13731. return 0;
  13732. ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
  13733. if (ret)
  13734. return ret;
  13735. if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
  13736. /* to make freplace equivalent to their targets, they need to
  13737. * inherit env->ops and expected_attach_type for the rest of the
  13738. * verification
  13739. */
  13740. env->ops = bpf_verifier_ops[tgt_prog->type];
  13741. prog->expected_attach_type = tgt_prog->expected_attach_type;
  13742. }
  13743. /* store info about the attachment target that will be used later */
  13744. prog->aux->attach_func_proto = tgt_info.tgt_type;
  13745. prog->aux->attach_func_name = tgt_info.tgt_name;
  13746. if (tgt_prog) {
  13747. prog->aux->saved_dst_prog_type = tgt_prog->type;
  13748. prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
  13749. }
  13750. if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
  13751. prog->aux->attach_btf_trace = true;
  13752. return 0;
  13753. } else if (prog->expected_attach_type == BPF_TRACE_ITER) {
  13754. if (!bpf_iter_prog_supported(prog))
  13755. return -EINVAL;
  13756. return 0;
  13757. }
  13758. if (prog->type == BPF_PROG_TYPE_LSM) {
  13759. ret = bpf_lsm_verify_prog(&env->log, prog);
  13760. if (ret < 0)
  13761. return ret;
  13762. } else if (prog->type == BPF_PROG_TYPE_TRACING &&
  13763. btf_id_set_contains(&btf_id_deny, btf_id)) {
  13764. return -EINVAL;
  13765. }
  13766. key = bpf_trampoline_compute_key(tgt_prog, prog->aux->attach_btf, btf_id);
  13767. tr = bpf_trampoline_get(key, &tgt_info);
  13768. if (!tr)
  13769. return -ENOMEM;
  13770. prog->aux->dst_trampoline = tr;
  13771. return 0;
  13772. }
  13773. struct btf *bpf_get_btf_vmlinux(void)
  13774. {
  13775. if (!btf_vmlinux && IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) {
  13776. mutex_lock(&bpf_verifier_lock);
  13777. if (!btf_vmlinux)
  13778. btf_vmlinux = btf_parse_vmlinux();
  13779. mutex_unlock(&bpf_verifier_lock);
  13780. }
  13781. return btf_vmlinux;
  13782. }
  13783. int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr)
  13784. {
  13785. u64 start_time = ktime_get_ns();
  13786. struct bpf_verifier_env *env;
  13787. struct bpf_verifier_log *log;
  13788. int i, len, ret = -EINVAL;
  13789. bool is_priv;
  13790. /* no program is valid */
  13791. if (ARRAY_SIZE(bpf_verifier_ops) == 0)
  13792. return -EINVAL;
  13793. /* 'struct bpf_verifier_env' can be global, but since it's not small,
  13794. * allocate/free it every time bpf_check() is called
  13795. */
  13796. env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
  13797. if (!env)
  13798. return -ENOMEM;
  13799. log = &env->log;
  13800. len = (*prog)->len;
  13801. env->insn_aux_data =
  13802. vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
  13803. ret = -ENOMEM;
  13804. if (!env->insn_aux_data)
  13805. goto err_free_env;
  13806. for (i = 0; i < len; i++)
  13807. env->insn_aux_data[i].orig_idx = i;
  13808. env->prog = *prog;
  13809. env->ops = bpf_verifier_ops[env->prog->type];
  13810. env->fd_array = make_bpfptr(attr->fd_array, uattr.is_kernel);
  13811. is_priv = bpf_capable();
  13812. bpf_get_btf_vmlinux();
  13813. /* grab the mutex to protect few globals used by verifier */
  13814. if (!is_priv)
  13815. mutex_lock(&bpf_verifier_lock);
  13816. if (attr->log_level || attr->log_buf || attr->log_size) {
  13817. /* user requested verbose verifier output
  13818. * and supplied buffer to store the verification trace
  13819. */
  13820. log->level = attr->log_level;
  13821. log->ubuf = (char __user *) (unsigned long) attr->log_buf;
  13822. log->len_total = attr->log_size;
  13823. /* log attributes have to be sane */
  13824. if (!bpf_verifier_log_attr_valid(log)) {
  13825. ret = -EINVAL;
  13826. goto err_unlock;
  13827. }
  13828. }
  13829. mark_verifier_state_clean(env);
  13830. if (IS_ERR(btf_vmlinux)) {
  13831. /* Either gcc or pahole or kernel are broken. */
  13832. verbose(env, "in-kernel BTF is malformed\n");
  13833. ret = PTR_ERR(btf_vmlinux);
  13834. goto skip_full_check;
  13835. }
  13836. env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
  13837. if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
  13838. env->strict_alignment = true;
  13839. if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
  13840. env->strict_alignment = false;
  13841. env->allow_ptr_leaks = bpf_allow_ptr_leaks();
  13842. env->allow_uninit_stack = bpf_allow_uninit_stack();
  13843. env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
  13844. env->bypass_spec_v1 = bpf_bypass_spec_v1();
  13845. env->bypass_spec_v4 = bpf_bypass_spec_v4();
  13846. env->bpf_capable = bpf_capable();
  13847. if (is_priv)
  13848. env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
  13849. env->explored_states = kvcalloc(state_htab_size(env),
  13850. sizeof(struct bpf_verifier_state_list *),
  13851. GFP_USER);
  13852. ret = -ENOMEM;
  13853. if (!env->explored_states)
  13854. goto skip_full_check;
  13855. ret = add_subprog_and_kfunc(env);
  13856. if (ret < 0)
  13857. goto skip_full_check;
  13858. ret = check_subprogs(env);
  13859. if (ret < 0)
  13860. goto skip_full_check;
  13861. ret = check_btf_info(env, attr, uattr);
  13862. if (ret < 0)
  13863. goto skip_full_check;
  13864. ret = check_attach_btf_id(env);
  13865. if (ret)
  13866. goto skip_full_check;
  13867. ret = resolve_pseudo_ldimm64(env);
  13868. if (ret < 0)
  13869. goto skip_full_check;
  13870. if (bpf_prog_is_dev_bound(env->prog->aux)) {
  13871. ret = bpf_prog_offload_verifier_prep(env->prog);
  13872. if (ret)
  13873. goto skip_full_check;
  13874. }
  13875. ret = check_cfg(env);
  13876. if (ret < 0)
  13877. goto skip_full_check;
  13878. ret = do_check_subprogs(env);
  13879. ret = ret ?: do_check_main(env);
  13880. if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
  13881. ret = bpf_prog_offload_finalize(env);
  13882. skip_full_check:
  13883. kvfree(env->explored_states);
  13884. if (ret == 0)
  13885. ret = check_max_stack_depth(env);
  13886. /* instruction rewrites happen after this point */
  13887. if (ret == 0)
  13888. ret = optimize_bpf_loop(env);
  13889. if (is_priv) {
  13890. if (ret == 0)
  13891. opt_hard_wire_dead_code_branches(env);
  13892. if (ret == 0)
  13893. ret = opt_remove_dead_code(env);
  13894. if (ret == 0)
  13895. ret = opt_remove_nops(env);
  13896. } else {
  13897. if (ret == 0)
  13898. sanitize_dead_code(env);
  13899. }
  13900. if (ret == 0)
  13901. /* program is valid, convert *(u32*)(ctx + off) accesses */
  13902. ret = convert_ctx_accesses(env);
  13903. if (ret == 0)
  13904. ret = do_misc_fixups(env);
  13905. /* do 32-bit optimization after insn patching has done so those patched
  13906. * insns could be handled correctly.
  13907. */
  13908. if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
  13909. ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
  13910. env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret
  13911. : false;
  13912. }
  13913. if (ret == 0)
  13914. ret = fixup_call_args(env);
  13915. env->verification_time = ktime_get_ns() - start_time;
  13916. print_verification_stats(env);
  13917. env->prog->aux->verified_insns = env->insn_processed;
  13918. if (log->level && bpf_verifier_log_full(log))
  13919. ret = -ENOSPC;
  13920. if (log->level && !log->ubuf) {
  13921. ret = -EFAULT;
  13922. goto err_release_maps;
  13923. }
  13924. if (ret)
  13925. goto err_release_maps;
  13926. if (env->used_map_cnt) {
  13927. /* if program passed verifier, update used_maps in bpf_prog_info */
  13928. env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
  13929. sizeof(env->used_maps[0]),
  13930. GFP_KERNEL);
  13931. if (!env->prog->aux->used_maps) {
  13932. ret = -ENOMEM;
  13933. goto err_release_maps;
  13934. }
  13935. memcpy(env->prog->aux->used_maps, env->used_maps,
  13936. sizeof(env->used_maps[0]) * env->used_map_cnt);
  13937. env->prog->aux->used_map_cnt = env->used_map_cnt;
  13938. }
  13939. if (env->used_btf_cnt) {
  13940. /* if program passed verifier, update used_btfs in bpf_prog_aux */
  13941. env->prog->aux->used_btfs = kmalloc_array(env->used_btf_cnt,
  13942. sizeof(env->used_btfs[0]),
  13943. GFP_KERNEL);
  13944. if (!env->prog->aux->used_btfs) {
  13945. ret = -ENOMEM;
  13946. goto err_release_maps;
  13947. }
  13948. memcpy(env->prog->aux->used_btfs, env->used_btfs,
  13949. sizeof(env->used_btfs[0]) * env->used_btf_cnt);
  13950. env->prog->aux->used_btf_cnt = env->used_btf_cnt;
  13951. }
  13952. if (env->used_map_cnt || env->used_btf_cnt) {
  13953. /* program is valid. Convert pseudo bpf_ld_imm64 into generic
  13954. * bpf_ld_imm64 instructions
  13955. */
  13956. convert_pseudo_ld_imm64(env);
  13957. }
  13958. adjust_btf_func(env);
  13959. err_release_maps:
  13960. if (!env->prog->aux->used_maps)
  13961. /* if we didn't copy map pointers into bpf_prog_info, release
  13962. * them now. Otherwise free_used_maps() will release them.
  13963. */
  13964. release_maps(env);
  13965. if (!env->prog->aux->used_btfs)
  13966. release_btfs(env);
  13967. /* extension progs temporarily inherit the attach_type of their targets
  13968. for verification purposes, so set it back to zero before returning
  13969. */
  13970. if (env->prog->type == BPF_PROG_TYPE_EXT)
  13971. env->prog->expected_attach_type = 0;
  13972. *prog = env->prog;
  13973. err_unlock:
  13974. if (!is_priv)
  13975. mutex_unlock(&bpf_verifier_lock);
  13976. vfree(env->insn_aux_data);
  13977. err_free_env:
  13978. kfree(env);
  13979. return ret;
  13980. }