raid5.c 257 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * raid5.c : Multiple Devices driver for Linux
  4. * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
  5. * Copyright (C) 1999, 2000 Ingo Molnar
  6. * Copyright (C) 2002, 2003 H. Peter Anvin
  7. *
  8. * RAID-4/5/6 management functions.
  9. * Thanks to Penguin Computing for making the RAID-6 development possible
  10. * by donating a test server!
  11. */
  12. /*
  13. * BITMAP UNPLUGGING:
  14. *
  15. * The sequencing for updating the bitmap reliably is a little
  16. * subtle (and I got it wrong the first time) so it deserves some
  17. * explanation.
  18. *
  19. * We group bitmap updates into batches. Each batch has a number.
  20. * We may write out several batches at once, but that isn't very important.
  21. * conf->seq_write is the number of the last batch successfully written.
  22. * conf->seq_flush is the number of the last batch that was closed to
  23. * new additions.
  24. * When we discover that we will need to write to any block in a stripe
  25. * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
  26. * the number of the batch it will be in. This is seq_flush+1.
  27. * When we are ready to do a write, if that batch hasn't been written yet,
  28. * we plug the array and queue the stripe for later.
  29. * When an unplug happens, we increment bm_flush, thus closing the current
  30. * batch.
  31. * When we notice that bm_flush > bm_write, we write out all pending updates
  32. * to the bitmap, and advance bm_write to where bm_flush was.
  33. * This may occasionally write a bit out twice, but is sure never to
  34. * miss any bits.
  35. */
  36. #include <linux/blkdev.h>
  37. #include <linux/delay.h>
  38. #include <linux/kthread.h>
  39. #include <linux/raid/pq.h>
  40. #include <linux/async_tx.h>
  41. #include <linux/module.h>
  42. #include <linux/async.h>
  43. #include <linux/seq_file.h>
  44. #include <linux/cpu.h>
  45. #include <linux/slab.h>
  46. #include <linux/ratelimit.h>
  47. #include <linux/nodemask.h>
  48. #include <trace/events/block.h>
  49. #include <linux/list_sort.h>
  50. #include "md.h"
  51. #include "raid5.h"
  52. #include "raid0.h"
  53. #include "md-bitmap.h"
  54. #include "raid5-log.h"
  55. #define UNSUPPORTED_MDDEV_FLAGS (1L << MD_FAILFAST_SUPPORTED)
  56. #define cpu_to_group(cpu) cpu_to_node(cpu)
  57. #define ANY_GROUP NUMA_NO_NODE
  58. #define RAID5_MAX_REQ_STRIPES 256
  59. static bool devices_handle_discard_safely = false;
  60. module_param(devices_handle_discard_safely, bool, 0644);
  61. MODULE_PARM_DESC(devices_handle_discard_safely,
  62. "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
  63. static struct workqueue_struct *raid5_wq;
  64. static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
  65. {
  66. int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
  67. return &conf->stripe_hashtbl[hash];
  68. }
  69. static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)
  70. {
  71. return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;
  72. }
  73. static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
  74. __acquires(&conf->device_lock)
  75. {
  76. spin_lock_irq(conf->hash_locks + hash);
  77. spin_lock(&conf->device_lock);
  78. }
  79. static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
  80. __releases(&conf->device_lock)
  81. {
  82. spin_unlock(&conf->device_lock);
  83. spin_unlock_irq(conf->hash_locks + hash);
  84. }
  85. static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
  86. __acquires(&conf->device_lock)
  87. {
  88. int i;
  89. spin_lock_irq(conf->hash_locks);
  90. for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
  91. spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
  92. spin_lock(&conf->device_lock);
  93. }
  94. static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
  95. __releases(&conf->device_lock)
  96. {
  97. int i;
  98. spin_unlock(&conf->device_lock);
  99. for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--)
  100. spin_unlock(conf->hash_locks + i);
  101. spin_unlock_irq(conf->hash_locks);
  102. }
  103. /* Find first data disk in a raid6 stripe */
  104. static inline int raid6_d0(struct stripe_head *sh)
  105. {
  106. if (sh->ddf_layout)
  107. /* ddf always start from first device */
  108. return 0;
  109. /* md starts just after Q block */
  110. if (sh->qd_idx == sh->disks - 1)
  111. return 0;
  112. else
  113. return sh->qd_idx + 1;
  114. }
  115. static inline int raid6_next_disk(int disk, int raid_disks)
  116. {
  117. disk++;
  118. return (disk < raid_disks) ? disk : 0;
  119. }
  120. /* When walking through the disks in a raid5, starting at raid6_d0,
  121. * We need to map each disk to a 'slot', where the data disks are slot
  122. * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
  123. * is raid_disks-1. This help does that mapping.
  124. */
  125. static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
  126. int *count, int syndrome_disks)
  127. {
  128. int slot = *count;
  129. if (sh->ddf_layout)
  130. (*count)++;
  131. if (idx == sh->pd_idx)
  132. return syndrome_disks;
  133. if (idx == sh->qd_idx)
  134. return syndrome_disks + 1;
  135. if (!sh->ddf_layout)
  136. (*count)++;
  137. return slot;
  138. }
  139. static void print_raid5_conf (struct r5conf *conf);
  140. static int stripe_operations_active(struct stripe_head *sh)
  141. {
  142. return sh->check_state || sh->reconstruct_state ||
  143. test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
  144. test_bit(STRIPE_COMPUTE_RUN, &sh->state);
  145. }
  146. static bool stripe_is_lowprio(struct stripe_head *sh)
  147. {
  148. return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||
  149. test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
  150. !test_bit(STRIPE_R5C_CACHING, &sh->state);
  151. }
  152. static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
  153. __must_hold(&sh->raid_conf->device_lock)
  154. {
  155. struct r5conf *conf = sh->raid_conf;
  156. struct r5worker_group *group;
  157. int thread_cnt;
  158. int i, cpu = sh->cpu;
  159. if (!cpu_online(cpu)) {
  160. cpu = cpumask_any(cpu_online_mask);
  161. sh->cpu = cpu;
  162. }
  163. if (list_empty(&sh->lru)) {
  164. struct r5worker_group *group;
  165. group = conf->worker_groups + cpu_to_group(cpu);
  166. if (stripe_is_lowprio(sh))
  167. list_add_tail(&sh->lru, &group->loprio_list);
  168. else
  169. list_add_tail(&sh->lru, &group->handle_list);
  170. group->stripes_cnt++;
  171. sh->group = group;
  172. }
  173. if (conf->worker_cnt_per_group == 0) {
  174. md_wakeup_thread(conf->mddev->thread);
  175. return;
  176. }
  177. group = conf->worker_groups + cpu_to_group(sh->cpu);
  178. group->workers[0].working = true;
  179. /* at least one worker should run to avoid race */
  180. queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
  181. thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
  182. /* wakeup more workers */
  183. for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
  184. if (group->workers[i].working == false) {
  185. group->workers[i].working = true;
  186. queue_work_on(sh->cpu, raid5_wq,
  187. &group->workers[i].work);
  188. thread_cnt--;
  189. }
  190. }
  191. }
  192. static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
  193. struct list_head *temp_inactive_list)
  194. __must_hold(&conf->device_lock)
  195. {
  196. int i;
  197. int injournal = 0; /* number of date pages with R5_InJournal */
  198. BUG_ON(!list_empty(&sh->lru));
  199. BUG_ON(atomic_read(&conf->active_stripes)==0);
  200. if (r5c_is_writeback(conf->log))
  201. for (i = sh->disks; i--; )
  202. if (test_bit(R5_InJournal, &sh->dev[i].flags))
  203. injournal++;
  204. /*
  205. * In the following cases, the stripe cannot be released to cached
  206. * lists. Therefore, we make the stripe write out and set
  207. * STRIPE_HANDLE:
  208. * 1. when quiesce in r5c write back;
  209. * 2. when resync is requested fot the stripe.
  210. */
  211. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||
  212. (conf->quiesce && r5c_is_writeback(conf->log) &&
  213. !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {
  214. if (test_bit(STRIPE_R5C_CACHING, &sh->state))
  215. r5c_make_stripe_write_out(sh);
  216. set_bit(STRIPE_HANDLE, &sh->state);
  217. }
  218. if (test_bit(STRIPE_HANDLE, &sh->state)) {
  219. if (test_bit(STRIPE_DELAYED, &sh->state) &&
  220. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  221. list_add_tail(&sh->lru, &conf->delayed_list);
  222. else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
  223. sh->bm_seq - conf->seq_write > 0)
  224. list_add_tail(&sh->lru, &conf->bitmap_list);
  225. else {
  226. clear_bit(STRIPE_DELAYED, &sh->state);
  227. clear_bit(STRIPE_BIT_DELAY, &sh->state);
  228. if (conf->worker_cnt_per_group == 0) {
  229. if (stripe_is_lowprio(sh))
  230. list_add_tail(&sh->lru,
  231. &conf->loprio_list);
  232. else
  233. list_add_tail(&sh->lru,
  234. &conf->handle_list);
  235. } else {
  236. raid5_wakeup_stripe_thread(sh);
  237. return;
  238. }
  239. }
  240. md_wakeup_thread(conf->mddev->thread);
  241. } else {
  242. BUG_ON(stripe_operations_active(sh));
  243. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  244. if (atomic_dec_return(&conf->preread_active_stripes)
  245. < IO_THRESHOLD)
  246. md_wakeup_thread(conf->mddev->thread);
  247. atomic_dec(&conf->active_stripes);
  248. if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
  249. if (!r5c_is_writeback(conf->log))
  250. list_add_tail(&sh->lru, temp_inactive_list);
  251. else {
  252. WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
  253. if (injournal == 0)
  254. list_add_tail(&sh->lru, temp_inactive_list);
  255. else if (injournal == conf->raid_disks - conf->max_degraded) {
  256. /* full stripe */
  257. if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
  258. atomic_inc(&conf->r5c_cached_full_stripes);
  259. if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
  260. atomic_dec(&conf->r5c_cached_partial_stripes);
  261. list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
  262. r5c_check_cached_full_stripe(conf);
  263. } else
  264. /*
  265. * STRIPE_R5C_PARTIAL_STRIPE is set in
  266. * r5c_try_caching_write(). No need to
  267. * set it again.
  268. */
  269. list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
  270. }
  271. }
  272. }
  273. }
  274. static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
  275. struct list_head *temp_inactive_list)
  276. __must_hold(&conf->device_lock)
  277. {
  278. if (atomic_dec_and_test(&sh->count))
  279. do_release_stripe(conf, sh, temp_inactive_list);
  280. }
  281. /*
  282. * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
  283. *
  284. * Be careful: Only one task can add/delete stripes from temp_inactive_list at
  285. * given time. Adding stripes only takes device lock, while deleting stripes
  286. * only takes hash lock.
  287. */
  288. static void release_inactive_stripe_list(struct r5conf *conf,
  289. struct list_head *temp_inactive_list,
  290. int hash)
  291. {
  292. int size;
  293. bool do_wakeup = false;
  294. unsigned long flags;
  295. if (hash == NR_STRIPE_HASH_LOCKS) {
  296. size = NR_STRIPE_HASH_LOCKS;
  297. hash = NR_STRIPE_HASH_LOCKS - 1;
  298. } else
  299. size = 1;
  300. while (size) {
  301. struct list_head *list = &temp_inactive_list[size - 1];
  302. /*
  303. * We don't hold any lock here yet, raid5_get_active_stripe() might
  304. * remove stripes from the list
  305. */
  306. if (!list_empty_careful(list)) {
  307. spin_lock_irqsave(conf->hash_locks + hash, flags);
  308. if (list_empty(conf->inactive_list + hash) &&
  309. !list_empty(list))
  310. atomic_dec(&conf->empty_inactive_list_nr);
  311. list_splice_tail_init(list, conf->inactive_list + hash);
  312. do_wakeup = true;
  313. spin_unlock_irqrestore(conf->hash_locks + hash, flags);
  314. }
  315. size--;
  316. hash--;
  317. }
  318. if (do_wakeup) {
  319. wake_up(&conf->wait_for_stripe);
  320. if (atomic_read(&conf->active_stripes) == 0)
  321. wake_up(&conf->wait_for_quiescent);
  322. if (conf->retry_read_aligned)
  323. md_wakeup_thread(conf->mddev->thread);
  324. }
  325. }
  326. static int release_stripe_list(struct r5conf *conf,
  327. struct list_head *temp_inactive_list)
  328. __must_hold(&conf->device_lock)
  329. {
  330. struct stripe_head *sh, *t;
  331. int count = 0;
  332. struct llist_node *head;
  333. head = llist_del_all(&conf->released_stripes);
  334. head = llist_reverse_order(head);
  335. llist_for_each_entry_safe(sh, t, head, release_list) {
  336. int hash;
  337. /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
  338. smp_mb();
  339. clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
  340. /*
  341. * Don't worry the bit is set here, because if the bit is set
  342. * again, the count is always > 1. This is true for
  343. * STRIPE_ON_UNPLUG_LIST bit too.
  344. */
  345. hash = sh->hash_lock_index;
  346. __release_stripe(conf, sh, &temp_inactive_list[hash]);
  347. count++;
  348. }
  349. return count;
  350. }
  351. void raid5_release_stripe(struct stripe_head *sh)
  352. {
  353. struct r5conf *conf = sh->raid_conf;
  354. unsigned long flags;
  355. struct list_head list;
  356. int hash;
  357. bool wakeup;
  358. /* Avoid release_list until the last reference.
  359. */
  360. if (atomic_add_unless(&sh->count, -1, 1))
  361. return;
  362. if (unlikely(!conf->mddev->thread) ||
  363. test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
  364. goto slow_path;
  365. wakeup = llist_add(&sh->release_list, &conf->released_stripes);
  366. if (wakeup)
  367. md_wakeup_thread(conf->mddev->thread);
  368. return;
  369. slow_path:
  370. /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
  371. if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {
  372. INIT_LIST_HEAD(&list);
  373. hash = sh->hash_lock_index;
  374. do_release_stripe(conf, sh, &list);
  375. spin_unlock_irqrestore(&conf->device_lock, flags);
  376. release_inactive_stripe_list(conf, &list, hash);
  377. }
  378. }
  379. static inline void remove_hash(struct stripe_head *sh)
  380. {
  381. pr_debug("remove_hash(), stripe %llu\n",
  382. (unsigned long long)sh->sector);
  383. hlist_del_init(&sh->hash);
  384. }
  385. static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
  386. {
  387. struct hlist_head *hp = stripe_hash(conf, sh->sector);
  388. pr_debug("insert_hash(), stripe %llu\n",
  389. (unsigned long long)sh->sector);
  390. hlist_add_head(&sh->hash, hp);
  391. }
  392. /* find an idle stripe, make sure it is unhashed, and return it. */
  393. static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
  394. {
  395. struct stripe_head *sh = NULL;
  396. struct list_head *first;
  397. if (list_empty(conf->inactive_list + hash))
  398. goto out;
  399. first = (conf->inactive_list + hash)->next;
  400. sh = list_entry(first, struct stripe_head, lru);
  401. list_del_init(first);
  402. remove_hash(sh);
  403. atomic_inc(&conf->active_stripes);
  404. BUG_ON(hash != sh->hash_lock_index);
  405. if (list_empty(conf->inactive_list + hash))
  406. atomic_inc(&conf->empty_inactive_list_nr);
  407. out:
  408. return sh;
  409. }
  410. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  411. static void free_stripe_pages(struct stripe_head *sh)
  412. {
  413. int i;
  414. struct page *p;
  415. /* Have not allocate page pool */
  416. if (!sh->pages)
  417. return;
  418. for (i = 0; i < sh->nr_pages; i++) {
  419. p = sh->pages[i];
  420. if (p)
  421. put_page(p);
  422. sh->pages[i] = NULL;
  423. }
  424. }
  425. static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
  426. {
  427. int i;
  428. struct page *p;
  429. for (i = 0; i < sh->nr_pages; i++) {
  430. /* The page have allocated. */
  431. if (sh->pages[i])
  432. continue;
  433. p = alloc_page(gfp);
  434. if (!p) {
  435. free_stripe_pages(sh);
  436. return -ENOMEM;
  437. }
  438. sh->pages[i] = p;
  439. }
  440. return 0;
  441. }
  442. static int
  443. init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)
  444. {
  445. int nr_pages, cnt;
  446. if (sh->pages)
  447. return 0;
  448. /* Each of the sh->dev[i] need one conf->stripe_size */
  449. cnt = PAGE_SIZE / conf->stripe_size;
  450. nr_pages = (disks + cnt - 1) / cnt;
  451. sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
  452. if (!sh->pages)
  453. return -ENOMEM;
  454. sh->nr_pages = nr_pages;
  455. sh->stripes_per_page = cnt;
  456. return 0;
  457. }
  458. #endif
  459. static void shrink_buffers(struct stripe_head *sh)
  460. {
  461. int i;
  462. int num = sh->raid_conf->pool_size;
  463. #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
  464. for (i = 0; i < num ; i++) {
  465. struct page *p;
  466. WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
  467. p = sh->dev[i].page;
  468. if (!p)
  469. continue;
  470. sh->dev[i].page = NULL;
  471. put_page(p);
  472. }
  473. #else
  474. for (i = 0; i < num; i++)
  475. sh->dev[i].page = NULL;
  476. free_stripe_pages(sh); /* Free pages */
  477. #endif
  478. }
  479. static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
  480. {
  481. int i;
  482. int num = sh->raid_conf->pool_size;
  483. #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
  484. for (i = 0; i < num; i++) {
  485. struct page *page;
  486. if (!(page = alloc_page(gfp))) {
  487. return 1;
  488. }
  489. sh->dev[i].page = page;
  490. sh->dev[i].orig_page = page;
  491. sh->dev[i].offset = 0;
  492. }
  493. #else
  494. if (alloc_stripe_pages(sh, gfp))
  495. return -ENOMEM;
  496. for (i = 0; i < num; i++) {
  497. sh->dev[i].page = raid5_get_dev_page(sh, i);
  498. sh->dev[i].orig_page = sh->dev[i].page;
  499. sh->dev[i].offset = raid5_get_page_offset(sh, i);
  500. }
  501. #endif
  502. return 0;
  503. }
  504. static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
  505. struct stripe_head *sh);
  506. static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
  507. {
  508. struct r5conf *conf = sh->raid_conf;
  509. int i, seq;
  510. BUG_ON(atomic_read(&sh->count) != 0);
  511. BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
  512. BUG_ON(stripe_operations_active(sh));
  513. BUG_ON(sh->batch_head);
  514. pr_debug("init_stripe called, stripe %llu\n",
  515. (unsigned long long)sector);
  516. retry:
  517. seq = read_seqcount_begin(&conf->gen_lock);
  518. sh->generation = conf->generation - previous;
  519. sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
  520. sh->sector = sector;
  521. stripe_set_idx(sector, conf, previous, sh);
  522. sh->state = 0;
  523. for (i = sh->disks; i--; ) {
  524. struct r5dev *dev = &sh->dev[i];
  525. if (dev->toread || dev->read || dev->towrite || dev->written ||
  526. test_bit(R5_LOCKED, &dev->flags)) {
  527. pr_err("sector=%llx i=%d %p %p %p %p %d\n",
  528. (unsigned long long)sh->sector, i, dev->toread,
  529. dev->read, dev->towrite, dev->written,
  530. test_bit(R5_LOCKED, &dev->flags));
  531. WARN_ON(1);
  532. }
  533. dev->flags = 0;
  534. dev->sector = raid5_compute_blocknr(sh, i, previous);
  535. }
  536. if (read_seqcount_retry(&conf->gen_lock, seq))
  537. goto retry;
  538. sh->overwrite_disks = 0;
  539. insert_hash(conf, sh);
  540. sh->cpu = smp_processor_id();
  541. set_bit(STRIPE_BATCH_READY, &sh->state);
  542. }
  543. static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
  544. short generation)
  545. {
  546. struct stripe_head *sh;
  547. pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
  548. hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
  549. if (sh->sector == sector && sh->generation == generation)
  550. return sh;
  551. pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
  552. return NULL;
  553. }
  554. static struct stripe_head *find_get_stripe(struct r5conf *conf,
  555. sector_t sector, short generation, int hash)
  556. {
  557. int inc_empty_inactive_list_flag;
  558. struct stripe_head *sh;
  559. sh = __find_stripe(conf, sector, generation);
  560. if (!sh)
  561. return NULL;
  562. if (atomic_inc_not_zero(&sh->count))
  563. return sh;
  564. /*
  565. * Slow path. The reference count is zero which means the stripe must
  566. * be on a list (sh->lru). Must remove the stripe from the list that
  567. * references it with the device_lock held.
  568. */
  569. spin_lock(&conf->device_lock);
  570. if (!atomic_read(&sh->count)) {
  571. if (!test_bit(STRIPE_HANDLE, &sh->state))
  572. atomic_inc(&conf->active_stripes);
  573. BUG_ON(list_empty(&sh->lru) &&
  574. !test_bit(STRIPE_EXPANDING, &sh->state));
  575. inc_empty_inactive_list_flag = 0;
  576. if (!list_empty(conf->inactive_list + hash))
  577. inc_empty_inactive_list_flag = 1;
  578. list_del_init(&sh->lru);
  579. if (list_empty(conf->inactive_list + hash) &&
  580. inc_empty_inactive_list_flag)
  581. atomic_inc(&conf->empty_inactive_list_nr);
  582. if (sh->group) {
  583. sh->group->stripes_cnt--;
  584. sh->group = NULL;
  585. }
  586. }
  587. atomic_inc(&sh->count);
  588. spin_unlock(&conf->device_lock);
  589. return sh;
  590. }
  591. /*
  592. * Need to check if array has failed when deciding whether to:
  593. * - start an array
  594. * - remove non-faulty devices
  595. * - add a spare
  596. * - allow a reshape
  597. * This determination is simple when no reshape is happening.
  598. * However if there is a reshape, we need to carefully check
  599. * both the before and after sections.
  600. * This is because some failed devices may only affect one
  601. * of the two sections, and some non-in_sync devices may
  602. * be insync in the section most affected by failed devices.
  603. *
  604. * Most calls to this function hold &conf->device_lock. Calls
  605. * in raid5_run() do not require the lock as no other threads
  606. * have been started yet.
  607. */
  608. int raid5_calc_degraded(struct r5conf *conf)
  609. {
  610. int degraded, degraded2;
  611. int i;
  612. rcu_read_lock();
  613. degraded = 0;
  614. for (i = 0; i < conf->previous_raid_disks; i++) {
  615. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  616. if (rdev && test_bit(Faulty, &rdev->flags))
  617. rdev = rcu_dereference(conf->disks[i].replacement);
  618. if (!rdev || test_bit(Faulty, &rdev->flags))
  619. degraded++;
  620. else if (test_bit(In_sync, &rdev->flags))
  621. ;
  622. else
  623. /* not in-sync or faulty.
  624. * If the reshape increases the number of devices,
  625. * this is being recovered by the reshape, so
  626. * this 'previous' section is not in_sync.
  627. * If the number of devices is being reduced however,
  628. * the device can only be part of the array if
  629. * we are reverting a reshape, so this section will
  630. * be in-sync.
  631. */
  632. if (conf->raid_disks >= conf->previous_raid_disks)
  633. degraded++;
  634. }
  635. rcu_read_unlock();
  636. if (conf->raid_disks == conf->previous_raid_disks)
  637. return degraded;
  638. rcu_read_lock();
  639. degraded2 = 0;
  640. for (i = 0; i < conf->raid_disks; i++) {
  641. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  642. if (rdev && test_bit(Faulty, &rdev->flags))
  643. rdev = rcu_dereference(conf->disks[i].replacement);
  644. if (!rdev || test_bit(Faulty, &rdev->flags))
  645. degraded2++;
  646. else if (test_bit(In_sync, &rdev->flags))
  647. ;
  648. else
  649. /* not in-sync or faulty.
  650. * If reshape increases the number of devices, this
  651. * section has already been recovered, else it
  652. * almost certainly hasn't.
  653. */
  654. if (conf->raid_disks <= conf->previous_raid_disks)
  655. degraded2++;
  656. }
  657. rcu_read_unlock();
  658. if (degraded2 > degraded)
  659. return degraded2;
  660. return degraded;
  661. }
  662. static bool has_failed(struct r5conf *conf)
  663. {
  664. int degraded = conf->mddev->degraded;
  665. if (test_bit(MD_BROKEN, &conf->mddev->flags))
  666. return true;
  667. if (conf->mddev->reshape_position != MaxSector)
  668. degraded = raid5_calc_degraded(conf);
  669. return degraded > conf->max_degraded;
  670. }
  671. enum stripe_result {
  672. STRIPE_SUCCESS = 0,
  673. STRIPE_RETRY,
  674. STRIPE_SCHEDULE_AND_RETRY,
  675. STRIPE_FAIL,
  676. };
  677. struct stripe_request_ctx {
  678. /* a reference to the last stripe_head for batching */
  679. struct stripe_head *batch_last;
  680. /* first sector in the request */
  681. sector_t first_sector;
  682. /* last sector in the request */
  683. sector_t last_sector;
  684. /*
  685. * bitmap to track stripe sectors that have been added to stripes
  686. * add one to account for unaligned requests
  687. */
  688. DECLARE_BITMAP(sectors_to_do, RAID5_MAX_REQ_STRIPES + 1);
  689. /* the request had REQ_PREFLUSH, cleared after the first stripe_head */
  690. bool do_flush;
  691. };
  692. /*
  693. * Block until another thread clears R5_INACTIVE_BLOCKED or
  694. * there are fewer than 3/4 the maximum number of active stripes
  695. * and there is an inactive stripe available.
  696. */
  697. static bool is_inactive_blocked(struct r5conf *conf, int hash)
  698. {
  699. if (list_empty(conf->inactive_list + hash))
  700. return false;
  701. if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
  702. return true;
  703. return (atomic_read(&conf->active_stripes) <
  704. (conf->max_nr_stripes * 3 / 4));
  705. }
  706. struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,
  707. struct stripe_request_ctx *ctx, sector_t sector,
  708. unsigned int flags)
  709. {
  710. struct stripe_head *sh;
  711. int hash = stripe_hash_locks_hash(conf, sector);
  712. int previous = !!(flags & R5_GAS_PREVIOUS);
  713. pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
  714. spin_lock_irq(conf->hash_locks + hash);
  715. for (;;) {
  716. if (!(flags & R5_GAS_NOQUIESCE) && conf->quiesce) {
  717. /*
  718. * Must release the reference to batch_last before
  719. * waiting, on quiesce, otherwise the batch_last will
  720. * hold a reference to a stripe and raid5_quiesce()
  721. * will deadlock waiting for active_stripes to go to
  722. * zero.
  723. */
  724. if (ctx && ctx->batch_last) {
  725. raid5_release_stripe(ctx->batch_last);
  726. ctx->batch_last = NULL;
  727. }
  728. wait_event_lock_irq(conf->wait_for_quiescent,
  729. !conf->quiesce,
  730. *(conf->hash_locks + hash));
  731. }
  732. sh = find_get_stripe(conf, sector, conf->generation - previous,
  733. hash);
  734. if (sh)
  735. break;
  736. if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
  737. sh = get_free_stripe(conf, hash);
  738. if (sh) {
  739. r5c_check_stripe_cache_usage(conf);
  740. init_stripe(sh, sector, previous);
  741. atomic_inc(&sh->count);
  742. break;
  743. }
  744. if (!test_bit(R5_DID_ALLOC, &conf->cache_state))
  745. set_bit(R5_ALLOC_MORE, &conf->cache_state);
  746. }
  747. if (flags & R5_GAS_NOBLOCK)
  748. break;
  749. set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
  750. r5l_wake_reclaim(conf->log, 0);
  751. /* release batch_last before wait to avoid risk of deadlock */
  752. if (ctx && ctx->batch_last) {
  753. raid5_release_stripe(ctx->batch_last);
  754. ctx->batch_last = NULL;
  755. }
  756. wait_event_lock_irq(conf->wait_for_stripe,
  757. is_inactive_blocked(conf, hash),
  758. *(conf->hash_locks + hash));
  759. clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);
  760. }
  761. spin_unlock_irq(conf->hash_locks + hash);
  762. return sh;
  763. }
  764. static bool is_full_stripe_write(struct stripe_head *sh)
  765. {
  766. BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
  767. return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
  768. }
  769. static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
  770. __acquires(&sh1->stripe_lock)
  771. __acquires(&sh2->stripe_lock)
  772. {
  773. if (sh1 > sh2) {
  774. spin_lock_irq(&sh2->stripe_lock);
  775. spin_lock_nested(&sh1->stripe_lock, 1);
  776. } else {
  777. spin_lock_irq(&sh1->stripe_lock);
  778. spin_lock_nested(&sh2->stripe_lock, 1);
  779. }
  780. }
  781. static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
  782. __releases(&sh1->stripe_lock)
  783. __releases(&sh2->stripe_lock)
  784. {
  785. spin_unlock(&sh1->stripe_lock);
  786. spin_unlock_irq(&sh2->stripe_lock);
  787. }
  788. /* Only freshly new full stripe normal write stripe can be added to a batch list */
  789. static bool stripe_can_batch(struct stripe_head *sh)
  790. {
  791. struct r5conf *conf = sh->raid_conf;
  792. if (raid5_has_log(conf) || raid5_has_ppl(conf))
  793. return false;
  794. return test_bit(STRIPE_BATCH_READY, &sh->state) &&
  795. !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
  796. is_full_stripe_write(sh);
  797. }
  798. /* we only do back search */
  799. static void stripe_add_to_batch_list(struct r5conf *conf,
  800. struct stripe_head *sh, struct stripe_head *last_sh)
  801. {
  802. struct stripe_head *head;
  803. sector_t head_sector, tmp_sec;
  804. int hash;
  805. int dd_idx;
  806. /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
  807. tmp_sec = sh->sector;
  808. if (!sector_div(tmp_sec, conf->chunk_sectors))
  809. return;
  810. head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
  811. if (last_sh && head_sector == last_sh->sector) {
  812. head = last_sh;
  813. atomic_inc(&head->count);
  814. } else {
  815. hash = stripe_hash_locks_hash(conf, head_sector);
  816. spin_lock_irq(conf->hash_locks + hash);
  817. head = find_get_stripe(conf, head_sector, conf->generation,
  818. hash);
  819. spin_unlock_irq(conf->hash_locks + hash);
  820. if (!head)
  821. return;
  822. if (!stripe_can_batch(head))
  823. goto out;
  824. }
  825. lock_two_stripes(head, sh);
  826. /* clear_batch_ready clear the flag */
  827. if (!stripe_can_batch(head) || !stripe_can_batch(sh))
  828. goto unlock_out;
  829. if (sh->batch_head)
  830. goto unlock_out;
  831. dd_idx = 0;
  832. while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
  833. dd_idx++;
  834. if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||
  835. bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))
  836. goto unlock_out;
  837. if (head->batch_head) {
  838. spin_lock(&head->batch_head->batch_lock);
  839. /* This batch list is already running */
  840. if (!stripe_can_batch(head)) {
  841. spin_unlock(&head->batch_head->batch_lock);
  842. goto unlock_out;
  843. }
  844. /*
  845. * We must assign batch_head of this stripe within the
  846. * batch_lock, otherwise clear_batch_ready of batch head
  847. * stripe could clear BATCH_READY bit of this stripe and
  848. * this stripe->batch_head doesn't get assigned, which
  849. * could confuse clear_batch_ready for this stripe
  850. */
  851. sh->batch_head = head->batch_head;
  852. /*
  853. * at this point, head's BATCH_READY could be cleared, but we
  854. * can still add the stripe to batch list
  855. */
  856. list_add(&sh->batch_list, &head->batch_list);
  857. spin_unlock(&head->batch_head->batch_lock);
  858. } else {
  859. head->batch_head = head;
  860. sh->batch_head = head->batch_head;
  861. spin_lock(&head->batch_lock);
  862. list_add_tail(&sh->batch_list, &head->batch_list);
  863. spin_unlock(&head->batch_lock);
  864. }
  865. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  866. if (atomic_dec_return(&conf->preread_active_stripes)
  867. < IO_THRESHOLD)
  868. md_wakeup_thread(conf->mddev->thread);
  869. if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
  870. int seq = sh->bm_seq;
  871. if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
  872. sh->batch_head->bm_seq > seq)
  873. seq = sh->batch_head->bm_seq;
  874. set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
  875. sh->batch_head->bm_seq = seq;
  876. }
  877. atomic_inc(&sh->count);
  878. unlock_out:
  879. unlock_two_stripes(head, sh);
  880. out:
  881. raid5_release_stripe(head);
  882. }
  883. /* Determine if 'data_offset' or 'new_data_offset' should be used
  884. * in this stripe_head.
  885. */
  886. static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
  887. {
  888. sector_t progress = conf->reshape_progress;
  889. /* Need a memory barrier to make sure we see the value
  890. * of conf->generation, or ->data_offset that was set before
  891. * reshape_progress was updated.
  892. */
  893. smp_rmb();
  894. if (progress == MaxSector)
  895. return 0;
  896. if (sh->generation == conf->generation - 1)
  897. return 0;
  898. /* We are in a reshape, and this is a new-generation stripe,
  899. * so use new_data_offset.
  900. */
  901. return 1;
  902. }
  903. static void dispatch_bio_list(struct bio_list *tmp)
  904. {
  905. struct bio *bio;
  906. while ((bio = bio_list_pop(tmp)))
  907. submit_bio_noacct(bio);
  908. }
  909. static int cmp_stripe(void *priv, const struct list_head *a,
  910. const struct list_head *b)
  911. {
  912. const struct r5pending_data *da = list_entry(a,
  913. struct r5pending_data, sibling);
  914. const struct r5pending_data *db = list_entry(b,
  915. struct r5pending_data, sibling);
  916. if (da->sector > db->sector)
  917. return 1;
  918. if (da->sector < db->sector)
  919. return -1;
  920. return 0;
  921. }
  922. static void dispatch_defer_bios(struct r5conf *conf, int target,
  923. struct bio_list *list)
  924. {
  925. struct r5pending_data *data;
  926. struct list_head *first, *next = NULL;
  927. int cnt = 0;
  928. if (conf->pending_data_cnt == 0)
  929. return;
  930. list_sort(NULL, &conf->pending_list, cmp_stripe);
  931. first = conf->pending_list.next;
  932. /* temporarily move the head */
  933. if (conf->next_pending_data)
  934. list_move_tail(&conf->pending_list,
  935. &conf->next_pending_data->sibling);
  936. while (!list_empty(&conf->pending_list)) {
  937. data = list_first_entry(&conf->pending_list,
  938. struct r5pending_data, sibling);
  939. if (&data->sibling == first)
  940. first = data->sibling.next;
  941. next = data->sibling.next;
  942. bio_list_merge(list, &data->bios);
  943. list_move(&data->sibling, &conf->free_list);
  944. cnt++;
  945. if (cnt >= target)
  946. break;
  947. }
  948. conf->pending_data_cnt -= cnt;
  949. BUG_ON(conf->pending_data_cnt < 0 || cnt < target);
  950. if (next != &conf->pending_list)
  951. conf->next_pending_data = list_entry(next,
  952. struct r5pending_data, sibling);
  953. else
  954. conf->next_pending_data = NULL;
  955. /* list isn't empty */
  956. if (first != &conf->pending_list)
  957. list_move_tail(&conf->pending_list, first);
  958. }
  959. static void flush_deferred_bios(struct r5conf *conf)
  960. {
  961. struct bio_list tmp = BIO_EMPTY_LIST;
  962. if (conf->pending_data_cnt == 0)
  963. return;
  964. spin_lock(&conf->pending_bios_lock);
  965. dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);
  966. BUG_ON(conf->pending_data_cnt != 0);
  967. spin_unlock(&conf->pending_bios_lock);
  968. dispatch_bio_list(&tmp);
  969. }
  970. static void defer_issue_bios(struct r5conf *conf, sector_t sector,
  971. struct bio_list *bios)
  972. {
  973. struct bio_list tmp = BIO_EMPTY_LIST;
  974. struct r5pending_data *ent;
  975. spin_lock(&conf->pending_bios_lock);
  976. ent = list_first_entry(&conf->free_list, struct r5pending_data,
  977. sibling);
  978. list_move_tail(&ent->sibling, &conf->pending_list);
  979. ent->sector = sector;
  980. bio_list_init(&ent->bios);
  981. bio_list_merge(&ent->bios, bios);
  982. conf->pending_data_cnt++;
  983. if (conf->pending_data_cnt >= PENDING_IO_MAX)
  984. dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);
  985. spin_unlock(&conf->pending_bios_lock);
  986. dispatch_bio_list(&tmp);
  987. }
  988. static void
  989. raid5_end_read_request(struct bio *bi);
  990. static void
  991. raid5_end_write_request(struct bio *bi);
  992. static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
  993. {
  994. struct r5conf *conf = sh->raid_conf;
  995. int i, disks = sh->disks;
  996. struct stripe_head *head_sh = sh;
  997. struct bio_list pending_bios = BIO_EMPTY_LIST;
  998. struct r5dev *dev;
  999. bool should_defer;
  1000. might_sleep();
  1001. if (log_stripe(sh, s) == 0)
  1002. return;
  1003. should_defer = conf->batch_bio_dispatch && conf->group_cnt;
  1004. for (i = disks; i--; ) {
  1005. enum req_op op;
  1006. blk_opf_t op_flags = 0;
  1007. int replace_only = 0;
  1008. struct bio *bi, *rbi;
  1009. struct md_rdev *rdev, *rrdev = NULL;
  1010. sh = head_sh;
  1011. if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
  1012. op = REQ_OP_WRITE;
  1013. if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
  1014. op_flags = REQ_FUA;
  1015. if (test_bit(R5_Discard, &sh->dev[i].flags))
  1016. op = REQ_OP_DISCARD;
  1017. } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
  1018. op = REQ_OP_READ;
  1019. else if (test_and_clear_bit(R5_WantReplace,
  1020. &sh->dev[i].flags)) {
  1021. op = REQ_OP_WRITE;
  1022. replace_only = 1;
  1023. } else
  1024. continue;
  1025. if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
  1026. op_flags |= REQ_SYNC;
  1027. again:
  1028. dev = &sh->dev[i];
  1029. bi = &dev->req;
  1030. rbi = &dev->rreq; /* For writing to replacement */
  1031. rcu_read_lock();
  1032. rrdev = rcu_dereference(conf->disks[i].replacement);
  1033. smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
  1034. rdev = rcu_dereference(conf->disks[i].rdev);
  1035. if (!rdev) {
  1036. rdev = rrdev;
  1037. rrdev = NULL;
  1038. }
  1039. if (op_is_write(op)) {
  1040. if (replace_only)
  1041. rdev = NULL;
  1042. if (rdev == rrdev)
  1043. /* We raced and saw duplicates */
  1044. rrdev = NULL;
  1045. } else {
  1046. if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
  1047. rdev = rrdev;
  1048. rrdev = NULL;
  1049. }
  1050. if (rdev && test_bit(Faulty, &rdev->flags))
  1051. rdev = NULL;
  1052. if (rdev)
  1053. atomic_inc(&rdev->nr_pending);
  1054. if (rrdev && test_bit(Faulty, &rrdev->flags))
  1055. rrdev = NULL;
  1056. if (rrdev)
  1057. atomic_inc(&rrdev->nr_pending);
  1058. rcu_read_unlock();
  1059. /* We have already checked bad blocks for reads. Now
  1060. * need to check for writes. We never accept write errors
  1061. * on the replacement, so we don't to check rrdev.
  1062. */
  1063. while (op_is_write(op) && rdev &&
  1064. test_bit(WriteErrorSeen, &rdev->flags)) {
  1065. sector_t first_bad;
  1066. int bad_sectors;
  1067. int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
  1068. &first_bad, &bad_sectors);
  1069. if (!bad)
  1070. break;
  1071. if (bad < 0) {
  1072. set_bit(BlockedBadBlocks, &rdev->flags);
  1073. if (!conf->mddev->external &&
  1074. conf->mddev->sb_flags) {
  1075. /* It is very unlikely, but we might
  1076. * still need to write out the
  1077. * bad block log - better give it
  1078. * a chance*/
  1079. md_check_recovery(conf->mddev);
  1080. }
  1081. /*
  1082. * Because md_wait_for_blocked_rdev
  1083. * will dec nr_pending, we must
  1084. * increment it first.
  1085. */
  1086. atomic_inc(&rdev->nr_pending);
  1087. md_wait_for_blocked_rdev(rdev, conf->mddev);
  1088. } else {
  1089. /* Acknowledged bad block - skip the write */
  1090. rdev_dec_pending(rdev, conf->mddev);
  1091. rdev = NULL;
  1092. }
  1093. }
  1094. if (rdev) {
  1095. if (s->syncing || s->expanding || s->expanded
  1096. || s->replacing)
  1097. md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));
  1098. set_bit(STRIPE_IO_STARTED, &sh->state);
  1099. bio_init(bi, rdev->bdev, &dev->vec, 1, op | op_flags);
  1100. bi->bi_end_io = op_is_write(op)
  1101. ? raid5_end_write_request
  1102. : raid5_end_read_request;
  1103. bi->bi_private = sh;
  1104. pr_debug("%s: for %llu schedule op %d on disc %d\n",
  1105. __func__, (unsigned long long)sh->sector,
  1106. bi->bi_opf, i);
  1107. atomic_inc(&sh->count);
  1108. if (sh != head_sh)
  1109. atomic_inc(&head_sh->count);
  1110. if (use_new_offset(conf, sh))
  1111. bi->bi_iter.bi_sector = (sh->sector
  1112. + rdev->new_data_offset);
  1113. else
  1114. bi->bi_iter.bi_sector = (sh->sector
  1115. + rdev->data_offset);
  1116. if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
  1117. bi->bi_opf |= REQ_NOMERGE;
  1118. if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
  1119. WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
  1120. if (!op_is_write(op) &&
  1121. test_bit(R5_InJournal, &sh->dev[i].flags))
  1122. /*
  1123. * issuing read for a page in journal, this
  1124. * must be preparing for prexor in rmw; read
  1125. * the data into orig_page
  1126. */
  1127. sh->dev[i].vec.bv_page = sh->dev[i].orig_page;
  1128. else
  1129. sh->dev[i].vec.bv_page = sh->dev[i].page;
  1130. bi->bi_vcnt = 1;
  1131. bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
  1132. bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
  1133. bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
  1134. /*
  1135. * If this is discard request, set bi_vcnt 0. We don't
  1136. * want to confuse SCSI because SCSI will replace payload
  1137. */
  1138. if (op == REQ_OP_DISCARD)
  1139. bi->bi_vcnt = 0;
  1140. if (rrdev)
  1141. set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
  1142. if (conf->mddev->gendisk)
  1143. trace_block_bio_remap(bi,
  1144. disk_devt(conf->mddev->gendisk),
  1145. sh->dev[i].sector);
  1146. if (should_defer && op_is_write(op))
  1147. bio_list_add(&pending_bios, bi);
  1148. else
  1149. submit_bio_noacct(bi);
  1150. }
  1151. if (rrdev) {
  1152. if (s->syncing || s->expanding || s->expanded
  1153. || s->replacing)
  1154. md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));
  1155. set_bit(STRIPE_IO_STARTED, &sh->state);
  1156. bio_init(rbi, rrdev->bdev, &dev->rvec, 1, op | op_flags);
  1157. BUG_ON(!op_is_write(op));
  1158. rbi->bi_end_io = raid5_end_write_request;
  1159. rbi->bi_private = sh;
  1160. pr_debug("%s: for %llu schedule op %d on "
  1161. "replacement disc %d\n",
  1162. __func__, (unsigned long long)sh->sector,
  1163. rbi->bi_opf, i);
  1164. atomic_inc(&sh->count);
  1165. if (sh != head_sh)
  1166. atomic_inc(&head_sh->count);
  1167. if (use_new_offset(conf, sh))
  1168. rbi->bi_iter.bi_sector = (sh->sector
  1169. + rrdev->new_data_offset);
  1170. else
  1171. rbi->bi_iter.bi_sector = (sh->sector
  1172. + rrdev->data_offset);
  1173. if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
  1174. WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
  1175. sh->dev[i].rvec.bv_page = sh->dev[i].page;
  1176. rbi->bi_vcnt = 1;
  1177. rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
  1178. rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
  1179. rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
  1180. /*
  1181. * If this is discard request, set bi_vcnt 0. We don't
  1182. * want to confuse SCSI because SCSI will replace payload
  1183. */
  1184. if (op == REQ_OP_DISCARD)
  1185. rbi->bi_vcnt = 0;
  1186. if (conf->mddev->gendisk)
  1187. trace_block_bio_remap(rbi,
  1188. disk_devt(conf->mddev->gendisk),
  1189. sh->dev[i].sector);
  1190. if (should_defer && op_is_write(op))
  1191. bio_list_add(&pending_bios, rbi);
  1192. else
  1193. submit_bio_noacct(rbi);
  1194. }
  1195. if (!rdev && !rrdev) {
  1196. if (op_is_write(op))
  1197. set_bit(STRIPE_DEGRADED, &sh->state);
  1198. pr_debug("skip op %d on disc %d for sector %llu\n",
  1199. bi->bi_opf, i, (unsigned long long)sh->sector);
  1200. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  1201. set_bit(STRIPE_HANDLE, &sh->state);
  1202. }
  1203. if (!head_sh->batch_head)
  1204. continue;
  1205. sh = list_first_entry(&sh->batch_list, struct stripe_head,
  1206. batch_list);
  1207. if (sh != head_sh)
  1208. goto again;
  1209. }
  1210. if (should_defer && !bio_list_empty(&pending_bios))
  1211. defer_issue_bios(conf, head_sh->sector, &pending_bios);
  1212. }
  1213. static struct dma_async_tx_descriptor *
  1214. async_copy_data(int frombio, struct bio *bio, struct page **page,
  1215. unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
  1216. struct stripe_head *sh, int no_skipcopy)
  1217. {
  1218. struct bio_vec bvl;
  1219. struct bvec_iter iter;
  1220. struct page *bio_page;
  1221. int page_offset;
  1222. struct async_submit_ctl submit;
  1223. enum async_tx_flags flags = 0;
  1224. struct r5conf *conf = sh->raid_conf;
  1225. if (bio->bi_iter.bi_sector >= sector)
  1226. page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
  1227. else
  1228. page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
  1229. if (frombio)
  1230. flags |= ASYNC_TX_FENCE;
  1231. init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
  1232. bio_for_each_segment(bvl, bio, iter) {
  1233. int len = bvl.bv_len;
  1234. int clen;
  1235. int b_offset = 0;
  1236. if (page_offset < 0) {
  1237. b_offset = -page_offset;
  1238. page_offset += b_offset;
  1239. len -= b_offset;
  1240. }
  1241. if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))
  1242. clen = RAID5_STRIPE_SIZE(conf) - page_offset;
  1243. else
  1244. clen = len;
  1245. if (clen > 0) {
  1246. b_offset += bvl.bv_offset;
  1247. bio_page = bvl.bv_page;
  1248. if (frombio) {
  1249. if (conf->skip_copy &&
  1250. b_offset == 0 && page_offset == 0 &&
  1251. clen == RAID5_STRIPE_SIZE(conf) &&
  1252. !no_skipcopy)
  1253. *page = bio_page;
  1254. else
  1255. tx = async_memcpy(*page, bio_page, page_offset + poff,
  1256. b_offset, clen, &submit);
  1257. } else
  1258. tx = async_memcpy(bio_page, *page, b_offset,
  1259. page_offset + poff, clen, &submit);
  1260. }
  1261. /* chain the operations */
  1262. submit.depend_tx = tx;
  1263. if (clen < len) /* hit end of page */
  1264. break;
  1265. page_offset += len;
  1266. }
  1267. return tx;
  1268. }
  1269. static void ops_complete_biofill(void *stripe_head_ref)
  1270. {
  1271. struct stripe_head *sh = stripe_head_ref;
  1272. int i;
  1273. struct r5conf *conf = sh->raid_conf;
  1274. pr_debug("%s: stripe %llu\n", __func__,
  1275. (unsigned long long)sh->sector);
  1276. /* clear completed biofills */
  1277. for (i = sh->disks; i--; ) {
  1278. struct r5dev *dev = &sh->dev[i];
  1279. /* acknowledge completion of a biofill operation */
  1280. /* and check if we need to reply to a read request,
  1281. * new R5_Wantfill requests are held off until
  1282. * !STRIPE_BIOFILL_RUN
  1283. */
  1284. if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
  1285. struct bio *rbi, *rbi2;
  1286. BUG_ON(!dev->read);
  1287. rbi = dev->read;
  1288. dev->read = NULL;
  1289. while (rbi && rbi->bi_iter.bi_sector <
  1290. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  1291. rbi2 = r5_next_bio(conf, rbi, dev->sector);
  1292. bio_endio(rbi);
  1293. rbi = rbi2;
  1294. }
  1295. }
  1296. }
  1297. clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
  1298. set_bit(STRIPE_HANDLE, &sh->state);
  1299. raid5_release_stripe(sh);
  1300. }
  1301. static void ops_run_biofill(struct stripe_head *sh)
  1302. {
  1303. struct dma_async_tx_descriptor *tx = NULL;
  1304. struct async_submit_ctl submit;
  1305. int i;
  1306. struct r5conf *conf = sh->raid_conf;
  1307. BUG_ON(sh->batch_head);
  1308. pr_debug("%s: stripe %llu\n", __func__,
  1309. (unsigned long long)sh->sector);
  1310. for (i = sh->disks; i--; ) {
  1311. struct r5dev *dev = &sh->dev[i];
  1312. if (test_bit(R5_Wantfill, &dev->flags)) {
  1313. struct bio *rbi;
  1314. spin_lock_irq(&sh->stripe_lock);
  1315. dev->read = rbi = dev->toread;
  1316. dev->toread = NULL;
  1317. spin_unlock_irq(&sh->stripe_lock);
  1318. while (rbi && rbi->bi_iter.bi_sector <
  1319. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  1320. tx = async_copy_data(0, rbi, &dev->page,
  1321. dev->offset,
  1322. dev->sector, tx, sh, 0);
  1323. rbi = r5_next_bio(conf, rbi, dev->sector);
  1324. }
  1325. }
  1326. }
  1327. atomic_inc(&sh->count);
  1328. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
  1329. async_trigger_callback(&submit);
  1330. }
  1331. static void mark_target_uptodate(struct stripe_head *sh, int target)
  1332. {
  1333. struct r5dev *tgt;
  1334. if (target < 0)
  1335. return;
  1336. tgt = &sh->dev[target];
  1337. set_bit(R5_UPTODATE, &tgt->flags);
  1338. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1339. clear_bit(R5_Wantcompute, &tgt->flags);
  1340. }
  1341. static void ops_complete_compute(void *stripe_head_ref)
  1342. {
  1343. struct stripe_head *sh = stripe_head_ref;
  1344. pr_debug("%s: stripe %llu\n", __func__,
  1345. (unsigned long long)sh->sector);
  1346. /* mark the computed target(s) as uptodate */
  1347. mark_target_uptodate(sh, sh->ops.target);
  1348. mark_target_uptodate(sh, sh->ops.target2);
  1349. clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
  1350. if (sh->check_state == check_state_compute_run)
  1351. sh->check_state = check_state_compute_result;
  1352. set_bit(STRIPE_HANDLE, &sh->state);
  1353. raid5_release_stripe(sh);
  1354. }
  1355. /* return a pointer to the address conversion region of the scribble buffer */
  1356. static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
  1357. {
  1358. return percpu->scribble + i * percpu->scribble_obj_size;
  1359. }
  1360. /* return a pointer to the address conversion region of the scribble buffer */
  1361. static addr_conv_t *to_addr_conv(struct stripe_head *sh,
  1362. struct raid5_percpu *percpu, int i)
  1363. {
  1364. return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
  1365. }
  1366. /*
  1367. * Return a pointer to record offset address.
  1368. */
  1369. static unsigned int *
  1370. to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
  1371. {
  1372. return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
  1373. }
  1374. static struct dma_async_tx_descriptor *
  1375. ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
  1376. {
  1377. int disks = sh->disks;
  1378. struct page **xor_srcs = to_addr_page(percpu, 0);
  1379. unsigned int *off_srcs = to_addr_offs(sh, percpu);
  1380. int target = sh->ops.target;
  1381. struct r5dev *tgt = &sh->dev[target];
  1382. struct page *xor_dest = tgt->page;
  1383. unsigned int off_dest = tgt->offset;
  1384. int count = 0;
  1385. struct dma_async_tx_descriptor *tx;
  1386. struct async_submit_ctl submit;
  1387. int i;
  1388. BUG_ON(sh->batch_head);
  1389. pr_debug("%s: stripe %llu block: %d\n",
  1390. __func__, (unsigned long long)sh->sector, target);
  1391. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1392. for (i = disks; i--; ) {
  1393. if (i != target) {
  1394. off_srcs[count] = sh->dev[i].offset;
  1395. xor_srcs[count++] = sh->dev[i].page;
  1396. }
  1397. }
  1398. atomic_inc(&sh->count);
  1399. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
  1400. ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
  1401. if (unlikely(count == 1))
  1402. tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
  1403. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1404. else
  1405. tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1406. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1407. return tx;
  1408. }
  1409. /* set_syndrome_sources - populate source buffers for gen_syndrome
  1410. * @srcs - (struct page *) array of size sh->disks
  1411. * @offs - (unsigned int) array of offset for each page
  1412. * @sh - stripe_head to parse
  1413. *
  1414. * Populates srcs in proper layout order for the stripe and returns the
  1415. * 'count' of sources to be used in a call to async_gen_syndrome. The P
  1416. * destination buffer is recorded in srcs[count] and the Q destination
  1417. * is recorded in srcs[count+1]].
  1418. */
  1419. static int set_syndrome_sources(struct page **srcs,
  1420. unsigned int *offs,
  1421. struct stripe_head *sh,
  1422. int srctype)
  1423. {
  1424. int disks = sh->disks;
  1425. int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
  1426. int d0_idx = raid6_d0(sh);
  1427. int count;
  1428. int i;
  1429. for (i = 0; i < disks; i++)
  1430. srcs[i] = NULL;
  1431. count = 0;
  1432. i = d0_idx;
  1433. do {
  1434. int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
  1435. struct r5dev *dev = &sh->dev[i];
  1436. if (i == sh->qd_idx || i == sh->pd_idx ||
  1437. (srctype == SYNDROME_SRC_ALL) ||
  1438. (srctype == SYNDROME_SRC_WANT_DRAIN &&
  1439. (test_bit(R5_Wantdrain, &dev->flags) ||
  1440. test_bit(R5_InJournal, &dev->flags))) ||
  1441. (srctype == SYNDROME_SRC_WRITTEN &&
  1442. (dev->written ||
  1443. test_bit(R5_InJournal, &dev->flags)))) {
  1444. if (test_bit(R5_InJournal, &dev->flags))
  1445. srcs[slot] = sh->dev[i].orig_page;
  1446. else
  1447. srcs[slot] = sh->dev[i].page;
  1448. /*
  1449. * For R5_InJournal, PAGE_SIZE must be 4KB and will
  1450. * not shared page. In that case, dev[i].offset
  1451. * is 0.
  1452. */
  1453. offs[slot] = sh->dev[i].offset;
  1454. }
  1455. i = raid6_next_disk(i, disks);
  1456. } while (i != d0_idx);
  1457. return syndrome_disks;
  1458. }
  1459. static struct dma_async_tx_descriptor *
  1460. ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
  1461. {
  1462. int disks = sh->disks;
  1463. struct page **blocks = to_addr_page(percpu, 0);
  1464. unsigned int *offs = to_addr_offs(sh, percpu);
  1465. int target;
  1466. int qd_idx = sh->qd_idx;
  1467. struct dma_async_tx_descriptor *tx;
  1468. struct async_submit_ctl submit;
  1469. struct r5dev *tgt;
  1470. struct page *dest;
  1471. unsigned int dest_off;
  1472. int i;
  1473. int count;
  1474. BUG_ON(sh->batch_head);
  1475. if (sh->ops.target < 0)
  1476. target = sh->ops.target2;
  1477. else if (sh->ops.target2 < 0)
  1478. target = sh->ops.target;
  1479. else
  1480. /* we should only have one valid target */
  1481. BUG();
  1482. BUG_ON(target < 0);
  1483. pr_debug("%s: stripe %llu block: %d\n",
  1484. __func__, (unsigned long long)sh->sector, target);
  1485. tgt = &sh->dev[target];
  1486. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1487. dest = tgt->page;
  1488. dest_off = tgt->offset;
  1489. atomic_inc(&sh->count);
  1490. if (target == qd_idx) {
  1491. count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
  1492. blocks[count] = NULL; /* regenerating p is not necessary */
  1493. BUG_ON(blocks[count+1] != dest); /* q should already be set */
  1494. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  1495. ops_complete_compute, sh,
  1496. to_addr_conv(sh, percpu, 0));
  1497. tx = async_gen_syndrome(blocks, offs, count+2,
  1498. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1499. } else {
  1500. /* Compute any data- or p-drive using XOR */
  1501. count = 0;
  1502. for (i = disks; i-- ; ) {
  1503. if (i == target || i == qd_idx)
  1504. continue;
  1505. offs[count] = sh->dev[i].offset;
  1506. blocks[count++] = sh->dev[i].page;
  1507. }
  1508. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
  1509. NULL, ops_complete_compute, sh,
  1510. to_addr_conv(sh, percpu, 0));
  1511. tx = async_xor_offs(dest, dest_off, blocks, offs, count,
  1512. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1513. }
  1514. return tx;
  1515. }
  1516. static struct dma_async_tx_descriptor *
  1517. ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
  1518. {
  1519. int i, count, disks = sh->disks;
  1520. int syndrome_disks = sh->ddf_layout ? disks : disks-2;
  1521. int d0_idx = raid6_d0(sh);
  1522. int faila = -1, failb = -1;
  1523. int target = sh->ops.target;
  1524. int target2 = sh->ops.target2;
  1525. struct r5dev *tgt = &sh->dev[target];
  1526. struct r5dev *tgt2 = &sh->dev[target2];
  1527. struct dma_async_tx_descriptor *tx;
  1528. struct page **blocks = to_addr_page(percpu, 0);
  1529. unsigned int *offs = to_addr_offs(sh, percpu);
  1530. struct async_submit_ctl submit;
  1531. BUG_ON(sh->batch_head);
  1532. pr_debug("%s: stripe %llu block1: %d block2: %d\n",
  1533. __func__, (unsigned long long)sh->sector, target, target2);
  1534. BUG_ON(target < 0 || target2 < 0);
  1535. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1536. BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
  1537. /* we need to open-code set_syndrome_sources to handle the
  1538. * slot number conversion for 'faila' and 'failb'
  1539. */
  1540. for (i = 0; i < disks ; i++) {
  1541. offs[i] = 0;
  1542. blocks[i] = NULL;
  1543. }
  1544. count = 0;
  1545. i = d0_idx;
  1546. do {
  1547. int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
  1548. offs[slot] = sh->dev[i].offset;
  1549. blocks[slot] = sh->dev[i].page;
  1550. if (i == target)
  1551. faila = slot;
  1552. if (i == target2)
  1553. failb = slot;
  1554. i = raid6_next_disk(i, disks);
  1555. } while (i != d0_idx);
  1556. BUG_ON(faila == failb);
  1557. if (failb < faila)
  1558. swap(faila, failb);
  1559. pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
  1560. __func__, (unsigned long long)sh->sector, faila, failb);
  1561. atomic_inc(&sh->count);
  1562. if (failb == syndrome_disks+1) {
  1563. /* Q disk is one of the missing disks */
  1564. if (faila == syndrome_disks) {
  1565. /* Missing P+Q, just recompute */
  1566. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  1567. ops_complete_compute, sh,
  1568. to_addr_conv(sh, percpu, 0));
  1569. return async_gen_syndrome(blocks, offs, syndrome_disks+2,
  1570. RAID5_STRIPE_SIZE(sh->raid_conf),
  1571. &submit);
  1572. } else {
  1573. struct page *dest;
  1574. unsigned int dest_off;
  1575. int data_target;
  1576. int qd_idx = sh->qd_idx;
  1577. /* Missing D+Q: recompute D from P, then recompute Q */
  1578. if (target == qd_idx)
  1579. data_target = target2;
  1580. else
  1581. data_target = target;
  1582. count = 0;
  1583. for (i = disks; i-- ; ) {
  1584. if (i == data_target || i == qd_idx)
  1585. continue;
  1586. offs[count] = sh->dev[i].offset;
  1587. blocks[count++] = sh->dev[i].page;
  1588. }
  1589. dest = sh->dev[data_target].page;
  1590. dest_off = sh->dev[data_target].offset;
  1591. init_async_submit(&submit,
  1592. ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
  1593. NULL, NULL, NULL,
  1594. to_addr_conv(sh, percpu, 0));
  1595. tx = async_xor_offs(dest, dest_off, blocks, offs, count,
  1596. RAID5_STRIPE_SIZE(sh->raid_conf),
  1597. &submit);
  1598. count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
  1599. init_async_submit(&submit, ASYNC_TX_FENCE, tx,
  1600. ops_complete_compute, sh,
  1601. to_addr_conv(sh, percpu, 0));
  1602. return async_gen_syndrome(blocks, offs, count+2,
  1603. RAID5_STRIPE_SIZE(sh->raid_conf),
  1604. &submit);
  1605. }
  1606. } else {
  1607. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  1608. ops_complete_compute, sh,
  1609. to_addr_conv(sh, percpu, 0));
  1610. if (failb == syndrome_disks) {
  1611. /* We're missing D+P. */
  1612. return async_raid6_datap_recov(syndrome_disks+2,
  1613. RAID5_STRIPE_SIZE(sh->raid_conf),
  1614. faila,
  1615. blocks, offs, &submit);
  1616. } else {
  1617. /* We're missing D+D. */
  1618. return async_raid6_2data_recov(syndrome_disks+2,
  1619. RAID5_STRIPE_SIZE(sh->raid_conf),
  1620. faila, failb,
  1621. blocks, offs, &submit);
  1622. }
  1623. }
  1624. }
  1625. static void ops_complete_prexor(void *stripe_head_ref)
  1626. {
  1627. struct stripe_head *sh = stripe_head_ref;
  1628. pr_debug("%s: stripe %llu\n", __func__,
  1629. (unsigned long long)sh->sector);
  1630. if (r5c_is_writeback(sh->raid_conf->log))
  1631. /*
  1632. * raid5-cache write back uses orig_page during prexor.
  1633. * After prexor, it is time to free orig_page
  1634. */
  1635. r5c_release_extra_page(sh);
  1636. }
  1637. static struct dma_async_tx_descriptor *
  1638. ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
  1639. struct dma_async_tx_descriptor *tx)
  1640. {
  1641. int disks = sh->disks;
  1642. struct page **xor_srcs = to_addr_page(percpu, 0);
  1643. unsigned int *off_srcs = to_addr_offs(sh, percpu);
  1644. int count = 0, pd_idx = sh->pd_idx, i;
  1645. struct async_submit_ctl submit;
  1646. /* existing parity data subtracted */
  1647. unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
  1648. struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
  1649. BUG_ON(sh->batch_head);
  1650. pr_debug("%s: stripe %llu\n", __func__,
  1651. (unsigned long long)sh->sector);
  1652. for (i = disks; i--; ) {
  1653. struct r5dev *dev = &sh->dev[i];
  1654. /* Only process blocks that are known to be uptodate */
  1655. if (test_bit(R5_InJournal, &dev->flags)) {
  1656. /*
  1657. * For this case, PAGE_SIZE must be equal to 4KB and
  1658. * page offset is zero.
  1659. */
  1660. off_srcs[count] = dev->offset;
  1661. xor_srcs[count++] = dev->orig_page;
  1662. } else if (test_bit(R5_Wantdrain, &dev->flags)) {
  1663. off_srcs[count] = dev->offset;
  1664. xor_srcs[count++] = dev->page;
  1665. }
  1666. }
  1667. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  1668. ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
  1669. tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1670. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1671. return tx;
  1672. }
  1673. static struct dma_async_tx_descriptor *
  1674. ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
  1675. struct dma_async_tx_descriptor *tx)
  1676. {
  1677. struct page **blocks = to_addr_page(percpu, 0);
  1678. unsigned int *offs = to_addr_offs(sh, percpu);
  1679. int count;
  1680. struct async_submit_ctl submit;
  1681. pr_debug("%s: stripe %llu\n", __func__,
  1682. (unsigned long long)sh->sector);
  1683. count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
  1684. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
  1685. ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
  1686. tx = async_gen_syndrome(blocks, offs, count+2,
  1687. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1688. return tx;
  1689. }
  1690. static struct dma_async_tx_descriptor *
  1691. ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
  1692. {
  1693. struct r5conf *conf = sh->raid_conf;
  1694. int disks = sh->disks;
  1695. int i;
  1696. struct stripe_head *head_sh = sh;
  1697. pr_debug("%s: stripe %llu\n", __func__,
  1698. (unsigned long long)sh->sector);
  1699. for (i = disks; i--; ) {
  1700. struct r5dev *dev;
  1701. struct bio *chosen;
  1702. sh = head_sh;
  1703. if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
  1704. struct bio *wbi;
  1705. again:
  1706. dev = &sh->dev[i];
  1707. /*
  1708. * clear R5_InJournal, so when rewriting a page in
  1709. * journal, it is not skipped by r5l_log_stripe()
  1710. */
  1711. clear_bit(R5_InJournal, &dev->flags);
  1712. spin_lock_irq(&sh->stripe_lock);
  1713. chosen = dev->towrite;
  1714. dev->towrite = NULL;
  1715. sh->overwrite_disks = 0;
  1716. BUG_ON(dev->written);
  1717. wbi = dev->written = chosen;
  1718. spin_unlock_irq(&sh->stripe_lock);
  1719. WARN_ON(dev->page != dev->orig_page);
  1720. while (wbi && wbi->bi_iter.bi_sector <
  1721. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  1722. if (wbi->bi_opf & REQ_FUA)
  1723. set_bit(R5_WantFUA, &dev->flags);
  1724. if (wbi->bi_opf & REQ_SYNC)
  1725. set_bit(R5_SyncIO, &dev->flags);
  1726. if (bio_op(wbi) == REQ_OP_DISCARD)
  1727. set_bit(R5_Discard, &dev->flags);
  1728. else {
  1729. tx = async_copy_data(1, wbi, &dev->page,
  1730. dev->offset,
  1731. dev->sector, tx, sh,
  1732. r5c_is_writeback(conf->log));
  1733. if (dev->page != dev->orig_page &&
  1734. !r5c_is_writeback(conf->log)) {
  1735. set_bit(R5_SkipCopy, &dev->flags);
  1736. clear_bit(R5_UPTODATE, &dev->flags);
  1737. clear_bit(R5_OVERWRITE, &dev->flags);
  1738. }
  1739. }
  1740. wbi = r5_next_bio(conf, wbi, dev->sector);
  1741. }
  1742. if (head_sh->batch_head) {
  1743. sh = list_first_entry(&sh->batch_list,
  1744. struct stripe_head,
  1745. batch_list);
  1746. if (sh == head_sh)
  1747. continue;
  1748. goto again;
  1749. }
  1750. }
  1751. }
  1752. return tx;
  1753. }
  1754. static void ops_complete_reconstruct(void *stripe_head_ref)
  1755. {
  1756. struct stripe_head *sh = stripe_head_ref;
  1757. int disks = sh->disks;
  1758. int pd_idx = sh->pd_idx;
  1759. int qd_idx = sh->qd_idx;
  1760. int i;
  1761. bool fua = false, sync = false, discard = false;
  1762. pr_debug("%s: stripe %llu\n", __func__,
  1763. (unsigned long long)sh->sector);
  1764. for (i = disks; i--; ) {
  1765. fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
  1766. sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
  1767. discard |= test_bit(R5_Discard, &sh->dev[i].flags);
  1768. }
  1769. for (i = disks; i--; ) {
  1770. struct r5dev *dev = &sh->dev[i];
  1771. if (dev->written || i == pd_idx || i == qd_idx) {
  1772. if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) {
  1773. set_bit(R5_UPTODATE, &dev->flags);
  1774. if (test_bit(STRIPE_EXPAND_READY, &sh->state))
  1775. set_bit(R5_Expanded, &dev->flags);
  1776. }
  1777. if (fua)
  1778. set_bit(R5_WantFUA, &dev->flags);
  1779. if (sync)
  1780. set_bit(R5_SyncIO, &dev->flags);
  1781. }
  1782. }
  1783. if (sh->reconstruct_state == reconstruct_state_drain_run)
  1784. sh->reconstruct_state = reconstruct_state_drain_result;
  1785. else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
  1786. sh->reconstruct_state = reconstruct_state_prexor_drain_result;
  1787. else {
  1788. BUG_ON(sh->reconstruct_state != reconstruct_state_run);
  1789. sh->reconstruct_state = reconstruct_state_result;
  1790. }
  1791. set_bit(STRIPE_HANDLE, &sh->state);
  1792. raid5_release_stripe(sh);
  1793. }
  1794. static void
  1795. ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
  1796. struct dma_async_tx_descriptor *tx)
  1797. {
  1798. int disks = sh->disks;
  1799. struct page **xor_srcs;
  1800. unsigned int *off_srcs;
  1801. struct async_submit_ctl submit;
  1802. int count, pd_idx = sh->pd_idx, i;
  1803. struct page *xor_dest;
  1804. unsigned int off_dest;
  1805. int prexor = 0;
  1806. unsigned long flags;
  1807. int j = 0;
  1808. struct stripe_head *head_sh = sh;
  1809. int last_stripe;
  1810. pr_debug("%s: stripe %llu\n", __func__,
  1811. (unsigned long long)sh->sector);
  1812. for (i = 0; i < sh->disks; i++) {
  1813. if (pd_idx == i)
  1814. continue;
  1815. if (!test_bit(R5_Discard, &sh->dev[i].flags))
  1816. break;
  1817. }
  1818. if (i >= sh->disks) {
  1819. atomic_inc(&sh->count);
  1820. set_bit(R5_Discard, &sh->dev[pd_idx].flags);
  1821. ops_complete_reconstruct(sh);
  1822. return;
  1823. }
  1824. again:
  1825. count = 0;
  1826. xor_srcs = to_addr_page(percpu, j);
  1827. off_srcs = to_addr_offs(sh, percpu);
  1828. /* check if prexor is active which means only process blocks
  1829. * that are part of a read-modify-write (written)
  1830. */
  1831. if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
  1832. prexor = 1;
  1833. off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
  1834. xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
  1835. for (i = disks; i--; ) {
  1836. struct r5dev *dev = &sh->dev[i];
  1837. if (head_sh->dev[i].written ||
  1838. test_bit(R5_InJournal, &head_sh->dev[i].flags)) {
  1839. off_srcs[count] = dev->offset;
  1840. xor_srcs[count++] = dev->page;
  1841. }
  1842. }
  1843. } else {
  1844. xor_dest = sh->dev[pd_idx].page;
  1845. off_dest = sh->dev[pd_idx].offset;
  1846. for (i = disks; i--; ) {
  1847. struct r5dev *dev = &sh->dev[i];
  1848. if (i != pd_idx) {
  1849. off_srcs[count] = dev->offset;
  1850. xor_srcs[count++] = dev->page;
  1851. }
  1852. }
  1853. }
  1854. /* 1/ if we prexor'd then the dest is reused as a source
  1855. * 2/ if we did not prexor then we are redoing the parity
  1856. * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
  1857. * for the synchronous xor case
  1858. */
  1859. last_stripe = !head_sh->batch_head ||
  1860. list_first_entry(&sh->batch_list,
  1861. struct stripe_head, batch_list) == head_sh;
  1862. if (last_stripe) {
  1863. flags = ASYNC_TX_ACK |
  1864. (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
  1865. atomic_inc(&head_sh->count);
  1866. init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
  1867. to_addr_conv(sh, percpu, j));
  1868. } else {
  1869. flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
  1870. init_async_submit(&submit, flags, tx, NULL, NULL,
  1871. to_addr_conv(sh, percpu, j));
  1872. }
  1873. if (unlikely(count == 1))
  1874. tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
  1875. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1876. else
  1877. tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1878. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1879. if (!last_stripe) {
  1880. j++;
  1881. sh = list_first_entry(&sh->batch_list, struct stripe_head,
  1882. batch_list);
  1883. goto again;
  1884. }
  1885. }
  1886. static void
  1887. ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
  1888. struct dma_async_tx_descriptor *tx)
  1889. {
  1890. struct async_submit_ctl submit;
  1891. struct page **blocks;
  1892. unsigned int *offs;
  1893. int count, i, j = 0;
  1894. struct stripe_head *head_sh = sh;
  1895. int last_stripe;
  1896. int synflags;
  1897. unsigned long txflags;
  1898. pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
  1899. for (i = 0; i < sh->disks; i++) {
  1900. if (sh->pd_idx == i || sh->qd_idx == i)
  1901. continue;
  1902. if (!test_bit(R5_Discard, &sh->dev[i].flags))
  1903. break;
  1904. }
  1905. if (i >= sh->disks) {
  1906. atomic_inc(&sh->count);
  1907. set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
  1908. set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
  1909. ops_complete_reconstruct(sh);
  1910. return;
  1911. }
  1912. again:
  1913. blocks = to_addr_page(percpu, j);
  1914. offs = to_addr_offs(sh, percpu);
  1915. if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
  1916. synflags = SYNDROME_SRC_WRITTEN;
  1917. txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
  1918. } else {
  1919. synflags = SYNDROME_SRC_ALL;
  1920. txflags = ASYNC_TX_ACK;
  1921. }
  1922. count = set_syndrome_sources(blocks, offs, sh, synflags);
  1923. last_stripe = !head_sh->batch_head ||
  1924. list_first_entry(&sh->batch_list,
  1925. struct stripe_head, batch_list) == head_sh;
  1926. if (last_stripe) {
  1927. atomic_inc(&head_sh->count);
  1928. init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
  1929. head_sh, to_addr_conv(sh, percpu, j));
  1930. } else
  1931. init_async_submit(&submit, 0, tx, NULL, NULL,
  1932. to_addr_conv(sh, percpu, j));
  1933. tx = async_gen_syndrome(blocks, offs, count+2,
  1934. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1935. if (!last_stripe) {
  1936. j++;
  1937. sh = list_first_entry(&sh->batch_list, struct stripe_head,
  1938. batch_list);
  1939. goto again;
  1940. }
  1941. }
  1942. static void ops_complete_check(void *stripe_head_ref)
  1943. {
  1944. struct stripe_head *sh = stripe_head_ref;
  1945. pr_debug("%s: stripe %llu\n", __func__,
  1946. (unsigned long long)sh->sector);
  1947. sh->check_state = check_state_check_result;
  1948. set_bit(STRIPE_HANDLE, &sh->state);
  1949. raid5_release_stripe(sh);
  1950. }
  1951. static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
  1952. {
  1953. int disks = sh->disks;
  1954. int pd_idx = sh->pd_idx;
  1955. int qd_idx = sh->qd_idx;
  1956. struct page *xor_dest;
  1957. unsigned int off_dest;
  1958. struct page **xor_srcs = to_addr_page(percpu, 0);
  1959. unsigned int *off_srcs = to_addr_offs(sh, percpu);
  1960. struct dma_async_tx_descriptor *tx;
  1961. struct async_submit_ctl submit;
  1962. int count;
  1963. int i;
  1964. pr_debug("%s: stripe %llu\n", __func__,
  1965. (unsigned long long)sh->sector);
  1966. BUG_ON(sh->batch_head);
  1967. count = 0;
  1968. xor_dest = sh->dev[pd_idx].page;
  1969. off_dest = sh->dev[pd_idx].offset;
  1970. off_srcs[count] = off_dest;
  1971. xor_srcs[count++] = xor_dest;
  1972. for (i = disks; i--; ) {
  1973. if (i == pd_idx || i == qd_idx)
  1974. continue;
  1975. off_srcs[count] = sh->dev[i].offset;
  1976. xor_srcs[count++] = sh->dev[i].page;
  1977. }
  1978. init_async_submit(&submit, 0, NULL, NULL, NULL,
  1979. to_addr_conv(sh, percpu, 0));
  1980. tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1981. RAID5_STRIPE_SIZE(sh->raid_conf),
  1982. &sh->ops.zero_sum_result, &submit);
  1983. atomic_inc(&sh->count);
  1984. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
  1985. tx = async_trigger_callback(&submit);
  1986. }
  1987. static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
  1988. {
  1989. struct page **srcs = to_addr_page(percpu, 0);
  1990. unsigned int *offs = to_addr_offs(sh, percpu);
  1991. struct async_submit_ctl submit;
  1992. int count;
  1993. pr_debug("%s: stripe %llu checkp: %d\n", __func__,
  1994. (unsigned long long)sh->sector, checkp);
  1995. BUG_ON(sh->batch_head);
  1996. count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
  1997. if (!checkp)
  1998. srcs[count] = NULL;
  1999. atomic_inc(&sh->count);
  2000. init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
  2001. sh, to_addr_conv(sh, percpu, 0));
  2002. async_syndrome_val(srcs, offs, count+2,
  2003. RAID5_STRIPE_SIZE(sh->raid_conf),
  2004. &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
  2005. }
  2006. static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
  2007. {
  2008. int overlap_clear = 0, i, disks = sh->disks;
  2009. struct dma_async_tx_descriptor *tx = NULL;
  2010. struct r5conf *conf = sh->raid_conf;
  2011. int level = conf->level;
  2012. struct raid5_percpu *percpu;
  2013. local_lock(&conf->percpu->lock);
  2014. percpu = this_cpu_ptr(conf->percpu);
  2015. if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
  2016. ops_run_biofill(sh);
  2017. overlap_clear++;
  2018. }
  2019. if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
  2020. if (level < 6)
  2021. tx = ops_run_compute5(sh, percpu);
  2022. else {
  2023. if (sh->ops.target2 < 0 || sh->ops.target < 0)
  2024. tx = ops_run_compute6_1(sh, percpu);
  2025. else
  2026. tx = ops_run_compute6_2(sh, percpu);
  2027. }
  2028. /* terminate the chain if reconstruct is not set to be run */
  2029. if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
  2030. async_tx_ack(tx);
  2031. }
  2032. if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
  2033. if (level < 6)
  2034. tx = ops_run_prexor5(sh, percpu, tx);
  2035. else
  2036. tx = ops_run_prexor6(sh, percpu, tx);
  2037. }
  2038. if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
  2039. tx = ops_run_partial_parity(sh, percpu, tx);
  2040. if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
  2041. tx = ops_run_biodrain(sh, tx);
  2042. overlap_clear++;
  2043. }
  2044. if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
  2045. if (level < 6)
  2046. ops_run_reconstruct5(sh, percpu, tx);
  2047. else
  2048. ops_run_reconstruct6(sh, percpu, tx);
  2049. }
  2050. if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
  2051. if (sh->check_state == check_state_run)
  2052. ops_run_check_p(sh, percpu);
  2053. else if (sh->check_state == check_state_run_q)
  2054. ops_run_check_pq(sh, percpu, 0);
  2055. else if (sh->check_state == check_state_run_pq)
  2056. ops_run_check_pq(sh, percpu, 1);
  2057. else
  2058. BUG();
  2059. }
  2060. if (overlap_clear && !sh->batch_head) {
  2061. for (i = disks; i--; ) {
  2062. struct r5dev *dev = &sh->dev[i];
  2063. if (test_and_clear_bit(R5_Overlap, &dev->flags))
  2064. wake_up(&sh->raid_conf->wait_for_overlap);
  2065. }
  2066. }
  2067. local_unlock(&conf->percpu->lock);
  2068. }
  2069. static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
  2070. {
  2071. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2072. kfree(sh->pages);
  2073. #endif
  2074. if (sh->ppl_page)
  2075. __free_page(sh->ppl_page);
  2076. kmem_cache_free(sc, sh);
  2077. }
  2078. static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
  2079. int disks, struct r5conf *conf)
  2080. {
  2081. struct stripe_head *sh;
  2082. sh = kmem_cache_zalloc(sc, gfp);
  2083. if (sh) {
  2084. spin_lock_init(&sh->stripe_lock);
  2085. spin_lock_init(&sh->batch_lock);
  2086. INIT_LIST_HEAD(&sh->batch_list);
  2087. INIT_LIST_HEAD(&sh->lru);
  2088. INIT_LIST_HEAD(&sh->r5c);
  2089. INIT_LIST_HEAD(&sh->log_list);
  2090. atomic_set(&sh->count, 1);
  2091. sh->raid_conf = conf;
  2092. sh->log_start = MaxSector;
  2093. if (raid5_has_ppl(conf)) {
  2094. sh->ppl_page = alloc_page(gfp);
  2095. if (!sh->ppl_page) {
  2096. free_stripe(sc, sh);
  2097. return NULL;
  2098. }
  2099. }
  2100. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2101. if (init_stripe_shared_pages(sh, conf, disks)) {
  2102. free_stripe(sc, sh);
  2103. return NULL;
  2104. }
  2105. #endif
  2106. }
  2107. return sh;
  2108. }
  2109. static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
  2110. {
  2111. struct stripe_head *sh;
  2112. sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
  2113. if (!sh)
  2114. return 0;
  2115. if (grow_buffers(sh, gfp)) {
  2116. shrink_buffers(sh);
  2117. free_stripe(conf->slab_cache, sh);
  2118. return 0;
  2119. }
  2120. sh->hash_lock_index =
  2121. conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
  2122. /* we just created an active stripe so... */
  2123. atomic_inc(&conf->active_stripes);
  2124. raid5_release_stripe(sh);
  2125. conf->max_nr_stripes++;
  2126. return 1;
  2127. }
  2128. static int grow_stripes(struct r5conf *conf, int num)
  2129. {
  2130. struct kmem_cache *sc;
  2131. size_t namelen = sizeof(conf->cache_name[0]);
  2132. int devs = max(conf->raid_disks, conf->previous_raid_disks);
  2133. if (conf->mddev->gendisk)
  2134. snprintf(conf->cache_name[0], namelen,
  2135. "raid%d-%s", conf->level, mdname(conf->mddev));
  2136. else
  2137. snprintf(conf->cache_name[0], namelen,
  2138. "raid%d-%p", conf->level, conf->mddev);
  2139. snprintf(conf->cache_name[1], namelen, "%.27s-alt", conf->cache_name[0]);
  2140. conf->active_name = 0;
  2141. sc = kmem_cache_create(conf->cache_name[conf->active_name],
  2142. sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
  2143. 0, 0, NULL);
  2144. if (!sc)
  2145. return 1;
  2146. conf->slab_cache = sc;
  2147. conf->pool_size = devs;
  2148. while (num--)
  2149. if (!grow_one_stripe(conf, GFP_KERNEL))
  2150. return 1;
  2151. return 0;
  2152. }
  2153. /**
  2154. * scribble_alloc - allocate percpu scribble buffer for required size
  2155. * of the scribble region
  2156. * @percpu: from for_each_present_cpu() of the caller
  2157. * @num: total number of disks in the array
  2158. * @cnt: scribble objs count for required size of the scribble region
  2159. *
  2160. * The scribble buffer size must be enough to contain:
  2161. * 1/ a struct page pointer for each device in the array +2
  2162. * 2/ room to convert each entry in (1) to its corresponding dma
  2163. * (dma_map_page()) or page (page_address()) address.
  2164. *
  2165. * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
  2166. * calculate over all devices (not just the data blocks), using zeros in place
  2167. * of the P and Q blocks.
  2168. */
  2169. static int scribble_alloc(struct raid5_percpu *percpu,
  2170. int num, int cnt)
  2171. {
  2172. size_t obj_size =
  2173. sizeof(struct page *) * (num + 2) +
  2174. sizeof(addr_conv_t) * (num + 2) +
  2175. sizeof(unsigned int) * (num + 2);
  2176. void *scribble;
  2177. /*
  2178. * If here is in raid array suspend context, it is in memalloc noio
  2179. * context as well, there is no potential recursive memory reclaim
  2180. * I/Os with the GFP_KERNEL flag.
  2181. */
  2182. scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
  2183. if (!scribble)
  2184. return -ENOMEM;
  2185. kvfree(percpu->scribble);
  2186. percpu->scribble = scribble;
  2187. percpu->scribble_obj_size = obj_size;
  2188. return 0;
  2189. }
  2190. static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
  2191. {
  2192. unsigned long cpu;
  2193. int err = 0;
  2194. /*
  2195. * Never shrink. And mddev_suspend() could deadlock if this is called
  2196. * from raid5d. In that case, scribble_disks and scribble_sectors
  2197. * should equal to new_disks and new_sectors
  2198. */
  2199. if (conf->scribble_disks >= new_disks &&
  2200. conf->scribble_sectors >= new_sectors)
  2201. return 0;
  2202. mddev_suspend(conf->mddev);
  2203. cpus_read_lock();
  2204. for_each_present_cpu(cpu) {
  2205. struct raid5_percpu *percpu;
  2206. percpu = per_cpu_ptr(conf->percpu, cpu);
  2207. err = scribble_alloc(percpu, new_disks,
  2208. new_sectors / RAID5_STRIPE_SECTORS(conf));
  2209. if (err)
  2210. break;
  2211. }
  2212. cpus_read_unlock();
  2213. mddev_resume(conf->mddev);
  2214. if (!err) {
  2215. conf->scribble_disks = new_disks;
  2216. conf->scribble_sectors = new_sectors;
  2217. }
  2218. return err;
  2219. }
  2220. static int resize_stripes(struct r5conf *conf, int newsize)
  2221. {
  2222. /* Make all the stripes able to hold 'newsize' devices.
  2223. * New slots in each stripe get 'page' set to a new page.
  2224. *
  2225. * This happens in stages:
  2226. * 1/ create a new kmem_cache and allocate the required number of
  2227. * stripe_heads.
  2228. * 2/ gather all the old stripe_heads and transfer the pages across
  2229. * to the new stripe_heads. This will have the side effect of
  2230. * freezing the array as once all stripe_heads have been collected,
  2231. * no IO will be possible. Old stripe heads are freed once their
  2232. * pages have been transferred over, and the old kmem_cache is
  2233. * freed when all stripes are done.
  2234. * 3/ reallocate conf->disks to be suitable bigger. If this fails,
  2235. * we simple return a failure status - no need to clean anything up.
  2236. * 4/ allocate new pages for the new slots in the new stripe_heads.
  2237. * If this fails, we don't bother trying the shrink the
  2238. * stripe_heads down again, we just leave them as they are.
  2239. * As each stripe_head is processed the new one is released into
  2240. * active service.
  2241. *
  2242. * Once step2 is started, we cannot afford to wait for a write,
  2243. * so we use GFP_NOIO allocations.
  2244. */
  2245. struct stripe_head *osh, *nsh;
  2246. LIST_HEAD(newstripes);
  2247. struct disk_info *ndisks;
  2248. int err = 0;
  2249. struct kmem_cache *sc;
  2250. int i;
  2251. int hash, cnt;
  2252. md_allow_write(conf->mddev);
  2253. /* Step 1 */
  2254. sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
  2255. sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
  2256. 0, 0, NULL);
  2257. if (!sc)
  2258. return -ENOMEM;
  2259. /* Need to ensure auto-resizing doesn't interfere */
  2260. mutex_lock(&conf->cache_size_mutex);
  2261. for (i = conf->max_nr_stripes; i; i--) {
  2262. nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);
  2263. if (!nsh)
  2264. break;
  2265. list_add(&nsh->lru, &newstripes);
  2266. }
  2267. if (i) {
  2268. /* didn't get enough, give up */
  2269. while (!list_empty(&newstripes)) {
  2270. nsh = list_entry(newstripes.next, struct stripe_head, lru);
  2271. list_del(&nsh->lru);
  2272. free_stripe(sc, nsh);
  2273. }
  2274. kmem_cache_destroy(sc);
  2275. mutex_unlock(&conf->cache_size_mutex);
  2276. return -ENOMEM;
  2277. }
  2278. /* Step 2 - Must use GFP_NOIO now.
  2279. * OK, we have enough stripes, start collecting inactive
  2280. * stripes and copying them over
  2281. */
  2282. hash = 0;
  2283. cnt = 0;
  2284. list_for_each_entry(nsh, &newstripes, lru) {
  2285. lock_device_hash_lock(conf, hash);
  2286. wait_event_cmd(conf->wait_for_stripe,
  2287. !list_empty(conf->inactive_list + hash),
  2288. unlock_device_hash_lock(conf, hash),
  2289. lock_device_hash_lock(conf, hash));
  2290. osh = get_free_stripe(conf, hash);
  2291. unlock_device_hash_lock(conf, hash);
  2292. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2293. for (i = 0; i < osh->nr_pages; i++) {
  2294. nsh->pages[i] = osh->pages[i];
  2295. osh->pages[i] = NULL;
  2296. }
  2297. #endif
  2298. for(i=0; i<conf->pool_size; i++) {
  2299. nsh->dev[i].page = osh->dev[i].page;
  2300. nsh->dev[i].orig_page = osh->dev[i].page;
  2301. nsh->dev[i].offset = osh->dev[i].offset;
  2302. }
  2303. nsh->hash_lock_index = hash;
  2304. free_stripe(conf->slab_cache, osh);
  2305. cnt++;
  2306. if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
  2307. !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
  2308. hash++;
  2309. cnt = 0;
  2310. }
  2311. }
  2312. kmem_cache_destroy(conf->slab_cache);
  2313. /* Step 3.
  2314. * At this point, we are holding all the stripes so the array
  2315. * is completely stalled, so now is a good time to resize
  2316. * conf->disks and the scribble region
  2317. */
  2318. ndisks = kcalloc(newsize, sizeof(struct disk_info), GFP_NOIO);
  2319. if (ndisks) {
  2320. for (i = 0; i < conf->pool_size; i++)
  2321. ndisks[i] = conf->disks[i];
  2322. for (i = conf->pool_size; i < newsize; i++) {
  2323. ndisks[i].extra_page = alloc_page(GFP_NOIO);
  2324. if (!ndisks[i].extra_page)
  2325. err = -ENOMEM;
  2326. }
  2327. if (err) {
  2328. for (i = conf->pool_size; i < newsize; i++)
  2329. if (ndisks[i].extra_page)
  2330. put_page(ndisks[i].extra_page);
  2331. kfree(ndisks);
  2332. } else {
  2333. kfree(conf->disks);
  2334. conf->disks = ndisks;
  2335. }
  2336. } else
  2337. err = -ENOMEM;
  2338. conf->slab_cache = sc;
  2339. conf->active_name = 1-conf->active_name;
  2340. /* Step 4, return new stripes to service */
  2341. while(!list_empty(&newstripes)) {
  2342. nsh = list_entry(newstripes.next, struct stripe_head, lru);
  2343. list_del_init(&nsh->lru);
  2344. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2345. for (i = 0; i < nsh->nr_pages; i++) {
  2346. if (nsh->pages[i])
  2347. continue;
  2348. nsh->pages[i] = alloc_page(GFP_NOIO);
  2349. if (!nsh->pages[i])
  2350. err = -ENOMEM;
  2351. }
  2352. for (i = conf->raid_disks; i < newsize; i++) {
  2353. if (nsh->dev[i].page)
  2354. continue;
  2355. nsh->dev[i].page = raid5_get_dev_page(nsh, i);
  2356. nsh->dev[i].orig_page = nsh->dev[i].page;
  2357. nsh->dev[i].offset = raid5_get_page_offset(nsh, i);
  2358. }
  2359. #else
  2360. for (i=conf->raid_disks; i < newsize; i++)
  2361. if (nsh->dev[i].page == NULL) {
  2362. struct page *p = alloc_page(GFP_NOIO);
  2363. nsh->dev[i].page = p;
  2364. nsh->dev[i].orig_page = p;
  2365. nsh->dev[i].offset = 0;
  2366. if (!p)
  2367. err = -ENOMEM;
  2368. }
  2369. #endif
  2370. raid5_release_stripe(nsh);
  2371. }
  2372. /* critical section pass, GFP_NOIO no longer needed */
  2373. if (!err)
  2374. conf->pool_size = newsize;
  2375. mutex_unlock(&conf->cache_size_mutex);
  2376. return err;
  2377. }
  2378. static int drop_one_stripe(struct r5conf *conf)
  2379. {
  2380. struct stripe_head *sh;
  2381. int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;
  2382. spin_lock_irq(conf->hash_locks + hash);
  2383. sh = get_free_stripe(conf, hash);
  2384. spin_unlock_irq(conf->hash_locks + hash);
  2385. if (!sh)
  2386. return 0;
  2387. BUG_ON(atomic_read(&sh->count));
  2388. shrink_buffers(sh);
  2389. free_stripe(conf->slab_cache, sh);
  2390. atomic_dec(&conf->active_stripes);
  2391. conf->max_nr_stripes--;
  2392. return 1;
  2393. }
  2394. static void shrink_stripes(struct r5conf *conf)
  2395. {
  2396. while (conf->max_nr_stripes &&
  2397. drop_one_stripe(conf))
  2398. ;
  2399. kmem_cache_destroy(conf->slab_cache);
  2400. conf->slab_cache = NULL;
  2401. }
  2402. /*
  2403. * This helper wraps rcu_dereference_protected() and can be used when
  2404. * it is known that the nr_pending of the rdev is elevated.
  2405. */
  2406. static struct md_rdev *rdev_pend_deref(struct md_rdev __rcu *rdev)
  2407. {
  2408. return rcu_dereference_protected(rdev,
  2409. atomic_read(&rcu_access_pointer(rdev)->nr_pending));
  2410. }
  2411. /*
  2412. * This helper wraps rcu_dereference_protected() and should be used
  2413. * when it is known that the mddev_lock() is held. This is safe
  2414. * seeing raid5_remove_disk() has the same lock held.
  2415. */
  2416. static struct md_rdev *rdev_mdlock_deref(struct mddev *mddev,
  2417. struct md_rdev __rcu *rdev)
  2418. {
  2419. return rcu_dereference_protected(rdev,
  2420. lockdep_is_held(&mddev->reconfig_mutex));
  2421. }
  2422. static void raid5_end_read_request(struct bio * bi)
  2423. {
  2424. struct stripe_head *sh = bi->bi_private;
  2425. struct r5conf *conf = sh->raid_conf;
  2426. int disks = sh->disks, i;
  2427. struct md_rdev *rdev = NULL;
  2428. sector_t s;
  2429. for (i=0 ; i<disks; i++)
  2430. if (bi == &sh->dev[i].req)
  2431. break;
  2432. pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",
  2433. (unsigned long long)sh->sector, i, atomic_read(&sh->count),
  2434. bi->bi_status);
  2435. if (i == disks) {
  2436. BUG();
  2437. return;
  2438. }
  2439. if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
  2440. /* If replacement finished while this request was outstanding,
  2441. * 'replacement' might be NULL already.
  2442. * In that case it moved down to 'rdev'.
  2443. * rdev is not removed until all requests are finished.
  2444. */
  2445. rdev = rdev_pend_deref(conf->disks[i].replacement);
  2446. if (!rdev)
  2447. rdev = rdev_pend_deref(conf->disks[i].rdev);
  2448. if (use_new_offset(conf, sh))
  2449. s = sh->sector + rdev->new_data_offset;
  2450. else
  2451. s = sh->sector + rdev->data_offset;
  2452. if (!bi->bi_status) {
  2453. set_bit(R5_UPTODATE, &sh->dev[i].flags);
  2454. if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
  2455. /* Note that this cannot happen on a
  2456. * replacement device. We just fail those on
  2457. * any error
  2458. */
  2459. pr_info_ratelimited(
  2460. "md/raid:%s: read error corrected (%lu sectors at %llu on %pg)\n",
  2461. mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),
  2462. (unsigned long long)s,
  2463. rdev->bdev);
  2464. atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);
  2465. clear_bit(R5_ReadError, &sh->dev[i].flags);
  2466. clear_bit(R5_ReWrite, &sh->dev[i].flags);
  2467. } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
  2468. clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  2469. if (test_bit(R5_InJournal, &sh->dev[i].flags))
  2470. /*
  2471. * end read for a page in journal, this
  2472. * must be preparing for prexor in rmw
  2473. */
  2474. set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
  2475. if (atomic_read(&rdev->read_errors))
  2476. atomic_set(&rdev->read_errors, 0);
  2477. } else {
  2478. int retry = 0;
  2479. int set_bad = 0;
  2480. clear_bit(R5_UPTODATE, &sh->dev[i].flags);
  2481. if (!(bi->bi_status == BLK_STS_PROTECTION))
  2482. atomic_inc(&rdev->read_errors);
  2483. if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
  2484. pr_warn_ratelimited(
  2485. "md/raid:%s: read error on replacement device (sector %llu on %pg).\n",
  2486. mdname(conf->mddev),
  2487. (unsigned long long)s,
  2488. rdev->bdev);
  2489. else if (conf->mddev->degraded >= conf->max_degraded) {
  2490. set_bad = 1;
  2491. pr_warn_ratelimited(
  2492. "md/raid:%s: read error not correctable (sector %llu on %pg).\n",
  2493. mdname(conf->mddev),
  2494. (unsigned long long)s,
  2495. rdev->bdev);
  2496. } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
  2497. /* Oh, no!!! */
  2498. set_bad = 1;
  2499. pr_warn_ratelimited(
  2500. "md/raid:%s: read error NOT corrected!! (sector %llu on %pg).\n",
  2501. mdname(conf->mddev),
  2502. (unsigned long long)s,
  2503. rdev->bdev);
  2504. } else if (atomic_read(&rdev->read_errors)
  2505. > conf->max_nr_stripes) {
  2506. if (!test_bit(Faulty, &rdev->flags)) {
  2507. pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
  2508. mdname(conf->mddev),
  2509. atomic_read(&rdev->read_errors),
  2510. conf->max_nr_stripes);
  2511. pr_warn("md/raid:%s: Too many read errors, failing device %pg.\n",
  2512. mdname(conf->mddev), rdev->bdev);
  2513. }
  2514. } else
  2515. retry = 1;
  2516. if (set_bad && test_bit(In_sync, &rdev->flags)
  2517. && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
  2518. retry = 1;
  2519. if (retry)
  2520. if (sh->qd_idx >= 0 && sh->pd_idx == i)
  2521. set_bit(R5_ReadError, &sh->dev[i].flags);
  2522. else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
  2523. set_bit(R5_ReadError, &sh->dev[i].flags);
  2524. clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  2525. } else
  2526. set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  2527. else {
  2528. clear_bit(R5_ReadError, &sh->dev[i].flags);
  2529. clear_bit(R5_ReWrite, &sh->dev[i].flags);
  2530. if (!(set_bad
  2531. && test_bit(In_sync, &rdev->flags)
  2532. && rdev_set_badblocks(
  2533. rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
  2534. md_error(conf->mddev, rdev);
  2535. }
  2536. }
  2537. rdev_dec_pending(rdev, conf->mddev);
  2538. bio_uninit(bi);
  2539. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  2540. set_bit(STRIPE_HANDLE, &sh->state);
  2541. raid5_release_stripe(sh);
  2542. }
  2543. static void raid5_end_write_request(struct bio *bi)
  2544. {
  2545. struct stripe_head *sh = bi->bi_private;
  2546. struct r5conf *conf = sh->raid_conf;
  2547. int disks = sh->disks, i;
  2548. struct md_rdev *rdev;
  2549. sector_t first_bad;
  2550. int bad_sectors;
  2551. int replacement = 0;
  2552. for (i = 0 ; i < disks; i++) {
  2553. if (bi == &sh->dev[i].req) {
  2554. rdev = rdev_pend_deref(conf->disks[i].rdev);
  2555. break;
  2556. }
  2557. if (bi == &sh->dev[i].rreq) {
  2558. rdev = rdev_pend_deref(conf->disks[i].replacement);
  2559. if (rdev)
  2560. replacement = 1;
  2561. else
  2562. /* rdev was removed and 'replacement'
  2563. * replaced it. rdev is not removed
  2564. * until all requests are finished.
  2565. */
  2566. rdev = rdev_pend_deref(conf->disks[i].rdev);
  2567. break;
  2568. }
  2569. }
  2570. pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",
  2571. (unsigned long long)sh->sector, i, atomic_read(&sh->count),
  2572. bi->bi_status);
  2573. if (i == disks) {
  2574. BUG();
  2575. return;
  2576. }
  2577. if (replacement) {
  2578. if (bi->bi_status)
  2579. md_error(conf->mddev, rdev);
  2580. else if (is_badblock(rdev, sh->sector,
  2581. RAID5_STRIPE_SECTORS(conf),
  2582. &first_bad, &bad_sectors))
  2583. set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
  2584. } else {
  2585. if (bi->bi_status) {
  2586. set_bit(STRIPE_DEGRADED, &sh->state);
  2587. set_bit(WriteErrorSeen, &rdev->flags);
  2588. set_bit(R5_WriteError, &sh->dev[i].flags);
  2589. if (!test_and_set_bit(WantReplacement, &rdev->flags))
  2590. set_bit(MD_RECOVERY_NEEDED,
  2591. &rdev->mddev->recovery);
  2592. } else if (is_badblock(rdev, sh->sector,
  2593. RAID5_STRIPE_SECTORS(conf),
  2594. &first_bad, &bad_sectors)) {
  2595. set_bit(R5_MadeGood, &sh->dev[i].flags);
  2596. if (test_bit(R5_ReadError, &sh->dev[i].flags))
  2597. /* That was a successful write so make
  2598. * sure it looks like we already did
  2599. * a re-write.
  2600. */
  2601. set_bit(R5_ReWrite, &sh->dev[i].flags);
  2602. }
  2603. }
  2604. rdev_dec_pending(rdev, conf->mddev);
  2605. if (sh->batch_head && bi->bi_status && !replacement)
  2606. set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
  2607. bio_uninit(bi);
  2608. if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
  2609. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  2610. set_bit(STRIPE_HANDLE, &sh->state);
  2611. if (sh->batch_head && sh != sh->batch_head)
  2612. raid5_release_stripe(sh->batch_head);
  2613. raid5_release_stripe(sh);
  2614. }
  2615. static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)
  2616. {
  2617. struct r5conf *conf = mddev->private;
  2618. unsigned long flags;
  2619. pr_debug("raid456: error called\n");
  2620. pr_crit("md/raid:%s: Disk failure on %pg, disabling device.\n",
  2621. mdname(mddev), rdev->bdev);
  2622. spin_lock_irqsave(&conf->device_lock, flags);
  2623. set_bit(Faulty, &rdev->flags);
  2624. clear_bit(In_sync, &rdev->flags);
  2625. mddev->degraded = raid5_calc_degraded(conf);
  2626. if (has_failed(conf)) {
  2627. set_bit(MD_BROKEN, &conf->mddev->flags);
  2628. conf->recovery_disabled = mddev->recovery_disabled;
  2629. pr_crit("md/raid:%s: Cannot continue operation (%d/%d failed).\n",
  2630. mdname(mddev), mddev->degraded, conf->raid_disks);
  2631. } else {
  2632. pr_crit("md/raid:%s: Operation continuing on %d devices.\n",
  2633. mdname(mddev), conf->raid_disks - mddev->degraded);
  2634. }
  2635. spin_unlock_irqrestore(&conf->device_lock, flags);
  2636. set_bit(MD_RECOVERY_INTR, &mddev->recovery);
  2637. set_bit(Blocked, &rdev->flags);
  2638. set_mask_bits(&mddev->sb_flags, 0,
  2639. BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
  2640. r5c_update_on_rdev_error(mddev, rdev);
  2641. }
  2642. /*
  2643. * Input: a 'big' sector number,
  2644. * Output: index of the data and parity disk, and the sector # in them.
  2645. */
  2646. sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
  2647. int previous, int *dd_idx,
  2648. struct stripe_head *sh)
  2649. {
  2650. sector_t stripe, stripe2;
  2651. sector_t chunk_number;
  2652. unsigned int chunk_offset;
  2653. int pd_idx, qd_idx;
  2654. int ddf_layout = 0;
  2655. sector_t new_sector;
  2656. int algorithm = previous ? conf->prev_algo
  2657. : conf->algorithm;
  2658. int sectors_per_chunk = previous ? conf->prev_chunk_sectors
  2659. : conf->chunk_sectors;
  2660. int raid_disks = previous ? conf->previous_raid_disks
  2661. : conf->raid_disks;
  2662. int data_disks = raid_disks - conf->max_degraded;
  2663. /* First compute the information on this sector */
  2664. /*
  2665. * Compute the chunk number and the sector offset inside the chunk
  2666. */
  2667. chunk_offset = sector_div(r_sector, sectors_per_chunk);
  2668. chunk_number = r_sector;
  2669. /*
  2670. * Compute the stripe number
  2671. */
  2672. stripe = chunk_number;
  2673. *dd_idx = sector_div(stripe, data_disks);
  2674. stripe2 = stripe;
  2675. /*
  2676. * Select the parity disk based on the user selected algorithm.
  2677. */
  2678. pd_idx = qd_idx = -1;
  2679. switch(conf->level) {
  2680. case 4:
  2681. pd_idx = data_disks;
  2682. break;
  2683. case 5:
  2684. switch (algorithm) {
  2685. case ALGORITHM_LEFT_ASYMMETRIC:
  2686. pd_idx = data_disks - sector_div(stripe2, raid_disks);
  2687. if (*dd_idx >= pd_idx)
  2688. (*dd_idx)++;
  2689. break;
  2690. case ALGORITHM_RIGHT_ASYMMETRIC:
  2691. pd_idx = sector_div(stripe2, raid_disks);
  2692. if (*dd_idx >= pd_idx)
  2693. (*dd_idx)++;
  2694. break;
  2695. case ALGORITHM_LEFT_SYMMETRIC:
  2696. pd_idx = data_disks - sector_div(stripe2, raid_disks);
  2697. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  2698. break;
  2699. case ALGORITHM_RIGHT_SYMMETRIC:
  2700. pd_idx = sector_div(stripe2, raid_disks);
  2701. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  2702. break;
  2703. case ALGORITHM_PARITY_0:
  2704. pd_idx = 0;
  2705. (*dd_idx)++;
  2706. break;
  2707. case ALGORITHM_PARITY_N:
  2708. pd_idx = data_disks;
  2709. break;
  2710. default:
  2711. BUG();
  2712. }
  2713. break;
  2714. case 6:
  2715. switch (algorithm) {
  2716. case ALGORITHM_LEFT_ASYMMETRIC:
  2717. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2718. qd_idx = pd_idx + 1;
  2719. if (pd_idx == raid_disks-1) {
  2720. (*dd_idx)++; /* Q D D D P */
  2721. qd_idx = 0;
  2722. } else if (*dd_idx >= pd_idx)
  2723. (*dd_idx) += 2; /* D D P Q D */
  2724. break;
  2725. case ALGORITHM_RIGHT_ASYMMETRIC:
  2726. pd_idx = sector_div(stripe2, raid_disks);
  2727. qd_idx = pd_idx + 1;
  2728. if (pd_idx == raid_disks-1) {
  2729. (*dd_idx)++; /* Q D D D P */
  2730. qd_idx = 0;
  2731. } else if (*dd_idx >= pd_idx)
  2732. (*dd_idx) += 2; /* D D P Q D */
  2733. break;
  2734. case ALGORITHM_LEFT_SYMMETRIC:
  2735. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2736. qd_idx = (pd_idx + 1) % raid_disks;
  2737. *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
  2738. break;
  2739. case ALGORITHM_RIGHT_SYMMETRIC:
  2740. pd_idx = sector_div(stripe2, raid_disks);
  2741. qd_idx = (pd_idx + 1) % raid_disks;
  2742. *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
  2743. break;
  2744. case ALGORITHM_PARITY_0:
  2745. pd_idx = 0;
  2746. qd_idx = 1;
  2747. (*dd_idx) += 2;
  2748. break;
  2749. case ALGORITHM_PARITY_N:
  2750. pd_idx = data_disks;
  2751. qd_idx = data_disks + 1;
  2752. break;
  2753. case ALGORITHM_ROTATING_ZERO_RESTART:
  2754. /* Exactly the same as RIGHT_ASYMMETRIC, but or
  2755. * of blocks for computing Q is different.
  2756. */
  2757. pd_idx = sector_div(stripe2, raid_disks);
  2758. qd_idx = pd_idx + 1;
  2759. if (pd_idx == raid_disks-1) {
  2760. (*dd_idx)++; /* Q D D D P */
  2761. qd_idx = 0;
  2762. } else if (*dd_idx >= pd_idx)
  2763. (*dd_idx) += 2; /* D D P Q D */
  2764. ddf_layout = 1;
  2765. break;
  2766. case ALGORITHM_ROTATING_N_RESTART:
  2767. /* Same a left_asymmetric, by first stripe is
  2768. * D D D P Q rather than
  2769. * Q D D D P
  2770. */
  2771. stripe2 += 1;
  2772. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2773. qd_idx = pd_idx + 1;
  2774. if (pd_idx == raid_disks-1) {
  2775. (*dd_idx)++; /* Q D D D P */
  2776. qd_idx = 0;
  2777. } else if (*dd_idx >= pd_idx)
  2778. (*dd_idx) += 2; /* D D P Q D */
  2779. ddf_layout = 1;
  2780. break;
  2781. case ALGORITHM_ROTATING_N_CONTINUE:
  2782. /* Same as left_symmetric but Q is before P */
  2783. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2784. qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
  2785. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  2786. ddf_layout = 1;
  2787. break;
  2788. case ALGORITHM_LEFT_ASYMMETRIC_6:
  2789. /* RAID5 left_asymmetric, with Q on last device */
  2790. pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
  2791. if (*dd_idx >= pd_idx)
  2792. (*dd_idx)++;
  2793. qd_idx = raid_disks - 1;
  2794. break;
  2795. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  2796. pd_idx = sector_div(stripe2, raid_disks-1);
  2797. if (*dd_idx >= pd_idx)
  2798. (*dd_idx)++;
  2799. qd_idx = raid_disks - 1;
  2800. break;
  2801. case ALGORITHM_LEFT_SYMMETRIC_6:
  2802. pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
  2803. *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
  2804. qd_idx = raid_disks - 1;
  2805. break;
  2806. case ALGORITHM_RIGHT_SYMMETRIC_6:
  2807. pd_idx = sector_div(stripe2, raid_disks-1);
  2808. *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
  2809. qd_idx = raid_disks - 1;
  2810. break;
  2811. case ALGORITHM_PARITY_0_6:
  2812. pd_idx = 0;
  2813. (*dd_idx)++;
  2814. qd_idx = raid_disks - 1;
  2815. break;
  2816. default:
  2817. BUG();
  2818. }
  2819. break;
  2820. }
  2821. if (sh) {
  2822. sh->pd_idx = pd_idx;
  2823. sh->qd_idx = qd_idx;
  2824. sh->ddf_layout = ddf_layout;
  2825. }
  2826. /*
  2827. * Finally, compute the new sector number
  2828. */
  2829. new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
  2830. return new_sector;
  2831. }
  2832. sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
  2833. {
  2834. struct r5conf *conf = sh->raid_conf;
  2835. int raid_disks = sh->disks;
  2836. int data_disks = raid_disks - conf->max_degraded;
  2837. sector_t new_sector = sh->sector, check;
  2838. int sectors_per_chunk = previous ? conf->prev_chunk_sectors
  2839. : conf->chunk_sectors;
  2840. int algorithm = previous ? conf->prev_algo
  2841. : conf->algorithm;
  2842. sector_t stripe;
  2843. int chunk_offset;
  2844. sector_t chunk_number;
  2845. int dummy1, dd_idx = i;
  2846. sector_t r_sector;
  2847. struct stripe_head sh2;
  2848. chunk_offset = sector_div(new_sector, sectors_per_chunk);
  2849. stripe = new_sector;
  2850. if (i == sh->pd_idx)
  2851. return 0;
  2852. switch(conf->level) {
  2853. case 4: break;
  2854. case 5:
  2855. switch (algorithm) {
  2856. case ALGORITHM_LEFT_ASYMMETRIC:
  2857. case ALGORITHM_RIGHT_ASYMMETRIC:
  2858. if (i > sh->pd_idx)
  2859. i--;
  2860. break;
  2861. case ALGORITHM_LEFT_SYMMETRIC:
  2862. case ALGORITHM_RIGHT_SYMMETRIC:
  2863. if (i < sh->pd_idx)
  2864. i += raid_disks;
  2865. i -= (sh->pd_idx + 1);
  2866. break;
  2867. case ALGORITHM_PARITY_0:
  2868. i -= 1;
  2869. break;
  2870. case ALGORITHM_PARITY_N:
  2871. break;
  2872. default:
  2873. BUG();
  2874. }
  2875. break;
  2876. case 6:
  2877. if (i == sh->qd_idx)
  2878. return 0; /* It is the Q disk */
  2879. switch (algorithm) {
  2880. case ALGORITHM_LEFT_ASYMMETRIC:
  2881. case ALGORITHM_RIGHT_ASYMMETRIC:
  2882. case ALGORITHM_ROTATING_ZERO_RESTART:
  2883. case ALGORITHM_ROTATING_N_RESTART:
  2884. if (sh->pd_idx == raid_disks-1)
  2885. i--; /* Q D D D P */
  2886. else if (i > sh->pd_idx)
  2887. i -= 2; /* D D P Q D */
  2888. break;
  2889. case ALGORITHM_LEFT_SYMMETRIC:
  2890. case ALGORITHM_RIGHT_SYMMETRIC:
  2891. if (sh->pd_idx == raid_disks-1)
  2892. i--; /* Q D D D P */
  2893. else {
  2894. /* D D P Q D */
  2895. if (i < sh->pd_idx)
  2896. i += raid_disks;
  2897. i -= (sh->pd_idx + 2);
  2898. }
  2899. break;
  2900. case ALGORITHM_PARITY_0:
  2901. i -= 2;
  2902. break;
  2903. case ALGORITHM_PARITY_N:
  2904. break;
  2905. case ALGORITHM_ROTATING_N_CONTINUE:
  2906. /* Like left_symmetric, but P is before Q */
  2907. if (sh->pd_idx == 0)
  2908. i--; /* P D D D Q */
  2909. else {
  2910. /* D D Q P D */
  2911. if (i < sh->pd_idx)
  2912. i += raid_disks;
  2913. i -= (sh->pd_idx + 1);
  2914. }
  2915. break;
  2916. case ALGORITHM_LEFT_ASYMMETRIC_6:
  2917. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  2918. if (i > sh->pd_idx)
  2919. i--;
  2920. break;
  2921. case ALGORITHM_LEFT_SYMMETRIC_6:
  2922. case ALGORITHM_RIGHT_SYMMETRIC_6:
  2923. if (i < sh->pd_idx)
  2924. i += data_disks + 1;
  2925. i -= (sh->pd_idx + 1);
  2926. break;
  2927. case ALGORITHM_PARITY_0_6:
  2928. i -= 1;
  2929. break;
  2930. default:
  2931. BUG();
  2932. }
  2933. break;
  2934. }
  2935. chunk_number = stripe * data_disks + i;
  2936. r_sector = chunk_number * sectors_per_chunk + chunk_offset;
  2937. check = raid5_compute_sector(conf, r_sector,
  2938. previous, &dummy1, &sh2);
  2939. if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
  2940. || sh2.qd_idx != sh->qd_idx) {
  2941. pr_warn("md/raid:%s: compute_blocknr: map not correct\n",
  2942. mdname(conf->mddev));
  2943. return 0;
  2944. }
  2945. return r_sector;
  2946. }
  2947. /*
  2948. * There are cases where we want handle_stripe_dirtying() and
  2949. * schedule_reconstruction() to delay towrite to some dev of a stripe.
  2950. *
  2951. * This function checks whether we want to delay the towrite. Specifically,
  2952. * we delay the towrite when:
  2953. *
  2954. * 1. degraded stripe has a non-overwrite to the missing dev, AND this
  2955. * stripe has data in journal (for other devices).
  2956. *
  2957. * In this case, when reading data for the non-overwrite dev, it is
  2958. * necessary to handle complex rmw of write back cache (prexor with
  2959. * orig_page, and xor with page). To keep read path simple, we would
  2960. * like to flush data in journal to RAID disks first, so complex rmw
  2961. * is handled in the write patch (handle_stripe_dirtying).
  2962. *
  2963. * 2. when journal space is critical (R5C_LOG_CRITICAL=1)
  2964. *
  2965. * It is important to be able to flush all stripes in raid5-cache.
  2966. * Therefore, we need reserve some space on the journal device for
  2967. * these flushes. If flush operation includes pending writes to the
  2968. * stripe, we need to reserve (conf->raid_disk + 1) pages per stripe
  2969. * for the flush out. If we exclude these pending writes from flush
  2970. * operation, we only need (conf->max_degraded + 1) pages per stripe.
  2971. * Therefore, excluding pending writes in these cases enables more
  2972. * efficient use of the journal device.
  2973. *
  2974. * Note: To make sure the stripe makes progress, we only delay
  2975. * towrite for stripes with data already in journal (injournal > 0).
  2976. * When LOG_CRITICAL, stripes with injournal == 0 will be sent to
  2977. * no_space_stripes list.
  2978. *
  2979. * 3. during journal failure
  2980. * In journal failure, we try to flush all cached data to raid disks
  2981. * based on data in stripe cache. The array is read-only to upper
  2982. * layers, so we would skip all pending writes.
  2983. *
  2984. */
  2985. static inline bool delay_towrite(struct r5conf *conf,
  2986. struct r5dev *dev,
  2987. struct stripe_head_state *s)
  2988. {
  2989. /* case 1 above */
  2990. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  2991. !test_bit(R5_Insync, &dev->flags) && s->injournal)
  2992. return true;
  2993. /* case 2 above */
  2994. if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
  2995. s->injournal > 0)
  2996. return true;
  2997. /* case 3 above */
  2998. if (s->log_failed && s->injournal)
  2999. return true;
  3000. return false;
  3001. }
  3002. static void
  3003. schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
  3004. int rcw, int expand)
  3005. {
  3006. int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
  3007. struct r5conf *conf = sh->raid_conf;
  3008. int level = conf->level;
  3009. if (rcw) {
  3010. /*
  3011. * In some cases, handle_stripe_dirtying initially decided to
  3012. * run rmw and allocates extra page for prexor. However, rcw is
  3013. * cheaper later on. We need to free the extra page now,
  3014. * because we won't be able to do that in ops_complete_prexor().
  3015. */
  3016. r5c_release_extra_page(sh);
  3017. for (i = disks; i--; ) {
  3018. struct r5dev *dev = &sh->dev[i];
  3019. if (dev->towrite && !delay_towrite(conf, dev, s)) {
  3020. set_bit(R5_LOCKED, &dev->flags);
  3021. set_bit(R5_Wantdrain, &dev->flags);
  3022. if (!expand)
  3023. clear_bit(R5_UPTODATE, &dev->flags);
  3024. s->locked++;
  3025. } else if (test_bit(R5_InJournal, &dev->flags)) {
  3026. set_bit(R5_LOCKED, &dev->flags);
  3027. s->locked++;
  3028. }
  3029. }
  3030. /* if we are not expanding this is a proper write request, and
  3031. * there will be bios with new data to be drained into the
  3032. * stripe cache
  3033. */
  3034. if (!expand) {
  3035. if (!s->locked)
  3036. /* False alarm, nothing to do */
  3037. return;
  3038. sh->reconstruct_state = reconstruct_state_drain_run;
  3039. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  3040. } else
  3041. sh->reconstruct_state = reconstruct_state_run;
  3042. set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
  3043. if (s->locked + conf->max_degraded == disks)
  3044. if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
  3045. atomic_inc(&conf->pending_full_writes);
  3046. } else {
  3047. BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
  3048. test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
  3049. BUG_ON(level == 6 &&
  3050. (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
  3051. test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
  3052. for (i = disks; i--; ) {
  3053. struct r5dev *dev = &sh->dev[i];
  3054. if (i == pd_idx || i == qd_idx)
  3055. continue;
  3056. if (dev->towrite &&
  3057. (test_bit(R5_UPTODATE, &dev->flags) ||
  3058. test_bit(R5_Wantcompute, &dev->flags))) {
  3059. set_bit(R5_Wantdrain, &dev->flags);
  3060. set_bit(R5_LOCKED, &dev->flags);
  3061. clear_bit(R5_UPTODATE, &dev->flags);
  3062. s->locked++;
  3063. } else if (test_bit(R5_InJournal, &dev->flags)) {
  3064. set_bit(R5_LOCKED, &dev->flags);
  3065. s->locked++;
  3066. }
  3067. }
  3068. if (!s->locked)
  3069. /* False alarm - nothing to do */
  3070. return;
  3071. sh->reconstruct_state = reconstruct_state_prexor_drain_run;
  3072. set_bit(STRIPE_OP_PREXOR, &s->ops_request);
  3073. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  3074. set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
  3075. }
  3076. /* keep the parity disk(s) locked while asynchronous operations
  3077. * are in flight
  3078. */
  3079. set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
  3080. clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
  3081. s->locked++;
  3082. if (level == 6) {
  3083. int qd_idx = sh->qd_idx;
  3084. struct r5dev *dev = &sh->dev[qd_idx];
  3085. set_bit(R5_LOCKED, &dev->flags);
  3086. clear_bit(R5_UPTODATE, &dev->flags);
  3087. s->locked++;
  3088. }
  3089. if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
  3090. test_bit(STRIPE_OP_BIODRAIN, &s->ops_request) &&
  3091. !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
  3092. test_bit(R5_Insync, &sh->dev[pd_idx].flags))
  3093. set_bit(STRIPE_OP_PARTIAL_PARITY, &s->ops_request);
  3094. pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
  3095. __func__, (unsigned long long)sh->sector,
  3096. s->locked, s->ops_request);
  3097. }
  3098. static bool stripe_bio_overlaps(struct stripe_head *sh, struct bio *bi,
  3099. int dd_idx, int forwrite)
  3100. {
  3101. struct r5conf *conf = sh->raid_conf;
  3102. struct bio **bip;
  3103. pr_debug("checking bi b#%llu to stripe s#%llu\n",
  3104. bi->bi_iter.bi_sector, sh->sector);
  3105. /* Don't allow new IO added to stripes in batch list */
  3106. if (sh->batch_head)
  3107. return true;
  3108. if (forwrite)
  3109. bip = &sh->dev[dd_idx].towrite;
  3110. else
  3111. bip = &sh->dev[dd_idx].toread;
  3112. while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
  3113. if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
  3114. return true;
  3115. bip = &(*bip)->bi_next;
  3116. }
  3117. if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
  3118. return true;
  3119. if (forwrite && raid5_has_ppl(conf)) {
  3120. /*
  3121. * With PPL only writes to consecutive data chunks within a
  3122. * stripe are allowed because for a single stripe_head we can
  3123. * only have one PPL entry at a time, which describes one data
  3124. * range. Not really an overlap, but wait_for_overlap can be
  3125. * used to handle this.
  3126. */
  3127. sector_t sector;
  3128. sector_t first = 0;
  3129. sector_t last = 0;
  3130. int count = 0;
  3131. int i;
  3132. for (i = 0; i < sh->disks; i++) {
  3133. if (i != sh->pd_idx &&
  3134. (i == dd_idx || sh->dev[i].towrite)) {
  3135. sector = sh->dev[i].sector;
  3136. if (count == 0 || sector < first)
  3137. first = sector;
  3138. if (sector > last)
  3139. last = sector;
  3140. count++;
  3141. }
  3142. }
  3143. if (first + conf->chunk_sectors * (count - 1) != last)
  3144. return true;
  3145. }
  3146. return false;
  3147. }
  3148. static void __add_stripe_bio(struct stripe_head *sh, struct bio *bi,
  3149. int dd_idx, int forwrite, int previous)
  3150. {
  3151. struct r5conf *conf = sh->raid_conf;
  3152. struct bio **bip;
  3153. int firstwrite = 0;
  3154. if (forwrite) {
  3155. bip = &sh->dev[dd_idx].towrite;
  3156. if (!*bip)
  3157. firstwrite = 1;
  3158. } else {
  3159. bip = &sh->dev[dd_idx].toread;
  3160. }
  3161. while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector)
  3162. bip = &(*bip)->bi_next;
  3163. if (!forwrite || previous)
  3164. clear_bit(STRIPE_BATCH_READY, &sh->state);
  3165. BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
  3166. if (*bip)
  3167. bi->bi_next = *bip;
  3168. *bip = bi;
  3169. bio_inc_remaining(bi);
  3170. md_write_inc(conf->mddev, bi);
  3171. if (forwrite) {
  3172. /* check if page is covered */
  3173. sector_t sector = sh->dev[dd_idx].sector;
  3174. for (bi=sh->dev[dd_idx].towrite;
  3175. sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
  3176. bi && bi->bi_iter.bi_sector <= sector;
  3177. bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
  3178. if (bio_end_sector(bi) >= sector)
  3179. sector = bio_end_sector(bi);
  3180. }
  3181. if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
  3182. if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
  3183. sh->overwrite_disks++;
  3184. }
  3185. pr_debug("added bi b#%llu to stripe s#%llu, disk %d, logical %llu\n",
  3186. (*bip)->bi_iter.bi_sector, sh->sector, dd_idx,
  3187. sh->dev[dd_idx].sector);
  3188. if (conf->mddev->bitmap && firstwrite) {
  3189. /* Cannot hold spinlock over bitmap_startwrite,
  3190. * but must ensure this isn't added to a batch until
  3191. * we have added to the bitmap and set bm_seq.
  3192. * So set STRIPE_BITMAP_PENDING to prevent
  3193. * batching.
  3194. * If multiple __add_stripe_bio() calls race here they
  3195. * much all set STRIPE_BITMAP_PENDING. So only the first one
  3196. * to complete "bitmap_startwrite" gets to set
  3197. * STRIPE_BIT_DELAY. This is important as once a stripe
  3198. * is added to a batch, STRIPE_BIT_DELAY cannot be changed
  3199. * any more.
  3200. */
  3201. set_bit(STRIPE_BITMAP_PENDING, &sh->state);
  3202. spin_unlock_irq(&sh->stripe_lock);
  3203. md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
  3204. RAID5_STRIPE_SECTORS(conf), 0);
  3205. spin_lock_irq(&sh->stripe_lock);
  3206. clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
  3207. if (!sh->batch_head) {
  3208. sh->bm_seq = conf->seq_flush+1;
  3209. set_bit(STRIPE_BIT_DELAY, &sh->state);
  3210. }
  3211. }
  3212. }
  3213. /*
  3214. * Each stripe/dev can have one or more bios attached.
  3215. * toread/towrite point to the first in a chain.
  3216. * The bi_next chain must be in order.
  3217. */
  3218. static bool add_stripe_bio(struct stripe_head *sh, struct bio *bi,
  3219. int dd_idx, int forwrite, int previous)
  3220. {
  3221. spin_lock_irq(&sh->stripe_lock);
  3222. if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {
  3223. set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
  3224. spin_unlock_irq(&sh->stripe_lock);
  3225. return false;
  3226. }
  3227. __add_stripe_bio(sh, bi, dd_idx, forwrite, previous);
  3228. spin_unlock_irq(&sh->stripe_lock);
  3229. return true;
  3230. }
  3231. static void end_reshape(struct r5conf *conf);
  3232. static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
  3233. struct stripe_head *sh)
  3234. {
  3235. int sectors_per_chunk =
  3236. previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
  3237. int dd_idx;
  3238. int chunk_offset = sector_div(stripe, sectors_per_chunk);
  3239. int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
  3240. raid5_compute_sector(conf,
  3241. stripe * (disks - conf->max_degraded)
  3242. *sectors_per_chunk + chunk_offset,
  3243. previous,
  3244. &dd_idx, sh);
  3245. }
  3246. static void
  3247. handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
  3248. struct stripe_head_state *s, int disks)
  3249. {
  3250. int i;
  3251. BUG_ON(sh->batch_head);
  3252. for (i = disks; i--; ) {
  3253. struct bio *bi;
  3254. int bitmap_end = 0;
  3255. if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
  3256. struct md_rdev *rdev;
  3257. rcu_read_lock();
  3258. rdev = rcu_dereference(conf->disks[i].rdev);
  3259. if (rdev && test_bit(In_sync, &rdev->flags) &&
  3260. !test_bit(Faulty, &rdev->flags))
  3261. atomic_inc(&rdev->nr_pending);
  3262. else
  3263. rdev = NULL;
  3264. rcu_read_unlock();
  3265. if (rdev) {
  3266. if (!rdev_set_badblocks(
  3267. rdev,
  3268. sh->sector,
  3269. RAID5_STRIPE_SECTORS(conf), 0))
  3270. md_error(conf->mddev, rdev);
  3271. rdev_dec_pending(rdev, conf->mddev);
  3272. }
  3273. }
  3274. spin_lock_irq(&sh->stripe_lock);
  3275. /* fail all writes first */
  3276. bi = sh->dev[i].towrite;
  3277. sh->dev[i].towrite = NULL;
  3278. sh->overwrite_disks = 0;
  3279. spin_unlock_irq(&sh->stripe_lock);
  3280. if (bi)
  3281. bitmap_end = 1;
  3282. log_stripe_write_finished(sh);
  3283. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  3284. wake_up(&conf->wait_for_overlap);
  3285. while (bi && bi->bi_iter.bi_sector <
  3286. sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
  3287. struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
  3288. md_write_end(conf->mddev);
  3289. bio_io_error(bi);
  3290. bi = nextbi;
  3291. }
  3292. if (bitmap_end)
  3293. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  3294. RAID5_STRIPE_SECTORS(conf), 0, 0);
  3295. bitmap_end = 0;
  3296. /* and fail all 'written' */
  3297. bi = sh->dev[i].written;
  3298. sh->dev[i].written = NULL;
  3299. if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
  3300. WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
  3301. sh->dev[i].page = sh->dev[i].orig_page;
  3302. }
  3303. if (bi) bitmap_end = 1;
  3304. while (bi && bi->bi_iter.bi_sector <
  3305. sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
  3306. struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
  3307. md_write_end(conf->mddev);
  3308. bio_io_error(bi);
  3309. bi = bi2;
  3310. }
  3311. /* fail any reads if this device is non-operational and
  3312. * the data has not reached the cache yet.
  3313. */
  3314. if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
  3315. s->failed > conf->max_degraded &&
  3316. (!test_bit(R5_Insync, &sh->dev[i].flags) ||
  3317. test_bit(R5_ReadError, &sh->dev[i].flags))) {
  3318. spin_lock_irq(&sh->stripe_lock);
  3319. bi = sh->dev[i].toread;
  3320. sh->dev[i].toread = NULL;
  3321. spin_unlock_irq(&sh->stripe_lock);
  3322. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  3323. wake_up(&conf->wait_for_overlap);
  3324. if (bi)
  3325. s->to_read--;
  3326. while (bi && bi->bi_iter.bi_sector <
  3327. sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
  3328. struct bio *nextbi =
  3329. r5_next_bio(conf, bi, sh->dev[i].sector);
  3330. bio_io_error(bi);
  3331. bi = nextbi;
  3332. }
  3333. }
  3334. if (bitmap_end)
  3335. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  3336. RAID5_STRIPE_SECTORS(conf), 0, 0);
  3337. /* If we were in the middle of a write the parity block might
  3338. * still be locked - so just clear all R5_LOCKED flags
  3339. */
  3340. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  3341. }
  3342. s->to_write = 0;
  3343. s->written = 0;
  3344. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  3345. if (atomic_dec_and_test(&conf->pending_full_writes))
  3346. md_wakeup_thread(conf->mddev->thread);
  3347. }
  3348. static void
  3349. handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
  3350. struct stripe_head_state *s)
  3351. {
  3352. int abort = 0;
  3353. int i;
  3354. BUG_ON(sh->batch_head);
  3355. clear_bit(STRIPE_SYNCING, &sh->state);
  3356. if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
  3357. wake_up(&conf->wait_for_overlap);
  3358. s->syncing = 0;
  3359. s->replacing = 0;
  3360. /* There is nothing more to do for sync/check/repair.
  3361. * Don't even need to abort as that is handled elsewhere
  3362. * if needed, and not always wanted e.g. if there is a known
  3363. * bad block here.
  3364. * For recover/replace we need to record a bad block on all
  3365. * non-sync devices, or abort the recovery
  3366. */
  3367. if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
  3368. /* During recovery devices cannot be removed, so
  3369. * locking and refcounting of rdevs is not needed
  3370. */
  3371. rcu_read_lock();
  3372. for (i = 0; i < conf->raid_disks; i++) {
  3373. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  3374. if (rdev
  3375. && !test_bit(Faulty, &rdev->flags)
  3376. && !test_bit(In_sync, &rdev->flags)
  3377. && !rdev_set_badblocks(rdev, sh->sector,
  3378. RAID5_STRIPE_SECTORS(conf), 0))
  3379. abort = 1;
  3380. rdev = rcu_dereference(conf->disks[i].replacement);
  3381. if (rdev
  3382. && !test_bit(Faulty, &rdev->flags)
  3383. && !test_bit(In_sync, &rdev->flags)
  3384. && !rdev_set_badblocks(rdev, sh->sector,
  3385. RAID5_STRIPE_SECTORS(conf), 0))
  3386. abort = 1;
  3387. }
  3388. rcu_read_unlock();
  3389. if (abort)
  3390. conf->recovery_disabled =
  3391. conf->mddev->recovery_disabled;
  3392. }
  3393. md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);
  3394. }
  3395. static int want_replace(struct stripe_head *sh, int disk_idx)
  3396. {
  3397. struct md_rdev *rdev;
  3398. int rv = 0;
  3399. rcu_read_lock();
  3400. rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);
  3401. if (rdev
  3402. && !test_bit(Faulty, &rdev->flags)
  3403. && !test_bit(In_sync, &rdev->flags)
  3404. && (rdev->recovery_offset <= sh->sector
  3405. || rdev->mddev->recovery_cp <= sh->sector))
  3406. rv = 1;
  3407. rcu_read_unlock();
  3408. return rv;
  3409. }
  3410. static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
  3411. int disk_idx, int disks)
  3412. {
  3413. struct r5dev *dev = &sh->dev[disk_idx];
  3414. struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
  3415. &sh->dev[s->failed_num[1]] };
  3416. int i;
  3417. bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
  3418. if (test_bit(R5_LOCKED, &dev->flags) ||
  3419. test_bit(R5_UPTODATE, &dev->flags))
  3420. /* No point reading this as we already have it or have
  3421. * decided to get it.
  3422. */
  3423. return 0;
  3424. if (dev->toread ||
  3425. (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
  3426. /* We need this block to directly satisfy a request */
  3427. return 1;
  3428. if (s->syncing || s->expanding ||
  3429. (s->replacing && want_replace(sh, disk_idx)))
  3430. /* When syncing, or expanding we read everything.
  3431. * When replacing, we need the replaced block.
  3432. */
  3433. return 1;
  3434. if ((s->failed >= 1 && fdev[0]->toread) ||
  3435. (s->failed >= 2 && fdev[1]->toread))
  3436. /* If we want to read from a failed device, then
  3437. * we need to actually read every other device.
  3438. */
  3439. return 1;
  3440. /* Sometimes neither read-modify-write nor reconstruct-write
  3441. * cycles can work. In those cases we read every block we
  3442. * can. Then the parity-update is certain to have enough to
  3443. * work with.
  3444. * This can only be a problem when we need to write something,
  3445. * and some device has failed. If either of those tests
  3446. * fail we need look no further.
  3447. */
  3448. if (!s->failed || !s->to_write)
  3449. return 0;
  3450. if (test_bit(R5_Insync, &dev->flags) &&
  3451. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3452. /* Pre-reads at not permitted until after short delay
  3453. * to gather multiple requests. However if this
  3454. * device is no Insync, the block could only be computed
  3455. * and there is no need to delay that.
  3456. */
  3457. return 0;
  3458. for (i = 0; i < s->failed && i < 2; i++) {
  3459. if (fdev[i]->towrite &&
  3460. !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
  3461. !test_bit(R5_OVERWRITE, &fdev[i]->flags))
  3462. /* If we have a partial write to a failed
  3463. * device, then we will need to reconstruct
  3464. * the content of that device, so all other
  3465. * devices must be read.
  3466. */
  3467. return 1;
  3468. if (s->failed >= 2 &&
  3469. (fdev[i]->towrite ||
  3470. s->failed_num[i] == sh->pd_idx ||
  3471. s->failed_num[i] == sh->qd_idx) &&
  3472. !test_bit(R5_UPTODATE, &fdev[i]->flags))
  3473. /* In max degraded raid6, If the failed disk is P, Q,
  3474. * or we want to read the failed disk, we need to do
  3475. * reconstruct-write.
  3476. */
  3477. force_rcw = true;
  3478. }
  3479. /* If we are forced to do a reconstruct-write, because parity
  3480. * cannot be trusted and we are currently recovering it, there
  3481. * is extra need to be careful.
  3482. * If one of the devices that we would need to read, because
  3483. * it is not being overwritten (and maybe not written at all)
  3484. * is missing/faulty, then we need to read everything we can.
  3485. */
  3486. if (!force_rcw &&
  3487. sh->sector < sh->raid_conf->mddev->recovery_cp)
  3488. /* reconstruct-write isn't being forced */
  3489. return 0;
  3490. for (i = 0; i < s->failed && i < 2; i++) {
  3491. if (s->failed_num[i] != sh->pd_idx &&
  3492. s->failed_num[i] != sh->qd_idx &&
  3493. !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
  3494. !test_bit(R5_OVERWRITE, &fdev[i]->flags))
  3495. return 1;
  3496. }
  3497. return 0;
  3498. }
  3499. /* fetch_block - checks the given member device to see if its data needs
  3500. * to be read or computed to satisfy a request.
  3501. *
  3502. * Returns 1 when no more member devices need to be checked, otherwise returns
  3503. * 0 to tell the loop in handle_stripe_fill to continue
  3504. */
  3505. static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
  3506. int disk_idx, int disks)
  3507. {
  3508. struct r5dev *dev = &sh->dev[disk_idx];
  3509. /* is the data in this block needed, and can we get it? */
  3510. if (need_this_block(sh, s, disk_idx, disks)) {
  3511. /* we would like to get this block, possibly by computing it,
  3512. * otherwise read it if the backing disk is insync
  3513. */
  3514. BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
  3515. BUG_ON(test_bit(R5_Wantread, &dev->flags));
  3516. BUG_ON(sh->batch_head);
  3517. /*
  3518. * In the raid6 case if the only non-uptodate disk is P
  3519. * then we already trusted P to compute the other failed
  3520. * drives. It is safe to compute rather than re-read P.
  3521. * In other cases we only compute blocks from failed
  3522. * devices, otherwise check/repair might fail to detect
  3523. * a real inconsistency.
  3524. */
  3525. if ((s->uptodate == disks - 1) &&
  3526. ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||
  3527. (s->failed && (disk_idx == s->failed_num[0] ||
  3528. disk_idx == s->failed_num[1])))) {
  3529. /* have disk failed, and we're requested to fetch it;
  3530. * do compute it
  3531. */
  3532. pr_debug("Computing stripe %llu block %d\n",
  3533. (unsigned long long)sh->sector, disk_idx);
  3534. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  3535. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  3536. set_bit(R5_Wantcompute, &dev->flags);
  3537. sh->ops.target = disk_idx;
  3538. sh->ops.target2 = -1; /* no 2nd target */
  3539. s->req_compute = 1;
  3540. /* Careful: from this point on 'uptodate' is in the eye
  3541. * of raid_run_ops which services 'compute' operations
  3542. * before writes. R5_Wantcompute flags a block that will
  3543. * be R5_UPTODATE by the time it is needed for a
  3544. * subsequent operation.
  3545. */
  3546. s->uptodate++;
  3547. return 1;
  3548. } else if (s->uptodate == disks-2 && s->failed >= 2) {
  3549. /* Computing 2-failure is *very* expensive; only
  3550. * do it if failed >= 2
  3551. */
  3552. int other;
  3553. for (other = disks; other--; ) {
  3554. if (other == disk_idx)
  3555. continue;
  3556. if (!test_bit(R5_UPTODATE,
  3557. &sh->dev[other].flags))
  3558. break;
  3559. }
  3560. BUG_ON(other < 0);
  3561. pr_debug("Computing stripe %llu blocks %d,%d\n",
  3562. (unsigned long long)sh->sector,
  3563. disk_idx, other);
  3564. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  3565. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  3566. set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
  3567. set_bit(R5_Wantcompute, &sh->dev[other].flags);
  3568. sh->ops.target = disk_idx;
  3569. sh->ops.target2 = other;
  3570. s->uptodate += 2;
  3571. s->req_compute = 1;
  3572. return 1;
  3573. } else if (test_bit(R5_Insync, &dev->flags)) {
  3574. set_bit(R5_LOCKED, &dev->flags);
  3575. set_bit(R5_Wantread, &dev->flags);
  3576. s->locked++;
  3577. pr_debug("Reading block %d (sync=%d)\n",
  3578. disk_idx, s->syncing);
  3579. }
  3580. }
  3581. return 0;
  3582. }
  3583. /*
  3584. * handle_stripe_fill - read or compute data to satisfy pending requests.
  3585. */
  3586. static void handle_stripe_fill(struct stripe_head *sh,
  3587. struct stripe_head_state *s,
  3588. int disks)
  3589. {
  3590. int i;
  3591. /* look for blocks to read/compute, skip this if a compute
  3592. * is already in flight, or if the stripe contents are in the
  3593. * midst of changing due to a write
  3594. */
  3595. if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
  3596. !sh->reconstruct_state) {
  3597. /*
  3598. * For degraded stripe with data in journal, do not handle
  3599. * read requests yet, instead, flush the stripe to raid
  3600. * disks first, this avoids handling complex rmw of write
  3601. * back cache (prexor with orig_page, and then xor with
  3602. * page) in the read path
  3603. */
  3604. if (s->to_read && s->injournal && s->failed) {
  3605. if (test_bit(STRIPE_R5C_CACHING, &sh->state))
  3606. r5c_make_stripe_write_out(sh);
  3607. goto out;
  3608. }
  3609. for (i = disks; i--; )
  3610. if (fetch_block(sh, s, i, disks))
  3611. break;
  3612. }
  3613. out:
  3614. set_bit(STRIPE_HANDLE, &sh->state);
  3615. }
  3616. static void break_stripe_batch_list(struct stripe_head *head_sh,
  3617. unsigned long handle_flags);
  3618. /* handle_stripe_clean_event
  3619. * any written block on an uptodate or failed drive can be returned.
  3620. * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
  3621. * never LOCKED, so we don't need to test 'failed' directly.
  3622. */
  3623. static void handle_stripe_clean_event(struct r5conf *conf,
  3624. struct stripe_head *sh, int disks)
  3625. {
  3626. int i;
  3627. struct r5dev *dev;
  3628. int discard_pending = 0;
  3629. struct stripe_head *head_sh = sh;
  3630. bool do_endio = false;
  3631. for (i = disks; i--; )
  3632. if (sh->dev[i].written) {
  3633. dev = &sh->dev[i];
  3634. if (!test_bit(R5_LOCKED, &dev->flags) &&
  3635. (test_bit(R5_UPTODATE, &dev->flags) ||
  3636. test_bit(R5_Discard, &dev->flags) ||
  3637. test_bit(R5_SkipCopy, &dev->flags))) {
  3638. /* We can return any write requests */
  3639. struct bio *wbi, *wbi2;
  3640. pr_debug("Return write for disc %d\n", i);
  3641. if (test_and_clear_bit(R5_Discard, &dev->flags))
  3642. clear_bit(R5_UPTODATE, &dev->flags);
  3643. if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
  3644. WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
  3645. }
  3646. do_endio = true;
  3647. returnbi:
  3648. dev->page = dev->orig_page;
  3649. wbi = dev->written;
  3650. dev->written = NULL;
  3651. while (wbi && wbi->bi_iter.bi_sector <
  3652. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  3653. wbi2 = r5_next_bio(conf, wbi, dev->sector);
  3654. md_write_end(conf->mddev);
  3655. bio_endio(wbi);
  3656. wbi = wbi2;
  3657. }
  3658. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  3659. RAID5_STRIPE_SECTORS(conf),
  3660. !test_bit(STRIPE_DEGRADED, &sh->state),
  3661. 0);
  3662. if (head_sh->batch_head) {
  3663. sh = list_first_entry(&sh->batch_list,
  3664. struct stripe_head,
  3665. batch_list);
  3666. if (sh != head_sh) {
  3667. dev = &sh->dev[i];
  3668. goto returnbi;
  3669. }
  3670. }
  3671. sh = head_sh;
  3672. dev = &sh->dev[i];
  3673. } else if (test_bit(R5_Discard, &dev->flags))
  3674. discard_pending = 1;
  3675. }
  3676. log_stripe_write_finished(sh);
  3677. if (!discard_pending &&
  3678. test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
  3679. int hash;
  3680. clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
  3681. clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
  3682. if (sh->qd_idx >= 0) {
  3683. clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
  3684. clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
  3685. }
  3686. /* now that discard is done we can proceed with any sync */
  3687. clear_bit(STRIPE_DISCARD, &sh->state);
  3688. /*
  3689. * SCSI discard will change some bio fields and the stripe has
  3690. * no updated data, so remove it from hash list and the stripe
  3691. * will be reinitialized
  3692. */
  3693. unhash:
  3694. hash = sh->hash_lock_index;
  3695. spin_lock_irq(conf->hash_locks + hash);
  3696. remove_hash(sh);
  3697. spin_unlock_irq(conf->hash_locks + hash);
  3698. if (head_sh->batch_head) {
  3699. sh = list_first_entry(&sh->batch_list,
  3700. struct stripe_head, batch_list);
  3701. if (sh != head_sh)
  3702. goto unhash;
  3703. }
  3704. sh = head_sh;
  3705. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
  3706. set_bit(STRIPE_HANDLE, &sh->state);
  3707. }
  3708. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  3709. if (atomic_dec_and_test(&conf->pending_full_writes))
  3710. md_wakeup_thread(conf->mddev->thread);
  3711. if (head_sh->batch_head && do_endio)
  3712. break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
  3713. }
  3714. /*
  3715. * For RMW in write back cache, we need extra page in prexor to store the
  3716. * old data. This page is stored in dev->orig_page.
  3717. *
  3718. * This function checks whether we have data for prexor. The exact logic
  3719. * is:
  3720. * R5_UPTODATE && (!R5_InJournal || R5_OrigPageUPTDODATE)
  3721. */
  3722. static inline bool uptodate_for_rmw(struct r5dev *dev)
  3723. {
  3724. return (test_bit(R5_UPTODATE, &dev->flags)) &&
  3725. (!test_bit(R5_InJournal, &dev->flags) ||
  3726. test_bit(R5_OrigPageUPTDODATE, &dev->flags));
  3727. }
  3728. static int handle_stripe_dirtying(struct r5conf *conf,
  3729. struct stripe_head *sh,
  3730. struct stripe_head_state *s,
  3731. int disks)
  3732. {
  3733. int rmw = 0, rcw = 0, i;
  3734. sector_t recovery_cp = conf->mddev->recovery_cp;
  3735. /* Check whether resync is now happening or should start.
  3736. * If yes, then the array is dirty (after unclean shutdown or
  3737. * initial creation), so parity in some stripes might be inconsistent.
  3738. * In this case, we need to always do reconstruct-write, to ensure
  3739. * that in case of drive failure or read-error correction, we
  3740. * generate correct data from the parity.
  3741. */
  3742. if (conf->rmw_level == PARITY_DISABLE_RMW ||
  3743. (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
  3744. s->failed == 0)) {
  3745. /* Calculate the real rcw later - for now make it
  3746. * look like rcw is cheaper
  3747. */
  3748. rcw = 1; rmw = 2;
  3749. pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
  3750. conf->rmw_level, (unsigned long long)recovery_cp,
  3751. (unsigned long long)sh->sector);
  3752. } else for (i = disks; i--; ) {
  3753. /* would I have to read this buffer for read_modify_write */
  3754. struct r5dev *dev = &sh->dev[i];
  3755. if (((dev->towrite && !delay_towrite(conf, dev, s)) ||
  3756. i == sh->pd_idx || i == sh->qd_idx ||
  3757. test_bit(R5_InJournal, &dev->flags)) &&
  3758. !test_bit(R5_LOCKED, &dev->flags) &&
  3759. !(uptodate_for_rmw(dev) ||
  3760. test_bit(R5_Wantcompute, &dev->flags))) {
  3761. if (test_bit(R5_Insync, &dev->flags))
  3762. rmw++;
  3763. else
  3764. rmw += 2*disks; /* cannot read it */
  3765. }
  3766. /* Would I have to read this buffer for reconstruct_write */
  3767. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  3768. i != sh->pd_idx && i != sh->qd_idx &&
  3769. !test_bit(R5_LOCKED, &dev->flags) &&
  3770. !(test_bit(R5_UPTODATE, &dev->flags) ||
  3771. test_bit(R5_Wantcompute, &dev->flags))) {
  3772. if (test_bit(R5_Insync, &dev->flags))
  3773. rcw++;
  3774. else
  3775. rcw += 2*disks;
  3776. }
  3777. }
  3778. pr_debug("for sector %llu state 0x%lx, rmw=%d rcw=%d\n",
  3779. (unsigned long long)sh->sector, sh->state, rmw, rcw);
  3780. set_bit(STRIPE_HANDLE, &sh->state);
  3781. if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_PREFER_RMW)) && rmw > 0) {
  3782. /* prefer read-modify-write, but need to get some data */
  3783. if (conf->mddev->queue)
  3784. blk_add_trace_msg(conf->mddev->queue,
  3785. "raid5 rmw %llu %d",
  3786. (unsigned long long)sh->sector, rmw);
  3787. for (i = disks; i--; ) {
  3788. struct r5dev *dev = &sh->dev[i];
  3789. if (test_bit(R5_InJournal, &dev->flags) &&
  3790. dev->page == dev->orig_page &&
  3791. !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
  3792. /* alloc page for prexor */
  3793. struct page *p = alloc_page(GFP_NOIO);
  3794. if (p) {
  3795. dev->orig_page = p;
  3796. continue;
  3797. }
  3798. /*
  3799. * alloc_page() failed, try use
  3800. * disk_info->extra_page
  3801. */
  3802. if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE,
  3803. &conf->cache_state)) {
  3804. r5c_use_extra_page(sh);
  3805. break;
  3806. }
  3807. /* extra_page in use, add to delayed_list */
  3808. set_bit(STRIPE_DELAYED, &sh->state);
  3809. s->waiting_extra_page = 1;
  3810. return -EAGAIN;
  3811. }
  3812. }
  3813. for (i = disks; i--; ) {
  3814. struct r5dev *dev = &sh->dev[i];
  3815. if (((dev->towrite && !delay_towrite(conf, dev, s)) ||
  3816. i == sh->pd_idx || i == sh->qd_idx ||
  3817. test_bit(R5_InJournal, &dev->flags)) &&
  3818. !test_bit(R5_LOCKED, &dev->flags) &&
  3819. !(uptodate_for_rmw(dev) ||
  3820. test_bit(R5_Wantcompute, &dev->flags)) &&
  3821. test_bit(R5_Insync, &dev->flags)) {
  3822. if (test_bit(STRIPE_PREREAD_ACTIVE,
  3823. &sh->state)) {
  3824. pr_debug("Read_old block %d for r-m-w\n",
  3825. i);
  3826. set_bit(R5_LOCKED, &dev->flags);
  3827. set_bit(R5_Wantread, &dev->flags);
  3828. s->locked++;
  3829. } else
  3830. set_bit(STRIPE_DELAYED, &sh->state);
  3831. }
  3832. }
  3833. }
  3834. if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_PREFER_RMW)) && rcw > 0) {
  3835. /* want reconstruct write, but need to get some data */
  3836. int qread =0;
  3837. rcw = 0;
  3838. for (i = disks; i--; ) {
  3839. struct r5dev *dev = &sh->dev[i];
  3840. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  3841. i != sh->pd_idx && i != sh->qd_idx &&
  3842. !test_bit(R5_LOCKED, &dev->flags) &&
  3843. !(test_bit(R5_UPTODATE, &dev->flags) ||
  3844. test_bit(R5_Wantcompute, &dev->flags))) {
  3845. rcw++;
  3846. if (test_bit(R5_Insync, &dev->flags) &&
  3847. test_bit(STRIPE_PREREAD_ACTIVE,
  3848. &sh->state)) {
  3849. pr_debug("Read_old block "
  3850. "%d for Reconstruct\n", i);
  3851. set_bit(R5_LOCKED, &dev->flags);
  3852. set_bit(R5_Wantread, &dev->flags);
  3853. s->locked++;
  3854. qread++;
  3855. } else
  3856. set_bit(STRIPE_DELAYED, &sh->state);
  3857. }
  3858. }
  3859. if (rcw && conf->mddev->queue)
  3860. blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
  3861. (unsigned long long)sh->sector,
  3862. rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
  3863. }
  3864. if (rcw > disks && rmw > disks &&
  3865. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3866. set_bit(STRIPE_DELAYED, &sh->state);
  3867. /* now if nothing is locked, and if we have enough data,
  3868. * we can start a write request
  3869. */
  3870. /* since handle_stripe can be called at any time we need to handle the
  3871. * case where a compute block operation has been submitted and then a
  3872. * subsequent call wants to start a write request. raid_run_ops only
  3873. * handles the case where compute block and reconstruct are requested
  3874. * simultaneously. If this is not the case then new writes need to be
  3875. * held off until the compute completes.
  3876. */
  3877. if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
  3878. (s->locked == 0 && (rcw == 0 || rmw == 0) &&
  3879. !test_bit(STRIPE_BIT_DELAY, &sh->state)))
  3880. schedule_reconstruction(sh, s, rcw == 0, 0);
  3881. return 0;
  3882. }
  3883. static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
  3884. struct stripe_head_state *s, int disks)
  3885. {
  3886. struct r5dev *dev = NULL;
  3887. BUG_ON(sh->batch_head);
  3888. set_bit(STRIPE_HANDLE, &sh->state);
  3889. switch (sh->check_state) {
  3890. case check_state_idle:
  3891. /* start a new check operation if there are no failures */
  3892. if (s->failed == 0) {
  3893. BUG_ON(s->uptodate != disks);
  3894. sh->check_state = check_state_run;
  3895. set_bit(STRIPE_OP_CHECK, &s->ops_request);
  3896. clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
  3897. s->uptodate--;
  3898. break;
  3899. }
  3900. dev = &sh->dev[s->failed_num[0]];
  3901. fallthrough;
  3902. case check_state_compute_result:
  3903. sh->check_state = check_state_idle;
  3904. if (!dev)
  3905. dev = &sh->dev[sh->pd_idx];
  3906. /* check that a write has not made the stripe insync */
  3907. if (test_bit(STRIPE_INSYNC, &sh->state))
  3908. break;
  3909. /* either failed parity check, or recovery is happening */
  3910. BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
  3911. BUG_ON(s->uptodate != disks);
  3912. set_bit(R5_LOCKED, &dev->flags);
  3913. s->locked++;
  3914. set_bit(R5_Wantwrite, &dev->flags);
  3915. clear_bit(STRIPE_DEGRADED, &sh->state);
  3916. set_bit(STRIPE_INSYNC, &sh->state);
  3917. break;
  3918. case check_state_run:
  3919. break; /* we will be called again upon completion */
  3920. case check_state_check_result:
  3921. sh->check_state = check_state_idle;
  3922. /* if a failure occurred during the check operation, leave
  3923. * STRIPE_INSYNC not set and let the stripe be handled again
  3924. */
  3925. if (s->failed)
  3926. break;
  3927. /* handle a successful check operation, if parity is correct
  3928. * we are done. Otherwise update the mismatch count and repair
  3929. * parity if !MD_RECOVERY_CHECK
  3930. */
  3931. if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
  3932. /* parity is correct (on disc,
  3933. * not in buffer any more)
  3934. */
  3935. set_bit(STRIPE_INSYNC, &sh->state);
  3936. else {
  3937. atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
  3938. if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
  3939. /* don't try to repair!! */
  3940. set_bit(STRIPE_INSYNC, &sh->state);
  3941. pr_warn_ratelimited("%s: mismatch sector in range "
  3942. "%llu-%llu\n", mdname(conf->mddev),
  3943. (unsigned long long) sh->sector,
  3944. (unsigned long long) sh->sector +
  3945. RAID5_STRIPE_SECTORS(conf));
  3946. } else {
  3947. sh->check_state = check_state_compute_run;
  3948. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  3949. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  3950. set_bit(R5_Wantcompute,
  3951. &sh->dev[sh->pd_idx].flags);
  3952. sh->ops.target = sh->pd_idx;
  3953. sh->ops.target2 = -1;
  3954. s->uptodate++;
  3955. }
  3956. }
  3957. break;
  3958. case check_state_compute_run:
  3959. break;
  3960. default:
  3961. pr_err("%s: unknown check_state: %d sector: %llu\n",
  3962. __func__, sh->check_state,
  3963. (unsigned long long) sh->sector);
  3964. BUG();
  3965. }
  3966. }
  3967. static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
  3968. struct stripe_head_state *s,
  3969. int disks)
  3970. {
  3971. int pd_idx = sh->pd_idx;
  3972. int qd_idx = sh->qd_idx;
  3973. struct r5dev *dev;
  3974. BUG_ON(sh->batch_head);
  3975. set_bit(STRIPE_HANDLE, &sh->state);
  3976. BUG_ON(s->failed > 2);
  3977. /* Want to check and possibly repair P and Q.
  3978. * However there could be one 'failed' device, in which
  3979. * case we can only check one of them, possibly using the
  3980. * other to generate missing data
  3981. */
  3982. switch (sh->check_state) {
  3983. case check_state_idle:
  3984. /* start a new check operation if there are < 2 failures */
  3985. if (s->failed == s->q_failed) {
  3986. /* The only possible failed device holds Q, so it
  3987. * makes sense to check P (If anything else were failed,
  3988. * we would have used P to recreate it).
  3989. */
  3990. sh->check_state = check_state_run;
  3991. }
  3992. if (!s->q_failed && s->failed < 2) {
  3993. /* Q is not failed, and we didn't use it to generate
  3994. * anything, so it makes sense to check it
  3995. */
  3996. if (sh->check_state == check_state_run)
  3997. sh->check_state = check_state_run_pq;
  3998. else
  3999. sh->check_state = check_state_run_q;
  4000. }
  4001. /* discard potentially stale zero_sum_result */
  4002. sh->ops.zero_sum_result = 0;
  4003. if (sh->check_state == check_state_run) {
  4004. /* async_xor_zero_sum destroys the contents of P */
  4005. clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
  4006. s->uptodate--;
  4007. }
  4008. if (sh->check_state >= check_state_run &&
  4009. sh->check_state <= check_state_run_pq) {
  4010. /* async_syndrome_zero_sum preserves P and Q, so
  4011. * no need to mark them !uptodate here
  4012. */
  4013. set_bit(STRIPE_OP_CHECK, &s->ops_request);
  4014. break;
  4015. }
  4016. /* we have 2-disk failure */
  4017. BUG_ON(s->failed != 2);
  4018. fallthrough;
  4019. case check_state_compute_result:
  4020. sh->check_state = check_state_idle;
  4021. /* check that a write has not made the stripe insync */
  4022. if (test_bit(STRIPE_INSYNC, &sh->state))
  4023. break;
  4024. /* now write out any block on a failed drive,
  4025. * or P or Q if they were recomputed
  4026. */
  4027. dev = NULL;
  4028. if (s->failed == 2) {
  4029. dev = &sh->dev[s->failed_num[1]];
  4030. s->locked++;
  4031. set_bit(R5_LOCKED, &dev->flags);
  4032. set_bit(R5_Wantwrite, &dev->flags);
  4033. }
  4034. if (s->failed >= 1) {
  4035. dev = &sh->dev[s->failed_num[0]];
  4036. s->locked++;
  4037. set_bit(R5_LOCKED, &dev->flags);
  4038. set_bit(R5_Wantwrite, &dev->flags);
  4039. }
  4040. if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
  4041. dev = &sh->dev[pd_idx];
  4042. s->locked++;
  4043. set_bit(R5_LOCKED, &dev->flags);
  4044. set_bit(R5_Wantwrite, &dev->flags);
  4045. }
  4046. if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
  4047. dev = &sh->dev[qd_idx];
  4048. s->locked++;
  4049. set_bit(R5_LOCKED, &dev->flags);
  4050. set_bit(R5_Wantwrite, &dev->flags);
  4051. }
  4052. if (WARN_ONCE(dev && !test_bit(R5_UPTODATE, &dev->flags),
  4053. "%s: disk%td not up to date\n",
  4054. mdname(conf->mddev),
  4055. dev - (struct r5dev *) &sh->dev)) {
  4056. clear_bit(R5_LOCKED, &dev->flags);
  4057. clear_bit(R5_Wantwrite, &dev->flags);
  4058. s->locked--;
  4059. }
  4060. clear_bit(STRIPE_DEGRADED, &sh->state);
  4061. set_bit(STRIPE_INSYNC, &sh->state);
  4062. break;
  4063. case check_state_run:
  4064. case check_state_run_q:
  4065. case check_state_run_pq:
  4066. break; /* we will be called again upon completion */
  4067. case check_state_check_result:
  4068. sh->check_state = check_state_idle;
  4069. /* handle a successful check operation, if parity is correct
  4070. * we are done. Otherwise update the mismatch count and repair
  4071. * parity if !MD_RECOVERY_CHECK
  4072. */
  4073. if (sh->ops.zero_sum_result == 0) {
  4074. /* both parities are correct */
  4075. if (!s->failed)
  4076. set_bit(STRIPE_INSYNC, &sh->state);
  4077. else {
  4078. /* in contrast to the raid5 case we can validate
  4079. * parity, but still have a failure to write
  4080. * back
  4081. */
  4082. sh->check_state = check_state_compute_result;
  4083. /* Returning at this point means that we may go
  4084. * off and bring p and/or q uptodate again so
  4085. * we make sure to check zero_sum_result again
  4086. * to verify if p or q need writeback
  4087. */
  4088. }
  4089. } else {
  4090. atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
  4091. if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
  4092. /* don't try to repair!! */
  4093. set_bit(STRIPE_INSYNC, &sh->state);
  4094. pr_warn_ratelimited("%s: mismatch sector in range "
  4095. "%llu-%llu\n", mdname(conf->mddev),
  4096. (unsigned long long) sh->sector,
  4097. (unsigned long long) sh->sector +
  4098. RAID5_STRIPE_SECTORS(conf));
  4099. } else {
  4100. int *target = &sh->ops.target;
  4101. sh->ops.target = -1;
  4102. sh->ops.target2 = -1;
  4103. sh->check_state = check_state_compute_run;
  4104. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  4105. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  4106. if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
  4107. set_bit(R5_Wantcompute,
  4108. &sh->dev[pd_idx].flags);
  4109. *target = pd_idx;
  4110. target = &sh->ops.target2;
  4111. s->uptodate++;
  4112. }
  4113. if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
  4114. set_bit(R5_Wantcompute,
  4115. &sh->dev[qd_idx].flags);
  4116. *target = qd_idx;
  4117. s->uptodate++;
  4118. }
  4119. }
  4120. }
  4121. break;
  4122. case check_state_compute_run:
  4123. break;
  4124. default:
  4125. pr_warn("%s: unknown check_state: %d sector: %llu\n",
  4126. __func__, sh->check_state,
  4127. (unsigned long long) sh->sector);
  4128. BUG();
  4129. }
  4130. }
  4131. static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
  4132. {
  4133. int i;
  4134. /* We have read all the blocks in this stripe and now we need to
  4135. * copy some of them into a target stripe for expand.
  4136. */
  4137. struct dma_async_tx_descriptor *tx = NULL;
  4138. BUG_ON(sh->batch_head);
  4139. clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  4140. for (i = 0; i < sh->disks; i++)
  4141. if (i != sh->pd_idx && i != sh->qd_idx) {
  4142. int dd_idx, j;
  4143. struct stripe_head *sh2;
  4144. struct async_submit_ctl submit;
  4145. sector_t bn = raid5_compute_blocknr(sh, i, 1);
  4146. sector_t s = raid5_compute_sector(conf, bn, 0,
  4147. &dd_idx, NULL);
  4148. sh2 = raid5_get_active_stripe(conf, NULL, s,
  4149. R5_GAS_NOBLOCK | R5_GAS_NOQUIESCE);
  4150. if (sh2 == NULL)
  4151. /* so far only the early blocks of this stripe
  4152. * have been requested. When later blocks
  4153. * get requested, we will try again
  4154. */
  4155. continue;
  4156. if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
  4157. test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
  4158. /* must have already done this block */
  4159. raid5_release_stripe(sh2);
  4160. continue;
  4161. }
  4162. /* place all the copies on one channel */
  4163. init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
  4164. tx = async_memcpy(sh2->dev[dd_idx].page,
  4165. sh->dev[i].page, sh2->dev[dd_idx].offset,
  4166. sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
  4167. &submit);
  4168. set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
  4169. set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
  4170. for (j = 0; j < conf->raid_disks; j++)
  4171. if (j != sh2->pd_idx &&
  4172. j != sh2->qd_idx &&
  4173. !test_bit(R5_Expanded, &sh2->dev[j].flags))
  4174. break;
  4175. if (j == conf->raid_disks) {
  4176. set_bit(STRIPE_EXPAND_READY, &sh2->state);
  4177. set_bit(STRIPE_HANDLE, &sh2->state);
  4178. }
  4179. raid5_release_stripe(sh2);
  4180. }
  4181. /* done submitting copies, wait for them to complete */
  4182. async_tx_quiesce(&tx);
  4183. }
  4184. /*
  4185. * handle_stripe - do things to a stripe.
  4186. *
  4187. * We lock the stripe by setting STRIPE_ACTIVE and then examine the
  4188. * state of various bits to see what needs to be done.
  4189. * Possible results:
  4190. * return some read requests which now have data
  4191. * return some write requests which are safely on storage
  4192. * schedule a read on some buffers
  4193. * schedule a write of some buffers
  4194. * return confirmation of parity correctness
  4195. *
  4196. */
  4197. static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
  4198. {
  4199. struct r5conf *conf = sh->raid_conf;
  4200. int disks = sh->disks;
  4201. struct r5dev *dev;
  4202. int i;
  4203. int do_recovery = 0;
  4204. memset(s, 0, sizeof(*s));
  4205. s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
  4206. s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
  4207. s->failed_num[0] = -1;
  4208. s->failed_num[1] = -1;
  4209. s->log_failed = r5l_log_disk_error(conf);
  4210. /* Now to look around and see what can be done */
  4211. rcu_read_lock();
  4212. for (i=disks; i--; ) {
  4213. struct md_rdev *rdev;
  4214. sector_t first_bad;
  4215. int bad_sectors;
  4216. int is_bad = 0;
  4217. dev = &sh->dev[i];
  4218. pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
  4219. i, dev->flags,
  4220. dev->toread, dev->towrite, dev->written);
  4221. /* maybe we can reply to a read
  4222. *
  4223. * new wantfill requests are only permitted while
  4224. * ops_complete_biofill is guaranteed to be inactive
  4225. */
  4226. if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
  4227. !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
  4228. set_bit(R5_Wantfill, &dev->flags);
  4229. /* now count some things */
  4230. if (test_bit(R5_LOCKED, &dev->flags))
  4231. s->locked++;
  4232. if (test_bit(R5_UPTODATE, &dev->flags))
  4233. s->uptodate++;
  4234. if (test_bit(R5_Wantcompute, &dev->flags)) {
  4235. s->compute++;
  4236. BUG_ON(s->compute > 2);
  4237. }
  4238. if (test_bit(R5_Wantfill, &dev->flags))
  4239. s->to_fill++;
  4240. else if (dev->toread)
  4241. s->to_read++;
  4242. if (dev->towrite) {
  4243. s->to_write++;
  4244. if (!test_bit(R5_OVERWRITE, &dev->flags))
  4245. s->non_overwrite++;
  4246. }
  4247. if (dev->written)
  4248. s->written++;
  4249. /* Prefer to use the replacement for reads, but only
  4250. * if it is recovered enough and has no bad blocks.
  4251. */
  4252. rdev = rcu_dereference(conf->disks[i].replacement);
  4253. if (rdev && !test_bit(Faulty, &rdev->flags) &&
  4254. rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
  4255. !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
  4256. &first_bad, &bad_sectors))
  4257. set_bit(R5_ReadRepl, &dev->flags);
  4258. else {
  4259. if (rdev && !test_bit(Faulty, &rdev->flags))
  4260. set_bit(R5_NeedReplace, &dev->flags);
  4261. else
  4262. clear_bit(R5_NeedReplace, &dev->flags);
  4263. rdev = rcu_dereference(conf->disks[i].rdev);
  4264. clear_bit(R5_ReadRepl, &dev->flags);
  4265. }
  4266. if (rdev && test_bit(Faulty, &rdev->flags))
  4267. rdev = NULL;
  4268. if (rdev) {
  4269. is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
  4270. &first_bad, &bad_sectors);
  4271. if (s->blocked_rdev == NULL
  4272. && (test_bit(Blocked, &rdev->flags)
  4273. || is_bad < 0)) {
  4274. if (is_bad < 0)
  4275. set_bit(BlockedBadBlocks,
  4276. &rdev->flags);
  4277. s->blocked_rdev = rdev;
  4278. atomic_inc(&rdev->nr_pending);
  4279. }
  4280. }
  4281. clear_bit(R5_Insync, &dev->flags);
  4282. if (!rdev)
  4283. /* Not in-sync */;
  4284. else if (is_bad) {
  4285. /* also not in-sync */
  4286. if (!test_bit(WriteErrorSeen, &rdev->flags) &&
  4287. test_bit(R5_UPTODATE, &dev->flags)) {
  4288. /* treat as in-sync, but with a read error
  4289. * which we can now try to correct
  4290. */
  4291. set_bit(R5_Insync, &dev->flags);
  4292. set_bit(R5_ReadError, &dev->flags);
  4293. }
  4294. } else if (test_bit(In_sync, &rdev->flags))
  4295. set_bit(R5_Insync, &dev->flags);
  4296. else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
  4297. /* in sync if before recovery_offset */
  4298. set_bit(R5_Insync, &dev->flags);
  4299. else if (test_bit(R5_UPTODATE, &dev->flags) &&
  4300. test_bit(R5_Expanded, &dev->flags))
  4301. /* If we've reshaped into here, we assume it is Insync.
  4302. * We will shortly update recovery_offset to make
  4303. * it official.
  4304. */
  4305. set_bit(R5_Insync, &dev->flags);
  4306. if (test_bit(R5_WriteError, &dev->flags)) {
  4307. /* This flag does not apply to '.replacement'
  4308. * only to .rdev, so make sure to check that*/
  4309. struct md_rdev *rdev2 = rcu_dereference(
  4310. conf->disks[i].rdev);
  4311. if (rdev2 == rdev)
  4312. clear_bit(R5_Insync, &dev->flags);
  4313. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  4314. s->handle_bad_blocks = 1;
  4315. atomic_inc(&rdev2->nr_pending);
  4316. } else
  4317. clear_bit(R5_WriteError, &dev->flags);
  4318. }
  4319. if (test_bit(R5_MadeGood, &dev->flags)) {
  4320. /* This flag does not apply to '.replacement'
  4321. * only to .rdev, so make sure to check that*/
  4322. struct md_rdev *rdev2 = rcu_dereference(
  4323. conf->disks[i].rdev);
  4324. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  4325. s->handle_bad_blocks = 1;
  4326. atomic_inc(&rdev2->nr_pending);
  4327. } else
  4328. clear_bit(R5_MadeGood, &dev->flags);
  4329. }
  4330. if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
  4331. struct md_rdev *rdev2 = rcu_dereference(
  4332. conf->disks[i].replacement);
  4333. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  4334. s->handle_bad_blocks = 1;
  4335. atomic_inc(&rdev2->nr_pending);
  4336. } else
  4337. clear_bit(R5_MadeGoodRepl, &dev->flags);
  4338. }
  4339. if (!test_bit(R5_Insync, &dev->flags)) {
  4340. /* The ReadError flag will just be confusing now */
  4341. clear_bit(R5_ReadError, &dev->flags);
  4342. clear_bit(R5_ReWrite, &dev->flags);
  4343. }
  4344. if (test_bit(R5_ReadError, &dev->flags))
  4345. clear_bit(R5_Insync, &dev->flags);
  4346. if (!test_bit(R5_Insync, &dev->flags)) {
  4347. if (s->failed < 2)
  4348. s->failed_num[s->failed] = i;
  4349. s->failed++;
  4350. if (rdev && !test_bit(Faulty, &rdev->flags))
  4351. do_recovery = 1;
  4352. else if (!rdev) {
  4353. rdev = rcu_dereference(
  4354. conf->disks[i].replacement);
  4355. if (rdev && !test_bit(Faulty, &rdev->flags))
  4356. do_recovery = 1;
  4357. }
  4358. }
  4359. if (test_bit(R5_InJournal, &dev->flags))
  4360. s->injournal++;
  4361. if (test_bit(R5_InJournal, &dev->flags) && dev->written)
  4362. s->just_cached++;
  4363. }
  4364. if (test_bit(STRIPE_SYNCING, &sh->state)) {
  4365. /* If there is a failed device being replaced,
  4366. * we must be recovering.
  4367. * else if we are after recovery_cp, we must be syncing
  4368. * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
  4369. * else we can only be replacing
  4370. * sync and recovery both need to read all devices, and so
  4371. * use the same flag.
  4372. */
  4373. if (do_recovery ||
  4374. sh->sector >= conf->mddev->recovery_cp ||
  4375. test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
  4376. s->syncing = 1;
  4377. else
  4378. s->replacing = 1;
  4379. }
  4380. rcu_read_unlock();
  4381. }
  4382. /*
  4383. * Return '1' if this is a member of batch, or '0' if it is a lone stripe or
  4384. * a head which can now be handled.
  4385. */
  4386. static int clear_batch_ready(struct stripe_head *sh)
  4387. {
  4388. struct stripe_head *tmp;
  4389. if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
  4390. return (sh->batch_head && sh->batch_head != sh);
  4391. spin_lock(&sh->stripe_lock);
  4392. if (!sh->batch_head) {
  4393. spin_unlock(&sh->stripe_lock);
  4394. return 0;
  4395. }
  4396. /*
  4397. * this stripe could be added to a batch list before we check
  4398. * BATCH_READY, skips it
  4399. */
  4400. if (sh->batch_head != sh) {
  4401. spin_unlock(&sh->stripe_lock);
  4402. return 1;
  4403. }
  4404. spin_lock(&sh->batch_lock);
  4405. list_for_each_entry(tmp, &sh->batch_list, batch_list)
  4406. clear_bit(STRIPE_BATCH_READY, &tmp->state);
  4407. spin_unlock(&sh->batch_lock);
  4408. spin_unlock(&sh->stripe_lock);
  4409. /*
  4410. * BATCH_READY is cleared, no new stripes can be added.
  4411. * batch_list can be accessed without lock
  4412. */
  4413. return 0;
  4414. }
  4415. static void break_stripe_batch_list(struct stripe_head *head_sh,
  4416. unsigned long handle_flags)
  4417. {
  4418. struct stripe_head *sh, *next;
  4419. int i;
  4420. int do_wakeup = 0;
  4421. list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
  4422. list_del_init(&sh->batch_list);
  4423. WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
  4424. (1 << STRIPE_SYNCING) |
  4425. (1 << STRIPE_REPLACED) |
  4426. (1 << STRIPE_DELAYED) |
  4427. (1 << STRIPE_BIT_DELAY) |
  4428. (1 << STRIPE_FULL_WRITE) |
  4429. (1 << STRIPE_BIOFILL_RUN) |
  4430. (1 << STRIPE_COMPUTE_RUN) |
  4431. (1 << STRIPE_DISCARD) |
  4432. (1 << STRIPE_BATCH_READY) |
  4433. (1 << STRIPE_BATCH_ERR) |
  4434. (1 << STRIPE_BITMAP_PENDING)),
  4435. "stripe state: %lx\n", sh->state);
  4436. WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) |
  4437. (1 << STRIPE_REPLACED)),
  4438. "head stripe state: %lx\n", head_sh->state);
  4439. set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
  4440. (1 << STRIPE_PREREAD_ACTIVE) |
  4441. (1 << STRIPE_DEGRADED) |
  4442. (1 << STRIPE_ON_UNPLUG_LIST)),
  4443. head_sh->state & (1 << STRIPE_INSYNC));
  4444. sh->check_state = head_sh->check_state;
  4445. sh->reconstruct_state = head_sh->reconstruct_state;
  4446. spin_lock_irq(&sh->stripe_lock);
  4447. sh->batch_head = NULL;
  4448. spin_unlock_irq(&sh->stripe_lock);
  4449. for (i = 0; i < sh->disks; i++) {
  4450. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  4451. do_wakeup = 1;
  4452. sh->dev[i].flags = head_sh->dev[i].flags &
  4453. (~((1 << R5_WriteError) | (1 << R5_Overlap)));
  4454. }
  4455. if (handle_flags == 0 ||
  4456. sh->state & handle_flags)
  4457. set_bit(STRIPE_HANDLE, &sh->state);
  4458. raid5_release_stripe(sh);
  4459. }
  4460. spin_lock_irq(&head_sh->stripe_lock);
  4461. head_sh->batch_head = NULL;
  4462. spin_unlock_irq(&head_sh->stripe_lock);
  4463. for (i = 0; i < head_sh->disks; i++)
  4464. if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
  4465. do_wakeup = 1;
  4466. if (head_sh->state & handle_flags)
  4467. set_bit(STRIPE_HANDLE, &head_sh->state);
  4468. if (do_wakeup)
  4469. wake_up(&head_sh->raid_conf->wait_for_overlap);
  4470. }
  4471. static void handle_stripe(struct stripe_head *sh)
  4472. {
  4473. struct stripe_head_state s;
  4474. struct r5conf *conf = sh->raid_conf;
  4475. int i;
  4476. int prexor;
  4477. int disks = sh->disks;
  4478. struct r5dev *pdev, *qdev;
  4479. clear_bit(STRIPE_HANDLE, &sh->state);
  4480. /*
  4481. * handle_stripe should not continue handle the batched stripe, only
  4482. * the head of batch list or lone stripe can continue. Otherwise we
  4483. * could see break_stripe_batch_list warns about the STRIPE_ACTIVE
  4484. * is set for the batched stripe.
  4485. */
  4486. if (clear_batch_ready(sh))
  4487. return;
  4488. if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
  4489. /* already being handled, ensure it gets handled
  4490. * again when current action finishes */
  4491. set_bit(STRIPE_HANDLE, &sh->state);
  4492. return;
  4493. }
  4494. if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
  4495. break_stripe_batch_list(sh, 0);
  4496. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
  4497. spin_lock(&sh->stripe_lock);
  4498. /*
  4499. * Cannot process 'sync' concurrently with 'discard'.
  4500. * Flush data in r5cache before 'sync'.
  4501. */
  4502. if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
  4503. !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&
  4504. !test_bit(STRIPE_DISCARD, &sh->state) &&
  4505. test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
  4506. set_bit(STRIPE_SYNCING, &sh->state);
  4507. clear_bit(STRIPE_INSYNC, &sh->state);
  4508. clear_bit(STRIPE_REPLACED, &sh->state);
  4509. }
  4510. spin_unlock(&sh->stripe_lock);
  4511. }
  4512. clear_bit(STRIPE_DELAYED, &sh->state);
  4513. pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
  4514. "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
  4515. (unsigned long long)sh->sector, sh->state,
  4516. atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
  4517. sh->check_state, sh->reconstruct_state);
  4518. analyse_stripe(sh, &s);
  4519. if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
  4520. goto finish;
  4521. if (s.handle_bad_blocks ||
  4522. test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
  4523. set_bit(STRIPE_HANDLE, &sh->state);
  4524. goto finish;
  4525. }
  4526. if (unlikely(s.blocked_rdev)) {
  4527. if (s.syncing || s.expanding || s.expanded ||
  4528. s.replacing || s.to_write || s.written) {
  4529. set_bit(STRIPE_HANDLE, &sh->state);
  4530. goto finish;
  4531. }
  4532. /* There is nothing for the blocked_rdev to block */
  4533. rdev_dec_pending(s.blocked_rdev, conf->mddev);
  4534. s.blocked_rdev = NULL;
  4535. }
  4536. if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
  4537. set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
  4538. set_bit(STRIPE_BIOFILL_RUN, &sh->state);
  4539. }
  4540. pr_debug("locked=%d uptodate=%d to_read=%d"
  4541. " to_write=%d failed=%d failed_num=%d,%d\n",
  4542. s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
  4543. s.failed_num[0], s.failed_num[1]);
  4544. /*
  4545. * check if the array has lost more than max_degraded devices and,
  4546. * if so, some requests might need to be failed.
  4547. *
  4548. * When journal device failed (log_failed), we will only process
  4549. * the stripe if there is data need write to raid disks
  4550. */
  4551. if (s.failed > conf->max_degraded ||
  4552. (s.log_failed && s.injournal == 0)) {
  4553. sh->check_state = 0;
  4554. sh->reconstruct_state = 0;
  4555. break_stripe_batch_list(sh, 0);
  4556. if (s.to_read+s.to_write+s.written)
  4557. handle_failed_stripe(conf, sh, &s, disks);
  4558. if (s.syncing + s.replacing)
  4559. handle_failed_sync(conf, sh, &s);
  4560. }
  4561. /* Now we check to see if any write operations have recently
  4562. * completed
  4563. */
  4564. prexor = 0;
  4565. if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
  4566. prexor = 1;
  4567. if (sh->reconstruct_state == reconstruct_state_drain_result ||
  4568. sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
  4569. sh->reconstruct_state = reconstruct_state_idle;
  4570. /* All the 'written' buffers and the parity block are ready to
  4571. * be written back to disk
  4572. */
  4573. BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
  4574. !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
  4575. BUG_ON(sh->qd_idx >= 0 &&
  4576. !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
  4577. !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
  4578. for (i = disks; i--; ) {
  4579. struct r5dev *dev = &sh->dev[i];
  4580. if (test_bit(R5_LOCKED, &dev->flags) &&
  4581. (i == sh->pd_idx || i == sh->qd_idx ||
  4582. dev->written || test_bit(R5_InJournal,
  4583. &dev->flags))) {
  4584. pr_debug("Writing block %d\n", i);
  4585. set_bit(R5_Wantwrite, &dev->flags);
  4586. if (prexor)
  4587. continue;
  4588. if (s.failed > 1)
  4589. continue;
  4590. if (!test_bit(R5_Insync, &dev->flags) ||
  4591. ((i == sh->pd_idx || i == sh->qd_idx) &&
  4592. s.failed == 0))
  4593. set_bit(STRIPE_INSYNC, &sh->state);
  4594. }
  4595. }
  4596. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  4597. s.dec_preread_active = 1;
  4598. }
  4599. /*
  4600. * might be able to return some write requests if the parity blocks
  4601. * are safe, or on a failed drive
  4602. */
  4603. pdev = &sh->dev[sh->pd_idx];
  4604. s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
  4605. || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
  4606. qdev = &sh->dev[sh->qd_idx];
  4607. s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
  4608. || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
  4609. || conf->level < 6;
  4610. if (s.written &&
  4611. (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
  4612. && !test_bit(R5_LOCKED, &pdev->flags)
  4613. && (test_bit(R5_UPTODATE, &pdev->flags) ||
  4614. test_bit(R5_Discard, &pdev->flags))))) &&
  4615. (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
  4616. && !test_bit(R5_LOCKED, &qdev->flags)
  4617. && (test_bit(R5_UPTODATE, &qdev->flags) ||
  4618. test_bit(R5_Discard, &qdev->flags))))))
  4619. handle_stripe_clean_event(conf, sh, disks);
  4620. if (s.just_cached)
  4621. r5c_handle_cached_data_endio(conf, sh, disks);
  4622. log_stripe_write_finished(sh);
  4623. /* Now we might consider reading some blocks, either to check/generate
  4624. * parity, or to satisfy requests
  4625. * or to load a block that is being partially written.
  4626. */
  4627. if (s.to_read || s.non_overwrite
  4628. || (s.to_write && s.failed)
  4629. || (s.syncing && (s.uptodate + s.compute < disks))
  4630. || s.replacing
  4631. || s.expanding)
  4632. handle_stripe_fill(sh, &s, disks);
  4633. /*
  4634. * When the stripe finishes full journal write cycle (write to journal
  4635. * and raid disk), this is the clean up procedure so it is ready for
  4636. * next operation.
  4637. */
  4638. r5c_finish_stripe_write_out(conf, sh, &s);
  4639. /*
  4640. * Now to consider new write requests, cache write back and what else,
  4641. * if anything should be read. We do not handle new writes when:
  4642. * 1/ A 'write' operation (copy+xor) is already in flight.
  4643. * 2/ A 'check' operation is in flight, as it may clobber the parity
  4644. * block.
  4645. * 3/ A r5c cache log write is in flight.
  4646. */
  4647. if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
  4648. if (!r5c_is_writeback(conf->log)) {
  4649. if (s.to_write)
  4650. handle_stripe_dirtying(conf, sh, &s, disks);
  4651. } else { /* write back cache */
  4652. int ret = 0;
  4653. /* First, try handle writes in caching phase */
  4654. if (s.to_write)
  4655. ret = r5c_try_caching_write(conf, sh, &s,
  4656. disks);
  4657. /*
  4658. * If caching phase failed: ret == -EAGAIN
  4659. * OR
  4660. * stripe under reclaim: !caching && injournal
  4661. *
  4662. * fall back to handle_stripe_dirtying()
  4663. */
  4664. if (ret == -EAGAIN ||
  4665. /* stripe under reclaim: !caching && injournal */
  4666. (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
  4667. s.injournal > 0)) {
  4668. ret = handle_stripe_dirtying(conf, sh, &s,
  4669. disks);
  4670. if (ret == -EAGAIN)
  4671. goto finish;
  4672. }
  4673. }
  4674. }
  4675. /* maybe we need to check and possibly fix the parity for this stripe
  4676. * Any reads will already have been scheduled, so we just see if enough
  4677. * data is available. The parity check is held off while parity
  4678. * dependent operations are in flight.
  4679. */
  4680. if (sh->check_state ||
  4681. (s.syncing && s.locked == 0 &&
  4682. !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
  4683. !test_bit(STRIPE_INSYNC, &sh->state))) {
  4684. if (conf->level == 6)
  4685. handle_parity_checks6(conf, sh, &s, disks);
  4686. else
  4687. handle_parity_checks5(conf, sh, &s, disks);
  4688. }
  4689. if ((s.replacing || s.syncing) && s.locked == 0
  4690. && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
  4691. && !test_bit(STRIPE_REPLACED, &sh->state)) {
  4692. /* Write out to replacement devices where possible */
  4693. for (i = 0; i < conf->raid_disks; i++)
  4694. if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
  4695. WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
  4696. set_bit(R5_WantReplace, &sh->dev[i].flags);
  4697. set_bit(R5_LOCKED, &sh->dev[i].flags);
  4698. s.locked++;
  4699. }
  4700. if (s.replacing)
  4701. set_bit(STRIPE_INSYNC, &sh->state);
  4702. set_bit(STRIPE_REPLACED, &sh->state);
  4703. }
  4704. if ((s.syncing || s.replacing) && s.locked == 0 &&
  4705. !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
  4706. test_bit(STRIPE_INSYNC, &sh->state)) {
  4707. md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
  4708. clear_bit(STRIPE_SYNCING, &sh->state);
  4709. if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
  4710. wake_up(&conf->wait_for_overlap);
  4711. }
  4712. /* If the failed drives are just a ReadError, then we might need
  4713. * to progress the repair/check process
  4714. */
  4715. if (s.failed <= conf->max_degraded && !conf->mddev->ro)
  4716. for (i = 0; i < s.failed; i++) {
  4717. struct r5dev *dev = &sh->dev[s.failed_num[i]];
  4718. if (test_bit(R5_ReadError, &dev->flags)
  4719. && !test_bit(R5_LOCKED, &dev->flags)
  4720. && test_bit(R5_UPTODATE, &dev->flags)
  4721. ) {
  4722. if (!test_bit(R5_ReWrite, &dev->flags)) {
  4723. set_bit(R5_Wantwrite, &dev->flags);
  4724. set_bit(R5_ReWrite, &dev->flags);
  4725. } else
  4726. /* let's read it back */
  4727. set_bit(R5_Wantread, &dev->flags);
  4728. set_bit(R5_LOCKED, &dev->flags);
  4729. s.locked++;
  4730. }
  4731. }
  4732. /* Finish reconstruct operations initiated by the expansion process */
  4733. if (sh->reconstruct_state == reconstruct_state_result) {
  4734. struct stripe_head *sh_src
  4735. = raid5_get_active_stripe(conf, NULL, sh->sector,
  4736. R5_GAS_PREVIOUS | R5_GAS_NOBLOCK |
  4737. R5_GAS_NOQUIESCE);
  4738. if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
  4739. /* sh cannot be written until sh_src has been read.
  4740. * so arrange for sh to be delayed a little
  4741. */
  4742. set_bit(STRIPE_DELAYED, &sh->state);
  4743. set_bit(STRIPE_HANDLE, &sh->state);
  4744. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
  4745. &sh_src->state))
  4746. atomic_inc(&conf->preread_active_stripes);
  4747. raid5_release_stripe(sh_src);
  4748. goto finish;
  4749. }
  4750. if (sh_src)
  4751. raid5_release_stripe(sh_src);
  4752. sh->reconstruct_state = reconstruct_state_idle;
  4753. clear_bit(STRIPE_EXPANDING, &sh->state);
  4754. for (i = conf->raid_disks; i--; ) {
  4755. set_bit(R5_Wantwrite, &sh->dev[i].flags);
  4756. set_bit(R5_LOCKED, &sh->dev[i].flags);
  4757. s.locked++;
  4758. }
  4759. }
  4760. if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
  4761. !sh->reconstruct_state) {
  4762. /* Need to write out all blocks after computing parity */
  4763. sh->disks = conf->raid_disks;
  4764. stripe_set_idx(sh->sector, conf, 0, sh);
  4765. schedule_reconstruction(sh, &s, 1, 1);
  4766. } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
  4767. clear_bit(STRIPE_EXPAND_READY, &sh->state);
  4768. atomic_dec(&conf->reshape_stripes);
  4769. wake_up(&conf->wait_for_overlap);
  4770. md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
  4771. }
  4772. if (s.expanding && s.locked == 0 &&
  4773. !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
  4774. handle_stripe_expansion(conf, sh);
  4775. finish:
  4776. /* wait for this device to become unblocked */
  4777. if (unlikely(s.blocked_rdev)) {
  4778. if (conf->mddev->external)
  4779. md_wait_for_blocked_rdev(s.blocked_rdev,
  4780. conf->mddev);
  4781. else
  4782. /* Internal metadata will immediately
  4783. * be written by raid5d, so we don't
  4784. * need to wait here.
  4785. */
  4786. rdev_dec_pending(s.blocked_rdev,
  4787. conf->mddev);
  4788. }
  4789. if (s.handle_bad_blocks)
  4790. for (i = disks; i--; ) {
  4791. struct md_rdev *rdev;
  4792. struct r5dev *dev = &sh->dev[i];
  4793. if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
  4794. /* We own a safe reference to the rdev */
  4795. rdev = rdev_pend_deref(conf->disks[i].rdev);
  4796. if (!rdev_set_badblocks(rdev, sh->sector,
  4797. RAID5_STRIPE_SECTORS(conf), 0))
  4798. md_error(conf->mddev, rdev);
  4799. rdev_dec_pending(rdev, conf->mddev);
  4800. }
  4801. if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
  4802. rdev = rdev_pend_deref(conf->disks[i].rdev);
  4803. rdev_clear_badblocks(rdev, sh->sector,
  4804. RAID5_STRIPE_SECTORS(conf), 0);
  4805. rdev_dec_pending(rdev, conf->mddev);
  4806. }
  4807. if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
  4808. rdev = rdev_pend_deref(conf->disks[i].replacement);
  4809. if (!rdev)
  4810. /* rdev have been moved down */
  4811. rdev = rdev_pend_deref(conf->disks[i].rdev);
  4812. rdev_clear_badblocks(rdev, sh->sector,
  4813. RAID5_STRIPE_SECTORS(conf), 0);
  4814. rdev_dec_pending(rdev, conf->mddev);
  4815. }
  4816. }
  4817. if (s.ops_request)
  4818. raid_run_ops(sh, s.ops_request);
  4819. ops_run_io(sh, &s);
  4820. if (s.dec_preread_active) {
  4821. /* We delay this until after ops_run_io so that if make_request
  4822. * is waiting on a flush, it won't continue until the writes
  4823. * have actually been submitted.
  4824. */
  4825. atomic_dec(&conf->preread_active_stripes);
  4826. if (atomic_read(&conf->preread_active_stripes) <
  4827. IO_THRESHOLD)
  4828. md_wakeup_thread(conf->mddev->thread);
  4829. }
  4830. clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
  4831. }
  4832. static void raid5_activate_delayed(struct r5conf *conf)
  4833. __must_hold(&conf->device_lock)
  4834. {
  4835. if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
  4836. while (!list_empty(&conf->delayed_list)) {
  4837. struct list_head *l = conf->delayed_list.next;
  4838. struct stripe_head *sh;
  4839. sh = list_entry(l, struct stripe_head, lru);
  4840. list_del_init(l);
  4841. clear_bit(STRIPE_DELAYED, &sh->state);
  4842. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  4843. atomic_inc(&conf->preread_active_stripes);
  4844. list_add_tail(&sh->lru, &conf->hold_list);
  4845. raid5_wakeup_stripe_thread(sh);
  4846. }
  4847. }
  4848. }
  4849. static void activate_bit_delay(struct r5conf *conf,
  4850. struct list_head *temp_inactive_list)
  4851. __must_hold(&conf->device_lock)
  4852. {
  4853. struct list_head head;
  4854. list_add(&head, &conf->bitmap_list);
  4855. list_del_init(&conf->bitmap_list);
  4856. while (!list_empty(&head)) {
  4857. struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
  4858. int hash;
  4859. list_del_init(&sh->lru);
  4860. atomic_inc(&sh->count);
  4861. hash = sh->hash_lock_index;
  4862. __release_stripe(conf, sh, &temp_inactive_list[hash]);
  4863. }
  4864. }
  4865. static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
  4866. {
  4867. struct r5conf *conf = mddev->private;
  4868. sector_t sector = bio->bi_iter.bi_sector;
  4869. unsigned int chunk_sectors;
  4870. unsigned int bio_sectors = bio_sectors(bio);
  4871. chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
  4872. return chunk_sectors >=
  4873. ((sector & (chunk_sectors - 1)) + bio_sectors);
  4874. }
  4875. /*
  4876. * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
  4877. * later sampled by raid5d.
  4878. */
  4879. static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
  4880. {
  4881. unsigned long flags;
  4882. spin_lock_irqsave(&conf->device_lock, flags);
  4883. bi->bi_next = conf->retry_read_aligned_list;
  4884. conf->retry_read_aligned_list = bi;
  4885. spin_unlock_irqrestore(&conf->device_lock, flags);
  4886. md_wakeup_thread(conf->mddev->thread);
  4887. }
  4888. static struct bio *remove_bio_from_retry(struct r5conf *conf,
  4889. unsigned int *offset)
  4890. {
  4891. struct bio *bi;
  4892. bi = conf->retry_read_aligned;
  4893. if (bi) {
  4894. *offset = conf->retry_read_offset;
  4895. conf->retry_read_aligned = NULL;
  4896. return bi;
  4897. }
  4898. bi = conf->retry_read_aligned_list;
  4899. if(bi) {
  4900. conf->retry_read_aligned_list = bi->bi_next;
  4901. bi->bi_next = NULL;
  4902. *offset = 0;
  4903. }
  4904. return bi;
  4905. }
  4906. /*
  4907. * The "raid5_align_endio" should check if the read succeeded and if it
  4908. * did, call bio_endio on the original bio (having bio_put the new bio
  4909. * first).
  4910. * If the read failed..
  4911. */
  4912. static void raid5_align_endio(struct bio *bi)
  4913. {
  4914. struct md_io_acct *md_io_acct = bi->bi_private;
  4915. struct bio *raid_bi = md_io_acct->orig_bio;
  4916. struct mddev *mddev;
  4917. struct r5conf *conf;
  4918. struct md_rdev *rdev;
  4919. blk_status_t error = bi->bi_status;
  4920. unsigned long start_time = md_io_acct->start_time;
  4921. bio_put(bi);
  4922. rdev = (void*)raid_bi->bi_next;
  4923. raid_bi->bi_next = NULL;
  4924. mddev = rdev->mddev;
  4925. conf = mddev->private;
  4926. rdev_dec_pending(rdev, conf->mddev);
  4927. if (!error) {
  4928. if (blk_queue_io_stat(raid_bi->bi_bdev->bd_disk->queue))
  4929. bio_end_io_acct(raid_bi, start_time);
  4930. bio_endio(raid_bi);
  4931. if (atomic_dec_and_test(&conf->active_aligned_reads))
  4932. wake_up(&conf->wait_for_quiescent);
  4933. return;
  4934. }
  4935. pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
  4936. add_bio_to_retry(raid_bi, conf);
  4937. }
  4938. static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
  4939. {
  4940. struct r5conf *conf = mddev->private;
  4941. struct bio *align_bio;
  4942. struct md_rdev *rdev;
  4943. sector_t sector, end_sector, first_bad;
  4944. int bad_sectors, dd_idx;
  4945. struct md_io_acct *md_io_acct;
  4946. bool did_inc;
  4947. if (!in_chunk_boundary(mddev, raid_bio)) {
  4948. pr_debug("%s: non aligned\n", __func__);
  4949. return 0;
  4950. }
  4951. sector = raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 0,
  4952. &dd_idx, NULL);
  4953. end_sector = sector + bio_sectors(raid_bio);
  4954. rcu_read_lock();
  4955. if (r5c_big_stripe_cached(conf, sector))
  4956. goto out_rcu_unlock;
  4957. rdev = rcu_dereference(conf->disks[dd_idx].replacement);
  4958. if (!rdev || test_bit(Faulty, &rdev->flags) ||
  4959. rdev->recovery_offset < end_sector) {
  4960. rdev = rcu_dereference(conf->disks[dd_idx].rdev);
  4961. if (!rdev)
  4962. goto out_rcu_unlock;
  4963. if (test_bit(Faulty, &rdev->flags) ||
  4964. !(test_bit(In_sync, &rdev->flags) ||
  4965. rdev->recovery_offset >= end_sector))
  4966. goto out_rcu_unlock;
  4967. }
  4968. atomic_inc(&rdev->nr_pending);
  4969. rcu_read_unlock();
  4970. if (is_badblock(rdev, sector, bio_sectors(raid_bio), &first_bad,
  4971. &bad_sectors)) {
  4972. rdev_dec_pending(rdev, mddev);
  4973. return 0;
  4974. }
  4975. align_bio = bio_alloc_clone(rdev->bdev, raid_bio, GFP_NOIO,
  4976. &mddev->io_acct_set);
  4977. md_io_acct = container_of(align_bio, struct md_io_acct, bio_clone);
  4978. raid_bio->bi_next = (void *)rdev;
  4979. if (blk_queue_io_stat(raid_bio->bi_bdev->bd_disk->queue))
  4980. md_io_acct->start_time = bio_start_io_acct(raid_bio);
  4981. md_io_acct->orig_bio = raid_bio;
  4982. align_bio->bi_end_io = raid5_align_endio;
  4983. align_bio->bi_private = md_io_acct;
  4984. align_bio->bi_iter.bi_sector = sector;
  4985. /* No reshape active, so we can trust rdev->data_offset */
  4986. align_bio->bi_iter.bi_sector += rdev->data_offset;
  4987. did_inc = false;
  4988. if (conf->quiesce == 0) {
  4989. atomic_inc(&conf->active_aligned_reads);
  4990. did_inc = true;
  4991. }
  4992. /* need a memory barrier to detect the race with raid5_quiesce() */
  4993. if (!did_inc || smp_load_acquire(&conf->quiesce) != 0) {
  4994. /* quiesce is in progress, so we need to undo io activation and wait
  4995. * for it to finish
  4996. */
  4997. if (did_inc && atomic_dec_and_test(&conf->active_aligned_reads))
  4998. wake_up(&conf->wait_for_quiescent);
  4999. spin_lock_irq(&conf->device_lock);
  5000. wait_event_lock_irq(conf->wait_for_quiescent, conf->quiesce == 0,
  5001. conf->device_lock);
  5002. atomic_inc(&conf->active_aligned_reads);
  5003. spin_unlock_irq(&conf->device_lock);
  5004. }
  5005. if (mddev->gendisk)
  5006. trace_block_bio_remap(align_bio, disk_devt(mddev->gendisk),
  5007. raid_bio->bi_iter.bi_sector);
  5008. submit_bio_noacct(align_bio);
  5009. return 1;
  5010. out_rcu_unlock:
  5011. rcu_read_unlock();
  5012. return 0;
  5013. }
  5014. static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
  5015. {
  5016. struct bio *split;
  5017. sector_t sector = raid_bio->bi_iter.bi_sector;
  5018. unsigned chunk_sects = mddev->chunk_sectors;
  5019. unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
  5020. if (sectors < bio_sectors(raid_bio)) {
  5021. struct r5conf *conf = mddev->private;
  5022. split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
  5023. bio_chain(split, raid_bio);
  5024. submit_bio_noacct(raid_bio);
  5025. raid_bio = split;
  5026. }
  5027. if (!raid5_read_one_chunk(mddev, raid_bio))
  5028. return raid_bio;
  5029. return NULL;
  5030. }
  5031. /* __get_priority_stripe - get the next stripe to process
  5032. *
  5033. * Full stripe writes are allowed to pass preread active stripes up until
  5034. * the bypass_threshold is exceeded. In general the bypass_count
  5035. * increments when the handle_list is handled before the hold_list; however, it
  5036. * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
  5037. * stripe with in flight i/o. The bypass_count will be reset when the
  5038. * head of the hold_list has changed, i.e. the head was promoted to the
  5039. * handle_list.
  5040. */
  5041. static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
  5042. __must_hold(&conf->device_lock)
  5043. {
  5044. struct stripe_head *sh, *tmp;
  5045. struct list_head *handle_list = NULL;
  5046. struct r5worker_group *wg;
  5047. bool second_try = !r5c_is_writeback(conf->log) &&
  5048. !r5l_log_disk_error(conf);
  5049. bool try_loprio = test_bit(R5C_LOG_TIGHT, &conf->cache_state) ||
  5050. r5l_log_disk_error(conf);
  5051. again:
  5052. wg = NULL;
  5053. sh = NULL;
  5054. if (conf->worker_cnt_per_group == 0) {
  5055. handle_list = try_loprio ? &conf->loprio_list :
  5056. &conf->handle_list;
  5057. } else if (group != ANY_GROUP) {
  5058. handle_list = try_loprio ? &conf->worker_groups[group].loprio_list :
  5059. &conf->worker_groups[group].handle_list;
  5060. wg = &conf->worker_groups[group];
  5061. } else {
  5062. int i;
  5063. for (i = 0; i < conf->group_cnt; i++) {
  5064. handle_list = try_loprio ? &conf->worker_groups[i].loprio_list :
  5065. &conf->worker_groups[i].handle_list;
  5066. wg = &conf->worker_groups[i];
  5067. if (!list_empty(handle_list))
  5068. break;
  5069. }
  5070. }
  5071. pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
  5072. __func__,
  5073. list_empty(handle_list) ? "empty" : "busy",
  5074. list_empty(&conf->hold_list) ? "empty" : "busy",
  5075. atomic_read(&conf->pending_full_writes), conf->bypass_count);
  5076. if (!list_empty(handle_list)) {
  5077. sh = list_entry(handle_list->next, typeof(*sh), lru);
  5078. if (list_empty(&conf->hold_list))
  5079. conf->bypass_count = 0;
  5080. else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
  5081. if (conf->hold_list.next == conf->last_hold)
  5082. conf->bypass_count++;
  5083. else {
  5084. conf->last_hold = conf->hold_list.next;
  5085. conf->bypass_count -= conf->bypass_threshold;
  5086. if (conf->bypass_count < 0)
  5087. conf->bypass_count = 0;
  5088. }
  5089. }
  5090. } else if (!list_empty(&conf->hold_list) &&
  5091. ((conf->bypass_threshold &&
  5092. conf->bypass_count > conf->bypass_threshold) ||
  5093. atomic_read(&conf->pending_full_writes) == 0)) {
  5094. list_for_each_entry(tmp, &conf->hold_list, lru) {
  5095. if (conf->worker_cnt_per_group == 0 ||
  5096. group == ANY_GROUP ||
  5097. !cpu_online(tmp->cpu) ||
  5098. cpu_to_group(tmp->cpu) == group) {
  5099. sh = tmp;
  5100. break;
  5101. }
  5102. }
  5103. if (sh) {
  5104. conf->bypass_count -= conf->bypass_threshold;
  5105. if (conf->bypass_count < 0)
  5106. conf->bypass_count = 0;
  5107. }
  5108. wg = NULL;
  5109. }
  5110. if (!sh) {
  5111. if (second_try)
  5112. return NULL;
  5113. second_try = true;
  5114. try_loprio = !try_loprio;
  5115. goto again;
  5116. }
  5117. if (wg) {
  5118. wg->stripes_cnt--;
  5119. sh->group = NULL;
  5120. }
  5121. list_del_init(&sh->lru);
  5122. BUG_ON(atomic_inc_return(&sh->count) != 1);
  5123. return sh;
  5124. }
  5125. struct raid5_plug_cb {
  5126. struct blk_plug_cb cb;
  5127. struct list_head list;
  5128. struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
  5129. };
  5130. static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
  5131. {
  5132. struct raid5_plug_cb *cb = container_of(
  5133. blk_cb, struct raid5_plug_cb, cb);
  5134. struct stripe_head *sh;
  5135. struct mddev *mddev = cb->cb.data;
  5136. struct r5conf *conf = mddev->private;
  5137. int cnt = 0;
  5138. int hash;
  5139. if (cb->list.next && !list_empty(&cb->list)) {
  5140. spin_lock_irq(&conf->device_lock);
  5141. while (!list_empty(&cb->list)) {
  5142. sh = list_first_entry(&cb->list, struct stripe_head, lru);
  5143. list_del_init(&sh->lru);
  5144. /*
  5145. * avoid race release_stripe_plug() sees
  5146. * STRIPE_ON_UNPLUG_LIST clear but the stripe
  5147. * is still in our list
  5148. */
  5149. smp_mb__before_atomic();
  5150. clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
  5151. /*
  5152. * STRIPE_ON_RELEASE_LIST could be set here. In that
  5153. * case, the count is always > 1 here
  5154. */
  5155. hash = sh->hash_lock_index;
  5156. __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
  5157. cnt++;
  5158. }
  5159. spin_unlock_irq(&conf->device_lock);
  5160. }
  5161. release_inactive_stripe_list(conf, cb->temp_inactive_list,
  5162. NR_STRIPE_HASH_LOCKS);
  5163. if (mddev->queue)
  5164. trace_block_unplug(mddev->queue, cnt, !from_schedule);
  5165. kfree(cb);
  5166. }
  5167. static void release_stripe_plug(struct mddev *mddev,
  5168. struct stripe_head *sh)
  5169. {
  5170. struct blk_plug_cb *blk_cb = blk_check_plugged(
  5171. raid5_unplug, mddev,
  5172. sizeof(struct raid5_plug_cb));
  5173. struct raid5_plug_cb *cb;
  5174. if (!blk_cb) {
  5175. raid5_release_stripe(sh);
  5176. return;
  5177. }
  5178. cb = container_of(blk_cb, struct raid5_plug_cb, cb);
  5179. if (cb->list.next == NULL) {
  5180. int i;
  5181. INIT_LIST_HEAD(&cb->list);
  5182. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  5183. INIT_LIST_HEAD(cb->temp_inactive_list + i);
  5184. }
  5185. if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
  5186. list_add_tail(&sh->lru, &cb->list);
  5187. else
  5188. raid5_release_stripe(sh);
  5189. }
  5190. static void make_discard_request(struct mddev *mddev, struct bio *bi)
  5191. {
  5192. struct r5conf *conf = mddev->private;
  5193. sector_t logical_sector, last_sector;
  5194. struct stripe_head *sh;
  5195. int stripe_sectors;
  5196. /* We need to handle this when io_uring supports discard/trim */
  5197. if (WARN_ON_ONCE(bi->bi_opf & REQ_NOWAIT))
  5198. return;
  5199. if (mddev->reshape_position != MaxSector)
  5200. /* Skip discard while reshape is happening */
  5201. return;
  5202. logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5203. last_sector = bio_end_sector(bi);
  5204. bi->bi_next = NULL;
  5205. stripe_sectors = conf->chunk_sectors *
  5206. (conf->raid_disks - conf->max_degraded);
  5207. logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
  5208. stripe_sectors);
  5209. sector_div(last_sector, stripe_sectors);
  5210. logical_sector *= conf->chunk_sectors;
  5211. last_sector *= conf->chunk_sectors;
  5212. for (; logical_sector < last_sector;
  5213. logical_sector += RAID5_STRIPE_SECTORS(conf)) {
  5214. DEFINE_WAIT(w);
  5215. int d;
  5216. again:
  5217. sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0);
  5218. prepare_to_wait(&conf->wait_for_overlap, &w,
  5219. TASK_UNINTERRUPTIBLE);
  5220. set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
  5221. if (test_bit(STRIPE_SYNCING, &sh->state)) {
  5222. raid5_release_stripe(sh);
  5223. schedule();
  5224. goto again;
  5225. }
  5226. clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
  5227. spin_lock_irq(&sh->stripe_lock);
  5228. for (d = 0; d < conf->raid_disks; d++) {
  5229. if (d == sh->pd_idx || d == sh->qd_idx)
  5230. continue;
  5231. if (sh->dev[d].towrite || sh->dev[d].toread) {
  5232. set_bit(R5_Overlap, &sh->dev[d].flags);
  5233. spin_unlock_irq(&sh->stripe_lock);
  5234. raid5_release_stripe(sh);
  5235. schedule();
  5236. goto again;
  5237. }
  5238. }
  5239. set_bit(STRIPE_DISCARD, &sh->state);
  5240. finish_wait(&conf->wait_for_overlap, &w);
  5241. sh->overwrite_disks = 0;
  5242. for (d = 0; d < conf->raid_disks; d++) {
  5243. if (d == sh->pd_idx || d == sh->qd_idx)
  5244. continue;
  5245. sh->dev[d].towrite = bi;
  5246. set_bit(R5_OVERWRITE, &sh->dev[d].flags);
  5247. bio_inc_remaining(bi);
  5248. md_write_inc(mddev, bi);
  5249. sh->overwrite_disks++;
  5250. }
  5251. spin_unlock_irq(&sh->stripe_lock);
  5252. if (conf->mddev->bitmap) {
  5253. for (d = 0;
  5254. d < conf->raid_disks - conf->max_degraded;
  5255. d++)
  5256. md_bitmap_startwrite(mddev->bitmap,
  5257. sh->sector,
  5258. RAID5_STRIPE_SECTORS(conf),
  5259. 0);
  5260. sh->bm_seq = conf->seq_flush + 1;
  5261. set_bit(STRIPE_BIT_DELAY, &sh->state);
  5262. }
  5263. set_bit(STRIPE_HANDLE, &sh->state);
  5264. clear_bit(STRIPE_DELAYED, &sh->state);
  5265. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  5266. atomic_inc(&conf->preread_active_stripes);
  5267. release_stripe_plug(mddev, sh);
  5268. }
  5269. bio_endio(bi);
  5270. }
  5271. static bool ahead_of_reshape(struct mddev *mddev, sector_t sector,
  5272. sector_t reshape_sector)
  5273. {
  5274. return mddev->reshape_backwards ? sector < reshape_sector :
  5275. sector >= reshape_sector;
  5276. }
  5277. static bool range_ahead_of_reshape(struct mddev *mddev, sector_t min,
  5278. sector_t max, sector_t reshape_sector)
  5279. {
  5280. return mddev->reshape_backwards ? max < reshape_sector :
  5281. min >= reshape_sector;
  5282. }
  5283. static bool stripe_ahead_of_reshape(struct mddev *mddev, struct r5conf *conf,
  5284. struct stripe_head *sh)
  5285. {
  5286. sector_t max_sector = 0, min_sector = MaxSector;
  5287. bool ret = false;
  5288. int dd_idx;
  5289. for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {
  5290. if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
  5291. continue;
  5292. min_sector = min(min_sector, sh->dev[dd_idx].sector);
  5293. max_sector = max(max_sector, sh->dev[dd_idx].sector);
  5294. }
  5295. spin_lock_irq(&conf->device_lock);
  5296. if (!range_ahead_of_reshape(mddev, min_sector, max_sector,
  5297. conf->reshape_progress))
  5298. /* mismatch, need to try again */
  5299. ret = true;
  5300. spin_unlock_irq(&conf->device_lock);
  5301. return ret;
  5302. }
  5303. static int add_all_stripe_bios(struct r5conf *conf,
  5304. struct stripe_request_ctx *ctx, struct stripe_head *sh,
  5305. struct bio *bi, int forwrite, int previous)
  5306. {
  5307. int dd_idx;
  5308. int ret = 1;
  5309. spin_lock_irq(&sh->stripe_lock);
  5310. for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {
  5311. struct r5dev *dev = &sh->dev[dd_idx];
  5312. if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
  5313. continue;
  5314. if (dev->sector < ctx->first_sector ||
  5315. dev->sector >= ctx->last_sector)
  5316. continue;
  5317. if (stripe_bio_overlaps(sh, bi, dd_idx, forwrite)) {
  5318. set_bit(R5_Overlap, &dev->flags);
  5319. ret = 0;
  5320. continue;
  5321. }
  5322. }
  5323. if (!ret)
  5324. goto out;
  5325. for (dd_idx = 0; dd_idx < sh->disks; dd_idx++) {
  5326. struct r5dev *dev = &sh->dev[dd_idx];
  5327. if (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
  5328. continue;
  5329. if (dev->sector < ctx->first_sector ||
  5330. dev->sector >= ctx->last_sector)
  5331. continue;
  5332. __add_stripe_bio(sh, bi, dd_idx, forwrite, previous);
  5333. clear_bit((dev->sector - ctx->first_sector) >>
  5334. RAID5_STRIPE_SHIFT(conf), ctx->sectors_to_do);
  5335. }
  5336. out:
  5337. spin_unlock_irq(&sh->stripe_lock);
  5338. return ret;
  5339. }
  5340. static enum stripe_result make_stripe_request(struct mddev *mddev,
  5341. struct r5conf *conf, struct stripe_request_ctx *ctx,
  5342. sector_t logical_sector, struct bio *bi)
  5343. {
  5344. const int rw = bio_data_dir(bi);
  5345. enum stripe_result ret;
  5346. struct stripe_head *sh;
  5347. sector_t new_sector;
  5348. int previous = 0, flags = 0;
  5349. int seq, dd_idx;
  5350. seq = read_seqcount_begin(&conf->gen_lock);
  5351. if (unlikely(conf->reshape_progress != MaxSector)) {
  5352. /*
  5353. * Spinlock is needed as reshape_progress may be
  5354. * 64bit on a 32bit platform, and so it might be
  5355. * possible to see a half-updated value
  5356. * Of course reshape_progress could change after
  5357. * the lock is dropped, so once we get a reference
  5358. * to the stripe that we think it is, we will have
  5359. * to check again.
  5360. */
  5361. spin_lock_irq(&conf->device_lock);
  5362. if (ahead_of_reshape(mddev, logical_sector,
  5363. conf->reshape_progress)) {
  5364. previous = 1;
  5365. } else {
  5366. if (ahead_of_reshape(mddev, logical_sector,
  5367. conf->reshape_safe)) {
  5368. spin_unlock_irq(&conf->device_lock);
  5369. return STRIPE_SCHEDULE_AND_RETRY;
  5370. }
  5371. }
  5372. spin_unlock_irq(&conf->device_lock);
  5373. }
  5374. new_sector = raid5_compute_sector(conf, logical_sector, previous,
  5375. &dd_idx, NULL);
  5376. pr_debug("raid456: %s, sector %llu logical %llu\n", __func__,
  5377. new_sector, logical_sector);
  5378. if (previous)
  5379. flags |= R5_GAS_PREVIOUS;
  5380. if (bi->bi_opf & REQ_RAHEAD)
  5381. flags |= R5_GAS_NOBLOCK;
  5382. sh = raid5_get_active_stripe(conf, ctx, new_sector, flags);
  5383. if (unlikely(!sh)) {
  5384. /* cannot get stripe, just give-up */
  5385. bi->bi_status = BLK_STS_IOERR;
  5386. return STRIPE_FAIL;
  5387. }
  5388. if (unlikely(previous) &&
  5389. stripe_ahead_of_reshape(mddev, conf, sh)) {
  5390. /*
  5391. * Expansion moved on while waiting for a stripe.
  5392. * Expansion could still move past after this
  5393. * test, but as we are holding a reference to
  5394. * 'sh', we know that if that happens,
  5395. * STRIPE_EXPANDING will get set and the expansion
  5396. * won't proceed until we finish with the stripe.
  5397. */
  5398. ret = STRIPE_SCHEDULE_AND_RETRY;
  5399. goto out_release;
  5400. }
  5401. if (read_seqcount_retry(&conf->gen_lock, seq)) {
  5402. /* Might have got the wrong stripe_head by accident */
  5403. ret = STRIPE_RETRY;
  5404. goto out_release;
  5405. }
  5406. if (test_bit(STRIPE_EXPANDING, &sh->state) ||
  5407. !add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {
  5408. /*
  5409. * Stripe is busy expanding or add failed due to
  5410. * overlap. Flush everything and wait a while.
  5411. */
  5412. md_wakeup_thread(mddev->thread);
  5413. ret = STRIPE_SCHEDULE_AND_RETRY;
  5414. goto out_release;
  5415. }
  5416. if (stripe_can_batch(sh)) {
  5417. stripe_add_to_batch_list(conf, sh, ctx->batch_last);
  5418. if (ctx->batch_last)
  5419. raid5_release_stripe(ctx->batch_last);
  5420. atomic_inc(&sh->count);
  5421. ctx->batch_last = sh;
  5422. }
  5423. if (ctx->do_flush) {
  5424. set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
  5425. /* we only need flush for one stripe */
  5426. ctx->do_flush = false;
  5427. }
  5428. set_bit(STRIPE_HANDLE, &sh->state);
  5429. clear_bit(STRIPE_DELAYED, &sh->state);
  5430. if ((!sh->batch_head || sh == sh->batch_head) &&
  5431. (bi->bi_opf & REQ_SYNC) &&
  5432. !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  5433. atomic_inc(&conf->preread_active_stripes);
  5434. release_stripe_plug(mddev, sh);
  5435. return STRIPE_SUCCESS;
  5436. out_release:
  5437. raid5_release_stripe(sh);
  5438. return ret;
  5439. }
  5440. /*
  5441. * If the bio covers multiple data disks, find sector within the bio that has
  5442. * the lowest chunk offset in the first chunk.
  5443. */
  5444. static sector_t raid5_bio_lowest_chunk_sector(struct r5conf *conf,
  5445. struct bio *bi)
  5446. {
  5447. int sectors_per_chunk = conf->chunk_sectors;
  5448. int raid_disks = conf->raid_disks;
  5449. int dd_idx;
  5450. struct stripe_head sh;
  5451. unsigned int chunk_offset;
  5452. sector_t r_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5453. sector_t sector;
  5454. /* We pass in fake stripe_head to get back parity disk numbers */
  5455. sector = raid5_compute_sector(conf, r_sector, 0, &dd_idx, &sh);
  5456. chunk_offset = sector_div(sector, sectors_per_chunk);
  5457. if (sectors_per_chunk - chunk_offset >= bio_sectors(bi))
  5458. return r_sector;
  5459. /*
  5460. * Bio crosses to the next data disk. Check whether it's in the same
  5461. * chunk.
  5462. */
  5463. dd_idx++;
  5464. while (dd_idx == sh.pd_idx || dd_idx == sh.qd_idx)
  5465. dd_idx++;
  5466. if (dd_idx >= raid_disks)
  5467. return r_sector;
  5468. return r_sector + sectors_per_chunk - chunk_offset;
  5469. }
  5470. static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
  5471. {
  5472. DEFINE_WAIT_FUNC(wait, woken_wake_function);
  5473. struct r5conf *conf = mddev->private;
  5474. sector_t logical_sector;
  5475. struct stripe_request_ctx ctx = {};
  5476. const int rw = bio_data_dir(bi);
  5477. enum stripe_result res;
  5478. int s, stripe_cnt;
  5479. if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
  5480. int ret = log_handle_flush_request(conf, bi);
  5481. if (ret == 0)
  5482. return true;
  5483. if (ret == -ENODEV) {
  5484. if (md_flush_request(mddev, bi))
  5485. return true;
  5486. }
  5487. /* ret == -EAGAIN, fallback */
  5488. /*
  5489. * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH,
  5490. * we need to flush journal device
  5491. */
  5492. ctx.do_flush = bi->bi_opf & REQ_PREFLUSH;
  5493. }
  5494. if (!md_write_start(mddev, bi))
  5495. return false;
  5496. /*
  5497. * If array is degraded, better not do chunk aligned read because
  5498. * later we might have to read it again in order to reconstruct
  5499. * data on failed drives.
  5500. */
  5501. if (rw == READ && mddev->degraded == 0 &&
  5502. mddev->reshape_position == MaxSector) {
  5503. bi = chunk_aligned_read(mddev, bi);
  5504. if (!bi)
  5505. return true;
  5506. }
  5507. if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) {
  5508. make_discard_request(mddev, bi);
  5509. md_write_end(mddev);
  5510. return true;
  5511. }
  5512. logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5513. ctx.first_sector = logical_sector;
  5514. ctx.last_sector = bio_end_sector(bi);
  5515. bi->bi_next = NULL;
  5516. stripe_cnt = DIV_ROUND_UP_SECTOR_T(ctx.last_sector - logical_sector,
  5517. RAID5_STRIPE_SECTORS(conf));
  5518. bitmap_set(ctx.sectors_to_do, 0, stripe_cnt);
  5519. pr_debug("raid456: %s, logical %llu to %llu\n", __func__,
  5520. bi->bi_iter.bi_sector, ctx.last_sector);
  5521. /* Bail out if conflicts with reshape and REQ_NOWAIT is set */
  5522. if ((bi->bi_opf & REQ_NOWAIT) &&
  5523. (conf->reshape_progress != MaxSector) &&
  5524. !ahead_of_reshape(mddev, logical_sector, conf->reshape_progress) &&
  5525. ahead_of_reshape(mddev, logical_sector, conf->reshape_safe)) {
  5526. bio_wouldblock_error(bi);
  5527. if (rw == WRITE)
  5528. md_write_end(mddev);
  5529. return true;
  5530. }
  5531. md_account_bio(mddev, &bi);
  5532. /*
  5533. * Lets start with the stripe with the lowest chunk offset in the first
  5534. * chunk. That has the best chances of creating IOs adjacent to
  5535. * previous IOs in case of sequential IO and thus creates the most
  5536. * sequential IO pattern. We don't bother with the optimization when
  5537. * reshaping as the performance benefit is not worth the complexity.
  5538. */
  5539. if (likely(conf->reshape_progress == MaxSector))
  5540. logical_sector = raid5_bio_lowest_chunk_sector(conf, bi);
  5541. s = (logical_sector - ctx.first_sector) >> RAID5_STRIPE_SHIFT(conf);
  5542. add_wait_queue(&conf->wait_for_overlap, &wait);
  5543. while (1) {
  5544. res = make_stripe_request(mddev, conf, &ctx, logical_sector,
  5545. bi);
  5546. if (res == STRIPE_FAIL)
  5547. break;
  5548. if (res == STRIPE_RETRY)
  5549. continue;
  5550. if (res == STRIPE_SCHEDULE_AND_RETRY) {
  5551. /*
  5552. * Must release the reference to batch_last before
  5553. * scheduling and waiting for work to be done,
  5554. * otherwise the batch_last stripe head could prevent
  5555. * raid5_activate_delayed() from making progress
  5556. * and thus deadlocking.
  5557. */
  5558. if (ctx.batch_last) {
  5559. raid5_release_stripe(ctx.batch_last);
  5560. ctx.batch_last = NULL;
  5561. }
  5562. wait_woken(&wait, TASK_UNINTERRUPTIBLE,
  5563. MAX_SCHEDULE_TIMEOUT);
  5564. continue;
  5565. }
  5566. s = find_next_bit_wrap(ctx.sectors_to_do, stripe_cnt, s);
  5567. if (s == stripe_cnt)
  5568. break;
  5569. logical_sector = ctx.first_sector +
  5570. (s << RAID5_STRIPE_SHIFT(conf));
  5571. }
  5572. remove_wait_queue(&conf->wait_for_overlap, &wait);
  5573. if (ctx.batch_last)
  5574. raid5_release_stripe(ctx.batch_last);
  5575. if (rw == WRITE)
  5576. md_write_end(mddev);
  5577. bio_endio(bi);
  5578. return true;
  5579. }
  5580. static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
  5581. static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
  5582. {
  5583. /* reshaping is quite different to recovery/resync so it is
  5584. * handled quite separately ... here.
  5585. *
  5586. * On each call to sync_request, we gather one chunk worth of
  5587. * destination stripes and flag them as expanding.
  5588. * Then we find all the source stripes and request reads.
  5589. * As the reads complete, handle_stripe will copy the data
  5590. * into the destination stripe and release that stripe.
  5591. */
  5592. struct r5conf *conf = mddev->private;
  5593. struct stripe_head *sh;
  5594. struct md_rdev *rdev;
  5595. sector_t first_sector, last_sector;
  5596. int raid_disks = conf->previous_raid_disks;
  5597. int data_disks = raid_disks - conf->max_degraded;
  5598. int new_data_disks = conf->raid_disks - conf->max_degraded;
  5599. int i;
  5600. int dd_idx;
  5601. sector_t writepos, readpos, safepos;
  5602. sector_t stripe_addr;
  5603. int reshape_sectors;
  5604. struct list_head stripes;
  5605. sector_t retn;
  5606. if (sector_nr == 0) {
  5607. /* If restarting in the middle, skip the initial sectors */
  5608. if (mddev->reshape_backwards &&
  5609. conf->reshape_progress < raid5_size(mddev, 0, 0)) {
  5610. sector_nr = raid5_size(mddev, 0, 0)
  5611. - conf->reshape_progress;
  5612. } else if (mddev->reshape_backwards &&
  5613. conf->reshape_progress == MaxSector) {
  5614. /* shouldn't happen, but just in case, finish up.*/
  5615. sector_nr = MaxSector;
  5616. } else if (!mddev->reshape_backwards &&
  5617. conf->reshape_progress > 0)
  5618. sector_nr = conf->reshape_progress;
  5619. sector_div(sector_nr, new_data_disks);
  5620. if (sector_nr) {
  5621. mddev->curr_resync_completed = sector_nr;
  5622. sysfs_notify_dirent_safe(mddev->sysfs_completed);
  5623. *skipped = 1;
  5624. retn = sector_nr;
  5625. goto finish;
  5626. }
  5627. }
  5628. /* We need to process a full chunk at a time.
  5629. * If old and new chunk sizes differ, we need to process the
  5630. * largest of these
  5631. */
  5632. reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors);
  5633. /* We update the metadata at least every 10 seconds, or when
  5634. * the data about to be copied would over-write the source of
  5635. * the data at the front of the range. i.e. one new_stripe
  5636. * along from reshape_progress new_maps to after where
  5637. * reshape_safe old_maps to
  5638. */
  5639. writepos = conf->reshape_progress;
  5640. sector_div(writepos, new_data_disks);
  5641. readpos = conf->reshape_progress;
  5642. sector_div(readpos, data_disks);
  5643. safepos = conf->reshape_safe;
  5644. sector_div(safepos, data_disks);
  5645. if (mddev->reshape_backwards) {
  5646. BUG_ON(writepos < reshape_sectors);
  5647. writepos -= reshape_sectors;
  5648. readpos += reshape_sectors;
  5649. safepos += reshape_sectors;
  5650. } else {
  5651. writepos += reshape_sectors;
  5652. /* readpos and safepos are worst-case calculations.
  5653. * A negative number is overly pessimistic, and causes
  5654. * obvious problems for unsigned storage. So clip to 0.
  5655. */
  5656. readpos -= min_t(sector_t, reshape_sectors, readpos);
  5657. safepos -= min_t(sector_t, reshape_sectors, safepos);
  5658. }
  5659. /* Having calculated the 'writepos' possibly use it
  5660. * to set 'stripe_addr' which is where we will write to.
  5661. */
  5662. if (mddev->reshape_backwards) {
  5663. BUG_ON(conf->reshape_progress == 0);
  5664. stripe_addr = writepos;
  5665. BUG_ON((mddev->dev_sectors &
  5666. ~((sector_t)reshape_sectors - 1))
  5667. - reshape_sectors - stripe_addr
  5668. != sector_nr);
  5669. } else {
  5670. BUG_ON(writepos != sector_nr + reshape_sectors);
  5671. stripe_addr = sector_nr;
  5672. }
  5673. /* 'writepos' is the most advanced device address we might write.
  5674. * 'readpos' is the least advanced device address we might read.
  5675. * 'safepos' is the least address recorded in the metadata as having
  5676. * been reshaped.
  5677. * If there is a min_offset_diff, these are adjusted either by
  5678. * increasing the safepos/readpos if diff is negative, or
  5679. * increasing writepos if diff is positive.
  5680. * If 'readpos' is then behind 'writepos', there is no way that we can
  5681. * ensure safety in the face of a crash - that must be done by userspace
  5682. * making a backup of the data. So in that case there is no particular
  5683. * rush to update metadata.
  5684. * Otherwise if 'safepos' is behind 'writepos', then we really need to
  5685. * update the metadata to advance 'safepos' to match 'readpos' so that
  5686. * we can be safe in the event of a crash.
  5687. * So we insist on updating metadata if safepos is behind writepos and
  5688. * readpos is beyond writepos.
  5689. * In any case, update the metadata every 10 seconds.
  5690. * Maybe that number should be configurable, but I'm not sure it is
  5691. * worth it.... maybe it could be a multiple of safemode_delay???
  5692. */
  5693. if (conf->min_offset_diff < 0) {
  5694. safepos += -conf->min_offset_diff;
  5695. readpos += -conf->min_offset_diff;
  5696. } else
  5697. writepos += conf->min_offset_diff;
  5698. if ((mddev->reshape_backwards
  5699. ? (safepos > writepos && readpos < writepos)
  5700. : (safepos < writepos && readpos > writepos)) ||
  5701. time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
  5702. /* Cannot proceed until we've updated the superblock... */
  5703. wait_event(conf->wait_for_overlap,
  5704. atomic_read(&conf->reshape_stripes)==0
  5705. || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5706. if (atomic_read(&conf->reshape_stripes) != 0)
  5707. return 0;
  5708. mddev->reshape_position = conf->reshape_progress;
  5709. mddev->curr_resync_completed = sector_nr;
  5710. if (!mddev->reshape_backwards)
  5711. /* Can update recovery_offset */
  5712. rdev_for_each(rdev, mddev)
  5713. if (rdev->raid_disk >= 0 &&
  5714. !test_bit(Journal, &rdev->flags) &&
  5715. !test_bit(In_sync, &rdev->flags) &&
  5716. rdev->recovery_offset < sector_nr)
  5717. rdev->recovery_offset = sector_nr;
  5718. conf->reshape_checkpoint = jiffies;
  5719. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  5720. md_wakeup_thread(mddev->thread);
  5721. wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
  5722. test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5723. if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
  5724. return 0;
  5725. spin_lock_irq(&conf->device_lock);
  5726. conf->reshape_safe = mddev->reshape_position;
  5727. spin_unlock_irq(&conf->device_lock);
  5728. wake_up(&conf->wait_for_overlap);
  5729. sysfs_notify_dirent_safe(mddev->sysfs_completed);
  5730. }
  5731. INIT_LIST_HEAD(&stripes);
  5732. for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {
  5733. int j;
  5734. int skipped_disk = 0;
  5735. sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i,
  5736. R5_GAS_NOQUIESCE);
  5737. set_bit(STRIPE_EXPANDING, &sh->state);
  5738. atomic_inc(&conf->reshape_stripes);
  5739. /* If any of this stripe is beyond the end of the old
  5740. * array, then we need to zero those blocks
  5741. */
  5742. for (j=sh->disks; j--;) {
  5743. sector_t s;
  5744. if (j == sh->pd_idx)
  5745. continue;
  5746. if (conf->level == 6 &&
  5747. j == sh->qd_idx)
  5748. continue;
  5749. s = raid5_compute_blocknr(sh, j, 0);
  5750. if (s < raid5_size(mddev, 0, 0)) {
  5751. skipped_disk = 1;
  5752. continue;
  5753. }
  5754. memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
  5755. set_bit(R5_Expanded, &sh->dev[j].flags);
  5756. set_bit(R5_UPTODATE, &sh->dev[j].flags);
  5757. }
  5758. if (!skipped_disk) {
  5759. set_bit(STRIPE_EXPAND_READY, &sh->state);
  5760. set_bit(STRIPE_HANDLE, &sh->state);
  5761. }
  5762. list_add(&sh->lru, &stripes);
  5763. }
  5764. spin_lock_irq(&conf->device_lock);
  5765. if (mddev->reshape_backwards)
  5766. conf->reshape_progress -= reshape_sectors * new_data_disks;
  5767. else
  5768. conf->reshape_progress += reshape_sectors * new_data_disks;
  5769. spin_unlock_irq(&conf->device_lock);
  5770. /* Ok, those stripe are ready. We can start scheduling
  5771. * reads on the source stripes.
  5772. * The source stripes are determined by mapping the first and last
  5773. * block on the destination stripes.
  5774. */
  5775. first_sector =
  5776. raid5_compute_sector(conf, stripe_addr*(new_data_disks),
  5777. 1, &dd_idx, NULL);
  5778. last_sector =
  5779. raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
  5780. * new_data_disks - 1),
  5781. 1, &dd_idx, NULL);
  5782. if (last_sector >= mddev->dev_sectors)
  5783. last_sector = mddev->dev_sectors - 1;
  5784. while (first_sector <= last_sector) {
  5785. sh = raid5_get_active_stripe(conf, NULL, first_sector,
  5786. R5_GAS_PREVIOUS | R5_GAS_NOQUIESCE);
  5787. set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  5788. set_bit(STRIPE_HANDLE, &sh->state);
  5789. raid5_release_stripe(sh);
  5790. first_sector += RAID5_STRIPE_SECTORS(conf);
  5791. }
  5792. /* Now that the sources are clearly marked, we can release
  5793. * the destination stripes
  5794. */
  5795. while (!list_empty(&stripes)) {
  5796. sh = list_entry(stripes.next, struct stripe_head, lru);
  5797. list_del_init(&sh->lru);
  5798. raid5_release_stripe(sh);
  5799. }
  5800. /* If this takes us to the resync_max point where we have to pause,
  5801. * then we need to write out the superblock.
  5802. */
  5803. sector_nr += reshape_sectors;
  5804. retn = reshape_sectors;
  5805. finish:
  5806. if (mddev->curr_resync_completed > mddev->resync_max ||
  5807. (sector_nr - mddev->curr_resync_completed) * 2
  5808. >= mddev->resync_max - mddev->curr_resync_completed) {
  5809. /* Cannot proceed until we've updated the superblock... */
  5810. wait_event(conf->wait_for_overlap,
  5811. atomic_read(&conf->reshape_stripes) == 0
  5812. || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5813. if (atomic_read(&conf->reshape_stripes) != 0)
  5814. goto ret;
  5815. mddev->reshape_position = conf->reshape_progress;
  5816. mddev->curr_resync_completed = sector_nr;
  5817. if (!mddev->reshape_backwards)
  5818. /* Can update recovery_offset */
  5819. rdev_for_each(rdev, mddev)
  5820. if (rdev->raid_disk >= 0 &&
  5821. !test_bit(Journal, &rdev->flags) &&
  5822. !test_bit(In_sync, &rdev->flags) &&
  5823. rdev->recovery_offset < sector_nr)
  5824. rdev->recovery_offset = sector_nr;
  5825. conf->reshape_checkpoint = jiffies;
  5826. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  5827. md_wakeup_thread(mddev->thread);
  5828. wait_event(mddev->sb_wait,
  5829. !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)
  5830. || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5831. if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
  5832. goto ret;
  5833. spin_lock_irq(&conf->device_lock);
  5834. conf->reshape_safe = mddev->reshape_position;
  5835. spin_unlock_irq(&conf->device_lock);
  5836. wake_up(&conf->wait_for_overlap);
  5837. sysfs_notify_dirent_safe(mddev->sysfs_completed);
  5838. }
  5839. ret:
  5840. return retn;
  5841. }
  5842. static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_nr,
  5843. int *skipped)
  5844. {
  5845. struct r5conf *conf = mddev->private;
  5846. struct stripe_head *sh;
  5847. sector_t max_sector = mddev->dev_sectors;
  5848. sector_t sync_blocks;
  5849. int still_degraded = 0;
  5850. int i;
  5851. if (sector_nr >= max_sector) {
  5852. /* just being told to finish up .. nothing much to do */
  5853. if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
  5854. end_reshape(conf);
  5855. return 0;
  5856. }
  5857. if (mddev->curr_resync < max_sector) /* aborted */
  5858. md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
  5859. &sync_blocks, 1);
  5860. else /* completed sync */
  5861. conf->fullsync = 0;
  5862. md_bitmap_close_sync(mddev->bitmap);
  5863. return 0;
  5864. }
  5865. /* Allow raid5_quiesce to complete */
  5866. wait_event(conf->wait_for_overlap, conf->quiesce != 2);
  5867. if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
  5868. return reshape_request(mddev, sector_nr, skipped);
  5869. /* No need to check resync_max as we never do more than one
  5870. * stripe, and as resync_max will always be on a chunk boundary,
  5871. * if the check in md_do_sync didn't fire, there is no chance
  5872. * of overstepping resync_max here
  5873. */
  5874. /* if there is too many failed drives and we are trying
  5875. * to resync, then assert that we are finished, because there is
  5876. * nothing we can do.
  5877. */
  5878. if (mddev->degraded >= conf->max_degraded &&
  5879. test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
  5880. sector_t rv = mddev->dev_sectors - sector_nr;
  5881. *skipped = 1;
  5882. return rv;
  5883. }
  5884. if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
  5885. !conf->fullsync &&
  5886. !md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
  5887. sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {
  5888. /* we can skip this block, and probably more */
  5889. do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));
  5890. *skipped = 1;
  5891. /* keep things rounded to whole stripes */
  5892. return sync_blocks * RAID5_STRIPE_SECTORS(conf);
  5893. }
  5894. md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
  5895. sh = raid5_get_active_stripe(conf, NULL, sector_nr,
  5896. R5_GAS_NOBLOCK);
  5897. if (sh == NULL) {
  5898. sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0);
  5899. /* make sure we don't swamp the stripe cache if someone else
  5900. * is trying to get access
  5901. */
  5902. schedule_timeout_uninterruptible(1);
  5903. }
  5904. /* Need to check if array will still be degraded after recovery/resync
  5905. * Note in case of > 1 drive failures it's possible we're rebuilding
  5906. * one drive while leaving another faulty drive in array.
  5907. */
  5908. rcu_read_lock();
  5909. for (i = 0; i < conf->raid_disks; i++) {
  5910. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  5911. if (rdev == NULL || test_bit(Faulty, &rdev->flags))
  5912. still_degraded = 1;
  5913. }
  5914. rcu_read_unlock();
  5915. md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
  5916. set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
  5917. set_bit(STRIPE_HANDLE, &sh->state);
  5918. raid5_release_stripe(sh);
  5919. return RAID5_STRIPE_SECTORS(conf);
  5920. }
  5921. static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio,
  5922. unsigned int offset)
  5923. {
  5924. /* We may not be able to submit a whole bio at once as there
  5925. * may not be enough stripe_heads available.
  5926. * We cannot pre-allocate enough stripe_heads as we may need
  5927. * more than exist in the cache (if we allow ever large chunks).
  5928. * So we do one stripe head at a time and record in
  5929. * ->bi_hw_segments how many have been done.
  5930. *
  5931. * We *know* that this entire raid_bio is in one chunk, so
  5932. * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
  5933. */
  5934. struct stripe_head *sh;
  5935. int dd_idx;
  5936. sector_t sector, logical_sector, last_sector;
  5937. int scnt = 0;
  5938. int handled = 0;
  5939. logical_sector = raid_bio->bi_iter.bi_sector &
  5940. ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5941. sector = raid5_compute_sector(conf, logical_sector,
  5942. 0, &dd_idx, NULL);
  5943. last_sector = bio_end_sector(raid_bio);
  5944. for (; logical_sector < last_sector;
  5945. logical_sector += RAID5_STRIPE_SECTORS(conf),
  5946. sector += RAID5_STRIPE_SECTORS(conf),
  5947. scnt++) {
  5948. if (scnt < offset)
  5949. /* already done this stripe */
  5950. continue;
  5951. sh = raid5_get_active_stripe(conf, NULL, sector,
  5952. R5_GAS_NOBLOCK | R5_GAS_NOQUIESCE);
  5953. if (!sh) {
  5954. /* failed to get a stripe - must wait */
  5955. conf->retry_read_aligned = raid_bio;
  5956. conf->retry_read_offset = scnt;
  5957. return handled;
  5958. }
  5959. if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
  5960. raid5_release_stripe(sh);
  5961. conf->retry_read_aligned = raid_bio;
  5962. conf->retry_read_offset = scnt;
  5963. return handled;
  5964. }
  5965. set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
  5966. handle_stripe(sh);
  5967. raid5_release_stripe(sh);
  5968. handled++;
  5969. }
  5970. bio_endio(raid_bio);
  5971. if (atomic_dec_and_test(&conf->active_aligned_reads))
  5972. wake_up(&conf->wait_for_quiescent);
  5973. return handled;
  5974. }
  5975. static int handle_active_stripes(struct r5conf *conf, int group,
  5976. struct r5worker *worker,
  5977. struct list_head *temp_inactive_list)
  5978. __must_hold(&conf->device_lock)
  5979. {
  5980. struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
  5981. int i, batch_size = 0, hash;
  5982. bool release_inactive = false;
  5983. while (batch_size < MAX_STRIPE_BATCH &&
  5984. (sh = __get_priority_stripe(conf, group)) != NULL)
  5985. batch[batch_size++] = sh;
  5986. if (batch_size == 0) {
  5987. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  5988. if (!list_empty(temp_inactive_list + i))
  5989. break;
  5990. if (i == NR_STRIPE_HASH_LOCKS) {
  5991. spin_unlock_irq(&conf->device_lock);
  5992. log_flush_stripe_to_raid(conf);
  5993. spin_lock_irq(&conf->device_lock);
  5994. return batch_size;
  5995. }
  5996. release_inactive = true;
  5997. }
  5998. spin_unlock_irq(&conf->device_lock);
  5999. release_inactive_stripe_list(conf, temp_inactive_list,
  6000. NR_STRIPE_HASH_LOCKS);
  6001. r5l_flush_stripe_to_raid(conf->log);
  6002. if (release_inactive) {
  6003. spin_lock_irq(&conf->device_lock);
  6004. return 0;
  6005. }
  6006. for (i = 0; i < batch_size; i++)
  6007. handle_stripe(batch[i]);
  6008. log_write_stripe_run(conf);
  6009. cond_resched();
  6010. spin_lock_irq(&conf->device_lock);
  6011. for (i = 0; i < batch_size; i++) {
  6012. hash = batch[i]->hash_lock_index;
  6013. __release_stripe(conf, batch[i], &temp_inactive_list[hash]);
  6014. }
  6015. return batch_size;
  6016. }
  6017. static void raid5_do_work(struct work_struct *work)
  6018. {
  6019. struct r5worker *worker = container_of(work, struct r5worker, work);
  6020. struct r5worker_group *group = worker->group;
  6021. struct r5conf *conf = group->conf;
  6022. struct mddev *mddev = conf->mddev;
  6023. int group_id = group - conf->worker_groups;
  6024. int handled;
  6025. struct blk_plug plug;
  6026. pr_debug("+++ raid5worker active\n");
  6027. blk_start_plug(&plug);
  6028. handled = 0;
  6029. spin_lock_irq(&conf->device_lock);
  6030. while (1) {
  6031. int batch_size, released;
  6032. released = release_stripe_list(conf, worker->temp_inactive_list);
  6033. batch_size = handle_active_stripes(conf, group_id, worker,
  6034. worker->temp_inactive_list);
  6035. worker->working = false;
  6036. if (!batch_size && !released)
  6037. break;
  6038. handled += batch_size;
  6039. wait_event_lock_irq(mddev->sb_wait,
  6040. !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
  6041. conf->device_lock);
  6042. }
  6043. pr_debug("%d stripes handled\n", handled);
  6044. spin_unlock_irq(&conf->device_lock);
  6045. flush_deferred_bios(conf);
  6046. r5l_flush_stripe_to_raid(conf->log);
  6047. async_tx_issue_pending_all();
  6048. blk_finish_plug(&plug);
  6049. pr_debug("--- raid5worker inactive\n");
  6050. }
  6051. /*
  6052. * This is our raid5 kernel thread.
  6053. *
  6054. * We scan the hash table for stripes which can be handled now.
  6055. * During the scan, completed stripes are saved for us by the interrupt
  6056. * handler, so that they will not have to wait for our next wakeup.
  6057. */
  6058. static void raid5d(struct md_thread *thread)
  6059. {
  6060. struct mddev *mddev = thread->mddev;
  6061. struct r5conf *conf = mddev->private;
  6062. int handled;
  6063. struct blk_plug plug;
  6064. pr_debug("+++ raid5d active\n");
  6065. md_check_recovery(mddev);
  6066. blk_start_plug(&plug);
  6067. handled = 0;
  6068. spin_lock_irq(&conf->device_lock);
  6069. while (1) {
  6070. struct bio *bio;
  6071. int batch_size, released;
  6072. unsigned int offset;
  6073. released = release_stripe_list(conf, conf->temp_inactive_list);
  6074. if (released)
  6075. clear_bit(R5_DID_ALLOC, &conf->cache_state);
  6076. if (
  6077. !list_empty(&conf->bitmap_list)) {
  6078. /* Now is a good time to flush some bitmap updates */
  6079. conf->seq_flush++;
  6080. spin_unlock_irq(&conf->device_lock);
  6081. md_bitmap_unplug(mddev->bitmap);
  6082. spin_lock_irq(&conf->device_lock);
  6083. conf->seq_write = conf->seq_flush;
  6084. activate_bit_delay(conf, conf->temp_inactive_list);
  6085. }
  6086. raid5_activate_delayed(conf);
  6087. while ((bio = remove_bio_from_retry(conf, &offset))) {
  6088. int ok;
  6089. spin_unlock_irq(&conf->device_lock);
  6090. ok = retry_aligned_read(conf, bio, offset);
  6091. spin_lock_irq(&conf->device_lock);
  6092. if (!ok)
  6093. break;
  6094. handled++;
  6095. }
  6096. batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
  6097. conf->temp_inactive_list);
  6098. if (!batch_size && !released)
  6099. break;
  6100. handled += batch_size;
  6101. if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) {
  6102. spin_unlock_irq(&conf->device_lock);
  6103. md_check_recovery(mddev);
  6104. spin_lock_irq(&conf->device_lock);
  6105. /*
  6106. * Waiting on MD_SB_CHANGE_PENDING below may deadlock
  6107. * seeing md_check_recovery() is needed to clear
  6108. * the flag when using mdmon.
  6109. */
  6110. continue;
  6111. }
  6112. wait_event_lock_irq(mddev->sb_wait,
  6113. !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
  6114. conf->device_lock);
  6115. }
  6116. pr_debug("%d stripes handled\n", handled);
  6117. spin_unlock_irq(&conf->device_lock);
  6118. if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&
  6119. mutex_trylock(&conf->cache_size_mutex)) {
  6120. grow_one_stripe(conf, __GFP_NOWARN);
  6121. /* Set flag even if allocation failed. This helps
  6122. * slow down allocation requests when mem is short
  6123. */
  6124. set_bit(R5_DID_ALLOC, &conf->cache_state);
  6125. mutex_unlock(&conf->cache_size_mutex);
  6126. }
  6127. flush_deferred_bios(conf);
  6128. r5l_flush_stripe_to_raid(conf->log);
  6129. async_tx_issue_pending_all();
  6130. blk_finish_plug(&plug);
  6131. pr_debug("--- raid5d inactive\n");
  6132. }
  6133. static ssize_t
  6134. raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
  6135. {
  6136. struct r5conf *conf;
  6137. int ret = 0;
  6138. spin_lock(&mddev->lock);
  6139. conf = mddev->private;
  6140. if (conf)
  6141. ret = sprintf(page, "%d\n", conf->min_nr_stripes);
  6142. spin_unlock(&mddev->lock);
  6143. return ret;
  6144. }
  6145. int
  6146. raid5_set_cache_size(struct mddev *mddev, int size)
  6147. {
  6148. int result = 0;
  6149. struct r5conf *conf = mddev->private;
  6150. if (size <= 16 || size > 32768)
  6151. return -EINVAL;
  6152. conf->min_nr_stripes = size;
  6153. mutex_lock(&conf->cache_size_mutex);
  6154. while (size < conf->max_nr_stripes &&
  6155. drop_one_stripe(conf))
  6156. ;
  6157. mutex_unlock(&conf->cache_size_mutex);
  6158. md_allow_write(mddev);
  6159. mutex_lock(&conf->cache_size_mutex);
  6160. while (size > conf->max_nr_stripes)
  6161. if (!grow_one_stripe(conf, GFP_KERNEL)) {
  6162. conf->min_nr_stripes = conf->max_nr_stripes;
  6163. result = -ENOMEM;
  6164. break;
  6165. }
  6166. mutex_unlock(&conf->cache_size_mutex);
  6167. return result;
  6168. }
  6169. EXPORT_SYMBOL(raid5_set_cache_size);
  6170. static ssize_t
  6171. raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
  6172. {
  6173. struct r5conf *conf;
  6174. unsigned long new;
  6175. int err;
  6176. if (len >= PAGE_SIZE)
  6177. return -EINVAL;
  6178. if (kstrtoul(page, 10, &new))
  6179. return -EINVAL;
  6180. err = mddev_lock(mddev);
  6181. if (err)
  6182. return err;
  6183. conf = mddev->private;
  6184. if (!conf)
  6185. err = -ENODEV;
  6186. else
  6187. err = raid5_set_cache_size(mddev, new);
  6188. mddev_unlock(mddev);
  6189. return err ?: len;
  6190. }
  6191. static struct md_sysfs_entry
  6192. raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
  6193. raid5_show_stripe_cache_size,
  6194. raid5_store_stripe_cache_size);
  6195. static ssize_t
  6196. raid5_show_rmw_level(struct mddev *mddev, char *page)
  6197. {
  6198. struct r5conf *conf = mddev->private;
  6199. if (conf)
  6200. return sprintf(page, "%d\n", conf->rmw_level);
  6201. else
  6202. return 0;
  6203. }
  6204. static ssize_t
  6205. raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)
  6206. {
  6207. struct r5conf *conf = mddev->private;
  6208. unsigned long new;
  6209. if (!conf)
  6210. return -ENODEV;
  6211. if (len >= PAGE_SIZE)
  6212. return -EINVAL;
  6213. if (kstrtoul(page, 10, &new))
  6214. return -EINVAL;
  6215. if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
  6216. return -EINVAL;
  6217. if (new != PARITY_DISABLE_RMW &&
  6218. new != PARITY_ENABLE_RMW &&
  6219. new != PARITY_PREFER_RMW)
  6220. return -EINVAL;
  6221. conf->rmw_level = new;
  6222. return len;
  6223. }
  6224. static struct md_sysfs_entry
  6225. raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
  6226. raid5_show_rmw_level,
  6227. raid5_store_rmw_level);
  6228. static ssize_t
  6229. raid5_show_stripe_size(struct mddev *mddev, char *page)
  6230. {
  6231. struct r5conf *conf;
  6232. int ret = 0;
  6233. spin_lock(&mddev->lock);
  6234. conf = mddev->private;
  6235. if (conf)
  6236. ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));
  6237. spin_unlock(&mddev->lock);
  6238. return ret;
  6239. }
  6240. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  6241. static ssize_t
  6242. raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
  6243. {
  6244. struct r5conf *conf;
  6245. unsigned long new;
  6246. int err;
  6247. int size;
  6248. if (len >= PAGE_SIZE)
  6249. return -EINVAL;
  6250. if (kstrtoul(page, 10, &new))
  6251. return -EINVAL;
  6252. /*
  6253. * The value should not be bigger than PAGE_SIZE. It requires to
  6254. * be multiple of DEFAULT_STRIPE_SIZE and the value should be power
  6255. * of two.
  6256. */
  6257. if (new % DEFAULT_STRIPE_SIZE != 0 ||
  6258. new > PAGE_SIZE || new == 0 ||
  6259. new != roundup_pow_of_two(new))
  6260. return -EINVAL;
  6261. err = mddev_lock(mddev);
  6262. if (err)
  6263. return err;
  6264. conf = mddev->private;
  6265. if (!conf) {
  6266. err = -ENODEV;
  6267. goto out_unlock;
  6268. }
  6269. if (new == conf->stripe_size)
  6270. goto out_unlock;
  6271. pr_debug("md/raid: change stripe_size from %lu to %lu\n",
  6272. conf->stripe_size, new);
  6273. if (mddev->sync_thread ||
  6274. test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||
  6275. mddev->reshape_position != MaxSector ||
  6276. mddev->sysfs_active) {
  6277. err = -EBUSY;
  6278. goto out_unlock;
  6279. }
  6280. mddev_suspend(mddev);
  6281. mutex_lock(&conf->cache_size_mutex);
  6282. size = conf->max_nr_stripes;
  6283. shrink_stripes(conf);
  6284. conf->stripe_size = new;
  6285. conf->stripe_shift = ilog2(new) - 9;
  6286. conf->stripe_sectors = new >> 9;
  6287. if (grow_stripes(conf, size)) {
  6288. pr_warn("md/raid:%s: couldn't allocate buffers\n",
  6289. mdname(mddev));
  6290. err = -ENOMEM;
  6291. }
  6292. mutex_unlock(&conf->cache_size_mutex);
  6293. mddev_resume(mddev);
  6294. out_unlock:
  6295. mddev_unlock(mddev);
  6296. return err ?: len;
  6297. }
  6298. static struct md_sysfs_entry
  6299. raid5_stripe_size = __ATTR(stripe_size, 0644,
  6300. raid5_show_stripe_size,
  6301. raid5_store_stripe_size);
  6302. #else
  6303. static struct md_sysfs_entry
  6304. raid5_stripe_size = __ATTR(stripe_size, 0444,
  6305. raid5_show_stripe_size,
  6306. NULL);
  6307. #endif
  6308. static ssize_t
  6309. raid5_show_preread_threshold(struct mddev *mddev, char *page)
  6310. {
  6311. struct r5conf *conf;
  6312. int ret = 0;
  6313. spin_lock(&mddev->lock);
  6314. conf = mddev->private;
  6315. if (conf)
  6316. ret = sprintf(page, "%d\n", conf->bypass_threshold);
  6317. spin_unlock(&mddev->lock);
  6318. return ret;
  6319. }
  6320. static ssize_t
  6321. raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
  6322. {
  6323. struct r5conf *conf;
  6324. unsigned long new;
  6325. int err;
  6326. if (len >= PAGE_SIZE)
  6327. return -EINVAL;
  6328. if (kstrtoul(page, 10, &new))
  6329. return -EINVAL;
  6330. err = mddev_lock(mddev);
  6331. if (err)
  6332. return err;
  6333. conf = mddev->private;
  6334. if (!conf)
  6335. err = -ENODEV;
  6336. else if (new > conf->min_nr_stripes)
  6337. err = -EINVAL;
  6338. else
  6339. conf->bypass_threshold = new;
  6340. mddev_unlock(mddev);
  6341. return err ?: len;
  6342. }
  6343. static struct md_sysfs_entry
  6344. raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
  6345. S_IRUGO | S_IWUSR,
  6346. raid5_show_preread_threshold,
  6347. raid5_store_preread_threshold);
  6348. static ssize_t
  6349. raid5_show_skip_copy(struct mddev *mddev, char *page)
  6350. {
  6351. struct r5conf *conf;
  6352. int ret = 0;
  6353. spin_lock(&mddev->lock);
  6354. conf = mddev->private;
  6355. if (conf)
  6356. ret = sprintf(page, "%d\n", conf->skip_copy);
  6357. spin_unlock(&mddev->lock);
  6358. return ret;
  6359. }
  6360. static ssize_t
  6361. raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
  6362. {
  6363. struct r5conf *conf;
  6364. unsigned long new;
  6365. int err;
  6366. if (len >= PAGE_SIZE)
  6367. return -EINVAL;
  6368. if (kstrtoul(page, 10, &new))
  6369. return -EINVAL;
  6370. new = !!new;
  6371. err = mddev_lock(mddev);
  6372. if (err)
  6373. return err;
  6374. conf = mddev->private;
  6375. if (!conf)
  6376. err = -ENODEV;
  6377. else if (new != conf->skip_copy) {
  6378. struct request_queue *q = mddev->queue;
  6379. mddev_suspend(mddev);
  6380. conf->skip_copy = new;
  6381. if (new)
  6382. blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
  6383. else
  6384. blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
  6385. mddev_resume(mddev);
  6386. }
  6387. mddev_unlock(mddev);
  6388. return err ?: len;
  6389. }
  6390. static struct md_sysfs_entry
  6391. raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
  6392. raid5_show_skip_copy,
  6393. raid5_store_skip_copy);
  6394. static ssize_t
  6395. stripe_cache_active_show(struct mddev *mddev, char *page)
  6396. {
  6397. struct r5conf *conf = mddev->private;
  6398. if (conf)
  6399. return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
  6400. else
  6401. return 0;
  6402. }
  6403. static struct md_sysfs_entry
  6404. raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
  6405. static ssize_t
  6406. raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
  6407. {
  6408. struct r5conf *conf;
  6409. int ret = 0;
  6410. spin_lock(&mddev->lock);
  6411. conf = mddev->private;
  6412. if (conf)
  6413. ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
  6414. spin_unlock(&mddev->lock);
  6415. return ret;
  6416. }
  6417. static int alloc_thread_groups(struct r5conf *conf, int cnt,
  6418. int *group_cnt,
  6419. struct r5worker_group **worker_groups);
  6420. static ssize_t
  6421. raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
  6422. {
  6423. struct r5conf *conf;
  6424. unsigned int new;
  6425. int err;
  6426. struct r5worker_group *new_groups, *old_groups;
  6427. int group_cnt;
  6428. if (len >= PAGE_SIZE)
  6429. return -EINVAL;
  6430. if (kstrtouint(page, 10, &new))
  6431. return -EINVAL;
  6432. /* 8192 should be big enough */
  6433. if (new > 8192)
  6434. return -EINVAL;
  6435. err = mddev_lock(mddev);
  6436. if (err)
  6437. return err;
  6438. conf = mddev->private;
  6439. if (!conf)
  6440. err = -ENODEV;
  6441. else if (new != conf->worker_cnt_per_group) {
  6442. mddev_suspend(mddev);
  6443. old_groups = conf->worker_groups;
  6444. if (old_groups)
  6445. flush_workqueue(raid5_wq);
  6446. err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
  6447. if (!err) {
  6448. spin_lock_irq(&conf->device_lock);
  6449. conf->group_cnt = group_cnt;
  6450. conf->worker_cnt_per_group = new;
  6451. conf->worker_groups = new_groups;
  6452. spin_unlock_irq(&conf->device_lock);
  6453. if (old_groups)
  6454. kfree(old_groups[0].workers);
  6455. kfree(old_groups);
  6456. }
  6457. mddev_resume(mddev);
  6458. }
  6459. mddev_unlock(mddev);
  6460. return err ?: len;
  6461. }
  6462. static struct md_sysfs_entry
  6463. raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
  6464. raid5_show_group_thread_cnt,
  6465. raid5_store_group_thread_cnt);
  6466. static struct attribute *raid5_attrs[] = {
  6467. &raid5_stripecache_size.attr,
  6468. &raid5_stripecache_active.attr,
  6469. &raid5_preread_bypass_threshold.attr,
  6470. &raid5_group_thread_cnt.attr,
  6471. &raid5_skip_copy.attr,
  6472. &raid5_rmw_level.attr,
  6473. &raid5_stripe_size.attr,
  6474. &r5c_journal_mode.attr,
  6475. &ppl_write_hint.attr,
  6476. NULL,
  6477. };
  6478. static const struct attribute_group raid5_attrs_group = {
  6479. .name = NULL,
  6480. .attrs = raid5_attrs,
  6481. };
  6482. static int alloc_thread_groups(struct r5conf *conf, int cnt, int *group_cnt,
  6483. struct r5worker_group **worker_groups)
  6484. {
  6485. int i, j, k;
  6486. ssize_t size;
  6487. struct r5worker *workers;
  6488. if (cnt == 0) {
  6489. *group_cnt = 0;
  6490. *worker_groups = NULL;
  6491. return 0;
  6492. }
  6493. *group_cnt = num_possible_nodes();
  6494. size = sizeof(struct r5worker) * cnt;
  6495. workers = kcalloc(size, *group_cnt, GFP_NOIO);
  6496. *worker_groups = kcalloc(*group_cnt, sizeof(struct r5worker_group),
  6497. GFP_NOIO);
  6498. if (!*worker_groups || !workers) {
  6499. kfree(workers);
  6500. kfree(*worker_groups);
  6501. return -ENOMEM;
  6502. }
  6503. for (i = 0; i < *group_cnt; i++) {
  6504. struct r5worker_group *group;
  6505. group = &(*worker_groups)[i];
  6506. INIT_LIST_HEAD(&group->handle_list);
  6507. INIT_LIST_HEAD(&group->loprio_list);
  6508. group->conf = conf;
  6509. group->workers = workers + i * cnt;
  6510. for (j = 0; j < cnt; j++) {
  6511. struct r5worker *worker = group->workers + j;
  6512. worker->group = group;
  6513. INIT_WORK(&worker->work, raid5_do_work);
  6514. for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
  6515. INIT_LIST_HEAD(worker->temp_inactive_list + k);
  6516. }
  6517. }
  6518. return 0;
  6519. }
  6520. static void free_thread_groups(struct r5conf *conf)
  6521. {
  6522. if (conf->worker_groups)
  6523. kfree(conf->worker_groups[0].workers);
  6524. kfree(conf->worker_groups);
  6525. conf->worker_groups = NULL;
  6526. }
  6527. static sector_t
  6528. raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
  6529. {
  6530. struct r5conf *conf = mddev->private;
  6531. if (!sectors)
  6532. sectors = mddev->dev_sectors;
  6533. if (!raid_disks)
  6534. /* size is defined by the smallest of previous and new size */
  6535. raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
  6536. sectors &= ~((sector_t)conf->chunk_sectors - 1);
  6537. sectors &= ~((sector_t)conf->prev_chunk_sectors - 1);
  6538. return sectors * (raid_disks - conf->max_degraded);
  6539. }
  6540. static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
  6541. {
  6542. safe_put_page(percpu->spare_page);
  6543. percpu->spare_page = NULL;
  6544. kvfree(percpu->scribble);
  6545. percpu->scribble = NULL;
  6546. }
  6547. static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
  6548. {
  6549. if (conf->level == 6 && !percpu->spare_page) {
  6550. percpu->spare_page = alloc_page(GFP_KERNEL);
  6551. if (!percpu->spare_page)
  6552. return -ENOMEM;
  6553. }
  6554. if (scribble_alloc(percpu,
  6555. max(conf->raid_disks,
  6556. conf->previous_raid_disks),
  6557. max(conf->chunk_sectors,
  6558. conf->prev_chunk_sectors)
  6559. / RAID5_STRIPE_SECTORS(conf))) {
  6560. free_scratch_buffer(conf, percpu);
  6561. return -ENOMEM;
  6562. }
  6563. local_lock_init(&percpu->lock);
  6564. return 0;
  6565. }
  6566. static int raid456_cpu_dead(unsigned int cpu, struct hlist_node *node)
  6567. {
  6568. struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
  6569. free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
  6570. return 0;
  6571. }
  6572. static void raid5_free_percpu(struct r5conf *conf)
  6573. {
  6574. if (!conf->percpu)
  6575. return;
  6576. cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
  6577. free_percpu(conf->percpu);
  6578. }
  6579. static void free_conf(struct r5conf *conf)
  6580. {
  6581. int i;
  6582. log_exit(conf);
  6583. unregister_shrinker(&conf->shrinker);
  6584. free_thread_groups(conf);
  6585. shrink_stripes(conf);
  6586. raid5_free_percpu(conf);
  6587. for (i = 0; i < conf->pool_size; i++)
  6588. if (conf->disks[i].extra_page)
  6589. put_page(conf->disks[i].extra_page);
  6590. kfree(conf->disks);
  6591. bioset_exit(&conf->bio_split);
  6592. kfree(conf->stripe_hashtbl);
  6593. kfree(conf->pending_data);
  6594. kfree(conf);
  6595. }
  6596. static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
  6597. {
  6598. struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
  6599. struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
  6600. if (alloc_scratch_buffer(conf, percpu)) {
  6601. pr_warn("%s: failed memory allocation for cpu%u\n",
  6602. __func__, cpu);
  6603. return -ENOMEM;
  6604. }
  6605. return 0;
  6606. }
  6607. static int raid5_alloc_percpu(struct r5conf *conf)
  6608. {
  6609. int err = 0;
  6610. conf->percpu = alloc_percpu(struct raid5_percpu);
  6611. if (!conf->percpu)
  6612. return -ENOMEM;
  6613. err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
  6614. if (!err) {
  6615. conf->scribble_disks = max(conf->raid_disks,
  6616. conf->previous_raid_disks);
  6617. conf->scribble_sectors = max(conf->chunk_sectors,
  6618. conf->prev_chunk_sectors);
  6619. }
  6620. return err;
  6621. }
  6622. static unsigned long raid5_cache_scan(struct shrinker *shrink,
  6623. struct shrink_control *sc)
  6624. {
  6625. struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
  6626. unsigned long ret = SHRINK_STOP;
  6627. if (mutex_trylock(&conf->cache_size_mutex)) {
  6628. ret= 0;
  6629. while (ret < sc->nr_to_scan &&
  6630. conf->max_nr_stripes > conf->min_nr_stripes) {
  6631. if (drop_one_stripe(conf) == 0) {
  6632. ret = SHRINK_STOP;
  6633. break;
  6634. }
  6635. ret++;
  6636. }
  6637. mutex_unlock(&conf->cache_size_mutex);
  6638. }
  6639. return ret;
  6640. }
  6641. static unsigned long raid5_cache_count(struct shrinker *shrink,
  6642. struct shrink_control *sc)
  6643. {
  6644. struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
  6645. if (conf->max_nr_stripes < conf->min_nr_stripes)
  6646. /* unlikely, but not impossible */
  6647. return 0;
  6648. return conf->max_nr_stripes - conf->min_nr_stripes;
  6649. }
  6650. static struct r5conf *setup_conf(struct mddev *mddev)
  6651. {
  6652. struct r5conf *conf;
  6653. int raid_disk, memory, max_disks;
  6654. struct md_rdev *rdev;
  6655. struct disk_info *disk;
  6656. char pers_name[6];
  6657. int i;
  6658. int group_cnt;
  6659. struct r5worker_group *new_group;
  6660. int ret = -ENOMEM;
  6661. if (mddev->new_level != 5
  6662. && mddev->new_level != 4
  6663. && mddev->new_level != 6) {
  6664. pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n",
  6665. mdname(mddev), mddev->new_level);
  6666. return ERR_PTR(-EIO);
  6667. }
  6668. if ((mddev->new_level == 5
  6669. && !algorithm_valid_raid5(mddev->new_layout)) ||
  6670. (mddev->new_level == 6
  6671. && !algorithm_valid_raid6(mddev->new_layout))) {
  6672. pr_warn("md/raid:%s: layout %d not supported\n",
  6673. mdname(mddev), mddev->new_layout);
  6674. return ERR_PTR(-EIO);
  6675. }
  6676. if (mddev->new_level == 6 && mddev->raid_disks < 4) {
  6677. pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",
  6678. mdname(mddev), mddev->raid_disks);
  6679. return ERR_PTR(-EINVAL);
  6680. }
  6681. if (!mddev->new_chunk_sectors ||
  6682. (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
  6683. !is_power_of_2(mddev->new_chunk_sectors)) {
  6684. pr_warn("md/raid:%s: invalid chunk size %d\n",
  6685. mdname(mddev), mddev->new_chunk_sectors << 9);
  6686. return ERR_PTR(-EINVAL);
  6687. }
  6688. conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
  6689. if (conf == NULL)
  6690. goto abort;
  6691. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  6692. conf->stripe_size = DEFAULT_STRIPE_SIZE;
  6693. conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
  6694. conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;
  6695. #endif
  6696. INIT_LIST_HEAD(&conf->free_list);
  6697. INIT_LIST_HEAD(&conf->pending_list);
  6698. conf->pending_data = kcalloc(PENDING_IO_MAX,
  6699. sizeof(struct r5pending_data),
  6700. GFP_KERNEL);
  6701. if (!conf->pending_data)
  6702. goto abort;
  6703. for (i = 0; i < PENDING_IO_MAX; i++)
  6704. list_add(&conf->pending_data[i].sibling, &conf->free_list);
  6705. /* Don't enable multi-threading by default*/
  6706. if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
  6707. conf->group_cnt = group_cnt;
  6708. conf->worker_cnt_per_group = 0;
  6709. conf->worker_groups = new_group;
  6710. } else
  6711. goto abort;
  6712. spin_lock_init(&conf->device_lock);
  6713. seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);
  6714. mutex_init(&conf->cache_size_mutex);
  6715. init_waitqueue_head(&conf->wait_for_quiescent);
  6716. init_waitqueue_head(&conf->wait_for_stripe);
  6717. init_waitqueue_head(&conf->wait_for_overlap);
  6718. INIT_LIST_HEAD(&conf->handle_list);
  6719. INIT_LIST_HEAD(&conf->loprio_list);
  6720. INIT_LIST_HEAD(&conf->hold_list);
  6721. INIT_LIST_HEAD(&conf->delayed_list);
  6722. INIT_LIST_HEAD(&conf->bitmap_list);
  6723. init_llist_head(&conf->released_stripes);
  6724. atomic_set(&conf->active_stripes, 0);
  6725. atomic_set(&conf->preread_active_stripes, 0);
  6726. atomic_set(&conf->active_aligned_reads, 0);
  6727. spin_lock_init(&conf->pending_bios_lock);
  6728. conf->batch_bio_dispatch = true;
  6729. rdev_for_each(rdev, mddev) {
  6730. if (test_bit(Journal, &rdev->flags))
  6731. continue;
  6732. if (bdev_nonrot(rdev->bdev)) {
  6733. conf->batch_bio_dispatch = false;
  6734. break;
  6735. }
  6736. }
  6737. conf->bypass_threshold = BYPASS_THRESHOLD;
  6738. conf->recovery_disabled = mddev->recovery_disabled - 1;
  6739. conf->raid_disks = mddev->raid_disks;
  6740. if (mddev->reshape_position == MaxSector)
  6741. conf->previous_raid_disks = mddev->raid_disks;
  6742. else
  6743. conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
  6744. max_disks = max(conf->raid_disks, conf->previous_raid_disks);
  6745. conf->disks = kcalloc(max_disks, sizeof(struct disk_info),
  6746. GFP_KERNEL);
  6747. if (!conf->disks)
  6748. goto abort;
  6749. for (i = 0; i < max_disks; i++) {
  6750. conf->disks[i].extra_page = alloc_page(GFP_KERNEL);
  6751. if (!conf->disks[i].extra_page)
  6752. goto abort;
  6753. }
  6754. ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
  6755. if (ret)
  6756. goto abort;
  6757. conf->mddev = mddev;
  6758. ret = -ENOMEM;
  6759. conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);
  6760. if (!conf->stripe_hashtbl)
  6761. goto abort;
  6762. /* We init hash_locks[0] separately to that it can be used
  6763. * as the reference lock in the spin_lock_nest_lock() call
  6764. * in lock_all_device_hash_locks_irq in order to convince
  6765. * lockdep that we know what we are doing.
  6766. */
  6767. spin_lock_init(conf->hash_locks);
  6768. for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
  6769. spin_lock_init(conf->hash_locks + i);
  6770. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  6771. INIT_LIST_HEAD(conf->inactive_list + i);
  6772. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  6773. INIT_LIST_HEAD(conf->temp_inactive_list + i);
  6774. atomic_set(&conf->r5c_cached_full_stripes, 0);
  6775. INIT_LIST_HEAD(&conf->r5c_full_stripe_list);
  6776. atomic_set(&conf->r5c_cached_partial_stripes, 0);
  6777. INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);
  6778. atomic_set(&conf->r5c_flushing_full_stripes, 0);
  6779. atomic_set(&conf->r5c_flushing_partial_stripes, 0);
  6780. conf->level = mddev->new_level;
  6781. conf->chunk_sectors = mddev->new_chunk_sectors;
  6782. ret = raid5_alloc_percpu(conf);
  6783. if (ret)
  6784. goto abort;
  6785. pr_debug("raid456: run(%s) called.\n", mdname(mddev));
  6786. ret = -EIO;
  6787. rdev_for_each(rdev, mddev) {
  6788. raid_disk = rdev->raid_disk;
  6789. if (raid_disk >= max_disks
  6790. || raid_disk < 0 || test_bit(Journal, &rdev->flags))
  6791. continue;
  6792. disk = conf->disks + raid_disk;
  6793. if (test_bit(Replacement, &rdev->flags)) {
  6794. if (disk->replacement)
  6795. goto abort;
  6796. RCU_INIT_POINTER(disk->replacement, rdev);
  6797. } else {
  6798. if (disk->rdev)
  6799. goto abort;
  6800. RCU_INIT_POINTER(disk->rdev, rdev);
  6801. }
  6802. if (test_bit(In_sync, &rdev->flags)) {
  6803. pr_info("md/raid:%s: device %pg operational as raid disk %d\n",
  6804. mdname(mddev), rdev->bdev, raid_disk);
  6805. } else if (rdev->saved_raid_disk != raid_disk)
  6806. /* Cannot rely on bitmap to complete recovery */
  6807. conf->fullsync = 1;
  6808. }
  6809. conf->level = mddev->new_level;
  6810. if (conf->level == 6) {
  6811. conf->max_degraded = 2;
  6812. if (raid6_call.xor_syndrome)
  6813. conf->rmw_level = PARITY_ENABLE_RMW;
  6814. else
  6815. conf->rmw_level = PARITY_DISABLE_RMW;
  6816. } else {
  6817. conf->max_degraded = 1;
  6818. conf->rmw_level = PARITY_ENABLE_RMW;
  6819. }
  6820. conf->algorithm = mddev->new_layout;
  6821. conf->reshape_progress = mddev->reshape_position;
  6822. if (conf->reshape_progress != MaxSector) {
  6823. conf->prev_chunk_sectors = mddev->chunk_sectors;
  6824. conf->prev_algo = mddev->layout;
  6825. } else {
  6826. conf->prev_chunk_sectors = conf->chunk_sectors;
  6827. conf->prev_algo = conf->algorithm;
  6828. }
  6829. conf->min_nr_stripes = NR_STRIPES;
  6830. if (mddev->reshape_position != MaxSector) {
  6831. int stripes = max_t(int,
  6832. ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,
  6833. ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);
  6834. conf->min_nr_stripes = max(NR_STRIPES, stripes);
  6835. if (conf->min_nr_stripes != NR_STRIPES)
  6836. pr_info("md/raid:%s: force stripe size %d for reshape\n",
  6837. mdname(mddev), conf->min_nr_stripes);
  6838. }
  6839. memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
  6840. max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
  6841. atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
  6842. if (grow_stripes(conf, conf->min_nr_stripes)) {
  6843. pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n",
  6844. mdname(mddev), memory);
  6845. ret = -ENOMEM;
  6846. goto abort;
  6847. } else
  6848. pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory);
  6849. /*
  6850. * Losing a stripe head costs more than the time to refill it,
  6851. * it reduces the queue depth and so can hurt throughput.
  6852. * So set it rather large, scaled by number of devices.
  6853. */
  6854. conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4;
  6855. conf->shrinker.scan_objects = raid5_cache_scan;
  6856. conf->shrinker.count_objects = raid5_cache_count;
  6857. conf->shrinker.batch = 128;
  6858. conf->shrinker.flags = 0;
  6859. ret = register_shrinker(&conf->shrinker, "md-raid5:%s", mdname(mddev));
  6860. if (ret) {
  6861. pr_warn("md/raid:%s: couldn't register shrinker.\n",
  6862. mdname(mddev));
  6863. goto abort;
  6864. }
  6865. sprintf(pers_name, "raid%d", mddev->new_level);
  6866. conf->thread = md_register_thread(raid5d, mddev, pers_name);
  6867. if (!conf->thread) {
  6868. pr_warn("md/raid:%s: couldn't allocate thread.\n",
  6869. mdname(mddev));
  6870. ret = -ENOMEM;
  6871. goto abort;
  6872. }
  6873. return conf;
  6874. abort:
  6875. if (conf)
  6876. free_conf(conf);
  6877. return ERR_PTR(ret);
  6878. }
  6879. static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
  6880. {
  6881. switch (algo) {
  6882. case ALGORITHM_PARITY_0:
  6883. if (raid_disk < max_degraded)
  6884. return 1;
  6885. break;
  6886. case ALGORITHM_PARITY_N:
  6887. if (raid_disk >= raid_disks - max_degraded)
  6888. return 1;
  6889. break;
  6890. case ALGORITHM_PARITY_0_6:
  6891. if (raid_disk == 0 ||
  6892. raid_disk == raid_disks - 1)
  6893. return 1;
  6894. break;
  6895. case ALGORITHM_LEFT_ASYMMETRIC_6:
  6896. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  6897. case ALGORITHM_LEFT_SYMMETRIC_6:
  6898. case ALGORITHM_RIGHT_SYMMETRIC_6:
  6899. if (raid_disk == raid_disks - 1)
  6900. return 1;
  6901. }
  6902. return 0;
  6903. }
  6904. static void raid5_set_io_opt(struct r5conf *conf)
  6905. {
  6906. blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
  6907. (conf->raid_disks - conf->max_degraded));
  6908. }
  6909. static int raid5_run(struct mddev *mddev)
  6910. {
  6911. struct r5conf *conf;
  6912. int working_disks = 0;
  6913. int dirty_parity_disks = 0;
  6914. struct md_rdev *rdev;
  6915. struct md_rdev *journal_dev = NULL;
  6916. sector_t reshape_offset = 0;
  6917. int i, ret = 0;
  6918. long long min_offset_diff = 0;
  6919. int first = 1;
  6920. if (acct_bioset_init(mddev)) {
  6921. pr_err("md/raid456:%s: alloc acct bioset failed.\n", mdname(mddev));
  6922. return -ENOMEM;
  6923. }
  6924. if (mddev_init_writes_pending(mddev) < 0) {
  6925. ret = -ENOMEM;
  6926. goto exit_acct_set;
  6927. }
  6928. if (mddev->recovery_cp != MaxSector)
  6929. pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
  6930. mdname(mddev));
  6931. rdev_for_each(rdev, mddev) {
  6932. long long diff;
  6933. if (test_bit(Journal, &rdev->flags)) {
  6934. journal_dev = rdev;
  6935. continue;
  6936. }
  6937. if (rdev->raid_disk < 0)
  6938. continue;
  6939. diff = (rdev->new_data_offset - rdev->data_offset);
  6940. if (first) {
  6941. min_offset_diff = diff;
  6942. first = 0;
  6943. } else if (mddev->reshape_backwards &&
  6944. diff < min_offset_diff)
  6945. min_offset_diff = diff;
  6946. else if (!mddev->reshape_backwards &&
  6947. diff > min_offset_diff)
  6948. min_offset_diff = diff;
  6949. }
  6950. if ((test_bit(MD_HAS_JOURNAL, &mddev->flags) || journal_dev) &&
  6951. (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {
  6952. pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",
  6953. mdname(mddev));
  6954. ret = -EINVAL;
  6955. goto exit_acct_set;
  6956. }
  6957. if (mddev->reshape_position != MaxSector) {
  6958. /* Check that we can continue the reshape.
  6959. * Difficulties arise if the stripe we would write to
  6960. * next is at or after the stripe we would read from next.
  6961. * For a reshape that changes the number of devices, this
  6962. * is only possible for a very short time, and mdadm makes
  6963. * sure that time appears to have past before assembling
  6964. * the array. So we fail if that time hasn't passed.
  6965. * For a reshape that keeps the number of devices the same
  6966. * mdadm must be monitoring the reshape can keeping the
  6967. * critical areas read-only and backed up. It will start
  6968. * the array in read-only mode, so we check for that.
  6969. */
  6970. sector_t here_new, here_old;
  6971. int old_disks;
  6972. int max_degraded = (mddev->level == 6 ? 2 : 1);
  6973. int chunk_sectors;
  6974. int new_data_disks;
  6975. if (journal_dev) {
  6976. pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
  6977. mdname(mddev));
  6978. ret = -EINVAL;
  6979. goto exit_acct_set;
  6980. }
  6981. if (mddev->new_level != mddev->level) {
  6982. pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
  6983. mdname(mddev));
  6984. ret = -EINVAL;
  6985. goto exit_acct_set;
  6986. }
  6987. old_disks = mddev->raid_disks - mddev->delta_disks;
  6988. /* reshape_position must be on a new-stripe boundary, and one
  6989. * further up in new geometry must map after here in old
  6990. * geometry.
  6991. * If the chunk sizes are different, then as we perform reshape
  6992. * in units of the largest of the two, reshape_position needs
  6993. * be a multiple of the largest chunk size times new data disks.
  6994. */
  6995. here_new = mddev->reshape_position;
  6996. chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);
  6997. new_data_disks = mddev->raid_disks - max_degraded;
  6998. if (sector_div(here_new, chunk_sectors * new_data_disks)) {
  6999. pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
  7000. mdname(mddev));
  7001. ret = -EINVAL;
  7002. goto exit_acct_set;
  7003. }
  7004. reshape_offset = here_new * chunk_sectors;
  7005. /* here_new is the stripe we will write to */
  7006. here_old = mddev->reshape_position;
  7007. sector_div(here_old, chunk_sectors * (old_disks-max_degraded));
  7008. /* here_old is the first stripe that we might need to read
  7009. * from */
  7010. if (mddev->delta_disks == 0) {
  7011. /* We cannot be sure it is safe to start an in-place
  7012. * reshape. It is only safe if user-space is monitoring
  7013. * and taking constant backups.
  7014. * mdadm always starts a situation like this in
  7015. * readonly mode so it can take control before
  7016. * allowing any writes. So just check for that.
  7017. */
  7018. if (abs(min_offset_diff) >= mddev->chunk_sectors &&
  7019. abs(min_offset_diff) >= mddev->new_chunk_sectors)
  7020. /* not really in-place - so OK */;
  7021. else if (mddev->ro == 0) {
  7022. pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
  7023. mdname(mddev));
  7024. ret = -EINVAL;
  7025. goto exit_acct_set;
  7026. }
  7027. } else if (mddev->reshape_backwards
  7028. ? (here_new * chunk_sectors + min_offset_diff <=
  7029. here_old * chunk_sectors)
  7030. : (here_new * chunk_sectors >=
  7031. here_old * chunk_sectors + (-min_offset_diff))) {
  7032. /* Reading from the same stripe as writing to - bad */
  7033. pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
  7034. mdname(mddev));
  7035. ret = -EINVAL;
  7036. goto exit_acct_set;
  7037. }
  7038. pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
  7039. /* OK, we should be able to continue; */
  7040. } else {
  7041. BUG_ON(mddev->level != mddev->new_level);
  7042. BUG_ON(mddev->layout != mddev->new_layout);
  7043. BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
  7044. BUG_ON(mddev->delta_disks != 0);
  7045. }
  7046. if (test_bit(MD_HAS_JOURNAL, &mddev->flags) &&
  7047. test_bit(MD_HAS_PPL, &mddev->flags)) {
  7048. pr_warn("md/raid:%s: using journal device and PPL not allowed - disabling PPL\n",
  7049. mdname(mddev));
  7050. clear_bit(MD_HAS_PPL, &mddev->flags);
  7051. clear_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags);
  7052. }
  7053. if (mddev->private == NULL)
  7054. conf = setup_conf(mddev);
  7055. else
  7056. conf = mddev->private;
  7057. if (IS_ERR(conf)) {
  7058. ret = PTR_ERR(conf);
  7059. goto exit_acct_set;
  7060. }
  7061. if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
  7062. if (!journal_dev) {
  7063. pr_warn("md/raid:%s: journal disk is missing, force array readonly\n",
  7064. mdname(mddev));
  7065. mddev->ro = 1;
  7066. set_disk_ro(mddev->gendisk, 1);
  7067. } else if (mddev->recovery_cp == MaxSector)
  7068. set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
  7069. }
  7070. conf->min_offset_diff = min_offset_diff;
  7071. mddev->thread = conf->thread;
  7072. conf->thread = NULL;
  7073. mddev->private = conf;
  7074. for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
  7075. i++) {
  7076. rdev = rdev_mdlock_deref(mddev, conf->disks[i].rdev);
  7077. if (!rdev && conf->disks[i].replacement) {
  7078. /* The replacement is all we have yet */
  7079. rdev = rdev_mdlock_deref(mddev,
  7080. conf->disks[i].replacement);
  7081. conf->disks[i].replacement = NULL;
  7082. clear_bit(Replacement, &rdev->flags);
  7083. rcu_assign_pointer(conf->disks[i].rdev, rdev);
  7084. }
  7085. if (!rdev)
  7086. continue;
  7087. if (rcu_access_pointer(conf->disks[i].replacement) &&
  7088. conf->reshape_progress != MaxSector) {
  7089. /* replacements and reshape simply do not mix. */
  7090. pr_warn("md: cannot handle concurrent replacement and reshape.\n");
  7091. goto abort;
  7092. }
  7093. if (test_bit(In_sync, &rdev->flags)) {
  7094. working_disks++;
  7095. continue;
  7096. }
  7097. /* This disc is not fully in-sync. However if it
  7098. * just stored parity (beyond the recovery_offset),
  7099. * when we don't need to be concerned about the
  7100. * array being dirty.
  7101. * When reshape goes 'backwards', we never have
  7102. * partially completed devices, so we only need
  7103. * to worry about reshape going forwards.
  7104. */
  7105. /* Hack because v0.91 doesn't store recovery_offset properly. */
  7106. if (mddev->major_version == 0 &&
  7107. mddev->minor_version > 90)
  7108. rdev->recovery_offset = reshape_offset;
  7109. if (rdev->recovery_offset < reshape_offset) {
  7110. /* We need to check old and new layout */
  7111. if (!only_parity(rdev->raid_disk,
  7112. conf->algorithm,
  7113. conf->raid_disks,
  7114. conf->max_degraded))
  7115. continue;
  7116. }
  7117. if (!only_parity(rdev->raid_disk,
  7118. conf->prev_algo,
  7119. conf->previous_raid_disks,
  7120. conf->max_degraded))
  7121. continue;
  7122. dirty_parity_disks++;
  7123. }
  7124. /*
  7125. * 0 for a fully functional array, 1 or 2 for a degraded array.
  7126. */
  7127. mddev->degraded = raid5_calc_degraded(conf);
  7128. if (has_failed(conf)) {
  7129. pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n",
  7130. mdname(mddev), mddev->degraded, conf->raid_disks);
  7131. goto abort;
  7132. }
  7133. /* device size must be a multiple of chunk size */
  7134. mddev->dev_sectors &= ~((sector_t)mddev->chunk_sectors - 1);
  7135. mddev->resync_max_sectors = mddev->dev_sectors;
  7136. if (mddev->degraded > dirty_parity_disks &&
  7137. mddev->recovery_cp != MaxSector) {
  7138. if (test_bit(MD_HAS_PPL, &mddev->flags))
  7139. pr_crit("md/raid:%s: starting dirty degraded array with PPL.\n",
  7140. mdname(mddev));
  7141. else if (mddev->ok_start_degraded)
  7142. pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",
  7143. mdname(mddev));
  7144. else {
  7145. pr_crit("md/raid:%s: cannot start dirty degraded array.\n",
  7146. mdname(mddev));
  7147. goto abort;
  7148. }
  7149. }
  7150. pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n",
  7151. mdname(mddev), conf->level,
  7152. mddev->raid_disks-mddev->degraded, mddev->raid_disks,
  7153. mddev->new_layout);
  7154. print_raid5_conf(conf);
  7155. if (conf->reshape_progress != MaxSector) {
  7156. conf->reshape_safe = conf->reshape_progress;
  7157. atomic_set(&conf->reshape_stripes, 0);
  7158. clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
  7159. clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
  7160. set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
  7161. set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
  7162. mddev->sync_thread = md_register_thread(md_do_sync, mddev,
  7163. "reshape");
  7164. if (!mddev->sync_thread)
  7165. goto abort;
  7166. }
  7167. /* Ok, everything is just fine now */
  7168. if (mddev->to_remove == &raid5_attrs_group)
  7169. mddev->to_remove = NULL;
  7170. else if (mddev->kobj.sd &&
  7171. sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
  7172. pr_warn("raid5: failed to create sysfs attributes for %s\n",
  7173. mdname(mddev));
  7174. md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
  7175. if (mddev->queue) {
  7176. int chunk_size;
  7177. /* read-ahead size must cover two whole stripes, which
  7178. * is 2 * (datadisks) * chunksize where 'n' is the
  7179. * number of raid devices
  7180. */
  7181. int data_disks = conf->previous_raid_disks - conf->max_degraded;
  7182. int stripe = data_disks *
  7183. ((mddev->chunk_sectors << 9) / PAGE_SIZE);
  7184. chunk_size = mddev->chunk_sectors << 9;
  7185. blk_queue_io_min(mddev->queue, chunk_size);
  7186. raid5_set_io_opt(conf);
  7187. mddev->queue->limits.raid_partial_stripes_expensive = 1;
  7188. /*
  7189. * We can only discard a whole stripe. It doesn't make sense to
  7190. * discard data disk but write parity disk
  7191. */
  7192. stripe = stripe * PAGE_SIZE;
  7193. stripe = roundup_pow_of_two(stripe);
  7194. mddev->queue->limits.discard_granularity = stripe;
  7195. blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
  7196. rdev_for_each(rdev, mddev) {
  7197. disk_stack_limits(mddev->gendisk, rdev->bdev,
  7198. rdev->data_offset << 9);
  7199. disk_stack_limits(mddev->gendisk, rdev->bdev,
  7200. rdev->new_data_offset << 9);
  7201. }
  7202. /*
  7203. * zeroing is required, otherwise data
  7204. * could be lost. Consider a scenario: discard a stripe
  7205. * (the stripe could be inconsistent if
  7206. * discard_zeroes_data is 0); write one disk of the
  7207. * stripe (the stripe could be inconsistent again
  7208. * depending on which disks are used to calculate
  7209. * parity); the disk is broken; The stripe data of this
  7210. * disk is lost.
  7211. *
  7212. * We only allow DISCARD if the sysadmin has confirmed that
  7213. * only safe devices are in use by setting a module parameter.
  7214. * A better idea might be to turn DISCARD into WRITE_ZEROES
  7215. * requests, as that is required to be safe.
  7216. */
  7217. if (!devices_handle_discard_safely ||
  7218. mddev->queue->limits.max_discard_sectors < (stripe >> 9) ||
  7219. mddev->queue->limits.discard_granularity < stripe)
  7220. blk_queue_max_discard_sectors(mddev->queue, 0);
  7221. /*
  7222. * Requests require having a bitmap for each stripe.
  7223. * Limit the max sectors based on this.
  7224. */
  7225. blk_queue_max_hw_sectors(mddev->queue,
  7226. RAID5_MAX_REQ_STRIPES << RAID5_STRIPE_SHIFT(conf));
  7227. /* No restrictions on the number of segments in the request */
  7228. blk_queue_max_segments(mddev->queue, USHRT_MAX);
  7229. }
  7230. if (log_init(conf, journal_dev, raid5_has_ppl(conf)))
  7231. goto abort;
  7232. return 0;
  7233. abort:
  7234. md_unregister_thread(&mddev->thread);
  7235. print_raid5_conf(conf);
  7236. free_conf(conf);
  7237. mddev->private = NULL;
  7238. pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
  7239. ret = -EIO;
  7240. exit_acct_set:
  7241. acct_bioset_exit(mddev);
  7242. return ret;
  7243. }
  7244. static void raid5_free(struct mddev *mddev, void *priv)
  7245. {
  7246. struct r5conf *conf = priv;
  7247. free_conf(conf);
  7248. acct_bioset_exit(mddev);
  7249. mddev->to_remove = &raid5_attrs_group;
  7250. }
  7251. static void raid5_status(struct seq_file *seq, struct mddev *mddev)
  7252. {
  7253. struct r5conf *conf = mddev->private;
  7254. int i;
  7255. seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
  7256. conf->chunk_sectors / 2, mddev->layout);
  7257. seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
  7258. rcu_read_lock();
  7259. for (i = 0; i < conf->raid_disks; i++) {
  7260. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  7261. seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
  7262. }
  7263. rcu_read_unlock();
  7264. seq_printf (seq, "]");
  7265. }
  7266. static void print_raid5_conf (struct r5conf *conf)
  7267. {
  7268. struct md_rdev *rdev;
  7269. int i;
  7270. pr_debug("RAID conf printout:\n");
  7271. if (!conf) {
  7272. pr_debug("(conf==NULL)\n");
  7273. return;
  7274. }
  7275. pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,
  7276. conf->raid_disks,
  7277. conf->raid_disks - conf->mddev->degraded);
  7278. rcu_read_lock();
  7279. for (i = 0; i < conf->raid_disks; i++) {
  7280. rdev = rcu_dereference(conf->disks[i].rdev);
  7281. if (rdev)
  7282. pr_debug(" disk %d, o:%d, dev:%pg\n",
  7283. i, !test_bit(Faulty, &rdev->flags),
  7284. rdev->bdev);
  7285. }
  7286. rcu_read_unlock();
  7287. }
  7288. static int raid5_spare_active(struct mddev *mddev)
  7289. {
  7290. int i;
  7291. struct r5conf *conf = mddev->private;
  7292. struct md_rdev *rdev, *replacement;
  7293. int count = 0;
  7294. unsigned long flags;
  7295. for (i = 0; i < conf->raid_disks; i++) {
  7296. rdev = rdev_mdlock_deref(mddev, conf->disks[i].rdev);
  7297. replacement = rdev_mdlock_deref(mddev,
  7298. conf->disks[i].replacement);
  7299. if (replacement
  7300. && replacement->recovery_offset == MaxSector
  7301. && !test_bit(Faulty, &replacement->flags)
  7302. && !test_and_set_bit(In_sync, &replacement->flags)) {
  7303. /* Replacement has just become active. */
  7304. if (!rdev
  7305. || !test_and_clear_bit(In_sync, &rdev->flags))
  7306. count++;
  7307. if (rdev) {
  7308. /* Replaced device not technically faulty,
  7309. * but we need to be sure it gets removed
  7310. * and never re-added.
  7311. */
  7312. set_bit(Faulty, &rdev->flags);
  7313. sysfs_notify_dirent_safe(
  7314. rdev->sysfs_state);
  7315. }
  7316. sysfs_notify_dirent_safe(replacement->sysfs_state);
  7317. } else if (rdev
  7318. && rdev->recovery_offset == MaxSector
  7319. && !test_bit(Faulty, &rdev->flags)
  7320. && !test_and_set_bit(In_sync, &rdev->flags)) {
  7321. count++;
  7322. sysfs_notify_dirent_safe(rdev->sysfs_state);
  7323. }
  7324. }
  7325. spin_lock_irqsave(&conf->device_lock, flags);
  7326. mddev->degraded = raid5_calc_degraded(conf);
  7327. spin_unlock_irqrestore(&conf->device_lock, flags);
  7328. print_raid5_conf(conf);
  7329. return count;
  7330. }
  7331. static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
  7332. {
  7333. struct r5conf *conf = mddev->private;
  7334. int err = 0;
  7335. int number = rdev->raid_disk;
  7336. struct md_rdev __rcu **rdevp;
  7337. struct disk_info *p;
  7338. struct md_rdev *tmp;
  7339. print_raid5_conf(conf);
  7340. if (test_bit(Journal, &rdev->flags) && conf->log) {
  7341. /*
  7342. * we can't wait pending write here, as this is called in
  7343. * raid5d, wait will deadlock.
  7344. * neilb: there is no locking about new writes here,
  7345. * so this cannot be safe.
  7346. */
  7347. if (atomic_read(&conf->active_stripes) ||
  7348. atomic_read(&conf->r5c_cached_full_stripes) ||
  7349. atomic_read(&conf->r5c_cached_partial_stripes)) {
  7350. return -EBUSY;
  7351. }
  7352. log_exit(conf);
  7353. return 0;
  7354. }
  7355. if (unlikely(number >= conf->pool_size))
  7356. return 0;
  7357. p = conf->disks + number;
  7358. if (rdev == rcu_access_pointer(p->rdev))
  7359. rdevp = &p->rdev;
  7360. else if (rdev == rcu_access_pointer(p->replacement))
  7361. rdevp = &p->replacement;
  7362. else
  7363. return 0;
  7364. if (number >= conf->raid_disks &&
  7365. conf->reshape_progress == MaxSector)
  7366. clear_bit(In_sync, &rdev->flags);
  7367. if (test_bit(In_sync, &rdev->flags) ||
  7368. atomic_read(&rdev->nr_pending)) {
  7369. err = -EBUSY;
  7370. goto abort;
  7371. }
  7372. /* Only remove non-faulty devices if recovery
  7373. * isn't possible.
  7374. */
  7375. if (!test_bit(Faulty, &rdev->flags) &&
  7376. mddev->recovery_disabled != conf->recovery_disabled &&
  7377. !has_failed(conf) &&
  7378. (!rcu_access_pointer(p->replacement) ||
  7379. rcu_access_pointer(p->replacement) == rdev) &&
  7380. number < conf->raid_disks) {
  7381. err = -EBUSY;
  7382. goto abort;
  7383. }
  7384. *rdevp = NULL;
  7385. if (!test_bit(RemoveSynchronized, &rdev->flags)) {
  7386. lockdep_assert_held(&mddev->reconfig_mutex);
  7387. synchronize_rcu();
  7388. if (atomic_read(&rdev->nr_pending)) {
  7389. /* lost the race, try later */
  7390. err = -EBUSY;
  7391. rcu_assign_pointer(*rdevp, rdev);
  7392. }
  7393. }
  7394. if (!err) {
  7395. err = log_modify(conf, rdev, false);
  7396. if (err)
  7397. goto abort;
  7398. }
  7399. tmp = rcu_access_pointer(p->replacement);
  7400. if (tmp) {
  7401. /* We must have just cleared 'rdev' */
  7402. rcu_assign_pointer(p->rdev, tmp);
  7403. clear_bit(Replacement, &tmp->flags);
  7404. smp_mb(); /* Make sure other CPUs may see both as identical
  7405. * but will never see neither - if they are careful
  7406. */
  7407. rcu_assign_pointer(p->replacement, NULL);
  7408. if (!err)
  7409. err = log_modify(conf, tmp, true);
  7410. }
  7411. clear_bit(WantReplacement, &rdev->flags);
  7412. abort:
  7413. print_raid5_conf(conf);
  7414. return err;
  7415. }
  7416. static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
  7417. {
  7418. struct r5conf *conf = mddev->private;
  7419. int ret, err = -EEXIST;
  7420. int disk;
  7421. struct disk_info *p;
  7422. struct md_rdev *tmp;
  7423. int first = 0;
  7424. int last = conf->raid_disks - 1;
  7425. if (test_bit(Journal, &rdev->flags)) {
  7426. if (conf->log)
  7427. return -EBUSY;
  7428. rdev->raid_disk = 0;
  7429. /*
  7430. * The array is in readonly mode if journal is missing, so no
  7431. * write requests running. We should be safe
  7432. */
  7433. ret = log_init(conf, rdev, false);
  7434. if (ret)
  7435. return ret;
  7436. ret = r5l_start(conf->log);
  7437. if (ret)
  7438. return ret;
  7439. return 0;
  7440. }
  7441. if (mddev->recovery_disabled == conf->recovery_disabled)
  7442. return -EBUSY;
  7443. if (rdev->saved_raid_disk < 0 && has_failed(conf))
  7444. /* no point adding a device */
  7445. return -EINVAL;
  7446. if (rdev->raid_disk >= 0)
  7447. first = last = rdev->raid_disk;
  7448. /*
  7449. * find the disk ... but prefer rdev->saved_raid_disk
  7450. * if possible.
  7451. */
  7452. if (rdev->saved_raid_disk >= first &&
  7453. rdev->saved_raid_disk <= last &&
  7454. conf->disks[rdev->saved_raid_disk].rdev == NULL)
  7455. first = rdev->saved_raid_disk;
  7456. for (disk = first; disk <= last; disk++) {
  7457. p = conf->disks + disk;
  7458. if (p->rdev == NULL) {
  7459. clear_bit(In_sync, &rdev->flags);
  7460. rdev->raid_disk = disk;
  7461. if (rdev->saved_raid_disk != disk)
  7462. conf->fullsync = 1;
  7463. rcu_assign_pointer(p->rdev, rdev);
  7464. err = log_modify(conf, rdev, true);
  7465. goto out;
  7466. }
  7467. }
  7468. for (disk = first; disk <= last; disk++) {
  7469. p = conf->disks + disk;
  7470. tmp = rdev_mdlock_deref(mddev, p->rdev);
  7471. if (test_bit(WantReplacement, &tmp->flags) &&
  7472. p->replacement == NULL) {
  7473. clear_bit(In_sync, &rdev->flags);
  7474. set_bit(Replacement, &rdev->flags);
  7475. rdev->raid_disk = disk;
  7476. err = 0;
  7477. conf->fullsync = 1;
  7478. rcu_assign_pointer(p->replacement, rdev);
  7479. break;
  7480. }
  7481. }
  7482. out:
  7483. print_raid5_conf(conf);
  7484. return err;
  7485. }
  7486. static int raid5_resize(struct mddev *mddev, sector_t sectors)
  7487. {
  7488. /* no resync is happening, and there is enough space
  7489. * on all devices, so we can resize.
  7490. * We need to make sure resync covers any new space.
  7491. * If the array is shrinking we should possibly wait until
  7492. * any io in the removed space completes, but it hardly seems
  7493. * worth it.
  7494. */
  7495. sector_t newsize;
  7496. struct r5conf *conf = mddev->private;
  7497. if (raid5_has_log(conf) || raid5_has_ppl(conf))
  7498. return -EINVAL;
  7499. sectors &= ~((sector_t)conf->chunk_sectors - 1);
  7500. newsize = raid5_size(mddev, sectors, mddev->raid_disks);
  7501. if (mddev->external_size &&
  7502. mddev->array_sectors > newsize)
  7503. return -EINVAL;
  7504. if (mddev->bitmap) {
  7505. int ret = md_bitmap_resize(mddev->bitmap, sectors, 0, 0);
  7506. if (ret)
  7507. return ret;
  7508. }
  7509. md_set_array_sectors(mddev, newsize);
  7510. if (sectors > mddev->dev_sectors &&
  7511. mddev->recovery_cp > mddev->dev_sectors) {
  7512. mddev->recovery_cp = mddev->dev_sectors;
  7513. set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
  7514. }
  7515. mddev->dev_sectors = sectors;
  7516. mddev->resync_max_sectors = sectors;
  7517. return 0;
  7518. }
  7519. static int check_stripe_cache(struct mddev *mddev)
  7520. {
  7521. /* Can only proceed if there are plenty of stripe_heads.
  7522. * We need a minimum of one full stripe,, and for sensible progress
  7523. * it is best to have about 4 times that.
  7524. * If we require 4 times, then the default 256 4K stripe_heads will
  7525. * allow for chunk sizes up to 256K, which is probably OK.
  7526. * If the chunk size is greater, user-space should request more
  7527. * stripe_heads first.
  7528. */
  7529. struct r5conf *conf = mddev->private;
  7530. if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
  7531. > conf->min_nr_stripes ||
  7532. ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
  7533. > conf->min_nr_stripes) {
  7534. pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
  7535. mdname(mddev),
  7536. ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
  7537. / RAID5_STRIPE_SIZE(conf))*4);
  7538. return 0;
  7539. }
  7540. return 1;
  7541. }
  7542. static int check_reshape(struct mddev *mddev)
  7543. {
  7544. struct r5conf *conf = mddev->private;
  7545. if (raid5_has_log(conf) || raid5_has_ppl(conf))
  7546. return -EINVAL;
  7547. if (mddev->delta_disks == 0 &&
  7548. mddev->new_layout == mddev->layout &&
  7549. mddev->new_chunk_sectors == mddev->chunk_sectors)
  7550. return 0; /* nothing to do */
  7551. if (has_failed(conf))
  7552. return -EINVAL;
  7553. if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
  7554. /* We might be able to shrink, but the devices must
  7555. * be made bigger first.
  7556. * For raid6, 4 is the minimum size.
  7557. * Otherwise 2 is the minimum
  7558. */
  7559. int min = 2;
  7560. if (mddev->level == 6)
  7561. min = 4;
  7562. if (mddev->raid_disks + mddev->delta_disks < min)
  7563. return -EINVAL;
  7564. }
  7565. if (!check_stripe_cache(mddev))
  7566. return -ENOSPC;
  7567. if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
  7568. mddev->delta_disks > 0)
  7569. if (resize_chunks(conf,
  7570. conf->previous_raid_disks
  7571. + max(0, mddev->delta_disks),
  7572. max(mddev->new_chunk_sectors,
  7573. mddev->chunk_sectors)
  7574. ) < 0)
  7575. return -ENOMEM;
  7576. if (conf->previous_raid_disks + mddev->delta_disks <= conf->pool_size)
  7577. return 0; /* never bother to shrink */
  7578. return resize_stripes(conf, (conf->previous_raid_disks
  7579. + mddev->delta_disks));
  7580. }
  7581. static int raid5_start_reshape(struct mddev *mddev)
  7582. {
  7583. struct r5conf *conf = mddev->private;
  7584. struct md_rdev *rdev;
  7585. int spares = 0;
  7586. unsigned long flags;
  7587. if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
  7588. return -EBUSY;
  7589. if (!check_stripe_cache(mddev))
  7590. return -ENOSPC;
  7591. if (has_failed(conf))
  7592. return -EINVAL;
  7593. rdev_for_each(rdev, mddev) {
  7594. if (!test_bit(In_sync, &rdev->flags)
  7595. && !test_bit(Faulty, &rdev->flags))
  7596. spares++;
  7597. }
  7598. if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
  7599. /* Not enough devices even to make a degraded array
  7600. * of that size
  7601. */
  7602. return -EINVAL;
  7603. /* Refuse to reduce size of the array. Any reductions in
  7604. * array size must be through explicit setting of array_size
  7605. * attribute.
  7606. */
  7607. if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
  7608. < mddev->array_sectors) {
  7609. pr_warn("md/raid:%s: array size must be reduced before number of disks\n",
  7610. mdname(mddev));
  7611. return -EINVAL;
  7612. }
  7613. atomic_set(&conf->reshape_stripes, 0);
  7614. spin_lock_irq(&conf->device_lock);
  7615. write_seqcount_begin(&conf->gen_lock);
  7616. conf->previous_raid_disks = conf->raid_disks;
  7617. conf->raid_disks += mddev->delta_disks;
  7618. conf->prev_chunk_sectors = conf->chunk_sectors;
  7619. conf->chunk_sectors = mddev->new_chunk_sectors;
  7620. conf->prev_algo = conf->algorithm;
  7621. conf->algorithm = mddev->new_layout;
  7622. conf->generation++;
  7623. /* Code that selects data_offset needs to see the generation update
  7624. * if reshape_progress has been set - so a memory barrier needed.
  7625. */
  7626. smp_mb();
  7627. if (mddev->reshape_backwards)
  7628. conf->reshape_progress = raid5_size(mddev, 0, 0);
  7629. else
  7630. conf->reshape_progress = 0;
  7631. conf->reshape_safe = conf->reshape_progress;
  7632. write_seqcount_end(&conf->gen_lock);
  7633. spin_unlock_irq(&conf->device_lock);
  7634. /* Now make sure any requests that proceeded on the assumption
  7635. * the reshape wasn't running - like Discard or Read - have
  7636. * completed.
  7637. */
  7638. mddev_suspend(mddev);
  7639. mddev_resume(mddev);
  7640. /* Add some new drives, as many as will fit.
  7641. * We know there are enough to make the newly sized array work.
  7642. * Don't add devices if we are reducing the number of
  7643. * devices in the array. This is because it is not possible
  7644. * to correctly record the "partially reconstructed" state of
  7645. * such devices during the reshape and confusion could result.
  7646. */
  7647. if (mddev->delta_disks >= 0) {
  7648. rdev_for_each(rdev, mddev)
  7649. if (rdev->raid_disk < 0 &&
  7650. !test_bit(Faulty, &rdev->flags)) {
  7651. if (raid5_add_disk(mddev, rdev) == 0) {
  7652. if (rdev->raid_disk
  7653. >= conf->previous_raid_disks)
  7654. set_bit(In_sync, &rdev->flags);
  7655. else
  7656. rdev->recovery_offset = 0;
  7657. /* Failure here is OK */
  7658. sysfs_link_rdev(mddev, rdev);
  7659. }
  7660. } else if (rdev->raid_disk >= conf->previous_raid_disks
  7661. && !test_bit(Faulty, &rdev->flags)) {
  7662. /* This is a spare that was manually added */
  7663. set_bit(In_sync, &rdev->flags);
  7664. }
  7665. /* When a reshape changes the number of devices,
  7666. * ->degraded is measured against the larger of the
  7667. * pre and post number of devices.
  7668. */
  7669. spin_lock_irqsave(&conf->device_lock, flags);
  7670. mddev->degraded = raid5_calc_degraded(conf);
  7671. spin_unlock_irqrestore(&conf->device_lock, flags);
  7672. }
  7673. mddev->raid_disks = conf->raid_disks;
  7674. mddev->reshape_position = conf->reshape_progress;
  7675. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  7676. clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
  7677. clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
  7678. clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
  7679. set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
  7680. set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
  7681. mddev->sync_thread = md_register_thread(md_do_sync, mddev,
  7682. "reshape");
  7683. if (!mddev->sync_thread) {
  7684. mddev->recovery = 0;
  7685. spin_lock_irq(&conf->device_lock);
  7686. write_seqcount_begin(&conf->gen_lock);
  7687. mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
  7688. mddev->new_chunk_sectors =
  7689. conf->chunk_sectors = conf->prev_chunk_sectors;
  7690. mddev->new_layout = conf->algorithm = conf->prev_algo;
  7691. rdev_for_each(rdev, mddev)
  7692. rdev->new_data_offset = rdev->data_offset;
  7693. smp_wmb();
  7694. conf->generation --;
  7695. conf->reshape_progress = MaxSector;
  7696. mddev->reshape_position = MaxSector;
  7697. write_seqcount_end(&conf->gen_lock);
  7698. spin_unlock_irq(&conf->device_lock);
  7699. return -EAGAIN;
  7700. }
  7701. conf->reshape_checkpoint = jiffies;
  7702. md_wakeup_thread(mddev->sync_thread);
  7703. md_new_event();
  7704. return 0;
  7705. }
  7706. /* This is called from the reshape thread and should make any
  7707. * changes needed in 'conf'
  7708. */
  7709. static void end_reshape(struct r5conf *conf)
  7710. {
  7711. if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
  7712. struct md_rdev *rdev;
  7713. spin_lock_irq(&conf->device_lock);
  7714. conf->previous_raid_disks = conf->raid_disks;
  7715. md_finish_reshape(conf->mddev);
  7716. smp_wmb();
  7717. conf->reshape_progress = MaxSector;
  7718. conf->mddev->reshape_position = MaxSector;
  7719. rdev_for_each(rdev, conf->mddev)
  7720. if (rdev->raid_disk >= 0 &&
  7721. !test_bit(Journal, &rdev->flags) &&
  7722. !test_bit(In_sync, &rdev->flags))
  7723. rdev->recovery_offset = MaxSector;
  7724. spin_unlock_irq(&conf->device_lock);
  7725. wake_up(&conf->wait_for_overlap);
  7726. if (conf->mddev->queue)
  7727. raid5_set_io_opt(conf);
  7728. }
  7729. }
  7730. /* This is called from the raid5d thread with mddev_lock held.
  7731. * It makes config changes to the device.
  7732. */
  7733. static void raid5_finish_reshape(struct mddev *mddev)
  7734. {
  7735. struct r5conf *conf = mddev->private;
  7736. struct md_rdev *rdev;
  7737. if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
  7738. if (mddev->delta_disks <= 0) {
  7739. int d;
  7740. spin_lock_irq(&conf->device_lock);
  7741. mddev->degraded = raid5_calc_degraded(conf);
  7742. spin_unlock_irq(&conf->device_lock);
  7743. for (d = conf->raid_disks ;
  7744. d < conf->raid_disks - mddev->delta_disks;
  7745. d++) {
  7746. rdev = rdev_mdlock_deref(mddev,
  7747. conf->disks[d].rdev);
  7748. if (rdev)
  7749. clear_bit(In_sync, &rdev->flags);
  7750. rdev = rdev_mdlock_deref(mddev,
  7751. conf->disks[d].replacement);
  7752. if (rdev)
  7753. clear_bit(In_sync, &rdev->flags);
  7754. }
  7755. }
  7756. mddev->layout = conf->algorithm;
  7757. mddev->chunk_sectors = conf->chunk_sectors;
  7758. mddev->reshape_position = MaxSector;
  7759. mddev->delta_disks = 0;
  7760. mddev->reshape_backwards = 0;
  7761. }
  7762. }
  7763. static void raid5_quiesce(struct mddev *mddev, int quiesce)
  7764. {
  7765. struct r5conf *conf = mddev->private;
  7766. if (quiesce) {
  7767. /* stop all writes */
  7768. lock_all_device_hash_locks_irq(conf);
  7769. /* '2' tells resync/reshape to pause so that all
  7770. * active stripes can drain
  7771. */
  7772. r5c_flush_cache(conf, INT_MAX);
  7773. /* need a memory barrier to make sure read_one_chunk() sees
  7774. * quiesce started and reverts to slow (locked) path.
  7775. */
  7776. smp_store_release(&conf->quiesce, 2);
  7777. wait_event_cmd(conf->wait_for_quiescent,
  7778. atomic_read(&conf->active_stripes) == 0 &&
  7779. atomic_read(&conf->active_aligned_reads) == 0,
  7780. unlock_all_device_hash_locks_irq(conf),
  7781. lock_all_device_hash_locks_irq(conf));
  7782. conf->quiesce = 1;
  7783. unlock_all_device_hash_locks_irq(conf);
  7784. /* allow reshape to continue */
  7785. wake_up(&conf->wait_for_overlap);
  7786. } else {
  7787. /* re-enable writes */
  7788. lock_all_device_hash_locks_irq(conf);
  7789. conf->quiesce = 0;
  7790. wake_up(&conf->wait_for_quiescent);
  7791. wake_up(&conf->wait_for_overlap);
  7792. unlock_all_device_hash_locks_irq(conf);
  7793. }
  7794. log_quiesce(conf, quiesce);
  7795. }
  7796. static void *raid45_takeover_raid0(struct mddev *mddev, int level)
  7797. {
  7798. struct r0conf *raid0_conf = mddev->private;
  7799. sector_t sectors;
  7800. /* for raid0 takeover only one zone is supported */
  7801. if (raid0_conf->nr_strip_zones > 1) {
  7802. pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n",
  7803. mdname(mddev));
  7804. return ERR_PTR(-EINVAL);
  7805. }
  7806. sectors = raid0_conf->strip_zone[0].zone_end;
  7807. sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
  7808. mddev->dev_sectors = sectors;
  7809. mddev->new_level = level;
  7810. mddev->new_layout = ALGORITHM_PARITY_N;
  7811. mddev->new_chunk_sectors = mddev->chunk_sectors;
  7812. mddev->raid_disks += 1;
  7813. mddev->delta_disks = 1;
  7814. /* make sure it will be not marked as dirty */
  7815. mddev->recovery_cp = MaxSector;
  7816. return setup_conf(mddev);
  7817. }
  7818. static void *raid5_takeover_raid1(struct mddev *mddev)
  7819. {
  7820. int chunksect;
  7821. void *ret;
  7822. if (mddev->raid_disks != 2 ||
  7823. mddev->degraded > 1)
  7824. return ERR_PTR(-EINVAL);
  7825. /* Should check if there are write-behind devices? */
  7826. chunksect = 64*2; /* 64K by default */
  7827. /* The array must be an exact multiple of chunksize */
  7828. while (chunksect && (mddev->array_sectors & (chunksect-1)))
  7829. chunksect >>= 1;
  7830. if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))
  7831. /* array size does not allow a suitable chunk size */
  7832. return ERR_PTR(-EINVAL);
  7833. mddev->new_level = 5;
  7834. mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
  7835. mddev->new_chunk_sectors = chunksect;
  7836. ret = setup_conf(mddev);
  7837. if (!IS_ERR(ret))
  7838. mddev_clear_unsupported_flags(mddev,
  7839. UNSUPPORTED_MDDEV_FLAGS);
  7840. return ret;
  7841. }
  7842. static void *raid5_takeover_raid6(struct mddev *mddev)
  7843. {
  7844. int new_layout;
  7845. switch (mddev->layout) {
  7846. case ALGORITHM_LEFT_ASYMMETRIC_6:
  7847. new_layout = ALGORITHM_LEFT_ASYMMETRIC;
  7848. break;
  7849. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  7850. new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
  7851. break;
  7852. case ALGORITHM_LEFT_SYMMETRIC_6:
  7853. new_layout = ALGORITHM_LEFT_SYMMETRIC;
  7854. break;
  7855. case ALGORITHM_RIGHT_SYMMETRIC_6:
  7856. new_layout = ALGORITHM_RIGHT_SYMMETRIC;
  7857. break;
  7858. case ALGORITHM_PARITY_0_6:
  7859. new_layout = ALGORITHM_PARITY_0;
  7860. break;
  7861. case ALGORITHM_PARITY_N:
  7862. new_layout = ALGORITHM_PARITY_N;
  7863. break;
  7864. default:
  7865. return ERR_PTR(-EINVAL);
  7866. }
  7867. mddev->new_level = 5;
  7868. mddev->new_layout = new_layout;
  7869. mddev->delta_disks = -1;
  7870. mddev->raid_disks -= 1;
  7871. return setup_conf(mddev);
  7872. }
  7873. static int raid5_check_reshape(struct mddev *mddev)
  7874. {
  7875. /* For a 2-drive array, the layout and chunk size can be changed
  7876. * immediately as not restriping is needed.
  7877. * For larger arrays we record the new value - after validation
  7878. * to be used by a reshape pass.
  7879. */
  7880. struct r5conf *conf = mddev->private;
  7881. int new_chunk = mddev->new_chunk_sectors;
  7882. if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
  7883. return -EINVAL;
  7884. if (new_chunk > 0) {
  7885. if (!is_power_of_2(new_chunk))
  7886. return -EINVAL;
  7887. if (new_chunk < (PAGE_SIZE>>9))
  7888. return -EINVAL;
  7889. if (mddev->array_sectors & (new_chunk-1))
  7890. /* not factor of array size */
  7891. return -EINVAL;
  7892. }
  7893. /* They look valid */
  7894. if (mddev->raid_disks == 2) {
  7895. /* can make the change immediately */
  7896. if (mddev->new_layout >= 0) {
  7897. conf->algorithm = mddev->new_layout;
  7898. mddev->layout = mddev->new_layout;
  7899. }
  7900. if (new_chunk > 0) {
  7901. conf->chunk_sectors = new_chunk ;
  7902. mddev->chunk_sectors = new_chunk;
  7903. }
  7904. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  7905. md_wakeup_thread(mddev->thread);
  7906. }
  7907. return check_reshape(mddev);
  7908. }
  7909. static int raid6_check_reshape(struct mddev *mddev)
  7910. {
  7911. int new_chunk = mddev->new_chunk_sectors;
  7912. if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
  7913. return -EINVAL;
  7914. if (new_chunk > 0) {
  7915. if (!is_power_of_2(new_chunk))
  7916. return -EINVAL;
  7917. if (new_chunk < (PAGE_SIZE >> 9))
  7918. return -EINVAL;
  7919. if (mddev->array_sectors & (new_chunk-1))
  7920. /* not factor of array size */
  7921. return -EINVAL;
  7922. }
  7923. /* They look valid */
  7924. return check_reshape(mddev);
  7925. }
  7926. static void *raid5_takeover(struct mddev *mddev)
  7927. {
  7928. /* raid5 can take over:
  7929. * raid0 - if there is only one strip zone - make it a raid4 layout
  7930. * raid1 - if there are two drives. We need to know the chunk size
  7931. * raid4 - trivial - just use a raid4 layout.
  7932. * raid6 - Providing it is a *_6 layout
  7933. */
  7934. if (mddev->level == 0)
  7935. return raid45_takeover_raid0(mddev, 5);
  7936. if (mddev->level == 1)
  7937. return raid5_takeover_raid1(mddev);
  7938. if (mddev->level == 4) {
  7939. mddev->new_layout = ALGORITHM_PARITY_N;
  7940. mddev->new_level = 5;
  7941. return setup_conf(mddev);
  7942. }
  7943. if (mddev->level == 6)
  7944. return raid5_takeover_raid6(mddev);
  7945. return ERR_PTR(-EINVAL);
  7946. }
  7947. static void *raid4_takeover(struct mddev *mddev)
  7948. {
  7949. /* raid4 can take over:
  7950. * raid0 - if there is only one strip zone
  7951. * raid5 - if layout is right
  7952. */
  7953. if (mddev->level == 0)
  7954. return raid45_takeover_raid0(mddev, 4);
  7955. if (mddev->level == 5 &&
  7956. mddev->layout == ALGORITHM_PARITY_N) {
  7957. mddev->new_layout = 0;
  7958. mddev->new_level = 4;
  7959. return setup_conf(mddev);
  7960. }
  7961. return ERR_PTR(-EINVAL);
  7962. }
  7963. static struct md_personality raid5_personality;
  7964. static void *raid6_takeover(struct mddev *mddev)
  7965. {
  7966. /* Currently can only take over a raid5. We map the
  7967. * personality to an equivalent raid6 personality
  7968. * with the Q block at the end.
  7969. */
  7970. int new_layout;
  7971. if (mddev->pers != &raid5_personality)
  7972. return ERR_PTR(-EINVAL);
  7973. if (mddev->degraded > 1)
  7974. return ERR_PTR(-EINVAL);
  7975. if (mddev->raid_disks > 253)
  7976. return ERR_PTR(-EINVAL);
  7977. if (mddev->raid_disks < 3)
  7978. return ERR_PTR(-EINVAL);
  7979. switch (mddev->layout) {
  7980. case ALGORITHM_LEFT_ASYMMETRIC:
  7981. new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
  7982. break;
  7983. case ALGORITHM_RIGHT_ASYMMETRIC:
  7984. new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
  7985. break;
  7986. case ALGORITHM_LEFT_SYMMETRIC:
  7987. new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
  7988. break;
  7989. case ALGORITHM_RIGHT_SYMMETRIC:
  7990. new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
  7991. break;
  7992. case ALGORITHM_PARITY_0:
  7993. new_layout = ALGORITHM_PARITY_0_6;
  7994. break;
  7995. case ALGORITHM_PARITY_N:
  7996. new_layout = ALGORITHM_PARITY_N;
  7997. break;
  7998. default:
  7999. return ERR_PTR(-EINVAL);
  8000. }
  8001. mddev->new_level = 6;
  8002. mddev->new_layout = new_layout;
  8003. mddev->delta_disks = 1;
  8004. mddev->raid_disks += 1;
  8005. return setup_conf(mddev);
  8006. }
  8007. static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
  8008. {
  8009. struct r5conf *conf;
  8010. int err;
  8011. err = mddev_lock(mddev);
  8012. if (err)
  8013. return err;
  8014. conf = mddev->private;
  8015. if (!conf) {
  8016. mddev_unlock(mddev);
  8017. return -ENODEV;
  8018. }
  8019. if (strncmp(buf, "ppl", 3) == 0) {
  8020. /* ppl only works with RAID 5 */
  8021. if (!raid5_has_ppl(conf) && conf->level == 5) {
  8022. err = log_init(conf, NULL, true);
  8023. if (!err) {
  8024. err = resize_stripes(conf, conf->pool_size);
  8025. if (err) {
  8026. mddev_suspend(mddev);
  8027. log_exit(conf);
  8028. mddev_resume(mddev);
  8029. }
  8030. }
  8031. } else
  8032. err = -EINVAL;
  8033. } else if (strncmp(buf, "resync", 6) == 0) {
  8034. if (raid5_has_ppl(conf)) {
  8035. mddev_suspend(mddev);
  8036. log_exit(conf);
  8037. mddev_resume(mddev);
  8038. err = resize_stripes(conf, conf->pool_size);
  8039. } else if (test_bit(MD_HAS_JOURNAL, &conf->mddev->flags) &&
  8040. r5l_log_disk_error(conf)) {
  8041. bool journal_dev_exists = false;
  8042. struct md_rdev *rdev;
  8043. rdev_for_each(rdev, mddev)
  8044. if (test_bit(Journal, &rdev->flags)) {
  8045. journal_dev_exists = true;
  8046. break;
  8047. }
  8048. if (!journal_dev_exists) {
  8049. mddev_suspend(mddev);
  8050. clear_bit(MD_HAS_JOURNAL, &mddev->flags);
  8051. mddev_resume(mddev);
  8052. } else /* need remove journal device first */
  8053. err = -EBUSY;
  8054. } else
  8055. err = -EINVAL;
  8056. } else {
  8057. err = -EINVAL;
  8058. }
  8059. if (!err)
  8060. md_update_sb(mddev, 1);
  8061. mddev_unlock(mddev);
  8062. return err;
  8063. }
  8064. static int raid5_start(struct mddev *mddev)
  8065. {
  8066. struct r5conf *conf = mddev->private;
  8067. return r5l_start(conf->log);
  8068. }
  8069. static struct md_personality raid6_personality =
  8070. {
  8071. .name = "raid6",
  8072. .level = 6,
  8073. .owner = THIS_MODULE,
  8074. .make_request = raid5_make_request,
  8075. .run = raid5_run,
  8076. .start = raid5_start,
  8077. .free = raid5_free,
  8078. .status = raid5_status,
  8079. .error_handler = raid5_error,
  8080. .hot_add_disk = raid5_add_disk,
  8081. .hot_remove_disk= raid5_remove_disk,
  8082. .spare_active = raid5_spare_active,
  8083. .sync_request = raid5_sync_request,
  8084. .resize = raid5_resize,
  8085. .size = raid5_size,
  8086. .check_reshape = raid6_check_reshape,
  8087. .start_reshape = raid5_start_reshape,
  8088. .finish_reshape = raid5_finish_reshape,
  8089. .quiesce = raid5_quiesce,
  8090. .takeover = raid6_takeover,
  8091. .change_consistency_policy = raid5_change_consistency_policy,
  8092. };
  8093. static struct md_personality raid5_personality =
  8094. {
  8095. .name = "raid5",
  8096. .level = 5,
  8097. .owner = THIS_MODULE,
  8098. .make_request = raid5_make_request,
  8099. .run = raid5_run,
  8100. .start = raid5_start,
  8101. .free = raid5_free,
  8102. .status = raid5_status,
  8103. .error_handler = raid5_error,
  8104. .hot_add_disk = raid5_add_disk,
  8105. .hot_remove_disk= raid5_remove_disk,
  8106. .spare_active = raid5_spare_active,
  8107. .sync_request = raid5_sync_request,
  8108. .resize = raid5_resize,
  8109. .size = raid5_size,
  8110. .check_reshape = raid5_check_reshape,
  8111. .start_reshape = raid5_start_reshape,
  8112. .finish_reshape = raid5_finish_reshape,
  8113. .quiesce = raid5_quiesce,
  8114. .takeover = raid5_takeover,
  8115. .change_consistency_policy = raid5_change_consistency_policy,
  8116. };
  8117. static struct md_personality raid4_personality =
  8118. {
  8119. .name = "raid4",
  8120. .level = 4,
  8121. .owner = THIS_MODULE,
  8122. .make_request = raid5_make_request,
  8123. .run = raid5_run,
  8124. .start = raid5_start,
  8125. .free = raid5_free,
  8126. .status = raid5_status,
  8127. .error_handler = raid5_error,
  8128. .hot_add_disk = raid5_add_disk,
  8129. .hot_remove_disk= raid5_remove_disk,
  8130. .spare_active = raid5_spare_active,
  8131. .sync_request = raid5_sync_request,
  8132. .resize = raid5_resize,
  8133. .size = raid5_size,
  8134. .check_reshape = raid5_check_reshape,
  8135. .start_reshape = raid5_start_reshape,
  8136. .finish_reshape = raid5_finish_reshape,
  8137. .quiesce = raid5_quiesce,
  8138. .takeover = raid4_takeover,
  8139. .change_consistency_policy = raid5_change_consistency_policy,
  8140. };
  8141. static int __init raid5_init(void)
  8142. {
  8143. int ret;
  8144. raid5_wq = alloc_workqueue("raid5wq",
  8145. WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
  8146. if (!raid5_wq)
  8147. return -ENOMEM;
  8148. ret = cpuhp_setup_state_multi(CPUHP_MD_RAID5_PREPARE,
  8149. "md/raid5:prepare",
  8150. raid456_cpu_up_prepare,
  8151. raid456_cpu_dead);
  8152. if (ret) {
  8153. destroy_workqueue(raid5_wq);
  8154. return ret;
  8155. }
  8156. register_md_personality(&raid6_personality);
  8157. register_md_personality(&raid5_personality);
  8158. register_md_personality(&raid4_personality);
  8159. return 0;
  8160. }
  8161. static void raid5_exit(void)
  8162. {
  8163. unregister_md_personality(&raid6_personality);
  8164. unregister_md_personality(&raid5_personality);
  8165. unregister_md_personality(&raid4_personality);
  8166. cpuhp_remove_multi_state(CPUHP_MD_RAID5_PREPARE);
  8167. destroy_workqueue(raid5_wq);
  8168. }
  8169. module_init(raid5_init);
  8170. module_exit(raid5_exit);
  8171. MODULE_LICENSE("GPL");
  8172. MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
  8173. MODULE_ALIAS("md-personality-4"); /* RAID5 */
  8174. MODULE_ALIAS("md-raid5");
  8175. MODULE_ALIAS("md-raid4");
  8176. MODULE_ALIAS("md-level-5");
  8177. MODULE_ALIAS("md-level-4");
  8178. MODULE_ALIAS("md-personality-8"); /* RAID6 */
  8179. MODULE_ALIAS("md-raid6");
  8180. MODULE_ALIAS("md-level-6");
  8181. /* This used to be two separate modules, they were: */
  8182. MODULE_ALIAS("raid5");
  8183. MODULE_ALIAS("raid6");