workqueue.c 169 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * kernel/workqueue.c - generic async execution with shared worker pool
  4. *
  5. * Copyright (C) 2002 Ingo Molnar
  6. *
  7. * Derived from the taskqueue/keventd code by:
  8. * David Woodhouse <[email protected]>
  9. * Andrew Morton
  10. * Kai Petzke <[email protected]>
  11. * Theodore Ts'o <[email protected]>
  12. *
  13. * Made to use alloc_percpu by Christoph Lameter.
  14. *
  15. * Copyright (C) 2010 SUSE Linux Products GmbH
  16. * Copyright (C) 2010 Tejun Heo <[email protected]>
  17. *
  18. * This is the generic async execution mechanism. Work items as are
  19. * executed in process context. The worker pool is shared and
  20. * automatically managed. There are two worker pools for each CPU (one for
  21. * normal work items and the other for high priority ones) and some extra
  22. * pools for workqueues which are not bound to any specific CPU - the
  23. * number of these backing pools is dynamic.
  24. *
  25. * Please read Documentation/core-api/workqueue.rst for details.
  26. */
  27. #include <linux/export.h>
  28. #include <linux/kernel.h>
  29. #include <linux/sched.h>
  30. #include <linux/init.h>
  31. #include <linux/signal.h>
  32. #include <linux/completion.h>
  33. #include <linux/workqueue.h>
  34. #include <linux/slab.h>
  35. #include <linux/cpu.h>
  36. #include <linux/notifier.h>
  37. #include <linux/kthread.h>
  38. #include <linux/hardirq.h>
  39. #include <linux/mempolicy.h>
  40. #include <linux/freezer.h>
  41. #include <linux/debug_locks.h>
  42. #include <linux/lockdep.h>
  43. #include <linux/idr.h>
  44. #include <linux/jhash.h>
  45. #include <linux/hashtable.h>
  46. #include <linux/rculist.h>
  47. #include <linux/nodemask.h>
  48. #include <linux/moduleparam.h>
  49. #include <linux/uaccess.h>
  50. #include <linux/sched/isolation.h>
  51. #include <linux/nmi.h>
  52. #include <linux/kvm_para.h>
  53. #include "workqueue_internal.h"
  54. #include <trace/hooks/wqlockup.h>
  55. /* events/workqueue.h uses default TRACE_INCLUDE_PATH */
  56. #undef TRACE_INCLUDE_PATH
  57. enum {
  58. /*
  59. * worker_pool flags
  60. *
  61. * A bound pool is either associated or disassociated with its CPU.
  62. * While associated (!DISASSOCIATED), all workers are bound to the
  63. * CPU and none has %WORKER_UNBOUND set and concurrency management
  64. * is in effect.
  65. *
  66. * While DISASSOCIATED, the cpu may be offline and all workers have
  67. * %WORKER_UNBOUND set and concurrency management disabled, and may
  68. * be executing on any CPU. The pool behaves as an unbound one.
  69. *
  70. * Note that DISASSOCIATED should be flipped only while holding
  71. * wq_pool_attach_mutex to avoid changing binding state while
  72. * worker_attach_to_pool() is in progress.
  73. */
  74. POOL_MANAGER_ACTIVE = 1 << 0, /* being managed */
  75. POOL_DISASSOCIATED = 1 << 2, /* cpu can't serve workers */
  76. /* worker flags */
  77. WORKER_DIE = 1 << 1, /* die die die */
  78. WORKER_IDLE = 1 << 2, /* is idle */
  79. WORKER_PREP = 1 << 3, /* preparing to run works */
  80. WORKER_CPU_INTENSIVE = 1 << 6, /* cpu intensive */
  81. WORKER_UNBOUND = 1 << 7, /* worker is unbound */
  82. WORKER_REBOUND = 1 << 8, /* worker was rebound */
  83. WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
  84. WORKER_UNBOUND | WORKER_REBOUND,
  85. NR_STD_WORKER_POOLS = 2, /* # standard pools per cpu */
  86. UNBOUND_POOL_HASH_ORDER = 6, /* hashed by pool->attrs */
  87. BUSY_WORKER_HASH_ORDER = 6, /* 64 pointers */
  88. MAX_IDLE_WORKERS_RATIO = 4, /* 1/4 of busy can be idle */
  89. IDLE_WORKER_TIMEOUT = 300 * HZ, /* keep idle ones for 5 mins */
  90. MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
  91. /* call for help after 10ms
  92. (min two ticks) */
  93. MAYDAY_INTERVAL = HZ / 10, /* and then every 100ms */
  94. CREATE_COOLDOWN = HZ, /* time to breath after fail */
  95. /*
  96. * Rescue workers are used only on emergencies and shared by
  97. * all cpus. Give MIN_NICE.
  98. */
  99. RESCUER_NICE_LEVEL = MIN_NICE,
  100. HIGHPRI_NICE_LEVEL = MIN_NICE,
  101. WQ_NAME_LEN = 24,
  102. };
  103. /*
  104. * Structure fields follow one of the following exclusion rules.
  105. *
  106. * I: Modifiable by initialization/destruction paths and read-only for
  107. * everyone else.
  108. *
  109. * P: Preemption protected. Disabling preemption is enough and should
  110. * only be modified and accessed from the local cpu.
  111. *
  112. * L: pool->lock protected. Access with pool->lock held.
  113. *
  114. * X: During normal operation, modification requires pool->lock and should
  115. * be done only from local cpu. Either disabling preemption on local
  116. * cpu or grabbing pool->lock is enough for read access. If
  117. * POOL_DISASSOCIATED is set, it's identical to L.
  118. *
  119. * A: wq_pool_attach_mutex protected.
  120. *
  121. * PL: wq_pool_mutex protected.
  122. *
  123. * PR: wq_pool_mutex protected for writes. RCU protected for reads.
  124. *
  125. * PW: wq_pool_mutex and wq->mutex protected for writes. Either for reads.
  126. *
  127. * PWR: wq_pool_mutex and wq->mutex protected for writes. Either or
  128. * RCU for reads.
  129. *
  130. * WQ: wq->mutex protected.
  131. *
  132. * WR: wq->mutex protected for writes. RCU protected for reads.
  133. *
  134. * MD: wq_mayday_lock protected.
  135. */
  136. /* struct worker is defined in workqueue_internal.h */
  137. struct worker_pool {
  138. raw_spinlock_t lock; /* the pool lock */
  139. int cpu; /* I: the associated cpu */
  140. int node; /* I: the associated node ID */
  141. int id; /* I: pool ID */
  142. unsigned int flags; /* X: flags */
  143. unsigned long watchdog_ts; /* L: watchdog timestamp */
  144. /*
  145. * The counter is incremented in a process context on the associated CPU
  146. * w/ preemption disabled, and decremented or reset in the same context
  147. * but w/ pool->lock held. The readers grab pool->lock and are
  148. * guaranteed to see if the counter reached zero.
  149. */
  150. int nr_running;
  151. struct list_head worklist; /* L: list of pending works */
  152. int nr_workers; /* L: total number of workers */
  153. int nr_idle; /* L: currently idle workers */
  154. struct list_head idle_list; /* L: list of idle workers */
  155. struct timer_list idle_timer; /* L: worker idle timeout */
  156. struct timer_list mayday_timer; /* L: SOS timer for workers */
  157. /* a workers is either on busy_hash or idle_list, or the manager */
  158. DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
  159. /* L: hash of busy workers */
  160. struct worker *manager; /* L: purely informational */
  161. struct list_head workers; /* A: attached workers */
  162. struct completion *detach_completion; /* all workers detached */
  163. struct ida worker_ida; /* worker IDs for task name */
  164. struct workqueue_attrs *attrs; /* I: worker attributes */
  165. struct hlist_node hash_node; /* PL: unbound_pool_hash node */
  166. int refcnt; /* PL: refcnt for unbound pools */
  167. /*
  168. * Destruction of pool is RCU protected to allow dereferences
  169. * from get_work_pool().
  170. */
  171. struct rcu_head rcu;
  172. };
  173. /*
  174. * The per-pool workqueue. While queued, the lower WORK_STRUCT_FLAG_BITS
  175. * of work_struct->data are used for flags and the remaining high bits
  176. * point to the pwq; thus, pwqs need to be aligned at two's power of the
  177. * number of flag bits.
  178. */
  179. struct pool_workqueue {
  180. struct worker_pool *pool; /* I: the associated pool */
  181. struct workqueue_struct *wq; /* I: the owning workqueue */
  182. int work_color; /* L: current color */
  183. int flush_color; /* L: flushing color */
  184. int refcnt; /* L: reference count */
  185. int nr_in_flight[WORK_NR_COLORS];
  186. /* L: nr of in_flight works */
  187. /*
  188. * nr_active management and WORK_STRUCT_INACTIVE:
  189. *
  190. * When pwq->nr_active >= max_active, new work item is queued to
  191. * pwq->inactive_works instead of pool->worklist and marked with
  192. * WORK_STRUCT_INACTIVE.
  193. *
  194. * All work items marked with WORK_STRUCT_INACTIVE do not participate
  195. * in pwq->nr_active and all work items in pwq->inactive_works are
  196. * marked with WORK_STRUCT_INACTIVE. But not all WORK_STRUCT_INACTIVE
  197. * work items are in pwq->inactive_works. Some of them are ready to
  198. * run in pool->worklist or worker->scheduled. Those work itmes are
  199. * only struct wq_barrier which is used for flush_work() and should
  200. * not participate in pwq->nr_active. For non-barrier work item, it
  201. * is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
  202. */
  203. int nr_active; /* L: nr of active works */
  204. int max_active; /* L: max active works */
  205. struct list_head inactive_works; /* L: inactive works */
  206. struct list_head pwqs_node; /* WR: node on wq->pwqs */
  207. struct list_head mayday_node; /* MD: node on wq->maydays */
  208. /*
  209. * Release of unbound pwq is punted to system_wq. See put_pwq()
  210. * and pwq_unbound_release_workfn() for details. pool_workqueue
  211. * itself is also RCU protected so that the first pwq can be
  212. * determined without grabbing wq->mutex.
  213. */
  214. struct work_struct unbound_release_work;
  215. struct rcu_head rcu;
  216. } __aligned(1 << WORK_STRUCT_FLAG_BITS);
  217. /*
  218. * Structure used to wait for workqueue flush.
  219. */
  220. struct wq_flusher {
  221. struct list_head list; /* WQ: list of flushers */
  222. int flush_color; /* WQ: flush color waiting for */
  223. struct completion done; /* flush completion */
  224. };
  225. struct wq_device;
  226. /*
  227. * The externally visible workqueue. It relays the issued work items to
  228. * the appropriate worker_pool through its pool_workqueues.
  229. */
  230. struct workqueue_struct {
  231. struct list_head pwqs; /* WR: all pwqs of this wq */
  232. struct list_head list; /* PR: list of all workqueues */
  233. struct mutex mutex; /* protects this wq */
  234. int work_color; /* WQ: current work color */
  235. int flush_color; /* WQ: current flush color */
  236. atomic_t nr_pwqs_to_flush; /* flush in progress */
  237. struct wq_flusher *first_flusher; /* WQ: first flusher */
  238. struct list_head flusher_queue; /* WQ: flush waiters */
  239. struct list_head flusher_overflow; /* WQ: flush overflow list */
  240. struct list_head maydays; /* MD: pwqs requesting rescue */
  241. struct worker *rescuer; /* MD: rescue worker */
  242. int nr_drainers; /* WQ: drain in progress */
  243. int saved_max_active; /* WQ: saved pwq max_active */
  244. struct workqueue_attrs *unbound_attrs; /* PW: only for unbound wqs */
  245. struct pool_workqueue *dfl_pwq; /* PW: only for unbound wqs */
  246. #ifdef CONFIG_SYSFS
  247. struct wq_device *wq_dev; /* I: for sysfs interface */
  248. #endif
  249. #ifdef CONFIG_LOCKDEP
  250. char *lock_name;
  251. struct lock_class_key key;
  252. struct lockdep_map lockdep_map;
  253. #endif
  254. char name[WQ_NAME_LEN]; /* I: workqueue name */
  255. /*
  256. * Destruction of workqueue_struct is RCU protected to allow walking
  257. * the workqueues list without grabbing wq_pool_mutex.
  258. * This is used to dump all workqueues from sysrq.
  259. */
  260. struct rcu_head rcu;
  261. /* hot fields used during command issue, aligned to cacheline */
  262. unsigned int flags ____cacheline_aligned; /* WQ: WQ_* flags */
  263. struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
  264. struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
  265. };
  266. static struct kmem_cache *pwq_cache;
  267. static cpumask_var_t *wq_numa_possible_cpumask;
  268. /* possible CPUs of each node */
  269. static bool wq_disable_numa;
  270. module_param_named(disable_numa, wq_disable_numa, bool, 0444);
  271. /* see the comment above the definition of WQ_POWER_EFFICIENT */
  272. static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
  273. module_param_named(power_efficient, wq_power_efficient, bool, 0444);
  274. static bool wq_online; /* can kworkers be created yet? */
  275. static bool wq_numa_enabled; /* unbound NUMA affinity enabled */
  276. /* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
  277. static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
  278. static DEFINE_MUTEX(wq_pool_mutex); /* protects pools and workqueues list */
  279. static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
  280. static DEFINE_RAW_SPINLOCK(wq_mayday_lock); /* protects wq->maydays list */
  281. /* wait for manager to go away */
  282. static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
  283. static LIST_HEAD(workqueues); /* PR: list of all workqueues */
  284. static bool workqueue_freezing; /* PL: have wqs started freezing? */
  285. /* PL&A: allowable cpus for unbound wqs and work items */
  286. static cpumask_var_t wq_unbound_cpumask;
  287. /* CPU where unbound work was last round robin scheduled from this CPU */
  288. static DEFINE_PER_CPU(int, wq_rr_cpu_last);
  289. /*
  290. * Local execution of unbound work items is no longer guaranteed. The
  291. * following always forces round-robin CPU selection on unbound work items
  292. * to uncover usages which depend on it.
  293. */
  294. #ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
  295. static bool wq_debug_force_rr_cpu = true;
  296. #else
  297. static bool wq_debug_force_rr_cpu = false;
  298. #endif
  299. module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
  300. /* the per-cpu worker pools */
  301. static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
  302. static DEFINE_IDR(worker_pool_idr); /* PR: idr of all pools */
  303. /* PL: hash of all unbound pools keyed by pool->attrs */
  304. static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
  305. /* I: attributes used when instantiating standard unbound pools on demand */
  306. static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
  307. /* I: attributes used when instantiating ordered pools on demand */
  308. static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
  309. struct workqueue_struct *system_wq __read_mostly;
  310. EXPORT_SYMBOL(system_wq);
  311. struct workqueue_struct *system_highpri_wq __read_mostly;
  312. EXPORT_SYMBOL_GPL(system_highpri_wq);
  313. struct workqueue_struct *system_long_wq __read_mostly;
  314. EXPORT_SYMBOL_GPL(system_long_wq);
  315. struct workqueue_struct *system_unbound_wq __read_mostly;
  316. EXPORT_SYMBOL_GPL(system_unbound_wq);
  317. struct workqueue_struct *system_freezable_wq __read_mostly;
  318. EXPORT_SYMBOL_GPL(system_freezable_wq);
  319. struct workqueue_struct *system_power_efficient_wq __read_mostly;
  320. EXPORT_SYMBOL_GPL(system_power_efficient_wq);
  321. struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
  322. EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
  323. static int worker_thread(void *__worker);
  324. static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
  325. static void show_pwq(struct pool_workqueue *pwq);
  326. static void show_one_worker_pool(struct worker_pool *pool);
  327. #define CREATE_TRACE_POINTS
  328. #include <trace/events/workqueue.h>
  329. EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_start);
  330. EXPORT_TRACEPOINT_SYMBOL_GPL(workqueue_execute_end);
  331. #define assert_rcu_or_pool_mutex() \
  332. RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  333. !lockdep_is_held(&wq_pool_mutex), \
  334. "RCU or wq_pool_mutex should be held")
  335. #define assert_rcu_or_wq_mutex_or_pool_mutex(wq) \
  336. RCU_LOCKDEP_WARN(!rcu_read_lock_held() && \
  337. !lockdep_is_held(&wq->mutex) && \
  338. !lockdep_is_held(&wq_pool_mutex), \
  339. "RCU, wq->mutex or wq_pool_mutex should be held")
  340. #define for_each_cpu_worker_pool(pool, cpu) \
  341. for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
  342. (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
  343. (pool)++)
  344. /**
  345. * for_each_pool - iterate through all worker_pools in the system
  346. * @pool: iteration cursor
  347. * @pi: integer used for iteration
  348. *
  349. * This must be called either with wq_pool_mutex held or RCU read
  350. * locked. If the pool needs to be used beyond the locking in effect, the
  351. * caller is responsible for guaranteeing that the pool stays online.
  352. *
  353. * The if/else clause exists only for the lockdep assertion and can be
  354. * ignored.
  355. */
  356. #define for_each_pool(pool, pi) \
  357. idr_for_each_entry(&worker_pool_idr, pool, pi) \
  358. if (({ assert_rcu_or_pool_mutex(); false; })) { } \
  359. else
  360. /**
  361. * for_each_pool_worker - iterate through all workers of a worker_pool
  362. * @worker: iteration cursor
  363. * @pool: worker_pool to iterate workers of
  364. *
  365. * This must be called with wq_pool_attach_mutex.
  366. *
  367. * The if/else clause exists only for the lockdep assertion and can be
  368. * ignored.
  369. */
  370. #define for_each_pool_worker(worker, pool) \
  371. list_for_each_entry((worker), &(pool)->workers, node) \
  372. if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
  373. else
  374. /**
  375. * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
  376. * @pwq: iteration cursor
  377. * @wq: the target workqueue
  378. *
  379. * This must be called either with wq->mutex held or RCU read locked.
  380. * If the pwq needs to be used beyond the locking in effect, the caller is
  381. * responsible for guaranteeing that the pwq stays online.
  382. *
  383. * The if/else clause exists only for the lockdep assertion and can be
  384. * ignored.
  385. */
  386. #define for_each_pwq(pwq, wq) \
  387. list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node, \
  388. lockdep_is_held(&(wq->mutex)))
  389. #ifdef CONFIG_DEBUG_OBJECTS_WORK
  390. static const struct debug_obj_descr work_debug_descr;
  391. static void *work_debug_hint(void *addr)
  392. {
  393. return ((struct work_struct *) addr)->func;
  394. }
  395. static bool work_is_static_object(void *addr)
  396. {
  397. struct work_struct *work = addr;
  398. return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
  399. }
  400. /*
  401. * fixup_init is called when:
  402. * - an active object is initialized
  403. */
  404. static bool work_fixup_init(void *addr, enum debug_obj_state state)
  405. {
  406. struct work_struct *work = addr;
  407. switch (state) {
  408. case ODEBUG_STATE_ACTIVE:
  409. cancel_work_sync(work);
  410. debug_object_init(work, &work_debug_descr);
  411. return true;
  412. default:
  413. return false;
  414. }
  415. }
  416. /*
  417. * fixup_free is called when:
  418. * - an active object is freed
  419. */
  420. static bool work_fixup_free(void *addr, enum debug_obj_state state)
  421. {
  422. struct work_struct *work = addr;
  423. switch (state) {
  424. case ODEBUG_STATE_ACTIVE:
  425. cancel_work_sync(work);
  426. debug_object_free(work, &work_debug_descr);
  427. return true;
  428. default:
  429. return false;
  430. }
  431. }
  432. static const struct debug_obj_descr work_debug_descr = {
  433. .name = "work_struct",
  434. .debug_hint = work_debug_hint,
  435. .is_static_object = work_is_static_object,
  436. .fixup_init = work_fixup_init,
  437. .fixup_free = work_fixup_free,
  438. };
  439. static inline void debug_work_activate(struct work_struct *work)
  440. {
  441. debug_object_activate(work, &work_debug_descr);
  442. }
  443. static inline void debug_work_deactivate(struct work_struct *work)
  444. {
  445. debug_object_deactivate(work, &work_debug_descr);
  446. }
  447. void __init_work(struct work_struct *work, int onstack)
  448. {
  449. if (onstack)
  450. debug_object_init_on_stack(work, &work_debug_descr);
  451. else
  452. debug_object_init(work, &work_debug_descr);
  453. }
  454. EXPORT_SYMBOL_GPL(__init_work);
  455. void destroy_work_on_stack(struct work_struct *work)
  456. {
  457. debug_object_free(work, &work_debug_descr);
  458. }
  459. EXPORT_SYMBOL_GPL(destroy_work_on_stack);
  460. void destroy_delayed_work_on_stack(struct delayed_work *work)
  461. {
  462. destroy_timer_on_stack(&work->timer);
  463. debug_object_free(&work->work, &work_debug_descr);
  464. }
  465. EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
  466. #else
  467. static inline void debug_work_activate(struct work_struct *work) { }
  468. static inline void debug_work_deactivate(struct work_struct *work) { }
  469. #endif
  470. /**
  471. * worker_pool_assign_id - allocate ID and assign it to @pool
  472. * @pool: the pool pointer of interest
  473. *
  474. * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
  475. * successfully, -errno on failure.
  476. */
  477. static int worker_pool_assign_id(struct worker_pool *pool)
  478. {
  479. int ret;
  480. lockdep_assert_held(&wq_pool_mutex);
  481. ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
  482. GFP_KERNEL);
  483. if (ret >= 0) {
  484. pool->id = ret;
  485. return 0;
  486. }
  487. return ret;
  488. }
  489. /**
  490. * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
  491. * @wq: the target workqueue
  492. * @node: the node ID
  493. *
  494. * This must be called with any of wq_pool_mutex, wq->mutex or RCU
  495. * read locked.
  496. * If the pwq needs to be used beyond the locking in effect, the caller is
  497. * responsible for guaranteeing that the pwq stays online.
  498. *
  499. * Return: The unbound pool_workqueue for @node.
  500. */
  501. static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
  502. int node)
  503. {
  504. assert_rcu_or_wq_mutex_or_pool_mutex(wq);
  505. /*
  506. * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
  507. * delayed item is pending. The plan is to keep CPU -> NODE
  508. * mapping valid and stable across CPU on/offlines. Once that
  509. * happens, this workaround can be removed.
  510. */
  511. if (unlikely(node == NUMA_NO_NODE))
  512. return wq->dfl_pwq;
  513. return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
  514. }
  515. static unsigned int work_color_to_flags(int color)
  516. {
  517. return color << WORK_STRUCT_COLOR_SHIFT;
  518. }
  519. static int get_work_color(unsigned long work_data)
  520. {
  521. return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
  522. ((1 << WORK_STRUCT_COLOR_BITS) - 1);
  523. }
  524. static int work_next_color(int color)
  525. {
  526. return (color + 1) % WORK_NR_COLORS;
  527. }
  528. /*
  529. * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
  530. * contain the pointer to the queued pwq. Once execution starts, the flag
  531. * is cleared and the high bits contain OFFQ flags and pool ID.
  532. *
  533. * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
  534. * and clear_work_data() can be used to set the pwq, pool or clear
  535. * work->data. These functions should only be called while the work is
  536. * owned - ie. while the PENDING bit is set.
  537. *
  538. * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
  539. * corresponding to a work. Pool is available once the work has been
  540. * queued anywhere after initialization until it is sync canceled. pwq is
  541. * available only while the work item is queued.
  542. *
  543. * %WORK_OFFQ_CANCELING is used to mark a work item which is being
  544. * canceled. While being canceled, a work item may have its PENDING set
  545. * but stay off timer and worklist for arbitrarily long and nobody should
  546. * try to steal the PENDING bit.
  547. */
  548. static inline void set_work_data(struct work_struct *work, unsigned long data,
  549. unsigned long flags)
  550. {
  551. WARN_ON_ONCE(!work_pending(work));
  552. atomic_long_set(&work->data, data | flags | work_static(work));
  553. }
  554. static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
  555. unsigned long extra_flags)
  556. {
  557. set_work_data(work, (unsigned long)pwq,
  558. WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
  559. }
  560. static void set_work_pool_and_keep_pending(struct work_struct *work,
  561. int pool_id)
  562. {
  563. set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
  564. WORK_STRUCT_PENDING);
  565. }
  566. static void set_work_pool_and_clear_pending(struct work_struct *work,
  567. int pool_id)
  568. {
  569. /*
  570. * The following wmb is paired with the implied mb in
  571. * test_and_set_bit(PENDING) and ensures all updates to @work made
  572. * here are visible to and precede any updates by the next PENDING
  573. * owner.
  574. */
  575. smp_wmb();
  576. set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
  577. /*
  578. * The following mb guarantees that previous clear of a PENDING bit
  579. * will not be reordered with any speculative LOADS or STORES from
  580. * work->current_func, which is executed afterwards. This possible
  581. * reordering can lead to a missed execution on attempt to queue
  582. * the same @work. E.g. consider this case:
  583. *
  584. * CPU#0 CPU#1
  585. * ---------------------------- --------------------------------
  586. *
  587. * 1 STORE event_indicated
  588. * 2 queue_work_on() {
  589. * 3 test_and_set_bit(PENDING)
  590. * 4 } set_..._and_clear_pending() {
  591. * 5 set_work_data() # clear bit
  592. * 6 smp_mb()
  593. * 7 work->current_func() {
  594. * 8 LOAD event_indicated
  595. * }
  596. *
  597. * Without an explicit full barrier speculative LOAD on line 8 can
  598. * be executed before CPU#0 does STORE on line 1. If that happens,
  599. * CPU#0 observes the PENDING bit is still set and new execution of
  600. * a @work is not queued in a hope, that CPU#1 will eventually
  601. * finish the queued @work. Meanwhile CPU#1 does not see
  602. * event_indicated is set, because speculative LOAD was executed
  603. * before actual STORE.
  604. */
  605. smp_mb();
  606. }
  607. static void clear_work_data(struct work_struct *work)
  608. {
  609. smp_wmb(); /* see set_work_pool_and_clear_pending() */
  610. set_work_data(work, WORK_STRUCT_NO_POOL, 0);
  611. }
  612. static inline struct pool_workqueue *work_struct_pwq(unsigned long data)
  613. {
  614. return (struct pool_workqueue *)(data & WORK_STRUCT_WQ_DATA_MASK);
  615. }
  616. static struct pool_workqueue *get_work_pwq(struct work_struct *work)
  617. {
  618. unsigned long data = atomic_long_read(&work->data);
  619. if (data & WORK_STRUCT_PWQ)
  620. return work_struct_pwq(data);
  621. else
  622. return NULL;
  623. }
  624. /**
  625. * get_work_pool - return the worker_pool a given work was associated with
  626. * @work: the work item of interest
  627. *
  628. * Pools are created and destroyed under wq_pool_mutex, and allows read
  629. * access under RCU read lock. As such, this function should be
  630. * called under wq_pool_mutex or inside of a rcu_read_lock() region.
  631. *
  632. * All fields of the returned pool are accessible as long as the above
  633. * mentioned locking is in effect. If the returned pool needs to be used
  634. * beyond the critical section, the caller is responsible for ensuring the
  635. * returned pool is and stays online.
  636. *
  637. * Return: The worker_pool @work was last associated with. %NULL if none.
  638. */
  639. static struct worker_pool *get_work_pool(struct work_struct *work)
  640. {
  641. unsigned long data = atomic_long_read(&work->data);
  642. int pool_id;
  643. assert_rcu_or_pool_mutex();
  644. if (data & WORK_STRUCT_PWQ)
  645. return work_struct_pwq(data)->pool;
  646. pool_id = data >> WORK_OFFQ_POOL_SHIFT;
  647. if (pool_id == WORK_OFFQ_POOL_NONE)
  648. return NULL;
  649. return idr_find(&worker_pool_idr, pool_id);
  650. }
  651. /**
  652. * get_work_pool_id - return the worker pool ID a given work is associated with
  653. * @work: the work item of interest
  654. *
  655. * Return: The worker_pool ID @work was last associated with.
  656. * %WORK_OFFQ_POOL_NONE if none.
  657. */
  658. static int get_work_pool_id(struct work_struct *work)
  659. {
  660. unsigned long data = atomic_long_read(&work->data);
  661. if (data & WORK_STRUCT_PWQ)
  662. return work_struct_pwq(data)->pool->id;
  663. return data >> WORK_OFFQ_POOL_SHIFT;
  664. }
  665. static void mark_work_canceling(struct work_struct *work)
  666. {
  667. unsigned long pool_id = get_work_pool_id(work);
  668. pool_id <<= WORK_OFFQ_POOL_SHIFT;
  669. set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
  670. }
  671. static bool work_is_canceling(struct work_struct *work)
  672. {
  673. unsigned long data = atomic_long_read(&work->data);
  674. return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
  675. }
  676. /*
  677. * Policy functions. These define the policies on how the global worker
  678. * pools are managed. Unless noted otherwise, these functions assume that
  679. * they're being called with pool->lock held.
  680. */
  681. static bool __need_more_worker(struct worker_pool *pool)
  682. {
  683. return !pool->nr_running;
  684. }
  685. /*
  686. * Need to wake up a worker? Called from anything but currently
  687. * running workers.
  688. *
  689. * Note that, because unbound workers never contribute to nr_running, this
  690. * function will always return %true for unbound pools as long as the
  691. * worklist isn't empty.
  692. */
  693. static bool need_more_worker(struct worker_pool *pool)
  694. {
  695. return !list_empty(&pool->worklist) && __need_more_worker(pool);
  696. }
  697. /* Can I start working? Called from busy but !running workers. */
  698. static bool may_start_working(struct worker_pool *pool)
  699. {
  700. return pool->nr_idle;
  701. }
  702. /* Do I need to keep working? Called from currently running workers. */
  703. static bool keep_working(struct worker_pool *pool)
  704. {
  705. return !list_empty(&pool->worklist) && (pool->nr_running <= 1);
  706. }
  707. /* Do we need a new worker? Called from manager. */
  708. static bool need_to_create_worker(struct worker_pool *pool)
  709. {
  710. return need_more_worker(pool) && !may_start_working(pool);
  711. }
  712. /* Do we have too many workers and should some go away? */
  713. static bool too_many_workers(struct worker_pool *pool)
  714. {
  715. bool managing = pool->flags & POOL_MANAGER_ACTIVE;
  716. int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
  717. int nr_busy = pool->nr_workers - nr_idle;
  718. return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
  719. }
  720. /*
  721. * Wake up functions.
  722. */
  723. /* Return the first idle worker. Called with pool->lock held. */
  724. static struct worker *first_idle_worker(struct worker_pool *pool)
  725. {
  726. if (unlikely(list_empty(&pool->idle_list)))
  727. return NULL;
  728. return list_first_entry(&pool->idle_list, struct worker, entry);
  729. }
  730. /**
  731. * wake_up_worker - wake up an idle worker
  732. * @pool: worker pool to wake worker from
  733. *
  734. * Wake up the first idle worker of @pool.
  735. *
  736. * CONTEXT:
  737. * raw_spin_lock_irq(pool->lock).
  738. */
  739. static void wake_up_worker(struct worker_pool *pool)
  740. {
  741. struct worker *worker = first_idle_worker(pool);
  742. if (likely(worker))
  743. wake_up_process(worker->task);
  744. }
  745. /**
  746. * wq_worker_running - a worker is running again
  747. * @task: task waking up
  748. *
  749. * This function is called when a worker returns from schedule()
  750. */
  751. void wq_worker_running(struct task_struct *task)
  752. {
  753. struct worker *worker = kthread_data(task);
  754. if (!worker->sleeping)
  755. return;
  756. /*
  757. * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check
  758. * and the nr_running increment below, we may ruin the nr_running reset
  759. * and leave with an unexpected pool->nr_running == 1 on the newly unbound
  760. * pool. Protect against such race.
  761. */
  762. preempt_disable();
  763. if (!(worker->flags & WORKER_NOT_RUNNING))
  764. worker->pool->nr_running++;
  765. preempt_enable();
  766. worker->sleeping = 0;
  767. }
  768. /**
  769. * wq_worker_sleeping - a worker is going to sleep
  770. * @task: task going to sleep
  771. *
  772. * This function is called from schedule() when a busy worker is
  773. * going to sleep.
  774. */
  775. void wq_worker_sleeping(struct task_struct *task)
  776. {
  777. struct worker *worker = kthread_data(task);
  778. struct worker_pool *pool;
  779. /*
  780. * Rescuers, which may not have all the fields set up like normal
  781. * workers, also reach here, let's not access anything before
  782. * checking NOT_RUNNING.
  783. */
  784. if (worker->flags & WORKER_NOT_RUNNING)
  785. return;
  786. pool = worker->pool;
  787. /* Return if preempted before wq_worker_running() was reached */
  788. if (worker->sleeping)
  789. return;
  790. worker->sleeping = 1;
  791. raw_spin_lock_irq(&pool->lock);
  792. /*
  793. * Recheck in case unbind_workers() preempted us. We don't
  794. * want to decrement nr_running after the worker is unbound
  795. * and nr_running has been reset.
  796. */
  797. if (worker->flags & WORKER_NOT_RUNNING) {
  798. raw_spin_unlock_irq(&pool->lock);
  799. return;
  800. }
  801. pool->nr_running--;
  802. if (need_more_worker(pool))
  803. wake_up_worker(pool);
  804. raw_spin_unlock_irq(&pool->lock);
  805. }
  806. /**
  807. * wq_worker_last_func - retrieve worker's last work function
  808. * @task: Task to retrieve last work function of.
  809. *
  810. * Determine the last function a worker executed. This is called from
  811. * the scheduler to get a worker's last known identity.
  812. *
  813. * CONTEXT:
  814. * raw_spin_lock_irq(rq->lock)
  815. *
  816. * This function is called during schedule() when a kworker is going
  817. * to sleep. It's used by psi to identify aggregation workers during
  818. * dequeuing, to allow periodic aggregation to shut-off when that
  819. * worker is the last task in the system or cgroup to go to sleep.
  820. *
  821. * As this function doesn't involve any workqueue-related locking, it
  822. * only returns stable values when called from inside the scheduler's
  823. * queuing and dequeuing paths, when @task, which must be a kworker,
  824. * is guaranteed to not be processing any works.
  825. *
  826. * Return:
  827. * The last work function %current executed as a worker, NULL if it
  828. * hasn't executed any work yet.
  829. */
  830. work_func_t wq_worker_last_func(struct task_struct *task)
  831. {
  832. struct worker *worker = kthread_data(task);
  833. return worker->last_func;
  834. }
  835. /**
  836. * worker_set_flags - set worker flags and adjust nr_running accordingly
  837. * @worker: self
  838. * @flags: flags to set
  839. *
  840. * Set @flags in @worker->flags and adjust nr_running accordingly.
  841. *
  842. * CONTEXT:
  843. * raw_spin_lock_irq(pool->lock)
  844. */
  845. static inline void worker_set_flags(struct worker *worker, unsigned int flags)
  846. {
  847. struct worker_pool *pool = worker->pool;
  848. WARN_ON_ONCE(worker->task != current);
  849. /* If transitioning into NOT_RUNNING, adjust nr_running. */
  850. if ((flags & WORKER_NOT_RUNNING) &&
  851. !(worker->flags & WORKER_NOT_RUNNING)) {
  852. pool->nr_running--;
  853. }
  854. worker->flags |= flags;
  855. }
  856. /**
  857. * worker_clr_flags - clear worker flags and adjust nr_running accordingly
  858. * @worker: self
  859. * @flags: flags to clear
  860. *
  861. * Clear @flags in @worker->flags and adjust nr_running accordingly.
  862. *
  863. * CONTEXT:
  864. * raw_spin_lock_irq(pool->lock)
  865. */
  866. static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
  867. {
  868. struct worker_pool *pool = worker->pool;
  869. unsigned int oflags = worker->flags;
  870. WARN_ON_ONCE(worker->task != current);
  871. worker->flags &= ~flags;
  872. /*
  873. * If transitioning out of NOT_RUNNING, increment nr_running. Note
  874. * that the nested NOT_RUNNING is not a noop. NOT_RUNNING is mask
  875. * of multiple flags, not a single flag.
  876. */
  877. if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
  878. if (!(worker->flags & WORKER_NOT_RUNNING))
  879. pool->nr_running++;
  880. }
  881. /**
  882. * find_worker_executing_work - find worker which is executing a work
  883. * @pool: pool of interest
  884. * @work: work to find worker for
  885. *
  886. * Find a worker which is executing @work on @pool by searching
  887. * @pool->busy_hash which is keyed by the address of @work. For a worker
  888. * to match, its current execution should match the address of @work and
  889. * its work function. This is to avoid unwanted dependency between
  890. * unrelated work executions through a work item being recycled while still
  891. * being executed.
  892. *
  893. * This is a bit tricky. A work item may be freed once its execution
  894. * starts and nothing prevents the freed area from being recycled for
  895. * another work item. If the same work item address ends up being reused
  896. * before the original execution finishes, workqueue will identify the
  897. * recycled work item as currently executing and make it wait until the
  898. * current execution finishes, introducing an unwanted dependency.
  899. *
  900. * This function checks the work item address and work function to avoid
  901. * false positives. Note that this isn't complete as one may construct a
  902. * work function which can introduce dependency onto itself through a
  903. * recycled work item. Well, if somebody wants to shoot oneself in the
  904. * foot that badly, there's only so much we can do, and if such deadlock
  905. * actually occurs, it should be easy to locate the culprit work function.
  906. *
  907. * CONTEXT:
  908. * raw_spin_lock_irq(pool->lock).
  909. *
  910. * Return:
  911. * Pointer to worker which is executing @work if found, %NULL
  912. * otherwise.
  913. */
  914. static struct worker *find_worker_executing_work(struct worker_pool *pool,
  915. struct work_struct *work)
  916. {
  917. struct worker *worker;
  918. hash_for_each_possible(pool->busy_hash, worker, hentry,
  919. (unsigned long)work)
  920. if (worker->current_work == work &&
  921. worker->current_func == work->func)
  922. return worker;
  923. return NULL;
  924. }
  925. /**
  926. * move_linked_works - move linked works to a list
  927. * @work: start of series of works to be scheduled
  928. * @head: target list to append @work to
  929. * @nextp: out parameter for nested worklist walking
  930. *
  931. * Schedule linked works starting from @work to @head. Work series to
  932. * be scheduled starts at @work and includes any consecutive work with
  933. * WORK_STRUCT_LINKED set in its predecessor.
  934. *
  935. * If @nextp is not NULL, it's updated to point to the next work of
  936. * the last scheduled work. This allows move_linked_works() to be
  937. * nested inside outer list_for_each_entry_safe().
  938. *
  939. * CONTEXT:
  940. * raw_spin_lock_irq(pool->lock).
  941. */
  942. static void move_linked_works(struct work_struct *work, struct list_head *head,
  943. struct work_struct **nextp)
  944. {
  945. struct work_struct *n;
  946. /*
  947. * Linked worklist will always end before the end of the list,
  948. * use NULL for list head.
  949. */
  950. list_for_each_entry_safe_from(work, n, NULL, entry) {
  951. list_move_tail(&work->entry, head);
  952. if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
  953. break;
  954. }
  955. /*
  956. * If we're already inside safe list traversal and have moved
  957. * multiple works to the scheduled queue, the next position
  958. * needs to be updated.
  959. */
  960. if (nextp)
  961. *nextp = n;
  962. }
  963. /**
  964. * get_pwq - get an extra reference on the specified pool_workqueue
  965. * @pwq: pool_workqueue to get
  966. *
  967. * Obtain an extra reference on @pwq. The caller should guarantee that
  968. * @pwq has positive refcnt and be holding the matching pool->lock.
  969. */
  970. static void get_pwq(struct pool_workqueue *pwq)
  971. {
  972. lockdep_assert_held(&pwq->pool->lock);
  973. WARN_ON_ONCE(pwq->refcnt <= 0);
  974. pwq->refcnt++;
  975. }
  976. /**
  977. * put_pwq - put a pool_workqueue reference
  978. * @pwq: pool_workqueue to put
  979. *
  980. * Drop a reference of @pwq. If its refcnt reaches zero, schedule its
  981. * destruction. The caller should be holding the matching pool->lock.
  982. */
  983. static void put_pwq(struct pool_workqueue *pwq)
  984. {
  985. lockdep_assert_held(&pwq->pool->lock);
  986. if (likely(--pwq->refcnt))
  987. return;
  988. if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
  989. return;
  990. /*
  991. * @pwq can't be released under pool->lock, bounce to
  992. * pwq_unbound_release_workfn(). This never recurses on the same
  993. * pool->lock as this path is taken only for unbound workqueues and
  994. * the release work item is scheduled on a per-cpu workqueue. To
  995. * avoid lockdep warning, unbound pool->locks are given lockdep
  996. * subclass of 1 in get_unbound_pool().
  997. */
  998. schedule_work(&pwq->unbound_release_work);
  999. }
  1000. /**
  1001. * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
  1002. * @pwq: pool_workqueue to put (can be %NULL)
  1003. *
  1004. * put_pwq() with locking. This function also allows %NULL @pwq.
  1005. */
  1006. static void put_pwq_unlocked(struct pool_workqueue *pwq)
  1007. {
  1008. if (pwq) {
  1009. /*
  1010. * As both pwqs and pools are RCU protected, the
  1011. * following lock operations are safe.
  1012. */
  1013. raw_spin_lock_irq(&pwq->pool->lock);
  1014. put_pwq(pwq);
  1015. raw_spin_unlock_irq(&pwq->pool->lock);
  1016. }
  1017. }
  1018. static void pwq_activate_inactive_work(struct work_struct *work)
  1019. {
  1020. struct pool_workqueue *pwq = get_work_pwq(work);
  1021. trace_workqueue_activate_work(work);
  1022. if (list_empty(&pwq->pool->worklist))
  1023. pwq->pool->watchdog_ts = jiffies;
  1024. move_linked_works(work, &pwq->pool->worklist, NULL);
  1025. __clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
  1026. pwq->nr_active++;
  1027. }
  1028. static void pwq_activate_first_inactive(struct pool_workqueue *pwq)
  1029. {
  1030. struct work_struct *work = list_first_entry(&pwq->inactive_works,
  1031. struct work_struct, entry);
  1032. pwq_activate_inactive_work(work);
  1033. }
  1034. /**
  1035. * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
  1036. * @pwq: pwq of interest
  1037. * @work_data: work_data of work which left the queue
  1038. *
  1039. * A work either has completed or is removed from pending queue,
  1040. * decrement nr_in_flight of its pwq and handle workqueue flushing.
  1041. *
  1042. * CONTEXT:
  1043. * raw_spin_lock_irq(pool->lock).
  1044. */
  1045. static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
  1046. {
  1047. int color = get_work_color(work_data);
  1048. if (!(work_data & WORK_STRUCT_INACTIVE)) {
  1049. pwq->nr_active--;
  1050. if (!list_empty(&pwq->inactive_works)) {
  1051. /* one down, submit an inactive one */
  1052. if (pwq->nr_active < pwq->max_active)
  1053. pwq_activate_first_inactive(pwq);
  1054. }
  1055. }
  1056. pwq->nr_in_flight[color]--;
  1057. /* is flush in progress and are we at the flushing tip? */
  1058. if (likely(pwq->flush_color != color))
  1059. goto out_put;
  1060. /* are there still in-flight works? */
  1061. if (pwq->nr_in_flight[color])
  1062. goto out_put;
  1063. /* this pwq is done, clear flush_color */
  1064. pwq->flush_color = -1;
  1065. /*
  1066. * If this was the last pwq, wake up the first flusher. It
  1067. * will handle the rest.
  1068. */
  1069. if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
  1070. complete(&pwq->wq->first_flusher->done);
  1071. out_put:
  1072. put_pwq(pwq);
  1073. }
  1074. /**
  1075. * try_to_grab_pending - steal work item from worklist and disable irq
  1076. * @work: work item to steal
  1077. * @is_dwork: @work is a delayed_work
  1078. * @flags: place to store irq state
  1079. *
  1080. * Try to grab PENDING bit of @work. This function can handle @work in any
  1081. * stable state - idle, on timer or on worklist.
  1082. *
  1083. * Return:
  1084. *
  1085. * ======== ================================================================
  1086. * 1 if @work was pending and we successfully stole PENDING
  1087. * 0 if @work was idle and we claimed PENDING
  1088. * -EAGAIN if PENDING couldn't be grabbed at the moment, safe to busy-retry
  1089. * -ENOENT if someone else is canceling @work, this state may persist
  1090. * for arbitrarily long
  1091. * ======== ================================================================
  1092. *
  1093. * Note:
  1094. * On >= 0 return, the caller owns @work's PENDING bit. To avoid getting
  1095. * interrupted while holding PENDING and @work off queue, irq must be
  1096. * disabled on entry. This, combined with delayed_work->timer being
  1097. * irqsafe, ensures that we return -EAGAIN for finite short period of time.
  1098. *
  1099. * On successful return, >= 0, irq is disabled and the caller is
  1100. * responsible for releasing it using local_irq_restore(*@flags).
  1101. *
  1102. * This function is safe to call from any context including IRQ handler.
  1103. */
  1104. static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
  1105. unsigned long *flags)
  1106. {
  1107. struct worker_pool *pool;
  1108. struct pool_workqueue *pwq;
  1109. local_irq_save(*flags);
  1110. /* try to steal the timer if it exists */
  1111. if (is_dwork) {
  1112. struct delayed_work *dwork = to_delayed_work(work);
  1113. /*
  1114. * dwork->timer is irqsafe. If del_timer() fails, it's
  1115. * guaranteed that the timer is not queued anywhere and not
  1116. * running on the local CPU.
  1117. */
  1118. if (likely(del_timer(&dwork->timer)))
  1119. return 1;
  1120. }
  1121. /* try to claim PENDING the normal way */
  1122. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
  1123. return 0;
  1124. rcu_read_lock();
  1125. /*
  1126. * The queueing is in progress, or it is already queued. Try to
  1127. * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
  1128. */
  1129. pool = get_work_pool(work);
  1130. if (!pool)
  1131. goto fail;
  1132. raw_spin_lock(&pool->lock);
  1133. /*
  1134. * work->data is guaranteed to point to pwq only while the work
  1135. * item is queued on pwq->wq, and both updating work->data to point
  1136. * to pwq on queueing and to pool on dequeueing are done under
  1137. * pwq->pool->lock. This in turn guarantees that, if work->data
  1138. * points to pwq which is associated with a locked pool, the work
  1139. * item is currently queued on that pool.
  1140. */
  1141. pwq = get_work_pwq(work);
  1142. if (pwq && pwq->pool == pool) {
  1143. debug_work_deactivate(work);
  1144. /*
  1145. * A cancelable inactive work item must be in the
  1146. * pwq->inactive_works since a queued barrier can't be
  1147. * canceled (see the comments in insert_wq_barrier()).
  1148. *
  1149. * An inactive work item cannot be grabbed directly because
  1150. * it might have linked barrier work items which, if left
  1151. * on the inactive_works list, will confuse pwq->nr_active
  1152. * management later on and cause stall. Make sure the work
  1153. * item is activated before grabbing.
  1154. */
  1155. if (*work_data_bits(work) & WORK_STRUCT_INACTIVE)
  1156. pwq_activate_inactive_work(work);
  1157. list_del_init(&work->entry);
  1158. pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
  1159. /* work->data points to pwq iff queued, point to pool */
  1160. set_work_pool_and_keep_pending(work, pool->id);
  1161. raw_spin_unlock(&pool->lock);
  1162. rcu_read_unlock();
  1163. return 1;
  1164. }
  1165. raw_spin_unlock(&pool->lock);
  1166. fail:
  1167. rcu_read_unlock();
  1168. local_irq_restore(*flags);
  1169. if (work_is_canceling(work))
  1170. return -ENOENT;
  1171. cpu_relax();
  1172. return -EAGAIN;
  1173. }
  1174. /**
  1175. * insert_work - insert a work into a pool
  1176. * @pwq: pwq @work belongs to
  1177. * @work: work to insert
  1178. * @head: insertion point
  1179. * @extra_flags: extra WORK_STRUCT_* flags to set
  1180. *
  1181. * Insert @work which belongs to @pwq after @head. @extra_flags is or'd to
  1182. * work_struct flags.
  1183. *
  1184. * CONTEXT:
  1185. * raw_spin_lock_irq(pool->lock).
  1186. */
  1187. static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
  1188. struct list_head *head, unsigned int extra_flags)
  1189. {
  1190. struct worker_pool *pool = pwq->pool;
  1191. /* record the work call stack in order to print it in KASAN reports */
  1192. kasan_record_aux_stack_noalloc(work);
  1193. /* we own @work, set data and link */
  1194. set_work_pwq(work, pwq, extra_flags);
  1195. list_add_tail(&work->entry, head);
  1196. get_pwq(pwq);
  1197. if (__need_more_worker(pool))
  1198. wake_up_worker(pool);
  1199. }
  1200. /*
  1201. * Test whether @work is being queued from another work executing on the
  1202. * same workqueue.
  1203. */
  1204. static bool is_chained_work(struct workqueue_struct *wq)
  1205. {
  1206. struct worker *worker;
  1207. worker = current_wq_worker();
  1208. /*
  1209. * Return %true iff I'm a worker executing a work item on @wq. If
  1210. * I'm @worker, it's safe to dereference it without locking.
  1211. */
  1212. return worker && worker->current_pwq->wq == wq;
  1213. }
  1214. /*
  1215. * When queueing an unbound work item to a wq, prefer local CPU if allowed
  1216. * by wq_unbound_cpumask. Otherwise, round robin among the allowed ones to
  1217. * avoid perturbing sensitive tasks.
  1218. */
  1219. static int wq_select_unbound_cpu(int cpu)
  1220. {
  1221. static bool printed_dbg_warning;
  1222. int new_cpu;
  1223. if (likely(!wq_debug_force_rr_cpu)) {
  1224. if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
  1225. return cpu;
  1226. } else if (!printed_dbg_warning) {
  1227. pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n");
  1228. printed_dbg_warning = true;
  1229. }
  1230. if (cpumask_empty(wq_unbound_cpumask))
  1231. return cpu;
  1232. new_cpu = __this_cpu_read(wq_rr_cpu_last);
  1233. new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
  1234. if (unlikely(new_cpu >= nr_cpu_ids)) {
  1235. new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
  1236. if (unlikely(new_cpu >= nr_cpu_ids))
  1237. return cpu;
  1238. }
  1239. __this_cpu_write(wq_rr_cpu_last, new_cpu);
  1240. return new_cpu;
  1241. }
  1242. static void __queue_work(int cpu, struct workqueue_struct *wq,
  1243. struct work_struct *work)
  1244. {
  1245. struct pool_workqueue *pwq;
  1246. struct worker_pool *last_pool;
  1247. struct list_head *worklist;
  1248. unsigned int work_flags;
  1249. unsigned int req_cpu = cpu;
  1250. /*
  1251. * While a work item is PENDING && off queue, a task trying to
  1252. * steal the PENDING will busy-loop waiting for it to either get
  1253. * queued or lose PENDING. Grabbing PENDING and queueing should
  1254. * happen with IRQ disabled.
  1255. */
  1256. lockdep_assert_irqs_disabled();
  1257. /* if draining, only works from the same workqueue are allowed */
  1258. if (unlikely(wq->flags & __WQ_DRAINING) &&
  1259. WARN_ON_ONCE(!is_chained_work(wq)))
  1260. return;
  1261. rcu_read_lock();
  1262. retry:
  1263. /* pwq which will be used unless @work is executing elsewhere */
  1264. if (wq->flags & WQ_UNBOUND) {
  1265. if (req_cpu == WORK_CPU_UNBOUND)
  1266. cpu = wq_select_unbound_cpu(raw_smp_processor_id());
  1267. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  1268. } else {
  1269. if (req_cpu == WORK_CPU_UNBOUND)
  1270. cpu = raw_smp_processor_id();
  1271. pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
  1272. }
  1273. /*
  1274. * If @work was previously on a different pool, it might still be
  1275. * running there, in which case the work needs to be queued on that
  1276. * pool to guarantee non-reentrancy.
  1277. */
  1278. last_pool = get_work_pool(work);
  1279. if (last_pool && last_pool != pwq->pool) {
  1280. struct worker *worker;
  1281. raw_spin_lock(&last_pool->lock);
  1282. worker = find_worker_executing_work(last_pool, work);
  1283. if (worker && worker->current_pwq->wq == wq) {
  1284. pwq = worker->current_pwq;
  1285. } else {
  1286. /* meh... not running there, queue here */
  1287. raw_spin_unlock(&last_pool->lock);
  1288. raw_spin_lock(&pwq->pool->lock);
  1289. }
  1290. } else {
  1291. raw_spin_lock(&pwq->pool->lock);
  1292. }
  1293. /*
  1294. * pwq is determined and locked. For unbound pools, we could have
  1295. * raced with pwq release and it could already be dead. If its
  1296. * refcnt is zero, repeat pwq selection. Note that pwqs never die
  1297. * without another pwq replacing it in the numa_pwq_tbl or while
  1298. * work items are executing on it, so the retrying is guaranteed to
  1299. * make forward-progress.
  1300. */
  1301. if (unlikely(!pwq->refcnt)) {
  1302. if (wq->flags & WQ_UNBOUND) {
  1303. raw_spin_unlock(&pwq->pool->lock);
  1304. cpu_relax();
  1305. goto retry;
  1306. }
  1307. /* oops */
  1308. WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
  1309. wq->name, cpu);
  1310. }
  1311. /* pwq determined, queue */
  1312. trace_workqueue_queue_work(req_cpu, pwq, work);
  1313. if (WARN_ON(!list_empty(&work->entry)))
  1314. goto out;
  1315. pwq->nr_in_flight[pwq->work_color]++;
  1316. work_flags = work_color_to_flags(pwq->work_color);
  1317. if (likely(pwq->nr_active < pwq->max_active)) {
  1318. trace_workqueue_activate_work(work);
  1319. pwq->nr_active++;
  1320. worklist = &pwq->pool->worklist;
  1321. if (list_empty(worklist))
  1322. pwq->pool->watchdog_ts = jiffies;
  1323. } else {
  1324. work_flags |= WORK_STRUCT_INACTIVE;
  1325. worklist = &pwq->inactive_works;
  1326. }
  1327. debug_work_activate(work);
  1328. insert_work(pwq, work, worklist, work_flags);
  1329. out:
  1330. raw_spin_unlock(&pwq->pool->lock);
  1331. rcu_read_unlock();
  1332. }
  1333. /**
  1334. * queue_work_on - queue work on specific cpu
  1335. * @cpu: CPU number to execute work on
  1336. * @wq: workqueue to use
  1337. * @work: work to queue
  1338. *
  1339. * We queue the work to a specific CPU, the caller must ensure it
  1340. * can't go away. Callers that fail to ensure that the specified
  1341. * CPU cannot go away will execute on a randomly chosen CPU.
  1342. *
  1343. * Return: %false if @work was already on a queue, %true otherwise.
  1344. */
  1345. bool queue_work_on(int cpu, struct workqueue_struct *wq,
  1346. struct work_struct *work)
  1347. {
  1348. bool ret = false;
  1349. unsigned long flags;
  1350. local_irq_save(flags);
  1351. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1352. __queue_work(cpu, wq, work);
  1353. ret = true;
  1354. }
  1355. local_irq_restore(flags);
  1356. return ret;
  1357. }
  1358. EXPORT_SYMBOL(queue_work_on);
  1359. /**
  1360. * workqueue_select_cpu_near - Select a CPU based on NUMA node
  1361. * @node: NUMA node ID that we want to select a CPU from
  1362. *
  1363. * This function will attempt to find a "random" cpu available on a given
  1364. * node. If there are no CPUs available on the given node it will return
  1365. * WORK_CPU_UNBOUND indicating that we should just schedule to any
  1366. * available CPU if we need to schedule this work.
  1367. */
  1368. static int workqueue_select_cpu_near(int node)
  1369. {
  1370. int cpu;
  1371. /* No point in doing this if NUMA isn't enabled for workqueues */
  1372. if (!wq_numa_enabled)
  1373. return WORK_CPU_UNBOUND;
  1374. /* Delay binding to CPU if node is not valid or online */
  1375. if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
  1376. return WORK_CPU_UNBOUND;
  1377. /* Use local node/cpu if we are already there */
  1378. cpu = raw_smp_processor_id();
  1379. if (node == cpu_to_node(cpu))
  1380. return cpu;
  1381. /* Use "random" otherwise know as "first" online CPU of node */
  1382. cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
  1383. /* If CPU is valid return that, otherwise just defer */
  1384. return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
  1385. }
  1386. /**
  1387. * queue_work_node - queue work on a "random" cpu for a given NUMA node
  1388. * @node: NUMA node that we are targeting the work for
  1389. * @wq: workqueue to use
  1390. * @work: work to queue
  1391. *
  1392. * We queue the work to a "random" CPU within a given NUMA node. The basic
  1393. * idea here is to provide a way to somehow associate work with a given
  1394. * NUMA node.
  1395. *
  1396. * This function will only make a best effort attempt at getting this onto
  1397. * the right NUMA node. If no node is requested or the requested node is
  1398. * offline then we just fall back to standard queue_work behavior.
  1399. *
  1400. * Currently the "random" CPU ends up being the first available CPU in the
  1401. * intersection of cpu_online_mask and the cpumask of the node, unless we
  1402. * are running on the node. In that case we just use the current CPU.
  1403. *
  1404. * Return: %false if @work was already on a queue, %true otherwise.
  1405. */
  1406. bool queue_work_node(int node, struct workqueue_struct *wq,
  1407. struct work_struct *work)
  1408. {
  1409. unsigned long flags;
  1410. bool ret = false;
  1411. /*
  1412. * This current implementation is specific to unbound workqueues.
  1413. * Specifically we only return the first available CPU for a given
  1414. * node instead of cycling through individual CPUs within the node.
  1415. *
  1416. * If this is used with a per-cpu workqueue then the logic in
  1417. * workqueue_select_cpu_near would need to be updated to allow for
  1418. * some round robin type logic.
  1419. */
  1420. WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
  1421. local_irq_save(flags);
  1422. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1423. int cpu = workqueue_select_cpu_near(node);
  1424. __queue_work(cpu, wq, work);
  1425. ret = true;
  1426. }
  1427. local_irq_restore(flags);
  1428. return ret;
  1429. }
  1430. EXPORT_SYMBOL_GPL(queue_work_node);
  1431. void delayed_work_timer_fn(struct timer_list *t)
  1432. {
  1433. struct delayed_work *dwork = from_timer(dwork, t, timer);
  1434. /* should have been called from irqsafe timer with irq already off */
  1435. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  1436. }
  1437. EXPORT_SYMBOL(delayed_work_timer_fn);
  1438. static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
  1439. struct delayed_work *dwork, unsigned long delay)
  1440. {
  1441. struct timer_list *timer = &dwork->timer;
  1442. struct work_struct *work = &dwork->work;
  1443. WARN_ON_ONCE(!wq);
  1444. WARN_ON_ONCE(timer->function != delayed_work_timer_fn);
  1445. WARN_ON_ONCE(timer_pending(timer));
  1446. WARN_ON_ONCE(!list_empty(&work->entry));
  1447. /*
  1448. * If @delay is 0, queue @dwork->work immediately. This is for
  1449. * both optimization and correctness. The earliest @timer can
  1450. * expire is on the closest next tick and delayed_work users depend
  1451. * on that there's no such delay when @delay is 0.
  1452. */
  1453. if (!delay) {
  1454. __queue_work(cpu, wq, &dwork->work);
  1455. return;
  1456. }
  1457. dwork->wq = wq;
  1458. dwork->cpu = cpu;
  1459. timer->expires = jiffies + delay;
  1460. if (unlikely(cpu != WORK_CPU_UNBOUND))
  1461. add_timer_on(timer, cpu);
  1462. else
  1463. add_timer(timer);
  1464. }
  1465. /**
  1466. * queue_delayed_work_on - queue work on specific CPU after delay
  1467. * @cpu: CPU number to execute work on
  1468. * @wq: workqueue to use
  1469. * @dwork: work to queue
  1470. * @delay: number of jiffies to wait before queueing
  1471. *
  1472. * Return: %false if @work was already on a queue, %true otherwise. If
  1473. * @delay is zero and @dwork is idle, it will be scheduled for immediate
  1474. * execution.
  1475. */
  1476. bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
  1477. struct delayed_work *dwork, unsigned long delay)
  1478. {
  1479. struct work_struct *work = &dwork->work;
  1480. bool ret = false;
  1481. unsigned long flags;
  1482. /* read the comment in __queue_work() */
  1483. local_irq_save(flags);
  1484. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1485. __queue_delayed_work(cpu, wq, dwork, delay);
  1486. ret = true;
  1487. }
  1488. local_irq_restore(flags);
  1489. return ret;
  1490. }
  1491. EXPORT_SYMBOL(queue_delayed_work_on);
  1492. /**
  1493. * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
  1494. * @cpu: CPU number to execute work on
  1495. * @wq: workqueue to use
  1496. * @dwork: work to queue
  1497. * @delay: number of jiffies to wait before queueing
  1498. *
  1499. * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
  1500. * modify @dwork's timer so that it expires after @delay. If @delay is
  1501. * zero, @work is guaranteed to be scheduled immediately regardless of its
  1502. * current state.
  1503. *
  1504. * Return: %false if @dwork was idle and queued, %true if @dwork was
  1505. * pending and its timer was modified.
  1506. *
  1507. * This function is safe to call from any context including IRQ handler.
  1508. * See try_to_grab_pending() for details.
  1509. */
  1510. bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
  1511. struct delayed_work *dwork, unsigned long delay)
  1512. {
  1513. unsigned long flags;
  1514. int ret;
  1515. do {
  1516. ret = try_to_grab_pending(&dwork->work, true, &flags);
  1517. } while (unlikely(ret == -EAGAIN));
  1518. if (likely(ret >= 0)) {
  1519. __queue_delayed_work(cpu, wq, dwork, delay);
  1520. local_irq_restore(flags);
  1521. }
  1522. /* -ENOENT from try_to_grab_pending() becomes %true */
  1523. return ret;
  1524. }
  1525. EXPORT_SYMBOL_GPL(mod_delayed_work_on);
  1526. static void rcu_work_rcufn(struct rcu_head *rcu)
  1527. {
  1528. struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
  1529. /* read the comment in __queue_work() */
  1530. local_irq_disable();
  1531. __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
  1532. local_irq_enable();
  1533. }
  1534. /**
  1535. * queue_rcu_work - queue work after a RCU grace period
  1536. * @wq: workqueue to use
  1537. * @rwork: work to queue
  1538. *
  1539. * Return: %false if @rwork was already pending, %true otherwise. Note
  1540. * that a full RCU grace period is guaranteed only after a %true return.
  1541. * While @rwork is guaranteed to be executed after a %false return, the
  1542. * execution may happen before a full RCU grace period has passed.
  1543. */
  1544. bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
  1545. {
  1546. struct work_struct *work = &rwork->work;
  1547. if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
  1548. rwork->wq = wq;
  1549. call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);
  1550. return true;
  1551. }
  1552. return false;
  1553. }
  1554. EXPORT_SYMBOL(queue_rcu_work);
  1555. /**
  1556. * worker_enter_idle - enter idle state
  1557. * @worker: worker which is entering idle state
  1558. *
  1559. * @worker is entering idle state. Update stats and idle timer if
  1560. * necessary.
  1561. *
  1562. * LOCKING:
  1563. * raw_spin_lock_irq(pool->lock).
  1564. */
  1565. static void worker_enter_idle(struct worker *worker)
  1566. {
  1567. struct worker_pool *pool = worker->pool;
  1568. if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
  1569. WARN_ON_ONCE(!list_empty(&worker->entry) &&
  1570. (worker->hentry.next || worker->hentry.pprev)))
  1571. return;
  1572. /* can't use worker_set_flags(), also called from create_worker() */
  1573. worker->flags |= WORKER_IDLE;
  1574. pool->nr_idle++;
  1575. worker->last_active = jiffies;
  1576. /* idle_list is LIFO */
  1577. list_add(&worker->entry, &pool->idle_list);
  1578. if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
  1579. mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
  1580. /* Sanity check nr_running. */
  1581. WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);
  1582. }
  1583. /**
  1584. * worker_leave_idle - leave idle state
  1585. * @worker: worker which is leaving idle state
  1586. *
  1587. * @worker is leaving idle state. Update stats.
  1588. *
  1589. * LOCKING:
  1590. * raw_spin_lock_irq(pool->lock).
  1591. */
  1592. static void worker_leave_idle(struct worker *worker)
  1593. {
  1594. struct worker_pool *pool = worker->pool;
  1595. if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
  1596. return;
  1597. worker_clr_flags(worker, WORKER_IDLE);
  1598. pool->nr_idle--;
  1599. list_del_init(&worker->entry);
  1600. }
  1601. static struct worker *alloc_worker(int node)
  1602. {
  1603. struct worker *worker;
  1604. worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
  1605. if (worker) {
  1606. INIT_LIST_HEAD(&worker->entry);
  1607. INIT_LIST_HEAD(&worker->scheduled);
  1608. INIT_LIST_HEAD(&worker->node);
  1609. /* on creation a worker is in !idle && prep state */
  1610. worker->flags = WORKER_PREP;
  1611. }
  1612. return worker;
  1613. }
  1614. /**
  1615. * worker_attach_to_pool() - attach a worker to a pool
  1616. * @worker: worker to be attached
  1617. * @pool: the target pool
  1618. *
  1619. * Attach @worker to @pool. Once attached, the %WORKER_UNBOUND flag and
  1620. * cpu-binding of @worker are kept coordinated with the pool across
  1621. * cpu-[un]hotplugs.
  1622. */
  1623. static void worker_attach_to_pool(struct worker *worker,
  1624. struct worker_pool *pool)
  1625. {
  1626. mutex_lock(&wq_pool_attach_mutex);
  1627. /*
  1628. * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains
  1629. * stable across this function. See the comments above the flag
  1630. * definition for details.
  1631. */
  1632. if (pool->flags & POOL_DISASSOCIATED)
  1633. worker->flags |= WORKER_UNBOUND;
  1634. else
  1635. kthread_set_per_cpu(worker->task, pool->cpu);
  1636. if (worker->rescue_wq)
  1637. set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
  1638. list_add_tail(&worker->node, &pool->workers);
  1639. worker->pool = pool;
  1640. mutex_unlock(&wq_pool_attach_mutex);
  1641. }
  1642. /**
  1643. * worker_detach_from_pool() - detach a worker from its pool
  1644. * @worker: worker which is attached to its pool
  1645. *
  1646. * Undo the attaching which had been done in worker_attach_to_pool(). The
  1647. * caller worker shouldn't access to the pool after detached except it has
  1648. * other reference to the pool.
  1649. */
  1650. static void worker_detach_from_pool(struct worker *worker)
  1651. {
  1652. struct worker_pool *pool = worker->pool;
  1653. struct completion *detach_completion = NULL;
  1654. mutex_lock(&wq_pool_attach_mutex);
  1655. kthread_set_per_cpu(worker->task, -1);
  1656. list_del(&worker->node);
  1657. worker->pool = NULL;
  1658. if (list_empty(&pool->workers))
  1659. detach_completion = pool->detach_completion;
  1660. mutex_unlock(&wq_pool_attach_mutex);
  1661. /* clear leftover flags without pool->lock after it is detached */
  1662. worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
  1663. if (detach_completion)
  1664. complete(detach_completion);
  1665. }
  1666. /**
  1667. * create_worker - create a new workqueue worker
  1668. * @pool: pool the new worker will belong to
  1669. *
  1670. * Create and start a new worker which is attached to @pool.
  1671. *
  1672. * CONTEXT:
  1673. * Might sleep. Does GFP_KERNEL allocations.
  1674. *
  1675. * Return:
  1676. * Pointer to the newly created worker.
  1677. */
  1678. static struct worker *create_worker(struct worker_pool *pool)
  1679. {
  1680. struct worker *worker;
  1681. int id;
  1682. char id_buf[16];
  1683. /* ID is needed to determine kthread name */
  1684. id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
  1685. if (id < 0)
  1686. return NULL;
  1687. worker = alloc_worker(pool->node);
  1688. if (!worker)
  1689. goto fail;
  1690. worker->id = id;
  1691. if (pool->cpu >= 0)
  1692. snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
  1693. pool->attrs->nice < 0 ? "H" : "");
  1694. else
  1695. snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
  1696. worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
  1697. "kworker/%s", id_buf);
  1698. if (IS_ERR(worker->task))
  1699. goto fail;
  1700. set_user_nice(worker->task, pool->attrs->nice);
  1701. kthread_bind_mask(worker->task, pool->attrs->cpumask);
  1702. /* successful, attach the worker to the pool */
  1703. worker_attach_to_pool(worker, pool);
  1704. /* start the newly created worker */
  1705. raw_spin_lock_irq(&pool->lock);
  1706. worker->pool->nr_workers++;
  1707. worker_enter_idle(worker);
  1708. wake_up_process(worker->task);
  1709. raw_spin_unlock_irq(&pool->lock);
  1710. return worker;
  1711. fail:
  1712. ida_free(&pool->worker_ida, id);
  1713. kfree(worker);
  1714. return NULL;
  1715. }
  1716. /**
  1717. * destroy_worker - destroy a workqueue worker
  1718. * @worker: worker to be destroyed
  1719. *
  1720. * Destroy @worker and adjust @pool stats accordingly. The worker should
  1721. * be idle.
  1722. *
  1723. * CONTEXT:
  1724. * raw_spin_lock_irq(pool->lock).
  1725. */
  1726. static void destroy_worker(struct worker *worker)
  1727. {
  1728. struct worker_pool *pool = worker->pool;
  1729. lockdep_assert_held(&pool->lock);
  1730. /* sanity check frenzy */
  1731. if (WARN_ON(worker->current_work) ||
  1732. WARN_ON(!list_empty(&worker->scheduled)) ||
  1733. WARN_ON(!(worker->flags & WORKER_IDLE)))
  1734. return;
  1735. pool->nr_workers--;
  1736. pool->nr_idle--;
  1737. list_del_init(&worker->entry);
  1738. worker->flags |= WORKER_DIE;
  1739. wake_up_process(worker->task);
  1740. }
  1741. static void idle_worker_timeout(struct timer_list *t)
  1742. {
  1743. struct worker_pool *pool = from_timer(pool, t, idle_timer);
  1744. raw_spin_lock_irq(&pool->lock);
  1745. while (too_many_workers(pool)) {
  1746. struct worker *worker;
  1747. unsigned long expires;
  1748. /* idle_list is kept in LIFO order, check the last one */
  1749. worker = list_entry(pool->idle_list.prev, struct worker, entry);
  1750. expires = worker->last_active + IDLE_WORKER_TIMEOUT;
  1751. if (time_before(jiffies, expires)) {
  1752. mod_timer(&pool->idle_timer, expires);
  1753. break;
  1754. }
  1755. destroy_worker(worker);
  1756. }
  1757. raw_spin_unlock_irq(&pool->lock);
  1758. }
  1759. static void send_mayday(struct work_struct *work)
  1760. {
  1761. struct pool_workqueue *pwq = get_work_pwq(work);
  1762. struct workqueue_struct *wq = pwq->wq;
  1763. lockdep_assert_held(&wq_mayday_lock);
  1764. if (!wq->rescuer)
  1765. return;
  1766. /* mayday mayday mayday */
  1767. if (list_empty(&pwq->mayday_node)) {
  1768. /*
  1769. * If @pwq is for an unbound wq, its base ref may be put at
  1770. * any time due to an attribute change. Pin @pwq until the
  1771. * rescuer is done with it.
  1772. */
  1773. get_pwq(pwq);
  1774. list_add_tail(&pwq->mayday_node, &wq->maydays);
  1775. wake_up_process(wq->rescuer->task);
  1776. }
  1777. }
  1778. static void pool_mayday_timeout(struct timer_list *t)
  1779. {
  1780. struct worker_pool *pool = from_timer(pool, t, mayday_timer);
  1781. struct work_struct *work;
  1782. raw_spin_lock_irq(&pool->lock);
  1783. raw_spin_lock(&wq_mayday_lock); /* for wq->maydays */
  1784. if (need_to_create_worker(pool)) {
  1785. /*
  1786. * We've been trying to create a new worker but
  1787. * haven't been successful. We might be hitting an
  1788. * allocation deadlock. Send distress signals to
  1789. * rescuers.
  1790. */
  1791. list_for_each_entry(work, &pool->worklist, entry)
  1792. send_mayday(work);
  1793. }
  1794. raw_spin_unlock(&wq_mayday_lock);
  1795. raw_spin_unlock_irq(&pool->lock);
  1796. mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
  1797. }
  1798. /**
  1799. * maybe_create_worker - create a new worker if necessary
  1800. * @pool: pool to create a new worker for
  1801. *
  1802. * Create a new worker for @pool if necessary. @pool is guaranteed to
  1803. * have at least one idle worker on return from this function. If
  1804. * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
  1805. * sent to all rescuers with works scheduled on @pool to resolve
  1806. * possible allocation deadlock.
  1807. *
  1808. * On return, need_to_create_worker() is guaranteed to be %false and
  1809. * may_start_working() %true.
  1810. *
  1811. * LOCKING:
  1812. * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
  1813. * multiple times. Does GFP_KERNEL allocations. Called only from
  1814. * manager.
  1815. */
  1816. static void maybe_create_worker(struct worker_pool *pool)
  1817. __releases(&pool->lock)
  1818. __acquires(&pool->lock)
  1819. {
  1820. restart:
  1821. raw_spin_unlock_irq(&pool->lock);
  1822. /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
  1823. mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
  1824. while (true) {
  1825. if (create_worker(pool) || !need_to_create_worker(pool))
  1826. break;
  1827. schedule_timeout_interruptible(CREATE_COOLDOWN);
  1828. if (!need_to_create_worker(pool))
  1829. break;
  1830. }
  1831. del_timer_sync(&pool->mayday_timer);
  1832. raw_spin_lock_irq(&pool->lock);
  1833. /*
  1834. * This is necessary even after a new worker was just successfully
  1835. * created as @pool->lock was dropped and the new worker might have
  1836. * already become busy.
  1837. */
  1838. if (need_to_create_worker(pool))
  1839. goto restart;
  1840. }
  1841. /**
  1842. * manage_workers - manage worker pool
  1843. * @worker: self
  1844. *
  1845. * Assume the manager role and manage the worker pool @worker belongs
  1846. * to. At any given time, there can be only zero or one manager per
  1847. * pool. The exclusion is handled automatically by this function.
  1848. *
  1849. * The caller can safely start processing works on false return. On
  1850. * true return, it's guaranteed that need_to_create_worker() is false
  1851. * and may_start_working() is true.
  1852. *
  1853. * CONTEXT:
  1854. * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
  1855. * multiple times. Does GFP_KERNEL allocations.
  1856. *
  1857. * Return:
  1858. * %false if the pool doesn't need management and the caller can safely
  1859. * start processing works, %true if management function was performed and
  1860. * the conditions that the caller verified before calling the function may
  1861. * no longer be true.
  1862. */
  1863. static bool manage_workers(struct worker *worker)
  1864. {
  1865. struct worker_pool *pool = worker->pool;
  1866. if (pool->flags & POOL_MANAGER_ACTIVE)
  1867. return false;
  1868. pool->flags |= POOL_MANAGER_ACTIVE;
  1869. pool->manager = worker;
  1870. maybe_create_worker(pool);
  1871. pool->manager = NULL;
  1872. pool->flags &= ~POOL_MANAGER_ACTIVE;
  1873. rcuwait_wake_up(&manager_wait);
  1874. return true;
  1875. }
  1876. /**
  1877. * process_one_work - process single work
  1878. * @worker: self
  1879. * @work: work to process
  1880. *
  1881. * Process @work. This function contains all the logics necessary to
  1882. * process a single work including synchronization against and
  1883. * interaction with other workers on the same cpu, queueing and
  1884. * flushing. As long as context requirement is met, any worker can
  1885. * call this function to process a work.
  1886. *
  1887. * CONTEXT:
  1888. * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
  1889. */
  1890. static void process_one_work(struct worker *worker, struct work_struct *work)
  1891. __releases(&pool->lock)
  1892. __acquires(&pool->lock)
  1893. {
  1894. struct pool_workqueue *pwq = get_work_pwq(work);
  1895. struct worker_pool *pool = worker->pool;
  1896. bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
  1897. unsigned long work_data;
  1898. struct worker *collision;
  1899. #ifdef CONFIG_LOCKDEP
  1900. /*
  1901. * It is permissible to free the struct work_struct from
  1902. * inside the function that is called from it, this we need to
  1903. * take into account for lockdep too. To avoid bogus "held
  1904. * lock freed" warnings as well as problems when looking into
  1905. * work->lockdep_map, make a copy and use that here.
  1906. */
  1907. struct lockdep_map lockdep_map;
  1908. lockdep_copy_map(&lockdep_map, &work->lockdep_map);
  1909. #endif
  1910. /* ensure we're on the correct CPU */
  1911. WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
  1912. raw_smp_processor_id() != pool->cpu);
  1913. /*
  1914. * A single work shouldn't be executed concurrently by
  1915. * multiple workers on a single cpu. Check whether anyone is
  1916. * already processing the work. If so, defer the work to the
  1917. * currently executing one.
  1918. */
  1919. collision = find_worker_executing_work(pool, work);
  1920. if (unlikely(collision)) {
  1921. move_linked_works(work, &collision->scheduled, NULL);
  1922. return;
  1923. }
  1924. /* claim and dequeue */
  1925. debug_work_deactivate(work);
  1926. hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
  1927. worker->current_work = work;
  1928. worker->current_func = work->func;
  1929. worker->current_pwq = pwq;
  1930. work_data = *work_data_bits(work);
  1931. worker->current_color = get_work_color(work_data);
  1932. /*
  1933. * Record wq name for cmdline and debug reporting, may get
  1934. * overridden through set_worker_desc().
  1935. */
  1936. strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
  1937. list_del_init(&work->entry);
  1938. /*
  1939. * CPU intensive works don't participate in concurrency management.
  1940. * They're the scheduler's responsibility. This takes @worker out
  1941. * of concurrency management and the next code block will chain
  1942. * execution of the pending work items.
  1943. */
  1944. if (unlikely(cpu_intensive))
  1945. worker_set_flags(worker, WORKER_CPU_INTENSIVE);
  1946. /*
  1947. * Wake up another worker if necessary. The condition is always
  1948. * false for normal per-cpu workers since nr_running would always
  1949. * be >= 1 at this point. This is used to chain execution of the
  1950. * pending work items for WORKER_NOT_RUNNING workers such as the
  1951. * UNBOUND and CPU_INTENSIVE ones.
  1952. */
  1953. if (need_more_worker(pool))
  1954. wake_up_worker(pool);
  1955. /*
  1956. * Record the last pool and clear PENDING which should be the last
  1957. * update to @work. Also, do this inside @pool->lock so that
  1958. * PENDING and queued state changes happen together while IRQ is
  1959. * disabled.
  1960. */
  1961. set_work_pool_and_clear_pending(work, pool->id);
  1962. raw_spin_unlock_irq(&pool->lock);
  1963. lock_map_acquire(&pwq->wq->lockdep_map);
  1964. lock_map_acquire(&lockdep_map);
  1965. /*
  1966. * Strictly speaking we should mark the invariant state without holding
  1967. * any locks, that is, before these two lock_map_acquire()'s.
  1968. *
  1969. * However, that would result in:
  1970. *
  1971. * A(W1)
  1972. * WFC(C)
  1973. * A(W1)
  1974. * C(C)
  1975. *
  1976. * Which would create W1->C->W1 dependencies, even though there is no
  1977. * actual deadlock possible. There are two solutions, using a
  1978. * read-recursive acquire on the work(queue) 'locks', but this will then
  1979. * hit the lockdep limitation on recursive locks, or simply discard
  1980. * these locks.
  1981. *
  1982. * AFAICT there is no possible deadlock scenario between the
  1983. * flush_work() and complete() primitives (except for single-threaded
  1984. * workqueues), so hiding them isn't a problem.
  1985. */
  1986. lockdep_invariant_state(true);
  1987. trace_workqueue_execute_start(work);
  1988. worker->current_func(work);
  1989. /*
  1990. * While we must be careful to not use "work" after this, the trace
  1991. * point will only record its address.
  1992. */
  1993. trace_workqueue_execute_end(work, worker->current_func);
  1994. lock_map_release(&lockdep_map);
  1995. lock_map_release(&pwq->wq->lockdep_map);
  1996. if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
  1997. pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
  1998. " last function: %ps\n",
  1999. current->comm, preempt_count(), task_pid_nr(current),
  2000. worker->current_func);
  2001. debug_show_held_locks(current);
  2002. dump_stack();
  2003. }
  2004. /*
  2005. * The following prevents a kworker from hogging CPU on !PREEMPTION
  2006. * kernels, where a requeueing work item waiting for something to
  2007. * happen could deadlock with stop_machine as such work item could
  2008. * indefinitely requeue itself while all other CPUs are trapped in
  2009. * stop_machine. At the same time, report a quiescent RCU state so
  2010. * the same condition doesn't freeze RCU.
  2011. */
  2012. cond_resched();
  2013. raw_spin_lock_irq(&pool->lock);
  2014. /* clear cpu intensive status */
  2015. if (unlikely(cpu_intensive))
  2016. worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
  2017. /* tag the worker for identification in schedule() */
  2018. worker->last_func = worker->current_func;
  2019. /* we're done with it, release */
  2020. hash_del(&worker->hentry);
  2021. worker->current_work = NULL;
  2022. worker->current_func = NULL;
  2023. worker->current_pwq = NULL;
  2024. worker->current_color = INT_MAX;
  2025. pwq_dec_nr_in_flight(pwq, work_data);
  2026. }
  2027. /**
  2028. * process_scheduled_works - process scheduled works
  2029. * @worker: self
  2030. *
  2031. * Process all scheduled works. Please note that the scheduled list
  2032. * may change while processing a work, so this function repeatedly
  2033. * fetches a work from the top and executes it.
  2034. *
  2035. * CONTEXT:
  2036. * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
  2037. * multiple times.
  2038. */
  2039. static void process_scheduled_works(struct worker *worker)
  2040. {
  2041. while (!list_empty(&worker->scheduled)) {
  2042. struct work_struct *work = list_first_entry(&worker->scheduled,
  2043. struct work_struct, entry);
  2044. process_one_work(worker, work);
  2045. }
  2046. }
  2047. static void set_pf_worker(bool val)
  2048. {
  2049. mutex_lock(&wq_pool_attach_mutex);
  2050. if (val)
  2051. current->flags |= PF_WQ_WORKER;
  2052. else
  2053. current->flags &= ~PF_WQ_WORKER;
  2054. mutex_unlock(&wq_pool_attach_mutex);
  2055. }
  2056. /**
  2057. * worker_thread - the worker thread function
  2058. * @__worker: self
  2059. *
  2060. * The worker thread function. All workers belong to a worker_pool -
  2061. * either a per-cpu one or dynamic unbound one. These workers process all
  2062. * work items regardless of their specific target workqueue. The only
  2063. * exception is work items which belong to workqueues with a rescuer which
  2064. * will be explained in rescuer_thread().
  2065. *
  2066. * Return: 0
  2067. */
  2068. static int worker_thread(void *__worker)
  2069. {
  2070. struct worker *worker = __worker;
  2071. struct worker_pool *pool = worker->pool;
  2072. /* tell the scheduler that this is a workqueue worker */
  2073. set_pf_worker(true);
  2074. woke_up:
  2075. raw_spin_lock_irq(&pool->lock);
  2076. /* am I supposed to die? */
  2077. if (unlikely(worker->flags & WORKER_DIE)) {
  2078. raw_spin_unlock_irq(&pool->lock);
  2079. WARN_ON_ONCE(!list_empty(&worker->entry));
  2080. set_pf_worker(false);
  2081. set_task_comm(worker->task, "kworker/dying");
  2082. ida_free(&pool->worker_ida, worker->id);
  2083. worker_detach_from_pool(worker);
  2084. kfree(worker);
  2085. return 0;
  2086. }
  2087. worker_leave_idle(worker);
  2088. recheck:
  2089. /* no more worker necessary? */
  2090. if (!need_more_worker(pool))
  2091. goto sleep;
  2092. /* do we need to manage? */
  2093. if (unlikely(!may_start_working(pool)) && manage_workers(worker))
  2094. goto recheck;
  2095. /*
  2096. * ->scheduled list can only be filled while a worker is
  2097. * preparing to process a work or actually processing it.
  2098. * Make sure nobody diddled with it while I was sleeping.
  2099. */
  2100. WARN_ON_ONCE(!list_empty(&worker->scheduled));
  2101. /*
  2102. * Finish PREP stage. We're guaranteed to have at least one idle
  2103. * worker or that someone else has already assumed the manager
  2104. * role. This is where @worker starts participating in concurrency
  2105. * management if applicable and concurrency management is restored
  2106. * after being rebound. See rebind_workers() for details.
  2107. */
  2108. worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
  2109. do {
  2110. struct work_struct *work =
  2111. list_first_entry(&pool->worklist,
  2112. struct work_struct, entry);
  2113. pool->watchdog_ts = jiffies;
  2114. if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
  2115. /* optimization path, not strictly necessary */
  2116. process_one_work(worker, work);
  2117. if (unlikely(!list_empty(&worker->scheduled)))
  2118. process_scheduled_works(worker);
  2119. } else {
  2120. move_linked_works(work, &worker->scheduled, NULL);
  2121. process_scheduled_works(worker);
  2122. }
  2123. } while (keep_working(pool));
  2124. worker_set_flags(worker, WORKER_PREP);
  2125. sleep:
  2126. /*
  2127. * pool->lock is held and there's no work to process and no need to
  2128. * manage, sleep. Workers are woken up only while holding
  2129. * pool->lock or from local cpu, so setting the current state
  2130. * before releasing pool->lock is enough to prevent losing any
  2131. * event.
  2132. */
  2133. worker_enter_idle(worker);
  2134. __set_current_state(TASK_IDLE);
  2135. raw_spin_unlock_irq(&pool->lock);
  2136. schedule();
  2137. goto woke_up;
  2138. }
  2139. /**
  2140. * rescuer_thread - the rescuer thread function
  2141. * @__rescuer: self
  2142. *
  2143. * Workqueue rescuer thread function. There's one rescuer for each
  2144. * workqueue which has WQ_MEM_RECLAIM set.
  2145. *
  2146. * Regular work processing on a pool may block trying to create a new
  2147. * worker which uses GFP_KERNEL allocation which has slight chance of
  2148. * developing into deadlock if some works currently on the same queue
  2149. * need to be processed to satisfy the GFP_KERNEL allocation. This is
  2150. * the problem rescuer solves.
  2151. *
  2152. * When such condition is possible, the pool summons rescuers of all
  2153. * workqueues which have works queued on the pool and let them process
  2154. * those works so that forward progress can be guaranteed.
  2155. *
  2156. * This should happen rarely.
  2157. *
  2158. * Return: 0
  2159. */
  2160. static int rescuer_thread(void *__rescuer)
  2161. {
  2162. struct worker *rescuer = __rescuer;
  2163. struct workqueue_struct *wq = rescuer->rescue_wq;
  2164. struct list_head *scheduled = &rescuer->scheduled;
  2165. bool should_stop;
  2166. set_user_nice(current, RESCUER_NICE_LEVEL);
  2167. /*
  2168. * Mark rescuer as worker too. As WORKER_PREP is never cleared, it
  2169. * doesn't participate in concurrency management.
  2170. */
  2171. set_pf_worker(true);
  2172. repeat:
  2173. set_current_state(TASK_IDLE);
  2174. /*
  2175. * By the time the rescuer is requested to stop, the workqueue
  2176. * shouldn't have any work pending, but @wq->maydays may still have
  2177. * pwq(s) queued. This can happen by non-rescuer workers consuming
  2178. * all the work items before the rescuer got to them. Go through
  2179. * @wq->maydays processing before acting on should_stop so that the
  2180. * list is always empty on exit.
  2181. */
  2182. should_stop = kthread_should_stop();
  2183. /* see whether any pwq is asking for help */
  2184. raw_spin_lock_irq(&wq_mayday_lock);
  2185. while (!list_empty(&wq->maydays)) {
  2186. struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
  2187. struct pool_workqueue, mayday_node);
  2188. struct worker_pool *pool = pwq->pool;
  2189. struct work_struct *work, *n;
  2190. bool first = true;
  2191. __set_current_state(TASK_RUNNING);
  2192. list_del_init(&pwq->mayday_node);
  2193. raw_spin_unlock_irq(&wq_mayday_lock);
  2194. worker_attach_to_pool(rescuer, pool);
  2195. raw_spin_lock_irq(&pool->lock);
  2196. /*
  2197. * Slurp in all works issued via this workqueue and
  2198. * process'em.
  2199. */
  2200. WARN_ON_ONCE(!list_empty(scheduled));
  2201. list_for_each_entry_safe(work, n, &pool->worklist, entry) {
  2202. if (get_work_pwq(work) == pwq) {
  2203. if (first)
  2204. pool->watchdog_ts = jiffies;
  2205. move_linked_works(work, scheduled, &n);
  2206. }
  2207. first = false;
  2208. }
  2209. if (!list_empty(scheduled)) {
  2210. process_scheduled_works(rescuer);
  2211. /*
  2212. * The above execution of rescued work items could
  2213. * have created more to rescue through
  2214. * pwq_activate_first_inactive() or chained
  2215. * queueing. Let's put @pwq back on mayday list so
  2216. * that such back-to-back work items, which may be
  2217. * being used to relieve memory pressure, don't
  2218. * incur MAYDAY_INTERVAL delay inbetween.
  2219. */
  2220. if (pwq->nr_active && need_to_create_worker(pool)) {
  2221. raw_spin_lock(&wq_mayday_lock);
  2222. /*
  2223. * Queue iff we aren't racing destruction
  2224. * and somebody else hasn't queued it already.
  2225. */
  2226. if (wq->rescuer && list_empty(&pwq->mayday_node)) {
  2227. get_pwq(pwq);
  2228. list_add_tail(&pwq->mayday_node, &wq->maydays);
  2229. }
  2230. raw_spin_unlock(&wq_mayday_lock);
  2231. }
  2232. }
  2233. /*
  2234. * Put the reference grabbed by send_mayday(). @pool won't
  2235. * go away while we're still attached to it.
  2236. */
  2237. put_pwq(pwq);
  2238. /*
  2239. * Leave this pool. If need_more_worker() is %true, notify a
  2240. * regular worker; otherwise, we end up with 0 concurrency
  2241. * and stalling the execution.
  2242. */
  2243. if (need_more_worker(pool))
  2244. wake_up_worker(pool);
  2245. raw_spin_unlock_irq(&pool->lock);
  2246. worker_detach_from_pool(rescuer);
  2247. raw_spin_lock_irq(&wq_mayday_lock);
  2248. }
  2249. raw_spin_unlock_irq(&wq_mayday_lock);
  2250. if (should_stop) {
  2251. __set_current_state(TASK_RUNNING);
  2252. set_pf_worker(false);
  2253. return 0;
  2254. }
  2255. /* rescuers should never participate in concurrency management */
  2256. WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
  2257. schedule();
  2258. goto repeat;
  2259. }
  2260. /**
  2261. * check_flush_dependency - check for flush dependency sanity
  2262. * @target_wq: workqueue being flushed
  2263. * @target_work: work item being flushed (NULL for workqueue flushes)
  2264. *
  2265. * %current is trying to flush the whole @target_wq or @target_work on it.
  2266. * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
  2267. * reclaiming memory or running on a workqueue which doesn't have
  2268. * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
  2269. * a deadlock.
  2270. */
  2271. static void check_flush_dependency(struct workqueue_struct *target_wq,
  2272. struct work_struct *target_work)
  2273. {
  2274. work_func_t target_func = target_work ? target_work->func : NULL;
  2275. struct worker *worker;
  2276. if (target_wq->flags & WQ_MEM_RECLAIM)
  2277. return;
  2278. worker = current_wq_worker();
  2279. WARN_ONCE(current->flags & PF_MEMALLOC,
  2280. "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
  2281. current->pid, current->comm, target_wq->name, target_func);
  2282. WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
  2283. (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
  2284. "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
  2285. worker->current_pwq->wq->name, worker->current_func,
  2286. target_wq->name, target_func);
  2287. }
  2288. struct wq_barrier {
  2289. struct work_struct work;
  2290. struct completion done;
  2291. struct task_struct *task; /* purely informational */
  2292. };
  2293. static void wq_barrier_func(struct work_struct *work)
  2294. {
  2295. struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
  2296. complete(&barr->done);
  2297. }
  2298. /**
  2299. * insert_wq_barrier - insert a barrier work
  2300. * @pwq: pwq to insert barrier into
  2301. * @barr: wq_barrier to insert
  2302. * @target: target work to attach @barr to
  2303. * @worker: worker currently executing @target, NULL if @target is not executing
  2304. *
  2305. * @barr is linked to @target such that @barr is completed only after
  2306. * @target finishes execution. Please note that the ordering
  2307. * guarantee is observed only with respect to @target and on the local
  2308. * cpu.
  2309. *
  2310. * Currently, a queued barrier can't be canceled. This is because
  2311. * try_to_grab_pending() can't determine whether the work to be
  2312. * grabbed is at the head of the queue and thus can't clear LINKED
  2313. * flag of the previous work while there must be a valid next work
  2314. * after a work with LINKED flag set.
  2315. *
  2316. * Note that when @worker is non-NULL, @target may be modified
  2317. * underneath us, so we can't reliably determine pwq from @target.
  2318. *
  2319. * CONTEXT:
  2320. * raw_spin_lock_irq(pool->lock).
  2321. */
  2322. static void insert_wq_barrier(struct pool_workqueue *pwq,
  2323. struct wq_barrier *barr,
  2324. struct work_struct *target, struct worker *worker)
  2325. {
  2326. unsigned int work_flags = 0;
  2327. unsigned int work_color;
  2328. struct list_head *head;
  2329. /*
  2330. * debugobject calls are safe here even with pool->lock locked
  2331. * as we know for sure that this will not trigger any of the
  2332. * checks and call back into the fixup functions where we
  2333. * might deadlock.
  2334. */
  2335. INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
  2336. __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
  2337. init_completion_map(&barr->done, &target->lockdep_map);
  2338. barr->task = current;
  2339. /* The barrier work item does not participate in pwq->nr_active. */
  2340. work_flags |= WORK_STRUCT_INACTIVE;
  2341. /*
  2342. * If @target is currently being executed, schedule the
  2343. * barrier to the worker; otherwise, put it after @target.
  2344. */
  2345. if (worker) {
  2346. head = worker->scheduled.next;
  2347. work_color = worker->current_color;
  2348. } else {
  2349. unsigned long *bits = work_data_bits(target);
  2350. head = target->entry.next;
  2351. /* there can already be other linked works, inherit and set */
  2352. work_flags |= *bits & WORK_STRUCT_LINKED;
  2353. work_color = get_work_color(*bits);
  2354. __set_bit(WORK_STRUCT_LINKED_BIT, bits);
  2355. }
  2356. pwq->nr_in_flight[work_color]++;
  2357. work_flags |= work_color_to_flags(work_color);
  2358. debug_work_activate(&barr->work);
  2359. insert_work(pwq, &barr->work, head, work_flags);
  2360. }
  2361. /**
  2362. * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
  2363. * @wq: workqueue being flushed
  2364. * @flush_color: new flush color, < 0 for no-op
  2365. * @work_color: new work color, < 0 for no-op
  2366. *
  2367. * Prepare pwqs for workqueue flushing.
  2368. *
  2369. * If @flush_color is non-negative, flush_color on all pwqs should be
  2370. * -1. If no pwq has in-flight commands at the specified color, all
  2371. * pwq->flush_color's stay at -1 and %false is returned. If any pwq
  2372. * has in flight commands, its pwq->flush_color is set to
  2373. * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
  2374. * wakeup logic is armed and %true is returned.
  2375. *
  2376. * The caller should have initialized @wq->first_flusher prior to
  2377. * calling this function with non-negative @flush_color. If
  2378. * @flush_color is negative, no flush color update is done and %false
  2379. * is returned.
  2380. *
  2381. * If @work_color is non-negative, all pwqs should have the same
  2382. * work_color which is previous to @work_color and all will be
  2383. * advanced to @work_color.
  2384. *
  2385. * CONTEXT:
  2386. * mutex_lock(wq->mutex).
  2387. *
  2388. * Return:
  2389. * %true if @flush_color >= 0 and there's something to flush. %false
  2390. * otherwise.
  2391. */
  2392. static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
  2393. int flush_color, int work_color)
  2394. {
  2395. bool wait = false;
  2396. struct pool_workqueue *pwq;
  2397. if (flush_color >= 0) {
  2398. WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
  2399. atomic_set(&wq->nr_pwqs_to_flush, 1);
  2400. }
  2401. for_each_pwq(pwq, wq) {
  2402. struct worker_pool *pool = pwq->pool;
  2403. raw_spin_lock_irq(&pool->lock);
  2404. if (flush_color >= 0) {
  2405. WARN_ON_ONCE(pwq->flush_color != -1);
  2406. if (pwq->nr_in_flight[flush_color]) {
  2407. pwq->flush_color = flush_color;
  2408. atomic_inc(&wq->nr_pwqs_to_flush);
  2409. wait = true;
  2410. }
  2411. }
  2412. if (work_color >= 0) {
  2413. WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
  2414. pwq->work_color = work_color;
  2415. }
  2416. raw_spin_unlock_irq(&pool->lock);
  2417. }
  2418. if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
  2419. complete(&wq->first_flusher->done);
  2420. return wait;
  2421. }
  2422. /**
  2423. * __flush_workqueue - ensure that any scheduled work has run to completion.
  2424. * @wq: workqueue to flush
  2425. *
  2426. * This function sleeps until all work items which were queued on entry
  2427. * have finished execution, but it is not livelocked by new incoming ones.
  2428. */
  2429. void __flush_workqueue(struct workqueue_struct *wq)
  2430. {
  2431. struct wq_flusher this_flusher = {
  2432. .list = LIST_HEAD_INIT(this_flusher.list),
  2433. .flush_color = -1,
  2434. .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
  2435. };
  2436. int next_color;
  2437. if (WARN_ON(!wq_online))
  2438. return;
  2439. lock_map_acquire(&wq->lockdep_map);
  2440. lock_map_release(&wq->lockdep_map);
  2441. mutex_lock(&wq->mutex);
  2442. /*
  2443. * Start-to-wait phase
  2444. */
  2445. next_color = work_next_color(wq->work_color);
  2446. if (next_color != wq->flush_color) {
  2447. /*
  2448. * Color space is not full. The current work_color
  2449. * becomes our flush_color and work_color is advanced
  2450. * by one.
  2451. */
  2452. WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
  2453. this_flusher.flush_color = wq->work_color;
  2454. wq->work_color = next_color;
  2455. if (!wq->first_flusher) {
  2456. /* no flush in progress, become the first flusher */
  2457. WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
  2458. wq->first_flusher = &this_flusher;
  2459. if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
  2460. wq->work_color)) {
  2461. /* nothing to flush, done */
  2462. wq->flush_color = next_color;
  2463. wq->first_flusher = NULL;
  2464. goto out_unlock;
  2465. }
  2466. } else {
  2467. /* wait in queue */
  2468. WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
  2469. list_add_tail(&this_flusher.list, &wq->flusher_queue);
  2470. flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
  2471. }
  2472. } else {
  2473. /*
  2474. * Oops, color space is full, wait on overflow queue.
  2475. * The next flush completion will assign us
  2476. * flush_color and transfer to flusher_queue.
  2477. */
  2478. list_add_tail(&this_flusher.list, &wq->flusher_overflow);
  2479. }
  2480. check_flush_dependency(wq, NULL);
  2481. mutex_unlock(&wq->mutex);
  2482. wait_for_completion(&this_flusher.done);
  2483. /*
  2484. * Wake-up-and-cascade phase
  2485. *
  2486. * First flushers are responsible for cascading flushes and
  2487. * handling overflow. Non-first flushers can simply return.
  2488. */
  2489. if (READ_ONCE(wq->first_flusher) != &this_flusher)
  2490. return;
  2491. mutex_lock(&wq->mutex);
  2492. /* we might have raced, check again with mutex held */
  2493. if (wq->first_flusher != &this_flusher)
  2494. goto out_unlock;
  2495. WRITE_ONCE(wq->first_flusher, NULL);
  2496. WARN_ON_ONCE(!list_empty(&this_flusher.list));
  2497. WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
  2498. while (true) {
  2499. struct wq_flusher *next, *tmp;
  2500. /* complete all the flushers sharing the current flush color */
  2501. list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
  2502. if (next->flush_color != wq->flush_color)
  2503. break;
  2504. list_del_init(&next->list);
  2505. complete(&next->done);
  2506. }
  2507. WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
  2508. wq->flush_color != work_next_color(wq->work_color));
  2509. /* this flush_color is finished, advance by one */
  2510. wq->flush_color = work_next_color(wq->flush_color);
  2511. /* one color has been freed, handle overflow queue */
  2512. if (!list_empty(&wq->flusher_overflow)) {
  2513. /*
  2514. * Assign the same color to all overflowed
  2515. * flushers, advance work_color and append to
  2516. * flusher_queue. This is the start-to-wait
  2517. * phase for these overflowed flushers.
  2518. */
  2519. list_for_each_entry(tmp, &wq->flusher_overflow, list)
  2520. tmp->flush_color = wq->work_color;
  2521. wq->work_color = work_next_color(wq->work_color);
  2522. list_splice_tail_init(&wq->flusher_overflow,
  2523. &wq->flusher_queue);
  2524. flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
  2525. }
  2526. if (list_empty(&wq->flusher_queue)) {
  2527. WARN_ON_ONCE(wq->flush_color != wq->work_color);
  2528. break;
  2529. }
  2530. /*
  2531. * Need to flush more colors. Make the next flusher
  2532. * the new first flusher and arm pwqs.
  2533. */
  2534. WARN_ON_ONCE(wq->flush_color == wq->work_color);
  2535. WARN_ON_ONCE(wq->flush_color != next->flush_color);
  2536. list_del_init(&next->list);
  2537. wq->first_flusher = next;
  2538. if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
  2539. break;
  2540. /*
  2541. * Meh... this color is already done, clear first
  2542. * flusher and repeat cascading.
  2543. */
  2544. wq->first_flusher = NULL;
  2545. }
  2546. out_unlock:
  2547. mutex_unlock(&wq->mutex);
  2548. }
  2549. EXPORT_SYMBOL(__flush_workqueue);
  2550. /**
  2551. * drain_workqueue - drain a workqueue
  2552. * @wq: workqueue to drain
  2553. *
  2554. * Wait until the workqueue becomes empty. While draining is in progress,
  2555. * only chain queueing is allowed. IOW, only currently pending or running
  2556. * work items on @wq can queue further work items on it. @wq is flushed
  2557. * repeatedly until it becomes empty. The number of flushing is determined
  2558. * by the depth of chaining and should be relatively short. Whine if it
  2559. * takes too long.
  2560. */
  2561. void drain_workqueue(struct workqueue_struct *wq)
  2562. {
  2563. unsigned int flush_cnt = 0;
  2564. struct pool_workqueue *pwq;
  2565. /*
  2566. * __queue_work() needs to test whether there are drainers, is much
  2567. * hotter than drain_workqueue() and already looks at @wq->flags.
  2568. * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
  2569. */
  2570. mutex_lock(&wq->mutex);
  2571. if (!wq->nr_drainers++)
  2572. wq->flags |= __WQ_DRAINING;
  2573. mutex_unlock(&wq->mutex);
  2574. reflush:
  2575. __flush_workqueue(wq);
  2576. mutex_lock(&wq->mutex);
  2577. for_each_pwq(pwq, wq) {
  2578. bool drained;
  2579. raw_spin_lock_irq(&pwq->pool->lock);
  2580. drained = !pwq->nr_active && list_empty(&pwq->inactive_works);
  2581. raw_spin_unlock_irq(&pwq->pool->lock);
  2582. if (drained)
  2583. continue;
  2584. if (++flush_cnt == 10 ||
  2585. (flush_cnt % 100 == 0 && flush_cnt <= 1000))
  2586. pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
  2587. wq->name, __func__, flush_cnt);
  2588. mutex_unlock(&wq->mutex);
  2589. goto reflush;
  2590. }
  2591. if (!--wq->nr_drainers)
  2592. wq->flags &= ~__WQ_DRAINING;
  2593. mutex_unlock(&wq->mutex);
  2594. }
  2595. EXPORT_SYMBOL_GPL(drain_workqueue);
  2596. static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
  2597. bool from_cancel)
  2598. {
  2599. struct worker *worker = NULL;
  2600. struct worker_pool *pool;
  2601. struct pool_workqueue *pwq;
  2602. might_sleep();
  2603. rcu_read_lock();
  2604. pool = get_work_pool(work);
  2605. if (!pool) {
  2606. rcu_read_unlock();
  2607. return false;
  2608. }
  2609. raw_spin_lock_irq(&pool->lock);
  2610. /* see the comment in try_to_grab_pending() with the same code */
  2611. pwq = get_work_pwq(work);
  2612. if (pwq) {
  2613. if (unlikely(pwq->pool != pool))
  2614. goto already_gone;
  2615. } else {
  2616. worker = find_worker_executing_work(pool, work);
  2617. if (!worker)
  2618. goto already_gone;
  2619. pwq = worker->current_pwq;
  2620. }
  2621. check_flush_dependency(pwq->wq, work);
  2622. insert_wq_barrier(pwq, barr, work, worker);
  2623. raw_spin_unlock_irq(&pool->lock);
  2624. /*
  2625. * Force a lock recursion deadlock when using flush_work() inside a
  2626. * single-threaded or rescuer equipped workqueue.
  2627. *
  2628. * For single threaded workqueues the deadlock happens when the work
  2629. * is after the work issuing the flush_work(). For rescuer equipped
  2630. * workqueues the deadlock happens when the rescuer stalls, blocking
  2631. * forward progress.
  2632. */
  2633. if (!from_cancel &&
  2634. (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {
  2635. lock_map_acquire(&pwq->wq->lockdep_map);
  2636. lock_map_release(&pwq->wq->lockdep_map);
  2637. }
  2638. rcu_read_unlock();
  2639. return true;
  2640. already_gone:
  2641. raw_spin_unlock_irq(&pool->lock);
  2642. rcu_read_unlock();
  2643. return false;
  2644. }
  2645. static bool __flush_work(struct work_struct *work, bool from_cancel)
  2646. {
  2647. struct wq_barrier barr;
  2648. if (WARN_ON(!wq_online))
  2649. return false;
  2650. if (WARN_ON(!work->func))
  2651. return false;
  2652. lock_map_acquire(&work->lockdep_map);
  2653. lock_map_release(&work->lockdep_map);
  2654. if (start_flush_work(work, &barr, from_cancel)) {
  2655. wait_for_completion(&barr.done);
  2656. destroy_work_on_stack(&barr.work);
  2657. return true;
  2658. } else {
  2659. return false;
  2660. }
  2661. }
  2662. /**
  2663. * flush_work - wait for a work to finish executing the last queueing instance
  2664. * @work: the work to flush
  2665. *
  2666. * Wait until @work has finished execution. @work is guaranteed to be idle
  2667. * on return if it hasn't been requeued since flush started.
  2668. *
  2669. * Return:
  2670. * %true if flush_work() waited for the work to finish execution,
  2671. * %false if it was already idle.
  2672. */
  2673. bool flush_work(struct work_struct *work)
  2674. {
  2675. return __flush_work(work, false);
  2676. }
  2677. EXPORT_SYMBOL_GPL(flush_work);
  2678. struct cwt_wait {
  2679. wait_queue_entry_t wait;
  2680. struct work_struct *work;
  2681. };
  2682. static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  2683. {
  2684. struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
  2685. if (cwait->work != key)
  2686. return 0;
  2687. return autoremove_wake_function(wait, mode, sync, key);
  2688. }
  2689. static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
  2690. {
  2691. static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
  2692. unsigned long flags;
  2693. int ret;
  2694. do {
  2695. ret = try_to_grab_pending(work, is_dwork, &flags);
  2696. /*
  2697. * If someone else is already canceling, wait for it to
  2698. * finish. flush_work() doesn't work for PREEMPT_NONE
  2699. * because we may get scheduled between @work's completion
  2700. * and the other canceling task resuming and clearing
  2701. * CANCELING - flush_work() will return false immediately
  2702. * as @work is no longer busy, try_to_grab_pending() will
  2703. * return -ENOENT as @work is still being canceled and the
  2704. * other canceling task won't be able to clear CANCELING as
  2705. * we're hogging the CPU.
  2706. *
  2707. * Let's wait for completion using a waitqueue. As this
  2708. * may lead to the thundering herd problem, use a custom
  2709. * wake function which matches @work along with exclusive
  2710. * wait and wakeup.
  2711. */
  2712. if (unlikely(ret == -ENOENT)) {
  2713. struct cwt_wait cwait;
  2714. init_wait(&cwait.wait);
  2715. cwait.wait.func = cwt_wakefn;
  2716. cwait.work = work;
  2717. prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
  2718. TASK_UNINTERRUPTIBLE);
  2719. if (work_is_canceling(work))
  2720. schedule();
  2721. finish_wait(&cancel_waitq, &cwait.wait);
  2722. }
  2723. } while (unlikely(ret < 0));
  2724. /* tell other tasks trying to grab @work to back off */
  2725. mark_work_canceling(work);
  2726. local_irq_restore(flags);
  2727. /*
  2728. * This allows canceling during early boot. We know that @work
  2729. * isn't executing.
  2730. */
  2731. if (wq_online)
  2732. __flush_work(work, true);
  2733. clear_work_data(work);
  2734. /*
  2735. * Paired with prepare_to_wait() above so that either
  2736. * waitqueue_active() is visible here or !work_is_canceling() is
  2737. * visible there.
  2738. */
  2739. smp_mb();
  2740. if (waitqueue_active(&cancel_waitq))
  2741. __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
  2742. return ret;
  2743. }
  2744. /**
  2745. * cancel_work_sync - cancel a work and wait for it to finish
  2746. * @work: the work to cancel
  2747. *
  2748. * Cancel @work and wait for its execution to finish. This function
  2749. * can be used even if the work re-queues itself or migrates to
  2750. * another workqueue. On return from this function, @work is
  2751. * guaranteed to be not pending or executing on any CPU.
  2752. *
  2753. * cancel_work_sync(&delayed_work->work) must not be used for
  2754. * delayed_work's. Use cancel_delayed_work_sync() instead.
  2755. *
  2756. * The caller must ensure that the workqueue on which @work was last
  2757. * queued can't be destroyed before this function returns.
  2758. *
  2759. * Return:
  2760. * %true if @work was pending, %false otherwise.
  2761. */
  2762. bool cancel_work_sync(struct work_struct *work)
  2763. {
  2764. return __cancel_work_timer(work, false);
  2765. }
  2766. EXPORT_SYMBOL_GPL(cancel_work_sync);
  2767. /**
  2768. * flush_delayed_work - wait for a dwork to finish executing the last queueing
  2769. * @dwork: the delayed work to flush
  2770. *
  2771. * Delayed timer is cancelled and the pending work is queued for
  2772. * immediate execution. Like flush_work(), this function only
  2773. * considers the last queueing instance of @dwork.
  2774. *
  2775. * Return:
  2776. * %true if flush_work() waited for the work to finish execution,
  2777. * %false if it was already idle.
  2778. */
  2779. bool flush_delayed_work(struct delayed_work *dwork)
  2780. {
  2781. local_irq_disable();
  2782. if (del_timer_sync(&dwork->timer))
  2783. __queue_work(dwork->cpu, dwork->wq, &dwork->work);
  2784. local_irq_enable();
  2785. return flush_work(&dwork->work);
  2786. }
  2787. EXPORT_SYMBOL(flush_delayed_work);
  2788. /**
  2789. * flush_rcu_work - wait for a rwork to finish executing the last queueing
  2790. * @rwork: the rcu work to flush
  2791. *
  2792. * Return:
  2793. * %true if flush_rcu_work() waited for the work to finish execution,
  2794. * %false if it was already idle.
  2795. */
  2796. bool flush_rcu_work(struct rcu_work *rwork)
  2797. {
  2798. if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
  2799. rcu_barrier();
  2800. flush_work(&rwork->work);
  2801. return true;
  2802. } else {
  2803. return flush_work(&rwork->work);
  2804. }
  2805. }
  2806. EXPORT_SYMBOL(flush_rcu_work);
  2807. static bool __cancel_work(struct work_struct *work, bool is_dwork)
  2808. {
  2809. unsigned long flags;
  2810. int ret;
  2811. do {
  2812. ret = try_to_grab_pending(work, is_dwork, &flags);
  2813. } while (unlikely(ret == -EAGAIN));
  2814. if (unlikely(ret < 0))
  2815. return false;
  2816. set_work_pool_and_clear_pending(work, get_work_pool_id(work));
  2817. local_irq_restore(flags);
  2818. return ret;
  2819. }
  2820. /*
  2821. * See cancel_delayed_work()
  2822. */
  2823. bool cancel_work(struct work_struct *work)
  2824. {
  2825. return __cancel_work(work, false);
  2826. }
  2827. EXPORT_SYMBOL(cancel_work);
  2828. /**
  2829. * cancel_delayed_work - cancel a delayed work
  2830. * @dwork: delayed_work to cancel
  2831. *
  2832. * Kill off a pending delayed_work.
  2833. *
  2834. * Return: %true if @dwork was pending and canceled; %false if it wasn't
  2835. * pending.
  2836. *
  2837. * Note:
  2838. * The work callback function may still be running on return, unless
  2839. * it returns %true and the work doesn't re-arm itself. Explicitly flush or
  2840. * use cancel_delayed_work_sync() to wait on it.
  2841. *
  2842. * This function is safe to call from any context including IRQ handler.
  2843. */
  2844. bool cancel_delayed_work(struct delayed_work *dwork)
  2845. {
  2846. return __cancel_work(&dwork->work, true);
  2847. }
  2848. EXPORT_SYMBOL(cancel_delayed_work);
  2849. /**
  2850. * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
  2851. * @dwork: the delayed work cancel
  2852. *
  2853. * This is cancel_work_sync() for delayed works.
  2854. *
  2855. * Return:
  2856. * %true if @dwork was pending, %false otherwise.
  2857. */
  2858. bool cancel_delayed_work_sync(struct delayed_work *dwork)
  2859. {
  2860. return __cancel_work_timer(&dwork->work, true);
  2861. }
  2862. EXPORT_SYMBOL(cancel_delayed_work_sync);
  2863. /**
  2864. * schedule_on_each_cpu - execute a function synchronously on each online CPU
  2865. * @func: the function to call
  2866. *
  2867. * schedule_on_each_cpu() executes @func on each online CPU using the
  2868. * system workqueue and blocks until all CPUs have completed.
  2869. * schedule_on_each_cpu() is very slow.
  2870. *
  2871. * Return:
  2872. * 0 on success, -errno on failure.
  2873. */
  2874. int schedule_on_each_cpu(work_func_t func)
  2875. {
  2876. int cpu;
  2877. struct work_struct __percpu *works;
  2878. works = alloc_percpu(struct work_struct);
  2879. if (!works)
  2880. return -ENOMEM;
  2881. cpus_read_lock();
  2882. for_each_online_cpu(cpu) {
  2883. struct work_struct *work = per_cpu_ptr(works, cpu);
  2884. INIT_WORK(work, func);
  2885. schedule_work_on(cpu, work);
  2886. }
  2887. for_each_online_cpu(cpu)
  2888. flush_work(per_cpu_ptr(works, cpu));
  2889. cpus_read_unlock();
  2890. free_percpu(works);
  2891. return 0;
  2892. }
  2893. /**
  2894. * execute_in_process_context - reliably execute the routine with user context
  2895. * @fn: the function to execute
  2896. * @ew: guaranteed storage for the execute work structure (must
  2897. * be available when the work executes)
  2898. *
  2899. * Executes the function immediately if process context is available,
  2900. * otherwise schedules the function for delayed execution.
  2901. *
  2902. * Return: 0 - function was executed
  2903. * 1 - function was scheduled for execution
  2904. */
  2905. int execute_in_process_context(work_func_t fn, struct execute_work *ew)
  2906. {
  2907. if (!in_interrupt()) {
  2908. fn(&ew->work);
  2909. return 0;
  2910. }
  2911. INIT_WORK(&ew->work, fn);
  2912. schedule_work(&ew->work);
  2913. return 1;
  2914. }
  2915. EXPORT_SYMBOL_GPL(execute_in_process_context);
  2916. /**
  2917. * free_workqueue_attrs - free a workqueue_attrs
  2918. * @attrs: workqueue_attrs to free
  2919. *
  2920. * Undo alloc_workqueue_attrs().
  2921. */
  2922. void free_workqueue_attrs(struct workqueue_attrs *attrs)
  2923. {
  2924. if (attrs) {
  2925. free_cpumask_var(attrs->cpumask);
  2926. kfree(attrs);
  2927. }
  2928. }
  2929. /**
  2930. * alloc_workqueue_attrs - allocate a workqueue_attrs
  2931. *
  2932. * Allocate a new workqueue_attrs, initialize with default settings and
  2933. * return it.
  2934. *
  2935. * Return: The allocated new workqueue_attr on success. %NULL on failure.
  2936. */
  2937. struct workqueue_attrs *alloc_workqueue_attrs(void)
  2938. {
  2939. struct workqueue_attrs *attrs;
  2940. attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
  2941. if (!attrs)
  2942. goto fail;
  2943. if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
  2944. goto fail;
  2945. cpumask_copy(attrs->cpumask, cpu_possible_mask);
  2946. return attrs;
  2947. fail:
  2948. free_workqueue_attrs(attrs);
  2949. return NULL;
  2950. }
  2951. static void copy_workqueue_attrs(struct workqueue_attrs *to,
  2952. const struct workqueue_attrs *from)
  2953. {
  2954. to->nice = from->nice;
  2955. cpumask_copy(to->cpumask, from->cpumask);
  2956. /*
  2957. * Unlike hash and equality test, this function doesn't ignore
  2958. * ->no_numa as it is used for both pool and wq attrs. Instead,
  2959. * get_unbound_pool() explicitly clears ->no_numa after copying.
  2960. */
  2961. to->no_numa = from->no_numa;
  2962. }
  2963. /* hash value of the content of @attr */
  2964. static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
  2965. {
  2966. u32 hash = 0;
  2967. hash = jhash_1word(attrs->nice, hash);
  2968. hash = jhash(cpumask_bits(attrs->cpumask),
  2969. BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
  2970. return hash;
  2971. }
  2972. /* content equality test */
  2973. static bool wqattrs_equal(const struct workqueue_attrs *a,
  2974. const struct workqueue_attrs *b)
  2975. {
  2976. if (a->nice != b->nice)
  2977. return false;
  2978. if (!cpumask_equal(a->cpumask, b->cpumask))
  2979. return false;
  2980. return true;
  2981. }
  2982. /**
  2983. * init_worker_pool - initialize a newly zalloc'd worker_pool
  2984. * @pool: worker_pool to initialize
  2985. *
  2986. * Initialize a newly zalloc'd @pool. It also allocates @pool->attrs.
  2987. *
  2988. * Return: 0 on success, -errno on failure. Even on failure, all fields
  2989. * inside @pool proper are initialized and put_unbound_pool() can be called
  2990. * on @pool safely to release it.
  2991. */
  2992. static int init_worker_pool(struct worker_pool *pool)
  2993. {
  2994. raw_spin_lock_init(&pool->lock);
  2995. pool->id = -1;
  2996. pool->cpu = -1;
  2997. pool->node = NUMA_NO_NODE;
  2998. pool->flags |= POOL_DISASSOCIATED;
  2999. pool->watchdog_ts = jiffies;
  3000. INIT_LIST_HEAD(&pool->worklist);
  3001. INIT_LIST_HEAD(&pool->idle_list);
  3002. hash_init(pool->busy_hash);
  3003. timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
  3004. timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
  3005. INIT_LIST_HEAD(&pool->workers);
  3006. ida_init(&pool->worker_ida);
  3007. INIT_HLIST_NODE(&pool->hash_node);
  3008. pool->refcnt = 1;
  3009. /* shouldn't fail above this point */
  3010. pool->attrs = alloc_workqueue_attrs();
  3011. if (!pool->attrs)
  3012. return -ENOMEM;
  3013. return 0;
  3014. }
  3015. #ifdef CONFIG_LOCKDEP
  3016. static void wq_init_lockdep(struct workqueue_struct *wq)
  3017. {
  3018. char *lock_name;
  3019. lockdep_register_key(&wq->key);
  3020. lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
  3021. if (!lock_name)
  3022. lock_name = wq->name;
  3023. wq->lock_name = lock_name;
  3024. lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
  3025. }
  3026. static void wq_unregister_lockdep(struct workqueue_struct *wq)
  3027. {
  3028. lockdep_unregister_key(&wq->key);
  3029. }
  3030. static void wq_free_lockdep(struct workqueue_struct *wq)
  3031. {
  3032. if (wq->lock_name != wq->name)
  3033. kfree(wq->lock_name);
  3034. }
  3035. #else
  3036. static void wq_init_lockdep(struct workqueue_struct *wq)
  3037. {
  3038. }
  3039. static void wq_unregister_lockdep(struct workqueue_struct *wq)
  3040. {
  3041. }
  3042. static void wq_free_lockdep(struct workqueue_struct *wq)
  3043. {
  3044. }
  3045. #endif
  3046. static void rcu_free_wq(struct rcu_head *rcu)
  3047. {
  3048. struct workqueue_struct *wq =
  3049. container_of(rcu, struct workqueue_struct, rcu);
  3050. wq_free_lockdep(wq);
  3051. if (!(wq->flags & WQ_UNBOUND))
  3052. free_percpu(wq->cpu_pwqs);
  3053. else
  3054. free_workqueue_attrs(wq->unbound_attrs);
  3055. kfree(wq);
  3056. }
  3057. static void rcu_free_pool(struct rcu_head *rcu)
  3058. {
  3059. struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
  3060. ida_destroy(&pool->worker_ida);
  3061. free_workqueue_attrs(pool->attrs);
  3062. kfree(pool);
  3063. }
  3064. /* This returns with the lock held on success (pool manager is inactive). */
  3065. static bool wq_manager_inactive(struct worker_pool *pool)
  3066. {
  3067. raw_spin_lock_irq(&pool->lock);
  3068. if (pool->flags & POOL_MANAGER_ACTIVE) {
  3069. raw_spin_unlock_irq(&pool->lock);
  3070. return false;
  3071. }
  3072. return true;
  3073. }
  3074. /**
  3075. * put_unbound_pool - put a worker_pool
  3076. * @pool: worker_pool to put
  3077. *
  3078. * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU
  3079. * safe manner. get_unbound_pool() calls this function on its failure path
  3080. * and this function should be able to release pools which went through,
  3081. * successfully or not, init_worker_pool().
  3082. *
  3083. * Should be called with wq_pool_mutex held.
  3084. */
  3085. static void put_unbound_pool(struct worker_pool *pool)
  3086. {
  3087. DECLARE_COMPLETION_ONSTACK(detach_completion);
  3088. struct worker *worker;
  3089. lockdep_assert_held(&wq_pool_mutex);
  3090. if (--pool->refcnt)
  3091. return;
  3092. /* sanity checks */
  3093. if (WARN_ON(!(pool->cpu < 0)) ||
  3094. WARN_ON(!list_empty(&pool->worklist)))
  3095. return;
  3096. /* release id and unhash */
  3097. if (pool->id >= 0)
  3098. idr_remove(&worker_pool_idr, pool->id);
  3099. hash_del(&pool->hash_node);
  3100. /*
  3101. * Become the manager and destroy all workers. This prevents
  3102. * @pool's workers from blocking on attach_mutex. We're the last
  3103. * manager and @pool gets freed with the flag set.
  3104. * Because of how wq_manager_inactive() works, we will hold the
  3105. * spinlock after a successful wait.
  3106. */
  3107. rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
  3108. TASK_UNINTERRUPTIBLE);
  3109. pool->flags |= POOL_MANAGER_ACTIVE;
  3110. while ((worker = first_idle_worker(pool)))
  3111. destroy_worker(worker);
  3112. WARN_ON(pool->nr_workers || pool->nr_idle);
  3113. raw_spin_unlock_irq(&pool->lock);
  3114. mutex_lock(&wq_pool_attach_mutex);
  3115. if (!list_empty(&pool->workers))
  3116. pool->detach_completion = &detach_completion;
  3117. mutex_unlock(&wq_pool_attach_mutex);
  3118. if (pool->detach_completion)
  3119. wait_for_completion(pool->detach_completion);
  3120. /* shut down the timers */
  3121. del_timer_sync(&pool->idle_timer);
  3122. del_timer_sync(&pool->mayday_timer);
  3123. /* RCU protected to allow dereferences from get_work_pool() */
  3124. call_rcu(&pool->rcu, rcu_free_pool);
  3125. }
  3126. /**
  3127. * get_unbound_pool - get a worker_pool with the specified attributes
  3128. * @attrs: the attributes of the worker_pool to get
  3129. *
  3130. * Obtain a worker_pool which has the same attributes as @attrs, bump the
  3131. * reference count and return it. If there already is a matching
  3132. * worker_pool, it will be used; otherwise, this function attempts to
  3133. * create a new one.
  3134. *
  3135. * Should be called with wq_pool_mutex held.
  3136. *
  3137. * Return: On success, a worker_pool with the same attributes as @attrs.
  3138. * On failure, %NULL.
  3139. */
  3140. static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
  3141. {
  3142. u32 hash = wqattrs_hash(attrs);
  3143. struct worker_pool *pool;
  3144. int node;
  3145. int target_node = NUMA_NO_NODE;
  3146. lockdep_assert_held(&wq_pool_mutex);
  3147. /* do we already have a matching pool? */
  3148. hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
  3149. if (wqattrs_equal(pool->attrs, attrs)) {
  3150. pool->refcnt++;
  3151. return pool;
  3152. }
  3153. }
  3154. /* if cpumask is contained inside a NUMA node, we belong to that node */
  3155. if (wq_numa_enabled) {
  3156. for_each_node(node) {
  3157. if (cpumask_subset(attrs->cpumask,
  3158. wq_numa_possible_cpumask[node])) {
  3159. target_node = node;
  3160. break;
  3161. }
  3162. }
  3163. }
  3164. /* nope, create a new one */
  3165. pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
  3166. if (!pool || init_worker_pool(pool) < 0)
  3167. goto fail;
  3168. lockdep_set_subclass(&pool->lock, 1); /* see put_pwq() */
  3169. copy_workqueue_attrs(pool->attrs, attrs);
  3170. pool->node = target_node;
  3171. /*
  3172. * no_numa isn't a worker_pool attribute, always clear it. See
  3173. * 'struct workqueue_attrs' comments for detail.
  3174. */
  3175. pool->attrs->no_numa = false;
  3176. if (worker_pool_assign_id(pool) < 0)
  3177. goto fail;
  3178. /* create and start the initial worker */
  3179. if (wq_online && !create_worker(pool))
  3180. goto fail;
  3181. /* install */
  3182. hash_add(unbound_pool_hash, &pool->hash_node, hash);
  3183. return pool;
  3184. fail:
  3185. if (pool)
  3186. put_unbound_pool(pool);
  3187. return NULL;
  3188. }
  3189. static void rcu_free_pwq(struct rcu_head *rcu)
  3190. {
  3191. kmem_cache_free(pwq_cache,
  3192. container_of(rcu, struct pool_workqueue, rcu));
  3193. }
  3194. /*
  3195. * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
  3196. * and needs to be destroyed.
  3197. */
  3198. static void pwq_unbound_release_workfn(struct work_struct *work)
  3199. {
  3200. struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
  3201. unbound_release_work);
  3202. struct workqueue_struct *wq = pwq->wq;
  3203. struct worker_pool *pool = pwq->pool;
  3204. bool is_last = false;
  3205. /*
  3206. * when @pwq is not linked, it doesn't hold any reference to the
  3207. * @wq, and @wq is invalid to access.
  3208. */
  3209. if (!list_empty(&pwq->pwqs_node)) {
  3210. if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
  3211. return;
  3212. mutex_lock(&wq->mutex);
  3213. list_del_rcu(&pwq->pwqs_node);
  3214. is_last = list_empty(&wq->pwqs);
  3215. mutex_unlock(&wq->mutex);
  3216. }
  3217. mutex_lock(&wq_pool_mutex);
  3218. put_unbound_pool(pool);
  3219. mutex_unlock(&wq_pool_mutex);
  3220. call_rcu(&pwq->rcu, rcu_free_pwq);
  3221. /*
  3222. * If we're the last pwq going away, @wq is already dead and no one
  3223. * is gonna access it anymore. Schedule RCU free.
  3224. */
  3225. if (is_last) {
  3226. wq_unregister_lockdep(wq);
  3227. call_rcu(&wq->rcu, rcu_free_wq);
  3228. }
  3229. }
  3230. /**
  3231. * pwq_adjust_max_active - update a pwq's max_active to the current setting
  3232. * @pwq: target pool_workqueue
  3233. *
  3234. * If @pwq isn't freezing, set @pwq->max_active to the associated
  3235. * workqueue's saved_max_active and activate inactive work items
  3236. * accordingly. If @pwq is freezing, clear @pwq->max_active to zero.
  3237. */
  3238. static void pwq_adjust_max_active(struct pool_workqueue *pwq)
  3239. {
  3240. struct workqueue_struct *wq = pwq->wq;
  3241. bool freezable = wq->flags & WQ_FREEZABLE;
  3242. unsigned long flags;
  3243. /* for @wq->saved_max_active */
  3244. lockdep_assert_held(&wq->mutex);
  3245. /* fast exit for non-freezable wqs */
  3246. if (!freezable && pwq->max_active == wq->saved_max_active)
  3247. return;
  3248. /* this function can be called during early boot w/ irq disabled */
  3249. raw_spin_lock_irqsave(&pwq->pool->lock, flags);
  3250. /*
  3251. * During [un]freezing, the caller is responsible for ensuring that
  3252. * this function is called at least once after @workqueue_freezing
  3253. * is updated and visible.
  3254. */
  3255. if (!freezable || !workqueue_freezing) {
  3256. bool kick = false;
  3257. pwq->max_active = wq->saved_max_active;
  3258. while (!list_empty(&pwq->inactive_works) &&
  3259. pwq->nr_active < pwq->max_active) {
  3260. pwq_activate_first_inactive(pwq);
  3261. kick = true;
  3262. }
  3263. /*
  3264. * Need to kick a worker after thawed or an unbound wq's
  3265. * max_active is bumped. In realtime scenarios, always kicking a
  3266. * worker will cause interference on the isolated cpu cores, so
  3267. * let's kick iff work items were activated.
  3268. */
  3269. if (kick)
  3270. wake_up_worker(pwq->pool);
  3271. } else {
  3272. pwq->max_active = 0;
  3273. }
  3274. raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
  3275. }
  3276. /* initialize newly allocated @pwq which is associated with @wq and @pool */
  3277. static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
  3278. struct worker_pool *pool)
  3279. {
  3280. BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
  3281. memset(pwq, 0, sizeof(*pwq));
  3282. pwq->pool = pool;
  3283. pwq->wq = wq;
  3284. pwq->flush_color = -1;
  3285. pwq->refcnt = 1;
  3286. INIT_LIST_HEAD(&pwq->inactive_works);
  3287. INIT_LIST_HEAD(&pwq->pwqs_node);
  3288. INIT_LIST_HEAD(&pwq->mayday_node);
  3289. INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
  3290. }
  3291. /* sync @pwq with the current state of its associated wq and link it */
  3292. static void link_pwq(struct pool_workqueue *pwq)
  3293. {
  3294. struct workqueue_struct *wq = pwq->wq;
  3295. lockdep_assert_held(&wq->mutex);
  3296. /* may be called multiple times, ignore if already linked */
  3297. if (!list_empty(&pwq->pwqs_node))
  3298. return;
  3299. /* set the matching work_color */
  3300. pwq->work_color = wq->work_color;
  3301. /* sync max_active to the current setting */
  3302. pwq_adjust_max_active(pwq);
  3303. /* link in @pwq */
  3304. list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
  3305. }
  3306. /* obtain a pool matching @attr and create a pwq associating the pool and @wq */
  3307. static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
  3308. const struct workqueue_attrs *attrs)
  3309. {
  3310. struct worker_pool *pool;
  3311. struct pool_workqueue *pwq;
  3312. lockdep_assert_held(&wq_pool_mutex);
  3313. pool = get_unbound_pool(attrs);
  3314. if (!pool)
  3315. return NULL;
  3316. pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
  3317. if (!pwq) {
  3318. put_unbound_pool(pool);
  3319. return NULL;
  3320. }
  3321. init_pwq(pwq, wq, pool);
  3322. return pwq;
  3323. }
  3324. /**
  3325. * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
  3326. * @attrs: the wq_attrs of the default pwq of the target workqueue
  3327. * @node: the target NUMA node
  3328. * @cpu_going_down: if >= 0, the CPU to consider as offline
  3329. * @cpumask: outarg, the resulting cpumask
  3330. *
  3331. * Calculate the cpumask a workqueue with @attrs should use on @node. If
  3332. * @cpu_going_down is >= 0, that cpu is considered offline during
  3333. * calculation. The result is stored in @cpumask.
  3334. *
  3335. * If NUMA affinity is not enabled, @attrs->cpumask is always used. If
  3336. * enabled and @node has online CPUs requested by @attrs, the returned
  3337. * cpumask is the intersection of the possible CPUs of @node and
  3338. * @attrs->cpumask.
  3339. *
  3340. * The caller is responsible for ensuring that the cpumask of @node stays
  3341. * stable.
  3342. *
  3343. * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
  3344. * %false if equal.
  3345. */
  3346. static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
  3347. int cpu_going_down, cpumask_t *cpumask)
  3348. {
  3349. if (!wq_numa_enabled || attrs->no_numa)
  3350. goto use_dfl;
  3351. /* does @node have any online CPUs @attrs wants? */
  3352. cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
  3353. if (cpu_going_down >= 0)
  3354. cpumask_clear_cpu(cpu_going_down, cpumask);
  3355. if (cpumask_empty(cpumask))
  3356. goto use_dfl;
  3357. /* yeap, return possible CPUs in @node that @attrs wants */
  3358. cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
  3359. if (cpumask_empty(cpumask)) {
  3360. pr_warn_once("WARNING: workqueue cpumask: online intersect > "
  3361. "possible intersect\n");
  3362. return false;
  3363. }
  3364. return !cpumask_equal(cpumask, attrs->cpumask);
  3365. use_dfl:
  3366. cpumask_copy(cpumask, attrs->cpumask);
  3367. return false;
  3368. }
  3369. /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
  3370. static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
  3371. int node,
  3372. struct pool_workqueue *pwq)
  3373. {
  3374. struct pool_workqueue *old_pwq;
  3375. lockdep_assert_held(&wq_pool_mutex);
  3376. lockdep_assert_held(&wq->mutex);
  3377. /* link_pwq() can handle duplicate calls */
  3378. link_pwq(pwq);
  3379. old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
  3380. rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
  3381. return old_pwq;
  3382. }
  3383. /* context to store the prepared attrs & pwqs before applying */
  3384. struct apply_wqattrs_ctx {
  3385. struct workqueue_struct *wq; /* target workqueue */
  3386. struct workqueue_attrs *attrs; /* attrs to apply */
  3387. struct list_head list; /* queued for batching commit */
  3388. struct pool_workqueue *dfl_pwq;
  3389. struct pool_workqueue *pwq_tbl[];
  3390. };
  3391. /* free the resources after success or abort */
  3392. static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
  3393. {
  3394. if (ctx) {
  3395. int node;
  3396. for_each_node(node)
  3397. put_pwq_unlocked(ctx->pwq_tbl[node]);
  3398. put_pwq_unlocked(ctx->dfl_pwq);
  3399. free_workqueue_attrs(ctx->attrs);
  3400. kfree(ctx);
  3401. }
  3402. }
  3403. /* allocate the attrs and pwqs for later installation */
  3404. static struct apply_wqattrs_ctx *
  3405. apply_wqattrs_prepare(struct workqueue_struct *wq,
  3406. const struct workqueue_attrs *attrs,
  3407. const cpumask_var_t unbound_cpumask)
  3408. {
  3409. struct apply_wqattrs_ctx *ctx;
  3410. struct workqueue_attrs *new_attrs, *tmp_attrs;
  3411. int node;
  3412. lockdep_assert_held(&wq_pool_mutex);
  3413. ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
  3414. new_attrs = alloc_workqueue_attrs();
  3415. tmp_attrs = alloc_workqueue_attrs();
  3416. if (!ctx || !new_attrs || !tmp_attrs)
  3417. goto out_free;
  3418. /*
  3419. * Calculate the attrs of the default pwq with unbound_cpumask
  3420. * which is wq_unbound_cpumask or to set to wq_unbound_cpumask.
  3421. * If the user configured cpumask doesn't overlap with the
  3422. * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
  3423. */
  3424. copy_workqueue_attrs(new_attrs, attrs);
  3425. cpumask_and(new_attrs->cpumask, new_attrs->cpumask, unbound_cpumask);
  3426. if (unlikely(cpumask_empty(new_attrs->cpumask)))
  3427. cpumask_copy(new_attrs->cpumask, unbound_cpumask);
  3428. /*
  3429. * We may create multiple pwqs with differing cpumasks. Make a
  3430. * copy of @new_attrs which will be modified and used to obtain
  3431. * pools.
  3432. */
  3433. copy_workqueue_attrs(tmp_attrs, new_attrs);
  3434. /*
  3435. * If something goes wrong during CPU up/down, we'll fall back to
  3436. * the default pwq covering whole @attrs->cpumask. Always create
  3437. * it even if we don't use it immediately.
  3438. */
  3439. ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
  3440. if (!ctx->dfl_pwq)
  3441. goto out_free;
  3442. for_each_node(node) {
  3443. if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
  3444. ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
  3445. if (!ctx->pwq_tbl[node])
  3446. goto out_free;
  3447. } else {
  3448. ctx->dfl_pwq->refcnt++;
  3449. ctx->pwq_tbl[node] = ctx->dfl_pwq;
  3450. }
  3451. }
  3452. /* save the user configured attrs and sanitize it. */
  3453. copy_workqueue_attrs(new_attrs, attrs);
  3454. cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
  3455. ctx->attrs = new_attrs;
  3456. ctx->wq = wq;
  3457. free_workqueue_attrs(tmp_attrs);
  3458. return ctx;
  3459. out_free:
  3460. free_workqueue_attrs(tmp_attrs);
  3461. free_workqueue_attrs(new_attrs);
  3462. apply_wqattrs_cleanup(ctx);
  3463. return NULL;
  3464. }
  3465. /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
  3466. static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
  3467. {
  3468. int node;
  3469. /* all pwqs have been created successfully, let's install'em */
  3470. mutex_lock(&ctx->wq->mutex);
  3471. copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
  3472. /* save the previous pwq and install the new one */
  3473. for_each_node(node)
  3474. ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
  3475. ctx->pwq_tbl[node]);
  3476. /* @dfl_pwq might not have been used, ensure it's linked */
  3477. link_pwq(ctx->dfl_pwq);
  3478. swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
  3479. mutex_unlock(&ctx->wq->mutex);
  3480. }
  3481. static void apply_wqattrs_lock(void)
  3482. {
  3483. /* CPUs should stay stable across pwq creations and installations */
  3484. cpus_read_lock();
  3485. mutex_lock(&wq_pool_mutex);
  3486. }
  3487. static void apply_wqattrs_unlock(void)
  3488. {
  3489. mutex_unlock(&wq_pool_mutex);
  3490. cpus_read_unlock();
  3491. }
  3492. static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
  3493. const struct workqueue_attrs *attrs)
  3494. {
  3495. struct apply_wqattrs_ctx *ctx;
  3496. /* only unbound workqueues can change attributes */
  3497. if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
  3498. return -EINVAL;
  3499. /* creating multiple pwqs breaks ordering guarantee */
  3500. if (!list_empty(&wq->pwqs)) {
  3501. if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
  3502. return -EINVAL;
  3503. wq->flags &= ~__WQ_ORDERED;
  3504. }
  3505. ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);
  3506. if (!ctx)
  3507. return -ENOMEM;
  3508. /* the ctx has been prepared successfully, let's commit it */
  3509. apply_wqattrs_commit(ctx);
  3510. apply_wqattrs_cleanup(ctx);
  3511. return 0;
  3512. }
  3513. /**
  3514. * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
  3515. * @wq: the target workqueue
  3516. * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
  3517. *
  3518. * Apply @attrs to an unbound workqueue @wq. Unless disabled, on NUMA
  3519. * machines, this function maps a separate pwq to each NUMA node with
  3520. * possibles CPUs in @attrs->cpumask so that work items are affine to the
  3521. * NUMA node it was issued on. Older pwqs are released as in-flight work
  3522. * items finish. Note that a work item which repeatedly requeues itself
  3523. * back-to-back will stay on its current pwq.
  3524. *
  3525. * Performs GFP_KERNEL allocations.
  3526. *
  3527. * Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
  3528. *
  3529. * Return: 0 on success and -errno on failure.
  3530. */
  3531. int apply_workqueue_attrs(struct workqueue_struct *wq,
  3532. const struct workqueue_attrs *attrs)
  3533. {
  3534. int ret;
  3535. lockdep_assert_cpus_held();
  3536. mutex_lock(&wq_pool_mutex);
  3537. ret = apply_workqueue_attrs_locked(wq, attrs);
  3538. mutex_unlock(&wq_pool_mutex);
  3539. return ret;
  3540. }
  3541. /**
  3542. * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
  3543. * @wq: the target workqueue
  3544. * @cpu: the CPU coming up or going down
  3545. * @online: whether @cpu is coming up or going down
  3546. *
  3547. * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
  3548. * %CPU_DOWN_FAILED. @cpu is being hot[un]plugged, update NUMA affinity of
  3549. * @wq accordingly.
  3550. *
  3551. * If NUMA affinity can't be adjusted due to memory allocation failure, it
  3552. * falls back to @wq->dfl_pwq which may not be optimal but is always
  3553. * correct.
  3554. *
  3555. * Note that when the last allowed CPU of a NUMA node goes offline for a
  3556. * workqueue with a cpumask spanning multiple nodes, the workers which were
  3557. * already executing the work items for the workqueue will lose their CPU
  3558. * affinity and may execute on any CPU. This is similar to how per-cpu
  3559. * workqueues behave on CPU_DOWN. If a workqueue user wants strict
  3560. * affinity, it's the user's responsibility to flush the work item from
  3561. * CPU_DOWN_PREPARE.
  3562. */
  3563. static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
  3564. bool online)
  3565. {
  3566. int node = cpu_to_node(cpu);
  3567. int cpu_off = online ? -1 : cpu;
  3568. struct pool_workqueue *old_pwq = NULL, *pwq;
  3569. struct workqueue_attrs *target_attrs;
  3570. cpumask_t *cpumask;
  3571. lockdep_assert_held(&wq_pool_mutex);
  3572. if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||
  3573. wq->unbound_attrs->no_numa)
  3574. return;
  3575. /*
  3576. * We don't wanna alloc/free wq_attrs for each wq for each CPU.
  3577. * Let's use a preallocated one. The following buf is protected by
  3578. * CPU hotplug exclusion.
  3579. */
  3580. target_attrs = wq_update_unbound_numa_attrs_buf;
  3581. cpumask = target_attrs->cpumask;
  3582. copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
  3583. pwq = unbound_pwq_by_node(wq, node);
  3584. /*
  3585. * Let's determine what needs to be done. If the target cpumask is
  3586. * different from the default pwq's, we need to compare it to @pwq's
  3587. * and create a new one if they don't match. If the target cpumask
  3588. * equals the default pwq's, the default pwq should be used.
  3589. */
  3590. if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
  3591. if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
  3592. return;
  3593. } else {
  3594. goto use_dfl_pwq;
  3595. }
  3596. /* create a new pwq */
  3597. pwq = alloc_unbound_pwq(wq, target_attrs);
  3598. if (!pwq) {
  3599. pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
  3600. wq->name);
  3601. goto use_dfl_pwq;
  3602. }
  3603. /* Install the new pwq. */
  3604. mutex_lock(&wq->mutex);
  3605. old_pwq = numa_pwq_tbl_install(wq, node, pwq);
  3606. goto out_unlock;
  3607. use_dfl_pwq:
  3608. mutex_lock(&wq->mutex);
  3609. raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
  3610. get_pwq(wq->dfl_pwq);
  3611. raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
  3612. old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
  3613. out_unlock:
  3614. mutex_unlock(&wq->mutex);
  3615. put_pwq_unlocked(old_pwq);
  3616. }
  3617. static int alloc_and_link_pwqs(struct workqueue_struct *wq)
  3618. {
  3619. bool highpri = wq->flags & WQ_HIGHPRI;
  3620. int cpu, ret;
  3621. if (!(wq->flags & WQ_UNBOUND)) {
  3622. wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
  3623. if (!wq->cpu_pwqs)
  3624. return -ENOMEM;
  3625. for_each_possible_cpu(cpu) {
  3626. struct pool_workqueue *pwq =
  3627. per_cpu_ptr(wq->cpu_pwqs, cpu);
  3628. struct worker_pool *cpu_pools =
  3629. per_cpu(cpu_worker_pools, cpu);
  3630. init_pwq(pwq, wq, &cpu_pools[highpri]);
  3631. mutex_lock(&wq->mutex);
  3632. link_pwq(pwq);
  3633. mutex_unlock(&wq->mutex);
  3634. }
  3635. return 0;
  3636. }
  3637. cpus_read_lock();
  3638. if (wq->flags & __WQ_ORDERED) {
  3639. ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
  3640. /* there should only be single pwq for ordering guarantee */
  3641. WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
  3642. wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
  3643. "ordering guarantee broken for workqueue %s\n", wq->name);
  3644. } else {
  3645. ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
  3646. }
  3647. cpus_read_unlock();
  3648. return ret;
  3649. }
  3650. static int wq_clamp_max_active(int max_active, unsigned int flags,
  3651. const char *name)
  3652. {
  3653. int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
  3654. if (max_active < 1 || max_active > lim)
  3655. pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
  3656. max_active, name, 1, lim);
  3657. return clamp_val(max_active, 1, lim);
  3658. }
  3659. /*
  3660. * Workqueues which may be used during memory reclaim should have a rescuer
  3661. * to guarantee forward progress.
  3662. */
  3663. static int init_rescuer(struct workqueue_struct *wq)
  3664. {
  3665. struct worker *rescuer;
  3666. int ret;
  3667. if (!(wq->flags & WQ_MEM_RECLAIM))
  3668. return 0;
  3669. rescuer = alloc_worker(NUMA_NO_NODE);
  3670. if (!rescuer)
  3671. return -ENOMEM;
  3672. rescuer->rescue_wq = wq;
  3673. rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
  3674. if (IS_ERR(rescuer->task)) {
  3675. ret = PTR_ERR(rescuer->task);
  3676. kfree(rescuer);
  3677. return ret;
  3678. }
  3679. wq->rescuer = rescuer;
  3680. kthread_bind_mask(rescuer->task, cpu_possible_mask);
  3681. wake_up_process(rescuer->task);
  3682. return 0;
  3683. }
  3684. __printf(1, 4)
  3685. struct workqueue_struct *alloc_workqueue(const char *fmt,
  3686. unsigned int flags,
  3687. int max_active, ...)
  3688. {
  3689. size_t tbl_size = 0;
  3690. va_list args;
  3691. struct workqueue_struct *wq;
  3692. struct pool_workqueue *pwq;
  3693. /*
  3694. * Unbound && max_active == 1 used to imply ordered, which is no
  3695. * longer the case on NUMA machines due to per-node pools. While
  3696. * alloc_ordered_workqueue() is the right way to create an ordered
  3697. * workqueue, keep the previous behavior to avoid subtle breakages
  3698. * on NUMA.
  3699. */
  3700. if ((flags & WQ_UNBOUND) && max_active == 1)
  3701. flags |= __WQ_ORDERED;
  3702. /* see the comment above the definition of WQ_POWER_EFFICIENT */
  3703. if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
  3704. flags |= WQ_UNBOUND;
  3705. /* allocate wq and format name */
  3706. if (flags & WQ_UNBOUND)
  3707. tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
  3708. wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
  3709. if (!wq)
  3710. return NULL;
  3711. if (flags & WQ_UNBOUND) {
  3712. wq->unbound_attrs = alloc_workqueue_attrs();
  3713. if (!wq->unbound_attrs)
  3714. goto err_free_wq;
  3715. }
  3716. va_start(args, max_active);
  3717. vsnprintf(wq->name, sizeof(wq->name), fmt, args);
  3718. va_end(args);
  3719. max_active = max_active ?: WQ_DFL_ACTIVE;
  3720. max_active = wq_clamp_max_active(max_active, flags, wq->name);
  3721. /* init wq */
  3722. wq->flags = flags;
  3723. wq->saved_max_active = max_active;
  3724. mutex_init(&wq->mutex);
  3725. atomic_set(&wq->nr_pwqs_to_flush, 0);
  3726. INIT_LIST_HEAD(&wq->pwqs);
  3727. INIT_LIST_HEAD(&wq->flusher_queue);
  3728. INIT_LIST_HEAD(&wq->flusher_overflow);
  3729. INIT_LIST_HEAD(&wq->maydays);
  3730. wq_init_lockdep(wq);
  3731. INIT_LIST_HEAD(&wq->list);
  3732. if (alloc_and_link_pwqs(wq) < 0)
  3733. goto err_unreg_lockdep;
  3734. if (wq_online && init_rescuer(wq) < 0)
  3735. goto err_destroy;
  3736. if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
  3737. goto err_destroy;
  3738. /*
  3739. * wq_pool_mutex protects global freeze state and workqueues list.
  3740. * Grab it, adjust max_active and add the new @wq to workqueues
  3741. * list.
  3742. */
  3743. mutex_lock(&wq_pool_mutex);
  3744. mutex_lock(&wq->mutex);
  3745. for_each_pwq(pwq, wq)
  3746. pwq_adjust_max_active(pwq);
  3747. mutex_unlock(&wq->mutex);
  3748. list_add_tail_rcu(&wq->list, &workqueues);
  3749. mutex_unlock(&wq_pool_mutex);
  3750. return wq;
  3751. err_unreg_lockdep:
  3752. wq_unregister_lockdep(wq);
  3753. wq_free_lockdep(wq);
  3754. err_free_wq:
  3755. free_workqueue_attrs(wq->unbound_attrs);
  3756. kfree(wq);
  3757. return NULL;
  3758. err_destroy:
  3759. destroy_workqueue(wq);
  3760. return NULL;
  3761. }
  3762. EXPORT_SYMBOL_GPL(alloc_workqueue);
  3763. static bool pwq_busy(struct pool_workqueue *pwq)
  3764. {
  3765. int i;
  3766. for (i = 0; i < WORK_NR_COLORS; i++)
  3767. if (pwq->nr_in_flight[i])
  3768. return true;
  3769. if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
  3770. return true;
  3771. if (pwq->nr_active || !list_empty(&pwq->inactive_works))
  3772. return true;
  3773. return false;
  3774. }
  3775. /**
  3776. * destroy_workqueue - safely terminate a workqueue
  3777. * @wq: target workqueue
  3778. *
  3779. * Safely destroy a workqueue. All work currently pending will be done first.
  3780. */
  3781. void destroy_workqueue(struct workqueue_struct *wq)
  3782. {
  3783. struct pool_workqueue *pwq;
  3784. int node;
  3785. /*
  3786. * Remove it from sysfs first so that sanity check failure doesn't
  3787. * lead to sysfs name conflicts.
  3788. */
  3789. workqueue_sysfs_unregister(wq);
  3790. /* drain it before proceeding with destruction */
  3791. drain_workqueue(wq);
  3792. /* kill rescuer, if sanity checks fail, leave it w/o rescuer */
  3793. if (wq->rescuer) {
  3794. struct worker *rescuer = wq->rescuer;
  3795. /* this prevents new queueing */
  3796. raw_spin_lock_irq(&wq_mayday_lock);
  3797. wq->rescuer = NULL;
  3798. raw_spin_unlock_irq(&wq_mayday_lock);
  3799. /* rescuer will empty maydays list before exiting */
  3800. kthread_stop(rescuer->task);
  3801. kfree(rescuer);
  3802. }
  3803. /*
  3804. * Sanity checks - grab all the locks so that we wait for all
  3805. * in-flight operations which may do put_pwq().
  3806. */
  3807. mutex_lock(&wq_pool_mutex);
  3808. mutex_lock(&wq->mutex);
  3809. for_each_pwq(pwq, wq) {
  3810. raw_spin_lock_irq(&pwq->pool->lock);
  3811. if (WARN_ON(pwq_busy(pwq))) {
  3812. pr_warn("%s: %s has the following busy pwq\n",
  3813. __func__, wq->name);
  3814. show_pwq(pwq);
  3815. raw_spin_unlock_irq(&pwq->pool->lock);
  3816. mutex_unlock(&wq->mutex);
  3817. mutex_unlock(&wq_pool_mutex);
  3818. show_one_workqueue(wq);
  3819. return;
  3820. }
  3821. raw_spin_unlock_irq(&pwq->pool->lock);
  3822. }
  3823. mutex_unlock(&wq->mutex);
  3824. /*
  3825. * wq list is used to freeze wq, remove from list after
  3826. * flushing is complete in case freeze races us.
  3827. */
  3828. list_del_rcu(&wq->list);
  3829. mutex_unlock(&wq_pool_mutex);
  3830. if (!(wq->flags & WQ_UNBOUND)) {
  3831. wq_unregister_lockdep(wq);
  3832. /*
  3833. * The base ref is never dropped on per-cpu pwqs. Directly
  3834. * schedule RCU free.
  3835. */
  3836. call_rcu(&wq->rcu, rcu_free_wq);
  3837. } else {
  3838. /*
  3839. * We're the sole accessor of @wq at this point. Directly
  3840. * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
  3841. * @wq will be freed when the last pwq is released.
  3842. */
  3843. for_each_node(node) {
  3844. pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
  3845. RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
  3846. put_pwq_unlocked(pwq);
  3847. }
  3848. /*
  3849. * Put dfl_pwq. @wq may be freed any time after dfl_pwq is
  3850. * put. Don't access it afterwards.
  3851. */
  3852. pwq = wq->dfl_pwq;
  3853. wq->dfl_pwq = NULL;
  3854. put_pwq_unlocked(pwq);
  3855. }
  3856. }
  3857. EXPORT_SYMBOL_GPL(destroy_workqueue);
  3858. /**
  3859. * workqueue_set_max_active - adjust max_active of a workqueue
  3860. * @wq: target workqueue
  3861. * @max_active: new max_active value.
  3862. *
  3863. * Set max_active of @wq to @max_active.
  3864. *
  3865. * CONTEXT:
  3866. * Don't call from IRQ context.
  3867. */
  3868. void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
  3869. {
  3870. struct pool_workqueue *pwq;
  3871. /* disallow meddling with max_active for ordered workqueues */
  3872. if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
  3873. return;
  3874. max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
  3875. mutex_lock(&wq->mutex);
  3876. wq->flags &= ~__WQ_ORDERED;
  3877. wq->saved_max_active = max_active;
  3878. for_each_pwq(pwq, wq)
  3879. pwq_adjust_max_active(pwq);
  3880. mutex_unlock(&wq->mutex);
  3881. }
  3882. EXPORT_SYMBOL_GPL(workqueue_set_max_active);
  3883. /**
  3884. * current_work - retrieve %current task's work struct
  3885. *
  3886. * Determine if %current task is a workqueue worker and what it's working on.
  3887. * Useful to find out the context that the %current task is running in.
  3888. *
  3889. * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
  3890. */
  3891. struct work_struct *current_work(void)
  3892. {
  3893. struct worker *worker = current_wq_worker();
  3894. return worker ? worker->current_work : NULL;
  3895. }
  3896. EXPORT_SYMBOL(current_work);
  3897. /**
  3898. * current_is_workqueue_rescuer - is %current workqueue rescuer?
  3899. *
  3900. * Determine whether %current is a workqueue rescuer. Can be used from
  3901. * work functions to determine whether it's being run off the rescuer task.
  3902. *
  3903. * Return: %true if %current is a workqueue rescuer. %false otherwise.
  3904. */
  3905. bool current_is_workqueue_rescuer(void)
  3906. {
  3907. struct worker *worker = current_wq_worker();
  3908. return worker && worker->rescue_wq;
  3909. }
  3910. /**
  3911. * workqueue_congested - test whether a workqueue is congested
  3912. * @cpu: CPU in question
  3913. * @wq: target workqueue
  3914. *
  3915. * Test whether @wq's cpu workqueue for @cpu is congested. There is
  3916. * no synchronization around this function and the test result is
  3917. * unreliable and only useful as advisory hints or for debugging.
  3918. *
  3919. * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
  3920. * Note that both per-cpu and unbound workqueues may be associated with
  3921. * multiple pool_workqueues which have separate congested states. A
  3922. * workqueue being congested on one CPU doesn't mean the workqueue is also
  3923. * contested on other CPUs / NUMA nodes.
  3924. *
  3925. * Return:
  3926. * %true if congested, %false otherwise.
  3927. */
  3928. bool workqueue_congested(int cpu, struct workqueue_struct *wq)
  3929. {
  3930. struct pool_workqueue *pwq;
  3931. bool ret;
  3932. rcu_read_lock();
  3933. preempt_disable();
  3934. if (cpu == WORK_CPU_UNBOUND)
  3935. cpu = smp_processor_id();
  3936. if (!(wq->flags & WQ_UNBOUND))
  3937. pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
  3938. else
  3939. pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
  3940. ret = !list_empty(&pwq->inactive_works);
  3941. preempt_enable();
  3942. rcu_read_unlock();
  3943. return ret;
  3944. }
  3945. EXPORT_SYMBOL_GPL(workqueue_congested);
  3946. /**
  3947. * work_busy - test whether a work is currently pending or running
  3948. * @work: the work to be tested
  3949. *
  3950. * Test whether @work is currently pending or running. There is no
  3951. * synchronization around this function and the test result is
  3952. * unreliable and only useful as advisory hints or for debugging.
  3953. *
  3954. * Return:
  3955. * OR'd bitmask of WORK_BUSY_* bits.
  3956. */
  3957. unsigned int work_busy(struct work_struct *work)
  3958. {
  3959. struct worker_pool *pool;
  3960. unsigned long flags;
  3961. unsigned int ret = 0;
  3962. if (work_pending(work))
  3963. ret |= WORK_BUSY_PENDING;
  3964. rcu_read_lock();
  3965. pool = get_work_pool(work);
  3966. if (pool) {
  3967. raw_spin_lock_irqsave(&pool->lock, flags);
  3968. if (find_worker_executing_work(pool, work))
  3969. ret |= WORK_BUSY_RUNNING;
  3970. raw_spin_unlock_irqrestore(&pool->lock, flags);
  3971. }
  3972. rcu_read_unlock();
  3973. return ret;
  3974. }
  3975. EXPORT_SYMBOL_GPL(work_busy);
  3976. /**
  3977. * set_worker_desc - set description for the current work item
  3978. * @fmt: printf-style format string
  3979. * @...: arguments for the format string
  3980. *
  3981. * This function can be called by a running work function to describe what
  3982. * the work item is about. If the worker task gets dumped, this
  3983. * information will be printed out together to help debugging. The
  3984. * description can be at most WORKER_DESC_LEN including the trailing '\0'.
  3985. */
  3986. void set_worker_desc(const char *fmt, ...)
  3987. {
  3988. struct worker *worker = current_wq_worker();
  3989. va_list args;
  3990. if (worker) {
  3991. va_start(args, fmt);
  3992. vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
  3993. va_end(args);
  3994. }
  3995. }
  3996. EXPORT_SYMBOL_GPL(set_worker_desc);
  3997. /**
  3998. * print_worker_info - print out worker information and description
  3999. * @log_lvl: the log level to use when printing
  4000. * @task: target task
  4001. *
  4002. * If @task is a worker and currently executing a work item, print out the
  4003. * name of the workqueue being serviced and worker description set with
  4004. * set_worker_desc() by the currently executing work item.
  4005. *
  4006. * This function can be safely called on any task as long as the
  4007. * task_struct itself is accessible. While safe, this function isn't
  4008. * synchronized and may print out mixups or garbages of limited length.
  4009. */
  4010. void print_worker_info(const char *log_lvl, struct task_struct *task)
  4011. {
  4012. work_func_t *fn = NULL;
  4013. char name[WQ_NAME_LEN] = { };
  4014. char desc[WORKER_DESC_LEN] = { };
  4015. struct pool_workqueue *pwq = NULL;
  4016. struct workqueue_struct *wq = NULL;
  4017. struct worker *worker;
  4018. if (!(task->flags & PF_WQ_WORKER))
  4019. return;
  4020. /*
  4021. * This function is called without any synchronization and @task
  4022. * could be in any state. Be careful with dereferences.
  4023. */
  4024. worker = kthread_probe_data(task);
  4025. /*
  4026. * Carefully copy the associated workqueue's workfn, name and desc.
  4027. * Keep the original last '\0' in case the original is garbage.
  4028. */
  4029. copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
  4030. copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
  4031. copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
  4032. copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
  4033. copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
  4034. if (fn || name[0] || desc[0]) {
  4035. printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
  4036. if (strcmp(name, desc))
  4037. pr_cont(" (%s)", desc);
  4038. pr_cont("\n");
  4039. }
  4040. }
  4041. static void pr_cont_pool_info(struct worker_pool *pool)
  4042. {
  4043. pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
  4044. if (pool->node != NUMA_NO_NODE)
  4045. pr_cont(" node=%d", pool->node);
  4046. pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
  4047. }
  4048. static void pr_cont_work(bool comma, struct work_struct *work)
  4049. {
  4050. if (work->func == wq_barrier_func) {
  4051. struct wq_barrier *barr;
  4052. barr = container_of(work, struct wq_barrier, work);
  4053. pr_cont("%s BAR(%d)", comma ? "," : "",
  4054. task_pid_nr(barr->task));
  4055. } else {
  4056. pr_cont("%s %ps", comma ? "," : "", work->func);
  4057. }
  4058. }
  4059. static void show_pwq(struct pool_workqueue *pwq)
  4060. {
  4061. struct worker_pool *pool = pwq->pool;
  4062. struct work_struct *work;
  4063. struct worker *worker;
  4064. bool has_in_flight = false, has_pending = false;
  4065. int bkt;
  4066. pr_info(" pwq %d:", pool->id);
  4067. pr_cont_pool_info(pool);
  4068. pr_cont(" active=%d/%d refcnt=%d%s\n",
  4069. pwq->nr_active, pwq->max_active, pwq->refcnt,
  4070. !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
  4071. hash_for_each(pool->busy_hash, bkt, worker, hentry) {
  4072. if (worker->current_pwq == pwq) {
  4073. has_in_flight = true;
  4074. break;
  4075. }
  4076. }
  4077. if (has_in_flight) {
  4078. bool comma = false;
  4079. pr_info(" in-flight:");
  4080. hash_for_each(pool->busy_hash, bkt, worker, hentry) {
  4081. if (worker->current_pwq != pwq)
  4082. continue;
  4083. pr_cont("%s %d%s:%ps", comma ? "," : "",
  4084. task_pid_nr(worker->task),
  4085. worker->rescue_wq ? "(RESCUER)" : "",
  4086. worker->current_func);
  4087. list_for_each_entry(work, &worker->scheduled, entry)
  4088. pr_cont_work(false, work);
  4089. comma = true;
  4090. }
  4091. pr_cont("\n");
  4092. }
  4093. list_for_each_entry(work, &pool->worklist, entry) {
  4094. if (get_work_pwq(work) == pwq) {
  4095. has_pending = true;
  4096. break;
  4097. }
  4098. }
  4099. if (has_pending) {
  4100. bool comma = false;
  4101. pr_info(" pending:");
  4102. list_for_each_entry(work, &pool->worklist, entry) {
  4103. if (get_work_pwq(work) != pwq)
  4104. continue;
  4105. pr_cont_work(comma, work);
  4106. comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
  4107. }
  4108. pr_cont("\n");
  4109. }
  4110. if (!list_empty(&pwq->inactive_works)) {
  4111. bool comma = false;
  4112. pr_info(" inactive:");
  4113. list_for_each_entry(work, &pwq->inactive_works, entry) {
  4114. pr_cont_work(comma, work);
  4115. comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
  4116. }
  4117. pr_cont("\n");
  4118. }
  4119. }
  4120. /**
  4121. * show_one_workqueue - dump state of specified workqueue
  4122. * @wq: workqueue whose state will be printed
  4123. */
  4124. void show_one_workqueue(struct workqueue_struct *wq)
  4125. {
  4126. struct pool_workqueue *pwq;
  4127. bool idle = true;
  4128. unsigned long flags;
  4129. for_each_pwq(pwq, wq) {
  4130. if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
  4131. idle = false;
  4132. break;
  4133. }
  4134. }
  4135. if (idle) /* Nothing to print for idle workqueue */
  4136. return;
  4137. pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
  4138. for_each_pwq(pwq, wq) {
  4139. raw_spin_lock_irqsave(&pwq->pool->lock, flags);
  4140. if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
  4141. /*
  4142. * Defer printing to avoid deadlocks in console
  4143. * drivers that queue work while holding locks
  4144. * also taken in their write paths.
  4145. */
  4146. printk_deferred_enter();
  4147. show_pwq(pwq);
  4148. printk_deferred_exit();
  4149. }
  4150. raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
  4151. /*
  4152. * We could be printing a lot from atomic context, e.g.
  4153. * sysrq-t -> show_all_workqueues(). Avoid triggering
  4154. * hard lockup.
  4155. */
  4156. touch_nmi_watchdog();
  4157. }
  4158. }
  4159. /**
  4160. * show_one_worker_pool - dump state of specified worker pool
  4161. * @pool: worker pool whose state will be printed
  4162. */
  4163. static void show_one_worker_pool(struct worker_pool *pool)
  4164. {
  4165. struct worker *worker;
  4166. bool first = true;
  4167. unsigned long flags;
  4168. unsigned long hung = 0;
  4169. raw_spin_lock_irqsave(&pool->lock, flags);
  4170. if (pool->nr_workers == pool->nr_idle)
  4171. goto next_pool;
  4172. /* How long the first pending work is waiting for a worker. */
  4173. if (!list_empty(&pool->worklist))
  4174. hung = jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000;
  4175. /*
  4176. * Defer printing to avoid deadlocks in console drivers that
  4177. * queue work while holding locks also taken in their write
  4178. * paths.
  4179. */
  4180. printk_deferred_enter();
  4181. pr_info("pool %d:", pool->id);
  4182. pr_cont_pool_info(pool);
  4183. pr_cont(" hung=%lus workers=%d", hung, pool->nr_workers);
  4184. if (pool->manager)
  4185. pr_cont(" manager: %d",
  4186. task_pid_nr(pool->manager->task));
  4187. list_for_each_entry(worker, &pool->idle_list, entry) {
  4188. pr_cont(" %s%d", first ? "idle: " : "",
  4189. task_pid_nr(worker->task));
  4190. first = false;
  4191. }
  4192. pr_cont("\n");
  4193. printk_deferred_exit();
  4194. next_pool:
  4195. raw_spin_unlock_irqrestore(&pool->lock, flags);
  4196. /*
  4197. * We could be printing a lot from atomic context, e.g.
  4198. * sysrq-t -> show_all_workqueues(). Avoid triggering
  4199. * hard lockup.
  4200. */
  4201. touch_nmi_watchdog();
  4202. }
  4203. /**
  4204. * show_all_workqueues - dump workqueue state
  4205. *
  4206. * Called from a sysrq handler or try_to_freeze_tasks() and prints out
  4207. * all busy workqueues and pools.
  4208. */
  4209. void show_all_workqueues(void)
  4210. {
  4211. struct workqueue_struct *wq;
  4212. struct worker_pool *pool;
  4213. int pi;
  4214. rcu_read_lock();
  4215. pr_info("Showing busy workqueues and worker pools:\n");
  4216. list_for_each_entry_rcu(wq, &workqueues, list)
  4217. show_one_workqueue(wq);
  4218. for_each_pool(pool, pi)
  4219. show_one_worker_pool(pool);
  4220. rcu_read_unlock();
  4221. }
  4222. /* used to show worker information through /proc/PID/{comm,stat,status} */
  4223. void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
  4224. {
  4225. int off;
  4226. /* always show the actual comm */
  4227. off = strscpy(buf, task->comm, size);
  4228. if (off < 0)
  4229. return;
  4230. /* stabilize PF_WQ_WORKER and worker pool association */
  4231. mutex_lock(&wq_pool_attach_mutex);
  4232. if (task->flags & PF_WQ_WORKER) {
  4233. struct worker *worker = kthread_data(task);
  4234. struct worker_pool *pool = worker->pool;
  4235. if (pool) {
  4236. raw_spin_lock_irq(&pool->lock);
  4237. /*
  4238. * ->desc tracks information (wq name or
  4239. * set_worker_desc()) for the latest execution. If
  4240. * current, prepend '+', otherwise '-'.
  4241. */
  4242. if (worker->desc[0] != '\0') {
  4243. if (worker->current_work)
  4244. scnprintf(buf + off, size - off, "+%s",
  4245. worker->desc);
  4246. else
  4247. scnprintf(buf + off, size - off, "-%s",
  4248. worker->desc);
  4249. }
  4250. raw_spin_unlock_irq(&pool->lock);
  4251. }
  4252. }
  4253. mutex_unlock(&wq_pool_attach_mutex);
  4254. }
  4255. EXPORT_SYMBOL_GPL(wq_worker_comm);
  4256. #ifdef CONFIG_SMP
  4257. /*
  4258. * CPU hotplug.
  4259. *
  4260. * There are two challenges in supporting CPU hotplug. Firstly, there
  4261. * are a lot of assumptions on strong associations among work, pwq and
  4262. * pool which make migrating pending and scheduled works very
  4263. * difficult to implement without impacting hot paths. Secondly,
  4264. * worker pools serve mix of short, long and very long running works making
  4265. * blocked draining impractical.
  4266. *
  4267. * This is solved by allowing the pools to be disassociated from the CPU
  4268. * running as an unbound one and allowing it to be reattached later if the
  4269. * cpu comes back online.
  4270. */
  4271. static void unbind_workers(int cpu)
  4272. {
  4273. struct worker_pool *pool;
  4274. struct worker *worker;
  4275. for_each_cpu_worker_pool(pool, cpu) {
  4276. mutex_lock(&wq_pool_attach_mutex);
  4277. raw_spin_lock_irq(&pool->lock);
  4278. /*
  4279. * We've blocked all attach/detach operations. Make all workers
  4280. * unbound and set DISASSOCIATED. Before this, all workers
  4281. * must be on the cpu. After this, they may become diasporas.
  4282. * And the preemption disabled section in their sched callbacks
  4283. * are guaranteed to see WORKER_UNBOUND since the code here
  4284. * is on the same cpu.
  4285. */
  4286. for_each_pool_worker(worker, pool)
  4287. worker->flags |= WORKER_UNBOUND;
  4288. pool->flags |= POOL_DISASSOCIATED;
  4289. /*
  4290. * The handling of nr_running in sched callbacks are disabled
  4291. * now. Zap nr_running. After this, nr_running stays zero and
  4292. * need_more_worker() and keep_working() are always true as
  4293. * long as the worklist is not empty. This pool now behaves as
  4294. * an unbound (in terms of concurrency management) pool which
  4295. * are served by workers tied to the pool.
  4296. */
  4297. pool->nr_running = 0;
  4298. /*
  4299. * With concurrency management just turned off, a busy
  4300. * worker blocking could lead to lengthy stalls. Kick off
  4301. * unbound chain execution of currently pending work items.
  4302. */
  4303. wake_up_worker(pool);
  4304. raw_spin_unlock_irq(&pool->lock);
  4305. for_each_pool_worker(worker, pool) {
  4306. kthread_set_per_cpu(worker->task, -1);
  4307. if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))
  4308. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);
  4309. else
  4310. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
  4311. }
  4312. mutex_unlock(&wq_pool_attach_mutex);
  4313. }
  4314. }
  4315. /**
  4316. * rebind_workers - rebind all workers of a pool to the associated CPU
  4317. * @pool: pool of interest
  4318. *
  4319. * @pool->cpu is coming online. Rebind all workers to the CPU.
  4320. */
  4321. static void rebind_workers(struct worker_pool *pool)
  4322. {
  4323. struct worker *worker;
  4324. lockdep_assert_held(&wq_pool_attach_mutex);
  4325. /*
  4326. * Restore CPU affinity of all workers. As all idle workers should
  4327. * be on the run-queue of the associated CPU before any local
  4328. * wake-ups for concurrency management happen, restore CPU affinity
  4329. * of all workers first and then clear UNBOUND. As we're called
  4330. * from CPU_ONLINE, the following shouldn't fail.
  4331. */
  4332. for_each_pool_worker(worker, pool) {
  4333. kthread_set_per_cpu(worker->task, pool->cpu);
  4334. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
  4335. pool->attrs->cpumask) < 0);
  4336. }
  4337. raw_spin_lock_irq(&pool->lock);
  4338. pool->flags &= ~POOL_DISASSOCIATED;
  4339. for_each_pool_worker(worker, pool) {
  4340. unsigned int worker_flags = worker->flags;
  4341. /*
  4342. * We want to clear UNBOUND but can't directly call
  4343. * worker_clr_flags() or adjust nr_running. Atomically
  4344. * replace UNBOUND with another NOT_RUNNING flag REBOUND.
  4345. * @worker will clear REBOUND using worker_clr_flags() when
  4346. * it initiates the next execution cycle thus restoring
  4347. * concurrency management. Note that when or whether
  4348. * @worker clears REBOUND doesn't affect correctness.
  4349. *
  4350. * WRITE_ONCE() is necessary because @worker->flags may be
  4351. * tested without holding any lock in
  4352. * wq_worker_running(). Without it, NOT_RUNNING test may
  4353. * fail incorrectly leading to premature concurrency
  4354. * management operations.
  4355. */
  4356. WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
  4357. worker_flags |= WORKER_REBOUND;
  4358. worker_flags &= ~WORKER_UNBOUND;
  4359. WRITE_ONCE(worker->flags, worker_flags);
  4360. }
  4361. raw_spin_unlock_irq(&pool->lock);
  4362. }
  4363. /**
  4364. * restore_unbound_workers_cpumask - restore cpumask of unbound workers
  4365. * @pool: unbound pool of interest
  4366. * @cpu: the CPU which is coming up
  4367. *
  4368. * An unbound pool may end up with a cpumask which doesn't have any online
  4369. * CPUs. When a worker of such pool get scheduled, the scheduler resets
  4370. * its cpus_allowed. If @cpu is in @pool's cpumask which didn't have any
  4371. * online CPU before, cpus_allowed of all its workers should be restored.
  4372. */
  4373. static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
  4374. {
  4375. static cpumask_t cpumask;
  4376. struct worker *worker;
  4377. lockdep_assert_held(&wq_pool_attach_mutex);
  4378. /* is @cpu allowed for @pool? */
  4379. if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
  4380. return;
  4381. cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
  4382. /* as we're called from CPU_ONLINE, the following shouldn't fail */
  4383. for_each_pool_worker(worker, pool)
  4384. WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
  4385. }
  4386. int workqueue_prepare_cpu(unsigned int cpu)
  4387. {
  4388. struct worker_pool *pool;
  4389. for_each_cpu_worker_pool(pool, cpu) {
  4390. if (pool->nr_workers)
  4391. continue;
  4392. if (!create_worker(pool))
  4393. return -ENOMEM;
  4394. }
  4395. return 0;
  4396. }
  4397. int workqueue_online_cpu(unsigned int cpu)
  4398. {
  4399. struct worker_pool *pool;
  4400. struct workqueue_struct *wq;
  4401. int pi;
  4402. mutex_lock(&wq_pool_mutex);
  4403. for_each_pool(pool, pi) {
  4404. mutex_lock(&wq_pool_attach_mutex);
  4405. if (pool->cpu == cpu)
  4406. rebind_workers(pool);
  4407. else if (pool->cpu < 0)
  4408. restore_unbound_workers_cpumask(pool, cpu);
  4409. mutex_unlock(&wq_pool_attach_mutex);
  4410. }
  4411. /* update NUMA affinity of unbound workqueues */
  4412. list_for_each_entry(wq, &workqueues, list)
  4413. wq_update_unbound_numa(wq, cpu, true);
  4414. mutex_unlock(&wq_pool_mutex);
  4415. return 0;
  4416. }
  4417. int workqueue_offline_cpu(unsigned int cpu)
  4418. {
  4419. struct workqueue_struct *wq;
  4420. /* unbinding per-cpu workers should happen on the local CPU */
  4421. if (WARN_ON(cpu != smp_processor_id()))
  4422. return -1;
  4423. unbind_workers(cpu);
  4424. /* update NUMA affinity of unbound workqueues */
  4425. mutex_lock(&wq_pool_mutex);
  4426. list_for_each_entry(wq, &workqueues, list)
  4427. wq_update_unbound_numa(wq, cpu, false);
  4428. mutex_unlock(&wq_pool_mutex);
  4429. return 0;
  4430. }
  4431. struct work_for_cpu {
  4432. struct work_struct work;
  4433. long (*fn)(void *);
  4434. void *arg;
  4435. long ret;
  4436. };
  4437. static void work_for_cpu_fn(struct work_struct *work)
  4438. {
  4439. struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
  4440. wfc->ret = wfc->fn(wfc->arg);
  4441. }
  4442. /**
  4443. * work_on_cpu - run a function in thread context on a particular cpu
  4444. * @cpu: the cpu to run on
  4445. * @fn: the function to run
  4446. * @arg: the function arg
  4447. *
  4448. * It is up to the caller to ensure that the cpu doesn't go offline.
  4449. * The caller must not hold any locks which would prevent @fn from completing.
  4450. *
  4451. * Return: The value @fn returns.
  4452. */
  4453. long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
  4454. {
  4455. struct work_for_cpu wfc = { .fn = fn, .arg = arg };
  4456. INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
  4457. schedule_work_on(cpu, &wfc.work);
  4458. flush_work(&wfc.work);
  4459. destroy_work_on_stack(&wfc.work);
  4460. return wfc.ret;
  4461. }
  4462. EXPORT_SYMBOL_GPL(work_on_cpu);
  4463. /**
  4464. * work_on_cpu_safe - run a function in thread context on a particular cpu
  4465. * @cpu: the cpu to run on
  4466. * @fn: the function to run
  4467. * @arg: the function argument
  4468. *
  4469. * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
  4470. * any locks which would prevent @fn from completing.
  4471. *
  4472. * Return: The value @fn returns.
  4473. */
  4474. long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg)
  4475. {
  4476. long ret = -ENODEV;
  4477. cpus_read_lock();
  4478. if (cpu_online(cpu))
  4479. ret = work_on_cpu(cpu, fn, arg);
  4480. cpus_read_unlock();
  4481. return ret;
  4482. }
  4483. EXPORT_SYMBOL_GPL(work_on_cpu_safe);
  4484. #endif /* CONFIG_SMP */
  4485. #ifdef CONFIG_FREEZER
  4486. /**
  4487. * freeze_workqueues_begin - begin freezing workqueues
  4488. *
  4489. * Start freezing workqueues. After this function returns, all freezable
  4490. * workqueues will queue new works to their inactive_works list instead of
  4491. * pool->worklist.
  4492. *
  4493. * CONTEXT:
  4494. * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
  4495. */
  4496. void freeze_workqueues_begin(void)
  4497. {
  4498. struct workqueue_struct *wq;
  4499. struct pool_workqueue *pwq;
  4500. mutex_lock(&wq_pool_mutex);
  4501. WARN_ON_ONCE(workqueue_freezing);
  4502. workqueue_freezing = true;
  4503. list_for_each_entry(wq, &workqueues, list) {
  4504. mutex_lock(&wq->mutex);
  4505. for_each_pwq(pwq, wq)
  4506. pwq_adjust_max_active(pwq);
  4507. mutex_unlock(&wq->mutex);
  4508. }
  4509. mutex_unlock(&wq_pool_mutex);
  4510. }
  4511. /**
  4512. * freeze_workqueues_busy - are freezable workqueues still busy?
  4513. *
  4514. * Check whether freezing is complete. This function must be called
  4515. * between freeze_workqueues_begin() and thaw_workqueues().
  4516. *
  4517. * CONTEXT:
  4518. * Grabs and releases wq_pool_mutex.
  4519. *
  4520. * Return:
  4521. * %true if some freezable workqueues are still busy. %false if freezing
  4522. * is complete.
  4523. */
  4524. bool freeze_workqueues_busy(void)
  4525. {
  4526. bool busy = false;
  4527. struct workqueue_struct *wq;
  4528. struct pool_workqueue *pwq;
  4529. mutex_lock(&wq_pool_mutex);
  4530. WARN_ON_ONCE(!workqueue_freezing);
  4531. list_for_each_entry(wq, &workqueues, list) {
  4532. if (!(wq->flags & WQ_FREEZABLE))
  4533. continue;
  4534. /*
  4535. * nr_active is monotonically decreasing. It's safe
  4536. * to peek without lock.
  4537. */
  4538. rcu_read_lock();
  4539. for_each_pwq(pwq, wq) {
  4540. WARN_ON_ONCE(pwq->nr_active < 0);
  4541. if (pwq->nr_active) {
  4542. busy = true;
  4543. rcu_read_unlock();
  4544. goto out_unlock;
  4545. }
  4546. }
  4547. rcu_read_unlock();
  4548. }
  4549. out_unlock:
  4550. mutex_unlock(&wq_pool_mutex);
  4551. return busy;
  4552. }
  4553. /**
  4554. * thaw_workqueues - thaw workqueues
  4555. *
  4556. * Thaw workqueues. Normal queueing is restored and all collected
  4557. * frozen works are transferred to their respective pool worklists.
  4558. *
  4559. * CONTEXT:
  4560. * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
  4561. */
  4562. void thaw_workqueues(void)
  4563. {
  4564. struct workqueue_struct *wq;
  4565. struct pool_workqueue *pwq;
  4566. mutex_lock(&wq_pool_mutex);
  4567. if (!workqueue_freezing)
  4568. goto out_unlock;
  4569. workqueue_freezing = false;
  4570. /* restore max_active and repopulate worklist */
  4571. list_for_each_entry(wq, &workqueues, list) {
  4572. mutex_lock(&wq->mutex);
  4573. for_each_pwq(pwq, wq)
  4574. pwq_adjust_max_active(pwq);
  4575. mutex_unlock(&wq->mutex);
  4576. }
  4577. out_unlock:
  4578. mutex_unlock(&wq_pool_mutex);
  4579. }
  4580. #endif /* CONFIG_FREEZER */
  4581. static int workqueue_apply_unbound_cpumask(const cpumask_var_t unbound_cpumask)
  4582. {
  4583. LIST_HEAD(ctxs);
  4584. int ret = 0;
  4585. struct workqueue_struct *wq;
  4586. struct apply_wqattrs_ctx *ctx, *n;
  4587. lockdep_assert_held(&wq_pool_mutex);
  4588. list_for_each_entry(wq, &workqueues, list) {
  4589. if (!(wq->flags & WQ_UNBOUND))
  4590. continue;
  4591. /* creating multiple pwqs breaks ordering guarantee */
  4592. if (!list_empty(&wq->pwqs)) {
  4593. if (wq->flags & __WQ_ORDERED_EXPLICIT)
  4594. continue;
  4595. wq->flags &= ~__WQ_ORDERED;
  4596. }
  4597. ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs, unbound_cpumask);
  4598. if (!ctx) {
  4599. ret = -ENOMEM;
  4600. break;
  4601. }
  4602. list_add_tail(&ctx->list, &ctxs);
  4603. }
  4604. list_for_each_entry_safe(ctx, n, &ctxs, list) {
  4605. if (!ret)
  4606. apply_wqattrs_commit(ctx);
  4607. apply_wqattrs_cleanup(ctx);
  4608. }
  4609. if (!ret) {
  4610. mutex_lock(&wq_pool_attach_mutex);
  4611. cpumask_copy(wq_unbound_cpumask, unbound_cpumask);
  4612. mutex_unlock(&wq_pool_attach_mutex);
  4613. }
  4614. return ret;
  4615. }
  4616. /**
  4617. * workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
  4618. * @cpumask: the cpumask to set
  4619. *
  4620. * The low-level workqueues cpumask is a global cpumask that limits
  4621. * the affinity of all unbound workqueues. This function check the @cpumask
  4622. * and apply it to all unbound workqueues and updates all pwqs of them.
  4623. *
  4624. * Return: 0 - Success
  4625. * -EINVAL - Invalid @cpumask
  4626. * -ENOMEM - Failed to allocate memory for attrs or pwqs.
  4627. */
  4628. int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
  4629. {
  4630. int ret = -EINVAL;
  4631. /*
  4632. * Not excluding isolated cpus on purpose.
  4633. * If the user wishes to include them, we allow that.
  4634. */
  4635. cpumask_and(cpumask, cpumask, cpu_possible_mask);
  4636. if (!cpumask_empty(cpumask)) {
  4637. apply_wqattrs_lock();
  4638. if (cpumask_equal(cpumask, wq_unbound_cpumask)) {
  4639. ret = 0;
  4640. goto out_unlock;
  4641. }
  4642. ret = workqueue_apply_unbound_cpumask(cpumask);
  4643. out_unlock:
  4644. apply_wqattrs_unlock();
  4645. }
  4646. return ret;
  4647. }
  4648. #ifdef CONFIG_SYSFS
  4649. /*
  4650. * Workqueues with WQ_SYSFS flag set is visible to userland via
  4651. * /sys/bus/workqueue/devices/WQ_NAME. All visible workqueues have the
  4652. * following attributes.
  4653. *
  4654. * per_cpu RO bool : whether the workqueue is per-cpu or unbound
  4655. * max_active RW int : maximum number of in-flight work items
  4656. *
  4657. * Unbound workqueues have the following extra attributes.
  4658. *
  4659. * pool_ids RO int : the associated pool IDs for each node
  4660. * nice RW int : nice value of the workers
  4661. * cpumask RW mask : bitmask of allowed CPUs for the workers
  4662. * numa RW bool : whether enable NUMA affinity
  4663. */
  4664. struct wq_device {
  4665. struct workqueue_struct *wq;
  4666. struct device dev;
  4667. };
  4668. static struct workqueue_struct *dev_to_wq(struct device *dev)
  4669. {
  4670. struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
  4671. return wq_dev->wq;
  4672. }
  4673. static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
  4674. char *buf)
  4675. {
  4676. struct workqueue_struct *wq = dev_to_wq(dev);
  4677. return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
  4678. }
  4679. static DEVICE_ATTR_RO(per_cpu);
  4680. static ssize_t max_active_show(struct device *dev,
  4681. struct device_attribute *attr, char *buf)
  4682. {
  4683. struct workqueue_struct *wq = dev_to_wq(dev);
  4684. return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
  4685. }
  4686. static ssize_t max_active_store(struct device *dev,
  4687. struct device_attribute *attr, const char *buf,
  4688. size_t count)
  4689. {
  4690. struct workqueue_struct *wq = dev_to_wq(dev);
  4691. int val;
  4692. if (sscanf(buf, "%d", &val) != 1 || val <= 0)
  4693. return -EINVAL;
  4694. workqueue_set_max_active(wq, val);
  4695. return count;
  4696. }
  4697. static DEVICE_ATTR_RW(max_active);
  4698. static struct attribute *wq_sysfs_attrs[] = {
  4699. &dev_attr_per_cpu.attr,
  4700. &dev_attr_max_active.attr,
  4701. NULL,
  4702. };
  4703. ATTRIBUTE_GROUPS(wq_sysfs);
  4704. static ssize_t wq_pool_ids_show(struct device *dev,
  4705. struct device_attribute *attr, char *buf)
  4706. {
  4707. struct workqueue_struct *wq = dev_to_wq(dev);
  4708. const char *delim = "";
  4709. int node, written = 0;
  4710. cpus_read_lock();
  4711. rcu_read_lock();
  4712. for_each_node(node) {
  4713. written += scnprintf(buf + written, PAGE_SIZE - written,
  4714. "%s%d:%d", delim, node,
  4715. unbound_pwq_by_node(wq, node)->pool->id);
  4716. delim = " ";
  4717. }
  4718. written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
  4719. rcu_read_unlock();
  4720. cpus_read_unlock();
  4721. return written;
  4722. }
  4723. static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
  4724. char *buf)
  4725. {
  4726. struct workqueue_struct *wq = dev_to_wq(dev);
  4727. int written;
  4728. mutex_lock(&wq->mutex);
  4729. written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
  4730. mutex_unlock(&wq->mutex);
  4731. return written;
  4732. }
  4733. /* prepare workqueue_attrs for sysfs store operations */
  4734. static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
  4735. {
  4736. struct workqueue_attrs *attrs;
  4737. lockdep_assert_held(&wq_pool_mutex);
  4738. attrs = alloc_workqueue_attrs();
  4739. if (!attrs)
  4740. return NULL;
  4741. copy_workqueue_attrs(attrs, wq->unbound_attrs);
  4742. return attrs;
  4743. }
  4744. static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
  4745. const char *buf, size_t count)
  4746. {
  4747. struct workqueue_struct *wq = dev_to_wq(dev);
  4748. struct workqueue_attrs *attrs;
  4749. int ret = -ENOMEM;
  4750. apply_wqattrs_lock();
  4751. attrs = wq_sysfs_prep_attrs(wq);
  4752. if (!attrs)
  4753. goto out_unlock;
  4754. if (sscanf(buf, "%d", &attrs->nice) == 1 &&
  4755. attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
  4756. ret = apply_workqueue_attrs_locked(wq, attrs);
  4757. else
  4758. ret = -EINVAL;
  4759. out_unlock:
  4760. apply_wqattrs_unlock();
  4761. free_workqueue_attrs(attrs);
  4762. return ret ?: count;
  4763. }
  4764. static ssize_t wq_cpumask_show(struct device *dev,
  4765. struct device_attribute *attr, char *buf)
  4766. {
  4767. struct workqueue_struct *wq = dev_to_wq(dev);
  4768. int written;
  4769. mutex_lock(&wq->mutex);
  4770. written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
  4771. cpumask_pr_args(wq->unbound_attrs->cpumask));
  4772. mutex_unlock(&wq->mutex);
  4773. return written;
  4774. }
  4775. static ssize_t wq_cpumask_store(struct device *dev,
  4776. struct device_attribute *attr,
  4777. const char *buf, size_t count)
  4778. {
  4779. struct workqueue_struct *wq = dev_to_wq(dev);
  4780. struct workqueue_attrs *attrs;
  4781. int ret = -ENOMEM;
  4782. apply_wqattrs_lock();
  4783. attrs = wq_sysfs_prep_attrs(wq);
  4784. if (!attrs)
  4785. goto out_unlock;
  4786. ret = cpumask_parse(buf, attrs->cpumask);
  4787. if (!ret)
  4788. ret = apply_workqueue_attrs_locked(wq, attrs);
  4789. out_unlock:
  4790. apply_wqattrs_unlock();
  4791. free_workqueue_attrs(attrs);
  4792. return ret ?: count;
  4793. }
  4794. static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
  4795. char *buf)
  4796. {
  4797. struct workqueue_struct *wq = dev_to_wq(dev);
  4798. int written;
  4799. mutex_lock(&wq->mutex);
  4800. written = scnprintf(buf, PAGE_SIZE, "%d\n",
  4801. !wq->unbound_attrs->no_numa);
  4802. mutex_unlock(&wq->mutex);
  4803. return written;
  4804. }
  4805. static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
  4806. const char *buf, size_t count)
  4807. {
  4808. struct workqueue_struct *wq = dev_to_wq(dev);
  4809. struct workqueue_attrs *attrs;
  4810. int v, ret = -ENOMEM;
  4811. apply_wqattrs_lock();
  4812. attrs = wq_sysfs_prep_attrs(wq);
  4813. if (!attrs)
  4814. goto out_unlock;
  4815. ret = -EINVAL;
  4816. if (sscanf(buf, "%d", &v) == 1) {
  4817. attrs->no_numa = !v;
  4818. ret = apply_workqueue_attrs_locked(wq, attrs);
  4819. }
  4820. out_unlock:
  4821. apply_wqattrs_unlock();
  4822. free_workqueue_attrs(attrs);
  4823. return ret ?: count;
  4824. }
  4825. static struct device_attribute wq_sysfs_unbound_attrs[] = {
  4826. __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
  4827. __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
  4828. __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
  4829. __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
  4830. __ATTR_NULL,
  4831. };
  4832. static struct bus_type wq_subsys = {
  4833. .name = "workqueue",
  4834. .dev_groups = wq_sysfs_groups,
  4835. };
  4836. static ssize_t wq_unbound_cpumask_show(struct device *dev,
  4837. struct device_attribute *attr, char *buf)
  4838. {
  4839. int written;
  4840. mutex_lock(&wq_pool_mutex);
  4841. written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
  4842. cpumask_pr_args(wq_unbound_cpumask));
  4843. mutex_unlock(&wq_pool_mutex);
  4844. return written;
  4845. }
  4846. static ssize_t wq_unbound_cpumask_store(struct device *dev,
  4847. struct device_attribute *attr, const char *buf, size_t count)
  4848. {
  4849. cpumask_var_t cpumask;
  4850. int ret;
  4851. if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
  4852. return -ENOMEM;
  4853. ret = cpumask_parse(buf, cpumask);
  4854. if (!ret)
  4855. ret = workqueue_set_unbound_cpumask(cpumask);
  4856. free_cpumask_var(cpumask);
  4857. return ret ? ret : count;
  4858. }
  4859. static struct device_attribute wq_sysfs_cpumask_attr =
  4860. __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
  4861. wq_unbound_cpumask_store);
  4862. static int __init wq_sysfs_init(void)
  4863. {
  4864. int err;
  4865. err = subsys_virtual_register(&wq_subsys, NULL);
  4866. if (err)
  4867. return err;
  4868. return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
  4869. }
  4870. core_initcall(wq_sysfs_init);
  4871. static void wq_device_release(struct device *dev)
  4872. {
  4873. struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
  4874. kfree(wq_dev);
  4875. }
  4876. /**
  4877. * workqueue_sysfs_register - make a workqueue visible in sysfs
  4878. * @wq: the workqueue to register
  4879. *
  4880. * Expose @wq in sysfs under /sys/bus/workqueue/devices.
  4881. * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
  4882. * which is the preferred method.
  4883. *
  4884. * Workqueue user should use this function directly iff it wants to apply
  4885. * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
  4886. * apply_workqueue_attrs() may race against userland updating the
  4887. * attributes.
  4888. *
  4889. * Return: 0 on success, -errno on failure.
  4890. */
  4891. int workqueue_sysfs_register(struct workqueue_struct *wq)
  4892. {
  4893. struct wq_device *wq_dev;
  4894. int ret;
  4895. /*
  4896. * Adjusting max_active or creating new pwqs by applying
  4897. * attributes breaks ordering guarantee. Disallow exposing ordered
  4898. * workqueues.
  4899. */
  4900. if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
  4901. return -EINVAL;
  4902. wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
  4903. if (!wq_dev)
  4904. return -ENOMEM;
  4905. wq_dev->wq = wq;
  4906. wq_dev->dev.bus = &wq_subsys;
  4907. wq_dev->dev.release = wq_device_release;
  4908. dev_set_name(&wq_dev->dev, "%s", wq->name);
  4909. /*
  4910. * unbound_attrs are created separately. Suppress uevent until
  4911. * everything is ready.
  4912. */
  4913. dev_set_uevent_suppress(&wq_dev->dev, true);
  4914. ret = device_register(&wq_dev->dev);
  4915. if (ret) {
  4916. put_device(&wq_dev->dev);
  4917. wq->wq_dev = NULL;
  4918. return ret;
  4919. }
  4920. if (wq->flags & WQ_UNBOUND) {
  4921. struct device_attribute *attr;
  4922. for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
  4923. ret = device_create_file(&wq_dev->dev, attr);
  4924. if (ret) {
  4925. device_unregister(&wq_dev->dev);
  4926. wq->wq_dev = NULL;
  4927. return ret;
  4928. }
  4929. }
  4930. }
  4931. dev_set_uevent_suppress(&wq_dev->dev, false);
  4932. kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
  4933. return 0;
  4934. }
  4935. /**
  4936. * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
  4937. * @wq: the workqueue to unregister
  4938. *
  4939. * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
  4940. */
  4941. static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
  4942. {
  4943. struct wq_device *wq_dev = wq->wq_dev;
  4944. if (!wq->wq_dev)
  4945. return;
  4946. wq->wq_dev = NULL;
  4947. device_unregister(&wq_dev->dev);
  4948. }
  4949. #else /* CONFIG_SYSFS */
  4950. static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
  4951. #endif /* CONFIG_SYSFS */
  4952. /*
  4953. * Workqueue watchdog.
  4954. *
  4955. * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
  4956. * flush dependency, a concurrency managed work item which stays RUNNING
  4957. * indefinitely. Workqueue stalls can be very difficult to debug as the
  4958. * usual warning mechanisms don't trigger and internal workqueue state is
  4959. * largely opaque.
  4960. *
  4961. * Workqueue watchdog monitors all worker pools periodically and dumps
  4962. * state if some pools failed to make forward progress for a while where
  4963. * forward progress is defined as the first item on ->worklist changing.
  4964. *
  4965. * This mechanism is controlled through the kernel parameter
  4966. * "workqueue.watchdog_thresh" which can be updated at runtime through the
  4967. * corresponding sysfs parameter file.
  4968. */
  4969. #ifdef CONFIG_WQ_WATCHDOG
  4970. static unsigned long wq_watchdog_thresh = 30;
  4971. static struct timer_list wq_watchdog_timer;
  4972. static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
  4973. static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
  4974. static void wq_watchdog_reset_touched(void)
  4975. {
  4976. int cpu;
  4977. wq_watchdog_touched = jiffies;
  4978. for_each_possible_cpu(cpu)
  4979. per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
  4980. }
  4981. static void wq_watchdog_timer_fn(struct timer_list *unused)
  4982. {
  4983. unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
  4984. bool lockup_detected = false;
  4985. unsigned long now = jiffies;
  4986. struct worker_pool *pool;
  4987. int pi;
  4988. if (!thresh)
  4989. return;
  4990. rcu_read_lock();
  4991. for_each_pool(pool, pi) {
  4992. unsigned long pool_ts, touched, ts;
  4993. if (list_empty(&pool->worklist))
  4994. continue;
  4995. /*
  4996. * If a virtual machine is stopped by the host it can look to
  4997. * the watchdog like a stall.
  4998. */
  4999. kvm_check_and_clear_guest_paused();
  5000. /* get the latest of pool and touched timestamps */
  5001. if (pool->cpu >= 0)
  5002. touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
  5003. else
  5004. touched = READ_ONCE(wq_watchdog_touched);
  5005. pool_ts = READ_ONCE(pool->watchdog_ts);
  5006. if (time_after(pool_ts, touched))
  5007. ts = pool_ts;
  5008. else
  5009. ts = touched;
  5010. /* did we stall? */
  5011. if (time_after(now, ts + thresh)) {
  5012. lockup_detected = true;
  5013. pr_emerg("BUG: workqueue lockup - pool");
  5014. pr_cont_pool_info(pool);
  5015. pr_cont(" stuck for %us!\n",
  5016. jiffies_to_msecs(now - pool_ts) / 1000);
  5017. trace_android_vh_wq_lockup_pool(pool->cpu, pool_ts);
  5018. }
  5019. }
  5020. rcu_read_unlock();
  5021. if (lockup_detected)
  5022. show_all_workqueues();
  5023. wq_watchdog_reset_touched();
  5024. mod_timer(&wq_watchdog_timer, jiffies + thresh);
  5025. }
  5026. notrace void wq_watchdog_touch(int cpu)
  5027. {
  5028. if (cpu >= 0)
  5029. per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
  5030. wq_watchdog_touched = jiffies;
  5031. }
  5032. static void wq_watchdog_set_thresh(unsigned long thresh)
  5033. {
  5034. wq_watchdog_thresh = 0;
  5035. del_timer_sync(&wq_watchdog_timer);
  5036. if (thresh) {
  5037. wq_watchdog_thresh = thresh;
  5038. wq_watchdog_reset_touched();
  5039. mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
  5040. }
  5041. }
  5042. static int wq_watchdog_param_set_thresh(const char *val,
  5043. const struct kernel_param *kp)
  5044. {
  5045. unsigned long thresh;
  5046. int ret;
  5047. ret = kstrtoul(val, 0, &thresh);
  5048. if (ret)
  5049. return ret;
  5050. if (system_wq)
  5051. wq_watchdog_set_thresh(thresh);
  5052. else
  5053. wq_watchdog_thresh = thresh;
  5054. return 0;
  5055. }
  5056. static const struct kernel_param_ops wq_watchdog_thresh_ops = {
  5057. .set = wq_watchdog_param_set_thresh,
  5058. .get = param_get_ulong,
  5059. };
  5060. module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
  5061. 0644);
  5062. static void wq_watchdog_init(void)
  5063. {
  5064. timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
  5065. wq_watchdog_set_thresh(wq_watchdog_thresh);
  5066. }
  5067. #else /* CONFIG_WQ_WATCHDOG */
  5068. static inline void wq_watchdog_init(void) { }
  5069. #endif /* CONFIG_WQ_WATCHDOG */
  5070. static void __init wq_numa_init(void)
  5071. {
  5072. cpumask_var_t *tbl;
  5073. int node, cpu;
  5074. if (num_possible_nodes() <= 1)
  5075. return;
  5076. if (wq_disable_numa) {
  5077. pr_info("workqueue: NUMA affinity support disabled\n");
  5078. return;
  5079. }
  5080. for_each_possible_cpu(cpu) {
  5081. if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
  5082. pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
  5083. return;
  5084. }
  5085. }
  5086. wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
  5087. BUG_ON(!wq_update_unbound_numa_attrs_buf);
  5088. /*
  5089. * We want masks of possible CPUs of each node which isn't readily
  5090. * available. Build one from cpu_to_node() which should have been
  5091. * fully initialized by now.
  5092. */
  5093. tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);
  5094. BUG_ON(!tbl);
  5095. for_each_node(node)
  5096. BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
  5097. node_online(node) ? node : NUMA_NO_NODE));
  5098. for_each_possible_cpu(cpu) {
  5099. node = cpu_to_node(cpu);
  5100. cpumask_set_cpu(cpu, tbl[node]);
  5101. }
  5102. wq_numa_possible_cpumask = tbl;
  5103. wq_numa_enabled = true;
  5104. }
  5105. /**
  5106. * workqueue_init_early - early init for workqueue subsystem
  5107. *
  5108. * This is the first half of two-staged workqueue subsystem initialization
  5109. * and invoked as soon as the bare basics - memory allocation, cpumasks and
  5110. * idr are up. It sets up all the data structures and system workqueues
  5111. * and allows early boot code to create workqueues and queue/cancel work
  5112. * items. Actual work item execution starts only after kthreads can be
  5113. * created and scheduled right before early initcalls.
  5114. */
  5115. void __init workqueue_init_early(void)
  5116. {
  5117. int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
  5118. int i, cpu;
  5119. BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
  5120. BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
  5121. cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_WQ));
  5122. cpumask_and(wq_unbound_cpumask, wq_unbound_cpumask, housekeeping_cpumask(HK_TYPE_DOMAIN));
  5123. pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
  5124. /* initialize CPU pools */
  5125. for_each_possible_cpu(cpu) {
  5126. struct worker_pool *pool;
  5127. i = 0;
  5128. for_each_cpu_worker_pool(pool, cpu) {
  5129. BUG_ON(init_worker_pool(pool));
  5130. pool->cpu = cpu;
  5131. cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
  5132. pool->attrs->nice = std_nice[i++];
  5133. pool->node = cpu_to_node(cpu);
  5134. /* alloc pool ID */
  5135. mutex_lock(&wq_pool_mutex);
  5136. BUG_ON(worker_pool_assign_id(pool));
  5137. mutex_unlock(&wq_pool_mutex);
  5138. }
  5139. }
  5140. /* create default unbound and ordered wq attrs */
  5141. for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
  5142. struct workqueue_attrs *attrs;
  5143. BUG_ON(!(attrs = alloc_workqueue_attrs()));
  5144. attrs->nice = std_nice[i];
  5145. unbound_std_wq_attrs[i] = attrs;
  5146. /*
  5147. * An ordered wq should have only one pwq as ordering is
  5148. * guaranteed by max_active which is enforced by pwqs.
  5149. * Turn off NUMA so that dfl_pwq is used for all nodes.
  5150. */
  5151. BUG_ON(!(attrs = alloc_workqueue_attrs()));
  5152. attrs->nice = std_nice[i];
  5153. attrs->no_numa = true;
  5154. ordered_wq_attrs[i] = attrs;
  5155. }
  5156. system_wq = alloc_workqueue("events", 0, 0);
  5157. system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
  5158. system_long_wq = alloc_workqueue("events_long", 0, 0);
  5159. system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
  5160. WQ_UNBOUND_MAX_ACTIVE);
  5161. system_freezable_wq = alloc_workqueue("events_freezable",
  5162. WQ_FREEZABLE, 0);
  5163. system_power_efficient_wq = alloc_workqueue("events_power_efficient",
  5164. WQ_POWER_EFFICIENT, 0);
  5165. system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
  5166. WQ_FREEZABLE | WQ_POWER_EFFICIENT,
  5167. 0);
  5168. BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
  5169. !system_unbound_wq || !system_freezable_wq ||
  5170. !system_power_efficient_wq ||
  5171. !system_freezable_power_efficient_wq);
  5172. }
  5173. /**
  5174. * workqueue_init - bring workqueue subsystem fully online
  5175. *
  5176. * This is the latter half of two-staged workqueue subsystem initialization
  5177. * and invoked as soon as kthreads can be created and scheduled.
  5178. * Workqueues have been created and work items queued on them, but there
  5179. * are no kworkers executing the work items yet. Populate the worker pools
  5180. * with the initial workers and enable future kworker creations.
  5181. */
  5182. void __init workqueue_init(void)
  5183. {
  5184. struct workqueue_struct *wq;
  5185. struct worker_pool *pool;
  5186. int cpu, bkt;
  5187. /*
  5188. * It'd be simpler to initialize NUMA in workqueue_init_early() but
  5189. * CPU to node mapping may not be available that early on some
  5190. * archs such as power and arm64. As per-cpu pools created
  5191. * previously could be missing node hint and unbound pools NUMA
  5192. * affinity, fix them up.
  5193. *
  5194. * Also, while iterating workqueues, create rescuers if requested.
  5195. */
  5196. wq_numa_init();
  5197. mutex_lock(&wq_pool_mutex);
  5198. for_each_possible_cpu(cpu) {
  5199. for_each_cpu_worker_pool(pool, cpu) {
  5200. pool->node = cpu_to_node(cpu);
  5201. }
  5202. }
  5203. list_for_each_entry(wq, &workqueues, list) {
  5204. wq_update_unbound_numa(wq, smp_processor_id(), true);
  5205. WARN(init_rescuer(wq),
  5206. "workqueue: failed to create early rescuer for %s",
  5207. wq->name);
  5208. }
  5209. mutex_unlock(&wq_pool_mutex);
  5210. /* create the initial workers */
  5211. for_each_online_cpu(cpu) {
  5212. for_each_cpu_worker_pool(pool, cpu) {
  5213. pool->flags &= ~POOL_DISASSOCIATED;
  5214. BUG_ON(!create_worker(pool));
  5215. }
  5216. }
  5217. hash_for_each(unbound_pool_hash, bkt, pool, hash_node)
  5218. BUG_ON(!create_worker(pool));
  5219. wq_online = true;
  5220. wq_watchdog_init();
  5221. }
  5222. /*
  5223. * Despite the naming, this is a no-op function which is here only for avoiding
  5224. * link error. Since compile-time warning may fail to catch, we will need to
  5225. * emit run-time warning from __flush_workqueue().
  5226. */
  5227. void __warn_flushing_systemwide_wq(void) { }
  5228. EXPORT_SYMBOL(__warn_flushing_systemwide_wq);