1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074 |
- /*
- * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
- *
- * Permission to use, copy, modify, and/or distribute this software for
- * any purpose with or without fee is hereby granted, provided that the
- * above copyright notice and this permission notice appear in all
- * copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
- * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
- * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
- * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
- * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- * PERFORMANCE OF THIS SOFTWARE.
- */
- #include "htt.h"
- #include "dp_htt.h"
- #include "hal_hw_headers.h"
- #include "dp_tx.h"
- #include "dp_tx_desc.h"
- #include "dp_peer.h"
- #include "dp_types.h"
- #include "hal_tx.h"
- #include "qdf_mem.h"
- #include "qdf_nbuf.h"
- #include "qdf_net_types.h"
- #include "qdf_module.h"
- #include <wlan_cfg.h>
- #include "dp_ipa.h"
- #if defined(MESH_MODE_SUPPORT) || defined(FEATURE_PERPKT_INFO)
- #include "if_meta_hdr.h"
- #endif
- #include "enet.h"
- #include "dp_internal.h"
- #ifdef ATH_SUPPORT_IQUE
- #include "dp_txrx_me.h"
- #endif
- #include "dp_hist.h"
- #ifdef WLAN_DP_FEATURE_SW_LATENCY_MGR
- #include <wlan_dp_swlm.h>
- #endif
- #ifdef WIFI_MONITOR_SUPPORT
- #include <dp_mon.h>
- #endif
- #ifdef FEATURE_WDS
- #include "dp_txrx_wds.h"
- #endif
- #include "cdp_txrx_cmn_reg.h"
- #ifdef CONFIG_SAWF
- #include <dp_sawf.h>
- #endif
- /* Flag to skip CCE classify when mesh or tid override enabled */
- #define DP_TX_SKIP_CCE_CLASSIFY \
- (DP_TXRX_HLOS_TID_OVERRIDE_ENABLED | DP_TX_MESH_ENABLED)
- /* TODO Add support in TSO */
- #define DP_DESC_NUM_FRAG(x) 0
- /* disable TQM_BYPASS */
- #define TQM_BYPASS_WAR 0
- #define DP_RETRY_COUNT 7
- #ifdef WLAN_PEER_JITTER
- #define DP_AVG_JITTER_WEIGHT_DENOM 4
- #define DP_AVG_DELAY_WEIGHT_DENOM 3
- #endif
- #ifdef QCA_DP_TX_FW_METADATA_V2
- #define DP_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)\
- HTT_TX_TCL_METADATA_V2_PDEV_ID_SET(_var, _val)
- #define DP_TX_TCL_METADATA_VALID_HTT_SET(_var, _val) \
- HTT_TX_TCL_METADATA_V2_VALID_HTT_SET(_var, _val)
- #define DP_TX_TCL_METADATA_TYPE_SET(_var, _val) \
- HTT_TX_TCL_METADATA_TYPE_V2_SET(_var, _val)
- #define DP_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val) \
- HTT_TX_TCL_METADATA_V2_HOST_INSPECTED_SET(_var, _val)
- #define DP_TX_TCL_METADATA_PEER_ID_SET(_var, _val) \
- HTT_TX_TCL_METADATA_V2_PEER_ID_SET(_var, _val)
- #define DP_TX_TCL_METADATA_VDEV_ID_SET(_var, _val) \
- HTT_TX_TCL_METADATA_V2_VDEV_ID_SET(_var, _val)
- #define DP_TCL_METADATA_TYPE_PEER_BASED \
- HTT_TCL_METADATA_V2_TYPE_PEER_BASED
- #define DP_TCL_METADATA_TYPE_VDEV_BASED \
- HTT_TCL_METADATA_V2_TYPE_VDEV_BASED
- #else
- #define DP_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)\
- HTT_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)
- #define DP_TX_TCL_METADATA_VALID_HTT_SET(_var, _val) \
- HTT_TX_TCL_METADATA_VALID_HTT_SET(_var, _val)
- #define DP_TX_TCL_METADATA_TYPE_SET(_var, _val) \
- HTT_TX_TCL_METADATA_TYPE_SET(_var, _val)
- #define DP_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val) \
- HTT_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val)
- #define DP_TX_TCL_METADATA_PEER_ID_SET(_var, _val) \
- HTT_TX_TCL_METADATA_PEER_ID_SET(_var, _val)
- #define DP_TX_TCL_METADATA_VDEV_ID_SET(_var, _val) \
- HTT_TX_TCL_METADATA_VDEV_ID_SET(_var, _val)
- #define DP_TCL_METADATA_TYPE_PEER_BASED \
- HTT_TCL_METADATA_TYPE_PEER_BASED
- #define DP_TCL_METADATA_TYPE_VDEV_BASED \
- HTT_TCL_METADATA_TYPE_VDEV_BASED
- #endif
- #define DP_GET_HW_LINK_ID_FRM_PPDU_ID(PPDU_ID, LINK_ID_OFFSET, LINK_ID_BITS) \
- (((PPDU_ID) >> (LINK_ID_OFFSET)) & ((1 << (LINK_ID_BITS)) - 1))
- /*mapping between hal encrypt type and cdp_sec_type*/
- uint8_t sec_type_map[MAX_CDP_SEC_TYPE] = {HAL_TX_ENCRYPT_TYPE_NO_CIPHER,
- HAL_TX_ENCRYPT_TYPE_WEP_128,
- HAL_TX_ENCRYPT_TYPE_WEP_104,
- HAL_TX_ENCRYPT_TYPE_WEP_40,
- HAL_TX_ENCRYPT_TYPE_TKIP_WITH_MIC,
- HAL_TX_ENCRYPT_TYPE_TKIP_NO_MIC,
- HAL_TX_ENCRYPT_TYPE_AES_CCMP_128,
- HAL_TX_ENCRYPT_TYPE_WAPI,
- HAL_TX_ENCRYPT_TYPE_AES_CCMP_256,
- HAL_TX_ENCRYPT_TYPE_AES_GCMP_128,
- HAL_TX_ENCRYPT_TYPE_AES_GCMP_256,
- HAL_TX_ENCRYPT_TYPE_WAPI_GCM_SM4};
- qdf_export_symbol(sec_type_map);
- #ifdef WLAN_FEATURE_DP_TX_DESC_HISTORY
- static inline enum dp_tx_event_type dp_tx_get_event_type(uint32_t flags)
- {
- enum dp_tx_event_type type;
- if (flags & DP_TX_DESC_FLAG_FLUSH)
- type = DP_TX_DESC_FLUSH;
- else if (flags & DP_TX_DESC_FLAG_TX_COMP_ERR)
- type = DP_TX_COMP_UNMAP_ERR;
- else if (flags & DP_TX_DESC_FLAG_COMPLETED_TX)
- type = DP_TX_COMP_UNMAP;
- else
- type = DP_TX_DESC_UNMAP;
- return type;
- }
- static inline void
- dp_tx_desc_history_add(struct dp_soc *soc, dma_addr_t paddr,
- qdf_nbuf_t skb, uint32_t sw_cookie,
- enum dp_tx_event_type type)
- {
- struct dp_tx_tcl_history *tx_tcl_history = &soc->tx_tcl_history;
- struct dp_tx_comp_history *tx_comp_history = &soc->tx_comp_history;
- struct dp_tx_desc_event *entry;
- uint32_t idx;
- uint16_t slot;
- switch (type) {
- case DP_TX_COMP_UNMAP:
- case DP_TX_COMP_UNMAP_ERR:
- case DP_TX_COMP_MSDU_EXT:
- if (qdf_unlikely(!tx_comp_history->allocated))
- return;
- dp_get_frag_hist_next_atomic_idx(&tx_comp_history->index, &idx,
- &slot,
- DP_TX_COMP_HIST_SLOT_SHIFT,
- DP_TX_COMP_HIST_PER_SLOT_MAX,
- DP_TX_COMP_HISTORY_SIZE);
- entry = &tx_comp_history->entry[slot][idx];
- break;
- case DP_TX_DESC_MAP:
- case DP_TX_DESC_UNMAP:
- case DP_TX_DESC_COOKIE:
- case DP_TX_DESC_FLUSH:
- if (qdf_unlikely(!tx_tcl_history->allocated))
- return;
- dp_get_frag_hist_next_atomic_idx(&tx_tcl_history->index, &idx,
- &slot,
- DP_TX_TCL_HIST_SLOT_SHIFT,
- DP_TX_TCL_HIST_PER_SLOT_MAX,
- DP_TX_TCL_HISTORY_SIZE);
- entry = &tx_tcl_history->entry[slot][idx];
- break;
- default:
- dp_info_rl("Invalid dp_tx_event_type: %d", type);
- return;
- }
- entry->skb = skb;
- entry->paddr = paddr;
- entry->sw_cookie = sw_cookie;
- entry->type = type;
- entry->ts = qdf_get_log_timestamp();
- }
- static inline void
- dp_tx_tso_seg_history_add(struct dp_soc *soc,
- struct qdf_tso_seg_elem_t *tso_seg,
- qdf_nbuf_t skb, uint32_t sw_cookie,
- enum dp_tx_event_type type)
- {
- int i;
- for (i = 1; i < tso_seg->seg.num_frags; i++) {
- dp_tx_desc_history_add(soc, tso_seg->seg.tso_frags[i].paddr,
- skb, sw_cookie, type);
- }
- if (!tso_seg->next)
- dp_tx_desc_history_add(soc, tso_seg->seg.tso_frags[0].paddr,
- skb, 0xFFFFFFFF, type);
- }
- static inline void
- dp_tx_tso_history_add(struct dp_soc *soc, struct qdf_tso_info_t tso_info,
- qdf_nbuf_t skb, uint32_t sw_cookie,
- enum dp_tx_event_type type)
- {
- struct qdf_tso_seg_elem_t *curr_seg = tso_info.tso_seg_list;
- uint32_t num_segs = tso_info.num_segs;
- while (num_segs) {
- dp_tx_tso_seg_history_add(soc, curr_seg, skb, sw_cookie, type);
- curr_seg = curr_seg->next;
- num_segs--;
- }
- }
- #else
- static inline enum dp_tx_event_type dp_tx_get_event_type(uint32_t flags)
- {
- return DP_TX_DESC_INVAL_EVT;
- }
- static inline void
- dp_tx_desc_history_add(struct dp_soc *soc, dma_addr_t paddr,
- qdf_nbuf_t skb, uint32_t sw_cookie,
- enum dp_tx_event_type type)
- {
- }
- static inline void
- dp_tx_tso_seg_history_add(struct dp_soc *soc,
- struct qdf_tso_seg_elem_t *tso_seg,
- qdf_nbuf_t skb, uint32_t sw_cookie,
- enum dp_tx_event_type type)
- {
- }
- static inline void
- dp_tx_tso_history_add(struct dp_soc *soc, struct qdf_tso_info_t tso_info,
- qdf_nbuf_t skb, uint32_t sw_cookie,
- enum dp_tx_event_type type)
- {
- }
- #endif /* WLAN_FEATURE_DP_TX_DESC_HISTORY */
- /**
- * dp_is_tput_high() - Check if throughput is high
- *
- * @soc: core txrx main context
- *
- * The current function is based of the RTPM tput policy variable where RTPM is
- * avoided based on throughput.
- */
- static inline int dp_is_tput_high(struct dp_soc *soc)
- {
- return dp_get_rtpm_tput_policy_requirement(soc);
- }
- #if defined(FEATURE_TSO)
- /**
- * dp_tx_tso_unmap_segment() - Unmap TSO segment
- *
- * @soc: core txrx main context
- * @seg_desc: tso segment descriptor
- * @num_seg_desc: tso number segment descriptor
- */
- static void dp_tx_tso_unmap_segment(
- struct dp_soc *soc,
- struct qdf_tso_seg_elem_t *seg_desc,
- struct qdf_tso_num_seg_elem_t *num_seg_desc)
- {
- TSO_DEBUG("%s: Unmap the tso segment", __func__);
- if (qdf_unlikely(!seg_desc)) {
- DP_TRACE(ERROR, "%s %d TSO desc is NULL!",
- __func__, __LINE__);
- qdf_assert(0);
- } else if (qdf_unlikely(!num_seg_desc)) {
- DP_TRACE(ERROR, "%s %d TSO num desc is NULL!",
- __func__, __LINE__);
- qdf_assert(0);
- } else {
- bool is_last_seg;
- /* no tso segment left to do dma unmap */
- if (num_seg_desc->num_seg.tso_cmn_num_seg < 1)
- return;
- is_last_seg = (num_seg_desc->num_seg.tso_cmn_num_seg == 1) ?
- true : false;
- qdf_nbuf_unmap_tso_segment(soc->osdev,
- seg_desc, is_last_seg);
- num_seg_desc->num_seg.tso_cmn_num_seg--;
- }
- }
- /**
- * dp_tx_tso_desc_release() - Release the tso segment and tso_cmn_num_seg
- * back to the freelist
- *
- * @soc: soc device handle
- * @tx_desc: Tx software descriptor
- */
- static void dp_tx_tso_desc_release(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc)
- {
- TSO_DEBUG("%s: Free the tso descriptor", __func__);
- if (qdf_unlikely(!tx_desc->msdu_ext_desc->tso_desc)) {
- dp_tx_err("SO desc is NULL!");
- qdf_assert(0);
- } else if (qdf_unlikely(!tx_desc->msdu_ext_desc->tso_num_desc)) {
- dp_tx_err("TSO num desc is NULL!");
- qdf_assert(0);
- } else {
- struct qdf_tso_num_seg_elem_t *tso_num_desc =
- (struct qdf_tso_num_seg_elem_t *)tx_desc->
- msdu_ext_desc->tso_num_desc;
- /* Add the tso num segment into the free list */
- if (tso_num_desc->num_seg.tso_cmn_num_seg == 0) {
- dp_tso_num_seg_free(soc, tx_desc->pool_id,
- tx_desc->msdu_ext_desc->
- tso_num_desc);
- tx_desc->msdu_ext_desc->tso_num_desc = NULL;
- DP_STATS_INC(tx_desc->pdev, tso_stats.tso_comp, 1);
- }
- /* Add the tso segment into the free list*/
- dp_tx_tso_desc_free(soc,
- tx_desc->pool_id, tx_desc->msdu_ext_desc->
- tso_desc);
- tx_desc->msdu_ext_desc->tso_desc = NULL;
- }
- }
- #else
- static void dp_tx_tso_unmap_segment(
- struct dp_soc *soc,
- struct qdf_tso_seg_elem_t *seg_desc,
- struct qdf_tso_num_seg_elem_t *num_seg_desc)
- {
- }
- static void dp_tx_tso_desc_release(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc)
- {
- }
- #endif
- #ifdef WLAN_SUPPORT_PPEDS
- static inline int
- dp_tx_release_ds_tx_desc(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
- uint8_t desc_pool_id)
- {
- if (tx_desc->flags & DP_TX_DESC_FLAG_PPEDS) {
- __dp_tx_outstanding_dec(soc);
- dp_tx_desc_free(soc, tx_desc, desc_pool_id);
- return 1;
- }
- return 0;
- }
- #else
- static inline int
- dp_tx_release_ds_tx_desc(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
- uint8_t desc_pool_id)
- {
- return 0;
- }
- #endif
- void
- dp_tx_desc_release(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
- uint8_t desc_pool_id)
- {
- struct dp_pdev *pdev = tx_desc->pdev;
- uint8_t comp_status = 0;
- if (dp_tx_release_ds_tx_desc(soc, tx_desc, desc_pool_id))
- return;
- qdf_assert(pdev);
- soc = pdev->soc;
- dp_tx_outstanding_dec(pdev);
- if (tx_desc->msdu_ext_desc) {
- if (tx_desc->frm_type == dp_tx_frm_tso)
- dp_tx_tso_desc_release(soc, tx_desc);
- if (tx_desc->flags & DP_TX_DESC_FLAG_ME)
- dp_tx_me_free_buf(tx_desc->pdev,
- tx_desc->msdu_ext_desc->me_buffer);
- dp_tx_ext_desc_free(soc, tx_desc->msdu_ext_desc, desc_pool_id);
- tx_desc->msdu_ext_desc = NULL;
- }
- if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)
- qdf_atomic_dec(&soc->num_tx_exception);
- if (HAL_TX_COMP_RELEASE_SOURCE_TQM ==
- tx_desc->buffer_src)
- comp_status = hal_tx_comp_get_release_reason(&tx_desc->comp,
- soc->hal_soc);
- else
- comp_status = HAL_TX_COMP_RELEASE_REASON_FW;
- if (soc->dp_debug_log_en) {
- dp_tx_debug("Tx Completion Release desc %d status %d outstanding %d",
- tx_desc->id, comp_status,
- qdf_atomic_read(&pdev->num_tx_outstanding));
- }
- if (tx_desc->flags & DP_TX_DESC_FLAG_SPECIAL)
- dp_tx_spcl_desc_free(soc, tx_desc, desc_pool_id);
- else
- dp_tx_desc_free(soc, tx_desc, desc_pool_id);
- return;
- }
- /**
- * dp_tx_prepare_htt_metadata() - Prepare HTT metadata for special frames
- * @vdev: DP vdev Handle
- * @nbuf: skb
- * @msdu_info: msdu_info required to create HTT metadata
- *
- * Prepares and fills HTT metadata in the frame pre-header for special frames
- * that should be transmitted using varying transmit parameters.
- * There are 2 VDEV modes that currently needs this special metadata -
- * 1) Mesh Mode
- * 2) DSRC Mode
- *
- * Return: HTT metadata size
- *
- */
- static uint8_t dp_tx_prepare_htt_metadata(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- uint32_t *meta_data = msdu_info->meta_data;
- struct htt_tx_msdu_desc_ext2_t *desc_ext =
- (struct htt_tx_msdu_desc_ext2_t *) meta_data;
- uint8_t htt_desc_size;
- /* Size rounded of multiple of 8 bytes */
- uint8_t htt_desc_size_aligned;
- uint8_t *hdr = NULL;
- /*
- * Metadata - HTT MSDU Extension header
- */
- htt_desc_size = sizeof(struct htt_tx_msdu_desc_ext2_t);
- htt_desc_size_aligned = (htt_desc_size + 7) & ~0x7;
- if (vdev->mesh_vdev || msdu_info->is_tx_sniffer ||
- HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_GET(msdu_info->
- meta_data[0]) ||
- msdu_info->exception_fw) {
- if (qdf_unlikely(qdf_nbuf_headroom(nbuf) <
- htt_desc_size_aligned)) {
- nbuf = qdf_nbuf_realloc_headroom(nbuf,
- htt_desc_size_aligned);
- if (!nbuf) {
- /*
- * qdf_nbuf_realloc_headroom won't do skb_clone
- * as skb_realloc_headroom does. so, no free is
- * needed here.
- */
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].dropped.headroom_insufficient,
- 1);
- qdf_print(" %s[%d] skb_realloc_headroom failed",
- __func__, __LINE__);
- return 0;
- }
- }
- /* Fill and add HTT metaheader */
- hdr = qdf_nbuf_push_head(nbuf, htt_desc_size_aligned);
- if (!hdr) {
- dp_tx_err("Error in filling HTT metadata");
- return 0;
- }
- qdf_mem_copy(hdr, desc_ext, htt_desc_size);
- } else if (vdev->opmode == wlan_op_mode_ocb) {
- /* Todo - Add support for DSRC */
- }
- return htt_desc_size_aligned;
- }
- /**
- * dp_tx_prepare_tso_ext_desc() - Prepare MSDU extension descriptor for TSO
- * @tso_seg: TSO segment to process
- * @ext_desc: Pointer to MSDU extension descriptor
- *
- * Return: void
- */
- #if defined(FEATURE_TSO)
- static void dp_tx_prepare_tso_ext_desc(struct qdf_tso_seg_t *tso_seg,
- void *ext_desc)
- {
- uint8_t num_frag;
- uint32_t tso_flags;
- /*
- * Set tso_en, tcp_flags(NS, CWR, ECE, URG, ACK, PSH, RST, SYN, FIN),
- * tcp_flag_mask
- *
- * Checksum enable flags are set in TCL descriptor and not in Extension
- * Descriptor (H/W ignores checksum_en flags in MSDU ext descriptor)
- */
- tso_flags = *(uint32_t *) &tso_seg->tso_flags;
- hal_tx_ext_desc_set_tso_flags(ext_desc, tso_flags);
- hal_tx_ext_desc_set_msdu_length(ext_desc, tso_seg->tso_flags.l2_len,
- tso_seg->tso_flags.ip_len);
- hal_tx_ext_desc_set_tcp_seq(ext_desc, tso_seg->tso_flags.tcp_seq_num);
- hal_tx_ext_desc_set_ip_id(ext_desc, tso_seg->tso_flags.ip_id);
- for (num_frag = 0; num_frag < tso_seg->num_frags; num_frag++) {
- uint32_t lo = 0;
- uint32_t hi = 0;
- qdf_assert_always((tso_seg->tso_frags[num_frag].paddr) &&
- (tso_seg->tso_frags[num_frag].length));
- qdf_dmaaddr_to_32s(
- tso_seg->tso_frags[num_frag].paddr, &lo, &hi);
- hal_tx_ext_desc_set_buffer(ext_desc, num_frag, lo, hi,
- tso_seg->tso_frags[num_frag].length);
- }
- return;
- }
- #else
- static void dp_tx_prepare_tso_ext_desc(struct qdf_tso_seg_t *tso_seg,
- void *ext_desc)
- {
- return;
- }
- #endif
- #if defined(FEATURE_TSO)
- /**
- * dp_tx_free_tso_seg_list() - Loop through the tso segments
- * allocated and free them
- * @soc: soc handle
- * @free_seg: list of tso segments
- * @msdu_info: msdu descriptor
- *
- * Return: void
- */
- static void dp_tx_free_tso_seg_list(
- struct dp_soc *soc,
- struct qdf_tso_seg_elem_t *free_seg,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- struct qdf_tso_seg_elem_t *next_seg;
- while (free_seg) {
- next_seg = free_seg->next;
- dp_tx_tso_desc_free(soc,
- msdu_info->tx_queue.desc_pool_id,
- free_seg);
- free_seg = next_seg;
- }
- }
- /**
- * dp_tx_free_tso_num_seg_list() - Loop through the tso num segments
- * allocated and free them
- * @soc: soc handle
- * @free_num_seg: list of tso number segments
- * @msdu_info: msdu descriptor
- *
- * Return: void
- */
- static void dp_tx_free_tso_num_seg_list(
- struct dp_soc *soc,
- struct qdf_tso_num_seg_elem_t *free_num_seg,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- struct qdf_tso_num_seg_elem_t *next_num_seg;
- while (free_num_seg) {
- next_num_seg = free_num_seg->next;
- dp_tso_num_seg_free(soc,
- msdu_info->tx_queue.desc_pool_id,
- free_num_seg);
- free_num_seg = next_num_seg;
- }
- }
- /**
- * dp_tx_unmap_tso_seg_list() - Loop through the tso segments
- * do dma unmap for each segment
- * @soc: soc handle
- * @free_seg: list of tso segments
- * @num_seg_desc: tso number segment descriptor
- *
- * Return: void
- */
- static void dp_tx_unmap_tso_seg_list(
- struct dp_soc *soc,
- struct qdf_tso_seg_elem_t *free_seg,
- struct qdf_tso_num_seg_elem_t *num_seg_desc)
- {
- struct qdf_tso_seg_elem_t *next_seg;
- if (qdf_unlikely(!num_seg_desc)) {
- DP_TRACE(ERROR, "TSO number seg desc is NULL!");
- return;
- }
- while (free_seg) {
- next_seg = free_seg->next;
- dp_tx_tso_unmap_segment(soc, free_seg, num_seg_desc);
- free_seg = next_seg;
- }
- }
- #ifdef FEATURE_TSO_STATS
- /**
- * dp_tso_get_stats_idx() - Retrieve the tso packet id
- * @pdev: pdev handle
- *
- * Return: id
- */
- static uint32_t dp_tso_get_stats_idx(struct dp_pdev *pdev)
- {
- uint32_t stats_idx;
- stats_idx = (((uint32_t)qdf_atomic_inc_return(&pdev->tso_idx))
- % CDP_MAX_TSO_PACKETS);
- return stats_idx;
- }
- #else
- static int dp_tso_get_stats_idx(struct dp_pdev *pdev)
- {
- return 0;
- }
- #endif /* FEATURE_TSO_STATS */
- /**
- * dp_tx_free_remaining_tso_desc() - do dma unmap for tso segments if any,
- * free the tso segments descriptor and
- * tso num segments descriptor
- * @soc: soc handle
- * @msdu_info: msdu descriptor
- * @tso_seg_unmap: flag to show if dma unmap is necessary
- *
- * Return: void
- */
- static void dp_tx_free_remaining_tso_desc(struct dp_soc *soc,
- struct dp_tx_msdu_info_s *msdu_info,
- bool tso_seg_unmap)
- {
- struct qdf_tso_info_t *tso_info = &msdu_info->u.tso_info;
- struct qdf_tso_seg_elem_t *free_seg = tso_info->tso_seg_list;
- struct qdf_tso_num_seg_elem_t *tso_num_desc =
- tso_info->tso_num_seg_list;
- /* do dma unmap for each segment */
- if (tso_seg_unmap)
- dp_tx_unmap_tso_seg_list(soc, free_seg, tso_num_desc);
- /* free all tso number segment descriptor though looks only have 1 */
- dp_tx_free_tso_num_seg_list(soc, tso_num_desc, msdu_info);
- /* free all tso segment descriptor */
- dp_tx_free_tso_seg_list(soc, free_seg, msdu_info);
- }
- /**
- * dp_tx_prepare_tso() - Given a jumbo msdu, prepare the TSO info
- * @vdev: virtual device handle
- * @msdu: network buffer
- * @msdu_info: meta data associated with the msdu
- *
- * Return: QDF_STATUS_SUCCESS success
- */
- static QDF_STATUS dp_tx_prepare_tso(struct dp_vdev *vdev,
- qdf_nbuf_t msdu, struct dp_tx_msdu_info_s *msdu_info)
- {
- struct qdf_tso_seg_elem_t *tso_seg;
- int num_seg = qdf_nbuf_get_tso_num_seg(msdu);
- struct dp_soc *soc = vdev->pdev->soc;
- struct dp_pdev *pdev = vdev->pdev;
- struct qdf_tso_info_t *tso_info;
- struct qdf_tso_num_seg_elem_t *tso_num_seg;
- tso_info = &msdu_info->u.tso_info;
- tso_info->curr_seg = NULL;
- tso_info->tso_seg_list = NULL;
- tso_info->num_segs = num_seg;
- msdu_info->frm_type = dp_tx_frm_tso;
- tso_info->tso_num_seg_list = NULL;
- TSO_DEBUG(" %s: num_seg: %d", __func__, num_seg);
- while (num_seg) {
- tso_seg = dp_tx_tso_desc_alloc(
- soc, msdu_info->tx_queue.desc_pool_id);
- if (tso_seg) {
- tso_seg->next = tso_info->tso_seg_list;
- tso_info->tso_seg_list = tso_seg;
- num_seg--;
- } else {
- dp_err_rl("Failed to alloc tso seg desc");
- DP_STATS_INC_PKT(vdev->pdev,
- tso_stats.tso_no_mem_dropped, 1,
- qdf_nbuf_len(msdu));
- dp_tx_free_remaining_tso_desc(soc, msdu_info, false);
- return QDF_STATUS_E_NOMEM;
- }
- }
- TSO_DEBUG(" %s: num_seg: %d", __func__, num_seg);
- tso_num_seg = dp_tso_num_seg_alloc(soc,
- msdu_info->tx_queue.desc_pool_id);
- if (tso_num_seg) {
- tso_num_seg->next = tso_info->tso_num_seg_list;
- tso_info->tso_num_seg_list = tso_num_seg;
- } else {
- DP_TRACE(ERROR, "%s: Failed to alloc - Number of segs desc",
- __func__);
- dp_tx_free_remaining_tso_desc(soc, msdu_info, false);
- return QDF_STATUS_E_NOMEM;
- }
- msdu_info->num_seg =
- qdf_nbuf_get_tso_info(soc->osdev, msdu, tso_info);
- TSO_DEBUG(" %s: msdu_info->num_seg: %d", __func__,
- msdu_info->num_seg);
- if (!(msdu_info->num_seg)) {
- /*
- * Free allocated TSO seg desc and number seg desc,
- * do unmap for segments if dma map has done.
- */
- DP_TRACE(ERROR, "%s: Failed to get tso info", __func__);
- dp_tx_free_remaining_tso_desc(soc, msdu_info, true);
- return QDF_STATUS_E_INVAL;
- }
- dp_tx_tso_history_add(soc, msdu_info->u.tso_info,
- msdu, 0, DP_TX_DESC_MAP);
- tso_info->curr_seg = tso_info->tso_seg_list;
- tso_info->msdu_stats_idx = dp_tso_get_stats_idx(pdev);
- dp_tso_packet_update(pdev, tso_info->msdu_stats_idx,
- msdu, msdu_info->num_seg);
- dp_tso_segment_stats_update(pdev, tso_info->tso_seg_list,
- tso_info->msdu_stats_idx);
- dp_stats_tso_segment_histogram_update(pdev, msdu_info->num_seg);
- return QDF_STATUS_SUCCESS;
- }
- #else
- static QDF_STATUS dp_tx_prepare_tso(struct dp_vdev *vdev,
- qdf_nbuf_t msdu, struct dp_tx_msdu_info_s *msdu_info)
- {
- return QDF_STATUS_E_NOMEM;
- }
- #endif
- QDF_COMPILE_TIME_ASSERT(dp_tx_htt_metadata_len_check,
- (DP_TX_MSDU_INFO_META_DATA_DWORDS * 4 >=
- sizeof(struct htt_tx_msdu_desc_ext2_t)));
- /**
- * dp_tx_prepare_ext_desc() - Allocate and prepare MSDU extension descriptor
- * @vdev: DP Vdev handle
- * @msdu_info: MSDU info to be setup in MSDU extension descriptor
- * @desc_pool_id: Descriptor Pool ID
- *
- * Return:
- */
- static
- struct dp_tx_ext_desc_elem_s *dp_tx_prepare_ext_desc(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info, uint8_t desc_pool_id)
- {
- uint8_t i;
- uint8_t cached_ext_desc[HAL_TX_EXT_DESC_WITH_META_DATA];
- struct dp_tx_seg_info_s *seg_info;
- struct dp_tx_ext_desc_elem_s *msdu_ext_desc;
- struct dp_soc *soc = vdev->pdev->soc;
- /* Allocate an extension descriptor */
- msdu_ext_desc = dp_tx_ext_desc_alloc(soc, desc_pool_id);
- qdf_mem_zero(&cached_ext_desc[0], HAL_TX_EXT_DESC_WITH_META_DATA);
- if (!msdu_ext_desc) {
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].dropped.desc_na.num, 1);
- return NULL;
- }
- if (msdu_info->exception_fw &&
- qdf_unlikely(vdev->mesh_vdev)) {
- qdf_mem_copy(&cached_ext_desc[HAL_TX_EXTENSION_DESC_LEN_BYTES],
- &msdu_info->meta_data[0],
- sizeof(struct htt_tx_msdu_desc_ext2_t));
- qdf_atomic_inc(&soc->num_tx_exception);
- msdu_ext_desc->flags |= DP_TX_EXT_DESC_FLAG_METADATA_VALID;
- }
- switch (msdu_info->frm_type) {
- case dp_tx_frm_sg:
- case dp_tx_frm_me:
- case dp_tx_frm_raw:
- seg_info = msdu_info->u.sg_info.curr_seg;
- /* Update the buffer pointers in MSDU Extension Descriptor */
- for (i = 0; i < seg_info->frag_cnt; i++) {
- hal_tx_ext_desc_set_buffer(&cached_ext_desc[0], i,
- seg_info->frags[i].paddr_lo,
- seg_info->frags[i].paddr_hi,
- seg_info->frags[i].len);
- }
- break;
- case dp_tx_frm_tso:
- dp_tx_prepare_tso_ext_desc(&msdu_info->u.tso_info.curr_seg->seg,
- &cached_ext_desc[0]);
- break;
- default:
- break;
- }
- QDF_TRACE_HEX_DUMP(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_DEBUG,
- cached_ext_desc, HAL_TX_EXT_DESC_WITH_META_DATA);
- hal_tx_ext_desc_sync(&cached_ext_desc[0],
- msdu_ext_desc->vaddr);
- return msdu_ext_desc;
- }
- /**
- * dp_tx_trace_pkt() - Trace TX packet at DP layer
- * @soc: datapath SOC
- * @skb: skb to be traced
- * @msdu_id: msdu_id of the packet
- * @vdev_id: vdev_id of the packet
- * @op_mode: Vdev Operation mode
- *
- * Return: None
- */
- #ifdef DP_DISABLE_TX_PKT_TRACE
- static void dp_tx_trace_pkt(struct dp_soc *soc,
- qdf_nbuf_t skb, uint16_t msdu_id,
- uint8_t vdev_id, enum QDF_OPMODE op_mode)
- {
- }
- #else
- static void dp_tx_trace_pkt(struct dp_soc *soc,
- qdf_nbuf_t skb, uint16_t msdu_id,
- uint8_t vdev_id, enum QDF_OPMODE op_mode)
- {
- if (dp_is_tput_high(soc))
- return;
- QDF_NBUF_CB_TX_PACKET_TRACK(skb) = QDF_NBUF_TX_PKT_DATA_TRACK;
- QDF_NBUF_CB_TX_DP_TRACE(skb) = 1;
- DPTRACE(qdf_dp_trace_ptr(skb,
- QDF_DP_TRACE_LI_DP_TX_PACKET_PTR_RECORD,
- QDF_TRACE_DEFAULT_PDEV_ID,
- qdf_nbuf_data_addr(skb),
- sizeof(qdf_nbuf_data(skb)),
- msdu_id, vdev_id, 0,
- op_mode));
- qdf_dp_trace_log_pkt(vdev_id, skb, QDF_TX, QDF_TRACE_DEFAULT_PDEV_ID,
- op_mode);
- DPTRACE(qdf_dp_trace_data_pkt(skb, QDF_TRACE_DEFAULT_PDEV_ID,
- QDF_DP_TRACE_LI_DP_TX_PACKET_RECORD,
- msdu_id, QDF_TX));
- }
- #endif
- #ifdef WLAN_DP_FEATURE_MARK_ICMP_REQ_TO_FW
- /**
- * dp_tx_is_nbuf_marked_exception() - Check if the packet has been marked as
- * exception by the upper layer (OS_IF)
- * @soc: DP soc handle
- * @nbuf: packet to be transmitted
- *
- * Return: 1 if the packet is marked as exception,
- * 0, if the packet is not marked as exception.
- */
- static inline int dp_tx_is_nbuf_marked_exception(struct dp_soc *soc,
- qdf_nbuf_t nbuf)
- {
- return QDF_NBUF_CB_TX_PACKET_TO_FW(nbuf);
- }
- #else
- static inline int dp_tx_is_nbuf_marked_exception(struct dp_soc *soc,
- qdf_nbuf_t nbuf)
- {
- return 0;
- }
- #endif
- #ifdef DP_TRAFFIC_END_INDICATION
- /**
- * dp_tx_get_traffic_end_indication_pkt() - Allocate and prepare packet to send
- * as indication to fw to inform that
- * data stream has ended
- * @vdev: DP vdev handle
- * @nbuf: original buffer from network stack
- *
- * Return: NULL on failure,
- * nbuf on success
- */
- static inline qdf_nbuf_t
- dp_tx_get_traffic_end_indication_pkt(struct dp_vdev *vdev,
- qdf_nbuf_t nbuf)
- {
- /* Packet length should be enough to copy upto L3 header */
- uint8_t end_nbuf_len = 64;
- uint8_t htt_desc_size_aligned;
- uint8_t htt_desc_size;
- qdf_nbuf_t end_nbuf;
- if (qdf_unlikely(QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) ==
- QDF_NBUF_CB_PACKET_TYPE_END_INDICATION)) {
- htt_desc_size = sizeof(struct htt_tx_msdu_desc_ext2_t);
- htt_desc_size_aligned = (htt_desc_size + 7) & ~0x7;
- end_nbuf = qdf_nbuf_queue_remove(&vdev->end_ind_pkt_q);
- if (!end_nbuf) {
- end_nbuf = qdf_nbuf_alloc(NULL,
- (htt_desc_size_aligned +
- end_nbuf_len),
- htt_desc_size_aligned,
- 8, false);
- if (!end_nbuf) {
- dp_err("Packet allocation failed");
- goto out;
- }
- } else {
- qdf_nbuf_reset(end_nbuf, htt_desc_size_aligned, 8);
- }
- qdf_mem_copy(qdf_nbuf_data(end_nbuf), qdf_nbuf_data(nbuf),
- end_nbuf_len);
- qdf_nbuf_set_pktlen(end_nbuf, end_nbuf_len);
- return end_nbuf;
- }
- out:
- return NULL;
- }
- /**
- * dp_tx_send_traffic_end_indication_pkt() - Send indication packet to FW
- * via exception path.
- * @vdev: DP vdev handle
- * @end_nbuf: skb to send as indication
- * @msdu_info: msdu_info of original nbuf
- * @peer_id: peer id
- *
- * Return: None
- */
- static inline void
- dp_tx_send_traffic_end_indication_pkt(struct dp_vdev *vdev,
- qdf_nbuf_t end_nbuf,
- struct dp_tx_msdu_info_s *msdu_info,
- uint16_t peer_id)
- {
- struct dp_tx_msdu_info_s e_msdu_info = {0};
- qdf_nbuf_t nbuf;
- struct htt_tx_msdu_desc_ext2_t *desc_ext =
- (struct htt_tx_msdu_desc_ext2_t *)(e_msdu_info.meta_data);
- e_msdu_info.tx_queue = msdu_info->tx_queue;
- e_msdu_info.tid = msdu_info->tid;
- e_msdu_info.exception_fw = 1;
- e_msdu_info.xmit_type = msdu_info->xmit_type;
- desc_ext->host_tx_desc_pool = 1;
- desc_ext->traffic_end_indication = 1;
- nbuf = dp_tx_send_msdu_single(vdev, end_nbuf, &e_msdu_info,
- peer_id, NULL);
- if (nbuf) {
- dp_err("Traffic end indication packet tx failed");
- qdf_nbuf_free(nbuf);
- }
- }
- /**
- * dp_tx_traffic_end_indication_set_desc_flag() - Set tx descriptor flag to
- * mark it traffic end indication
- * packet.
- * @tx_desc: Tx descriptor pointer
- * @msdu_info: msdu_info structure pointer
- *
- * Return: None
- */
- static inline void
- dp_tx_traffic_end_indication_set_desc_flag(struct dp_tx_desc_s *tx_desc,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- struct htt_tx_msdu_desc_ext2_t *desc_ext =
- (struct htt_tx_msdu_desc_ext2_t *)(msdu_info->meta_data);
- if (qdf_unlikely(desc_ext->traffic_end_indication))
- tx_desc->flags |= DP_TX_DESC_FLAG_TRAFFIC_END_IND;
- }
- /**
- * dp_tx_traffic_end_indication_enq_ind_pkt() - Enqueue the packet instead of
- * freeing which are associated
- * with traffic end indication
- * flagged descriptor.
- * @soc: dp soc handle
- * @desc: Tx descriptor pointer
- * @nbuf: buffer pointer
- *
- * Return: True if packet gets enqueued else false
- */
- static bool
- dp_tx_traffic_end_indication_enq_ind_pkt(struct dp_soc *soc,
- struct dp_tx_desc_s *desc,
- qdf_nbuf_t nbuf)
- {
- struct dp_vdev *vdev = NULL;
- if (qdf_unlikely((desc->flags &
- DP_TX_DESC_FLAG_TRAFFIC_END_IND) != 0)) {
- vdev = dp_vdev_get_ref_by_id(soc, desc->vdev_id,
- DP_MOD_ID_TX_COMP);
- if (vdev) {
- qdf_nbuf_queue_add(&vdev->end_ind_pkt_q, nbuf);
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_COMP);
- return true;
- }
- }
- return false;
- }
- /**
- * dp_tx_traffic_end_indication_is_enabled() - get the feature
- * enable/disable status
- * @vdev: dp vdev handle
- *
- * Return: True if feature is enable else false
- */
- static inline bool
- dp_tx_traffic_end_indication_is_enabled(struct dp_vdev *vdev)
- {
- return qdf_unlikely(vdev->traffic_end_ind_en);
- }
- static inline qdf_nbuf_t
- dp_tx_send_msdu_single_wrapper(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info,
- uint16_t peer_id, qdf_nbuf_t end_nbuf)
- {
- if (dp_tx_traffic_end_indication_is_enabled(vdev))
- end_nbuf = dp_tx_get_traffic_end_indication_pkt(vdev, nbuf);
- nbuf = dp_tx_send_msdu_single(vdev, nbuf, msdu_info, peer_id, NULL);
- if (qdf_unlikely(end_nbuf))
- dp_tx_send_traffic_end_indication_pkt(vdev, end_nbuf,
- msdu_info, peer_id);
- return nbuf;
- }
- #else
- static inline qdf_nbuf_t
- dp_tx_get_traffic_end_indication_pkt(struct dp_vdev *vdev,
- qdf_nbuf_t nbuf)
- {
- return NULL;
- }
- static inline void
- dp_tx_send_traffic_end_indication_pkt(struct dp_vdev *vdev,
- qdf_nbuf_t end_nbuf,
- struct dp_tx_msdu_info_s *msdu_info,
- uint16_t peer_id)
- {}
- static inline void
- dp_tx_traffic_end_indication_set_desc_flag(struct dp_tx_desc_s *tx_desc,
- struct dp_tx_msdu_info_s *msdu_info)
- {}
- static inline bool
- dp_tx_traffic_end_indication_enq_ind_pkt(struct dp_soc *soc,
- struct dp_tx_desc_s *desc,
- qdf_nbuf_t nbuf)
- {
- return false;
- }
- static inline bool
- dp_tx_traffic_end_indication_is_enabled(struct dp_vdev *vdev)
- {
- return false;
- }
- static inline qdf_nbuf_t
- dp_tx_send_msdu_single_wrapper(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info,
- uint16_t peer_id, qdf_nbuf_t end_nbuf)
- {
- return dp_tx_send_msdu_single(vdev, nbuf, msdu_info, peer_id, NULL);
- }
- #endif
- #if defined(QCA_SUPPORT_WDS_EXTENDED)
- static bool
- dp_tx_is_wds_ast_override_en(struct dp_soc *soc,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- if (soc->features.wds_ext_ast_override_enable &&
- tx_exc_metadata && tx_exc_metadata->is_wds_extended)
- return true;
- return false;
- }
- #else
- static bool
- dp_tx_is_wds_ast_override_en(struct dp_soc *soc,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- return false;
- }
- #endif
- /**
- * dp_tx_prepare_desc_single() - Allocate and prepare Tx descriptor
- * @vdev: DP vdev handle
- * @nbuf: skb
- * @desc_pool_id: Descriptor pool ID
- * @msdu_info: Metadata to the fw
- * @tx_exc_metadata: Handle that holds exception path metadata
- *
- * Allocate and prepare Tx descriptor with msdu information.
- *
- * Return: Pointer to Tx Descriptor on success,
- * NULL on failure
- */
- static
- struct dp_tx_desc_s *dp_tx_prepare_desc_single(struct dp_vdev *vdev,
- qdf_nbuf_t nbuf, uint8_t desc_pool_id,
- struct dp_tx_msdu_info_s *msdu_info,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- uint8_t align_pad;
- uint8_t is_exception = 0;
- uint8_t htt_hdr_size;
- struct dp_tx_desc_s *tx_desc;
- struct dp_pdev *pdev = vdev->pdev;
- struct dp_soc *soc = pdev->soc;
- uint8_t xmit_type = msdu_info->xmit_type;
- if (dp_tx_limit_check(vdev, nbuf))
- return NULL;
- /* Allocate software Tx descriptor */
- if (nbuf->protocol == QDF_NBUF_TRAC_EAPOL_ETH_TYPE)
- tx_desc = dp_tx_spcl_desc_alloc(soc, desc_pool_id);
- else
- tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
- if (qdf_unlikely(!tx_desc)) {
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.desc_na.num, 1);
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.desc_na_exc_alloc_fail.num,
- 1);
- return NULL;
- }
- dp_tx_outstanding_inc(pdev);
- /* Initialize the SW tx descriptor */
- tx_desc->nbuf = nbuf;
- tx_desc->frm_type = dp_tx_frm_std;
- tx_desc->tx_encap_type = ((tx_exc_metadata &&
- (tx_exc_metadata->tx_encap_type != CDP_INVALID_TX_ENCAP_TYPE)) ?
- tx_exc_metadata->tx_encap_type : vdev->tx_encap_type);
- tx_desc->vdev_id = vdev->vdev_id;
- tx_desc->pdev = pdev;
- tx_desc->msdu_ext_desc = NULL;
- tx_desc->pkt_offset = 0;
- tx_desc->length = qdf_nbuf_headlen(nbuf);
- dp_tx_trace_pkt(soc, nbuf, tx_desc->id, vdev->vdev_id,
- vdev->qdf_opmode);
- if (qdf_unlikely(vdev->multipass_en)) {
- if (!dp_tx_multipass_process(soc, vdev, nbuf, msdu_info))
- goto failure;
- }
- /* Packets marked by upper layer (OS-IF) to be sent to FW */
- if (dp_tx_is_nbuf_marked_exception(soc, nbuf))
- is_exception = 1;
- /* for BE chipsets if wds extension was enbled will not mark FW
- * in desc will mark ast index based search for ast index.
- */
- if (dp_tx_is_wds_ast_override_en(soc, tx_exc_metadata))
- return tx_desc;
- /*
- * For special modes (vdev_type == ocb or mesh), data frames should be
- * transmitted using varying transmit parameters (tx spec) which include
- * transmit rate, power, priority, channel, channel bandwidth , nss etc.
- * These are filled in HTT MSDU descriptor and sent in frame pre-header.
- * These frames are sent as exception packets to firmware.
- *
- * HW requirement is that metadata should always point to a
- * 8-byte aligned address. So we add alignment pad to start of buffer.
- * HTT Metadata should be ensured to be multiple of 8-bytes,
- * to get 8-byte aligned start address along with align_pad added
- *
- * |-----------------------------|
- * | |
- * |-----------------------------| <-----Buffer Pointer Address given
- * | | ^ in HW descriptor (aligned)
- * | HTT Metadata | |
- * | | |
- * | | | Packet Offset given in descriptor
- * | | |
- * |-----------------------------| |
- * | Alignment Pad | v
- * |-----------------------------| <----- Actual buffer start address
- * | SKB Data | (Unaligned)
- * | |
- * | |
- * | |
- * | |
- * | |
- * |-----------------------------|
- */
- if (qdf_unlikely((msdu_info->exception_fw)) ||
- (vdev->opmode == wlan_op_mode_ocb) ||
- (tx_exc_metadata &&
- tx_exc_metadata->is_tx_sniffer)) {
- align_pad = ((unsigned long) qdf_nbuf_data(nbuf)) & 0x7;
- if (qdf_unlikely(qdf_nbuf_headroom(nbuf) < align_pad)) {
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.headroom_insufficient,
- 1);
- goto failure;
- }
- if (qdf_nbuf_push_head(nbuf, align_pad) == NULL) {
- dp_tx_err("qdf_nbuf_push_head failed");
- goto failure;
- }
- htt_hdr_size = dp_tx_prepare_htt_metadata(vdev, nbuf,
- msdu_info);
- if (htt_hdr_size == 0)
- goto failure;
- tx_desc->length = qdf_nbuf_headlen(nbuf);
- tx_desc->pkt_offset = align_pad + htt_hdr_size;
- tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
- dp_tx_traffic_end_indication_set_desc_flag(tx_desc,
- msdu_info);
- is_exception = 1;
- tx_desc->length -= tx_desc->pkt_offset;
- }
- #if !TQM_BYPASS_WAR
- if (is_exception || tx_exc_metadata)
- #endif
- {
- /* Temporary WAR due to TQM VP issues */
- tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
- qdf_atomic_inc(&soc->num_tx_exception);
- }
- return tx_desc;
- failure:
- dp_tx_desc_release(soc, tx_desc, desc_pool_id);
- return NULL;
- }
- /**
- * dp_tx_prepare_desc() - Allocate and prepare Tx descriptor for multisegment
- * frame
- * @vdev: DP vdev handle
- * @nbuf: skb
- * @msdu_info: Info to be setup in MSDU descriptor and MSDU extension descriptor
- * @desc_pool_id : Descriptor Pool ID
- *
- * Allocate and prepare Tx descriptor with msdu and fragment descritor
- * information. For frames with fragments, allocate and prepare
- * an MSDU extension descriptor
- *
- * Return: Pointer to Tx Descriptor on success,
- * NULL on failure
- */
- static struct dp_tx_desc_s *dp_tx_prepare_desc(struct dp_vdev *vdev,
- qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info,
- uint8_t desc_pool_id)
- {
- struct dp_tx_desc_s *tx_desc;
- struct dp_tx_ext_desc_elem_s *msdu_ext_desc;
- struct dp_pdev *pdev = vdev->pdev;
- struct dp_soc *soc = pdev->soc;
- if (dp_tx_limit_check(vdev, nbuf))
- return NULL;
- /* Allocate software Tx descriptor */
- if (nbuf->protocol == QDF_NBUF_TRAC_EAPOL_ETH_TYPE)
- tx_desc = dp_tx_spcl_desc_alloc(soc, desc_pool_id);
- else
- tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
- if (!tx_desc) {
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].dropped.desc_na.num, 1);
- return NULL;
- }
- dp_tx_tso_seg_history_add(soc, msdu_info->u.tso_info.curr_seg,
- nbuf, tx_desc->id, DP_TX_DESC_COOKIE);
- dp_tx_outstanding_inc(pdev);
- /* Initialize the SW tx descriptor */
- tx_desc->nbuf = nbuf;
- tx_desc->frm_type = msdu_info->frm_type;
- tx_desc->tx_encap_type = vdev->tx_encap_type;
- tx_desc->vdev_id = vdev->vdev_id;
- tx_desc->pdev = pdev;
- tx_desc->pkt_offset = 0;
- dp_tx_trace_pkt(soc, nbuf, tx_desc->id, vdev->vdev_id,
- vdev->qdf_opmode);
- /* Handle scattered frames - TSO/SG/ME */
- /* Allocate and prepare an extension descriptor for scattered frames */
- msdu_ext_desc = dp_tx_prepare_ext_desc(vdev, msdu_info, desc_pool_id);
- if (!msdu_ext_desc) {
- dp_tx_info("Tx Extension Descriptor Alloc Fail");
- goto failure;
- }
- #if !TQM_BYPASS_WAR
- if (qdf_unlikely(msdu_info->exception_fw) ||
- dp_tx_is_nbuf_marked_exception(soc, nbuf))
- #endif
- {
- /* Temporary WAR due to TQM VP issues */
- tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
- qdf_atomic_inc(&soc->num_tx_exception);
- }
- tx_desc->msdu_ext_desc = msdu_ext_desc;
- tx_desc->flags |= DP_TX_DESC_FLAG_FRAG;
- msdu_ext_desc->tso_desc = msdu_info->u.tso_info.curr_seg;
- msdu_ext_desc->tso_num_desc = msdu_info->u.tso_info.tso_num_seg_list;
- tx_desc->dma_addr = msdu_ext_desc->paddr;
- if (msdu_ext_desc->flags & DP_TX_EXT_DESC_FLAG_METADATA_VALID)
- tx_desc->length = HAL_TX_EXT_DESC_WITH_META_DATA;
- else
- tx_desc->length = HAL_TX_EXTENSION_DESC_LEN_BYTES;
- return tx_desc;
- failure:
- dp_tx_desc_release(soc, tx_desc, desc_pool_id);
- return NULL;
- }
- /**
- * dp_tx_prepare_raw() - Prepare RAW packet TX
- * @vdev: DP vdev handle
- * @nbuf: buffer pointer
- * @seg_info: Pointer to Segment info Descriptor to be prepared
- * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension
- * descriptor
- *
- * Return:
- */
- static qdf_nbuf_t dp_tx_prepare_raw(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_seg_info_s *seg_info, struct dp_tx_msdu_info_s *msdu_info)
- {
- qdf_nbuf_t curr_nbuf = NULL;
- uint16_t total_len = 0;
- qdf_dma_addr_t paddr;
- int32_t i;
- int32_t mapped_buf_num = 0;
- struct dp_tx_sg_info_s *sg_info = &msdu_info->u.sg_info;
- qdf_dot3_qosframe_t *qos_wh = (qdf_dot3_qosframe_t *) nbuf->data;
- DP_STATS_INC_PKT(vdev, tx_i[msdu_info->xmit_type].raw.raw_pkt,
- 1, qdf_nbuf_len(nbuf));
- /* Continue only if frames are of DATA type */
- if (!DP_FRAME_IS_DATA(qos_wh)) {
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].raw.invalid_raw_pkt_datatype,
- 1);
- dp_tx_debug("Pkt. recd is of not data type");
- goto error;
- }
- /* SWAR for HW: Enable WEP bit in the AMSDU frames for RAW mode */
- if (vdev->raw_mode_war &&
- (qos_wh->i_fc[0] & QDF_IEEE80211_FC0_SUBTYPE_QOS) &&
- (qos_wh->i_qos[0] & IEEE80211_QOS_AMSDU))
- qos_wh->i_fc[1] |= IEEE80211_FC1_WEP;
- for (curr_nbuf = nbuf, i = 0; curr_nbuf;
- curr_nbuf = qdf_nbuf_next(curr_nbuf), i++) {
- /*
- * Number of nbuf's must not exceed the size of the frags
- * array in seg_info.
- */
- if (i >= DP_TX_MAX_NUM_FRAGS) {
- dp_err_rl("nbuf cnt exceeds the max number of segs");
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].raw.num_frags_overflow_err,
- 1);
- goto error;
- }
- if (QDF_STATUS_SUCCESS !=
- qdf_nbuf_map_nbytes_single(vdev->osdev,
- curr_nbuf,
- QDF_DMA_TO_DEVICE,
- curr_nbuf->len)) {
- dp_tx_err("%s dma map error ", __func__);
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].raw.dma_map_error,
- 1);
- goto error;
- }
- /* Update the count of mapped nbuf's */
- mapped_buf_num++;
- paddr = qdf_nbuf_get_frag_paddr(curr_nbuf, 0);
- seg_info->frags[i].paddr_lo = paddr;
- seg_info->frags[i].paddr_hi = ((uint64_t)paddr >> 32);
- seg_info->frags[i].len = qdf_nbuf_len(curr_nbuf);
- seg_info->frags[i].vaddr = (void *) curr_nbuf;
- total_len += qdf_nbuf_len(curr_nbuf);
- }
- seg_info->frag_cnt = i;
- seg_info->total_len = total_len;
- seg_info->next = NULL;
- sg_info->curr_seg = seg_info;
- msdu_info->frm_type = dp_tx_frm_raw;
- msdu_info->num_seg = 1;
- return nbuf;
- error:
- i = 0;
- while (nbuf) {
- curr_nbuf = nbuf;
- if (i < mapped_buf_num) {
- qdf_nbuf_unmap_nbytes_single(vdev->osdev, curr_nbuf,
- QDF_DMA_TO_DEVICE,
- curr_nbuf->len);
- i++;
- }
- nbuf = qdf_nbuf_next(nbuf);
- qdf_nbuf_free(curr_nbuf);
- }
- return NULL;
- }
- /**
- * dp_tx_raw_prepare_unset() - unmap the chain of nbufs belonging to RAW frame.
- * @soc: DP soc handle
- * @nbuf: Buffer pointer
- *
- * unmap the chain of nbufs that belong to this RAW frame.
- *
- * Return: None
- */
- static void dp_tx_raw_prepare_unset(struct dp_soc *soc,
- qdf_nbuf_t nbuf)
- {
- qdf_nbuf_t cur_nbuf = nbuf;
- do {
- qdf_nbuf_unmap_nbytes_single(soc->osdev, cur_nbuf,
- QDF_DMA_TO_DEVICE,
- cur_nbuf->len);
- cur_nbuf = qdf_nbuf_next(cur_nbuf);
- } while (cur_nbuf);
- }
- #ifdef VDEV_PEER_PROTOCOL_COUNT
- void dp_vdev_peer_stats_update_protocol_cnt_tx(struct dp_vdev *vdev_hdl,
- qdf_nbuf_t nbuf)
- {
- qdf_nbuf_t nbuf_local;
- struct dp_vdev *vdev_local = vdev_hdl;
- do {
- if (qdf_likely(!((vdev_local)->peer_protocol_count_track)))
- break;
- nbuf_local = nbuf;
- if (qdf_unlikely(((vdev_local)->tx_encap_type) ==
- htt_cmn_pkt_type_raw))
- break;
- else if (qdf_unlikely(qdf_nbuf_is_nonlinear((nbuf_local))))
- break;
- else if (qdf_nbuf_is_tso((nbuf_local)))
- break;
- dp_vdev_peer_stats_update_protocol_cnt((vdev_local),
- (nbuf_local),
- NULL, 1, 0);
- } while (0);
- }
- #endif
- #ifdef WLAN_DP_FEATURE_SW_LATENCY_MGR
- void dp_tx_update_stats(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc,
- uint8_t ring_id)
- {
- uint32_t stats_len = dp_tx_get_pkt_len(tx_desc);
- DP_STATS_INC_PKT(soc, tx.egress[ring_id], 1, stats_len);
- }
- int
- dp_tx_attempt_coalescing(struct dp_soc *soc, struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- uint8_t tid,
- struct dp_tx_msdu_info_s *msdu_info,
- uint8_t ring_id)
- {
- struct dp_swlm *swlm = &soc->swlm;
- union swlm_data swlm_query_data;
- struct dp_swlm_tcl_data tcl_data;
- QDF_STATUS status;
- int ret;
- if (!swlm->is_enabled)
- return msdu_info->skip_hp_update;
- tcl_data.nbuf = tx_desc->nbuf;
- tcl_data.tid = tid;
- tcl_data.ring_id = ring_id;
- tcl_data.pkt_len = dp_tx_get_pkt_len(tx_desc);
- tcl_data.num_ll_connections = vdev->num_latency_critical_conn;
- swlm_query_data.tcl_data = &tcl_data;
- status = dp_swlm_tcl_pre_check(soc, &tcl_data);
- if (QDF_IS_STATUS_ERROR(status)) {
- dp_swlm_tcl_reset_session_data(soc, ring_id);
- DP_STATS_INC(swlm, tcl[ring_id].coalesce_fail, 1);
- return 0;
- }
- ret = dp_swlm_query_policy(soc, TCL_DATA, swlm_query_data);
- if (ret) {
- DP_STATS_INC(swlm, tcl[ring_id].coalesce_success, 1);
- } else {
- DP_STATS_INC(swlm, tcl[ring_id].coalesce_fail, 1);
- }
- return ret;
- }
- void
- dp_tx_ring_access_end(struct dp_soc *soc, hal_ring_handle_t hal_ring_hdl,
- int coalesce)
- {
- if (coalesce)
- dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
- else
- dp_tx_hal_ring_access_end(soc, hal_ring_hdl);
- }
- static inline void
- dp_tx_is_hp_update_required(uint32_t i, struct dp_tx_msdu_info_s *msdu_info)
- {
- if (((i + 1) < msdu_info->num_seg))
- msdu_info->skip_hp_update = 1;
- else
- msdu_info->skip_hp_update = 0;
- }
- static inline void
- dp_flush_tcp_hp(struct dp_soc *soc, uint8_t ring_id)
- {
- hal_ring_handle_t hal_ring_hdl =
- dp_tx_get_hal_ring_hdl(soc, ring_id);
- if (dp_tx_hal_ring_access_start(soc, hal_ring_hdl)) {
- dp_err("Fillmore: SRNG access start failed");
- return;
- }
- dp_tx_ring_access_end_wrapper(soc, hal_ring_hdl, 0);
- }
- static inline void
- dp_tx_check_and_flush_hp(struct dp_soc *soc,
- QDF_STATUS status,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- if (QDF_IS_STATUS_ERROR(status) && !msdu_info->skip_hp_update) {
- dp_flush_tcp_hp(soc,
- (msdu_info->tx_queue.ring_id & DP_TX_QUEUE_MASK));
- }
- }
- #else
- static inline void
- dp_tx_is_hp_update_required(uint32_t i, struct dp_tx_msdu_info_s *msdu_info)
- {
- }
- static inline void
- dp_tx_check_and_flush_hp(struct dp_soc *soc,
- QDF_STATUS status,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- }
- #endif
- #ifdef FEATURE_RUNTIME_PM
- void
- dp_tx_ring_access_end_wrapper(struct dp_soc *soc,
- hal_ring_handle_t hal_ring_hdl,
- int coalesce)
- {
- int ret;
- /*
- * Avoid runtime get and put APIs under high throughput scenarios.
- */
- if (dp_get_rtpm_tput_policy_requirement(soc)) {
- dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
- return;
- }
- ret = hif_rtpm_get(HIF_RTPM_GET_ASYNC, HIF_RTPM_ID_DP);
- if (QDF_IS_STATUS_SUCCESS(ret)) {
- if (hif_system_pm_state_check(soc->hif_handle)) {
- dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
- hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
- hal_srng_inc_flush_cnt(hal_ring_hdl);
- } else {
- dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
- }
- hif_rtpm_put(HIF_RTPM_PUT_ASYNC, HIF_RTPM_ID_DP);
- } else {
- dp_runtime_get(soc);
- dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
- hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
- qdf_atomic_inc(&soc->tx_pending_rtpm);
- hal_srng_inc_flush_cnt(hal_ring_hdl);
- dp_runtime_put(soc);
- }
- }
- #else
- #ifdef DP_POWER_SAVE
- void
- dp_tx_ring_access_end_wrapper(struct dp_soc *soc,
- hal_ring_handle_t hal_ring_hdl,
- int coalesce)
- {
- if (hif_system_pm_state_check(soc->hif_handle)) {
- dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
- hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
- hal_srng_inc_flush_cnt(hal_ring_hdl);
- } else {
- dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
- }
- }
- #endif
- #endif
- /**
- * dp_tx_get_tid() - Obtain TID to be used for this frame
- * @vdev: DP vdev handle
- * @nbuf: skb
- * @msdu_info: msdu descriptor
- *
- * Extract the DSCP or PCP information from frame and map into TID value.
- *
- * Return: void
- */
- static void dp_tx_get_tid(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- uint8_t tos = 0, dscp_tid_override = 0;
- uint8_t *hdr_ptr, *L3datap;
- uint8_t is_mcast = 0;
- qdf_ether_header_t *eh = NULL;
- qdf_ethervlan_header_t *evh = NULL;
- uint16_t ether_type;
- qdf_llc_t *llcHdr;
- struct dp_pdev *pdev = (struct dp_pdev *)vdev->pdev;
- DP_TX_TID_OVERRIDE(msdu_info, nbuf);
- if (qdf_likely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
- eh = (qdf_ether_header_t *)nbuf->data;
- hdr_ptr = (uint8_t *)(eh->ether_dhost);
- L3datap = hdr_ptr + sizeof(qdf_ether_header_t);
- } else {
- qdf_dot3_qosframe_t *qos_wh =
- (qdf_dot3_qosframe_t *) nbuf->data;
- msdu_info->tid = qos_wh->i_fc[0] & DP_FC0_SUBTYPE_QOS ?
- qos_wh->i_qos[0] & DP_QOS_TID : 0;
- return;
- }
- is_mcast = DP_FRAME_IS_MULTICAST(hdr_ptr);
- ether_type = eh->ether_type;
- llcHdr = (qdf_llc_t *)(nbuf->data + sizeof(qdf_ether_header_t));
- /*
- * Check if packet is dot3 or eth2 type.
- */
- if (DP_FRAME_IS_LLC(ether_type) && DP_FRAME_IS_SNAP(llcHdr)) {
- ether_type = (uint16_t)*(nbuf->data + 2*QDF_MAC_ADDR_SIZE +
- sizeof(*llcHdr));
- if (ether_type == htons(ETHERTYPE_VLAN)) {
- L3datap = hdr_ptr + sizeof(qdf_ethervlan_header_t) +
- sizeof(*llcHdr);
- ether_type = (uint16_t)*(nbuf->data + 2*QDF_MAC_ADDR_SIZE
- + sizeof(*llcHdr) +
- sizeof(qdf_net_vlanhdr_t));
- } else {
- L3datap = hdr_ptr + sizeof(qdf_ether_header_t) +
- sizeof(*llcHdr);
- }
- } else {
- if (ether_type == htons(ETHERTYPE_VLAN)) {
- evh = (qdf_ethervlan_header_t *) eh;
- ether_type = evh->ether_type;
- L3datap = hdr_ptr + sizeof(qdf_ethervlan_header_t);
- }
- }
- /*
- * Find priority from IP TOS DSCP field
- */
- if (qdf_nbuf_is_ipv4_pkt(nbuf)) {
- qdf_net_iphdr_t *ip = (qdf_net_iphdr_t *) L3datap;
- if (qdf_nbuf_is_ipv4_dhcp_pkt(nbuf)) {
- /* Only for unicast frames */
- if (!is_mcast) {
- /* send it on VO queue */
- msdu_info->tid = DP_VO_TID;
- }
- } else {
- /*
- * IP frame: exclude ECN bits 0-1 and map DSCP bits 2-7
- * from TOS byte.
- */
- tos = ip->ip_tos;
- dscp_tid_override = 1;
- }
- } else if (qdf_nbuf_is_ipv6_pkt(nbuf)) {
- /* TODO
- * use flowlabel
- *igmpmld cases to be handled in phase 2
- */
- unsigned long ver_pri_flowlabel;
- unsigned long pri;
- ver_pri_flowlabel = *(unsigned long *) L3datap;
- pri = (ntohl(ver_pri_flowlabel) & IPV6_FLOWINFO_PRIORITY) >>
- DP_IPV6_PRIORITY_SHIFT;
- tos = pri;
- dscp_tid_override = 1;
- } else if (qdf_nbuf_is_ipv4_eapol_pkt(nbuf))
- msdu_info->tid = DP_VO_TID;
- else if (qdf_nbuf_is_ipv4_arp_pkt(nbuf)) {
- /* Only for unicast frames */
- if (!is_mcast) {
- /* send ucast arp on VO queue */
- msdu_info->tid = DP_VO_TID;
- }
- }
- /*
- * Assign all MCAST packets to BE
- */
- if (qdf_unlikely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
- if (is_mcast) {
- tos = 0;
- dscp_tid_override = 1;
- }
- }
- if (dscp_tid_override == 1) {
- tos = (tos >> DP_IP_DSCP_SHIFT) & DP_IP_DSCP_MASK;
- msdu_info->tid = pdev->dscp_tid_map[vdev->dscp_tid_map_id][tos];
- }
- if (msdu_info->tid >= CDP_MAX_DATA_TIDS)
- msdu_info->tid = CDP_MAX_DATA_TIDS - 1;
- return;
- }
- /**
- * dp_tx_classify_tid() - Obtain TID to be used for this frame
- * @vdev: DP vdev handle
- * @nbuf: skb
- * @msdu_info: msdu descriptor
- *
- * Software based TID classification is required when more than 2 DSCP-TID
- * mapping tables are needed.
- * Hardware supports 2 DSCP-TID mapping tables for HKv1 and 48 for HKv2.
- *
- * Return: void
- */
- static inline void dp_tx_classify_tid(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- DP_TX_TID_OVERRIDE(msdu_info, nbuf);
- /*
- * skip_sw_tid_classification flag will set in below cases-
- * 1. vdev->dscp_tid_map_id < pdev->soc->num_hw_dscp_tid_map
- * 2. hlos_tid_override enabled for vdev
- * 3. mesh mode enabled for vdev
- */
- if (qdf_likely(vdev->skip_sw_tid_classification)) {
- /* Update tid in msdu_info from skb priority */
- if (qdf_unlikely(vdev->skip_sw_tid_classification
- & DP_TXRX_HLOS_TID_OVERRIDE_ENABLED)) {
- uint32_t tid = qdf_nbuf_get_priority(nbuf);
- if (tid == DP_TX_INVALID_QOS_TAG)
- return;
- msdu_info->tid = tid;
- return;
- }
- return;
- }
- dp_tx_get_tid(vdev, nbuf, msdu_info);
- }
- #ifdef FEATURE_WLAN_TDLS
- /**
- * dp_tx_update_tdls_flags() - Update descriptor flags for TDLS frame
- * @soc: datapath SOC
- * @vdev: datapath vdev
- * @tx_desc: TX descriptor
- *
- * Return: None
- */
- static void dp_tx_update_tdls_flags(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- if (vdev) {
- if (vdev->is_tdls_frame) {
- tx_desc->flags |= DP_TX_DESC_FLAG_TDLS_FRAME;
- vdev->is_tdls_frame = false;
- }
- }
- }
- static uint8_t dp_htt_tx_comp_get_status(struct dp_soc *soc, char *htt_desc)
- {
- uint8_t tx_status = HTT_TX_FW2WBM_TX_STATUS_MAX;
- switch (soc->arch_id) {
- case CDP_ARCH_TYPE_LI:
- tx_status = HTT_TX_WBM_COMPLETION_V2_TX_STATUS_GET(htt_desc[0]);
- break;
- case CDP_ARCH_TYPE_BE:
- tx_status = HTT_TX_WBM_COMPLETION_V3_TX_STATUS_GET(htt_desc[0]);
- break;
- case CDP_ARCH_TYPE_RH:
- {
- uint32_t *msg_word = (uint32_t *)htt_desc;
- tx_status = HTT_TX_MSDU_INFO_RELEASE_REASON_GET(
- *(msg_word + 3));
- }
- break;
- default:
- dp_err("Incorrect CDP_ARCH %d", soc->arch_id);
- QDF_BUG(0);
- }
- return tx_status;
- }
- /**
- * dp_non_std_htt_tx_comp_free_buff() - Free the non std tx packet buffer
- * @soc: dp_soc handle
- * @tx_desc: TX descriptor
- *
- * Return: None
- */
- static void dp_non_std_htt_tx_comp_free_buff(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc)
- {
- uint8_t tx_status = 0;
- uint8_t htt_tx_status[HAL_TX_COMP_HTT_STATUS_LEN];
- qdf_nbuf_t nbuf = tx_desc->nbuf;
- struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id,
- DP_MOD_ID_TDLS);
- if (qdf_unlikely(!vdev)) {
- dp_err_rl("vdev is null!");
- goto error;
- }
- hal_tx_comp_get_htt_desc(&tx_desc->comp, htt_tx_status);
- tx_status = dp_htt_tx_comp_get_status(soc, htt_tx_status);
- dp_debug("vdev_id: %d tx_status: %d", tx_desc->vdev_id, tx_status);
- if (vdev->tx_non_std_data_callback.func) {
- qdf_nbuf_set_next(nbuf, NULL);
- vdev->tx_non_std_data_callback.func(
- vdev->tx_non_std_data_callback.ctxt,
- nbuf, tx_status);
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
- return;
- } else {
- dp_err_rl("callback func is null");
- }
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
- error:
- qdf_nbuf_unmap_single(soc->osdev, nbuf, QDF_DMA_TO_DEVICE);
- qdf_nbuf_free(nbuf);
- }
- /**
- * dp_tx_msdu_single_map() - do nbuf map
- * @vdev: DP vdev handle
- * @tx_desc: DP TX descriptor pointer
- * @nbuf: skb pointer
- *
- * For TDLS frame, use qdf_nbuf_map_single() to align with the unmap
- * operation done in other component.
- *
- * Return: QDF_STATUS
- */
- static inline QDF_STATUS dp_tx_msdu_single_map(struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf)
- {
- if (qdf_likely(!(tx_desc->flags & DP_TX_DESC_FLAG_TDLS_FRAME)))
- return qdf_nbuf_map_nbytes_single(vdev->osdev,
- nbuf,
- QDF_DMA_TO_DEVICE,
- nbuf->len);
- else
- return qdf_nbuf_map_single(vdev->osdev, nbuf,
- QDF_DMA_TO_DEVICE);
- }
- #else
- static inline void dp_tx_update_tdls_flags(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- }
- static inline void dp_non_std_htt_tx_comp_free_buff(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc)
- {
- }
- static inline QDF_STATUS dp_tx_msdu_single_map(struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf)
- {
- return qdf_nbuf_map_nbytes_single(vdev->osdev,
- nbuf,
- QDF_DMA_TO_DEVICE,
- nbuf->len);
- }
- #endif
- static inline
- qdf_dma_addr_t dp_tx_nbuf_map_regular(struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf)
- {
- QDF_STATUS ret = QDF_STATUS_E_FAILURE;
- ret = dp_tx_msdu_single_map(vdev, tx_desc, nbuf);
- if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret)))
- return 0;
- return qdf_nbuf_mapped_paddr_get(nbuf);
- }
- static inline
- void dp_tx_nbuf_unmap_regular(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- qdf_nbuf_unmap_nbytes_single_paddr(soc->osdev,
- desc->nbuf,
- desc->dma_addr,
- QDF_DMA_TO_DEVICE,
- desc->length);
- }
- #ifdef QCA_DP_TX_RMNET_OPTIMIZATION
- static inline bool
- is_nbuf_frm_rmnet(qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info)
- {
- struct net_device *ingress_dev;
- skb_frag_t *frag;
- uint16_t buf_len = 0;
- uint16_t linear_data_len = 0;
- uint8_t *payload_addr = NULL;
- ingress_dev = dev_get_by_index(dev_net(nbuf->dev), nbuf->skb_iif);
- if (!ingress_dev)
- return false;
- if ((ingress_dev->priv_flags & IFF_PHONY_HEADROOM)) {
- qdf_net_if_release_dev((struct qdf_net_if *)ingress_dev);
- frag = &(skb_shinfo(nbuf)->frags[0]);
- buf_len = skb_frag_size(frag);
- payload_addr = (uint8_t *)skb_frag_address(frag);
- linear_data_len = skb_headlen(nbuf);
- buf_len += linear_data_len;
- payload_addr = payload_addr - linear_data_len;
- memcpy(payload_addr, nbuf->data, linear_data_len);
- msdu_info->frm_type = dp_tx_frm_rmnet;
- msdu_info->buf_len = buf_len;
- msdu_info->payload_addr = payload_addr;
- return true;
- }
- qdf_net_if_release_dev((struct qdf_net_if *)ingress_dev);
- return false;
- }
- static inline
- qdf_dma_addr_t dp_tx_rmnet_nbuf_map(struct dp_tx_msdu_info_s *msdu_info,
- struct dp_tx_desc_s *tx_desc)
- {
- qdf_dma_addr_t paddr;
- paddr = (qdf_dma_addr_t)qdf_mem_virt_to_phys(msdu_info->payload_addr);
- tx_desc->length = msdu_info->buf_len;
- qdf_nbuf_dma_clean_range((void *)msdu_info->payload_addr,
- (void *)(msdu_info->payload_addr +
- msdu_info->buf_len));
- tx_desc->flags |= DP_TX_DESC_FLAG_RMNET;
- return paddr;
- }
- #else
- static inline bool
- is_nbuf_frm_rmnet(qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info)
- {
- return false;
- }
- static inline
- qdf_dma_addr_t dp_tx_rmnet_nbuf_map(struct dp_tx_msdu_info_s *msdu_info,
- struct dp_tx_desc_s *tx_desc)
- {
- return 0;
- }
- #endif
- #if defined(QCA_DP_TX_NBUF_NO_MAP_UNMAP) && !defined(BUILD_X86)
- static inline
- qdf_dma_addr_t dp_tx_nbuf_map(struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf)
- {
- if (qdf_likely(tx_desc->flags & DP_TX_DESC_FLAG_FAST)) {
- qdf_nbuf_dma_clean_range((void *)nbuf->data,
- (void *)(nbuf->data + nbuf->len));
- return (qdf_dma_addr_t)qdf_mem_virt_to_phys(nbuf->data);
- } else {
- return dp_tx_nbuf_map_regular(vdev, tx_desc, nbuf);
- }
- }
- static inline
- void dp_tx_nbuf_unmap(struct dp_soc *soc,
- struct dp_tx_desc_s *desc)
- {
- if (qdf_unlikely(!(desc->flags &
- (DP_TX_DESC_FLAG_SIMPLE | DP_TX_DESC_FLAG_RMNET))))
- return dp_tx_nbuf_unmap_regular(soc, desc);
- }
- #else
- static inline
- qdf_dma_addr_t dp_tx_nbuf_map(struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf)
- {
- return dp_tx_nbuf_map_regular(vdev, tx_desc, nbuf);
- }
- static inline
- void dp_tx_nbuf_unmap(struct dp_soc *soc,
- struct dp_tx_desc_s *desc)
- {
- return dp_tx_nbuf_unmap_regular(soc, desc);
- }
- #endif
- #if defined(WLAN_TX_PKT_CAPTURE_ENH) || defined(FEATURE_PERPKT_INFO)
- static inline
- void dp_tx_enh_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- if (qdf_likely(!(desc->flags & DP_TX_DESC_FLAG_UNMAP_DONE))) {
- dp_tx_nbuf_unmap(soc, desc);
- desc->flags |= DP_TX_DESC_FLAG_UNMAP_DONE;
- }
- }
- static inline void dp_tx_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- if (qdf_likely(!(desc->flags & DP_TX_DESC_FLAG_UNMAP_DONE)))
- dp_tx_nbuf_unmap(soc, desc);
- }
- #else
- static inline
- void dp_tx_enh_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- }
- static inline void dp_tx_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- dp_tx_nbuf_unmap(soc, desc);
- }
- #endif
- #ifdef MESH_MODE_SUPPORT
- /**
- * dp_tx_update_mesh_flags() - Update descriptor flags for mesh VAP
- * @soc: datapath SOC
- * @vdev: datapath vdev
- * @tx_desc: TX descriptor
- *
- * Return: None
- */
- static inline void dp_tx_update_mesh_flags(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- if (qdf_unlikely(vdev->mesh_vdev))
- tx_desc->flags |= DP_TX_DESC_FLAG_MESH_MODE;
- }
- /**
- * dp_mesh_tx_comp_free_buff() - Free the mesh tx packet buffer
- * @soc: dp_soc handle
- * @tx_desc: TX descriptor
- * @delayed_free: delay the nbuf free
- *
- * Return: nbuf to be freed late
- */
- static inline qdf_nbuf_t dp_mesh_tx_comp_free_buff(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc,
- bool delayed_free)
- {
- qdf_nbuf_t nbuf = tx_desc->nbuf;
- struct dp_vdev *vdev = NULL;
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id, DP_MOD_ID_MESH);
- if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW) {
- if (vdev)
- DP_STATS_INC(vdev,
- tx_i[xmit_type].mesh.completion_fw, 1);
- if (delayed_free)
- return nbuf;
- qdf_nbuf_free(nbuf);
- } else {
- if (vdev && vdev->osif_tx_free_ext) {
- vdev->osif_tx_free_ext((nbuf));
- } else {
- if (delayed_free)
- return nbuf;
- qdf_nbuf_free(nbuf);
- }
- }
- if (vdev)
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
- return NULL;
- }
- #else
- static inline void dp_tx_update_mesh_flags(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- }
- static inline qdf_nbuf_t dp_mesh_tx_comp_free_buff(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc,
- bool delayed_free)
- {
- return NULL;
- }
- #endif
- int dp_tx_frame_is_drop(struct dp_vdev *vdev, uint8_t *srcmac, uint8_t *dstmac)
- {
- struct dp_pdev *pdev = NULL;
- struct dp_ast_entry *src_ast_entry = NULL;
- struct dp_ast_entry *dst_ast_entry = NULL;
- struct dp_soc *soc = NULL;
- qdf_assert(vdev);
- pdev = vdev->pdev;
- qdf_assert(pdev);
- soc = pdev->soc;
- dst_ast_entry = dp_peer_ast_hash_find_by_pdevid
- (soc, dstmac, vdev->pdev->pdev_id);
- src_ast_entry = dp_peer_ast_hash_find_by_pdevid
- (soc, srcmac, vdev->pdev->pdev_id);
- if (dst_ast_entry && src_ast_entry) {
- if (dst_ast_entry->peer_id ==
- src_ast_entry->peer_id)
- return 1;
- }
- return 0;
- }
- #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
- defined(WLAN_MCAST_MLO)
- /* MLO peer id for reinject*/
- #define DP_MLO_MCAST_REINJECT_PEER_ID 0XFFFD
- /* MLO vdev id inc offset */
- #define DP_MLO_VDEV_ID_OFFSET 0x80
- #ifdef QCA_SUPPORT_WDS_EXTENDED
- static inline bool
- dp_tx_wds_ext_check(struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- if (tx_exc_metadata && tx_exc_metadata->is_wds_extended)
- return true;
- return false;
- }
- #else
- static inline bool
- dp_tx_wds_ext_check(struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- return false;
- }
- #endif
- static inline void
- dp_tx_bypass_reinjection(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- /* wds ext enabled will not set the TO_FW bit */
- if (dp_tx_wds_ext_check(tx_exc_metadata))
- return;
- if (!(tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)) {
- tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
- qdf_atomic_inc(&soc->num_tx_exception);
- }
- }
- static inline void
- dp_tx_update_mcast_param(uint16_t peer_id,
- uint16_t *htt_tcl_metadata,
- struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- if (peer_id == DP_MLO_MCAST_REINJECT_PEER_ID) {
- *htt_tcl_metadata = 0;
- DP_TX_TCL_METADATA_TYPE_SET(
- *htt_tcl_metadata,
- HTT_TCL_METADATA_V2_TYPE_GLOBAL_SEQ_BASED);
- HTT_TX_TCL_METADATA_GLBL_SEQ_NO_SET(*htt_tcl_metadata,
- msdu_info->gsn);
- msdu_info->vdev_id = vdev->vdev_id + DP_MLO_VDEV_ID_OFFSET;
- HTT_TX_TCL_METADATA_GLBL_SEQ_HOST_INSPECTED_SET(
- *htt_tcl_metadata, 1);
- } else {
- msdu_info->vdev_id = vdev->vdev_id;
- }
- }
- #else
- static inline void
- dp_tx_bypass_reinjection(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- }
- static inline void
- dp_tx_update_mcast_param(uint16_t peer_id,
- uint16_t *htt_tcl_metadata,
- struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- }
- #endif
- #ifdef DP_TX_SW_DROP_STATS_INC
- static void tx_sw_drop_stats_inc(struct dp_pdev *pdev,
- qdf_nbuf_t nbuf,
- enum cdp_tx_sw_drop drop_code)
- {
- /* EAPOL Drop stats */
- if (qdf_nbuf_is_ipv4_eapol_pkt(nbuf)) {
- switch (drop_code) {
- case TX_DESC_ERR:
- DP_STATS_INC(pdev, eap_drop_stats.tx_desc_err, 1);
- break;
- case TX_HAL_RING_ACCESS_ERR:
- DP_STATS_INC(pdev,
- eap_drop_stats.tx_hal_ring_access_err, 1);
- break;
- case TX_DMA_MAP_ERR:
- DP_STATS_INC(pdev, eap_drop_stats.tx_dma_map_err, 1);
- break;
- case TX_HW_ENQUEUE:
- DP_STATS_INC(pdev, eap_drop_stats.tx_hw_enqueue, 1);
- break;
- case TX_SW_ENQUEUE:
- DP_STATS_INC(pdev, eap_drop_stats.tx_sw_enqueue, 1);
- break;
- default:
- dp_info_rl("Invalid eapol_drop code: %d", drop_code);
- break;
- }
- }
- }
- #else
- static void tx_sw_drop_stats_inc(struct dp_pdev *pdev,
- qdf_nbuf_t nbuf,
- enum cdp_tx_sw_drop drop_code)
- {
- }
- #endif
- #ifdef WLAN_FEATURE_TX_LATENCY_STATS
- /**
- * dp_tx_latency_stats_enabled() - check enablement of transmit latency
- * statistics
- * @vdev: DP vdev handle
- *
- * Return: true if transmit latency statistics is enabled, false otherwise.
- */
- static inline bool dp_tx_latency_stats_enabled(struct dp_vdev *vdev)
- {
- return qdf_atomic_read(&vdev->tx_latency_cfg.enabled);
- }
- /**
- * dp_tx_latency_stats_report_enabled() - check enablement of async report
- * for transmit latency statistics
- * @vdev: DP vdev handle
- *
- * Return: true if transmit latency statistics is enabled, false otherwise.
- */
- static inline bool dp_tx_latency_stats_report_enabled(struct dp_vdev *vdev)
- {
- return qdf_atomic_read(&vdev->tx_latency_cfg.report);
- }
- /**
- * dp_tx_get_driver_ingress_ts() - get driver ingress timestamp from nbuf
- * @vdev: DP vdev handle
- * @msdu_info: pointer to MSDU Descriptor
- * @nbuf: original buffer from network stack
- *
- * Return: None
- */
- static inline void
- dp_tx_get_driver_ingress_ts(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- qdf_nbuf_t nbuf)
- {
- if (!dp_tx_latency_stats_enabled(vdev))
- return;
- msdu_info->driver_ingress_ts = qdf_nbuf_get_tx_ts(nbuf, true);
- }
- /**
- * dp_tx_update_ts_on_enqueued() - set driver ingress/egress timestamp in
- * tx descriptor
- * @vdev: DP vdev handle
- * @msdu_info: pointer to MSDU Descriptor
- * @tx_desc: pointer to tx descriptor
- *
- * Return: None
- */
- static inline void
- dp_tx_update_ts_on_enqueued(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- struct dp_tx_desc_s *tx_desc)
- {
- if (!dp_tx_latency_stats_enabled(vdev))
- return;
- tx_desc->driver_ingress_ts = msdu_info->driver_ingress_ts;
- tx_desc->driver_egress_ts = qdf_ktime_real_get();
- }
- /**
- * dp_tx_latency_stats_update_bucket() - update transmit latency statistics
- * for specified type
- * @vdev: DP vdev handle
- * @tx_latency: pointer to transmit latency stats
- * @idx: index of the statistics
- * @type: transmit latency type
- * @value: latency to be recorded
- *
- * Return: None
- */
- static inline void
- dp_tx_latency_stats_update_bucket(struct dp_vdev *vdev,
- struct dp_tx_latency *tx_latency,
- int idx, enum cdp_tx_latency_type type,
- uint32_t value)
- {
- int32_t granularity;
- int lvl;
- granularity =
- qdf_atomic_read(&vdev->tx_latency_cfg.granularity[type]);
- if (qdf_unlikely(!granularity))
- return;
- lvl = value / granularity;
- if (lvl >= CDP_TX_LATENCY_DISTR_LV_MAX)
- lvl = CDP_TX_LATENCY_DISTR_LV_MAX - 1;
- qdf_atomic_inc(&tx_latency->stats[idx][type].msdus_accum);
- qdf_atomic_add(value, &tx_latency->stats[idx][type].latency_accum);
- qdf_atomic_inc(&tx_latency->stats[idx][type].distribution[lvl]);
- }
- /**
- * dp_tx_latency_stats_update() - update transmit latency statistics on
- * msdu transmit completed
- * @soc: dp soc handle
- * @txrx_peer: txrx peer handle
- * @tx_desc: pointer to tx descriptor
- * @ts: tx completion status
- * @link_id: link id
- *
- * Return: None
- */
- static inline void
- dp_tx_latency_stats_update(struct dp_soc *soc,
- struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t link_id)
- {
- uint32_t driver_latency, ring_buf_latency, hw_latency;
- QDF_STATUS status = QDF_STATUS_E_INVAL;
- int64_t current_ts, ingress, egress;
- struct dp_vdev *vdev = txrx_peer->vdev;
- struct dp_tx_latency *tx_latency;
- uint8_t idx;
- if (!dp_tx_latency_stats_enabled(vdev))
- return;
- if (!tx_desc->driver_ingress_ts || !tx_desc->driver_egress_ts)
- return;
- status = dp_tx_compute_hw_delay_us(ts, vdev->delta_tsf, &hw_latency);
- if (QDF_IS_STATUS_ERROR(status))
- return;
- ingress = qdf_ktime_to_us(tx_desc->driver_ingress_ts);
- egress = qdf_ktime_to_us(tx_desc->driver_egress_ts);
- driver_latency = (uint32_t)(egress - ingress);
- current_ts = qdf_ktime_to_us(qdf_ktime_real_get());
- ring_buf_latency = (uint32_t)(current_ts - egress);
- tx_latency = &txrx_peer->stats[link_id].tx_latency;
- idx = tx_latency->cur_idx;
- dp_tx_latency_stats_update_bucket(txrx_peer->vdev, tx_latency, idx,
- CDP_TX_LATENCY_TYPE_DRIVER,
- driver_latency);
- dp_tx_latency_stats_update_bucket(txrx_peer->vdev, tx_latency, idx,
- CDP_TX_LATENCY_TYPE_RING_BUF,
- ring_buf_latency);
- dp_tx_latency_stats_update_bucket(txrx_peer->vdev, tx_latency, idx,
- CDP_TX_LATENCY_TYPE_HW, hw_latency);
- }
- /**
- * dp_tx_latency_stats_clear_bucket() - clear specified transmit latency
- * statistics for specified type
- * @tx_latency: pointer to transmit latency stats
- * @idx: index of the statistics
- * @type: transmit latency type
- *
- * Return: None
- */
- static inline void
- dp_tx_latency_stats_clear_bucket(struct dp_tx_latency *tx_latency,
- int idx, enum cdp_tx_latency_type type)
- {
- int lvl;
- struct dp_tx_latency_stats *stats;
- stats = &tx_latency->stats[idx][type];
- qdf_atomic_init(&stats->msdus_accum);
- qdf_atomic_init(&stats->latency_accum);
- for (lvl = 0; lvl < CDP_TX_LATENCY_DISTR_LV_MAX; lvl++)
- qdf_atomic_init(&stats->distribution[lvl]);
- }
- /**
- * dp_tx_latency_stats_clear_buckets() - clear specified transmit latency
- * statistics
- * @tx_latency: pointer to transmit latency stats
- * @idx: index of the statistics
- *
- * Return: None
- */
- static void
- dp_tx_latency_stats_clear_buckets(struct dp_tx_latency *tx_latency,
- int idx)
- {
- int type;
- for (type = 0; type < CDP_TX_LATENCY_TYPE_MAX; type++)
- dp_tx_latency_stats_clear_bucket(tx_latency, idx, type);
- }
- /**
- * dp_tx_latency_stats_update_cca() - update transmit latency statistics for
- * CCA
- * @soc: dp soc handle
- * @peer_id: peer id
- * @granularity: granularity of distribution
- * @distribution: distribution of transmit latency statistics
- * @avg: average of CCA latency(in microseconds) within a cycle
- *
- * Return: None
- */
- void
- dp_tx_latency_stats_update_cca(struct dp_soc *soc, uint16_t peer_id,
- uint32_t granularity, uint32_t *distribution,
- uint32_t avg)
- {
- int lvl, idx;
- uint8_t link_id;
- struct dp_tx_latency *tx_latency;
- struct dp_tx_latency_stats *stats;
- int32_t cur_granularity;
- struct dp_vdev *vdev;
- struct dp_tx_latency_config *cfg;
- struct dp_txrx_peer *txrx_peer;
- struct dp_peer *peer;
- peer = dp_peer_get_ref_by_id(soc, peer_id, DP_MOD_ID_HTT);
- if (!peer) {
- dp_err_rl("Peer not found peer id %d", peer_id);
- return;
- }
- if (IS_MLO_DP_MLD_PEER(peer))
- goto out;
- vdev = peer->vdev;
- if (!dp_tx_latency_stats_enabled(vdev))
- goto out;
- cfg = &vdev->tx_latency_cfg;
- cur_granularity =
- qdf_atomic_read(&cfg->granularity[CDP_TX_LATENCY_TYPE_CCA]);
- /* in unit of ms */
- cur_granularity /= 1000;
- if (cur_granularity != granularity) {
- dp_info_rl("invalid granularity, cur %d report %d",
- cur_granularity, granularity);
- goto out;
- }
- txrx_peer = dp_get_txrx_peer(peer);
- if (qdf_unlikely(!txrx_peer)) {
- dp_err_rl("txrx_peer NULL for MAC: " QDF_MAC_ADDR_FMT,
- QDF_MAC_ADDR_REF(peer->mac_addr.raw));
- goto out;
- }
- link_id = dp_get_peer_link_id(peer);
- if (link_id >= txrx_peer->stats_arr_size)
- goto out;
- tx_latency = &txrx_peer->stats[link_id].tx_latency;
- idx = tx_latency->cur_idx;
- stats = &tx_latency->stats[idx][CDP_TX_LATENCY_TYPE_CCA];
- qdf_atomic_set(&stats->latency_accum, avg);
- qdf_atomic_set(&stats->msdus_accum, (avg ? 1 : 0));
- for (lvl = 0; lvl < CDP_TX_LATENCY_DISTR_LV_MAX; lvl++)
- qdf_atomic_set(&stats->distribution[lvl],
- distribution[lvl]);
- /* prepare for the next cycle */
- tx_latency->cur_idx = 1 - idx;
- dp_tx_latency_stats_clear_buckets(tx_latency, tx_latency->cur_idx);
- out:
- dp_peer_unref_delete(peer, DP_MOD_ID_HTT);
- }
- /**
- * dp_tx_latency_stats_get_per_peer() - get transmit latency statistics for a
- * peer
- * @soc: dp soc handle
- * @peer: dp peer Handle
- * @latency: buffer to hold transmit latency statistics
- *
- * Return: QDF_STATUS
- */
- static QDF_STATUS
- dp_tx_latency_stats_get_per_peer(struct dp_soc *soc, struct dp_peer *peer,
- struct cdp_tx_latency *latency)
- {
- int lvl, type, link_id;
- int32_t latency_accum, msdus_accum;
- struct dp_vdev *vdev;
- struct dp_txrx_peer *txrx_peer;
- struct dp_tx_latency *tx_latency;
- struct dp_tx_latency_config *cfg;
- struct dp_tx_latency_stats *stats;
- uint8_t last_idx;
- if (unlikely(!latency))
- return QDF_STATUS_E_INVAL;
- /* Authenticated link/legacy peer only */
- if (IS_MLO_DP_MLD_PEER(peer) || peer->state != OL_TXRX_PEER_STATE_AUTH)
- return QDF_STATUS_E_INVAL;
- vdev = peer->vdev;
- if (peer->bss_peer && vdev->opmode == wlan_op_mode_ap)
- return QDF_STATUS_E_INVAL;
- txrx_peer = dp_get_txrx_peer(peer);
- if (!txrx_peer)
- return QDF_STATUS_E_INVAL;
- link_id = dp_get_peer_link_id(peer);
- if (link_id >= txrx_peer->stats_arr_size)
- return QDF_STATUS_E_INVAL;
- tx_latency = &txrx_peer->stats[link_id].tx_latency;
- qdf_mem_zero(latency, sizeof(*latency));
- qdf_mem_copy(latency->mac_remote.bytes,
- peer->mac_addr.raw, QDF_MAC_ADDR_SIZE);
- last_idx = 1 - tx_latency->cur_idx;
- cfg = &vdev->tx_latency_cfg;
- for (type = 0; type < CDP_TX_LATENCY_TYPE_MAX; type++) {
- latency->stats[type].granularity =
- qdf_atomic_read(&cfg->granularity[type]);
- stats = &tx_latency->stats[last_idx][type];
- msdus_accum = qdf_atomic_read(&stats->msdus_accum);
- if (!msdus_accum)
- continue;
- latency_accum = qdf_atomic_read(&stats->latency_accum);
- latency->stats[type].average = latency_accum / msdus_accum;
- for (lvl = 0; lvl < CDP_TX_LATENCY_DISTR_LV_MAX; lvl++) {
- latency->stats[type].distribution[lvl] =
- qdf_atomic_read(&stats->distribution[lvl]);
- }
- }
- return QDF_STATUS_SUCCESS;
- }
- /**
- * dp_tx_latency_stats_get_peer_iter() - iterator to get transmit latency
- * statistics for specified peer
- * @soc: dp soc handle
- * @peer: dp peer Handle
- * @arg: list to hold transmit latency statistics for peers
- *
- * Return: None
- */
- static void
- dp_tx_latency_stats_get_peer_iter(struct dp_soc *soc,
- struct dp_peer *peer,
- void *arg)
- {
- struct dp_vdev *vdev;
- struct dp_txrx_peer *txrx_peer;
- struct cdp_tx_latency *latency;
- QDF_STATUS status;
- qdf_list_t *stats_list = (qdf_list_t *)arg;
- /* Authenticated link/legacy peer only */
- if (IS_MLO_DP_MLD_PEER(peer) || peer->state != OL_TXRX_PEER_STATE_AUTH)
- return;
- txrx_peer = dp_get_txrx_peer(peer);
- if (!txrx_peer)
- return;
- vdev = peer->vdev;
- latency = qdf_mem_malloc(sizeof(*latency));
- if (!latency)
- return;
- status = dp_tx_latency_stats_get_per_peer(soc, peer, latency);
- if (QDF_IS_STATUS_ERROR(status))
- goto out;
- status = qdf_list_insert_back(stats_list, &latency->node);
- if (QDF_IS_STATUS_ERROR(status))
- goto out;
- return;
- out:
- qdf_mem_free(latency);
- }
- /**
- * dp_tx_latency_stats_rpt_per_vdev() - report transmit latency statistics for
- * specified vdev
- * @soc: dp soc handle
- * @vdev: dp vdev Handle
- *
- * Return: None
- */
- static void
- dp_tx_latency_stats_rpt_per_vdev(struct dp_soc *soc, struct dp_vdev *vdev)
- {
- qdf_list_t stats_list;
- struct cdp_tx_latency *entry, *next;
- if (!soc->tx_latency_cb || !dp_tx_latency_stats_report_enabled(vdev))
- return;
- qdf_list_create(&stats_list, 0);
- dp_vdev_iterate_peer(vdev, dp_tx_latency_stats_get_peer_iter,
- &stats_list, DP_MOD_ID_CDP);
- if (qdf_list_empty(&stats_list))
- goto out;
- soc->tx_latency_cb(vdev->vdev_id, &stats_list);
- qdf_list_for_each_del(&stats_list, entry, next, node) {
- qdf_list_remove_node(&stats_list, &entry->node);
- qdf_mem_free(entry);
- }
- out:
- qdf_list_destroy(&stats_list);
- }
- /**
- * dp_tx_latency_stats_report() - report transmit latency statistics for each
- * vdev of specified pdev
- * @soc: dp soc handle
- * @pdev: dp pdev Handle
- *
- * Return: None
- */
- void dp_tx_latency_stats_report(struct dp_soc *soc, struct dp_pdev *pdev)
- {
- struct dp_vdev *vdev;
- if (!soc->tx_latency_cb)
- return;
- qdf_spin_lock_bh(&pdev->vdev_list_lock);
- DP_PDEV_ITERATE_VDEV_LIST(pdev, vdev) {
- dp_tx_latency_stats_rpt_per_vdev(soc, vdev);
- }
- qdf_spin_unlock_bh(&pdev->vdev_list_lock);
- }
- /**
- * dp_tx_latency_stats_clear_per_peer() - iterator to clear transmit latency
- * statistics for specified peer
- * @soc: dp soc handle
- * @peer: dp pdev Handle
- * @arg: argument from iterator
- *
- * Return: None
- */
- static void
- dp_tx_latency_stats_clear_per_peer(struct dp_soc *soc, struct dp_peer *peer,
- void *arg)
- {
- int link_id;
- struct dp_tx_latency *tx_latency;
- struct dp_txrx_peer *txrx_peer = dp_get_txrx_peer(peer);
- if (!txrx_peer) {
- dp_err("no txrx peer, skip");
- return;
- }
- for (link_id = 0; link_id < txrx_peer->stats_arr_size; link_id++) {
- tx_latency = &txrx_peer->stats[link_id].tx_latency;
- dp_tx_latency_stats_clear_buckets(tx_latency, 0);
- dp_tx_latency_stats_clear_buckets(tx_latency, 1);
- }
- }
- /**
- * dp_tx_latency_stats_clear_per_vdev() - clear transmit latency statistics
- * for specified vdev
- * @vdev: dp vdev handle
- *
- * Return: None
- */
- static inline void dp_tx_latency_stats_clear_per_vdev(struct dp_vdev *vdev)
- {
- dp_vdev_iterate_peer(vdev, dp_tx_latency_stats_clear_per_peer,
- NULL, DP_MOD_ID_CDP);
- }
- /**
- * dp_tx_latency_stats_fetch() - fetch transmit latency statistics for
- * specified link mac address
- * @soc_hdl: Handle to struct dp_soc
- * @vdev_id: vdev id
- * @mac: link mac address of remote peer
- * @latency: buffer to hold per-link transmit latency statistics
- *
- * Return: QDF_STATUS
- */
- QDF_STATUS
- dp_tx_latency_stats_fetch(struct cdp_soc_t *soc_hdl,
- uint8_t vdev_id, uint8_t *mac,
- struct cdp_tx_latency *latency)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct cdp_peer_info peer_info = {0};
- struct dp_peer *peer;
- QDF_STATUS status;
- /* MAC addr of link peer may be the same as MLD peer,
- * so specify the type as CDP_LINK_PEER_TYPE here to
- * get link peer explicitly.
- */
- DP_PEER_INFO_PARAMS_INIT(&peer_info, vdev_id, mac, false,
- CDP_LINK_PEER_TYPE);
- peer = dp_peer_hash_find_wrapper(soc, &peer_info, DP_MOD_ID_CDP);
- if (!peer) {
- dp_err_rl("peer(vdev id %d mac " QDF_MAC_ADDR_FMT ") not found",
- vdev_id, QDF_MAC_ADDR_REF(mac));
- return QDF_STATUS_E_INVAL;
- }
- status = dp_tx_latency_stats_get_per_peer(soc, peer, latency);
- dp_peer_unref_delete(peer, DP_MOD_ID_CDP);
- return status;
- }
- /**
- * dp_tx_latency_stats_config() - config transmit latency statistics for
- * specified vdev
- * @soc_hdl: Handle to struct dp_soc
- * @vdev_id: vdev id
- * @cfg: configuration for transmit latency statistics
- *
- * Return: QDF_STATUS
- */
- QDF_STATUS
- dp_tx_latency_stats_config(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- struct cdp_tx_latency_config *cfg)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev;
- QDF_STATUS status = QDF_STATUS_E_INVAL;
- uint32_t cca_granularity;
- int type;
- vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_CDP);
- if (!vdev) {
- dp_err_rl("vdev %d does not exist", vdev_id);
- return QDF_STATUS_E_FAILURE;
- }
- /* disable to ignore upcoming updates */
- qdf_atomic_set(&vdev->tx_latency_cfg.enabled, 0);
- dp_tx_latency_stats_clear_per_vdev(vdev);
- if (!cfg->enable)
- goto send_htt;
- qdf_atomic_set(&vdev->tx_latency_cfg.report, (cfg->report ? 1 : 0));
- for (type = 0; type < CDP_TX_LATENCY_TYPE_MAX; type++)
- qdf_atomic_set(&vdev->tx_latency_cfg.granularity[type],
- cfg->granularity[type]);
- send_htt:
- /* in units of ms */
- cca_granularity = cfg->granularity[CDP_TX_LATENCY_TYPE_CCA] / 1000;
- status = dp_h2t_tx_latency_stats_cfg_msg_send(soc, vdev_id,
- cfg->enable, cfg->period,
- cca_granularity);
- if (QDF_IS_STATUS_ERROR(status)) {
- dp_err_rl("failed to send htt msg: %d", status);
- goto out;
- }
- qdf_atomic_set(&vdev->tx_latency_cfg.enabled, (cfg->enable ? 1 : 0));
- out:
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
- return status;
- }
- /**
- * dp_tx_latency_stats_register_cb() - register transmit latency statistics
- * callback
- * @handle: Handle to struct dp_soc
- * @cb: callback function for transmit latency statistics
- *
- * Return: QDF_STATUS
- */
- QDF_STATUS
- dp_tx_latency_stats_register_cb(struct cdp_soc_t *handle, cdp_tx_latency_cb cb)
- {
- struct dp_soc *soc = (struct dp_soc *)handle;
- if (!soc || !cb) {
- dp_err("soc or cb is NULL");
- return QDF_STATUS_E_INVAL;
- }
- soc->tx_latency_cb = cb;
- return QDF_STATUS_SUCCESS;
- }
- #else
- static inline void
- dp_tx_get_driver_ingress_ts(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- qdf_nbuf_t nbuf)
- {
- }
- static inline void
- dp_tx_update_ts_on_enqueued(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- struct dp_tx_desc_s *tx_desc)
- {
- }
- static inline void
- dp_tx_latency_stats_update(struct dp_soc *soc,
- struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t link_id)
- {
- }
- #endif
- qdf_nbuf_t
- dp_tx_send_msdu_single(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info, uint16_t peer_id,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- struct dp_pdev *pdev = vdev->pdev;
- struct dp_soc *soc = pdev->soc;
- struct dp_tx_desc_s *tx_desc;
- QDF_STATUS status;
- struct dp_tx_queue *tx_q = &(msdu_info->tx_queue);
- uint16_t htt_tcl_metadata = 0;
- enum cdp_tx_sw_drop drop_code = TX_MAX_DROP;
- uint8_t tid = msdu_info->tid;
- struct cdp_tid_tx_stats *tid_stats = NULL;
- qdf_dma_addr_t paddr;
- /* Setup Tx descriptor for an MSDU, and MSDU extension descriptor */
- tx_desc = dp_tx_prepare_desc_single(vdev, nbuf, tx_q->desc_pool_id,
- msdu_info, tx_exc_metadata);
- if (!tx_desc) {
- dp_err_rl("Tx_desc prepare Fail vdev_id %d vdev %pK queue %d",
- vdev->vdev_id, vdev, tx_q->desc_pool_id);
- drop_code = TX_DESC_ERR;
- goto fail_return;
- }
- dp_tx_update_tdls_flags(soc, vdev, tx_desc);
- if (qdf_unlikely(peer_id == DP_INVALID_PEER)) {
- htt_tcl_metadata = vdev->htt_tcl_metadata;
- DP_TX_TCL_METADATA_HOST_INSPECTED_SET(htt_tcl_metadata, 1);
- } else if (qdf_unlikely(peer_id != HTT_INVALID_PEER)) {
- DP_TX_TCL_METADATA_TYPE_SET(htt_tcl_metadata,
- DP_TCL_METADATA_TYPE_PEER_BASED);
- DP_TX_TCL_METADATA_PEER_ID_SET(htt_tcl_metadata,
- peer_id);
- dp_tx_bypass_reinjection(soc, tx_desc, tx_exc_metadata);
- } else
- htt_tcl_metadata = vdev->htt_tcl_metadata;
- if (msdu_info->exception_fw)
- DP_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1);
- dp_tx_desc_update_fast_comp_flag(soc, tx_desc,
- !pdev->enhanced_stats_en);
- dp_tx_update_mesh_flags(soc, vdev, tx_desc);
- if (qdf_unlikely(msdu_info->frm_type == dp_tx_frm_rmnet))
- paddr = dp_tx_rmnet_nbuf_map(msdu_info, tx_desc);
- else
- paddr = dp_tx_nbuf_map(vdev, tx_desc, nbuf);
- if (!paddr) {
- /* Handle failure */
- dp_err("qdf_nbuf_map failed");
- DP_STATS_INC(vdev,
- tx_i[msdu_info->xmit_type].dropped.dma_error, 1);
- drop_code = TX_DMA_MAP_ERR;
- goto release_desc;
- }
- tx_desc->dma_addr = paddr;
- dp_tx_desc_history_add(soc, tx_desc->dma_addr, nbuf,
- tx_desc->id, DP_TX_DESC_MAP);
- dp_tx_update_mcast_param(peer_id, &htt_tcl_metadata, vdev, msdu_info);
- /* Enqueue the Tx MSDU descriptor to HW for transmit */
- status = soc->arch_ops.tx_hw_enqueue(soc, vdev, tx_desc,
- htt_tcl_metadata,
- tx_exc_metadata, msdu_info);
- if (status != QDF_STATUS_SUCCESS) {
- dp_tx_err_rl("Tx_hw_enqueue Fail tx_desc %pK queue %d",
- tx_desc, tx_q->ring_id);
- dp_tx_desc_history_add(soc, tx_desc->dma_addr, nbuf,
- tx_desc->id, DP_TX_DESC_UNMAP);
- dp_tx_nbuf_unmap(soc, tx_desc);
- drop_code = TX_HW_ENQUEUE;
- goto release_desc;
- }
- dp_tx_update_ts_on_enqueued(vdev, msdu_info, tx_desc);
- tx_sw_drop_stats_inc(pdev, nbuf, drop_code);
- return NULL;
- release_desc:
- dp_tx_desc_release(soc, tx_desc, tx_q->desc_pool_id);
- fail_return:
- dp_tx_get_tid(vdev, nbuf, msdu_info);
- tx_sw_drop_stats_inc(pdev, nbuf, drop_code);
- tid_stats = &pdev->stats.tid_stats.
- tid_tx_stats[tx_q->ring_id][tid];
- tid_stats->swdrop_cnt[drop_code]++;
- return nbuf;
- }
- /**
- * dp_tdls_tx_comp_free_buff() - Free non std buffer when TDLS flag is set
- * @soc: Soc handle
- * @desc: software Tx descriptor to be processed
- *
- * Return: 0 if Success
- */
- #ifdef FEATURE_WLAN_TDLS
- static inline int
- dp_tdls_tx_comp_free_buff(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- /* If it is TDLS mgmt, don't unmap or free the frame */
- if (desc->flags & DP_TX_DESC_FLAG_TDLS_FRAME) {
- dp_non_std_htt_tx_comp_free_buff(soc, desc);
- return 0;
- }
- return 1;
- }
- #else
- static inline int
- dp_tdls_tx_comp_free_buff(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- return 1;
- }
- #endif
- qdf_nbuf_t dp_tx_comp_free_buf(struct dp_soc *soc, struct dp_tx_desc_s *desc,
- bool delayed_free)
- {
- qdf_nbuf_t nbuf = desc->nbuf;
- enum dp_tx_event_type type = dp_tx_get_event_type(desc->flags);
- /* nbuf already freed in vdev detach path */
- if (!nbuf)
- return NULL;
- if (!dp_tdls_tx_comp_free_buff(soc, desc))
- return NULL;
- /* 0 : MSDU buffer, 1 : MLE */
- if (desc->msdu_ext_desc) {
- /* TSO free */
- if (hal_tx_ext_desc_get_tso_enable(
- desc->msdu_ext_desc->vaddr)) {
- dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf,
- desc->id, DP_TX_COMP_MSDU_EXT);
- dp_tx_tso_seg_history_add(soc,
- desc->msdu_ext_desc->tso_desc,
- desc->nbuf, desc->id, type);
- /* unmap eash TSO seg before free the nbuf */
- dp_tx_tso_unmap_segment(soc,
- desc->msdu_ext_desc->tso_desc,
- desc->msdu_ext_desc->
- tso_num_desc);
- goto nbuf_free;
- }
- if (qdf_unlikely(desc->frm_type == dp_tx_frm_sg)) {
- void *msdu_ext_desc = desc->msdu_ext_desc->vaddr;
- qdf_dma_addr_t iova;
- uint32_t frag_len;
- uint32_t i;
- qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf,
- QDF_DMA_TO_DEVICE,
- qdf_nbuf_headlen(nbuf));
- for (i = 1; i < DP_TX_MAX_NUM_FRAGS; i++) {
- hal_tx_ext_desc_get_frag_info(msdu_ext_desc, i,
- &iova,
- &frag_len);
- if (!iova || !frag_len)
- break;
- qdf_mem_unmap_page(soc->osdev, iova, frag_len,
- QDF_DMA_TO_DEVICE);
- }
- goto nbuf_free;
- }
- }
- /* If it's ME frame, dont unmap the cloned nbuf's */
- if ((desc->flags & DP_TX_DESC_FLAG_ME) && qdf_nbuf_is_cloned(nbuf))
- goto nbuf_free;
- dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf, desc->id, type);
- dp_tx_unmap(soc, desc);
- if (desc->flags & DP_TX_DESC_FLAG_MESH_MODE)
- return dp_mesh_tx_comp_free_buff(soc, desc, delayed_free);
- if (dp_tx_traffic_end_indication_enq_ind_pkt(soc, desc, nbuf))
- return NULL;
- nbuf_free:
- if (delayed_free)
- return nbuf;
- qdf_nbuf_free(nbuf);
- return NULL;
- }
- /**
- * dp_tx_sg_unmap_buf() - Unmap scatter gather fragments
- * @soc: DP soc handle
- * @nbuf: skb
- * @msdu_info: MSDU info
- *
- * Return: None
- */
- static inline void
- dp_tx_sg_unmap_buf(struct dp_soc *soc, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- uint32_t cur_idx;
- struct dp_tx_seg_info_s *seg = msdu_info->u.sg_info.curr_seg;
- qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf, QDF_DMA_TO_DEVICE,
- qdf_nbuf_headlen(nbuf));
- for (cur_idx = 1; cur_idx < seg->frag_cnt; cur_idx++)
- qdf_mem_unmap_page(soc->osdev, (qdf_dma_addr_t)
- (seg->frags[cur_idx].paddr_lo | ((uint64_t)
- seg->frags[cur_idx].paddr_hi) << 32),
- seg->frags[cur_idx].len,
- QDF_DMA_TO_DEVICE);
- }
- #if QDF_LOCK_STATS
- noinline
- #else
- #endif
- qdf_nbuf_t dp_tx_send_msdu_multiple(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- uint32_t i;
- struct dp_pdev *pdev = vdev->pdev;
- struct dp_soc *soc = pdev->soc;
- struct dp_tx_desc_s *tx_desc;
- bool is_cce_classified = false;
- QDF_STATUS status;
- uint16_t htt_tcl_metadata = 0;
- struct dp_tx_queue *tx_q = &msdu_info->tx_queue;
- struct cdp_tid_tx_stats *tid_stats = NULL;
- uint8_t prep_desc_fail = 0, hw_enq_fail = 0;
- if (msdu_info->frm_type == dp_tx_frm_me)
- nbuf = msdu_info->u.sg_info.curr_seg->nbuf;
- i = 0;
- /* Print statement to track i and num_seg */
- /*
- * For each segment (maps to 1 MSDU) , prepare software and hardware
- * descriptors using information in msdu_info
- */
- while (i < msdu_info->num_seg) {
- /*
- * Setup Tx descriptor for an MSDU, and MSDU extension
- * descriptor
- */
- tx_desc = dp_tx_prepare_desc(vdev, nbuf, msdu_info,
- tx_q->desc_pool_id);
- if (!tx_desc) {
- if (msdu_info->frm_type == dp_tx_frm_me) {
- prep_desc_fail++;
- dp_tx_me_free_buf(pdev,
- (void *)(msdu_info->u.sg_info
- .curr_seg->frags[0].vaddr));
- if (prep_desc_fail == msdu_info->num_seg) {
- /*
- * Unmap is needed only if descriptor
- * preparation failed for all segments.
- */
- qdf_nbuf_unmap(soc->osdev,
- msdu_info->u.sg_info.
- curr_seg->nbuf,
- QDF_DMA_TO_DEVICE);
- }
- /*
- * Free the nbuf for the current segment
- * and make it point to the next in the list.
- * For me, there are as many segments as there
- * are no of clients.
- */
- qdf_nbuf_free(msdu_info->u.sg_info
- .curr_seg->nbuf);
- if (msdu_info->u.sg_info.curr_seg->next) {
- msdu_info->u.sg_info.curr_seg =
- msdu_info->u.sg_info
- .curr_seg->next;
- nbuf = msdu_info->u.sg_info
- .curr_seg->nbuf;
- }
- i++;
- continue;
- }
- if (msdu_info->frm_type == dp_tx_frm_tso) {
- dp_tx_tso_seg_history_add(
- soc,
- msdu_info->u.tso_info.curr_seg,
- nbuf, 0, DP_TX_DESC_UNMAP);
- dp_tx_tso_unmap_segment(soc,
- msdu_info->u.tso_info.
- curr_seg,
- msdu_info->u.tso_info.
- tso_num_seg_list);
- if (msdu_info->u.tso_info.curr_seg->next) {
- msdu_info->u.tso_info.curr_seg =
- msdu_info->u.tso_info.curr_seg->next;
- i++;
- continue;
- }
- }
- if (msdu_info->frm_type == dp_tx_frm_sg)
- dp_tx_sg_unmap_buf(soc, nbuf, msdu_info);
- goto done;
- }
- if (msdu_info->frm_type == dp_tx_frm_me) {
- tx_desc->msdu_ext_desc->me_buffer =
- (struct dp_tx_me_buf_t *)msdu_info->
- u.sg_info.curr_seg->frags[0].vaddr;
- tx_desc->flags |= DP_TX_DESC_FLAG_ME;
- }
- if (is_cce_classified)
- tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
- htt_tcl_metadata = vdev->htt_tcl_metadata;
- if (msdu_info->exception_fw) {
- DP_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1);
- }
- dp_tx_is_hp_update_required(i, msdu_info);
- /*
- * For frames with multiple segments (TSO, ME), jump to next
- * segment.
- */
- if (msdu_info->frm_type == dp_tx_frm_tso) {
- if (msdu_info->u.tso_info.curr_seg->next) {
- msdu_info->u.tso_info.curr_seg =
- msdu_info->u.tso_info.curr_seg->next;
- /*
- * If this is a jumbo nbuf, then increment the
- * number of nbuf users for each additional
- * segment of the msdu. This will ensure that
- * the skb is freed only after receiving tx
- * completion for all segments of an nbuf
- */
- qdf_nbuf_inc_users(nbuf);
- /* Check with MCL if this is needed */
- /* nbuf = msdu_info->u.tso_info.curr_seg->nbuf;
- */
- }
- }
- dp_tx_update_mcast_param(DP_INVALID_PEER,
- &htt_tcl_metadata,
- vdev,
- msdu_info);
- /*
- * Enqueue the Tx MSDU descriptor to HW for transmit
- */
- status = soc->arch_ops.tx_hw_enqueue(soc, vdev, tx_desc,
- htt_tcl_metadata,
- NULL, msdu_info);
- dp_tx_check_and_flush_hp(soc, status, msdu_info);
- if (status != QDF_STATUS_SUCCESS) {
- dp_info_rl("Tx_hw_enqueue Fail tx_desc %pK queue %d",
- tx_desc, tx_q->ring_id);
- dp_tx_get_tid(vdev, nbuf, msdu_info);
- tid_stats = &pdev->stats.tid_stats.
- tid_tx_stats[tx_q->ring_id][msdu_info->tid];
- tid_stats->swdrop_cnt[TX_HW_ENQUEUE]++;
- if (msdu_info->frm_type == dp_tx_frm_me) {
- hw_enq_fail++;
- if (hw_enq_fail == msdu_info->num_seg) {
- /*
- * Unmap is needed only if enqueue
- * failed for all segments.
- */
- qdf_nbuf_unmap(soc->osdev,
- msdu_info->u.sg_info.
- curr_seg->nbuf,
- QDF_DMA_TO_DEVICE);
- }
- /*
- * Free the nbuf for the current segment
- * and make it point to the next in the list.
- * For me, there are as many segments as there
- * are no of clients.
- */
- qdf_nbuf_free(msdu_info->u.sg_info
- .curr_seg->nbuf);
- dp_tx_desc_release(soc, tx_desc,
- tx_q->desc_pool_id);
- if (msdu_info->u.sg_info.curr_seg->next) {
- msdu_info->u.sg_info.curr_seg =
- msdu_info->u.sg_info
- .curr_seg->next;
- nbuf = msdu_info->u.sg_info
- .curr_seg->nbuf;
- } else
- break;
- i++;
- continue;
- }
- /*
- * For TSO frames, the nbuf users increment done for
- * the current segment has to be reverted, since the
- * hw enqueue for this segment failed
- */
- if (msdu_info->frm_type == dp_tx_frm_tso &&
- msdu_info->u.tso_info.curr_seg) {
- /*
- * unmap and free current,
- * retransmit remaining segments
- */
- dp_tx_comp_free_buf(soc, tx_desc, false);
- i++;
- dp_tx_desc_release(soc, tx_desc,
- tx_q->desc_pool_id);
- continue;
- }
- if (msdu_info->frm_type == dp_tx_frm_sg)
- dp_tx_sg_unmap_buf(soc, nbuf, msdu_info);
- dp_tx_desc_release(soc, tx_desc, tx_q->desc_pool_id);
- goto done;
- }
- dp_tx_update_ts_on_enqueued(vdev, msdu_info, tx_desc);
- /*
- * TODO
- * if tso_info structure can be modified to have curr_seg
- * as first element, following 2 blocks of code (for TSO and SG)
- * can be combined into 1
- */
- /*
- * For Multicast-Unicast converted packets,
- * each converted frame (for a client) is represented as
- * 1 segment
- */
- if ((msdu_info->frm_type == dp_tx_frm_sg) ||
- (msdu_info->frm_type == dp_tx_frm_me)) {
- if (msdu_info->u.sg_info.curr_seg->next) {
- msdu_info->u.sg_info.curr_seg =
- msdu_info->u.sg_info.curr_seg->next;
- nbuf = msdu_info->u.sg_info.curr_seg->nbuf;
- } else
- break;
- }
- i++;
- }
- nbuf = NULL;
- done:
- return nbuf;
- }
- /**
- * dp_tx_prepare_sg()- Extract SG info from NBUF and prepare msdu_info
- * for SG frames
- * @vdev: DP vdev handle
- * @nbuf: skb
- * @seg_info: Pointer to Segment info Descriptor to be prepared
- * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
- *
- * Return: NULL on success,
- * nbuf when it fails to send
- */
- static qdf_nbuf_t dp_tx_prepare_sg(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_seg_info_s *seg_info, struct dp_tx_msdu_info_s *msdu_info)
- {
- uint32_t cur_frag, nr_frags, i;
- qdf_dma_addr_t paddr;
- struct dp_tx_sg_info_s *sg_info;
- uint8_t xmit_type = msdu_info->xmit_type;
- sg_info = &msdu_info->u.sg_info;
- nr_frags = qdf_nbuf_get_nr_frags(nbuf);
- if (QDF_STATUS_SUCCESS !=
- qdf_nbuf_map_nbytes_single(vdev->osdev, nbuf,
- QDF_DMA_TO_DEVICE,
- qdf_nbuf_headlen(nbuf))) {
- dp_tx_err("dma map error");
- DP_STATS_INC(vdev, tx_i[xmit_type].sg.dma_map_error,
- 1);
- qdf_nbuf_free(nbuf);
- return NULL;
- }
- paddr = qdf_nbuf_mapped_paddr_get(nbuf);
- seg_info->frags[0].paddr_lo = paddr;
- seg_info->frags[0].paddr_hi = ((uint64_t) paddr) >> 32;
- seg_info->frags[0].len = qdf_nbuf_headlen(nbuf);
- seg_info->frags[0].vaddr = (void *) nbuf;
- for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {
- if (QDF_STATUS_SUCCESS != qdf_nbuf_frag_map(vdev->osdev,
- nbuf, 0,
- QDF_DMA_TO_DEVICE,
- cur_frag)) {
- dp_tx_err("frag dma map error");
- DP_STATS_INC(vdev,
- tx_i[xmit_type].sg.dma_map_error,
- 1);
- goto map_err;
- }
- paddr = qdf_nbuf_get_tx_frag_paddr(nbuf);
- seg_info->frags[cur_frag + 1].paddr_lo = paddr;
- seg_info->frags[cur_frag + 1].paddr_hi =
- ((uint64_t) paddr) >> 32;
- seg_info->frags[cur_frag + 1].len =
- qdf_nbuf_get_frag_size(nbuf, cur_frag);
- }
- seg_info->frag_cnt = (cur_frag + 1);
- seg_info->total_len = qdf_nbuf_len(nbuf);
- seg_info->next = NULL;
- sg_info->curr_seg = seg_info;
- msdu_info->frm_type = dp_tx_frm_sg;
- msdu_info->num_seg = 1;
- return nbuf;
- map_err:
- /* restore paddr into nbuf before calling unmap */
- qdf_nbuf_mapped_paddr_set(nbuf,
- (qdf_dma_addr_t)(seg_info->frags[0].paddr_lo |
- ((uint64_t)
- seg_info->frags[0].paddr_hi) << 32));
- qdf_nbuf_unmap_nbytes_single(vdev->osdev, nbuf,
- QDF_DMA_TO_DEVICE,
- seg_info->frags[0].len);
- for (i = 1; i <= cur_frag; i++) {
- qdf_mem_unmap_page(vdev->osdev, (qdf_dma_addr_t)
- (seg_info->frags[i].paddr_lo | ((uint64_t)
- seg_info->frags[i].paddr_hi) << 32),
- seg_info->frags[i].len,
- QDF_DMA_TO_DEVICE);
- }
- qdf_nbuf_free(nbuf);
- return NULL;
- }
- /**
- * dp_tx_add_tx_sniffer_meta_data()- Add tx_sniffer meta hdr info
- * @vdev: DP vdev handle
- * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
- * @ppdu_cookie: PPDU cookie that should be replayed in the ppdu completions
- *
- * Return: NULL on failure,
- * nbuf when extracted successfully
- */
- static
- void dp_tx_add_tx_sniffer_meta_data(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- uint16_t ppdu_cookie)
- {
- struct htt_tx_msdu_desc_ext2_t *meta_data =
- (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
- qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
- HTT_TX_MSDU_EXT2_DESC_FLAG_SEND_AS_STANDALONE_SET
- (msdu_info->meta_data[5], 1);
- HTT_TX_MSDU_EXT2_DESC_FLAG_HOST_OPAQUE_VALID_SET
- (msdu_info->meta_data[5], 1);
- HTT_TX_MSDU_EXT2_DESC_HOST_OPAQUE_COOKIE_SET
- (msdu_info->meta_data[6], ppdu_cookie);
- msdu_info->exception_fw = 1;
- msdu_info->is_tx_sniffer = 1;
- }
- #ifdef MESH_MODE_SUPPORT
- /**
- * dp_tx_extract_mesh_meta_data()- Extract mesh meta hdr info from nbuf
- * and prepare msdu_info for mesh frames.
- * @vdev: DP vdev handle
- * @nbuf: skb
- * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
- *
- * Return: NULL on failure,
- * nbuf when extracted successfully
- */
- static
- qdf_nbuf_t dp_tx_extract_mesh_meta_data(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- struct meta_hdr_s *mhdr;
- struct htt_tx_msdu_desc_ext2_t *meta_data =
- (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
- mhdr = (struct meta_hdr_s *)qdf_nbuf_data(nbuf);
- if (CB_FTYPE_MESH_TX_INFO != qdf_nbuf_get_tx_ftype(nbuf)) {
- msdu_info->exception_fw = 0;
- goto remove_meta_hdr;
- }
- msdu_info->exception_fw = 1;
- qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
- meta_data->host_tx_desc_pool = 1;
- meta_data->update_peer_cache = 1;
- meta_data->learning_frame = 1;
- if (!(mhdr->flags & METAHDR_FLAG_AUTO_RATE)) {
- meta_data->power = mhdr->power;
- meta_data->mcs_mask = 1 << mhdr->rate_info[0].mcs;
- meta_data->nss_mask = 1 << mhdr->rate_info[0].nss;
- meta_data->pream_type = mhdr->rate_info[0].preamble_type;
- meta_data->retry_limit = mhdr->rate_info[0].max_tries;
- meta_data->dyn_bw = 1;
- meta_data->valid_pwr = 1;
- meta_data->valid_mcs_mask = 1;
- meta_data->valid_nss_mask = 1;
- meta_data->valid_preamble_type = 1;
- meta_data->valid_retries = 1;
- meta_data->valid_bw_info = 1;
- }
- if (mhdr->flags & METAHDR_FLAG_NOENCRYPT) {
- meta_data->encrypt_type = 0;
- meta_data->valid_encrypt_type = 1;
- meta_data->learning_frame = 0;
- }
- meta_data->valid_key_flags = 1;
- meta_data->key_flags = (mhdr->keyix & 0x3);
- remove_meta_hdr:
- if (qdf_nbuf_pull_head(nbuf, sizeof(struct meta_hdr_s)) == NULL) {
- dp_tx_err("qdf_nbuf_pull_head failed");
- qdf_nbuf_free(nbuf);
- return NULL;
- }
- msdu_info->tid = qdf_nbuf_get_priority(nbuf);
- dp_tx_info("Meta hdr %0x %0x %0x %0x %0x %0x"
- " tid %d to_fw %d",
- msdu_info->meta_data[0],
- msdu_info->meta_data[1],
- msdu_info->meta_data[2],
- msdu_info->meta_data[3],
- msdu_info->meta_data[4],
- msdu_info->meta_data[5],
- msdu_info->tid, msdu_info->exception_fw);
- return nbuf;
- }
- #else
- static
- qdf_nbuf_t dp_tx_extract_mesh_meta_data(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- return nbuf;
- }
- #endif
- /**
- * dp_check_exc_metadata() - Checks if parameters are valid
- * @tx_exc: holds all exception path parameters
- *
- * Return: true when all the parameters are valid else false
- *
- */
- static bool dp_check_exc_metadata(struct cdp_tx_exception_metadata *tx_exc)
- {
- bool invalid_tid = (tx_exc->tid >= DP_MAX_TIDS && tx_exc->tid !=
- HTT_INVALID_TID);
- bool invalid_encap_type =
- (tx_exc->tx_encap_type > htt_cmn_pkt_num_types &&
- tx_exc->tx_encap_type != CDP_INVALID_TX_ENCAP_TYPE);
- bool invalid_sec_type = (tx_exc->sec_type > cdp_num_sec_types &&
- tx_exc->sec_type != CDP_INVALID_SEC_TYPE);
- bool invalid_cookie = (tx_exc->is_tx_sniffer == 1 &&
- tx_exc->ppdu_cookie == 0);
- if (tx_exc->is_intrabss_fwd)
- return true;
- if (invalid_tid || invalid_encap_type || invalid_sec_type ||
- invalid_cookie) {
- return false;
- }
- return true;
- }
- #ifdef ATH_SUPPORT_IQUE
- bool dp_tx_mcast_enhance(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
- {
- qdf_ether_header_t *eh;
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- /* Mcast to Ucast Conversion*/
- if (qdf_likely(!vdev->mcast_enhancement_en))
- return true;
- eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
- if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost) &&
- !DP_FRAME_IS_BROADCAST((eh)->ether_dhost)) {
- dp_verbose_debug("Mcast frm for ME %pK", vdev);
- qdf_nbuf_set_next(nbuf, NULL);
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].mcast_en.mcast_pkt, 1,
- qdf_nbuf_len(nbuf));
- if (dp_tx_prepare_send_me(vdev, nbuf) ==
- QDF_STATUS_SUCCESS) {
- return false;
- }
- if (qdf_unlikely(vdev->igmp_mcast_enhanc_en > 0)) {
- if (dp_tx_prepare_send_igmp_me(vdev, nbuf) ==
- QDF_STATUS_SUCCESS) {
- return false;
- }
- }
- }
- return true;
- }
- #else
- bool dp_tx_mcast_enhance(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
- {
- return true;
- }
- #endif
- #ifdef QCA_SUPPORT_WDS_EXTENDED
- /**
- * dp_tx_mcast_drop() - Drop mcast frame if drop_tx_mcast is set in WDS_EXT
- * @vdev: vdev handle
- * @nbuf: skb
- *
- * Return: true if frame is dropped, false otherwise
- */
- static inline bool dp_tx_mcast_drop(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
- {
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- /* Drop tx mcast and WDS Extended feature check */
- if (qdf_unlikely((vdev->drop_tx_mcast) && (vdev->wds_ext_enabled))) {
- qdf_ether_header_t *eh = (qdf_ether_header_t *)
- qdf_nbuf_data(nbuf);
- if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost)) {
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.tx_mcast_drop, 1);
- return true;
- }
- }
- return false;
- }
- #else
- static inline bool dp_tx_mcast_drop(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
- {
- return false;
- }
- #endif
- /**
- * dp_tx_per_pkt_vdev_id_check() - vdev id check for frame
- * @nbuf: qdf_nbuf_t
- * @vdev: struct dp_vdev *
- *
- * Allow packet for processing only if it is for peer client which is
- * connected with same vap. Drop packet if client is connected to
- * different vap.
- *
- * Return: QDF_STATUS
- */
- static inline QDF_STATUS
- dp_tx_per_pkt_vdev_id_check(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
- {
- struct dp_ast_entry *dst_ast_entry = NULL;
- qdf_ether_header_t *eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
- if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost) ||
- DP_FRAME_IS_BROADCAST((eh)->ether_dhost))
- return QDF_STATUS_SUCCESS;
- qdf_spin_lock_bh(&vdev->pdev->soc->ast_lock);
- dst_ast_entry = dp_peer_ast_hash_find_by_vdevid(vdev->pdev->soc,
- eh->ether_dhost,
- vdev->vdev_id);
- /* If there is no ast entry, return failure */
- if (qdf_unlikely(!dst_ast_entry)) {
- qdf_spin_unlock_bh(&vdev->pdev->soc->ast_lock);
- return QDF_STATUS_E_FAILURE;
- }
- qdf_spin_unlock_bh(&vdev->pdev->soc->ast_lock);
- return QDF_STATUS_SUCCESS;
- }
- /**
- * dp_tx_nawds_handler() - NAWDS handler
- *
- * @soc: DP soc handle
- * @vdev: DP vdev handle
- * @msdu_info: msdu_info required to create HTT metadata
- * @nbuf: skb
- * @sa_peer_id:
- *
- * This API transfers the multicast frames with the peer id
- * on NAWDS enabled peer.
- *
- * Return: none
- */
- void dp_tx_nawds_handler(struct dp_soc *soc, struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- qdf_nbuf_t nbuf, uint16_t sa_peer_id)
- {
- struct dp_peer *peer = NULL;
- qdf_nbuf_t nbuf_clone = NULL;
- uint16_t peer_id = DP_INVALID_PEER;
- struct dp_txrx_peer *txrx_peer;
- uint8_t link_id = 0;
- /* This check avoids pkt forwarding which is entered
- * in the ast table but still doesn't have valid peerid.
- */
- if (sa_peer_id == HTT_INVALID_PEER)
- return;
- qdf_spin_lock_bh(&vdev->peer_list_lock);
- TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
- txrx_peer = dp_get_txrx_peer(peer);
- if (!txrx_peer)
- continue;
- if (!txrx_peer->bss_peer && txrx_peer->nawds_enabled) {
- peer_id = peer->peer_id;
- if (!dp_peer_is_primary_link_peer(peer))
- continue;
- /* In the case of wds ext peer mcast traffic will be
- * sent as part of VLAN interface
- */
- if (dp_peer_is_wds_ext_peer(txrx_peer))
- continue;
- /* Multicast packets needs to be
- * dropped in case of intra bss forwarding
- */
- if (sa_peer_id == txrx_peer->peer_id) {
- dp_tx_debug("multicast packet");
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.nawds_mcast_drop,
- 1, link_id);
- continue;
- }
- nbuf_clone = qdf_nbuf_clone(nbuf);
- if (!nbuf_clone) {
- QDF_TRACE(QDF_MODULE_ID_DP,
- QDF_TRACE_LEVEL_ERROR,
- FL("nbuf clone failed"));
- break;
- }
- nbuf_clone = dp_tx_send_msdu_single(vdev, nbuf_clone,
- msdu_info, peer_id,
- NULL);
- if (nbuf_clone) {
- dp_tx_debug("pkt send failed");
- qdf_nbuf_free(nbuf_clone);
- } else {
- if (peer_id != DP_INVALID_PEER)
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
- tx.nawds_mcast,
- 1, qdf_nbuf_len(nbuf), link_id);
- }
- }
- }
- qdf_spin_unlock_bh(&vdev->peer_list_lock);
- }
- #ifdef WLAN_MCAST_MLO
- static inline bool
- dp_tx_check_mesh_vdev(struct dp_vdev *vdev,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- if (!tx_exc_metadata->is_mlo_mcast && qdf_unlikely(vdev->mesh_vdev))
- return true;
- return false;
- }
- #else
- static inline bool
- dp_tx_check_mesh_vdev(struct dp_vdev *vdev,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- if (qdf_unlikely(vdev->mesh_vdev))
- return true;
- return false;
- }
- #endif
- qdf_nbuf_t
- dp_tx_send_exception(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- qdf_nbuf_t nbuf,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_tx_msdu_info_s msdu_info;
- struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
- DP_MOD_ID_TX_EXCEPTION);
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- if (qdf_unlikely(!vdev))
- goto fail;
- qdf_mem_zero(&msdu_info, sizeof(msdu_info));
- if (!tx_exc_metadata)
- goto fail;
- msdu_info.tid = tx_exc_metadata->tid;
- msdu_info.xmit_type = xmit_type;
- dp_verbose_debug("skb "QDF_MAC_ADDR_FMT,
- QDF_MAC_ADDR_REF(nbuf->data));
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].rcvd, 1, qdf_nbuf_len(nbuf));
- if (qdf_unlikely(!dp_check_exc_metadata(tx_exc_metadata))) {
- dp_tx_err("Invalid parameters in exception path");
- goto fail;
- }
- /* for peer based metadata check if peer is valid */
- if (tx_exc_metadata->peer_id != CDP_INVALID_PEER) {
- struct dp_peer *peer = NULL;
- peer = dp_peer_get_ref_by_id(vdev->pdev->soc,
- tx_exc_metadata->peer_id,
- DP_MOD_ID_TX_EXCEPTION);
- if (qdf_unlikely(!peer)) {
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.invalid_peer_id_in_exc_path,
- 1);
- goto fail;
- }
- dp_peer_unref_delete(peer, DP_MOD_ID_TX_EXCEPTION);
- }
- /* Basic sanity checks for unsupported packets */
- /* MESH mode */
- if (dp_tx_check_mesh_vdev(vdev, tx_exc_metadata)) {
- dp_tx_err("Mesh mode is not supported in exception path");
- goto fail;
- }
- /*
- * Classify the frame and call corresponding
- * "prepare" function which extracts the segment (TSO)
- * and fragmentation information (for TSO , SG, ME, or Raw)
- * into MSDU_INFO structure which is later used to fill
- * SW and HW descriptors.
- */
- if (qdf_nbuf_is_tso(nbuf)) {
- dp_verbose_debug("TSO frame %pK", vdev);
- DP_STATS_INC_PKT(vdev->pdev, tso_stats.num_tso_pkts, 1,
- qdf_nbuf_len(nbuf));
- if (dp_tx_prepare_tso(vdev, nbuf, &msdu_info)) {
- DP_STATS_INC_PKT(vdev->pdev, tso_stats.dropped_host, 1,
- qdf_nbuf_len(nbuf));
- goto fail;
- }
- DP_STATS_INC(vdev,
- tx_i[xmit_type].rcvd.num, msdu_info.num_seg - 1);
- goto send_multiple;
- }
- /* SG */
- if (qdf_unlikely(qdf_nbuf_is_nonlinear(nbuf))) {
- struct dp_tx_seg_info_s seg_info = {0};
- nbuf = dp_tx_prepare_sg(vdev, nbuf, &seg_info, &msdu_info);
- if (!nbuf)
- goto fail;
- dp_verbose_debug("non-TSO SG frame %pK", vdev);
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].sg.sg_pkt, 1,
- qdf_nbuf_len(nbuf));
- goto send_multiple;
- }
- if (qdf_likely(tx_exc_metadata->is_tx_sniffer)) {
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].sniffer_rcvd, 1,
- qdf_nbuf_len(nbuf));
- dp_tx_add_tx_sniffer_meta_data(vdev, &msdu_info,
- tx_exc_metadata->ppdu_cookie);
- }
- /*
- * Get HW Queue to use for this frame.
- * TCL supports upto 4 DMA rings, out of which 3 rings are
- * dedicated for data and 1 for command.
- * "queue_id" maps to one hardware ring.
- * With each ring, we also associate a unique Tx descriptor pool
- * to minimize lock contention for these resources.
- */
- dp_tx_get_queue(vdev, nbuf, &msdu_info.tx_queue);
- DP_STATS_INC(vdev,
- tx_i[xmit_type].rcvd_per_core[msdu_info.tx_queue.desc_pool_id],
- 1);
- /*
- * if the packet is mcast packet send through mlo_macst handler
- * for all prnt_vdevs
- */
- if (soc->arch_ops.dp_tx_mlo_mcast_send) {
- nbuf = soc->arch_ops.dp_tx_mlo_mcast_send(soc, vdev,
- nbuf,
- tx_exc_metadata);
- if (!nbuf)
- goto fail;
- }
- if (qdf_likely(tx_exc_metadata->is_intrabss_fwd)) {
- if (qdf_unlikely(vdev->nawds_enabled)) {
- /*
- * This is a multicast packet
- */
- dp_tx_nawds_handler(soc, vdev, &msdu_info, nbuf,
- tx_exc_metadata->peer_id);
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].nawds_mcast,
- 1, qdf_nbuf_len(nbuf));
- }
- nbuf = dp_tx_send_msdu_single(vdev, nbuf, &msdu_info,
- DP_INVALID_PEER, NULL);
- } else {
- /*
- * Check exception descriptors
- */
- if (dp_tx_exception_limit_check(vdev, xmit_type))
- goto fail;
- /* Single linear frame */
- /*
- * If nbuf is a simple linear frame, use send_single function to
- * prepare direct-buffer type TCL descriptor and enqueue to TCL
- * SRNG. There is no need to setup a MSDU extension descriptor.
- */
- nbuf = dp_tx_send_msdu_single(vdev, nbuf, &msdu_info,
- tx_exc_metadata->peer_id,
- tx_exc_metadata);
- }
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
- return nbuf;
- send_multiple:
- nbuf = dp_tx_send_msdu_multiple(vdev, nbuf, &msdu_info);
- fail:
- if (vdev)
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
- dp_verbose_debug("pkt send failed");
- return nbuf;
- }
- qdf_nbuf_t
- dp_tx_send_exception_vdev_id_check(struct cdp_soc_t *soc_hdl,
- uint8_t vdev_id, qdf_nbuf_t nbuf,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
- DP_MOD_ID_TX_EXCEPTION);
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- if (qdf_unlikely(!vdev))
- goto fail;
- if (qdf_unlikely(dp_tx_per_pkt_vdev_id_check(nbuf, vdev)
- == QDF_STATUS_E_FAILURE)) {
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.fail_per_pkt_vdev_id_check,
- 1);
- goto fail;
- }
- /* Unref count as it will again be taken inside dp_tx_exception */
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
- return dp_tx_send_exception(soc_hdl, vdev_id, nbuf, tx_exc_metadata);
- fail:
- if (vdev)
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
- dp_verbose_debug("pkt send failed");
- return nbuf;
- }
- #ifdef MESH_MODE_SUPPORT
- qdf_nbuf_t dp_tx_send_mesh(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- qdf_nbuf_t nbuf)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct meta_hdr_s *mhdr;
- qdf_nbuf_t nbuf_mesh = NULL;
- qdf_nbuf_t nbuf_clone = NULL;
- struct dp_vdev *vdev;
- uint8_t no_enc_frame = 0;
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- nbuf_mesh = qdf_nbuf_unshare(nbuf);
- if (!nbuf_mesh) {
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
- "qdf_nbuf_unshare failed");
- return nbuf;
- }
- vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_MESH);
- if (!vdev) {
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
- "vdev is NULL for vdev_id %d", vdev_id);
- return nbuf;
- }
- nbuf = nbuf_mesh;
- mhdr = (struct meta_hdr_s *)qdf_nbuf_data(nbuf);
- if ((vdev->sec_type != cdp_sec_type_none) &&
- (mhdr->flags & METAHDR_FLAG_NOENCRYPT))
- no_enc_frame = 1;
- if (mhdr->flags & METAHDR_FLAG_NOQOS)
- qdf_nbuf_set_priority(nbuf, HTT_TX_EXT_TID_NON_QOS_MCAST_BCAST);
- if ((mhdr->flags & METAHDR_FLAG_INFO_UPDATED) &&
- !no_enc_frame) {
- nbuf_clone = qdf_nbuf_clone(nbuf);
- if (!nbuf_clone) {
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
- "qdf_nbuf_clone failed");
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
- return nbuf;
- }
- qdf_nbuf_set_tx_ftype(nbuf_clone, CB_FTYPE_MESH_TX_INFO);
- }
- if (nbuf_clone) {
- if (!dp_tx_send(soc_hdl, vdev_id, nbuf_clone)) {
- DP_STATS_INC(vdev, tx_i[xmit_type].mesh.exception_fw,
- 1);
- } else {
- qdf_nbuf_free(nbuf_clone);
- }
- }
- if (no_enc_frame)
- qdf_nbuf_set_tx_ftype(nbuf, CB_FTYPE_MESH_TX_INFO);
- else
- qdf_nbuf_set_tx_ftype(nbuf, CB_FTYPE_INVALID);
- nbuf = dp_tx_send(soc_hdl, vdev_id, nbuf);
- if ((!nbuf) && no_enc_frame) {
- DP_STATS_INC(vdev, tx_i[xmit_type].mesh.exception_fw, 1);
- }
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
- return nbuf;
- }
- #else
- qdf_nbuf_t dp_tx_send_mesh(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- qdf_nbuf_t nbuf)
- {
- return dp_tx_send(soc_hdl, vdev_id, nbuf);
- }
- #endif
- #ifdef DP_UMAC_HW_RESET_SUPPORT
- qdf_nbuf_t dp_tx_drop(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- qdf_nbuf_t nbuf)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev = NULL;
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- vdev = soc->vdev_id_map[vdev_id];
- if (qdf_unlikely(!vdev))
- return nbuf;
- DP_STATS_INC(vdev, tx_i[xmit_type].dropped.drop_ingress, 1);
- return nbuf;
- }
- qdf_nbuf_t dp_tx_exc_drop(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- qdf_nbuf_t nbuf,
- struct cdp_tx_exception_metadata *tx_exc_metadata)
- {
- return dp_tx_drop(soc_hdl, vdev_id, nbuf);
- }
- #endif
- #ifdef FEATURE_DIRECT_LINK
- /**
- * dp_vdev_tx_mark_to_fw() - Mark to_fw bit for the tx packet
- * @nbuf: skb
- * @vdev: DP vdev handle
- *
- * Return: None
- */
- static inline void dp_vdev_tx_mark_to_fw(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
- {
- if (qdf_unlikely(vdev->to_fw))
- QDF_NBUF_CB_TX_PACKET_TO_FW(nbuf) = 1;
- }
- #else
- static inline void dp_vdev_tx_mark_to_fw(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
- {
- }
- #endif
- qdf_nbuf_t dp_tx_send(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- qdf_nbuf_t nbuf)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- uint16_t peer_id = HTT_INVALID_PEER;
- /*
- * doing a memzero is causing additional function call overhead
- * so doing static stack clearing
- */
- struct dp_tx_msdu_info_s msdu_info = {0};
- struct dp_vdev *vdev = NULL;
- qdf_nbuf_t end_nbuf = NULL;
- uint8_t xmit_type;
- if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
- return nbuf;
- /*
- * dp_vdev_get_ref_by_id does does a atomic operation avoid using
- * this in per packet path.
- *
- * As in this path vdev memory is already protected with netdev
- * tx lock
- */
- vdev = soc->vdev_id_map[vdev_id];
- if (qdf_unlikely(!vdev))
- return nbuf;
- dp_tx_get_driver_ingress_ts(vdev, &msdu_info, nbuf);
- dp_vdev_tx_mark_to_fw(nbuf, vdev);
- /*
- * Set Default Host TID value to invalid TID
- * (TID override disabled)
- */
- msdu_info.tid = HTT_TX_EXT_TID_INVALID;
- xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- msdu_info.xmit_type = xmit_type;
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].rcvd, 1, qdf_nbuf_len(nbuf));
- if (qdf_unlikely(vdev->mesh_vdev)) {
- qdf_nbuf_t nbuf_mesh = dp_tx_extract_mesh_meta_data(vdev, nbuf,
- &msdu_info);
- if (!nbuf_mesh) {
- dp_verbose_debug("Extracting mesh metadata failed");
- return nbuf;
- }
- nbuf = nbuf_mesh;
- }
- /*
- * Get HW Queue to use for this frame.
- * TCL supports upto 4 DMA rings, out of which 3 rings are
- * dedicated for data and 1 for command.
- * "queue_id" maps to one hardware ring.
- * With each ring, we also associate a unique Tx descriptor pool
- * to minimize lock contention for these resources.
- */
- dp_tx_get_queue(vdev, nbuf, &msdu_info.tx_queue);
- DP_STATS_INC(vdev,
- tx_i[xmit_type].rcvd_per_core[msdu_info.tx_queue.desc_pool_id],
- 1);
- /*
- * TCL H/W supports 2 DSCP-TID mapping tables.
- * Table 1 - Default DSCP-TID mapping table
- * Table 2 - 1 DSCP-TID override table
- *
- * If we need a different DSCP-TID mapping for this vap,
- * call tid_classify to extract DSCP/ToS from frame and
- * map to a TID and store in msdu_info. This is later used
- * to fill in TCL Input descriptor (per-packet TID override).
- */
- dp_tx_classify_tid(vdev, nbuf, &msdu_info);
- /*
- * Classify the frame and call corresponding
- * "prepare" function which extracts the segment (TSO)
- * and fragmentation information (for TSO , SG, ME, or Raw)
- * into MSDU_INFO structure which is later used to fill
- * SW and HW descriptors.
- */
- if (qdf_nbuf_is_tso(nbuf)) {
- dp_verbose_debug("TSO frame %pK", vdev);
- DP_STATS_INC_PKT(vdev->pdev, tso_stats.num_tso_pkts, 1,
- qdf_nbuf_len(nbuf));
- if (dp_tx_prepare_tso(vdev, nbuf, &msdu_info)) {
- DP_STATS_INC_PKT(vdev->pdev, tso_stats.dropped_host, 1,
- qdf_nbuf_len(nbuf));
- return nbuf;
- }
- DP_STATS_INC(vdev, tx_i[xmit_type].rcvd.num,
- msdu_info.num_seg - 1);
- goto send_multiple;
- }
- /* SG */
- if (qdf_unlikely(qdf_nbuf_is_nonlinear(nbuf))) {
- if (qdf_nbuf_get_nr_frags(nbuf) > DP_TX_MAX_NUM_FRAGS - 1) {
- if (qdf_unlikely(qdf_nbuf_linearize(nbuf)))
- return nbuf;
- } else {
- struct dp_tx_seg_info_s seg_info = {0};
- if (qdf_unlikely(is_nbuf_frm_rmnet(nbuf, &msdu_info)))
- goto send_single;
- nbuf = dp_tx_prepare_sg(vdev, nbuf, &seg_info,
- &msdu_info);
- if (!nbuf)
- return NULL;
- dp_verbose_debug("non-TSO SG frame %pK", vdev);
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].sg.sg_pkt, 1,
- qdf_nbuf_len(nbuf));
- goto send_multiple;
- }
- }
- if (qdf_unlikely(!dp_tx_mcast_enhance(vdev, nbuf)))
- return NULL;
- if (qdf_unlikely(dp_tx_mcast_drop(vdev, nbuf)))
- return nbuf;
- /* RAW */
- if (qdf_unlikely(vdev->tx_encap_type == htt_cmn_pkt_type_raw)) {
- struct dp_tx_seg_info_s seg_info = {0};
- nbuf = dp_tx_prepare_raw(vdev, nbuf, &seg_info, &msdu_info);
- if (!nbuf)
- return NULL;
- dp_verbose_debug("Raw frame %pK", vdev);
- goto send_multiple;
- }
- if (qdf_unlikely(vdev->nawds_enabled)) {
- qdf_ether_header_t *eh = (qdf_ether_header_t *)
- qdf_nbuf_data(nbuf);
- if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost)) {
- uint16_t sa_peer_id = DP_INVALID_PEER;
- if (!soc->ast_offload_support) {
- struct dp_ast_entry *ast_entry = NULL;
- qdf_spin_lock_bh(&soc->ast_lock);
- ast_entry = dp_peer_ast_hash_find_by_pdevid
- (soc,
- (uint8_t *)(eh->ether_shost),
- vdev->pdev->pdev_id);
- if (ast_entry)
- sa_peer_id = ast_entry->peer_id;
- qdf_spin_unlock_bh(&soc->ast_lock);
- }
- dp_tx_nawds_handler(soc, vdev, &msdu_info, nbuf,
- sa_peer_id);
- }
- peer_id = DP_INVALID_PEER;
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].nawds_mcast,
- 1, qdf_nbuf_len(nbuf));
- }
- send_single:
- /* Single linear frame */
- /*
- * If nbuf is a simple linear frame, use send_single function to
- * prepare direct-buffer type TCL descriptor and enqueue to TCL
- * SRNG. There is no need to setup a MSDU extension descriptor.
- */
- nbuf = dp_tx_send_msdu_single_wrapper(vdev, nbuf, &msdu_info,
- peer_id, end_nbuf);
- return nbuf;
- send_multiple:
- nbuf = dp_tx_send_msdu_multiple(vdev, nbuf, &msdu_info);
- if (qdf_unlikely(nbuf && msdu_info.frm_type == dp_tx_frm_raw))
- dp_tx_raw_prepare_unset(vdev->pdev->soc, nbuf);
- return nbuf;
- }
- qdf_nbuf_t dp_tx_send_vdev_id_check(struct cdp_soc_t *soc_hdl,
- uint8_t vdev_id, qdf_nbuf_t nbuf)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev = NULL;
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
- return nbuf;
- /*
- * dp_vdev_get_ref_by_id does does a atomic operation avoid using
- * this in per packet path.
- *
- * As in this path vdev memory is already protected with netdev
- * tx lock
- */
- vdev = soc->vdev_id_map[vdev_id];
- if (qdf_unlikely(!vdev))
- return nbuf;
- if (qdf_unlikely(dp_tx_per_pkt_vdev_id_check(nbuf, vdev)
- == QDF_STATUS_E_FAILURE)) {
- DP_STATS_INC(vdev,
- tx_i[xmit_type].dropped.fail_per_pkt_vdev_id_check,
- 1);
- return nbuf;
- }
- return dp_tx_send(soc_hdl, vdev_id, nbuf);
- }
- #ifdef UMAC_SUPPORT_PROXY_ARP
- /**
- * dp_tx_proxy_arp() - Tx proxy arp handler
- * @vdev: datapath vdev handle
- * @nbuf: sk buffer
- *
- * Return: status
- */
- int dp_tx_proxy_arp(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
- {
- if (vdev->osif_proxy_arp)
- return vdev->osif_proxy_arp(vdev->osif_vdev, nbuf);
- /*
- * when UMAC_SUPPORT_PROXY_ARP is defined, we expect
- * osif_proxy_arp has a valid function pointer assigned
- * to it
- */
- dp_tx_err("valid function pointer for osif_proxy_arp is expected!!\n");
- return QDF_STATUS_NOT_INITIALIZED;
- }
- #else
- int dp_tx_proxy_arp(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
- {
- return QDF_STATUS_SUCCESS;
- }
- #endif
- #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
- !defined(CONFIG_MLO_SINGLE_DEV)
- #ifdef WLAN_MCAST_MLO
- static bool
- dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf,
- uint8_t reinject_reason)
- {
- if (reinject_reason == HTT_TX_FW2WBM_REINJECT_REASON_MLO_MCAST) {
- if (soc->arch_ops.dp_tx_mcast_handler)
- soc->arch_ops.dp_tx_mcast_handler(soc, vdev, nbuf);
- dp_tx_desc_release(soc, tx_desc, tx_desc->pool_id);
- return true;
- }
- return false;
- }
- #else /* WLAN_MCAST_MLO */
- static inline bool
- dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf,
- uint8_t reinject_reason)
- {
- return false;
- }
- #endif /* WLAN_MCAST_MLO */
- #else
- static inline bool
- dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t nbuf,
- uint8_t reinject_reason)
- {
- return false;
- }
- #endif
- void dp_tx_reinject_handler(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- uint8_t *status,
- uint8_t reinject_reason)
- {
- struct dp_peer *peer = NULL;
- uint32_t peer_id = HTT_INVALID_PEER;
- qdf_nbuf_t nbuf = tx_desc->nbuf;
- qdf_nbuf_t nbuf_copy = NULL;
- struct dp_tx_msdu_info_s msdu_info;
- #ifdef WDS_VENDOR_EXTENSION
- int is_mcast = 0, is_ucast = 0;
- int num_peers_3addr = 0;
- qdf_ether_header_t *eth_hdr = (qdf_ether_header_t *)(qdf_nbuf_data(nbuf));
- struct ieee80211_frame_addr4 *wh = (struct ieee80211_frame_addr4 *)(qdf_nbuf_data(nbuf));
- #endif
- struct dp_txrx_peer *txrx_peer;
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
- qdf_assert(vdev);
- dp_tx_debug("Tx reinject path");
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].reinject_pkts, 1,
- qdf_nbuf_len(tx_desc->nbuf));
- if (dp_tx_reinject_mlo_hdl(soc, vdev, tx_desc, nbuf, reinject_reason))
- return;
- #ifdef WDS_VENDOR_EXTENSION
- if (qdf_unlikely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
- is_mcast = (IS_MULTICAST(wh->i_addr1)) ? 1 : 0;
- } else {
- is_mcast = (IS_MULTICAST(eth_hdr->ether_dhost)) ? 1 : 0;
- }
- is_ucast = !is_mcast;
- qdf_spin_lock_bh(&vdev->peer_list_lock);
- TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
- txrx_peer = dp_get_txrx_peer(peer);
- if (!txrx_peer || txrx_peer->bss_peer)
- continue;
- /* Detect wds peers that use 3-addr framing for mcast.
- * if there are any, the bss_peer is used to send the
- * the mcast frame using 3-addr format. all wds enabled
- * peers that use 4-addr framing for mcast frames will
- * be duplicated and sent as 4-addr frames below.
- */
- if (!txrx_peer->wds_enabled ||
- !txrx_peer->wds_ecm.wds_tx_mcast_4addr) {
- num_peers_3addr = 1;
- break;
- }
- }
- qdf_spin_unlock_bh(&vdev->peer_list_lock);
- #endif
- if (qdf_unlikely(vdev->mesh_vdev)) {
- DP_TX_FREE_SINGLE_BUF(vdev->pdev->soc, tx_desc->nbuf);
- } else {
- qdf_spin_lock_bh(&vdev->peer_list_lock);
- TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
- txrx_peer = dp_get_txrx_peer(peer);
- if (!txrx_peer)
- continue;
- if ((txrx_peer->peer_id != HTT_INVALID_PEER) &&
- #ifdef WDS_VENDOR_EXTENSION
- /*
- * . if 3-addr STA, then send on BSS Peer
- * . if Peer WDS enabled and accept 4-addr mcast,
- * send mcast on that peer only
- * . if Peer WDS enabled and accept 4-addr ucast,
- * send ucast on that peer only
- */
- ((txrx_peer->bss_peer && num_peers_3addr && is_mcast) ||
- (txrx_peer->wds_enabled &&
- ((is_mcast && txrx_peer->wds_ecm.wds_tx_mcast_4addr) ||
- (is_ucast &&
- txrx_peer->wds_ecm.wds_tx_ucast_4addr))))) {
- #else
- (txrx_peer->bss_peer &&
- (dp_tx_proxy_arp(vdev, nbuf) == QDF_STATUS_SUCCESS))) {
- #endif
- peer_id = DP_INVALID_PEER;
- nbuf_copy = qdf_nbuf_copy(nbuf);
- if (!nbuf_copy) {
- dp_tx_debug("nbuf copy failed");
- break;
- }
- qdf_mem_zero(&msdu_info, sizeof(msdu_info));
- dp_tx_get_queue(vdev, nbuf,
- &msdu_info.tx_queue);
- msdu_info.xmit_type =
- qdf_nbuf_get_vdev_xmit_type(nbuf);
- nbuf_copy = dp_tx_send_msdu_single(vdev,
- nbuf_copy,
- &msdu_info,
- peer_id,
- NULL);
- if (nbuf_copy) {
- dp_tx_debug("pkt send failed");
- qdf_nbuf_free(nbuf_copy);
- }
- }
- }
- qdf_spin_unlock_bh(&vdev->peer_list_lock);
- qdf_nbuf_unmap_nbytes_single(vdev->osdev, nbuf,
- QDF_DMA_TO_DEVICE, nbuf->len);
- qdf_nbuf_free(nbuf);
- }
- dp_tx_desc_release(soc, tx_desc, tx_desc->pool_id);
- }
- void dp_tx_inspect_handler(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- uint8_t *status)
- {
- uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(tx_desc->nbuf);
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO,
- "%s Tx inspect path",
- __func__);
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].inspect_pkts, 1,
- qdf_nbuf_len(tx_desc->nbuf));
- DP_TX_FREE_SINGLE_BUF(soc, tx_desc->nbuf);
- dp_tx_desc_release(soc, tx_desc, tx_desc->pool_id);
- }
- #ifdef MESH_MODE_SUPPORT
- /**
- * dp_tx_comp_fill_tx_completion_stats() - Fill per packet Tx completion stats
- * in mesh meta header
- * @tx_desc: software descriptor head pointer
- * @ts: pointer to tx completion stats
- * Return: none
- */
- static
- void dp_tx_comp_fill_tx_completion_stats(struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts)
- {
- qdf_nbuf_t netbuf = tx_desc->nbuf;
- if (!tx_desc->msdu_ext_desc) {
- if (qdf_nbuf_pull_head(netbuf, tx_desc->pkt_offset) == NULL) {
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
- "netbuf %pK offset %d",
- netbuf, tx_desc->pkt_offset);
- return;
- }
- }
- }
- #else
- static
- void dp_tx_comp_fill_tx_completion_stats(struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts)
- {
- }
- #endif
- #ifdef CONFIG_SAWF
- static void dp_tx_update_peer_sawf_stats(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t tid)
- {
- dp_sawf_tx_compl_update_peer_stats(soc, vdev, txrx_peer, tx_desc,
- ts, tid);
- }
- static void dp_tx_compute_delay_avg(struct cdp_delay_tx_stats *tx_delay,
- uint32_t nw_delay,
- uint32_t sw_delay,
- uint32_t hw_delay)
- {
- dp_peer_tid_delay_avg(tx_delay,
- nw_delay,
- sw_delay,
- hw_delay);
- }
- #else
- static void dp_tx_update_peer_sawf_stats(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t tid)
- {
- }
- static inline void
- dp_tx_compute_delay_avg(struct cdp_delay_tx_stats *tx_delay,
- uint32_t nw_delay, uint32_t sw_delay,
- uint32_t hw_delay)
- {
- }
- #endif
- #ifdef QCA_PEER_EXT_STATS
- #ifdef WLAN_CONFIG_TX_DELAY
- static void dp_tx_compute_tid_delay(struct cdp_delay_tid_stats *stats,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- struct dp_vdev *vdev)
- {
- struct dp_soc *soc = vdev->pdev->soc;
- struct cdp_delay_tx_stats *tx_delay = &stats->tx_delay;
- int64_t timestamp_ingress, timestamp_hw_enqueue;
- uint32_t sw_enqueue_delay, fwhw_transmit_delay = 0;
- if (!ts->valid)
- return;
- timestamp_ingress = qdf_nbuf_get_timestamp_us(tx_desc->nbuf);
- timestamp_hw_enqueue = qdf_ktime_to_us(tx_desc->timestamp);
- sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
- dp_hist_update_stats(&tx_delay->tx_swq_delay, sw_enqueue_delay);
- if (soc->arch_ops.dp_tx_compute_hw_delay)
- if (!soc->arch_ops.dp_tx_compute_hw_delay(soc, vdev, ts,
- &fwhw_transmit_delay))
- dp_hist_update_stats(&tx_delay->hwtx_delay,
- fwhw_transmit_delay);
- dp_tx_compute_delay_avg(tx_delay, 0, sw_enqueue_delay,
- fwhw_transmit_delay);
- }
- #else
- /**
- * dp_tx_compute_tid_delay() - Compute per TID delay
- * @stats: Per TID delay stats
- * @tx_desc: Software Tx descriptor
- * @ts: Tx completion status
- * @vdev: vdev
- *
- * Compute the software enqueue and hw enqueue delays and
- * update the respective histograms
- *
- * Return: void
- */
- static void dp_tx_compute_tid_delay(struct cdp_delay_tid_stats *stats,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- struct dp_vdev *vdev)
- {
- struct cdp_delay_tx_stats *tx_delay = &stats->tx_delay;
- int64_t current_timestamp, timestamp_ingress, timestamp_hw_enqueue;
- uint32_t sw_enqueue_delay, fwhw_transmit_delay;
- current_timestamp = qdf_ktime_to_ms(qdf_ktime_real_get());
- timestamp_ingress = qdf_nbuf_get_timestamp(tx_desc->nbuf);
- timestamp_hw_enqueue = qdf_ktime_to_ms(tx_desc->timestamp);
- sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
- fwhw_transmit_delay = (uint32_t)(current_timestamp -
- timestamp_hw_enqueue);
- /*
- * Update the Tx software enqueue delay and HW enque-Completion delay.
- */
- dp_hist_update_stats(&tx_delay->tx_swq_delay, sw_enqueue_delay);
- dp_hist_update_stats(&tx_delay->hwtx_delay, fwhw_transmit_delay);
- }
- #endif
- /**
- * dp_tx_update_peer_delay_stats() - Update the peer delay stats
- * @txrx_peer: DP peer context
- * @tx_desc: Tx software descriptor
- * @ts: Tx completion status
- * @ring_id: Rx CPU context ID/CPU_ID
- *
- * Update the peer extended stats. These are enhanced other
- * delay stats per msdu level.
- *
- * Return: void
- */
- static void dp_tx_update_peer_delay_stats(struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t ring_id)
- {
- struct dp_pdev *pdev = txrx_peer->vdev->pdev;
- struct dp_soc *soc = NULL;
- struct dp_peer_delay_stats *delay_stats = NULL;
- uint8_t tid;
- soc = pdev->soc;
- if (qdf_likely(!wlan_cfg_is_peer_ext_stats_enabled(soc->wlan_cfg_ctx)))
- return;
- if (!txrx_peer->delay_stats)
- return;
- tid = ts->tid;
- delay_stats = txrx_peer->delay_stats;
- /*
- * For non-TID packets use the TID 9
- */
- if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
- tid = CDP_MAX_DATA_TIDS - 1;
- dp_tx_compute_tid_delay(&delay_stats->delay_tid_stats[tid][ring_id],
- tx_desc, ts, txrx_peer->vdev);
- }
- #else
- static inline
- void dp_tx_update_peer_delay_stats(struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t ring_id)
- {
- }
- #endif
- #ifdef WLAN_PEER_JITTER
- /**
- * dp_tx_jitter_get_avg_jitter() - compute the average jitter
- * @curr_delay: Current delay
- * @prev_delay: Previous delay
- * @avg_jitter: Average Jitter
- * Return: Newly Computed Average Jitter
- */
- static uint32_t dp_tx_jitter_get_avg_jitter(uint32_t curr_delay,
- uint32_t prev_delay,
- uint32_t avg_jitter)
- {
- uint32_t curr_jitter;
- int32_t jitter_diff;
- curr_jitter = qdf_abs(curr_delay - prev_delay);
- if (!avg_jitter)
- return curr_jitter;
- jitter_diff = curr_jitter - avg_jitter;
- if (jitter_diff < 0)
- avg_jitter = avg_jitter -
- (qdf_abs(jitter_diff) >> DP_AVG_JITTER_WEIGHT_DENOM);
- else
- avg_jitter = avg_jitter +
- (qdf_abs(jitter_diff) >> DP_AVG_JITTER_WEIGHT_DENOM);
- return avg_jitter;
- }
- /**
- * dp_tx_jitter_get_avg_delay() - compute the average delay
- * @curr_delay: Current delay
- * @avg_delay: Average delay
- * Return: Newly Computed Average Delay
- */
- static uint32_t dp_tx_jitter_get_avg_delay(uint32_t curr_delay,
- uint32_t avg_delay)
- {
- int32_t delay_diff;
- if (!avg_delay)
- return curr_delay;
- delay_diff = curr_delay - avg_delay;
- if (delay_diff < 0)
- avg_delay = avg_delay - (qdf_abs(delay_diff) >>
- DP_AVG_DELAY_WEIGHT_DENOM);
- else
- avg_delay = avg_delay + (qdf_abs(delay_diff) >>
- DP_AVG_DELAY_WEIGHT_DENOM);
- return avg_delay;
- }
- #ifdef WLAN_CONFIG_TX_DELAY
- /**
- * dp_tx_compute_cur_delay() - get the current delay
- * @soc: soc handle
- * @vdev: vdev structure for data path state
- * @ts: Tx completion status
- * @curr_delay: current delay
- * @tx_desc: tx descriptor
- * Return: void
- */
- static
- QDF_STATUS dp_tx_compute_cur_delay(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct hal_tx_completion_status *ts,
- uint32_t *curr_delay,
- struct dp_tx_desc_s *tx_desc)
- {
- QDF_STATUS status = QDF_STATUS_E_FAILURE;
- if (soc->arch_ops.dp_tx_compute_hw_delay)
- status = soc->arch_ops.dp_tx_compute_hw_delay(soc, vdev, ts,
- curr_delay);
- return status;
- }
- #else
- static
- QDF_STATUS dp_tx_compute_cur_delay(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct hal_tx_completion_status *ts,
- uint32_t *curr_delay,
- struct dp_tx_desc_s *tx_desc)
- {
- int64_t current_timestamp, timestamp_hw_enqueue;
- current_timestamp = qdf_ktime_to_us(qdf_ktime_real_get());
- timestamp_hw_enqueue = qdf_ktime_to_us(tx_desc->timestamp);
- *curr_delay = (uint32_t)(current_timestamp - timestamp_hw_enqueue);
- return QDF_STATUS_SUCCESS;
- }
- #endif
- /**
- * dp_tx_compute_tid_jitter() - compute per tid per ring jitter
- * @jitter: per tid per ring jitter stats
- * @ts: Tx completion status
- * @vdev: vdev structure for data path state
- * @tx_desc: tx descriptor
- * Return: void
- */
- static void dp_tx_compute_tid_jitter(struct cdp_peer_tid_stats *jitter,
- struct hal_tx_completion_status *ts,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- uint32_t curr_delay, avg_delay, avg_jitter, prev_delay;
- struct dp_soc *soc = vdev->pdev->soc;
- QDF_STATUS status = QDF_STATUS_E_FAILURE;
- if (ts->status != HAL_TX_TQM_RR_FRAME_ACKED) {
- jitter->tx_drop += 1;
- return;
- }
- status = dp_tx_compute_cur_delay(soc, vdev, ts, &curr_delay,
- tx_desc);
- if (QDF_IS_STATUS_SUCCESS(status)) {
- avg_delay = jitter->tx_avg_delay;
- avg_jitter = jitter->tx_avg_jitter;
- prev_delay = jitter->tx_prev_delay;
- avg_jitter = dp_tx_jitter_get_avg_jitter(curr_delay,
- prev_delay,
- avg_jitter);
- avg_delay = dp_tx_jitter_get_avg_delay(curr_delay, avg_delay);
- jitter->tx_avg_delay = avg_delay;
- jitter->tx_avg_jitter = avg_jitter;
- jitter->tx_prev_delay = curr_delay;
- jitter->tx_total_success += 1;
- } else if (status == QDF_STATUS_E_FAILURE) {
- jitter->tx_avg_err += 1;
- }
- }
- /* dp_tx_update_peer_jitter_stats() - Update the peer jitter stats
- * @txrx_peer: DP peer context
- * @tx_desc: Tx software descriptor
- * @ts: Tx completion status
- * @ring_id: Rx CPU context ID/CPU_ID
- * Return: void
- */
- static void dp_tx_update_peer_jitter_stats(struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t ring_id)
- {
- struct dp_pdev *pdev = txrx_peer->vdev->pdev;
- struct dp_soc *soc = pdev->soc;
- struct cdp_peer_tid_stats *jitter_stats = NULL;
- uint8_t tid;
- struct cdp_peer_tid_stats *rx_tid = NULL;
- if (qdf_likely(!wlan_cfg_is_peer_jitter_stats_enabled(soc->wlan_cfg_ctx)))
- return;
- if (!txrx_peer->jitter_stats)
- return;
- tid = ts->tid;
- jitter_stats = txrx_peer->jitter_stats;
- /*
- * For non-TID packets use the TID 9
- */
- if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
- tid = CDP_MAX_DATA_TIDS - 1;
- rx_tid = &jitter_stats[tid * CDP_MAX_TXRX_CTX + ring_id];
- dp_tx_compute_tid_jitter(rx_tid,
- ts, txrx_peer->vdev, tx_desc);
- }
- #else
- static void dp_tx_update_peer_jitter_stats(struct dp_txrx_peer *txrx_peer,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- uint8_t ring_id)
- {
- }
- #endif
- #ifdef HW_TX_DELAY_STATS_ENABLE
- /**
- * dp_update_tx_delay_stats() - update the delay stats
- * @vdev: vdev handle
- * @delay: delay in ms or us based on the flag delay_in_us
- * @tid: tid value
- * @mode: type of tx delay mode
- * @ring_id: ring number
- * @delay_in_us: flag to indicate whether the delay is in ms or us
- *
- * Return: none
- */
- static inline
- void dp_update_tx_delay_stats(struct dp_vdev *vdev, uint32_t delay, uint8_t tid,
- uint8_t mode, uint8_t ring_id, bool delay_in_us)
- {
- struct cdp_tid_tx_stats *tstats =
- &vdev->stats.tid_tx_stats[ring_id][tid];
- dp_update_delay_stats(tstats, NULL, delay, tid, mode, ring_id,
- delay_in_us);
- }
- #else
- static inline
- void dp_update_tx_delay_stats(struct dp_vdev *vdev, uint32_t delay, uint8_t tid,
- uint8_t mode, uint8_t ring_id, bool delay_in_us)
- {
- struct cdp_tid_tx_stats *tstats =
- &vdev->pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
- dp_update_delay_stats(tstats, NULL, delay, tid, mode, ring_id,
- delay_in_us);
- }
- #endif
- void dp_tx_compute_delay(struct dp_vdev *vdev, struct dp_tx_desc_s *tx_desc,
- uint8_t tid, uint8_t ring_id)
- {
- int64_t current_timestamp, timestamp_ingress, timestamp_hw_enqueue;
- uint32_t sw_enqueue_delay, fwhw_transmit_delay, interframe_delay;
- uint32_t fwhw_transmit_delay_us;
- if (qdf_likely(!vdev->pdev->delay_stats_flag) &&
- qdf_likely(!dp_is_vdev_tx_delay_stats_enabled(vdev)))
- return;
- if (dp_is_vdev_tx_delay_stats_enabled(vdev)) {
- fwhw_transmit_delay_us =
- qdf_ktime_to_us(qdf_ktime_real_get()) -
- qdf_ktime_to_us(tx_desc->timestamp);
- /*
- * Delay between packet enqueued to HW and Tx completion in us
- */
- dp_update_tx_delay_stats(vdev, fwhw_transmit_delay_us, tid,
- CDP_DELAY_STATS_FW_HW_TRANSMIT,
- ring_id, true);
- /*
- * For MCL, only enqueue to completion delay is required
- * so return if the vdev flag is enabled.
- */
- return;
- }
- current_timestamp = qdf_ktime_to_ms(qdf_ktime_real_get());
- timestamp_hw_enqueue = qdf_ktime_to_ms(tx_desc->timestamp);
- fwhw_transmit_delay = (uint32_t)(current_timestamp -
- timestamp_hw_enqueue);
- if (!timestamp_hw_enqueue)
- return;
- /*
- * Delay between packet enqueued to HW and Tx completion in ms
- */
- dp_update_tx_delay_stats(vdev, fwhw_transmit_delay, tid,
- CDP_DELAY_STATS_FW_HW_TRANSMIT, ring_id,
- false);
- timestamp_ingress = qdf_nbuf_get_timestamp(tx_desc->nbuf);
- sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
- interframe_delay = (uint32_t)(timestamp_ingress -
- vdev->prev_tx_enq_tstamp);
- /*
- * Delay in software enqueue
- */
- dp_update_tx_delay_stats(vdev, sw_enqueue_delay, tid,
- CDP_DELAY_STATS_SW_ENQ, ring_id,
- false);
- /*
- * Update interframe delay stats calculated at hardstart receive point.
- * Value of vdev->prev_tx_enq_tstamp will be 0 for 1st frame, so
- * interframe delay will not be calculate correctly for 1st frame.
- * On the other side, this will help in avoiding extra per packet check
- * of !vdev->prev_tx_enq_tstamp.
- */
- dp_update_tx_delay_stats(vdev, interframe_delay, tid,
- CDP_DELAY_STATS_TX_INTERFRAME, ring_id,
- false);
- vdev->prev_tx_enq_tstamp = timestamp_ingress;
- }
- #ifdef DISABLE_DP_STATS
- static
- inline void dp_update_no_ack_stats(qdf_nbuf_t nbuf,
- struct dp_txrx_peer *txrx_peer,
- uint8_t link_id)
- {
- }
- #else
- static inline void
- dp_update_no_ack_stats(qdf_nbuf_t nbuf, struct dp_txrx_peer *txrx_peer,
- uint8_t link_id)
- {
- enum qdf_proto_subtype subtype = QDF_PROTO_INVALID;
- DPTRACE(qdf_dp_track_noack_check(nbuf, &subtype));
- if (subtype != QDF_PROTO_INVALID)
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.no_ack_count[subtype],
- 1, link_id);
- }
- #endif
- #ifndef QCA_ENHANCED_STATS_SUPPORT
- #ifdef DP_PEER_EXTENDED_API
- static inline uint8_t
- dp_tx_get_mpdu_retry_threshold(struct dp_txrx_peer *txrx_peer)
- {
- return txrx_peer->mpdu_retry_threshold;
- }
- #else
- static inline uint8_t
- dp_tx_get_mpdu_retry_threshold(struct dp_txrx_peer *txrx_peer)
- {
- return 0;
- }
- #endif
- /**
- * dp_tx_update_peer_extd_stats()- Update Tx extended path stats for peer
- *
- * @ts: Tx compltion status
- * @txrx_peer: datapath txrx_peer handle
- * @link_id: Link id
- *
- * Return: void
- */
- static inline void
- dp_tx_update_peer_extd_stats(struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer, uint8_t link_id)
- {
- uint8_t mcs, pkt_type, dst_mcs_idx;
- uint8_t retry_threshold = dp_tx_get_mpdu_retry_threshold(txrx_peer);
- mcs = ts->mcs;
- pkt_type = ts->pkt_type;
- /* do HW to SW pkt type conversion */
- pkt_type = (pkt_type >= HAL_DOT11_MAX ? DOT11_MAX :
- hal_2_dp_pkt_type_map[pkt_type]);
- dst_mcs_idx = dp_get_mcs_array_index_by_pkt_type_mcs(pkt_type, mcs);
- if (MCS_INVALID_ARRAY_INDEX != dst_mcs_idx)
- DP_PEER_EXTD_STATS_INC(txrx_peer,
- tx.pkt_type[pkt_type].mcs_count[dst_mcs_idx],
- 1, link_id);
- DP_PEER_EXTD_STATS_INC(txrx_peer, tx.sgi_count[ts->sgi], 1, link_id);
- DP_PEER_EXTD_STATS_INC(txrx_peer, tx.bw[ts->bw], 1, link_id);
- DP_PEER_EXTD_STATS_UPD(txrx_peer, tx.last_ack_rssi, ts->ack_frame_rssi,
- link_id);
- DP_PEER_EXTD_STATS_INC(txrx_peer,
- tx.wme_ac_type[TID_TO_WME_AC(ts->tid)], 1,
- link_id);
- DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.stbc, 1, ts->stbc, link_id);
- DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.ldpc, 1, ts->ldpc, link_id);
- DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.retries, 1, ts->transmit_cnt > 1,
- link_id);
- if (ts->first_msdu) {
- DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.retries_mpdu, 1,
- ts->transmit_cnt > 1, link_id);
- if (!retry_threshold)
- return;
- DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.mpdu_success_with_retries,
- qdf_do_div(ts->transmit_cnt,
- retry_threshold),
- ts->transmit_cnt > retry_threshold,
- link_id);
- }
- }
- #else
- static inline void
- dp_tx_update_peer_extd_stats(struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer, uint8_t link_id)
- {
- }
- #endif
- #if defined(WLAN_FEATURE_11BE_MLO) && \
- (defined(QCA_ENHANCED_STATS_SUPPORT) || \
- defined(DP_MLO_LINK_STATS_SUPPORT))
- static inline uint8_t
- dp_tx_get_link_id_from_ppdu_id(struct dp_soc *soc,
- struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer,
- struct dp_vdev *vdev)
- {
- uint8_t hw_link_id = 0;
- uint32_t ppdu_id;
- uint8_t link_id_offset, link_id_bits;
- if (!txrx_peer->is_mld_peer || !vdev->pdev->link_peer_stats)
- return 0;
- link_id_offset = soc->link_id_offset;
- link_id_bits = soc->link_id_bits;
- ppdu_id = ts->ppdu_id;
- hw_link_id = ((DP_GET_HW_LINK_ID_FRM_PPDU_ID(ppdu_id, link_id_offset,
- link_id_bits)) + 1);
- if (hw_link_id > DP_MAX_MLO_LINKS) {
- hw_link_id = 0;
- DP_PEER_PER_PKT_STATS_INC(
- txrx_peer,
- tx.inval_link_id_pkt_cnt, 1, hw_link_id);
- }
- return hw_link_id;
- }
- #else
- static inline uint8_t
- dp_tx_get_link_id_from_ppdu_id(struct dp_soc *soc,
- struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer,
- struct dp_vdev *vdev)
- {
- return 0;
- }
- #endif
- /**
- * dp_tx_update_peer_stats() - Update peer stats from Tx completion indications
- * per wbm ring
- *
- * @tx_desc: software descriptor head pointer
- * @ts: Tx completion status
- * @txrx_peer: peer handle
- * @ring_id: ring number
- * @link_id: Link id
- *
- * Return: None
- */
- static inline void
- dp_tx_update_peer_stats(struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer, uint8_t ring_id,
- uint8_t link_id)
- {
- struct dp_pdev *pdev = txrx_peer->vdev->pdev;
- uint8_t tid = ts->tid;
- uint32_t length;
- struct cdp_tid_tx_stats *tid_stats;
- if (!pdev)
- return;
- if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
- tid = CDP_MAX_DATA_TIDS - 1;
- tid_stats = &pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
- if (ts->release_src != HAL_TX_COMP_RELEASE_SOURCE_TQM) {
- dp_err_rl("Release source:%d is not from TQM", ts->release_src);
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.release_src_not_tqm, 1,
- link_id);
- return;
- }
- length = qdf_nbuf_len(tx_desc->nbuf);
- DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
- if (qdf_unlikely(pdev->delay_stats_flag) ||
- qdf_unlikely(dp_is_vdev_tx_delay_stats_enabled(txrx_peer->vdev)))
- dp_tx_compute_delay(txrx_peer->vdev, tx_desc, tid, ring_id);
- if (ts->status < CDP_MAX_TX_TQM_STATUS) {
- tid_stats->tqm_status_cnt[ts->status]++;
- }
- if (qdf_likely(ts->status == HAL_TX_TQM_RR_FRAME_ACKED)) {
- DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.retry_count, 1,
- ts->transmit_cnt > 1, link_id);
- DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.multiple_retry_count,
- 1, ts->transmit_cnt > 2, link_id);
- DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.ofdma, 1, ts->ofdma,
- link_id);
- DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.amsdu_cnt, 1,
- ts->msdu_part_of_amsdu, link_id);
- DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.non_amsdu_cnt, 1,
- !ts->msdu_part_of_amsdu, link_id);
- txrx_peer->stats[link_id].per_pkt_stats.tx.last_tx_ts =
- qdf_system_ticks();
- dp_tx_update_peer_extd_stats(ts, txrx_peer, link_id);
- return;
- }
- /*
- * tx_failed is ideally supposed to be updated from HTT ppdu
- * completion stats. But in IPQ807X/IPQ6018 chipsets owing to
- * hw limitation there are no completions for failed cases.
- * Hence updating tx_failed from data path. Please note that
- * if tx_failed is fixed to be from ppdu, then this has to be
- * removed
- */
- DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
- DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.failed_retry_count, 1,
- ts->transmit_cnt > DP_RETRY_COUNT,
- link_id);
- dp_update_no_ack_stats(tx_desc->nbuf, txrx_peer, link_id);
- if (ts->status == HAL_TX_TQM_RR_REM_CMD_AGED) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.age_out, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_REM) {
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.dropped.fw_rem, 1,
- length, link_id);
- } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_NOTX) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_rem_notx, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_TX) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_rem_tx, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_FW_REASON1) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason1, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_FW_REASON2) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason2, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_FW_REASON3) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason3, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_DISABLE_QUEUE) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.dropped.fw_rem_queue_disable, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_TILL_NONMATCHING) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.dropped.fw_rem_no_match, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_DROP_THRESHOLD) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.dropped.drop_threshold, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_LINK_DESC_UNAVAILABLE) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.dropped.drop_link_desc_na, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_DROP_OR_INVALID_MSDU) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.dropped.invalid_drop, 1,
- link_id);
- } else if (ts->status == HAL_TX_TQM_RR_MULTICAST_DROP) {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer,
- tx.dropped.mcast_vdev_drop, 1,
- link_id);
- } else {
- DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.invalid_rr, 1,
- link_id);
- }
- }
- #ifdef QCA_LL_TX_FLOW_CONTROL_V2
- /**
- * dp_tx_flow_pool_lock() - take flow pool lock
- * @soc: core txrx main context
- * @tx_desc: tx desc
- *
- * Return: None
- */
- static inline
- void dp_tx_flow_pool_lock(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc)
- {
- struct dp_tx_desc_pool_s *pool;
- uint8_t desc_pool_id;
- desc_pool_id = tx_desc->pool_id;
- pool = &soc->tx_desc[desc_pool_id];
- qdf_spin_lock_bh(&pool->flow_pool_lock);
- }
- /**
- * dp_tx_flow_pool_unlock() - release flow pool lock
- * @soc: core txrx main context
- * @tx_desc: tx desc
- *
- * Return: None
- */
- static inline
- void dp_tx_flow_pool_unlock(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc)
- {
- struct dp_tx_desc_pool_s *pool;
- uint8_t desc_pool_id;
- desc_pool_id = tx_desc->pool_id;
- pool = &soc->tx_desc[desc_pool_id];
- qdf_spin_unlock_bh(&pool->flow_pool_lock);
- }
- #else
- static inline
- void dp_tx_flow_pool_lock(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
- {
- }
- static inline
- void dp_tx_flow_pool_unlock(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
- {
- }
- #endif
- /**
- * dp_tx_notify_completion() - Notify tx completion for this desc
- * @soc: core txrx main context
- * @vdev: datapath vdev handle
- * @tx_desc: tx desc
- * @netbuf: buffer
- * @status: tx status
- *
- * Return: none
- */
- static inline void dp_tx_notify_completion(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- qdf_nbuf_t netbuf,
- uint8_t status)
- {
- void *osif_dev;
- ol_txrx_completion_fp tx_compl_cbk = NULL;
- uint16_t flag = BIT(QDF_TX_RX_STATUS_DOWNLOAD_SUCC);
- qdf_assert(tx_desc);
- if (!vdev ||
- !vdev->osif_vdev) {
- return;
- }
- osif_dev = vdev->osif_vdev;
- tx_compl_cbk = vdev->tx_comp;
- if (status == HAL_TX_TQM_RR_FRAME_ACKED)
- flag |= BIT(QDF_TX_RX_STATUS_OK);
- if (tx_compl_cbk)
- tx_compl_cbk(netbuf, osif_dev, flag);
- }
- /**
- * dp_tx_sojourn_stats_process() - Collect sojourn stats
- * @pdev: pdev handle
- * @txrx_peer: DP peer context
- * @tid: tid value
- * @txdesc_ts: timestamp from txdesc
- * @ppdu_id: ppdu id
- * @link_id: link id
- *
- * Return: none
- */
- #ifdef FEATURE_PERPKT_INFO
- static inline void dp_tx_sojourn_stats_process(struct dp_pdev *pdev,
- struct dp_txrx_peer *txrx_peer,
- uint8_t tid,
- uint64_t txdesc_ts,
- uint32_t ppdu_id,
- uint8_t link_id)
- {
- uint64_t delta_ms;
- struct cdp_tx_sojourn_stats *sojourn_stats;
- struct dp_peer *primary_link_peer = NULL;
- struct dp_soc *link_peer_soc = NULL;
- if (qdf_unlikely(!pdev->enhanced_stats_en))
- return;
- if (qdf_unlikely(tid == HTT_INVALID_TID ||
- tid >= CDP_DATA_TID_MAX))
- return;
- if (qdf_unlikely(!pdev->sojourn_buf))
- return;
- primary_link_peer = dp_get_primary_link_peer_by_id(pdev->soc,
- txrx_peer->peer_id,
- DP_MOD_ID_TX_COMP);
- if (qdf_unlikely(!primary_link_peer))
- return;
- sojourn_stats = (struct cdp_tx_sojourn_stats *)
- qdf_nbuf_data(pdev->sojourn_buf);
- link_peer_soc = primary_link_peer->vdev->pdev->soc;
- sojourn_stats->cookie = (void *)
- dp_monitor_peer_get_peerstats_ctx(link_peer_soc,
- primary_link_peer);
- delta_ms = qdf_ktime_to_ms(qdf_ktime_real_get()) -
- txdesc_ts;
- qdf_ewma_tx_lag_add(&txrx_peer->stats[link_id].per_pkt_stats.tx.avg_sojourn_msdu[tid],
- delta_ms);
- sojourn_stats->sum_sojourn_msdu[tid] = delta_ms;
- sojourn_stats->num_msdus[tid] = 1;
- sojourn_stats->avg_sojourn_msdu[tid].internal =
- txrx_peer->stats[link_id].
- per_pkt_stats.tx.avg_sojourn_msdu[tid].internal;
- dp_wdi_event_handler(WDI_EVENT_TX_SOJOURN_STAT, pdev->soc,
- pdev->sojourn_buf, HTT_INVALID_PEER,
- WDI_NO_VAL, pdev->pdev_id);
- sojourn_stats->sum_sojourn_msdu[tid] = 0;
- sojourn_stats->num_msdus[tid] = 0;
- sojourn_stats->avg_sojourn_msdu[tid].internal = 0;
- dp_peer_unref_delete(primary_link_peer, DP_MOD_ID_TX_COMP);
- }
- #else
- static inline void dp_tx_sojourn_stats_process(struct dp_pdev *pdev,
- struct dp_txrx_peer *txrx_peer,
- uint8_t tid,
- uint64_t txdesc_ts,
- uint32_t ppdu_id)
- {
- }
- #endif
- #ifdef WLAN_FEATURE_PKT_CAPTURE_V2
- void dp_send_completion_to_pkt_capture(struct dp_soc *soc,
- struct dp_tx_desc_s *desc,
- struct hal_tx_completion_status *ts)
- {
- dp_wdi_event_handler(WDI_EVENT_PKT_CAPTURE_TX_DATA, soc,
- desc, ts->peer_id,
- WDI_NO_VAL, desc->pdev->pdev_id);
- }
- #endif
- void
- dp_tx_comp_process_desc(struct dp_soc *soc,
- struct dp_tx_desc_s *desc,
- struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer)
- {
- uint64_t time_latency = 0;
- uint16_t peer_id = DP_INVALID_PEER_ID;
- /*
- * m_copy/tx_capture modes are not supported for
- * scatter gather packets
- */
- if (qdf_unlikely(!!desc->pdev->latency_capture_enable)) {
- time_latency = (qdf_ktime_to_ms(qdf_ktime_real_get()) -
- qdf_ktime_to_ms(desc->timestamp));
- }
- dp_send_completion_to_pkt_capture(soc, desc, ts);
- if (dp_tx_pkt_tracepoints_enabled())
- qdf_trace_dp_packet(desc->nbuf, QDF_TX,
- desc->msdu_ext_desc ?
- desc->msdu_ext_desc->tso_desc : NULL,
- qdf_ktime_to_us(desc->timestamp));
- if (!(desc->msdu_ext_desc)) {
- dp_tx_enh_unmap(soc, desc);
- if (txrx_peer)
- peer_id = txrx_peer->peer_id;
- if (QDF_STATUS_SUCCESS ==
- dp_monitor_tx_add_to_comp_queue(soc, desc, ts, peer_id)) {
- return;
- }
- if (QDF_STATUS_SUCCESS ==
- dp_get_completion_indication_for_stack(soc,
- desc->pdev,
- txrx_peer, ts,
- desc->nbuf,
- time_latency)) {
- dp_send_completion_to_stack(soc,
- desc->pdev,
- ts->peer_id,
- ts->ppdu_id,
- desc->nbuf);
- return;
- }
- }
- desc->flags |= DP_TX_DESC_FLAG_COMPLETED_TX;
- dp_tx_comp_free_buf(soc, desc, false);
- }
- #ifdef DISABLE_DP_STATS
- /**
- * dp_tx_update_connectivity_stats() - update tx connectivity stats
- * @soc: core txrx main context
- * @vdev: virtual device instance
- * @tx_desc: tx desc
- * @status: tx status
- *
- * Return: none
- */
- static inline
- void dp_tx_update_connectivity_stats(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- uint8_t status)
- {
- }
- #else
- static inline
- void dp_tx_update_connectivity_stats(struct dp_soc *soc,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc,
- uint8_t status)
- {
- void *osif_dev;
- ol_txrx_stats_rx_fp stats_cbk;
- uint8_t pkt_type;
- qdf_assert(tx_desc);
- if (!vdev || vdev->delete.pending || !vdev->osif_vdev ||
- !vdev->stats_cb)
- return;
- osif_dev = vdev->osif_vdev;
- stats_cbk = vdev->stats_cb;
- stats_cbk(tx_desc->nbuf, osif_dev, PKT_TYPE_TX_HOST_FW_SENT, &pkt_type);
- if (status == HAL_TX_TQM_RR_FRAME_ACKED)
- stats_cbk(tx_desc->nbuf, osif_dev, PKT_TYPE_TX_ACK_CNT,
- &pkt_type);
- }
- #endif
- #if defined(WLAN_FEATURE_TSF_AUTO_REPORT) || defined(WLAN_CONFIG_TX_DELAY)
- /* Mask for bit29 ~ bit31 */
- #define DP_TX_TS_BIT29_31_MASK 0xE0000000
- /* Timestamp value (unit us) if bit29 is set */
- #define DP_TX_TS_BIT29_SET_VALUE BIT(29)
- /**
- * dp_tx_adjust_enqueue_buffer_ts() - adjust the enqueue buffer_timestamp
- * @ack_ts: OTA ack timestamp, unit us.
- * @enqueue_ts: TCL enqueue TX data to TQM timestamp, unit us.
- * @base_delta_ts: base timestamp delta for ack_ts and enqueue_ts
- *
- * this function will restore the bit29 ~ bit31 3 bits value for
- * buffer_timestamp in wbm2sw ring entry, currently buffer_timestamp only
- * can support 0x7FFF * 1024 us (29 bits), but if the timestamp is >
- * 0x7FFF * 1024 us, bit29~ bit31 will be lost.
- *
- * Return: the adjusted buffer_timestamp value
- */
- static inline
- uint32_t dp_tx_adjust_enqueue_buffer_ts(uint32_t ack_ts,
- uint32_t enqueue_ts,
- uint32_t base_delta_ts)
- {
- uint32_t ack_buffer_ts;
- uint32_t ack_buffer_ts_bit29_31;
- uint32_t adjusted_enqueue_ts;
- /* corresponding buffer_timestamp value when receive OTA Ack */
- ack_buffer_ts = ack_ts - base_delta_ts;
- ack_buffer_ts_bit29_31 = ack_buffer_ts & DP_TX_TS_BIT29_31_MASK;
- /* restore the bit29 ~ bit31 value */
- adjusted_enqueue_ts = ack_buffer_ts_bit29_31 | enqueue_ts;
- /*
- * if actual enqueue_ts value occupied 29 bits only, this enqueue_ts
- * value + real UL delay overflow 29 bits, then 30th bit (bit-29)
- * should not be marked, otherwise extra 0x20000000 us is added to
- * enqueue_ts.
- */
- if (qdf_unlikely(adjusted_enqueue_ts > ack_buffer_ts))
- adjusted_enqueue_ts -= DP_TX_TS_BIT29_SET_VALUE;
- return adjusted_enqueue_ts;
- }
- QDF_STATUS
- dp_tx_compute_hw_delay_us(struct hal_tx_completion_status *ts,
- uint32_t delta_tsf,
- uint32_t *delay_us)
- {
- uint32_t buffer_ts;
- uint32_t delay;
- if (!delay_us)
- return QDF_STATUS_E_INVAL;
- /* Tx_rate_stats_info_valid is 0 and tsf is invalid then */
- if (!ts->valid)
- return QDF_STATUS_E_INVAL;
- /* buffer_timestamp is in units of 1024 us and is [31:13] of
- * WBM_RELEASE_RING_4. After left shift 10 bits, it's
- * valid up to 29 bits.
- */
- buffer_ts = ts->buffer_timestamp << 10;
- buffer_ts = dp_tx_adjust_enqueue_buffer_ts(ts->tsf,
- buffer_ts, delta_tsf);
- delay = ts->tsf - buffer_ts - delta_tsf;
- if (qdf_unlikely(delay & 0x80000000)) {
- dp_err_rl("delay = 0x%x (-ve)\n"
- "release_src = %d\n"
- "ppdu_id = 0x%x\n"
- "peer_id = 0x%x\n"
- "tid = 0x%x\n"
- "release_reason = %d\n"
- "tsf = %u (0x%x)\n"
- "buffer_timestamp = %u (0x%x)\n"
- "delta_tsf = %u (0x%x)\n",
- delay, ts->release_src, ts->ppdu_id, ts->peer_id,
- ts->tid, ts->status, ts->tsf, ts->tsf,
- ts->buffer_timestamp, ts->buffer_timestamp,
- delta_tsf, delta_tsf);
- delay = 0;
- goto end;
- }
- delay &= 0x1FFFFFFF; /* mask 29 BITS */
- if (delay > 0x1000000) {
- dp_info_rl("----------------------\n"
- "Tx completion status:\n"
- "----------------------\n"
- "release_src = %d\n"
- "ppdu_id = 0x%x\n"
- "release_reason = %d\n"
- "tsf = %u (0x%x)\n"
- "buffer_timestamp = %u (0x%x)\n"
- "delta_tsf = %u (0x%x)\n",
- ts->release_src, ts->ppdu_id, ts->status,
- ts->tsf, ts->tsf, ts->buffer_timestamp,
- ts->buffer_timestamp, delta_tsf, delta_tsf);
- return QDF_STATUS_E_FAILURE;
- }
- end:
- *delay_us = delay;
- return QDF_STATUS_SUCCESS;
- }
- void dp_set_delta_tsf(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- uint32_t delta_tsf)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
- DP_MOD_ID_CDP);
- if (!vdev) {
- dp_err_rl("vdev %d does not exist", vdev_id);
- return;
- }
- vdev->delta_tsf = delta_tsf;
- dp_debug("vdev id %u delta_tsf %u", vdev_id, delta_tsf);
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
- }
- #endif
- #ifdef WLAN_FEATURE_TSF_UPLINK_DELAY
- QDF_STATUS dp_set_tsf_ul_delay_report(struct cdp_soc_t *soc_hdl,
- uint8_t vdev_id, bool enable)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
- DP_MOD_ID_CDP);
- if (!vdev) {
- dp_err_rl("vdev %d does not exist", vdev_id);
- return QDF_STATUS_E_FAILURE;
- }
- qdf_atomic_set(&vdev->ul_delay_report, enable);
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
- return QDF_STATUS_SUCCESS;
- }
- QDF_STATUS dp_get_uplink_delay(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- uint32_t *val)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev;
- uint32_t delay_accum;
- uint32_t pkts_accum;
- vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_CDP);
- if (!vdev) {
- dp_err_rl("vdev %d does not exist", vdev_id);
- return QDF_STATUS_E_FAILURE;
- }
- if (!qdf_atomic_read(&vdev->ul_delay_report)) {
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
- return QDF_STATUS_E_FAILURE;
- }
- /* Average uplink delay based on current accumulated values */
- delay_accum = qdf_atomic_read(&vdev->ul_delay_accum);
- pkts_accum = qdf_atomic_read(&vdev->ul_pkts_accum);
- *val = delay_accum / pkts_accum;
- dp_debug("uplink_delay %u delay_accum %u pkts_accum %u", *val,
- delay_accum, pkts_accum);
- /* Reset accumulated values to 0 */
- qdf_atomic_set(&vdev->ul_delay_accum, 0);
- qdf_atomic_set(&vdev->ul_pkts_accum, 0);
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
- return QDF_STATUS_SUCCESS;
- }
- static void dp_tx_update_uplink_delay(struct dp_soc *soc, struct dp_vdev *vdev,
- struct hal_tx_completion_status *ts)
- {
- uint32_t ul_delay;
- if (qdf_unlikely(!vdev)) {
- dp_info_rl("vdev is null or delete in progress");
- return;
- }
- if (!qdf_atomic_read(&vdev->ul_delay_report))
- return;
- if (QDF_IS_STATUS_ERROR(dp_tx_compute_hw_delay_us(ts,
- vdev->delta_tsf,
- &ul_delay)))
- return;
- ul_delay /= 1000; /* in unit of ms */
- qdf_atomic_add(ul_delay, &vdev->ul_delay_accum);
- qdf_atomic_inc(&vdev->ul_pkts_accum);
- }
- #else /* !WLAN_FEATURE_TSF_UPLINK_DELAY */
- static inline
- void dp_tx_update_uplink_delay(struct dp_soc *soc, struct dp_vdev *vdev,
- struct hal_tx_completion_status *ts)
- {
- }
- #endif /* WLAN_FEATURE_TSF_UPLINK_DELAY */
- #ifndef CONFIG_AP_PLATFORM
- /**
- * dp_update_mcast_stats() - Update Tx Mcast stats
- * @txrx_peer: txrx_peer pointer
- * @link_id: Link ID
- * @length: packet length
- * @nbuf: nbuf handle
- *
- * Return: None
- */
- static inline void
- dp_update_mcast_stats(struct dp_txrx_peer *txrx_peer, uint8_t link_id,
- uint32_t length, qdf_nbuf_t nbuf)
- {
- if (QDF_NBUF_CB_GET_IS_MCAST(nbuf))
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.mcast, 1,
- length, link_id);
- }
- #else
- static inline void
- dp_update_mcast_stats(struct dp_txrx_peer *txrx_peer, uint8_t link_id,
- uint32_t length, qdf_nbuf_t nbuf)
- {
- }
- #endif
- #if defined(WLAN_FEATURE_11BE_MLO) && defined(DP_MLO_LINK_STATS_SUPPORT)
- /**
- * dp_tx_comp_set_nbuf_band() - set nbuf band.
- * @soc: dp soc handle
- * @nbuf: nbuf handle
- * @ts: tx completion status
- *
- * Return: None
- */
- static inline void
- dp_tx_comp_set_nbuf_band(struct dp_soc *soc, qdf_nbuf_t nbuf,
- struct hal_tx_completion_status *ts)
- {
- struct qdf_mac_addr *mac_addr;
- struct dp_peer *peer;
- struct dp_txrx_peer *txrx_peer;
- uint8_t link_id;
- if ((QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) !=
- QDF_NBUF_CB_PACKET_TYPE_EAPOL &&
- QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) !=
- QDF_NBUF_CB_PACKET_TYPE_DHCP &&
- QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) !=
- QDF_NBUF_CB_PACKET_TYPE_DHCPV6) ||
- QDF_NBUF_CB_GET_IS_BCAST(nbuf))
- return;
- mac_addr = (struct qdf_mac_addr *)(qdf_nbuf_data(nbuf) +
- QDF_NBUF_DEST_MAC_OFFSET);
- peer = dp_mld_peer_find_hash_find(soc, mac_addr->bytes, 0,
- DP_VDEV_ALL, DP_MOD_ID_TX_COMP);
- if (qdf_likely(peer)) {
- txrx_peer = dp_get_txrx_peer(peer);
- if (qdf_likely(txrx_peer)) {
- link_id =
- dp_tx_get_link_id_from_ppdu_id(soc, ts,
- txrx_peer,
- txrx_peer->vdev);
- qdf_nbuf_tx_set_band(nbuf, txrx_peer->ll_band[link_id]);
- }
- dp_peer_unref_delete(peer, DP_MOD_ID_TX_COMP);
- }
- }
- #else
- static inline void
- dp_tx_comp_set_nbuf_band(struct dp_soc *soc, qdf_nbuf_t nbuf,
- struct hal_tx_completion_status *ts)
- {
- }
- #endif
- void dp_tx_comp_process_tx_status(struct dp_soc *soc,
- struct dp_tx_desc_s *tx_desc,
- struct hal_tx_completion_status *ts,
- struct dp_txrx_peer *txrx_peer,
- uint8_t ring_id)
- {
- uint32_t length;
- qdf_ether_header_t *eh;
- struct dp_vdev *vdev = NULL;
- qdf_nbuf_t nbuf = tx_desc->nbuf;
- enum qdf_dp_tx_rx_status dp_status;
- uint8_t link_id = 0;
- enum QDF_OPMODE op_mode = QDF_MAX_NO_OF_MODE;
- if (!nbuf) {
- dp_info_rl("invalid tx descriptor. nbuf NULL");
- goto out;
- }
- eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
- length = dp_tx_get_pkt_len(tx_desc);
- dp_status = dp_tx_hw_to_qdf(ts->status);
- if (soc->dp_debug_log_en) {
- dp_tx_comp_debug("--------------------\n"
- "Tx Completion Stats:\n"
- "--------------------\n"
- "ack_frame_rssi = %d\n"
- "first_msdu = %d\n"
- "last_msdu = %d\n"
- "msdu_part_of_amsdu = %d\n"
- "rate_stats valid = %d\n"
- "bw = %d\n"
- "pkt_type = %d\n"
- "stbc = %d\n"
- "ldpc = %d\n"
- "sgi = %d\n"
- "mcs = %d\n"
- "ofdma = %d\n"
- "tones_in_ru = %d\n"
- "tsf = %d\n"
- "ppdu_id = %d\n"
- "transmit_cnt = %d\n"
- "tid = %d\n"
- "peer_id = %d\n"
- "tx_status = %d\n"
- "tx_release_source = %d\n",
- ts->ack_frame_rssi, ts->first_msdu,
- ts->last_msdu, ts->msdu_part_of_amsdu,
- ts->valid, ts->bw, ts->pkt_type, ts->stbc,
- ts->ldpc, ts->sgi, ts->mcs, ts->ofdma,
- ts->tones_in_ru, ts->tsf, ts->ppdu_id,
- ts->transmit_cnt, ts->tid, ts->peer_id,
- ts->status, ts->release_src);
- }
- /* Update SoC level stats */
- DP_STATS_INCC(soc, tx.dropped_fw_removed, 1,
- (ts->status == HAL_TX_TQM_RR_REM_CMD_REM));
- if (!txrx_peer) {
- dp_tx_comp_set_nbuf_band(soc, nbuf, ts);
- dp_info_rl("peer is null or deletion in progress");
- DP_STATS_INC_PKT(soc, tx.tx_invalid_peer, 1, length);
- vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id,
- DP_MOD_ID_CDP);
- if (qdf_likely(vdev)) {
- op_mode = vdev->qdf_opmode;
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
- }
- goto out_log;
- }
- vdev = txrx_peer->vdev;
- link_id = dp_tx_get_link_id_from_ppdu_id(soc, ts, txrx_peer, vdev);
- dp_tx_set_nbuf_band(nbuf, txrx_peer, link_id);
- op_mode = vdev->qdf_opmode;
- dp_tx_update_connectivity_stats(soc, vdev, tx_desc, ts->status);
- dp_tx_update_uplink_delay(soc, vdev, ts);
- /* check tx complete notification */
- if (qdf_nbuf_tx_notify_comp_get(nbuf))
- dp_tx_notify_completion(soc, vdev, tx_desc,
- nbuf, ts->status);
- /* Update per-packet stats for mesh mode */
- if (qdf_unlikely(vdev->mesh_vdev) &&
- !(tx_desc->flags & DP_TX_DESC_FLAG_TO_FW))
- dp_tx_comp_fill_tx_completion_stats(tx_desc, ts);
- /* Update peer level stats */
- if (qdf_unlikely(txrx_peer->bss_peer &&
- vdev->opmode == wlan_op_mode_ap)) {
- if (ts->status != HAL_TX_TQM_RR_REM_CMD_REM) {
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.mcast, 1,
- length, link_id);
- if (txrx_peer->vdev->tx_encap_type ==
- htt_cmn_pkt_type_ethernet &&
- QDF_IS_ADDR_BROADCAST(eh->ether_dhost)) {
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
- tx.bcast, 1,
- length, link_id);
- }
- }
- } else {
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.ucast, 1, length,
- link_id);
- if (ts->status == HAL_TX_TQM_RR_FRAME_ACKED) {
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.tx_success,
- 1, length, link_id);
- if (qdf_unlikely(txrx_peer->in_twt)) {
- DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
- tx.tx_success_twt,
- 1, length,
- link_id);
- }
- dp_update_mcast_stats(txrx_peer, link_id, length, nbuf);
- }
- }
- dp_tx_update_peer_stats(tx_desc, ts, txrx_peer, ring_id, link_id);
- dp_tx_update_peer_delay_stats(txrx_peer, tx_desc, ts, ring_id);
- dp_tx_update_peer_jitter_stats(txrx_peer, tx_desc, ts, ring_id);
- dp_tx_update_peer_sawf_stats(soc, vdev, txrx_peer, tx_desc,
- ts, ts->tid);
- dp_tx_send_pktlog(soc, vdev->pdev, tx_desc, nbuf, dp_status);
- dp_tx_latency_stats_update(soc, txrx_peer, tx_desc, ts, link_id);
- #ifdef QCA_SUPPORT_RDK_STATS
- if (soc->peerstats_enabled)
- dp_tx_sojourn_stats_process(vdev->pdev, txrx_peer, ts->tid,
- qdf_ktime_to_ms(tx_desc->timestamp),
- ts->ppdu_id, link_id);
- #endif
- out_log:
- DPTRACE(qdf_dp_trace_ptr(tx_desc->nbuf,
- QDF_DP_TRACE_LI_DP_FREE_PACKET_PTR_RECORD,
- QDF_TRACE_DEFAULT_PDEV_ID,
- qdf_nbuf_data_addr(nbuf),
- sizeof(qdf_nbuf_data(nbuf)),
- tx_desc->id, ts->status, dp_status, op_mode));
- out:
- return;
- }
- #if defined(QCA_VDEV_STATS_HW_OFFLOAD_SUPPORT) && \
- defined(QCA_ENHANCED_STATS_SUPPORT)
- void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
- uint32_t length, uint8_t tx_status,
- bool update)
- {
- if (update || (!txrx_peer->hw_txrx_stats_en)) {
- DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
- if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
- DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
- }
- }
- #elif defined(QCA_VDEV_STATS_HW_OFFLOAD_SUPPORT)
- void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
- uint32_t length, uint8_t tx_status,
- bool update)
- {
- if (!txrx_peer->hw_txrx_stats_en) {
- DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
- if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
- DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
- }
- }
- #else
- void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
- uint32_t length, uint8_t tx_status,
- bool update)
- {
- DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
- if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
- DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
- }
- #endif
- /**
- * dp_tx_prefetch_next_nbuf_data(): Prefetch nbuf and nbuf data
- * @next: descriptor of the nrxt buffer
- *
- * Return: none
- */
- #ifdef QCA_DP_RX_NBUF_AND_NBUF_DATA_PREFETCH
- static inline
- void dp_tx_prefetch_next_nbuf_data(struct dp_tx_desc_s *next)
- {
- qdf_nbuf_t nbuf = NULL;
- if (next)
- nbuf = next->nbuf;
- if (nbuf)
- qdf_prefetch(nbuf);
- }
- #else
- static inline
- void dp_tx_prefetch_next_nbuf_data(struct dp_tx_desc_s *next)
- {
- }
- #endif
- /**
- * dp_tx_mcast_reinject_handler() - Tx reinjected multicast packets handler
- * @soc: core txrx main context
- * @desc: software descriptor
- *
- * Return: true when packet is reinjected
- */
- #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
- defined(WLAN_MCAST_MLO) && !defined(CONFIG_MLO_SINGLE_DEV)
- static inline bool
- dp_tx_mcast_reinject_handler(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- struct dp_vdev *vdev = NULL;
- uint8_t xmit_type;
- if (desc->tx_status == HAL_TX_TQM_RR_MULTICAST_DROP) {
- if (!soc->arch_ops.dp_tx_mcast_handler ||
- !soc->arch_ops.dp_tx_is_mcast_primary)
- return false;
- vdev = dp_vdev_get_ref_by_id(soc, desc->vdev_id,
- DP_MOD_ID_REINJECT);
- if (qdf_unlikely(!vdev)) {
- dp_tx_comp_info_rl("Unable to get vdev ref %d",
- desc->id);
- return false;
- }
- if (!(soc->arch_ops.dp_tx_is_mcast_primary(soc, vdev))) {
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_REINJECT);
- return false;
- }
- xmit_type = qdf_nbuf_get_vdev_xmit_type(desc->nbuf);
- DP_STATS_INC_PKT(vdev, tx_i[xmit_type].reinject_pkts, 1,
- qdf_nbuf_len(desc->nbuf));
- soc->arch_ops.dp_tx_mcast_handler(soc, vdev, desc->nbuf);
- dp_tx_desc_release(soc, desc, desc->pool_id);
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_REINJECT);
- return true;
- }
- return false;
- }
- #else
- static inline bool
- dp_tx_mcast_reinject_handler(struct dp_soc *soc, struct dp_tx_desc_s *desc)
- {
- return false;
- }
- #endif
- #ifdef QCA_DP_TX_NBUF_LIST_FREE
- static inline void
- dp_tx_nbuf_queue_head_init(qdf_nbuf_queue_head_t *nbuf_queue_head)
- {
- qdf_nbuf_queue_head_init(nbuf_queue_head);
- }
- static inline void
- dp_tx_nbuf_dev_queue_free(qdf_nbuf_queue_head_t *nbuf_queue_head,
- struct dp_tx_desc_s *desc)
- {
- qdf_nbuf_t nbuf = NULL;
- nbuf = desc->nbuf;
- if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_FAST))
- qdf_nbuf_dev_queue_head(nbuf_queue_head, nbuf);
- else
- qdf_nbuf_free(nbuf);
- }
- static inline void
- dp_tx_nbuf_dev_queue_free_no_flag(qdf_nbuf_queue_head_t *nbuf_queue_head,
- qdf_nbuf_t nbuf)
- {
- if (!nbuf)
- return;
- if (nbuf->is_from_recycler)
- qdf_nbuf_dev_queue_head(nbuf_queue_head, nbuf);
- else
- qdf_nbuf_free(nbuf);
- }
- static inline void
- dp_tx_nbuf_dev_kfree_list(qdf_nbuf_queue_head_t *nbuf_queue_head)
- {
- qdf_nbuf_dev_kfree_list(nbuf_queue_head);
- }
- #else
- static inline void
- dp_tx_nbuf_queue_head_init(qdf_nbuf_queue_head_t *nbuf_queue_head)
- {
- }
- static inline void
- dp_tx_nbuf_dev_queue_free(qdf_nbuf_queue_head_t *nbuf_queue_head,
- struct dp_tx_desc_s *desc)
- {
- qdf_nbuf_free(desc->nbuf);
- }
- static inline void
- dp_tx_nbuf_dev_queue_free_no_flag(qdf_nbuf_queue_head_t *nbuf_queue_head,
- qdf_nbuf_t nbuf)
- {
- qdf_nbuf_free(nbuf);
- }
- static inline void
- dp_tx_nbuf_dev_kfree_list(qdf_nbuf_queue_head_t *nbuf_queue_head)
- {
- }
- #endif
- #ifdef WLAN_SUPPORT_PPEDS
- static inline void
- dp_tx_update_ppeds_tx_comp_stats(struct dp_soc *soc,
- struct dp_txrx_peer *txrx_peer,
- struct hal_tx_completion_status *ts,
- struct dp_tx_desc_s *desc,
- uint8_t ring_id)
- {
- uint8_t link_id = 0;
- struct dp_vdev *vdev = NULL;
- if (qdf_likely(txrx_peer)) {
- if (!(desc->flags & DP_TX_DESC_FLAG_SIMPLE)) {
- hal_tx_comp_get_status(&desc->comp,
- ts,
- soc->hal_soc);
- vdev = txrx_peer->vdev;
- link_id = dp_tx_get_link_id_from_ppdu_id(soc,
- ts,
- txrx_peer,
- vdev);
- if (link_id < 1 || link_id > DP_MAX_MLO_LINKS)
- link_id = 0;
- dp_tx_update_peer_stats(desc, ts,
- txrx_peer,
- ring_id,
- link_id);
- } else {
- dp_tx_update_peer_basic_stats(txrx_peer, desc->length,
- desc->tx_status, false);
- }
- }
- }
- #else
- static inline void
- dp_tx_update_ppeds_tx_comp_stats(struct dp_soc *soc,
- struct dp_txrx_peer *txrx_peer,
- struct hal_tx_completion_status *ts,
- struct dp_tx_desc_s *desc,
- uint8_t ring_id)
- {
- }
- #endif
- void
- dp_tx_comp_process_desc_list_fast(struct dp_soc *soc,
- struct dp_tx_desc_s *head_desc,
- struct dp_tx_desc_s *tail_desc,
- uint8_t ring_id,
- uint32_t fast_desc_count)
- {
- struct dp_tx_desc_pool_s *pool = NULL;
- pool = dp_get_tx_desc_pool(soc, head_desc->pool_id);
- dp_tx_outstanding_sub(head_desc->pdev, fast_desc_count);
- dp_tx_desc_free_list(pool, head_desc, tail_desc, fast_desc_count);
- }
- void
- dp_tx_comp_process_desc_list(struct dp_soc *soc,
- struct dp_tx_desc_s *comp_head, uint8_t ring_id)
- {
- struct dp_tx_desc_s *desc;
- struct dp_tx_desc_s *next;
- struct hal_tx_completion_status ts;
- struct dp_txrx_peer *txrx_peer = NULL;
- uint16_t peer_id = DP_INVALID_PEER;
- dp_txrx_ref_handle txrx_ref_handle = NULL;
- qdf_nbuf_queue_head_t h;
- desc = comp_head;
- dp_tx_nbuf_queue_head_init(&h);
- while (desc) {
- next = desc->next;
- dp_tx_prefetch_next_nbuf_data(next);
- if (peer_id != desc->peer_id) {
- if (txrx_peer)
- dp_txrx_peer_unref_delete(txrx_ref_handle,
- DP_MOD_ID_TX_COMP);
- peer_id = desc->peer_id;
- txrx_peer =
- dp_txrx_peer_get_ref_by_id(soc, peer_id,
- &txrx_ref_handle,
- DP_MOD_ID_TX_COMP);
- }
- if (dp_tx_mcast_reinject_handler(soc, desc)) {
- desc = next;
- continue;
- }
- if (desc->flags & DP_TX_DESC_FLAG_PPEDS) {
- qdf_nbuf_t nbuf;
- dp_tx_update_ppeds_tx_comp_stats(soc, txrx_peer, &ts,
- desc, ring_id);
- if (desc->pool_id != DP_TX_PPEDS_POOL_ID) {
- nbuf = desc->nbuf;
- dp_tx_nbuf_dev_queue_free_no_flag(&h, nbuf);
- if (desc->flags & DP_TX_DESC_FLAG_SPECIAL)
- dp_tx_spcl_desc_free(soc, desc,
- desc->pool_id);
- else
- dp_tx_desc_free(soc, desc,
- desc->pool_id);
- __dp_tx_outstanding_dec(soc);
- } else {
- nbuf = dp_ppeds_tx_desc_free(soc, desc);
- dp_tx_nbuf_dev_queue_free_no_flag(&h, nbuf);
- }
- desc = next;
- continue;
- }
- if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_SIMPLE)) {
- struct dp_pdev *pdev = desc->pdev;
- if (qdf_likely(txrx_peer))
- dp_tx_update_peer_basic_stats(txrx_peer,
- desc->length,
- desc->tx_status,
- false);
- qdf_assert(pdev);
- dp_tx_outstanding_dec(pdev);
- /*
- * Calling a QDF WRAPPER here is creating significant
- * performance impact so avoided the wrapper call here
- */
- dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf,
- desc->id, DP_TX_COMP_UNMAP);
- dp_tx_nbuf_unmap(soc, desc);
- dp_tx_nbuf_dev_queue_free(&h, desc);
- dp_tx_desc_free(soc, desc, desc->pool_id);
- desc = next;
- continue;
- }
- hal_tx_comp_get_status(&desc->comp, &ts, soc->hal_soc);
- dp_tx_comp_process_tx_status(soc, desc, &ts, txrx_peer,
- ring_id);
- dp_tx_comp_process_desc(soc, desc, &ts, txrx_peer);
- dp_tx_desc_release(soc, desc, desc->pool_id);
- desc = next;
- }
- dp_tx_nbuf_dev_kfree_list(&h);
- if (txrx_peer)
- dp_txrx_peer_unref_delete(txrx_ref_handle, DP_MOD_ID_TX_COMP);
- }
- #ifndef WLAN_SOFTUMAC_SUPPORT
- /**
- * dp_tx_dump_tx_desc() - Dump tx desc for debugging
- * @tx_desc: software descriptor head pointer
- *
- * This function will dump tx desc for further debugging
- *
- * Return: none
- */
- static
- void dp_tx_dump_tx_desc(struct dp_tx_desc_s *tx_desc)
- {
- if (tx_desc) {
- dp_tx_comp_warn("tx_desc->nbuf: %pK", tx_desc->nbuf);
- dp_tx_comp_warn("tx_desc->flags: 0x%x", tx_desc->flags);
- dp_tx_comp_warn("tx_desc->id: %u", tx_desc->id);
- dp_tx_comp_warn("tx_desc->dma_addr: 0x%x",
- (unsigned int)tx_desc->dma_addr);
- dp_tx_comp_warn("tx_desc->vdev_id: %u",
- tx_desc->vdev_id);
- dp_tx_comp_warn("tx_desc->tx_status: %u",
- tx_desc->tx_status);
- dp_tx_comp_warn("tx_desc->pdev: %pK",
- tx_desc->pdev);
- dp_tx_comp_warn("tx_desc->tx_encap_type: %u",
- tx_desc->tx_encap_type);
- dp_tx_comp_warn("tx_desc->buffer_src: %u",
- tx_desc->buffer_src);
- dp_tx_comp_warn("tx_desc->frm_type: %u",
- tx_desc->frm_type);
- dp_tx_comp_warn("tx_desc->pkt_offset: %u",
- tx_desc->pkt_offset);
- dp_tx_comp_warn("tx_desc->pool_id: %u",
- tx_desc->pool_id);
- }
- }
- #endif
- #ifdef WLAN_FEATURE_RX_SOFTIRQ_TIME_LIMIT
- static inline
- bool dp_tx_comp_loop_pkt_limit_hit(struct dp_soc *soc, int num_reaped,
- int max_reap_limit)
- {
- bool limit_hit = false;
- limit_hit =
- (num_reaped >= max_reap_limit) ? true : false;
- if (limit_hit)
- DP_STATS_INC(soc, tx.tx_comp_loop_pkt_limit_hit, 1);
- return limit_hit;
- }
- static inline bool dp_tx_comp_enable_eol_data_check(struct dp_soc *soc)
- {
- return soc->wlan_cfg_ctx->tx_comp_enable_eol_data_check;
- }
- static inline int dp_tx_comp_get_loop_pkt_limit(struct dp_soc *soc)
- {
- struct wlan_cfg_dp_soc_ctxt *cfg = soc->wlan_cfg_ctx;
- return cfg->tx_comp_loop_pkt_limit;
- }
- #else
- static inline
- bool dp_tx_comp_loop_pkt_limit_hit(struct dp_soc *soc, int num_reaped,
- int max_reap_limit)
- {
- return false;
- }
- static inline bool dp_tx_comp_enable_eol_data_check(struct dp_soc *soc)
- {
- return false;
- }
- static inline int dp_tx_comp_get_loop_pkt_limit(struct dp_soc *soc)
- {
- return 0;
- }
- #endif
- #ifdef WLAN_FEATURE_NEAR_FULL_IRQ
- static inline int
- dp_srng_test_and_update_nf_params(struct dp_soc *soc, struct dp_srng *dp_srng,
- int *max_reap_limit)
- {
- return soc->arch_ops.dp_srng_test_and_update_nf_params(soc, dp_srng,
- max_reap_limit);
- }
- #else
- static inline int
- dp_srng_test_and_update_nf_params(struct dp_soc *soc, struct dp_srng *dp_srng,
- int *max_reap_limit)
- {
- return 0;
- }
- #endif
- #ifdef DP_TX_TRACKING
- void dp_tx_desc_check_corruption(struct dp_tx_desc_s *tx_desc)
- {
- if ((tx_desc->magic != DP_TX_MAGIC_PATTERN_INUSE) &&
- (tx_desc->magic != DP_TX_MAGIC_PATTERN_FREE)) {
- dp_err_rl("tx_desc %u is corrupted", tx_desc->id);
- qdf_trigger_self_recovery(NULL, QDF_TX_DESC_LEAK);
- }
- }
- #endif
- #ifndef WLAN_SOFTUMAC_SUPPORT
- #ifdef DP_TX_COMP_RING_DESC_SANITY_CHECK
- /* Increasing this value, runs the risk of srng backpressure */
- #define DP_STALE_TX_COMP_WAIT_TIMEOUT_US 1000
- static inline void
- dp_tx_comp_reset_stale_entry_detection(struct dp_soc *soc, uint32_t ring_num)
- {
- soc->stale_entry[ring_num].detected = 0;
- }
- /**
- * dp_tx_comp_stale_entry_handle() - Detect stale entry condition in tx
- * completion srng.
- * @soc: DP SoC handle
- * @ring_num: tx completion ring number
- * @status: QDF_STATUS from tx_comp_get_params_from_hal_desc arch ops
- *
- * Return: QDF_STATUS_SUCCESS if stale entry is detected and handled
- * QDF_STATUS error code in other cases.
- */
- static inline QDF_STATUS
- dp_tx_comp_stale_entry_handle(struct dp_soc *soc, uint32_t ring_num,
- QDF_STATUS status)
- {
- uint64_t curr_timestamp = qdf_get_log_timestamp_usecs();
- uint64_t delta_us;
- if (status != QDF_STATUS_E_PENDING) {
- dp_tx_comp_reset_stale_entry_detection(soc, ring_num);
- return QDF_STATUS_E_INVAL;
- }
- if (soc->stale_entry[ring_num].detected) {
- /* stale entry process continuation */
- delta_us = curr_timestamp -
- soc->stale_entry[ring_num].start_time;
- if (delta_us > DP_STALE_TX_COMP_WAIT_TIMEOUT_US) {
- dp_err("Stale tx comp desc, waited %d us", delta_us);
- return QDF_STATUS_E_FAILURE;
- }
- } else {
- /* This is the start of stale entry detection */
- soc->stale_entry[ring_num].detected = 1;
- soc->stale_entry[ring_num].start_time = curr_timestamp;
- }
- return QDF_STATUS_SUCCESS;
- }
- #else
- static inline void
- dp_tx_comp_reset_stale_entry_detection(struct dp_soc *soc, uint32_t ring_num)
- {
- }
- static inline QDF_STATUS
- dp_tx_comp_stale_entry_handle(struct dp_soc *soc, uint32_t ring_num,
- QDF_STATUS status)
- {
- return QDF_STATUS_SUCCESS;
- }
- #endif
- uint32_t dp_tx_comp_handler(struct dp_intr *int_ctx, struct dp_soc *soc,
- hal_ring_handle_t hal_ring_hdl, uint8_t ring_id,
- uint32_t quota)
- {
- void *tx_comp_hal_desc;
- void *last_prefetched_hw_desc = NULL;
- void *last_hw_desc = NULL;
- struct dp_tx_desc_s *last_prefetched_sw_desc = NULL;
- hal_soc_handle_t hal_soc;
- uint8_t buffer_src;
- struct dp_tx_desc_s *tx_desc = NULL;
- struct dp_tx_desc_s *head_desc = NULL;
- struct dp_tx_desc_s *tail_desc = NULL;
- struct dp_tx_desc_s *fast_head_desc = NULL;
- struct dp_tx_desc_s *fast_tail_desc = NULL;
- uint32_t num_processed = 0;
- uint32_t fast_desc_count = 0;
- uint32_t count;
- uint32_t num_avail_for_reap = 0;
- bool force_break = false;
- struct dp_srng *tx_comp_ring = &soc->tx_comp_ring[ring_id];
- int max_reap_limit, ring_near_full;
- uint32_t num_entries;
- qdf_nbuf_queue_head_t h;
- QDF_STATUS status;
- DP_HIST_INIT();
- num_entries = hal_srng_get_num_entries(soc->hal_soc, hal_ring_hdl);
- more_data:
- hal_soc = soc->hal_soc;
- /* Re-initialize local variables to be re-used */
- head_desc = NULL;
- tail_desc = NULL;
- count = 0;
- max_reap_limit = dp_tx_comp_get_loop_pkt_limit(soc);
- ring_near_full = dp_srng_test_and_update_nf_params(soc, tx_comp_ring,
- &max_reap_limit);
- if (qdf_unlikely(dp_srng_access_start(int_ctx, soc, hal_ring_hdl))) {
- dp_err("HAL RING Access Failed -- %pK", hal_ring_hdl);
- return 0;
- }
- hal_srng_update_ring_usage_wm_no_lock(soc->hal_soc, hal_ring_hdl);
- if (!num_avail_for_reap)
- num_avail_for_reap = hal_srng_dst_num_valid(hal_soc,
- hal_ring_hdl, 0);
- if (num_avail_for_reap >= quota)
- num_avail_for_reap = quota;
- last_hw_desc = dp_srng_dst_inv_cached_descs(soc, hal_ring_hdl,
- num_avail_for_reap);
- last_prefetched_hw_desc = dp_srng_dst_prefetch_32_byte_desc(hal_soc,
- hal_ring_hdl,
- num_avail_for_reap);
- dp_tx_nbuf_queue_head_init(&h);
- /* Find head descriptor from completion ring */
- while (qdf_likely(num_avail_for_reap--)) {
- tx_comp_hal_desc = dp_srng_dst_get_next(soc, hal_ring_hdl);
- if (qdf_unlikely(!tx_comp_hal_desc))
- break;
- buffer_src = hal_tx_comp_get_buffer_source(hal_soc,
- tx_comp_hal_desc);
- /* If this buffer was not released by TQM or FW, then it is not
- * Tx completion indication, assert */
- if (qdf_unlikely(buffer_src !=
- HAL_TX_COMP_RELEASE_SOURCE_TQM) &&
- (qdf_unlikely(buffer_src !=
- HAL_TX_COMP_RELEASE_SOURCE_FW))) {
- uint8_t wbm_internal_error;
- dp_err_rl(
- "Tx comp release_src != TQM | FW but from %d",
- buffer_src);
- hal_dump_comp_desc(tx_comp_hal_desc);
- DP_STATS_INC(soc, tx.invalid_release_source, 1);
- /* When WBM sees NULL buffer_addr_info in any of
- * ingress rings it sends an error indication,
- * with wbm_internal_error=1, to a specific ring.
- * The WBM2SW ring used to indicate these errors is
- * fixed in HW, and that ring is being used as Tx
- * completion ring. These errors are not related to
- * Tx completions, and should just be ignored
- */
- wbm_internal_error = hal_get_wbm_internal_error(
- hal_soc,
- tx_comp_hal_desc);
- if (wbm_internal_error) {
- dp_err_rl("Tx comp wbm_internal_error!!");
- DP_STATS_INC(soc, tx.wbm_internal_error[WBM_INT_ERROR_ALL], 1);
- if (HAL_TX_COMP_RELEASE_SOURCE_REO ==
- buffer_src)
- dp_handle_wbm_internal_error(
- soc,
- tx_comp_hal_desc,
- hal_tx_comp_get_buffer_type(
- tx_comp_hal_desc));
- } else {
- dp_err_rl("Tx comp wbm_internal_error false");
- DP_STATS_INC(soc, tx.non_wbm_internal_err, 1);
- }
- continue;
- }
- status = soc->arch_ops.tx_comp_get_params_from_hal_desc(
- soc, tx_comp_hal_desc,
- &tx_desc);
- if (qdf_unlikely(!tx_desc)) {
- if (QDF_IS_STATUS_SUCCESS(
- dp_tx_comp_stale_entry_handle(soc, ring_id,
- status))) {
- hal_srng_dst_dec_tp(hal_soc, hal_ring_hdl);
- break;
- }
- dp_err("unable to retrieve tx_desc!");
- hal_dump_comp_desc(tx_comp_hal_desc);
- DP_STATS_INC(soc, tx.invalid_tx_comp_desc, 1);
- QDF_BUG(0);
- continue;
- }
- dp_tx_comp_reset_stale_entry_detection(soc, ring_id);
- tx_desc->buffer_src = buffer_src;
- /*
- * If the release source is FW, process the HTT status
- */
- if (qdf_unlikely(buffer_src ==
- HAL_TX_COMP_RELEASE_SOURCE_FW)) {
- uint8_t htt_tx_status[HAL_TX_COMP_HTT_STATUS_LEN];
- hal_tx_comp_get_htt_desc(tx_comp_hal_desc,
- htt_tx_status);
- /* Collect hw completion contents */
- hal_tx_comp_desc_sync(tx_comp_hal_desc,
- &tx_desc->comp, 1);
- soc->arch_ops.dp_tx_process_htt_completion(
- soc,
- tx_desc,
- htt_tx_status,
- ring_id);
- if (qdf_unlikely(!tx_desc->pdev)) {
- dp_tx_dump_tx_desc(tx_desc);
- }
- } else {
- if (tx_desc->flags & DP_TX_DESC_FLAG_FASTPATH_SIMPLE ||
- tx_desc->flags & DP_TX_DESC_FLAG_PPEDS)
- goto add_to_pool2;
- tx_desc->tx_status =
- hal_tx_comp_get_tx_status(tx_comp_hal_desc);
- tx_desc->buffer_src = buffer_src;
- /*
- * If the fast completion mode is enabled extended
- * metadata from descriptor is not copied
- */
- if (qdf_likely(tx_desc->flags &
- DP_TX_DESC_FLAG_SIMPLE))
- goto add_to_pool;
- /*
- * If the descriptor is already freed in vdev_detach,
- * continue to next descriptor
- */
- if (qdf_unlikely
- ((tx_desc->vdev_id == DP_INVALID_VDEV_ID) &&
- !tx_desc->flags)) {
- dp_tx_comp_info_rl("Descriptor freed in vdev_detach %d",
- tx_desc->id);
- DP_STATS_INC(soc, tx.tx_comp_exception, 1);
- dp_tx_desc_check_corruption(tx_desc);
- continue;
- }
- if (qdf_unlikely(!tx_desc->pdev)) {
- dp_tx_comp_warn("The pdev is NULL in TX desc, ignored.");
- dp_tx_dump_tx_desc(tx_desc);
- DP_STATS_INC(soc, tx.tx_comp_exception, 1);
- continue;
- }
- if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) {
- dp_tx_comp_info_rl("pdev in down state %d",
- tx_desc->id);
- tx_desc->flags |= DP_TX_DESC_FLAG_TX_COMP_ERR;
- dp_tx_comp_free_buf(soc, tx_desc, false);
- dp_tx_desc_release(soc, tx_desc,
- tx_desc->pool_id);
- goto next_desc;
- }
- if (!(tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED) ||
- !(tx_desc->flags & DP_TX_DESC_FLAG_QUEUED_TX)) {
- dp_tx_comp_alert("Txdesc invalid, flgs = %x,id = %d",
- tx_desc->flags, tx_desc->id);
- qdf_assert_always(0);
- }
- /* Collect hw completion contents */
- hal_tx_comp_desc_sync(tx_comp_hal_desc,
- &tx_desc->comp, 1);
- add_to_pool:
- DP_HIST_PACKET_COUNT_INC(tx_desc->pdev->pdev_id);
- add_to_pool2:
- /* First ring descriptor on the cycle */
- if (tx_desc->flags & DP_TX_DESC_FLAG_FASTPATH_SIMPLE ||
- tx_desc->flags & DP_TX_DESC_FLAG_PPEDS) {
- dp_tx_nbuf_dev_queue_free(&h, tx_desc);
- fast_desc_count++;
- if (!fast_head_desc) {
- fast_head_desc = tx_desc;
- fast_tail_desc = tx_desc;
- }
- fast_tail_desc->next = tx_desc;
- fast_tail_desc = tx_desc;
- dp_tx_desc_clear(tx_desc);
- } else {
- if (!head_desc) {
- head_desc = tx_desc;
- tail_desc = tx_desc;
- }
- tail_desc->next = tx_desc;
- tx_desc->next = NULL;
- tail_desc = tx_desc;
- }
- }
- next_desc:
- num_processed += !(count & DP_TX_NAPI_BUDGET_DIV_MASK);
- /*
- * Processed packet count is more than given quota
- * stop to processing
- */
- count++;
- dp_tx_prefetch_hw_sw_nbuf_desc(soc, hal_soc,
- num_avail_for_reap,
- hal_ring_hdl,
- &last_prefetched_hw_desc,
- &last_prefetched_sw_desc,
- last_hw_desc);
- if (dp_tx_comp_loop_pkt_limit_hit(soc, count, max_reap_limit))
- break;
- }
- dp_srng_access_end(int_ctx, soc, hal_ring_hdl);
- /* Process the reaped descriptors that were sent via fast path */
- if (fast_head_desc) {
- dp_tx_comp_process_desc_list_fast(soc, fast_head_desc,
- fast_tail_desc, ring_id,
- fast_desc_count);
- dp_tx_nbuf_dev_kfree_list(&h);
- }
- /* Process the reaped descriptors */
- if (head_desc)
- dp_tx_comp_process_desc_list(soc, head_desc, ring_id);
- DP_STATS_INC(soc, tx.tx_comp[ring_id], count);
- /*
- * If we are processing in near-full condition, there are 3 scenario
- * 1) Ring entries has reached critical state
- * 2) Ring entries are still near high threshold
- * 3) Ring entries are below the safe level
- *
- * One more loop will move the state to normal processing and yield
- */
- if (ring_near_full)
- goto more_data;
- if (dp_tx_comp_enable_eol_data_check(soc)) {
- if (num_processed >= quota)
- force_break = true;
- if (!force_break &&
- hal_srng_dst_peek_sync_locked(soc->hal_soc,
- hal_ring_hdl)) {
- DP_STATS_INC(soc, tx.hp_oos2, 1);
- if (!hif_exec_should_yield(soc->hif_handle,
- int_ctx->dp_intr_id))
- goto more_data;
- num_avail_for_reap =
- hal_srng_dst_num_valid_locked(soc->hal_soc,
- hal_ring_hdl,
- true);
- if (qdf_unlikely(num_entries &&
- (num_avail_for_reap >=
- num_entries >> 1))) {
- DP_STATS_INC(soc, tx.near_full, 1);
- goto more_data;
- }
- }
- }
- DP_TX_HIST_STATS_PER_PDEV();
- return num_processed;
- }
- #endif
- #ifdef FEATURE_WLAN_TDLS
- qdf_nbuf_t dp_tx_non_std(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
- enum ol_tx_spec tx_spec, qdf_nbuf_t msdu_list)
- {
- struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
- struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
- DP_MOD_ID_TDLS);
- if (!vdev) {
- dp_err("vdev handle for id %d is NULL", vdev_id);
- return NULL;
- }
- if (tx_spec & OL_TX_SPEC_NO_FREE)
- vdev->is_tdls_frame = true;
- dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
- return dp_tx_send(soc_hdl, vdev_id, msdu_list);
- }
- #endif
- QDF_STATUS dp_tx_vdev_attach(struct dp_vdev *vdev)
- {
- int pdev_id;
- /*
- * Fill HTT TCL Metadata with Vdev ID and MAC ID
- */
- DP_TX_TCL_METADATA_TYPE_SET(vdev->htt_tcl_metadata,
- DP_TCL_METADATA_TYPE_VDEV_BASED);
- DP_TX_TCL_METADATA_VDEV_ID_SET(vdev->htt_tcl_metadata,
- vdev->vdev_id);
- pdev_id =
- dp_get_target_pdev_id_for_host_pdev_id(vdev->pdev->soc,
- vdev->pdev->pdev_id);
- DP_TX_TCL_METADATA_PDEV_ID_SET(vdev->htt_tcl_metadata, pdev_id);
- /*
- * Set HTT Extension Valid bit to 0 by default
- */
- DP_TX_TCL_METADATA_VALID_HTT_SET(vdev->htt_tcl_metadata, 0);
- dp_tx_vdev_update_search_flags(vdev);
- return QDF_STATUS_SUCCESS;
- }
- #ifndef FEATURE_WDS
- static inline bool dp_tx_da_search_override(struct dp_vdev *vdev)
- {
- return false;
- }
- #endif
- void dp_tx_vdev_update_search_flags(struct dp_vdev *vdev)
- {
- struct dp_soc *soc = vdev->pdev->soc;
- /*
- * Enable both AddrY (SA based search) and AddrX (Da based search)
- * for TDLS link
- *
- * Enable AddrY (SA based search) only for non-WDS STA and
- * ProxySTA VAP (in HKv1) modes.
- *
- * In all other VAP modes, only DA based search should be
- * enabled
- */
- if (vdev->opmode == wlan_op_mode_sta &&
- vdev->tdls_link_connected)
- vdev->hal_desc_addr_search_flags =
- (HAL_TX_DESC_ADDRX_EN | HAL_TX_DESC_ADDRY_EN);
- else if ((vdev->opmode == wlan_op_mode_sta) &&
- !dp_tx_da_search_override(vdev))
- vdev->hal_desc_addr_search_flags = HAL_TX_DESC_ADDRY_EN;
- else
- vdev->hal_desc_addr_search_flags = HAL_TX_DESC_ADDRX_EN;
- if (vdev->opmode == wlan_op_mode_sta && !vdev->tdls_link_connected)
- vdev->search_type = soc->sta_mode_search_policy;
- else
- vdev->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
- }
- #ifdef WLAN_SUPPORT_PPEDS
- static inline bool
- dp_is_tx_desc_flush_match(struct dp_pdev *pdev,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- if (!(tx_desc && (tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED)))
- return false;
- if (tx_desc->flags & DP_TX_DESC_FLAG_PPEDS)
- return true;
- /*
- * if vdev is given, then only check whether desc
- * vdev match. if vdev is NULL, then check whether
- * desc pdev match.
- */
- return vdev ? (tx_desc->vdev_id == vdev->vdev_id) :
- (tx_desc->pdev == pdev);
- }
- #else
- static inline bool
- dp_is_tx_desc_flush_match(struct dp_pdev *pdev,
- struct dp_vdev *vdev,
- struct dp_tx_desc_s *tx_desc)
- {
- if (!(tx_desc && (tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED)))
- return false;
- /*
- * if vdev is given, then only check whether desc
- * vdev match. if vdev is NULL, then check whether
- * desc pdev match.
- */
- return vdev ? (tx_desc->vdev_id == vdev->vdev_id) :
- (tx_desc->pdev == pdev);
- }
- #endif
- #ifdef QCA_LL_TX_FLOW_CONTROL_V2
- void dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
- bool force_free)
- {
- uint8_t i;
- uint32_t j;
- uint32_t num_desc, page_id, offset;
- uint16_t num_desc_per_page;
- struct dp_soc *soc = pdev->soc;
- struct dp_tx_desc_s *tx_desc = NULL;
- struct dp_tx_desc_pool_s *tx_desc_pool = NULL;
- if (!vdev && !force_free) {
- dp_err("Reset TX desc vdev, Vdev param is required!");
- return;
- }
- for (i = 0; i < MAX_TXDESC_POOLS; i++) {
- tx_desc_pool = &soc->tx_desc[i];
- if (!(tx_desc_pool->pool_size) ||
- IS_TX_DESC_POOL_STATUS_INACTIVE(tx_desc_pool) ||
- !(tx_desc_pool->desc_pages.cacheable_pages))
- continue;
- /*
- * Add flow pool lock protection in case pool is freed
- * due to all tx_desc is recycled when handle TX completion.
- * this is not necessary when do force flush as:
- * a. double lock will happen if dp_tx_desc_release is
- * also trying to acquire it.
- * b. dp interrupt has been disabled before do force TX desc
- * flush in dp_pdev_deinit().
- */
- if (!force_free)
- qdf_spin_lock_bh(&tx_desc_pool->flow_pool_lock);
- num_desc = tx_desc_pool->pool_size;
- num_desc_per_page =
- tx_desc_pool->desc_pages.num_element_per_page;
- for (j = 0; j < num_desc; j++) {
- page_id = j / num_desc_per_page;
- offset = j % num_desc_per_page;
- if (qdf_unlikely(!(tx_desc_pool->
- desc_pages.cacheable_pages)))
- break;
- tx_desc = dp_tx_desc_find(soc, i, page_id, offset,
- false);
- if (dp_is_tx_desc_flush_match(pdev, vdev, tx_desc)) {
- /*
- * Free TX desc if force free is
- * required, otherwise only reset vdev
- * in this TX desc.
- */
- if (force_free) {
- tx_desc->flags |= DP_TX_DESC_FLAG_FLUSH;
- dp_tx_comp_free_buf(soc, tx_desc,
- false);
- dp_tx_desc_release(soc, tx_desc, i);
- } else {
- tx_desc->vdev_id = DP_INVALID_VDEV_ID;
- }
- }
- }
- if (!force_free)
- qdf_spin_unlock_bh(&tx_desc_pool->flow_pool_lock);
- }
- }
- #else /* QCA_LL_TX_FLOW_CONTROL_V2! */
- /**
- * dp_tx_desc_reset_vdev() - reset vdev to NULL in TX Desc
- *
- * @soc: Handle to DP soc structure
- * @tx_desc: pointer of one TX desc
- * @desc_pool_id: TX Desc pool id
- * @spcl_pool: Special pool
- */
- static inline void
- dp_tx_desc_reset_vdev(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
- uint8_t desc_pool_id, bool spcl_pool)
- {
- struct dp_tx_desc_pool_s *pool = NULL;
- pool = spcl_pool ? dp_get_spcl_tx_desc_pool(soc, desc_pool_id) :
- dp_get_tx_desc_pool(soc, desc_pool_id);
- TX_DESC_LOCK_LOCK(&pool->lock);
- tx_desc->vdev_id = DP_INVALID_VDEV_ID;
- TX_DESC_LOCK_UNLOCK(&pool->lock);
- }
- void __dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
- bool force_free, bool spcl_pool)
- {
- uint8_t i, num_pool;
- uint32_t j;
- uint32_t num_desc_t, page_id, offset;
- uint16_t num_desc_per_page;
- struct dp_soc *soc = pdev->soc;
- struct dp_tx_desc_s *tx_desc = NULL;
- struct dp_tx_desc_pool_s *tx_desc_pool = NULL;
- if (!vdev && !force_free) {
- dp_err("Reset TX desc vdev, Vdev param is required!");
- return;
- }
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- for (i = 0; i < num_pool; i++) {
- tx_desc_pool = spcl_pool ? dp_get_spcl_tx_desc_pool(soc, i) :
- dp_get_tx_desc_pool(soc, i);
- num_desc_t = tx_desc_pool->elem_count;
- if (!tx_desc_pool->desc_pages.cacheable_pages)
- continue;
- num_desc_per_page =
- tx_desc_pool->desc_pages.num_element_per_page;
- for (j = 0; j < num_desc_t; j++) {
- page_id = j / num_desc_per_page;
- offset = j % num_desc_per_page;
- tx_desc = dp_tx_desc_find(soc, i, page_id, offset,
- spcl_pool);
- if (dp_is_tx_desc_flush_match(pdev, vdev, tx_desc)) {
- if (force_free) {
- dp_tx_comp_free_buf(soc, tx_desc,
- false);
- dp_tx_desc_release(soc, tx_desc, i);
- } else {
- dp_tx_desc_reset_vdev(soc, tx_desc,
- i, spcl_pool);
- }
- }
- }
- }
- }
- void dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
- bool force_free)
- {
- __dp_tx_desc_flush(pdev, vdev, force_free, false);
- __dp_tx_desc_flush(pdev, vdev, force_free, true);
- }
- #endif /* !QCA_LL_TX_FLOW_CONTROL_V2 */
- QDF_STATUS dp_tx_vdev_detach(struct dp_vdev *vdev)
- {
- struct dp_pdev *pdev = vdev->pdev;
- /* Reset TX desc associated to this Vdev as NULL */
- dp_tx_desc_flush(pdev, vdev, false);
- return QDF_STATUS_SUCCESS;
- }
- #ifdef QCA_LL_TX_FLOW_CONTROL_V2
- /* Pools will be allocated dynamically */
- static QDF_STATUS dp_tx_alloc_static_pools(struct dp_soc *soc, int num_pool,
- int num_desc)
- {
- uint8_t i;
- for (i = 0; i < num_pool; i++) {
- qdf_spinlock_create(&soc->tx_desc[i].flow_pool_lock);
- soc->tx_desc[i].status = FLOW_POOL_INACTIVE;
- }
- return QDF_STATUS_SUCCESS;
- }
- static QDF_STATUS dp_tx_spcl_alloc_static_pools(struct dp_soc *soc,
- int num_pool,
- int num_spcl_desc)
- {
- return QDF_STATUS_SUCCESS;
- }
- static QDF_STATUS dp_tx_init_static_pools(struct dp_soc *soc, int num_pool,
- uint32_t num_desc)
- {
- return QDF_STATUS_SUCCESS;
- }
- static QDF_STATUS dp_tx_spcl_init_static_pools(struct dp_soc *soc, int num_pool,
- uint32_t num_spcl_desc)
- {
- return QDF_STATUS_SUCCESS;
- }
- static void dp_tx_deinit_static_pools(struct dp_soc *soc, int num_pool)
- {
- }
- static void dp_tx_spcl_deinit_static_pools(struct dp_soc *soc, int num_pool)
- {
- }
- static void dp_tx_delete_static_pools(struct dp_soc *soc, int num_pool)
- {
- uint8_t i;
- for (i = 0; i < num_pool; i++)
- qdf_spinlock_destroy(&soc->tx_desc[i].flow_pool_lock);
- }
- static void dp_tx_spcl_delete_static_pools(struct dp_soc *soc, int num_pool)
- {
- }
- #else /* QCA_LL_TX_FLOW_CONTROL_V2! */
- static QDF_STATUS dp_tx_alloc_static_pools(struct dp_soc *soc, int num_pool,
- uint32_t num_desc)
- {
- uint8_t i, count;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- /* Allocate software Tx descriptor pools */
- if (dp_global->tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++) {
- if (dp_tx_desc_pool_alloc(soc, i, num_desc, false)) {
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
- FL("Tx Desc Pool alloc %d failed %pK"),
- i, soc);
- goto fail;
- }
- }
- }
- dp_global->tx_desc_pool_alloc_cnt[soc->arch_id]++;
- return QDF_STATUS_SUCCESS;
- fail:
- for (count = 0; count < i; count++)
- dp_tx_desc_pool_free(soc, count, false);
- return QDF_STATUS_E_NOMEM;
- }
- static QDF_STATUS dp_tx_spcl_alloc_static_pools(struct dp_soc *soc,
- int num_pool,
- uint32_t num_spcl_desc)
- {
- uint8_t j, count;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- /* Allocate software Tx descriptor pools */
- if (dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
- for (j = 0; j < num_pool; j++) {
- if (dp_tx_desc_pool_alloc(soc, j, num_spcl_desc, true)) {
- QDF_TRACE(QDF_MODULE_ID_DP,
- QDF_TRACE_LEVEL_ERROR,
- FL("Tx special Desc Pool alloc %d failed %pK"),
- j, soc);
- goto fail;
- }
- }
- }
- dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id]++;
- return QDF_STATUS_SUCCESS;
- fail:
- for (count = 0; count < j; count++)
- dp_tx_desc_pool_free(soc, count, true);
- return QDF_STATUS_E_NOMEM;
- }
- static QDF_STATUS dp_tx_init_static_pools(struct dp_soc *soc, int num_pool,
- uint32_t num_desc)
- {
- uint8_t i;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- if (dp_global->tx_desc_pool_init_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++) {
- if (dp_tx_desc_pool_init(soc, i, num_desc, false)) {
- QDF_TRACE(QDF_MODULE_ID_DP,
- QDF_TRACE_LEVEL_ERROR,
- FL("Tx Desc Pool init %d failed %pK"),
- i, soc);
- return QDF_STATUS_E_NOMEM;
- }
- }
- }
- dp_global->tx_desc_pool_init_cnt[soc->arch_id]++;
- return QDF_STATUS_SUCCESS;
- }
- static QDF_STATUS dp_tx_spcl_init_static_pools(struct dp_soc *soc, int num_pool,
- uint32_t num_spcl_desc)
- {
- uint8_t i;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- if (dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++) {
- if (dp_tx_desc_pool_init(soc, i, num_spcl_desc, true)) {
- QDF_TRACE(QDF_MODULE_ID_DP,
- QDF_TRACE_LEVEL_ERROR,
- FL("Tx special Desc Pool init %d failed %pK"),
- i, soc);
- return QDF_STATUS_E_NOMEM;
- }
- }
- }
- dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id]++;
- return QDF_STATUS_SUCCESS;
- }
- static void dp_tx_deinit_static_pools(struct dp_soc *soc, int num_pool)
- {
- uint8_t i;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- dp_global->tx_desc_pool_init_cnt[soc->arch_id]--;
- if (dp_global->tx_desc_pool_init_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++)
- dp_tx_desc_pool_deinit(soc, i, false);
- }
- }
- static void dp_tx_spcl_deinit_static_pools(struct dp_soc *soc, int num_pool)
- {
- uint8_t i;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id]--;
- if (dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++)
- dp_tx_desc_pool_deinit(soc, i, true);
- }
- }
- static void dp_tx_delete_static_pools(struct dp_soc *soc, int num_pool)
- {
- uint8_t i;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- dp_global->tx_desc_pool_alloc_cnt[soc->arch_id]--;
- if (dp_global->tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++)
- dp_tx_desc_pool_free(soc, i, false);
- }
- }
- static void dp_tx_spcl_delete_static_pools(struct dp_soc *soc, int num_pool)
- {
- uint8_t i;
- struct dp_global_context *dp_global;
- dp_global = wlan_objmgr_get_global_ctx();
- dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id]--;
- if (dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
- for (i = 0; i < num_pool; i++)
- dp_tx_desc_pool_free(soc, i, true);
- }
- }
- #endif /* !QCA_LL_TX_FLOW_CONTROL_V2 */
- /**
- * dp_tx_tso_cmn_desc_pool_deinit() - de-initialize TSO descriptors
- * @soc: core txrx main context
- * @num_pool: number of pools
- *
- */
- static void dp_tx_tso_cmn_desc_pool_deinit(struct dp_soc *soc, uint8_t num_pool)
- {
- dp_tx_tso_desc_pool_deinit(soc, num_pool);
- dp_tx_tso_num_seg_pool_deinit(soc, num_pool);
- }
- /**
- * dp_tx_tso_cmn_desc_pool_free() - free TSO descriptors
- * @soc: core txrx main context
- * @num_pool: number of pools
- *
- */
- static void dp_tx_tso_cmn_desc_pool_free(struct dp_soc *soc, uint8_t num_pool)
- {
- dp_tx_tso_desc_pool_free(soc, num_pool);
- dp_tx_tso_num_seg_pool_free(soc, num_pool);
- }
- #ifndef WLAN_SOFTUMAC_SUPPORT
- void dp_soc_tx_desc_sw_pools_free(struct dp_soc *soc)
- {
- uint8_t num_pool, num_ext_pool;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
- dp_tx_tso_cmn_desc_pool_free(soc, num_pool);
- dp_tx_ext_desc_pool_free(soc, num_ext_pool);
- dp_tx_delete_static_pools(soc, num_pool);
- dp_tx_spcl_delete_static_pools(soc, num_pool);
- }
- void dp_soc_tx_desc_sw_pools_deinit(struct dp_soc *soc)
- {
- uint8_t num_pool, num_ext_pool;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
- dp_tx_flow_control_deinit(soc);
- dp_tx_tso_cmn_desc_pool_deinit(soc, num_pool);
- dp_tx_ext_desc_pool_deinit(soc, num_ext_pool);
- dp_tx_deinit_static_pools(soc, num_pool);
- dp_tx_spcl_deinit_static_pools(soc, num_pool);
- }
- #else
- void dp_soc_tx_desc_sw_pools_free(struct dp_soc *soc)
- {
- uint8_t num_pool;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- dp_tx_delete_static_pools(soc, num_pool);
- dp_tx_spcl_delete_static_pools(soc, num_pool);
- }
- void dp_soc_tx_desc_sw_pools_deinit(struct dp_soc *soc)
- {
- uint8_t num_pool;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- dp_tx_flow_control_deinit(soc);
- dp_tx_deinit_static_pools(soc, num_pool);
- dp_tx_spcl_deinit_static_pools(soc, num_pool);
- }
- #endif /*WLAN_SOFTUMAC_SUPPORT*/
- /**
- * dp_tx_tso_cmn_desc_pool_alloc() - TSO cmn desc pool allocator
- * @soc: DP soc handle
- * @num_pool: Number of pools
- * @num_desc: Number of descriptors
- *
- * Reserve TSO descriptor buffers
- *
- * Return: QDF_STATUS_E_FAILURE on failure or
- * QDF_STATUS_SUCCESS on success
- */
- static QDF_STATUS dp_tx_tso_cmn_desc_pool_alloc(struct dp_soc *soc,
- uint8_t num_pool,
- uint32_t num_desc)
- {
- if (dp_tx_tso_desc_pool_alloc(soc, num_pool, num_desc)) {
- dp_err("TSO Desc Pool alloc %d failed %pK", num_pool, soc);
- return QDF_STATUS_E_FAILURE;
- }
- if (dp_tx_tso_num_seg_pool_alloc(soc, num_pool, num_desc)) {
- dp_err("TSO Num of seg Pool alloc %d failed %pK",
- num_pool, soc);
- return QDF_STATUS_E_FAILURE;
- }
- return QDF_STATUS_SUCCESS;
- }
- /**
- * dp_tx_tso_cmn_desc_pool_init() - TSO cmn desc pool init
- * @soc: DP soc handle
- * @num_pool: Number of pools
- * @num_desc: Number of descriptors
- *
- * Initialize TSO descriptor pools
- *
- * Return: QDF_STATUS_E_FAILURE on failure or
- * QDF_STATUS_SUCCESS on success
- */
- static QDF_STATUS dp_tx_tso_cmn_desc_pool_init(struct dp_soc *soc,
- uint8_t num_pool,
- uint32_t num_desc)
- {
- if (dp_tx_tso_desc_pool_init(soc, num_pool, num_desc)) {
- dp_err("TSO Desc Pool alloc %d failed %pK", num_pool, soc);
- return QDF_STATUS_E_FAILURE;
- }
- if (dp_tx_tso_num_seg_pool_init(soc, num_pool, num_desc)) {
- dp_err("TSO Num of seg Pool alloc %d failed %pK",
- num_pool, soc);
- return QDF_STATUS_E_FAILURE;
- }
- return QDF_STATUS_SUCCESS;
- }
- #ifndef WLAN_SOFTUMAC_SUPPORT
- QDF_STATUS dp_soc_tx_desc_sw_pools_alloc(struct dp_soc *soc)
- {
- uint8_t num_pool, num_ext_pool;
- uint32_t num_desc;
- uint32_t num_spcl_desc;
- uint32_t num_ext_desc;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return QDF_STATUS_SUCCESS;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
- num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
- num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
- num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
- dp_info("Tx Desc Alloc num_pool: %d descs: %d", num_pool, num_desc);
- if ((num_pool > MAX_TXDESC_POOLS) ||
- (num_ext_pool > MAX_TXDESC_POOLS) ||
- (num_desc > WLAN_CFG_NUM_TX_DESC_MAX) ||
- (num_spcl_desc > WLAN_CFG_NUM_TX_SPL_DESC_MAX))
- goto fail1;
- if (dp_tx_alloc_static_pools(soc, num_pool, num_desc))
- goto fail1;
- if (dp_tx_spcl_alloc_static_pools(soc, num_pool, num_spcl_desc))
- goto fail2;
- if (dp_tx_ext_desc_pool_alloc(soc, num_ext_pool, num_ext_desc))
- goto fail3;
- if (wlan_cfg_is_tso_desc_attach_defer(soc->wlan_cfg_ctx))
- return QDF_STATUS_SUCCESS;
- if (dp_tx_tso_cmn_desc_pool_alloc(soc, num_ext_pool, num_ext_desc))
- goto fail4;
- return QDF_STATUS_SUCCESS;
- fail4:
- dp_tx_ext_desc_pool_free(soc, num_ext_pool);
- fail3:
- dp_tx_spcl_delete_static_pools(soc, num_pool);
- fail2:
- dp_tx_delete_static_pools(soc, num_pool);
- fail1:
- return QDF_STATUS_E_RESOURCES;
- }
- QDF_STATUS dp_soc_tx_desc_sw_pools_init(struct dp_soc *soc)
- {
- uint8_t num_pool, num_ext_pool;
- uint32_t num_desc;
- uint32_t num_spcl_desc;
- uint32_t num_ext_desc;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return QDF_STATUS_SUCCESS;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
- num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
- num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
- num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
- if (dp_tx_init_static_pools(soc, num_pool, num_desc))
- goto fail1;
- if (dp_tx_spcl_init_static_pools(soc, num_pool, num_spcl_desc))
- goto fail2;
- if (dp_tx_ext_desc_pool_init(soc, num_ext_pool, num_ext_desc))
- goto fail3;
- if (wlan_cfg_is_tso_desc_attach_defer(soc->wlan_cfg_ctx))
- return QDF_STATUS_SUCCESS;
- if (dp_tx_tso_cmn_desc_pool_init(soc, num_ext_pool, num_ext_desc))
- goto fail4;
- dp_tx_flow_control_init(soc);
- soc->process_tx_status = CONFIG_PROCESS_TX_STATUS;
- return QDF_STATUS_SUCCESS;
- fail4:
- dp_tx_ext_desc_pool_deinit(soc, num_ext_pool);
- fail3:
- dp_tx_spcl_deinit_static_pools(soc, num_pool);
- fail2:
- dp_tx_deinit_static_pools(soc, num_pool);
- fail1:
- return QDF_STATUS_E_RESOURCES;
- }
- #else
- QDF_STATUS dp_soc_tx_desc_sw_pools_alloc(struct dp_soc *soc)
- {
- uint8_t num_pool;
- uint32_t num_desc;
- uint32_t num_spcl_desc;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return QDF_STATUS_SUCCESS;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
- num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
- QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO,
- "%s Tx Desc Alloc num_pool = %d, descs = %d",
- __func__, num_pool, num_desc);
- if ((num_pool > MAX_TXDESC_POOLS) ||
- (num_desc > WLAN_CFG_NUM_TX_DESC_MAX) ||
- (num_spcl_desc > WLAN_CFG_NUM_TX_SPL_DESC_MAX))
- goto fail1;
- if (dp_tx_alloc_static_pools(soc, num_pool, num_desc))
- goto fail1;
- if (dp_tx_spcl_alloc_static_pools(soc, num_pool, num_spcl_desc))
- goto fail2;
- return QDF_STATUS_SUCCESS;
- fail2:
- dp_tx_delete_static_pools(soc, num_pool);
- fail1:
- return QDF_STATUS_E_RESOURCES;
- }
- QDF_STATUS dp_soc_tx_desc_sw_pools_init(struct dp_soc *soc)
- {
- uint8_t num_pool;
- uint32_t num_desc;
- uint32_t num_spcl_desc;
- if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
- return QDF_STATUS_SUCCESS;
- num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
- num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
- num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
- if (dp_tx_init_static_pools(soc, num_pool, num_desc))
- goto fail;
- if (dp_tx_spcl_init_static_pools(soc, num_pool, num_spcl_desc))
- goto fail1;
- dp_tx_flow_control_init(soc);
- soc->process_tx_status = CONFIG_PROCESS_TX_STATUS;
- return QDF_STATUS_SUCCESS;
- fail1:
- dp_tx_deinit_static_pools(soc, num_pool);
- fail:
- return QDF_STATUS_E_RESOURCES;
- }
- #endif
- QDF_STATUS dp_tso_soc_attach(struct cdp_soc_t *txrx_soc)
- {
- struct dp_soc *soc = (struct dp_soc *)txrx_soc;
- uint8_t num_ext_desc_pool;
- uint32_t num_ext_desc;
- num_ext_desc_pool = dp_get_ext_tx_desc_pool_num(soc);
- num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
- if (dp_tx_tso_cmn_desc_pool_alloc(soc, num_ext_desc_pool, num_ext_desc))
- return QDF_STATUS_E_FAILURE;
- if (dp_tx_tso_cmn_desc_pool_init(soc, num_ext_desc_pool, num_ext_desc))
- return QDF_STATUS_E_FAILURE;
- return QDF_STATUS_SUCCESS;
- }
- QDF_STATUS dp_tso_soc_detach(struct cdp_soc_t *txrx_soc)
- {
- struct dp_soc *soc = (struct dp_soc *)txrx_soc;
- uint8_t num_ext_desc_pool = dp_get_ext_tx_desc_pool_num(soc);
- dp_tx_tso_cmn_desc_pool_deinit(soc, num_ext_desc_pool);
- dp_tx_tso_cmn_desc_pool_free(soc, num_ext_desc_pool);
- return QDF_STATUS_SUCCESS;
- }
- #ifdef CONFIG_DP_PKT_ADD_TIMESTAMP
- void dp_pkt_add_timestamp(struct dp_vdev *vdev,
- enum qdf_pkt_timestamp_index index, uint64_t time,
- qdf_nbuf_t nbuf)
- {
- if (qdf_unlikely(qdf_is_dp_pkt_timestamp_enabled())) {
- uint64_t tsf_time;
- if (vdev->get_tsf_time) {
- vdev->get_tsf_time(vdev->osif_vdev, time, &tsf_time);
- qdf_add_dp_pkt_timestamp(nbuf, index, tsf_time);
- }
- }
- }
- void dp_pkt_get_timestamp(uint64_t *time)
- {
- if (qdf_unlikely(qdf_is_dp_pkt_timestamp_enabled()))
- *time = qdf_get_log_timestamp();
- }
- #endif
- #ifdef QCA_MULTIPASS_SUPPORT
- void dp_tx_add_groupkey_metadata(struct dp_vdev *vdev,
- struct dp_tx_msdu_info_s *msdu_info,
- uint16_t group_key)
- {
- struct htt_tx_msdu_desc_ext2_t *meta_data =
- (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
- qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
- /*
- * When attempting to send a multicast packet with multi-passphrase,
- * host shall add HTT EXT meta data "struct htt_tx_msdu_desc_ext2_t"
- * ref htt.h indicating the group_id field in "key_flags" also having
- * "valid_key_flags" as 1. Assign “key_flags = group_key_ix”.
- */
- HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_SET(msdu_info->meta_data[0],
- 1);
- HTT_TX_MSDU_EXT2_DESC_KEY_FLAGS_SET(msdu_info->meta_data[2], group_key);
- }
- #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
- defined(WLAN_MCAST_MLO)
- /**
- * dp_tx_need_mcast_reinject() - If frame needs to be processed in reinject path
- * @vdev: DP vdev handle
- *
- * Return: true if reinject handling is required else false
- */
- static inline bool
- dp_tx_need_mcast_reinject(struct dp_vdev *vdev)
- {
- if (vdev->mlo_vdev && vdev->opmode == wlan_op_mode_ap)
- return true;
- return false;
- }
- #else
- static inline bool
- dp_tx_need_mcast_reinject(struct dp_vdev *vdev)
- {
- return false;
- }
- #endif
- /**
- * dp_tx_need_multipass_process() - If frame needs multipass phrase processing
- * @soc: dp soc handle
- * @vdev: DP vdev handle
- * @buf: frame
- * @vlan_id: vlan id of frame
- *
- * Return: whether peer is special or classic
- */
- static
- uint8_t dp_tx_need_multipass_process(struct dp_soc *soc, struct dp_vdev *vdev,
- qdf_nbuf_t buf, uint16_t *vlan_id)
- {
- struct dp_txrx_peer *txrx_peer = NULL;
- struct dp_peer *peer = NULL;
- qdf_ether_header_t *eh = (qdf_ether_header_t *)qdf_nbuf_data(buf);
- struct vlan_ethhdr *veh = NULL;
- bool not_vlan = ((vdev->tx_encap_type == htt_cmn_pkt_type_raw) ||
- (htons(eh->ether_type) != ETH_P_8021Q));
- struct cdp_peer_info peer_info = { 0 };
- if (qdf_unlikely(not_vlan))
- return DP_VLAN_UNTAGGED;
- veh = (struct vlan_ethhdr *)eh;
- *vlan_id = (ntohs(veh->h_vlan_TCI) & VLAN_VID_MASK);
- if (qdf_unlikely(DP_FRAME_IS_MULTICAST((eh)->ether_dhost))) {
- /* look for handling of multicast packets in reinject path */
- if (dp_tx_need_mcast_reinject(vdev))
- return DP_VLAN_UNTAGGED;
- qdf_spin_lock_bh(&vdev->mpass_peer_mutex);
- TAILQ_FOREACH(txrx_peer, &vdev->mpass_peer_list,
- mpass_peer_list_elem) {
- if (*vlan_id == txrx_peer->vlan_id) {
- qdf_spin_unlock_bh(&vdev->mpass_peer_mutex);
- return DP_VLAN_TAGGED_MULTICAST;
- }
- }
- qdf_spin_unlock_bh(&vdev->mpass_peer_mutex);
- return DP_VLAN_UNTAGGED;
- }
- DP_PEER_INFO_PARAMS_INIT(&peer_info, DP_VDEV_ALL, eh->ether_dhost,
- false, CDP_WILD_PEER_TYPE);
- peer = dp_peer_hash_find_wrapper((struct dp_soc *)soc, &peer_info,
- DP_MOD_ID_TX_MULTIPASS);
- if (qdf_unlikely(!peer))
- return DP_VLAN_UNTAGGED;
- /*
- * Do not drop the frame when vlan_id doesn't match.
- * Send the frame as it is.
- */
- if (*vlan_id == peer->txrx_peer->vlan_id) {
- dp_peer_unref_delete(peer, DP_MOD_ID_TX_MULTIPASS);
- return DP_VLAN_TAGGED_UNICAST;
- }
- dp_peer_unref_delete(peer, DP_MOD_ID_TX_MULTIPASS);
- return DP_VLAN_UNTAGGED;
- }
- #ifndef WLAN_REPEATER_NOT_SUPPORTED
- static inline void
- dp_tx_multipass_send_pkt_to_repeater(struct dp_soc *soc, struct dp_vdev *vdev,
- qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- qdf_nbuf_t nbuf_copy = NULL;
- /* AP can have classic clients, special clients &
- * classic repeaters.
- * 1. Classic clients & special client:
- * Remove vlan header, find corresponding group key
- * index, fill in metaheader and enqueue multicast
- * frame to TCL.
- * 2. Classic repeater:
- * Pass through to classic repeater with vlan tag
- * intact without any group key index. Hardware
- * will know which key to use to send frame to
- * repeater.
- */
- nbuf_copy = qdf_nbuf_copy(nbuf);
- /*
- * Send multicast frame to special peers even
- * if pass through to classic repeater fails.
- */
- if (nbuf_copy) {
- struct dp_tx_msdu_info_s msdu_info_copy;
- qdf_mem_zero(&msdu_info_copy, sizeof(msdu_info_copy));
- msdu_info_copy.tid = HTT_TX_EXT_TID_INVALID;
- msdu_info_copy.xmit_type =
- qdf_nbuf_get_vdev_xmit_type(nbuf);
- HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_SET(msdu_info_copy.meta_data[0], 1);
- nbuf_copy = dp_tx_send_msdu_single(vdev, nbuf_copy,
- &msdu_info_copy,
- HTT_INVALID_PEER, NULL);
- if (nbuf_copy) {
- qdf_nbuf_free(nbuf_copy);
- dp_info_rl("nbuf_copy send failed");
- }
- }
- }
- #else
- static inline void
- dp_tx_multipass_send_pkt_to_repeater(struct dp_soc *soc, struct dp_vdev *vdev,
- qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- }
- #endif
- bool dp_tx_multipass_process(struct dp_soc *soc, struct dp_vdev *vdev,
- qdf_nbuf_t nbuf,
- struct dp_tx_msdu_info_s *msdu_info)
- {
- uint16_t vlan_id = 0;
- uint16_t group_key = 0;
- uint8_t is_spcl_peer = DP_VLAN_UNTAGGED;
- if (HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_GET(msdu_info->meta_data[0]))
- return true;
- is_spcl_peer = dp_tx_need_multipass_process(soc, vdev, nbuf, &vlan_id);
- if ((is_spcl_peer != DP_VLAN_TAGGED_MULTICAST) &&
- (is_spcl_peer != DP_VLAN_TAGGED_UNICAST))
- return true;
- if (is_spcl_peer == DP_VLAN_TAGGED_UNICAST) {
- dp_tx_remove_vlan_tag(vdev, nbuf);
- return true;
- }
- dp_tx_multipass_send_pkt_to_repeater(soc, vdev, nbuf, msdu_info);
- group_key = vdev->iv_vlan_map[vlan_id];
- /*
- * If group key is not installed, drop the frame.
- */
- if (!group_key)
- return false;
- dp_tx_remove_vlan_tag(vdev, nbuf);
- dp_tx_add_groupkey_metadata(vdev, msdu_info, group_key);
- msdu_info->exception_fw = 1;
- return true;
- }
- #endif /* QCA_MULTIPASS_SUPPORT */
|