dp_tx.c 214 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074
  1. /*
  2. * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved.
  3. * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
  4. *
  5. * Permission to use, copy, modify, and/or distribute this software for
  6. * any purpose with or without fee is hereby granted, provided that the
  7. * above copyright notice and this permission notice appear in all
  8. * copies.
  9. *
  10. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
  11. * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  12. * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
  13. * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  14. * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  15. * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  16. * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  17. * PERFORMANCE OF THIS SOFTWARE.
  18. */
  19. #include "htt.h"
  20. #include "dp_htt.h"
  21. #include "hal_hw_headers.h"
  22. #include "dp_tx.h"
  23. #include "dp_tx_desc.h"
  24. #include "dp_peer.h"
  25. #include "dp_types.h"
  26. #include "hal_tx.h"
  27. #include "qdf_mem.h"
  28. #include "qdf_nbuf.h"
  29. #include "qdf_net_types.h"
  30. #include "qdf_module.h"
  31. #include <wlan_cfg.h>
  32. #include "dp_ipa.h"
  33. #if defined(MESH_MODE_SUPPORT) || defined(FEATURE_PERPKT_INFO)
  34. #include "if_meta_hdr.h"
  35. #endif
  36. #include "enet.h"
  37. #include "dp_internal.h"
  38. #ifdef ATH_SUPPORT_IQUE
  39. #include "dp_txrx_me.h"
  40. #endif
  41. #include "dp_hist.h"
  42. #ifdef WLAN_DP_FEATURE_SW_LATENCY_MGR
  43. #include <wlan_dp_swlm.h>
  44. #endif
  45. #ifdef WIFI_MONITOR_SUPPORT
  46. #include <dp_mon.h>
  47. #endif
  48. #ifdef FEATURE_WDS
  49. #include "dp_txrx_wds.h"
  50. #endif
  51. #include "cdp_txrx_cmn_reg.h"
  52. #ifdef CONFIG_SAWF
  53. #include <dp_sawf.h>
  54. #endif
  55. /* Flag to skip CCE classify when mesh or tid override enabled */
  56. #define DP_TX_SKIP_CCE_CLASSIFY \
  57. (DP_TXRX_HLOS_TID_OVERRIDE_ENABLED | DP_TX_MESH_ENABLED)
  58. /* TODO Add support in TSO */
  59. #define DP_DESC_NUM_FRAG(x) 0
  60. /* disable TQM_BYPASS */
  61. #define TQM_BYPASS_WAR 0
  62. #define DP_RETRY_COUNT 7
  63. #ifdef WLAN_PEER_JITTER
  64. #define DP_AVG_JITTER_WEIGHT_DENOM 4
  65. #define DP_AVG_DELAY_WEIGHT_DENOM 3
  66. #endif
  67. #ifdef QCA_DP_TX_FW_METADATA_V2
  68. #define DP_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)\
  69. HTT_TX_TCL_METADATA_V2_PDEV_ID_SET(_var, _val)
  70. #define DP_TX_TCL_METADATA_VALID_HTT_SET(_var, _val) \
  71. HTT_TX_TCL_METADATA_V2_VALID_HTT_SET(_var, _val)
  72. #define DP_TX_TCL_METADATA_TYPE_SET(_var, _val) \
  73. HTT_TX_TCL_METADATA_TYPE_V2_SET(_var, _val)
  74. #define DP_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val) \
  75. HTT_TX_TCL_METADATA_V2_HOST_INSPECTED_SET(_var, _val)
  76. #define DP_TX_TCL_METADATA_PEER_ID_SET(_var, _val) \
  77. HTT_TX_TCL_METADATA_V2_PEER_ID_SET(_var, _val)
  78. #define DP_TX_TCL_METADATA_VDEV_ID_SET(_var, _val) \
  79. HTT_TX_TCL_METADATA_V2_VDEV_ID_SET(_var, _val)
  80. #define DP_TCL_METADATA_TYPE_PEER_BASED \
  81. HTT_TCL_METADATA_V2_TYPE_PEER_BASED
  82. #define DP_TCL_METADATA_TYPE_VDEV_BASED \
  83. HTT_TCL_METADATA_V2_TYPE_VDEV_BASED
  84. #else
  85. #define DP_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)\
  86. HTT_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)
  87. #define DP_TX_TCL_METADATA_VALID_HTT_SET(_var, _val) \
  88. HTT_TX_TCL_METADATA_VALID_HTT_SET(_var, _val)
  89. #define DP_TX_TCL_METADATA_TYPE_SET(_var, _val) \
  90. HTT_TX_TCL_METADATA_TYPE_SET(_var, _val)
  91. #define DP_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val) \
  92. HTT_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val)
  93. #define DP_TX_TCL_METADATA_PEER_ID_SET(_var, _val) \
  94. HTT_TX_TCL_METADATA_PEER_ID_SET(_var, _val)
  95. #define DP_TX_TCL_METADATA_VDEV_ID_SET(_var, _val) \
  96. HTT_TX_TCL_METADATA_VDEV_ID_SET(_var, _val)
  97. #define DP_TCL_METADATA_TYPE_PEER_BASED \
  98. HTT_TCL_METADATA_TYPE_PEER_BASED
  99. #define DP_TCL_METADATA_TYPE_VDEV_BASED \
  100. HTT_TCL_METADATA_TYPE_VDEV_BASED
  101. #endif
  102. #define DP_GET_HW_LINK_ID_FRM_PPDU_ID(PPDU_ID, LINK_ID_OFFSET, LINK_ID_BITS) \
  103. (((PPDU_ID) >> (LINK_ID_OFFSET)) & ((1 << (LINK_ID_BITS)) - 1))
  104. /*mapping between hal encrypt type and cdp_sec_type*/
  105. uint8_t sec_type_map[MAX_CDP_SEC_TYPE] = {HAL_TX_ENCRYPT_TYPE_NO_CIPHER,
  106. HAL_TX_ENCRYPT_TYPE_WEP_128,
  107. HAL_TX_ENCRYPT_TYPE_WEP_104,
  108. HAL_TX_ENCRYPT_TYPE_WEP_40,
  109. HAL_TX_ENCRYPT_TYPE_TKIP_WITH_MIC,
  110. HAL_TX_ENCRYPT_TYPE_TKIP_NO_MIC,
  111. HAL_TX_ENCRYPT_TYPE_AES_CCMP_128,
  112. HAL_TX_ENCRYPT_TYPE_WAPI,
  113. HAL_TX_ENCRYPT_TYPE_AES_CCMP_256,
  114. HAL_TX_ENCRYPT_TYPE_AES_GCMP_128,
  115. HAL_TX_ENCRYPT_TYPE_AES_GCMP_256,
  116. HAL_TX_ENCRYPT_TYPE_WAPI_GCM_SM4};
  117. qdf_export_symbol(sec_type_map);
  118. #ifdef WLAN_FEATURE_DP_TX_DESC_HISTORY
  119. static inline enum dp_tx_event_type dp_tx_get_event_type(uint32_t flags)
  120. {
  121. enum dp_tx_event_type type;
  122. if (flags & DP_TX_DESC_FLAG_FLUSH)
  123. type = DP_TX_DESC_FLUSH;
  124. else if (flags & DP_TX_DESC_FLAG_TX_COMP_ERR)
  125. type = DP_TX_COMP_UNMAP_ERR;
  126. else if (flags & DP_TX_DESC_FLAG_COMPLETED_TX)
  127. type = DP_TX_COMP_UNMAP;
  128. else
  129. type = DP_TX_DESC_UNMAP;
  130. return type;
  131. }
  132. static inline void
  133. dp_tx_desc_history_add(struct dp_soc *soc, dma_addr_t paddr,
  134. qdf_nbuf_t skb, uint32_t sw_cookie,
  135. enum dp_tx_event_type type)
  136. {
  137. struct dp_tx_tcl_history *tx_tcl_history = &soc->tx_tcl_history;
  138. struct dp_tx_comp_history *tx_comp_history = &soc->tx_comp_history;
  139. struct dp_tx_desc_event *entry;
  140. uint32_t idx;
  141. uint16_t slot;
  142. switch (type) {
  143. case DP_TX_COMP_UNMAP:
  144. case DP_TX_COMP_UNMAP_ERR:
  145. case DP_TX_COMP_MSDU_EXT:
  146. if (qdf_unlikely(!tx_comp_history->allocated))
  147. return;
  148. dp_get_frag_hist_next_atomic_idx(&tx_comp_history->index, &idx,
  149. &slot,
  150. DP_TX_COMP_HIST_SLOT_SHIFT,
  151. DP_TX_COMP_HIST_PER_SLOT_MAX,
  152. DP_TX_COMP_HISTORY_SIZE);
  153. entry = &tx_comp_history->entry[slot][idx];
  154. break;
  155. case DP_TX_DESC_MAP:
  156. case DP_TX_DESC_UNMAP:
  157. case DP_TX_DESC_COOKIE:
  158. case DP_TX_DESC_FLUSH:
  159. if (qdf_unlikely(!tx_tcl_history->allocated))
  160. return;
  161. dp_get_frag_hist_next_atomic_idx(&tx_tcl_history->index, &idx,
  162. &slot,
  163. DP_TX_TCL_HIST_SLOT_SHIFT,
  164. DP_TX_TCL_HIST_PER_SLOT_MAX,
  165. DP_TX_TCL_HISTORY_SIZE);
  166. entry = &tx_tcl_history->entry[slot][idx];
  167. break;
  168. default:
  169. dp_info_rl("Invalid dp_tx_event_type: %d", type);
  170. return;
  171. }
  172. entry->skb = skb;
  173. entry->paddr = paddr;
  174. entry->sw_cookie = sw_cookie;
  175. entry->type = type;
  176. entry->ts = qdf_get_log_timestamp();
  177. }
  178. static inline void
  179. dp_tx_tso_seg_history_add(struct dp_soc *soc,
  180. struct qdf_tso_seg_elem_t *tso_seg,
  181. qdf_nbuf_t skb, uint32_t sw_cookie,
  182. enum dp_tx_event_type type)
  183. {
  184. int i;
  185. for (i = 1; i < tso_seg->seg.num_frags; i++) {
  186. dp_tx_desc_history_add(soc, tso_seg->seg.tso_frags[i].paddr,
  187. skb, sw_cookie, type);
  188. }
  189. if (!tso_seg->next)
  190. dp_tx_desc_history_add(soc, tso_seg->seg.tso_frags[0].paddr,
  191. skb, 0xFFFFFFFF, type);
  192. }
  193. static inline void
  194. dp_tx_tso_history_add(struct dp_soc *soc, struct qdf_tso_info_t tso_info,
  195. qdf_nbuf_t skb, uint32_t sw_cookie,
  196. enum dp_tx_event_type type)
  197. {
  198. struct qdf_tso_seg_elem_t *curr_seg = tso_info.tso_seg_list;
  199. uint32_t num_segs = tso_info.num_segs;
  200. while (num_segs) {
  201. dp_tx_tso_seg_history_add(soc, curr_seg, skb, sw_cookie, type);
  202. curr_seg = curr_seg->next;
  203. num_segs--;
  204. }
  205. }
  206. #else
  207. static inline enum dp_tx_event_type dp_tx_get_event_type(uint32_t flags)
  208. {
  209. return DP_TX_DESC_INVAL_EVT;
  210. }
  211. static inline void
  212. dp_tx_desc_history_add(struct dp_soc *soc, dma_addr_t paddr,
  213. qdf_nbuf_t skb, uint32_t sw_cookie,
  214. enum dp_tx_event_type type)
  215. {
  216. }
  217. static inline void
  218. dp_tx_tso_seg_history_add(struct dp_soc *soc,
  219. struct qdf_tso_seg_elem_t *tso_seg,
  220. qdf_nbuf_t skb, uint32_t sw_cookie,
  221. enum dp_tx_event_type type)
  222. {
  223. }
  224. static inline void
  225. dp_tx_tso_history_add(struct dp_soc *soc, struct qdf_tso_info_t tso_info,
  226. qdf_nbuf_t skb, uint32_t sw_cookie,
  227. enum dp_tx_event_type type)
  228. {
  229. }
  230. #endif /* WLAN_FEATURE_DP_TX_DESC_HISTORY */
  231. /**
  232. * dp_is_tput_high() - Check if throughput is high
  233. *
  234. * @soc: core txrx main context
  235. *
  236. * The current function is based of the RTPM tput policy variable where RTPM is
  237. * avoided based on throughput.
  238. */
  239. static inline int dp_is_tput_high(struct dp_soc *soc)
  240. {
  241. return dp_get_rtpm_tput_policy_requirement(soc);
  242. }
  243. #if defined(FEATURE_TSO)
  244. /**
  245. * dp_tx_tso_unmap_segment() - Unmap TSO segment
  246. *
  247. * @soc: core txrx main context
  248. * @seg_desc: tso segment descriptor
  249. * @num_seg_desc: tso number segment descriptor
  250. */
  251. static void dp_tx_tso_unmap_segment(
  252. struct dp_soc *soc,
  253. struct qdf_tso_seg_elem_t *seg_desc,
  254. struct qdf_tso_num_seg_elem_t *num_seg_desc)
  255. {
  256. TSO_DEBUG("%s: Unmap the tso segment", __func__);
  257. if (qdf_unlikely(!seg_desc)) {
  258. DP_TRACE(ERROR, "%s %d TSO desc is NULL!",
  259. __func__, __LINE__);
  260. qdf_assert(0);
  261. } else if (qdf_unlikely(!num_seg_desc)) {
  262. DP_TRACE(ERROR, "%s %d TSO num desc is NULL!",
  263. __func__, __LINE__);
  264. qdf_assert(0);
  265. } else {
  266. bool is_last_seg;
  267. /* no tso segment left to do dma unmap */
  268. if (num_seg_desc->num_seg.tso_cmn_num_seg < 1)
  269. return;
  270. is_last_seg = (num_seg_desc->num_seg.tso_cmn_num_seg == 1) ?
  271. true : false;
  272. qdf_nbuf_unmap_tso_segment(soc->osdev,
  273. seg_desc, is_last_seg);
  274. num_seg_desc->num_seg.tso_cmn_num_seg--;
  275. }
  276. }
  277. /**
  278. * dp_tx_tso_desc_release() - Release the tso segment and tso_cmn_num_seg
  279. * back to the freelist
  280. *
  281. * @soc: soc device handle
  282. * @tx_desc: Tx software descriptor
  283. */
  284. static void dp_tx_tso_desc_release(struct dp_soc *soc,
  285. struct dp_tx_desc_s *tx_desc)
  286. {
  287. TSO_DEBUG("%s: Free the tso descriptor", __func__);
  288. if (qdf_unlikely(!tx_desc->msdu_ext_desc->tso_desc)) {
  289. dp_tx_err("SO desc is NULL!");
  290. qdf_assert(0);
  291. } else if (qdf_unlikely(!tx_desc->msdu_ext_desc->tso_num_desc)) {
  292. dp_tx_err("TSO num desc is NULL!");
  293. qdf_assert(0);
  294. } else {
  295. struct qdf_tso_num_seg_elem_t *tso_num_desc =
  296. (struct qdf_tso_num_seg_elem_t *)tx_desc->
  297. msdu_ext_desc->tso_num_desc;
  298. /* Add the tso num segment into the free list */
  299. if (tso_num_desc->num_seg.tso_cmn_num_seg == 0) {
  300. dp_tso_num_seg_free(soc, tx_desc->pool_id,
  301. tx_desc->msdu_ext_desc->
  302. tso_num_desc);
  303. tx_desc->msdu_ext_desc->tso_num_desc = NULL;
  304. DP_STATS_INC(tx_desc->pdev, tso_stats.tso_comp, 1);
  305. }
  306. /* Add the tso segment into the free list*/
  307. dp_tx_tso_desc_free(soc,
  308. tx_desc->pool_id, tx_desc->msdu_ext_desc->
  309. tso_desc);
  310. tx_desc->msdu_ext_desc->tso_desc = NULL;
  311. }
  312. }
  313. #else
  314. static void dp_tx_tso_unmap_segment(
  315. struct dp_soc *soc,
  316. struct qdf_tso_seg_elem_t *seg_desc,
  317. struct qdf_tso_num_seg_elem_t *num_seg_desc)
  318. {
  319. }
  320. static void dp_tx_tso_desc_release(struct dp_soc *soc,
  321. struct dp_tx_desc_s *tx_desc)
  322. {
  323. }
  324. #endif
  325. #ifdef WLAN_SUPPORT_PPEDS
  326. static inline int
  327. dp_tx_release_ds_tx_desc(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  328. uint8_t desc_pool_id)
  329. {
  330. if (tx_desc->flags & DP_TX_DESC_FLAG_PPEDS) {
  331. __dp_tx_outstanding_dec(soc);
  332. dp_tx_desc_free(soc, tx_desc, desc_pool_id);
  333. return 1;
  334. }
  335. return 0;
  336. }
  337. #else
  338. static inline int
  339. dp_tx_release_ds_tx_desc(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  340. uint8_t desc_pool_id)
  341. {
  342. return 0;
  343. }
  344. #endif
  345. void
  346. dp_tx_desc_release(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  347. uint8_t desc_pool_id)
  348. {
  349. struct dp_pdev *pdev = tx_desc->pdev;
  350. uint8_t comp_status = 0;
  351. if (dp_tx_release_ds_tx_desc(soc, tx_desc, desc_pool_id))
  352. return;
  353. qdf_assert(pdev);
  354. soc = pdev->soc;
  355. dp_tx_outstanding_dec(pdev);
  356. if (tx_desc->msdu_ext_desc) {
  357. if (tx_desc->frm_type == dp_tx_frm_tso)
  358. dp_tx_tso_desc_release(soc, tx_desc);
  359. if (tx_desc->flags & DP_TX_DESC_FLAG_ME)
  360. dp_tx_me_free_buf(tx_desc->pdev,
  361. tx_desc->msdu_ext_desc->me_buffer);
  362. dp_tx_ext_desc_free(soc, tx_desc->msdu_ext_desc, desc_pool_id);
  363. tx_desc->msdu_ext_desc = NULL;
  364. }
  365. if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)
  366. qdf_atomic_dec(&soc->num_tx_exception);
  367. if (HAL_TX_COMP_RELEASE_SOURCE_TQM ==
  368. tx_desc->buffer_src)
  369. comp_status = hal_tx_comp_get_release_reason(&tx_desc->comp,
  370. soc->hal_soc);
  371. else
  372. comp_status = HAL_TX_COMP_RELEASE_REASON_FW;
  373. if (soc->dp_debug_log_en) {
  374. dp_tx_debug("Tx Completion Release desc %d status %d outstanding %d",
  375. tx_desc->id, comp_status,
  376. qdf_atomic_read(&pdev->num_tx_outstanding));
  377. }
  378. if (tx_desc->flags & DP_TX_DESC_FLAG_SPECIAL)
  379. dp_tx_spcl_desc_free(soc, tx_desc, desc_pool_id);
  380. else
  381. dp_tx_desc_free(soc, tx_desc, desc_pool_id);
  382. return;
  383. }
  384. /**
  385. * dp_tx_prepare_htt_metadata() - Prepare HTT metadata for special frames
  386. * @vdev: DP vdev Handle
  387. * @nbuf: skb
  388. * @msdu_info: msdu_info required to create HTT metadata
  389. *
  390. * Prepares and fills HTT metadata in the frame pre-header for special frames
  391. * that should be transmitted using varying transmit parameters.
  392. * There are 2 VDEV modes that currently needs this special metadata -
  393. * 1) Mesh Mode
  394. * 2) DSRC Mode
  395. *
  396. * Return: HTT metadata size
  397. *
  398. */
  399. static uint8_t dp_tx_prepare_htt_metadata(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  400. struct dp_tx_msdu_info_s *msdu_info)
  401. {
  402. uint32_t *meta_data = msdu_info->meta_data;
  403. struct htt_tx_msdu_desc_ext2_t *desc_ext =
  404. (struct htt_tx_msdu_desc_ext2_t *) meta_data;
  405. uint8_t htt_desc_size;
  406. /* Size rounded of multiple of 8 bytes */
  407. uint8_t htt_desc_size_aligned;
  408. uint8_t *hdr = NULL;
  409. /*
  410. * Metadata - HTT MSDU Extension header
  411. */
  412. htt_desc_size = sizeof(struct htt_tx_msdu_desc_ext2_t);
  413. htt_desc_size_aligned = (htt_desc_size + 7) & ~0x7;
  414. if (vdev->mesh_vdev || msdu_info->is_tx_sniffer ||
  415. HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_GET(msdu_info->
  416. meta_data[0]) ||
  417. msdu_info->exception_fw) {
  418. if (qdf_unlikely(qdf_nbuf_headroom(nbuf) <
  419. htt_desc_size_aligned)) {
  420. nbuf = qdf_nbuf_realloc_headroom(nbuf,
  421. htt_desc_size_aligned);
  422. if (!nbuf) {
  423. /*
  424. * qdf_nbuf_realloc_headroom won't do skb_clone
  425. * as skb_realloc_headroom does. so, no free is
  426. * needed here.
  427. */
  428. DP_STATS_INC(vdev,
  429. tx_i[msdu_info->xmit_type].dropped.headroom_insufficient,
  430. 1);
  431. qdf_print(" %s[%d] skb_realloc_headroom failed",
  432. __func__, __LINE__);
  433. return 0;
  434. }
  435. }
  436. /* Fill and add HTT metaheader */
  437. hdr = qdf_nbuf_push_head(nbuf, htt_desc_size_aligned);
  438. if (!hdr) {
  439. dp_tx_err("Error in filling HTT metadata");
  440. return 0;
  441. }
  442. qdf_mem_copy(hdr, desc_ext, htt_desc_size);
  443. } else if (vdev->opmode == wlan_op_mode_ocb) {
  444. /* Todo - Add support for DSRC */
  445. }
  446. return htt_desc_size_aligned;
  447. }
  448. /**
  449. * dp_tx_prepare_tso_ext_desc() - Prepare MSDU extension descriptor for TSO
  450. * @tso_seg: TSO segment to process
  451. * @ext_desc: Pointer to MSDU extension descriptor
  452. *
  453. * Return: void
  454. */
  455. #if defined(FEATURE_TSO)
  456. static void dp_tx_prepare_tso_ext_desc(struct qdf_tso_seg_t *tso_seg,
  457. void *ext_desc)
  458. {
  459. uint8_t num_frag;
  460. uint32_t tso_flags;
  461. /*
  462. * Set tso_en, tcp_flags(NS, CWR, ECE, URG, ACK, PSH, RST, SYN, FIN),
  463. * tcp_flag_mask
  464. *
  465. * Checksum enable flags are set in TCL descriptor and not in Extension
  466. * Descriptor (H/W ignores checksum_en flags in MSDU ext descriptor)
  467. */
  468. tso_flags = *(uint32_t *) &tso_seg->tso_flags;
  469. hal_tx_ext_desc_set_tso_flags(ext_desc, tso_flags);
  470. hal_tx_ext_desc_set_msdu_length(ext_desc, tso_seg->tso_flags.l2_len,
  471. tso_seg->tso_flags.ip_len);
  472. hal_tx_ext_desc_set_tcp_seq(ext_desc, tso_seg->tso_flags.tcp_seq_num);
  473. hal_tx_ext_desc_set_ip_id(ext_desc, tso_seg->tso_flags.ip_id);
  474. for (num_frag = 0; num_frag < tso_seg->num_frags; num_frag++) {
  475. uint32_t lo = 0;
  476. uint32_t hi = 0;
  477. qdf_assert_always((tso_seg->tso_frags[num_frag].paddr) &&
  478. (tso_seg->tso_frags[num_frag].length));
  479. qdf_dmaaddr_to_32s(
  480. tso_seg->tso_frags[num_frag].paddr, &lo, &hi);
  481. hal_tx_ext_desc_set_buffer(ext_desc, num_frag, lo, hi,
  482. tso_seg->tso_frags[num_frag].length);
  483. }
  484. return;
  485. }
  486. #else
  487. static void dp_tx_prepare_tso_ext_desc(struct qdf_tso_seg_t *tso_seg,
  488. void *ext_desc)
  489. {
  490. return;
  491. }
  492. #endif
  493. #if defined(FEATURE_TSO)
  494. /**
  495. * dp_tx_free_tso_seg_list() - Loop through the tso segments
  496. * allocated and free them
  497. * @soc: soc handle
  498. * @free_seg: list of tso segments
  499. * @msdu_info: msdu descriptor
  500. *
  501. * Return: void
  502. */
  503. static void dp_tx_free_tso_seg_list(
  504. struct dp_soc *soc,
  505. struct qdf_tso_seg_elem_t *free_seg,
  506. struct dp_tx_msdu_info_s *msdu_info)
  507. {
  508. struct qdf_tso_seg_elem_t *next_seg;
  509. while (free_seg) {
  510. next_seg = free_seg->next;
  511. dp_tx_tso_desc_free(soc,
  512. msdu_info->tx_queue.desc_pool_id,
  513. free_seg);
  514. free_seg = next_seg;
  515. }
  516. }
  517. /**
  518. * dp_tx_free_tso_num_seg_list() - Loop through the tso num segments
  519. * allocated and free them
  520. * @soc: soc handle
  521. * @free_num_seg: list of tso number segments
  522. * @msdu_info: msdu descriptor
  523. *
  524. * Return: void
  525. */
  526. static void dp_tx_free_tso_num_seg_list(
  527. struct dp_soc *soc,
  528. struct qdf_tso_num_seg_elem_t *free_num_seg,
  529. struct dp_tx_msdu_info_s *msdu_info)
  530. {
  531. struct qdf_tso_num_seg_elem_t *next_num_seg;
  532. while (free_num_seg) {
  533. next_num_seg = free_num_seg->next;
  534. dp_tso_num_seg_free(soc,
  535. msdu_info->tx_queue.desc_pool_id,
  536. free_num_seg);
  537. free_num_seg = next_num_seg;
  538. }
  539. }
  540. /**
  541. * dp_tx_unmap_tso_seg_list() - Loop through the tso segments
  542. * do dma unmap for each segment
  543. * @soc: soc handle
  544. * @free_seg: list of tso segments
  545. * @num_seg_desc: tso number segment descriptor
  546. *
  547. * Return: void
  548. */
  549. static void dp_tx_unmap_tso_seg_list(
  550. struct dp_soc *soc,
  551. struct qdf_tso_seg_elem_t *free_seg,
  552. struct qdf_tso_num_seg_elem_t *num_seg_desc)
  553. {
  554. struct qdf_tso_seg_elem_t *next_seg;
  555. if (qdf_unlikely(!num_seg_desc)) {
  556. DP_TRACE(ERROR, "TSO number seg desc is NULL!");
  557. return;
  558. }
  559. while (free_seg) {
  560. next_seg = free_seg->next;
  561. dp_tx_tso_unmap_segment(soc, free_seg, num_seg_desc);
  562. free_seg = next_seg;
  563. }
  564. }
  565. #ifdef FEATURE_TSO_STATS
  566. /**
  567. * dp_tso_get_stats_idx() - Retrieve the tso packet id
  568. * @pdev: pdev handle
  569. *
  570. * Return: id
  571. */
  572. static uint32_t dp_tso_get_stats_idx(struct dp_pdev *pdev)
  573. {
  574. uint32_t stats_idx;
  575. stats_idx = (((uint32_t)qdf_atomic_inc_return(&pdev->tso_idx))
  576. % CDP_MAX_TSO_PACKETS);
  577. return stats_idx;
  578. }
  579. #else
  580. static int dp_tso_get_stats_idx(struct dp_pdev *pdev)
  581. {
  582. return 0;
  583. }
  584. #endif /* FEATURE_TSO_STATS */
  585. /**
  586. * dp_tx_free_remaining_tso_desc() - do dma unmap for tso segments if any,
  587. * free the tso segments descriptor and
  588. * tso num segments descriptor
  589. * @soc: soc handle
  590. * @msdu_info: msdu descriptor
  591. * @tso_seg_unmap: flag to show if dma unmap is necessary
  592. *
  593. * Return: void
  594. */
  595. static void dp_tx_free_remaining_tso_desc(struct dp_soc *soc,
  596. struct dp_tx_msdu_info_s *msdu_info,
  597. bool tso_seg_unmap)
  598. {
  599. struct qdf_tso_info_t *tso_info = &msdu_info->u.tso_info;
  600. struct qdf_tso_seg_elem_t *free_seg = tso_info->tso_seg_list;
  601. struct qdf_tso_num_seg_elem_t *tso_num_desc =
  602. tso_info->tso_num_seg_list;
  603. /* do dma unmap for each segment */
  604. if (tso_seg_unmap)
  605. dp_tx_unmap_tso_seg_list(soc, free_seg, tso_num_desc);
  606. /* free all tso number segment descriptor though looks only have 1 */
  607. dp_tx_free_tso_num_seg_list(soc, tso_num_desc, msdu_info);
  608. /* free all tso segment descriptor */
  609. dp_tx_free_tso_seg_list(soc, free_seg, msdu_info);
  610. }
  611. /**
  612. * dp_tx_prepare_tso() - Given a jumbo msdu, prepare the TSO info
  613. * @vdev: virtual device handle
  614. * @msdu: network buffer
  615. * @msdu_info: meta data associated with the msdu
  616. *
  617. * Return: QDF_STATUS_SUCCESS success
  618. */
  619. static QDF_STATUS dp_tx_prepare_tso(struct dp_vdev *vdev,
  620. qdf_nbuf_t msdu, struct dp_tx_msdu_info_s *msdu_info)
  621. {
  622. struct qdf_tso_seg_elem_t *tso_seg;
  623. int num_seg = qdf_nbuf_get_tso_num_seg(msdu);
  624. struct dp_soc *soc = vdev->pdev->soc;
  625. struct dp_pdev *pdev = vdev->pdev;
  626. struct qdf_tso_info_t *tso_info;
  627. struct qdf_tso_num_seg_elem_t *tso_num_seg;
  628. tso_info = &msdu_info->u.tso_info;
  629. tso_info->curr_seg = NULL;
  630. tso_info->tso_seg_list = NULL;
  631. tso_info->num_segs = num_seg;
  632. msdu_info->frm_type = dp_tx_frm_tso;
  633. tso_info->tso_num_seg_list = NULL;
  634. TSO_DEBUG(" %s: num_seg: %d", __func__, num_seg);
  635. while (num_seg) {
  636. tso_seg = dp_tx_tso_desc_alloc(
  637. soc, msdu_info->tx_queue.desc_pool_id);
  638. if (tso_seg) {
  639. tso_seg->next = tso_info->tso_seg_list;
  640. tso_info->tso_seg_list = tso_seg;
  641. num_seg--;
  642. } else {
  643. dp_err_rl("Failed to alloc tso seg desc");
  644. DP_STATS_INC_PKT(vdev->pdev,
  645. tso_stats.tso_no_mem_dropped, 1,
  646. qdf_nbuf_len(msdu));
  647. dp_tx_free_remaining_tso_desc(soc, msdu_info, false);
  648. return QDF_STATUS_E_NOMEM;
  649. }
  650. }
  651. TSO_DEBUG(" %s: num_seg: %d", __func__, num_seg);
  652. tso_num_seg = dp_tso_num_seg_alloc(soc,
  653. msdu_info->tx_queue.desc_pool_id);
  654. if (tso_num_seg) {
  655. tso_num_seg->next = tso_info->tso_num_seg_list;
  656. tso_info->tso_num_seg_list = tso_num_seg;
  657. } else {
  658. DP_TRACE(ERROR, "%s: Failed to alloc - Number of segs desc",
  659. __func__);
  660. dp_tx_free_remaining_tso_desc(soc, msdu_info, false);
  661. return QDF_STATUS_E_NOMEM;
  662. }
  663. msdu_info->num_seg =
  664. qdf_nbuf_get_tso_info(soc->osdev, msdu, tso_info);
  665. TSO_DEBUG(" %s: msdu_info->num_seg: %d", __func__,
  666. msdu_info->num_seg);
  667. if (!(msdu_info->num_seg)) {
  668. /*
  669. * Free allocated TSO seg desc and number seg desc,
  670. * do unmap for segments if dma map has done.
  671. */
  672. DP_TRACE(ERROR, "%s: Failed to get tso info", __func__);
  673. dp_tx_free_remaining_tso_desc(soc, msdu_info, true);
  674. return QDF_STATUS_E_INVAL;
  675. }
  676. dp_tx_tso_history_add(soc, msdu_info->u.tso_info,
  677. msdu, 0, DP_TX_DESC_MAP);
  678. tso_info->curr_seg = tso_info->tso_seg_list;
  679. tso_info->msdu_stats_idx = dp_tso_get_stats_idx(pdev);
  680. dp_tso_packet_update(pdev, tso_info->msdu_stats_idx,
  681. msdu, msdu_info->num_seg);
  682. dp_tso_segment_stats_update(pdev, tso_info->tso_seg_list,
  683. tso_info->msdu_stats_idx);
  684. dp_stats_tso_segment_histogram_update(pdev, msdu_info->num_seg);
  685. return QDF_STATUS_SUCCESS;
  686. }
  687. #else
  688. static QDF_STATUS dp_tx_prepare_tso(struct dp_vdev *vdev,
  689. qdf_nbuf_t msdu, struct dp_tx_msdu_info_s *msdu_info)
  690. {
  691. return QDF_STATUS_E_NOMEM;
  692. }
  693. #endif
  694. QDF_COMPILE_TIME_ASSERT(dp_tx_htt_metadata_len_check,
  695. (DP_TX_MSDU_INFO_META_DATA_DWORDS * 4 >=
  696. sizeof(struct htt_tx_msdu_desc_ext2_t)));
  697. /**
  698. * dp_tx_prepare_ext_desc() - Allocate and prepare MSDU extension descriptor
  699. * @vdev: DP Vdev handle
  700. * @msdu_info: MSDU info to be setup in MSDU extension descriptor
  701. * @desc_pool_id: Descriptor Pool ID
  702. *
  703. * Return:
  704. */
  705. static
  706. struct dp_tx_ext_desc_elem_s *dp_tx_prepare_ext_desc(struct dp_vdev *vdev,
  707. struct dp_tx_msdu_info_s *msdu_info, uint8_t desc_pool_id)
  708. {
  709. uint8_t i;
  710. uint8_t cached_ext_desc[HAL_TX_EXT_DESC_WITH_META_DATA];
  711. struct dp_tx_seg_info_s *seg_info;
  712. struct dp_tx_ext_desc_elem_s *msdu_ext_desc;
  713. struct dp_soc *soc = vdev->pdev->soc;
  714. /* Allocate an extension descriptor */
  715. msdu_ext_desc = dp_tx_ext_desc_alloc(soc, desc_pool_id);
  716. qdf_mem_zero(&cached_ext_desc[0], HAL_TX_EXT_DESC_WITH_META_DATA);
  717. if (!msdu_ext_desc) {
  718. DP_STATS_INC(vdev,
  719. tx_i[msdu_info->xmit_type].dropped.desc_na.num, 1);
  720. return NULL;
  721. }
  722. if (msdu_info->exception_fw &&
  723. qdf_unlikely(vdev->mesh_vdev)) {
  724. qdf_mem_copy(&cached_ext_desc[HAL_TX_EXTENSION_DESC_LEN_BYTES],
  725. &msdu_info->meta_data[0],
  726. sizeof(struct htt_tx_msdu_desc_ext2_t));
  727. qdf_atomic_inc(&soc->num_tx_exception);
  728. msdu_ext_desc->flags |= DP_TX_EXT_DESC_FLAG_METADATA_VALID;
  729. }
  730. switch (msdu_info->frm_type) {
  731. case dp_tx_frm_sg:
  732. case dp_tx_frm_me:
  733. case dp_tx_frm_raw:
  734. seg_info = msdu_info->u.sg_info.curr_seg;
  735. /* Update the buffer pointers in MSDU Extension Descriptor */
  736. for (i = 0; i < seg_info->frag_cnt; i++) {
  737. hal_tx_ext_desc_set_buffer(&cached_ext_desc[0], i,
  738. seg_info->frags[i].paddr_lo,
  739. seg_info->frags[i].paddr_hi,
  740. seg_info->frags[i].len);
  741. }
  742. break;
  743. case dp_tx_frm_tso:
  744. dp_tx_prepare_tso_ext_desc(&msdu_info->u.tso_info.curr_seg->seg,
  745. &cached_ext_desc[0]);
  746. break;
  747. default:
  748. break;
  749. }
  750. QDF_TRACE_HEX_DUMP(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_DEBUG,
  751. cached_ext_desc, HAL_TX_EXT_DESC_WITH_META_DATA);
  752. hal_tx_ext_desc_sync(&cached_ext_desc[0],
  753. msdu_ext_desc->vaddr);
  754. return msdu_ext_desc;
  755. }
  756. /**
  757. * dp_tx_trace_pkt() - Trace TX packet at DP layer
  758. * @soc: datapath SOC
  759. * @skb: skb to be traced
  760. * @msdu_id: msdu_id of the packet
  761. * @vdev_id: vdev_id of the packet
  762. * @op_mode: Vdev Operation mode
  763. *
  764. * Return: None
  765. */
  766. #ifdef DP_DISABLE_TX_PKT_TRACE
  767. static void dp_tx_trace_pkt(struct dp_soc *soc,
  768. qdf_nbuf_t skb, uint16_t msdu_id,
  769. uint8_t vdev_id, enum QDF_OPMODE op_mode)
  770. {
  771. }
  772. #else
  773. static void dp_tx_trace_pkt(struct dp_soc *soc,
  774. qdf_nbuf_t skb, uint16_t msdu_id,
  775. uint8_t vdev_id, enum QDF_OPMODE op_mode)
  776. {
  777. if (dp_is_tput_high(soc))
  778. return;
  779. QDF_NBUF_CB_TX_PACKET_TRACK(skb) = QDF_NBUF_TX_PKT_DATA_TRACK;
  780. QDF_NBUF_CB_TX_DP_TRACE(skb) = 1;
  781. DPTRACE(qdf_dp_trace_ptr(skb,
  782. QDF_DP_TRACE_LI_DP_TX_PACKET_PTR_RECORD,
  783. QDF_TRACE_DEFAULT_PDEV_ID,
  784. qdf_nbuf_data_addr(skb),
  785. sizeof(qdf_nbuf_data(skb)),
  786. msdu_id, vdev_id, 0,
  787. op_mode));
  788. qdf_dp_trace_log_pkt(vdev_id, skb, QDF_TX, QDF_TRACE_DEFAULT_PDEV_ID,
  789. op_mode);
  790. DPTRACE(qdf_dp_trace_data_pkt(skb, QDF_TRACE_DEFAULT_PDEV_ID,
  791. QDF_DP_TRACE_LI_DP_TX_PACKET_RECORD,
  792. msdu_id, QDF_TX));
  793. }
  794. #endif
  795. #ifdef WLAN_DP_FEATURE_MARK_ICMP_REQ_TO_FW
  796. /**
  797. * dp_tx_is_nbuf_marked_exception() - Check if the packet has been marked as
  798. * exception by the upper layer (OS_IF)
  799. * @soc: DP soc handle
  800. * @nbuf: packet to be transmitted
  801. *
  802. * Return: 1 if the packet is marked as exception,
  803. * 0, if the packet is not marked as exception.
  804. */
  805. static inline int dp_tx_is_nbuf_marked_exception(struct dp_soc *soc,
  806. qdf_nbuf_t nbuf)
  807. {
  808. return QDF_NBUF_CB_TX_PACKET_TO_FW(nbuf);
  809. }
  810. #else
  811. static inline int dp_tx_is_nbuf_marked_exception(struct dp_soc *soc,
  812. qdf_nbuf_t nbuf)
  813. {
  814. return 0;
  815. }
  816. #endif
  817. #ifdef DP_TRAFFIC_END_INDICATION
  818. /**
  819. * dp_tx_get_traffic_end_indication_pkt() - Allocate and prepare packet to send
  820. * as indication to fw to inform that
  821. * data stream has ended
  822. * @vdev: DP vdev handle
  823. * @nbuf: original buffer from network stack
  824. *
  825. * Return: NULL on failure,
  826. * nbuf on success
  827. */
  828. static inline qdf_nbuf_t
  829. dp_tx_get_traffic_end_indication_pkt(struct dp_vdev *vdev,
  830. qdf_nbuf_t nbuf)
  831. {
  832. /* Packet length should be enough to copy upto L3 header */
  833. uint8_t end_nbuf_len = 64;
  834. uint8_t htt_desc_size_aligned;
  835. uint8_t htt_desc_size;
  836. qdf_nbuf_t end_nbuf;
  837. if (qdf_unlikely(QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) ==
  838. QDF_NBUF_CB_PACKET_TYPE_END_INDICATION)) {
  839. htt_desc_size = sizeof(struct htt_tx_msdu_desc_ext2_t);
  840. htt_desc_size_aligned = (htt_desc_size + 7) & ~0x7;
  841. end_nbuf = qdf_nbuf_queue_remove(&vdev->end_ind_pkt_q);
  842. if (!end_nbuf) {
  843. end_nbuf = qdf_nbuf_alloc(NULL,
  844. (htt_desc_size_aligned +
  845. end_nbuf_len),
  846. htt_desc_size_aligned,
  847. 8, false);
  848. if (!end_nbuf) {
  849. dp_err("Packet allocation failed");
  850. goto out;
  851. }
  852. } else {
  853. qdf_nbuf_reset(end_nbuf, htt_desc_size_aligned, 8);
  854. }
  855. qdf_mem_copy(qdf_nbuf_data(end_nbuf), qdf_nbuf_data(nbuf),
  856. end_nbuf_len);
  857. qdf_nbuf_set_pktlen(end_nbuf, end_nbuf_len);
  858. return end_nbuf;
  859. }
  860. out:
  861. return NULL;
  862. }
  863. /**
  864. * dp_tx_send_traffic_end_indication_pkt() - Send indication packet to FW
  865. * via exception path.
  866. * @vdev: DP vdev handle
  867. * @end_nbuf: skb to send as indication
  868. * @msdu_info: msdu_info of original nbuf
  869. * @peer_id: peer id
  870. *
  871. * Return: None
  872. */
  873. static inline void
  874. dp_tx_send_traffic_end_indication_pkt(struct dp_vdev *vdev,
  875. qdf_nbuf_t end_nbuf,
  876. struct dp_tx_msdu_info_s *msdu_info,
  877. uint16_t peer_id)
  878. {
  879. struct dp_tx_msdu_info_s e_msdu_info = {0};
  880. qdf_nbuf_t nbuf;
  881. struct htt_tx_msdu_desc_ext2_t *desc_ext =
  882. (struct htt_tx_msdu_desc_ext2_t *)(e_msdu_info.meta_data);
  883. e_msdu_info.tx_queue = msdu_info->tx_queue;
  884. e_msdu_info.tid = msdu_info->tid;
  885. e_msdu_info.exception_fw = 1;
  886. e_msdu_info.xmit_type = msdu_info->xmit_type;
  887. desc_ext->host_tx_desc_pool = 1;
  888. desc_ext->traffic_end_indication = 1;
  889. nbuf = dp_tx_send_msdu_single(vdev, end_nbuf, &e_msdu_info,
  890. peer_id, NULL);
  891. if (nbuf) {
  892. dp_err("Traffic end indication packet tx failed");
  893. qdf_nbuf_free(nbuf);
  894. }
  895. }
  896. /**
  897. * dp_tx_traffic_end_indication_set_desc_flag() - Set tx descriptor flag to
  898. * mark it traffic end indication
  899. * packet.
  900. * @tx_desc: Tx descriptor pointer
  901. * @msdu_info: msdu_info structure pointer
  902. *
  903. * Return: None
  904. */
  905. static inline void
  906. dp_tx_traffic_end_indication_set_desc_flag(struct dp_tx_desc_s *tx_desc,
  907. struct dp_tx_msdu_info_s *msdu_info)
  908. {
  909. struct htt_tx_msdu_desc_ext2_t *desc_ext =
  910. (struct htt_tx_msdu_desc_ext2_t *)(msdu_info->meta_data);
  911. if (qdf_unlikely(desc_ext->traffic_end_indication))
  912. tx_desc->flags |= DP_TX_DESC_FLAG_TRAFFIC_END_IND;
  913. }
  914. /**
  915. * dp_tx_traffic_end_indication_enq_ind_pkt() - Enqueue the packet instead of
  916. * freeing which are associated
  917. * with traffic end indication
  918. * flagged descriptor.
  919. * @soc: dp soc handle
  920. * @desc: Tx descriptor pointer
  921. * @nbuf: buffer pointer
  922. *
  923. * Return: True if packet gets enqueued else false
  924. */
  925. static bool
  926. dp_tx_traffic_end_indication_enq_ind_pkt(struct dp_soc *soc,
  927. struct dp_tx_desc_s *desc,
  928. qdf_nbuf_t nbuf)
  929. {
  930. struct dp_vdev *vdev = NULL;
  931. if (qdf_unlikely((desc->flags &
  932. DP_TX_DESC_FLAG_TRAFFIC_END_IND) != 0)) {
  933. vdev = dp_vdev_get_ref_by_id(soc, desc->vdev_id,
  934. DP_MOD_ID_TX_COMP);
  935. if (vdev) {
  936. qdf_nbuf_queue_add(&vdev->end_ind_pkt_q, nbuf);
  937. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_COMP);
  938. return true;
  939. }
  940. }
  941. return false;
  942. }
  943. /**
  944. * dp_tx_traffic_end_indication_is_enabled() - get the feature
  945. * enable/disable status
  946. * @vdev: dp vdev handle
  947. *
  948. * Return: True if feature is enable else false
  949. */
  950. static inline bool
  951. dp_tx_traffic_end_indication_is_enabled(struct dp_vdev *vdev)
  952. {
  953. return qdf_unlikely(vdev->traffic_end_ind_en);
  954. }
  955. static inline qdf_nbuf_t
  956. dp_tx_send_msdu_single_wrapper(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  957. struct dp_tx_msdu_info_s *msdu_info,
  958. uint16_t peer_id, qdf_nbuf_t end_nbuf)
  959. {
  960. if (dp_tx_traffic_end_indication_is_enabled(vdev))
  961. end_nbuf = dp_tx_get_traffic_end_indication_pkt(vdev, nbuf);
  962. nbuf = dp_tx_send_msdu_single(vdev, nbuf, msdu_info, peer_id, NULL);
  963. if (qdf_unlikely(end_nbuf))
  964. dp_tx_send_traffic_end_indication_pkt(vdev, end_nbuf,
  965. msdu_info, peer_id);
  966. return nbuf;
  967. }
  968. #else
  969. static inline qdf_nbuf_t
  970. dp_tx_get_traffic_end_indication_pkt(struct dp_vdev *vdev,
  971. qdf_nbuf_t nbuf)
  972. {
  973. return NULL;
  974. }
  975. static inline void
  976. dp_tx_send_traffic_end_indication_pkt(struct dp_vdev *vdev,
  977. qdf_nbuf_t end_nbuf,
  978. struct dp_tx_msdu_info_s *msdu_info,
  979. uint16_t peer_id)
  980. {}
  981. static inline void
  982. dp_tx_traffic_end_indication_set_desc_flag(struct dp_tx_desc_s *tx_desc,
  983. struct dp_tx_msdu_info_s *msdu_info)
  984. {}
  985. static inline bool
  986. dp_tx_traffic_end_indication_enq_ind_pkt(struct dp_soc *soc,
  987. struct dp_tx_desc_s *desc,
  988. qdf_nbuf_t nbuf)
  989. {
  990. return false;
  991. }
  992. static inline bool
  993. dp_tx_traffic_end_indication_is_enabled(struct dp_vdev *vdev)
  994. {
  995. return false;
  996. }
  997. static inline qdf_nbuf_t
  998. dp_tx_send_msdu_single_wrapper(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  999. struct dp_tx_msdu_info_s *msdu_info,
  1000. uint16_t peer_id, qdf_nbuf_t end_nbuf)
  1001. {
  1002. return dp_tx_send_msdu_single(vdev, nbuf, msdu_info, peer_id, NULL);
  1003. }
  1004. #endif
  1005. #if defined(QCA_SUPPORT_WDS_EXTENDED)
  1006. static bool
  1007. dp_tx_is_wds_ast_override_en(struct dp_soc *soc,
  1008. struct cdp_tx_exception_metadata *tx_exc_metadata)
  1009. {
  1010. if (soc->features.wds_ext_ast_override_enable &&
  1011. tx_exc_metadata && tx_exc_metadata->is_wds_extended)
  1012. return true;
  1013. return false;
  1014. }
  1015. #else
  1016. static bool
  1017. dp_tx_is_wds_ast_override_en(struct dp_soc *soc,
  1018. struct cdp_tx_exception_metadata *tx_exc_metadata)
  1019. {
  1020. return false;
  1021. }
  1022. #endif
  1023. /**
  1024. * dp_tx_prepare_desc_single() - Allocate and prepare Tx descriptor
  1025. * @vdev: DP vdev handle
  1026. * @nbuf: skb
  1027. * @desc_pool_id: Descriptor pool ID
  1028. * @msdu_info: Metadata to the fw
  1029. * @tx_exc_metadata: Handle that holds exception path metadata
  1030. *
  1031. * Allocate and prepare Tx descriptor with msdu information.
  1032. *
  1033. * Return: Pointer to Tx Descriptor on success,
  1034. * NULL on failure
  1035. */
  1036. static
  1037. struct dp_tx_desc_s *dp_tx_prepare_desc_single(struct dp_vdev *vdev,
  1038. qdf_nbuf_t nbuf, uint8_t desc_pool_id,
  1039. struct dp_tx_msdu_info_s *msdu_info,
  1040. struct cdp_tx_exception_metadata *tx_exc_metadata)
  1041. {
  1042. uint8_t align_pad;
  1043. uint8_t is_exception = 0;
  1044. uint8_t htt_hdr_size;
  1045. struct dp_tx_desc_s *tx_desc;
  1046. struct dp_pdev *pdev = vdev->pdev;
  1047. struct dp_soc *soc = pdev->soc;
  1048. uint8_t xmit_type = msdu_info->xmit_type;
  1049. if (dp_tx_limit_check(vdev, nbuf))
  1050. return NULL;
  1051. /* Allocate software Tx descriptor */
  1052. if (nbuf->protocol == QDF_NBUF_TRAC_EAPOL_ETH_TYPE)
  1053. tx_desc = dp_tx_spcl_desc_alloc(soc, desc_pool_id);
  1054. else
  1055. tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
  1056. if (qdf_unlikely(!tx_desc)) {
  1057. DP_STATS_INC(vdev,
  1058. tx_i[xmit_type].dropped.desc_na.num, 1);
  1059. DP_STATS_INC(vdev,
  1060. tx_i[xmit_type].dropped.desc_na_exc_alloc_fail.num,
  1061. 1);
  1062. return NULL;
  1063. }
  1064. dp_tx_outstanding_inc(pdev);
  1065. /* Initialize the SW tx descriptor */
  1066. tx_desc->nbuf = nbuf;
  1067. tx_desc->frm_type = dp_tx_frm_std;
  1068. tx_desc->tx_encap_type = ((tx_exc_metadata &&
  1069. (tx_exc_metadata->tx_encap_type != CDP_INVALID_TX_ENCAP_TYPE)) ?
  1070. tx_exc_metadata->tx_encap_type : vdev->tx_encap_type);
  1071. tx_desc->vdev_id = vdev->vdev_id;
  1072. tx_desc->pdev = pdev;
  1073. tx_desc->msdu_ext_desc = NULL;
  1074. tx_desc->pkt_offset = 0;
  1075. tx_desc->length = qdf_nbuf_headlen(nbuf);
  1076. dp_tx_trace_pkt(soc, nbuf, tx_desc->id, vdev->vdev_id,
  1077. vdev->qdf_opmode);
  1078. if (qdf_unlikely(vdev->multipass_en)) {
  1079. if (!dp_tx_multipass_process(soc, vdev, nbuf, msdu_info))
  1080. goto failure;
  1081. }
  1082. /* Packets marked by upper layer (OS-IF) to be sent to FW */
  1083. if (dp_tx_is_nbuf_marked_exception(soc, nbuf))
  1084. is_exception = 1;
  1085. /* for BE chipsets if wds extension was enbled will not mark FW
  1086. * in desc will mark ast index based search for ast index.
  1087. */
  1088. if (dp_tx_is_wds_ast_override_en(soc, tx_exc_metadata))
  1089. return tx_desc;
  1090. /*
  1091. * For special modes (vdev_type == ocb or mesh), data frames should be
  1092. * transmitted using varying transmit parameters (tx spec) which include
  1093. * transmit rate, power, priority, channel, channel bandwidth , nss etc.
  1094. * These are filled in HTT MSDU descriptor and sent in frame pre-header.
  1095. * These frames are sent as exception packets to firmware.
  1096. *
  1097. * HW requirement is that metadata should always point to a
  1098. * 8-byte aligned address. So we add alignment pad to start of buffer.
  1099. * HTT Metadata should be ensured to be multiple of 8-bytes,
  1100. * to get 8-byte aligned start address along with align_pad added
  1101. *
  1102. * |-----------------------------|
  1103. * | |
  1104. * |-----------------------------| <-----Buffer Pointer Address given
  1105. * | | ^ in HW descriptor (aligned)
  1106. * | HTT Metadata | |
  1107. * | | |
  1108. * | | | Packet Offset given in descriptor
  1109. * | | |
  1110. * |-----------------------------| |
  1111. * | Alignment Pad | v
  1112. * |-----------------------------| <----- Actual buffer start address
  1113. * | SKB Data | (Unaligned)
  1114. * | |
  1115. * | |
  1116. * | |
  1117. * | |
  1118. * | |
  1119. * |-----------------------------|
  1120. */
  1121. if (qdf_unlikely((msdu_info->exception_fw)) ||
  1122. (vdev->opmode == wlan_op_mode_ocb) ||
  1123. (tx_exc_metadata &&
  1124. tx_exc_metadata->is_tx_sniffer)) {
  1125. align_pad = ((unsigned long) qdf_nbuf_data(nbuf)) & 0x7;
  1126. if (qdf_unlikely(qdf_nbuf_headroom(nbuf) < align_pad)) {
  1127. DP_STATS_INC(vdev,
  1128. tx_i[xmit_type].dropped.headroom_insufficient,
  1129. 1);
  1130. goto failure;
  1131. }
  1132. if (qdf_nbuf_push_head(nbuf, align_pad) == NULL) {
  1133. dp_tx_err("qdf_nbuf_push_head failed");
  1134. goto failure;
  1135. }
  1136. htt_hdr_size = dp_tx_prepare_htt_metadata(vdev, nbuf,
  1137. msdu_info);
  1138. if (htt_hdr_size == 0)
  1139. goto failure;
  1140. tx_desc->length = qdf_nbuf_headlen(nbuf);
  1141. tx_desc->pkt_offset = align_pad + htt_hdr_size;
  1142. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1143. dp_tx_traffic_end_indication_set_desc_flag(tx_desc,
  1144. msdu_info);
  1145. is_exception = 1;
  1146. tx_desc->length -= tx_desc->pkt_offset;
  1147. }
  1148. #if !TQM_BYPASS_WAR
  1149. if (is_exception || tx_exc_metadata)
  1150. #endif
  1151. {
  1152. /* Temporary WAR due to TQM VP issues */
  1153. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1154. qdf_atomic_inc(&soc->num_tx_exception);
  1155. }
  1156. return tx_desc;
  1157. failure:
  1158. dp_tx_desc_release(soc, tx_desc, desc_pool_id);
  1159. return NULL;
  1160. }
  1161. /**
  1162. * dp_tx_prepare_desc() - Allocate and prepare Tx descriptor for multisegment
  1163. * frame
  1164. * @vdev: DP vdev handle
  1165. * @nbuf: skb
  1166. * @msdu_info: Info to be setup in MSDU descriptor and MSDU extension descriptor
  1167. * @desc_pool_id : Descriptor Pool ID
  1168. *
  1169. * Allocate and prepare Tx descriptor with msdu and fragment descritor
  1170. * information. For frames with fragments, allocate and prepare
  1171. * an MSDU extension descriptor
  1172. *
  1173. * Return: Pointer to Tx Descriptor on success,
  1174. * NULL on failure
  1175. */
  1176. static struct dp_tx_desc_s *dp_tx_prepare_desc(struct dp_vdev *vdev,
  1177. qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info,
  1178. uint8_t desc_pool_id)
  1179. {
  1180. struct dp_tx_desc_s *tx_desc;
  1181. struct dp_tx_ext_desc_elem_s *msdu_ext_desc;
  1182. struct dp_pdev *pdev = vdev->pdev;
  1183. struct dp_soc *soc = pdev->soc;
  1184. if (dp_tx_limit_check(vdev, nbuf))
  1185. return NULL;
  1186. /* Allocate software Tx descriptor */
  1187. if (nbuf->protocol == QDF_NBUF_TRAC_EAPOL_ETH_TYPE)
  1188. tx_desc = dp_tx_spcl_desc_alloc(soc, desc_pool_id);
  1189. else
  1190. tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
  1191. if (!tx_desc) {
  1192. DP_STATS_INC(vdev,
  1193. tx_i[msdu_info->xmit_type].dropped.desc_na.num, 1);
  1194. return NULL;
  1195. }
  1196. dp_tx_tso_seg_history_add(soc, msdu_info->u.tso_info.curr_seg,
  1197. nbuf, tx_desc->id, DP_TX_DESC_COOKIE);
  1198. dp_tx_outstanding_inc(pdev);
  1199. /* Initialize the SW tx descriptor */
  1200. tx_desc->nbuf = nbuf;
  1201. tx_desc->frm_type = msdu_info->frm_type;
  1202. tx_desc->tx_encap_type = vdev->tx_encap_type;
  1203. tx_desc->vdev_id = vdev->vdev_id;
  1204. tx_desc->pdev = pdev;
  1205. tx_desc->pkt_offset = 0;
  1206. dp_tx_trace_pkt(soc, nbuf, tx_desc->id, vdev->vdev_id,
  1207. vdev->qdf_opmode);
  1208. /* Handle scattered frames - TSO/SG/ME */
  1209. /* Allocate and prepare an extension descriptor for scattered frames */
  1210. msdu_ext_desc = dp_tx_prepare_ext_desc(vdev, msdu_info, desc_pool_id);
  1211. if (!msdu_ext_desc) {
  1212. dp_tx_info("Tx Extension Descriptor Alloc Fail");
  1213. goto failure;
  1214. }
  1215. #if !TQM_BYPASS_WAR
  1216. if (qdf_unlikely(msdu_info->exception_fw) ||
  1217. dp_tx_is_nbuf_marked_exception(soc, nbuf))
  1218. #endif
  1219. {
  1220. /* Temporary WAR due to TQM VP issues */
  1221. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1222. qdf_atomic_inc(&soc->num_tx_exception);
  1223. }
  1224. tx_desc->msdu_ext_desc = msdu_ext_desc;
  1225. tx_desc->flags |= DP_TX_DESC_FLAG_FRAG;
  1226. msdu_ext_desc->tso_desc = msdu_info->u.tso_info.curr_seg;
  1227. msdu_ext_desc->tso_num_desc = msdu_info->u.tso_info.tso_num_seg_list;
  1228. tx_desc->dma_addr = msdu_ext_desc->paddr;
  1229. if (msdu_ext_desc->flags & DP_TX_EXT_DESC_FLAG_METADATA_VALID)
  1230. tx_desc->length = HAL_TX_EXT_DESC_WITH_META_DATA;
  1231. else
  1232. tx_desc->length = HAL_TX_EXTENSION_DESC_LEN_BYTES;
  1233. return tx_desc;
  1234. failure:
  1235. dp_tx_desc_release(soc, tx_desc, desc_pool_id);
  1236. return NULL;
  1237. }
  1238. /**
  1239. * dp_tx_prepare_raw() - Prepare RAW packet TX
  1240. * @vdev: DP vdev handle
  1241. * @nbuf: buffer pointer
  1242. * @seg_info: Pointer to Segment info Descriptor to be prepared
  1243. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension
  1244. * descriptor
  1245. *
  1246. * Return:
  1247. */
  1248. static qdf_nbuf_t dp_tx_prepare_raw(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  1249. struct dp_tx_seg_info_s *seg_info, struct dp_tx_msdu_info_s *msdu_info)
  1250. {
  1251. qdf_nbuf_t curr_nbuf = NULL;
  1252. uint16_t total_len = 0;
  1253. qdf_dma_addr_t paddr;
  1254. int32_t i;
  1255. int32_t mapped_buf_num = 0;
  1256. struct dp_tx_sg_info_s *sg_info = &msdu_info->u.sg_info;
  1257. qdf_dot3_qosframe_t *qos_wh = (qdf_dot3_qosframe_t *) nbuf->data;
  1258. DP_STATS_INC_PKT(vdev, tx_i[msdu_info->xmit_type].raw.raw_pkt,
  1259. 1, qdf_nbuf_len(nbuf));
  1260. /* Continue only if frames are of DATA type */
  1261. if (!DP_FRAME_IS_DATA(qos_wh)) {
  1262. DP_STATS_INC(vdev,
  1263. tx_i[msdu_info->xmit_type].raw.invalid_raw_pkt_datatype,
  1264. 1);
  1265. dp_tx_debug("Pkt. recd is of not data type");
  1266. goto error;
  1267. }
  1268. /* SWAR for HW: Enable WEP bit in the AMSDU frames for RAW mode */
  1269. if (vdev->raw_mode_war &&
  1270. (qos_wh->i_fc[0] & QDF_IEEE80211_FC0_SUBTYPE_QOS) &&
  1271. (qos_wh->i_qos[0] & IEEE80211_QOS_AMSDU))
  1272. qos_wh->i_fc[1] |= IEEE80211_FC1_WEP;
  1273. for (curr_nbuf = nbuf, i = 0; curr_nbuf;
  1274. curr_nbuf = qdf_nbuf_next(curr_nbuf), i++) {
  1275. /*
  1276. * Number of nbuf's must not exceed the size of the frags
  1277. * array in seg_info.
  1278. */
  1279. if (i >= DP_TX_MAX_NUM_FRAGS) {
  1280. dp_err_rl("nbuf cnt exceeds the max number of segs");
  1281. DP_STATS_INC(vdev,
  1282. tx_i[msdu_info->xmit_type].raw.num_frags_overflow_err,
  1283. 1);
  1284. goto error;
  1285. }
  1286. if (QDF_STATUS_SUCCESS !=
  1287. qdf_nbuf_map_nbytes_single(vdev->osdev,
  1288. curr_nbuf,
  1289. QDF_DMA_TO_DEVICE,
  1290. curr_nbuf->len)) {
  1291. dp_tx_err("%s dma map error ", __func__);
  1292. DP_STATS_INC(vdev,
  1293. tx_i[msdu_info->xmit_type].raw.dma_map_error,
  1294. 1);
  1295. goto error;
  1296. }
  1297. /* Update the count of mapped nbuf's */
  1298. mapped_buf_num++;
  1299. paddr = qdf_nbuf_get_frag_paddr(curr_nbuf, 0);
  1300. seg_info->frags[i].paddr_lo = paddr;
  1301. seg_info->frags[i].paddr_hi = ((uint64_t)paddr >> 32);
  1302. seg_info->frags[i].len = qdf_nbuf_len(curr_nbuf);
  1303. seg_info->frags[i].vaddr = (void *) curr_nbuf;
  1304. total_len += qdf_nbuf_len(curr_nbuf);
  1305. }
  1306. seg_info->frag_cnt = i;
  1307. seg_info->total_len = total_len;
  1308. seg_info->next = NULL;
  1309. sg_info->curr_seg = seg_info;
  1310. msdu_info->frm_type = dp_tx_frm_raw;
  1311. msdu_info->num_seg = 1;
  1312. return nbuf;
  1313. error:
  1314. i = 0;
  1315. while (nbuf) {
  1316. curr_nbuf = nbuf;
  1317. if (i < mapped_buf_num) {
  1318. qdf_nbuf_unmap_nbytes_single(vdev->osdev, curr_nbuf,
  1319. QDF_DMA_TO_DEVICE,
  1320. curr_nbuf->len);
  1321. i++;
  1322. }
  1323. nbuf = qdf_nbuf_next(nbuf);
  1324. qdf_nbuf_free(curr_nbuf);
  1325. }
  1326. return NULL;
  1327. }
  1328. /**
  1329. * dp_tx_raw_prepare_unset() - unmap the chain of nbufs belonging to RAW frame.
  1330. * @soc: DP soc handle
  1331. * @nbuf: Buffer pointer
  1332. *
  1333. * unmap the chain of nbufs that belong to this RAW frame.
  1334. *
  1335. * Return: None
  1336. */
  1337. static void dp_tx_raw_prepare_unset(struct dp_soc *soc,
  1338. qdf_nbuf_t nbuf)
  1339. {
  1340. qdf_nbuf_t cur_nbuf = nbuf;
  1341. do {
  1342. qdf_nbuf_unmap_nbytes_single(soc->osdev, cur_nbuf,
  1343. QDF_DMA_TO_DEVICE,
  1344. cur_nbuf->len);
  1345. cur_nbuf = qdf_nbuf_next(cur_nbuf);
  1346. } while (cur_nbuf);
  1347. }
  1348. #ifdef VDEV_PEER_PROTOCOL_COUNT
  1349. void dp_vdev_peer_stats_update_protocol_cnt_tx(struct dp_vdev *vdev_hdl,
  1350. qdf_nbuf_t nbuf)
  1351. {
  1352. qdf_nbuf_t nbuf_local;
  1353. struct dp_vdev *vdev_local = vdev_hdl;
  1354. do {
  1355. if (qdf_likely(!((vdev_local)->peer_protocol_count_track)))
  1356. break;
  1357. nbuf_local = nbuf;
  1358. if (qdf_unlikely(((vdev_local)->tx_encap_type) ==
  1359. htt_cmn_pkt_type_raw))
  1360. break;
  1361. else if (qdf_unlikely(qdf_nbuf_is_nonlinear((nbuf_local))))
  1362. break;
  1363. else if (qdf_nbuf_is_tso((nbuf_local)))
  1364. break;
  1365. dp_vdev_peer_stats_update_protocol_cnt((vdev_local),
  1366. (nbuf_local),
  1367. NULL, 1, 0);
  1368. } while (0);
  1369. }
  1370. #endif
  1371. #ifdef WLAN_DP_FEATURE_SW_LATENCY_MGR
  1372. void dp_tx_update_stats(struct dp_soc *soc,
  1373. struct dp_tx_desc_s *tx_desc,
  1374. uint8_t ring_id)
  1375. {
  1376. uint32_t stats_len = dp_tx_get_pkt_len(tx_desc);
  1377. DP_STATS_INC_PKT(soc, tx.egress[ring_id], 1, stats_len);
  1378. }
  1379. int
  1380. dp_tx_attempt_coalescing(struct dp_soc *soc, struct dp_vdev *vdev,
  1381. struct dp_tx_desc_s *tx_desc,
  1382. uint8_t tid,
  1383. struct dp_tx_msdu_info_s *msdu_info,
  1384. uint8_t ring_id)
  1385. {
  1386. struct dp_swlm *swlm = &soc->swlm;
  1387. union swlm_data swlm_query_data;
  1388. struct dp_swlm_tcl_data tcl_data;
  1389. QDF_STATUS status;
  1390. int ret;
  1391. if (!swlm->is_enabled)
  1392. return msdu_info->skip_hp_update;
  1393. tcl_data.nbuf = tx_desc->nbuf;
  1394. tcl_data.tid = tid;
  1395. tcl_data.ring_id = ring_id;
  1396. tcl_data.pkt_len = dp_tx_get_pkt_len(tx_desc);
  1397. tcl_data.num_ll_connections = vdev->num_latency_critical_conn;
  1398. swlm_query_data.tcl_data = &tcl_data;
  1399. status = dp_swlm_tcl_pre_check(soc, &tcl_data);
  1400. if (QDF_IS_STATUS_ERROR(status)) {
  1401. dp_swlm_tcl_reset_session_data(soc, ring_id);
  1402. DP_STATS_INC(swlm, tcl[ring_id].coalesce_fail, 1);
  1403. return 0;
  1404. }
  1405. ret = dp_swlm_query_policy(soc, TCL_DATA, swlm_query_data);
  1406. if (ret) {
  1407. DP_STATS_INC(swlm, tcl[ring_id].coalesce_success, 1);
  1408. } else {
  1409. DP_STATS_INC(swlm, tcl[ring_id].coalesce_fail, 1);
  1410. }
  1411. return ret;
  1412. }
  1413. void
  1414. dp_tx_ring_access_end(struct dp_soc *soc, hal_ring_handle_t hal_ring_hdl,
  1415. int coalesce)
  1416. {
  1417. if (coalesce)
  1418. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1419. else
  1420. dp_tx_hal_ring_access_end(soc, hal_ring_hdl);
  1421. }
  1422. static inline void
  1423. dp_tx_is_hp_update_required(uint32_t i, struct dp_tx_msdu_info_s *msdu_info)
  1424. {
  1425. if (((i + 1) < msdu_info->num_seg))
  1426. msdu_info->skip_hp_update = 1;
  1427. else
  1428. msdu_info->skip_hp_update = 0;
  1429. }
  1430. static inline void
  1431. dp_flush_tcp_hp(struct dp_soc *soc, uint8_t ring_id)
  1432. {
  1433. hal_ring_handle_t hal_ring_hdl =
  1434. dp_tx_get_hal_ring_hdl(soc, ring_id);
  1435. if (dp_tx_hal_ring_access_start(soc, hal_ring_hdl)) {
  1436. dp_err("Fillmore: SRNG access start failed");
  1437. return;
  1438. }
  1439. dp_tx_ring_access_end_wrapper(soc, hal_ring_hdl, 0);
  1440. }
  1441. static inline void
  1442. dp_tx_check_and_flush_hp(struct dp_soc *soc,
  1443. QDF_STATUS status,
  1444. struct dp_tx_msdu_info_s *msdu_info)
  1445. {
  1446. if (QDF_IS_STATUS_ERROR(status) && !msdu_info->skip_hp_update) {
  1447. dp_flush_tcp_hp(soc,
  1448. (msdu_info->tx_queue.ring_id & DP_TX_QUEUE_MASK));
  1449. }
  1450. }
  1451. #else
  1452. static inline void
  1453. dp_tx_is_hp_update_required(uint32_t i, struct dp_tx_msdu_info_s *msdu_info)
  1454. {
  1455. }
  1456. static inline void
  1457. dp_tx_check_and_flush_hp(struct dp_soc *soc,
  1458. QDF_STATUS status,
  1459. struct dp_tx_msdu_info_s *msdu_info)
  1460. {
  1461. }
  1462. #endif
  1463. #ifdef FEATURE_RUNTIME_PM
  1464. void
  1465. dp_tx_ring_access_end_wrapper(struct dp_soc *soc,
  1466. hal_ring_handle_t hal_ring_hdl,
  1467. int coalesce)
  1468. {
  1469. int ret;
  1470. /*
  1471. * Avoid runtime get and put APIs under high throughput scenarios.
  1472. */
  1473. if (dp_get_rtpm_tput_policy_requirement(soc)) {
  1474. dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
  1475. return;
  1476. }
  1477. ret = hif_rtpm_get(HIF_RTPM_GET_ASYNC, HIF_RTPM_ID_DP);
  1478. if (QDF_IS_STATUS_SUCCESS(ret)) {
  1479. if (hif_system_pm_state_check(soc->hif_handle)) {
  1480. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1481. hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
  1482. hal_srng_inc_flush_cnt(hal_ring_hdl);
  1483. } else {
  1484. dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
  1485. }
  1486. hif_rtpm_put(HIF_RTPM_PUT_ASYNC, HIF_RTPM_ID_DP);
  1487. } else {
  1488. dp_runtime_get(soc);
  1489. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1490. hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
  1491. qdf_atomic_inc(&soc->tx_pending_rtpm);
  1492. hal_srng_inc_flush_cnt(hal_ring_hdl);
  1493. dp_runtime_put(soc);
  1494. }
  1495. }
  1496. #else
  1497. #ifdef DP_POWER_SAVE
  1498. void
  1499. dp_tx_ring_access_end_wrapper(struct dp_soc *soc,
  1500. hal_ring_handle_t hal_ring_hdl,
  1501. int coalesce)
  1502. {
  1503. if (hif_system_pm_state_check(soc->hif_handle)) {
  1504. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1505. hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
  1506. hal_srng_inc_flush_cnt(hal_ring_hdl);
  1507. } else {
  1508. dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
  1509. }
  1510. }
  1511. #endif
  1512. #endif
  1513. /**
  1514. * dp_tx_get_tid() - Obtain TID to be used for this frame
  1515. * @vdev: DP vdev handle
  1516. * @nbuf: skb
  1517. * @msdu_info: msdu descriptor
  1518. *
  1519. * Extract the DSCP or PCP information from frame and map into TID value.
  1520. *
  1521. * Return: void
  1522. */
  1523. static void dp_tx_get_tid(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  1524. struct dp_tx_msdu_info_s *msdu_info)
  1525. {
  1526. uint8_t tos = 0, dscp_tid_override = 0;
  1527. uint8_t *hdr_ptr, *L3datap;
  1528. uint8_t is_mcast = 0;
  1529. qdf_ether_header_t *eh = NULL;
  1530. qdf_ethervlan_header_t *evh = NULL;
  1531. uint16_t ether_type;
  1532. qdf_llc_t *llcHdr;
  1533. struct dp_pdev *pdev = (struct dp_pdev *)vdev->pdev;
  1534. DP_TX_TID_OVERRIDE(msdu_info, nbuf);
  1535. if (qdf_likely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
  1536. eh = (qdf_ether_header_t *)nbuf->data;
  1537. hdr_ptr = (uint8_t *)(eh->ether_dhost);
  1538. L3datap = hdr_ptr + sizeof(qdf_ether_header_t);
  1539. } else {
  1540. qdf_dot3_qosframe_t *qos_wh =
  1541. (qdf_dot3_qosframe_t *) nbuf->data;
  1542. msdu_info->tid = qos_wh->i_fc[0] & DP_FC0_SUBTYPE_QOS ?
  1543. qos_wh->i_qos[0] & DP_QOS_TID : 0;
  1544. return;
  1545. }
  1546. is_mcast = DP_FRAME_IS_MULTICAST(hdr_ptr);
  1547. ether_type = eh->ether_type;
  1548. llcHdr = (qdf_llc_t *)(nbuf->data + sizeof(qdf_ether_header_t));
  1549. /*
  1550. * Check if packet is dot3 or eth2 type.
  1551. */
  1552. if (DP_FRAME_IS_LLC(ether_type) && DP_FRAME_IS_SNAP(llcHdr)) {
  1553. ether_type = (uint16_t)*(nbuf->data + 2*QDF_MAC_ADDR_SIZE +
  1554. sizeof(*llcHdr));
  1555. if (ether_type == htons(ETHERTYPE_VLAN)) {
  1556. L3datap = hdr_ptr + sizeof(qdf_ethervlan_header_t) +
  1557. sizeof(*llcHdr);
  1558. ether_type = (uint16_t)*(nbuf->data + 2*QDF_MAC_ADDR_SIZE
  1559. + sizeof(*llcHdr) +
  1560. sizeof(qdf_net_vlanhdr_t));
  1561. } else {
  1562. L3datap = hdr_ptr + sizeof(qdf_ether_header_t) +
  1563. sizeof(*llcHdr);
  1564. }
  1565. } else {
  1566. if (ether_type == htons(ETHERTYPE_VLAN)) {
  1567. evh = (qdf_ethervlan_header_t *) eh;
  1568. ether_type = evh->ether_type;
  1569. L3datap = hdr_ptr + sizeof(qdf_ethervlan_header_t);
  1570. }
  1571. }
  1572. /*
  1573. * Find priority from IP TOS DSCP field
  1574. */
  1575. if (qdf_nbuf_is_ipv4_pkt(nbuf)) {
  1576. qdf_net_iphdr_t *ip = (qdf_net_iphdr_t *) L3datap;
  1577. if (qdf_nbuf_is_ipv4_dhcp_pkt(nbuf)) {
  1578. /* Only for unicast frames */
  1579. if (!is_mcast) {
  1580. /* send it on VO queue */
  1581. msdu_info->tid = DP_VO_TID;
  1582. }
  1583. } else {
  1584. /*
  1585. * IP frame: exclude ECN bits 0-1 and map DSCP bits 2-7
  1586. * from TOS byte.
  1587. */
  1588. tos = ip->ip_tos;
  1589. dscp_tid_override = 1;
  1590. }
  1591. } else if (qdf_nbuf_is_ipv6_pkt(nbuf)) {
  1592. /* TODO
  1593. * use flowlabel
  1594. *igmpmld cases to be handled in phase 2
  1595. */
  1596. unsigned long ver_pri_flowlabel;
  1597. unsigned long pri;
  1598. ver_pri_flowlabel = *(unsigned long *) L3datap;
  1599. pri = (ntohl(ver_pri_flowlabel) & IPV6_FLOWINFO_PRIORITY) >>
  1600. DP_IPV6_PRIORITY_SHIFT;
  1601. tos = pri;
  1602. dscp_tid_override = 1;
  1603. } else if (qdf_nbuf_is_ipv4_eapol_pkt(nbuf))
  1604. msdu_info->tid = DP_VO_TID;
  1605. else if (qdf_nbuf_is_ipv4_arp_pkt(nbuf)) {
  1606. /* Only for unicast frames */
  1607. if (!is_mcast) {
  1608. /* send ucast arp on VO queue */
  1609. msdu_info->tid = DP_VO_TID;
  1610. }
  1611. }
  1612. /*
  1613. * Assign all MCAST packets to BE
  1614. */
  1615. if (qdf_unlikely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
  1616. if (is_mcast) {
  1617. tos = 0;
  1618. dscp_tid_override = 1;
  1619. }
  1620. }
  1621. if (dscp_tid_override == 1) {
  1622. tos = (tos >> DP_IP_DSCP_SHIFT) & DP_IP_DSCP_MASK;
  1623. msdu_info->tid = pdev->dscp_tid_map[vdev->dscp_tid_map_id][tos];
  1624. }
  1625. if (msdu_info->tid >= CDP_MAX_DATA_TIDS)
  1626. msdu_info->tid = CDP_MAX_DATA_TIDS - 1;
  1627. return;
  1628. }
  1629. /**
  1630. * dp_tx_classify_tid() - Obtain TID to be used for this frame
  1631. * @vdev: DP vdev handle
  1632. * @nbuf: skb
  1633. * @msdu_info: msdu descriptor
  1634. *
  1635. * Software based TID classification is required when more than 2 DSCP-TID
  1636. * mapping tables are needed.
  1637. * Hardware supports 2 DSCP-TID mapping tables for HKv1 and 48 for HKv2.
  1638. *
  1639. * Return: void
  1640. */
  1641. static inline void dp_tx_classify_tid(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  1642. struct dp_tx_msdu_info_s *msdu_info)
  1643. {
  1644. DP_TX_TID_OVERRIDE(msdu_info, nbuf);
  1645. /*
  1646. * skip_sw_tid_classification flag will set in below cases-
  1647. * 1. vdev->dscp_tid_map_id < pdev->soc->num_hw_dscp_tid_map
  1648. * 2. hlos_tid_override enabled for vdev
  1649. * 3. mesh mode enabled for vdev
  1650. */
  1651. if (qdf_likely(vdev->skip_sw_tid_classification)) {
  1652. /* Update tid in msdu_info from skb priority */
  1653. if (qdf_unlikely(vdev->skip_sw_tid_classification
  1654. & DP_TXRX_HLOS_TID_OVERRIDE_ENABLED)) {
  1655. uint32_t tid = qdf_nbuf_get_priority(nbuf);
  1656. if (tid == DP_TX_INVALID_QOS_TAG)
  1657. return;
  1658. msdu_info->tid = tid;
  1659. return;
  1660. }
  1661. return;
  1662. }
  1663. dp_tx_get_tid(vdev, nbuf, msdu_info);
  1664. }
  1665. #ifdef FEATURE_WLAN_TDLS
  1666. /**
  1667. * dp_tx_update_tdls_flags() - Update descriptor flags for TDLS frame
  1668. * @soc: datapath SOC
  1669. * @vdev: datapath vdev
  1670. * @tx_desc: TX descriptor
  1671. *
  1672. * Return: None
  1673. */
  1674. static void dp_tx_update_tdls_flags(struct dp_soc *soc,
  1675. struct dp_vdev *vdev,
  1676. struct dp_tx_desc_s *tx_desc)
  1677. {
  1678. if (vdev) {
  1679. if (vdev->is_tdls_frame) {
  1680. tx_desc->flags |= DP_TX_DESC_FLAG_TDLS_FRAME;
  1681. vdev->is_tdls_frame = false;
  1682. }
  1683. }
  1684. }
  1685. static uint8_t dp_htt_tx_comp_get_status(struct dp_soc *soc, char *htt_desc)
  1686. {
  1687. uint8_t tx_status = HTT_TX_FW2WBM_TX_STATUS_MAX;
  1688. switch (soc->arch_id) {
  1689. case CDP_ARCH_TYPE_LI:
  1690. tx_status = HTT_TX_WBM_COMPLETION_V2_TX_STATUS_GET(htt_desc[0]);
  1691. break;
  1692. case CDP_ARCH_TYPE_BE:
  1693. tx_status = HTT_TX_WBM_COMPLETION_V3_TX_STATUS_GET(htt_desc[0]);
  1694. break;
  1695. case CDP_ARCH_TYPE_RH:
  1696. {
  1697. uint32_t *msg_word = (uint32_t *)htt_desc;
  1698. tx_status = HTT_TX_MSDU_INFO_RELEASE_REASON_GET(
  1699. *(msg_word + 3));
  1700. }
  1701. break;
  1702. default:
  1703. dp_err("Incorrect CDP_ARCH %d", soc->arch_id);
  1704. QDF_BUG(0);
  1705. }
  1706. return tx_status;
  1707. }
  1708. /**
  1709. * dp_non_std_htt_tx_comp_free_buff() - Free the non std tx packet buffer
  1710. * @soc: dp_soc handle
  1711. * @tx_desc: TX descriptor
  1712. *
  1713. * Return: None
  1714. */
  1715. static void dp_non_std_htt_tx_comp_free_buff(struct dp_soc *soc,
  1716. struct dp_tx_desc_s *tx_desc)
  1717. {
  1718. uint8_t tx_status = 0;
  1719. uint8_t htt_tx_status[HAL_TX_COMP_HTT_STATUS_LEN];
  1720. qdf_nbuf_t nbuf = tx_desc->nbuf;
  1721. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id,
  1722. DP_MOD_ID_TDLS);
  1723. if (qdf_unlikely(!vdev)) {
  1724. dp_err_rl("vdev is null!");
  1725. goto error;
  1726. }
  1727. hal_tx_comp_get_htt_desc(&tx_desc->comp, htt_tx_status);
  1728. tx_status = dp_htt_tx_comp_get_status(soc, htt_tx_status);
  1729. dp_debug("vdev_id: %d tx_status: %d", tx_desc->vdev_id, tx_status);
  1730. if (vdev->tx_non_std_data_callback.func) {
  1731. qdf_nbuf_set_next(nbuf, NULL);
  1732. vdev->tx_non_std_data_callback.func(
  1733. vdev->tx_non_std_data_callback.ctxt,
  1734. nbuf, tx_status);
  1735. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
  1736. return;
  1737. } else {
  1738. dp_err_rl("callback func is null");
  1739. }
  1740. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
  1741. error:
  1742. qdf_nbuf_unmap_single(soc->osdev, nbuf, QDF_DMA_TO_DEVICE);
  1743. qdf_nbuf_free(nbuf);
  1744. }
  1745. /**
  1746. * dp_tx_msdu_single_map() - do nbuf map
  1747. * @vdev: DP vdev handle
  1748. * @tx_desc: DP TX descriptor pointer
  1749. * @nbuf: skb pointer
  1750. *
  1751. * For TDLS frame, use qdf_nbuf_map_single() to align with the unmap
  1752. * operation done in other component.
  1753. *
  1754. * Return: QDF_STATUS
  1755. */
  1756. static inline QDF_STATUS dp_tx_msdu_single_map(struct dp_vdev *vdev,
  1757. struct dp_tx_desc_s *tx_desc,
  1758. qdf_nbuf_t nbuf)
  1759. {
  1760. if (qdf_likely(!(tx_desc->flags & DP_TX_DESC_FLAG_TDLS_FRAME)))
  1761. return qdf_nbuf_map_nbytes_single(vdev->osdev,
  1762. nbuf,
  1763. QDF_DMA_TO_DEVICE,
  1764. nbuf->len);
  1765. else
  1766. return qdf_nbuf_map_single(vdev->osdev, nbuf,
  1767. QDF_DMA_TO_DEVICE);
  1768. }
  1769. #else
  1770. static inline void dp_tx_update_tdls_flags(struct dp_soc *soc,
  1771. struct dp_vdev *vdev,
  1772. struct dp_tx_desc_s *tx_desc)
  1773. {
  1774. }
  1775. static inline void dp_non_std_htt_tx_comp_free_buff(struct dp_soc *soc,
  1776. struct dp_tx_desc_s *tx_desc)
  1777. {
  1778. }
  1779. static inline QDF_STATUS dp_tx_msdu_single_map(struct dp_vdev *vdev,
  1780. struct dp_tx_desc_s *tx_desc,
  1781. qdf_nbuf_t nbuf)
  1782. {
  1783. return qdf_nbuf_map_nbytes_single(vdev->osdev,
  1784. nbuf,
  1785. QDF_DMA_TO_DEVICE,
  1786. nbuf->len);
  1787. }
  1788. #endif
  1789. static inline
  1790. qdf_dma_addr_t dp_tx_nbuf_map_regular(struct dp_vdev *vdev,
  1791. struct dp_tx_desc_s *tx_desc,
  1792. qdf_nbuf_t nbuf)
  1793. {
  1794. QDF_STATUS ret = QDF_STATUS_E_FAILURE;
  1795. ret = dp_tx_msdu_single_map(vdev, tx_desc, nbuf);
  1796. if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret)))
  1797. return 0;
  1798. return qdf_nbuf_mapped_paddr_get(nbuf);
  1799. }
  1800. static inline
  1801. void dp_tx_nbuf_unmap_regular(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1802. {
  1803. qdf_nbuf_unmap_nbytes_single_paddr(soc->osdev,
  1804. desc->nbuf,
  1805. desc->dma_addr,
  1806. QDF_DMA_TO_DEVICE,
  1807. desc->length);
  1808. }
  1809. #ifdef QCA_DP_TX_RMNET_OPTIMIZATION
  1810. static inline bool
  1811. is_nbuf_frm_rmnet(qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info)
  1812. {
  1813. struct net_device *ingress_dev;
  1814. skb_frag_t *frag;
  1815. uint16_t buf_len = 0;
  1816. uint16_t linear_data_len = 0;
  1817. uint8_t *payload_addr = NULL;
  1818. ingress_dev = dev_get_by_index(dev_net(nbuf->dev), nbuf->skb_iif);
  1819. if (!ingress_dev)
  1820. return false;
  1821. if ((ingress_dev->priv_flags & IFF_PHONY_HEADROOM)) {
  1822. qdf_net_if_release_dev((struct qdf_net_if *)ingress_dev);
  1823. frag = &(skb_shinfo(nbuf)->frags[0]);
  1824. buf_len = skb_frag_size(frag);
  1825. payload_addr = (uint8_t *)skb_frag_address(frag);
  1826. linear_data_len = skb_headlen(nbuf);
  1827. buf_len += linear_data_len;
  1828. payload_addr = payload_addr - linear_data_len;
  1829. memcpy(payload_addr, nbuf->data, linear_data_len);
  1830. msdu_info->frm_type = dp_tx_frm_rmnet;
  1831. msdu_info->buf_len = buf_len;
  1832. msdu_info->payload_addr = payload_addr;
  1833. return true;
  1834. }
  1835. qdf_net_if_release_dev((struct qdf_net_if *)ingress_dev);
  1836. return false;
  1837. }
  1838. static inline
  1839. qdf_dma_addr_t dp_tx_rmnet_nbuf_map(struct dp_tx_msdu_info_s *msdu_info,
  1840. struct dp_tx_desc_s *tx_desc)
  1841. {
  1842. qdf_dma_addr_t paddr;
  1843. paddr = (qdf_dma_addr_t)qdf_mem_virt_to_phys(msdu_info->payload_addr);
  1844. tx_desc->length = msdu_info->buf_len;
  1845. qdf_nbuf_dma_clean_range((void *)msdu_info->payload_addr,
  1846. (void *)(msdu_info->payload_addr +
  1847. msdu_info->buf_len));
  1848. tx_desc->flags |= DP_TX_DESC_FLAG_RMNET;
  1849. return paddr;
  1850. }
  1851. #else
  1852. static inline bool
  1853. is_nbuf_frm_rmnet(qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info)
  1854. {
  1855. return false;
  1856. }
  1857. static inline
  1858. qdf_dma_addr_t dp_tx_rmnet_nbuf_map(struct dp_tx_msdu_info_s *msdu_info,
  1859. struct dp_tx_desc_s *tx_desc)
  1860. {
  1861. return 0;
  1862. }
  1863. #endif
  1864. #if defined(QCA_DP_TX_NBUF_NO_MAP_UNMAP) && !defined(BUILD_X86)
  1865. static inline
  1866. qdf_dma_addr_t dp_tx_nbuf_map(struct dp_vdev *vdev,
  1867. struct dp_tx_desc_s *tx_desc,
  1868. qdf_nbuf_t nbuf)
  1869. {
  1870. if (qdf_likely(tx_desc->flags & DP_TX_DESC_FLAG_FAST)) {
  1871. qdf_nbuf_dma_clean_range((void *)nbuf->data,
  1872. (void *)(nbuf->data + nbuf->len));
  1873. return (qdf_dma_addr_t)qdf_mem_virt_to_phys(nbuf->data);
  1874. } else {
  1875. return dp_tx_nbuf_map_regular(vdev, tx_desc, nbuf);
  1876. }
  1877. }
  1878. static inline
  1879. void dp_tx_nbuf_unmap(struct dp_soc *soc,
  1880. struct dp_tx_desc_s *desc)
  1881. {
  1882. if (qdf_unlikely(!(desc->flags &
  1883. (DP_TX_DESC_FLAG_SIMPLE | DP_TX_DESC_FLAG_RMNET))))
  1884. return dp_tx_nbuf_unmap_regular(soc, desc);
  1885. }
  1886. #else
  1887. static inline
  1888. qdf_dma_addr_t dp_tx_nbuf_map(struct dp_vdev *vdev,
  1889. struct dp_tx_desc_s *tx_desc,
  1890. qdf_nbuf_t nbuf)
  1891. {
  1892. return dp_tx_nbuf_map_regular(vdev, tx_desc, nbuf);
  1893. }
  1894. static inline
  1895. void dp_tx_nbuf_unmap(struct dp_soc *soc,
  1896. struct dp_tx_desc_s *desc)
  1897. {
  1898. return dp_tx_nbuf_unmap_regular(soc, desc);
  1899. }
  1900. #endif
  1901. #if defined(WLAN_TX_PKT_CAPTURE_ENH) || defined(FEATURE_PERPKT_INFO)
  1902. static inline
  1903. void dp_tx_enh_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1904. {
  1905. if (qdf_likely(!(desc->flags & DP_TX_DESC_FLAG_UNMAP_DONE))) {
  1906. dp_tx_nbuf_unmap(soc, desc);
  1907. desc->flags |= DP_TX_DESC_FLAG_UNMAP_DONE;
  1908. }
  1909. }
  1910. static inline void dp_tx_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1911. {
  1912. if (qdf_likely(!(desc->flags & DP_TX_DESC_FLAG_UNMAP_DONE)))
  1913. dp_tx_nbuf_unmap(soc, desc);
  1914. }
  1915. #else
  1916. static inline
  1917. void dp_tx_enh_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1918. {
  1919. }
  1920. static inline void dp_tx_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1921. {
  1922. dp_tx_nbuf_unmap(soc, desc);
  1923. }
  1924. #endif
  1925. #ifdef MESH_MODE_SUPPORT
  1926. /**
  1927. * dp_tx_update_mesh_flags() - Update descriptor flags for mesh VAP
  1928. * @soc: datapath SOC
  1929. * @vdev: datapath vdev
  1930. * @tx_desc: TX descriptor
  1931. *
  1932. * Return: None
  1933. */
  1934. static inline void dp_tx_update_mesh_flags(struct dp_soc *soc,
  1935. struct dp_vdev *vdev,
  1936. struct dp_tx_desc_s *tx_desc)
  1937. {
  1938. if (qdf_unlikely(vdev->mesh_vdev))
  1939. tx_desc->flags |= DP_TX_DESC_FLAG_MESH_MODE;
  1940. }
  1941. /**
  1942. * dp_mesh_tx_comp_free_buff() - Free the mesh tx packet buffer
  1943. * @soc: dp_soc handle
  1944. * @tx_desc: TX descriptor
  1945. * @delayed_free: delay the nbuf free
  1946. *
  1947. * Return: nbuf to be freed late
  1948. */
  1949. static inline qdf_nbuf_t dp_mesh_tx_comp_free_buff(struct dp_soc *soc,
  1950. struct dp_tx_desc_s *tx_desc,
  1951. bool delayed_free)
  1952. {
  1953. qdf_nbuf_t nbuf = tx_desc->nbuf;
  1954. struct dp_vdev *vdev = NULL;
  1955. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  1956. vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id, DP_MOD_ID_MESH);
  1957. if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW) {
  1958. if (vdev)
  1959. DP_STATS_INC(vdev,
  1960. tx_i[xmit_type].mesh.completion_fw, 1);
  1961. if (delayed_free)
  1962. return nbuf;
  1963. qdf_nbuf_free(nbuf);
  1964. } else {
  1965. if (vdev && vdev->osif_tx_free_ext) {
  1966. vdev->osif_tx_free_ext((nbuf));
  1967. } else {
  1968. if (delayed_free)
  1969. return nbuf;
  1970. qdf_nbuf_free(nbuf);
  1971. }
  1972. }
  1973. if (vdev)
  1974. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
  1975. return NULL;
  1976. }
  1977. #else
  1978. static inline void dp_tx_update_mesh_flags(struct dp_soc *soc,
  1979. struct dp_vdev *vdev,
  1980. struct dp_tx_desc_s *tx_desc)
  1981. {
  1982. }
  1983. static inline qdf_nbuf_t dp_mesh_tx_comp_free_buff(struct dp_soc *soc,
  1984. struct dp_tx_desc_s *tx_desc,
  1985. bool delayed_free)
  1986. {
  1987. return NULL;
  1988. }
  1989. #endif
  1990. int dp_tx_frame_is_drop(struct dp_vdev *vdev, uint8_t *srcmac, uint8_t *dstmac)
  1991. {
  1992. struct dp_pdev *pdev = NULL;
  1993. struct dp_ast_entry *src_ast_entry = NULL;
  1994. struct dp_ast_entry *dst_ast_entry = NULL;
  1995. struct dp_soc *soc = NULL;
  1996. qdf_assert(vdev);
  1997. pdev = vdev->pdev;
  1998. qdf_assert(pdev);
  1999. soc = pdev->soc;
  2000. dst_ast_entry = dp_peer_ast_hash_find_by_pdevid
  2001. (soc, dstmac, vdev->pdev->pdev_id);
  2002. src_ast_entry = dp_peer_ast_hash_find_by_pdevid
  2003. (soc, srcmac, vdev->pdev->pdev_id);
  2004. if (dst_ast_entry && src_ast_entry) {
  2005. if (dst_ast_entry->peer_id ==
  2006. src_ast_entry->peer_id)
  2007. return 1;
  2008. }
  2009. return 0;
  2010. }
  2011. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  2012. defined(WLAN_MCAST_MLO)
  2013. /* MLO peer id for reinject*/
  2014. #define DP_MLO_MCAST_REINJECT_PEER_ID 0XFFFD
  2015. /* MLO vdev id inc offset */
  2016. #define DP_MLO_VDEV_ID_OFFSET 0x80
  2017. #ifdef QCA_SUPPORT_WDS_EXTENDED
  2018. static inline bool
  2019. dp_tx_wds_ext_check(struct cdp_tx_exception_metadata *tx_exc_metadata)
  2020. {
  2021. if (tx_exc_metadata && tx_exc_metadata->is_wds_extended)
  2022. return true;
  2023. return false;
  2024. }
  2025. #else
  2026. static inline bool
  2027. dp_tx_wds_ext_check(struct cdp_tx_exception_metadata *tx_exc_metadata)
  2028. {
  2029. return false;
  2030. }
  2031. #endif
  2032. static inline void
  2033. dp_tx_bypass_reinjection(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  2034. struct cdp_tx_exception_metadata *tx_exc_metadata)
  2035. {
  2036. /* wds ext enabled will not set the TO_FW bit */
  2037. if (dp_tx_wds_ext_check(tx_exc_metadata))
  2038. return;
  2039. if (!(tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)) {
  2040. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  2041. qdf_atomic_inc(&soc->num_tx_exception);
  2042. }
  2043. }
  2044. static inline void
  2045. dp_tx_update_mcast_param(uint16_t peer_id,
  2046. uint16_t *htt_tcl_metadata,
  2047. struct dp_vdev *vdev,
  2048. struct dp_tx_msdu_info_s *msdu_info)
  2049. {
  2050. if (peer_id == DP_MLO_MCAST_REINJECT_PEER_ID) {
  2051. *htt_tcl_metadata = 0;
  2052. DP_TX_TCL_METADATA_TYPE_SET(
  2053. *htt_tcl_metadata,
  2054. HTT_TCL_METADATA_V2_TYPE_GLOBAL_SEQ_BASED);
  2055. HTT_TX_TCL_METADATA_GLBL_SEQ_NO_SET(*htt_tcl_metadata,
  2056. msdu_info->gsn);
  2057. msdu_info->vdev_id = vdev->vdev_id + DP_MLO_VDEV_ID_OFFSET;
  2058. HTT_TX_TCL_METADATA_GLBL_SEQ_HOST_INSPECTED_SET(
  2059. *htt_tcl_metadata, 1);
  2060. } else {
  2061. msdu_info->vdev_id = vdev->vdev_id;
  2062. }
  2063. }
  2064. #else
  2065. static inline void
  2066. dp_tx_bypass_reinjection(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  2067. struct cdp_tx_exception_metadata *tx_exc_metadata)
  2068. {
  2069. }
  2070. static inline void
  2071. dp_tx_update_mcast_param(uint16_t peer_id,
  2072. uint16_t *htt_tcl_metadata,
  2073. struct dp_vdev *vdev,
  2074. struct dp_tx_msdu_info_s *msdu_info)
  2075. {
  2076. }
  2077. #endif
  2078. #ifdef DP_TX_SW_DROP_STATS_INC
  2079. static void tx_sw_drop_stats_inc(struct dp_pdev *pdev,
  2080. qdf_nbuf_t nbuf,
  2081. enum cdp_tx_sw_drop drop_code)
  2082. {
  2083. /* EAPOL Drop stats */
  2084. if (qdf_nbuf_is_ipv4_eapol_pkt(nbuf)) {
  2085. switch (drop_code) {
  2086. case TX_DESC_ERR:
  2087. DP_STATS_INC(pdev, eap_drop_stats.tx_desc_err, 1);
  2088. break;
  2089. case TX_HAL_RING_ACCESS_ERR:
  2090. DP_STATS_INC(pdev,
  2091. eap_drop_stats.tx_hal_ring_access_err, 1);
  2092. break;
  2093. case TX_DMA_MAP_ERR:
  2094. DP_STATS_INC(pdev, eap_drop_stats.tx_dma_map_err, 1);
  2095. break;
  2096. case TX_HW_ENQUEUE:
  2097. DP_STATS_INC(pdev, eap_drop_stats.tx_hw_enqueue, 1);
  2098. break;
  2099. case TX_SW_ENQUEUE:
  2100. DP_STATS_INC(pdev, eap_drop_stats.tx_sw_enqueue, 1);
  2101. break;
  2102. default:
  2103. dp_info_rl("Invalid eapol_drop code: %d", drop_code);
  2104. break;
  2105. }
  2106. }
  2107. }
  2108. #else
  2109. static void tx_sw_drop_stats_inc(struct dp_pdev *pdev,
  2110. qdf_nbuf_t nbuf,
  2111. enum cdp_tx_sw_drop drop_code)
  2112. {
  2113. }
  2114. #endif
  2115. #ifdef WLAN_FEATURE_TX_LATENCY_STATS
  2116. /**
  2117. * dp_tx_latency_stats_enabled() - check enablement of transmit latency
  2118. * statistics
  2119. * @vdev: DP vdev handle
  2120. *
  2121. * Return: true if transmit latency statistics is enabled, false otherwise.
  2122. */
  2123. static inline bool dp_tx_latency_stats_enabled(struct dp_vdev *vdev)
  2124. {
  2125. return qdf_atomic_read(&vdev->tx_latency_cfg.enabled);
  2126. }
  2127. /**
  2128. * dp_tx_latency_stats_report_enabled() - check enablement of async report
  2129. * for transmit latency statistics
  2130. * @vdev: DP vdev handle
  2131. *
  2132. * Return: true if transmit latency statistics is enabled, false otherwise.
  2133. */
  2134. static inline bool dp_tx_latency_stats_report_enabled(struct dp_vdev *vdev)
  2135. {
  2136. return qdf_atomic_read(&vdev->tx_latency_cfg.report);
  2137. }
  2138. /**
  2139. * dp_tx_get_driver_ingress_ts() - get driver ingress timestamp from nbuf
  2140. * @vdev: DP vdev handle
  2141. * @msdu_info: pointer to MSDU Descriptor
  2142. * @nbuf: original buffer from network stack
  2143. *
  2144. * Return: None
  2145. */
  2146. static inline void
  2147. dp_tx_get_driver_ingress_ts(struct dp_vdev *vdev,
  2148. struct dp_tx_msdu_info_s *msdu_info,
  2149. qdf_nbuf_t nbuf)
  2150. {
  2151. if (!dp_tx_latency_stats_enabled(vdev))
  2152. return;
  2153. msdu_info->driver_ingress_ts = qdf_nbuf_get_tx_ts(nbuf, true);
  2154. }
  2155. /**
  2156. * dp_tx_update_ts_on_enqueued() - set driver ingress/egress timestamp in
  2157. * tx descriptor
  2158. * @vdev: DP vdev handle
  2159. * @msdu_info: pointer to MSDU Descriptor
  2160. * @tx_desc: pointer to tx descriptor
  2161. *
  2162. * Return: None
  2163. */
  2164. static inline void
  2165. dp_tx_update_ts_on_enqueued(struct dp_vdev *vdev,
  2166. struct dp_tx_msdu_info_s *msdu_info,
  2167. struct dp_tx_desc_s *tx_desc)
  2168. {
  2169. if (!dp_tx_latency_stats_enabled(vdev))
  2170. return;
  2171. tx_desc->driver_ingress_ts = msdu_info->driver_ingress_ts;
  2172. tx_desc->driver_egress_ts = qdf_ktime_real_get();
  2173. }
  2174. /**
  2175. * dp_tx_latency_stats_update_bucket() - update transmit latency statistics
  2176. * for specified type
  2177. * @vdev: DP vdev handle
  2178. * @tx_latency: pointer to transmit latency stats
  2179. * @idx: index of the statistics
  2180. * @type: transmit latency type
  2181. * @value: latency to be recorded
  2182. *
  2183. * Return: None
  2184. */
  2185. static inline void
  2186. dp_tx_latency_stats_update_bucket(struct dp_vdev *vdev,
  2187. struct dp_tx_latency *tx_latency,
  2188. int idx, enum cdp_tx_latency_type type,
  2189. uint32_t value)
  2190. {
  2191. int32_t granularity;
  2192. int lvl;
  2193. granularity =
  2194. qdf_atomic_read(&vdev->tx_latency_cfg.granularity[type]);
  2195. if (qdf_unlikely(!granularity))
  2196. return;
  2197. lvl = value / granularity;
  2198. if (lvl >= CDP_TX_LATENCY_DISTR_LV_MAX)
  2199. lvl = CDP_TX_LATENCY_DISTR_LV_MAX - 1;
  2200. qdf_atomic_inc(&tx_latency->stats[idx][type].msdus_accum);
  2201. qdf_atomic_add(value, &tx_latency->stats[idx][type].latency_accum);
  2202. qdf_atomic_inc(&tx_latency->stats[idx][type].distribution[lvl]);
  2203. }
  2204. /**
  2205. * dp_tx_latency_stats_update() - update transmit latency statistics on
  2206. * msdu transmit completed
  2207. * @soc: dp soc handle
  2208. * @txrx_peer: txrx peer handle
  2209. * @tx_desc: pointer to tx descriptor
  2210. * @ts: tx completion status
  2211. * @link_id: link id
  2212. *
  2213. * Return: None
  2214. */
  2215. static inline void
  2216. dp_tx_latency_stats_update(struct dp_soc *soc,
  2217. struct dp_txrx_peer *txrx_peer,
  2218. struct dp_tx_desc_s *tx_desc,
  2219. struct hal_tx_completion_status *ts,
  2220. uint8_t link_id)
  2221. {
  2222. uint32_t driver_latency, ring_buf_latency, hw_latency;
  2223. QDF_STATUS status = QDF_STATUS_E_INVAL;
  2224. int64_t current_ts, ingress, egress;
  2225. struct dp_vdev *vdev = txrx_peer->vdev;
  2226. struct dp_tx_latency *tx_latency;
  2227. uint8_t idx;
  2228. if (!dp_tx_latency_stats_enabled(vdev))
  2229. return;
  2230. if (!tx_desc->driver_ingress_ts || !tx_desc->driver_egress_ts)
  2231. return;
  2232. status = dp_tx_compute_hw_delay_us(ts, vdev->delta_tsf, &hw_latency);
  2233. if (QDF_IS_STATUS_ERROR(status))
  2234. return;
  2235. ingress = qdf_ktime_to_us(tx_desc->driver_ingress_ts);
  2236. egress = qdf_ktime_to_us(tx_desc->driver_egress_ts);
  2237. driver_latency = (uint32_t)(egress - ingress);
  2238. current_ts = qdf_ktime_to_us(qdf_ktime_real_get());
  2239. ring_buf_latency = (uint32_t)(current_ts - egress);
  2240. tx_latency = &txrx_peer->stats[link_id].tx_latency;
  2241. idx = tx_latency->cur_idx;
  2242. dp_tx_latency_stats_update_bucket(txrx_peer->vdev, tx_latency, idx,
  2243. CDP_TX_LATENCY_TYPE_DRIVER,
  2244. driver_latency);
  2245. dp_tx_latency_stats_update_bucket(txrx_peer->vdev, tx_latency, idx,
  2246. CDP_TX_LATENCY_TYPE_RING_BUF,
  2247. ring_buf_latency);
  2248. dp_tx_latency_stats_update_bucket(txrx_peer->vdev, tx_latency, idx,
  2249. CDP_TX_LATENCY_TYPE_HW, hw_latency);
  2250. }
  2251. /**
  2252. * dp_tx_latency_stats_clear_bucket() - clear specified transmit latency
  2253. * statistics for specified type
  2254. * @tx_latency: pointer to transmit latency stats
  2255. * @idx: index of the statistics
  2256. * @type: transmit latency type
  2257. *
  2258. * Return: None
  2259. */
  2260. static inline void
  2261. dp_tx_latency_stats_clear_bucket(struct dp_tx_latency *tx_latency,
  2262. int idx, enum cdp_tx_latency_type type)
  2263. {
  2264. int lvl;
  2265. struct dp_tx_latency_stats *stats;
  2266. stats = &tx_latency->stats[idx][type];
  2267. qdf_atomic_init(&stats->msdus_accum);
  2268. qdf_atomic_init(&stats->latency_accum);
  2269. for (lvl = 0; lvl < CDP_TX_LATENCY_DISTR_LV_MAX; lvl++)
  2270. qdf_atomic_init(&stats->distribution[lvl]);
  2271. }
  2272. /**
  2273. * dp_tx_latency_stats_clear_buckets() - clear specified transmit latency
  2274. * statistics
  2275. * @tx_latency: pointer to transmit latency stats
  2276. * @idx: index of the statistics
  2277. *
  2278. * Return: None
  2279. */
  2280. static void
  2281. dp_tx_latency_stats_clear_buckets(struct dp_tx_latency *tx_latency,
  2282. int idx)
  2283. {
  2284. int type;
  2285. for (type = 0; type < CDP_TX_LATENCY_TYPE_MAX; type++)
  2286. dp_tx_latency_stats_clear_bucket(tx_latency, idx, type);
  2287. }
  2288. /**
  2289. * dp_tx_latency_stats_update_cca() - update transmit latency statistics for
  2290. * CCA
  2291. * @soc: dp soc handle
  2292. * @peer_id: peer id
  2293. * @granularity: granularity of distribution
  2294. * @distribution: distribution of transmit latency statistics
  2295. * @avg: average of CCA latency(in microseconds) within a cycle
  2296. *
  2297. * Return: None
  2298. */
  2299. void
  2300. dp_tx_latency_stats_update_cca(struct dp_soc *soc, uint16_t peer_id,
  2301. uint32_t granularity, uint32_t *distribution,
  2302. uint32_t avg)
  2303. {
  2304. int lvl, idx;
  2305. uint8_t link_id;
  2306. struct dp_tx_latency *tx_latency;
  2307. struct dp_tx_latency_stats *stats;
  2308. int32_t cur_granularity;
  2309. struct dp_vdev *vdev;
  2310. struct dp_tx_latency_config *cfg;
  2311. struct dp_txrx_peer *txrx_peer;
  2312. struct dp_peer *peer;
  2313. peer = dp_peer_get_ref_by_id(soc, peer_id, DP_MOD_ID_HTT);
  2314. if (!peer) {
  2315. dp_err_rl("Peer not found peer id %d", peer_id);
  2316. return;
  2317. }
  2318. if (IS_MLO_DP_MLD_PEER(peer))
  2319. goto out;
  2320. vdev = peer->vdev;
  2321. if (!dp_tx_latency_stats_enabled(vdev))
  2322. goto out;
  2323. cfg = &vdev->tx_latency_cfg;
  2324. cur_granularity =
  2325. qdf_atomic_read(&cfg->granularity[CDP_TX_LATENCY_TYPE_CCA]);
  2326. /* in unit of ms */
  2327. cur_granularity /= 1000;
  2328. if (cur_granularity != granularity) {
  2329. dp_info_rl("invalid granularity, cur %d report %d",
  2330. cur_granularity, granularity);
  2331. goto out;
  2332. }
  2333. txrx_peer = dp_get_txrx_peer(peer);
  2334. if (qdf_unlikely(!txrx_peer)) {
  2335. dp_err_rl("txrx_peer NULL for MAC: " QDF_MAC_ADDR_FMT,
  2336. QDF_MAC_ADDR_REF(peer->mac_addr.raw));
  2337. goto out;
  2338. }
  2339. link_id = dp_get_peer_link_id(peer);
  2340. if (link_id >= txrx_peer->stats_arr_size)
  2341. goto out;
  2342. tx_latency = &txrx_peer->stats[link_id].tx_latency;
  2343. idx = tx_latency->cur_idx;
  2344. stats = &tx_latency->stats[idx][CDP_TX_LATENCY_TYPE_CCA];
  2345. qdf_atomic_set(&stats->latency_accum, avg);
  2346. qdf_atomic_set(&stats->msdus_accum, (avg ? 1 : 0));
  2347. for (lvl = 0; lvl < CDP_TX_LATENCY_DISTR_LV_MAX; lvl++)
  2348. qdf_atomic_set(&stats->distribution[lvl],
  2349. distribution[lvl]);
  2350. /* prepare for the next cycle */
  2351. tx_latency->cur_idx = 1 - idx;
  2352. dp_tx_latency_stats_clear_buckets(tx_latency, tx_latency->cur_idx);
  2353. out:
  2354. dp_peer_unref_delete(peer, DP_MOD_ID_HTT);
  2355. }
  2356. /**
  2357. * dp_tx_latency_stats_get_per_peer() - get transmit latency statistics for a
  2358. * peer
  2359. * @soc: dp soc handle
  2360. * @peer: dp peer Handle
  2361. * @latency: buffer to hold transmit latency statistics
  2362. *
  2363. * Return: QDF_STATUS
  2364. */
  2365. static QDF_STATUS
  2366. dp_tx_latency_stats_get_per_peer(struct dp_soc *soc, struct dp_peer *peer,
  2367. struct cdp_tx_latency *latency)
  2368. {
  2369. int lvl, type, link_id;
  2370. int32_t latency_accum, msdus_accum;
  2371. struct dp_vdev *vdev;
  2372. struct dp_txrx_peer *txrx_peer;
  2373. struct dp_tx_latency *tx_latency;
  2374. struct dp_tx_latency_config *cfg;
  2375. struct dp_tx_latency_stats *stats;
  2376. uint8_t last_idx;
  2377. if (unlikely(!latency))
  2378. return QDF_STATUS_E_INVAL;
  2379. /* Authenticated link/legacy peer only */
  2380. if (IS_MLO_DP_MLD_PEER(peer) || peer->state != OL_TXRX_PEER_STATE_AUTH)
  2381. return QDF_STATUS_E_INVAL;
  2382. vdev = peer->vdev;
  2383. if (peer->bss_peer && vdev->opmode == wlan_op_mode_ap)
  2384. return QDF_STATUS_E_INVAL;
  2385. txrx_peer = dp_get_txrx_peer(peer);
  2386. if (!txrx_peer)
  2387. return QDF_STATUS_E_INVAL;
  2388. link_id = dp_get_peer_link_id(peer);
  2389. if (link_id >= txrx_peer->stats_arr_size)
  2390. return QDF_STATUS_E_INVAL;
  2391. tx_latency = &txrx_peer->stats[link_id].tx_latency;
  2392. qdf_mem_zero(latency, sizeof(*latency));
  2393. qdf_mem_copy(latency->mac_remote.bytes,
  2394. peer->mac_addr.raw, QDF_MAC_ADDR_SIZE);
  2395. last_idx = 1 - tx_latency->cur_idx;
  2396. cfg = &vdev->tx_latency_cfg;
  2397. for (type = 0; type < CDP_TX_LATENCY_TYPE_MAX; type++) {
  2398. latency->stats[type].granularity =
  2399. qdf_atomic_read(&cfg->granularity[type]);
  2400. stats = &tx_latency->stats[last_idx][type];
  2401. msdus_accum = qdf_atomic_read(&stats->msdus_accum);
  2402. if (!msdus_accum)
  2403. continue;
  2404. latency_accum = qdf_atomic_read(&stats->latency_accum);
  2405. latency->stats[type].average = latency_accum / msdus_accum;
  2406. for (lvl = 0; lvl < CDP_TX_LATENCY_DISTR_LV_MAX; lvl++) {
  2407. latency->stats[type].distribution[lvl] =
  2408. qdf_atomic_read(&stats->distribution[lvl]);
  2409. }
  2410. }
  2411. return QDF_STATUS_SUCCESS;
  2412. }
  2413. /**
  2414. * dp_tx_latency_stats_get_peer_iter() - iterator to get transmit latency
  2415. * statistics for specified peer
  2416. * @soc: dp soc handle
  2417. * @peer: dp peer Handle
  2418. * @arg: list to hold transmit latency statistics for peers
  2419. *
  2420. * Return: None
  2421. */
  2422. static void
  2423. dp_tx_latency_stats_get_peer_iter(struct dp_soc *soc,
  2424. struct dp_peer *peer,
  2425. void *arg)
  2426. {
  2427. struct dp_vdev *vdev;
  2428. struct dp_txrx_peer *txrx_peer;
  2429. struct cdp_tx_latency *latency;
  2430. QDF_STATUS status;
  2431. qdf_list_t *stats_list = (qdf_list_t *)arg;
  2432. /* Authenticated link/legacy peer only */
  2433. if (IS_MLO_DP_MLD_PEER(peer) || peer->state != OL_TXRX_PEER_STATE_AUTH)
  2434. return;
  2435. txrx_peer = dp_get_txrx_peer(peer);
  2436. if (!txrx_peer)
  2437. return;
  2438. vdev = peer->vdev;
  2439. latency = qdf_mem_malloc(sizeof(*latency));
  2440. if (!latency)
  2441. return;
  2442. status = dp_tx_latency_stats_get_per_peer(soc, peer, latency);
  2443. if (QDF_IS_STATUS_ERROR(status))
  2444. goto out;
  2445. status = qdf_list_insert_back(stats_list, &latency->node);
  2446. if (QDF_IS_STATUS_ERROR(status))
  2447. goto out;
  2448. return;
  2449. out:
  2450. qdf_mem_free(latency);
  2451. }
  2452. /**
  2453. * dp_tx_latency_stats_rpt_per_vdev() - report transmit latency statistics for
  2454. * specified vdev
  2455. * @soc: dp soc handle
  2456. * @vdev: dp vdev Handle
  2457. *
  2458. * Return: None
  2459. */
  2460. static void
  2461. dp_tx_latency_stats_rpt_per_vdev(struct dp_soc *soc, struct dp_vdev *vdev)
  2462. {
  2463. qdf_list_t stats_list;
  2464. struct cdp_tx_latency *entry, *next;
  2465. if (!soc->tx_latency_cb || !dp_tx_latency_stats_report_enabled(vdev))
  2466. return;
  2467. qdf_list_create(&stats_list, 0);
  2468. dp_vdev_iterate_peer(vdev, dp_tx_latency_stats_get_peer_iter,
  2469. &stats_list, DP_MOD_ID_CDP);
  2470. if (qdf_list_empty(&stats_list))
  2471. goto out;
  2472. soc->tx_latency_cb(vdev->vdev_id, &stats_list);
  2473. qdf_list_for_each_del(&stats_list, entry, next, node) {
  2474. qdf_list_remove_node(&stats_list, &entry->node);
  2475. qdf_mem_free(entry);
  2476. }
  2477. out:
  2478. qdf_list_destroy(&stats_list);
  2479. }
  2480. /**
  2481. * dp_tx_latency_stats_report() - report transmit latency statistics for each
  2482. * vdev of specified pdev
  2483. * @soc: dp soc handle
  2484. * @pdev: dp pdev Handle
  2485. *
  2486. * Return: None
  2487. */
  2488. void dp_tx_latency_stats_report(struct dp_soc *soc, struct dp_pdev *pdev)
  2489. {
  2490. struct dp_vdev *vdev;
  2491. if (!soc->tx_latency_cb)
  2492. return;
  2493. qdf_spin_lock_bh(&pdev->vdev_list_lock);
  2494. DP_PDEV_ITERATE_VDEV_LIST(pdev, vdev) {
  2495. dp_tx_latency_stats_rpt_per_vdev(soc, vdev);
  2496. }
  2497. qdf_spin_unlock_bh(&pdev->vdev_list_lock);
  2498. }
  2499. /**
  2500. * dp_tx_latency_stats_clear_per_peer() - iterator to clear transmit latency
  2501. * statistics for specified peer
  2502. * @soc: dp soc handle
  2503. * @peer: dp pdev Handle
  2504. * @arg: argument from iterator
  2505. *
  2506. * Return: None
  2507. */
  2508. static void
  2509. dp_tx_latency_stats_clear_per_peer(struct dp_soc *soc, struct dp_peer *peer,
  2510. void *arg)
  2511. {
  2512. int link_id;
  2513. struct dp_tx_latency *tx_latency;
  2514. struct dp_txrx_peer *txrx_peer = dp_get_txrx_peer(peer);
  2515. if (!txrx_peer) {
  2516. dp_err("no txrx peer, skip");
  2517. return;
  2518. }
  2519. for (link_id = 0; link_id < txrx_peer->stats_arr_size; link_id++) {
  2520. tx_latency = &txrx_peer->stats[link_id].tx_latency;
  2521. dp_tx_latency_stats_clear_buckets(tx_latency, 0);
  2522. dp_tx_latency_stats_clear_buckets(tx_latency, 1);
  2523. }
  2524. }
  2525. /**
  2526. * dp_tx_latency_stats_clear_per_vdev() - clear transmit latency statistics
  2527. * for specified vdev
  2528. * @vdev: dp vdev handle
  2529. *
  2530. * Return: None
  2531. */
  2532. static inline void dp_tx_latency_stats_clear_per_vdev(struct dp_vdev *vdev)
  2533. {
  2534. dp_vdev_iterate_peer(vdev, dp_tx_latency_stats_clear_per_peer,
  2535. NULL, DP_MOD_ID_CDP);
  2536. }
  2537. /**
  2538. * dp_tx_latency_stats_fetch() - fetch transmit latency statistics for
  2539. * specified link mac address
  2540. * @soc_hdl: Handle to struct dp_soc
  2541. * @vdev_id: vdev id
  2542. * @mac: link mac address of remote peer
  2543. * @latency: buffer to hold per-link transmit latency statistics
  2544. *
  2545. * Return: QDF_STATUS
  2546. */
  2547. QDF_STATUS
  2548. dp_tx_latency_stats_fetch(struct cdp_soc_t *soc_hdl,
  2549. uint8_t vdev_id, uint8_t *mac,
  2550. struct cdp_tx_latency *latency)
  2551. {
  2552. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  2553. struct cdp_peer_info peer_info = {0};
  2554. struct dp_peer *peer;
  2555. QDF_STATUS status;
  2556. /* MAC addr of link peer may be the same as MLD peer,
  2557. * so specify the type as CDP_LINK_PEER_TYPE here to
  2558. * get link peer explicitly.
  2559. */
  2560. DP_PEER_INFO_PARAMS_INIT(&peer_info, vdev_id, mac, false,
  2561. CDP_LINK_PEER_TYPE);
  2562. peer = dp_peer_hash_find_wrapper(soc, &peer_info, DP_MOD_ID_CDP);
  2563. if (!peer) {
  2564. dp_err_rl("peer(vdev id %d mac " QDF_MAC_ADDR_FMT ") not found",
  2565. vdev_id, QDF_MAC_ADDR_REF(mac));
  2566. return QDF_STATUS_E_INVAL;
  2567. }
  2568. status = dp_tx_latency_stats_get_per_peer(soc, peer, latency);
  2569. dp_peer_unref_delete(peer, DP_MOD_ID_CDP);
  2570. return status;
  2571. }
  2572. /**
  2573. * dp_tx_latency_stats_config() - config transmit latency statistics for
  2574. * specified vdev
  2575. * @soc_hdl: Handle to struct dp_soc
  2576. * @vdev_id: vdev id
  2577. * @cfg: configuration for transmit latency statistics
  2578. *
  2579. * Return: QDF_STATUS
  2580. */
  2581. QDF_STATUS
  2582. dp_tx_latency_stats_config(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  2583. struct cdp_tx_latency_config *cfg)
  2584. {
  2585. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  2586. struct dp_vdev *vdev;
  2587. QDF_STATUS status = QDF_STATUS_E_INVAL;
  2588. uint32_t cca_granularity;
  2589. int type;
  2590. vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_CDP);
  2591. if (!vdev) {
  2592. dp_err_rl("vdev %d does not exist", vdev_id);
  2593. return QDF_STATUS_E_FAILURE;
  2594. }
  2595. /* disable to ignore upcoming updates */
  2596. qdf_atomic_set(&vdev->tx_latency_cfg.enabled, 0);
  2597. dp_tx_latency_stats_clear_per_vdev(vdev);
  2598. if (!cfg->enable)
  2599. goto send_htt;
  2600. qdf_atomic_set(&vdev->tx_latency_cfg.report, (cfg->report ? 1 : 0));
  2601. for (type = 0; type < CDP_TX_LATENCY_TYPE_MAX; type++)
  2602. qdf_atomic_set(&vdev->tx_latency_cfg.granularity[type],
  2603. cfg->granularity[type]);
  2604. send_htt:
  2605. /* in units of ms */
  2606. cca_granularity = cfg->granularity[CDP_TX_LATENCY_TYPE_CCA] / 1000;
  2607. status = dp_h2t_tx_latency_stats_cfg_msg_send(soc, vdev_id,
  2608. cfg->enable, cfg->period,
  2609. cca_granularity);
  2610. if (QDF_IS_STATUS_ERROR(status)) {
  2611. dp_err_rl("failed to send htt msg: %d", status);
  2612. goto out;
  2613. }
  2614. qdf_atomic_set(&vdev->tx_latency_cfg.enabled, (cfg->enable ? 1 : 0));
  2615. out:
  2616. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  2617. return status;
  2618. }
  2619. /**
  2620. * dp_tx_latency_stats_register_cb() - register transmit latency statistics
  2621. * callback
  2622. * @handle: Handle to struct dp_soc
  2623. * @cb: callback function for transmit latency statistics
  2624. *
  2625. * Return: QDF_STATUS
  2626. */
  2627. QDF_STATUS
  2628. dp_tx_latency_stats_register_cb(struct cdp_soc_t *handle, cdp_tx_latency_cb cb)
  2629. {
  2630. struct dp_soc *soc = (struct dp_soc *)handle;
  2631. if (!soc || !cb) {
  2632. dp_err("soc or cb is NULL");
  2633. return QDF_STATUS_E_INVAL;
  2634. }
  2635. soc->tx_latency_cb = cb;
  2636. return QDF_STATUS_SUCCESS;
  2637. }
  2638. #else
  2639. static inline void
  2640. dp_tx_get_driver_ingress_ts(struct dp_vdev *vdev,
  2641. struct dp_tx_msdu_info_s *msdu_info,
  2642. qdf_nbuf_t nbuf)
  2643. {
  2644. }
  2645. static inline void
  2646. dp_tx_update_ts_on_enqueued(struct dp_vdev *vdev,
  2647. struct dp_tx_msdu_info_s *msdu_info,
  2648. struct dp_tx_desc_s *tx_desc)
  2649. {
  2650. }
  2651. static inline void
  2652. dp_tx_latency_stats_update(struct dp_soc *soc,
  2653. struct dp_txrx_peer *txrx_peer,
  2654. struct dp_tx_desc_s *tx_desc,
  2655. struct hal_tx_completion_status *ts,
  2656. uint8_t link_id)
  2657. {
  2658. }
  2659. #endif
  2660. qdf_nbuf_t
  2661. dp_tx_send_msdu_single(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2662. struct dp_tx_msdu_info_s *msdu_info, uint16_t peer_id,
  2663. struct cdp_tx_exception_metadata *tx_exc_metadata)
  2664. {
  2665. struct dp_pdev *pdev = vdev->pdev;
  2666. struct dp_soc *soc = pdev->soc;
  2667. struct dp_tx_desc_s *tx_desc;
  2668. QDF_STATUS status;
  2669. struct dp_tx_queue *tx_q = &(msdu_info->tx_queue);
  2670. uint16_t htt_tcl_metadata = 0;
  2671. enum cdp_tx_sw_drop drop_code = TX_MAX_DROP;
  2672. uint8_t tid = msdu_info->tid;
  2673. struct cdp_tid_tx_stats *tid_stats = NULL;
  2674. qdf_dma_addr_t paddr;
  2675. /* Setup Tx descriptor for an MSDU, and MSDU extension descriptor */
  2676. tx_desc = dp_tx_prepare_desc_single(vdev, nbuf, tx_q->desc_pool_id,
  2677. msdu_info, tx_exc_metadata);
  2678. if (!tx_desc) {
  2679. dp_err_rl("Tx_desc prepare Fail vdev_id %d vdev %pK queue %d",
  2680. vdev->vdev_id, vdev, tx_q->desc_pool_id);
  2681. drop_code = TX_DESC_ERR;
  2682. goto fail_return;
  2683. }
  2684. dp_tx_update_tdls_flags(soc, vdev, tx_desc);
  2685. if (qdf_unlikely(peer_id == DP_INVALID_PEER)) {
  2686. htt_tcl_metadata = vdev->htt_tcl_metadata;
  2687. DP_TX_TCL_METADATA_HOST_INSPECTED_SET(htt_tcl_metadata, 1);
  2688. } else if (qdf_unlikely(peer_id != HTT_INVALID_PEER)) {
  2689. DP_TX_TCL_METADATA_TYPE_SET(htt_tcl_metadata,
  2690. DP_TCL_METADATA_TYPE_PEER_BASED);
  2691. DP_TX_TCL_METADATA_PEER_ID_SET(htt_tcl_metadata,
  2692. peer_id);
  2693. dp_tx_bypass_reinjection(soc, tx_desc, tx_exc_metadata);
  2694. } else
  2695. htt_tcl_metadata = vdev->htt_tcl_metadata;
  2696. if (msdu_info->exception_fw)
  2697. DP_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1);
  2698. dp_tx_desc_update_fast_comp_flag(soc, tx_desc,
  2699. !pdev->enhanced_stats_en);
  2700. dp_tx_update_mesh_flags(soc, vdev, tx_desc);
  2701. if (qdf_unlikely(msdu_info->frm_type == dp_tx_frm_rmnet))
  2702. paddr = dp_tx_rmnet_nbuf_map(msdu_info, tx_desc);
  2703. else
  2704. paddr = dp_tx_nbuf_map(vdev, tx_desc, nbuf);
  2705. if (!paddr) {
  2706. /* Handle failure */
  2707. dp_err("qdf_nbuf_map failed");
  2708. DP_STATS_INC(vdev,
  2709. tx_i[msdu_info->xmit_type].dropped.dma_error, 1);
  2710. drop_code = TX_DMA_MAP_ERR;
  2711. goto release_desc;
  2712. }
  2713. tx_desc->dma_addr = paddr;
  2714. dp_tx_desc_history_add(soc, tx_desc->dma_addr, nbuf,
  2715. tx_desc->id, DP_TX_DESC_MAP);
  2716. dp_tx_update_mcast_param(peer_id, &htt_tcl_metadata, vdev, msdu_info);
  2717. /* Enqueue the Tx MSDU descriptor to HW for transmit */
  2718. status = soc->arch_ops.tx_hw_enqueue(soc, vdev, tx_desc,
  2719. htt_tcl_metadata,
  2720. tx_exc_metadata, msdu_info);
  2721. if (status != QDF_STATUS_SUCCESS) {
  2722. dp_tx_err_rl("Tx_hw_enqueue Fail tx_desc %pK queue %d",
  2723. tx_desc, tx_q->ring_id);
  2724. dp_tx_desc_history_add(soc, tx_desc->dma_addr, nbuf,
  2725. tx_desc->id, DP_TX_DESC_UNMAP);
  2726. dp_tx_nbuf_unmap(soc, tx_desc);
  2727. drop_code = TX_HW_ENQUEUE;
  2728. goto release_desc;
  2729. }
  2730. dp_tx_update_ts_on_enqueued(vdev, msdu_info, tx_desc);
  2731. tx_sw_drop_stats_inc(pdev, nbuf, drop_code);
  2732. return NULL;
  2733. release_desc:
  2734. dp_tx_desc_release(soc, tx_desc, tx_q->desc_pool_id);
  2735. fail_return:
  2736. dp_tx_get_tid(vdev, nbuf, msdu_info);
  2737. tx_sw_drop_stats_inc(pdev, nbuf, drop_code);
  2738. tid_stats = &pdev->stats.tid_stats.
  2739. tid_tx_stats[tx_q->ring_id][tid];
  2740. tid_stats->swdrop_cnt[drop_code]++;
  2741. return nbuf;
  2742. }
  2743. /**
  2744. * dp_tdls_tx_comp_free_buff() - Free non std buffer when TDLS flag is set
  2745. * @soc: Soc handle
  2746. * @desc: software Tx descriptor to be processed
  2747. *
  2748. * Return: 0 if Success
  2749. */
  2750. #ifdef FEATURE_WLAN_TDLS
  2751. static inline int
  2752. dp_tdls_tx_comp_free_buff(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  2753. {
  2754. /* If it is TDLS mgmt, don't unmap or free the frame */
  2755. if (desc->flags & DP_TX_DESC_FLAG_TDLS_FRAME) {
  2756. dp_non_std_htt_tx_comp_free_buff(soc, desc);
  2757. return 0;
  2758. }
  2759. return 1;
  2760. }
  2761. #else
  2762. static inline int
  2763. dp_tdls_tx_comp_free_buff(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  2764. {
  2765. return 1;
  2766. }
  2767. #endif
  2768. qdf_nbuf_t dp_tx_comp_free_buf(struct dp_soc *soc, struct dp_tx_desc_s *desc,
  2769. bool delayed_free)
  2770. {
  2771. qdf_nbuf_t nbuf = desc->nbuf;
  2772. enum dp_tx_event_type type = dp_tx_get_event_type(desc->flags);
  2773. /* nbuf already freed in vdev detach path */
  2774. if (!nbuf)
  2775. return NULL;
  2776. if (!dp_tdls_tx_comp_free_buff(soc, desc))
  2777. return NULL;
  2778. /* 0 : MSDU buffer, 1 : MLE */
  2779. if (desc->msdu_ext_desc) {
  2780. /* TSO free */
  2781. if (hal_tx_ext_desc_get_tso_enable(
  2782. desc->msdu_ext_desc->vaddr)) {
  2783. dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf,
  2784. desc->id, DP_TX_COMP_MSDU_EXT);
  2785. dp_tx_tso_seg_history_add(soc,
  2786. desc->msdu_ext_desc->tso_desc,
  2787. desc->nbuf, desc->id, type);
  2788. /* unmap eash TSO seg before free the nbuf */
  2789. dp_tx_tso_unmap_segment(soc,
  2790. desc->msdu_ext_desc->tso_desc,
  2791. desc->msdu_ext_desc->
  2792. tso_num_desc);
  2793. goto nbuf_free;
  2794. }
  2795. if (qdf_unlikely(desc->frm_type == dp_tx_frm_sg)) {
  2796. void *msdu_ext_desc = desc->msdu_ext_desc->vaddr;
  2797. qdf_dma_addr_t iova;
  2798. uint32_t frag_len;
  2799. uint32_t i;
  2800. qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf,
  2801. QDF_DMA_TO_DEVICE,
  2802. qdf_nbuf_headlen(nbuf));
  2803. for (i = 1; i < DP_TX_MAX_NUM_FRAGS; i++) {
  2804. hal_tx_ext_desc_get_frag_info(msdu_ext_desc, i,
  2805. &iova,
  2806. &frag_len);
  2807. if (!iova || !frag_len)
  2808. break;
  2809. qdf_mem_unmap_page(soc->osdev, iova, frag_len,
  2810. QDF_DMA_TO_DEVICE);
  2811. }
  2812. goto nbuf_free;
  2813. }
  2814. }
  2815. /* If it's ME frame, dont unmap the cloned nbuf's */
  2816. if ((desc->flags & DP_TX_DESC_FLAG_ME) && qdf_nbuf_is_cloned(nbuf))
  2817. goto nbuf_free;
  2818. dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf, desc->id, type);
  2819. dp_tx_unmap(soc, desc);
  2820. if (desc->flags & DP_TX_DESC_FLAG_MESH_MODE)
  2821. return dp_mesh_tx_comp_free_buff(soc, desc, delayed_free);
  2822. if (dp_tx_traffic_end_indication_enq_ind_pkt(soc, desc, nbuf))
  2823. return NULL;
  2824. nbuf_free:
  2825. if (delayed_free)
  2826. return nbuf;
  2827. qdf_nbuf_free(nbuf);
  2828. return NULL;
  2829. }
  2830. /**
  2831. * dp_tx_sg_unmap_buf() - Unmap scatter gather fragments
  2832. * @soc: DP soc handle
  2833. * @nbuf: skb
  2834. * @msdu_info: MSDU info
  2835. *
  2836. * Return: None
  2837. */
  2838. static inline void
  2839. dp_tx_sg_unmap_buf(struct dp_soc *soc, qdf_nbuf_t nbuf,
  2840. struct dp_tx_msdu_info_s *msdu_info)
  2841. {
  2842. uint32_t cur_idx;
  2843. struct dp_tx_seg_info_s *seg = msdu_info->u.sg_info.curr_seg;
  2844. qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf, QDF_DMA_TO_DEVICE,
  2845. qdf_nbuf_headlen(nbuf));
  2846. for (cur_idx = 1; cur_idx < seg->frag_cnt; cur_idx++)
  2847. qdf_mem_unmap_page(soc->osdev, (qdf_dma_addr_t)
  2848. (seg->frags[cur_idx].paddr_lo | ((uint64_t)
  2849. seg->frags[cur_idx].paddr_hi) << 32),
  2850. seg->frags[cur_idx].len,
  2851. QDF_DMA_TO_DEVICE);
  2852. }
  2853. #if QDF_LOCK_STATS
  2854. noinline
  2855. #else
  2856. #endif
  2857. qdf_nbuf_t dp_tx_send_msdu_multiple(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2858. struct dp_tx_msdu_info_s *msdu_info)
  2859. {
  2860. uint32_t i;
  2861. struct dp_pdev *pdev = vdev->pdev;
  2862. struct dp_soc *soc = pdev->soc;
  2863. struct dp_tx_desc_s *tx_desc;
  2864. bool is_cce_classified = false;
  2865. QDF_STATUS status;
  2866. uint16_t htt_tcl_metadata = 0;
  2867. struct dp_tx_queue *tx_q = &msdu_info->tx_queue;
  2868. struct cdp_tid_tx_stats *tid_stats = NULL;
  2869. uint8_t prep_desc_fail = 0, hw_enq_fail = 0;
  2870. if (msdu_info->frm_type == dp_tx_frm_me)
  2871. nbuf = msdu_info->u.sg_info.curr_seg->nbuf;
  2872. i = 0;
  2873. /* Print statement to track i and num_seg */
  2874. /*
  2875. * For each segment (maps to 1 MSDU) , prepare software and hardware
  2876. * descriptors using information in msdu_info
  2877. */
  2878. while (i < msdu_info->num_seg) {
  2879. /*
  2880. * Setup Tx descriptor for an MSDU, and MSDU extension
  2881. * descriptor
  2882. */
  2883. tx_desc = dp_tx_prepare_desc(vdev, nbuf, msdu_info,
  2884. tx_q->desc_pool_id);
  2885. if (!tx_desc) {
  2886. if (msdu_info->frm_type == dp_tx_frm_me) {
  2887. prep_desc_fail++;
  2888. dp_tx_me_free_buf(pdev,
  2889. (void *)(msdu_info->u.sg_info
  2890. .curr_seg->frags[0].vaddr));
  2891. if (prep_desc_fail == msdu_info->num_seg) {
  2892. /*
  2893. * Unmap is needed only if descriptor
  2894. * preparation failed for all segments.
  2895. */
  2896. qdf_nbuf_unmap(soc->osdev,
  2897. msdu_info->u.sg_info.
  2898. curr_seg->nbuf,
  2899. QDF_DMA_TO_DEVICE);
  2900. }
  2901. /*
  2902. * Free the nbuf for the current segment
  2903. * and make it point to the next in the list.
  2904. * For me, there are as many segments as there
  2905. * are no of clients.
  2906. */
  2907. qdf_nbuf_free(msdu_info->u.sg_info
  2908. .curr_seg->nbuf);
  2909. if (msdu_info->u.sg_info.curr_seg->next) {
  2910. msdu_info->u.sg_info.curr_seg =
  2911. msdu_info->u.sg_info
  2912. .curr_seg->next;
  2913. nbuf = msdu_info->u.sg_info
  2914. .curr_seg->nbuf;
  2915. }
  2916. i++;
  2917. continue;
  2918. }
  2919. if (msdu_info->frm_type == dp_tx_frm_tso) {
  2920. dp_tx_tso_seg_history_add(
  2921. soc,
  2922. msdu_info->u.tso_info.curr_seg,
  2923. nbuf, 0, DP_TX_DESC_UNMAP);
  2924. dp_tx_tso_unmap_segment(soc,
  2925. msdu_info->u.tso_info.
  2926. curr_seg,
  2927. msdu_info->u.tso_info.
  2928. tso_num_seg_list);
  2929. if (msdu_info->u.tso_info.curr_seg->next) {
  2930. msdu_info->u.tso_info.curr_seg =
  2931. msdu_info->u.tso_info.curr_seg->next;
  2932. i++;
  2933. continue;
  2934. }
  2935. }
  2936. if (msdu_info->frm_type == dp_tx_frm_sg)
  2937. dp_tx_sg_unmap_buf(soc, nbuf, msdu_info);
  2938. goto done;
  2939. }
  2940. if (msdu_info->frm_type == dp_tx_frm_me) {
  2941. tx_desc->msdu_ext_desc->me_buffer =
  2942. (struct dp_tx_me_buf_t *)msdu_info->
  2943. u.sg_info.curr_seg->frags[0].vaddr;
  2944. tx_desc->flags |= DP_TX_DESC_FLAG_ME;
  2945. }
  2946. if (is_cce_classified)
  2947. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  2948. htt_tcl_metadata = vdev->htt_tcl_metadata;
  2949. if (msdu_info->exception_fw) {
  2950. DP_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1);
  2951. }
  2952. dp_tx_is_hp_update_required(i, msdu_info);
  2953. /*
  2954. * For frames with multiple segments (TSO, ME), jump to next
  2955. * segment.
  2956. */
  2957. if (msdu_info->frm_type == dp_tx_frm_tso) {
  2958. if (msdu_info->u.tso_info.curr_seg->next) {
  2959. msdu_info->u.tso_info.curr_seg =
  2960. msdu_info->u.tso_info.curr_seg->next;
  2961. /*
  2962. * If this is a jumbo nbuf, then increment the
  2963. * number of nbuf users for each additional
  2964. * segment of the msdu. This will ensure that
  2965. * the skb is freed only after receiving tx
  2966. * completion for all segments of an nbuf
  2967. */
  2968. qdf_nbuf_inc_users(nbuf);
  2969. /* Check with MCL if this is needed */
  2970. /* nbuf = msdu_info->u.tso_info.curr_seg->nbuf;
  2971. */
  2972. }
  2973. }
  2974. dp_tx_update_mcast_param(DP_INVALID_PEER,
  2975. &htt_tcl_metadata,
  2976. vdev,
  2977. msdu_info);
  2978. /*
  2979. * Enqueue the Tx MSDU descriptor to HW for transmit
  2980. */
  2981. status = soc->arch_ops.tx_hw_enqueue(soc, vdev, tx_desc,
  2982. htt_tcl_metadata,
  2983. NULL, msdu_info);
  2984. dp_tx_check_and_flush_hp(soc, status, msdu_info);
  2985. if (status != QDF_STATUS_SUCCESS) {
  2986. dp_info_rl("Tx_hw_enqueue Fail tx_desc %pK queue %d",
  2987. tx_desc, tx_q->ring_id);
  2988. dp_tx_get_tid(vdev, nbuf, msdu_info);
  2989. tid_stats = &pdev->stats.tid_stats.
  2990. tid_tx_stats[tx_q->ring_id][msdu_info->tid];
  2991. tid_stats->swdrop_cnt[TX_HW_ENQUEUE]++;
  2992. if (msdu_info->frm_type == dp_tx_frm_me) {
  2993. hw_enq_fail++;
  2994. if (hw_enq_fail == msdu_info->num_seg) {
  2995. /*
  2996. * Unmap is needed only if enqueue
  2997. * failed for all segments.
  2998. */
  2999. qdf_nbuf_unmap(soc->osdev,
  3000. msdu_info->u.sg_info.
  3001. curr_seg->nbuf,
  3002. QDF_DMA_TO_DEVICE);
  3003. }
  3004. /*
  3005. * Free the nbuf for the current segment
  3006. * and make it point to the next in the list.
  3007. * For me, there are as many segments as there
  3008. * are no of clients.
  3009. */
  3010. qdf_nbuf_free(msdu_info->u.sg_info
  3011. .curr_seg->nbuf);
  3012. dp_tx_desc_release(soc, tx_desc,
  3013. tx_q->desc_pool_id);
  3014. if (msdu_info->u.sg_info.curr_seg->next) {
  3015. msdu_info->u.sg_info.curr_seg =
  3016. msdu_info->u.sg_info
  3017. .curr_seg->next;
  3018. nbuf = msdu_info->u.sg_info
  3019. .curr_seg->nbuf;
  3020. } else
  3021. break;
  3022. i++;
  3023. continue;
  3024. }
  3025. /*
  3026. * For TSO frames, the nbuf users increment done for
  3027. * the current segment has to be reverted, since the
  3028. * hw enqueue for this segment failed
  3029. */
  3030. if (msdu_info->frm_type == dp_tx_frm_tso &&
  3031. msdu_info->u.tso_info.curr_seg) {
  3032. /*
  3033. * unmap and free current,
  3034. * retransmit remaining segments
  3035. */
  3036. dp_tx_comp_free_buf(soc, tx_desc, false);
  3037. i++;
  3038. dp_tx_desc_release(soc, tx_desc,
  3039. tx_q->desc_pool_id);
  3040. continue;
  3041. }
  3042. if (msdu_info->frm_type == dp_tx_frm_sg)
  3043. dp_tx_sg_unmap_buf(soc, nbuf, msdu_info);
  3044. dp_tx_desc_release(soc, tx_desc, tx_q->desc_pool_id);
  3045. goto done;
  3046. }
  3047. dp_tx_update_ts_on_enqueued(vdev, msdu_info, tx_desc);
  3048. /*
  3049. * TODO
  3050. * if tso_info structure can be modified to have curr_seg
  3051. * as first element, following 2 blocks of code (for TSO and SG)
  3052. * can be combined into 1
  3053. */
  3054. /*
  3055. * For Multicast-Unicast converted packets,
  3056. * each converted frame (for a client) is represented as
  3057. * 1 segment
  3058. */
  3059. if ((msdu_info->frm_type == dp_tx_frm_sg) ||
  3060. (msdu_info->frm_type == dp_tx_frm_me)) {
  3061. if (msdu_info->u.sg_info.curr_seg->next) {
  3062. msdu_info->u.sg_info.curr_seg =
  3063. msdu_info->u.sg_info.curr_seg->next;
  3064. nbuf = msdu_info->u.sg_info.curr_seg->nbuf;
  3065. } else
  3066. break;
  3067. }
  3068. i++;
  3069. }
  3070. nbuf = NULL;
  3071. done:
  3072. return nbuf;
  3073. }
  3074. /**
  3075. * dp_tx_prepare_sg()- Extract SG info from NBUF and prepare msdu_info
  3076. * for SG frames
  3077. * @vdev: DP vdev handle
  3078. * @nbuf: skb
  3079. * @seg_info: Pointer to Segment info Descriptor to be prepared
  3080. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
  3081. *
  3082. * Return: NULL on success,
  3083. * nbuf when it fails to send
  3084. */
  3085. static qdf_nbuf_t dp_tx_prepare_sg(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  3086. struct dp_tx_seg_info_s *seg_info, struct dp_tx_msdu_info_s *msdu_info)
  3087. {
  3088. uint32_t cur_frag, nr_frags, i;
  3089. qdf_dma_addr_t paddr;
  3090. struct dp_tx_sg_info_s *sg_info;
  3091. uint8_t xmit_type = msdu_info->xmit_type;
  3092. sg_info = &msdu_info->u.sg_info;
  3093. nr_frags = qdf_nbuf_get_nr_frags(nbuf);
  3094. if (QDF_STATUS_SUCCESS !=
  3095. qdf_nbuf_map_nbytes_single(vdev->osdev, nbuf,
  3096. QDF_DMA_TO_DEVICE,
  3097. qdf_nbuf_headlen(nbuf))) {
  3098. dp_tx_err("dma map error");
  3099. DP_STATS_INC(vdev, tx_i[xmit_type].sg.dma_map_error,
  3100. 1);
  3101. qdf_nbuf_free(nbuf);
  3102. return NULL;
  3103. }
  3104. paddr = qdf_nbuf_mapped_paddr_get(nbuf);
  3105. seg_info->frags[0].paddr_lo = paddr;
  3106. seg_info->frags[0].paddr_hi = ((uint64_t) paddr) >> 32;
  3107. seg_info->frags[0].len = qdf_nbuf_headlen(nbuf);
  3108. seg_info->frags[0].vaddr = (void *) nbuf;
  3109. for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {
  3110. if (QDF_STATUS_SUCCESS != qdf_nbuf_frag_map(vdev->osdev,
  3111. nbuf, 0,
  3112. QDF_DMA_TO_DEVICE,
  3113. cur_frag)) {
  3114. dp_tx_err("frag dma map error");
  3115. DP_STATS_INC(vdev,
  3116. tx_i[xmit_type].sg.dma_map_error,
  3117. 1);
  3118. goto map_err;
  3119. }
  3120. paddr = qdf_nbuf_get_tx_frag_paddr(nbuf);
  3121. seg_info->frags[cur_frag + 1].paddr_lo = paddr;
  3122. seg_info->frags[cur_frag + 1].paddr_hi =
  3123. ((uint64_t) paddr) >> 32;
  3124. seg_info->frags[cur_frag + 1].len =
  3125. qdf_nbuf_get_frag_size(nbuf, cur_frag);
  3126. }
  3127. seg_info->frag_cnt = (cur_frag + 1);
  3128. seg_info->total_len = qdf_nbuf_len(nbuf);
  3129. seg_info->next = NULL;
  3130. sg_info->curr_seg = seg_info;
  3131. msdu_info->frm_type = dp_tx_frm_sg;
  3132. msdu_info->num_seg = 1;
  3133. return nbuf;
  3134. map_err:
  3135. /* restore paddr into nbuf before calling unmap */
  3136. qdf_nbuf_mapped_paddr_set(nbuf,
  3137. (qdf_dma_addr_t)(seg_info->frags[0].paddr_lo |
  3138. ((uint64_t)
  3139. seg_info->frags[0].paddr_hi) << 32));
  3140. qdf_nbuf_unmap_nbytes_single(vdev->osdev, nbuf,
  3141. QDF_DMA_TO_DEVICE,
  3142. seg_info->frags[0].len);
  3143. for (i = 1; i <= cur_frag; i++) {
  3144. qdf_mem_unmap_page(vdev->osdev, (qdf_dma_addr_t)
  3145. (seg_info->frags[i].paddr_lo | ((uint64_t)
  3146. seg_info->frags[i].paddr_hi) << 32),
  3147. seg_info->frags[i].len,
  3148. QDF_DMA_TO_DEVICE);
  3149. }
  3150. qdf_nbuf_free(nbuf);
  3151. return NULL;
  3152. }
  3153. /**
  3154. * dp_tx_add_tx_sniffer_meta_data()- Add tx_sniffer meta hdr info
  3155. * @vdev: DP vdev handle
  3156. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
  3157. * @ppdu_cookie: PPDU cookie that should be replayed in the ppdu completions
  3158. *
  3159. * Return: NULL on failure,
  3160. * nbuf when extracted successfully
  3161. */
  3162. static
  3163. void dp_tx_add_tx_sniffer_meta_data(struct dp_vdev *vdev,
  3164. struct dp_tx_msdu_info_s *msdu_info,
  3165. uint16_t ppdu_cookie)
  3166. {
  3167. struct htt_tx_msdu_desc_ext2_t *meta_data =
  3168. (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
  3169. qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
  3170. HTT_TX_MSDU_EXT2_DESC_FLAG_SEND_AS_STANDALONE_SET
  3171. (msdu_info->meta_data[5], 1);
  3172. HTT_TX_MSDU_EXT2_DESC_FLAG_HOST_OPAQUE_VALID_SET
  3173. (msdu_info->meta_data[5], 1);
  3174. HTT_TX_MSDU_EXT2_DESC_HOST_OPAQUE_COOKIE_SET
  3175. (msdu_info->meta_data[6], ppdu_cookie);
  3176. msdu_info->exception_fw = 1;
  3177. msdu_info->is_tx_sniffer = 1;
  3178. }
  3179. #ifdef MESH_MODE_SUPPORT
  3180. /**
  3181. * dp_tx_extract_mesh_meta_data()- Extract mesh meta hdr info from nbuf
  3182. * and prepare msdu_info for mesh frames.
  3183. * @vdev: DP vdev handle
  3184. * @nbuf: skb
  3185. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
  3186. *
  3187. * Return: NULL on failure,
  3188. * nbuf when extracted successfully
  3189. */
  3190. static
  3191. qdf_nbuf_t dp_tx_extract_mesh_meta_data(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  3192. struct dp_tx_msdu_info_s *msdu_info)
  3193. {
  3194. struct meta_hdr_s *mhdr;
  3195. struct htt_tx_msdu_desc_ext2_t *meta_data =
  3196. (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
  3197. mhdr = (struct meta_hdr_s *)qdf_nbuf_data(nbuf);
  3198. if (CB_FTYPE_MESH_TX_INFO != qdf_nbuf_get_tx_ftype(nbuf)) {
  3199. msdu_info->exception_fw = 0;
  3200. goto remove_meta_hdr;
  3201. }
  3202. msdu_info->exception_fw = 1;
  3203. qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
  3204. meta_data->host_tx_desc_pool = 1;
  3205. meta_data->update_peer_cache = 1;
  3206. meta_data->learning_frame = 1;
  3207. if (!(mhdr->flags & METAHDR_FLAG_AUTO_RATE)) {
  3208. meta_data->power = mhdr->power;
  3209. meta_data->mcs_mask = 1 << mhdr->rate_info[0].mcs;
  3210. meta_data->nss_mask = 1 << mhdr->rate_info[0].nss;
  3211. meta_data->pream_type = mhdr->rate_info[0].preamble_type;
  3212. meta_data->retry_limit = mhdr->rate_info[0].max_tries;
  3213. meta_data->dyn_bw = 1;
  3214. meta_data->valid_pwr = 1;
  3215. meta_data->valid_mcs_mask = 1;
  3216. meta_data->valid_nss_mask = 1;
  3217. meta_data->valid_preamble_type = 1;
  3218. meta_data->valid_retries = 1;
  3219. meta_data->valid_bw_info = 1;
  3220. }
  3221. if (mhdr->flags & METAHDR_FLAG_NOENCRYPT) {
  3222. meta_data->encrypt_type = 0;
  3223. meta_data->valid_encrypt_type = 1;
  3224. meta_data->learning_frame = 0;
  3225. }
  3226. meta_data->valid_key_flags = 1;
  3227. meta_data->key_flags = (mhdr->keyix & 0x3);
  3228. remove_meta_hdr:
  3229. if (qdf_nbuf_pull_head(nbuf, sizeof(struct meta_hdr_s)) == NULL) {
  3230. dp_tx_err("qdf_nbuf_pull_head failed");
  3231. qdf_nbuf_free(nbuf);
  3232. return NULL;
  3233. }
  3234. msdu_info->tid = qdf_nbuf_get_priority(nbuf);
  3235. dp_tx_info("Meta hdr %0x %0x %0x %0x %0x %0x"
  3236. " tid %d to_fw %d",
  3237. msdu_info->meta_data[0],
  3238. msdu_info->meta_data[1],
  3239. msdu_info->meta_data[2],
  3240. msdu_info->meta_data[3],
  3241. msdu_info->meta_data[4],
  3242. msdu_info->meta_data[5],
  3243. msdu_info->tid, msdu_info->exception_fw);
  3244. return nbuf;
  3245. }
  3246. #else
  3247. static
  3248. qdf_nbuf_t dp_tx_extract_mesh_meta_data(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  3249. struct dp_tx_msdu_info_s *msdu_info)
  3250. {
  3251. return nbuf;
  3252. }
  3253. #endif
  3254. /**
  3255. * dp_check_exc_metadata() - Checks if parameters are valid
  3256. * @tx_exc: holds all exception path parameters
  3257. *
  3258. * Return: true when all the parameters are valid else false
  3259. *
  3260. */
  3261. static bool dp_check_exc_metadata(struct cdp_tx_exception_metadata *tx_exc)
  3262. {
  3263. bool invalid_tid = (tx_exc->tid >= DP_MAX_TIDS && tx_exc->tid !=
  3264. HTT_INVALID_TID);
  3265. bool invalid_encap_type =
  3266. (tx_exc->tx_encap_type > htt_cmn_pkt_num_types &&
  3267. tx_exc->tx_encap_type != CDP_INVALID_TX_ENCAP_TYPE);
  3268. bool invalid_sec_type = (tx_exc->sec_type > cdp_num_sec_types &&
  3269. tx_exc->sec_type != CDP_INVALID_SEC_TYPE);
  3270. bool invalid_cookie = (tx_exc->is_tx_sniffer == 1 &&
  3271. tx_exc->ppdu_cookie == 0);
  3272. if (tx_exc->is_intrabss_fwd)
  3273. return true;
  3274. if (invalid_tid || invalid_encap_type || invalid_sec_type ||
  3275. invalid_cookie) {
  3276. return false;
  3277. }
  3278. return true;
  3279. }
  3280. #ifdef ATH_SUPPORT_IQUE
  3281. bool dp_tx_mcast_enhance(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3282. {
  3283. qdf_ether_header_t *eh;
  3284. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3285. /* Mcast to Ucast Conversion*/
  3286. if (qdf_likely(!vdev->mcast_enhancement_en))
  3287. return true;
  3288. eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
  3289. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost) &&
  3290. !DP_FRAME_IS_BROADCAST((eh)->ether_dhost)) {
  3291. dp_verbose_debug("Mcast frm for ME %pK", vdev);
  3292. qdf_nbuf_set_next(nbuf, NULL);
  3293. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].mcast_en.mcast_pkt, 1,
  3294. qdf_nbuf_len(nbuf));
  3295. if (dp_tx_prepare_send_me(vdev, nbuf) ==
  3296. QDF_STATUS_SUCCESS) {
  3297. return false;
  3298. }
  3299. if (qdf_unlikely(vdev->igmp_mcast_enhanc_en > 0)) {
  3300. if (dp_tx_prepare_send_igmp_me(vdev, nbuf) ==
  3301. QDF_STATUS_SUCCESS) {
  3302. return false;
  3303. }
  3304. }
  3305. }
  3306. return true;
  3307. }
  3308. #else
  3309. bool dp_tx_mcast_enhance(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3310. {
  3311. return true;
  3312. }
  3313. #endif
  3314. #ifdef QCA_SUPPORT_WDS_EXTENDED
  3315. /**
  3316. * dp_tx_mcast_drop() - Drop mcast frame if drop_tx_mcast is set in WDS_EXT
  3317. * @vdev: vdev handle
  3318. * @nbuf: skb
  3319. *
  3320. * Return: true if frame is dropped, false otherwise
  3321. */
  3322. static inline bool dp_tx_mcast_drop(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3323. {
  3324. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3325. /* Drop tx mcast and WDS Extended feature check */
  3326. if (qdf_unlikely((vdev->drop_tx_mcast) && (vdev->wds_ext_enabled))) {
  3327. qdf_ether_header_t *eh = (qdf_ether_header_t *)
  3328. qdf_nbuf_data(nbuf);
  3329. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost)) {
  3330. DP_STATS_INC(vdev,
  3331. tx_i[xmit_type].dropped.tx_mcast_drop, 1);
  3332. return true;
  3333. }
  3334. }
  3335. return false;
  3336. }
  3337. #else
  3338. static inline bool dp_tx_mcast_drop(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3339. {
  3340. return false;
  3341. }
  3342. #endif
  3343. /**
  3344. * dp_tx_per_pkt_vdev_id_check() - vdev id check for frame
  3345. * @nbuf: qdf_nbuf_t
  3346. * @vdev: struct dp_vdev *
  3347. *
  3348. * Allow packet for processing only if it is for peer client which is
  3349. * connected with same vap. Drop packet if client is connected to
  3350. * different vap.
  3351. *
  3352. * Return: QDF_STATUS
  3353. */
  3354. static inline QDF_STATUS
  3355. dp_tx_per_pkt_vdev_id_check(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
  3356. {
  3357. struct dp_ast_entry *dst_ast_entry = NULL;
  3358. qdf_ether_header_t *eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
  3359. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost) ||
  3360. DP_FRAME_IS_BROADCAST((eh)->ether_dhost))
  3361. return QDF_STATUS_SUCCESS;
  3362. qdf_spin_lock_bh(&vdev->pdev->soc->ast_lock);
  3363. dst_ast_entry = dp_peer_ast_hash_find_by_vdevid(vdev->pdev->soc,
  3364. eh->ether_dhost,
  3365. vdev->vdev_id);
  3366. /* If there is no ast entry, return failure */
  3367. if (qdf_unlikely(!dst_ast_entry)) {
  3368. qdf_spin_unlock_bh(&vdev->pdev->soc->ast_lock);
  3369. return QDF_STATUS_E_FAILURE;
  3370. }
  3371. qdf_spin_unlock_bh(&vdev->pdev->soc->ast_lock);
  3372. return QDF_STATUS_SUCCESS;
  3373. }
  3374. /**
  3375. * dp_tx_nawds_handler() - NAWDS handler
  3376. *
  3377. * @soc: DP soc handle
  3378. * @vdev: DP vdev handle
  3379. * @msdu_info: msdu_info required to create HTT metadata
  3380. * @nbuf: skb
  3381. * @sa_peer_id:
  3382. *
  3383. * This API transfers the multicast frames with the peer id
  3384. * on NAWDS enabled peer.
  3385. *
  3386. * Return: none
  3387. */
  3388. void dp_tx_nawds_handler(struct dp_soc *soc, struct dp_vdev *vdev,
  3389. struct dp_tx_msdu_info_s *msdu_info,
  3390. qdf_nbuf_t nbuf, uint16_t sa_peer_id)
  3391. {
  3392. struct dp_peer *peer = NULL;
  3393. qdf_nbuf_t nbuf_clone = NULL;
  3394. uint16_t peer_id = DP_INVALID_PEER;
  3395. struct dp_txrx_peer *txrx_peer;
  3396. uint8_t link_id = 0;
  3397. /* This check avoids pkt forwarding which is entered
  3398. * in the ast table but still doesn't have valid peerid.
  3399. */
  3400. if (sa_peer_id == HTT_INVALID_PEER)
  3401. return;
  3402. qdf_spin_lock_bh(&vdev->peer_list_lock);
  3403. TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
  3404. txrx_peer = dp_get_txrx_peer(peer);
  3405. if (!txrx_peer)
  3406. continue;
  3407. if (!txrx_peer->bss_peer && txrx_peer->nawds_enabled) {
  3408. peer_id = peer->peer_id;
  3409. if (!dp_peer_is_primary_link_peer(peer))
  3410. continue;
  3411. /* In the case of wds ext peer mcast traffic will be
  3412. * sent as part of VLAN interface
  3413. */
  3414. if (dp_peer_is_wds_ext_peer(txrx_peer))
  3415. continue;
  3416. /* Multicast packets needs to be
  3417. * dropped in case of intra bss forwarding
  3418. */
  3419. if (sa_peer_id == txrx_peer->peer_id) {
  3420. dp_tx_debug("multicast packet");
  3421. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  3422. tx.nawds_mcast_drop,
  3423. 1, link_id);
  3424. continue;
  3425. }
  3426. nbuf_clone = qdf_nbuf_clone(nbuf);
  3427. if (!nbuf_clone) {
  3428. QDF_TRACE(QDF_MODULE_ID_DP,
  3429. QDF_TRACE_LEVEL_ERROR,
  3430. FL("nbuf clone failed"));
  3431. break;
  3432. }
  3433. nbuf_clone = dp_tx_send_msdu_single(vdev, nbuf_clone,
  3434. msdu_info, peer_id,
  3435. NULL);
  3436. if (nbuf_clone) {
  3437. dp_tx_debug("pkt send failed");
  3438. qdf_nbuf_free(nbuf_clone);
  3439. } else {
  3440. if (peer_id != DP_INVALID_PEER)
  3441. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
  3442. tx.nawds_mcast,
  3443. 1, qdf_nbuf_len(nbuf), link_id);
  3444. }
  3445. }
  3446. }
  3447. qdf_spin_unlock_bh(&vdev->peer_list_lock);
  3448. }
  3449. #ifdef WLAN_MCAST_MLO
  3450. static inline bool
  3451. dp_tx_check_mesh_vdev(struct dp_vdev *vdev,
  3452. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3453. {
  3454. if (!tx_exc_metadata->is_mlo_mcast && qdf_unlikely(vdev->mesh_vdev))
  3455. return true;
  3456. return false;
  3457. }
  3458. #else
  3459. static inline bool
  3460. dp_tx_check_mesh_vdev(struct dp_vdev *vdev,
  3461. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3462. {
  3463. if (qdf_unlikely(vdev->mesh_vdev))
  3464. return true;
  3465. return false;
  3466. }
  3467. #endif
  3468. qdf_nbuf_t
  3469. dp_tx_send_exception(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3470. qdf_nbuf_t nbuf,
  3471. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3472. {
  3473. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3474. struct dp_tx_msdu_info_s msdu_info;
  3475. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  3476. DP_MOD_ID_TX_EXCEPTION);
  3477. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3478. if (qdf_unlikely(!vdev))
  3479. goto fail;
  3480. qdf_mem_zero(&msdu_info, sizeof(msdu_info));
  3481. if (!tx_exc_metadata)
  3482. goto fail;
  3483. msdu_info.tid = tx_exc_metadata->tid;
  3484. msdu_info.xmit_type = xmit_type;
  3485. dp_verbose_debug("skb "QDF_MAC_ADDR_FMT,
  3486. QDF_MAC_ADDR_REF(nbuf->data));
  3487. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].rcvd, 1, qdf_nbuf_len(nbuf));
  3488. if (qdf_unlikely(!dp_check_exc_metadata(tx_exc_metadata))) {
  3489. dp_tx_err("Invalid parameters in exception path");
  3490. goto fail;
  3491. }
  3492. /* for peer based metadata check if peer is valid */
  3493. if (tx_exc_metadata->peer_id != CDP_INVALID_PEER) {
  3494. struct dp_peer *peer = NULL;
  3495. peer = dp_peer_get_ref_by_id(vdev->pdev->soc,
  3496. tx_exc_metadata->peer_id,
  3497. DP_MOD_ID_TX_EXCEPTION);
  3498. if (qdf_unlikely(!peer)) {
  3499. DP_STATS_INC(vdev,
  3500. tx_i[xmit_type].dropped.invalid_peer_id_in_exc_path,
  3501. 1);
  3502. goto fail;
  3503. }
  3504. dp_peer_unref_delete(peer, DP_MOD_ID_TX_EXCEPTION);
  3505. }
  3506. /* Basic sanity checks for unsupported packets */
  3507. /* MESH mode */
  3508. if (dp_tx_check_mesh_vdev(vdev, tx_exc_metadata)) {
  3509. dp_tx_err("Mesh mode is not supported in exception path");
  3510. goto fail;
  3511. }
  3512. /*
  3513. * Classify the frame and call corresponding
  3514. * "prepare" function which extracts the segment (TSO)
  3515. * and fragmentation information (for TSO , SG, ME, or Raw)
  3516. * into MSDU_INFO structure which is later used to fill
  3517. * SW and HW descriptors.
  3518. */
  3519. if (qdf_nbuf_is_tso(nbuf)) {
  3520. dp_verbose_debug("TSO frame %pK", vdev);
  3521. DP_STATS_INC_PKT(vdev->pdev, tso_stats.num_tso_pkts, 1,
  3522. qdf_nbuf_len(nbuf));
  3523. if (dp_tx_prepare_tso(vdev, nbuf, &msdu_info)) {
  3524. DP_STATS_INC_PKT(vdev->pdev, tso_stats.dropped_host, 1,
  3525. qdf_nbuf_len(nbuf));
  3526. goto fail;
  3527. }
  3528. DP_STATS_INC(vdev,
  3529. tx_i[xmit_type].rcvd.num, msdu_info.num_seg - 1);
  3530. goto send_multiple;
  3531. }
  3532. /* SG */
  3533. if (qdf_unlikely(qdf_nbuf_is_nonlinear(nbuf))) {
  3534. struct dp_tx_seg_info_s seg_info = {0};
  3535. nbuf = dp_tx_prepare_sg(vdev, nbuf, &seg_info, &msdu_info);
  3536. if (!nbuf)
  3537. goto fail;
  3538. dp_verbose_debug("non-TSO SG frame %pK", vdev);
  3539. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].sg.sg_pkt, 1,
  3540. qdf_nbuf_len(nbuf));
  3541. goto send_multiple;
  3542. }
  3543. if (qdf_likely(tx_exc_metadata->is_tx_sniffer)) {
  3544. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].sniffer_rcvd, 1,
  3545. qdf_nbuf_len(nbuf));
  3546. dp_tx_add_tx_sniffer_meta_data(vdev, &msdu_info,
  3547. tx_exc_metadata->ppdu_cookie);
  3548. }
  3549. /*
  3550. * Get HW Queue to use for this frame.
  3551. * TCL supports upto 4 DMA rings, out of which 3 rings are
  3552. * dedicated for data and 1 for command.
  3553. * "queue_id" maps to one hardware ring.
  3554. * With each ring, we also associate a unique Tx descriptor pool
  3555. * to minimize lock contention for these resources.
  3556. */
  3557. dp_tx_get_queue(vdev, nbuf, &msdu_info.tx_queue);
  3558. DP_STATS_INC(vdev,
  3559. tx_i[xmit_type].rcvd_per_core[msdu_info.tx_queue.desc_pool_id],
  3560. 1);
  3561. /*
  3562. * if the packet is mcast packet send through mlo_macst handler
  3563. * for all prnt_vdevs
  3564. */
  3565. if (soc->arch_ops.dp_tx_mlo_mcast_send) {
  3566. nbuf = soc->arch_ops.dp_tx_mlo_mcast_send(soc, vdev,
  3567. nbuf,
  3568. tx_exc_metadata);
  3569. if (!nbuf)
  3570. goto fail;
  3571. }
  3572. if (qdf_likely(tx_exc_metadata->is_intrabss_fwd)) {
  3573. if (qdf_unlikely(vdev->nawds_enabled)) {
  3574. /*
  3575. * This is a multicast packet
  3576. */
  3577. dp_tx_nawds_handler(soc, vdev, &msdu_info, nbuf,
  3578. tx_exc_metadata->peer_id);
  3579. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].nawds_mcast,
  3580. 1, qdf_nbuf_len(nbuf));
  3581. }
  3582. nbuf = dp_tx_send_msdu_single(vdev, nbuf, &msdu_info,
  3583. DP_INVALID_PEER, NULL);
  3584. } else {
  3585. /*
  3586. * Check exception descriptors
  3587. */
  3588. if (dp_tx_exception_limit_check(vdev, xmit_type))
  3589. goto fail;
  3590. /* Single linear frame */
  3591. /*
  3592. * If nbuf is a simple linear frame, use send_single function to
  3593. * prepare direct-buffer type TCL descriptor and enqueue to TCL
  3594. * SRNG. There is no need to setup a MSDU extension descriptor.
  3595. */
  3596. nbuf = dp_tx_send_msdu_single(vdev, nbuf, &msdu_info,
  3597. tx_exc_metadata->peer_id,
  3598. tx_exc_metadata);
  3599. }
  3600. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3601. return nbuf;
  3602. send_multiple:
  3603. nbuf = dp_tx_send_msdu_multiple(vdev, nbuf, &msdu_info);
  3604. fail:
  3605. if (vdev)
  3606. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3607. dp_verbose_debug("pkt send failed");
  3608. return nbuf;
  3609. }
  3610. qdf_nbuf_t
  3611. dp_tx_send_exception_vdev_id_check(struct cdp_soc_t *soc_hdl,
  3612. uint8_t vdev_id, qdf_nbuf_t nbuf,
  3613. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3614. {
  3615. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3616. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  3617. DP_MOD_ID_TX_EXCEPTION);
  3618. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3619. if (qdf_unlikely(!vdev))
  3620. goto fail;
  3621. if (qdf_unlikely(dp_tx_per_pkt_vdev_id_check(nbuf, vdev)
  3622. == QDF_STATUS_E_FAILURE)) {
  3623. DP_STATS_INC(vdev,
  3624. tx_i[xmit_type].dropped.fail_per_pkt_vdev_id_check,
  3625. 1);
  3626. goto fail;
  3627. }
  3628. /* Unref count as it will again be taken inside dp_tx_exception */
  3629. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3630. return dp_tx_send_exception(soc_hdl, vdev_id, nbuf, tx_exc_metadata);
  3631. fail:
  3632. if (vdev)
  3633. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3634. dp_verbose_debug("pkt send failed");
  3635. return nbuf;
  3636. }
  3637. #ifdef MESH_MODE_SUPPORT
  3638. qdf_nbuf_t dp_tx_send_mesh(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3639. qdf_nbuf_t nbuf)
  3640. {
  3641. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3642. struct meta_hdr_s *mhdr;
  3643. qdf_nbuf_t nbuf_mesh = NULL;
  3644. qdf_nbuf_t nbuf_clone = NULL;
  3645. struct dp_vdev *vdev;
  3646. uint8_t no_enc_frame = 0;
  3647. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3648. nbuf_mesh = qdf_nbuf_unshare(nbuf);
  3649. if (!nbuf_mesh) {
  3650. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3651. "qdf_nbuf_unshare failed");
  3652. return nbuf;
  3653. }
  3654. vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_MESH);
  3655. if (!vdev) {
  3656. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3657. "vdev is NULL for vdev_id %d", vdev_id);
  3658. return nbuf;
  3659. }
  3660. nbuf = nbuf_mesh;
  3661. mhdr = (struct meta_hdr_s *)qdf_nbuf_data(nbuf);
  3662. if ((vdev->sec_type != cdp_sec_type_none) &&
  3663. (mhdr->flags & METAHDR_FLAG_NOENCRYPT))
  3664. no_enc_frame = 1;
  3665. if (mhdr->flags & METAHDR_FLAG_NOQOS)
  3666. qdf_nbuf_set_priority(nbuf, HTT_TX_EXT_TID_NON_QOS_MCAST_BCAST);
  3667. if ((mhdr->flags & METAHDR_FLAG_INFO_UPDATED) &&
  3668. !no_enc_frame) {
  3669. nbuf_clone = qdf_nbuf_clone(nbuf);
  3670. if (!nbuf_clone) {
  3671. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3672. "qdf_nbuf_clone failed");
  3673. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
  3674. return nbuf;
  3675. }
  3676. qdf_nbuf_set_tx_ftype(nbuf_clone, CB_FTYPE_MESH_TX_INFO);
  3677. }
  3678. if (nbuf_clone) {
  3679. if (!dp_tx_send(soc_hdl, vdev_id, nbuf_clone)) {
  3680. DP_STATS_INC(vdev, tx_i[xmit_type].mesh.exception_fw,
  3681. 1);
  3682. } else {
  3683. qdf_nbuf_free(nbuf_clone);
  3684. }
  3685. }
  3686. if (no_enc_frame)
  3687. qdf_nbuf_set_tx_ftype(nbuf, CB_FTYPE_MESH_TX_INFO);
  3688. else
  3689. qdf_nbuf_set_tx_ftype(nbuf, CB_FTYPE_INVALID);
  3690. nbuf = dp_tx_send(soc_hdl, vdev_id, nbuf);
  3691. if ((!nbuf) && no_enc_frame) {
  3692. DP_STATS_INC(vdev, tx_i[xmit_type].mesh.exception_fw, 1);
  3693. }
  3694. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
  3695. return nbuf;
  3696. }
  3697. #else
  3698. qdf_nbuf_t dp_tx_send_mesh(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3699. qdf_nbuf_t nbuf)
  3700. {
  3701. return dp_tx_send(soc_hdl, vdev_id, nbuf);
  3702. }
  3703. #endif
  3704. #ifdef DP_UMAC_HW_RESET_SUPPORT
  3705. qdf_nbuf_t dp_tx_drop(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3706. qdf_nbuf_t nbuf)
  3707. {
  3708. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3709. struct dp_vdev *vdev = NULL;
  3710. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3711. vdev = soc->vdev_id_map[vdev_id];
  3712. if (qdf_unlikely(!vdev))
  3713. return nbuf;
  3714. DP_STATS_INC(vdev, tx_i[xmit_type].dropped.drop_ingress, 1);
  3715. return nbuf;
  3716. }
  3717. qdf_nbuf_t dp_tx_exc_drop(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3718. qdf_nbuf_t nbuf,
  3719. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3720. {
  3721. return dp_tx_drop(soc_hdl, vdev_id, nbuf);
  3722. }
  3723. #endif
  3724. #ifdef FEATURE_DIRECT_LINK
  3725. /**
  3726. * dp_vdev_tx_mark_to_fw() - Mark to_fw bit for the tx packet
  3727. * @nbuf: skb
  3728. * @vdev: DP vdev handle
  3729. *
  3730. * Return: None
  3731. */
  3732. static inline void dp_vdev_tx_mark_to_fw(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
  3733. {
  3734. if (qdf_unlikely(vdev->to_fw))
  3735. QDF_NBUF_CB_TX_PACKET_TO_FW(nbuf) = 1;
  3736. }
  3737. #else
  3738. static inline void dp_vdev_tx_mark_to_fw(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
  3739. {
  3740. }
  3741. #endif
  3742. qdf_nbuf_t dp_tx_send(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3743. qdf_nbuf_t nbuf)
  3744. {
  3745. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3746. uint16_t peer_id = HTT_INVALID_PEER;
  3747. /*
  3748. * doing a memzero is causing additional function call overhead
  3749. * so doing static stack clearing
  3750. */
  3751. struct dp_tx_msdu_info_s msdu_info = {0};
  3752. struct dp_vdev *vdev = NULL;
  3753. qdf_nbuf_t end_nbuf = NULL;
  3754. uint8_t xmit_type;
  3755. if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
  3756. return nbuf;
  3757. /*
  3758. * dp_vdev_get_ref_by_id does does a atomic operation avoid using
  3759. * this in per packet path.
  3760. *
  3761. * As in this path vdev memory is already protected with netdev
  3762. * tx lock
  3763. */
  3764. vdev = soc->vdev_id_map[vdev_id];
  3765. if (qdf_unlikely(!vdev))
  3766. return nbuf;
  3767. dp_tx_get_driver_ingress_ts(vdev, &msdu_info, nbuf);
  3768. dp_vdev_tx_mark_to_fw(nbuf, vdev);
  3769. /*
  3770. * Set Default Host TID value to invalid TID
  3771. * (TID override disabled)
  3772. */
  3773. msdu_info.tid = HTT_TX_EXT_TID_INVALID;
  3774. xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3775. msdu_info.xmit_type = xmit_type;
  3776. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].rcvd, 1, qdf_nbuf_len(nbuf));
  3777. if (qdf_unlikely(vdev->mesh_vdev)) {
  3778. qdf_nbuf_t nbuf_mesh = dp_tx_extract_mesh_meta_data(vdev, nbuf,
  3779. &msdu_info);
  3780. if (!nbuf_mesh) {
  3781. dp_verbose_debug("Extracting mesh metadata failed");
  3782. return nbuf;
  3783. }
  3784. nbuf = nbuf_mesh;
  3785. }
  3786. /*
  3787. * Get HW Queue to use for this frame.
  3788. * TCL supports upto 4 DMA rings, out of which 3 rings are
  3789. * dedicated for data and 1 for command.
  3790. * "queue_id" maps to one hardware ring.
  3791. * With each ring, we also associate a unique Tx descriptor pool
  3792. * to minimize lock contention for these resources.
  3793. */
  3794. dp_tx_get_queue(vdev, nbuf, &msdu_info.tx_queue);
  3795. DP_STATS_INC(vdev,
  3796. tx_i[xmit_type].rcvd_per_core[msdu_info.tx_queue.desc_pool_id],
  3797. 1);
  3798. /*
  3799. * TCL H/W supports 2 DSCP-TID mapping tables.
  3800. * Table 1 - Default DSCP-TID mapping table
  3801. * Table 2 - 1 DSCP-TID override table
  3802. *
  3803. * If we need a different DSCP-TID mapping for this vap,
  3804. * call tid_classify to extract DSCP/ToS from frame and
  3805. * map to a TID and store in msdu_info. This is later used
  3806. * to fill in TCL Input descriptor (per-packet TID override).
  3807. */
  3808. dp_tx_classify_tid(vdev, nbuf, &msdu_info);
  3809. /*
  3810. * Classify the frame and call corresponding
  3811. * "prepare" function which extracts the segment (TSO)
  3812. * and fragmentation information (for TSO , SG, ME, or Raw)
  3813. * into MSDU_INFO structure which is later used to fill
  3814. * SW and HW descriptors.
  3815. */
  3816. if (qdf_nbuf_is_tso(nbuf)) {
  3817. dp_verbose_debug("TSO frame %pK", vdev);
  3818. DP_STATS_INC_PKT(vdev->pdev, tso_stats.num_tso_pkts, 1,
  3819. qdf_nbuf_len(nbuf));
  3820. if (dp_tx_prepare_tso(vdev, nbuf, &msdu_info)) {
  3821. DP_STATS_INC_PKT(vdev->pdev, tso_stats.dropped_host, 1,
  3822. qdf_nbuf_len(nbuf));
  3823. return nbuf;
  3824. }
  3825. DP_STATS_INC(vdev, tx_i[xmit_type].rcvd.num,
  3826. msdu_info.num_seg - 1);
  3827. goto send_multiple;
  3828. }
  3829. /* SG */
  3830. if (qdf_unlikely(qdf_nbuf_is_nonlinear(nbuf))) {
  3831. if (qdf_nbuf_get_nr_frags(nbuf) > DP_TX_MAX_NUM_FRAGS - 1) {
  3832. if (qdf_unlikely(qdf_nbuf_linearize(nbuf)))
  3833. return nbuf;
  3834. } else {
  3835. struct dp_tx_seg_info_s seg_info = {0};
  3836. if (qdf_unlikely(is_nbuf_frm_rmnet(nbuf, &msdu_info)))
  3837. goto send_single;
  3838. nbuf = dp_tx_prepare_sg(vdev, nbuf, &seg_info,
  3839. &msdu_info);
  3840. if (!nbuf)
  3841. return NULL;
  3842. dp_verbose_debug("non-TSO SG frame %pK", vdev);
  3843. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].sg.sg_pkt, 1,
  3844. qdf_nbuf_len(nbuf));
  3845. goto send_multiple;
  3846. }
  3847. }
  3848. if (qdf_unlikely(!dp_tx_mcast_enhance(vdev, nbuf)))
  3849. return NULL;
  3850. if (qdf_unlikely(dp_tx_mcast_drop(vdev, nbuf)))
  3851. return nbuf;
  3852. /* RAW */
  3853. if (qdf_unlikely(vdev->tx_encap_type == htt_cmn_pkt_type_raw)) {
  3854. struct dp_tx_seg_info_s seg_info = {0};
  3855. nbuf = dp_tx_prepare_raw(vdev, nbuf, &seg_info, &msdu_info);
  3856. if (!nbuf)
  3857. return NULL;
  3858. dp_verbose_debug("Raw frame %pK", vdev);
  3859. goto send_multiple;
  3860. }
  3861. if (qdf_unlikely(vdev->nawds_enabled)) {
  3862. qdf_ether_header_t *eh = (qdf_ether_header_t *)
  3863. qdf_nbuf_data(nbuf);
  3864. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost)) {
  3865. uint16_t sa_peer_id = DP_INVALID_PEER;
  3866. if (!soc->ast_offload_support) {
  3867. struct dp_ast_entry *ast_entry = NULL;
  3868. qdf_spin_lock_bh(&soc->ast_lock);
  3869. ast_entry = dp_peer_ast_hash_find_by_pdevid
  3870. (soc,
  3871. (uint8_t *)(eh->ether_shost),
  3872. vdev->pdev->pdev_id);
  3873. if (ast_entry)
  3874. sa_peer_id = ast_entry->peer_id;
  3875. qdf_spin_unlock_bh(&soc->ast_lock);
  3876. }
  3877. dp_tx_nawds_handler(soc, vdev, &msdu_info, nbuf,
  3878. sa_peer_id);
  3879. }
  3880. peer_id = DP_INVALID_PEER;
  3881. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].nawds_mcast,
  3882. 1, qdf_nbuf_len(nbuf));
  3883. }
  3884. send_single:
  3885. /* Single linear frame */
  3886. /*
  3887. * If nbuf is a simple linear frame, use send_single function to
  3888. * prepare direct-buffer type TCL descriptor and enqueue to TCL
  3889. * SRNG. There is no need to setup a MSDU extension descriptor.
  3890. */
  3891. nbuf = dp_tx_send_msdu_single_wrapper(vdev, nbuf, &msdu_info,
  3892. peer_id, end_nbuf);
  3893. return nbuf;
  3894. send_multiple:
  3895. nbuf = dp_tx_send_msdu_multiple(vdev, nbuf, &msdu_info);
  3896. if (qdf_unlikely(nbuf && msdu_info.frm_type == dp_tx_frm_raw))
  3897. dp_tx_raw_prepare_unset(vdev->pdev->soc, nbuf);
  3898. return nbuf;
  3899. }
  3900. qdf_nbuf_t dp_tx_send_vdev_id_check(struct cdp_soc_t *soc_hdl,
  3901. uint8_t vdev_id, qdf_nbuf_t nbuf)
  3902. {
  3903. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3904. struct dp_vdev *vdev = NULL;
  3905. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  3906. if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
  3907. return nbuf;
  3908. /*
  3909. * dp_vdev_get_ref_by_id does does a atomic operation avoid using
  3910. * this in per packet path.
  3911. *
  3912. * As in this path vdev memory is already protected with netdev
  3913. * tx lock
  3914. */
  3915. vdev = soc->vdev_id_map[vdev_id];
  3916. if (qdf_unlikely(!vdev))
  3917. return nbuf;
  3918. if (qdf_unlikely(dp_tx_per_pkt_vdev_id_check(nbuf, vdev)
  3919. == QDF_STATUS_E_FAILURE)) {
  3920. DP_STATS_INC(vdev,
  3921. tx_i[xmit_type].dropped.fail_per_pkt_vdev_id_check,
  3922. 1);
  3923. return nbuf;
  3924. }
  3925. return dp_tx_send(soc_hdl, vdev_id, nbuf);
  3926. }
  3927. #ifdef UMAC_SUPPORT_PROXY_ARP
  3928. /**
  3929. * dp_tx_proxy_arp() - Tx proxy arp handler
  3930. * @vdev: datapath vdev handle
  3931. * @nbuf: sk buffer
  3932. *
  3933. * Return: status
  3934. */
  3935. int dp_tx_proxy_arp(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3936. {
  3937. if (vdev->osif_proxy_arp)
  3938. return vdev->osif_proxy_arp(vdev->osif_vdev, nbuf);
  3939. /*
  3940. * when UMAC_SUPPORT_PROXY_ARP is defined, we expect
  3941. * osif_proxy_arp has a valid function pointer assigned
  3942. * to it
  3943. */
  3944. dp_tx_err("valid function pointer for osif_proxy_arp is expected!!\n");
  3945. return QDF_STATUS_NOT_INITIALIZED;
  3946. }
  3947. #else
  3948. int dp_tx_proxy_arp(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3949. {
  3950. return QDF_STATUS_SUCCESS;
  3951. }
  3952. #endif
  3953. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  3954. !defined(CONFIG_MLO_SINGLE_DEV)
  3955. #ifdef WLAN_MCAST_MLO
  3956. static bool
  3957. dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
  3958. struct dp_tx_desc_s *tx_desc,
  3959. qdf_nbuf_t nbuf,
  3960. uint8_t reinject_reason)
  3961. {
  3962. if (reinject_reason == HTT_TX_FW2WBM_REINJECT_REASON_MLO_MCAST) {
  3963. if (soc->arch_ops.dp_tx_mcast_handler)
  3964. soc->arch_ops.dp_tx_mcast_handler(soc, vdev, nbuf);
  3965. dp_tx_desc_release(soc, tx_desc, tx_desc->pool_id);
  3966. return true;
  3967. }
  3968. return false;
  3969. }
  3970. #else /* WLAN_MCAST_MLO */
  3971. static inline bool
  3972. dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
  3973. struct dp_tx_desc_s *tx_desc,
  3974. qdf_nbuf_t nbuf,
  3975. uint8_t reinject_reason)
  3976. {
  3977. return false;
  3978. }
  3979. #endif /* WLAN_MCAST_MLO */
  3980. #else
  3981. static inline bool
  3982. dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
  3983. struct dp_tx_desc_s *tx_desc,
  3984. qdf_nbuf_t nbuf,
  3985. uint8_t reinject_reason)
  3986. {
  3987. return false;
  3988. }
  3989. #endif
  3990. void dp_tx_reinject_handler(struct dp_soc *soc,
  3991. struct dp_vdev *vdev,
  3992. struct dp_tx_desc_s *tx_desc,
  3993. uint8_t *status,
  3994. uint8_t reinject_reason)
  3995. {
  3996. struct dp_peer *peer = NULL;
  3997. uint32_t peer_id = HTT_INVALID_PEER;
  3998. qdf_nbuf_t nbuf = tx_desc->nbuf;
  3999. qdf_nbuf_t nbuf_copy = NULL;
  4000. struct dp_tx_msdu_info_s msdu_info;
  4001. #ifdef WDS_VENDOR_EXTENSION
  4002. int is_mcast = 0, is_ucast = 0;
  4003. int num_peers_3addr = 0;
  4004. qdf_ether_header_t *eth_hdr = (qdf_ether_header_t *)(qdf_nbuf_data(nbuf));
  4005. struct ieee80211_frame_addr4 *wh = (struct ieee80211_frame_addr4 *)(qdf_nbuf_data(nbuf));
  4006. #endif
  4007. struct dp_txrx_peer *txrx_peer;
  4008. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(nbuf);
  4009. qdf_assert(vdev);
  4010. dp_tx_debug("Tx reinject path");
  4011. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].reinject_pkts, 1,
  4012. qdf_nbuf_len(tx_desc->nbuf));
  4013. if (dp_tx_reinject_mlo_hdl(soc, vdev, tx_desc, nbuf, reinject_reason))
  4014. return;
  4015. #ifdef WDS_VENDOR_EXTENSION
  4016. if (qdf_unlikely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
  4017. is_mcast = (IS_MULTICAST(wh->i_addr1)) ? 1 : 0;
  4018. } else {
  4019. is_mcast = (IS_MULTICAST(eth_hdr->ether_dhost)) ? 1 : 0;
  4020. }
  4021. is_ucast = !is_mcast;
  4022. qdf_spin_lock_bh(&vdev->peer_list_lock);
  4023. TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
  4024. txrx_peer = dp_get_txrx_peer(peer);
  4025. if (!txrx_peer || txrx_peer->bss_peer)
  4026. continue;
  4027. /* Detect wds peers that use 3-addr framing for mcast.
  4028. * if there are any, the bss_peer is used to send the
  4029. * the mcast frame using 3-addr format. all wds enabled
  4030. * peers that use 4-addr framing for mcast frames will
  4031. * be duplicated and sent as 4-addr frames below.
  4032. */
  4033. if (!txrx_peer->wds_enabled ||
  4034. !txrx_peer->wds_ecm.wds_tx_mcast_4addr) {
  4035. num_peers_3addr = 1;
  4036. break;
  4037. }
  4038. }
  4039. qdf_spin_unlock_bh(&vdev->peer_list_lock);
  4040. #endif
  4041. if (qdf_unlikely(vdev->mesh_vdev)) {
  4042. DP_TX_FREE_SINGLE_BUF(vdev->pdev->soc, tx_desc->nbuf);
  4043. } else {
  4044. qdf_spin_lock_bh(&vdev->peer_list_lock);
  4045. TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
  4046. txrx_peer = dp_get_txrx_peer(peer);
  4047. if (!txrx_peer)
  4048. continue;
  4049. if ((txrx_peer->peer_id != HTT_INVALID_PEER) &&
  4050. #ifdef WDS_VENDOR_EXTENSION
  4051. /*
  4052. * . if 3-addr STA, then send on BSS Peer
  4053. * . if Peer WDS enabled and accept 4-addr mcast,
  4054. * send mcast on that peer only
  4055. * . if Peer WDS enabled and accept 4-addr ucast,
  4056. * send ucast on that peer only
  4057. */
  4058. ((txrx_peer->bss_peer && num_peers_3addr && is_mcast) ||
  4059. (txrx_peer->wds_enabled &&
  4060. ((is_mcast && txrx_peer->wds_ecm.wds_tx_mcast_4addr) ||
  4061. (is_ucast &&
  4062. txrx_peer->wds_ecm.wds_tx_ucast_4addr))))) {
  4063. #else
  4064. (txrx_peer->bss_peer &&
  4065. (dp_tx_proxy_arp(vdev, nbuf) == QDF_STATUS_SUCCESS))) {
  4066. #endif
  4067. peer_id = DP_INVALID_PEER;
  4068. nbuf_copy = qdf_nbuf_copy(nbuf);
  4069. if (!nbuf_copy) {
  4070. dp_tx_debug("nbuf copy failed");
  4071. break;
  4072. }
  4073. qdf_mem_zero(&msdu_info, sizeof(msdu_info));
  4074. dp_tx_get_queue(vdev, nbuf,
  4075. &msdu_info.tx_queue);
  4076. msdu_info.xmit_type =
  4077. qdf_nbuf_get_vdev_xmit_type(nbuf);
  4078. nbuf_copy = dp_tx_send_msdu_single(vdev,
  4079. nbuf_copy,
  4080. &msdu_info,
  4081. peer_id,
  4082. NULL);
  4083. if (nbuf_copy) {
  4084. dp_tx_debug("pkt send failed");
  4085. qdf_nbuf_free(nbuf_copy);
  4086. }
  4087. }
  4088. }
  4089. qdf_spin_unlock_bh(&vdev->peer_list_lock);
  4090. qdf_nbuf_unmap_nbytes_single(vdev->osdev, nbuf,
  4091. QDF_DMA_TO_DEVICE, nbuf->len);
  4092. qdf_nbuf_free(nbuf);
  4093. }
  4094. dp_tx_desc_release(soc, tx_desc, tx_desc->pool_id);
  4095. }
  4096. void dp_tx_inspect_handler(struct dp_soc *soc,
  4097. struct dp_vdev *vdev,
  4098. struct dp_tx_desc_s *tx_desc,
  4099. uint8_t *status)
  4100. {
  4101. uint8_t xmit_type = qdf_nbuf_get_vdev_xmit_type(tx_desc->nbuf);
  4102. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO,
  4103. "%s Tx inspect path",
  4104. __func__);
  4105. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].inspect_pkts, 1,
  4106. qdf_nbuf_len(tx_desc->nbuf));
  4107. DP_TX_FREE_SINGLE_BUF(soc, tx_desc->nbuf);
  4108. dp_tx_desc_release(soc, tx_desc, tx_desc->pool_id);
  4109. }
  4110. #ifdef MESH_MODE_SUPPORT
  4111. /**
  4112. * dp_tx_comp_fill_tx_completion_stats() - Fill per packet Tx completion stats
  4113. * in mesh meta header
  4114. * @tx_desc: software descriptor head pointer
  4115. * @ts: pointer to tx completion stats
  4116. * Return: none
  4117. */
  4118. static
  4119. void dp_tx_comp_fill_tx_completion_stats(struct dp_tx_desc_s *tx_desc,
  4120. struct hal_tx_completion_status *ts)
  4121. {
  4122. qdf_nbuf_t netbuf = tx_desc->nbuf;
  4123. if (!tx_desc->msdu_ext_desc) {
  4124. if (qdf_nbuf_pull_head(netbuf, tx_desc->pkt_offset) == NULL) {
  4125. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  4126. "netbuf %pK offset %d",
  4127. netbuf, tx_desc->pkt_offset);
  4128. return;
  4129. }
  4130. }
  4131. }
  4132. #else
  4133. static
  4134. void dp_tx_comp_fill_tx_completion_stats(struct dp_tx_desc_s *tx_desc,
  4135. struct hal_tx_completion_status *ts)
  4136. {
  4137. }
  4138. #endif
  4139. #ifdef CONFIG_SAWF
  4140. static void dp_tx_update_peer_sawf_stats(struct dp_soc *soc,
  4141. struct dp_vdev *vdev,
  4142. struct dp_txrx_peer *txrx_peer,
  4143. struct dp_tx_desc_s *tx_desc,
  4144. struct hal_tx_completion_status *ts,
  4145. uint8_t tid)
  4146. {
  4147. dp_sawf_tx_compl_update_peer_stats(soc, vdev, txrx_peer, tx_desc,
  4148. ts, tid);
  4149. }
  4150. static void dp_tx_compute_delay_avg(struct cdp_delay_tx_stats *tx_delay,
  4151. uint32_t nw_delay,
  4152. uint32_t sw_delay,
  4153. uint32_t hw_delay)
  4154. {
  4155. dp_peer_tid_delay_avg(tx_delay,
  4156. nw_delay,
  4157. sw_delay,
  4158. hw_delay);
  4159. }
  4160. #else
  4161. static void dp_tx_update_peer_sawf_stats(struct dp_soc *soc,
  4162. struct dp_vdev *vdev,
  4163. struct dp_txrx_peer *txrx_peer,
  4164. struct dp_tx_desc_s *tx_desc,
  4165. struct hal_tx_completion_status *ts,
  4166. uint8_t tid)
  4167. {
  4168. }
  4169. static inline void
  4170. dp_tx_compute_delay_avg(struct cdp_delay_tx_stats *tx_delay,
  4171. uint32_t nw_delay, uint32_t sw_delay,
  4172. uint32_t hw_delay)
  4173. {
  4174. }
  4175. #endif
  4176. #ifdef QCA_PEER_EXT_STATS
  4177. #ifdef WLAN_CONFIG_TX_DELAY
  4178. static void dp_tx_compute_tid_delay(struct cdp_delay_tid_stats *stats,
  4179. struct dp_tx_desc_s *tx_desc,
  4180. struct hal_tx_completion_status *ts,
  4181. struct dp_vdev *vdev)
  4182. {
  4183. struct dp_soc *soc = vdev->pdev->soc;
  4184. struct cdp_delay_tx_stats *tx_delay = &stats->tx_delay;
  4185. int64_t timestamp_ingress, timestamp_hw_enqueue;
  4186. uint32_t sw_enqueue_delay, fwhw_transmit_delay = 0;
  4187. if (!ts->valid)
  4188. return;
  4189. timestamp_ingress = qdf_nbuf_get_timestamp_us(tx_desc->nbuf);
  4190. timestamp_hw_enqueue = qdf_ktime_to_us(tx_desc->timestamp);
  4191. sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
  4192. dp_hist_update_stats(&tx_delay->tx_swq_delay, sw_enqueue_delay);
  4193. if (soc->arch_ops.dp_tx_compute_hw_delay)
  4194. if (!soc->arch_ops.dp_tx_compute_hw_delay(soc, vdev, ts,
  4195. &fwhw_transmit_delay))
  4196. dp_hist_update_stats(&tx_delay->hwtx_delay,
  4197. fwhw_transmit_delay);
  4198. dp_tx_compute_delay_avg(tx_delay, 0, sw_enqueue_delay,
  4199. fwhw_transmit_delay);
  4200. }
  4201. #else
  4202. /**
  4203. * dp_tx_compute_tid_delay() - Compute per TID delay
  4204. * @stats: Per TID delay stats
  4205. * @tx_desc: Software Tx descriptor
  4206. * @ts: Tx completion status
  4207. * @vdev: vdev
  4208. *
  4209. * Compute the software enqueue and hw enqueue delays and
  4210. * update the respective histograms
  4211. *
  4212. * Return: void
  4213. */
  4214. static void dp_tx_compute_tid_delay(struct cdp_delay_tid_stats *stats,
  4215. struct dp_tx_desc_s *tx_desc,
  4216. struct hal_tx_completion_status *ts,
  4217. struct dp_vdev *vdev)
  4218. {
  4219. struct cdp_delay_tx_stats *tx_delay = &stats->tx_delay;
  4220. int64_t current_timestamp, timestamp_ingress, timestamp_hw_enqueue;
  4221. uint32_t sw_enqueue_delay, fwhw_transmit_delay;
  4222. current_timestamp = qdf_ktime_to_ms(qdf_ktime_real_get());
  4223. timestamp_ingress = qdf_nbuf_get_timestamp(tx_desc->nbuf);
  4224. timestamp_hw_enqueue = qdf_ktime_to_ms(tx_desc->timestamp);
  4225. sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
  4226. fwhw_transmit_delay = (uint32_t)(current_timestamp -
  4227. timestamp_hw_enqueue);
  4228. /*
  4229. * Update the Tx software enqueue delay and HW enque-Completion delay.
  4230. */
  4231. dp_hist_update_stats(&tx_delay->tx_swq_delay, sw_enqueue_delay);
  4232. dp_hist_update_stats(&tx_delay->hwtx_delay, fwhw_transmit_delay);
  4233. }
  4234. #endif
  4235. /**
  4236. * dp_tx_update_peer_delay_stats() - Update the peer delay stats
  4237. * @txrx_peer: DP peer context
  4238. * @tx_desc: Tx software descriptor
  4239. * @ts: Tx completion status
  4240. * @ring_id: Rx CPU context ID/CPU_ID
  4241. *
  4242. * Update the peer extended stats. These are enhanced other
  4243. * delay stats per msdu level.
  4244. *
  4245. * Return: void
  4246. */
  4247. static void dp_tx_update_peer_delay_stats(struct dp_txrx_peer *txrx_peer,
  4248. struct dp_tx_desc_s *tx_desc,
  4249. struct hal_tx_completion_status *ts,
  4250. uint8_t ring_id)
  4251. {
  4252. struct dp_pdev *pdev = txrx_peer->vdev->pdev;
  4253. struct dp_soc *soc = NULL;
  4254. struct dp_peer_delay_stats *delay_stats = NULL;
  4255. uint8_t tid;
  4256. soc = pdev->soc;
  4257. if (qdf_likely(!wlan_cfg_is_peer_ext_stats_enabled(soc->wlan_cfg_ctx)))
  4258. return;
  4259. if (!txrx_peer->delay_stats)
  4260. return;
  4261. tid = ts->tid;
  4262. delay_stats = txrx_peer->delay_stats;
  4263. /*
  4264. * For non-TID packets use the TID 9
  4265. */
  4266. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  4267. tid = CDP_MAX_DATA_TIDS - 1;
  4268. dp_tx_compute_tid_delay(&delay_stats->delay_tid_stats[tid][ring_id],
  4269. tx_desc, ts, txrx_peer->vdev);
  4270. }
  4271. #else
  4272. static inline
  4273. void dp_tx_update_peer_delay_stats(struct dp_txrx_peer *txrx_peer,
  4274. struct dp_tx_desc_s *tx_desc,
  4275. struct hal_tx_completion_status *ts,
  4276. uint8_t ring_id)
  4277. {
  4278. }
  4279. #endif
  4280. #ifdef WLAN_PEER_JITTER
  4281. /**
  4282. * dp_tx_jitter_get_avg_jitter() - compute the average jitter
  4283. * @curr_delay: Current delay
  4284. * @prev_delay: Previous delay
  4285. * @avg_jitter: Average Jitter
  4286. * Return: Newly Computed Average Jitter
  4287. */
  4288. static uint32_t dp_tx_jitter_get_avg_jitter(uint32_t curr_delay,
  4289. uint32_t prev_delay,
  4290. uint32_t avg_jitter)
  4291. {
  4292. uint32_t curr_jitter;
  4293. int32_t jitter_diff;
  4294. curr_jitter = qdf_abs(curr_delay - prev_delay);
  4295. if (!avg_jitter)
  4296. return curr_jitter;
  4297. jitter_diff = curr_jitter - avg_jitter;
  4298. if (jitter_diff < 0)
  4299. avg_jitter = avg_jitter -
  4300. (qdf_abs(jitter_diff) >> DP_AVG_JITTER_WEIGHT_DENOM);
  4301. else
  4302. avg_jitter = avg_jitter +
  4303. (qdf_abs(jitter_diff) >> DP_AVG_JITTER_WEIGHT_DENOM);
  4304. return avg_jitter;
  4305. }
  4306. /**
  4307. * dp_tx_jitter_get_avg_delay() - compute the average delay
  4308. * @curr_delay: Current delay
  4309. * @avg_delay: Average delay
  4310. * Return: Newly Computed Average Delay
  4311. */
  4312. static uint32_t dp_tx_jitter_get_avg_delay(uint32_t curr_delay,
  4313. uint32_t avg_delay)
  4314. {
  4315. int32_t delay_diff;
  4316. if (!avg_delay)
  4317. return curr_delay;
  4318. delay_diff = curr_delay - avg_delay;
  4319. if (delay_diff < 0)
  4320. avg_delay = avg_delay - (qdf_abs(delay_diff) >>
  4321. DP_AVG_DELAY_WEIGHT_DENOM);
  4322. else
  4323. avg_delay = avg_delay + (qdf_abs(delay_diff) >>
  4324. DP_AVG_DELAY_WEIGHT_DENOM);
  4325. return avg_delay;
  4326. }
  4327. #ifdef WLAN_CONFIG_TX_DELAY
  4328. /**
  4329. * dp_tx_compute_cur_delay() - get the current delay
  4330. * @soc: soc handle
  4331. * @vdev: vdev structure for data path state
  4332. * @ts: Tx completion status
  4333. * @curr_delay: current delay
  4334. * @tx_desc: tx descriptor
  4335. * Return: void
  4336. */
  4337. static
  4338. QDF_STATUS dp_tx_compute_cur_delay(struct dp_soc *soc,
  4339. struct dp_vdev *vdev,
  4340. struct hal_tx_completion_status *ts,
  4341. uint32_t *curr_delay,
  4342. struct dp_tx_desc_s *tx_desc)
  4343. {
  4344. QDF_STATUS status = QDF_STATUS_E_FAILURE;
  4345. if (soc->arch_ops.dp_tx_compute_hw_delay)
  4346. status = soc->arch_ops.dp_tx_compute_hw_delay(soc, vdev, ts,
  4347. curr_delay);
  4348. return status;
  4349. }
  4350. #else
  4351. static
  4352. QDF_STATUS dp_tx_compute_cur_delay(struct dp_soc *soc,
  4353. struct dp_vdev *vdev,
  4354. struct hal_tx_completion_status *ts,
  4355. uint32_t *curr_delay,
  4356. struct dp_tx_desc_s *tx_desc)
  4357. {
  4358. int64_t current_timestamp, timestamp_hw_enqueue;
  4359. current_timestamp = qdf_ktime_to_us(qdf_ktime_real_get());
  4360. timestamp_hw_enqueue = qdf_ktime_to_us(tx_desc->timestamp);
  4361. *curr_delay = (uint32_t)(current_timestamp - timestamp_hw_enqueue);
  4362. return QDF_STATUS_SUCCESS;
  4363. }
  4364. #endif
  4365. /**
  4366. * dp_tx_compute_tid_jitter() - compute per tid per ring jitter
  4367. * @jitter: per tid per ring jitter stats
  4368. * @ts: Tx completion status
  4369. * @vdev: vdev structure for data path state
  4370. * @tx_desc: tx descriptor
  4371. * Return: void
  4372. */
  4373. static void dp_tx_compute_tid_jitter(struct cdp_peer_tid_stats *jitter,
  4374. struct hal_tx_completion_status *ts,
  4375. struct dp_vdev *vdev,
  4376. struct dp_tx_desc_s *tx_desc)
  4377. {
  4378. uint32_t curr_delay, avg_delay, avg_jitter, prev_delay;
  4379. struct dp_soc *soc = vdev->pdev->soc;
  4380. QDF_STATUS status = QDF_STATUS_E_FAILURE;
  4381. if (ts->status != HAL_TX_TQM_RR_FRAME_ACKED) {
  4382. jitter->tx_drop += 1;
  4383. return;
  4384. }
  4385. status = dp_tx_compute_cur_delay(soc, vdev, ts, &curr_delay,
  4386. tx_desc);
  4387. if (QDF_IS_STATUS_SUCCESS(status)) {
  4388. avg_delay = jitter->tx_avg_delay;
  4389. avg_jitter = jitter->tx_avg_jitter;
  4390. prev_delay = jitter->tx_prev_delay;
  4391. avg_jitter = dp_tx_jitter_get_avg_jitter(curr_delay,
  4392. prev_delay,
  4393. avg_jitter);
  4394. avg_delay = dp_tx_jitter_get_avg_delay(curr_delay, avg_delay);
  4395. jitter->tx_avg_delay = avg_delay;
  4396. jitter->tx_avg_jitter = avg_jitter;
  4397. jitter->tx_prev_delay = curr_delay;
  4398. jitter->tx_total_success += 1;
  4399. } else if (status == QDF_STATUS_E_FAILURE) {
  4400. jitter->tx_avg_err += 1;
  4401. }
  4402. }
  4403. /* dp_tx_update_peer_jitter_stats() - Update the peer jitter stats
  4404. * @txrx_peer: DP peer context
  4405. * @tx_desc: Tx software descriptor
  4406. * @ts: Tx completion status
  4407. * @ring_id: Rx CPU context ID/CPU_ID
  4408. * Return: void
  4409. */
  4410. static void dp_tx_update_peer_jitter_stats(struct dp_txrx_peer *txrx_peer,
  4411. struct dp_tx_desc_s *tx_desc,
  4412. struct hal_tx_completion_status *ts,
  4413. uint8_t ring_id)
  4414. {
  4415. struct dp_pdev *pdev = txrx_peer->vdev->pdev;
  4416. struct dp_soc *soc = pdev->soc;
  4417. struct cdp_peer_tid_stats *jitter_stats = NULL;
  4418. uint8_t tid;
  4419. struct cdp_peer_tid_stats *rx_tid = NULL;
  4420. if (qdf_likely(!wlan_cfg_is_peer_jitter_stats_enabled(soc->wlan_cfg_ctx)))
  4421. return;
  4422. if (!txrx_peer->jitter_stats)
  4423. return;
  4424. tid = ts->tid;
  4425. jitter_stats = txrx_peer->jitter_stats;
  4426. /*
  4427. * For non-TID packets use the TID 9
  4428. */
  4429. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  4430. tid = CDP_MAX_DATA_TIDS - 1;
  4431. rx_tid = &jitter_stats[tid * CDP_MAX_TXRX_CTX + ring_id];
  4432. dp_tx_compute_tid_jitter(rx_tid,
  4433. ts, txrx_peer->vdev, tx_desc);
  4434. }
  4435. #else
  4436. static void dp_tx_update_peer_jitter_stats(struct dp_txrx_peer *txrx_peer,
  4437. struct dp_tx_desc_s *tx_desc,
  4438. struct hal_tx_completion_status *ts,
  4439. uint8_t ring_id)
  4440. {
  4441. }
  4442. #endif
  4443. #ifdef HW_TX_DELAY_STATS_ENABLE
  4444. /**
  4445. * dp_update_tx_delay_stats() - update the delay stats
  4446. * @vdev: vdev handle
  4447. * @delay: delay in ms or us based on the flag delay_in_us
  4448. * @tid: tid value
  4449. * @mode: type of tx delay mode
  4450. * @ring_id: ring number
  4451. * @delay_in_us: flag to indicate whether the delay is in ms or us
  4452. *
  4453. * Return: none
  4454. */
  4455. static inline
  4456. void dp_update_tx_delay_stats(struct dp_vdev *vdev, uint32_t delay, uint8_t tid,
  4457. uint8_t mode, uint8_t ring_id, bool delay_in_us)
  4458. {
  4459. struct cdp_tid_tx_stats *tstats =
  4460. &vdev->stats.tid_tx_stats[ring_id][tid];
  4461. dp_update_delay_stats(tstats, NULL, delay, tid, mode, ring_id,
  4462. delay_in_us);
  4463. }
  4464. #else
  4465. static inline
  4466. void dp_update_tx_delay_stats(struct dp_vdev *vdev, uint32_t delay, uint8_t tid,
  4467. uint8_t mode, uint8_t ring_id, bool delay_in_us)
  4468. {
  4469. struct cdp_tid_tx_stats *tstats =
  4470. &vdev->pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
  4471. dp_update_delay_stats(tstats, NULL, delay, tid, mode, ring_id,
  4472. delay_in_us);
  4473. }
  4474. #endif
  4475. void dp_tx_compute_delay(struct dp_vdev *vdev, struct dp_tx_desc_s *tx_desc,
  4476. uint8_t tid, uint8_t ring_id)
  4477. {
  4478. int64_t current_timestamp, timestamp_ingress, timestamp_hw_enqueue;
  4479. uint32_t sw_enqueue_delay, fwhw_transmit_delay, interframe_delay;
  4480. uint32_t fwhw_transmit_delay_us;
  4481. if (qdf_likely(!vdev->pdev->delay_stats_flag) &&
  4482. qdf_likely(!dp_is_vdev_tx_delay_stats_enabled(vdev)))
  4483. return;
  4484. if (dp_is_vdev_tx_delay_stats_enabled(vdev)) {
  4485. fwhw_transmit_delay_us =
  4486. qdf_ktime_to_us(qdf_ktime_real_get()) -
  4487. qdf_ktime_to_us(tx_desc->timestamp);
  4488. /*
  4489. * Delay between packet enqueued to HW and Tx completion in us
  4490. */
  4491. dp_update_tx_delay_stats(vdev, fwhw_transmit_delay_us, tid,
  4492. CDP_DELAY_STATS_FW_HW_TRANSMIT,
  4493. ring_id, true);
  4494. /*
  4495. * For MCL, only enqueue to completion delay is required
  4496. * so return if the vdev flag is enabled.
  4497. */
  4498. return;
  4499. }
  4500. current_timestamp = qdf_ktime_to_ms(qdf_ktime_real_get());
  4501. timestamp_hw_enqueue = qdf_ktime_to_ms(tx_desc->timestamp);
  4502. fwhw_transmit_delay = (uint32_t)(current_timestamp -
  4503. timestamp_hw_enqueue);
  4504. if (!timestamp_hw_enqueue)
  4505. return;
  4506. /*
  4507. * Delay between packet enqueued to HW and Tx completion in ms
  4508. */
  4509. dp_update_tx_delay_stats(vdev, fwhw_transmit_delay, tid,
  4510. CDP_DELAY_STATS_FW_HW_TRANSMIT, ring_id,
  4511. false);
  4512. timestamp_ingress = qdf_nbuf_get_timestamp(tx_desc->nbuf);
  4513. sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
  4514. interframe_delay = (uint32_t)(timestamp_ingress -
  4515. vdev->prev_tx_enq_tstamp);
  4516. /*
  4517. * Delay in software enqueue
  4518. */
  4519. dp_update_tx_delay_stats(vdev, sw_enqueue_delay, tid,
  4520. CDP_DELAY_STATS_SW_ENQ, ring_id,
  4521. false);
  4522. /*
  4523. * Update interframe delay stats calculated at hardstart receive point.
  4524. * Value of vdev->prev_tx_enq_tstamp will be 0 for 1st frame, so
  4525. * interframe delay will not be calculate correctly for 1st frame.
  4526. * On the other side, this will help in avoiding extra per packet check
  4527. * of !vdev->prev_tx_enq_tstamp.
  4528. */
  4529. dp_update_tx_delay_stats(vdev, interframe_delay, tid,
  4530. CDP_DELAY_STATS_TX_INTERFRAME, ring_id,
  4531. false);
  4532. vdev->prev_tx_enq_tstamp = timestamp_ingress;
  4533. }
  4534. #ifdef DISABLE_DP_STATS
  4535. static
  4536. inline void dp_update_no_ack_stats(qdf_nbuf_t nbuf,
  4537. struct dp_txrx_peer *txrx_peer,
  4538. uint8_t link_id)
  4539. {
  4540. }
  4541. #else
  4542. static inline void
  4543. dp_update_no_ack_stats(qdf_nbuf_t nbuf, struct dp_txrx_peer *txrx_peer,
  4544. uint8_t link_id)
  4545. {
  4546. enum qdf_proto_subtype subtype = QDF_PROTO_INVALID;
  4547. DPTRACE(qdf_dp_track_noack_check(nbuf, &subtype));
  4548. if (subtype != QDF_PROTO_INVALID)
  4549. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.no_ack_count[subtype],
  4550. 1, link_id);
  4551. }
  4552. #endif
  4553. #ifndef QCA_ENHANCED_STATS_SUPPORT
  4554. #ifdef DP_PEER_EXTENDED_API
  4555. static inline uint8_t
  4556. dp_tx_get_mpdu_retry_threshold(struct dp_txrx_peer *txrx_peer)
  4557. {
  4558. return txrx_peer->mpdu_retry_threshold;
  4559. }
  4560. #else
  4561. static inline uint8_t
  4562. dp_tx_get_mpdu_retry_threshold(struct dp_txrx_peer *txrx_peer)
  4563. {
  4564. return 0;
  4565. }
  4566. #endif
  4567. /**
  4568. * dp_tx_update_peer_extd_stats()- Update Tx extended path stats for peer
  4569. *
  4570. * @ts: Tx compltion status
  4571. * @txrx_peer: datapath txrx_peer handle
  4572. * @link_id: Link id
  4573. *
  4574. * Return: void
  4575. */
  4576. static inline void
  4577. dp_tx_update_peer_extd_stats(struct hal_tx_completion_status *ts,
  4578. struct dp_txrx_peer *txrx_peer, uint8_t link_id)
  4579. {
  4580. uint8_t mcs, pkt_type, dst_mcs_idx;
  4581. uint8_t retry_threshold = dp_tx_get_mpdu_retry_threshold(txrx_peer);
  4582. mcs = ts->mcs;
  4583. pkt_type = ts->pkt_type;
  4584. /* do HW to SW pkt type conversion */
  4585. pkt_type = (pkt_type >= HAL_DOT11_MAX ? DOT11_MAX :
  4586. hal_2_dp_pkt_type_map[pkt_type]);
  4587. dst_mcs_idx = dp_get_mcs_array_index_by_pkt_type_mcs(pkt_type, mcs);
  4588. if (MCS_INVALID_ARRAY_INDEX != dst_mcs_idx)
  4589. DP_PEER_EXTD_STATS_INC(txrx_peer,
  4590. tx.pkt_type[pkt_type].mcs_count[dst_mcs_idx],
  4591. 1, link_id);
  4592. DP_PEER_EXTD_STATS_INC(txrx_peer, tx.sgi_count[ts->sgi], 1, link_id);
  4593. DP_PEER_EXTD_STATS_INC(txrx_peer, tx.bw[ts->bw], 1, link_id);
  4594. DP_PEER_EXTD_STATS_UPD(txrx_peer, tx.last_ack_rssi, ts->ack_frame_rssi,
  4595. link_id);
  4596. DP_PEER_EXTD_STATS_INC(txrx_peer,
  4597. tx.wme_ac_type[TID_TO_WME_AC(ts->tid)], 1,
  4598. link_id);
  4599. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.stbc, 1, ts->stbc, link_id);
  4600. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.ldpc, 1, ts->ldpc, link_id);
  4601. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.retries, 1, ts->transmit_cnt > 1,
  4602. link_id);
  4603. if (ts->first_msdu) {
  4604. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.retries_mpdu, 1,
  4605. ts->transmit_cnt > 1, link_id);
  4606. if (!retry_threshold)
  4607. return;
  4608. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.mpdu_success_with_retries,
  4609. qdf_do_div(ts->transmit_cnt,
  4610. retry_threshold),
  4611. ts->transmit_cnt > retry_threshold,
  4612. link_id);
  4613. }
  4614. }
  4615. #else
  4616. static inline void
  4617. dp_tx_update_peer_extd_stats(struct hal_tx_completion_status *ts,
  4618. struct dp_txrx_peer *txrx_peer, uint8_t link_id)
  4619. {
  4620. }
  4621. #endif
  4622. #if defined(WLAN_FEATURE_11BE_MLO) && \
  4623. (defined(QCA_ENHANCED_STATS_SUPPORT) || \
  4624. defined(DP_MLO_LINK_STATS_SUPPORT))
  4625. static inline uint8_t
  4626. dp_tx_get_link_id_from_ppdu_id(struct dp_soc *soc,
  4627. struct hal_tx_completion_status *ts,
  4628. struct dp_txrx_peer *txrx_peer,
  4629. struct dp_vdev *vdev)
  4630. {
  4631. uint8_t hw_link_id = 0;
  4632. uint32_t ppdu_id;
  4633. uint8_t link_id_offset, link_id_bits;
  4634. if (!txrx_peer->is_mld_peer || !vdev->pdev->link_peer_stats)
  4635. return 0;
  4636. link_id_offset = soc->link_id_offset;
  4637. link_id_bits = soc->link_id_bits;
  4638. ppdu_id = ts->ppdu_id;
  4639. hw_link_id = ((DP_GET_HW_LINK_ID_FRM_PPDU_ID(ppdu_id, link_id_offset,
  4640. link_id_bits)) + 1);
  4641. if (hw_link_id > DP_MAX_MLO_LINKS) {
  4642. hw_link_id = 0;
  4643. DP_PEER_PER_PKT_STATS_INC(
  4644. txrx_peer,
  4645. tx.inval_link_id_pkt_cnt, 1, hw_link_id);
  4646. }
  4647. return hw_link_id;
  4648. }
  4649. #else
  4650. static inline uint8_t
  4651. dp_tx_get_link_id_from_ppdu_id(struct dp_soc *soc,
  4652. struct hal_tx_completion_status *ts,
  4653. struct dp_txrx_peer *txrx_peer,
  4654. struct dp_vdev *vdev)
  4655. {
  4656. return 0;
  4657. }
  4658. #endif
  4659. /**
  4660. * dp_tx_update_peer_stats() - Update peer stats from Tx completion indications
  4661. * per wbm ring
  4662. *
  4663. * @tx_desc: software descriptor head pointer
  4664. * @ts: Tx completion status
  4665. * @txrx_peer: peer handle
  4666. * @ring_id: ring number
  4667. * @link_id: Link id
  4668. *
  4669. * Return: None
  4670. */
  4671. static inline void
  4672. dp_tx_update_peer_stats(struct dp_tx_desc_s *tx_desc,
  4673. struct hal_tx_completion_status *ts,
  4674. struct dp_txrx_peer *txrx_peer, uint8_t ring_id,
  4675. uint8_t link_id)
  4676. {
  4677. struct dp_pdev *pdev = txrx_peer->vdev->pdev;
  4678. uint8_t tid = ts->tid;
  4679. uint32_t length;
  4680. struct cdp_tid_tx_stats *tid_stats;
  4681. if (!pdev)
  4682. return;
  4683. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  4684. tid = CDP_MAX_DATA_TIDS - 1;
  4685. tid_stats = &pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
  4686. if (ts->release_src != HAL_TX_COMP_RELEASE_SOURCE_TQM) {
  4687. dp_err_rl("Release source:%d is not from TQM", ts->release_src);
  4688. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.release_src_not_tqm, 1,
  4689. link_id);
  4690. return;
  4691. }
  4692. length = qdf_nbuf_len(tx_desc->nbuf);
  4693. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  4694. if (qdf_unlikely(pdev->delay_stats_flag) ||
  4695. qdf_unlikely(dp_is_vdev_tx_delay_stats_enabled(txrx_peer->vdev)))
  4696. dp_tx_compute_delay(txrx_peer->vdev, tx_desc, tid, ring_id);
  4697. if (ts->status < CDP_MAX_TX_TQM_STATUS) {
  4698. tid_stats->tqm_status_cnt[ts->status]++;
  4699. }
  4700. if (qdf_likely(ts->status == HAL_TX_TQM_RR_FRAME_ACKED)) {
  4701. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.retry_count, 1,
  4702. ts->transmit_cnt > 1, link_id);
  4703. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.multiple_retry_count,
  4704. 1, ts->transmit_cnt > 2, link_id);
  4705. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.ofdma, 1, ts->ofdma,
  4706. link_id);
  4707. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.amsdu_cnt, 1,
  4708. ts->msdu_part_of_amsdu, link_id);
  4709. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.non_amsdu_cnt, 1,
  4710. !ts->msdu_part_of_amsdu, link_id);
  4711. txrx_peer->stats[link_id].per_pkt_stats.tx.last_tx_ts =
  4712. qdf_system_ticks();
  4713. dp_tx_update_peer_extd_stats(ts, txrx_peer, link_id);
  4714. return;
  4715. }
  4716. /*
  4717. * tx_failed is ideally supposed to be updated from HTT ppdu
  4718. * completion stats. But in IPQ807X/IPQ6018 chipsets owing to
  4719. * hw limitation there are no completions for failed cases.
  4720. * Hence updating tx_failed from data path. Please note that
  4721. * if tx_failed is fixed to be from ppdu, then this has to be
  4722. * removed
  4723. */
  4724. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  4725. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.failed_retry_count, 1,
  4726. ts->transmit_cnt > DP_RETRY_COUNT,
  4727. link_id);
  4728. dp_update_no_ack_stats(tx_desc->nbuf, txrx_peer, link_id);
  4729. if (ts->status == HAL_TX_TQM_RR_REM_CMD_AGED) {
  4730. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.age_out, 1,
  4731. link_id);
  4732. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_REM) {
  4733. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.dropped.fw_rem, 1,
  4734. length, link_id);
  4735. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_NOTX) {
  4736. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_rem_notx, 1,
  4737. link_id);
  4738. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_TX) {
  4739. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_rem_tx, 1,
  4740. link_id);
  4741. } else if (ts->status == HAL_TX_TQM_RR_FW_REASON1) {
  4742. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason1, 1,
  4743. link_id);
  4744. } else if (ts->status == HAL_TX_TQM_RR_FW_REASON2) {
  4745. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason2, 1,
  4746. link_id);
  4747. } else if (ts->status == HAL_TX_TQM_RR_FW_REASON3) {
  4748. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason3, 1,
  4749. link_id);
  4750. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_DISABLE_QUEUE) {
  4751. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4752. tx.dropped.fw_rem_queue_disable, 1,
  4753. link_id);
  4754. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_TILL_NONMATCHING) {
  4755. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4756. tx.dropped.fw_rem_no_match, 1,
  4757. link_id);
  4758. } else if (ts->status == HAL_TX_TQM_RR_DROP_THRESHOLD) {
  4759. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4760. tx.dropped.drop_threshold, 1,
  4761. link_id);
  4762. } else if (ts->status == HAL_TX_TQM_RR_LINK_DESC_UNAVAILABLE) {
  4763. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4764. tx.dropped.drop_link_desc_na, 1,
  4765. link_id);
  4766. } else if (ts->status == HAL_TX_TQM_RR_DROP_OR_INVALID_MSDU) {
  4767. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4768. tx.dropped.invalid_drop, 1,
  4769. link_id);
  4770. } else if (ts->status == HAL_TX_TQM_RR_MULTICAST_DROP) {
  4771. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4772. tx.dropped.mcast_vdev_drop, 1,
  4773. link_id);
  4774. } else {
  4775. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.invalid_rr, 1,
  4776. link_id);
  4777. }
  4778. }
  4779. #ifdef QCA_LL_TX_FLOW_CONTROL_V2
  4780. /**
  4781. * dp_tx_flow_pool_lock() - take flow pool lock
  4782. * @soc: core txrx main context
  4783. * @tx_desc: tx desc
  4784. *
  4785. * Return: None
  4786. */
  4787. static inline
  4788. void dp_tx_flow_pool_lock(struct dp_soc *soc,
  4789. struct dp_tx_desc_s *tx_desc)
  4790. {
  4791. struct dp_tx_desc_pool_s *pool;
  4792. uint8_t desc_pool_id;
  4793. desc_pool_id = tx_desc->pool_id;
  4794. pool = &soc->tx_desc[desc_pool_id];
  4795. qdf_spin_lock_bh(&pool->flow_pool_lock);
  4796. }
  4797. /**
  4798. * dp_tx_flow_pool_unlock() - release flow pool lock
  4799. * @soc: core txrx main context
  4800. * @tx_desc: tx desc
  4801. *
  4802. * Return: None
  4803. */
  4804. static inline
  4805. void dp_tx_flow_pool_unlock(struct dp_soc *soc,
  4806. struct dp_tx_desc_s *tx_desc)
  4807. {
  4808. struct dp_tx_desc_pool_s *pool;
  4809. uint8_t desc_pool_id;
  4810. desc_pool_id = tx_desc->pool_id;
  4811. pool = &soc->tx_desc[desc_pool_id];
  4812. qdf_spin_unlock_bh(&pool->flow_pool_lock);
  4813. }
  4814. #else
  4815. static inline
  4816. void dp_tx_flow_pool_lock(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
  4817. {
  4818. }
  4819. static inline
  4820. void dp_tx_flow_pool_unlock(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
  4821. {
  4822. }
  4823. #endif
  4824. /**
  4825. * dp_tx_notify_completion() - Notify tx completion for this desc
  4826. * @soc: core txrx main context
  4827. * @vdev: datapath vdev handle
  4828. * @tx_desc: tx desc
  4829. * @netbuf: buffer
  4830. * @status: tx status
  4831. *
  4832. * Return: none
  4833. */
  4834. static inline void dp_tx_notify_completion(struct dp_soc *soc,
  4835. struct dp_vdev *vdev,
  4836. struct dp_tx_desc_s *tx_desc,
  4837. qdf_nbuf_t netbuf,
  4838. uint8_t status)
  4839. {
  4840. void *osif_dev;
  4841. ol_txrx_completion_fp tx_compl_cbk = NULL;
  4842. uint16_t flag = BIT(QDF_TX_RX_STATUS_DOWNLOAD_SUCC);
  4843. qdf_assert(tx_desc);
  4844. if (!vdev ||
  4845. !vdev->osif_vdev) {
  4846. return;
  4847. }
  4848. osif_dev = vdev->osif_vdev;
  4849. tx_compl_cbk = vdev->tx_comp;
  4850. if (status == HAL_TX_TQM_RR_FRAME_ACKED)
  4851. flag |= BIT(QDF_TX_RX_STATUS_OK);
  4852. if (tx_compl_cbk)
  4853. tx_compl_cbk(netbuf, osif_dev, flag);
  4854. }
  4855. /**
  4856. * dp_tx_sojourn_stats_process() - Collect sojourn stats
  4857. * @pdev: pdev handle
  4858. * @txrx_peer: DP peer context
  4859. * @tid: tid value
  4860. * @txdesc_ts: timestamp from txdesc
  4861. * @ppdu_id: ppdu id
  4862. * @link_id: link id
  4863. *
  4864. * Return: none
  4865. */
  4866. #ifdef FEATURE_PERPKT_INFO
  4867. static inline void dp_tx_sojourn_stats_process(struct dp_pdev *pdev,
  4868. struct dp_txrx_peer *txrx_peer,
  4869. uint8_t tid,
  4870. uint64_t txdesc_ts,
  4871. uint32_t ppdu_id,
  4872. uint8_t link_id)
  4873. {
  4874. uint64_t delta_ms;
  4875. struct cdp_tx_sojourn_stats *sojourn_stats;
  4876. struct dp_peer *primary_link_peer = NULL;
  4877. struct dp_soc *link_peer_soc = NULL;
  4878. if (qdf_unlikely(!pdev->enhanced_stats_en))
  4879. return;
  4880. if (qdf_unlikely(tid == HTT_INVALID_TID ||
  4881. tid >= CDP_DATA_TID_MAX))
  4882. return;
  4883. if (qdf_unlikely(!pdev->sojourn_buf))
  4884. return;
  4885. primary_link_peer = dp_get_primary_link_peer_by_id(pdev->soc,
  4886. txrx_peer->peer_id,
  4887. DP_MOD_ID_TX_COMP);
  4888. if (qdf_unlikely(!primary_link_peer))
  4889. return;
  4890. sojourn_stats = (struct cdp_tx_sojourn_stats *)
  4891. qdf_nbuf_data(pdev->sojourn_buf);
  4892. link_peer_soc = primary_link_peer->vdev->pdev->soc;
  4893. sojourn_stats->cookie = (void *)
  4894. dp_monitor_peer_get_peerstats_ctx(link_peer_soc,
  4895. primary_link_peer);
  4896. delta_ms = qdf_ktime_to_ms(qdf_ktime_real_get()) -
  4897. txdesc_ts;
  4898. qdf_ewma_tx_lag_add(&txrx_peer->stats[link_id].per_pkt_stats.tx.avg_sojourn_msdu[tid],
  4899. delta_ms);
  4900. sojourn_stats->sum_sojourn_msdu[tid] = delta_ms;
  4901. sojourn_stats->num_msdus[tid] = 1;
  4902. sojourn_stats->avg_sojourn_msdu[tid].internal =
  4903. txrx_peer->stats[link_id].
  4904. per_pkt_stats.tx.avg_sojourn_msdu[tid].internal;
  4905. dp_wdi_event_handler(WDI_EVENT_TX_SOJOURN_STAT, pdev->soc,
  4906. pdev->sojourn_buf, HTT_INVALID_PEER,
  4907. WDI_NO_VAL, pdev->pdev_id);
  4908. sojourn_stats->sum_sojourn_msdu[tid] = 0;
  4909. sojourn_stats->num_msdus[tid] = 0;
  4910. sojourn_stats->avg_sojourn_msdu[tid].internal = 0;
  4911. dp_peer_unref_delete(primary_link_peer, DP_MOD_ID_TX_COMP);
  4912. }
  4913. #else
  4914. static inline void dp_tx_sojourn_stats_process(struct dp_pdev *pdev,
  4915. struct dp_txrx_peer *txrx_peer,
  4916. uint8_t tid,
  4917. uint64_t txdesc_ts,
  4918. uint32_t ppdu_id)
  4919. {
  4920. }
  4921. #endif
  4922. #ifdef WLAN_FEATURE_PKT_CAPTURE_V2
  4923. void dp_send_completion_to_pkt_capture(struct dp_soc *soc,
  4924. struct dp_tx_desc_s *desc,
  4925. struct hal_tx_completion_status *ts)
  4926. {
  4927. dp_wdi_event_handler(WDI_EVENT_PKT_CAPTURE_TX_DATA, soc,
  4928. desc, ts->peer_id,
  4929. WDI_NO_VAL, desc->pdev->pdev_id);
  4930. }
  4931. #endif
  4932. void
  4933. dp_tx_comp_process_desc(struct dp_soc *soc,
  4934. struct dp_tx_desc_s *desc,
  4935. struct hal_tx_completion_status *ts,
  4936. struct dp_txrx_peer *txrx_peer)
  4937. {
  4938. uint64_t time_latency = 0;
  4939. uint16_t peer_id = DP_INVALID_PEER_ID;
  4940. /*
  4941. * m_copy/tx_capture modes are not supported for
  4942. * scatter gather packets
  4943. */
  4944. if (qdf_unlikely(!!desc->pdev->latency_capture_enable)) {
  4945. time_latency = (qdf_ktime_to_ms(qdf_ktime_real_get()) -
  4946. qdf_ktime_to_ms(desc->timestamp));
  4947. }
  4948. dp_send_completion_to_pkt_capture(soc, desc, ts);
  4949. if (dp_tx_pkt_tracepoints_enabled())
  4950. qdf_trace_dp_packet(desc->nbuf, QDF_TX,
  4951. desc->msdu_ext_desc ?
  4952. desc->msdu_ext_desc->tso_desc : NULL,
  4953. qdf_ktime_to_us(desc->timestamp));
  4954. if (!(desc->msdu_ext_desc)) {
  4955. dp_tx_enh_unmap(soc, desc);
  4956. if (txrx_peer)
  4957. peer_id = txrx_peer->peer_id;
  4958. if (QDF_STATUS_SUCCESS ==
  4959. dp_monitor_tx_add_to_comp_queue(soc, desc, ts, peer_id)) {
  4960. return;
  4961. }
  4962. if (QDF_STATUS_SUCCESS ==
  4963. dp_get_completion_indication_for_stack(soc,
  4964. desc->pdev,
  4965. txrx_peer, ts,
  4966. desc->nbuf,
  4967. time_latency)) {
  4968. dp_send_completion_to_stack(soc,
  4969. desc->pdev,
  4970. ts->peer_id,
  4971. ts->ppdu_id,
  4972. desc->nbuf);
  4973. return;
  4974. }
  4975. }
  4976. desc->flags |= DP_TX_DESC_FLAG_COMPLETED_TX;
  4977. dp_tx_comp_free_buf(soc, desc, false);
  4978. }
  4979. #ifdef DISABLE_DP_STATS
  4980. /**
  4981. * dp_tx_update_connectivity_stats() - update tx connectivity stats
  4982. * @soc: core txrx main context
  4983. * @vdev: virtual device instance
  4984. * @tx_desc: tx desc
  4985. * @status: tx status
  4986. *
  4987. * Return: none
  4988. */
  4989. static inline
  4990. void dp_tx_update_connectivity_stats(struct dp_soc *soc,
  4991. struct dp_vdev *vdev,
  4992. struct dp_tx_desc_s *tx_desc,
  4993. uint8_t status)
  4994. {
  4995. }
  4996. #else
  4997. static inline
  4998. void dp_tx_update_connectivity_stats(struct dp_soc *soc,
  4999. struct dp_vdev *vdev,
  5000. struct dp_tx_desc_s *tx_desc,
  5001. uint8_t status)
  5002. {
  5003. void *osif_dev;
  5004. ol_txrx_stats_rx_fp stats_cbk;
  5005. uint8_t pkt_type;
  5006. qdf_assert(tx_desc);
  5007. if (!vdev || vdev->delete.pending || !vdev->osif_vdev ||
  5008. !vdev->stats_cb)
  5009. return;
  5010. osif_dev = vdev->osif_vdev;
  5011. stats_cbk = vdev->stats_cb;
  5012. stats_cbk(tx_desc->nbuf, osif_dev, PKT_TYPE_TX_HOST_FW_SENT, &pkt_type);
  5013. if (status == HAL_TX_TQM_RR_FRAME_ACKED)
  5014. stats_cbk(tx_desc->nbuf, osif_dev, PKT_TYPE_TX_ACK_CNT,
  5015. &pkt_type);
  5016. }
  5017. #endif
  5018. #if defined(WLAN_FEATURE_TSF_AUTO_REPORT) || defined(WLAN_CONFIG_TX_DELAY)
  5019. /* Mask for bit29 ~ bit31 */
  5020. #define DP_TX_TS_BIT29_31_MASK 0xE0000000
  5021. /* Timestamp value (unit us) if bit29 is set */
  5022. #define DP_TX_TS_BIT29_SET_VALUE BIT(29)
  5023. /**
  5024. * dp_tx_adjust_enqueue_buffer_ts() - adjust the enqueue buffer_timestamp
  5025. * @ack_ts: OTA ack timestamp, unit us.
  5026. * @enqueue_ts: TCL enqueue TX data to TQM timestamp, unit us.
  5027. * @base_delta_ts: base timestamp delta for ack_ts and enqueue_ts
  5028. *
  5029. * this function will restore the bit29 ~ bit31 3 bits value for
  5030. * buffer_timestamp in wbm2sw ring entry, currently buffer_timestamp only
  5031. * can support 0x7FFF * 1024 us (29 bits), but if the timestamp is >
  5032. * 0x7FFF * 1024 us, bit29~ bit31 will be lost.
  5033. *
  5034. * Return: the adjusted buffer_timestamp value
  5035. */
  5036. static inline
  5037. uint32_t dp_tx_adjust_enqueue_buffer_ts(uint32_t ack_ts,
  5038. uint32_t enqueue_ts,
  5039. uint32_t base_delta_ts)
  5040. {
  5041. uint32_t ack_buffer_ts;
  5042. uint32_t ack_buffer_ts_bit29_31;
  5043. uint32_t adjusted_enqueue_ts;
  5044. /* corresponding buffer_timestamp value when receive OTA Ack */
  5045. ack_buffer_ts = ack_ts - base_delta_ts;
  5046. ack_buffer_ts_bit29_31 = ack_buffer_ts & DP_TX_TS_BIT29_31_MASK;
  5047. /* restore the bit29 ~ bit31 value */
  5048. adjusted_enqueue_ts = ack_buffer_ts_bit29_31 | enqueue_ts;
  5049. /*
  5050. * if actual enqueue_ts value occupied 29 bits only, this enqueue_ts
  5051. * value + real UL delay overflow 29 bits, then 30th bit (bit-29)
  5052. * should not be marked, otherwise extra 0x20000000 us is added to
  5053. * enqueue_ts.
  5054. */
  5055. if (qdf_unlikely(adjusted_enqueue_ts > ack_buffer_ts))
  5056. adjusted_enqueue_ts -= DP_TX_TS_BIT29_SET_VALUE;
  5057. return adjusted_enqueue_ts;
  5058. }
  5059. QDF_STATUS
  5060. dp_tx_compute_hw_delay_us(struct hal_tx_completion_status *ts,
  5061. uint32_t delta_tsf,
  5062. uint32_t *delay_us)
  5063. {
  5064. uint32_t buffer_ts;
  5065. uint32_t delay;
  5066. if (!delay_us)
  5067. return QDF_STATUS_E_INVAL;
  5068. /* Tx_rate_stats_info_valid is 0 and tsf is invalid then */
  5069. if (!ts->valid)
  5070. return QDF_STATUS_E_INVAL;
  5071. /* buffer_timestamp is in units of 1024 us and is [31:13] of
  5072. * WBM_RELEASE_RING_4. After left shift 10 bits, it's
  5073. * valid up to 29 bits.
  5074. */
  5075. buffer_ts = ts->buffer_timestamp << 10;
  5076. buffer_ts = dp_tx_adjust_enqueue_buffer_ts(ts->tsf,
  5077. buffer_ts, delta_tsf);
  5078. delay = ts->tsf - buffer_ts - delta_tsf;
  5079. if (qdf_unlikely(delay & 0x80000000)) {
  5080. dp_err_rl("delay = 0x%x (-ve)\n"
  5081. "release_src = %d\n"
  5082. "ppdu_id = 0x%x\n"
  5083. "peer_id = 0x%x\n"
  5084. "tid = 0x%x\n"
  5085. "release_reason = %d\n"
  5086. "tsf = %u (0x%x)\n"
  5087. "buffer_timestamp = %u (0x%x)\n"
  5088. "delta_tsf = %u (0x%x)\n",
  5089. delay, ts->release_src, ts->ppdu_id, ts->peer_id,
  5090. ts->tid, ts->status, ts->tsf, ts->tsf,
  5091. ts->buffer_timestamp, ts->buffer_timestamp,
  5092. delta_tsf, delta_tsf);
  5093. delay = 0;
  5094. goto end;
  5095. }
  5096. delay &= 0x1FFFFFFF; /* mask 29 BITS */
  5097. if (delay > 0x1000000) {
  5098. dp_info_rl("----------------------\n"
  5099. "Tx completion status:\n"
  5100. "----------------------\n"
  5101. "release_src = %d\n"
  5102. "ppdu_id = 0x%x\n"
  5103. "release_reason = %d\n"
  5104. "tsf = %u (0x%x)\n"
  5105. "buffer_timestamp = %u (0x%x)\n"
  5106. "delta_tsf = %u (0x%x)\n",
  5107. ts->release_src, ts->ppdu_id, ts->status,
  5108. ts->tsf, ts->tsf, ts->buffer_timestamp,
  5109. ts->buffer_timestamp, delta_tsf, delta_tsf);
  5110. return QDF_STATUS_E_FAILURE;
  5111. }
  5112. end:
  5113. *delay_us = delay;
  5114. return QDF_STATUS_SUCCESS;
  5115. }
  5116. void dp_set_delta_tsf(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  5117. uint32_t delta_tsf)
  5118. {
  5119. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  5120. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  5121. DP_MOD_ID_CDP);
  5122. if (!vdev) {
  5123. dp_err_rl("vdev %d does not exist", vdev_id);
  5124. return;
  5125. }
  5126. vdev->delta_tsf = delta_tsf;
  5127. dp_debug("vdev id %u delta_tsf %u", vdev_id, delta_tsf);
  5128. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  5129. }
  5130. #endif
  5131. #ifdef WLAN_FEATURE_TSF_UPLINK_DELAY
  5132. QDF_STATUS dp_set_tsf_ul_delay_report(struct cdp_soc_t *soc_hdl,
  5133. uint8_t vdev_id, bool enable)
  5134. {
  5135. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  5136. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  5137. DP_MOD_ID_CDP);
  5138. if (!vdev) {
  5139. dp_err_rl("vdev %d does not exist", vdev_id);
  5140. return QDF_STATUS_E_FAILURE;
  5141. }
  5142. qdf_atomic_set(&vdev->ul_delay_report, enable);
  5143. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  5144. return QDF_STATUS_SUCCESS;
  5145. }
  5146. QDF_STATUS dp_get_uplink_delay(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  5147. uint32_t *val)
  5148. {
  5149. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  5150. struct dp_vdev *vdev;
  5151. uint32_t delay_accum;
  5152. uint32_t pkts_accum;
  5153. vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_CDP);
  5154. if (!vdev) {
  5155. dp_err_rl("vdev %d does not exist", vdev_id);
  5156. return QDF_STATUS_E_FAILURE;
  5157. }
  5158. if (!qdf_atomic_read(&vdev->ul_delay_report)) {
  5159. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  5160. return QDF_STATUS_E_FAILURE;
  5161. }
  5162. /* Average uplink delay based on current accumulated values */
  5163. delay_accum = qdf_atomic_read(&vdev->ul_delay_accum);
  5164. pkts_accum = qdf_atomic_read(&vdev->ul_pkts_accum);
  5165. *val = delay_accum / pkts_accum;
  5166. dp_debug("uplink_delay %u delay_accum %u pkts_accum %u", *val,
  5167. delay_accum, pkts_accum);
  5168. /* Reset accumulated values to 0 */
  5169. qdf_atomic_set(&vdev->ul_delay_accum, 0);
  5170. qdf_atomic_set(&vdev->ul_pkts_accum, 0);
  5171. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  5172. return QDF_STATUS_SUCCESS;
  5173. }
  5174. static void dp_tx_update_uplink_delay(struct dp_soc *soc, struct dp_vdev *vdev,
  5175. struct hal_tx_completion_status *ts)
  5176. {
  5177. uint32_t ul_delay;
  5178. if (qdf_unlikely(!vdev)) {
  5179. dp_info_rl("vdev is null or delete in progress");
  5180. return;
  5181. }
  5182. if (!qdf_atomic_read(&vdev->ul_delay_report))
  5183. return;
  5184. if (QDF_IS_STATUS_ERROR(dp_tx_compute_hw_delay_us(ts,
  5185. vdev->delta_tsf,
  5186. &ul_delay)))
  5187. return;
  5188. ul_delay /= 1000; /* in unit of ms */
  5189. qdf_atomic_add(ul_delay, &vdev->ul_delay_accum);
  5190. qdf_atomic_inc(&vdev->ul_pkts_accum);
  5191. }
  5192. #else /* !WLAN_FEATURE_TSF_UPLINK_DELAY */
  5193. static inline
  5194. void dp_tx_update_uplink_delay(struct dp_soc *soc, struct dp_vdev *vdev,
  5195. struct hal_tx_completion_status *ts)
  5196. {
  5197. }
  5198. #endif /* WLAN_FEATURE_TSF_UPLINK_DELAY */
  5199. #ifndef CONFIG_AP_PLATFORM
  5200. /**
  5201. * dp_update_mcast_stats() - Update Tx Mcast stats
  5202. * @txrx_peer: txrx_peer pointer
  5203. * @link_id: Link ID
  5204. * @length: packet length
  5205. * @nbuf: nbuf handle
  5206. *
  5207. * Return: None
  5208. */
  5209. static inline void
  5210. dp_update_mcast_stats(struct dp_txrx_peer *txrx_peer, uint8_t link_id,
  5211. uint32_t length, qdf_nbuf_t nbuf)
  5212. {
  5213. if (QDF_NBUF_CB_GET_IS_MCAST(nbuf))
  5214. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.mcast, 1,
  5215. length, link_id);
  5216. }
  5217. #else
  5218. static inline void
  5219. dp_update_mcast_stats(struct dp_txrx_peer *txrx_peer, uint8_t link_id,
  5220. uint32_t length, qdf_nbuf_t nbuf)
  5221. {
  5222. }
  5223. #endif
  5224. #if defined(WLAN_FEATURE_11BE_MLO) && defined(DP_MLO_LINK_STATS_SUPPORT)
  5225. /**
  5226. * dp_tx_comp_set_nbuf_band() - set nbuf band.
  5227. * @soc: dp soc handle
  5228. * @nbuf: nbuf handle
  5229. * @ts: tx completion status
  5230. *
  5231. * Return: None
  5232. */
  5233. static inline void
  5234. dp_tx_comp_set_nbuf_band(struct dp_soc *soc, qdf_nbuf_t nbuf,
  5235. struct hal_tx_completion_status *ts)
  5236. {
  5237. struct qdf_mac_addr *mac_addr;
  5238. struct dp_peer *peer;
  5239. struct dp_txrx_peer *txrx_peer;
  5240. uint8_t link_id;
  5241. if ((QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) !=
  5242. QDF_NBUF_CB_PACKET_TYPE_EAPOL &&
  5243. QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) !=
  5244. QDF_NBUF_CB_PACKET_TYPE_DHCP &&
  5245. QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) !=
  5246. QDF_NBUF_CB_PACKET_TYPE_DHCPV6) ||
  5247. QDF_NBUF_CB_GET_IS_BCAST(nbuf))
  5248. return;
  5249. mac_addr = (struct qdf_mac_addr *)(qdf_nbuf_data(nbuf) +
  5250. QDF_NBUF_DEST_MAC_OFFSET);
  5251. peer = dp_mld_peer_find_hash_find(soc, mac_addr->bytes, 0,
  5252. DP_VDEV_ALL, DP_MOD_ID_TX_COMP);
  5253. if (qdf_likely(peer)) {
  5254. txrx_peer = dp_get_txrx_peer(peer);
  5255. if (qdf_likely(txrx_peer)) {
  5256. link_id =
  5257. dp_tx_get_link_id_from_ppdu_id(soc, ts,
  5258. txrx_peer,
  5259. txrx_peer->vdev);
  5260. qdf_nbuf_tx_set_band(nbuf, txrx_peer->ll_band[link_id]);
  5261. }
  5262. dp_peer_unref_delete(peer, DP_MOD_ID_TX_COMP);
  5263. }
  5264. }
  5265. #else
  5266. static inline void
  5267. dp_tx_comp_set_nbuf_band(struct dp_soc *soc, qdf_nbuf_t nbuf,
  5268. struct hal_tx_completion_status *ts)
  5269. {
  5270. }
  5271. #endif
  5272. void dp_tx_comp_process_tx_status(struct dp_soc *soc,
  5273. struct dp_tx_desc_s *tx_desc,
  5274. struct hal_tx_completion_status *ts,
  5275. struct dp_txrx_peer *txrx_peer,
  5276. uint8_t ring_id)
  5277. {
  5278. uint32_t length;
  5279. qdf_ether_header_t *eh;
  5280. struct dp_vdev *vdev = NULL;
  5281. qdf_nbuf_t nbuf = tx_desc->nbuf;
  5282. enum qdf_dp_tx_rx_status dp_status;
  5283. uint8_t link_id = 0;
  5284. enum QDF_OPMODE op_mode = QDF_MAX_NO_OF_MODE;
  5285. if (!nbuf) {
  5286. dp_info_rl("invalid tx descriptor. nbuf NULL");
  5287. goto out;
  5288. }
  5289. eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
  5290. length = dp_tx_get_pkt_len(tx_desc);
  5291. dp_status = dp_tx_hw_to_qdf(ts->status);
  5292. if (soc->dp_debug_log_en) {
  5293. dp_tx_comp_debug("--------------------\n"
  5294. "Tx Completion Stats:\n"
  5295. "--------------------\n"
  5296. "ack_frame_rssi = %d\n"
  5297. "first_msdu = %d\n"
  5298. "last_msdu = %d\n"
  5299. "msdu_part_of_amsdu = %d\n"
  5300. "rate_stats valid = %d\n"
  5301. "bw = %d\n"
  5302. "pkt_type = %d\n"
  5303. "stbc = %d\n"
  5304. "ldpc = %d\n"
  5305. "sgi = %d\n"
  5306. "mcs = %d\n"
  5307. "ofdma = %d\n"
  5308. "tones_in_ru = %d\n"
  5309. "tsf = %d\n"
  5310. "ppdu_id = %d\n"
  5311. "transmit_cnt = %d\n"
  5312. "tid = %d\n"
  5313. "peer_id = %d\n"
  5314. "tx_status = %d\n"
  5315. "tx_release_source = %d\n",
  5316. ts->ack_frame_rssi, ts->first_msdu,
  5317. ts->last_msdu, ts->msdu_part_of_amsdu,
  5318. ts->valid, ts->bw, ts->pkt_type, ts->stbc,
  5319. ts->ldpc, ts->sgi, ts->mcs, ts->ofdma,
  5320. ts->tones_in_ru, ts->tsf, ts->ppdu_id,
  5321. ts->transmit_cnt, ts->tid, ts->peer_id,
  5322. ts->status, ts->release_src);
  5323. }
  5324. /* Update SoC level stats */
  5325. DP_STATS_INCC(soc, tx.dropped_fw_removed, 1,
  5326. (ts->status == HAL_TX_TQM_RR_REM_CMD_REM));
  5327. if (!txrx_peer) {
  5328. dp_tx_comp_set_nbuf_band(soc, nbuf, ts);
  5329. dp_info_rl("peer is null or deletion in progress");
  5330. DP_STATS_INC_PKT(soc, tx.tx_invalid_peer, 1, length);
  5331. vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id,
  5332. DP_MOD_ID_CDP);
  5333. if (qdf_likely(vdev)) {
  5334. op_mode = vdev->qdf_opmode;
  5335. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  5336. }
  5337. goto out_log;
  5338. }
  5339. vdev = txrx_peer->vdev;
  5340. link_id = dp_tx_get_link_id_from_ppdu_id(soc, ts, txrx_peer, vdev);
  5341. dp_tx_set_nbuf_band(nbuf, txrx_peer, link_id);
  5342. op_mode = vdev->qdf_opmode;
  5343. dp_tx_update_connectivity_stats(soc, vdev, tx_desc, ts->status);
  5344. dp_tx_update_uplink_delay(soc, vdev, ts);
  5345. /* check tx complete notification */
  5346. if (qdf_nbuf_tx_notify_comp_get(nbuf))
  5347. dp_tx_notify_completion(soc, vdev, tx_desc,
  5348. nbuf, ts->status);
  5349. /* Update per-packet stats for mesh mode */
  5350. if (qdf_unlikely(vdev->mesh_vdev) &&
  5351. !(tx_desc->flags & DP_TX_DESC_FLAG_TO_FW))
  5352. dp_tx_comp_fill_tx_completion_stats(tx_desc, ts);
  5353. /* Update peer level stats */
  5354. if (qdf_unlikely(txrx_peer->bss_peer &&
  5355. vdev->opmode == wlan_op_mode_ap)) {
  5356. if (ts->status != HAL_TX_TQM_RR_REM_CMD_REM) {
  5357. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.mcast, 1,
  5358. length, link_id);
  5359. if (txrx_peer->vdev->tx_encap_type ==
  5360. htt_cmn_pkt_type_ethernet &&
  5361. QDF_IS_ADDR_BROADCAST(eh->ether_dhost)) {
  5362. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
  5363. tx.bcast, 1,
  5364. length, link_id);
  5365. }
  5366. }
  5367. } else {
  5368. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.ucast, 1, length,
  5369. link_id);
  5370. if (ts->status == HAL_TX_TQM_RR_FRAME_ACKED) {
  5371. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.tx_success,
  5372. 1, length, link_id);
  5373. if (qdf_unlikely(txrx_peer->in_twt)) {
  5374. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
  5375. tx.tx_success_twt,
  5376. 1, length,
  5377. link_id);
  5378. }
  5379. dp_update_mcast_stats(txrx_peer, link_id, length, nbuf);
  5380. }
  5381. }
  5382. dp_tx_update_peer_stats(tx_desc, ts, txrx_peer, ring_id, link_id);
  5383. dp_tx_update_peer_delay_stats(txrx_peer, tx_desc, ts, ring_id);
  5384. dp_tx_update_peer_jitter_stats(txrx_peer, tx_desc, ts, ring_id);
  5385. dp_tx_update_peer_sawf_stats(soc, vdev, txrx_peer, tx_desc,
  5386. ts, ts->tid);
  5387. dp_tx_send_pktlog(soc, vdev->pdev, tx_desc, nbuf, dp_status);
  5388. dp_tx_latency_stats_update(soc, txrx_peer, tx_desc, ts, link_id);
  5389. #ifdef QCA_SUPPORT_RDK_STATS
  5390. if (soc->peerstats_enabled)
  5391. dp_tx_sojourn_stats_process(vdev->pdev, txrx_peer, ts->tid,
  5392. qdf_ktime_to_ms(tx_desc->timestamp),
  5393. ts->ppdu_id, link_id);
  5394. #endif
  5395. out_log:
  5396. DPTRACE(qdf_dp_trace_ptr(tx_desc->nbuf,
  5397. QDF_DP_TRACE_LI_DP_FREE_PACKET_PTR_RECORD,
  5398. QDF_TRACE_DEFAULT_PDEV_ID,
  5399. qdf_nbuf_data_addr(nbuf),
  5400. sizeof(qdf_nbuf_data(nbuf)),
  5401. tx_desc->id, ts->status, dp_status, op_mode));
  5402. out:
  5403. return;
  5404. }
  5405. #if defined(QCA_VDEV_STATS_HW_OFFLOAD_SUPPORT) && \
  5406. defined(QCA_ENHANCED_STATS_SUPPORT)
  5407. void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
  5408. uint32_t length, uint8_t tx_status,
  5409. bool update)
  5410. {
  5411. if (update || (!txrx_peer->hw_txrx_stats_en)) {
  5412. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  5413. if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
  5414. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  5415. }
  5416. }
  5417. #elif defined(QCA_VDEV_STATS_HW_OFFLOAD_SUPPORT)
  5418. void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
  5419. uint32_t length, uint8_t tx_status,
  5420. bool update)
  5421. {
  5422. if (!txrx_peer->hw_txrx_stats_en) {
  5423. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  5424. if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
  5425. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  5426. }
  5427. }
  5428. #else
  5429. void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
  5430. uint32_t length, uint8_t tx_status,
  5431. bool update)
  5432. {
  5433. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  5434. if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
  5435. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  5436. }
  5437. #endif
  5438. /**
  5439. * dp_tx_prefetch_next_nbuf_data(): Prefetch nbuf and nbuf data
  5440. * @next: descriptor of the nrxt buffer
  5441. *
  5442. * Return: none
  5443. */
  5444. #ifdef QCA_DP_RX_NBUF_AND_NBUF_DATA_PREFETCH
  5445. static inline
  5446. void dp_tx_prefetch_next_nbuf_data(struct dp_tx_desc_s *next)
  5447. {
  5448. qdf_nbuf_t nbuf = NULL;
  5449. if (next)
  5450. nbuf = next->nbuf;
  5451. if (nbuf)
  5452. qdf_prefetch(nbuf);
  5453. }
  5454. #else
  5455. static inline
  5456. void dp_tx_prefetch_next_nbuf_data(struct dp_tx_desc_s *next)
  5457. {
  5458. }
  5459. #endif
  5460. /**
  5461. * dp_tx_mcast_reinject_handler() - Tx reinjected multicast packets handler
  5462. * @soc: core txrx main context
  5463. * @desc: software descriptor
  5464. *
  5465. * Return: true when packet is reinjected
  5466. */
  5467. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  5468. defined(WLAN_MCAST_MLO) && !defined(CONFIG_MLO_SINGLE_DEV)
  5469. static inline bool
  5470. dp_tx_mcast_reinject_handler(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  5471. {
  5472. struct dp_vdev *vdev = NULL;
  5473. uint8_t xmit_type;
  5474. if (desc->tx_status == HAL_TX_TQM_RR_MULTICAST_DROP) {
  5475. if (!soc->arch_ops.dp_tx_mcast_handler ||
  5476. !soc->arch_ops.dp_tx_is_mcast_primary)
  5477. return false;
  5478. vdev = dp_vdev_get_ref_by_id(soc, desc->vdev_id,
  5479. DP_MOD_ID_REINJECT);
  5480. if (qdf_unlikely(!vdev)) {
  5481. dp_tx_comp_info_rl("Unable to get vdev ref %d",
  5482. desc->id);
  5483. return false;
  5484. }
  5485. if (!(soc->arch_ops.dp_tx_is_mcast_primary(soc, vdev))) {
  5486. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_REINJECT);
  5487. return false;
  5488. }
  5489. xmit_type = qdf_nbuf_get_vdev_xmit_type(desc->nbuf);
  5490. DP_STATS_INC_PKT(vdev, tx_i[xmit_type].reinject_pkts, 1,
  5491. qdf_nbuf_len(desc->nbuf));
  5492. soc->arch_ops.dp_tx_mcast_handler(soc, vdev, desc->nbuf);
  5493. dp_tx_desc_release(soc, desc, desc->pool_id);
  5494. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_REINJECT);
  5495. return true;
  5496. }
  5497. return false;
  5498. }
  5499. #else
  5500. static inline bool
  5501. dp_tx_mcast_reinject_handler(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  5502. {
  5503. return false;
  5504. }
  5505. #endif
  5506. #ifdef QCA_DP_TX_NBUF_LIST_FREE
  5507. static inline void
  5508. dp_tx_nbuf_queue_head_init(qdf_nbuf_queue_head_t *nbuf_queue_head)
  5509. {
  5510. qdf_nbuf_queue_head_init(nbuf_queue_head);
  5511. }
  5512. static inline void
  5513. dp_tx_nbuf_dev_queue_free(qdf_nbuf_queue_head_t *nbuf_queue_head,
  5514. struct dp_tx_desc_s *desc)
  5515. {
  5516. qdf_nbuf_t nbuf = NULL;
  5517. nbuf = desc->nbuf;
  5518. if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_FAST))
  5519. qdf_nbuf_dev_queue_head(nbuf_queue_head, nbuf);
  5520. else
  5521. qdf_nbuf_free(nbuf);
  5522. }
  5523. static inline void
  5524. dp_tx_nbuf_dev_queue_free_no_flag(qdf_nbuf_queue_head_t *nbuf_queue_head,
  5525. qdf_nbuf_t nbuf)
  5526. {
  5527. if (!nbuf)
  5528. return;
  5529. if (nbuf->is_from_recycler)
  5530. qdf_nbuf_dev_queue_head(nbuf_queue_head, nbuf);
  5531. else
  5532. qdf_nbuf_free(nbuf);
  5533. }
  5534. static inline void
  5535. dp_tx_nbuf_dev_kfree_list(qdf_nbuf_queue_head_t *nbuf_queue_head)
  5536. {
  5537. qdf_nbuf_dev_kfree_list(nbuf_queue_head);
  5538. }
  5539. #else
  5540. static inline void
  5541. dp_tx_nbuf_queue_head_init(qdf_nbuf_queue_head_t *nbuf_queue_head)
  5542. {
  5543. }
  5544. static inline void
  5545. dp_tx_nbuf_dev_queue_free(qdf_nbuf_queue_head_t *nbuf_queue_head,
  5546. struct dp_tx_desc_s *desc)
  5547. {
  5548. qdf_nbuf_free(desc->nbuf);
  5549. }
  5550. static inline void
  5551. dp_tx_nbuf_dev_queue_free_no_flag(qdf_nbuf_queue_head_t *nbuf_queue_head,
  5552. qdf_nbuf_t nbuf)
  5553. {
  5554. qdf_nbuf_free(nbuf);
  5555. }
  5556. static inline void
  5557. dp_tx_nbuf_dev_kfree_list(qdf_nbuf_queue_head_t *nbuf_queue_head)
  5558. {
  5559. }
  5560. #endif
  5561. #ifdef WLAN_SUPPORT_PPEDS
  5562. static inline void
  5563. dp_tx_update_ppeds_tx_comp_stats(struct dp_soc *soc,
  5564. struct dp_txrx_peer *txrx_peer,
  5565. struct hal_tx_completion_status *ts,
  5566. struct dp_tx_desc_s *desc,
  5567. uint8_t ring_id)
  5568. {
  5569. uint8_t link_id = 0;
  5570. struct dp_vdev *vdev = NULL;
  5571. if (qdf_likely(txrx_peer)) {
  5572. if (!(desc->flags & DP_TX_DESC_FLAG_SIMPLE)) {
  5573. hal_tx_comp_get_status(&desc->comp,
  5574. ts,
  5575. soc->hal_soc);
  5576. vdev = txrx_peer->vdev;
  5577. link_id = dp_tx_get_link_id_from_ppdu_id(soc,
  5578. ts,
  5579. txrx_peer,
  5580. vdev);
  5581. if (link_id < 1 || link_id > DP_MAX_MLO_LINKS)
  5582. link_id = 0;
  5583. dp_tx_update_peer_stats(desc, ts,
  5584. txrx_peer,
  5585. ring_id,
  5586. link_id);
  5587. } else {
  5588. dp_tx_update_peer_basic_stats(txrx_peer, desc->length,
  5589. desc->tx_status, false);
  5590. }
  5591. }
  5592. }
  5593. #else
  5594. static inline void
  5595. dp_tx_update_ppeds_tx_comp_stats(struct dp_soc *soc,
  5596. struct dp_txrx_peer *txrx_peer,
  5597. struct hal_tx_completion_status *ts,
  5598. struct dp_tx_desc_s *desc,
  5599. uint8_t ring_id)
  5600. {
  5601. }
  5602. #endif
  5603. void
  5604. dp_tx_comp_process_desc_list_fast(struct dp_soc *soc,
  5605. struct dp_tx_desc_s *head_desc,
  5606. struct dp_tx_desc_s *tail_desc,
  5607. uint8_t ring_id,
  5608. uint32_t fast_desc_count)
  5609. {
  5610. struct dp_tx_desc_pool_s *pool = NULL;
  5611. pool = dp_get_tx_desc_pool(soc, head_desc->pool_id);
  5612. dp_tx_outstanding_sub(head_desc->pdev, fast_desc_count);
  5613. dp_tx_desc_free_list(pool, head_desc, tail_desc, fast_desc_count);
  5614. }
  5615. void
  5616. dp_tx_comp_process_desc_list(struct dp_soc *soc,
  5617. struct dp_tx_desc_s *comp_head, uint8_t ring_id)
  5618. {
  5619. struct dp_tx_desc_s *desc;
  5620. struct dp_tx_desc_s *next;
  5621. struct hal_tx_completion_status ts;
  5622. struct dp_txrx_peer *txrx_peer = NULL;
  5623. uint16_t peer_id = DP_INVALID_PEER;
  5624. dp_txrx_ref_handle txrx_ref_handle = NULL;
  5625. qdf_nbuf_queue_head_t h;
  5626. desc = comp_head;
  5627. dp_tx_nbuf_queue_head_init(&h);
  5628. while (desc) {
  5629. next = desc->next;
  5630. dp_tx_prefetch_next_nbuf_data(next);
  5631. if (peer_id != desc->peer_id) {
  5632. if (txrx_peer)
  5633. dp_txrx_peer_unref_delete(txrx_ref_handle,
  5634. DP_MOD_ID_TX_COMP);
  5635. peer_id = desc->peer_id;
  5636. txrx_peer =
  5637. dp_txrx_peer_get_ref_by_id(soc, peer_id,
  5638. &txrx_ref_handle,
  5639. DP_MOD_ID_TX_COMP);
  5640. }
  5641. if (dp_tx_mcast_reinject_handler(soc, desc)) {
  5642. desc = next;
  5643. continue;
  5644. }
  5645. if (desc->flags & DP_TX_DESC_FLAG_PPEDS) {
  5646. qdf_nbuf_t nbuf;
  5647. dp_tx_update_ppeds_tx_comp_stats(soc, txrx_peer, &ts,
  5648. desc, ring_id);
  5649. if (desc->pool_id != DP_TX_PPEDS_POOL_ID) {
  5650. nbuf = desc->nbuf;
  5651. dp_tx_nbuf_dev_queue_free_no_flag(&h, nbuf);
  5652. if (desc->flags & DP_TX_DESC_FLAG_SPECIAL)
  5653. dp_tx_spcl_desc_free(soc, desc,
  5654. desc->pool_id);
  5655. else
  5656. dp_tx_desc_free(soc, desc,
  5657. desc->pool_id);
  5658. __dp_tx_outstanding_dec(soc);
  5659. } else {
  5660. nbuf = dp_ppeds_tx_desc_free(soc, desc);
  5661. dp_tx_nbuf_dev_queue_free_no_flag(&h, nbuf);
  5662. }
  5663. desc = next;
  5664. continue;
  5665. }
  5666. if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_SIMPLE)) {
  5667. struct dp_pdev *pdev = desc->pdev;
  5668. if (qdf_likely(txrx_peer))
  5669. dp_tx_update_peer_basic_stats(txrx_peer,
  5670. desc->length,
  5671. desc->tx_status,
  5672. false);
  5673. qdf_assert(pdev);
  5674. dp_tx_outstanding_dec(pdev);
  5675. /*
  5676. * Calling a QDF WRAPPER here is creating significant
  5677. * performance impact so avoided the wrapper call here
  5678. */
  5679. dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf,
  5680. desc->id, DP_TX_COMP_UNMAP);
  5681. dp_tx_nbuf_unmap(soc, desc);
  5682. dp_tx_nbuf_dev_queue_free(&h, desc);
  5683. dp_tx_desc_free(soc, desc, desc->pool_id);
  5684. desc = next;
  5685. continue;
  5686. }
  5687. hal_tx_comp_get_status(&desc->comp, &ts, soc->hal_soc);
  5688. dp_tx_comp_process_tx_status(soc, desc, &ts, txrx_peer,
  5689. ring_id);
  5690. dp_tx_comp_process_desc(soc, desc, &ts, txrx_peer);
  5691. dp_tx_desc_release(soc, desc, desc->pool_id);
  5692. desc = next;
  5693. }
  5694. dp_tx_nbuf_dev_kfree_list(&h);
  5695. if (txrx_peer)
  5696. dp_txrx_peer_unref_delete(txrx_ref_handle, DP_MOD_ID_TX_COMP);
  5697. }
  5698. #ifndef WLAN_SOFTUMAC_SUPPORT
  5699. /**
  5700. * dp_tx_dump_tx_desc() - Dump tx desc for debugging
  5701. * @tx_desc: software descriptor head pointer
  5702. *
  5703. * This function will dump tx desc for further debugging
  5704. *
  5705. * Return: none
  5706. */
  5707. static
  5708. void dp_tx_dump_tx_desc(struct dp_tx_desc_s *tx_desc)
  5709. {
  5710. if (tx_desc) {
  5711. dp_tx_comp_warn("tx_desc->nbuf: %pK", tx_desc->nbuf);
  5712. dp_tx_comp_warn("tx_desc->flags: 0x%x", tx_desc->flags);
  5713. dp_tx_comp_warn("tx_desc->id: %u", tx_desc->id);
  5714. dp_tx_comp_warn("tx_desc->dma_addr: 0x%x",
  5715. (unsigned int)tx_desc->dma_addr);
  5716. dp_tx_comp_warn("tx_desc->vdev_id: %u",
  5717. tx_desc->vdev_id);
  5718. dp_tx_comp_warn("tx_desc->tx_status: %u",
  5719. tx_desc->tx_status);
  5720. dp_tx_comp_warn("tx_desc->pdev: %pK",
  5721. tx_desc->pdev);
  5722. dp_tx_comp_warn("tx_desc->tx_encap_type: %u",
  5723. tx_desc->tx_encap_type);
  5724. dp_tx_comp_warn("tx_desc->buffer_src: %u",
  5725. tx_desc->buffer_src);
  5726. dp_tx_comp_warn("tx_desc->frm_type: %u",
  5727. tx_desc->frm_type);
  5728. dp_tx_comp_warn("tx_desc->pkt_offset: %u",
  5729. tx_desc->pkt_offset);
  5730. dp_tx_comp_warn("tx_desc->pool_id: %u",
  5731. tx_desc->pool_id);
  5732. }
  5733. }
  5734. #endif
  5735. #ifdef WLAN_FEATURE_RX_SOFTIRQ_TIME_LIMIT
  5736. static inline
  5737. bool dp_tx_comp_loop_pkt_limit_hit(struct dp_soc *soc, int num_reaped,
  5738. int max_reap_limit)
  5739. {
  5740. bool limit_hit = false;
  5741. limit_hit =
  5742. (num_reaped >= max_reap_limit) ? true : false;
  5743. if (limit_hit)
  5744. DP_STATS_INC(soc, tx.tx_comp_loop_pkt_limit_hit, 1);
  5745. return limit_hit;
  5746. }
  5747. static inline bool dp_tx_comp_enable_eol_data_check(struct dp_soc *soc)
  5748. {
  5749. return soc->wlan_cfg_ctx->tx_comp_enable_eol_data_check;
  5750. }
  5751. static inline int dp_tx_comp_get_loop_pkt_limit(struct dp_soc *soc)
  5752. {
  5753. struct wlan_cfg_dp_soc_ctxt *cfg = soc->wlan_cfg_ctx;
  5754. return cfg->tx_comp_loop_pkt_limit;
  5755. }
  5756. #else
  5757. static inline
  5758. bool dp_tx_comp_loop_pkt_limit_hit(struct dp_soc *soc, int num_reaped,
  5759. int max_reap_limit)
  5760. {
  5761. return false;
  5762. }
  5763. static inline bool dp_tx_comp_enable_eol_data_check(struct dp_soc *soc)
  5764. {
  5765. return false;
  5766. }
  5767. static inline int dp_tx_comp_get_loop_pkt_limit(struct dp_soc *soc)
  5768. {
  5769. return 0;
  5770. }
  5771. #endif
  5772. #ifdef WLAN_FEATURE_NEAR_FULL_IRQ
  5773. static inline int
  5774. dp_srng_test_and_update_nf_params(struct dp_soc *soc, struct dp_srng *dp_srng,
  5775. int *max_reap_limit)
  5776. {
  5777. return soc->arch_ops.dp_srng_test_and_update_nf_params(soc, dp_srng,
  5778. max_reap_limit);
  5779. }
  5780. #else
  5781. static inline int
  5782. dp_srng_test_and_update_nf_params(struct dp_soc *soc, struct dp_srng *dp_srng,
  5783. int *max_reap_limit)
  5784. {
  5785. return 0;
  5786. }
  5787. #endif
  5788. #ifdef DP_TX_TRACKING
  5789. void dp_tx_desc_check_corruption(struct dp_tx_desc_s *tx_desc)
  5790. {
  5791. if ((tx_desc->magic != DP_TX_MAGIC_PATTERN_INUSE) &&
  5792. (tx_desc->magic != DP_TX_MAGIC_PATTERN_FREE)) {
  5793. dp_err_rl("tx_desc %u is corrupted", tx_desc->id);
  5794. qdf_trigger_self_recovery(NULL, QDF_TX_DESC_LEAK);
  5795. }
  5796. }
  5797. #endif
  5798. #ifndef WLAN_SOFTUMAC_SUPPORT
  5799. #ifdef DP_TX_COMP_RING_DESC_SANITY_CHECK
  5800. /* Increasing this value, runs the risk of srng backpressure */
  5801. #define DP_STALE_TX_COMP_WAIT_TIMEOUT_US 1000
  5802. static inline void
  5803. dp_tx_comp_reset_stale_entry_detection(struct dp_soc *soc, uint32_t ring_num)
  5804. {
  5805. soc->stale_entry[ring_num].detected = 0;
  5806. }
  5807. /**
  5808. * dp_tx_comp_stale_entry_handle() - Detect stale entry condition in tx
  5809. * completion srng.
  5810. * @soc: DP SoC handle
  5811. * @ring_num: tx completion ring number
  5812. * @status: QDF_STATUS from tx_comp_get_params_from_hal_desc arch ops
  5813. *
  5814. * Return: QDF_STATUS_SUCCESS if stale entry is detected and handled
  5815. * QDF_STATUS error code in other cases.
  5816. */
  5817. static inline QDF_STATUS
  5818. dp_tx_comp_stale_entry_handle(struct dp_soc *soc, uint32_t ring_num,
  5819. QDF_STATUS status)
  5820. {
  5821. uint64_t curr_timestamp = qdf_get_log_timestamp_usecs();
  5822. uint64_t delta_us;
  5823. if (status != QDF_STATUS_E_PENDING) {
  5824. dp_tx_comp_reset_stale_entry_detection(soc, ring_num);
  5825. return QDF_STATUS_E_INVAL;
  5826. }
  5827. if (soc->stale_entry[ring_num].detected) {
  5828. /* stale entry process continuation */
  5829. delta_us = curr_timestamp -
  5830. soc->stale_entry[ring_num].start_time;
  5831. if (delta_us > DP_STALE_TX_COMP_WAIT_TIMEOUT_US) {
  5832. dp_err("Stale tx comp desc, waited %d us", delta_us);
  5833. return QDF_STATUS_E_FAILURE;
  5834. }
  5835. } else {
  5836. /* This is the start of stale entry detection */
  5837. soc->stale_entry[ring_num].detected = 1;
  5838. soc->stale_entry[ring_num].start_time = curr_timestamp;
  5839. }
  5840. return QDF_STATUS_SUCCESS;
  5841. }
  5842. #else
  5843. static inline void
  5844. dp_tx_comp_reset_stale_entry_detection(struct dp_soc *soc, uint32_t ring_num)
  5845. {
  5846. }
  5847. static inline QDF_STATUS
  5848. dp_tx_comp_stale_entry_handle(struct dp_soc *soc, uint32_t ring_num,
  5849. QDF_STATUS status)
  5850. {
  5851. return QDF_STATUS_SUCCESS;
  5852. }
  5853. #endif
  5854. uint32_t dp_tx_comp_handler(struct dp_intr *int_ctx, struct dp_soc *soc,
  5855. hal_ring_handle_t hal_ring_hdl, uint8_t ring_id,
  5856. uint32_t quota)
  5857. {
  5858. void *tx_comp_hal_desc;
  5859. void *last_prefetched_hw_desc = NULL;
  5860. void *last_hw_desc = NULL;
  5861. struct dp_tx_desc_s *last_prefetched_sw_desc = NULL;
  5862. hal_soc_handle_t hal_soc;
  5863. uint8_t buffer_src;
  5864. struct dp_tx_desc_s *tx_desc = NULL;
  5865. struct dp_tx_desc_s *head_desc = NULL;
  5866. struct dp_tx_desc_s *tail_desc = NULL;
  5867. struct dp_tx_desc_s *fast_head_desc = NULL;
  5868. struct dp_tx_desc_s *fast_tail_desc = NULL;
  5869. uint32_t num_processed = 0;
  5870. uint32_t fast_desc_count = 0;
  5871. uint32_t count;
  5872. uint32_t num_avail_for_reap = 0;
  5873. bool force_break = false;
  5874. struct dp_srng *tx_comp_ring = &soc->tx_comp_ring[ring_id];
  5875. int max_reap_limit, ring_near_full;
  5876. uint32_t num_entries;
  5877. qdf_nbuf_queue_head_t h;
  5878. QDF_STATUS status;
  5879. DP_HIST_INIT();
  5880. num_entries = hal_srng_get_num_entries(soc->hal_soc, hal_ring_hdl);
  5881. more_data:
  5882. hal_soc = soc->hal_soc;
  5883. /* Re-initialize local variables to be re-used */
  5884. head_desc = NULL;
  5885. tail_desc = NULL;
  5886. count = 0;
  5887. max_reap_limit = dp_tx_comp_get_loop_pkt_limit(soc);
  5888. ring_near_full = dp_srng_test_and_update_nf_params(soc, tx_comp_ring,
  5889. &max_reap_limit);
  5890. if (qdf_unlikely(dp_srng_access_start(int_ctx, soc, hal_ring_hdl))) {
  5891. dp_err("HAL RING Access Failed -- %pK", hal_ring_hdl);
  5892. return 0;
  5893. }
  5894. hal_srng_update_ring_usage_wm_no_lock(soc->hal_soc, hal_ring_hdl);
  5895. if (!num_avail_for_reap)
  5896. num_avail_for_reap = hal_srng_dst_num_valid(hal_soc,
  5897. hal_ring_hdl, 0);
  5898. if (num_avail_for_reap >= quota)
  5899. num_avail_for_reap = quota;
  5900. last_hw_desc = dp_srng_dst_inv_cached_descs(soc, hal_ring_hdl,
  5901. num_avail_for_reap);
  5902. last_prefetched_hw_desc = dp_srng_dst_prefetch_32_byte_desc(hal_soc,
  5903. hal_ring_hdl,
  5904. num_avail_for_reap);
  5905. dp_tx_nbuf_queue_head_init(&h);
  5906. /* Find head descriptor from completion ring */
  5907. while (qdf_likely(num_avail_for_reap--)) {
  5908. tx_comp_hal_desc = dp_srng_dst_get_next(soc, hal_ring_hdl);
  5909. if (qdf_unlikely(!tx_comp_hal_desc))
  5910. break;
  5911. buffer_src = hal_tx_comp_get_buffer_source(hal_soc,
  5912. tx_comp_hal_desc);
  5913. /* If this buffer was not released by TQM or FW, then it is not
  5914. * Tx completion indication, assert */
  5915. if (qdf_unlikely(buffer_src !=
  5916. HAL_TX_COMP_RELEASE_SOURCE_TQM) &&
  5917. (qdf_unlikely(buffer_src !=
  5918. HAL_TX_COMP_RELEASE_SOURCE_FW))) {
  5919. uint8_t wbm_internal_error;
  5920. dp_err_rl(
  5921. "Tx comp release_src != TQM | FW but from %d",
  5922. buffer_src);
  5923. hal_dump_comp_desc(tx_comp_hal_desc);
  5924. DP_STATS_INC(soc, tx.invalid_release_source, 1);
  5925. /* When WBM sees NULL buffer_addr_info in any of
  5926. * ingress rings it sends an error indication,
  5927. * with wbm_internal_error=1, to a specific ring.
  5928. * The WBM2SW ring used to indicate these errors is
  5929. * fixed in HW, and that ring is being used as Tx
  5930. * completion ring. These errors are not related to
  5931. * Tx completions, and should just be ignored
  5932. */
  5933. wbm_internal_error = hal_get_wbm_internal_error(
  5934. hal_soc,
  5935. tx_comp_hal_desc);
  5936. if (wbm_internal_error) {
  5937. dp_err_rl("Tx comp wbm_internal_error!!");
  5938. DP_STATS_INC(soc, tx.wbm_internal_error[WBM_INT_ERROR_ALL], 1);
  5939. if (HAL_TX_COMP_RELEASE_SOURCE_REO ==
  5940. buffer_src)
  5941. dp_handle_wbm_internal_error(
  5942. soc,
  5943. tx_comp_hal_desc,
  5944. hal_tx_comp_get_buffer_type(
  5945. tx_comp_hal_desc));
  5946. } else {
  5947. dp_err_rl("Tx comp wbm_internal_error false");
  5948. DP_STATS_INC(soc, tx.non_wbm_internal_err, 1);
  5949. }
  5950. continue;
  5951. }
  5952. status = soc->arch_ops.tx_comp_get_params_from_hal_desc(
  5953. soc, tx_comp_hal_desc,
  5954. &tx_desc);
  5955. if (qdf_unlikely(!tx_desc)) {
  5956. if (QDF_IS_STATUS_SUCCESS(
  5957. dp_tx_comp_stale_entry_handle(soc, ring_id,
  5958. status))) {
  5959. hal_srng_dst_dec_tp(hal_soc, hal_ring_hdl);
  5960. break;
  5961. }
  5962. dp_err("unable to retrieve tx_desc!");
  5963. hal_dump_comp_desc(tx_comp_hal_desc);
  5964. DP_STATS_INC(soc, tx.invalid_tx_comp_desc, 1);
  5965. QDF_BUG(0);
  5966. continue;
  5967. }
  5968. dp_tx_comp_reset_stale_entry_detection(soc, ring_id);
  5969. tx_desc->buffer_src = buffer_src;
  5970. /*
  5971. * If the release source is FW, process the HTT status
  5972. */
  5973. if (qdf_unlikely(buffer_src ==
  5974. HAL_TX_COMP_RELEASE_SOURCE_FW)) {
  5975. uint8_t htt_tx_status[HAL_TX_COMP_HTT_STATUS_LEN];
  5976. hal_tx_comp_get_htt_desc(tx_comp_hal_desc,
  5977. htt_tx_status);
  5978. /* Collect hw completion contents */
  5979. hal_tx_comp_desc_sync(tx_comp_hal_desc,
  5980. &tx_desc->comp, 1);
  5981. soc->arch_ops.dp_tx_process_htt_completion(
  5982. soc,
  5983. tx_desc,
  5984. htt_tx_status,
  5985. ring_id);
  5986. if (qdf_unlikely(!tx_desc->pdev)) {
  5987. dp_tx_dump_tx_desc(tx_desc);
  5988. }
  5989. } else {
  5990. if (tx_desc->flags & DP_TX_DESC_FLAG_FASTPATH_SIMPLE ||
  5991. tx_desc->flags & DP_TX_DESC_FLAG_PPEDS)
  5992. goto add_to_pool2;
  5993. tx_desc->tx_status =
  5994. hal_tx_comp_get_tx_status(tx_comp_hal_desc);
  5995. tx_desc->buffer_src = buffer_src;
  5996. /*
  5997. * If the fast completion mode is enabled extended
  5998. * metadata from descriptor is not copied
  5999. */
  6000. if (qdf_likely(tx_desc->flags &
  6001. DP_TX_DESC_FLAG_SIMPLE))
  6002. goto add_to_pool;
  6003. /*
  6004. * If the descriptor is already freed in vdev_detach,
  6005. * continue to next descriptor
  6006. */
  6007. if (qdf_unlikely
  6008. ((tx_desc->vdev_id == DP_INVALID_VDEV_ID) &&
  6009. !tx_desc->flags)) {
  6010. dp_tx_comp_info_rl("Descriptor freed in vdev_detach %d",
  6011. tx_desc->id);
  6012. DP_STATS_INC(soc, tx.tx_comp_exception, 1);
  6013. dp_tx_desc_check_corruption(tx_desc);
  6014. continue;
  6015. }
  6016. if (qdf_unlikely(!tx_desc->pdev)) {
  6017. dp_tx_comp_warn("The pdev is NULL in TX desc, ignored.");
  6018. dp_tx_dump_tx_desc(tx_desc);
  6019. DP_STATS_INC(soc, tx.tx_comp_exception, 1);
  6020. continue;
  6021. }
  6022. if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) {
  6023. dp_tx_comp_info_rl("pdev in down state %d",
  6024. tx_desc->id);
  6025. tx_desc->flags |= DP_TX_DESC_FLAG_TX_COMP_ERR;
  6026. dp_tx_comp_free_buf(soc, tx_desc, false);
  6027. dp_tx_desc_release(soc, tx_desc,
  6028. tx_desc->pool_id);
  6029. goto next_desc;
  6030. }
  6031. if (!(tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED) ||
  6032. !(tx_desc->flags & DP_TX_DESC_FLAG_QUEUED_TX)) {
  6033. dp_tx_comp_alert("Txdesc invalid, flgs = %x,id = %d",
  6034. tx_desc->flags, tx_desc->id);
  6035. qdf_assert_always(0);
  6036. }
  6037. /* Collect hw completion contents */
  6038. hal_tx_comp_desc_sync(tx_comp_hal_desc,
  6039. &tx_desc->comp, 1);
  6040. add_to_pool:
  6041. DP_HIST_PACKET_COUNT_INC(tx_desc->pdev->pdev_id);
  6042. add_to_pool2:
  6043. /* First ring descriptor on the cycle */
  6044. if (tx_desc->flags & DP_TX_DESC_FLAG_FASTPATH_SIMPLE ||
  6045. tx_desc->flags & DP_TX_DESC_FLAG_PPEDS) {
  6046. dp_tx_nbuf_dev_queue_free(&h, tx_desc);
  6047. fast_desc_count++;
  6048. if (!fast_head_desc) {
  6049. fast_head_desc = tx_desc;
  6050. fast_tail_desc = tx_desc;
  6051. }
  6052. fast_tail_desc->next = tx_desc;
  6053. fast_tail_desc = tx_desc;
  6054. dp_tx_desc_clear(tx_desc);
  6055. } else {
  6056. if (!head_desc) {
  6057. head_desc = tx_desc;
  6058. tail_desc = tx_desc;
  6059. }
  6060. tail_desc->next = tx_desc;
  6061. tx_desc->next = NULL;
  6062. tail_desc = tx_desc;
  6063. }
  6064. }
  6065. next_desc:
  6066. num_processed += !(count & DP_TX_NAPI_BUDGET_DIV_MASK);
  6067. /*
  6068. * Processed packet count is more than given quota
  6069. * stop to processing
  6070. */
  6071. count++;
  6072. dp_tx_prefetch_hw_sw_nbuf_desc(soc, hal_soc,
  6073. num_avail_for_reap,
  6074. hal_ring_hdl,
  6075. &last_prefetched_hw_desc,
  6076. &last_prefetched_sw_desc,
  6077. last_hw_desc);
  6078. if (dp_tx_comp_loop_pkt_limit_hit(soc, count, max_reap_limit))
  6079. break;
  6080. }
  6081. dp_srng_access_end(int_ctx, soc, hal_ring_hdl);
  6082. /* Process the reaped descriptors that were sent via fast path */
  6083. if (fast_head_desc) {
  6084. dp_tx_comp_process_desc_list_fast(soc, fast_head_desc,
  6085. fast_tail_desc, ring_id,
  6086. fast_desc_count);
  6087. dp_tx_nbuf_dev_kfree_list(&h);
  6088. }
  6089. /* Process the reaped descriptors */
  6090. if (head_desc)
  6091. dp_tx_comp_process_desc_list(soc, head_desc, ring_id);
  6092. DP_STATS_INC(soc, tx.tx_comp[ring_id], count);
  6093. /*
  6094. * If we are processing in near-full condition, there are 3 scenario
  6095. * 1) Ring entries has reached critical state
  6096. * 2) Ring entries are still near high threshold
  6097. * 3) Ring entries are below the safe level
  6098. *
  6099. * One more loop will move the state to normal processing and yield
  6100. */
  6101. if (ring_near_full)
  6102. goto more_data;
  6103. if (dp_tx_comp_enable_eol_data_check(soc)) {
  6104. if (num_processed >= quota)
  6105. force_break = true;
  6106. if (!force_break &&
  6107. hal_srng_dst_peek_sync_locked(soc->hal_soc,
  6108. hal_ring_hdl)) {
  6109. DP_STATS_INC(soc, tx.hp_oos2, 1);
  6110. if (!hif_exec_should_yield(soc->hif_handle,
  6111. int_ctx->dp_intr_id))
  6112. goto more_data;
  6113. num_avail_for_reap =
  6114. hal_srng_dst_num_valid_locked(soc->hal_soc,
  6115. hal_ring_hdl,
  6116. true);
  6117. if (qdf_unlikely(num_entries &&
  6118. (num_avail_for_reap >=
  6119. num_entries >> 1))) {
  6120. DP_STATS_INC(soc, tx.near_full, 1);
  6121. goto more_data;
  6122. }
  6123. }
  6124. }
  6125. DP_TX_HIST_STATS_PER_PDEV();
  6126. return num_processed;
  6127. }
  6128. #endif
  6129. #ifdef FEATURE_WLAN_TDLS
  6130. qdf_nbuf_t dp_tx_non_std(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  6131. enum ol_tx_spec tx_spec, qdf_nbuf_t msdu_list)
  6132. {
  6133. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  6134. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  6135. DP_MOD_ID_TDLS);
  6136. if (!vdev) {
  6137. dp_err("vdev handle for id %d is NULL", vdev_id);
  6138. return NULL;
  6139. }
  6140. if (tx_spec & OL_TX_SPEC_NO_FREE)
  6141. vdev->is_tdls_frame = true;
  6142. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
  6143. return dp_tx_send(soc_hdl, vdev_id, msdu_list);
  6144. }
  6145. #endif
  6146. QDF_STATUS dp_tx_vdev_attach(struct dp_vdev *vdev)
  6147. {
  6148. int pdev_id;
  6149. /*
  6150. * Fill HTT TCL Metadata with Vdev ID and MAC ID
  6151. */
  6152. DP_TX_TCL_METADATA_TYPE_SET(vdev->htt_tcl_metadata,
  6153. DP_TCL_METADATA_TYPE_VDEV_BASED);
  6154. DP_TX_TCL_METADATA_VDEV_ID_SET(vdev->htt_tcl_metadata,
  6155. vdev->vdev_id);
  6156. pdev_id =
  6157. dp_get_target_pdev_id_for_host_pdev_id(vdev->pdev->soc,
  6158. vdev->pdev->pdev_id);
  6159. DP_TX_TCL_METADATA_PDEV_ID_SET(vdev->htt_tcl_metadata, pdev_id);
  6160. /*
  6161. * Set HTT Extension Valid bit to 0 by default
  6162. */
  6163. DP_TX_TCL_METADATA_VALID_HTT_SET(vdev->htt_tcl_metadata, 0);
  6164. dp_tx_vdev_update_search_flags(vdev);
  6165. return QDF_STATUS_SUCCESS;
  6166. }
  6167. #ifndef FEATURE_WDS
  6168. static inline bool dp_tx_da_search_override(struct dp_vdev *vdev)
  6169. {
  6170. return false;
  6171. }
  6172. #endif
  6173. void dp_tx_vdev_update_search_flags(struct dp_vdev *vdev)
  6174. {
  6175. struct dp_soc *soc = vdev->pdev->soc;
  6176. /*
  6177. * Enable both AddrY (SA based search) and AddrX (Da based search)
  6178. * for TDLS link
  6179. *
  6180. * Enable AddrY (SA based search) only for non-WDS STA and
  6181. * ProxySTA VAP (in HKv1) modes.
  6182. *
  6183. * In all other VAP modes, only DA based search should be
  6184. * enabled
  6185. */
  6186. if (vdev->opmode == wlan_op_mode_sta &&
  6187. vdev->tdls_link_connected)
  6188. vdev->hal_desc_addr_search_flags =
  6189. (HAL_TX_DESC_ADDRX_EN | HAL_TX_DESC_ADDRY_EN);
  6190. else if ((vdev->opmode == wlan_op_mode_sta) &&
  6191. !dp_tx_da_search_override(vdev))
  6192. vdev->hal_desc_addr_search_flags = HAL_TX_DESC_ADDRY_EN;
  6193. else
  6194. vdev->hal_desc_addr_search_flags = HAL_TX_DESC_ADDRX_EN;
  6195. if (vdev->opmode == wlan_op_mode_sta && !vdev->tdls_link_connected)
  6196. vdev->search_type = soc->sta_mode_search_policy;
  6197. else
  6198. vdev->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
  6199. }
  6200. #ifdef WLAN_SUPPORT_PPEDS
  6201. static inline bool
  6202. dp_is_tx_desc_flush_match(struct dp_pdev *pdev,
  6203. struct dp_vdev *vdev,
  6204. struct dp_tx_desc_s *tx_desc)
  6205. {
  6206. if (!(tx_desc && (tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED)))
  6207. return false;
  6208. if (tx_desc->flags & DP_TX_DESC_FLAG_PPEDS)
  6209. return true;
  6210. /*
  6211. * if vdev is given, then only check whether desc
  6212. * vdev match. if vdev is NULL, then check whether
  6213. * desc pdev match.
  6214. */
  6215. return vdev ? (tx_desc->vdev_id == vdev->vdev_id) :
  6216. (tx_desc->pdev == pdev);
  6217. }
  6218. #else
  6219. static inline bool
  6220. dp_is_tx_desc_flush_match(struct dp_pdev *pdev,
  6221. struct dp_vdev *vdev,
  6222. struct dp_tx_desc_s *tx_desc)
  6223. {
  6224. if (!(tx_desc && (tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED)))
  6225. return false;
  6226. /*
  6227. * if vdev is given, then only check whether desc
  6228. * vdev match. if vdev is NULL, then check whether
  6229. * desc pdev match.
  6230. */
  6231. return vdev ? (tx_desc->vdev_id == vdev->vdev_id) :
  6232. (tx_desc->pdev == pdev);
  6233. }
  6234. #endif
  6235. #ifdef QCA_LL_TX_FLOW_CONTROL_V2
  6236. void dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
  6237. bool force_free)
  6238. {
  6239. uint8_t i;
  6240. uint32_t j;
  6241. uint32_t num_desc, page_id, offset;
  6242. uint16_t num_desc_per_page;
  6243. struct dp_soc *soc = pdev->soc;
  6244. struct dp_tx_desc_s *tx_desc = NULL;
  6245. struct dp_tx_desc_pool_s *tx_desc_pool = NULL;
  6246. if (!vdev && !force_free) {
  6247. dp_err("Reset TX desc vdev, Vdev param is required!");
  6248. return;
  6249. }
  6250. for (i = 0; i < MAX_TXDESC_POOLS; i++) {
  6251. tx_desc_pool = &soc->tx_desc[i];
  6252. if (!(tx_desc_pool->pool_size) ||
  6253. IS_TX_DESC_POOL_STATUS_INACTIVE(tx_desc_pool) ||
  6254. !(tx_desc_pool->desc_pages.cacheable_pages))
  6255. continue;
  6256. /*
  6257. * Add flow pool lock protection in case pool is freed
  6258. * due to all tx_desc is recycled when handle TX completion.
  6259. * this is not necessary when do force flush as:
  6260. * a. double lock will happen if dp_tx_desc_release is
  6261. * also trying to acquire it.
  6262. * b. dp interrupt has been disabled before do force TX desc
  6263. * flush in dp_pdev_deinit().
  6264. */
  6265. if (!force_free)
  6266. qdf_spin_lock_bh(&tx_desc_pool->flow_pool_lock);
  6267. num_desc = tx_desc_pool->pool_size;
  6268. num_desc_per_page =
  6269. tx_desc_pool->desc_pages.num_element_per_page;
  6270. for (j = 0; j < num_desc; j++) {
  6271. page_id = j / num_desc_per_page;
  6272. offset = j % num_desc_per_page;
  6273. if (qdf_unlikely(!(tx_desc_pool->
  6274. desc_pages.cacheable_pages)))
  6275. break;
  6276. tx_desc = dp_tx_desc_find(soc, i, page_id, offset,
  6277. false);
  6278. if (dp_is_tx_desc_flush_match(pdev, vdev, tx_desc)) {
  6279. /*
  6280. * Free TX desc if force free is
  6281. * required, otherwise only reset vdev
  6282. * in this TX desc.
  6283. */
  6284. if (force_free) {
  6285. tx_desc->flags |= DP_TX_DESC_FLAG_FLUSH;
  6286. dp_tx_comp_free_buf(soc, tx_desc,
  6287. false);
  6288. dp_tx_desc_release(soc, tx_desc, i);
  6289. } else {
  6290. tx_desc->vdev_id = DP_INVALID_VDEV_ID;
  6291. }
  6292. }
  6293. }
  6294. if (!force_free)
  6295. qdf_spin_unlock_bh(&tx_desc_pool->flow_pool_lock);
  6296. }
  6297. }
  6298. #else /* QCA_LL_TX_FLOW_CONTROL_V2! */
  6299. /**
  6300. * dp_tx_desc_reset_vdev() - reset vdev to NULL in TX Desc
  6301. *
  6302. * @soc: Handle to DP soc structure
  6303. * @tx_desc: pointer of one TX desc
  6304. * @desc_pool_id: TX Desc pool id
  6305. * @spcl_pool: Special pool
  6306. */
  6307. static inline void
  6308. dp_tx_desc_reset_vdev(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  6309. uint8_t desc_pool_id, bool spcl_pool)
  6310. {
  6311. struct dp_tx_desc_pool_s *pool = NULL;
  6312. pool = spcl_pool ? dp_get_spcl_tx_desc_pool(soc, desc_pool_id) :
  6313. dp_get_tx_desc_pool(soc, desc_pool_id);
  6314. TX_DESC_LOCK_LOCK(&pool->lock);
  6315. tx_desc->vdev_id = DP_INVALID_VDEV_ID;
  6316. TX_DESC_LOCK_UNLOCK(&pool->lock);
  6317. }
  6318. void __dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
  6319. bool force_free, bool spcl_pool)
  6320. {
  6321. uint8_t i, num_pool;
  6322. uint32_t j;
  6323. uint32_t num_desc_t, page_id, offset;
  6324. uint16_t num_desc_per_page;
  6325. struct dp_soc *soc = pdev->soc;
  6326. struct dp_tx_desc_s *tx_desc = NULL;
  6327. struct dp_tx_desc_pool_s *tx_desc_pool = NULL;
  6328. if (!vdev && !force_free) {
  6329. dp_err("Reset TX desc vdev, Vdev param is required!");
  6330. return;
  6331. }
  6332. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6333. for (i = 0; i < num_pool; i++) {
  6334. tx_desc_pool = spcl_pool ? dp_get_spcl_tx_desc_pool(soc, i) :
  6335. dp_get_tx_desc_pool(soc, i);
  6336. num_desc_t = tx_desc_pool->elem_count;
  6337. if (!tx_desc_pool->desc_pages.cacheable_pages)
  6338. continue;
  6339. num_desc_per_page =
  6340. tx_desc_pool->desc_pages.num_element_per_page;
  6341. for (j = 0; j < num_desc_t; j++) {
  6342. page_id = j / num_desc_per_page;
  6343. offset = j % num_desc_per_page;
  6344. tx_desc = dp_tx_desc_find(soc, i, page_id, offset,
  6345. spcl_pool);
  6346. if (dp_is_tx_desc_flush_match(pdev, vdev, tx_desc)) {
  6347. if (force_free) {
  6348. dp_tx_comp_free_buf(soc, tx_desc,
  6349. false);
  6350. dp_tx_desc_release(soc, tx_desc, i);
  6351. } else {
  6352. dp_tx_desc_reset_vdev(soc, tx_desc,
  6353. i, spcl_pool);
  6354. }
  6355. }
  6356. }
  6357. }
  6358. }
  6359. void dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
  6360. bool force_free)
  6361. {
  6362. __dp_tx_desc_flush(pdev, vdev, force_free, false);
  6363. __dp_tx_desc_flush(pdev, vdev, force_free, true);
  6364. }
  6365. #endif /* !QCA_LL_TX_FLOW_CONTROL_V2 */
  6366. QDF_STATUS dp_tx_vdev_detach(struct dp_vdev *vdev)
  6367. {
  6368. struct dp_pdev *pdev = vdev->pdev;
  6369. /* Reset TX desc associated to this Vdev as NULL */
  6370. dp_tx_desc_flush(pdev, vdev, false);
  6371. return QDF_STATUS_SUCCESS;
  6372. }
  6373. #ifdef QCA_LL_TX_FLOW_CONTROL_V2
  6374. /* Pools will be allocated dynamically */
  6375. static QDF_STATUS dp_tx_alloc_static_pools(struct dp_soc *soc, int num_pool,
  6376. int num_desc)
  6377. {
  6378. uint8_t i;
  6379. for (i = 0; i < num_pool; i++) {
  6380. qdf_spinlock_create(&soc->tx_desc[i].flow_pool_lock);
  6381. soc->tx_desc[i].status = FLOW_POOL_INACTIVE;
  6382. }
  6383. return QDF_STATUS_SUCCESS;
  6384. }
  6385. static QDF_STATUS dp_tx_spcl_alloc_static_pools(struct dp_soc *soc,
  6386. int num_pool,
  6387. int num_spcl_desc)
  6388. {
  6389. return QDF_STATUS_SUCCESS;
  6390. }
  6391. static QDF_STATUS dp_tx_init_static_pools(struct dp_soc *soc, int num_pool,
  6392. uint32_t num_desc)
  6393. {
  6394. return QDF_STATUS_SUCCESS;
  6395. }
  6396. static QDF_STATUS dp_tx_spcl_init_static_pools(struct dp_soc *soc, int num_pool,
  6397. uint32_t num_spcl_desc)
  6398. {
  6399. return QDF_STATUS_SUCCESS;
  6400. }
  6401. static void dp_tx_deinit_static_pools(struct dp_soc *soc, int num_pool)
  6402. {
  6403. }
  6404. static void dp_tx_spcl_deinit_static_pools(struct dp_soc *soc, int num_pool)
  6405. {
  6406. }
  6407. static void dp_tx_delete_static_pools(struct dp_soc *soc, int num_pool)
  6408. {
  6409. uint8_t i;
  6410. for (i = 0; i < num_pool; i++)
  6411. qdf_spinlock_destroy(&soc->tx_desc[i].flow_pool_lock);
  6412. }
  6413. static void dp_tx_spcl_delete_static_pools(struct dp_soc *soc, int num_pool)
  6414. {
  6415. }
  6416. #else /* QCA_LL_TX_FLOW_CONTROL_V2! */
  6417. static QDF_STATUS dp_tx_alloc_static_pools(struct dp_soc *soc, int num_pool,
  6418. uint32_t num_desc)
  6419. {
  6420. uint8_t i, count;
  6421. struct dp_global_context *dp_global;
  6422. dp_global = wlan_objmgr_get_global_ctx();
  6423. /* Allocate software Tx descriptor pools */
  6424. if (dp_global->tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
  6425. for (i = 0; i < num_pool; i++) {
  6426. if (dp_tx_desc_pool_alloc(soc, i, num_desc, false)) {
  6427. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  6428. FL("Tx Desc Pool alloc %d failed %pK"),
  6429. i, soc);
  6430. goto fail;
  6431. }
  6432. }
  6433. }
  6434. dp_global->tx_desc_pool_alloc_cnt[soc->arch_id]++;
  6435. return QDF_STATUS_SUCCESS;
  6436. fail:
  6437. for (count = 0; count < i; count++)
  6438. dp_tx_desc_pool_free(soc, count, false);
  6439. return QDF_STATUS_E_NOMEM;
  6440. }
  6441. static QDF_STATUS dp_tx_spcl_alloc_static_pools(struct dp_soc *soc,
  6442. int num_pool,
  6443. uint32_t num_spcl_desc)
  6444. {
  6445. uint8_t j, count;
  6446. struct dp_global_context *dp_global;
  6447. dp_global = wlan_objmgr_get_global_ctx();
  6448. /* Allocate software Tx descriptor pools */
  6449. if (dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
  6450. for (j = 0; j < num_pool; j++) {
  6451. if (dp_tx_desc_pool_alloc(soc, j, num_spcl_desc, true)) {
  6452. QDF_TRACE(QDF_MODULE_ID_DP,
  6453. QDF_TRACE_LEVEL_ERROR,
  6454. FL("Tx special Desc Pool alloc %d failed %pK"),
  6455. j, soc);
  6456. goto fail;
  6457. }
  6458. }
  6459. }
  6460. dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id]++;
  6461. return QDF_STATUS_SUCCESS;
  6462. fail:
  6463. for (count = 0; count < j; count++)
  6464. dp_tx_desc_pool_free(soc, count, true);
  6465. return QDF_STATUS_E_NOMEM;
  6466. }
  6467. static QDF_STATUS dp_tx_init_static_pools(struct dp_soc *soc, int num_pool,
  6468. uint32_t num_desc)
  6469. {
  6470. uint8_t i;
  6471. struct dp_global_context *dp_global;
  6472. dp_global = wlan_objmgr_get_global_ctx();
  6473. if (dp_global->tx_desc_pool_init_cnt[soc->arch_id] == 0) {
  6474. for (i = 0; i < num_pool; i++) {
  6475. if (dp_tx_desc_pool_init(soc, i, num_desc, false)) {
  6476. QDF_TRACE(QDF_MODULE_ID_DP,
  6477. QDF_TRACE_LEVEL_ERROR,
  6478. FL("Tx Desc Pool init %d failed %pK"),
  6479. i, soc);
  6480. return QDF_STATUS_E_NOMEM;
  6481. }
  6482. }
  6483. }
  6484. dp_global->tx_desc_pool_init_cnt[soc->arch_id]++;
  6485. return QDF_STATUS_SUCCESS;
  6486. }
  6487. static QDF_STATUS dp_tx_spcl_init_static_pools(struct dp_soc *soc, int num_pool,
  6488. uint32_t num_spcl_desc)
  6489. {
  6490. uint8_t i;
  6491. struct dp_global_context *dp_global;
  6492. dp_global = wlan_objmgr_get_global_ctx();
  6493. if (dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id] == 0) {
  6494. for (i = 0; i < num_pool; i++) {
  6495. if (dp_tx_desc_pool_init(soc, i, num_spcl_desc, true)) {
  6496. QDF_TRACE(QDF_MODULE_ID_DP,
  6497. QDF_TRACE_LEVEL_ERROR,
  6498. FL("Tx special Desc Pool init %d failed %pK"),
  6499. i, soc);
  6500. return QDF_STATUS_E_NOMEM;
  6501. }
  6502. }
  6503. }
  6504. dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id]++;
  6505. return QDF_STATUS_SUCCESS;
  6506. }
  6507. static void dp_tx_deinit_static_pools(struct dp_soc *soc, int num_pool)
  6508. {
  6509. uint8_t i;
  6510. struct dp_global_context *dp_global;
  6511. dp_global = wlan_objmgr_get_global_ctx();
  6512. dp_global->tx_desc_pool_init_cnt[soc->arch_id]--;
  6513. if (dp_global->tx_desc_pool_init_cnt[soc->arch_id] == 0) {
  6514. for (i = 0; i < num_pool; i++)
  6515. dp_tx_desc_pool_deinit(soc, i, false);
  6516. }
  6517. }
  6518. static void dp_tx_spcl_deinit_static_pools(struct dp_soc *soc, int num_pool)
  6519. {
  6520. uint8_t i;
  6521. struct dp_global_context *dp_global;
  6522. dp_global = wlan_objmgr_get_global_ctx();
  6523. dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id]--;
  6524. if (dp_global->spcl_tx_desc_pool_init_cnt[soc->arch_id] == 0) {
  6525. for (i = 0; i < num_pool; i++)
  6526. dp_tx_desc_pool_deinit(soc, i, true);
  6527. }
  6528. }
  6529. static void dp_tx_delete_static_pools(struct dp_soc *soc, int num_pool)
  6530. {
  6531. uint8_t i;
  6532. struct dp_global_context *dp_global;
  6533. dp_global = wlan_objmgr_get_global_ctx();
  6534. dp_global->tx_desc_pool_alloc_cnt[soc->arch_id]--;
  6535. if (dp_global->tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
  6536. for (i = 0; i < num_pool; i++)
  6537. dp_tx_desc_pool_free(soc, i, false);
  6538. }
  6539. }
  6540. static void dp_tx_spcl_delete_static_pools(struct dp_soc *soc, int num_pool)
  6541. {
  6542. uint8_t i;
  6543. struct dp_global_context *dp_global;
  6544. dp_global = wlan_objmgr_get_global_ctx();
  6545. dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id]--;
  6546. if (dp_global->spcl_tx_desc_pool_alloc_cnt[soc->arch_id] == 0) {
  6547. for (i = 0; i < num_pool; i++)
  6548. dp_tx_desc_pool_free(soc, i, true);
  6549. }
  6550. }
  6551. #endif /* !QCA_LL_TX_FLOW_CONTROL_V2 */
  6552. /**
  6553. * dp_tx_tso_cmn_desc_pool_deinit() - de-initialize TSO descriptors
  6554. * @soc: core txrx main context
  6555. * @num_pool: number of pools
  6556. *
  6557. */
  6558. static void dp_tx_tso_cmn_desc_pool_deinit(struct dp_soc *soc, uint8_t num_pool)
  6559. {
  6560. dp_tx_tso_desc_pool_deinit(soc, num_pool);
  6561. dp_tx_tso_num_seg_pool_deinit(soc, num_pool);
  6562. }
  6563. /**
  6564. * dp_tx_tso_cmn_desc_pool_free() - free TSO descriptors
  6565. * @soc: core txrx main context
  6566. * @num_pool: number of pools
  6567. *
  6568. */
  6569. static void dp_tx_tso_cmn_desc_pool_free(struct dp_soc *soc, uint8_t num_pool)
  6570. {
  6571. dp_tx_tso_desc_pool_free(soc, num_pool);
  6572. dp_tx_tso_num_seg_pool_free(soc, num_pool);
  6573. }
  6574. #ifndef WLAN_SOFTUMAC_SUPPORT
  6575. void dp_soc_tx_desc_sw_pools_free(struct dp_soc *soc)
  6576. {
  6577. uint8_t num_pool, num_ext_pool;
  6578. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6579. return;
  6580. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6581. num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
  6582. dp_tx_tso_cmn_desc_pool_free(soc, num_pool);
  6583. dp_tx_ext_desc_pool_free(soc, num_ext_pool);
  6584. dp_tx_delete_static_pools(soc, num_pool);
  6585. dp_tx_spcl_delete_static_pools(soc, num_pool);
  6586. }
  6587. void dp_soc_tx_desc_sw_pools_deinit(struct dp_soc *soc)
  6588. {
  6589. uint8_t num_pool, num_ext_pool;
  6590. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6591. return;
  6592. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6593. num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
  6594. dp_tx_flow_control_deinit(soc);
  6595. dp_tx_tso_cmn_desc_pool_deinit(soc, num_pool);
  6596. dp_tx_ext_desc_pool_deinit(soc, num_ext_pool);
  6597. dp_tx_deinit_static_pools(soc, num_pool);
  6598. dp_tx_spcl_deinit_static_pools(soc, num_pool);
  6599. }
  6600. #else
  6601. void dp_soc_tx_desc_sw_pools_free(struct dp_soc *soc)
  6602. {
  6603. uint8_t num_pool;
  6604. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6605. return;
  6606. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6607. dp_tx_delete_static_pools(soc, num_pool);
  6608. dp_tx_spcl_delete_static_pools(soc, num_pool);
  6609. }
  6610. void dp_soc_tx_desc_sw_pools_deinit(struct dp_soc *soc)
  6611. {
  6612. uint8_t num_pool;
  6613. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6614. return;
  6615. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6616. dp_tx_flow_control_deinit(soc);
  6617. dp_tx_deinit_static_pools(soc, num_pool);
  6618. dp_tx_spcl_deinit_static_pools(soc, num_pool);
  6619. }
  6620. #endif /*WLAN_SOFTUMAC_SUPPORT*/
  6621. /**
  6622. * dp_tx_tso_cmn_desc_pool_alloc() - TSO cmn desc pool allocator
  6623. * @soc: DP soc handle
  6624. * @num_pool: Number of pools
  6625. * @num_desc: Number of descriptors
  6626. *
  6627. * Reserve TSO descriptor buffers
  6628. *
  6629. * Return: QDF_STATUS_E_FAILURE on failure or
  6630. * QDF_STATUS_SUCCESS on success
  6631. */
  6632. static QDF_STATUS dp_tx_tso_cmn_desc_pool_alloc(struct dp_soc *soc,
  6633. uint8_t num_pool,
  6634. uint32_t num_desc)
  6635. {
  6636. if (dp_tx_tso_desc_pool_alloc(soc, num_pool, num_desc)) {
  6637. dp_err("TSO Desc Pool alloc %d failed %pK", num_pool, soc);
  6638. return QDF_STATUS_E_FAILURE;
  6639. }
  6640. if (dp_tx_tso_num_seg_pool_alloc(soc, num_pool, num_desc)) {
  6641. dp_err("TSO Num of seg Pool alloc %d failed %pK",
  6642. num_pool, soc);
  6643. return QDF_STATUS_E_FAILURE;
  6644. }
  6645. return QDF_STATUS_SUCCESS;
  6646. }
  6647. /**
  6648. * dp_tx_tso_cmn_desc_pool_init() - TSO cmn desc pool init
  6649. * @soc: DP soc handle
  6650. * @num_pool: Number of pools
  6651. * @num_desc: Number of descriptors
  6652. *
  6653. * Initialize TSO descriptor pools
  6654. *
  6655. * Return: QDF_STATUS_E_FAILURE on failure or
  6656. * QDF_STATUS_SUCCESS on success
  6657. */
  6658. static QDF_STATUS dp_tx_tso_cmn_desc_pool_init(struct dp_soc *soc,
  6659. uint8_t num_pool,
  6660. uint32_t num_desc)
  6661. {
  6662. if (dp_tx_tso_desc_pool_init(soc, num_pool, num_desc)) {
  6663. dp_err("TSO Desc Pool alloc %d failed %pK", num_pool, soc);
  6664. return QDF_STATUS_E_FAILURE;
  6665. }
  6666. if (dp_tx_tso_num_seg_pool_init(soc, num_pool, num_desc)) {
  6667. dp_err("TSO Num of seg Pool alloc %d failed %pK",
  6668. num_pool, soc);
  6669. return QDF_STATUS_E_FAILURE;
  6670. }
  6671. return QDF_STATUS_SUCCESS;
  6672. }
  6673. #ifndef WLAN_SOFTUMAC_SUPPORT
  6674. QDF_STATUS dp_soc_tx_desc_sw_pools_alloc(struct dp_soc *soc)
  6675. {
  6676. uint8_t num_pool, num_ext_pool;
  6677. uint32_t num_desc;
  6678. uint32_t num_spcl_desc;
  6679. uint32_t num_ext_desc;
  6680. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6681. return QDF_STATUS_SUCCESS;
  6682. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6683. num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
  6684. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  6685. num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
  6686. num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
  6687. dp_info("Tx Desc Alloc num_pool: %d descs: %d", num_pool, num_desc);
  6688. if ((num_pool > MAX_TXDESC_POOLS) ||
  6689. (num_ext_pool > MAX_TXDESC_POOLS) ||
  6690. (num_desc > WLAN_CFG_NUM_TX_DESC_MAX) ||
  6691. (num_spcl_desc > WLAN_CFG_NUM_TX_SPL_DESC_MAX))
  6692. goto fail1;
  6693. if (dp_tx_alloc_static_pools(soc, num_pool, num_desc))
  6694. goto fail1;
  6695. if (dp_tx_spcl_alloc_static_pools(soc, num_pool, num_spcl_desc))
  6696. goto fail2;
  6697. if (dp_tx_ext_desc_pool_alloc(soc, num_ext_pool, num_ext_desc))
  6698. goto fail3;
  6699. if (wlan_cfg_is_tso_desc_attach_defer(soc->wlan_cfg_ctx))
  6700. return QDF_STATUS_SUCCESS;
  6701. if (dp_tx_tso_cmn_desc_pool_alloc(soc, num_ext_pool, num_ext_desc))
  6702. goto fail4;
  6703. return QDF_STATUS_SUCCESS;
  6704. fail4:
  6705. dp_tx_ext_desc_pool_free(soc, num_ext_pool);
  6706. fail3:
  6707. dp_tx_spcl_delete_static_pools(soc, num_pool);
  6708. fail2:
  6709. dp_tx_delete_static_pools(soc, num_pool);
  6710. fail1:
  6711. return QDF_STATUS_E_RESOURCES;
  6712. }
  6713. QDF_STATUS dp_soc_tx_desc_sw_pools_init(struct dp_soc *soc)
  6714. {
  6715. uint8_t num_pool, num_ext_pool;
  6716. uint32_t num_desc;
  6717. uint32_t num_spcl_desc;
  6718. uint32_t num_ext_desc;
  6719. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6720. return QDF_STATUS_SUCCESS;
  6721. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6722. num_ext_pool = dp_get_ext_tx_desc_pool_num(soc);
  6723. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  6724. num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
  6725. num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
  6726. if (dp_tx_init_static_pools(soc, num_pool, num_desc))
  6727. goto fail1;
  6728. if (dp_tx_spcl_init_static_pools(soc, num_pool, num_spcl_desc))
  6729. goto fail2;
  6730. if (dp_tx_ext_desc_pool_init(soc, num_ext_pool, num_ext_desc))
  6731. goto fail3;
  6732. if (wlan_cfg_is_tso_desc_attach_defer(soc->wlan_cfg_ctx))
  6733. return QDF_STATUS_SUCCESS;
  6734. if (dp_tx_tso_cmn_desc_pool_init(soc, num_ext_pool, num_ext_desc))
  6735. goto fail4;
  6736. dp_tx_flow_control_init(soc);
  6737. soc->process_tx_status = CONFIG_PROCESS_TX_STATUS;
  6738. return QDF_STATUS_SUCCESS;
  6739. fail4:
  6740. dp_tx_ext_desc_pool_deinit(soc, num_ext_pool);
  6741. fail3:
  6742. dp_tx_spcl_deinit_static_pools(soc, num_pool);
  6743. fail2:
  6744. dp_tx_deinit_static_pools(soc, num_pool);
  6745. fail1:
  6746. return QDF_STATUS_E_RESOURCES;
  6747. }
  6748. #else
  6749. QDF_STATUS dp_soc_tx_desc_sw_pools_alloc(struct dp_soc *soc)
  6750. {
  6751. uint8_t num_pool;
  6752. uint32_t num_desc;
  6753. uint32_t num_spcl_desc;
  6754. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6755. return QDF_STATUS_SUCCESS;
  6756. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6757. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  6758. num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
  6759. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO,
  6760. "%s Tx Desc Alloc num_pool = %d, descs = %d",
  6761. __func__, num_pool, num_desc);
  6762. if ((num_pool > MAX_TXDESC_POOLS) ||
  6763. (num_desc > WLAN_CFG_NUM_TX_DESC_MAX) ||
  6764. (num_spcl_desc > WLAN_CFG_NUM_TX_SPL_DESC_MAX))
  6765. goto fail1;
  6766. if (dp_tx_alloc_static_pools(soc, num_pool, num_desc))
  6767. goto fail1;
  6768. if (dp_tx_spcl_alloc_static_pools(soc, num_pool, num_spcl_desc))
  6769. goto fail2;
  6770. return QDF_STATUS_SUCCESS;
  6771. fail2:
  6772. dp_tx_delete_static_pools(soc, num_pool);
  6773. fail1:
  6774. return QDF_STATUS_E_RESOURCES;
  6775. }
  6776. QDF_STATUS dp_soc_tx_desc_sw_pools_init(struct dp_soc *soc)
  6777. {
  6778. uint8_t num_pool;
  6779. uint32_t num_desc;
  6780. uint32_t num_spcl_desc;
  6781. if (wlan_cfg_get_dp_soc_nss_cfg(soc->wlan_cfg_ctx))
  6782. return QDF_STATUS_SUCCESS;
  6783. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  6784. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  6785. num_spcl_desc = wlan_cfg_get_num_tx_spl_desc(soc->wlan_cfg_ctx);
  6786. if (dp_tx_init_static_pools(soc, num_pool, num_desc))
  6787. goto fail;
  6788. if (dp_tx_spcl_init_static_pools(soc, num_pool, num_spcl_desc))
  6789. goto fail1;
  6790. dp_tx_flow_control_init(soc);
  6791. soc->process_tx_status = CONFIG_PROCESS_TX_STATUS;
  6792. return QDF_STATUS_SUCCESS;
  6793. fail1:
  6794. dp_tx_deinit_static_pools(soc, num_pool);
  6795. fail:
  6796. return QDF_STATUS_E_RESOURCES;
  6797. }
  6798. #endif
  6799. QDF_STATUS dp_tso_soc_attach(struct cdp_soc_t *txrx_soc)
  6800. {
  6801. struct dp_soc *soc = (struct dp_soc *)txrx_soc;
  6802. uint8_t num_ext_desc_pool;
  6803. uint32_t num_ext_desc;
  6804. num_ext_desc_pool = dp_get_ext_tx_desc_pool_num(soc);
  6805. num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
  6806. if (dp_tx_tso_cmn_desc_pool_alloc(soc, num_ext_desc_pool, num_ext_desc))
  6807. return QDF_STATUS_E_FAILURE;
  6808. if (dp_tx_tso_cmn_desc_pool_init(soc, num_ext_desc_pool, num_ext_desc))
  6809. return QDF_STATUS_E_FAILURE;
  6810. return QDF_STATUS_SUCCESS;
  6811. }
  6812. QDF_STATUS dp_tso_soc_detach(struct cdp_soc_t *txrx_soc)
  6813. {
  6814. struct dp_soc *soc = (struct dp_soc *)txrx_soc;
  6815. uint8_t num_ext_desc_pool = dp_get_ext_tx_desc_pool_num(soc);
  6816. dp_tx_tso_cmn_desc_pool_deinit(soc, num_ext_desc_pool);
  6817. dp_tx_tso_cmn_desc_pool_free(soc, num_ext_desc_pool);
  6818. return QDF_STATUS_SUCCESS;
  6819. }
  6820. #ifdef CONFIG_DP_PKT_ADD_TIMESTAMP
  6821. void dp_pkt_add_timestamp(struct dp_vdev *vdev,
  6822. enum qdf_pkt_timestamp_index index, uint64_t time,
  6823. qdf_nbuf_t nbuf)
  6824. {
  6825. if (qdf_unlikely(qdf_is_dp_pkt_timestamp_enabled())) {
  6826. uint64_t tsf_time;
  6827. if (vdev->get_tsf_time) {
  6828. vdev->get_tsf_time(vdev->osif_vdev, time, &tsf_time);
  6829. qdf_add_dp_pkt_timestamp(nbuf, index, tsf_time);
  6830. }
  6831. }
  6832. }
  6833. void dp_pkt_get_timestamp(uint64_t *time)
  6834. {
  6835. if (qdf_unlikely(qdf_is_dp_pkt_timestamp_enabled()))
  6836. *time = qdf_get_log_timestamp();
  6837. }
  6838. #endif
  6839. #ifdef QCA_MULTIPASS_SUPPORT
  6840. void dp_tx_add_groupkey_metadata(struct dp_vdev *vdev,
  6841. struct dp_tx_msdu_info_s *msdu_info,
  6842. uint16_t group_key)
  6843. {
  6844. struct htt_tx_msdu_desc_ext2_t *meta_data =
  6845. (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
  6846. qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
  6847. /*
  6848. * When attempting to send a multicast packet with multi-passphrase,
  6849. * host shall add HTT EXT meta data "struct htt_tx_msdu_desc_ext2_t"
  6850. * ref htt.h indicating the group_id field in "key_flags" also having
  6851. * "valid_key_flags" as 1. Assign “key_flags = group_key_ix”.
  6852. */
  6853. HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_SET(msdu_info->meta_data[0],
  6854. 1);
  6855. HTT_TX_MSDU_EXT2_DESC_KEY_FLAGS_SET(msdu_info->meta_data[2], group_key);
  6856. }
  6857. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  6858. defined(WLAN_MCAST_MLO)
  6859. /**
  6860. * dp_tx_need_mcast_reinject() - If frame needs to be processed in reinject path
  6861. * @vdev: DP vdev handle
  6862. *
  6863. * Return: true if reinject handling is required else false
  6864. */
  6865. static inline bool
  6866. dp_tx_need_mcast_reinject(struct dp_vdev *vdev)
  6867. {
  6868. if (vdev->mlo_vdev && vdev->opmode == wlan_op_mode_ap)
  6869. return true;
  6870. return false;
  6871. }
  6872. #else
  6873. static inline bool
  6874. dp_tx_need_mcast_reinject(struct dp_vdev *vdev)
  6875. {
  6876. return false;
  6877. }
  6878. #endif
  6879. /**
  6880. * dp_tx_need_multipass_process() - If frame needs multipass phrase processing
  6881. * @soc: dp soc handle
  6882. * @vdev: DP vdev handle
  6883. * @buf: frame
  6884. * @vlan_id: vlan id of frame
  6885. *
  6886. * Return: whether peer is special or classic
  6887. */
  6888. static
  6889. uint8_t dp_tx_need_multipass_process(struct dp_soc *soc, struct dp_vdev *vdev,
  6890. qdf_nbuf_t buf, uint16_t *vlan_id)
  6891. {
  6892. struct dp_txrx_peer *txrx_peer = NULL;
  6893. struct dp_peer *peer = NULL;
  6894. qdf_ether_header_t *eh = (qdf_ether_header_t *)qdf_nbuf_data(buf);
  6895. struct vlan_ethhdr *veh = NULL;
  6896. bool not_vlan = ((vdev->tx_encap_type == htt_cmn_pkt_type_raw) ||
  6897. (htons(eh->ether_type) != ETH_P_8021Q));
  6898. struct cdp_peer_info peer_info = { 0 };
  6899. if (qdf_unlikely(not_vlan))
  6900. return DP_VLAN_UNTAGGED;
  6901. veh = (struct vlan_ethhdr *)eh;
  6902. *vlan_id = (ntohs(veh->h_vlan_TCI) & VLAN_VID_MASK);
  6903. if (qdf_unlikely(DP_FRAME_IS_MULTICAST((eh)->ether_dhost))) {
  6904. /* look for handling of multicast packets in reinject path */
  6905. if (dp_tx_need_mcast_reinject(vdev))
  6906. return DP_VLAN_UNTAGGED;
  6907. qdf_spin_lock_bh(&vdev->mpass_peer_mutex);
  6908. TAILQ_FOREACH(txrx_peer, &vdev->mpass_peer_list,
  6909. mpass_peer_list_elem) {
  6910. if (*vlan_id == txrx_peer->vlan_id) {
  6911. qdf_spin_unlock_bh(&vdev->mpass_peer_mutex);
  6912. return DP_VLAN_TAGGED_MULTICAST;
  6913. }
  6914. }
  6915. qdf_spin_unlock_bh(&vdev->mpass_peer_mutex);
  6916. return DP_VLAN_UNTAGGED;
  6917. }
  6918. DP_PEER_INFO_PARAMS_INIT(&peer_info, DP_VDEV_ALL, eh->ether_dhost,
  6919. false, CDP_WILD_PEER_TYPE);
  6920. peer = dp_peer_hash_find_wrapper((struct dp_soc *)soc, &peer_info,
  6921. DP_MOD_ID_TX_MULTIPASS);
  6922. if (qdf_unlikely(!peer))
  6923. return DP_VLAN_UNTAGGED;
  6924. /*
  6925. * Do not drop the frame when vlan_id doesn't match.
  6926. * Send the frame as it is.
  6927. */
  6928. if (*vlan_id == peer->txrx_peer->vlan_id) {
  6929. dp_peer_unref_delete(peer, DP_MOD_ID_TX_MULTIPASS);
  6930. return DP_VLAN_TAGGED_UNICAST;
  6931. }
  6932. dp_peer_unref_delete(peer, DP_MOD_ID_TX_MULTIPASS);
  6933. return DP_VLAN_UNTAGGED;
  6934. }
  6935. #ifndef WLAN_REPEATER_NOT_SUPPORTED
  6936. static inline void
  6937. dp_tx_multipass_send_pkt_to_repeater(struct dp_soc *soc, struct dp_vdev *vdev,
  6938. qdf_nbuf_t nbuf,
  6939. struct dp_tx_msdu_info_s *msdu_info)
  6940. {
  6941. qdf_nbuf_t nbuf_copy = NULL;
  6942. /* AP can have classic clients, special clients &
  6943. * classic repeaters.
  6944. * 1. Classic clients & special client:
  6945. * Remove vlan header, find corresponding group key
  6946. * index, fill in metaheader and enqueue multicast
  6947. * frame to TCL.
  6948. * 2. Classic repeater:
  6949. * Pass through to classic repeater with vlan tag
  6950. * intact without any group key index. Hardware
  6951. * will know which key to use to send frame to
  6952. * repeater.
  6953. */
  6954. nbuf_copy = qdf_nbuf_copy(nbuf);
  6955. /*
  6956. * Send multicast frame to special peers even
  6957. * if pass through to classic repeater fails.
  6958. */
  6959. if (nbuf_copy) {
  6960. struct dp_tx_msdu_info_s msdu_info_copy;
  6961. qdf_mem_zero(&msdu_info_copy, sizeof(msdu_info_copy));
  6962. msdu_info_copy.tid = HTT_TX_EXT_TID_INVALID;
  6963. msdu_info_copy.xmit_type =
  6964. qdf_nbuf_get_vdev_xmit_type(nbuf);
  6965. HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_SET(msdu_info_copy.meta_data[0], 1);
  6966. nbuf_copy = dp_tx_send_msdu_single(vdev, nbuf_copy,
  6967. &msdu_info_copy,
  6968. HTT_INVALID_PEER, NULL);
  6969. if (nbuf_copy) {
  6970. qdf_nbuf_free(nbuf_copy);
  6971. dp_info_rl("nbuf_copy send failed");
  6972. }
  6973. }
  6974. }
  6975. #else
  6976. static inline void
  6977. dp_tx_multipass_send_pkt_to_repeater(struct dp_soc *soc, struct dp_vdev *vdev,
  6978. qdf_nbuf_t nbuf,
  6979. struct dp_tx_msdu_info_s *msdu_info)
  6980. {
  6981. }
  6982. #endif
  6983. bool dp_tx_multipass_process(struct dp_soc *soc, struct dp_vdev *vdev,
  6984. qdf_nbuf_t nbuf,
  6985. struct dp_tx_msdu_info_s *msdu_info)
  6986. {
  6987. uint16_t vlan_id = 0;
  6988. uint16_t group_key = 0;
  6989. uint8_t is_spcl_peer = DP_VLAN_UNTAGGED;
  6990. if (HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_GET(msdu_info->meta_data[0]))
  6991. return true;
  6992. is_spcl_peer = dp_tx_need_multipass_process(soc, vdev, nbuf, &vlan_id);
  6993. if ((is_spcl_peer != DP_VLAN_TAGGED_MULTICAST) &&
  6994. (is_spcl_peer != DP_VLAN_TAGGED_UNICAST))
  6995. return true;
  6996. if (is_spcl_peer == DP_VLAN_TAGGED_UNICAST) {
  6997. dp_tx_remove_vlan_tag(vdev, nbuf);
  6998. return true;
  6999. }
  7000. dp_tx_multipass_send_pkt_to_repeater(soc, vdev, nbuf, msdu_info);
  7001. group_key = vdev->iv_vlan_map[vlan_id];
  7002. /*
  7003. * If group key is not installed, drop the frame.
  7004. */
  7005. if (!group_key)
  7006. return false;
  7007. dp_tx_remove_vlan_tag(vdev, nbuf);
  7008. dp_tx_add_groupkey_metadata(vdev, msdu_info, group_key);
  7009. msdu_info->exception_fw = 1;
  7010. return true;
  7011. }
  7012. #endif /* QCA_MULTIPASS_SUPPORT */