dp_tx.c 178 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754
  1. /*
  2. * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved.
  3. * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  4. *
  5. * Permission to use, copy, modify, and/or distribute this software for
  6. * any purpose with or without fee is hereby granted, provided that the
  7. * above copyright notice and this permission notice appear in all
  8. * copies.
  9. *
  10. * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL
  11. * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED
  12. * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE
  13. * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL
  14. * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR
  15. * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
  16. * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
  17. * PERFORMANCE OF THIS SOFTWARE.
  18. */
  19. #include "htt.h"
  20. #include "dp_htt.h"
  21. #include "hal_hw_headers.h"
  22. #include "dp_tx.h"
  23. #include "dp_tx_desc.h"
  24. #include "dp_peer.h"
  25. #include "dp_types.h"
  26. #include "hal_tx.h"
  27. #include "qdf_mem.h"
  28. #include "qdf_nbuf.h"
  29. #include "qdf_net_types.h"
  30. #include "qdf_module.h"
  31. #include <wlan_cfg.h>
  32. #include "dp_ipa.h"
  33. #if defined(MESH_MODE_SUPPORT) || defined(FEATURE_PERPKT_INFO)
  34. #include "if_meta_hdr.h"
  35. #endif
  36. #include "enet.h"
  37. #include "dp_internal.h"
  38. #ifdef ATH_SUPPORT_IQUE
  39. #include "dp_txrx_me.h"
  40. #endif
  41. #include "dp_hist.h"
  42. #ifdef WLAN_DP_FEATURE_SW_LATENCY_MGR
  43. #include <wlan_dp_swlm.h>
  44. #endif
  45. #ifdef WIFI_MONITOR_SUPPORT
  46. #include <dp_mon.h>
  47. #endif
  48. #ifdef FEATURE_WDS
  49. #include "dp_txrx_wds.h"
  50. #endif
  51. #include "cdp_txrx_cmn_reg.h"
  52. #ifdef CONFIG_SAWF
  53. #include <dp_sawf.h>
  54. #endif
  55. /* Flag to skip CCE classify when mesh or tid override enabled */
  56. #define DP_TX_SKIP_CCE_CLASSIFY \
  57. (DP_TXRX_HLOS_TID_OVERRIDE_ENABLED | DP_TX_MESH_ENABLED)
  58. /* TODO Add support in TSO */
  59. #define DP_DESC_NUM_FRAG(x) 0
  60. /* disable TQM_BYPASS */
  61. #define TQM_BYPASS_WAR 0
  62. /* invalid peer id for reinject*/
  63. #define DP_INVALID_PEER 0XFFFE
  64. #define DP_RETRY_COUNT 7
  65. #ifdef WLAN_PEER_JITTER
  66. #define DP_AVG_JITTER_WEIGHT_DENOM 4
  67. #define DP_AVG_DELAY_WEIGHT_DENOM 3
  68. #endif
  69. #ifdef QCA_DP_TX_FW_METADATA_V2
  70. #define DP_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)\
  71. HTT_TX_TCL_METADATA_V2_PDEV_ID_SET(_var, _val)
  72. #define DP_TX_TCL_METADATA_VALID_HTT_SET(_var, _val) \
  73. HTT_TX_TCL_METADATA_V2_VALID_HTT_SET(_var, _val)
  74. #define DP_TX_TCL_METADATA_TYPE_SET(_var, _val) \
  75. HTT_TX_TCL_METADATA_TYPE_V2_SET(_var, _val)
  76. #define DP_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val) \
  77. HTT_TX_TCL_METADATA_V2_HOST_INSPECTED_SET(_var, _val)
  78. #define DP_TX_TCL_METADATA_PEER_ID_SET(_var, _val) \
  79. HTT_TX_TCL_METADATA_V2_PEER_ID_SET(_var, _val)
  80. #define DP_TX_TCL_METADATA_VDEV_ID_SET(_var, _val) \
  81. HTT_TX_TCL_METADATA_V2_VDEV_ID_SET(_var, _val)
  82. #define DP_TCL_METADATA_TYPE_PEER_BASED \
  83. HTT_TCL_METADATA_V2_TYPE_PEER_BASED
  84. #define DP_TCL_METADATA_TYPE_VDEV_BASED \
  85. HTT_TCL_METADATA_V2_TYPE_VDEV_BASED
  86. #else
  87. #define DP_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)\
  88. HTT_TX_TCL_METADATA_PDEV_ID_SET(_var, _val)
  89. #define DP_TX_TCL_METADATA_VALID_HTT_SET(_var, _val) \
  90. HTT_TX_TCL_METADATA_VALID_HTT_SET(_var, _val)
  91. #define DP_TX_TCL_METADATA_TYPE_SET(_var, _val) \
  92. HTT_TX_TCL_METADATA_TYPE_SET(_var, _val)
  93. #define DP_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val) \
  94. HTT_TX_TCL_METADATA_HOST_INSPECTED_SET(_var, _val)
  95. #define DP_TX_TCL_METADATA_PEER_ID_SET(_var, _val) \
  96. HTT_TX_TCL_METADATA_PEER_ID_SET(_var, _val)
  97. #define DP_TX_TCL_METADATA_VDEV_ID_SET(_var, _val) \
  98. HTT_TX_TCL_METADATA_VDEV_ID_SET(_var, _val)
  99. #define DP_TCL_METADATA_TYPE_PEER_BASED \
  100. HTT_TCL_METADATA_TYPE_PEER_BASED
  101. #define DP_TCL_METADATA_TYPE_VDEV_BASED \
  102. HTT_TCL_METADATA_TYPE_VDEV_BASED
  103. #endif
  104. /*mapping between hal encrypt type and cdp_sec_type*/
  105. uint8_t sec_type_map[MAX_CDP_SEC_TYPE] = {HAL_TX_ENCRYPT_TYPE_NO_CIPHER,
  106. HAL_TX_ENCRYPT_TYPE_WEP_128,
  107. HAL_TX_ENCRYPT_TYPE_WEP_104,
  108. HAL_TX_ENCRYPT_TYPE_WEP_40,
  109. HAL_TX_ENCRYPT_TYPE_TKIP_WITH_MIC,
  110. HAL_TX_ENCRYPT_TYPE_TKIP_NO_MIC,
  111. HAL_TX_ENCRYPT_TYPE_AES_CCMP_128,
  112. HAL_TX_ENCRYPT_TYPE_WAPI,
  113. HAL_TX_ENCRYPT_TYPE_AES_CCMP_256,
  114. HAL_TX_ENCRYPT_TYPE_AES_GCMP_128,
  115. HAL_TX_ENCRYPT_TYPE_AES_GCMP_256,
  116. HAL_TX_ENCRYPT_TYPE_WAPI_GCM_SM4};
  117. qdf_export_symbol(sec_type_map);
  118. #ifdef WLAN_FEATURE_DP_TX_DESC_HISTORY
  119. static inline enum dp_tx_event_type dp_tx_get_event_type(uint32_t flags)
  120. {
  121. enum dp_tx_event_type type;
  122. if (flags & DP_TX_DESC_FLAG_FLUSH)
  123. type = DP_TX_DESC_FLUSH;
  124. else if (flags & DP_TX_DESC_FLAG_TX_COMP_ERR)
  125. type = DP_TX_COMP_UNMAP_ERR;
  126. else if (flags & DP_TX_DESC_FLAG_COMPLETED_TX)
  127. type = DP_TX_COMP_UNMAP;
  128. else
  129. type = DP_TX_DESC_UNMAP;
  130. return type;
  131. }
  132. static inline void
  133. dp_tx_desc_history_add(struct dp_soc *soc, dma_addr_t paddr,
  134. qdf_nbuf_t skb, uint32_t sw_cookie,
  135. enum dp_tx_event_type type)
  136. {
  137. struct dp_tx_tcl_history *tx_tcl_history = &soc->tx_tcl_history;
  138. struct dp_tx_comp_history *tx_comp_history = &soc->tx_comp_history;
  139. struct dp_tx_desc_event *entry;
  140. uint32_t idx;
  141. uint16_t slot;
  142. switch (type) {
  143. case DP_TX_COMP_UNMAP:
  144. case DP_TX_COMP_UNMAP_ERR:
  145. case DP_TX_COMP_MSDU_EXT:
  146. if (qdf_unlikely(!tx_comp_history->allocated))
  147. return;
  148. dp_get_frag_hist_next_atomic_idx(&tx_comp_history->index, &idx,
  149. &slot,
  150. DP_TX_COMP_HIST_SLOT_SHIFT,
  151. DP_TX_COMP_HIST_PER_SLOT_MAX,
  152. DP_TX_COMP_HISTORY_SIZE);
  153. entry = &tx_comp_history->entry[slot][idx];
  154. break;
  155. case DP_TX_DESC_MAP:
  156. case DP_TX_DESC_UNMAP:
  157. case DP_TX_DESC_COOKIE:
  158. case DP_TX_DESC_FLUSH:
  159. if (qdf_unlikely(!tx_tcl_history->allocated))
  160. return;
  161. dp_get_frag_hist_next_atomic_idx(&tx_tcl_history->index, &idx,
  162. &slot,
  163. DP_TX_TCL_HIST_SLOT_SHIFT,
  164. DP_TX_TCL_HIST_PER_SLOT_MAX,
  165. DP_TX_TCL_HISTORY_SIZE);
  166. entry = &tx_tcl_history->entry[slot][idx];
  167. break;
  168. default:
  169. dp_info_rl("Invalid dp_tx_event_type: %d", type);
  170. return;
  171. }
  172. entry->skb = skb;
  173. entry->paddr = paddr;
  174. entry->sw_cookie = sw_cookie;
  175. entry->type = type;
  176. entry->ts = qdf_get_log_timestamp();
  177. }
  178. static inline void
  179. dp_tx_tso_seg_history_add(struct dp_soc *soc,
  180. struct qdf_tso_seg_elem_t *tso_seg,
  181. qdf_nbuf_t skb, uint32_t sw_cookie,
  182. enum dp_tx_event_type type)
  183. {
  184. int i;
  185. for (i = 1; i < tso_seg->seg.num_frags; i++) {
  186. dp_tx_desc_history_add(soc, tso_seg->seg.tso_frags[i].paddr,
  187. skb, sw_cookie, type);
  188. }
  189. if (!tso_seg->next)
  190. dp_tx_desc_history_add(soc, tso_seg->seg.tso_frags[0].paddr,
  191. skb, 0xFFFFFFFF, type);
  192. }
  193. static inline void
  194. dp_tx_tso_history_add(struct dp_soc *soc, struct qdf_tso_info_t tso_info,
  195. qdf_nbuf_t skb, uint32_t sw_cookie,
  196. enum dp_tx_event_type type)
  197. {
  198. struct qdf_tso_seg_elem_t *curr_seg = tso_info.tso_seg_list;
  199. uint32_t num_segs = tso_info.num_segs;
  200. while (num_segs) {
  201. dp_tx_tso_seg_history_add(soc, curr_seg, skb, sw_cookie, type);
  202. curr_seg = curr_seg->next;
  203. num_segs--;
  204. }
  205. }
  206. #else
  207. static inline enum dp_tx_event_type dp_tx_get_event_type(uint32_t flags)
  208. {
  209. return DP_TX_DESC_INVAL_EVT;
  210. }
  211. static inline void
  212. dp_tx_desc_history_add(struct dp_soc *soc, dma_addr_t paddr,
  213. qdf_nbuf_t skb, uint32_t sw_cookie,
  214. enum dp_tx_event_type type)
  215. {
  216. }
  217. static inline void
  218. dp_tx_tso_seg_history_add(struct dp_soc *soc,
  219. struct qdf_tso_seg_elem_t *tso_seg,
  220. qdf_nbuf_t skb, uint32_t sw_cookie,
  221. enum dp_tx_event_type type)
  222. {
  223. }
  224. static inline void
  225. dp_tx_tso_history_add(struct dp_soc *soc, struct qdf_tso_info_t tso_info,
  226. qdf_nbuf_t skb, uint32_t sw_cookie,
  227. enum dp_tx_event_type type)
  228. {
  229. }
  230. #endif /* WLAN_FEATURE_DP_TX_DESC_HISTORY */
  231. static int dp_get_rtpm_tput_policy_requirement(struct dp_soc *soc);
  232. /**
  233. * dp_is_tput_high() - Check if throughput is high
  234. *
  235. * @soc - core txrx main context
  236. *
  237. * The current function is based of the RTPM tput policy variable where RTPM is
  238. * avoided based on throughput.
  239. */
  240. static inline int dp_is_tput_high(struct dp_soc *soc)
  241. {
  242. return dp_get_rtpm_tput_policy_requirement(soc);
  243. }
  244. #if defined(FEATURE_TSO)
  245. /**
  246. * dp_tx_tso_unmap_segment() - Unmap TSO segment
  247. *
  248. * @soc - core txrx main context
  249. * @seg_desc - tso segment descriptor
  250. * @num_seg_desc - tso number segment descriptor
  251. */
  252. static void dp_tx_tso_unmap_segment(
  253. struct dp_soc *soc,
  254. struct qdf_tso_seg_elem_t *seg_desc,
  255. struct qdf_tso_num_seg_elem_t *num_seg_desc)
  256. {
  257. TSO_DEBUG("%s: Unmap the tso segment", __func__);
  258. if (qdf_unlikely(!seg_desc)) {
  259. DP_TRACE(ERROR, "%s %d TSO desc is NULL!",
  260. __func__, __LINE__);
  261. qdf_assert(0);
  262. } else if (qdf_unlikely(!num_seg_desc)) {
  263. DP_TRACE(ERROR, "%s %d TSO num desc is NULL!",
  264. __func__, __LINE__);
  265. qdf_assert(0);
  266. } else {
  267. bool is_last_seg;
  268. /* no tso segment left to do dma unmap */
  269. if (num_seg_desc->num_seg.tso_cmn_num_seg < 1)
  270. return;
  271. is_last_seg = (num_seg_desc->num_seg.tso_cmn_num_seg == 1) ?
  272. true : false;
  273. qdf_nbuf_unmap_tso_segment(soc->osdev,
  274. seg_desc, is_last_seg);
  275. num_seg_desc->num_seg.tso_cmn_num_seg--;
  276. }
  277. }
  278. /**
  279. * dp_tx_tso_desc_release() - Release the tso segment and tso_cmn_num_seg
  280. * back to the freelist
  281. *
  282. * @soc - soc device handle
  283. * @tx_desc - Tx software descriptor
  284. */
  285. static void dp_tx_tso_desc_release(struct dp_soc *soc,
  286. struct dp_tx_desc_s *tx_desc)
  287. {
  288. TSO_DEBUG("%s: Free the tso descriptor", __func__);
  289. if (qdf_unlikely(!tx_desc->msdu_ext_desc->tso_desc)) {
  290. dp_tx_err("SO desc is NULL!");
  291. qdf_assert(0);
  292. } else if (qdf_unlikely(!tx_desc->msdu_ext_desc->tso_num_desc)) {
  293. dp_tx_err("TSO num desc is NULL!");
  294. qdf_assert(0);
  295. } else {
  296. struct qdf_tso_num_seg_elem_t *tso_num_desc =
  297. (struct qdf_tso_num_seg_elem_t *)tx_desc->
  298. msdu_ext_desc->tso_num_desc;
  299. /* Add the tso num segment into the free list */
  300. if (tso_num_desc->num_seg.tso_cmn_num_seg == 0) {
  301. dp_tso_num_seg_free(soc, tx_desc->pool_id,
  302. tx_desc->msdu_ext_desc->
  303. tso_num_desc);
  304. tx_desc->msdu_ext_desc->tso_num_desc = NULL;
  305. DP_STATS_INC(tx_desc->pdev, tso_stats.tso_comp, 1);
  306. }
  307. /* Add the tso segment into the free list*/
  308. dp_tx_tso_desc_free(soc,
  309. tx_desc->pool_id, tx_desc->msdu_ext_desc->
  310. tso_desc);
  311. tx_desc->msdu_ext_desc->tso_desc = NULL;
  312. }
  313. }
  314. #else
  315. static void dp_tx_tso_unmap_segment(
  316. struct dp_soc *soc,
  317. struct qdf_tso_seg_elem_t *seg_desc,
  318. struct qdf_tso_num_seg_elem_t *num_seg_desc)
  319. {
  320. }
  321. static void dp_tx_tso_desc_release(struct dp_soc *soc,
  322. struct dp_tx_desc_s *tx_desc)
  323. {
  324. }
  325. #endif
  326. /**
  327. * dp_tx_desc_release() - Release Tx Descriptor
  328. * @tx_desc : Tx Descriptor
  329. * @desc_pool_id: Descriptor Pool ID
  330. *
  331. * Deallocate all resources attached to Tx descriptor and free the Tx
  332. * descriptor.
  333. *
  334. * Return:
  335. */
  336. void
  337. dp_tx_desc_release(struct dp_tx_desc_s *tx_desc, uint8_t desc_pool_id)
  338. {
  339. struct dp_pdev *pdev = tx_desc->pdev;
  340. struct dp_soc *soc;
  341. uint8_t comp_status = 0;
  342. qdf_assert(pdev);
  343. soc = pdev->soc;
  344. dp_tx_outstanding_dec(pdev);
  345. if (tx_desc->msdu_ext_desc) {
  346. if (tx_desc->frm_type == dp_tx_frm_tso)
  347. dp_tx_tso_desc_release(soc, tx_desc);
  348. if (tx_desc->flags & DP_TX_DESC_FLAG_ME)
  349. dp_tx_me_free_buf(tx_desc->pdev,
  350. tx_desc->msdu_ext_desc->me_buffer);
  351. dp_tx_ext_desc_free(soc, tx_desc->msdu_ext_desc, desc_pool_id);
  352. }
  353. if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)
  354. qdf_atomic_dec(&soc->num_tx_exception);
  355. if (HAL_TX_COMP_RELEASE_SOURCE_TQM ==
  356. tx_desc->buffer_src)
  357. comp_status = hal_tx_comp_get_release_reason(&tx_desc->comp,
  358. soc->hal_soc);
  359. else
  360. comp_status = HAL_TX_COMP_RELEASE_REASON_FW;
  361. dp_tx_debug("Tx Completion Release desc %d status %d outstanding %d",
  362. tx_desc->id, comp_status,
  363. qdf_atomic_read(&pdev->num_tx_outstanding));
  364. dp_tx_desc_free(soc, tx_desc, desc_pool_id);
  365. return;
  366. }
  367. /**
  368. * dp_tx_htt_metadata_prepare() - Prepare HTT metadata for special frames
  369. * @vdev: DP vdev Handle
  370. * @nbuf: skb
  371. * @msdu_info: msdu_info required to create HTT metadata
  372. *
  373. * Prepares and fills HTT metadata in the frame pre-header for special frames
  374. * that should be transmitted using varying transmit parameters.
  375. * There are 2 VDEV modes that currently needs this special metadata -
  376. * 1) Mesh Mode
  377. * 2) DSRC Mode
  378. *
  379. * Return: HTT metadata size
  380. *
  381. */
  382. static uint8_t dp_tx_prepare_htt_metadata(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  383. struct dp_tx_msdu_info_s *msdu_info)
  384. {
  385. uint32_t *meta_data = msdu_info->meta_data;
  386. struct htt_tx_msdu_desc_ext2_t *desc_ext =
  387. (struct htt_tx_msdu_desc_ext2_t *) meta_data;
  388. uint8_t htt_desc_size;
  389. /* Size rounded of multiple of 8 bytes */
  390. uint8_t htt_desc_size_aligned;
  391. uint8_t *hdr = NULL;
  392. /*
  393. * Metadata - HTT MSDU Extension header
  394. */
  395. htt_desc_size = sizeof(struct htt_tx_msdu_desc_ext2_t);
  396. htt_desc_size_aligned = (htt_desc_size + 7) & ~0x7;
  397. if (vdev->mesh_vdev || msdu_info->is_tx_sniffer ||
  398. HTT_TX_MSDU_EXT2_DESC_FLAG_VALID_KEY_FLAGS_GET(msdu_info->
  399. meta_data[0]) ||
  400. msdu_info->exception_fw) {
  401. if (qdf_unlikely(qdf_nbuf_headroom(nbuf) <
  402. htt_desc_size_aligned)) {
  403. nbuf = qdf_nbuf_realloc_headroom(nbuf,
  404. htt_desc_size_aligned);
  405. if (!nbuf) {
  406. /*
  407. * qdf_nbuf_realloc_headroom won't do skb_clone
  408. * as skb_realloc_headroom does. so, no free is
  409. * needed here.
  410. */
  411. DP_STATS_INC(vdev,
  412. tx_i.dropped.headroom_insufficient,
  413. 1);
  414. qdf_print(" %s[%d] skb_realloc_headroom failed",
  415. __func__, __LINE__);
  416. return 0;
  417. }
  418. }
  419. /* Fill and add HTT metaheader */
  420. hdr = qdf_nbuf_push_head(nbuf, htt_desc_size_aligned);
  421. if (!hdr) {
  422. dp_tx_err("Error in filling HTT metadata");
  423. return 0;
  424. }
  425. qdf_mem_copy(hdr, desc_ext, htt_desc_size);
  426. } else if (vdev->opmode == wlan_op_mode_ocb) {
  427. /* Todo - Add support for DSRC */
  428. }
  429. return htt_desc_size_aligned;
  430. }
  431. /**
  432. * dp_tx_prepare_tso_ext_desc() - Prepare MSDU extension descriptor for TSO
  433. * @tso_seg: TSO segment to process
  434. * @ext_desc: Pointer to MSDU extension descriptor
  435. *
  436. * Return: void
  437. */
  438. #if defined(FEATURE_TSO)
  439. static void dp_tx_prepare_tso_ext_desc(struct qdf_tso_seg_t *tso_seg,
  440. void *ext_desc)
  441. {
  442. uint8_t num_frag;
  443. uint32_t tso_flags;
  444. /*
  445. * Set tso_en, tcp_flags(NS, CWR, ECE, URG, ACK, PSH, RST, SYN, FIN),
  446. * tcp_flag_mask
  447. *
  448. * Checksum enable flags are set in TCL descriptor and not in Extension
  449. * Descriptor (H/W ignores checksum_en flags in MSDU ext descriptor)
  450. */
  451. tso_flags = *(uint32_t *) &tso_seg->tso_flags;
  452. hal_tx_ext_desc_set_tso_flags(ext_desc, tso_flags);
  453. hal_tx_ext_desc_set_msdu_length(ext_desc, tso_seg->tso_flags.l2_len,
  454. tso_seg->tso_flags.ip_len);
  455. hal_tx_ext_desc_set_tcp_seq(ext_desc, tso_seg->tso_flags.tcp_seq_num);
  456. hal_tx_ext_desc_set_ip_id(ext_desc, tso_seg->tso_flags.ip_id);
  457. for (num_frag = 0; num_frag < tso_seg->num_frags; num_frag++) {
  458. uint32_t lo = 0;
  459. uint32_t hi = 0;
  460. qdf_assert_always((tso_seg->tso_frags[num_frag].paddr) &&
  461. (tso_seg->tso_frags[num_frag].length));
  462. qdf_dmaaddr_to_32s(
  463. tso_seg->tso_frags[num_frag].paddr, &lo, &hi);
  464. hal_tx_ext_desc_set_buffer(ext_desc, num_frag, lo, hi,
  465. tso_seg->tso_frags[num_frag].length);
  466. }
  467. return;
  468. }
  469. #else
  470. static void dp_tx_prepare_tso_ext_desc(struct qdf_tso_seg_t *tso_seg,
  471. void *ext_desc)
  472. {
  473. return;
  474. }
  475. #endif
  476. #if defined(FEATURE_TSO)
  477. /**
  478. * dp_tx_free_tso_seg_list() - Loop through the tso segments
  479. * allocated and free them
  480. *
  481. * @soc: soc handle
  482. * @free_seg: list of tso segments
  483. * @msdu_info: msdu descriptor
  484. *
  485. * Return - void
  486. */
  487. static void dp_tx_free_tso_seg_list(
  488. struct dp_soc *soc,
  489. struct qdf_tso_seg_elem_t *free_seg,
  490. struct dp_tx_msdu_info_s *msdu_info)
  491. {
  492. struct qdf_tso_seg_elem_t *next_seg;
  493. while (free_seg) {
  494. next_seg = free_seg->next;
  495. dp_tx_tso_desc_free(soc,
  496. msdu_info->tx_queue.desc_pool_id,
  497. free_seg);
  498. free_seg = next_seg;
  499. }
  500. }
  501. /**
  502. * dp_tx_free_tso_num_seg_list() - Loop through the tso num segments
  503. * allocated and free them
  504. *
  505. * @soc: soc handle
  506. * @free_num_seg: list of tso number segments
  507. * @msdu_info: msdu descriptor
  508. * Return - void
  509. */
  510. static void dp_tx_free_tso_num_seg_list(
  511. struct dp_soc *soc,
  512. struct qdf_tso_num_seg_elem_t *free_num_seg,
  513. struct dp_tx_msdu_info_s *msdu_info)
  514. {
  515. struct qdf_tso_num_seg_elem_t *next_num_seg;
  516. while (free_num_seg) {
  517. next_num_seg = free_num_seg->next;
  518. dp_tso_num_seg_free(soc,
  519. msdu_info->tx_queue.desc_pool_id,
  520. free_num_seg);
  521. free_num_seg = next_num_seg;
  522. }
  523. }
  524. /**
  525. * dp_tx_unmap_tso_seg_list() - Loop through the tso segments
  526. * do dma unmap for each segment
  527. *
  528. * @soc: soc handle
  529. * @free_seg: list of tso segments
  530. * @num_seg_desc: tso number segment descriptor
  531. *
  532. * Return - void
  533. */
  534. static void dp_tx_unmap_tso_seg_list(
  535. struct dp_soc *soc,
  536. struct qdf_tso_seg_elem_t *free_seg,
  537. struct qdf_tso_num_seg_elem_t *num_seg_desc)
  538. {
  539. struct qdf_tso_seg_elem_t *next_seg;
  540. if (qdf_unlikely(!num_seg_desc)) {
  541. DP_TRACE(ERROR, "TSO number seg desc is NULL!");
  542. return;
  543. }
  544. while (free_seg) {
  545. next_seg = free_seg->next;
  546. dp_tx_tso_unmap_segment(soc, free_seg, num_seg_desc);
  547. free_seg = next_seg;
  548. }
  549. }
  550. #ifdef FEATURE_TSO_STATS
  551. /**
  552. * dp_tso_get_stats_idx: Retrieve the tso packet id
  553. * @pdev - pdev handle
  554. *
  555. * Return: id
  556. */
  557. static uint32_t dp_tso_get_stats_idx(struct dp_pdev *pdev)
  558. {
  559. uint32_t stats_idx;
  560. stats_idx = (((uint32_t)qdf_atomic_inc_return(&pdev->tso_idx))
  561. % CDP_MAX_TSO_PACKETS);
  562. return stats_idx;
  563. }
  564. #else
  565. static int dp_tso_get_stats_idx(struct dp_pdev *pdev)
  566. {
  567. return 0;
  568. }
  569. #endif /* FEATURE_TSO_STATS */
  570. /**
  571. * dp_tx_free_remaining_tso_desc() - do dma unmap for tso segments if any,
  572. * free the tso segments descriptor and
  573. * tso num segments descriptor
  574. *
  575. * @soc: soc handle
  576. * @msdu_info: msdu descriptor
  577. * @tso_seg_unmap: flag to show if dma unmap is necessary
  578. *
  579. * Return - void
  580. */
  581. static void dp_tx_free_remaining_tso_desc(struct dp_soc *soc,
  582. struct dp_tx_msdu_info_s *msdu_info,
  583. bool tso_seg_unmap)
  584. {
  585. struct qdf_tso_info_t *tso_info = &msdu_info->u.tso_info;
  586. struct qdf_tso_seg_elem_t *free_seg = tso_info->tso_seg_list;
  587. struct qdf_tso_num_seg_elem_t *tso_num_desc =
  588. tso_info->tso_num_seg_list;
  589. /* do dma unmap for each segment */
  590. if (tso_seg_unmap)
  591. dp_tx_unmap_tso_seg_list(soc, free_seg, tso_num_desc);
  592. /* free all tso number segment descriptor though looks only have 1 */
  593. dp_tx_free_tso_num_seg_list(soc, tso_num_desc, msdu_info);
  594. /* free all tso segment descriptor */
  595. dp_tx_free_tso_seg_list(soc, free_seg, msdu_info);
  596. }
  597. /**
  598. * dp_tx_prepare_tso() - Given a jumbo msdu, prepare the TSO info
  599. * @vdev: virtual device handle
  600. * @msdu: network buffer
  601. * @msdu_info: meta data associated with the msdu
  602. *
  603. * Return: QDF_STATUS_SUCCESS success
  604. */
  605. static QDF_STATUS dp_tx_prepare_tso(struct dp_vdev *vdev,
  606. qdf_nbuf_t msdu, struct dp_tx_msdu_info_s *msdu_info)
  607. {
  608. struct qdf_tso_seg_elem_t *tso_seg;
  609. int num_seg = qdf_nbuf_get_tso_num_seg(msdu);
  610. struct dp_soc *soc = vdev->pdev->soc;
  611. struct dp_pdev *pdev = vdev->pdev;
  612. struct qdf_tso_info_t *tso_info;
  613. struct qdf_tso_num_seg_elem_t *tso_num_seg;
  614. tso_info = &msdu_info->u.tso_info;
  615. tso_info->curr_seg = NULL;
  616. tso_info->tso_seg_list = NULL;
  617. tso_info->num_segs = num_seg;
  618. msdu_info->frm_type = dp_tx_frm_tso;
  619. tso_info->tso_num_seg_list = NULL;
  620. TSO_DEBUG(" %s: num_seg: %d", __func__, num_seg);
  621. while (num_seg) {
  622. tso_seg = dp_tx_tso_desc_alloc(
  623. soc, msdu_info->tx_queue.desc_pool_id);
  624. if (tso_seg) {
  625. tso_seg->next = tso_info->tso_seg_list;
  626. tso_info->tso_seg_list = tso_seg;
  627. num_seg--;
  628. } else {
  629. dp_err_rl("Failed to alloc tso seg desc");
  630. DP_STATS_INC_PKT(vdev->pdev,
  631. tso_stats.tso_no_mem_dropped, 1,
  632. qdf_nbuf_len(msdu));
  633. dp_tx_free_remaining_tso_desc(soc, msdu_info, false);
  634. return QDF_STATUS_E_NOMEM;
  635. }
  636. }
  637. TSO_DEBUG(" %s: num_seg: %d", __func__, num_seg);
  638. tso_num_seg = dp_tso_num_seg_alloc(soc,
  639. msdu_info->tx_queue.desc_pool_id);
  640. if (tso_num_seg) {
  641. tso_num_seg->next = tso_info->tso_num_seg_list;
  642. tso_info->tso_num_seg_list = tso_num_seg;
  643. } else {
  644. DP_TRACE(ERROR, "%s: Failed to alloc - Number of segs desc",
  645. __func__);
  646. dp_tx_free_remaining_tso_desc(soc, msdu_info, false);
  647. return QDF_STATUS_E_NOMEM;
  648. }
  649. msdu_info->num_seg =
  650. qdf_nbuf_get_tso_info(soc->osdev, msdu, tso_info);
  651. TSO_DEBUG(" %s: msdu_info->num_seg: %d", __func__,
  652. msdu_info->num_seg);
  653. if (!(msdu_info->num_seg)) {
  654. /*
  655. * Free allocated TSO seg desc and number seg desc,
  656. * do unmap for segments if dma map has done.
  657. */
  658. DP_TRACE(ERROR, "%s: Failed to get tso info", __func__);
  659. dp_tx_free_remaining_tso_desc(soc, msdu_info, true);
  660. return QDF_STATUS_E_INVAL;
  661. }
  662. dp_tx_tso_history_add(soc, msdu_info->u.tso_info,
  663. msdu, 0, DP_TX_DESC_MAP);
  664. tso_info->curr_seg = tso_info->tso_seg_list;
  665. tso_info->msdu_stats_idx = dp_tso_get_stats_idx(pdev);
  666. dp_tso_packet_update(pdev, tso_info->msdu_stats_idx,
  667. msdu, msdu_info->num_seg);
  668. dp_tso_segment_stats_update(pdev, tso_info->tso_seg_list,
  669. tso_info->msdu_stats_idx);
  670. dp_stats_tso_segment_histogram_update(pdev, msdu_info->num_seg);
  671. return QDF_STATUS_SUCCESS;
  672. }
  673. #else
  674. static QDF_STATUS dp_tx_prepare_tso(struct dp_vdev *vdev,
  675. qdf_nbuf_t msdu, struct dp_tx_msdu_info_s *msdu_info)
  676. {
  677. return QDF_STATUS_E_NOMEM;
  678. }
  679. #endif
  680. QDF_COMPILE_TIME_ASSERT(dp_tx_htt_metadata_len_check,
  681. (DP_TX_MSDU_INFO_META_DATA_DWORDS * 4 >=
  682. sizeof(struct htt_tx_msdu_desc_ext2_t)));
  683. /**
  684. * dp_tx_prepare_ext_desc() - Allocate and prepare MSDU extension descriptor
  685. * @vdev: DP Vdev handle
  686. * @msdu_info: MSDU info to be setup in MSDU extension descriptor
  687. * @desc_pool_id: Descriptor Pool ID
  688. *
  689. * Return:
  690. */
  691. static
  692. struct dp_tx_ext_desc_elem_s *dp_tx_prepare_ext_desc(struct dp_vdev *vdev,
  693. struct dp_tx_msdu_info_s *msdu_info, uint8_t desc_pool_id)
  694. {
  695. uint8_t i;
  696. uint8_t cached_ext_desc[HAL_TX_EXT_DESC_WITH_META_DATA];
  697. struct dp_tx_seg_info_s *seg_info;
  698. struct dp_tx_ext_desc_elem_s *msdu_ext_desc;
  699. struct dp_soc *soc = vdev->pdev->soc;
  700. /* Allocate an extension descriptor */
  701. msdu_ext_desc = dp_tx_ext_desc_alloc(soc, desc_pool_id);
  702. qdf_mem_zero(&cached_ext_desc[0], HAL_TX_EXT_DESC_WITH_META_DATA);
  703. if (!msdu_ext_desc) {
  704. DP_STATS_INC(vdev, tx_i.dropped.desc_na.num, 1);
  705. return NULL;
  706. }
  707. if (msdu_info->exception_fw &&
  708. qdf_unlikely(vdev->mesh_vdev)) {
  709. qdf_mem_copy(&cached_ext_desc[HAL_TX_EXTENSION_DESC_LEN_BYTES],
  710. &msdu_info->meta_data[0],
  711. sizeof(struct htt_tx_msdu_desc_ext2_t));
  712. qdf_atomic_inc(&soc->num_tx_exception);
  713. msdu_ext_desc->flags |= DP_TX_EXT_DESC_FLAG_METADATA_VALID;
  714. }
  715. switch (msdu_info->frm_type) {
  716. case dp_tx_frm_sg:
  717. case dp_tx_frm_me:
  718. case dp_tx_frm_raw:
  719. seg_info = msdu_info->u.sg_info.curr_seg;
  720. /* Update the buffer pointers in MSDU Extension Descriptor */
  721. for (i = 0; i < seg_info->frag_cnt; i++) {
  722. hal_tx_ext_desc_set_buffer(&cached_ext_desc[0], i,
  723. seg_info->frags[i].paddr_lo,
  724. seg_info->frags[i].paddr_hi,
  725. seg_info->frags[i].len);
  726. }
  727. break;
  728. case dp_tx_frm_tso:
  729. dp_tx_prepare_tso_ext_desc(&msdu_info->u.tso_info.curr_seg->seg,
  730. &cached_ext_desc[0]);
  731. break;
  732. default:
  733. break;
  734. }
  735. QDF_TRACE_HEX_DUMP(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_DEBUG,
  736. cached_ext_desc, HAL_TX_EXT_DESC_WITH_META_DATA);
  737. hal_tx_ext_desc_sync(&cached_ext_desc[0],
  738. msdu_ext_desc->vaddr);
  739. return msdu_ext_desc;
  740. }
  741. /**
  742. * dp_tx_trace_pkt() - Trace TX packet at DP layer
  743. *
  744. * @skb: skb to be traced
  745. * @msdu_id: msdu_id of the packet
  746. * @vdev_id: vdev_id of the packet
  747. *
  748. * Return: None
  749. */
  750. #ifdef DP_DISABLE_TX_PKT_TRACE
  751. static void dp_tx_trace_pkt(struct dp_soc *soc,
  752. qdf_nbuf_t skb, uint16_t msdu_id,
  753. uint8_t vdev_id)
  754. {
  755. }
  756. #else
  757. static void dp_tx_trace_pkt(struct dp_soc *soc,
  758. qdf_nbuf_t skb, uint16_t msdu_id,
  759. uint8_t vdev_id)
  760. {
  761. if (dp_is_tput_high(soc))
  762. return;
  763. QDF_NBUF_CB_TX_PACKET_TRACK(skb) = QDF_NBUF_TX_PKT_DATA_TRACK;
  764. QDF_NBUF_CB_TX_DP_TRACE(skb) = 1;
  765. DPTRACE(qdf_dp_trace_ptr(skb,
  766. QDF_DP_TRACE_LI_DP_TX_PACKET_PTR_RECORD,
  767. QDF_TRACE_DEFAULT_PDEV_ID,
  768. qdf_nbuf_data_addr(skb),
  769. sizeof(qdf_nbuf_data(skb)),
  770. msdu_id, vdev_id, 0));
  771. qdf_dp_trace_log_pkt(vdev_id, skb, QDF_TX, QDF_TRACE_DEFAULT_PDEV_ID);
  772. DPTRACE(qdf_dp_trace_data_pkt(skb, QDF_TRACE_DEFAULT_PDEV_ID,
  773. QDF_DP_TRACE_LI_DP_TX_PACKET_RECORD,
  774. msdu_id, QDF_TX));
  775. }
  776. #endif
  777. #ifdef WLAN_DP_FEATURE_MARK_ICMP_REQ_TO_FW
  778. /**
  779. * dp_tx_is_nbuf_marked_exception() - Check if the packet has been marked as
  780. * exception by the upper layer (OS_IF)
  781. * @soc: DP soc handle
  782. * @nbuf: packet to be transmitted
  783. *
  784. * Returns: 1 if the packet is marked as exception,
  785. * 0, if the packet is not marked as exception.
  786. */
  787. static inline int dp_tx_is_nbuf_marked_exception(struct dp_soc *soc,
  788. qdf_nbuf_t nbuf)
  789. {
  790. return QDF_NBUF_CB_TX_PACKET_TO_FW(nbuf);
  791. }
  792. #else
  793. static inline int dp_tx_is_nbuf_marked_exception(struct dp_soc *soc,
  794. qdf_nbuf_t nbuf)
  795. {
  796. return 0;
  797. }
  798. #endif
  799. #ifdef DP_TRAFFIC_END_INDICATION
  800. /**
  801. * dp_tx_get_traffic_end_indication_pkt() - Allocate and prepare packet to send
  802. * as indication to fw to inform that
  803. * data stream has ended
  804. * @vdev: DP vdev handle
  805. * @nbuf: original buffer from network stack
  806. *
  807. * Return: NULL on failure,
  808. * nbuf on success
  809. */
  810. static inline qdf_nbuf_t
  811. dp_tx_get_traffic_end_indication_pkt(struct dp_vdev *vdev,
  812. qdf_nbuf_t nbuf)
  813. {
  814. /* Packet length should be enough to copy upto L3 header */
  815. uint8_t end_nbuf_len = 64;
  816. uint8_t htt_desc_size_aligned;
  817. uint8_t htt_desc_size;
  818. qdf_nbuf_t end_nbuf;
  819. if (qdf_unlikely(QDF_NBUF_CB_GET_PACKET_TYPE(nbuf) ==
  820. QDF_NBUF_CB_PACKET_TYPE_END_INDICATION)) {
  821. htt_desc_size = sizeof(struct htt_tx_msdu_desc_ext2_t);
  822. htt_desc_size_aligned = (htt_desc_size + 7) & ~0x7;
  823. end_nbuf = qdf_nbuf_queue_remove(&vdev->end_ind_pkt_q);
  824. if (!end_nbuf) {
  825. end_nbuf = qdf_nbuf_alloc(NULL,
  826. (htt_desc_size_aligned +
  827. end_nbuf_len),
  828. htt_desc_size_aligned,
  829. 8, false);
  830. if (!end_nbuf) {
  831. dp_err("Packet allocation failed");
  832. goto out;
  833. }
  834. } else {
  835. qdf_nbuf_reset(end_nbuf, htt_desc_size_aligned, 8);
  836. }
  837. qdf_mem_copy(qdf_nbuf_data(end_nbuf), qdf_nbuf_data(nbuf),
  838. end_nbuf_len);
  839. qdf_nbuf_set_pktlen(end_nbuf, end_nbuf_len);
  840. return end_nbuf;
  841. }
  842. out:
  843. return NULL;
  844. }
  845. /**
  846. * dp_tx_send_traffic_end_indication_pkt() - Send indication packet to FW
  847. * via exception path.
  848. * @vdev: DP vdev handle
  849. * @end_nbuf: skb to send as indication
  850. * @msdu_info: msdu_info of original nbuf
  851. * @peer_id: peer id
  852. *
  853. * Return: None
  854. */
  855. static inline void
  856. dp_tx_send_traffic_end_indication_pkt(struct dp_vdev *vdev,
  857. qdf_nbuf_t end_nbuf,
  858. struct dp_tx_msdu_info_s *msdu_info,
  859. uint16_t peer_id)
  860. {
  861. struct dp_tx_msdu_info_s e_msdu_info = {0};
  862. qdf_nbuf_t nbuf;
  863. struct htt_tx_msdu_desc_ext2_t *desc_ext =
  864. (struct htt_tx_msdu_desc_ext2_t *)(e_msdu_info.meta_data);
  865. e_msdu_info.tx_queue = msdu_info->tx_queue;
  866. e_msdu_info.tid = msdu_info->tid;
  867. e_msdu_info.exception_fw = 1;
  868. desc_ext->host_tx_desc_pool = 1;
  869. desc_ext->traffic_end_indication = 1;
  870. nbuf = dp_tx_send_msdu_single(vdev, end_nbuf, &e_msdu_info,
  871. peer_id, NULL);
  872. if (nbuf) {
  873. dp_err("Traffic end indication packet tx failed");
  874. qdf_nbuf_free(nbuf);
  875. }
  876. }
  877. /**
  878. * dp_tx_traffic_end_indication_set_desc_flag() - Set tx descriptor flag to
  879. * mark it traffic end indication
  880. * packet.
  881. * @tx_desc: Tx descriptor pointer
  882. * @msdu_info: msdu_info structure pointer
  883. *
  884. * Return: None
  885. */
  886. static inline void
  887. dp_tx_traffic_end_indication_set_desc_flag(struct dp_tx_desc_s *tx_desc,
  888. struct dp_tx_msdu_info_s *msdu_info)
  889. {
  890. struct htt_tx_msdu_desc_ext2_t *desc_ext =
  891. (struct htt_tx_msdu_desc_ext2_t *)(msdu_info->meta_data);
  892. if (qdf_unlikely(desc_ext->traffic_end_indication))
  893. tx_desc->flags |= DP_TX_DESC_FLAG_TRAFFIC_END_IND;
  894. }
  895. /**
  896. * dp_tx_traffic_end_indication_enq_ind_pkt() - Enqueue the packet instead of
  897. * freeing which are associated
  898. * with traffic end indication
  899. * flagged descriptor.
  900. * @soc: dp soc handle
  901. * @desc: Tx descriptor pointer
  902. * @nbuf: buffer pointer
  903. *
  904. * Return: True if packet gets enqueued else false
  905. */
  906. static bool
  907. dp_tx_traffic_end_indication_enq_ind_pkt(struct dp_soc *soc,
  908. struct dp_tx_desc_s *desc,
  909. qdf_nbuf_t nbuf)
  910. {
  911. struct dp_vdev *vdev = NULL;
  912. if (qdf_unlikely((desc->flags &
  913. DP_TX_DESC_FLAG_TRAFFIC_END_IND) != 0)) {
  914. vdev = dp_vdev_get_ref_by_id(soc, desc->vdev_id,
  915. DP_MOD_ID_TX_COMP);
  916. if (vdev) {
  917. qdf_nbuf_queue_add(&vdev->end_ind_pkt_q, nbuf);
  918. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_COMP);
  919. return true;
  920. }
  921. }
  922. return false;
  923. }
  924. /**
  925. * dp_tx_traffic_end_indication_is_enabled() - get the feature
  926. * enable/disable status
  927. * @vdev: dp vdev handle
  928. *
  929. * Return: True if feature is enable else false
  930. */
  931. static inline bool
  932. dp_tx_traffic_end_indication_is_enabled(struct dp_vdev *vdev)
  933. {
  934. return qdf_unlikely(vdev->traffic_end_ind_en);
  935. }
  936. static inline qdf_nbuf_t
  937. dp_tx_send_msdu_single_wrapper(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  938. struct dp_tx_msdu_info_s *msdu_info,
  939. uint16_t peer_id, qdf_nbuf_t end_nbuf)
  940. {
  941. if (dp_tx_traffic_end_indication_is_enabled(vdev))
  942. end_nbuf = dp_tx_get_traffic_end_indication_pkt(vdev, nbuf);
  943. nbuf = dp_tx_send_msdu_single(vdev, nbuf, msdu_info, peer_id, NULL);
  944. if (qdf_unlikely(end_nbuf))
  945. dp_tx_send_traffic_end_indication_pkt(vdev, end_nbuf,
  946. msdu_info, peer_id);
  947. return nbuf;
  948. }
  949. #else
  950. static inline qdf_nbuf_t
  951. dp_tx_get_traffic_end_indication_pkt(struct dp_vdev *vdev,
  952. qdf_nbuf_t nbuf)
  953. {
  954. return NULL;
  955. }
  956. static inline void
  957. dp_tx_send_traffic_end_indication_pkt(struct dp_vdev *vdev,
  958. qdf_nbuf_t end_nbuf,
  959. struct dp_tx_msdu_info_s *msdu_info,
  960. uint16_t peer_id)
  961. {}
  962. static inline void
  963. dp_tx_traffic_end_indication_set_desc_flag(struct dp_tx_desc_s *tx_desc,
  964. struct dp_tx_msdu_info_s *msdu_info)
  965. {}
  966. static inline bool
  967. dp_tx_traffic_end_indication_enq_ind_pkt(struct dp_soc *soc,
  968. struct dp_tx_desc_s *desc,
  969. qdf_nbuf_t nbuf)
  970. {
  971. return false;
  972. }
  973. static inline bool
  974. dp_tx_traffic_end_indication_is_enabled(struct dp_vdev *vdev)
  975. {
  976. return false;
  977. }
  978. static inline qdf_nbuf_t
  979. dp_tx_send_msdu_single_wrapper(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  980. struct dp_tx_msdu_info_s *msdu_info,
  981. uint16_t peer_id, qdf_nbuf_t end_nbuf)
  982. {
  983. return dp_tx_send_msdu_single(vdev, nbuf, msdu_info, peer_id, NULL);
  984. }
  985. #endif
  986. #if defined(QCA_SUPPORT_WDS_EXTENDED)
  987. static bool
  988. dp_tx_is_wds_ast_override_en(struct dp_soc *soc,
  989. struct cdp_tx_exception_metadata *tx_exc_metadata)
  990. {
  991. if (soc->features.wds_ext_ast_override_enable &&
  992. tx_exc_metadata && tx_exc_metadata->is_wds_extended)
  993. return true;
  994. return false;
  995. }
  996. #else
  997. static bool
  998. dp_tx_is_wds_ast_override_en(struct dp_soc *soc,
  999. struct cdp_tx_exception_metadata *tx_exc_metadata)
  1000. {
  1001. return false;
  1002. }
  1003. #endif
  1004. /**
  1005. * dp_tx_desc_prepare_single - Allocate and prepare Tx descriptor
  1006. * @vdev: DP vdev handle
  1007. * @nbuf: skb
  1008. * @desc_pool_id: Descriptor pool ID
  1009. * @meta_data: Metadata to the fw
  1010. * @tx_exc_metadata: Handle that holds exception path metadata
  1011. * Allocate and prepare Tx descriptor with msdu information.
  1012. *
  1013. * Return: Pointer to Tx Descriptor on success,
  1014. * NULL on failure
  1015. */
  1016. static
  1017. struct dp_tx_desc_s *dp_tx_prepare_desc_single(struct dp_vdev *vdev,
  1018. qdf_nbuf_t nbuf, uint8_t desc_pool_id,
  1019. struct dp_tx_msdu_info_s *msdu_info,
  1020. struct cdp_tx_exception_metadata *tx_exc_metadata)
  1021. {
  1022. uint8_t align_pad;
  1023. uint8_t is_exception = 0;
  1024. uint8_t htt_hdr_size;
  1025. struct dp_tx_desc_s *tx_desc;
  1026. struct dp_pdev *pdev = vdev->pdev;
  1027. struct dp_soc *soc = pdev->soc;
  1028. if (dp_tx_limit_check(vdev, nbuf))
  1029. return NULL;
  1030. /* Allocate software Tx descriptor */
  1031. tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
  1032. if (qdf_unlikely(!tx_desc)) {
  1033. DP_STATS_INC(vdev, tx_i.dropped.desc_na.num, 1);
  1034. DP_STATS_INC(vdev, tx_i.dropped.desc_na_exc_alloc_fail.num, 1);
  1035. return NULL;
  1036. }
  1037. dp_tx_outstanding_inc(pdev);
  1038. /* Initialize the SW tx descriptor */
  1039. tx_desc->nbuf = nbuf;
  1040. tx_desc->frm_type = dp_tx_frm_std;
  1041. tx_desc->tx_encap_type = ((tx_exc_metadata &&
  1042. (tx_exc_metadata->tx_encap_type != CDP_INVALID_TX_ENCAP_TYPE)) ?
  1043. tx_exc_metadata->tx_encap_type : vdev->tx_encap_type);
  1044. tx_desc->vdev_id = vdev->vdev_id;
  1045. tx_desc->pdev = pdev;
  1046. tx_desc->msdu_ext_desc = NULL;
  1047. tx_desc->pkt_offset = 0;
  1048. tx_desc->length = qdf_nbuf_headlen(nbuf);
  1049. tx_desc->shinfo_addr = skb_end_pointer(nbuf);
  1050. dp_tx_trace_pkt(soc, nbuf, tx_desc->id, vdev->vdev_id);
  1051. if (qdf_unlikely(vdev->multipass_en)) {
  1052. if (!dp_tx_multipass_process(soc, vdev, nbuf, msdu_info))
  1053. goto failure;
  1054. }
  1055. /* Packets marked by upper layer (OS-IF) to be sent to FW */
  1056. if (dp_tx_is_nbuf_marked_exception(soc, nbuf))
  1057. is_exception = 1;
  1058. /* for BE chipsets if wds extension was enbled will not mark FW
  1059. * in desc will mark ast index based search for ast index.
  1060. */
  1061. if (dp_tx_is_wds_ast_override_en(soc, tx_exc_metadata))
  1062. return tx_desc;
  1063. /*
  1064. * For special modes (vdev_type == ocb or mesh), data frames should be
  1065. * transmitted using varying transmit parameters (tx spec) which include
  1066. * transmit rate, power, priority, channel, channel bandwidth , nss etc.
  1067. * These are filled in HTT MSDU descriptor and sent in frame pre-header.
  1068. * These frames are sent as exception packets to firmware.
  1069. *
  1070. * HW requirement is that metadata should always point to a
  1071. * 8-byte aligned address. So we add alignment pad to start of buffer.
  1072. * HTT Metadata should be ensured to be multiple of 8-bytes,
  1073. * to get 8-byte aligned start address along with align_pad added
  1074. *
  1075. * |-----------------------------|
  1076. * | |
  1077. * |-----------------------------| <-----Buffer Pointer Address given
  1078. * | | ^ in HW descriptor (aligned)
  1079. * | HTT Metadata | |
  1080. * | | |
  1081. * | | | Packet Offset given in descriptor
  1082. * | | |
  1083. * |-----------------------------| |
  1084. * | Alignment Pad | v
  1085. * |-----------------------------| <----- Actual buffer start address
  1086. * | SKB Data | (Unaligned)
  1087. * | |
  1088. * | |
  1089. * | |
  1090. * | |
  1091. * | |
  1092. * |-----------------------------|
  1093. */
  1094. if (qdf_unlikely((msdu_info->exception_fw)) ||
  1095. (vdev->opmode == wlan_op_mode_ocb) ||
  1096. (tx_exc_metadata &&
  1097. tx_exc_metadata->is_tx_sniffer)) {
  1098. align_pad = ((unsigned long) qdf_nbuf_data(nbuf)) & 0x7;
  1099. if (qdf_unlikely(qdf_nbuf_headroom(nbuf) < align_pad)) {
  1100. DP_STATS_INC(vdev,
  1101. tx_i.dropped.headroom_insufficient, 1);
  1102. goto failure;
  1103. }
  1104. if (qdf_nbuf_push_head(nbuf, align_pad) == NULL) {
  1105. dp_tx_err("qdf_nbuf_push_head failed");
  1106. goto failure;
  1107. }
  1108. htt_hdr_size = dp_tx_prepare_htt_metadata(vdev, nbuf,
  1109. msdu_info);
  1110. if (htt_hdr_size == 0)
  1111. goto failure;
  1112. tx_desc->length = qdf_nbuf_headlen(nbuf);
  1113. tx_desc->pkt_offset = align_pad + htt_hdr_size;
  1114. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1115. dp_tx_traffic_end_indication_set_desc_flag(tx_desc,
  1116. msdu_info);
  1117. is_exception = 1;
  1118. tx_desc->length -= tx_desc->pkt_offset;
  1119. }
  1120. #if !TQM_BYPASS_WAR
  1121. if (is_exception || tx_exc_metadata)
  1122. #endif
  1123. {
  1124. /* Temporary WAR due to TQM VP issues */
  1125. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1126. qdf_atomic_inc(&soc->num_tx_exception);
  1127. }
  1128. return tx_desc;
  1129. failure:
  1130. dp_tx_desc_release(tx_desc, desc_pool_id);
  1131. return NULL;
  1132. }
  1133. /**
  1134. * dp_tx_prepare_desc() - Allocate and prepare Tx descriptor for multisegment frame
  1135. * @vdev: DP vdev handle
  1136. * @nbuf: skb
  1137. * @msdu_info: Info to be setup in MSDU descriptor and MSDU extension descriptor
  1138. * @desc_pool_id : Descriptor Pool ID
  1139. *
  1140. * Allocate and prepare Tx descriptor with msdu and fragment descritor
  1141. * information. For frames with fragments, allocate and prepare
  1142. * an MSDU extension descriptor
  1143. *
  1144. * Return: Pointer to Tx Descriptor on success,
  1145. * NULL on failure
  1146. */
  1147. static struct dp_tx_desc_s *dp_tx_prepare_desc(struct dp_vdev *vdev,
  1148. qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info,
  1149. uint8_t desc_pool_id)
  1150. {
  1151. struct dp_tx_desc_s *tx_desc;
  1152. struct dp_tx_ext_desc_elem_s *msdu_ext_desc;
  1153. struct dp_pdev *pdev = vdev->pdev;
  1154. struct dp_soc *soc = pdev->soc;
  1155. if (dp_tx_limit_check(vdev, nbuf))
  1156. return NULL;
  1157. /* Allocate software Tx descriptor */
  1158. tx_desc = dp_tx_desc_alloc(soc, desc_pool_id);
  1159. if (!tx_desc) {
  1160. DP_STATS_INC(vdev, tx_i.dropped.desc_na.num, 1);
  1161. return NULL;
  1162. }
  1163. dp_tx_tso_seg_history_add(soc, msdu_info->u.tso_info.curr_seg,
  1164. nbuf, tx_desc->id, DP_TX_DESC_COOKIE);
  1165. dp_tx_outstanding_inc(pdev);
  1166. /* Initialize the SW tx descriptor */
  1167. tx_desc->nbuf = nbuf;
  1168. tx_desc->frm_type = msdu_info->frm_type;
  1169. tx_desc->tx_encap_type = vdev->tx_encap_type;
  1170. tx_desc->vdev_id = vdev->vdev_id;
  1171. tx_desc->pdev = pdev;
  1172. tx_desc->pkt_offset = 0;
  1173. dp_tx_trace_pkt(soc, nbuf, tx_desc->id, vdev->vdev_id);
  1174. /* Handle scattered frames - TSO/SG/ME */
  1175. /* Allocate and prepare an extension descriptor for scattered frames */
  1176. msdu_ext_desc = dp_tx_prepare_ext_desc(vdev, msdu_info, desc_pool_id);
  1177. if (!msdu_ext_desc) {
  1178. dp_tx_info("Tx Extension Descriptor Alloc Fail");
  1179. goto failure;
  1180. }
  1181. #if TQM_BYPASS_WAR
  1182. /* Temporary WAR due to TQM VP issues */
  1183. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1184. qdf_atomic_inc(&soc->num_tx_exception);
  1185. #endif
  1186. if (qdf_unlikely(msdu_info->exception_fw))
  1187. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  1188. tx_desc->msdu_ext_desc = msdu_ext_desc;
  1189. tx_desc->flags |= DP_TX_DESC_FLAG_FRAG;
  1190. msdu_ext_desc->tso_desc = msdu_info->u.tso_info.curr_seg;
  1191. msdu_ext_desc->tso_num_desc = msdu_info->u.tso_info.tso_num_seg_list;
  1192. tx_desc->dma_addr = msdu_ext_desc->paddr;
  1193. if (msdu_ext_desc->flags & DP_TX_EXT_DESC_FLAG_METADATA_VALID)
  1194. tx_desc->length = HAL_TX_EXT_DESC_WITH_META_DATA;
  1195. else
  1196. tx_desc->length = HAL_TX_EXTENSION_DESC_LEN_BYTES;
  1197. return tx_desc;
  1198. failure:
  1199. dp_tx_desc_release(tx_desc, desc_pool_id);
  1200. return NULL;
  1201. }
  1202. /**
  1203. * dp_tx_prepare_raw() - Prepare RAW packet TX
  1204. * @vdev: DP vdev handle
  1205. * @nbuf: buffer pointer
  1206. * @seg_info: Pointer to Segment info Descriptor to be prepared
  1207. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension
  1208. * descriptor
  1209. *
  1210. * Return:
  1211. */
  1212. static qdf_nbuf_t dp_tx_prepare_raw(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  1213. struct dp_tx_seg_info_s *seg_info, struct dp_tx_msdu_info_s *msdu_info)
  1214. {
  1215. qdf_nbuf_t curr_nbuf = NULL;
  1216. uint16_t total_len = 0;
  1217. qdf_dma_addr_t paddr;
  1218. int32_t i;
  1219. int32_t mapped_buf_num = 0;
  1220. struct dp_tx_sg_info_s *sg_info = &msdu_info->u.sg_info;
  1221. qdf_dot3_qosframe_t *qos_wh = (qdf_dot3_qosframe_t *) nbuf->data;
  1222. DP_STATS_INC_PKT(vdev, tx_i.raw.raw_pkt, 1, qdf_nbuf_len(nbuf));
  1223. /* Continue only if frames are of DATA type */
  1224. if (!DP_FRAME_IS_DATA(qos_wh)) {
  1225. DP_STATS_INC(vdev, tx_i.raw.invalid_raw_pkt_datatype, 1);
  1226. dp_tx_debug("Pkt. recd is of not data type");
  1227. goto error;
  1228. }
  1229. /* SWAR for HW: Enable WEP bit in the AMSDU frames for RAW mode */
  1230. if (vdev->raw_mode_war &&
  1231. (qos_wh->i_fc[0] & QDF_IEEE80211_FC0_SUBTYPE_QOS) &&
  1232. (qos_wh->i_qos[0] & IEEE80211_QOS_AMSDU))
  1233. qos_wh->i_fc[1] |= IEEE80211_FC1_WEP;
  1234. for (curr_nbuf = nbuf, i = 0; curr_nbuf;
  1235. curr_nbuf = qdf_nbuf_next(curr_nbuf), i++) {
  1236. /*
  1237. * Number of nbuf's must not exceed the size of the frags
  1238. * array in seg_info.
  1239. */
  1240. if (i >= DP_TX_MAX_NUM_FRAGS) {
  1241. dp_err_rl("nbuf cnt exceeds the max number of segs");
  1242. DP_STATS_INC(vdev, tx_i.raw.num_frags_overflow_err, 1);
  1243. goto error;
  1244. }
  1245. if (QDF_STATUS_SUCCESS !=
  1246. qdf_nbuf_map_nbytes_single(vdev->osdev,
  1247. curr_nbuf,
  1248. QDF_DMA_TO_DEVICE,
  1249. curr_nbuf->len)) {
  1250. dp_tx_err("%s dma map error ", __func__);
  1251. DP_STATS_INC(vdev, tx_i.raw.dma_map_error, 1);
  1252. goto error;
  1253. }
  1254. /* Update the count of mapped nbuf's */
  1255. mapped_buf_num++;
  1256. paddr = qdf_nbuf_get_frag_paddr(curr_nbuf, 0);
  1257. seg_info->frags[i].paddr_lo = paddr;
  1258. seg_info->frags[i].paddr_hi = ((uint64_t)paddr >> 32);
  1259. seg_info->frags[i].len = qdf_nbuf_len(curr_nbuf);
  1260. seg_info->frags[i].vaddr = (void *) curr_nbuf;
  1261. total_len += qdf_nbuf_len(curr_nbuf);
  1262. }
  1263. seg_info->frag_cnt = i;
  1264. seg_info->total_len = total_len;
  1265. seg_info->next = NULL;
  1266. sg_info->curr_seg = seg_info;
  1267. msdu_info->frm_type = dp_tx_frm_raw;
  1268. msdu_info->num_seg = 1;
  1269. return nbuf;
  1270. error:
  1271. i = 0;
  1272. while (nbuf) {
  1273. curr_nbuf = nbuf;
  1274. if (i < mapped_buf_num) {
  1275. qdf_nbuf_unmap_nbytes_single(vdev->osdev, curr_nbuf,
  1276. QDF_DMA_TO_DEVICE,
  1277. curr_nbuf->len);
  1278. i++;
  1279. }
  1280. nbuf = qdf_nbuf_next(nbuf);
  1281. qdf_nbuf_free(curr_nbuf);
  1282. }
  1283. return NULL;
  1284. }
  1285. /**
  1286. * dp_tx_raw_prepare_unset() - unmap the chain of nbufs belonging to RAW frame.
  1287. * @soc: DP soc handle
  1288. * @nbuf: Buffer pointer
  1289. *
  1290. * unmap the chain of nbufs that belong to this RAW frame.
  1291. *
  1292. * Return: None
  1293. */
  1294. static void dp_tx_raw_prepare_unset(struct dp_soc *soc,
  1295. qdf_nbuf_t nbuf)
  1296. {
  1297. qdf_nbuf_t cur_nbuf = nbuf;
  1298. do {
  1299. qdf_nbuf_unmap_nbytes_single(soc->osdev, cur_nbuf,
  1300. QDF_DMA_TO_DEVICE,
  1301. cur_nbuf->len);
  1302. cur_nbuf = qdf_nbuf_next(cur_nbuf);
  1303. } while (cur_nbuf);
  1304. }
  1305. #ifdef VDEV_PEER_PROTOCOL_COUNT
  1306. void dp_vdev_peer_stats_update_protocol_cnt_tx(struct dp_vdev *vdev_hdl,
  1307. qdf_nbuf_t nbuf)
  1308. {
  1309. qdf_nbuf_t nbuf_local;
  1310. struct dp_vdev *vdev_local = vdev_hdl;
  1311. do {
  1312. if (qdf_likely(!((vdev_local)->peer_protocol_count_track)))
  1313. break;
  1314. nbuf_local = nbuf;
  1315. if (qdf_unlikely(((vdev_local)->tx_encap_type) ==
  1316. htt_cmn_pkt_type_raw))
  1317. break;
  1318. else if (qdf_unlikely(qdf_nbuf_is_nonlinear((nbuf_local))))
  1319. break;
  1320. else if (qdf_nbuf_is_tso((nbuf_local)))
  1321. break;
  1322. dp_vdev_peer_stats_update_protocol_cnt((vdev_local),
  1323. (nbuf_local),
  1324. NULL, 1, 0);
  1325. } while (0);
  1326. }
  1327. #endif
  1328. #ifdef WLAN_DP_FEATURE_SW_LATENCY_MGR
  1329. /**
  1330. * dp_tx_update_stats() - Update soc level tx stats
  1331. * @soc: DP soc handle
  1332. * @tx_desc: TX descriptor reference
  1333. * @ring_id: TCL ring id
  1334. *
  1335. * Returns: none
  1336. */
  1337. void dp_tx_update_stats(struct dp_soc *soc,
  1338. struct dp_tx_desc_s *tx_desc,
  1339. uint8_t ring_id)
  1340. {
  1341. uint32_t stats_len = 0;
  1342. if (tx_desc->frm_type == dp_tx_frm_tso)
  1343. stats_len = tx_desc->msdu_ext_desc->tso_desc->seg.total_len;
  1344. else
  1345. stats_len = qdf_nbuf_len(tx_desc->nbuf);
  1346. DP_STATS_INC_PKT(soc, tx.egress[ring_id], 1, stats_len);
  1347. }
  1348. int
  1349. dp_tx_attempt_coalescing(struct dp_soc *soc, struct dp_vdev *vdev,
  1350. struct dp_tx_desc_s *tx_desc,
  1351. uint8_t tid,
  1352. struct dp_tx_msdu_info_s *msdu_info,
  1353. uint8_t ring_id)
  1354. {
  1355. struct dp_swlm *swlm = &soc->swlm;
  1356. union swlm_data swlm_query_data;
  1357. struct dp_swlm_tcl_data tcl_data;
  1358. QDF_STATUS status;
  1359. int ret;
  1360. if (!swlm->is_enabled)
  1361. return msdu_info->skip_hp_update;
  1362. tcl_data.nbuf = tx_desc->nbuf;
  1363. tcl_data.tid = tid;
  1364. tcl_data.ring_id = ring_id;
  1365. if (tx_desc->frm_type == dp_tx_frm_tso) {
  1366. tcl_data.pkt_len =
  1367. tx_desc->msdu_ext_desc->tso_desc->seg.total_len;
  1368. } else {
  1369. tcl_data.pkt_len = qdf_nbuf_len(tx_desc->nbuf);
  1370. }
  1371. tcl_data.num_ll_connections = vdev->num_latency_critical_conn;
  1372. swlm_query_data.tcl_data = &tcl_data;
  1373. status = dp_swlm_tcl_pre_check(soc, &tcl_data);
  1374. if (QDF_IS_STATUS_ERROR(status)) {
  1375. dp_swlm_tcl_reset_session_data(soc, ring_id);
  1376. DP_STATS_INC(swlm, tcl[ring_id].coalesce_fail, 1);
  1377. return 0;
  1378. }
  1379. ret = dp_swlm_query_policy(soc, TCL_DATA, swlm_query_data);
  1380. if (ret) {
  1381. DP_STATS_INC(swlm, tcl[ring_id].coalesce_success, 1);
  1382. } else {
  1383. DP_STATS_INC(swlm, tcl[ring_id].coalesce_fail, 1);
  1384. }
  1385. return ret;
  1386. }
  1387. void
  1388. dp_tx_ring_access_end(struct dp_soc *soc, hal_ring_handle_t hal_ring_hdl,
  1389. int coalesce)
  1390. {
  1391. if (coalesce)
  1392. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1393. else
  1394. dp_tx_hal_ring_access_end(soc, hal_ring_hdl);
  1395. }
  1396. static inline void
  1397. dp_tx_is_hp_update_required(uint32_t i, struct dp_tx_msdu_info_s *msdu_info)
  1398. {
  1399. if (((i + 1) < msdu_info->num_seg))
  1400. msdu_info->skip_hp_update = 1;
  1401. else
  1402. msdu_info->skip_hp_update = 0;
  1403. }
  1404. static inline void
  1405. dp_flush_tcp_hp(struct dp_soc *soc, uint8_t ring_id)
  1406. {
  1407. hal_ring_handle_t hal_ring_hdl =
  1408. dp_tx_get_hal_ring_hdl(soc, ring_id);
  1409. if (dp_tx_hal_ring_access_start(soc, hal_ring_hdl)) {
  1410. dp_err("Fillmore: SRNG access start failed");
  1411. return;
  1412. }
  1413. dp_tx_ring_access_end_wrapper(soc, hal_ring_hdl, 0);
  1414. }
  1415. static inline void
  1416. dp_tx_check_and_flush_hp(struct dp_soc *soc,
  1417. QDF_STATUS status,
  1418. struct dp_tx_msdu_info_s *msdu_info)
  1419. {
  1420. if (QDF_IS_STATUS_ERROR(status) && !msdu_info->skip_hp_update) {
  1421. dp_flush_tcp_hp(soc,
  1422. (msdu_info->tx_queue.ring_id & DP_TX_QUEUE_MASK));
  1423. }
  1424. }
  1425. #else
  1426. static inline void
  1427. dp_tx_is_hp_update_required(uint32_t i, struct dp_tx_msdu_info_s *msdu_info)
  1428. {
  1429. }
  1430. static inline void
  1431. dp_tx_check_and_flush_hp(struct dp_soc *soc,
  1432. QDF_STATUS status,
  1433. struct dp_tx_msdu_info_s *msdu_info)
  1434. {
  1435. }
  1436. #endif
  1437. #ifdef FEATURE_RUNTIME_PM
  1438. static inline int dp_get_rtpm_tput_policy_requirement(struct dp_soc *soc)
  1439. {
  1440. int ret;
  1441. ret = qdf_atomic_read(&soc->rtpm_high_tput_flag) &&
  1442. (hif_rtpm_get_state() <= HIF_RTPM_STATE_ON);
  1443. return ret;
  1444. }
  1445. /**
  1446. * dp_tx_ring_access_end_wrapper() - Wrapper for ring access end
  1447. * @soc: Datapath soc handle
  1448. * @hal_ring_hdl: HAL ring handle
  1449. * @coalesce: Coalesce the current write or not
  1450. *
  1451. * Wrapper for HAL ring access end for data transmission for
  1452. * FEATURE_RUNTIME_PM
  1453. *
  1454. * Returns: none
  1455. */
  1456. void
  1457. dp_tx_ring_access_end_wrapper(struct dp_soc *soc,
  1458. hal_ring_handle_t hal_ring_hdl,
  1459. int coalesce)
  1460. {
  1461. int ret;
  1462. /*
  1463. * Avoid runtime get and put APIs under high throughput scenarios.
  1464. */
  1465. if (dp_get_rtpm_tput_policy_requirement(soc)) {
  1466. dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
  1467. return;
  1468. }
  1469. ret = hif_rtpm_get(HIF_RTPM_GET_ASYNC, HIF_RTPM_ID_DP);
  1470. if (QDF_IS_STATUS_SUCCESS(ret)) {
  1471. if (hif_system_pm_state_check(soc->hif_handle)) {
  1472. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1473. hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
  1474. hal_srng_inc_flush_cnt(hal_ring_hdl);
  1475. } else {
  1476. dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
  1477. }
  1478. hif_rtpm_put(HIF_RTPM_PUT_ASYNC, HIF_RTPM_ID_DP);
  1479. } else {
  1480. dp_runtime_get(soc);
  1481. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1482. hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
  1483. qdf_atomic_inc(&soc->tx_pending_rtpm);
  1484. hal_srng_inc_flush_cnt(hal_ring_hdl);
  1485. dp_runtime_put(soc);
  1486. }
  1487. }
  1488. #else
  1489. #ifdef DP_POWER_SAVE
  1490. void
  1491. dp_tx_ring_access_end_wrapper(struct dp_soc *soc,
  1492. hal_ring_handle_t hal_ring_hdl,
  1493. int coalesce)
  1494. {
  1495. if (hif_system_pm_state_check(soc->hif_handle)) {
  1496. dp_tx_hal_ring_access_end_reap(soc, hal_ring_hdl);
  1497. hal_srng_set_event(hal_ring_hdl, HAL_SRNG_FLUSH_EVENT);
  1498. hal_srng_inc_flush_cnt(hal_ring_hdl);
  1499. } else {
  1500. dp_tx_ring_access_end(soc, hal_ring_hdl, coalesce);
  1501. }
  1502. }
  1503. #endif
  1504. static inline int dp_get_rtpm_tput_policy_requirement(struct dp_soc *soc)
  1505. {
  1506. return 0;
  1507. }
  1508. #endif
  1509. /**
  1510. * dp_tx_get_tid() - Obtain TID to be used for this frame
  1511. * @vdev: DP vdev handle
  1512. * @nbuf: skb
  1513. *
  1514. * Extract the DSCP or PCP information from frame and map into TID value.
  1515. *
  1516. * Return: void
  1517. */
  1518. static void dp_tx_get_tid(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  1519. struct dp_tx_msdu_info_s *msdu_info)
  1520. {
  1521. uint8_t tos = 0, dscp_tid_override = 0;
  1522. uint8_t *hdr_ptr, *L3datap;
  1523. uint8_t is_mcast = 0;
  1524. qdf_ether_header_t *eh = NULL;
  1525. qdf_ethervlan_header_t *evh = NULL;
  1526. uint16_t ether_type;
  1527. qdf_llc_t *llcHdr;
  1528. struct dp_pdev *pdev = (struct dp_pdev *)vdev->pdev;
  1529. DP_TX_TID_OVERRIDE(msdu_info, nbuf);
  1530. if (qdf_likely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
  1531. eh = (qdf_ether_header_t *)nbuf->data;
  1532. hdr_ptr = (uint8_t *)(eh->ether_dhost);
  1533. L3datap = hdr_ptr + sizeof(qdf_ether_header_t);
  1534. } else {
  1535. qdf_dot3_qosframe_t *qos_wh =
  1536. (qdf_dot3_qosframe_t *) nbuf->data;
  1537. msdu_info->tid = qos_wh->i_fc[0] & DP_FC0_SUBTYPE_QOS ?
  1538. qos_wh->i_qos[0] & DP_QOS_TID : 0;
  1539. return;
  1540. }
  1541. is_mcast = DP_FRAME_IS_MULTICAST(hdr_ptr);
  1542. ether_type = eh->ether_type;
  1543. llcHdr = (qdf_llc_t *)(nbuf->data + sizeof(qdf_ether_header_t));
  1544. /*
  1545. * Check if packet is dot3 or eth2 type.
  1546. */
  1547. if (DP_FRAME_IS_LLC(ether_type) && DP_FRAME_IS_SNAP(llcHdr)) {
  1548. ether_type = (uint16_t)*(nbuf->data + 2*QDF_MAC_ADDR_SIZE +
  1549. sizeof(*llcHdr));
  1550. if (ether_type == htons(ETHERTYPE_VLAN)) {
  1551. L3datap = hdr_ptr + sizeof(qdf_ethervlan_header_t) +
  1552. sizeof(*llcHdr);
  1553. ether_type = (uint16_t)*(nbuf->data + 2*QDF_MAC_ADDR_SIZE
  1554. + sizeof(*llcHdr) +
  1555. sizeof(qdf_net_vlanhdr_t));
  1556. } else {
  1557. L3datap = hdr_ptr + sizeof(qdf_ether_header_t) +
  1558. sizeof(*llcHdr);
  1559. }
  1560. } else {
  1561. if (ether_type == htons(ETHERTYPE_VLAN)) {
  1562. evh = (qdf_ethervlan_header_t *) eh;
  1563. ether_type = evh->ether_type;
  1564. L3datap = hdr_ptr + sizeof(qdf_ethervlan_header_t);
  1565. }
  1566. }
  1567. /*
  1568. * Find priority from IP TOS DSCP field
  1569. */
  1570. if (qdf_nbuf_is_ipv4_pkt(nbuf)) {
  1571. qdf_net_iphdr_t *ip = (qdf_net_iphdr_t *) L3datap;
  1572. if (qdf_nbuf_is_ipv4_dhcp_pkt(nbuf)) {
  1573. /* Only for unicast frames */
  1574. if (!is_mcast) {
  1575. /* send it on VO queue */
  1576. msdu_info->tid = DP_VO_TID;
  1577. }
  1578. } else {
  1579. /*
  1580. * IP frame: exclude ECN bits 0-1 and map DSCP bits 2-7
  1581. * from TOS byte.
  1582. */
  1583. tos = ip->ip_tos;
  1584. dscp_tid_override = 1;
  1585. }
  1586. } else if (qdf_nbuf_is_ipv6_pkt(nbuf)) {
  1587. /* TODO
  1588. * use flowlabel
  1589. *igmpmld cases to be handled in phase 2
  1590. */
  1591. unsigned long ver_pri_flowlabel;
  1592. unsigned long pri;
  1593. ver_pri_flowlabel = *(unsigned long *) L3datap;
  1594. pri = (ntohl(ver_pri_flowlabel) & IPV6_FLOWINFO_PRIORITY) >>
  1595. DP_IPV6_PRIORITY_SHIFT;
  1596. tos = pri;
  1597. dscp_tid_override = 1;
  1598. } else if (qdf_nbuf_is_ipv4_eapol_pkt(nbuf))
  1599. msdu_info->tid = DP_VO_TID;
  1600. else if (qdf_nbuf_is_ipv4_arp_pkt(nbuf)) {
  1601. /* Only for unicast frames */
  1602. if (!is_mcast) {
  1603. /* send ucast arp on VO queue */
  1604. msdu_info->tid = DP_VO_TID;
  1605. }
  1606. }
  1607. /*
  1608. * Assign all MCAST packets to BE
  1609. */
  1610. if (qdf_unlikely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
  1611. if (is_mcast) {
  1612. tos = 0;
  1613. dscp_tid_override = 1;
  1614. }
  1615. }
  1616. if (dscp_tid_override == 1) {
  1617. tos = (tos >> DP_IP_DSCP_SHIFT) & DP_IP_DSCP_MASK;
  1618. msdu_info->tid = pdev->dscp_tid_map[vdev->dscp_tid_map_id][tos];
  1619. }
  1620. if (msdu_info->tid >= CDP_MAX_DATA_TIDS)
  1621. msdu_info->tid = CDP_MAX_DATA_TIDS - 1;
  1622. return;
  1623. }
  1624. /**
  1625. * dp_tx_classify_tid() - Obtain TID to be used for this frame
  1626. * @vdev: DP vdev handle
  1627. * @nbuf: skb
  1628. *
  1629. * Software based TID classification is required when more than 2 DSCP-TID
  1630. * mapping tables are needed.
  1631. * Hardware supports 2 DSCP-TID mapping tables for HKv1 and 48 for HKv2.
  1632. *
  1633. * Return: void
  1634. */
  1635. static inline void dp_tx_classify_tid(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  1636. struct dp_tx_msdu_info_s *msdu_info)
  1637. {
  1638. DP_TX_TID_OVERRIDE(msdu_info, nbuf);
  1639. /*
  1640. * skip_sw_tid_classification flag will set in below cases-
  1641. * 1. vdev->dscp_tid_map_id < pdev->soc->num_hw_dscp_tid_map
  1642. * 2. hlos_tid_override enabled for vdev
  1643. * 3. mesh mode enabled for vdev
  1644. */
  1645. if (qdf_likely(vdev->skip_sw_tid_classification)) {
  1646. /* Update tid in msdu_info from skb priority */
  1647. if (qdf_unlikely(vdev->skip_sw_tid_classification
  1648. & DP_TXRX_HLOS_TID_OVERRIDE_ENABLED)) {
  1649. uint32_t tid = qdf_nbuf_get_priority(nbuf);
  1650. if (tid == DP_TX_INVALID_QOS_TAG)
  1651. return;
  1652. msdu_info->tid = tid;
  1653. return;
  1654. }
  1655. return;
  1656. }
  1657. dp_tx_get_tid(vdev, nbuf, msdu_info);
  1658. }
  1659. #ifdef FEATURE_WLAN_TDLS
  1660. /**
  1661. * dp_tx_update_tdls_flags() - Update descriptor flags for TDLS frame
  1662. * @soc: datapath SOC
  1663. * @vdev: datapath vdev
  1664. * @tx_desc: TX descriptor
  1665. *
  1666. * Return: None
  1667. */
  1668. static void dp_tx_update_tdls_flags(struct dp_soc *soc,
  1669. struct dp_vdev *vdev,
  1670. struct dp_tx_desc_s *tx_desc)
  1671. {
  1672. if (vdev) {
  1673. if (vdev->is_tdls_frame) {
  1674. tx_desc->flags |= DP_TX_DESC_FLAG_TDLS_FRAME;
  1675. vdev->is_tdls_frame = false;
  1676. }
  1677. }
  1678. }
  1679. static uint8_t dp_htt_tx_comp_get_status(struct dp_soc *soc, char *htt_desc)
  1680. {
  1681. uint8_t tx_status = HTT_TX_FW2WBM_TX_STATUS_MAX;
  1682. switch (soc->arch_id) {
  1683. case CDP_ARCH_TYPE_LI:
  1684. tx_status = HTT_TX_WBM_COMPLETION_V2_TX_STATUS_GET(htt_desc[0]);
  1685. break;
  1686. case CDP_ARCH_TYPE_BE:
  1687. tx_status = HTT_TX_WBM_COMPLETION_V3_TX_STATUS_GET(htt_desc[0]);
  1688. break;
  1689. default:
  1690. dp_err("Incorrect CDP_ARCH %d", soc->arch_id);
  1691. QDF_BUG(0);
  1692. }
  1693. return tx_status;
  1694. }
  1695. /**
  1696. * dp_non_std_htt_tx_comp_free_buff() - Free the non std tx packet buffer
  1697. * @soc: dp_soc handle
  1698. * @tx_desc: TX descriptor
  1699. * @vdev: datapath vdev handle
  1700. *
  1701. * Return: None
  1702. */
  1703. static void dp_non_std_htt_tx_comp_free_buff(struct dp_soc *soc,
  1704. struct dp_tx_desc_s *tx_desc)
  1705. {
  1706. uint8_t tx_status = 0;
  1707. uint8_t htt_tx_status[HAL_TX_COMP_HTT_STATUS_LEN];
  1708. qdf_nbuf_t nbuf = tx_desc->nbuf;
  1709. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id,
  1710. DP_MOD_ID_TDLS);
  1711. if (qdf_unlikely(!vdev)) {
  1712. dp_err_rl("vdev is null!");
  1713. goto error;
  1714. }
  1715. hal_tx_comp_get_htt_desc(&tx_desc->comp, htt_tx_status);
  1716. tx_status = dp_htt_tx_comp_get_status(soc, htt_tx_status);
  1717. dp_debug("vdev_id: %d tx_status: %d", tx_desc->vdev_id, tx_status);
  1718. if (vdev->tx_non_std_data_callback.func) {
  1719. qdf_nbuf_set_next(nbuf, NULL);
  1720. vdev->tx_non_std_data_callback.func(
  1721. vdev->tx_non_std_data_callback.ctxt,
  1722. nbuf, tx_status);
  1723. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
  1724. return;
  1725. } else {
  1726. dp_err_rl("callback func is null");
  1727. }
  1728. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
  1729. error:
  1730. qdf_nbuf_unmap_single(soc->osdev, nbuf, QDF_DMA_TO_DEVICE);
  1731. qdf_nbuf_free(nbuf);
  1732. }
  1733. /**
  1734. * dp_tx_msdu_single_map() - do nbuf map
  1735. * @vdev: DP vdev handle
  1736. * @tx_desc: DP TX descriptor pointer
  1737. * @nbuf: skb pointer
  1738. *
  1739. * For TDLS frame, use qdf_nbuf_map_single() to align with the unmap
  1740. * operation done in other component.
  1741. *
  1742. * Return: QDF_STATUS
  1743. */
  1744. static inline QDF_STATUS dp_tx_msdu_single_map(struct dp_vdev *vdev,
  1745. struct dp_tx_desc_s *tx_desc,
  1746. qdf_nbuf_t nbuf)
  1747. {
  1748. if (qdf_likely(!(tx_desc->flags & DP_TX_DESC_FLAG_TDLS_FRAME)))
  1749. return qdf_nbuf_map_nbytes_single(vdev->osdev,
  1750. nbuf,
  1751. QDF_DMA_TO_DEVICE,
  1752. nbuf->len);
  1753. else
  1754. return qdf_nbuf_map_single(vdev->osdev, nbuf,
  1755. QDF_DMA_TO_DEVICE);
  1756. }
  1757. #else
  1758. static inline void dp_tx_update_tdls_flags(struct dp_soc *soc,
  1759. struct dp_vdev *vdev,
  1760. struct dp_tx_desc_s *tx_desc)
  1761. {
  1762. }
  1763. static inline void dp_non_std_htt_tx_comp_free_buff(struct dp_soc *soc,
  1764. struct dp_tx_desc_s *tx_desc)
  1765. {
  1766. }
  1767. static inline QDF_STATUS dp_tx_msdu_single_map(struct dp_vdev *vdev,
  1768. struct dp_tx_desc_s *tx_desc,
  1769. qdf_nbuf_t nbuf)
  1770. {
  1771. return qdf_nbuf_map_nbytes_single(vdev->osdev,
  1772. nbuf,
  1773. QDF_DMA_TO_DEVICE,
  1774. nbuf->len);
  1775. }
  1776. #endif
  1777. static inline
  1778. qdf_dma_addr_t dp_tx_nbuf_map_regular(struct dp_vdev *vdev,
  1779. struct dp_tx_desc_s *tx_desc,
  1780. qdf_nbuf_t nbuf)
  1781. {
  1782. QDF_STATUS ret = QDF_STATUS_E_FAILURE;
  1783. ret = dp_tx_msdu_single_map(vdev, tx_desc, nbuf);
  1784. if (qdf_unlikely(QDF_IS_STATUS_ERROR(ret)))
  1785. return 0;
  1786. return qdf_nbuf_mapped_paddr_get(nbuf);
  1787. }
  1788. static inline
  1789. void dp_tx_nbuf_unmap_regular(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1790. {
  1791. qdf_nbuf_unmap_nbytes_single_paddr(soc->osdev,
  1792. desc->nbuf,
  1793. desc->dma_addr,
  1794. QDF_DMA_TO_DEVICE,
  1795. desc->length);
  1796. }
  1797. #ifdef QCA_DP_TX_RMNET_OPTIMIZATION
  1798. static inline bool
  1799. is_nbuf_frm_rmnet(qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info)
  1800. {
  1801. struct net_device *ingress_dev;
  1802. skb_frag_t *frag;
  1803. uint16_t buf_len = 0;
  1804. uint16_t linear_data_len = 0;
  1805. uint8_t *payload_addr = NULL;
  1806. ingress_dev = dev_get_by_index(dev_net(nbuf->dev), nbuf->skb_iif);
  1807. if ((ingress_dev->priv_flags & IFF_PHONY_HEADROOM)) {
  1808. dev_put(ingress_dev);
  1809. frag = &(skb_shinfo(nbuf)->frags[0]);
  1810. buf_len = skb_frag_size(frag);
  1811. payload_addr = (uint8_t *)skb_frag_address(frag);
  1812. linear_data_len = skb_headlen(nbuf);
  1813. buf_len += linear_data_len;
  1814. payload_addr = payload_addr - linear_data_len;
  1815. memcpy(payload_addr, nbuf->data, linear_data_len);
  1816. msdu_info->frm_type = dp_tx_frm_rmnet;
  1817. msdu_info->buf_len = buf_len;
  1818. msdu_info->payload_addr = payload_addr;
  1819. return true;
  1820. }
  1821. dev_put(ingress_dev);
  1822. return false;
  1823. }
  1824. static inline
  1825. qdf_dma_addr_t dp_tx_rmnet_nbuf_map(struct dp_tx_msdu_info_s *msdu_info,
  1826. struct dp_tx_desc_s *tx_desc)
  1827. {
  1828. qdf_dma_addr_t paddr;
  1829. paddr = (qdf_dma_addr_t)qdf_mem_virt_to_phys(msdu_info->payload_addr);
  1830. tx_desc->length = msdu_info->buf_len;
  1831. qdf_nbuf_dma_clean_range((void *)msdu_info->payload_addr,
  1832. (void *)(msdu_info->payload_addr +
  1833. msdu_info->buf_len));
  1834. tx_desc->flags |= DP_TX_DESC_FLAG_RMNET;
  1835. return paddr;
  1836. }
  1837. #else
  1838. static inline bool
  1839. is_nbuf_frm_rmnet(qdf_nbuf_t nbuf, struct dp_tx_msdu_info_s *msdu_info)
  1840. {
  1841. return false;
  1842. }
  1843. static inline
  1844. qdf_dma_addr_t dp_tx_rmnet_nbuf_map(struct dp_tx_msdu_info_s *msdu_info,
  1845. struct dp_tx_desc_s *tx_desc)
  1846. {
  1847. return 0;
  1848. }
  1849. #endif
  1850. #if defined(QCA_DP_TX_NBUF_NO_MAP_UNMAP) && !defined(BUILD_X86)
  1851. static inline
  1852. qdf_dma_addr_t dp_tx_nbuf_map(struct dp_vdev *vdev,
  1853. struct dp_tx_desc_s *tx_desc,
  1854. qdf_nbuf_t nbuf)
  1855. {
  1856. if (qdf_likely(tx_desc->flags & DP_TX_DESC_FLAG_SIMPLE)) {
  1857. qdf_nbuf_dma_clean_range((void *)nbuf->data,
  1858. (void *)(nbuf->data + nbuf->len));
  1859. return (qdf_dma_addr_t)qdf_mem_virt_to_phys(nbuf->data);
  1860. } else {
  1861. return dp_tx_nbuf_map_regular(vdev, tx_desc, nbuf);
  1862. }
  1863. }
  1864. static inline
  1865. void dp_tx_nbuf_unmap(struct dp_soc *soc,
  1866. struct dp_tx_desc_s *desc)
  1867. {
  1868. if (qdf_unlikely(!(desc->flags &
  1869. (DP_TX_DESC_FLAG_SIMPLE | DP_TX_DESC_FLAG_RMNET))))
  1870. return dp_tx_nbuf_unmap_regular(soc, desc);
  1871. }
  1872. #else
  1873. static inline
  1874. qdf_dma_addr_t dp_tx_nbuf_map(struct dp_vdev *vdev,
  1875. struct dp_tx_desc_s *tx_desc,
  1876. qdf_nbuf_t nbuf)
  1877. {
  1878. return dp_tx_nbuf_map_regular(vdev, tx_desc, nbuf);
  1879. }
  1880. static inline
  1881. void dp_tx_nbuf_unmap(struct dp_soc *soc,
  1882. struct dp_tx_desc_s *desc)
  1883. {
  1884. return dp_tx_nbuf_unmap_regular(soc, desc);
  1885. }
  1886. #endif
  1887. #if defined(WLAN_TX_PKT_CAPTURE_ENH) || defined(FEATURE_PERPKT_INFO)
  1888. static inline
  1889. void dp_tx_enh_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1890. {
  1891. dp_tx_nbuf_unmap(soc, desc);
  1892. desc->flags |= DP_TX_DESC_FLAG_UNMAP_DONE;
  1893. }
  1894. static inline void dp_tx_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1895. {
  1896. if (qdf_likely(!(desc->flags & DP_TX_DESC_FLAG_UNMAP_DONE)))
  1897. dp_tx_nbuf_unmap(soc, desc);
  1898. }
  1899. #else
  1900. static inline
  1901. void dp_tx_enh_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1902. {
  1903. }
  1904. static inline void dp_tx_unmap(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  1905. {
  1906. dp_tx_nbuf_unmap(soc, desc);
  1907. }
  1908. #endif
  1909. #ifdef MESH_MODE_SUPPORT
  1910. /**
  1911. * dp_tx_update_mesh_flags() - Update descriptor flags for mesh VAP
  1912. * @soc: datapath SOC
  1913. * @vdev: datapath vdev
  1914. * @tx_desc: TX descriptor
  1915. *
  1916. * Return: None
  1917. */
  1918. static inline void dp_tx_update_mesh_flags(struct dp_soc *soc,
  1919. struct dp_vdev *vdev,
  1920. struct dp_tx_desc_s *tx_desc)
  1921. {
  1922. if (qdf_unlikely(vdev->mesh_vdev))
  1923. tx_desc->flags |= DP_TX_DESC_FLAG_MESH_MODE;
  1924. }
  1925. /**
  1926. * dp_mesh_tx_comp_free_buff() - Free the mesh tx packet buffer
  1927. * @soc: dp_soc handle
  1928. * @tx_desc: TX descriptor
  1929. * @delayed_free: delay the nbuf free
  1930. *
  1931. * Return: nbuf to be freed late
  1932. */
  1933. static inline qdf_nbuf_t dp_mesh_tx_comp_free_buff(struct dp_soc *soc,
  1934. struct dp_tx_desc_s *tx_desc,
  1935. bool delayed_free)
  1936. {
  1937. qdf_nbuf_t nbuf = tx_desc->nbuf;
  1938. struct dp_vdev *vdev = NULL;
  1939. vdev = dp_vdev_get_ref_by_id(soc, tx_desc->vdev_id, DP_MOD_ID_MESH);
  1940. if (tx_desc->flags & DP_TX_DESC_FLAG_TO_FW) {
  1941. if (vdev)
  1942. DP_STATS_INC(vdev, tx_i.mesh.completion_fw, 1);
  1943. if (delayed_free)
  1944. return nbuf;
  1945. qdf_nbuf_free(nbuf);
  1946. } else {
  1947. if (vdev && vdev->osif_tx_free_ext) {
  1948. vdev->osif_tx_free_ext((nbuf));
  1949. } else {
  1950. if (delayed_free)
  1951. return nbuf;
  1952. qdf_nbuf_free(nbuf);
  1953. }
  1954. }
  1955. if (vdev)
  1956. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
  1957. return NULL;
  1958. }
  1959. #else
  1960. static inline void dp_tx_update_mesh_flags(struct dp_soc *soc,
  1961. struct dp_vdev *vdev,
  1962. struct dp_tx_desc_s *tx_desc)
  1963. {
  1964. }
  1965. static inline qdf_nbuf_t dp_mesh_tx_comp_free_buff(struct dp_soc *soc,
  1966. struct dp_tx_desc_s *tx_desc,
  1967. bool delayed_free)
  1968. {
  1969. return NULL;
  1970. }
  1971. #endif
  1972. /**
  1973. * dp_tx_frame_is_drop() - checks if the packet is loopback
  1974. * @vdev: DP vdev handle
  1975. * @nbuf: skb
  1976. *
  1977. * Return: 1 if frame needs to be dropped else 0
  1978. */
  1979. int dp_tx_frame_is_drop(struct dp_vdev *vdev, uint8_t *srcmac, uint8_t *dstmac)
  1980. {
  1981. struct dp_pdev *pdev = NULL;
  1982. struct dp_ast_entry *src_ast_entry = NULL;
  1983. struct dp_ast_entry *dst_ast_entry = NULL;
  1984. struct dp_soc *soc = NULL;
  1985. qdf_assert(vdev);
  1986. pdev = vdev->pdev;
  1987. qdf_assert(pdev);
  1988. soc = pdev->soc;
  1989. dst_ast_entry = dp_peer_ast_hash_find_by_pdevid
  1990. (soc, dstmac, vdev->pdev->pdev_id);
  1991. src_ast_entry = dp_peer_ast_hash_find_by_pdevid
  1992. (soc, srcmac, vdev->pdev->pdev_id);
  1993. if (dst_ast_entry && src_ast_entry) {
  1994. if (dst_ast_entry->peer_id ==
  1995. src_ast_entry->peer_id)
  1996. return 1;
  1997. }
  1998. return 0;
  1999. }
  2000. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  2001. defined(WLAN_MCAST_MLO)
  2002. /* MLO peer id for reinject*/
  2003. #define DP_MLO_MCAST_REINJECT_PEER_ID 0XFFFD
  2004. /* MLO vdev id inc offset */
  2005. #define DP_MLO_VDEV_ID_OFFSET 0x80
  2006. static inline void
  2007. dp_tx_bypass_reinjection(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
  2008. {
  2009. if (!(tx_desc->flags & DP_TX_DESC_FLAG_TO_FW)) {
  2010. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  2011. qdf_atomic_inc(&soc->num_tx_exception);
  2012. }
  2013. }
  2014. static inline void
  2015. dp_tx_update_mcast_param(uint16_t peer_id,
  2016. uint16_t *htt_tcl_metadata,
  2017. struct dp_vdev *vdev,
  2018. struct dp_tx_msdu_info_s *msdu_info)
  2019. {
  2020. if (peer_id == DP_MLO_MCAST_REINJECT_PEER_ID) {
  2021. *htt_tcl_metadata = 0;
  2022. DP_TX_TCL_METADATA_TYPE_SET(
  2023. *htt_tcl_metadata,
  2024. HTT_TCL_METADATA_V2_TYPE_GLOBAL_SEQ_BASED);
  2025. HTT_TX_TCL_METADATA_GLBL_SEQ_NO_SET(*htt_tcl_metadata,
  2026. msdu_info->gsn);
  2027. msdu_info->vdev_id = vdev->vdev_id + DP_MLO_VDEV_ID_OFFSET;
  2028. if (qdf_unlikely(vdev->nawds_enabled))
  2029. HTT_TX_TCL_METADATA_GLBL_SEQ_HOST_INSPECTED_SET(
  2030. *htt_tcl_metadata, 1);
  2031. } else {
  2032. msdu_info->vdev_id = vdev->vdev_id;
  2033. }
  2034. }
  2035. #else
  2036. static inline void
  2037. dp_tx_bypass_reinjection(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
  2038. {
  2039. }
  2040. static inline void
  2041. dp_tx_update_mcast_param(uint16_t peer_id,
  2042. uint16_t *htt_tcl_metadata,
  2043. struct dp_vdev *vdev,
  2044. struct dp_tx_msdu_info_s *msdu_info)
  2045. {
  2046. }
  2047. #endif
  2048. #ifdef DP_TX_SW_DROP_STATS_INC
  2049. static void tx_sw_drop_stats_inc(struct dp_pdev *pdev,
  2050. qdf_nbuf_t nbuf,
  2051. enum cdp_tx_sw_drop drop_code)
  2052. {
  2053. /* EAPOL Drop stats */
  2054. if (qdf_nbuf_is_ipv4_eapol_pkt(nbuf)) {
  2055. switch (drop_code) {
  2056. case TX_DESC_ERR:
  2057. DP_STATS_INC(pdev, eap_drop_stats.tx_desc_err, 1);
  2058. break;
  2059. case TX_HAL_RING_ACCESS_ERR:
  2060. DP_STATS_INC(pdev,
  2061. eap_drop_stats.tx_hal_ring_access_err, 1);
  2062. break;
  2063. case TX_DMA_MAP_ERR:
  2064. DP_STATS_INC(pdev, eap_drop_stats.tx_dma_map_err, 1);
  2065. break;
  2066. case TX_HW_ENQUEUE:
  2067. DP_STATS_INC(pdev, eap_drop_stats.tx_hw_enqueue, 1);
  2068. break;
  2069. case TX_SW_ENQUEUE:
  2070. DP_STATS_INC(pdev, eap_drop_stats.tx_sw_enqueue, 1);
  2071. break;
  2072. default:
  2073. dp_info_rl("Invalid eapol_drop code: %d", drop_code);
  2074. break;
  2075. }
  2076. }
  2077. }
  2078. #else
  2079. static void tx_sw_drop_stats_inc(struct dp_pdev *pdev,
  2080. qdf_nbuf_t nbuf,
  2081. enum cdp_tx_sw_drop drop_code)
  2082. {
  2083. }
  2084. #endif
  2085. /**
  2086. * dp_tx_send_msdu_single() - Setup descriptor and enqueue single MSDU to TCL
  2087. * @vdev: DP vdev handle
  2088. * @nbuf: skb
  2089. * @tid: TID from HLOS for overriding default DSCP-TID mapping
  2090. * @meta_data: Metadata to the fw
  2091. * @tx_q: Tx queue to be used for this Tx frame
  2092. * @peer_id: peer_id of the peer in case of NAWDS frames
  2093. * @tx_exc_metadata: Handle that holds exception path metadata
  2094. *
  2095. * Return: NULL on success,
  2096. * nbuf when it fails to send
  2097. */
  2098. qdf_nbuf_t
  2099. dp_tx_send_msdu_single(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2100. struct dp_tx_msdu_info_s *msdu_info, uint16_t peer_id,
  2101. struct cdp_tx_exception_metadata *tx_exc_metadata)
  2102. {
  2103. struct dp_pdev *pdev = vdev->pdev;
  2104. struct dp_soc *soc = pdev->soc;
  2105. struct dp_tx_desc_s *tx_desc;
  2106. QDF_STATUS status;
  2107. struct dp_tx_queue *tx_q = &(msdu_info->tx_queue);
  2108. uint16_t htt_tcl_metadata = 0;
  2109. enum cdp_tx_sw_drop drop_code = TX_MAX_DROP;
  2110. uint8_t tid = msdu_info->tid;
  2111. struct cdp_tid_tx_stats *tid_stats = NULL;
  2112. qdf_dma_addr_t paddr;
  2113. /* Setup Tx descriptor for an MSDU, and MSDU extension descriptor */
  2114. tx_desc = dp_tx_prepare_desc_single(vdev, nbuf, tx_q->desc_pool_id,
  2115. msdu_info, tx_exc_metadata);
  2116. if (!tx_desc) {
  2117. dp_err_rl("Tx_desc prepare Fail vdev_id %d vdev %pK queue %d",
  2118. vdev->vdev_id, vdev, tx_q->desc_pool_id);
  2119. drop_code = TX_DESC_ERR;
  2120. goto fail_return;
  2121. }
  2122. dp_tx_update_tdls_flags(soc, vdev, tx_desc);
  2123. if (qdf_unlikely(peer_id == DP_INVALID_PEER)) {
  2124. htt_tcl_metadata = vdev->htt_tcl_metadata;
  2125. DP_TX_TCL_METADATA_HOST_INSPECTED_SET(htt_tcl_metadata, 1);
  2126. } else if (qdf_unlikely(peer_id != HTT_INVALID_PEER)) {
  2127. DP_TX_TCL_METADATA_TYPE_SET(htt_tcl_metadata,
  2128. DP_TCL_METADATA_TYPE_PEER_BASED);
  2129. DP_TX_TCL_METADATA_PEER_ID_SET(htt_tcl_metadata,
  2130. peer_id);
  2131. dp_tx_bypass_reinjection(soc, tx_desc);
  2132. } else
  2133. htt_tcl_metadata = vdev->htt_tcl_metadata;
  2134. if (msdu_info->exception_fw)
  2135. DP_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1);
  2136. dp_tx_desc_update_fast_comp_flag(soc, tx_desc,
  2137. !pdev->enhanced_stats_en);
  2138. dp_tx_update_mesh_flags(soc, vdev, tx_desc);
  2139. if (qdf_unlikely(msdu_info->frm_type == dp_tx_frm_rmnet))
  2140. paddr = dp_tx_rmnet_nbuf_map(msdu_info, tx_desc);
  2141. else
  2142. paddr = dp_tx_nbuf_map(vdev, tx_desc, nbuf);
  2143. if (!paddr) {
  2144. /* Handle failure */
  2145. dp_err("qdf_nbuf_map failed");
  2146. DP_STATS_INC(vdev, tx_i.dropped.dma_error, 1);
  2147. drop_code = TX_DMA_MAP_ERR;
  2148. goto release_desc;
  2149. }
  2150. tx_desc->dma_addr = paddr;
  2151. dp_tx_desc_history_add(soc, tx_desc->dma_addr, nbuf,
  2152. tx_desc->id, DP_TX_DESC_MAP);
  2153. dp_tx_update_mcast_param(peer_id, &htt_tcl_metadata, vdev, msdu_info);
  2154. /* Enqueue the Tx MSDU descriptor to HW for transmit */
  2155. status = soc->arch_ops.tx_hw_enqueue(soc, vdev, tx_desc,
  2156. htt_tcl_metadata,
  2157. tx_exc_metadata, msdu_info);
  2158. if (status != QDF_STATUS_SUCCESS) {
  2159. dp_tx_err_rl("Tx_hw_enqueue Fail tx_desc %pK queue %d",
  2160. tx_desc, tx_q->ring_id);
  2161. dp_tx_desc_history_add(soc, tx_desc->dma_addr, nbuf,
  2162. tx_desc->id, DP_TX_DESC_UNMAP);
  2163. dp_tx_nbuf_unmap(soc, tx_desc);
  2164. drop_code = TX_HW_ENQUEUE;
  2165. goto release_desc;
  2166. }
  2167. tx_sw_drop_stats_inc(pdev, nbuf, drop_code);
  2168. return NULL;
  2169. release_desc:
  2170. dp_tx_desc_release(tx_desc, tx_q->desc_pool_id);
  2171. fail_return:
  2172. dp_tx_get_tid(vdev, nbuf, msdu_info);
  2173. tx_sw_drop_stats_inc(pdev, nbuf, drop_code);
  2174. tid_stats = &pdev->stats.tid_stats.
  2175. tid_tx_stats[tx_q->ring_id][tid];
  2176. tid_stats->swdrop_cnt[drop_code]++;
  2177. return nbuf;
  2178. }
  2179. /**
  2180. * dp_tdls_tx_comp_free_buff() - Free non std buffer when TDLS flag is set
  2181. * @soc: Soc handle
  2182. * @desc: software Tx descriptor to be processed
  2183. *
  2184. * Return: 0 if Success
  2185. */
  2186. #ifdef FEATURE_WLAN_TDLS
  2187. static inline int
  2188. dp_tdls_tx_comp_free_buff(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  2189. {
  2190. /* If it is TDLS mgmt, don't unmap or free the frame */
  2191. if (desc->flags & DP_TX_DESC_FLAG_TDLS_FRAME) {
  2192. dp_non_std_htt_tx_comp_free_buff(soc, desc);
  2193. return 0;
  2194. }
  2195. return 1;
  2196. }
  2197. #else
  2198. static inline int
  2199. dp_tdls_tx_comp_free_buff(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  2200. {
  2201. return 1;
  2202. }
  2203. #endif
  2204. /**
  2205. * dp_tx_comp_free_buf() - Free nbuf associated with the Tx Descriptor
  2206. * @soc: Soc handle
  2207. * @desc: software Tx descriptor to be processed
  2208. * @delayed_free: defer freeing of nbuf
  2209. *
  2210. * Return: nbuf to be freed later
  2211. */
  2212. qdf_nbuf_t dp_tx_comp_free_buf(struct dp_soc *soc, struct dp_tx_desc_s *desc,
  2213. bool delayed_free)
  2214. {
  2215. qdf_nbuf_t nbuf = desc->nbuf;
  2216. enum dp_tx_event_type type = dp_tx_get_event_type(desc->flags);
  2217. /* nbuf already freed in vdev detach path */
  2218. if (!nbuf)
  2219. return NULL;
  2220. if (!dp_tdls_tx_comp_free_buff(soc, desc))
  2221. return NULL;
  2222. /* 0 : MSDU buffer, 1 : MLE */
  2223. if (desc->msdu_ext_desc) {
  2224. /* TSO free */
  2225. if (hal_tx_ext_desc_get_tso_enable(
  2226. desc->msdu_ext_desc->vaddr)) {
  2227. dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf,
  2228. desc->id, DP_TX_COMP_MSDU_EXT);
  2229. dp_tx_tso_seg_history_add(soc,
  2230. desc->msdu_ext_desc->tso_desc,
  2231. desc->nbuf, desc->id, type);
  2232. /* unmap eash TSO seg before free the nbuf */
  2233. dp_tx_tso_unmap_segment(soc,
  2234. desc->msdu_ext_desc->tso_desc,
  2235. desc->msdu_ext_desc->
  2236. tso_num_desc);
  2237. goto nbuf_free;
  2238. }
  2239. if (qdf_unlikely(desc->frm_type == dp_tx_frm_sg)) {
  2240. void *msdu_ext_desc = desc->msdu_ext_desc->vaddr;
  2241. qdf_dma_addr_t iova;
  2242. uint32_t frag_len;
  2243. uint32_t i;
  2244. qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf,
  2245. QDF_DMA_TO_DEVICE,
  2246. qdf_nbuf_headlen(nbuf));
  2247. for (i = 1; i < DP_TX_MAX_NUM_FRAGS; i++) {
  2248. hal_tx_ext_desc_get_frag_info(msdu_ext_desc, i,
  2249. &iova,
  2250. &frag_len);
  2251. if (!iova || !frag_len)
  2252. break;
  2253. qdf_mem_unmap_page(soc->osdev, iova, frag_len,
  2254. QDF_DMA_TO_DEVICE);
  2255. }
  2256. goto nbuf_free;
  2257. }
  2258. }
  2259. /* If it's ME frame, dont unmap the cloned nbuf's */
  2260. if ((desc->flags & DP_TX_DESC_FLAG_ME) && qdf_nbuf_is_cloned(nbuf))
  2261. goto nbuf_free;
  2262. dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf, desc->id, type);
  2263. dp_tx_unmap(soc, desc);
  2264. if (desc->flags & DP_TX_DESC_FLAG_MESH_MODE)
  2265. return dp_mesh_tx_comp_free_buff(soc, desc, delayed_free);
  2266. if (dp_tx_traffic_end_indication_enq_ind_pkt(soc, desc, nbuf))
  2267. return NULL;
  2268. nbuf_free:
  2269. if (delayed_free)
  2270. return nbuf;
  2271. qdf_nbuf_free(nbuf);
  2272. return NULL;
  2273. }
  2274. /**
  2275. * dp_tx_sg_unmap_buf() - Unmap scatter gather fragments
  2276. * @soc: DP soc handle
  2277. * @nbuf: skb
  2278. * @msdu_info: MSDU info
  2279. *
  2280. * Return: None
  2281. */
  2282. static inline void
  2283. dp_tx_sg_unmap_buf(struct dp_soc *soc, qdf_nbuf_t nbuf,
  2284. struct dp_tx_msdu_info_s *msdu_info)
  2285. {
  2286. uint32_t cur_idx;
  2287. struct dp_tx_seg_info_s *seg = msdu_info->u.sg_info.curr_seg;
  2288. qdf_nbuf_unmap_nbytes_single(soc->osdev, nbuf, QDF_DMA_TO_DEVICE,
  2289. qdf_nbuf_headlen(nbuf));
  2290. for (cur_idx = 1; cur_idx < seg->frag_cnt; cur_idx++)
  2291. qdf_mem_unmap_page(soc->osdev, (qdf_dma_addr_t)
  2292. (seg->frags[cur_idx].paddr_lo | ((uint64_t)
  2293. seg->frags[cur_idx].paddr_hi) << 32),
  2294. seg->frags[cur_idx].len,
  2295. QDF_DMA_TO_DEVICE);
  2296. }
  2297. /**
  2298. * dp_tx_send_msdu_multiple() - Enqueue multiple MSDUs
  2299. * @vdev: DP vdev handle
  2300. * @nbuf: skb
  2301. * @msdu_info: MSDU info to be setup in MSDU extension descriptor
  2302. *
  2303. * Prepare descriptors for multiple MSDUs (TSO segments) and enqueue to TCL
  2304. *
  2305. * Return: NULL on success,
  2306. * nbuf when it fails to send
  2307. */
  2308. #if QDF_LOCK_STATS
  2309. noinline
  2310. #else
  2311. #endif
  2312. qdf_nbuf_t dp_tx_send_msdu_multiple(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2313. struct dp_tx_msdu_info_s *msdu_info)
  2314. {
  2315. uint32_t i;
  2316. struct dp_pdev *pdev = vdev->pdev;
  2317. struct dp_soc *soc = pdev->soc;
  2318. struct dp_tx_desc_s *tx_desc;
  2319. bool is_cce_classified = false;
  2320. QDF_STATUS status;
  2321. uint16_t htt_tcl_metadata = 0;
  2322. struct dp_tx_queue *tx_q = &msdu_info->tx_queue;
  2323. struct cdp_tid_tx_stats *tid_stats = NULL;
  2324. uint8_t prep_desc_fail = 0, hw_enq_fail = 0;
  2325. if (msdu_info->frm_type == dp_tx_frm_me)
  2326. nbuf = msdu_info->u.sg_info.curr_seg->nbuf;
  2327. i = 0;
  2328. /* Print statement to track i and num_seg */
  2329. /*
  2330. * For each segment (maps to 1 MSDU) , prepare software and hardware
  2331. * descriptors using information in msdu_info
  2332. */
  2333. while (i < msdu_info->num_seg) {
  2334. /*
  2335. * Setup Tx descriptor for an MSDU, and MSDU extension
  2336. * descriptor
  2337. */
  2338. tx_desc = dp_tx_prepare_desc(vdev, nbuf, msdu_info,
  2339. tx_q->desc_pool_id);
  2340. if (!tx_desc) {
  2341. if (msdu_info->frm_type == dp_tx_frm_me) {
  2342. prep_desc_fail++;
  2343. dp_tx_me_free_buf(pdev,
  2344. (void *)(msdu_info->u.sg_info
  2345. .curr_seg->frags[0].vaddr));
  2346. if (prep_desc_fail == msdu_info->num_seg) {
  2347. /*
  2348. * Unmap is needed only if descriptor
  2349. * preparation failed for all segments.
  2350. */
  2351. qdf_nbuf_unmap(soc->osdev,
  2352. msdu_info->u.sg_info.
  2353. curr_seg->nbuf,
  2354. QDF_DMA_TO_DEVICE);
  2355. }
  2356. /*
  2357. * Free the nbuf for the current segment
  2358. * and make it point to the next in the list.
  2359. * For me, there are as many segments as there
  2360. * are no of clients.
  2361. */
  2362. qdf_nbuf_free(msdu_info->u.sg_info
  2363. .curr_seg->nbuf);
  2364. if (msdu_info->u.sg_info.curr_seg->next) {
  2365. msdu_info->u.sg_info.curr_seg =
  2366. msdu_info->u.sg_info
  2367. .curr_seg->next;
  2368. nbuf = msdu_info->u.sg_info
  2369. .curr_seg->nbuf;
  2370. }
  2371. i++;
  2372. continue;
  2373. }
  2374. if (msdu_info->frm_type == dp_tx_frm_tso) {
  2375. dp_tx_tso_seg_history_add(
  2376. soc,
  2377. msdu_info->u.tso_info.curr_seg,
  2378. nbuf, 0, DP_TX_DESC_UNMAP);
  2379. dp_tx_tso_unmap_segment(soc,
  2380. msdu_info->u.tso_info.
  2381. curr_seg,
  2382. msdu_info->u.tso_info.
  2383. tso_num_seg_list);
  2384. if (msdu_info->u.tso_info.curr_seg->next) {
  2385. msdu_info->u.tso_info.curr_seg =
  2386. msdu_info->u.tso_info.curr_seg->next;
  2387. i++;
  2388. continue;
  2389. }
  2390. }
  2391. if (msdu_info->frm_type == dp_tx_frm_sg)
  2392. dp_tx_sg_unmap_buf(soc, nbuf, msdu_info);
  2393. goto done;
  2394. }
  2395. if (msdu_info->frm_type == dp_tx_frm_me) {
  2396. tx_desc->msdu_ext_desc->me_buffer =
  2397. (struct dp_tx_me_buf_t *)msdu_info->
  2398. u.sg_info.curr_seg->frags[0].vaddr;
  2399. tx_desc->flags |= DP_TX_DESC_FLAG_ME;
  2400. }
  2401. if (is_cce_classified)
  2402. tx_desc->flags |= DP_TX_DESC_FLAG_TO_FW;
  2403. htt_tcl_metadata = vdev->htt_tcl_metadata;
  2404. if (msdu_info->exception_fw) {
  2405. DP_TX_TCL_METADATA_VALID_HTT_SET(htt_tcl_metadata, 1);
  2406. }
  2407. dp_tx_is_hp_update_required(i, msdu_info);
  2408. /*
  2409. * For frames with multiple segments (TSO, ME), jump to next
  2410. * segment.
  2411. */
  2412. if (msdu_info->frm_type == dp_tx_frm_tso) {
  2413. if (msdu_info->u.tso_info.curr_seg->next) {
  2414. msdu_info->u.tso_info.curr_seg =
  2415. msdu_info->u.tso_info.curr_seg->next;
  2416. /*
  2417. * If this is a jumbo nbuf, then increment the
  2418. * number of nbuf users for each additional
  2419. * segment of the msdu. This will ensure that
  2420. * the skb is freed only after receiving tx
  2421. * completion for all segments of an nbuf
  2422. */
  2423. qdf_nbuf_inc_users(nbuf);
  2424. /* Check with MCL if this is needed */
  2425. /* nbuf = msdu_info->u.tso_info.curr_seg->nbuf;
  2426. */
  2427. }
  2428. }
  2429. dp_tx_update_mcast_param(DP_INVALID_PEER,
  2430. &htt_tcl_metadata,
  2431. vdev,
  2432. msdu_info);
  2433. /*
  2434. * Enqueue the Tx MSDU descriptor to HW for transmit
  2435. */
  2436. status = soc->arch_ops.tx_hw_enqueue(soc, vdev, tx_desc,
  2437. htt_tcl_metadata,
  2438. NULL, msdu_info);
  2439. dp_tx_check_and_flush_hp(soc, status, msdu_info);
  2440. if (status != QDF_STATUS_SUCCESS) {
  2441. dp_info_rl("Tx_hw_enqueue Fail tx_desc %pK queue %d",
  2442. tx_desc, tx_q->ring_id);
  2443. dp_tx_get_tid(vdev, nbuf, msdu_info);
  2444. tid_stats = &pdev->stats.tid_stats.
  2445. tid_tx_stats[tx_q->ring_id][msdu_info->tid];
  2446. tid_stats->swdrop_cnt[TX_HW_ENQUEUE]++;
  2447. if (msdu_info->frm_type == dp_tx_frm_me) {
  2448. hw_enq_fail++;
  2449. if (hw_enq_fail == msdu_info->num_seg) {
  2450. /*
  2451. * Unmap is needed only if enqueue
  2452. * failed for all segments.
  2453. */
  2454. qdf_nbuf_unmap(soc->osdev,
  2455. msdu_info->u.sg_info.
  2456. curr_seg->nbuf,
  2457. QDF_DMA_TO_DEVICE);
  2458. }
  2459. /*
  2460. * Free the nbuf for the current segment
  2461. * and make it point to the next in the list.
  2462. * For me, there are as many segments as there
  2463. * are no of clients.
  2464. */
  2465. qdf_nbuf_free(msdu_info->u.sg_info
  2466. .curr_seg->nbuf);
  2467. dp_tx_desc_release(tx_desc, tx_q->desc_pool_id);
  2468. if (msdu_info->u.sg_info.curr_seg->next) {
  2469. msdu_info->u.sg_info.curr_seg =
  2470. msdu_info->u.sg_info
  2471. .curr_seg->next;
  2472. nbuf = msdu_info->u.sg_info
  2473. .curr_seg->nbuf;
  2474. } else
  2475. break;
  2476. i++;
  2477. continue;
  2478. }
  2479. /*
  2480. * For TSO frames, the nbuf users increment done for
  2481. * the current segment has to be reverted, since the
  2482. * hw enqueue for this segment failed
  2483. */
  2484. if (msdu_info->frm_type == dp_tx_frm_tso &&
  2485. msdu_info->u.tso_info.curr_seg) {
  2486. /*
  2487. * unmap and free current,
  2488. * retransmit remaining segments
  2489. */
  2490. dp_tx_comp_free_buf(soc, tx_desc, false);
  2491. i++;
  2492. dp_tx_desc_release(tx_desc, tx_q->desc_pool_id);
  2493. continue;
  2494. }
  2495. if (msdu_info->frm_type == dp_tx_frm_sg)
  2496. dp_tx_sg_unmap_buf(soc, nbuf, msdu_info);
  2497. dp_tx_desc_release(tx_desc, tx_q->desc_pool_id);
  2498. goto done;
  2499. }
  2500. /*
  2501. * TODO
  2502. * if tso_info structure can be modified to have curr_seg
  2503. * as first element, following 2 blocks of code (for TSO and SG)
  2504. * can be combined into 1
  2505. */
  2506. /*
  2507. * For Multicast-Unicast converted packets,
  2508. * each converted frame (for a client) is represented as
  2509. * 1 segment
  2510. */
  2511. if ((msdu_info->frm_type == dp_tx_frm_sg) ||
  2512. (msdu_info->frm_type == dp_tx_frm_me)) {
  2513. if (msdu_info->u.sg_info.curr_seg->next) {
  2514. msdu_info->u.sg_info.curr_seg =
  2515. msdu_info->u.sg_info.curr_seg->next;
  2516. nbuf = msdu_info->u.sg_info.curr_seg->nbuf;
  2517. } else
  2518. break;
  2519. }
  2520. i++;
  2521. }
  2522. nbuf = NULL;
  2523. done:
  2524. return nbuf;
  2525. }
  2526. /**
  2527. * dp_tx_prepare_sg()- Extract SG info from NBUF and prepare msdu_info
  2528. * for SG frames
  2529. * @vdev: DP vdev handle
  2530. * @nbuf: skb
  2531. * @seg_info: Pointer to Segment info Descriptor to be prepared
  2532. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
  2533. *
  2534. * Return: NULL on success,
  2535. * nbuf when it fails to send
  2536. */
  2537. static qdf_nbuf_t dp_tx_prepare_sg(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2538. struct dp_tx_seg_info_s *seg_info, struct dp_tx_msdu_info_s *msdu_info)
  2539. {
  2540. uint32_t cur_frag, nr_frags, i;
  2541. qdf_dma_addr_t paddr;
  2542. struct dp_tx_sg_info_s *sg_info;
  2543. sg_info = &msdu_info->u.sg_info;
  2544. nr_frags = qdf_nbuf_get_nr_frags(nbuf);
  2545. if (QDF_STATUS_SUCCESS !=
  2546. qdf_nbuf_map_nbytes_single(vdev->osdev, nbuf,
  2547. QDF_DMA_TO_DEVICE,
  2548. qdf_nbuf_headlen(nbuf))) {
  2549. dp_tx_err("dma map error");
  2550. DP_STATS_INC(vdev, tx_i.sg.dma_map_error, 1);
  2551. qdf_nbuf_free(nbuf);
  2552. return NULL;
  2553. }
  2554. paddr = qdf_nbuf_mapped_paddr_get(nbuf);
  2555. seg_info->frags[0].paddr_lo = paddr;
  2556. seg_info->frags[0].paddr_hi = ((uint64_t) paddr) >> 32;
  2557. seg_info->frags[0].len = qdf_nbuf_headlen(nbuf);
  2558. seg_info->frags[0].vaddr = (void *) nbuf;
  2559. for (cur_frag = 0; cur_frag < nr_frags; cur_frag++) {
  2560. if (QDF_STATUS_SUCCESS != qdf_nbuf_frag_map(vdev->osdev,
  2561. nbuf, 0,
  2562. QDF_DMA_TO_DEVICE,
  2563. cur_frag)) {
  2564. dp_tx_err("frag dma map error");
  2565. DP_STATS_INC(vdev, tx_i.sg.dma_map_error, 1);
  2566. goto map_err;
  2567. }
  2568. paddr = qdf_nbuf_get_tx_frag_paddr(nbuf);
  2569. seg_info->frags[cur_frag + 1].paddr_lo = paddr;
  2570. seg_info->frags[cur_frag + 1].paddr_hi =
  2571. ((uint64_t) paddr) >> 32;
  2572. seg_info->frags[cur_frag + 1].len =
  2573. qdf_nbuf_get_frag_size(nbuf, cur_frag);
  2574. }
  2575. seg_info->frag_cnt = (cur_frag + 1);
  2576. seg_info->total_len = qdf_nbuf_len(nbuf);
  2577. seg_info->next = NULL;
  2578. sg_info->curr_seg = seg_info;
  2579. msdu_info->frm_type = dp_tx_frm_sg;
  2580. msdu_info->num_seg = 1;
  2581. return nbuf;
  2582. map_err:
  2583. /* restore paddr into nbuf before calling unmap */
  2584. qdf_nbuf_mapped_paddr_set(nbuf,
  2585. (qdf_dma_addr_t)(seg_info->frags[0].paddr_lo |
  2586. ((uint64_t)
  2587. seg_info->frags[0].paddr_hi) << 32));
  2588. qdf_nbuf_unmap_nbytes_single(vdev->osdev, nbuf,
  2589. QDF_DMA_TO_DEVICE,
  2590. seg_info->frags[0].len);
  2591. for (i = 1; i <= cur_frag; i++) {
  2592. qdf_mem_unmap_page(vdev->osdev, (qdf_dma_addr_t)
  2593. (seg_info->frags[i].paddr_lo | ((uint64_t)
  2594. seg_info->frags[i].paddr_hi) << 32),
  2595. seg_info->frags[i].len,
  2596. QDF_DMA_TO_DEVICE);
  2597. }
  2598. qdf_nbuf_free(nbuf);
  2599. return NULL;
  2600. }
  2601. /**
  2602. * dp_tx_add_tx_sniffer_meta_data()- Add tx_sniffer meta hdr info
  2603. * @vdev: DP vdev handle
  2604. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
  2605. * @ppdu_cookie: PPDU cookie that should be replayed in the ppdu completions
  2606. *
  2607. * Return: NULL on failure,
  2608. * nbuf when extracted successfully
  2609. */
  2610. static
  2611. void dp_tx_add_tx_sniffer_meta_data(struct dp_vdev *vdev,
  2612. struct dp_tx_msdu_info_s *msdu_info,
  2613. uint16_t ppdu_cookie)
  2614. {
  2615. struct htt_tx_msdu_desc_ext2_t *meta_data =
  2616. (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
  2617. qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
  2618. HTT_TX_MSDU_EXT2_DESC_FLAG_SEND_AS_STANDALONE_SET
  2619. (msdu_info->meta_data[5], 1);
  2620. HTT_TX_MSDU_EXT2_DESC_FLAG_HOST_OPAQUE_VALID_SET
  2621. (msdu_info->meta_data[5], 1);
  2622. HTT_TX_MSDU_EXT2_DESC_HOST_OPAQUE_COOKIE_SET
  2623. (msdu_info->meta_data[6], ppdu_cookie);
  2624. msdu_info->exception_fw = 1;
  2625. msdu_info->is_tx_sniffer = 1;
  2626. }
  2627. #ifdef MESH_MODE_SUPPORT
  2628. /**
  2629. * dp_tx_extract_mesh_meta_data()- Extract mesh meta hdr info from nbuf
  2630. and prepare msdu_info for mesh frames.
  2631. * @vdev: DP vdev handle
  2632. * @nbuf: skb
  2633. * @msdu_info: MSDU info to be setup in MSDU descriptor and MSDU extension desc.
  2634. *
  2635. * Return: NULL on failure,
  2636. * nbuf when extracted successfully
  2637. */
  2638. static
  2639. qdf_nbuf_t dp_tx_extract_mesh_meta_data(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2640. struct dp_tx_msdu_info_s *msdu_info)
  2641. {
  2642. struct meta_hdr_s *mhdr;
  2643. struct htt_tx_msdu_desc_ext2_t *meta_data =
  2644. (struct htt_tx_msdu_desc_ext2_t *)&msdu_info->meta_data[0];
  2645. mhdr = (struct meta_hdr_s *)qdf_nbuf_data(nbuf);
  2646. if (CB_FTYPE_MESH_TX_INFO != qdf_nbuf_get_tx_ftype(nbuf)) {
  2647. msdu_info->exception_fw = 0;
  2648. goto remove_meta_hdr;
  2649. }
  2650. msdu_info->exception_fw = 1;
  2651. qdf_mem_zero(meta_data, sizeof(struct htt_tx_msdu_desc_ext2_t));
  2652. meta_data->host_tx_desc_pool = 1;
  2653. meta_data->update_peer_cache = 1;
  2654. meta_data->learning_frame = 1;
  2655. if (!(mhdr->flags & METAHDR_FLAG_AUTO_RATE)) {
  2656. meta_data->power = mhdr->power;
  2657. meta_data->mcs_mask = 1 << mhdr->rate_info[0].mcs;
  2658. meta_data->nss_mask = 1 << mhdr->rate_info[0].nss;
  2659. meta_data->pream_type = mhdr->rate_info[0].preamble_type;
  2660. meta_data->retry_limit = mhdr->rate_info[0].max_tries;
  2661. meta_data->dyn_bw = 1;
  2662. meta_data->valid_pwr = 1;
  2663. meta_data->valid_mcs_mask = 1;
  2664. meta_data->valid_nss_mask = 1;
  2665. meta_data->valid_preamble_type = 1;
  2666. meta_data->valid_retries = 1;
  2667. meta_data->valid_bw_info = 1;
  2668. }
  2669. if (mhdr->flags & METAHDR_FLAG_NOENCRYPT) {
  2670. meta_data->encrypt_type = 0;
  2671. meta_data->valid_encrypt_type = 1;
  2672. meta_data->learning_frame = 0;
  2673. }
  2674. meta_data->valid_key_flags = 1;
  2675. meta_data->key_flags = (mhdr->keyix & 0x3);
  2676. remove_meta_hdr:
  2677. if (qdf_nbuf_pull_head(nbuf, sizeof(struct meta_hdr_s)) == NULL) {
  2678. dp_tx_err("qdf_nbuf_pull_head failed");
  2679. qdf_nbuf_free(nbuf);
  2680. return NULL;
  2681. }
  2682. msdu_info->tid = qdf_nbuf_get_priority(nbuf);
  2683. dp_tx_info("Meta hdr %0x %0x %0x %0x %0x %0x"
  2684. " tid %d to_fw %d",
  2685. msdu_info->meta_data[0],
  2686. msdu_info->meta_data[1],
  2687. msdu_info->meta_data[2],
  2688. msdu_info->meta_data[3],
  2689. msdu_info->meta_data[4],
  2690. msdu_info->meta_data[5],
  2691. msdu_info->tid, msdu_info->exception_fw);
  2692. return nbuf;
  2693. }
  2694. #else
  2695. static
  2696. qdf_nbuf_t dp_tx_extract_mesh_meta_data(struct dp_vdev *vdev, qdf_nbuf_t nbuf,
  2697. struct dp_tx_msdu_info_s *msdu_info)
  2698. {
  2699. return nbuf;
  2700. }
  2701. #endif
  2702. /**
  2703. * dp_check_exc_metadata() - Checks if parameters are valid
  2704. * @tx_exc - holds all exception path parameters
  2705. *
  2706. * Returns true when all the parameters are valid else false
  2707. *
  2708. */
  2709. static bool dp_check_exc_metadata(struct cdp_tx_exception_metadata *tx_exc)
  2710. {
  2711. bool invalid_tid = (tx_exc->tid >= DP_MAX_TIDS && tx_exc->tid !=
  2712. HTT_INVALID_TID);
  2713. bool invalid_encap_type =
  2714. (tx_exc->tx_encap_type > htt_cmn_pkt_num_types &&
  2715. tx_exc->tx_encap_type != CDP_INVALID_TX_ENCAP_TYPE);
  2716. bool invalid_sec_type = (tx_exc->sec_type > cdp_num_sec_types &&
  2717. tx_exc->sec_type != CDP_INVALID_SEC_TYPE);
  2718. bool invalid_cookie = (tx_exc->is_tx_sniffer == 1 &&
  2719. tx_exc->ppdu_cookie == 0);
  2720. if (tx_exc->is_intrabss_fwd)
  2721. return true;
  2722. if (invalid_tid || invalid_encap_type || invalid_sec_type ||
  2723. invalid_cookie) {
  2724. return false;
  2725. }
  2726. return true;
  2727. }
  2728. #ifdef ATH_SUPPORT_IQUE
  2729. /**
  2730. * dp_tx_mcast_enhance() - Multicast enhancement on TX
  2731. * @vdev: vdev handle
  2732. * @nbuf: skb
  2733. *
  2734. * Return: true on success,
  2735. * false on failure
  2736. */
  2737. static inline bool dp_tx_mcast_enhance(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  2738. {
  2739. qdf_ether_header_t *eh;
  2740. /* Mcast to Ucast Conversion*/
  2741. if (qdf_likely(!vdev->mcast_enhancement_en))
  2742. return true;
  2743. eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
  2744. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost) &&
  2745. !DP_FRAME_IS_BROADCAST((eh)->ether_dhost)) {
  2746. dp_verbose_debug("Mcast frm for ME %pK", vdev);
  2747. qdf_nbuf_set_next(nbuf, NULL);
  2748. DP_STATS_INC_PKT(vdev, tx_i.mcast_en.mcast_pkt, 1,
  2749. qdf_nbuf_len(nbuf));
  2750. if (dp_tx_prepare_send_me(vdev, nbuf) ==
  2751. QDF_STATUS_SUCCESS) {
  2752. return false;
  2753. }
  2754. if (qdf_unlikely(vdev->igmp_mcast_enhanc_en > 0)) {
  2755. if (dp_tx_prepare_send_igmp_me(vdev, nbuf) ==
  2756. QDF_STATUS_SUCCESS) {
  2757. return false;
  2758. }
  2759. }
  2760. }
  2761. return true;
  2762. }
  2763. #else
  2764. static inline bool dp_tx_mcast_enhance(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  2765. {
  2766. return true;
  2767. }
  2768. #endif
  2769. #ifdef QCA_SUPPORT_WDS_EXTENDED
  2770. /**
  2771. * dp_tx_mcast_drop() - Drop mcast frame if drop_tx_mcast is set in WDS_EXT
  2772. * @vdev: vdev handle
  2773. * @nbuf: skb
  2774. *
  2775. * Return: true if frame is dropped, false otherwise
  2776. */
  2777. static inline bool dp_tx_mcast_drop(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  2778. {
  2779. /* Drop tx mcast and WDS Extended feature check */
  2780. if (qdf_unlikely((vdev->drop_tx_mcast) && (vdev->wds_ext_enabled))) {
  2781. qdf_ether_header_t *eh = (qdf_ether_header_t *)
  2782. qdf_nbuf_data(nbuf);
  2783. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost)) {
  2784. DP_STATS_INC(vdev, tx_i.dropped.tx_mcast_drop, 1);
  2785. return true;
  2786. }
  2787. }
  2788. return false;
  2789. }
  2790. #else
  2791. static inline bool dp_tx_mcast_drop(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  2792. {
  2793. return false;
  2794. }
  2795. #endif
  2796. /**
  2797. * dp_tx_per_pkt_vdev_id_check() - vdev id check for frame
  2798. * @nbuf: qdf_nbuf_t
  2799. * @vdev: struct dp_vdev *
  2800. *
  2801. * Allow packet for processing only if it is for peer client which is
  2802. * connected with same vap. Drop packet if client is connected to
  2803. * different vap.
  2804. *
  2805. * Return: QDF_STATUS
  2806. */
  2807. static inline QDF_STATUS
  2808. dp_tx_per_pkt_vdev_id_check(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
  2809. {
  2810. struct dp_ast_entry *dst_ast_entry = NULL;
  2811. qdf_ether_header_t *eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
  2812. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost) ||
  2813. DP_FRAME_IS_BROADCAST((eh)->ether_dhost))
  2814. return QDF_STATUS_SUCCESS;
  2815. qdf_spin_lock_bh(&vdev->pdev->soc->ast_lock);
  2816. dst_ast_entry = dp_peer_ast_hash_find_by_vdevid(vdev->pdev->soc,
  2817. eh->ether_dhost,
  2818. vdev->vdev_id);
  2819. /* If there is no ast entry, return failure */
  2820. if (qdf_unlikely(!dst_ast_entry)) {
  2821. qdf_spin_unlock_bh(&vdev->pdev->soc->ast_lock);
  2822. return QDF_STATUS_E_FAILURE;
  2823. }
  2824. qdf_spin_unlock_bh(&vdev->pdev->soc->ast_lock);
  2825. return QDF_STATUS_SUCCESS;
  2826. }
  2827. /**
  2828. * dp_tx_nawds_handler() - NAWDS handler
  2829. *
  2830. * @soc: DP soc handle
  2831. * @vdev_id: id of DP vdev handle
  2832. * @msdu_info: msdu_info required to create HTT metadata
  2833. * @nbuf: skb
  2834. *
  2835. * This API transfers the multicast frames with the peer id
  2836. * on NAWDS enabled peer.
  2837. * Return: none
  2838. */
  2839. static inline
  2840. void dp_tx_nawds_handler(struct dp_soc *soc, struct dp_vdev *vdev,
  2841. struct dp_tx_msdu_info_s *msdu_info,
  2842. qdf_nbuf_t nbuf, uint16_t sa_peer_id)
  2843. {
  2844. struct dp_peer *peer = NULL;
  2845. qdf_nbuf_t nbuf_clone = NULL;
  2846. uint16_t peer_id = DP_INVALID_PEER;
  2847. struct dp_txrx_peer *txrx_peer;
  2848. /* This check avoids pkt forwarding which is entered
  2849. * in the ast table but still doesn't have valid peerid.
  2850. */
  2851. if (sa_peer_id == HTT_INVALID_PEER)
  2852. return;
  2853. qdf_spin_lock_bh(&vdev->peer_list_lock);
  2854. TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
  2855. txrx_peer = dp_get_txrx_peer(peer);
  2856. if (!txrx_peer)
  2857. continue;
  2858. if (!txrx_peer->bss_peer && txrx_peer->nawds_enabled) {
  2859. peer_id = peer->peer_id;
  2860. if (!dp_peer_is_primary_link_peer(peer))
  2861. continue;
  2862. /* Multicast packets needs to be
  2863. * dropped in case of intra bss forwarding
  2864. */
  2865. if (sa_peer_id == txrx_peer->peer_id) {
  2866. dp_tx_debug("multicast packet");
  2867. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  2868. tx.nawds_mcast_drop,
  2869. 1);
  2870. continue;
  2871. }
  2872. nbuf_clone = qdf_nbuf_clone(nbuf);
  2873. if (!nbuf_clone) {
  2874. QDF_TRACE(QDF_MODULE_ID_DP,
  2875. QDF_TRACE_LEVEL_ERROR,
  2876. FL("nbuf clone failed"));
  2877. break;
  2878. }
  2879. nbuf_clone = dp_tx_send_msdu_single(vdev, nbuf_clone,
  2880. msdu_info, peer_id,
  2881. NULL);
  2882. if (nbuf_clone) {
  2883. dp_tx_debug("pkt send failed");
  2884. qdf_nbuf_free(nbuf_clone);
  2885. } else {
  2886. if (peer_id != DP_INVALID_PEER)
  2887. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
  2888. tx.nawds_mcast,
  2889. 1, qdf_nbuf_len(nbuf));
  2890. }
  2891. }
  2892. }
  2893. qdf_spin_unlock_bh(&vdev->peer_list_lock);
  2894. }
  2895. /**
  2896. * dp_tx_send_exception() - Transmit a frame on a given VAP in exception path
  2897. * @soc: DP soc handle
  2898. * @vdev_id: id of DP vdev handle
  2899. * @nbuf: skb
  2900. * @tx_exc_metadata: Handle that holds exception path meta data
  2901. *
  2902. * Entry point for Core Tx layer (DP_TX) invoked from
  2903. * hard_start_xmit in OSIF/HDD to transmit frames through fw
  2904. *
  2905. * Return: NULL on success,
  2906. * nbuf when it fails to send
  2907. */
  2908. qdf_nbuf_t
  2909. dp_tx_send_exception(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  2910. qdf_nbuf_t nbuf,
  2911. struct cdp_tx_exception_metadata *tx_exc_metadata)
  2912. {
  2913. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  2914. struct dp_tx_msdu_info_s msdu_info;
  2915. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  2916. DP_MOD_ID_TX_EXCEPTION);
  2917. if (qdf_unlikely(!vdev))
  2918. goto fail;
  2919. qdf_mem_zero(&msdu_info, sizeof(msdu_info));
  2920. if (!tx_exc_metadata)
  2921. goto fail;
  2922. msdu_info.tid = tx_exc_metadata->tid;
  2923. dp_verbose_debug("skb "QDF_MAC_ADDR_FMT,
  2924. QDF_MAC_ADDR_REF(nbuf->data));
  2925. DP_STATS_INC_PKT(vdev, tx_i.rcvd, 1, qdf_nbuf_len(nbuf));
  2926. if (qdf_unlikely(!dp_check_exc_metadata(tx_exc_metadata))) {
  2927. dp_tx_err("Invalid parameters in exception path");
  2928. goto fail;
  2929. }
  2930. /* for peer based metadata check if peer is valid */
  2931. if (tx_exc_metadata->peer_id != CDP_INVALID_PEER) {
  2932. struct dp_peer *peer = NULL;
  2933. peer = dp_peer_get_ref_by_id(vdev->pdev->soc,
  2934. tx_exc_metadata->peer_id,
  2935. DP_MOD_ID_TX_EXCEPTION);
  2936. if (qdf_unlikely(!peer)) {
  2937. DP_STATS_INC(vdev,
  2938. tx_i.dropped.invalid_peer_id_in_exc_path,
  2939. 1);
  2940. goto fail;
  2941. }
  2942. dp_peer_unref_delete(peer, DP_MOD_ID_TX_EXCEPTION);
  2943. }
  2944. /* Basic sanity checks for unsupported packets */
  2945. /* MESH mode */
  2946. if (qdf_unlikely(vdev->mesh_vdev)) {
  2947. dp_tx_err("Mesh mode is not supported in exception path");
  2948. goto fail;
  2949. }
  2950. /*
  2951. * Classify the frame and call corresponding
  2952. * "prepare" function which extracts the segment (TSO)
  2953. * and fragmentation information (for TSO , SG, ME, or Raw)
  2954. * into MSDU_INFO structure which is later used to fill
  2955. * SW and HW descriptors.
  2956. */
  2957. if (qdf_nbuf_is_tso(nbuf)) {
  2958. dp_verbose_debug("TSO frame %pK", vdev);
  2959. DP_STATS_INC_PKT(vdev->pdev, tso_stats.num_tso_pkts, 1,
  2960. qdf_nbuf_len(nbuf));
  2961. if (dp_tx_prepare_tso(vdev, nbuf, &msdu_info)) {
  2962. DP_STATS_INC_PKT(vdev->pdev, tso_stats.dropped_host, 1,
  2963. qdf_nbuf_len(nbuf));
  2964. goto fail;
  2965. }
  2966. DP_STATS_INC(vdev, tx_i.rcvd.num, msdu_info.num_seg - 1);
  2967. goto send_multiple;
  2968. }
  2969. /* SG */
  2970. if (qdf_unlikely(qdf_nbuf_is_nonlinear(nbuf))) {
  2971. struct dp_tx_seg_info_s seg_info = {0};
  2972. nbuf = dp_tx_prepare_sg(vdev, nbuf, &seg_info, &msdu_info);
  2973. if (!nbuf)
  2974. goto fail;
  2975. dp_verbose_debug("non-TSO SG frame %pK", vdev);
  2976. DP_STATS_INC_PKT(vdev, tx_i.sg.sg_pkt, 1,
  2977. qdf_nbuf_len(nbuf));
  2978. goto send_multiple;
  2979. }
  2980. if (qdf_likely(tx_exc_metadata->is_tx_sniffer)) {
  2981. DP_STATS_INC_PKT(vdev, tx_i.sniffer_rcvd, 1,
  2982. qdf_nbuf_len(nbuf));
  2983. dp_tx_add_tx_sniffer_meta_data(vdev, &msdu_info,
  2984. tx_exc_metadata->ppdu_cookie);
  2985. }
  2986. /*
  2987. * Get HW Queue to use for this frame.
  2988. * TCL supports upto 4 DMA rings, out of which 3 rings are
  2989. * dedicated for data and 1 for command.
  2990. * "queue_id" maps to one hardware ring.
  2991. * With each ring, we also associate a unique Tx descriptor pool
  2992. * to minimize lock contention for these resources.
  2993. */
  2994. dp_tx_get_queue(vdev, nbuf, &msdu_info.tx_queue);
  2995. if (qdf_likely(tx_exc_metadata->is_intrabss_fwd)) {
  2996. if (qdf_unlikely(vdev->nawds_enabled)) {
  2997. /*
  2998. * This is a multicast packet
  2999. */
  3000. dp_tx_nawds_handler(soc, vdev, &msdu_info, nbuf,
  3001. tx_exc_metadata->peer_id);
  3002. DP_STATS_INC_PKT(vdev, tx_i.nawds_mcast,
  3003. 1, qdf_nbuf_len(nbuf));
  3004. }
  3005. nbuf = dp_tx_send_msdu_single(vdev, nbuf, &msdu_info,
  3006. DP_INVALID_PEER, NULL);
  3007. } else {
  3008. /*
  3009. * Check exception descriptors
  3010. */
  3011. if (dp_tx_exception_limit_check(vdev))
  3012. goto fail;
  3013. /* Single linear frame */
  3014. /*
  3015. * If nbuf is a simple linear frame, use send_single function to
  3016. * prepare direct-buffer type TCL descriptor and enqueue to TCL
  3017. * SRNG. There is no need to setup a MSDU extension descriptor.
  3018. */
  3019. nbuf = dp_tx_send_msdu_single(vdev, nbuf, &msdu_info,
  3020. tx_exc_metadata->peer_id,
  3021. tx_exc_metadata);
  3022. }
  3023. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3024. return nbuf;
  3025. send_multiple:
  3026. nbuf = dp_tx_send_msdu_multiple(vdev, nbuf, &msdu_info);
  3027. fail:
  3028. if (vdev)
  3029. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3030. dp_verbose_debug("pkt send failed");
  3031. return nbuf;
  3032. }
  3033. /**
  3034. * dp_tx_send_exception_vdev_id_check() - Transmit a frame on a given VAP
  3035. * in exception path in special case to avoid regular exception path chk.
  3036. * @soc: DP soc handle
  3037. * @vdev_id: id of DP vdev handle
  3038. * @nbuf: skb
  3039. * @tx_exc_metadata: Handle that holds exception path meta data
  3040. *
  3041. * Entry point for Core Tx layer (DP_TX) invoked from
  3042. * hard_start_xmit in OSIF/HDD to transmit frames through fw
  3043. *
  3044. * Return: NULL on success,
  3045. * nbuf when it fails to send
  3046. */
  3047. qdf_nbuf_t
  3048. dp_tx_send_exception_vdev_id_check(struct cdp_soc_t *soc_hdl,
  3049. uint8_t vdev_id, qdf_nbuf_t nbuf,
  3050. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3051. {
  3052. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3053. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  3054. DP_MOD_ID_TX_EXCEPTION);
  3055. if (qdf_unlikely(!vdev))
  3056. goto fail;
  3057. if (qdf_unlikely(dp_tx_per_pkt_vdev_id_check(nbuf, vdev)
  3058. == QDF_STATUS_E_FAILURE)) {
  3059. DP_STATS_INC(vdev, tx_i.dropped.fail_per_pkt_vdev_id_check, 1);
  3060. goto fail;
  3061. }
  3062. /* Unref count as it will again be taken inside dp_tx_exception */
  3063. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3064. return dp_tx_send_exception(soc_hdl, vdev_id, nbuf, tx_exc_metadata);
  3065. fail:
  3066. if (vdev)
  3067. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TX_EXCEPTION);
  3068. dp_verbose_debug("pkt send failed");
  3069. return nbuf;
  3070. }
  3071. /**
  3072. * dp_tx_send_mesh() - Transmit mesh frame on a given VAP
  3073. * @soc: DP soc handle
  3074. * @vdev_id: DP vdev handle
  3075. * @nbuf: skb
  3076. *
  3077. * Entry point for Core Tx layer (DP_TX) invoked from
  3078. * hard_start_xmit in OSIF/HDD
  3079. *
  3080. * Return: NULL on success,
  3081. * nbuf when it fails to send
  3082. */
  3083. #ifdef MESH_MODE_SUPPORT
  3084. qdf_nbuf_t dp_tx_send_mesh(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3085. qdf_nbuf_t nbuf)
  3086. {
  3087. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3088. struct meta_hdr_s *mhdr;
  3089. qdf_nbuf_t nbuf_mesh = NULL;
  3090. qdf_nbuf_t nbuf_clone = NULL;
  3091. struct dp_vdev *vdev;
  3092. uint8_t no_enc_frame = 0;
  3093. nbuf_mesh = qdf_nbuf_unshare(nbuf);
  3094. if (!nbuf_mesh) {
  3095. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3096. "qdf_nbuf_unshare failed");
  3097. return nbuf;
  3098. }
  3099. vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_MESH);
  3100. if (!vdev) {
  3101. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3102. "vdev is NULL for vdev_id %d", vdev_id);
  3103. return nbuf;
  3104. }
  3105. nbuf = nbuf_mesh;
  3106. mhdr = (struct meta_hdr_s *)qdf_nbuf_data(nbuf);
  3107. if ((vdev->sec_type != cdp_sec_type_none) &&
  3108. (mhdr->flags & METAHDR_FLAG_NOENCRYPT))
  3109. no_enc_frame = 1;
  3110. if (mhdr->flags & METAHDR_FLAG_NOQOS)
  3111. qdf_nbuf_set_priority(nbuf, HTT_TX_EXT_TID_NON_QOS_MCAST_BCAST);
  3112. if ((mhdr->flags & METAHDR_FLAG_INFO_UPDATED) &&
  3113. !no_enc_frame) {
  3114. nbuf_clone = qdf_nbuf_clone(nbuf);
  3115. if (!nbuf_clone) {
  3116. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3117. "qdf_nbuf_clone failed");
  3118. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
  3119. return nbuf;
  3120. }
  3121. qdf_nbuf_set_tx_ftype(nbuf_clone, CB_FTYPE_MESH_TX_INFO);
  3122. }
  3123. if (nbuf_clone) {
  3124. if (!dp_tx_send(soc_hdl, vdev_id, nbuf_clone)) {
  3125. DP_STATS_INC(vdev, tx_i.mesh.exception_fw, 1);
  3126. } else {
  3127. qdf_nbuf_free(nbuf_clone);
  3128. }
  3129. }
  3130. if (no_enc_frame)
  3131. qdf_nbuf_set_tx_ftype(nbuf, CB_FTYPE_MESH_TX_INFO);
  3132. else
  3133. qdf_nbuf_set_tx_ftype(nbuf, CB_FTYPE_INVALID);
  3134. nbuf = dp_tx_send(soc_hdl, vdev_id, nbuf);
  3135. if ((!nbuf) && no_enc_frame) {
  3136. DP_STATS_INC(vdev, tx_i.mesh.exception_fw, 1);
  3137. }
  3138. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_MESH);
  3139. return nbuf;
  3140. }
  3141. #else
  3142. qdf_nbuf_t dp_tx_send_mesh(struct cdp_soc_t *soc, uint8_t vdev_id,
  3143. qdf_nbuf_t nbuf)
  3144. {
  3145. return dp_tx_send(soc, vdev_id, nbuf);
  3146. }
  3147. #endif
  3148. #ifdef QCA_DP_TX_NBUF_AND_NBUF_DATA_PREFETCH
  3149. static inline
  3150. void dp_tx_prefetch_nbuf_data(qdf_nbuf_t nbuf)
  3151. {
  3152. if (nbuf) {
  3153. qdf_prefetch(&nbuf->len);
  3154. qdf_prefetch(&nbuf->data);
  3155. }
  3156. }
  3157. #else
  3158. static inline
  3159. void dp_tx_prefetch_nbuf_data(qdf_nbuf_t nbuf)
  3160. {
  3161. }
  3162. #endif
  3163. #ifdef DP_UMAC_HW_RESET_SUPPORT
  3164. /*
  3165. * dp_tx_drop() - Drop the frame on a given VAP
  3166. * @soc: DP soc handle
  3167. * @vdev_id: id of DP vdev handle
  3168. * @nbuf: skb
  3169. *
  3170. * Drop all the incoming packets
  3171. *
  3172. * Return: nbuf
  3173. *
  3174. */
  3175. qdf_nbuf_t dp_tx_drop(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3176. qdf_nbuf_t nbuf)
  3177. {
  3178. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3179. struct dp_vdev *vdev = NULL;
  3180. vdev = soc->vdev_id_map[vdev_id];
  3181. if (qdf_unlikely(!vdev))
  3182. return nbuf;
  3183. DP_STATS_INC(vdev, tx_i.dropped.drop_ingress, 1);
  3184. return nbuf;
  3185. }
  3186. /*
  3187. * dp_tx_exc_drop() - Drop the frame on a given VAP
  3188. * @soc: DP soc handle
  3189. * @vdev_id: id of DP vdev handle
  3190. * @nbuf: skb
  3191. * @tx_exc_metadata: Handle that holds exception path meta data
  3192. *
  3193. * Drop all the incoming packets
  3194. *
  3195. * Return: nbuf
  3196. *
  3197. */
  3198. qdf_nbuf_t dp_tx_exc_drop(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3199. qdf_nbuf_t nbuf,
  3200. struct cdp_tx_exception_metadata *tx_exc_metadata)
  3201. {
  3202. return dp_tx_drop(soc_hdl, vdev_id, nbuf);
  3203. }
  3204. #endif
  3205. #ifdef FEATURE_DIRECT_LINK
  3206. /*
  3207. * dp_vdev_tx_mark_to_fw() - Mark to_fw bit for the tx packet
  3208. * @nbuf: skb
  3209. * @vdev: DP vdev handle
  3210. *
  3211. * Return: None
  3212. */
  3213. static inline void dp_vdev_tx_mark_to_fw(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
  3214. {
  3215. if (qdf_unlikely(vdev->to_fw))
  3216. QDF_NBUF_CB_TX_PACKET_TO_FW(nbuf) = 1;
  3217. }
  3218. #else
  3219. static inline void dp_vdev_tx_mark_to_fw(qdf_nbuf_t nbuf, struct dp_vdev *vdev)
  3220. {
  3221. }
  3222. #endif
  3223. /*
  3224. * dp_tx_send() - Transmit a frame on a given VAP
  3225. * @soc: DP soc handle
  3226. * @vdev_id: id of DP vdev handle
  3227. * @nbuf: skb
  3228. *
  3229. * Entry point for Core Tx layer (DP_TX) invoked from
  3230. * hard_start_xmit in OSIF/HDD or from dp_rx_process for intravap forwarding
  3231. * cases
  3232. *
  3233. * Return: NULL on success,
  3234. * nbuf when it fails to send
  3235. */
  3236. qdf_nbuf_t dp_tx_send(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  3237. qdf_nbuf_t nbuf)
  3238. {
  3239. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3240. uint16_t peer_id = HTT_INVALID_PEER;
  3241. /*
  3242. * doing a memzero is causing additional function call overhead
  3243. * so doing static stack clearing
  3244. */
  3245. struct dp_tx_msdu_info_s msdu_info = {0};
  3246. struct dp_vdev *vdev = NULL;
  3247. qdf_nbuf_t end_nbuf = NULL;
  3248. if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
  3249. return nbuf;
  3250. /*
  3251. * dp_vdev_get_ref_by_id does does a atomic operation avoid using
  3252. * this in per packet path.
  3253. *
  3254. * As in this path vdev memory is already protected with netdev
  3255. * tx lock
  3256. */
  3257. vdev = soc->vdev_id_map[vdev_id];
  3258. if (qdf_unlikely(!vdev))
  3259. return nbuf;
  3260. dp_vdev_tx_mark_to_fw(nbuf, vdev);
  3261. /*
  3262. * Set Default Host TID value to invalid TID
  3263. * (TID override disabled)
  3264. */
  3265. msdu_info.tid = HTT_TX_EXT_TID_INVALID;
  3266. DP_STATS_INC_PKT(vdev, tx_i.rcvd, 1, qdf_nbuf_len(nbuf));
  3267. if (qdf_unlikely(vdev->mesh_vdev)) {
  3268. qdf_nbuf_t nbuf_mesh = dp_tx_extract_mesh_meta_data(vdev, nbuf,
  3269. &msdu_info);
  3270. if (!nbuf_mesh) {
  3271. dp_verbose_debug("Extracting mesh metadata failed");
  3272. return nbuf;
  3273. }
  3274. nbuf = nbuf_mesh;
  3275. }
  3276. /*
  3277. * Get HW Queue to use for this frame.
  3278. * TCL supports upto 4 DMA rings, out of which 3 rings are
  3279. * dedicated for data and 1 for command.
  3280. * "queue_id" maps to one hardware ring.
  3281. * With each ring, we also associate a unique Tx descriptor pool
  3282. * to minimize lock contention for these resources.
  3283. */
  3284. dp_tx_get_queue(vdev, nbuf, &msdu_info.tx_queue);
  3285. DP_STATS_INC(vdev, tx_i.rcvd_per_core[msdu_info.tx_queue.desc_pool_id],
  3286. 1);
  3287. /*
  3288. * TCL H/W supports 2 DSCP-TID mapping tables.
  3289. * Table 1 - Default DSCP-TID mapping table
  3290. * Table 2 - 1 DSCP-TID override table
  3291. *
  3292. * If we need a different DSCP-TID mapping for this vap,
  3293. * call tid_classify to extract DSCP/ToS from frame and
  3294. * map to a TID and store in msdu_info. This is later used
  3295. * to fill in TCL Input descriptor (per-packet TID override).
  3296. */
  3297. dp_tx_classify_tid(vdev, nbuf, &msdu_info);
  3298. /*
  3299. * Classify the frame and call corresponding
  3300. * "prepare" function which extracts the segment (TSO)
  3301. * and fragmentation information (for TSO , SG, ME, or Raw)
  3302. * into MSDU_INFO structure which is later used to fill
  3303. * SW and HW descriptors.
  3304. */
  3305. if (qdf_nbuf_is_tso(nbuf)) {
  3306. dp_verbose_debug("TSO frame %pK", vdev);
  3307. DP_STATS_INC_PKT(vdev->pdev, tso_stats.num_tso_pkts, 1,
  3308. qdf_nbuf_len(nbuf));
  3309. if (dp_tx_prepare_tso(vdev, nbuf, &msdu_info)) {
  3310. DP_STATS_INC_PKT(vdev->pdev, tso_stats.dropped_host, 1,
  3311. qdf_nbuf_len(nbuf));
  3312. return nbuf;
  3313. }
  3314. DP_STATS_INC(vdev, tx_i.rcvd.num, msdu_info.num_seg - 1);
  3315. goto send_multiple;
  3316. }
  3317. /* SG */
  3318. if (qdf_unlikely(qdf_nbuf_is_nonlinear(nbuf))) {
  3319. if (qdf_nbuf_get_nr_frags(nbuf) > DP_TX_MAX_NUM_FRAGS - 1) {
  3320. if (qdf_unlikely(qdf_nbuf_linearize(nbuf)))
  3321. return nbuf;
  3322. } else {
  3323. struct dp_tx_seg_info_s seg_info = {0};
  3324. if (qdf_unlikely(is_nbuf_frm_rmnet(nbuf, &msdu_info)))
  3325. goto send_single;
  3326. nbuf = dp_tx_prepare_sg(vdev, nbuf, &seg_info,
  3327. &msdu_info);
  3328. if (!nbuf)
  3329. return NULL;
  3330. dp_verbose_debug("non-TSO SG frame %pK", vdev);
  3331. DP_STATS_INC_PKT(vdev, tx_i.sg.sg_pkt, 1,
  3332. qdf_nbuf_len(nbuf));
  3333. goto send_multiple;
  3334. }
  3335. }
  3336. if (qdf_unlikely(!dp_tx_mcast_enhance(vdev, nbuf)))
  3337. return NULL;
  3338. if (qdf_unlikely(dp_tx_mcast_drop(vdev, nbuf)))
  3339. return nbuf;
  3340. /* RAW */
  3341. if (qdf_unlikely(vdev->tx_encap_type == htt_cmn_pkt_type_raw)) {
  3342. struct dp_tx_seg_info_s seg_info = {0};
  3343. nbuf = dp_tx_prepare_raw(vdev, nbuf, &seg_info, &msdu_info);
  3344. if (!nbuf)
  3345. return NULL;
  3346. dp_verbose_debug("Raw frame %pK", vdev);
  3347. goto send_multiple;
  3348. }
  3349. if (qdf_unlikely(vdev->nawds_enabled)) {
  3350. qdf_ether_header_t *eh = (qdf_ether_header_t *)
  3351. qdf_nbuf_data(nbuf);
  3352. if (DP_FRAME_IS_MULTICAST((eh)->ether_dhost)) {
  3353. uint16_t sa_peer_id = DP_INVALID_PEER;
  3354. if (!soc->ast_offload_support) {
  3355. struct dp_ast_entry *ast_entry = NULL;
  3356. qdf_spin_lock_bh(&soc->ast_lock);
  3357. ast_entry = dp_peer_ast_hash_find_by_pdevid
  3358. (soc,
  3359. (uint8_t *)(eh->ether_shost),
  3360. vdev->pdev->pdev_id);
  3361. if (ast_entry)
  3362. sa_peer_id = ast_entry->peer_id;
  3363. qdf_spin_unlock_bh(&soc->ast_lock);
  3364. }
  3365. dp_tx_nawds_handler(soc, vdev, &msdu_info, nbuf,
  3366. sa_peer_id);
  3367. }
  3368. peer_id = DP_INVALID_PEER;
  3369. DP_STATS_INC_PKT(vdev, tx_i.nawds_mcast,
  3370. 1, qdf_nbuf_len(nbuf));
  3371. }
  3372. send_single:
  3373. /* Single linear frame */
  3374. /*
  3375. * If nbuf is a simple linear frame, use send_single function to
  3376. * prepare direct-buffer type TCL descriptor and enqueue to TCL
  3377. * SRNG. There is no need to setup a MSDU extension descriptor.
  3378. */
  3379. dp_tx_prefetch_nbuf_data(nbuf);
  3380. nbuf = dp_tx_send_msdu_single_wrapper(vdev, nbuf, &msdu_info,
  3381. peer_id, end_nbuf);
  3382. return nbuf;
  3383. send_multiple:
  3384. nbuf = dp_tx_send_msdu_multiple(vdev, nbuf, &msdu_info);
  3385. if (qdf_unlikely(nbuf && msdu_info.frm_type == dp_tx_frm_raw))
  3386. dp_tx_raw_prepare_unset(vdev->pdev->soc, nbuf);
  3387. return nbuf;
  3388. }
  3389. /**
  3390. * dp_tx_send_vdev_id_check() - Transmit a frame on a given VAP in special
  3391. * case to vaoid check in perpkt path.
  3392. * @soc: DP soc handle
  3393. * @vdev_id: id of DP vdev handle
  3394. * @nbuf: skb
  3395. *
  3396. * Entry point for Core Tx layer (DP_TX) invoked from
  3397. * hard_start_xmit in OSIF/HDD to transmit packet through dp_tx_send
  3398. * with special condition to avoid per pkt check in dp_tx_send
  3399. *
  3400. * Return: NULL on success,
  3401. * nbuf when it fails to send
  3402. */
  3403. qdf_nbuf_t dp_tx_send_vdev_id_check(struct cdp_soc_t *soc_hdl,
  3404. uint8_t vdev_id, qdf_nbuf_t nbuf)
  3405. {
  3406. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  3407. struct dp_vdev *vdev = NULL;
  3408. if (qdf_unlikely(vdev_id >= MAX_VDEV_CNT))
  3409. return nbuf;
  3410. /*
  3411. * dp_vdev_get_ref_by_id does does a atomic operation avoid using
  3412. * this in per packet path.
  3413. *
  3414. * As in this path vdev memory is already protected with netdev
  3415. * tx lock
  3416. */
  3417. vdev = soc->vdev_id_map[vdev_id];
  3418. if (qdf_unlikely(!vdev))
  3419. return nbuf;
  3420. if (qdf_unlikely(dp_tx_per_pkt_vdev_id_check(nbuf, vdev)
  3421. == QDF_STATUS_E_FAILURE)) {
  3422. DP_STATS_INC(vdev, tx_i.dropped.fail_per_pkt_vdev_id_check, 1);
  3423. return nbuf;
  3424. }
  3425. return dp_tx_send(soc_hdl, vdev_id, nbuf);
  3426. }
  3427. #ifdef UMAC_SUPPORT_PROXY_ARP
  3428. /**
  3429. * dp_tx_proxy_arp() - Tx proxy arp handler
  3430. * @vdev: datapath vdev handle
  3431. * @buf: sk buffer
  3432. *
  3433. * Return: status
  3434. */
  3435. static inline
  3436. int dp_tx_proxy_arp(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3437. {
  3438. if (vdev->osif_proxy_arp)
  3439. return vdev->osif_proxy_arp(vdev->osif_vdev, nbuf);
  3440. /*
  3441. * when UMAC_SUPPORT_PROXY_ARP is defined, we expect
  3442. * osif_proxy_arp has a valid function pointer assigned
  3443. * to it
  3444. */
  3445. dp_tx_err("valid function pointer for osif_proxy_arp is expected!!\n");
  3446. return QDF_STATUS_NOT_INITIALIZED;
  3447. }
  3448. #else
  3449. /**
  3450. * dp_tx_proxy_arp() - Tx proxy arp handler
  3451. * @vdev: datapath vdev handle
  3452. * @buf: sk buffer
  3453. *
  3454. * This function always return 0 when UMAC_SUPPORT_PROXY_ARP
  3455. * is not defined.
  3456. *
  3457. * Return: status
  3458. */
  3459. static inline
  3460. int dp_tx_proxy_arp(struct dp_vdev *vdev, qdf_nbuf_t nbuf)
  3461. {
  3462. return QDF_STATUS_SUCCESS;
  3463. }
  3464. #endif
  3465. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP)
  3466. #ifdef WLAN_MCAST_MLO
  3467. static bool
  3468. dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
  3469. struct dp_tx_desc_s *tx_desc,
  3470. qdf_nbuf_t nbuf,
  3471. uint8_t reinject_reason)
  3472. {
  3473. if (reinject_reason == HTT_TX_FW2WBM_REINJECT_REASON_MLO_MCAST) {
  3474. if (soc->arch_ops.dp_tx_mcast_handler)
  3475. soc->arch_ops.dp_tx_mcast_handler(soc, vdev, nbuf);
  3476. dp_tx_desc_release(tx_desc, tx_desc->pool_id);
  3477. return true;
  3478. }
  3479. return false;
  3480. }
  3481. #else /* WLAN_MCAST_MLO */
  3482. static inline bool
  3483. dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
  3484. struct dp_tx_desc_s *tx_desc,
  3485. qdf_nbuf_t nbuf,
  3486. uint8_t reinject_reason)
  3487. {
  3488. return false;
  3489. }
  3490. #endif /* WLAN_MCAST_MLO */
  3491. #else
  3492. static inline bool
  3493. dp_tx_reinject_mlo_hdl(struct dp_soc *soc, struct dp_vdev *vdev,
  3494. struct dp_tx_desc_s *tx_desc,
  3495. qdf_nbuf_t nbuf,
  3496. uint8_t reinject_reason)
  3497. {
  3498. return false;
  3499. }
  3500. #endif
  3501. /**
  3502. * dp_tx_reinject_handler() - Tx Reinject Handler
  3503. * @soc: datapath soc handle
  3504. * @vdev: datapath vdev handle
  3505. * @tx_desc: software descriptor head pointer
  3506. * @status : Tx completion status from HTT descriptor
  3507. * @reinject_reason : reinject reason from HTT descriptor
  3508. *
  3509. * This function reinjects frames back to Target.
  3510. * Todo - Host queue needs to be added
  3511. *
  3512. * Return: none
  3513. */
  3514. void dp_tx_reinject_handler(struct dp_soc *soc,
  3515. struct dp_vdev *vdev,
  3516. struct dp_tx_desc_s *tx_desc,
  3517. uint8_t *status,
  3518. uint8_t reinject_reason)
  3519. {
  3520. struct dp_peer *peer = NULL;
  3521. uint32_t peer_id = HTT_INVALID_PEER;
  3522. qdf_nbuf_t nbuf = tx_desc->nbuf;
  3523. qdf_nbuf_t nbuf_copy = NULL;
  3524. struct dp_tx_msdu_info_s msdu_info;
  3525. #ifdef WDS_VENDOR_EXTENSION
  3526. int is_mcast = 0, is_ucast = 0;
  3527. int num_peers_3addr = 0;
  3528. qdf_ether_header_t *eth_hdr = (qdf_ether_header_t *)(qdf_nbuf_data(nbuf));
  3529. struct ieee80211_frame_addr4 *wh = (struct ieee80211_frame_addr4 *)(qdf_nbuf_data(nbuf));
  3530. #endif
  3531. struct dp_txrx_peer *txrx_peer;
  3532. qdf_assert(vdev);
  3533. dp_tx_debug("Tx reinject path");
  3534. DP_STATS_INC_PKT(vdev, tx_i.reinject_pkts, 1,
  3535. qdf_nbuf_len(tx_desc->nbuf));
  3536. if (dp_tx_reinject_mlo_hdl(soc, vdev, tx_desc, nbuf, reinject_reason))
  3537. return;
  3538. #ifdef WDS_VENDOR_EXTENSION
  3539. if (qdf_unlikely(vdev->tx_encap_type != htt_cmn_pkt_type_raw)) {
  3540. is_mcast = (IS_MULTICAST(wh->i_addr1)) ? 1 : 0;
  3541. } else {
  3542. is_mcast = (IS_MULTICAST(eth_hdr->ether_dhost)) ? 1 : 0;
  3543. }
  3544. is_ucast = !is_mcast;
  3545. qdf_spin_lock_bh(&vdev->peer_list_lock);
  3546. TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
  3547. txrx_peer = dp_get_txrx_peer(peer);
  3548. if (!txrx_peer || txrx_peer->bss_peer)
  3549. continue;
  3550. /* Detect wds peers that use 3-addr framing for mcast.
  3551. * if there are any, the bss_peer is used to send the
  3552. * the mcast frame using 3-addr format. all wds enabled
  3553. * peers that use 4-addr framing for mcast frames will
  3554. * be duplicated and sent as 4-addr frames below.
  3555. */
  3556. if (!txrx_peer->wds_enabled ||
  3557. !txrx_peer->wds_ecm.wds_tx_mcast_4addr) {
  3558. num_peers_3addr = 1;
  3559. break;
  3560. }
  3561. }
  3562. qdf_spin_unlock_bh(&vdev->peer_list_lock);
  3563. #endif
  3564. if (qdf_unlikely(vdev->mesh_vdev)) {
  3565. DP_TX_FREE_SINGLE_BUF(vdev->pdev->soc, tx_desc->nbuf);
  3566. } else {
  3567. qdf_spin_lock_bh(&vdev->peer_list_lock);
  3568. TAILQ_FOREACH(peer, &vdev->peer_list, peer_list_elem) {
  3569. txrx_peer = dp_get_txrx_peer(peer);
  3570. if (!txrx_peer)
  3571. continue;
  3572. if ((txrx_peer->peer_id != HTT_INVALID_PEER) &&
  3573. #ifdef WDS_VENDOR_EXTENSION
  3574. /*
  3575. * . if 3-addr STA, then send on BSS Peer
  3576. * . if Peer WDS enabled and accept 4-addr mcast,
  3577. * send mcast on that peer only
  3578. * . if Peer WDS enabled and accept 4-addr ucast,
  3579. * send ucast on that peer only
  3580. */
  3581. ((txrx_peer->bss_peer && num_peers_3addr && is_mcast) ||
  3582. (txrx_peer->wds_enabled &&
  3583. ((is_mcast && txrx_peer->wds_ecm.wds_tx_mcast_4addr) ||
  3584. (is_ucast &&
  3585. txrx_peer->wds_ecm.wds_tx_ucast_4addr))))) {
  3586. #else
  3587. (txrx_peer->bss_peer &&
  3588. (dp_tx_proxy_arp(vdev, nbuf) == QDF_STATUS_SUCCESS))) {
  3589. #endif
  3590. peer_id = DP_INVALID_PEER;
  3591. nbuf_copy = qdf_nbuf_copy(nbuf);
  3592. if (!nbuf_copy) {
  3593. dp_tx_debug("nbuf copy failed");
  3594. break;
  3595. }
  3596. qdf_mem_zero(&msdu_info, sizeof(msdu_info));
  3597. dp_tx_get_queue(vdev, nbuf,
  3598. &msdu_info.tx_queue);
  3599. nbuf_copy = dp_tx_send_msdu_single(vdev,
  3600. nbuf_copy,
  3601. &msdu_info,
  3602. peer_id,
  3603. NULL);
  3604. if (nbuf_copy) {
  3605. dp_tx_debug("pkt send failed");
  3606. qdf_nbuf_free(nbuf_copy);
  3607. }
  3608. }
  3609. }
  3610. qdf_spin_unlock_bh(&vdev->peer_list_lock);
  3611. qdf_nbuf_unmap_nbytes_single(vdev->osdev, nbuf,
  3612. QDF_DMA_TO_DEVICE, nbuf->len);
  3613. qdf_nbuf_free(nbuf);
  3614. }
  3615. dp_tx_desc_release(tx_desc, tx_desc->pool_id);
  3616. }
  3617. /**
  3618. * dp_tx_inspect_handler() - Tx Inspect Handler
  3619. * @soc: datapath soc handle
  3620. * @vdev: datapath vdev handle
  3621. * @tx_desc: software descriptor head pointer
  3622. * @status : Tx completion status from HTT descriptor
  3623. *
  3624. * Handles Tx frames sent back to Host for inspection
  3625. * (ProxyARP)
  3626. *
  3627. * Return: none
  3628. */
  3629. void dp_tx_inspect_handler(struct dp_soc *soc,
  3630. struct dp_vdev *vdev,
  3631. struct dp_tx_desc_s *tx_desc,
  3632. uint8_t *status)
  3633. {
  3634. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO,
  3635. "%s Tx inspect path",
  3636. __func__);
  3637. DP_STATS_INC_PKT(vdev, tx_i.inspect_pkts, 1,
  3638. qdf_nbuf_len(tx_desc->nbuf));
  3639. DP_TX_FREE_SINGLE_BUF(soc, tx_desc->nbuf);
  3640. dp_tx_desc_release(tx_desc, tx_desc->pool_id);
  3641. }
  3642. #ifdef MESH_MODE_SUPPORT
  3643. /**
  3644. * dp_tx_comp_fill_tx_completion_stats() - Fill per packet Tx completion stats
  3645. * in mesh meta header
  3646. * @tx_desc: software descriptor head pointer
  3647. * @ts: pointer to tx completion stats
  3648. * Return: none
  3649. */
  3650. static
  3651. void dp_tx_comp_fill_tx_completion_stats(struct dp_tx_desc_s *tx_desc,
  3652. struct hal_tx_completion_status *ts)
  3653. {
  3654. qdf_nbuf_t netbuf = tx_desc->nbuf;
  3655. if (!tx_desc->msdu_ext_desc) {
  3656. if (qdf_nbuf_pull_head(netbuf, tx_desc->pkt_offset) == NULL) {
  3657. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  3658. "netbuf %pK offset %d",
  3659. netbuf, tx_desc->pkt_offset);
  3660. return;
  3661. }
  3662. }
  3663. }
  3664. #else
  3665. static
  3666. void dp_tx_comp_fill_tx_completion_stats(struct dp_tx_desc_s *tx_desc,
  3667. struct hal_tx_completion_status *ts)
  3668. {
  3669. }
  3670. #endif
  3671. #ifdef CONFIG_SAWF
  3672. static void dp_tx_update_peer_sawf_stats(struct dp_soc *soc,
  3673. struct dp_vdev *vdev,
  3674. struct dp_txrx_peer *txrx_peer,
  3675. struct dp_tx_desc_s *tx_desc,
  3676. struct hal_tx_completion_status *ts,
  3677. uint8_t tid)
  3678. {
  3679. dp_sawf_tx_compl_update_peer_stats(soc, vdev, txrx_peer, tx_desc,
  3680. ts, tid);
  3681. }
  3682. static void dp_tx_compute_delay_avg(struct cdp_delay_tx_stats *tx_delay,
  3683. uint32_t nw_delay,
  3684. uint32_t sw_delay,
  3685. uint32_t hw_delay)
  3686. {
  3687. dp_peer_tid_delay_avg(tx_delay,
  3688. nw_delay,
  3689. sw_delay,
  3690. hw_delay);
  3691. }
  3692. #else
  3693. static void dp_tx_update_peer_sawf_stats(struct dp_soc *soc,
  3694. struct dp_vdev *vdev,
  3695. struct dp_txrx_peer *txrx_peer,
  3696. struct dp_tx_desc_s *tx_desc,
  3697. struct hal_tx_completion_status *ts,
  3698. uint8_t tid)
  3699. {
  3700. }
  3701. static inline void
  3702. dp_tx_compute_delay_avg(struct cdp_delay_tx_stats *tx_delay,
  3703. uint32_t nw_delay, uint32_t sw_delay,
  3704. uint32_t hw_delay)
  3705. {
  3706. }
  3707. #endif
  3708. #ifdef QCA_PEER_EXT_STATS
  3709. #ifdef WLAN_CONFIG_TX_DELAY
  3710. static void dp_tx_compute_tid_delay(struct cdp_delay_tid_stats *stats,
  3711. struct dp_tx_desc_s *tx_desc,
  3712. struct hal_tx_completion_status *ts,
  3713. struct dp_vdev *vdev)
  3714. {
  3715. struct dp_soc *soc = vdev->pdev->soc;
  3716. struct cdp_delay_tx_stats *tx_delay = &stats->tx_delay;
  3717. int64_t timestamp_ingress, timestamp_hw_enqueue;
  3718. uint32_t sw_enqueue_delay, fwhw_transmit_delay = 0;
  3719. if (!ts->valid)
  3720. return;
  3721. timestamp_ingress = qdf_nbuf_get_timestamp_us(tx_desc->nbuf);
  3722. timestamp_hw_enqueue = qdf_ktime_to_us(tx_desc->timestamp);
  3723. sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
  3724. dp_hist_update_stats(&tx_delay->tx_swq_delay, sw_enqueue_delay);
  3725. if (soc->arch_ops.dp_tx_compute_hw_delay)
  3726. if (!soc->arch_ops.dp_tx_compute_hw_delay(soc, vdev, ts,
  3727. &fwhw_transmit_delay))
  3728. dp_hist_update_stats(&tx_delay->hwtx_delay,
  3729. fwhw_transmit_delay);
  3730. dp_tx_compute_delay_avg(tx_delay, 0, sw_enqueue_delay,
  3731. fwhw_transmit_delay);
  3732. }
  3733. #else
  3734. /*
  3735. * dp_tx_compute_tid_delay() - Compute per TID delay
  3736. * @stats: Per TID delay stats
  3737. * @tx_desc: Software Tx descriptor
  3738. * @ts: Tx completion status
  3739. * @vdev: vdev
  3740. *
  3741. * Compute the software enqueue and hw enqueue delays and
  3742. * update the respective histograms
  3743. *
  3744. * Return: void
  3745. */
  3746. static void dp_tx_compute_tid_delay(struct cdp_delay_tid_stats *stats,
  3747. struct dp_tx_desc_s *tx_desc,
  3748. struct hal_tx_completion_status *ts,
  3749. struct dp_vdev *vdev)
  3750. {
  3751. struct cdp_delay_tx_stats *tx_delay = &stats->tx_delay;
  3752. int64_t current_timestamp, timestamp_ingress, timestamp_hw_enqueue;
  3753. uint32_t sw_enqueue_delay, fwhw_transmit_delay;
  3754. current_timestamp = qdf_ktime_to_ms(qdf_ktime_real_get());
  3755. timestamp_ingress = qdf_nbuf_get_timestamp(tx_desc->nbuf);
  3756. timestamp_hw_enqueue = qdf_ktime_to_ms(tx_desc->timestamp);
  3757. sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
  3758. fwhw_transmit_delay = (uint32_t)(current_timestamp -
  3759. timestamp_hw_enqueue);
  3760. /*
  3761. * Update the Tx software enqueue delay and HW enque-Completion delay.
  3762. */
  3763. dp_hist_update_stats(&tx_delay->tx_swq_delay, sw_enqueue_delay);
  3764. dp_hist_update_stats(&tx_delay->hwtx_delay, fwhw_transmit_delay);
  3765. }
  3766. #endif
  3767. /*
  3768. * dp_tx_update_peer_delay_stats() - Update the peer delay stats
  3769. * @txrx_peer: DP peer context
  3770. * @tx_desc: Tx software descriptor
  3771. * @tid: Transmission ID
  3772. * @ring_id: Rx CPU context ID/CPU_ID
  3773. *
  3774. * Update the peer extended stats. These are enhanced other
  3775. * delay stats per msdu level.
  3776. *
  3777. * Return: void
  3778. */
  3779. static void dp_tx_update_peer_delay_stats(struct dp_txrx_peer *txrx_peer,
  3780. struct dp_tx_desc_s *tx_desc,
  3781. struct hal_tx_completion_status *ts,
  3782. uint8_t ring_id)
  3783. {
  3784. struct dp_pdev *pdev = txrx_peer->vdev->pdev;
  3785. struct dp_soc *soc = NULL;
  3786. struct dp_peer_delay_stats *delay_stats = NULL;
  3787. uint8_t tid;
  3788. soc = pdev->soc;
  3789. if (qdf_likely(!wlan_cfg_is_peer_ext_stats_enabled(soc->wlan_cfg_ctx)))
  3790. return;
  3791. if (!txrx_peer->delay_stats)
  3792. return;
  3793. tid = ts->tid;
  3794. delay_stats = txrx_peer->delay_stats;
  3795. qdf_assert(ring < CDP_MAX_TXRX_CTX);
  3796. /*
  3797. * For non-TID packets use the TID 9
  3798. */
  3799. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  3800. tid = CDP_MAX_DATA_TIDS - 1;
  3801. dp_tx_compute_tid_delay(&delay_stats->delay_tid_stats[tid][ring_id],
  3802. tx_desc, ts, txrx_peer->vdev);
  3803. }
  3804. #else
  3805. static inline
  3806. void dp_tx_update_peer_delay_stats(struct dp_txrx_peer *txrx_peer,
  3807. struct dp_tx_desc_s *tx_desc,
  3808. struct hal_tx_completion_status *ts,
  3809. uint8_t ring_id)
  3810. {
  3811. }
  3812. #endif
  3813. #ifdef WLAN_PEER_JITTER
  3814. /*
  3815. * dp_tx_jitter_get_avg_jitter() - compute the average jitter
  3816. * @curr_delay: Current delay
  3817. * @prev_Delay: Previous delay
  3818. * @avg_jitter: Average Jitter
  3819. * Return: Newly Computed Average Jitter
  3820. */
  3821. static uint32_t dp_tx_jitter_get_avg_jitter(uint32_t curr_delay,
  3822. uint32_t prev_delay,
  3823. uint32_t avg_jitter)
  3824. {
  3825. uint32_t curr_jitter;
  3826. int32_t jitter_diff;
  3827. curr_jitter = qdf_abs(curr_delay - prev_delay);
  3828. if (!avg_jitter)
  3829. return curr_jitter;
  3830. jitter_diff = curr_jitter - avg_jitter;
  3831. if (jitter_diff < 0)
  3832. avg_jitter = avg_jitter -
  3833. (qdf_abs(jitter_diff) >> DP_AVG_JITTER_WEIGHT_DENOM);
  3834. else
  3835. avg_jitter = avg_jitter +
  3836. (qdf_abs(jitter_diff) >> DP_AVG_JITTER_WEIGHT_DENOM);
  3837. return avg_jitter;
  3838. }
  3839. /*
  3840. * dp_tx_jitter_get_avg_delay() - compute the average delay
  3841. * @curr_delay: Current delay
  3842. * @avg_Delay: Average delay
  3843. * Return: Newly Computed Average Delay
  3844. */
  3845. static uint32_t dp_tx_jitter_get_avg_delay(uint32_t curr_delay,
  3846. uint32_t avg_delay)
  3847. {
  3848. int32_t delay_diff;
  3849. if (!avg_delay)
  3850. return curr_delay;
  3851. delay_diff = curr_delay - avg_delay;
  3852. if (delay_diff < 0)
  3853. avg_delay = avg_delay - (qdf_abs(delay_diff) >>
  3854. DP_AVG_DELAY_WEIGHT_DENOM);
  3855. else
  3856. avg_delay = avg_delay + (qdf_abs(delay_diff) >>
  3857. DP_AVG_DELAY_WEIGHT_DENOM);
  3858. return avg_delay;
  3859. }
  3860. #ifdef WLAN_CONFIG_TX_DELAY
  3861. /*
  3862. * dp_tx_compute_cur_delay() - get the current delay
  3863. * @soc: soc handle
  3864. * @vdev: vdev structure for data path state
  3865. * @ts: Tx completion status
  3866. * @curr_delay: current delay
  3867. * @tx_desc: tx descriptor
  3868. * Return: void
  3869. */
  3870. static
  3871. QDF_STATUS dp_tx_compute_cur_delay(struct dp_soc *soc,
  3872. struct dp_vdev *vdev,
  3873. struct hal_tx_completion_status *ts,
  3874. uint32_t *curr_delay,
  3875. struct dp_tx_desc_s *tx_desc)
  3876. {
  3877. QDF_STATUS status = QDF_STATUS_E_FAILURE;
  3878. if (soc->arch_ops.dp_tx_compute_hw_delay)
  3879. status = soc->arch_ops.dp_tx_compute_hw_delay(soc, vdev, ts,
  3880. curr_delay);
  3881. return status;
  3882. }
  3883. #else
  3884. static
  3885. QDF_STATUS dp_tx_compute_cur_delay(struct dp_soc *soc,
  3886. struct dp_vdev *vdev,
  3887. struct hal_tx_completion_status *ts,
  3888. uint32_t *curr_delay,
  3889. struct dp_tx_desc_s *tx_desc)
  3890. {
  3891. int64_t current_timestamp, timestamp_hw_enqueue;
  3892. current_timestamp = qdf_ktime_to_us(qdf_ktime_real_get());
  3893. timestamp_hw_enqueue = qdf_ktime_to_us(tx_desc->timestamp);
  3894. *curr_delay = (uint32_t)(current_timestamp - timestamp_hw_enqueue);
  3895. return QDF_STATUS_SUCCESS;
  3896. }
  3897. #endif
  3898. /* dp_tx_compute_tid_jitter() - compute per tid per ring jitter
  3899. * @jiiter - per tid per ring jitter stats
  3900. * @ts: Tx completion status
  3901. * @vdev - vdev structure for data path state
  3902. * @tx_desc - tx descriptor
  3903. * Return: void
  3904. */
  3905. static void dp_tx_compute_tid_jitter(struct cdp_peer_tid_stats *jitter,
  3906. struct hal_tx_completion_status *ts,
  3907. struct dp_vdev *vdev,
  3908. struct dp_tx_desc_s *tx_desc)
  3909. {
  3910. uint32_t curr_delay, avg_delay, avg_jitter, prev_delay;
  3911. struct dp_soc *soc = vdev->pdev->soc;
  3912. QDF_STATUS status = QDF_STATUS_E_FAILURE;
  3913. if (ts->status != HAL_TX_TQM_RR_FRAME_ACKED) {
  3914. jitter->tx_drop += 1;
  3915. return;
  3916. }
  3917. status = dp_tx_compute_cur_delay(soc, vdev, ts, &curr_delay,
  3918. tx_desc);
  3919. if (QDF_IS_STATUS_SUCCESS(status)) {
  3920. avg_delay = jitter->tx_avg_delay;
  3921. avg_jitter = jitter->tx_avg_jitter;
  3922. prev_delay = jitter->tx_prev_delay;
  3923. avg_jitter = dp_tx_jitter_get_avg_jitter(curr_delay,
  3924. prev_delay,
  3925. avg_jitter);
  3926. avg_delay = dp_tx_jitter_get_avg_delay(curr_delay, avg_delay);
  3927. jitter->tx_avg_delay = avg_delay;
  3928. jitter->tx_avg_jitter = avg_jitter;
  3929. jitter->tx_prev_delay = curr_delay;
  3930. jitter->tx_total_success += 1;
  3931. } else if (status == QDF_STATUS_E_FAILURE) {
  3932. jitter->tx_avg_err += 1;
  3933. }
  3934. }
  3935. /* dp_tx_update_peer_jitter_stats() - Update the peer jitter stats
  3936. * @txrx_peer: DP peer context
  3937. * @tx_desc: Tx software descriptor
  3938. * @ts: Tx completion status
  3939. * @ring_id: Rx CPU context ID/CPU_ID
  3940. * Return: void
  3941. */
  3942. static void dp_tx_update_peer_jitter_stats(struct dp_txrx_peer *txrx_peer,
  3943. struct dp_tx_desc_s *tx_desc,
  3944. struct hal_tx_completion_status *ts,
  3945. uint8_t ring_id)
  3946. {
  3947. struct dp_pdev *pdev = txrx_peer->vdev->pdev;
  3948. struct dp_soc *soc = pdev->soc;
  3949. struct cdp_peer_tid_stats *jitter_stats = NULL;
  3950. uint8_t tid;
  3951. struct cdp_peer_tid_stats *rx_tid = NULL;
  3952. if (qdf_likely(!wlan_cfg_is_peer_jitter_stats_enabled(soc->wlan_cfg_ctx)))
  3953. return;
  3954. tid = ts->tid;
  3955. jitter_stats = txrx_peer->jitter_stats;
  3956. qdf_assert_always(jitter_stats);
  3957. qdf_assert(ring < CDP_MAX_TXRX_CTX);
  3958. /*
  3959. * For non-TID packets use the TID 9
  3960. */
  3961. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  3962. tid = CDP_MAX_DATA_TIDS - 1;
  3963. rx_tid = &jitter_stats[tid * CDP_MAX_TXRX_CTX + ring_id];
  3964. dp_tx_compute_tid_jitter(rx_tid,
  3965. ts, txrx_peer->vdev, tx_desc);
  3966. }
  3967. #else
  3968. static void dp_tx_update_peer_jitter_stats(struct dp_txrx_peer *txrx_peer,
  3969. struct dp_tx_desc_s *tx_desc,
  3970. struct hal_tx_completion_status *ts,
  3971. uint8_t ring_id)
  3972. {
  3973. }
  3974. #endif
  3975. #ifdef HW_TX_DELAY_STATS_ENABLE
  3976. /**
  3977. * dp_update_tx_delay_stats() - update the delay stats
  3978. * @vdev: vdev handle
  3979. * @delay: delay in ms or us based on the flag delay_in_us
  3980. * @tid: tid value
  3981. * @mode: type of tx delay mode
  3982. * @ring id: ring number
  3983. * @delay_in_us: flag to indicate whether the delay is in ms or us
  3984. *
  3985. * Return: none
  3986. */
  3987. static inline
  3988. void dp_update_tx_delay_stats(struct dp_vdev *vdev, uint32_t delay, uint8_t tid,
  3989. uint8_t mode, uint8_t ring_id, bool delay_in_us)
  3990. {
  3991. struct cdp_tid_tx_stats *tstats =
  3992. &vdev->stats.tid_tx_stats[ring_id][tid];
  3993. dp_update_delay_stats(tstats, NULL, delay, tid, mode, ring_id,
  3994. delay_in_us);
  3995. }
  3996. #else
  3997. static inline
  3998. void dp_update_tx_delay_stats(struct dp_vdev *vdev, uint32_t delay, uint8_t tid,
  3999. uint8_t mode, uint8_t ring_id, bool delay_in_us)
  4000. {
  4001. struct cdp_tid_tx_stats *tstats =
  4002. &vdev->pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
  4003. dp_update_delay_stats(tstats, NULL, delay, tid, mode, ring_id,
  4004. delay_in_us);
  4005. }
  4006. #endif
  4007. /**
  4008. * dp_tx_compute_delay() - Compute and fill in all timestamps
  4009. * to pass in correct fields
  4010. *
  4011. * @vdev: pdev handle
  4012. * @tx_desc: tx descriptor
  4013. * @tid: tid value
  4014. * @ring_id: TCL or WBM ring number for transmit path
  4015. * Return: none
  4016. */
  4017. void dp_tx_compute_delay(struct dp_vdev *vdev, struct dp_tx_desc_s *tx_desc,
  4018. uint8_t tid, uint8_t ring_id)
  4019. {
  4020. int64_t current_timestamp, timestamp_ingress, timestamp_hw_enqueue;
  4021. uint32_t sw_enqueue_delay, fwhw_transmit_delay, interframe_delay;
  4022. uint32_t fwhw_transmit_delay_us;
  4023. if (qdf_likely(!vdev->pdev->delay_stats_flag) &&
  4024. qdf_likely(!dp_is_vdev_tx_delay_stats_enabled(vdev)))
  4025. return;
  4026. if (dp_is_vdev_tx_delay_stats_enabled(vdev)) {
  4027. fwhw_transmit_delay_us =
  4028. qdf_ktime_to_us(qdf_ktime_real_get()) -
  4029. qdf_ktime_to_us(tx_desc->timestamp);
  4030. /*
  4031. * Delay between packet enqueued to HW and Tx completion in us
  4032. */
  4033. dp_update_tx_delay_stats(vdev, fwhw_transmit_delay_us, tid,
  4034. CDP_DELAY_STATS_FW_HW_TRANSMIT,
  4035. ring_id, true);
  4036. /*
  4037. * For MCL, only enqueue to completion delay is required
  4038. * so return if the vdev flag is enabled.
  4039. */
  4040. return;
  4041. }
  4042. current_timestamp = qdf_ktime_to_ms(qdf_ktime_real_get());
  4043. timestamp_hw_enqueue = qdf_ktime_to_ms(tx_desc->timestamp);
  4044. fwhw_transmit_delay = (uint32_t)(current_timestamp -
  4045. timestamp_hw_enqueue);
  4046. /*
  4047. * Delay between packet enqueued to HW and Tx completion in ms
  4048. */
  4049. dp_update_tx_delay_stats(vdev, fwhw_transmit_delay, tid,
  4050. CDP_DELAY_STATS_FW_HW_TRANSMIT, ring_id,
  4051. false);
  4052. timestamp_ingress = qdf_nbuf_get_timestamp(tx_desc->nbuf);
  4053. sw_enqueue_delay = (uint32_t)(timestamp_hw_enqueue - timestamp_ingress);
  4054. interframe_delay = (uint32_t)(timestamp_ingress -
  4055. vdev->prev_tx_enq_tstamp);
  4056. /*
  4057. * Delay in software enqueue
  4058. */
  4059. dp_update_tx_delay_stats(vdev, sw_enqueue_delay, tid,
  4060. CDP_DELAY_STATS_SW_ENQ, ring_id,
  4061. false);
  4062. /*
  4063. * Update interframe delay stats calculated at hardstart receive point.
  4064. * Value of vdev->prev_tx_enq_tstamp will be 0 for 1st frame, so
  4065. * interframe delay will not be calculate correctly for 1st frame.
  4066. * On the other side, this will help in avoiding extra per packet check
  4067. * of !vdev->prev_tx_enq_tstamp.
  4068. */
  4069. dp_update_tx_delay_stats(vdev, interframe_delay, tid,
  4070. CDP_DELAY_STATS_TX_INTERFRAME, ring_id,
  4071. false);
  4072. vdev->prev_tx_enq_tstamp = timestamp_ingress;
  4073. }
  4074. #ifdef DISABLE_DP_STATS
  4075. static
  4076. inline void dp_update_no_ack_stats(qdf_nbuf_t nbuf,
  4077. struct dp_txrx_peer *txrx_peer)
  4078. {
  4079. }
  4080. #else
  4081. static inline void
  4082. dp_update_no_ack_stats(qdf_nbuf_t nbuf, struct dp_txrx_peer *txrx_peer)
  4083. {
  4084. enum qdf_proto_subtype subtype = QDF_PROTO_INVALID;
  4085. DPTRACE(qdf_dp_track_noack_check(nbuf, &subtype));
  4086. if (subtype != QDF_PROTO_INVALID)
  4087. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.no_ack_count[subtype],
  4088. 1);
  4089. }
  4090. #endif
  4091. #ifndef QCA_ENHANCED_STATS_SUPPORT
  4092. #ifdef DP_PEER_EXTENDED_API
  4093. static inline uint8_t
  4094. dp_tx_get_mpdu_retry_threshold(struct dp_txrx_peer *txrx_peer)
  4095. {
  4096. return txrx_peer->mpdu_retry_threshold;
  4097. }
  4098. #else
  4099. static inline uint8_t
  4100. dp_tx_get_mpdu_retry_threshold(struct dp_txrx_peer *txrx_peer)
  4101. {
  4102. return 0;
  4103. }
  4104. #endif
  4105. /**
  4106. * dp_tx_update_peer_extd_stats()- Update Tx extended path stats for peer
  4107. *
  4108. * @ts: Tx compltion status
  4109. * @txrx_peer: datapath txrx_peer handle
  4110. *
  4111. * Return: void
  4112. */
  4113. static inline void
  4114. dp_tx_update_peer_extd_stats(struct hal_tx_completion_status *ts,
  4115. struct dp_txrx_peer *txrx_peer)
  4116. {
  4117. uint8_t mcs, pkt_type, dst_mcs_idx;
  4118. uint8_t retry_threshold = dp_tx_get_mpdu_retry_threshold(txrx_peer);
  4119. mcs = ts->mcs;
  4120. pkt_type = ts->pkt_type;
  4121. /* do HW to SW pkt type conversion */
  4122. pkt_type = (pkt_type >= HAL_DOT11_MAX ? DOT11_MAX :
  4123. hal_2_dp_pkt_type_map[pkt_type]);
  4124. dst_mcs_idx = dp_get_mcs_array_index_by_pkt_type_mcs(pkt_type, mcs);
  4125. if (MCS_INVALID_ARRAY_INDEX != dst_mcs_idx)
  4126. DP_PEER_EXTD_STATS_INC(txrx_peer,
  4127. tx.pkt_type[pkt_type].mcs_count[dst_mcs_idx],
  4128. 1);
  4129. DP_PEER_EXTD_STATS_INC(txrx_peer, tx.sgi_count[ts->sgi], 1);
  4130. DP_PEER_EXTD_STATS_INC(txrx_peer, tx.bw[ts->bw], 1);
  4131. DP_PEER_EXTD_STATS_UPD(txrx_peer, tx.last_ack_rssi, ts->ack_frame_rssi);
  4132. DP_PEER_EXTD_STATS_INC(txrx_peer,
  4133. tx.wme_ac_type[TID_TO_WME_AC(ts->tid)], 1);
  4134. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.stbc, 1, ts->stbc);
  4135. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.ldpc, 1, ts->ldpc);
  4136. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.retries, 1, ts->transmit_cnt > 1);
  4137. if (ts->first_msdu) {
  4138. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.retries_mpdu, 1,
  4139. ts->transmit_cnt > 1);
  4140. if (!retry_threshold)
  4141. return;
  4142. DP_PEER_EXTD_STATS_INCC(txrx_peer, tx.mpdu_success_with_retries,
  4143. qdf_do_div(ts->transmit_cnt,
  4144. retry_threshold),
  4145. ts->transmit_cnt > retry_threshold);
  4146. }
  4147. }
  4148. #else
  4149. static inline void
  4150. dp_tx_update_peer_extd_stats(struct hal_tx_completion_status *ts,
  4151. struct dp_txrx_peer *txrx_peer)
  4152. {
  4153. }
  4154. #endif
  4155. /**
  4156. * dp_tx_update_peer_stats() - Update peer stats from Tx completion indications
  4157. * per wbm ring
  4158. *
  4159. * @tx_desc: software descriptor head pointer
  4160. * @ts: Tx completion status
  4161. * @peer: peer handle
  4162. * @ring_id: ring number
  4163. *
  4164. * Return: None
  4165. */
  4166. static inline void
  4167. dp_tx_update_peer_stats(struct dp_tx_desc_s *tx_desc,
  4168. struct hal_tx_completion_status *ts,
  4169. struct dp_txrx_peer *txrx_peer, uint8_t ring_id)
  4170. {
  4171. struct dp_pdev *pdev = txrx_peer->vdev->pdev;
  4172. uint8_t tid = ts->tid;
  4173. uint32_t length;
  4174. struct cdp_tid_tx_stats *tid_stats;
  4175. if (!pdev)
  4176. return;
  4177. if (qdf_unlikely(tid >= CDP_MAX_DATA_TIDS))
  4178. tid = CDP_MAX_DATA_TIDS - 1;
  4179. tid_stats = &pdev->stats.tid_stats.tid_tx_stats[ring_id][tid];
  4180. if (ts->release_src != HAL_TX_COMP_RELEASE_SOURCE_TQM) {
  4181. dp_err_rl("Release source:%d is not from TQM", ts->release_src);
  4182. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.release_src_not_tqm, 1);
  4183. return;
  4184. }
  4185. length = qdf_nbuf_len(tx_desc->nbuf);
  4186. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  4187. if (qdf_unlikely(pdev->delay_stats_flag) ||
  4188. qdf_unlikely(dp_is_vdev_tx_delay_stats_enabled(txrx_peer->vdev)))
  4189. dp_tx_compute_delay(txrx_peer->vdev, tx_desc, tid, ring_id);
  4190. if (ts->status < CDP_MAX_TX_TQM_STATUS) {
  4191. tid_stats->tqm_status_cnt[ts->status]++;
  4192. }
  4193. if (qdf_likely(ts->status == HAL_TX_TQM_RR_FRAME_ACKED)) {
  4194. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.retry_count, 1,
  4195. ts->transmit_cnt > 1);
  4196. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.multiple_retry_count,
  4197. 1, ts->transmit_cnt > 2);
  4198. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.ofdma, 1, ts->ofdma);
  4199. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.amsdu_cnt, 1,
  4200. ts->msdu_part_of_amsdu);
  4201. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.non_amsdu_cnt, 1,
  4202. !ts->msdu_part_of_amsdu);
  4203. txrx_peer->stats.per_pkt_stats.tx.last_tx_ts =
  4204. qdf_system_ticks();
  4205. dp_tx_update_peer_extd_stats(ts, txrx_peer);
  4206. return;
  4207. }
  4208. /*
  4209. * tx_failed is ideally supposed to be updated from HTT ppdu
  4210. * completion stats. But in IPQ807X/IPQ6018 chipsets owing to
  4211. * hw limitation there are no completions for failed cases.
  4212. * Hence updating tx_failed from data path. Please note that
  4213. * if tx_failed is fixed to be from ppdu, then this has to be
  4214. * removed
  4215. */
  4216. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  4217. DP_PEER_PER_PKT_STATS_INCC(txrx_peer, tx.failed_retry_count, 1,
  4218. ts->transmit_cnt > DP_RETRY_COUNT);
  4219. dp_update_no_ack_stats(tx_desc->nbuf, txrx_peer);
  4220. if (ts->status == HAL_TX_TQM_RR_REM_CMD_AGED) {
  4221. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.age_out, 1);
  4222. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_REM) {
  4223. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.dropped.fw_rem, 1,
  4224. length);
  4225. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_NOTX) {
  4226. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_rem_notx, 1);
  4227. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_TX) {
  4228. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_rem_tx, 1);
  4229. } else if (ts->status == HAL_TX_TQM_RR_FW_REASON1) {
  4230. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason1, 1);
  4231. } else if (ts->status == HAL_TX_TQM_RR_FW_REASON2) {
  4232. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason2, 1);
  4233. } else if (ts->status == HAL_TX_TQM_RR_FW_REASON3) {
  4234. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.fw_reason3, 1);
  4235. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_DISABLE_QUEUE) {
  4236. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4237. tx.dropped.fw_rem_queue_disable, 1);
  4238. } else if (ts->status == HAL_TX_TQM_RR_REM_CMD_TILL_NONMATCHING) {
  4239. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4240. tx.dropped.fw_rem_no_match, 1);
  4241. } else if (ts->status == HAL_TX_TQM_RR_DROP_THRESHOLD) {
  4242. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4243. tx.dropped.drop_threshold, 1);
  4244. } else if (ts->status == HAL_TX_TQM_RR_LINK_DESC_UNAVAILABLE) {
  4245. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4246. tx.dropped.drop_link_desc_na, 1);
  4247. } else if (ts->status == HAL_TX_TQM_RR_DROP_OR_INVALID_MSDU) {
  4248. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4249. tx.dropped.invalid_drop, 1);
  4250. } else if (ts->status == HAL_TX_TQM_RR_MULTICAST_DROP) {
  4251. DP_PEER_PER_PKT_STATS_INC(txrx_peer,
  4252. tx.dropped.mcast_vdev_drop, 1);
  4253. } else {
  4254. DP_PEER_PER_PKT_STATS_INC(txrx_peer, tx.dropped.invalid_rr, 1);
  4255. }
  4256. }
  4257. #ifdef QCA_LL_TX_FLOW_CONTROL_V2
  4258. /**
  4259. * dp_tx_flow_pool_lock() - take flow pool lock
  4260. * @soc: core txrx main context
  4261. * @tx_desc: tx desc
  4262. *
  4263. * Return: None
  4264. */
  4265. static inline
  4266. void dp_tx_flow_pool_lock(struct dp_soc *soc,
  4267. struct dp_tx_desc_s *tx_desc)
  4268. {
  4269. struct dp_tx_desc_pool_s *pool;
  4270. uint8_t desc_pool_id;
  4271. desc_pool_id = tx_desc->pool_id;
  4272. pool = &soc->tx_desc[desc_pool_id];
  4273. qdf_spin_lock_bh(&pool->flow_pool_lock);
  4274. }
  4275. /**
  4276. * dp_tx_flow_pool_unlock() - release flow pool lock
  4277. * @soc: core txrx main context
  4278. * @tx_desc: tx desc
  4279. *
  4280. * Return: None
  4281. */
  4282. static inline
  4283. void dp_tx_flow_pool_unlock(struct dp_soc *soc,
  4284. struct dp_tx_desc_s *tx_desc)
  4285. {
  4286. struct dp_tx_desc_pool_s *pool;
  4287. uint8_t desc_pool_id;
  4288. desc_pool_id = tx_desc->pool_id;
  4289. pool = &soc->tx_desc[desc_pool_id];
  4290. qdf_spin_unlock_bh(&pool->flow_pool_lock);
  4291. }
  4292. #else
  4293. static inline
  4294. void dp_tx_flow_pool_lock(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
  4295. {
  4296. }
  4297. static inline
  4298. void dp_tx_flow_pool_unlock(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc)
  4299. {
  4300. }
  4301. #endif
  4302. /**
  4303. * dp_tx_notify_completion() - Notify tx completion for this desc
  4304. * @soc: core txrx main context
  4305. * @vdev: datapath vdev handle
  4306. * @tx_desc: tx desc
  4307. * @netbuf: buffer
  4308. * @status: tx status
  4309. *
  4310. * Return: none
  4311. */
  4312. static inline void dp_tx_notify_completion(struct dp_soc *soc,
  4313. struct dp_vdev *vdev,
  4314. struct dp_tx_desc_s *tx_desc,
  4315. qdf_nbuf_t netbuf,
  4316. uint8_t status)
  4317. {
  4318. void *osif_dev;
  4319. ol_txrx_completion_fp tx_compl_cbk = NULL;
  4320. uint16_t flag = BIT(QDF_TX_RX_STATUS_DOWNLOAD_SUCC);
  4321. qdf_assert(tx_desc);
  4322. if (!vdev ||
  4323. !vdev->osif_vdev) {
  4324. return;
  4325. }
  4326. osif_dev = vdev->osif_vdev;
  4327. tx_compl_cbk = vdev->tx_comp;
  4328. if (status == HAL_TX_TQM_RR_FRAME_ACKED)
  4329. flag |= BIT(QDF_TX_RX_STATUS_OK);
  4330. if (tx_compl_cbk)
  4331. tx_compl_cbk(netbuf, osif_dev, flag);
  4332. }
  4333. /** dp_tx_sojourn_stats_process() - Collect sojourn stats
  4334. * @pdev: pdev handle
  4335. * @tid: tid value
  4336. * @txdesc_ts: timestamp from txdesc
  4337. * @ppdu_id: ppdu id
  4338. *
  4339. * Return: none
  4340. */
  4341. #ifdef FEATURE_PERPKT_INFO
  4342. static inline void dp_tx_sojourn_stats_process(struct dp_pdev *pdev,
  4343. struct dp_txrx_peer *txrx_peer,
  4344. uint8_t tid,
  4345. uint64_t txdesc_ts,
  4346. uint32_t ppdu_id)
  4347. {
  4348. uint64_t delta_ms;
  4349. struct cdp_tx_sojourn_stats *sojourn_stats;
  4350. struct dp_peer *primary_link_peer = NULL;
  4351. struct dp_soc *link_peer_soc = NULL;
  4352. if (qdf_unlikely(!pdev->enhanced_stats_en))
  4353. return;
  4354. if (qdf_unlikely(tid == HTT_INVALID_TID ||
  4355. tid >= CDP_DATA_TID_MAX))
  4356. return;
  4357. if (qdf_unlikely(!pdev->sojourn_buf))
  4358. return;
  4359. primary_link_peer = dp_get_primary_link_peer_by_id(pdev->soc,
  4360. txrx_peer->peer_id,
  4361. DP_MOD_ID_TX_COMP);
  4362. if (qdf_unlikely(!primary_link_peer))
  4363. return;
  4364. sojourn_stats = (struct cdp_tx_sojourn_stats *)
  4365. qdf_nbuf_data(pdev->sojourn_buf);
  4366. link_peer_soc = primary_link_peer->vdev->pdev->soc;
  4367. sojourn_stats->cookie = (void *)
  4368. dp_monitor_peer_get_peerstats_ctx(link_peer_soc,
  4369. primary_link_peer);
  4370. delta_ms = qdf_ktime_to_ms(qdf_ktime_real_get()) -
  4371. txdesc_ts;
  4372. qdf_ewma_tx_lag_add(&txrx_peer->stats.per_pkt_stats.tx.avg_sojourn_msdu[tid],
  4373. delta_ms);
  4374. sojourn_stats->sum_sojourn_msdu[tid] = delta_ms;
  4375. sojourn_stats->num_msdus[tid] = 1;
  4376. sojourn_stats->avg_sojourn_msdu[tid].internal =
  4377. txrx_peer->stats.per_pkt_stats.tx.avg_sojourn_msdu[tid].internal;
  4378. dp_wdi_event_handler(WDI_EVENT_TX_SOJOURN_STAT, pdev->soc,
  4379. pdev->sojourn_buf, HTT_INVALID_PEER,
  4380. WDI_NO_VAL, pdev->pdev_id);
  4381. sojourn_stats->sum_sojourn_msdu[tid] = 0;
  4382. sojourn_stats->num_msdus[tid] = 0;
  4383. sojourn_stats->avg_sojourn_msdu[tid].internal = 0;
  4384. dp_peer_unref_delete(primary_link_peer, DP_MOD_ID_TX_COMP);
  4385. }
  4386. #else
  4387. static inline void dp_tx_sojourn_stats_process(struct dp_pdev *pdev,
  4388. struct dp_txrx_peer *txrx_peer,
  4389. uint8_t tid,
  4390. uint64_t txdesc_ts,
  4391. uint32_t ppdu_id)
  4392. {
  4393. }
  4394. #endif
  4395. #ifdef WLAN_FEATURE_PKT_CAPTURE_V2
  4396. /**
  4397. * dp_send_completion_to_pkt_capture() - send tx completion to packet capture
  4398. * @soc: dp_soc handle
  4399. * @desc: Tx Descriptor
  4400. * @ts: HAL Tx completion descriptor contents
  4401. *
  4402. * This function is used to send tx completion to packet capture
  4403. */
  4404. void dp_send_completion_to_pkt_capture(struct dp_soc *soc,
  4405. struct dp_tx_desc_s *desc,
  4406. struct hal_tx_completion_status *ts)
  4407. {
  4408. dp_wdi_event_handler(WDI_EVENT_PKT_CAPTURE_TX_DATA, soc,
  4409. desc, ts->peer_id,
  4410. WDI_NO_VAL, desc->pdev->pdev_id);
  4411. }
  4412. #endif
  4413. /**
  4414. * dp_tx_comp_process_desc() - Process tx descriptor and free associated nbuf
  4415. * @soc: DP Soc handle
  4416. * @tx_desc: software Tx descriptor
  4417. * @ts : Tx completion status from HAL/HTT descriptor
  4418. *
  4419. * Return: none
  4420. */
  4421. void
  4422. dp_tx_comp_process_desc(struct dp_soc *soc,
  4423. struct dp_tx_desc_s *desc,
  4424. struct hal_tx_completion_status *ts,
  4425. struct dp_txrx_peer *txrx_peer)
  4426. {
  4427. uint64_t time_latency = 0;
  4428. uint16_t peer_id = DP_INVALID_PEER_ID;
  4429. /*
  4430. * m_copy/tx_capture modes are not supported for
  4431. * scatter gather packets
  4432. */
  4433. if (qdf_unlikely(!!desc->pdev->latency_capture_enable)) {
  4434. time_latency = (qdf_ktime_to_ms(qdf_ktime_real_get()) -
  4435. qdf_ktime_to_ms(desc->timestamp));
  4436. }
  4437. dp_send_completion_to_pkt_capture(soc, desc, ts);
  4438. if (dp_tx_pkt_tracepoints_enabled())
  4439. qdf_trace_dp_packet(desc->nbuf, QDF_TX,
  4440. desc->msdu_ext_desc ?
  4441. desc->msdu_ext_desc->tso_desc : NULL,
  4442. qdf_ktime_to_ms(desc->timestamp));
  4443. if (!(desc->msdu_ext_desc)) {
  4444. dp_tx_enh_unmap(soc, desc);
  4445. if (txrx_peer)
  4446. peer_id = txrx_peer->peer_id;
  4447. if (QDF_STATUS_SUCCESS ==
  4448. dp_monitor_tx_add_to_comp_queue(soc, desc, ts, peer_id)) {
  4449. return;
  4450. }
  4451. if (QDF_STATUS_SUCCESS ==
  4452. dp_get_completion_indication_for_stack(soc,
  4453. desc->pdev,
  4454. txrx_peer, ts,
  4455. desc->nbuf,
  4456. time_latency)) {
  4457. dp_send_completion_to_stack(soc,
  4458. desc->pdev,
  4459. ts->peer_id,
  4460. ts->ppdu_id,
  4461. desc->nbuf);
  4462. return;
  4463. }
  4464. }
  4465. desc->flags |= DP_TX_DESC_FLAG_COMPLETED_TX;
  4466. dp_tx_comp_free_buf(soc, desc, false);
  4467. }
  4468. #ifdef DISABLE_DP_STATS
  4469. /**
  4470. * dp_tx_update_connectivity_stats() - update tx connectivity stats
  4471. * @soc: core txrx main context
  4472. * @tx_desc: tx desc
  4473. * @status: tx status
  4474. *
  4475. * Return: none
  4476. */
  4477. static inline
  4478. void dp_tx_update_connectivity_stats(struct dp_soc *soc,
  4479. struct dp_vdev *vdev,
  4480. struct dp_tx_desc_s *tx_desc,
  4481. uint8_t status)
  4482. {
  4483. }
  4484. #else
  4485. static inline
  4486. void dp_tx_update_connectivity_stats(struct dp_soc *soc,
  4487. struct dp_vdev *vdev,
  4488. struct dp_tx_desc_s *tx_desc,
  4489. uint8_t status)
  4490. {
  4491. void *osif_dev;
  4492. ol_txrx_stats_rx_fp stats_cbk;
  4493. uint8_t pkt_type;
  4494. qdf_assert(tx_desc);
  4495. if (!vdev ||
  4496. !vdev->osif_vdev ||
  4497. !vdev->stats_cb)
  4498. return;
  4499. osif_dev = vdev->osif_vdev;
  4500. stats_cbk = vdev->stats_cb;
  4501. stats_cbk(tx_desc->nbuf, osif_dev, PKT_TYPE_TX_HOST_FW_SENT, &pkt_type);
  4502. if (status == HAL_TX_TQM_RR_FRAME_ACKED)
  4503. stats_cbk(tx_desc->nbuf, osif_dev, PKT_TYPE_TX_ACK_CNT,
  4504. &pkt_type);
  4505. }
  4506. #endif
  4507. #if defined(WLAN_FEATURE_TSF_UPLINK_DELAY) || defined(WLAN_CONFIG_TX_DELAY)
  4508. /* Mask for bit29 ~ bit31 */
  4509. #define DP_TX_TS_BIT29_31_MASK 0xE0000000
  4510. /* Timestamp value (unit us) if bit29 is set */
  4511. #define DP_TX_TS_BIT29_SET_VALUE BIT(29)
  4512. /**
  4513. * dp_tx_adjust_enqueue_buffer_ts() - adjust the enqueue buffer_timestamp
  4514. * @ack_ts: OTA ack timestamp, unit us.
  4515. * @enqueue_ts: TCL enqueue TX data to TQM timestamp, unit us.
  4516. * @base_delta_ts: base timestamp delta for ack_ts and enqueue_ts
  4517. *
  4518. * this function will restore the bit29 ~ bit31 3 bits value for
  4519. * buffer_timestamp in wbm2sw ring entry, currently buffer_timestamp only
  4520. * can support 0x7FFF * 1024 us (29 bits), but if the timestamp is >
  4521. * 0x7FFF * 1024 us, bit29~ bit31 will be lost.
  4522. *
  4523. * Return: the adjusted buffer_timestamp value
  4524. */
  4525. static inline
  4526. uint32_t dp_tx_adjust_enqueue_buffer_ts(uint32_t ack_ts,
  4527. uint32_t enqueue_ts,
  4528. uint32_t base_delta_ts)
  4529. {
  4530. uint32_t ack_buffer_ts;
  4531. uint32_t ack_buffer_ts_bit29_31;
  4532. uint32_t adjusted_enqueue_ts;
  4533. /* corresponding buffer_timestamp value when receive OTA Ack */
  4534. ack_buffer_ts = ack_ts - base_delta_ts;
  4535. ack_buffer_ts_bit29_31 = ack_buffer_ts & DP_TX_TS_BIT29_31_MASK;
  4536. /* restore the bit29 ~ bit31 value */
  4537. adjusted_enqueue_ts = ack_buffer_ts_bit29_31 | enqueue_ts;
  4538. /*
  4539. * if actual enqueue_ts value occupied 29 bits only, this enqueue_ts
  4540. * value + real UL delay overflow 29 bits, then 30th bit (bit-29)
  4541. * should not be marked, otherwise extra 0x20000000 us is added to
  4542. * enqueue_ts.
  4543. */
  4544. if (qdf_unlikely(adjusted_enqueue_ts > ack_buffer_ts))
  4545. adjusted_enqueue_ts -= DP_TX_TS_BIT29_SET_VALUE;
  4546. return adjusted_enqueue_ts;
  4547. }
  4548. QDF_STATUS
  4549. dp_tx_compute_hw_delay_us(struct hal_tx_completion_status *ts,
  4550. uint32_t delta_tsf,
  4551. uint32_t *delay_us)
  4552. {
  4553. uint32_t buffer_ts;
  4554. uint32_t delay;
  4555. if (!delay_us)
  4556. return QDF_STATUS_E_INVAL;
  4557. /* Tx_rate_stats_info_valid is 0 and tsf is invalid then */
  4558. if (!ts->valid)
  4559. return QDF_STATUS_E_INVAL;
  4560. /* buffer_timestamp is in units of 1024 us and is [31:13] of
  4561. * WBM_RELEASE_RING_4. After left shift 10 bits, it's
  4562. * valid up to 29 bits.
  4563. */
  4564. buffer_ts = ts->buffer_timestamp << 10;
  4565. buffer_ts = dp_tx_adjust_enqueue_buffer_ts(ts->tsf,
  4566. buffer_ts, delta_tsf);
  4567. delay = ts->tsf - buffer_ts - delta_tsf;
  4568. if (qdf_unlikely(delay & 0x80000000)) {
  4569. dp_err_rl("delay = 0x%x (-ve)\n"
  4570. "release_src = %d\n"
  4571. "ppdu_id = 0x%x\n"
  4572. "peer_id = 0x%x\n"
  4573. "tid = 0x%x\n"
  4574. "release_reason = %d\n"
  4575. "tsf = %u (0x%x)\n"
  4576. "buffer_timestamp = %u (0x%x)\n"
  4577. "delta_tsf = %u (0x%x)\n",
  4578. delay, ts->release_src, ts->ppdu_id, ts->peer_id,
  4579. ts->tid, ts->status, ts->tsf, ts->tsf,
  4580. ts->buffer_timestamp, ts->buffer_timestamp,
  4581. delta_tsf, delta_tsf);
  4582. delay = 0;
  4583. goto end;
  4584. }
  4585. delay &= 0x1FFFFFFF; /* mask 29 BITS */
  4586. if (delay > 0x1000000) {
  4587. dp_info_rl("----------------------\n"
  4588. "Tx completion status:\n"
  4589. "----------------------\n"
  4590. "release_src = %d\n"
  4591. "ppdu_id = 0x%x\n"
  4592. "release_reason = %d\n"
  4593. "tsf = %u (0x%x)\n"
  4594. "buffer_timestamp = %u (0x%x)\n"
  4595. "delta_tsf = %u (0x%x)\n",
  4596. ts->release_src, ts->ppdu_id, ts->status,
  4597. ts->tsf, ts->tsf, ts->buffer_timestamp,
  4598. ts->buffer_timestamp, delta_tsf, delta_tsf);
  4599. return QDF_STATUS_E_FAILURE;
  4600. }
  4601. end:
  4602. *delay_us = delay;
  4603. return QDF_STATUS_SUCCESS;
  4604. }
  4605. void dp_set_delta_tsf(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  4606. uint32_t delta_tsf)
  4607. {
  4608. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  4609. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  4610. DP_MOD_ID_CDP);
  4611. if (!vdev) {
  4612. dp_err_rl("vdev %d does not exist", vdev_id);
  4613. return;
  4614. }
  4615. vdev->delta_tsf = delta_tsf;
  4616. dp_debug("vdev id %u delta_tsf %u", vdev_id, delta_tsf);
  4617. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  4618. }
  4619. #endif
  4620. #ifdef WLAN_FEATURE_TSF_UPLINK_DELAY
  4621. QDF_STATUS dp_set_tsf_ul_delay_report(struct cdp_soc_t *soc_hdl,
  4622. uint8_t vdev_id, bool enable)
  4623. {
  4624. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  4625. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  4626. DP_MOD_ID_CDP);
  4627. if (!vdev) {
  4628. dp_err_rl("vdev %d does not exist", vdev_id);
  4629. return QDF_STATUS_E_FAILURE;
  4630. }
  4631. qdf_atomic_set(&vdev->ul_delay_report, enable);
  4632. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  4633. return QDF_STATUS_SUCCESS;
  4634. }
  4635. QDF_STATUS dp_get_uplink_delay(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  4636. uint32_t *val)
  4637. {
  4638. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  4639. struct dp_vdev *vdev;
  4640. uint32_t delay_accum;
  4641. uint32_t pkts_accum;
  4642. vdev = dp_vdev_get_ref_by_id(soc, vdev_id, DP_MOD_ID_CDP);
  4643. if (!vdev) {
  4644. dp_err_rl("vdev %d does not exist", vdev_id);
  4645. return QDF_STATUS_E_FAILURE;
  4646. }
  4647. if (!qdf_atomic_read(&vdev->ul_delay_report)) {
  4648. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  4649. return QDF_STATUS_E_FAILURE;
  4650. }
  4651. /* Average uplink delay based on current accumulated values */
  4652. delay_accum = qdf_atomic_read(&vdev->ul_delay_accum);
  4653. pkts_accum = qdf_atomic_read(&vdev->ul_pkts_accum);
  4654. *val = delay_accum / pkts_accum;
  4655. dp_debug("uplink_delay %u delay_accum %u pkts_accum %u", *val,
  4656. delay_accum, pkts_accum);
  4657. /* Reset accumulated values to 0 */
  4658. qdf_atomic_set(&vdev->ul_delay_accum, 0);
  4659. qdf_atomic_set(&vdev->ul_pkts_accum, 0);
  4660. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_CDP);
  4661. return QDF_STATUS_SUCCESS;
  4662. }
  4663. static void dp_tx_update_uplink_delay(struct dp_soc *soc, struct dp_vdev *vdev,
  4664. struct hal_tx_completion_status *ts)
  4665. {
  4666. uint32_t ul_delay;
  4667. if (qdf_unlikely(!vdev)) {
  4668. dp_info_rl("vdev is null or delete in progress");
  4669. return;
  4670. }
  4671. if (!qdf_atomic_read(&vdev->ul_delay_report))
  4672. return;
  4673. if (QDF_IS_STATUS_ERROR(dp_tx_compute_hw_delay_us(ts,
  4674. vdev->delta_tsf,
  4675. &ul_delay)))
  4676. return;
  4677. ul_delay /= 1000; /* in unit of ms */
  4678. qdf_atomic_add(ul_delay, &vdev->ul_delay_accum);
  4679. qdf_atomic_inc(&vdev->ul_pkts_accum);
  4680. }
  4681. #else /* !WLAN_FEATURE_TSF_UPLINK_DELAY */
  4682. static inline
  4683. void dp_tx_update_uplink_delay(struct dp_soc *soc, struct dp_vdev *vdev,
  4684. struct hal_tx_completion_status *ts)
  4685. {
  4686. }
  4687. #endif /* WLAN_FEATURE_TSF_UPLINK_DELAY */
  4688. /**
  4689. * dp_tx_comp_process_tx_status() - Parse and Dump Tx completion status info
  4690. * @soc: DP soc handle
  4691. * @tx_desc: software descriptor head pointer
  4692. * @ts: Tx completion status
  4693. * @txrx_peer: txrx peer handle
  4694. * @ring_id: ring number
  4695. *
  4696. * Return: none
  4697. */
  4698. void dp_tx_comp_process_tx_status(struct dp_soc *soc,
  4699. struct dp_tx_desc_s *tx_desc,
  4700. struct hal_tx_completion_status *ts,
  4701. struct dp_txrx_peer *txrx_peer,
  4702. uint8_t ring_id)
  4703. {
  4704. uint32_t length;
  4705. qdf_ether_header_t *eh;
  4706. struct dp_vdev *vdev = NULL;
  4707. qdf_nbuf_t nbuf = tx_desc->nbuf;
  4708. enum qdf_dp_tx_rx_status dp_status;
  4709. if (!nbuf) {
  4710. dp_info_rl("invalid tx descriptor. nbuf NULL");
  4711. goto out;
  4712. }
  4713. eh = (qdf_ether_header_t *)qdf_nbuf_data(nbuf);
  4714. length = dp_tx_get_pkt_len(tx_desc);
  4715. dp_status = dp_tx_hw_to_qdf(ts->status);
  4716. DPTRACE(qdf_dp_trace_ptr(tx_desc->nbuf,
  4717. QDF_DP_TRACE_LI_DP_FREE_PACKET_PTR_RECORD,
  4718. QDF_TRACE_DEFAULT_PDEV_ID,
  4719. qdf_nbuf_data_addr(nbuf),
  4720. sizeof(qdf_nbuf_data(nbuf)),
  4721. tx_desc->id, ts->status, dp_status));
  4722. dp_tx_comp_debug("-------------------- \n"
  4723. "Tx Completion Stats: \n"
  4724. "-------------------- \n"
  4725. "ack_frame_rssi = %d \n"
  4726. "first_msdu = %d \n"
  4727. "last_msdu = %d \n"
  4728. "msdu_part_of_amsdu = %d \n"
  4729. "rate_stats valid = %d \n"
  4730. "bw = %d \n"
  4731. "pkt_type = %d \n"
  4732. "stbc = %d \n"
  4733. "ldpc = %d \n"
  4734. "sgi = %d \n"
  4735. "mcs = %d \n"
  4736. "ofdma = %d \n"
  4737. "tones_in_ru = %d \n"
  4738. "tsf = %d \n"
  4739. "ppdu_id = %d \n"
  4740. "transmit_cnt = %d \n"
  4741. "tid = %d \n"
  4742. "peer_id = %d\n"
  4743. "tx_status = %d\n",
  4744. ts->ack_frame_rssi, ts->first_msdu,
  4745. ts->last_msdu, ts->msdu_part_of_amsdu,
  4746. ts->valid, ts->bw, ts->pkt_type, ts->stbc,
  4747. ts->ldpc, ts->sgi, ts->mcs, ts->ofdma,
  4748. ts->tones_in_ru, ts->tsf, ts->ppdu_id,
  4749. ts->transmit_cnt, ts->tid, ts->peer_id,
  4750. ts->status);
  4751. /* Update SoC level stats */
  4752. DP_STATS_INCC(soc, tx.dropped_fw_removed, 1,
  4753. (ts->status == HAL_TX_TQM_RR_REM_CMD_REM));
  4754. if (!txrx_peer) {
  4755. dp_info_rl("peer is null or deletion in progress");
  4756. DP_STATS_INC_PKT(soc, tx.tx_invalid_peer, 1, length);
  4757. goto out;
  4758. }
  4759. vdev = txrx_peer->vdev;
  4760. dp_tx_update_connectivity_stats(soc, vdev, tx_desc, ts->status);
  4761. dp_tx_update_uplink_delay(soc, vdev, ts);
  4762. /* check tx complete notification */
  4763. if (qdf_nbuf_tx_notify_comp_get(nbuf))
  4764. dp_tx_notify_completion(soc, vdev, tx_desc,
  4765. nbuf, ts->status);
  4766. /* Update per-packet stats for mesh mode */
  4767. if (qdf_unlikely(vdev->mesh_vdev) &&
  4768. !(tx_desc->flags & DP_TX_DESC_FLAG_TO_FW))
  4769. dp_tx_comp_fill_tx_completion_stats(tx_desc, ts);
  4770. /* Update peer level stats */
  4771. if (qdf_unlikely(txrx_peer->bss_peer &&
  4772. vdev->opmode == wlan_op_mode_ap)) {
  4773. if (ts->status != HAL_TX_TQM_RR_REM_CMD_REM) {
  4774. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.mcast, 1,
  4775. length);
  4776. if (txrx_peer->vdev->tx_encap_type ==
  4777. htt_cmn_pkt_type_ethernet &&
  4778. QDF_IS_ADDR_BROADCAST(eh->ether_dhost)) {
  4779. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
  4780. tx.bcast, 1,
  4781. length);
  4782. }
  4783. }
  4784. } else {
  4785. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.ucast, 1, length);
  4786. if (ts->status == HAL_TX_TQM_RR_FRAME_ACKED) {
  4787. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer, tx.tx_success,
  4788. 1, length);
  4789. if (qdf_unlikely(txrx_peer->in_twt)) {
  4790. DP_PEER_PER_PKT_STATS_INC_PKT(txrx_peer,
  4791. tx.tx_success_twt,
  4792. 1, length);
  4793. }
  4794. }
  4795. }
  4796. dp_tx_update_peer_stats(tx_desc, ts, txrx_peer, ring_id);
  4797. dp_tx_update_peer_delay_stats(txrx_peer, tx_desc, ts, ring_id);
  4798. dp_tx_update_peer_jitter_stats(txrx_peer, tx_desc, ts, ring_id);
  4799. dp_tx_update_peer_sawf_stats(soc, vdev, txrx_peer, tx_desc,
  4800. ts, ts->tid);
  4801. dp_tx_send_pktlog(soc, vdev->pdev, tx_desc, nbuf, dp_status);
  4802. #ifdef QCA_SUPPORT_RDK_STATS
  4803. if (soc->peerstats_enabled)
  4804. dp_tx_sojourn_stats_process(vdev->pdev, txrx_peer, ts->tid,
  4805. qdf_ktime_to_ms(tx_desc->timestamp),
  4806. ts->ppdu_id);
  4807. #endif
  4808. out:
  4809. return;
  4810. }
  4811. #if defined(QCA_VDEV_STATS_HW_OFFLOAD_SUPPORT) && \
  4812. defined(QCA_ENHANCED_STATS_SUPPORT)
  4813. /*
  4814. * dp_tx_update_peer_basic_stats(): Update peer basic stats
  4815. * @txrx_peer: Datapath txrx_peer handle
  4816. * @length: Length of the packet
  4817. * @tx_status: Tx status from TQM/FW
  4818. * @update: enhanced flag value present in dp_pdev
  4819. *
  4820. * Return: none
  4821. */
  4822. void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
  4823. uint32_t length, uint8_t tx_status,
  4824. bool update)
  4825. {
  4826. if (update || (!txrx_peer->hw_txrx_stats_en)) {
  4827. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  4828. if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
  4829. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  4830. }
  4831. }
  4832. #elif defined(QCA_VDEV_STATS_HW_OFFLOAD_SUPPORT)
  4833. void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
  4834. uint32_t length, uint8_t tx_status,
  4835. bool update)
  4836. {
  4837. if (!txrx_peer->hw_txrx_stats_en) {
  4838. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  4839. if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
  4840. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  4841. }
  4842. }
  4843. #else
  4844. void dp_tx_update_peer_basic_stats(struct dp_txrx_peer *txrx_peer,
  4845. uint32_t length, uint8_t tx_status,
  4846. bool update)
  4847. {
  4848. DP_PEER_STATS_FLAT_INC_PKT(txrx_peer, comp_pkt, 1, length);
  4849. if (tx_status != HAL_TX_TQM_RR_FRAME_ACKED)
  4850. DP_PEER_STATS_FLAT_INC(txrx_peer, tx_failed, 1);
  4851. }
  4852. #endif
  4853. /*
  4854. * dp_tx_prefetch_next_nbuf_data(): Prefetch nbuf and nbuf data
  4855. * @nbuf: skb buffer
  4856. *
  4857. * Return: none
  4858. */
  4859. #ifdef QCA_DP_RX_NBUF_AND_NBUF_DATA_PREFETCH
  4860. static inline
  4861. void dp_tx_prefetch_next_nbuf_data(struct dp_tx_desc_s *next)
  4862. {
  4863. qdf_nbuf_t nbuf = NULL;
  4864. if (next)
  4865. nbuf = next->nbuf;
  4866. if (nbuf)
  4867. qdf_prefetch(nbuf);
  4868. }
  4869. #else
  4870. static inline
  4871. void dp_tx_prefetch_next_nbuf_data(struct dp_tx_desc_s *next)
  4872. {
  4873. }
  4874. #endif
  4875. /**
  4876. * dp_tx_mcast_reinject_handler() - Tx reinjected multicast packets handler
  4877. * @soc: core txrx main context
  4878. * @desc: software descriptor
  4879. *
  4880. * Return: true when packet is reinjected
  4881. */
  4882. #if defined(WLAN_FEATURE_11BE_MLO) && defined(WLAN_MLO_MULTI_CHIP) && \
  4883. defined(WLAN_MCAST_MLO)
  4884. static inline bool
  4885. dp_tx_mcast_reinject_handler(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  4886. {
  4887. struct dp_vdev *vdev = NULL;
  4888. if (desc->tx_status == HAL_TX_TQM_RR_MULTICAST_DROP) {
  4889. if (!soc->arch_ops.dp_tx_mcast_handler ||
  4890. !soc->arch_ops.dp_tx_is_mcast_primary)
  4891. return false;
  4892. vdev = dp_vdev_get_ref_by_id(soc, desc->vdev_id,
  4893. DP_MOD_ID_REINJECT);
  4894. if (qdf_unlikely(!vdev)) {
  4895. dp_tx_comp_info_rl("Unable to get vdev ref %d",
  4896. desc->id);
  4897. return false;
  4898. }
  4899. if (!(soc->arch_ops.dp_tx_is_mcast_primary(soc, vdev))) {
  4900. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_REINJECT);
  4901. return false;
  4902. }
  4903. DP_STATS_INC_PKT(vdev, tx_i.reinject_pkts, 1,
  4904. qdf_nbuf_len(desc->nbuf));
  4905. soc->arch_ops.dp_tx_mcast_handler(soc, vdev, desc->nbuf);
  4906. dp_tx_desc_release(desc, desc->pool_id);
  4907. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_REINJECT);
  4908. return true;
  4909. }
  4910. return false;
  4911. }
  4912. #else
  4913. static inline bool
  4914. dp_tx_mcast_reinject_handler(struct dp_soc *soc, struct dp_tx_desc_s *desc)
  4915. {
  4916. return false;
  4917. }
  4918. #endif
  4919. #ifdef QCA_DP_TX_NBUF_LIST_FREE
  4920. static inline void
  4921. dp_tx_nbuf_queue_head_init(qdf_nbuf_queue_head_t *nbuf_queue_head)
  4922. {
  4923. qdf_nbuf_queue_head_init(nbuf_queue_head);
  4924. }
  4925. static inline void
  4926. dp_tx_nbuf_dev_queue_free(qdf_nbuf_queue_head_t *nbuf_queue_head,
  4927. struct dp_tx_desc_s *desc)
  4928. {
  4929. qdf_nbuf_t nbuf = NULL;
  4930. nbuf = desc->nbuf;
  4931. if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_FAST))
  4932. qdf_nbuf_dev_queue_head(nbuf_queue_head, nbuf);
  4933. else
  4934. qdf_nbuf_free(nbuf);
  4935. }
  4936. static inline void
  4937. dp_tx_nbuf_dev_kfree_list(qdf_nbuf_queue_head_t *nbuf_queue_head)
  4938. {
  4939. qdf_nbuf_dev_kfree_list(nbuf_queue_head);
  4940. }
  4941. #else
  4942. static inline void
  4943. dp_tx_nbuf_queue_head_init(qdf_nbuf_queue_head_t *nbuf_queue_head)
  4944. {
  4945. }
  4946. static inline void
  4947. dp_tx_nbuf_dev_queue_free(qdf_nbuf_queue_head_t *nbuf_queue_head,
  4948. struct dp_tx_desc_s *desc)
  4949. {
  4950. qdf_nbuf_free(desc->nbuf);
  4951. }
  4952. static inline void
  4953. dp_tx_nbuf_dev_kfree_list(qdf_nbuf_queue_head_t *nbuf_queue_head)
  4954. {
  4955. }
  4956. #endif
  4957. /**
  4958. * dp_tx_comp_process_desc_list() - Tx complete software descriptor handler
  4959. * @soc: core txrx main context
  4960. * @comp_head: software descriptor head pointer
  4961. * @ring_id: ring number
  4962. *
  4963. * This function will process batch of descriptors reaped by dp_tx_comp_handler
  4964. * and release the software descriptors after processing is complete
  4965. *
  4966. * Return: none
  4967. */
  4968. void
  4969. dp_tx_comp_process_desc_list(struct dp_soc *soc,
  4970. struct dp_tx_desc_s *comp_head, uint8_t ring_id)
  4971. {
  4972. struct dp_tx_desc_s *desc;
  4973. struct dp_tx_desc_s *next;
  4974. struct hal_tx_completion_status ts;
  4975. struct dp_txrx_peer *txrx_peer = NULL;
  4976. uint16_t peer_id = DP_INVALID_PEER;
  4977. dp_txrx_ref_handle txrx_ref_handle = NULL;
  4978. qdf_nbuf_queue_head_t h;
  4979. desc = comp_head;
  4980. dp_tx_nbuf_queue_head_init(&h);
  4981. while (desc) {
  4982. next = desc->next;
  4983. dp_tx_prefetch_next_nbuf_data(next);
  4984. if (peer_id != desc->peer_id) {
  4985. if (txrx_peer)
  4986. dp_txrx_peer_unref_delete(txrx_ref_handle,
  4987. DP_MOD_ID_TX_COMP);
  4988. peer_id = desc->peer_id;
  4989. txrx_peer =
  4990. dp_txrx_peer_get_ref_by_id(soc, peer_id,
  4991. &txrx_ref_handle,
  4992. DP_MOD_ID_TX_COMP);
  4993. }
  4994. if (dp_tx_mcast_reinject_handler(soc, desc)) {
  4995. desc = next;
  4996. continue;
  4997. }
  4998. if (desc->flags & DP_TX_DESC_FLAG_PPEDS) {
  4999. if (qdf_likely(txrx_peer))
  5000. dp_tx_update_peer_basic_stats(txrx_peer,
  5001. desc->length,
  5002. desc->tx_status,
  5003. false);
  5004. dp_tx_nbuf_dev_queue_free(&h, desc);
  5005. dp_ppeds_tx_desc_free(soc, desc);
  5006. desc = next;
  5007. continue;
  5008. }
  5009. if (qdf_likely(desc->flags & DP_TX_DESC_FLAG_SIMPLE)) {
  5010. struct dp_pdev *pdev = desc->pdev;
  5011. if (qdf_likely(txrx_peer))
  5012. dp_tx_update_peer_basic_stats(txrx_peer,
  5013. desc->length,
  5014. desc->tx_status,
  5015. false);
  5016. qdf_assert(pdev);
  5017. dp_tx_outstanding_dec(pdev);
  5018. /*
  5019. * Calling a QDF WRAPPER here is creating significant
  5020. * performance impact so avoided the wrapper call here
  5021. */
  5022. dp_tx_desc_history_add(soc, desc->dma_addr, desc->nbuf,
  5023. desc->id, DP_TX_COMP_UNMAP);
  5024. dp_tx_nbuf_unmap(soc, desc);
  5025. dp_tx_nbuf_dev_queue_free(&h, desc);
  5026. dp_tx_desc_free(soc, desc, desc->pool_id);
  5027. desc = next;
  5028. continue;
  5029. }
  5030. hal_tx_comp_get_status(&desc->comp, &ts, soc->hal_soc);
  5031. dp_tx_comp_process_tx_status(soc, desc, &ts, txrx_peer,
  5032. ring_id);
  5033. dp_tx_comp_process_desc(soc, desc, &ts, txrx_peer);
  5034. dp_tx_desc_release(desc, desc->pool_id);
  5035. desc = next;
  5036. }
  5037. dp_tx_nbuf_dev_kfree_list(&h);
  5038. if (txrx_peer)
  5039. dp_txrx_peer_unref_delete(txrx_ref_handle, DP_MOD_ID_TX_COMP);
  5040. }
  5041. #ifdef WLAN_FEATURE_RX_SOFTIRQ_TIME_LIMIT
  5042. static inline
  5043. bool dp_tx_comp_loop_pkt_limit_hit(struct dp_soc *soc, int num_reaped,
  5044. int max_reap_limit)
  5045. {
  5046. bool limit_hit = false;
  5047. limit_hit =
  5048. (num_reaped >= max_reap_limit) ? true : false;
  5049. if (limit_hit)
  5050. DP_STATS_INC(soc, tx.tx_comp_loop_pkt_limit_hit, 1);
  5051. return limit_hit;
  5052. }
  5053. static inline bool dp_tx_comp_enable_eol_data_check(struct dp_soc *soc)
  5054. {
  5055. return soc->wlan_cfg_ctx->tx_comp_enable_eol_data_check;
  5056. }
  5057. static inline int dp_tx_comp_get_loop_pkt_limit(struct dp_soc *soc)
  5058. {
  5059. struct wlan_cfg_dp_soc_ctxt *cfg = soc->wlan_cfg_ctx;
  5060. return cfg->tx_comp_loop_pkt_limit;
  5061. }
  5062. #else
  5063. static inline
  5064. bool dp_tx_comp_loop_pkt_limit_hit(struct dp_soc *soc, int num_reaped,
  5065. int max_reap_limit)
  5066. {
  5067. return false;
  5068. }
  5069. static inline bool dp_tx_comp_enable_eol_data_check(struct dp_soc *soc)
  5070. {
  5071. return false;
  5072. }
  5073. static inline int dp_tx_comp_get_loop_pkt_limit(struct dp_soc *soc)
  5074. {
  5075. return 0;
  5076. }
  5077. #endif
  5078. #ifdef WLAN_FEATURE_NEAR_FULL_IRQ
  5079. static inline int
  5080. dp_srng_test_and_update_nf_params(struct dp_soc *soc, struct dp_srng *dp_srng,
  5081. int *max_reap_limit)
  5082. {
  5083. return soc->arch_ops.dp_srng_test_and_update_nf_params(soc, dp_srng,
  5084. max_reap_limit);
  5085. }
  5086. #else
  5087. static inline int
  5088. dp_srng_test_and_update_nf_params(struct dp_soc *soc, struct dp_srng *dp_srng,
  5089. int *max_reap_limit)
  5090. {
  5091. return 0;
  5092. }
  5093. #endif
  5094. #ifdef DP_TX_TRACKING
  5095. void dp_tx_desc_check_corruption(struct dp_tx_desc_s *tx_desc)
  5096. {
  5097. if ((tx_desc->magic != DP_TX_MAGIC_PATTERN_INUSE) &&
  5098. (tx_desc->magic != DP_TX_MAGIC_PATTERN_FREE)) {
  5099. dp_err_rl("tx_desc %u is corrupted", tx_desc->id);
  5100. qdf_trigger_self_recovery(NULL, QDF_TX_DESC_LEAK);
  5101. }
  5102. }
  5103. #endif
  5104. uint32_t dp_tx_comp_handler(struct dp_intr *int_ctx, struct dp_soc *soc,
  5105. hal_ring_handle_t hal_ring_hdl, uint8_t ring_id,
  5106. uint32_t quota)
  5107. {
  5108. void *tx_comp_hal_desc;
  5109. void *last_prefetched_hw_desc = NULL;
  5110. struct dp_tx_desc_s *last_prefetched_sw_desc = NULL;
  5111. hal_soc_handle_t hal_soc;
  5112. uint8_t buffer_src;
  5113. struct dp_tx_desc_s *tx_desc = NULL;
  5114. struct dp_tx_desc_s *head_desc = NULL;
  5115. struct dp_tx_desc_s *tail_desc = NULL;
  5116. uint32_t num_processed = 0;
  5117. uint32_t count;
  5118. uint32_t num_avail_for_reap = 0;
  5119. bool force_break = false;
  5120. struct dp_srng *tx_comp_ring = &soc->tx_comp_ring[ring_id];
  5121. int max_reap_limit, ring_near_full;
  5122. uint32_t num_entries;
  5123. DP_HIST_INIT();
  5124. num_entries = hal_srng_get_num_entries(soc->hal_soc, hal_ring_hdl);
  5125. more_data:
  5126. hal_soc = soc->hal_soc;
  5127. /* Re-initialize local variables to be re-used */
  5128. head_desc = NULL;
  5129. tail_desc = NULL;
  5130. count = 0;
  5131. max_reap_limit = dp_tx_comp_get_loop_pkt_limit(soc);
  5132. ring_near_full = dp_srng_test_and_update_nf_params(soc, tx_comp_ring,
  5133. &max_reap_limit);
  5134. if (qdf_unlikely(dp_srng_access_start(int_ctx, soc, hal_ring_hdl))) {
  5135. dp_err("HAL RING Access Failed -- %pK", hal_ring_hdl);
  5136. return 0;
  5137. }
  5138. if (!num_avail_for_reap)
  5139. num_avail_for_reap = hal_srng_dst_num_valid(hal_soc,
  5140. hal_ring_hdl, 0);
  5141. if (num_avail_for_reap >= quota)
  5142. num_avail_for_reap = quota;
  5143. dp_srng_dst_inv_cached_descs(soc, hal_ring_hdl, num_avail_for_reap);
  5144. last_prefetched_hw_desc = dp_srng_dst_prefetch_32_byte_desc(hal_soc,
  5145. hal_ring_hdl,
  5146. num_avail_for_reap);
  5147. /* Find head descriptor from completion ring */
  5148. while (qdf_likely(num_avail_for_reap--)) {
  5149. tx_comp_hal_desc = dp_srng_dst_get_next(soc, hal_ring_hdl);
  5150. if (qdf_unlikely(!tx_comp_hal_desc))
  5151. break;
  5152. buffer_src = hal_tx_comp_get_buffer_source(hal_soc,
  5153. tx_comp_hal_desc);
  5154. /* If this buffer was not released by TQM or FW, then it is not
  5155. * Tx completion indication, assert */
  5156. if (qdf_unlikely(buffer_src !=
  5157. HAL_TX_COMP_RELEASE_SOURCE_TQM) &&
  5158. (qdf_unlikely(buffer_src !=
  5159. HAL_TX_COMP_RELEASE_SOURCE_FW))) {
  5160. uint8_t wbm_internal_error;
  5161. dp_err_rl(
  5162. "Tx comp release_src != TQM | FW but from %d",
  5163. buffer_src);
  5164. hal_dump_comp_desc(tx_comp_hal_desc);
  5165. DP_STATS_INC(soc, tx.invalid_release_source, 1);
  5166. /* When WBM sees NULL buffer_addr_info in any of
  5167. * ingress rings it sends an error indication,
  5168. * with wbm_internal_error=1, to a specific ring.
  5169. * The WBM2SW ring used to indicate these errors is
  5170. * fixed in HW, and that ring is being used as Tx
  5171. * completion ring. These errors are not related to
  5172. * Tx completions, and should just be ignored
  5173. */
  5174. wbm_internal_error = hal_get_wbm_internal_error(
  5175. hal_soc,
  5176. tx_comp_hal_desc);
  5177. if (wbm_internal_error) {
  5178. dp_err_rl("Tx comp wbm_internal_error!!");
  5179. DP_STATS_INC(soc, tx.wbm_internal_error[WBM_INT_ERROR_ALL], 1);
  5180. if (HAL_TX_COMP_RELEASE_SOURCE_REO ==
  5181. buffer_src)
  5182. dp_handle_wbm_internal_error(
  5183. soc,
  5184. tx_comp_hal_desc,
  5185. hal_tx_comp_get_buffer_type(
  5186. tx_comp_hal_desc));
  5187. } else {
  5188. dp_err_rl("Tx comp wbm_internal_error false");
  5189. DP_STATS_INC(soc, tx.non_wbm_internal_err, 1);
  5190. }
  5191. continue;
  5192. }
  5193. soc->arch_ops.tx_comp_get_params_from_hal_desc(soc,
  5194. tx_comp_hal_desc,
  5195. &tx_desc);
  5196. if (qdf_unlikely(!tx_desc)) {
  5197. dp_err("unable to retrieve tx_desc!");
  5198. hal_dump_comp_desc(tx_comp_hal_desc);
  5199. DP_STATS_INC(soc, tx.invalid_tx_comp_desc, 1);
  5200. QDF_BUG(0);
  5201. continue;
  5202. }
  5203. tx_desc->buffer_src = buffer_src;
  5204. if (tx_desc->flags & DP_TX_DESC_FLAG_PPEDS)
  5205. goto add_to_pool2;
  5206. /*
  5207. * If the release source is FW, process the HTT status
  5208. */
  5209. if (qdf_unlikely(buffer_src ==
  5210. HAL_TX_COMP_RELEASE_SOURCE_FW)) {
  5211. uint8_t htt_tx_status[HAL_TX_COMP_HTT_STATUS_LEN];
  5212. hal_tx_comp_get_htt_desc(tx_comp_hal_desc,
  5213. htt_tx_status);
  5214. /* Collect hw completion contents */
  5215. hal_tx_comp_desc_sync(tx_comp_hal_desc,
  5216. &tx_desc->comp, 1);
  5217. soc->arch_ops.dp_tx_process_htt_completion(
  5218. soc,
  5219. tx_desc,
  5220. htt_tx_status,
  5221. ring_id);
  5222. } else {
  5223. tx_desc->tx_status =
  5224. hal_tx_comp_get_tx_status(tx_comp_hal_desc);
  5225. tx_desc->buffer_src = buffer_src;
  5226. /*
  5227. * If the fast completion mode is enabled extended
  5228. * metadata from descriptor is not copied
  5229. */
  5230. if (qdf_likely(tx_desc->flags &
  5231. DP_TX_DESC_FLAG_SIMPLE))
  5232. goto add_to_pool;
  5233. /*
  5234. * If the descriptor is already freed in vdev_detach,
  5235. * continue to next descriptor
  5236. */
  5237. if (qdf_unlikely
  5238. ((tx_desc->vdev_id == DP_INVALID_VDEV_ID) &&
  5239. !tx_desc->flags)) {
  5240. dp_tx_comp_info_rl("Descriptor freed in vdev_detach %d",
  5241. tx_desc->id);
  5242. DP_STATS_INC(soc, tx.tx_comp_exception, 1);
  5243. dp_tx_desc_check_corruption(tx_desc);
  5244. continue;
  5245. }
  5246. if (qdf_unlikely(tx_desc->pdev->is_pdev_down)) {
  5247. dp_tx_comp_info_rl("pdev in down state %d",
  5248. tx_desc->id);
  5249. tx_desc->flags |= DP_TX_DESC_FLAG_TX_COMP_ERR;
  5250. dp_tx_comp_free_buf(soc, tx_desc, false);
  5251. dp_tx_desc_release(tx_desc, tx_desc->pool_id);
  5252. goto next_desc;
  5253. }
  5254. if (!(tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED) ||
  5255. !(tx_desc->flags & DP_TX_DESC_FLAG_QUEUED_TX)) {
  5256. dp_tx_comp_alert("Txdesc invalid, flgs = %x,id = %d",
  5257. tx_desc->flags, tx_desc->id);
  5258. qdf_assert_always(0);
  5259. }
  5260. /* Collect hw completion contents */
  5261. hal_tx_comp_desc_sync(tx_comp_hal_desc,
  5262. &tx_desc->comp, 1);
  5263. add_to_pool:
  5264. DP_HIST_PACKET_COUNT_INC(tx_desc->pdev->pdev_id);
  5265. add_to_pool2:
  5266. /* First ring descriptor on the cycle */
  5267. if (!head_desc) {
  5268. head_desc = tx_desc;
  5269. tail_desc = tx_desc;
  5270. }
  5271. tail_desc->next = tx_desc;
  5272. tx_desc->next = NULL;
  5273. tail_desc = tx_desc;
  5274. }
  5275. next_desc:
  5276. num_processed += !(count & DP_TX_NAPI_BUDGET_DIV_MASK);
  5277. /*
  5278. * Processed packet count is more than given quota
  5279. * stop to processing
  5280. */
  5281. count++;
  5282. dp_tx_prefetch_hw_sw_nbuf_desc(soc, hal_soc,
  5283. num_avail_for_reap,
  5284. hal_ring_hdl,
  5285. &last_prefetched_hw_desc,
  5286. &last_prefetched_sw_desc);
  5287. if (dp_tx_comp_loop_pkt_limit_hit(soc, count, max_reap_limit))
  5288. break;
  5289. }
  5290. dp_srng_access_end(int_ctx, soc, hal_ring_hdl);
  5291. /* Process the reaped descriptors */
  5292. if (head_desc)
  5293. dp_tx_comp_process_desc_list(soc, head_desc, ring_id);
  5294. DP_STATS_INC(soc, tx.tx_comp[ring_id], count);
  5295. /*
  5296. * If we are processing in near-full condition, there are 3 scenario
  5297. * 1) Ring entries has reached critical state
  5298. * 2) Ring entries are still near high threshold
  5299. * 3) Ring entries are below the safe level
  5300. *
  5301. * One more loop will move the state to normal processing and yield
  5302. */
  5303. if (ring_near_full)
  5304. goto more_data;
  5305. if (dp_tx_comp_enable_eol_data_check(soc)) {
  5306. if (num_processed >= quota)
  5307. force_break = true;
  5308. if (!force_break &&
  5309. hal_srng_dst_peek_sync_locked(soc->hal_soc,
  5310. hal_ring_hdl)) {
  5311. DP_STATS_INC(soc, tx.hp_oos2, 1);
  5312. if (!hif_exec_should_yield(soc->hif_handle,
  5313. int_ctx->dp_intr_id))
  5314. goto more_data;
  5315. num_avail_for_reap =
  5316. hal_srng_dst_num_valid_locked(soc->hal_soc,
  5317. hal_ring_hdl,
  5318. true);
  5319. if (qdf_unlikely(num_entries &&
  5320. (num_avail_for_reap >=
  5321. num_entries >> 1))) {
  5322. DP_STATS_INC(soc, tx.near_full, 1);
  5323. goto more_data;
  5324. }
  5325. }
  5326. }
  5327. DP_TX_HIST_STATS_PER_PDEV();
  5328. return num_processed;
  5329. }
  5330. #ifdef FEATURE_WLAN_TDLS
  5331. qdf_nbuf_t dp_tx_non_std(struct cdp_soc_t *soc_hdl, uint8_t vdev_id,
  5332. enum ol_tx_spec tx_spec, qdf_nbuf_t msdu_list)
  5333. {
  5334. struct dp_soc *soc = cdp_soc_t_to_dp_soc(soc_hdl);
  5335. struct dp_vdev *vdev = dp_vdev_get_ref_by_id(soc, vdev_id,
  5336. DP_MOD_ID_TDLS);
  5337. if (!vdev) {
  5338. dp_err("vdev handle for id %d is NULL", vdev_id);
  5339. return NULL;
  5340. }
  5341. if (tx_spec & OL_TX_SPEC_NO_FREE)
  5342. vdev->is_tdls_frame = true;
  5343. dp_vdev_unref_delete(soc, vdev, DP_MOD_ID_TDLS);
  5344. return dp_tx_send(soc_hdl, vdev_id, msdu_list);
  5345. }
  5346. #endif
  5347. /**
  5348. * dp_tx_vdev_attach() - attach vdev to dp tx
  5349. * @vdev: virtual device instance
  5350. *
  5351. * Return: QDF_STATUS_SUCCESS: success
  5352. * QDF_STATUS_E_RESOURCES: Error return
  5353. */
  5354. QDF_STATUS dp_tx_vdev_attach(struct dp_vdev *vdev)
  5355. {
  5356. int pdev_id;
  5357. /*
  5358. * Fill HTT TCL Metadata with Vdev ID and MAC ID
  5359. */
  5360. DP_TX_TCL_METADATA_TYPE_SET(vdev->htt_tcl_metadata,
  5361. DP_TCL_METADATA_TYPE_VDEV_BASED);
  5362. DP_TX_TCL_METADATA_VDEV_ID_SET(vdev->htt_tcl_metadata,
  5363. vdev->vdev_id);
  5364. pdev_id =
  5365. dp_get_target_pdev_id_for_host_pdev_id(vdev->pdev->soc,
  5366. vdev->pdev->pdev_id);
  5367. DP_TX_TCL_METADATA_PDEV_ID_SET(vdev->htt_tcl_metadata, pdev_id);
  5368. /*
  5369. * Set HTT Extension Valid bit to 0 by default
  5370. */
  5371. DP_TX_TCL_METADATA_VALID_HTT_SET(vdev->htt_tcl_metadata, 0);
  5372. dp_tx_vdev_update_search_flags(vdev);
  5373. return QDF_STATUS_SUCCESS;
  5374. }
  5375. #ifndef FEATURE_WDS
  5376. static inline bool dp_tx_da_search_override(struct dp_vdev *vdev)
  5377. {
  5378. return false;
  5379. }
  5380. #endif
  5381. /**
  5382. * dp_tx_vdev_update_search_flags() - Update vdev flags as per opmode
  5383. * @vdev: virtual device instance
  5384. *
  5385. * Return: void
  5386. *
  5387. */
  5388. void dp_tx_vdev_update_search_flags(struct dp_vdev *vdev)
  5389. {
  5390. struct dp_soc *soc = vdev->pdev->soc;
  5391. /*
  5392. * Enable both AddrY (SA based search) and AddrX (Da based search)
  5393. * for TDLS link
  5394. *
  5395. * Enable AddrY (SA based search) only for non-WDS STA and
  5396. * ProxySTA VAP (in HKv1) modes.
  5397. *
  5398. * In all other VAP modes, only DA based search should be
  5399. * enabled
  5400. */
  5401. if (vdev->opmode == wlan_op_mode_sta &&
  5402. vdev->tdls_link_connected)
  5403. vdev->hal_desc_addr_search_flags =
  5404. (HAL_TX_DESC_ADDRX_EN | HAL_TX_DESC_ADDRY_EN);
  5405. else if ((vdev->opmode == wlan_op_mode_sta) &&
  5406. !dp_tx_da_search_override(vdev))
  5407. vdev->hal_desc_addr_search_flags = HAL_TX_DESC_ADDRY_EN;
  5408. else
  5409. vdev->hal_desc_addr_search_flags = HAL_TX_DESC_ADDRX_EN;
  5410. if (vdev->opmode == wlan_op_mode_sta && !vdev->tdls_link_connected)
  5411. vdev->search_type = soc->sta_mode_search_policy;
  5412. else
  5413. vdev->search_type = HAL_TX_ADDR_SEARCH_DEFAULT;
  5414. }
  5415. static inline bool
  5416. dp_is_tx_desc_flush_match(struct dp_pdev *pdev,
  5417. struct dp_vdev *vdev,
  5418. struct dp_tx_desc_s *tx_desc)
  5419. {
  5420. if (!(tx_desc && (tx_desc->flags & DP_TX_DESC_FLAG_ALLOCATED)))
  5421. return false;
  5422. /*
  5423. * if vdev is given, then only check whether desc
  5424. * vdev match. if vdev is NULL, then check whether
  5425. * desc pdev match.
  5426. */
  5427. return vdev ? (tx_desc->vdev_id == vdev->vdev_id) :
  5428. (tx_desc->pdev == pdev);
  5429. }
  5430. #ifdef QCA_LL_TX_FLOW_CONTROL_V2
  5431. /**
  5432. * dp_tx_desc_flush() - release resources associated
  5433. * to TX Desc
  5434. *
  5435. * @dp_pdev: Handle to DP pdev structure
  5436. * @vdev: virtual device instance
  5437. * NULL: no specific Vdev is required and check all allcated TX desc
  5438. * on this pdev.
  5439. * Non-NULL: only check the allocated TX Desc associated to this Vdev.
  5440. *
  5441. * @force_free:
  5442. * true: flush the TX desc.
  5443. * false: only reset the Vdev in each allocated TX desc
  5444. * that associated to current Vdev.
  5445. *
  5446. * This function will go through the TX desc pool to flush
  5447. * the outstanding TX data or reset Vdev to NULL in associated TX
  5448. * Desc.
  5449. */
  5450. void dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
  5451. bool force_free)
  5452. {
  5453. uint8_t i;
  5454. uint32_t j;
  5455. uint32_t num_desc, page_id, offset;
  5456. uint16_t num_desc_per_page;
  5457. struct dp_soc *soc = pdev->soc;
  5458. struct dp_tx_desc_s *tx_desc = NULL;
  5459. struct dp_tx_desc_pool_s *tx_desc_pool = NULL;
  5460. if (!vdev && !force_free) {
  5461. dp_err("Reset TX desc vdev, Vdev param is required!");
  5462. return;
  5463. }
  5464. for (i = 0; i < MAX_TXDESC_POOLS; i++) {
  5465. tx_desc_pool = &soc->tx_desc[i];
  5466. if (!(tx_desc_pool->pool_size) ||
  5467. IS_TX_DESC_POOL_STATUS_INACTIVE(tx_desc_pool) ||
  5468. !(tx_desc_pool->desc_pages.cacheable_pages))
  5469. continue;
  5470. /*
  5471. * Add flow pool lock protection in case pool is freed
  5472. * due to all tx_desc is recycled when handle TX completion.
  5473. * this is not necessary when do force flush as:
  5474. * a. double lock will happen if dp_tx_desc_release is
  5475. * also trying to acquire it.
  5476. * b. dp interrupt has been disabled before do force TX desc
  5477. * flush in dp_pdev_deinit().
  5478. */
  5479. if (!force_free)
  5480. qdf_spin_lock_bh(&tx_desc_pool->flow_pool_lock);
  5481. num_desc = tx_desc_pool->pool_size;
  5482. num_desc_per_page =
  5483. tx_desc_pool->desc_pages.num_element_per_page;
  5484. for (j = 0; j < num_desc; j++) {
  5485. page_id = j / num_desc_per_page;
  5486. offset = j % num_desc_per_page;
  5487. if (qdf_unlikely(!(tx_desc_pool->
  5488. desc_pages.cacheable_pages)))
  5489. break;
  5490. tx_desc = dp_tx_desc_find(soc, i, page_id, offset);
  5491. if (dp_is_tx_desc_flush_match(pdev, vdev, tx_desc)) {
  5492. /*
  5493. * Free TX desc if force free is
  5494. * required, otherwise only reset vdev
  5495. * in this TX desc.
  5496. */
  5497. if (force_free) {
  5498. tx_desc->flags |= DP_TX_DESC_FLAG_FLUSH;
  5499. dp_tx_comp_free_buf(soc, tx_desc,
  5500. false);
  5501. dp_tx_desc_release(tx_desc, i);
  5502. } else {
  5503. tx_desc->vdev_id = DP_INVALID_VDEV_ID;
  5504. }
  5505. }
  5506. }
  5507. if (!force_free)
  5508. qdf_spin_unlock_bh(&tx_desc_pool->flow_pool_lock);
  5509. }
  5510. }
  5511. #else /* QCA_LL_TX_FLOW_CONTROL_V2! */
  5512. /**
  5513. * dp_tx_desc_reset_vdev() - reset vdev to NULL in TX Desc
  5514. *
  5515. * @soc: Handle to DP soc structure
  5516. * @tx_desc: pointer of one TX desc
  5517. * @desc_pool_id: TX Desc pool id
  5518. */
  5519. static inline void
  5520. dp_tx_desc_reset_vdev(struct dp_soc *soc, struct dp_tx_desc_s *tx_desc,
  5521. uint8_t desc_pool_id)
  5522. {
  5523. TX_DESC_LOCK_LOCK(&soc->tx_desc[desc_pool_id].lock);
  5524. tx_desc->vdev_id = DP_INVALID_VDEV_ID;
  5525. TX_DESC_LOCK_UNLOCK(&soc->tx_desc[desc_pool_id].lock);
  5526. }
  5527. void dp_tx_desc_flush(struct dp_pdev *pdev, struct dp_vdev *vdev,
  5528. bool force_free)
  5529. {
  5530. uint8_t i, num_pool;
  5531. uint32_t j;
  5532. uint32_t num_desc, page_id, offset;
  5533. uint16_t num_desc_per_page;
  5534. struct dp_soc *soc = pdev->soc;
  5535. struct dp_tx_desc_s *tx_desc = NULL;
  5536. struct dp_tx_desc_pool_s *tx_desc_pool = NULL;
  5537. if (!vdev && !force_free) {
  5538. dp_err("Reset TX desc vdev, Vdev param is required!");
  5539. return;
  5540. }
  5541. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  5542. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5543. for (i = 0; i < num_pool; i++) {
  5544. tx_desc_pool = &soc->tx_desc[i];
  5545. if (!tx_desc_pool->desc_pages.cacheable_pages)
  5546. continue;
  5547. num_desc_per_page =
  5548. tx_desc_pool->desc_pages.num_element_per_page;
  5549. for (j = 0; j < num_desc; j++) {
  5550. page_id = j / num_desc_per_page;
  5551. offset = j % num_desc_per_page;
  5552. tx_desc = dp_tx_desc_find(soc, i, page_id, offset);
  5553. if (dp_is_tx_desc_flush_match(pdev, vdev, tx_desc)) {
  5554. if (force_free) {
  5555. tx_desc->flags |= DP_TX_DESC_FLAG_FLUSH;
  5556. dp_tx_comp_free_buf(soc, tx_desc,
  5557. false);
  5558. dp_tx_desc_release(tx_desc, i);
  5559. } else {
  5560. dp_tx_desc_reset_vdev(soc, tx_desc,
  5561. i);
  5562. }
  5563. }
  5564. }
  5565. }
  5566. }
  5567. #endif /* !QCA_LL_TX_FLOW_CONTROL_V2 */
  5568. /**
  5569. * dp_tx_vdev_detach() - detach vdev from dp tx
  5570. * @vdev: virtual device instance
  5571. *
  5572. * Return: QDF_STATUS_SUCCESS: success
  5573. * QDF_STATUS_E_RESOURCES: Error return
  5574. */
  5575. QDF_STATUS dp_tx_vdev_detach(struct dp_vdev *vdev)
  5576. {
  5577. struct dp_pdev *pdev = vdev->pdev;
  5578. /* Reset TX desc associated to this Vdev as NULL */
  5579. dp_tx_desc_flush(pdev, vdev, false);
  5580. return QDF_STATUS_SUCCESS;
  5581. }
  5582. #ifdef QCA_LL_TX_FLOW_CONTROL_V2
  5583. /* Pools will be allocated dynamically */
  5584. static QDF_STATUS dp_tx_alloc_static_pools(struct dp_soc *soc, int num_pool,
  5585. int num_desc)
  5586. {
  5587. uint8_t i;
  5588. for (i = 0; i < num_pool; i++) {
  5589. qdf_spinlock_create(&soc->tx_desc[i].flow_pool_lock);
  5590. soc->tx_desc[i].status = FLOW_POOL_INACTIVE;
  5591. }
  5592. return QDF_STATUS_SUCCESS;
  5593. }
  5594. static QDF_STATUS dp_tx_init_static_pools(struct dp_soc *soc, int num_pool,
  5595. uint32_t num_desc)
  5596. {
  5597. return QDF_STATUS_SUCCESS;
  5598. }
  5599. static void dp_tx_deinit_static_pools(struct dp_soc *soc, int num_pool)
  5600. {
  5601. }
  5602. static void dp_tx_delete_static_pools(struct dp_soc *soc, int num_pool)
  5603. {
  5604. uint8_t i;
  5605. for (i = 0; i < num_pool; i++)
  5606. qdf_spinlock_destroy(&soc->tx_desc[i].flow_pool_lock);
  5607. }
  5608. #else /* QCA_LL_TX_FLOW_CONTROL_V2! */
  5609. static QDF_STATUS dp_tx_alloc_static_pools(struct dp_soc *soc, int num_pool,
  5610. uint32_t num_desc)
  5611. {
  5612. uint8_t i, count;
  5613. /* Allocate software Tx descriptor pools */
  5614. for (i = 0; i < num_pool; i++) {
  5615. if (dp_tx_desc_pool_alloc(soc, i, num_desc)) {
  5616. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  5617. FL("Tx Desc Pool alloc %d failed %pK"),
  5618. i, soc);
  5619. goto fail;
  5620. }
  5621. }
  5622. return QDF_STATUS_SUCCESS;
  5623. fail:
  5624. for (count = 0; count < i; count++)
  5625. dp_tx_desc_pool_free(soc, count);
  5626. return QDF_STATUS_E_NOMEM;
  5627. }
  5628. static QDF_STATUS dp_tx_init_static_pools(struct dp_soc *soc, int num_pool,
  5629. uint32_t num_desc)
  5630. {
  5631. uint8_t i;
  5632. for (i = 0; i < num_pool; i++) {
  5633. if (dp_tx_desc_pool_init(soc, i, num_desc)) {
  5634. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR,
  5635. FL("Tx Desc Pool init %d failed %pK"),
  5636. i, soc);
  5637. return QDF_STATUS_E_NOMEM;
  5638. }
  5639. }
  5640. return QDF_STATUS_SUCCESS;
  5641. }
  5642. static void dp_tx_deinit_static_pools(struct dp_soc *soc, int num_pool)
  5643. {
  5644. uint8_t i;
  5645. for (i = 0; i < num_pool; i++)
  5646. dp_tx_desc_pool_deinit(soc, i);
  5647. }
  5648. static void dp_tx_delete_static_pools(struct dp_soc *soc, int num_pool)
  5649. {
  5650. uint8_t i;
  5651. for (i = 0; i < num_pool; i++)
  5652. dp_tx_desc_pool_free(soc, i);
  5653. }
  5654. #endif /* !QCA_LL_TX_FLOW_CONTROL_V2 */
  5655. /**
  5656. * dp_tx_tso_cmn_desc_pool_deinit() - de-initialize TSO descriptors
  5657. * @soc: core txrx main context
  5658. * @num_pool: number of pools
  5659. *
  5660. */
  5661. void dp_tx_tso_cmn_desc_pool_deinit(struct dp_soc *soc, uint8_t num_pool)
  5662. {
  5663. dp_tx_tso_desc_pool_deinit(soc, num_pool);
  5664. dp_tx_tso_num_seg_pool_deinit(soc, num_pool);
  5665. }
  5666. /**
  5667. * dp_tx_tso_cmn_desc_pool_free() - free TSO descriptors
  5668. * @soc: core txrx main context
  5669. * @num_pool: number of pools
  5670. *
  5671. */
  5672. void dp_tx_tso_cmn_desc_pool_free(struct dp_soc *soc, uint8_t num_pool)
  5673. {
  5674. dp_tx_tso_desc_pool_free(soc, num_pool);
  5675. dp_tx_tso_num_seg_pool_free(soc, num_pool);
  5676. }
  5677. /**
  5678. * dp_soc_tx_desc_sw_pools_free() - free all TX descriptors
  5679. * @soc: core txrx main context
  5680. *
  5681. * This function frees all tx related descriptors as below
  5682. * 1. Regular TX descriptors (static pools)
  5683. * 2. extension TX descriptors (used for ME, RAW, TSO etc...)
  5684. * 3. TSO descriptors
  5685. *
  5686. */
  5687. void dp_soc_tx_desc_sw_pools_free(struct dp_soc *soc)
  5688. {
  5689. uint8_t num_pool;
  5690. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5691. dp_tx_tso_cmn_desc_pool_free(soc, num_pool);
  5692. dp_tx_ext_desc_pool_free(soc, num_pool);
  5693. dp_tx_delete_static_pools(soc, num_pool);
  5694. }
  5695. /**
  5696. * dp_soc_tx_desc_sw_pools_deinit() - de-initialize all TX descriptors
  5697. * @soc: core txrx main context
  5698. *
  5699. * This function de-initializes all tx related descriptors as below
  5700. * 1. Regular TX descriptors (static pools)
  5701. * 2. extension TX descriptors (used for ME, RAW, TSO etc...)
  5702. * 3. TSO descriptors
  5703. *
  5704. */
  5705. void dp_soc_tx_desc_sw_pools_deinit(struct dp_soc *soc)
  5706. {
  5707. uint8_t num_pool;
  5708. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5709. dp_tx_flow_control_deinit(soc);
  5710. dp_tx_tso_cmn_desc_pool_deinit(soc, num_pool);
  5711. dp_tx_ext_desc_pool_deinit(soc, num_pool);
  5712. dp_tx_deinit_static_pools(soc, num_pool);
  5713. }
  5714. /**
  5715. * dp_tso_attach() - TSO attach handler
  5716. * @txrx_soc: Opaque Dp handle
  5717. *
  5718. * Reserve TSO descriptor buffers
  5719. *
  5720. * Return: QDF_STATUS_E_FAILURE on failure or
  5721. * QDF_STATUS_SUCCESS on success
  5722. */
  5723. QDF_STATUS dp_tx_tso_cmn_desc_pool_alloc(struct dp_soc *soc,
  5724. uint8_t num_pool,
  5725. uint32_t num_desc)
  5726. {
  5727. if (dp_tx_tso_desc_pool_alloc(soc, num_pool, num_desc)) {
  5728. dp_err("TSO Desc Pool alloc %d failed %pK", num_pool, soc);
  5729. return QDF_STATUS_E_FAILURE;
  5730. }
  5731. if (dp_tx_tso_num_seg_pool_alloc(soc, num_pool, num_desc)) {
  5732. dp_err("TSO Num of seg Pool alloc %d failed %pK",
  5733. num_pool, soc);
  5734. return QDF_STATUS_E_FAILURE;
  5735. }
  5736. return QDF_STATUS_SUCCESS;
  5737. }
  5738. /**
  5739. * dp_tx_tso_cmn_desc_pool_init() - TSO cmn desc pool init
  5740. * @soc: DP soc handle
  5741. * @num_pool: Number of pools
  5742. * @num_desc: Number of descriptors
  5743. *
  5744. * Initialize TSO descriptor pools
  5745. *
  5746. * Return: QDF_STATUS_E_FAILURE on failure or
  5747. * QDF_STATUS_SUCCESS on success
  5748. */
  5749. QDF_STATUS dp_tx_tso_cmn_desc_pool_init(struct dp_soc *soc,
  5750. uint8_t num_pool,
  5751. uint32_t num_desc)
  5752. {
  5753. if (dp_tx_tso_desc_pool_init(soc, num_pool, num_desc)) {
  5754. dp_err("TSO Desc Pool alloc %d failed %pK", num_pool, soc);
  5755. return QDF_STATUS_E_FAILURE;
  5756. }
  5757. if (dp_tx_tso_num_seg_pool_init(soc, num_pool, num_desc)) {
  5758. dp_err("TSO Num of seg Pool alloc %d failed %pK",
  5759. num_pool, soc);
  5760. return QDF_STATUS_E_FAILURE;
  5761. }
  5762. return QDF_STATUS_SUCCESS;
  5763. }
  5764. /**
  5765. * dp_soc_tx_desc_sw_pools_alloc() - Allocate tx descriptor pool memory
  5766. * @soc: core txrx main context
  5767. *
  5768. * This function allocates memory for following descriptor pools
  5769. * 1. regular sw tx descriptor pools (static pools)
  5770. * 2. TX extension descriptor pools (ME, RAW, TSO etc...)
  5771. * 3. TSO descriptor pools
  5772. *
  5773. * Return: QDF_STATUS_SUCCESS: success
  5774. * QDF_STATUS_E_RESOURCES: Error return
  5775. */
  5776. QDF_STATUS dp_soc_tx_desc_sw_pools_alloc(struct dp_soc *soc)
  5777. {
  5778. uint8_t num_pool;
  5779. uint32_t num_desc;
  5780. uint32_t num_ext_desc;
  5781. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5782. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  5783. num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
  5784. QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO,
  5785. "%s Tx Desc Alloc num_pool = %d, descs = %d",
  5786. __func__, num_pool, num_desc);
  5787. if ((num_pool > MAX_TXDESC_POOLS) ||
  5788. (num_desc > WLAN_CFG_NUM_TX_DESC_MAX))
  5789. goto fail1;
  5790. if (dp_tx_alloc_static_pools(soc, num_pool, num_desc))
  5791. goto fail1;
  5792. if (dp_tx_ext_desc_pool_alloc(soc, num_pool, num_ext_desc))
  5793. goto fail2;
  5794. if (wlan_cfg_is_tso_desc_attach_defer(soc->wlan_cfg_ctx))
  5795. return QDF_STATUS_SUCCESS;
  5796. if (dp_tx_tso_cmn_desc_pool_alloc(soc, num_pool, num_ext_desc))
  5797. goto fail3;
  5798. return QDF_STATUS_SUCCESS;
  5799. fail3:
  5800. dp_tx_ext_desc_pool_free(soc, num_pool);
  5801. fail2:
  5802. dp_tx_delete_static_pools(soc, num_pool);
  5803. fail1:
  5804. return QDF_STATUS_E_RESOURCES;
  5805. }
  5806. /**
  5807. * dp_soc_tx_desc_sw_pools_init() - Initialise TX descriptor pools
  5808. * @soc: core txrx main context
  5809. *
  5810. * This function initializes the following TX descriptor pools
  5811. * 1. regular sw tx descriptor pools (static pools)
  5812. * 2. TX extension descriptor pools (ME, RAW, TSO etc...)
  5813. * 3. TSO descriptor pools
  5814. *
  5815. * Return: QDF_STATUS_SUCCESS: success
  5816. * QDF_STATUS_E_RESOURCES: Error return
  5817. */
  5818. QDF_STATUS dp_soc_tx_desc_sw_pools_init(struct dp_soc *soc)
  5819. {
  5820. uint8_t num_pool;
  5821. uint32_t num_desc;
  5822. uint32_t num_ext_desc;
  5823. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5824. num_desc = wlan_cfg_get_num_tx_desc(soc->wlan_cfg_ctx);
  5825. num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
  5826. if (dp_tx_init_static_pools(soc, num_pool, num_desc))
  5827. goto fail1;
  5828. if (dp_tx_ext_desc_pool_init(soc, num_pool, num_ext_desc))
  5829. goto fail2;
  5830. if (wlan_cfg_is_tso_desc_attach_defer(soc->wlan_cfg_ctx))
  5831. return QDF_STATUS_SUCCESS;
  5832. if (dp_tx_tso_cmn_desc_pool_init(soc, num_pool, num_ext_desc))
  5833. goto fail3;
  5834. dp_tx_flow_control_init(soc);
  5835. soc->process_tx_status = CONFIG_PROCESS_TX_STATUS;
  5836. return QDF_STATUS_SUCCESS;
  5837. fail3:
  5838. dp_tx_ext_desc_pool_deinit(soc, num_pool);
  5839. fail2:
  5840. dp_tx_deinit_static_pools(soc, num_pool);
  5841. fail1:
  5842. return QDF_STATUS_E_RESOURCES;
  5843. }
  5844. /**
  5845. * dp_tso_soc_attach() - Allocate and initialize TSO descriptors
  5846. * @txrx_soc: dp soc handle
  5847. *
  5848. * Return: QDF_STATUS - QDF_STATUS_SUCCESS
  5849. * QDF_STATUS_E_FAILURE
  5850. */
  5851. QDF_STATUS dp_tso_soc_attach(struct cdp_soc_t *txrx_soc)
  5852. {
  5853. struct dp_soc *soc = (struct dp_soc *)txrx_soc;
  5854. uint8_t num_pool;
  5855. uint32_t num_ext_desc;
  5856. num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5857. num_ext_desc = wlan_cfg_get_num_tx_ext_desc(soc->wlan_cfg_ctx);
  5858. if (dp_tx_tso_cmn_desc_pool_alloc(soc, num_pool, num_ext_desc))
  5859. return QDF_STATUS_E_FAILURE;
  5860. if (dp_tx_tso_cmn_desc_pool_init(soc, num_pool, num_ext_desc))
  5861. return QDF_STATUS_E_FAILURE;
  5862. return QDF_STATUS_SUCCESS;
  5863. }
  5864. /**
  5865. * dp_tso_soc_detach() - de-initialize and free the TSO descriptors
  5866. * @txrx_soc: dp soc handle
  5867. *
  5868. * Return: QDF_STATUS - QDF_STATUS_SUCCESS
  5869. */
  5870. QDF_STATUS dp_tso_soc_detach(struct cdp_soc_t *txrx_soc)
  5871. {
  5872. struct dp_soc *soc = (struct dp_soc *)txrx_soc;
  5873. uint8_t num_pool = wlan_cfg_get_num_tx_desc_pool(soc->wlan_cfg_ctx);
  5874. dp_tx_tso_cmn_desc_pool_deinit(soc, num_pool);
  5875. dp_tx_tso_cmn_desc_pool_free(soc, num_pool);
  5876. return QDF_STATUS_SUCCESS;
  5877. }
  5878. #ifdef CONFIG_DP_PKT_ADD_TIMESTAMP
  5879. void dp_pkt_add_timestamp(struct dp_vdev *vdev,
  5880. enum qdf_pkt_timestamp_index index, uint64_t time,
  5881. qdf_nbuf_t nbuf)
  5882. {
  5883. if (qdf_unlikely(qdf_is_dp_pkt_timestamp_enabled())) {
  5884. uint64_t tsf_time;
  5885. if (vdev->get_tsf_time) {
  5886. vdev->get_tsf_time(vdev->osif_vdev, time, &tsf_time);
  5887. qdf_add_dp_pkt_timestamp(nbuf, index, tsf_time);
  5888. }
  5889. }
  5890. }
  5891. void dp_pkt_get_timestamp(uint64_t *time)
  5892. {
  5893. if (qdf_unlikely(qdf_is_dp_pkt_timestamp_enabled()))
  5894. *time = qdf_get_log_timestamp();
  5895. }
  5896. #endif