lpfc_nvmet.c 108 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645
  1. /*******************************************************************
  2. * This file is part of the Emulex Linux Device Driver for *
  3. * Fibre Channel Host Bus Adapters. *
  4. * Copyright (C) 2017-2022 Broadcom. All Rights Reserved. The term *
  5. * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries. *
  6. * Copyright (C) 2004-2016 Emulex. All rights reserved. *
  7. * EMULEX and SLI are trademarks of Emulex. *
  8. * www.broadcom.com *
  9. * Portions Copyright (C) 2004-2005 Christoph Hellwig *
  10. * *
  11. * This program is free software; you can redistribute it and/or *
  12. * modify it under the terms of version 2 of the GNU General *
  13. * Public License as published by the Free Software Foundation. *
  14. * This program is distributed in the hope that it will be useful. *
  15. * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND *
  16. * WARRANTIES, INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, *
  17. * FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, ARE *
  18. * DISCLAIMED, EXCEPT TO THE EXTENT THAT SUCH DISCLAIMERS ARE HELD *
  19. * TO BE LEGALLY INVALID. See the GNU General Public License for *
  20. * more details, a copy of which can be found in the file COPYING *
  21. * included with this package. *
  22. ********************************************************************/
  23. #include <linux/pci.h>
  24. #include <linux/slab.h>
  25. #include <linux/interrupt.h>
  26. #include <linux/delay.h>
  27. #include <asm/unaligned.h>
  28. #include <linux/crc-t10dif.h>
  29. #include <net/checksum.h>
  30. #include <scsi/scsi.h>
  31. #include <scsi/scsi_device.h>
  32. #include <scsi/scsi_eh.h>
  33. #include <scsi/scsi_host.h>
  34. #include <scsi/scsi_tcq.h>
  35. #include <scsi/scsi_transport_fc.h>
  36. #include <scsi/fc/fc_fs.h>
  37. #include "lpfc_version.h"
  38. #include "lpfc_hw4.h"
  39. #include "lpfc_hw.h"
  40. #include "lpfc_sli.h"
  41. #include "lpfc_sli4.h"
  42. #include "lpfc_nl.h"
  43. #include "lpfc_disc.h"
  44. #include "lpfc.h"
  45. #include "lpfc_scsi.h"
  46. #include "lpfc_nvme.h"
  47. #include "lpfc_logmsg.h"
  48. #include "lpfc_crtn.h"
  49. #include "lpfc_vport.h"
  50. #include "lpfc_debugfs.h"
  51. static struct lpfc_iocbq *lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *,
  52. struct lpfc_async_xchg_ctx *,
  53. dma_addr_t rspbuf,
  54. uint16_t rspsize);
  55. static struct lpfc_iocbq *lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *,
  56. struct lpfc_async_xchg_ctx *);
  57. static int lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *,
  58. struct lpfc_async_xchg_ctx *,
  59. uint32_t, uint16_t);
  60. static int lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *,
  61. struct lpfc_async_xchg_ctx *,
  62. uint32_t, uint16_t);
  63. static void lpfc_nvmet_wqfull_flush(struct lpfc_hba *, struct lpfc_queue *,
  64. struct lpfc_async_xchg_ctx *);
  65. static void lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *);
  66. static void lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf);
  67. static union lpfc_wqe128 lpfc_tsend_cmd_template;
  68. static union lpfc_wqe128 lpfc_treceive_cmd_template;
  69. static union lpfc_wqe128 lpfc_trsp_cmd_template;
  70. /* Setup WQE templates for NVME IOs */
  71. void
  72. lpfc_nvmet_cmd_template(void)
  73. {
  74. union lpfc_wqe128 *wqe;
  75. /* TSEND template */
  76. wqe = &lpfc_tsend_cmd_template;
  77. memset(wqe, 0, sizeof(union lpfc_wqe128));
  78. /* Word 0, 1, 2 - BDE is variable */
  79. /* Word 3 - payload_offset_len is zero */
  80. /* Word 4 - relative_offset is variable */
  81. /* Word 5 - is zero */
  82. /* Word 6 - ctxt_tag, xri_tag is variable */
  83. /* Word 7 - wqe_ar is variable */
  84. bf_set(wqe_cmnd, &wqe->fcp_tsend.wqe_com, CMD_FCP_TSEND64_WQE);
  85. bf_set(wqe_pu, &wqe->fcp_tsend.wqe_com, PARM_REL_OFF);
  86. bf_set(wqe_class, &wqe->fcp_tsend.wqe_com, CLASS3);
  87. bf_set(wqe_ct, &wqe->fcp_tsend.wqe_com, SLI4_CT_RPI);
  88. bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 1);
  89. /* Word 8 - abort_tag is variable */
  90. /* Word 9 - reqtag, rcvoxid is variable */
  91. /* Word 10 - wqes, xc is variable */
  92. bf_set(wqe_xchg, &wqe->fcp_tsend.wqe_com, LPFC_NVME_XCHG);
  93. bf_set(wqe_dbde, &wqe->fcp_tsend.wqe_com, 1);
  94. bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 0);
  95. bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
  96. bf_set(wqe_iod, &wqe->fcp_tsend.wqe_com, LPFC_WQE_IOD_WRITE);
  97. bf_set(wqe_lenloc, &wqe->fcp_tsend.wqe_com, LPFC_WQE_LENLOC_WORD12);
  98. /* Word 11 - sup, irsp, irsplen is variable */
  99. bf_set(wqe_cmd_type, &wqe->fcp_tsend.wqe_com, FCP_COMMAND_TSEND);
  100. bf_set(wqe_cqid, &wqe->fcp_tsend.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
  101. bf_set(wqe_sup, &wqe->fcp_tsend.wqe_com, 0);
  102. bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 0);
  103. bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com, 0);
  104. bf_set(wqe_pbde, &wqe->fcp_tsend.wqe_com, 0);
  105. /* Word 12 - fcp_data_len is variable */
  106. /* Word 13, 14, 15 - PBDE is zero */
  107. /* TRECEIVE template */
  108. wqe = &lpfc_treceive_cmd_template;
  109. memset(wqe, 0, sizeof(union lpfc_wqe128));
  110. /* Word 0, 1, 2 - BDE is variable */
  111. /* Word 3 */
  112. wqe->fcp_treceive.payload_offset_len = TXRDY_PAYLOAD_LEN;
  113. /* Word 4 - relative_offset is variable */
  114. /* Word 5 - is zero */
  115. /* Word 6 - ctxt_tag, xri_tag is variable */
  116. /* Word 7 */
  117. bf_set(wqe_cmnd, &wqe->fcp_treceive.wqe_com, CMD_FCP_TRECEIVE64_WQE);
  118. bf_set(wqe_pu, &wqe->fcp_treceive.wqe_com, PARM_REL_OFF);
  119. bf_set(wqe_class, &wqe->fcp_treceive.wqe_com, CLASS3);
  120. bf_set(wqe_ct, &wqe->fcp_treceive.wqe_com, SLI4_CT_RPI);
  121. bf_set(wqe_ar, &wqe->fcp_treceive.wqe_com, 0);
  122. /* Word 8 - abort_tag is variable */
  123. /* Word 9 - reqtag, rcvoxid is variable */
  124. /* Word 10 - xc is variable */
  125. bf_set(wqe_dbde, &wqe->fcp_treceive.wqe_com, 1);
  126. bf_set(wqe_wqes, &wqe->fcp_treceive.wqe_com, 0);
  127. bf_set(wqe_xchg, &wqe->fcp_treceive.wqe_com, LPFC_NVME_XCHG);
  128. bf_set(wqe_iod, &wqe->fcp_treceive.wqe_com, LPFC_WQE_IOD_READ);
  129. bf_set(wqe_lenloc, &wqe->fcp_treceive.wqe_com, LPFC_WQE_LENLOC_WORD12);
  130. bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 1);
  131. /* Word 11 - pbde is variable */
  132. bf_set(wqe_cmd_type, &wqe->fcp_treceive.wqe_com, FCP_COMMAND_TRECEIVE);
  133. bf_set(wqe_cqid, &wqe->fcp_treceive.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
  134. bf_set(wqe_sup, &wqe->fcp_treceive.wqe_com, 0);
  135. bf_set(wqe_irsp, &wqe->fcp_treceive.wqe_com, 0);
  136. bf_set(wqe_irsplen, &wqe->fcp_treceive.wqe_com, 0);
  137. bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 1);
  138. /* Word 12 - fcp_data_len is variable */
  139. /* Word 13, 14, 15 - PBDE is variable */
  140. /* TRSP template */
  141. wqe = &lpfc_trsp_cmd_template;
  142. memset(wqe, 0, sizeof(union lpfc_wqe128));
  143. /* Word 0, 1, 2 - BDE is variable */
  144. /* Word 3 - response_len is variable */
  145. /* Word 4, 5 - is zero */
  146. /* Word 6 - ctxt_tag, xri_tag is variable */
  147. /* Word 7 */
  148. bf_set(wqe_cmnd, &wqe->fcp_trsp.wqe_com, CMD_FCP_TRSP64_WQE);
  149. bf_set(wqe_pu, &wqe->fcp_trsp.wqe_com, PARM_UNUSED);
  150. bf_set(wqe_class, &wqe->fcp_trsp.wqe_com, CLASS3);
  151. bf_set(wqe_ct, &wqe->fcp_trsp.wqe_com, SLI4_CT_RPI);
  152. bf_set(wqe_ag, &wqe->fcp_trsp.wqe_com, 1); /* wqe_ar */
  153. /* Word 8 - abort_tag is variable */
  154. /* Word 9 - reqtag is variable */
  155. /* Word 10 wqes, xc is variable */
  156. bf_set(wqe_dbde, &wqe->fcp_trsp.wqe_com, 1);
  157. bf_set(wqe_xchg, &wqe->fcp_trsp.wqe_com, LPFC_NVME_XCHG);
  158. bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 0);
  159. bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 0);
  160. bf_set(wqe_iod, &wqe->fcp_trsp.wqe_com, LPFC_WQE_IOD_NONE);
  161. bf_set(wqe_lenloc, &wqe->fcp_trsp.wqe_com, LPFC_WQE_LENLOC_WORD3);
  162. /* Word 11 irsp, irsplen is variable */
  163. bf_set(wqe_cmd_type, &wqe->fcp_trsp.wqe_com, FCP_COMMAND_TRSP);
  164. bf_set(wqe_cqid, &wqe->fcp_trsp.wqe_com, LPFC_WQE_CQ_ID_DEFAULT);
  165. bf_set(wqe_sup, &wqe->fcp_trsp.wqe_com, 0);
  166. bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 0);
  167. bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com, 0);
  168. bf_set(wqe_pbde, &wqe->fcp_trsp.wqe_com, 0);
  169. /* Word 12, 13, 14, 15 - is zero */
  170. }
  171. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  172. static struct lpfc_async_xchg_ctx *
  173. lpfc_nvmet_get_ctx_for_xri(struct lpfc_hba *phba, u16 xri)
  174. {
  175. struct lpfc_async_xchg_ctx *ctxp;
  176. unsigned long iflag;
  177. bool found = false;
  178. spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
  179. list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) {
  180. if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
  181. continue;
  182. found = true;
  183. break;
  184. }
  185. spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);
  186. if (found)
  187. return ctxp;
  188. return NULL;
  189. }
  190. static struct lpfc_async_xchg_ctx *
  191. lpfc_nvmet_get_ctx_for_oxid(struct lpfc_hba *phba, u16 oxid, u32 sid)
  192. {
  193. struct lpfc_async_xchg_ctx *ctxp;
  194. unsigned long iflag;
  195. bool found = false;
  196. spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
  197. list_for_each_entry(ctxp, &phba->sli4_hba.t_active_ctx_list, list) {
  198. if (ctxp->oxid != oxid || ctxp->sid != sid)
  199. continue;
  200. found = true;
  201. break;
  202. }
  203. spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);
  204. if (found)
  205. return ctxp;
  206. return NULL;
  207. }
  208. #endif
  209. static void
  210. lpfc_nvmet_defer_release(struct lpfc_hba *phba,
  211. struct lpfc_async_xchg_ctx *ctxp)
  212. {
  213. lockdep_assert_held(&ctxp->ctxlock);
  214. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  215. "6313 NVMET Defer ctx release oxid x%x flg x%x\n",
  216. ctxp->oxid, ctxp->flag);
  217. if (ctxp->flag & LPFC_NVME_CTX_RLS)
  218. return;
  219. ctxp->flag |= LPFC_NVME_CTX_RLS;
  220. spin_lock(&phba->sli4_hba.t_active_list_lock);
  221. list_del(&ctxp->list);
  222. spin_unlock(&phba->sli4_hba.t_active_list_lock);
  223. spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  224. list_add_tail(&ctxp->list, &phba->sli4_hba.lpfc_abts_nvmet_ctx_list);
  225. spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  226. }
  227. /**
  228. * __lpfc_nvme_xmt_ls_rsp_cmp - Generic completion handler for the
  229. * transmission of an NVME LS response.
  230. * @phba: Pointer to HBA context object.
  231. * @cmdwqe: Pointer to driver command WQE object.
  232. * @rspwqe: Pointer to driver response WQE object.
  233. *
  234. * The function is called from SLI ring event handler with no
  235. * lock held. The function frees memory resources used for the command
  236. * used to send the NVME LS RSP.
  237. **/
  238. void
  239. __lpfc_nvme_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  240. struct lpfc_iocbq *rspwqe)
  241. {
  242. struct lpfc_async_xchg_ctx *axchg = cmdwqe->context_un.axchg;
  243. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  244. struct nvmefc_ls_rsp *ls_rsp = &axchg->ls_rsp;
  245. uint32_t status, result;
  246. status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
  247. result = wcqe->parameter;
  248. if (axchg->state != LPFC_NVME_STE_LS_RSP || axchg->entry_cnt != 2) {
  249. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  250. "6410 NVMEx LS cmpl state mismatch IO x%x: "
  251. "%d %d\n",
  252. axchg->oxid, axchg->state, axchg->entry_cnt);
  253. }
  254. lpfc_nvmeio_data(phba, "NVMEx LS CMPL: xri x%x stat x%x result x%x\n",
  255. axchg->oxid, status, result);
  256. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
  257. "6038 NVMEx LS rsp cmpl: %d %d oxid x%x\n",
  258. status, result, axchg->oxid);
  259. lpfc_nlp_put(cmdwqe->ndlp);
  260. cmdwqe->context_un.axchg = NULL;
  261. cmdwqe->bpl_dmabuf = NULL;
  262. lpfc_sli_release_iocbq(phba, cmdwqe);
  263. ls_rsp->done(ls_rsp);
  264. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
  265. "6200 NVMEx LS rsp cmpl done status %d oxid x%x\n",
  266. status, axchg->oxid);
  267. kfree(axchg);
  268. }
  269. /**
  270. * lpfc_nvmet_xmt_ls_rsp_cmp - Completion handler for LS Response
  271. * @phba: Pointer to HBA context object.
  272. * @cmdwqe: Pointer to driver command WQE object.
  273. * @rspwqe: Pointer to driver response WQE object.
  274. *
  275. * The function is called from SLI ring event handler with no
  276. * lock held. This function is the completion handler for NVME LS commands
  277. * The function updates any states and statistics, then calls the
  278. * generic completion handler to free resources.
  279. **/
  280. static void
  281. lpfc_nvmet_xmt_ls_rsp_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  282. struct lpfc_iocbq *rspwqe)
  283. {
  284. struct lpfc_nvmet_tgtport *tgtp;
  285. uint32_t status, result;
  286. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  287. if (!phba->targetport)
  288. goto finish;
  289. status = bf_get(lpfc_wcqe_c_status, wcqe) & LPFC_IOCB_STATUS_MASK;
  290. result = wcqe->parameter;
  291. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  292. if (tgtp) {
  293. if (status) {
  294. atomic_inc(&tgtp->xmt_ls_rsp_error);
  295. if (result == IOERR_ABORT_REQUESTED)
  296. atomic_inc(&tgtp->xmt_ls_rsp_aborted);
  297. if (bf_get(lpfc_wcqe_c_xb, wcqe))
  298. atomic_inc(&tgtp->xmt_ls_rsp_xb_set);
  299. } else {
  300. atomic_inc(&tgtp->xmt_ls_rsp_cmpl);
  301. }
  302. }
  303. finish:
  304. __lpfc_nvme_xmt_ls_rsp_cmp(phba, cmdwqe, rspwqe);
  305. }
  306. /**
  307. * lpfc_nvmet_ctxbuf_post - Repost a NVMET RQ DMA buffer and clean up context
  308. * @phba: HBA buffer is associated with
  309. * @ctx_buf: ctx buffer context
  310. *
  311. * Description: Frees the given DMA buffer in the appropriate way given by
  312. * reposting it to its associated RQ so it can be reused.
  313. *
  314. * Notes: Takes phba->hbalock. Can be called with or without other locks held.
  315. *
  316. * Returns: None
  317. **/
  318. void
  319. lpfc_nvmet_ctxbuf_post(struct lpfc_hba *phba, struct lpfc_nvmet_ctxbuf *ctx_buf)
  320. {
  321. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  322. struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context;
  323. struct lpfc_nvmet_tgtport *tgtp;
  324. struct fc_frame_header *fc_hdr;
  325. struct rqb_dmabuf *nvmebuf;
  326. struct lpfc_nvmet_ctx_info *infop;
  327. uint32_t size, oxid, sid;
  328. int cpu;
  329. unsigned long iflag;
  330. if (ctxp->state == LPFC_NVME_STE_FREE) {
  331. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  332. "6411 NVMET free, already free IO x%x: %d %d\n",
  333. ctxp->oxid, ctxp->state, ctxp->entry_cnt);
  334. }
  335. if (ctxp->rqb_buffer) {
  336. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  337. nvmebuf = ctxp->rqb_buffer;
  338. /* check if freed in another path whilst acquiring lock */
  339. if (nvmebuf) {
  340. ctxp->rqb_buffer = NULL;
  341. if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) {
  342. ctxp->flag &= ~LPFC_NVME_CTX_REUSE_WQ;
  343. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  344. nvmebuf->hrq->rqbp->rqb_free_buffer(phba,
  345. nvmebuf);
  346. } else {
  347. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  348. /* repost */
  349. lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
  350. }
  351. } else {
  352. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  353. }
  354. }
  355. ctxp->state = LPFC_NVME_STE_FREE;
  356. spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
  357. if (phba->sli4_hba.nvmet_io_wait_cnt) {
  358. list_remove_head(&phba->sli4_hba.lpfc_nvmet_io_wait_list,
  359. nvmebuf, struct rqb_dmabuf,
  360. hbuf.list);
  361. phba->sli4_hba.nvmet_io_wait_cnt--;
  362. spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
  363. iflag);
  364. fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
  365. oxid = be16_to_cpu(fc_hdr->fh_ox_id);
  366. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  367. size = nvmebuf->bytes_recv;
  368. sid = sli4_sid_from_fc_hdr(fc_hdr);
  369. ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context;
  370. ctxp->wqeq = NULL;
  371. ctxp->offset = 0;
  372. ctxp->phba = phba;
  373. ctxp->size = size;
  374. ctxp->oxid = oxid;
  375. ctxp->sid = sid;
  376. ctxp->state = LPFC_NVME_STE_RCV;
  377. ctxp->entry_cnt = 1;
  378. ctxp->flag = 0;
  379. ctxp->ctxbuf = ctx_buf;
  380. ctxp->rqb_buffer = (void *)nvmebuf;
  381. spin_lock_init(&ctxp->ctxlock);
  382. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  383. /* NOTE: isr time stamp is stale when context is re-assigned*/
  384. if (ctxp->ts_isr_cmd) {
  385. ctxp->ts_cmd_nvme = 0;
  386. ctxp->ts_nvme_data = 0;
  387. ctxp->ts_data_wqput = 0;
  388. ctxp->ts_isr_data = 0;
  389. ctxp->ts_data_nvme = 0;
  390. ctxp->ts_nvme_status = 0;
  391. ctxp->ts_status_wqput = 0;
  392. ctxp->ts_isr_status = 0;
  393. ctxp->ts_status_nvme = 0;
  394. }
  395. #endif
  396. atomic_inc(&tgtp->rcv_fcp_cmd_in);
  397. /* Indicate that a replacement buffer has been posted */
  398. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  399. ctxp->flag |= LPFC_NVME_CTX_REUSE_WQ;
  400. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  401. if (!queue_work(phba->wq, &ctx_buf->defer_work)) {
  402. atomic_inc(&tgtp->rcv_fcp_cmd_drop);
  403. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  404. "6181 Unable to queue deferred work "
  405. "for oxid x%x. "
  406. "FCP Drop IO [x%x x%x x%x]\n",
  407. ctxp->oxid,
  408. atomic_read(&tgtp->rcv_fcp_cmd_in),
  409. atomic_read(&tgtp->rcv_fcp_cmd_out),
  410. atomic_read(&tgtp->xmt_fcp_release));
  411. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  412. lpfc_nvmet_defer_release(phba, ctxp);
  413. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  414. lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
  415. }
  416. return;
  417. }
  418. spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
  419. /*
  420. * Use the CPU context list, from the MRQ the IO was received on
  421. * (ctxp->idx), to save context structure.
  422. */
  423. spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
  424. list_del_init(&ctxp->list);
  425. spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);
  426. cpu = raw_smp_processor_id();
  427. infop = lpfc_get_ctx_list(phba, cpu, ctxp->idx);
  428. spin_lock_irqsave(&infop->nvmet_ctx_list_lock, iflag);
  429. list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
  430. infop->nvmet_ctx_list_cnt++;
  431. spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, iflag);
  432. #endif
  433. }
  434. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  435. static void
  436. lpfc_nvmet_ktime(struct lpfc_hba *phba,
  437. struct lpfc_async_xchg_ctx *ctxp)
  438. {
  439. uint64_t seg1, seg2, seg3, seg4, seg5;
  440. uint64_t seg6, seg7, seg8, seg9, seg10;
  441. uint64_t segsum;
  442. if (!ctxp->ts_isr_cmd || !ctxp->ts_cmd_nvme ||
  443. !ctxp->ts_nvme_data || !ctxp->ts_data_wqput ||
  444. !ctxp->ts_isr_data || !ctxp->ts_data_nvme ||
  445. !ctxp->ts_nvme_status || !ctxp->ts_status_wqput ||
  446. !ctxp->ts_isr_status || !ctxp->ts_status_nvme)
  447. return;
  448. if (ctxp->ts_status_nvme < ctxp->ts_isr_cmd)
  449. return;
  450. if (ctxp->ts_isr_cmd > ctxp->ts_cmd_nvme)
  451. return;
  452. if (ctxp->ts_cmd_nvme > ctxp->ts_nvme_data)
  453. return;
  454. if (ctxp->ts_nvme_data > ctxp->ts_data_wqput)
  455. return;
  456. if (ctxp->ts_data_wqput > ctxp->ts_isr_data)
  457. return;
  458. if (ctxp->ts_isr_data > ctxp->ts_data_nvme)
  459. return;
  460. if (ctxp->ts_data_nvme > ctxp->ts_nvme_status)
  461. return;
  462. if (ctxp->ts_nvme_status > ctxp->ts_status_wqput)
  463. return;
  464. if (ctxp->ts_status_wqput > ctxp->ts_isr_status)
  465. return;
  466. if (ctxp->ts_isr_status > ctxp->ts_status_nvme)
  467. return;
  468. /*
  469. * Segment 1 - Time from FCP command received by MSI-X ISR
  470. * to FCP command is passed to NVME Layer.
  471. * Segment 2 - Time from FCP command payload handed
  472. * off to NVME Layer to Driver receives a Command op
  473. * from NVME Layer.
  474. * Segment 3 - Time from Driver receives a Command op
  475. * from NVME Layer to Command is put on WQ.
  476. * Segment 4 - Time from Driver WQ put is done
  477. * to MSI-X ISR for Command cmpl.
  478. * Segment 5 - Time from MSI-X ISR for Command cmpl to
  479. * Command cmpl is passed to NVME Layer.
  480. * Segment 6 - Time from Command cmpl is passed to NVME
  481. * Layer to Driver receives a RSP op from NVME Layer.
  482. * Segment 7 - Time from Driver receives a RSP op from
  483. * NVME Layer to WQ put is done on TRSP FCP Status.
  484. * Segment 8 - Time from Driver WQ put is done on TRSP
  485. * FCP Status to MSI-X ISR for TRSP cmpl.
  486. * Segment 9 - Time from MSI-X ISR for TRSP cmpl to
  487. * TRSP cmpl is passed to NVME Layer.
  488. * Segment 10 - Time from FCP command received by
  489. * MSI-X ISR to command is completed on wire.
  490. * (Segments 1 thru 8) for READDATA / WRITEDATA
  491. * (Segments 1 thru 4) for READDATA_RSP
  492. */
  493. seg1 = ctxp->ts_cmd_nvme - ctxp->ts_isr_cmd;
  494. segsum = seg1;
  495. seg2 = ctxp->ts_nvme_data - ctxp->ts_isr_cmd;
  496. if (segsum > seg2)
  497. return;
  498. seg2 -= segsum;
  499. segsum += seg2;
  500. seg3 = ctxp->ts_data_wqput - ctxp->ts_isr_cmd;
  501. if (segsum > seg3)
  502. return;
  503. seg3 -= segsum;
  504. segsum += seg3;
  505. seg4 = ctxp->ts_isr_data - ctxp->ts_isr_cmd;
  506. if (segsum > seg4)
  507. return;
  508. seg4 -= segsum;
  509. segsum += seg4;
  510. seg5 = ctxp->ts_data_nvme - ctxp->ts_isr_cmd;
  511. if (segsum > seg5)
  512. return;
  513. seg5 -= segsum;
  514. segsum += seg5;
  515. /* For auto rsp commands seg6 thru seg10 will be 0 */
  516. if (ctxp->ts_nvme_status > ctxp->ts_data_nvme) {
  517. seg6 = ctxp->ts_nvme_status - ctxp->ts_isr_cmd;
  518. if (segsum > seg6)
  519. return;
  520. seg6 -= segsum;
  521. segsum += seg6;
  522. seg7 = ctxp->ts_status_wqput - ctxp->ts_isr_cmd;
  523. if (segsum > seg7)
  524. return;
  525. seg7 -= segsum;
  526. segsum += seg7;
  527. seg8 = ctxp->ts_isr_status - ctxp->ts_isr_cmd;
  528. if (segsum > seg8)
  529. return;
  530. seg8 -= segsum;
  531. segsum += seg8;
  532. seg9 = ctxp->ts_status_nvme - ctxp->ts_isr_cmd;
  533. if (segsum > seg9)
  534. return;
  535. seg9 -= segsum;
  536. segsum += seg9;
  537. if (ctxp->ts_isr_status < ctxp->ts_isr_cmd)
  538. return;
  539. seg10 = (ctxp->ts_isr_status -
  540. ctxp->ts_isr_cmd);
  541. } else {
  542. if (ctxp->ts_isr_data < ctxp->ts_isr_cmd)
  543. return;
  544. seg6 = 0;
  545. seg7 = 0;
  546. seg8 = 0;
  547. seg9 = 0;
  548. seg10 = (ctxp->ts_isr_data - ctxp->ts_isr_cmd);
  549. }
  550. phba->ktime_seg1_total += seg1;
  551. if (seg1 < phba->ktime_seg1_min)
  552. phba->ktime_seg1_min = seg1;
  553. else if (seg1 > phba->ktime_seg1_max)
  554. phba->ktime_seg1_max = seg1;
  555. phba->ktime_seg2_total += seg2;
  556. if (seg2 < phba->ktime_seg2_min)
  557. phba->ktime_seg2_min = seg2;
  558. else if (seg2 > phba->ktime_seg2_max)
  559. phba->ktime_seg2_max = seg2;
  560. phba->ktime_seg3_total += seg3;
  561. if (seg3 < phba->ktime_seg3_min)
  562. phba->ktime_seg3_min = seg3;
  563. else if (seg3 > phba->ktime_seg3_max)
  564. phba->ktime_seg3_max = seg3;
  565. phba->ktime_seg4_total += seg4;
  566. if (seg4 < phba->ktime_seg4_min)
  567. phba->ktime_seg4_min = seg4;
  568. else if (seg4 > phba->ktime_seg4_max)
  569. phba->ktime_seg4_max = seg4;
  570. phba->ktime_seg5_total += seg5;
  571. if (seg5 < phba->ktime_seg5_min)
  572. phba->ktime_seg5_min = seg5;
  573. else if (seg5 > phba->ktime_seg5_max)
  574. phba->ktime_seg5_max = seg5;
  575. phba->ktime_data_samples++;
  576. if (!seg6)
  577. goto out;
  578. phba->ktime_seg6_total += seg6;
  579. if (seg6 < phba->ktime_seg6_min)
  580. phba->ktime_seg6_min = seg6;
  581. else if (seg6 > phba->ktime_seg6_max)
  582. phba->ktime_seg6_max = seg6;
  583. phba->ktime_seg7_total += seg7;
  584. if (seg7 < phba->ktime_seg7_min)
  585. phba->ktime_seg7_min = seg7;
  586. else if (seg7 > phba->ktime_seg7_max)
  587. phba->ktime_seg7_max = seg7;
  588. phba->ktime_seg8_total += seg8;
  589. if (seg8 < phba->ktime_seg8_min)
  590. phba->ktime_seg8_min = seg8;
  591. else if (seg8 > phba->ktime_seg8_max)
  592. phba->ktime_seg8_max = seg8;
  593. phba->ktime_seg9_total += seg9;
  594. if (seg9 < phba->ktime_seg9_min)
  595. phba->ktime_seg9_min = seg9;
  596. else if (seg9 > phba->ktime_seg9_max)
  597. phba->ktime_seg9_max = seg9;
  598. out:
  599. phba->ktime_seg10_total += seg10;
  600. if (seg10 < phba->ktime_seg10_min)
  601. phba->ktime_seg10_min = seg10;
  602. else if (seg10 > phba->ktime_seg10_max)
  603. phba->ktime_seg10_max = seg10;
  604. phba->ktime_status_samples++;
  605. }
  606. #endif
  607. /**
  608. * lpfc_nvmet_xmt_fcp_op_cmp - Completion handler for FCP Response
  609. * @phba: Pointer to HBA context object.
  610. * @cmdwqe: Pointer to driver command WQE object.
  611. * @rspwqe: Pointer to driver response WQE object.
  612. *
  613. * The function is called from SLI ring event handler with no
  614. * lock held. This function is the completion handler for NVME FCP commands
  615. * The function frees memory resources used for the NVME commands.
  616. **/
  617. static void
  618. lpfc_nvmet_xmt_fcp_op_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  619. struct lpfc_iocbq *rspwqe)
  620. {
  621. struct lpfc_nvmet_tgtport *tgtp;
  622. struct nvmefc_tgt_fcp_req *rsp;
  623. struct lpfc_async_xchg_ctx *ctxp;
  624. uint32_t status, result, op, logerr;
  625. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  626. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  627. int id;
  628. #endif
  629. ctxp = cmdwqe->context_un.axchg;
  630. ctxp->flag &= ~LPFC_NVME_IO_INP;
  631. rsp = &ctxp->hdlrctx.fcp_req;
  632. op = rsp->op;
  633. status = bf_get(lpfc_wcqe_c_status, wcqe);
  634. result = wcqe->parameter;
  635. if (phba->targetport)
  636. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  637. else
  638. tgtp = NULL;
  639. lpfc_nvmeio_data(phba, "NVMET FCP CMPL: xri x%x op x%x status x%x\n",
  640. ctxp->oxid, op, status);
  641. if (status) {
  642. rsp->fcp_error = NVME_SC_DATA_XFER_ERROR;
  643. rsp->transferred_length = 0;
  644. if (tgtp) {
  645. atomic_inc(&tgtp->xmt_fcp_rsp_error);
  646. if (result == IOERR_ABORT_REQUESTED)
  647. atomic_inc(&tgtp->xmt_fcp_rsp_aborted);
  648. }
  649. logerr = LOG_NVME_IOERR;
  650. /* pick up SLI4 exhange busy condition */
  651. if (bf_get(lpfc_wcqe_c_xb, wcqe)) {
  652. ctxp->flag |= LPFC_NVME_XBUSY;
  653. logerr |= LOG_NVME_ABTS;
  654. if (tgtp)
  655. atomic_inc(&tgtp->xmt_fcp_rsp_xb_set);
  656. } else {
  657. ctxp->flag &= ~LPFC_NVME_XBUSY;
  658. }
  659. lpfc_printf_log(phba, KERN_INFO, logerr,
  660. "6315 IO Error Cmpl oxid: x%x xri: x%x %x/%x "
  661. "XBUSY:x%x\n",
  662. ctxp->oxid, ctxp->ctxbuf->sglq->sli4_xritag,
  663. status, result, ctxp->flag);
  664. } else {
  665. rsp->fcp_error = NVME_SC_SUCCESS;
  666. if (op == NVMET_FCOP_RSP)
  667. rsp->transferred_length = rsp->rsplen;
  668. else
  669. rsp->transferred_length = rsp->transfer_length;
  670. if (tgtp)
  671. atomic_inc(&tgtp->xmt_fcp_rsp_cmpl);
  672. }
  673. if ((op == NVMET_FCOP_READDATA_RSP) ||
  674. (op == NVMET_FCOP_RSP)) {
  675. /* Sanity check */
  676. ctxp->state = LPFC_NVME_STE_DONE;
  677. ctxp->entry_cnt++;
  678. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  679. if (ctxp->ts_cmd_nvme) {
  680. if (rsp->op == NVMET_FCOP_READDATA_RSP) {
  681. ctxp->ts_isr_data =
  682. cmdwqe->isr_timestamp;
  683. ctxp->ts_data_nvme =
  684. ktime_get_ns();
  685. ctxp->ts_nvme_status =
  686. ctxp->ts_data_nvme;
  687. ctxp->ts_status_wqput =
  688. ctxp->ts_data_nvme;
  689. ctxp->ts_isr_status =
  690. ctxp->ts_data_nvme;
  691. ctxp->ts_status_nvme =
  692. ctxp->ts_data_nvme;
  693. } else {
  694. ctxp->ts_isr_status =
  695. cmdwqe->isr_timestamp;
  696. ctxp->ts_status_nvme =
  697. ktime_get_ns();
  698. }
  699. }
  700. #endif
  701. rsp->done(rsp);
  702. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  703. if (ctxp->ts_cmd_nvme)
  704. lpfc_nvmet_ktime(phba, ctxp);
  705. #endif
  706. /* lpfc_nvmet_xmt_fcp_release() will recycle the context */
  707. } else {
  708. ctxp->entry_cnt++;
  709. memset_startat(cmdwqe, 0, cmd_flag);
  710. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  711. if (ctxp->ts_cmd_nvme) {
  712. ctxp->ts_isr_data = cmdwqe->isr_timestamp;
  713. ctxp->ts_data_nvme = ktime_get_ns();
  714. }
  715. #endif
  716. rsp->done(rsp);
  717. }
  718. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  719. if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {
  720. id = raw_smp_processor_id();
  721. this_cpu_inc(phba->sli4_hba.c_stat->cmpl_io);
  722. if (ctxp->cpu != id)
  723. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
  724. "6704 CPU Check cmdcmpl: "
  725. "cpu %d expect %d\n",
  726. id, ctxp->cpu);
  727. }
  728. #endif
  729. }
  730. /**
  731. * __lpfc_nvme_xmt_ls_rsp - Generic service routine to issue transmit
  732. * an NVME LS rsp for a prior NVME LS request that was received.
  733. * @axchg: pointer to exchange context for the NVME LS request the response
  734. * is for.
  735. * @ls_rsp: pointer to the transport LS RSP that is to be sent
  736. * @xmt_ls_rsp_cmp: completion routine to call upon RSP transmit done
  737. *
  738. * This routine is used to format and send a WQE to transmit a NVME LS
  739. * Response. The response is for a prior NVME LS request that was
  740. * received and posted to the transport.
  741. *
  742. * Returns:
  743. * 0 : if response successfully transmit
  744. * non-zero : if response failed to transmit, of the form -Exxx.
  745. **/
  746. int
  747. __lpfc_nvme_xmt_ls_rsp(struct lpfc_async_xchg_ctx *axchg,
  748. struct nvmefc_ls_rsp *ls_rsp,
  749. void (*xmt_ls_rsp_cmp)(struct lpfc_hba *phba,
  750. struct lpfc_iocbq *cmdwqe,
  751. struct lpfc_iocbq *rspwqe))
  752. {
  753. struct lpfc_hba *phba = axchg->phba;
  754. struct hbq_dmabuf *nvmebuf = (struct hbq_dmabuf *)axchg->rqb_buffer;
  755. struct lpfc_iocbq *nvmewqeq;
  756. struct lpfc_dmabuf dmabuf;
  757. struct ulp_bde64 bpl;
  758. int rc;
  759. if (phba->pport->load_flag & FC_UNLOADING)
  760. return -ENODEV;
  761. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
  762. "6023 NVMEx LS rsp oxid x%x\n", axchg->oxid);
  763. if (axchg->state != LPFC_NVME_STE_LS_RCV || axchg->entry_cnt != 1) {
  764. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  765. "6412 NVMEx LS rsp state mismatch "
  766. "oxid x%x: %d %d\n",
  767. axchg->oxid, axchg->state, axchg->entry_cnt);
  768. return -EALREADY;
  769. }
  770. axchg->state = LPFC_NVME_STE_LS_RSP;
  771. axchg->entry_cnt++;
  772. nvmewqeq = lpfc_nvmet_prep_ls_wqe(phba, axchg, ls_rsp->rspdma,
  773. ls_rsp->rsplen);
  774. if (nvmewqeq == NULL) {
  775. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  776. "6150 NVMEx LS Drop Rsp x%x: Prep\n",
  777. axchg->oxid);
  778. rc = -ENOMEM;
  779. goto out_free_buf;
  780. }
  781. /* Save numBdes for bpl2sgl */
  782. nvmewqeq->num_bdes = 1;
  783. nvmewqeq->hba_wqidx = 0;
  784. nvmewqeq->bpl_dmabuf = &dmabuf;
  785. dmabuf.virt = &bpl;
  786. bpl.addrLow = nvmewqeq->wqe.xmit_sequence.bde.addrLow;
  787. bpl.addrHigh = nvmewqeq->wqe.xmit_sequence.bde.addrHigh;
  788. bpl.tus.f.bdeSize = ls_rsp->rsplen;
  789. bpl.tus.f.bdeFlags = 0;
  790. bpl.tus.w = le32_to_cpu(bpl.tus.w);
  791. /*
  792. * Note: although we're using stack space for the dmabuf, the
  793. * call to lpfc_sli4_issue_wqe is synchronous, so it will not
  794. * be referenced after it returns back to this routine.
  795. */
  796. nvmewqeq->cmd_cmpl = xmt_ls_rsp_cmp;
  797. nvmewqeq->context_un.axchg = axchg;
  798. lpfc_nvmeio_data(phba, "NVMEx LS RSP: xri x%x wqidx x%x len x%x\n",
  799. axchg->oxid, nvmewqeq->hba_wqidx, ls_rsp->rsplen);
  800. rc = lpfc_sli4_issue_wqe(phba, axchg->hdwq, nvmewqeq);
  801. /* clear to be sure there's no reference */
  802. nvmewqeq->bpl_dmabuf = NULL;
  803. if (rc == WQE_SUCCESS) {
  804. /*
  805. * Okay to repost buffer here, but wait till cmpl
  806. * before freeing ctxp and iocbq.
  807. */
  808. lpfc_in_buf_free(phba, &nvmebuf->dbuf);
  809. return 0;
  810. }
  811. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  812. "6151 NVMEx LS RSP x%x: failed to transmit %d\n",
  813. axchg->oxid, rc);
  814. rc = -ENXIO;
  815. lpfc_nlp_put(nvmewqeq->ndlp);
  816. out_free_buf:
  817. /* Give back resources */
  818. lpfc_in_buf_free(phba, &nvmebuf->dbuf);
  819. /*
  820. * As transport doesn't track completions of responses, if the rsp
  821. * fails to send, the transport will effectively ignore the rsp
  822. * and consider the LS done. However, the driver has an active
  823. * exchange open for the LS - so be sure to abort the exchange
  824. * if the response isn't sent.
  825. */
  826. lpfc_nvme_unsol_ls_issue_abort(phba, axchg, axchg->sid, axchg->oxid);
  827. return rc;
  828. }
  829. /**
  830. * lpfc_nvmet_xmt_ls_rsp - Transmit NVME LS response
  831. * @tgtport: pointer to target port that NVME LS is to be transmit from.
  832. * @ls_rsp: pointer to the transport LS RSP that is to be sent
  833. *
  834. * Driver registers this routine to transmit responses for received NVME
  835. * LS requests.
  836. *
  837. * This routine is used to format and send a WQE to transmit a NVME LS
  838. * Response. The ls_rsp is used to reverse-map the LS to the original
  839. * NVME LS request sequence, which provides addressing information for
  840. * the remote port the LS to be sent to, as well as the exchange id
  841. * that is the LS is bound to.
  842. *
  843. * Returns:
  844. * 0 : if response successfully transmit
  845. * non-zero : if response failed to transmit, of the form -Exxx.
  846. **/
  847. static int
  848. lpfc_nvmet_xmt_ls_rsp(struct nvmet_fc_target_port *tgtport,
  849. struct nvmefc_ls_rsp *ls_rsp)
  850. {
  851. struct lpfc_async_xchg_ctx *axchg =
  852. container_of(ls_rsp, struct lpfc_async_xchg_ctx, ls_rsp);
  853. struct lpfc_nvmet_tgtport *nvmep = tgtport->private;
  854. int rc;
  855. if (axchg->phba->pport->load_flag & FC_UNLOADING)
  856. return -ENODEV;
  857. rc = __lpfc_nvme_xmt_ls_rsp(axchg, ls_rsp, lpfc_nvmet_xmt_ls_rsp_cmp);
  858. if (rc) {
  859. atomic_inc(&nvmep->xmt_ls_drop);
  860. /*
  861. * unless the failure is due to having already sent
  862. * the response, an abort will be generated for the
  863. * exchange if the rsp can't be sent.
  864. */
  865. if (rc != -EALREADY)
  866. atomic_inc(&nvmep->xmt_ls_abort);
  867. return rc;
  868. }
  869. atomic_inc(&nvmep->xmt_ls_rsp);
  870. return 0;
  871. }
  872. static int
  873. lpfc_nvmet_xmt_fcp_op(struct nvmet_fc_target_port *tgtport,
  874. struct nvmefc_tgt_fcp_req *rsp)
  875. {
  876. struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
  877. struct lpfc_async_xchg_ctx *ctxp =
  878. container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
  879. struct lpfc_hba *phba = ctxp->phba;
  880. struct lpfc_queue *wq;
  881. struct lpfc_iocbq *nvmewqeq;
  882. struct lpfc_sli_ring *pring;
  883. unsigned long iflags;
  884. int rc;
  885. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  886. int id;
  887. #endif
  888. if (phba->pport->load_flag & FC_UNLOADING) {
  889. rc = -ENODEV;
  890. goto aerr;
  891. }
  892. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  893. if (ctxp->ts_cmd_nvme) {
  894. if (rsp->op == NVMET_FCOP_RSP)
  895. ctxp->ts_nvme_status = ktime_get_ns();
  896. else
  897. ctxp->ts_nvme_data = ktime_get_ns();
  898. }
  899. /* Setup the hdw queue if not already set */
  900. if (!ctxp->hdwq)
  901. ctxp->hdwq = &phba->sli4_hba.hdwq[rsp->hwqid];
  902. if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {
  903. id = raw_smp_processor_id();
  904. this_cpu_inc(phba->sli4_hba.c_stat->xmt_io);
  905. if (rsp->hwqid != id)
  906. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
  907. "6705 CPU Check OP: "
  908. "cpu %d expect %d\n",
  909. id, rsp->hwqid);
  910. ctxp->cpu = id; /* Setup cpu for cmpl check */
  911. }
  912. #endif
  913. /* Sanity check */
  914. if ((ctxp->flag & LPFC_NVME_ABTS_RCV) ||
  915. (ctxp->state == LPFC_NVME_STE_ABORT)) {
  916. atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
  917. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  918. "6102 IO oxid x%x aborted\n",
  919. ctxp->oxid);
  920. rc = -ENXIO;
  921. goto aerr;
  922. }
  923. nvmewqeq = lpfc_nvmet_prep_fcp_wqe(phba, ctxp);
  924. if (nvmewqeq == NULL) {
  925. atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
  926. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  927. "6152 FCP Drop IO x%x: Prep\n",
  928. ctxp->oxid);
  929. rc = -ENXIO;
  930. goto aerr;
  931. }
  932. nvmewqeq->cmd_cmpl = lpfc_nvmet_xmt_fcp_op_cmp;
  933. nvmewqeq->context_un.axchg = ctxp;
  934. nvmewqeq->cmd_flag |= LPFC_IO_NVMET;
  935. ctxp->wqeq->hba_wqidx = rsp->hwqid;
  936. lpfc_nvmeio_data(phba, "NVMET FCP CMND: xri x%x op x%x len x%x\n",
  937. ctxp->oxid, rsp->op, rsp->rsplen);
  938. ctxp->flag |= LPFC_NVME_IO_INP;
  939. rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq);
  940. if (rc == WQE_SUCCESS) {
  941. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  942. if (!ctxp->ts_cmd_nvme)
  943. return 0;
  944. if (rsp->op == NVMET_FCOP_RSP)
  945. ctxp->ts_status_wqput = ktime_get_ns();
  946. else
  947. ctxp->ts_data_wqput = ktime_get_ns();
  948. #endif
  949. return 0;
  950. }
  951. if (rc == -EBUSY) {
  952. /*
  953. * WQ was full, so queue nvmewqeq to be sent after
  954. * WQE release CQE
  955. */
  956. ctxp->flag |= LPFC_NVME_DEFER_WQFULL;
  957. wq = ctxp->hdwq->io_wq;
  958. pring = wq->pring;
  959. spin_lock_irqsave(&pring->ring_lock, iflags);
  960. list_add_tail(&nvmewqeq->list, &wq->wqfull_list);
  961. wq->q_flag |= HBA_NVMET_WQFULL;
  962. spin_unlock_irqrestore(&pring->ring_lock, iflags);
  963. atomic_inc(&lpfc_nvmep->defer_wqfull);
  964. return 0;
  965. }
  966. /* Give back resources */
  967. atomic_inc(&lpfc_nvmep->xmt_fcp_drop);
  968. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  969. "6153 FCP Drop IO x%x: Issue: %d\n",
  970. ctxp->oxid, rc);
  971. ctxp->wqeq->hba_wqidx = 0;
  972. nvmewqeq->context_un.axchg = NULL;
  973. nvmewqeq->bpl_dmabuf = NULL;
  974. rc = -EBUSY;
  975. aerr:
  976. return rc;
  977. }
  978. static void
  979. lpfc_nvmet_targetport_delete(struct nvmet_fc_target_port *targetport)
  980. {
  981. struct lpfc_nvmet_tgtport *tport = targetport->private;
  982. /* release any threads waiting for the unreg to complete */
  983. if (tport->phba->targetport)
  984. complete(tport->tport_unreg_cmp);
  985. }
  986. static void
  987. lpfc_nvmet_xmt_fcp_abort(struct nvmet_fc_target_port *tgtport,
  988. struct nvmefc_tgt_fcp_req *req)
  989. {
  990. struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
  991. struct lpfc_async_xchg_ctx *ctxp =
  992. container_of(req, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
  993. struct lpfc_hba *phba = ctxp->phba;
  994. struct lpfc_queue *wq;
  995. unsigned long flags;
  996. if (phba->pport->load_flag & FC_UNLOADING)
  997. return;
  998. if (!ctxp->hdwq)
  999. ctxp->hdwq = &phba->sli4_hba.hdwq[0];
  1000. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1001. "6103 NVMET Abort op: oxid x%x flg x%x ste %d\n",
  1002. ctxp->oxid, ctxp->flag, ctxp->state);
  1003. lpfc_nvmeio_data(phba, "NVMET FCP ABRT: xri x%x flg x%x ste x%x\n",
  1004. ctxp->oxid, ctxp->flag, ctxp->state);
  1005. atomic_inc(&lpfc_nvmep->xmt_fcp_abort);
  1006. spin_lock_irqsave(&ctxp->ctxlock, flags);
  1007. /* Since iaab/iaar are NOT set, we need to check
  1008. * if the firmware is in process of aborting IO
  1009. */
  1010. if (ctxp->flag & (LPFC_NVME_XBUSY | LPFC_NVME_ABORT_OP)) {
  1011. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  1012. return;
  1013. }
  1014. ctxp->flag |= LPFC_NVME_ABORT_OP;
  1015. if (ctxp->flag & LPFC_NVME_DEFER_WQFULL) {
  1016. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  1017. lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
  1018. ctxp->oxid);
  1019. wq = ctxp->hdwq->io_wq;
  1020. lpfc_nvmet_wqfull_flush(phba, wq, ctxp);
  1021. return;
  1022. }
  1023. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  1024. /* A state of LPFC_NVME_STE_RCV means we have just received
  1025. * the NVME command and have not started processing it.
  1026. * (by issuing any IO WQEs on this exchange yet)
  1027. */
  1028. if (ctxp->state == LPFC_NVME_STE_RCV)
  1029. lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
  1030. ctxp->oxid);
  1031. else
  1032. lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
  1033. ctxp->oxid);
  1034. }
  1035. static void
  1036. lpfc_nvmet_xmt_fcp_release(struct nvmet_fc_target_port *tgtport,
  1037. struct nvmefc_tgt_fcp_req *rsp)
  1038. {
  1039. struct lpfc_nvmet_tgtport *lpfc_nvmep = tgtport->private;
  1040. struct lpfc_async_xchg_ctx *ctxp =
  1041. container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
  1042. struct lpfc_hba *phba = ctxp->phba;
  1043. unsigned long flags;
  1044. bool aborting = false;
  1045. spin_lock_irqsave(&ctxp->ctxlock, flags);
  1046. if (ctxp->flag & LPFC_NVME_XBUSY)
  1047. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
  1048. "6027 NVMET release with XBUSY flag x%x"
  1049. " oxid x%x\n",
  1050. ctxp->flag, ctxp->oxid);
  1051. else if (ctxp->state != LPFC_NVME_STE_DONE &&
  1052. ctxp->state != LPFC_NVME_STE_ABORT)
  1053. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1054. "6413 NVMET release bad state %d %d oxid x%x\n",
  1055. ctxp->state, ctxp->entry_cnt, ctxp->oxid);
  1056. if ((ctxp->flag & LPFC_NVME_ABORT_OP) ||
  1057. (ctxp->flag & LPFC_NVME_XBUSY)) {
  1058. aborting = true;
  1059. /* let the abort path do the real release */
  1060. lpfc_nvmet_defer_release(phba, ctxp);
  1061. }
  1062. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  1063. lpfc_nvmeio_data(phba, "NVMET FCP FREE: xri x%x ste %d abt %d\n", ctxp->oxid,
  1064. ctxp->state, aborting);
  1065. atomic_inc(&lpfc_nvmep->xmt_fcp_release);
  1066. ctxp->flag &= ~LPFC_NVME_TNOTIFY;
  1067. if (aborting)
  1068. return;
  1069. lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
  1070. }
  1071. static void
  1072. lpfc_nvmet_defer_rcv(struct nvmet_fc_target_port *tgtport,
  1073. struct nvmefc_tgt_fcp_req *rsp)
  1074. {
  1075. struct lpfc_nvmet_tgtport *tgtp;
  1076. struct lpfc_async_xchg_ctx *ctxp =
  1077. container_of(rsp, struct lpfc_async_xchg_ctx, hdlrctx.fcp_req);
  1078. struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;
  1079. struct lpfc_hba *phba = ctxp->phba;
  1080. unsigned long iflag;
  1081. lpfc_nvmeio_data(phba, "NVMET DEFERRCV: xri x%x sz %d CPU %02x\n",
  1082. ctxp->oxid, ctxp->size, raw_smp_processor_id());
  1083. if (!nvmebuf) {
  1084. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
  1085. "6425 Defer rcv: no buffer oxid x%x: "
  1086. "flg %x ste %x\n",
  1087. ctxp->oxid, ctxp->flag, ctxp->state);
  1088. return;
  1089. }
  1090. tgtp = phba->targetport->private;
  1091. if (tgtp)
  1092. atomic_inc(&tgtp->rcv_fcp_cmd_defer);
  1093. /* Free the nvmebuf since a new buffer already replaced it */
  1094. nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
  1095. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  1096. ctxp->rqb_buffer = NULL;
  1097. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  1098. }
  1099. /**
  1100. * lpfc_nvmet_ls_req_cmp - completion handler for a nvme ls request
  1101. * @phba: Pointer to HBA context object
  1102. * @cmdwqe: Pointer to driver command WQE object.
  1103. * @rspwqe: Pointer to driver response WQE object.
  1104. *
  1105. * This function is the completion handler for NVME LS requests.
  1106. * The function updates any states and statistics, then calls the
  1107. * generic completion handler to finish completion of the request.
  1108. **/
  1109. static void
  1110. lpfc_nvmet_ls_req_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  1111. struct lpfc_iocbq *rspwqe)
  1112. {
  1113. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  1114. __lpfc_nvme_ls_req_cmp(phba, cmdwqe->vport, cmdwqe, wcqe);
  1115. }
  1116. /**
  1117. * lpfc_nvmet_ls_req - Issue an Link Service request
  1118. * @targetport: pointer to target instance registered with nvmet transport.
  1119. * @hosthandle: hosthandle set by the driver in a prior ls_rqst_rcv.
  1120. * Driver sets this value to the ndlp pointer.
  1121. * @pnvme_lsreq: the transport nvme_ls_req structure for the LS
  1122. *
  1123. * Driver registers this routine to handle any link service request
  1124. * from the nvme_fc transport to a remote nvme-aware port.
  1125. *
  1126. * Return value :
  1127. * 0 - Success
  1128. * non-zero: various error codes, in form of -Exxx
  1129. **/
  1130. static int
  1131. lpfc_nvmet_ls_req(struct nvmet_fc_target_port *targetport,
  1132. void *hosthandle,
  1133. struct nvmefc_ls_req *pnvme_lsreq)
  1134. {
  1135. struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private;
  1136. struct lpfc_hba *phba;
  1137. struct lpfc_nodelist *ndlp;
  1138. int ret;
  1139. u32 hstate;
  1140. if (!lpfc_nvmet)
  1141. return -EINVAL;
  1142. phba = lpfc_nvmet->phba;
  1143. if (phba->pport->load_flag & FC_UNLOADING)
  1144. return -EINVAL;
  1145. hstate = atomic_read(&lpfc_nvmet->state);
  1146. if (hstate == LPFC_NVMET_INV_HOST_ACTIVE)
  1147. return -EACCES;
  1148. ndlp = (struct lpfc_nodelist *)hosthandle;
  1149. ret = __lpfc_nvme_ls_req(phba->pport, ndlp, pnvme_lsreq,
  1150. lpfc_nvmet_ls_req_cmp);
  1151. return ret;
  1152. }
  1153. /**
  1154. * lpfc_nvmet_ls_abort - Abort a prior NVME LS request
  1155. * @targetport: Transport targetport, that LS was issued from.
  1156. * @hosthandle: hosthandle set by the driver in a prior ls_rqst_rcv.
  1157. * Driver sets this value to the ndlp pointer.
  1158. * @pnvme_lsreq: the transport nvme_ls_req structure for LS to be aborted
  1159. *
  1160. * Driver registers this routine to abort an NVME LS request that is
  1161. * in progress (from the transports perspective).
  1162. **/
  1163. static void
  1164. lpfc_nvmet_ls_abort(struct nvmet_fc_target_port *targetport,
  1165. void *hosthandle,
  1166. struct nvmefc_ls_req *pnvme_lsreq)
  1167. {
  1168. struct lpfc_nvmet_tgtport *lpfc_nvmet = targetport->private;
  1169. struct lpfc_hba *phba;
  1170. struct lpfc_nodelist *ndlp;
  1171. int ret;
  1172. phba = lpfc_nvmet->phba;
  1173. if (phba->pport->load_flag & FC_UNLOADING)
  1174. return;
  1175. ndlp = (struct lpfc_nodelist *)hosthandle;
  1176. ret = __lpfc_nvme_ls_abort(phba->pport, ndlp, pnvme_lsreq);
  1177. if (!ret)
  1178. atomic_inc(&lpfc_nvmet->xmt_ls_abort);
  1179. }
  1180. static void
  1181. lpfc_nvmet_host_release(void *hosthandle)
  1182. {
  1183. struct lpfc_nodelist *ndlp = hosthandle;
  1184. struct lpfc_hba *phba = ndlp->phba;
  1185. struct lpfc_nvmet_tgtport *tgtp;
  1186. if (!phba->targetport || !phba->targetport->private)
  1187. return;
  1188. lpfc_printf_log(phba, KERN_ERR, LOG_NVME,
  1189. "6202 NVMET XPT releasing hosthandle x%px "
  1190. "DID x%x xflags x%x refcnt %d\n",
  1191. hosthandle, ndlp->nlp_DID, ndlp->fc4_xpt_flags,
  1192. kref_read(&ndlp->kref));
  1193. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  1194. spin_lock_irq(&ndlp->lock);
  1195. ndlp->fc4_xpt_flags &= ~NLP_XPT_HAS_HH;
  1196. spin_unlock_irq(&ndlp->lock);
  1197. lpfc_nlp_put(ndlp);
  1198. atomic_set(&tgtp->state, 0);
  1199. }
  1200. static void
  1201. lpfc_nvmet_discovery_event(struct nvmet_fc_target_port *tgtport)
  1202. {
  1203. struct lpfc_nvmet_tgtport *tgtp;
  1204. struct lpfc_hba *phba;
  1205. uint32_t rc;
  1206. tgtp = tgtport->private;
  1207. phba = tgtp->phba;
  1208. rc = lpfc_issue_els_rscn(phba->pport, 0);
  1209. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1210. "6420 NVMET subsystem change: Notification %s\n",
  1211. (rc) ? "Failed" : "Sent");
  1212. }
  1213. static struct nvmet_fc_target_template lpfc_tgttemplate = {
  1214. .targetport_delete = lpfc_nvmet_targetport_delete,
  1215. .xmt_ls_rsp = lpfc_nvmet_xmt_ls_rsp,
  1216. .fcp_op = lpfc_nvmet_xmt_fcp_op,
  1217. .fcp_abort = lpfc_nvmet_xmt_fcp_abort,
  1218. .fcp_req_release = lpfc_nvmet_xmt_fcp_release,
  1219. .defer_rcv = lpfc_nvmet_defer_rcv,
  1220. .discovery_event = lpfc_nvmet_discovery_event,
  1221. .ls_req = lpfc_nvmet_ls_req,
  1222. .ls_abort = lpfc_nvmet_ls_abort,
  1223. .host_release = lpfc_nvmet_host_release,
  1224. .max_hw_queues = 1,
  1225. .max_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
  1226. .max_dif_sgl_segments = LPFC_NVMET_DEFAULT_SEGS,
  1227. .dma_boundary = 0xFFFFFFFF,
  1228. /* optional features */
  1229. .target_features = 0,
  1230. /* sizes of additional private data for data structures */
  1231. .target_priv_sz = sizeof(struct lpfc_nvmet_tgtport),
  1232. .lsrqst_priv_sz = 0,
  1233. };
  1234. static void
  1235. __lpfc_nvmet_clean_io_for_cpu(struct lpfc_hba *phba,
  1236. struct lpfc_nvmet_ctx_info *infop)
  1237. {
  1238. struct lpfc_nvmet_ctxbuf *ctx_buf, *next_ctx_buf;
  1239. unsigned long flags;
  1240. spin_lock_irqsave(&infop->nvmet_ctx_list_lock, flags);
  1241. list_for_each_entry_safe(ctx_buf, next_ctx_buf,
  1242. &infop->nvmet_ctx_list, list) {
  1243. spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  1244. list_del_init(&ctx_buf->list);
  1245. spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  1246. spin_lock(&phba->hbalock);
  1247. __lpfc_clear_active_sglq(phba, ctx_buf->sglq->sli4_lxritag);
  1248. spin_unlock(&phba->hbalock);
  1249. ctx_buf->sglq->state = SGL_FREED;
  1250. ctx_buf->sglq->ndlp = NULL;
  1251. spin_lock(&phba->sli4_hba.sgl_list_lock);
  1252. list_add_tail(&ctx_buf->sglq->list,
  1253. &phba->sli4_hba.lpfc_nvmet_sgl_list);
  1254. spin_unlock(&phba->sli4_hba.sgl_list_lock);
  1255. lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
  1256. kfree(ctx_buf->context);
  1257. }
  1258. spin_unlock_irqrestore(&infop->nvmet_ctx_list_lock, flags);
  1259. }
  1260. static void
  1261. lpfc_nvmet_cleanup_io_context(struct lpfc_hba *phba)
  1262. {
  1263. struct lpfc_nvmet_ctx_info *infop;
  1264. int i, j;
  1265. /* The first context list, MRQ 0 CPU 0 */
  1266. infop = phba->sli4_hba.nvmet_ctx_info;
  1267. if (!infop)
  1268. return;
  1269. /* Cycle the the entire CPU context list for every MRQ */
  1270. for (i = 0; i < phba->cfg_nvmet_mrq; i++) {
  1271. for_each_present_cpu(j) {
  1272. infop = lpfc_get_ctx_list(phba, j, i);
  1273. __lpfc_nvmet_clean_io_for_cpu(phba, infop);
  1274. }
  1275. }
  1276. kfree(phba->sli4_hba.nvmet_ctx_info);
  1277. phba->sli4_hba.nvmet_ctx_info = NULL;
  1278. }
  1279. static int
  1280. lpfc_nvmet_setup_io_context(struct lpfc_hba *phba)
  1281. {
  1282. struct lpfc_nvmet_ctxbuf *ctx_buf;
  1283. struct lpfc_iocbq *nvmewqe;
  1284. union lpfc_wqe128 *wqe;
  1285. struct lpfc_nvmet_ctx_info *last_infop;
  1286. struct lpfc_nvmet_ctx_info *infop;
  1287. int i, j, idx, cpu;
  1288. lpfc_printf_log(phba, KERN_INFO, LOG_NVME,
  1289. "6403 Allocate NVMET resources for %d XRIs\n",
  1290. phba->sli4_hba.nvmet_xri_cnt);
  1291. phba->sli4_hba.nvmet_ctx_info = kcalloc(
  1292. phba->sli4_hba.num_possible_cpu * phba->cfg_nvmet_mrq,
  1293. sizeof(struct lpfc_nvmet_ctx_info), GFP_KERNEL);
  1294. if (!phba->sli4_hba.nvmet_ctx_info) {
  1295. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1296. "6419 Failed allocate memory for "
  1297. "nvmet context lists\n");
  1298. return -ENOMEM;
  1299. }
  1300. /*
  1301. * Assuming X CPUs in the system, and Y MRQs, allocate some
  1302. * lpfc_nvmet_ctx_info structures as follows:
  1303. *
  1304. * cpu0/mrq0 cpu1/mrq0 ... cpuX/mrq0
  1305. * cpu0/mrq1 cpu1/mrq1 ... cpuX/mrq1
  1306. * ...
  1307. * cpuX/mrqY cpuX/mrqY ... cpuX/mrqY
  1308. *
  1309. * Each line represents a MRQ "silo" containing an entry for
  1310. * every CPU.
  1311. *
  1312. * MRQ X is initially assumed to be associated with CPU X, thus
  1313. * contexts are initially distributed across all MRQs using
  1314. * the MRQ index (N) as follows cpuN/mrqN. When contexts are
  1315. * freed, the are freed to the MRQ silo based on the CPU number
  1316. * of the IO completion. Thus a context that was allocated for MRQ A
  1317. * whose IO completed on CPU B will be freed to cpuB/mrqA.
  1318. */
  1319. for_each_possible_cpu(i) {
  1320. for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
  1321. infop = lpfc_get_ctx_list(phba, i, j);
  1322. INIT_LIST_HEAD(&infop->nvmet_ctx_list);
  1323. spin_lock_init(&infop->nvmet_ctx_list_lock);
  1324. infop->nvmet_ctx_list_cnt = 0;
  1325. }
  1326. }
  1327. /*
  1328. * Setup the next CPU context info ptr for each MRQ.
  1329. * MRQ 0 will cycle thru CPUs 0 - X separately from
  1330. * MRQ 1 cycling thru CPUs 0 - X, and so on.
  1331. */
  1332. for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
  1333. last_infop = lpfc_get_ctx_list(phba,
  1334. cpumask_first(cpu_present_mask),
  1335. j);
  1336. for (i = phba->sli4_hba.num_possible_cpu - 1; i >= 0; i--) {
  1337. infop = lpfc_get_ctx_list(phba, i, j);
  1338. infop->nvmet_ctx_next_cpu = last_infop;
  1339. last_infop = infop;
  1340. }
  1341. }
  1342. /* For all nvmet xris, allocate resources needed to process a
  1343. * received command on a per xri basis.
  1344. */
  1345. idx = 0;
  1346. cpu = cpumask_first(cpu_present_mask);
  1347. for (i = 0; i < phba->sli4_hba.nvmet_xri_cnt; i++) {
  1348. ctx_buf = kzalloc(sizeof(*ctx_buf), GFP_KERNEL);
  1349. if (!ctx_buf) {
  1350. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1351. "6404 Ran out of memory for NVMET\n");
  1352. return -ENOMEM;
  1353. }
  1354. ctx_buf->context = kzalloc(sizeof(*ctx_buf->context),
  1355. GFP_KERNEL);
  1356. if (!ctx_buf->context) {
  1357. kfree(ctx_buf);
  1358. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1359. "6405 Ran out of NVMET "
  1360. "context memory\n");
  1361. return -ENOMEM;
  1362. }
  1363. ctx_buf->context->ctxbuf = ctx_buf;
  1364. ctx_buf->context->state = LPFC_NVME_STE_FREE;
  1365. ctx_buf->iocbq = lpfc_sli_get_iocbq(phba);
  1366. if (!ctx_buf->iocbq) {
  1367. kfree(ctx_buf->context);
  1368. kfree(ctx_buf);
  1369. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1370. "6406 Ran out of NVMET iocb/WQEs\n");
  1371. return -ENOMEM;
  1372. }
  1373. ctx_buf->iocbq->cmd_flag = LPFC_IO_NVMET;
  1374. nvmewqe = ctx_buf->iocbq;
  1375. wqe = &nvmewqe->wqe;
  1376. /* Initialize WQE */
  1377. memset(wqe, 0, sizeof(union lpfc_wqe));
  1378. ctx_buf->iocbq->cmd_dmabuf = NULL;
  1379. spin_lock(&phba->sli4_hba.sgl_list_lock);
  1380. ctx_buf->sglq = __lpfc_sli_get_nvmet_sglq(phba, ctx_buf->iocbq);
  1381. spin_unlock(&phba->sli4_hba.sgl_list_lock);
  1382. if (!ctx_buf->sglq) {
  1383. lpfc_sli_release_iocbq(phba, ctx_buf->iocbq);
  1384. kfree(ctx_buf->context);
  1385. kfree(ctx_buf);
  1386. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1387. "6407 Ran out of NVMET XRIs\n");
  1388. return -ENOMEM;
  1389. }
  1390. INIT_WORK(&ctx_buf->defer_work, lpfc_nvmet_fcp_rqst_defer_work);
  1391. /*
  1392. * Add ctx to MRQidx context list. Our initial assumption
  1393. * is MRQidx will be associated with CPUidx. This association
  1394. * can change on the fly.
  1395. */
  1396. infop = lpfc_get_ctx_list(phba, cpu, idx);
  1397. spin_lock(&infop->nvmet_ctx_list_lock);
  1398. list_add_tail(&ctx_buf->list, &infop->nvmet_ctx_list);
  1399. infop->nvmet_ctx_list_cnt++;
  1400. spin_unlock(&infop->nvmet_ctx_list_lock);
  1401. /* Spread ctx structures evenly across all MRQs */
  1402. idx++;
  1403. if (idx >= phba->cfg_nvmet_mrq) {
  1404. idx = 0;
  1405. cpu = cpumask_first(cpu_present_mask);
  1406. continue;
  1407. }
  1408. cpu = cpumask_next(cpu, cpu_present_mask);
  1409. if (cpu == nr_cpu_ids)
  1410. cpu = cpumask_first(cpu_present_mask);
  1411. }
  1412. for_each_present_cpu(i) {
  1413. for (j = 0; j < phba->cfg_nvmet_mrq; j++) {
  1414. infop = lpfc_get_ctx_list(phba, i, j);
  1415. lpfc_printf_log(phba, KERN_INFO, LOG_NVME | LOG_INIT,
  1416. "6408 TOTAL NVMET ctx for CPU %d "
  1417. "MRQ %d: cnt %d nextcpu x%px\n",
  1418. i, j, infop->nvmet_ctx_list_cnt,
  1419. infop->nvmet_ctx_next_cpu);
  1420. }
  1421. }
  1422. return 0;
  1423. }
  1424. int
  1425. lpfc_nvmet_create_targetport(struct lpfc_hba *phba)
  1426. {
  1427. struct lpfc_vport *vport = phba->pport;
  1428. struct lpfc_nvmet_tgtport *tgtp;
  1429. struct nvmet_fc_port_info pinfo;
  1430. int error;
  1431. if (phba->targetport)
  1432. return 0;
  1433. error = lpfc_nvmet_setup_io_context(phba);
  1434. if (error)
  1435. return error;
  1436. memset(&pinfo, 0, sizeof(struct nvmet_fc_port_info));
  1437. pinfo.node_name = wwn_to_u64(vport->fc_nodename.u.wwn);
  1438. pinfo.port_name = wwn_to_u64(vport->fc_portname.u.wwn);
  1439. pinfo.port_id = vport->fc_myDID;
  1440. /* We need to tell the transport layer + 1 because it takes page
  1441. * alignment into account. When space for the SGL is allocated we
  1442. * allocate + 3, one for cmd, one for rsp and one for this alignment
  1443. */
  1444. lpfc_tgttemplate.max_sgl_segments = phba->cfg_nvme_seg_cnt + 1;
  1445. lpfc_tgttemplate.max_hw_queues = phba->cfg_hdw_queue;
  1446. lpfc_tgttemplate.target_features = NVMET_FCTGTFEAT_READDATA_RSP;
  1447. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1448. error = nvmet_fc_register_targetport(&pinfo, &lpfc_tgttemplate,
  1449. &phba->pcidev->dev,
  1450. &phba->targetport);
  1451. #else
  1452. error = -ENOENT;
  1453. #endif
  1454. if (error) {
  1455. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1456. "6025 Cannot register NVME targetport x%x: "
  1457. "portnm %llx nodenm %llx segs %d qs %d\n",
  1458. error,
  1459. pinfo.port_name, pinfo.node_name,
  1460. lpfc_tgttemplate.max_sgl_segments,
  1461. lpfc_tgttemplate.max_hw_queues);
  1462. phba->targetport = NULL;
  1463. phba->nvmet_support = 0;
  1464. lpfc_nvmet_cleanup_io_context(phba);
  1465. } else {
  1466. tgtp = (struct lpfc_nvmet_tgtport *)
  1467. phba->targetport->private;
  1468. tgtp->phba = phba;
  1469. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
  1470. "6026 Registered NVME "
  1471. "targetport: x%px, private x%px "
  1472. "portnm %llx nodenm %llx segs %d qs %d\n",
  1473. phba->targetport, tgtp,
  1474. pinfo.port_name, pinfo.node_name,
  1475. lpfc_tgttemplate.max_sgl_segments,
  1476. lpfc_tgttemplate.max_hw_queues);
  1477. atomic_set(&tgtp->rcv_ls_req_in, 0);
  1478. atomic_set(&tgtp->rcv_ls_req_out, 0);
  1479. atomic_set(&tgtp->rcv_ls_req_drop, 0);
  1480. atomic_set(&tgtp->xmt_ls_abort, 0);
  1481. atomic_set(&tgtp->xmt_ls_abort_cmpl, 0);
  1482. atomic_set(&tgtp->xmt_ls_rsp, 0);
  1483. atomic_set(&tgtp->xmt_ls_drop, 0);
  1484. atomic_set(&tgtp->xmt_ls_rsp_error, 0);
  1485. atomic_set(&tgtp->xmt_ls_rsp_xb_set, 0);
  1486. atomic_set(&tgtp->xmt_ls_rsp_aborted, 0);
  1487. atomic_set(&tgtp->xmt_ls_rsp_cmpl, 0);
  1488. atomic_set(&tgtp->rcv_fcp_cmd_in, 0);
  1489. atomic_set(&tgtp->rcv_fcp_cmd_out, 0);
  1490. atomic_set(&tgtp->rcv_fcp_cmd_drop, 0);
  1491. atomic_set(&tgtp->xmt_fcp_drop, 0);
  1492. atomic_set(&tgtp->xmt_fcp_read_rsp, 0);
  1493. atomic_set(&tgtp->xmt_fcp_read, 0);
  1494. atomic_set(&tgtp->xmt_fcp_write, 0);
  1495. atomic_set(&tgtp->xmt_fcp_rsp, 0);
  1496. atomic_set(&tgtp->xmt_fcp_release, 0);
  1497. atomic_set(&tgtp->xmt_fcp_rsp_cmpl, 0);
  1498. atomic_set(&tgtp->xmt_fcp_rsp_error, 0);
  1499. atomic_set(&tgtp->xmt_fcp_rsp_xb_set, 0);
  1500. atomic_set(&tgtp->xmt_fcp_rsp_aborted, 0);
  1501. atomic_set(&tgtp->xmt_fcp_rsp_drop, 0);
  1502. atomic_set(&tgtp->xmt_fcp_xri_abort_cqe, 0);
  1503. atomic_set(&tgtp->xmt_fcp_abort, 0);
  1504. atomic_set(&tgtp->xmt_fcp_abort_cmpl, 0);
  1505. atomic_set(&tgtp->xmt_abort_unsol, 0);
  1506. atomic_set(&tgtp->xmt_abort_sol, 0);
  1507. atomic_set(&tgtp->xmt_abort_rsp, 0);
  1508. atomic_set(&tgtp->xmt_abort_rsp_error, 0);
  1509. atomic_set(&tgtp->defer_ctx, 0);
  1510. atomic_set(&tgtp->defer_fod, 0);
  1511. atomic_set(&tgtp->defer_wqfull, 0);
  1512. }
  1513. return error;
  1514. }
  1515. int
  1516. lpfc_nvmet_update_targetport(struct lpfc_hba *phba)
  1517. {
  1518. struct lpfc_vport *vport = phba->pport;
  1519. if (!phba->targetport)
  1520. return 0;
  1521. lpfc_printf_vlog(vport, KERN_INFO, LOG_NVME,
  1522. "6007 Update NVMET port x%px did x%x\n",
  1523. phba->targetport, vport->fc_myDID);
  1524. phba->targetport->port_id = vport->fc_myDID;
  1525. return 0;
  1526. }
  1527. /**
  1528. * lpfc_sli4_nvmet_xri_aborted - Fast-path process of nvmet xri abort
  1529. * @phba: pointer to lpfc hba data structure.
  1530. * @axri: pointer to the nvmet xri abort wcqe structure.
  1531. *
  1532. * This routine is invoked by the worker thread to process a SLI4 fast-path
  1533. * NVMET aborted xri.
  1534. **/
  1535. void
  1536. lpfc_sli4_nvmet_xri_aborted(struct lpfc_hba *phba,
  1537. struct sli4_wcqe_xri_aborted *axri)
  1538. {
  1539. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1540. uint16_t xri = bf_get(lpfc_wcqe_xa_xri, axri);
  1541. uint16_t rxid = bf_get(lpfc_wcqe_xa_remote_xid, axri);
  1542. struct lpfc_async_xchg_ctx *ctxp, *next_ctxp;
  1543. struct lpfc_nvmet_tgtport *tgtp;
  1544. struct nvmefc_tgt_fcp_req *req = NULL;
  1545. struct lpfc_nodelist *ndlp;
  1546. unsigned long iflag = 0;
  1547. int rrq_empty = 0;
  1548. bool released = false;
  1549. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1550. "6317 XB aborted xri x%x rxid x%x\n", xri, rxid);
  1551. if (!(phba->cfg_enable_fc4_type & LPFC_ENABLE_NVME))
  1552. return;
  1553. if (phba->targetport) {
  1554. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  1555. atomic_inc(&tgtp->xmt_fcp_xri_abort_cqe);
  1556. }
  1557. spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);
  1558. list_for_each_entry_safe(ctxp, next_ctxp,
  1559. &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
  1560. list) {
  1561. if (ctxp->ctxbuf->sglq->sli4_xritag != xri)
  1562. continue;
  1563. spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock,
  1564. iflag);
  1565. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  1566. /* Check if we already received a free context call
  1567. * and we have completed processing an abort situation.
  1568. */
  1569. if (ctxp->flag & LPFC_NVME_CTX_RLS &&
  1570. !(ctxp->flag & LPFC_NVME_ABORT_OP)) {
  1571. spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  1572. list_del_init(&ctxp->list);
  1573. spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  1574. released = true;
  1575. }
  1576. ctxp->flag &= ~LPFC_NVME_XBUSY;
  1577. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  1578. rrq_empty = list_empty(&phba->active_rrq_list);
  1579. ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
  1580. if (ndlp &&
  1581. (ndlp->nlp_state == NLP_STE_UNMAPPED_NODE ||
  1582. ndlp->nlp_state == NLP_STE_MAPPED_NODE)) {
  1583. lpfc_set_rrq_active(phba, ndlp,
  1584. ctxp->ctxbuf->sglq->sli4_lxritag,
  1585. rxid, 1);
  1586. lpfc_sli4_abts_err_handler(phba, ndlp, axri);
  1587. }
  1588. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1589. "6318 XB aborted oxid x%x flg x%x (%x)\n",
  1590. ctxp->oxid, ctxp->flag, released);
  1591. if (released)
  1592. lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
  1593. if (rrq_empty)
  1594. lpfc_worker_wake_up(phba);
  1595. return;
  1596. }
  1597. spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);
  1598. ctxp = lpfc_nvmet_get_ctx_for_xri(phba, xri);
  1599. if (ctxp) {
  1600. /*
  1601. * Abort already done by FW, so BA_ACC sent.
  1602. * However, the transport may be unaware.
  1603. */
  1604. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1605. "6323 NVMET Rcv ABTS xri x%x ctxp state x%x "
  1606. "flag x%x oxid x%x rxid x%x\n",
  1607. xri, ctxp->state, ctxp->flag, ctxp->oxid,
  1608. rxid);
  1609. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  1610. ctxp->flag |= LPFC_NVME_ABTS_RCV;
  1611. ctxp->state = LPFC_NVME_STE_ABORT;
  1612. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  1613. lpfc_nvmeio_data(phba,
  1614. "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
  1615. xri, raw_smp_processor_id(), 0);
  1616. req = &ctxp->hdlrctx.fcp_req;
  1617. if (req)
  1618. nvmet_fc_rcv_fcp_abort(phba->targetport, req);
  1619. }
  1620. #endif
  1621. }
  1622. int
  1623. lpfc_nvmet_rcv_unsol_abort(struct lpfc_vport *vport,
  1624. struct fc_frame_header *fc_hdr)
  1625. {
  1626. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1627. struct lpfc_hba *phba = vport->phba;
  1628. struct lpfc_async_xchg_ctx *ctxp, *next_ctxp;
  1629. struct nvmefc_tgt_fcp_req *rsp;
  1630. uint32_t sid;
  1631. uint16_t oxid, xri;
  1632. unsigned long iflag = 0;
  1633. sid = sli4_sid_from_fc_hdr(fc_hdr);
  1634. oxid = be16_to_cpu(fc_hdr->fh_ox_id);
  1635. spin_lock_irqsave(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);
  1636. list_for_each_entry_safe(ctxp, next_ctxp,
  1637. &phba->sli4_hba.lpfc_abts_nvmet_ctx_list,
  1638. list) {
  1639. if (ctxp->oxid != oxid || ctxp->sid != sid)
  1640. continue;
  1641. xri = ctxp->ctxbuf->sglq->sli4_xritag;
  1642. spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock,
  1643. iflag);
  1644. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  1645. ctxp->flag |= LPFC_NVME_ABTS_RCV;
  1646. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  1647. lpfc_nvmeio_data(phba,
  1648. "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
  1649. xri, raw_smp_processor_id(), 0);
  1650. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1651. "6319 NVMET Rcv ABTS:acc xri x%x\n", xri);
  1652. rsp = &ctxp->hdlrctx.fcp_req;
  1653. nvmet_fc_rcv_fcp_abort(phba->targetport, rsp);
  1654. /* Respond with BA_ACC accordingly */
  1655. lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
  1656. return 0;
  1657. }
  1658. spin_unlock_irqrestore(&phba->sli4_hba.abts_nvmet_buf_list_lock, iflag);
  1659. /* check the wait list */
  1660. if (phba->sli4_hba.nvmet_io_wait_cnt) {
  1661. struct rqb_dmabuf *nvmebuf;
  1662. struct fc_frame_header *fc_hdr_tmp;
  1663. u32 sid_tmp;
  1664. u16 oxid_tmp;
  1665. bool found = false;
  1666. spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
  1667. /* match by oxid and s_id */
  1668. list_for_each_entry(nvmebuf,
  1669. &phba->sli4_hba.lpfc_nvmet_io_wait_list,
  1670. hbuf.list) {
  1671. fc_hdr_tmp = (struct fc_frame_header *)
  1672. (nvmebuf->hbuf.virt);
  1673. oxid_tmp = be16_to_cpu(fc_hdr_tmp->fh_ox_id);
  1674. sid_tmp = sli4_sid_from_fc_hdr(fc_hdr_tmp);
  1675. if (oxid_tmp != oxid || sid_tmp != sid)
  1676. continue;
  1677. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1678. "6321 NVMET Rcv ABTS oxid x%x from x%x "
  1679. "is waiting for a ctxp\n",
  1680. oxid, sid);
  1681. list_del_init(&nvmebuf->hbuf.list);
  1682. phba->sli4_hba.nvmet_io_wait_cnt--;
  1683. found = true;
  1684. break;
  1685. }
  1686. spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
  1687. iflag);
  1688. /* free buffer since already posted a new DMA buffer to RQ */
  1689. if (found) {
  1690. nvmebuf->hrq->rqbp->rqb_free_buffer(phba, nvmebuf);
  1691. /* Respond with BA_ACC accordingly */
  1692. lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
  1693. return 0;
  1694. }
  1695. }
  1696. /* check active list */
  1697. ctxp = lpfc_nvmet_get_ctx_for_oxid(phba, oxid, sid);
  1698. if (ctxp) {
  1699. xri = ctxp->ctxbuf->sglq->sli4_xritag;
  1700. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  1701. ctxp->flag |= (LPFC_NVME_ABTS_RCV | LPFC_NVME_ABORT_OP);
  1702. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  1703. lpfc_nvmeio_data(phba,
  1704. "NVMET ABTS RCV: xri x%x CPU %02x rjt %d\n",
  1705. xri, raw_smp_processor_id(), 0);
  1706. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1707. "6322 NVMET Rcv ABTS:acc oxid x%x xri x%x "
  1708. "flag x%x state x%x\n",
  1709. ctxp->oxid, xri, ctxp->flag, ctxp->state);
  1710. if (ctxp->flag & LPFC_NVME_TNOTIFY) {
  1711. /* Notify the transport */
  1712. nvmet_fc_rcv_fcp_abort(phba->targetport,
  1713. &ctxp->hdlrctx.fcp_req);
  1714. } else {
  1715. cancel_work_sync(&ctxp->ctxbuf->defer_work);
  1716. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  1717. lpfc_nvmet_defer_release(phba, ctxp);
  1718. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  1719. }
  1720. lpfc_nvmet_sol_fcp_issue_abort(phba, ctxp, ctxp->sid,
  1721. ctxp->oxid);
  1722. lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 1);
  1723. return 0;
  1724. }
  1725. lpfc_nvmeio_data(phba, "NVMET ABTS RCV: oxid x%x CPU %02x rjt %d\n",
  1726. oxid, raw_smp_processor_id(), 1);
  1727. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  1728. "6320 NVMET Rcv ABTS:rjt oxid x%x\n", oxid);
  1729. /* Respond with BA_RJT accordingly */
  1730. lpfc_sli4_seq_abort_rsp(vport, fc_hdr, 0);
  1731. #endif
  1732. return 0;
  1733. }
  1734. static void
  1735. lpfc_nvmet_wqfull_flush(struct lpfc_hba *phba, struct lpfc_queue *wq,
  1736. struct lpfc_async_xchg_ctx *ctxp)
  1737. {
  1738. struct lpfc_sli_ring *pring;
  1739. struct lpfc_iocbq *nvmewqeq;
  1740. struct lpfc_iocbq *next_nvmewqeq;
  1741. unsigned long iflags;
  1742. struct lpfc_wcqe_complete wcqe;
  1743. struct lpfc_wcqe_complete *wcqep;
  1744. pring = wq->pring;
  1745. wcqep = &wcqe;
  1746. /* Fake an ABORT error code back to cmpl routine */
  1747. memset(wcqep, 0, sizeof(struct lpfc_wcqe_complete));
  1748. bf_set(lpfc_wcqe_c_status, wcqep, IOSTAT_LOCAL_REJECT);
  1749. wcqep->parameter = IOERR_ABORT_REQUESTED;
  1750. spin_lock_irqsave(&pring->ring_lock, iflags);
  1751. list_for_each_entry_safe(nvmewqeq, next_nvmewqeq,
  1752. &wq->wqfull_list, list) {
  1753. if (ctxp) {
  1754. /* Checking for a specific IO to flush */
  1755. if (nvmewqeq->context_un.axchg == ctxp) {
  1756. list_del(&nvmewqeq->list);
  1757. spin_unlock_irqrestore(&pring->ring_lock,
  1758. iflags);
  1759. memcpy(&nvmewqeq->wcqe_cmpl, wcqep,
  1760. sizeof(*wcqep));
  1761. lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq,
  1762. nvmewqeq);
  1763. return;
  1764. }
  1765. continue;
  1766. } else {
  1767. /* Flush all IOs */
  1768. list_del(&nvmewqeq->list);
  1769. spin_unlock_irqrestore(&pring->ring_lock, iflags);
  1770. memcpy(&nvmewqeq->wcqe_cmpl, wcqep, sizeof(*wcqep));
  1771. lpfc_nvmet_xmt_fcp_op_cmp(phba, nvmewqeq, nvmewqeq);
  1772. spin_lock_irqsave(&pring->ring_lock, iflags);
  1773. }
  1774. }
  1775. if (!ctxp)
  1776. wq->q_flag &= ~HBA_NVMET_WQFULL;
  1777. spin_unlock_irqrestore(&pring->ring_lock, iflags);
  1778. }
  1779. void
  1780. lpfc_nvmet_wqfull_process(struct lpfc_hba *phba,
  1781. struct lpfc_queue *wq)
  1782. {
  1783. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1784. struct lpfc_sli_ring *pring;
  1785. struct lpfc_iocbq *nvmewqeq;
  1786. struct lpfc_async_xchg_ctx *ctxp;
  1787. unsigned long iflags;
  1788. int rc;
  1789. /*
  1790. * Some WQE slots are available, so try to re-issue anything
  1791. * on the WQ wqfull_list.
  1792. */
  1793. pring = wq->pring;
  1794. spin_lock_irqsave(&pring->ring_lock, iflags);
  1795. while (!list_empty(&wq->wqfull_list)) {
  1796. list_remove_head(&wq->wqfull_list, nvmewqeq, struct lpfc_iocbq,
  1797. list);
  1798. spin_unlock_irqrestore(&pring->ring_lock, iflags);
  1799. ctxp = nvmewqeq->context_un.axchg;
  1800. rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, nvmewqeq);
  1801. spin_lock_irqsave(&pring->ring_lock, iflags);
  1802. if (rc == -EBUSY) {
  1803. /* WQ was full again, so put it back on the list */
  1804. list_add(&nvmewqeq->list, &wq->wqfull_list);
  1805. spin_unlock_irqrestore(&pring->ring_lock, iflags);
  1806. return;
  1807. }
  1808. if (rc == WQE_SUCCESS) {
  1809. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  1810. if (ctxp->ts_cmd_nvme) {
  1811. if (ctxp->hdlrctx.fcp_req.op == NVMET_FCOP_RSP)
  1812. ctxp->ts_status_wqput = ktime_get_ns();
  1813. else
  1814. ctxp->ts_data_wqput = ktime_get_ns();
  1815. }
  1816. #endif
  1817. } else {
  1818. WARN_ON(rc);
  1819. }
  1820. }
  1821. wq->q_flag &= ~HBA_NVMET_WQFULL;
  1822. spin_unlock_irqrestore(&pring->ring_lock, iflags);
  1823. #endif
  1824. }
  1825. void
  1826. lpfc_nvmet_destroy_targetport(struct lpfc_hba *phba)
  1827. {
  1828. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1829. struct lpfc_nvmet_tgtport *tgtp;
  1830. struct lpfc_queue *wq;
  1831. uint32_t qidx;
  1832. DECLARE_COMPLETION_ONSTACK(tport_unreg_cmp);
  1833. if (phba->nvmet_support == 0)
  1834. return;
  1835. if (phba->targetport) {
  1836. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  1837. for (qidx = 0; qidx < phba->cfg_hdw_queue; qidx++) {
  1838. wq = phba->sli4_hba.hdwq[qidx].io_wq;
  1839. lpfc_nvmet_wqfull_flush(phba, wq, NULL);
  1840. }
  1841. tgtp->tport_unreg_cmp = &tport_unreg_cmp;
  1842. nvmet_fc_unregister_targetport(phba->targetport);
  1843. if (!wait_for_completion_timeout(&tport_unreg_cmp,
  1844. msecs_to_jiffies(LPFC_NVMET_WAIT_TMO)))
  1845. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1846. "6179 Unreg targetport x%px timeout "
  1847. "reached.\n", phba->targetport);
  1848. lpfc_nvmet_cleanup_io_context(phba);
  1849. }
  1850. phba->targetport = NULL;
  1851. #endif
  1852. }
  1853. /**
  1854. * lpfc_nvmet_handle_lsreq - Process an NVME LS request
  1855. * @phba: pointer to lpfc hba data structure.
  1856. * @axchg: pointer to exchange context for the NVME LS request
  1857. *
  1858. * This routine is used for processing an asychronously received NVME LS
  1859. * request. Any remaining validation is done and the LS is then forwarded
  1860. * to the nvmet-fc transport via nvmet_fc_rcv_ls_req().
  1861. *
  1862. * The calling sequence should be: nvmet_fc_rcv_ls_req() -> (processing)
  1863. * -> lpfc_nvmet_xmt_ls_rsp/cmp -> req->done.
  1864. * lpfc_nvme_xmt_ls_rsp_cmp should free the allocated axchg.
  1865. *
  1866. * Returns 0 if LS was handled and delivered to the transport
  1867. * Returns 1 if LS failed to be handled and should be dropped
  1868. */
  1869. int
  1870. lpfc_nvmet_handle_lsreq(struct lpfc_hba *phba,
  1871. struct lpfc_async_xchg_ctx *axchg)
  1872. {
  1873. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1874. struct lpfc_nvmet_tgtport *tgtp = phba->targetport->private;
  1875. uint32_t *payload = axchg->payload;
  1876. int rc;
  1877. atomic_inc(&tgtp->rcv_ls_req_in);
  1878. /*
  1879. * Driver passes the ndlp as the hosthandle argument allowing
  1880. * the transport to generate LS requests for any associateions
  1881. * that are created.
  1882. */
  1883. rc = nvmet_fc_rcv_ls_req(phba->targetport, axchg->ndlp, &axchg->ls_rsp,
  1884. axchg->payload, axchg->size);
  1885. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
  1886. "6037 NVMET Unsol rcv: sz %d rc %d: %08x %08x %08x "
  1887. "%08x %08x %08x\n", axchg->size, rc,
  1888. *payload, *(payload+1), *(payload+2),
  1889. *(payload+3), *(payload+4), *(payload+5));
  1890. if (!rc) {
  1891. atomic_inc(&tgtp->rcv_ls_req_out);
  1892. return 0;
  1893. }
  1894. atomic_inc(&tgtp->rcv_ls_req_drop);
  1895. #endif
  1896. return 1;
  1897. }
  1898. static void
  1899. lpfc_nvmet_process_rcv_fcp_req(struct lpfc_nvmet_ctxbuf *ctx_buf)
  1900. {
  1901. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  1902. struct lpfc_async_xchg_ctx *ctxp = ctx_buf->context;
  1903. struct lpfc_hba *phba = ctxp->phba;
  1904. struct rqb_dmabuf *nvmebuf = ctxp->rqb_buffer;
  1905. struct lpfc_nvmet_tgtport *tgtp;
  1906. uint32_t *payload, qno;
  1907. uint32_t rc;
  1908. unsigned long iflags;
  1909. if (!nvmebuf) {
  1910. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1911. "6159 process_rcv_fcp_req, nvmebuf is NULL, "
  1912. "oxid: x%x flg: x%x state: x%x\n",
  1913. ctxp->oxid, ctxp->flag, ctxp->state);
  1914. spin_lock_irqsave(&ctxp->ctxlock, iflags);
  1915. lpfc_nvmet_defer_release(phba, ctxp);
  1916. spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
  1917. lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid,
  1918. ctxp->oxid);
  1919. return;
  1920. }
  1921. if (ctxp->flag & LPFC_NVME_ABTS_RCV) {
  1922. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1923. "6324 IO oxid x%x aborted\n",
  1924. ctxp->oxid);
  1925. return;
  1926. }
  1927. payload = (uint32_t *)(nvmebuf->dbuf.virt);
  1928. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  1929. ctxp->flag |= LPFC_NVME_TNOTIFY;
  1930. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  1931. if (ctxp->ts_isr_cmd)
  1932. ctxp->ts_cmd_nvme = ktime_get_ns();
  1933. #endif
  1934. /*
  1935. * The calling sequence should be:
  1936. * nvmet_fc_rcv_fcp_req->lpfc_nvmet_xmt_fcp_op/cmp- req->done
  1937. * lpfc_nvmet_xmt_fcp_op_cmp should free the allocated ctxp.
  1938. * When we return from nvmet_fc_rcv_fcp_req, all relevant info
  1939. * the NVME command / FC header is stored.
  1940. * A buffer has already been reposted for this IO, so just free
  1941. * the nvmebuf.
  1942. */
  1943. rc = nvmet_fc_rcv_fcp_req(phba->targetport, &ctxp->hdlrctx.fcp_req,
  1944. payload, ctxp->size);
  1945. /* Process FCP command */
  1946. if (rc == 0) {
  1947. atomic_inc(&tgtp->rcv_fcp_cmd_out);
  1948. spin_lock_irqsave(&ctxp->ctxlock, iflags);
  1949. if ((ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) ||
  1950. (nvmebuf != ctxp->rqb_buffer)) {
  1951. spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
  1952. return;
  1953. }
  1954. ctxp->rqb_buffer = NULL;
  1955. spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
  1956. lpfc_rq_buf_free(phba, &nvmebuf->hbuf); /* repost */
  1957. return;
  1958. }
  1959. /* Processing of FCP command is deferred */
  1960. if (rc == -EOVERFLOW) {
  1961. lpfc_nvmeio_data(phba, "NVMET RCV BUSY: xri x%x sz %d "
  1962. "from %06x\n",
  1963. ctxp->oxid, ctxp->size, ctxp->sid);
  1964. atomic_inc(&tgtp->rcv_fcp_cmd_out);
  1965. atomic_inc(&tgtp->defer_fod);
  1966. spin_lock_irqsave(&ctxp->ctxlock, iflags);
  1967. if (ctxp->flag & LPFC_NVME_CTX_REUSE_WQ) {
  1968. spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
  1969. return;
  1970. }
  1971. spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
  1972. /*
  1973. * Post a replacement DMA buffer to RQ and defer
  1974. * freeing rcv buffer till .defer_rcv callback
  1975. */
  1976. qno = nvmebuf->idx;
  1977. lpfc_post_rq_buffer(
  1978. phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
  1979. phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
  1980. return;
  1981. }
  1982. ctxp->flag &= ~LPFC_NVME_TNOTIFY;
  1983. atomic_inc(&tgtp->rcv_fcp_cmd_drop);
  1984. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  1985. "2582 FCP Drop IO x%x: err x%x: x%x x%x x%x\n",
  1986. ctxp->oxid, rc,
  1987. atomic_read(&tgtp->rcv_fcp_cmd_in),
  1988. atomic_read(&tgtp->rcv_fcp_cmd_out),
  1989. atomic_read(&tgtp->xmt_fcp_release));
  1990. lpfc_nvmeio_data(phba, "NVMET FCP DROP: xri x%x sz %d from %06x\n",
  1991. ctxp->oxid, ctxp->size, ctxp->sid);
  1992. spin_lock_irqsave(&ctxp->ctxlock, iflags);
  1993. lpfc_nvmet_defer_release(phba, ctxp);
  1994. spin_unlock_irqrestore(&ctxp->ctxlock, iflags);
  1995. lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, ctxp->sid, ctxp->oxid);
  1996. #endif
  1997. }
  1998. static void
  1999. lpfc_nvmet_fcp_rqst_defer_work(struct work_struct *work)
  2000. {
  2001. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  2002. struct lpfc_nvmet_ctxbuf *ctx_buf =
  2003. container_of(work, struct lpfc_nvmet_ctxbuf, defer_work);
  2004. lpfc_nvmet_process_rcv_fcp_req(ctx_buf);
  2005. #endif
  2006. }
  2007. static struct lpfc_nvmet_ctxbuf *
  2008. lpfc_nvmet_replenish_context(struct lpfc_hba *phba,
  2009. struct lpfc_nvmet_ctx_info *current_infop)
  2010. {
  2011. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  2012. struct lpfc_nvmet_ctxbuf *ctx_buf = NULL;
  2013. struct lpfc_nvmet_ctx_info *get_infop;
  2014. int i;
  2015. /*
  2016. * The current_infop for the MRQ a NVME command IU was received
  2017. * on is empty. Our goal is to replenish this MRQs context
  2018. * list from a another CPUs.
  2019. *
  2020. * First we need to pick a context list to start looking on.
  2021. * nvmet_ctx_start_cpu has available context the last time
  2022. * we needed to replenish this CPU where nvmet_ctx_next_cpu
  2023. * is just the next sequential CPU for this MRQ.
  2024. */
  2025. if (current_infop->nvmet_ctx_start_cpu)
  2026. get_infop = current_infop->nvmet_ctx_start_cpu;
  2027. else
  2028. get_infop = current_infop->nvmet_ctx_next_cpu;
  2029. for (i = 0; i < phba->sli4_hba.num_possible_cpu; i++) {
  2030. if (get_infop == current_infop) {
  2031. get_infop = get_infop->nvmet_ctx_next_cpu;
  2032. continue;
  2033. }
  2034. spin_lock(&get_infop->nvmet_ctx_list_lock);
  2035. /* Just take the entire context list, if there are any */
  2036. if (get_infop->nvmet_ctx_list_cnt) {
  2037. list_splice_init(&get_infop->nvmet_ctx_list,
  2038. &current_infop->nvmet_ctx_list);
  2039. current_infop->nvmet_ctx_list_cnt =
  2040. get_infop->nvmet_ctx_list_cnt - 1;
  2041. get_infop->nvmet_ctx_list_cnt = 0;
  2042. spin_unlock(&get_infop->nvmet_ctx_list_lock);
  2043. current_infop->nvmet_ctx_start_cpu = get_infop;
  2044. list_remove_head(&current_infop->nvmet_ctx_list,
  2045. ctx_buf, struct lpfc_nvmet_ctxbuf,
  2046. list);
  2047. return ctx_buf;
  2048. }
  2049. /* Otherwise, move on to the next CPU for this MRQ */
  2050. spin_unlock(&get_infop->nvmet_ctx_list_lock);
  2051. get_infop = get_infop->nvmet_ctx_next_cpu;
  2052. }
  2053. #endif
  2054. /* Nothing found, all contexts for the MRQ are in-flight */
  2055. return NULL;
  2056. }
  2057. /**
  2058. * lpfc_nvmet_unsol_fcp_buffer - Process an unsolicited event data buffer
  2059. * @phba: pointer to lpfc hba data structure.
  2060. * @idx: relative index of MRQ vector
  2061. * @nvmebuf: pointer to lpfc nvme command HBQ data structure.
  2062. * @isr_timestamp: in jiffies.
  2063. * @cqflag: cq processing information regarding workload.
  2064. *
  2065. * This routine is used for processing the WQE associated with a unsolicited
  2066. * event. It first determines whether there is an existing ndlp that matches
  2067. * the DID from the unsolicited WQE. If not, it will create a new one with
  2068. * the DID from the unsolicited WQE. The ELS command from the unsolicited
  2069. * WQE is then used to invoke the proper routine and to set up proper state
  2070. * of the discovery state machine.
  2071. **/
  2072. static void
  2073. lpfc_nvmet_unsol_fcp_buffer(struct lpfc_hba *phba,
  2074. uint32_t idx,
  2075. struct rqb_dmabuf *nvmebuf,
  2076. uint64_t isr_timestamp,
  2077. uint8_t cqflag)
  2078. {
  2079. struct lpfc_async_xchg_ctx *ctxp;
  2080. struct lpfc_nvmet_tgtport *tgtp;
  2081. struct fc_frame_header *fc_hdr;
  2082. struct lpfc_nvmet_ctxbuf *ctx_buf;
  2083. struct lpfc_nvmet_ctx_info *current_infop;
  2084. uint32_t size, oxid, sid, qno;
  2085. unsigned long iflag;
  2086. int current_cpu;
  2087. if (!IS_ENABLED(CONFIG_NVME_TARGET_FC))
  2088. return;
  2089. ctx_buf = NULL;
  2090. if (!nvmebuf || !phba->targetport) {
  2091. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2092. "6157 NVMET FCP Drop IO\n");
  2093. if (nvmebuf)
  2094. lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
  2095. return;
  2096. }
  2097. /*
  2098. * Get a pointer to the context list for this MRQ based on
  2099. * the CPU this MRQ IRQ is associated with. If the CPU association
  2100. * changes from our initial assumption, the context list could
  2101. * be empty, thus it would need to be replenished with the
  2102. * context list from another CPU for this MRQ.
  2103. */
  2104. current_cpu = raw_smp_processor_id();
  2105. current_infop = lpfc_get_ctx_list(phba, current_cpu, idx);
  2106. spin_lock_irqsave(&current_infop->nvmet_ctx_list_lock, iflag);
  2107. if (current_infop->nvmet_ctx_list_cnt) {
  2108. list_remove_head(&current_infop->nvmet_ctx_list,
  2109. ctx_buf, struct lpfc_nvmet_ctxbuf, list);
  2110. current_infop->nvmet_ctx_list_cnt--;
  2111. } else {
  2112. ctx_buf = lpfc_nvmet_replenish_context(phba, current_infop);
  2113. }
  2114. spin_unlock_irqrestore(&current_infop->nvmet_ctx_list_lock, iflag);
  2115. fc_hdr = (struct fc_frame_header *)(nvmebuf->hbuf.virt);
  2116. oxid = be16_to_cpu(fc_hdr->fh_ox_id);
  2117. size = nvmebuf->bytes_recv;
  2118. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  2119. if (phba->hdwqstat_on & LPFC_CHECK_NVMET_IO) {
  2120. this_cpu_inc(phba->sli4_hba.c_stat->rcv_io);
  2121. if (idx != current_cpu)
  2122. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
  2123. "6703 CPU Check rcv: "
  2124. "cpu %d expect %d\n",
  2125. current_cpu, idx);
  2126. }
  2127. #endif
  2128. lpfc_nvmeio_data(phba, "NVMET FCP RCV: xri x%x sz %d CPU %02x\n",
  2129. oxid, size, raw_smp_processor_id());
  2130. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2131. if (!ctx_buf) {
  2132. /* Queue this NVME IO to process later */
  2133. spin_lock_irqsave(&phba->sli4_hba.nvmet_io_wait_lock, iflag);
  2134. list_add_tail(&nvmebuf->hbuf.list,
  2135. &phba->sli4_hba.lpfc_nvmet_io_wait_list);
  2136. phba->sli4_hba.nvmet_io_wait_cnt++;
  2137. phba->sli4_hba.nvmet_io_wait_total++;
  2138. spin_unlock_irqrestore(&phba->sli4_hba.nvmet_io_wait_lock,
  2139. iflag);
  2140. /* Post a brand new DMA buffer to RQ */
  2141. qno = nvmebuf->idx;
  2142. lpfc_post_rq_buffer(
  2143. phba, phba->sli4_hba.nvmet_mrq_hdr[qno],
  2144. phba->sli4_hba.nvmet_mrq_data[qno], 1, qno);
  2145. atomic_inc(&tgtp->defer_ctx);
  2146. return;
  2147. }
  2148. sid = sli4_sid_from_fc_hdr(fc_hdr);
  2149. ctxp = (struct lpfc_async_xchg_ctx *)ctx_buf->context;
  2150. spin_lock_irqsave(&phba->sli4_hba.t_active_list_lock, iflag);
  2151. list_add_tail(&ctxp->list, &phba->sli4_hba.t_active_ctx_list);
  2152. spin_unlock_irqrestore(&phba->sli4_hba.t_active_list_lock, iflag);
  2153. if (ctxp->state != LPFC_NVME_STE_FREE) {
  2154. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2155. "6414 NVMET Context corrupt %d %d oxid x%x\n",
  2156. ctxp->state, ctxp->entry_cnt, ctxp->oxid);
  2157. }
  2158. ctxp->wqeq = NULL;
  2159. ctxp->offset = 0;
  2160. ctxp->phba = phba;
  2161. ctxp->size = size;
  2162. ctxp->oxid = oxid;
  2163. ctxp->sid = sid;
  2164. ctxp->idx = idx;
  2165. ctxp->state = LPFC_NVME_STE_RCV;
  2166. ctxp->entry_cnt = 1;
  2167. ctxp->flag = 0;
  2168. ctxp->ctxbuf = ctx_buf;
  2169. ctxp->rqb_buffer = (void *)nvmebuf;
  2170. ctxp->hdwq = NULL;
  2171. spin_lock_init(&ctxp->ctxlock);
  2172. #ifdef CONFIG_SCSI_LPFC_DEBUG_FS
  2173. if (isr_timestamp)
  2174. ctxp->ts_isr_cmd = isr_timestamp;
  2175. ctxp->ts_cmd_nvme = 0;
  2176. ctxp->ts_nvme_data = 0;
  2177. ctxp->ts_data_wqput = 0;
  2178. ctxp->ts_isr_data = 0;
  2179. ctxp->ts_data_nvme = 0;
  2180. ctxp->ts_nvme_status = 0;
  2181. ctxp->ts_status_wqput = 0;
  2182. ctxp->ts_isr_status = 0;
  2183. ctxp->ts_status_nvme = 0;
  2184. #endif
  2185. atomic_inc(&tgtp->rcv_fcp_cmd_in);
  2186. /* check for cq processing load */
  2187. if (!cqflag) {
  2188. lpfc_nvmet_process_rcv_fcp_req(ctx_buf);
  2189. return;
  2190. }
  2191. if (!queue_work(phba->wq, &ctx_buf->defer_work)) {
  2192. atomic_inc(&tgtp->rcv_fcp_cmd_drop);
  2193. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2194. "6325 Unable to queue work for oxid x%x. "
  2195. "FCP Drop IO [x%x x%x x%x]\n",
  2196. ctxp->oxid,
  2197. atomic_read(&tgtp->rcv_fcp_cmd_in),
  2198. atomic_read(&tgtp->rcv_fcp_cmd_out),
  2199. atomic_read(&tgtp->xmt_fcp_release));
  2200. spin_lock_irqsave(&ctxp->ctxlock, iflag);
  2201. lpfc_nvmet_defer_release(phba, ctxp);
  2202. spin_unlock_irqrestore(&ctxp->ctxlock, iflag);
  2203. lpfc_nvmet_unsol_fcp_issue_abort(phba, ctxp, sid, oxid);
  2204. }
  2205. }
  2206. /**
  2207. * lpfc_nvmet_unsol_fcp_event - Process an unsolicited event from an nvme nport
  2208. * @phba: pointer to lpfc hba data structure.
  2209. * @idx: relative index of MRQ vector
  2210. * @nvmebuf: pointer to received nvme data structure.
  2211. * @isr_timestamp: in jiffies.
  2212. * @cqflag: cq processing information regarding workload.
  2213. *
  2214. * This routine is used to process an unsolicited event received from a SLI
  2215. * (Service Level Interface) ring. The actual processing of the data buffer
  2216. * associated with the unsolicited event is done by invoking the routine
  2217. * lpfc_nvmet_unsol_fcp_buffer() after properly set up the buffer from the
  2218. * SLI RQ on which the unsolicited event was received.
  2219. **/
  2220. void
  2221. lpfc_nvmet_unsol_fcp_event(struct lpfc_hba *phba,
  2222. uint32_t idx,
  2223. struct rqb_dmabuf *nvmebuf,
  2224. uint64_t isr_timestamp,
  2225. uint8_t cqflag)
  2226. {
  2227. if (!nvmebuf) {
  2228. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2229. "3167 NVMET FCP Drop IO\n");
  2230. return;
  2231. }
  2232. if (phba->nvmet_support == 0) {
  2233. lpfc_rq_buf_free(phba, &nvmebuf->hbuf);
  2234. return;
  2235. }
  2236. lpfc_nvmet_unsol_fcp_buffer(phba, idx, nvmebuf, isr_timestamp, cqflag);
  2237. }
  2238. /**
  2239. * lpfc_nvmet_prep_ls_wqe - Allocate and prepare a lpfc wqe data structure
  2240. * @phba: pointer to a host N_Port data structure.
  2241. * @ctxp: Context info for NVME LS Request
  2242. * @rspbuf: DMA buffer of NVME command.
  2243. * @rspsize: size of the NVME command.
  2244. *
  2245. * This routine is used for allocating a lpfc-WQE data structure from
  2246. * the driver lpfc-WQE free-list and prepare the WQE with the parameters
  2247. * passed into the routine for discovery state machine to issue an Extended
  2248. * Link Service (NVME) commands. It is a generic lpfc-WQE allocation
  2249. * and preparation routine that is used by all the discovery state machine
  2250. * routines and the NVME command-specific fields will be later set up by
  2251. * the individual discovery machine routines after calling this routine
  2252. * allocating and preparing a generic WQE data structure. It fills in the
  2253. * Buffer Descriptor Entries (BDEs), allocates buffers for both command
  2254. * payload and response payload (if expected). The reference count on the
  2255. * ndlp is incremented by 1 and the reference to the ndlp is put into
  2256. * context1 of the WQE data structure for this WQE to hold the ndlp
  2257. * reference for the command's callback function to access later.
  2258. *
  2259. * Return code
  2260. * Pointer to the newly allocated/prepared nvme wqe data structure
  2261. * NULL - when nvme wqe data structure allocation/preparation failed
  2262. **/
  2263. static struct lpfc_iocbq *
  2264. lpfc_nvmet_prep_ls_wqe(struct lpfc_hba *phba,
  2265. struct lpfc_async_xchg_ctx *ctxp,
  2266. dma_addr_t rspbuf, uint16_t rspsize)
  2267. {
  2268. struct lpfc_nodelist *ndlp;
  2269. struct lpfc_iocbq *nvmewqe;
  2270. union lpfc_wqe128 *wqe;
  2271. if (!lpfc_is_link_up(phba)) {
  2272. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2273. "6104 NVMET prep LS wqe: link err: "
  2274. "NPORT x%x oxid:x%x ste %d\n",
  2275. ctxp->sid, ctxp->oxid, ctxp->state);
  2276. return NULL;
  2277. }
  2278. /* Allocate buffer for command wqe */
  2279. nvmewqe = lpfc_sli_get_iocbq(phba);
  2280. if (nvmewqe == NULL) {
  2281. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2282. "6105 NVMET prep LS wqe: No WQE: "
  2283. "NPORT x%x oxid x%x ste %d\n",
  2284. ctxp->sid, ctxp->oxid, ctxp->state);
  2285. return NULL;
  2286. }
  2287. ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
  2288. if (!ndlp ||
  2289. ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
  2290. (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
  2291. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2292. "6106 NVMET prep LS wqe: No ndlp: "
  2293. "NPORT x%x oxid x%x ste %d\n",
  2294. ctxp->sid, ctxp->oxid, ctxp->state);
  2295. goto nvme_wqe_free_wqeq_exit;
  2296. }
  2297. ctxp->wqeq = nvmewqe;
  2298. /* prevent preparing wqe with NULL ndlp reference */
  2299. nvmewqe->ndlp = lpfc_nlp_get(ndlp);
  2300. if (!nvmewqe->ndlp)
  2301. goto nvme_wqe_free_wqeq_exit;
  2302. nvmewqe->context_un.axchg = ctxp;
  2303. wqe = &nvmewqe->wqe;
  2304. memset(wqe, 0, sizeof(union lpfc_wqe));
  2305. /* Words 0 - 2 */
  2306. wqe->xmit_sequence.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
  2307. wqe->xmit_sequence.bde.tus.f.bdeSize = rspsize;
  2308. wqe->xmit_sequence.bde.addrLow = le32_to_cpu(putPaddrLow(rspbuf));
  2309. wqe->xmit_sequence.bde.addrHigh = le32_to_cpu(putPaddrHigh(rspbuf));
  2310. /* Word 3 */
  2311. /* Word 4 */
  2312. /* Word 5 */
  2313. bf_set(wqe_dfctl, &wqe->xmit_sequence.wge_ctl, 0);
  2314. bf_set(wqe_ls, &wqe->xmit_sequence.wge_ctl, 1);
  2315. bf_set(wqe_la, &wqe->xmit_sequence.wge_ctl, 0);
  2316. bf_set(wqe_rctl, &wqe->xmit_sequence.wge_ctl, FC_RCTL_ELS4_REP);
  2317. bf_set(wqe_type, &wqe->xmit_sequence.wge_ctl, FC_TYPE_NVME);
  2318. /* Word 6 */
  2319. bf_set(wqe_ctxt_tag, &wqe->xmit_sequence.wqe_com,
  2320. phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
  2321. bf_set(wqe_xri_tag, &wqe->xmit_sequence.wqe_com, nvmewqe->sli4_xritag);
  2322. /* Word 7 */
  2323. bf_set(wqe_cmnd, &wqe->xmit_sequence.wqe_com,
  2324. CMD_XMIT_SEQUENCE64_WQE);
  2325. bf_set(wqe_ct, &wqe->xmit_sequence.wqe_com, SLI4_CT_RPI);
  2326. bf_set(wqe_class, &wqe->xmit_sequence.wqe_com, CLASS3);
  2327. bf_set(wqe_pu, &wqe->xmit_sequence.wqe_com, 0);
  2328. /* Word 8 */
  2329. wqe->xmit_sequence.wqe_com.abort_tag = nvmewqe->iotag;
  2330. /* Word 9 */
  2331. bf_set(wqe_reqtag, &wqe->xmit_sequence.wqe_com, nvmewqe->iotag);
  2332. /* Needs to be set by caller */
  2333. bf_set(wqe_rcvoxid, &wqe->xmit_sequence.wqe_com, ctxp->oxid);
  2334. /* Word 10 */
  2335. bf_set(wqe_dbde, &wqe->xmit_sequence.wqe_com, 1);
  2336. bf_set(wqe_iod, &wqe->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
  2337. bf_set(wqe_lenloc, &wqe->xmit_sequence.wqe_com,
  2338. LPFC_WQE_LENLOC_WORD12);
  2339. bf_set(wqe_ebde_cnt, &wqe->xmit_sequence.wqe_com, 0);
  2340. /* Word 11 */
  2341. bf_set(wqe_cqid, &wqe->xmit_sequence.wqe_com,
  2342. LPFC_WQE_CQ_ID_DEFAULT);
  2343. bf_set(wqe_cmd_type, &wqe->xmit_sequence.wqe_com,
  2344. OTHER_COMMAND);
  2345. /* Word 12 */
  2346. wqe->xmit_sequence.xmit_len = rspsize;
  2347. nvmewqe->retry = 1;
  2348. nvmewqe->vport = phba->pport;
  2349. nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
  2350. nvmewqe->cmd_flag |= LPFC_IO_NVME_LS;
  2351. /* Xmit NVMET response to remote NPORT <did> */
  2352. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_DISC,
  2353. "6039 Xmit NVMET LS response to remote "
  2354. "NPORT x%x iotag:x%x oxid:x%x size:x%x\n",
  2355. ndlp->nlp_DID, nvmewqe->iotag, ctxp->oxid,
  2356. rspsize);
  2357. return nvmewqe;
  2358. nvme_wqe_free_wqeq_exit:
  2359. nvmewqe->context_un.axchg = NULL;
  2360. nvmewqe->ndlp = NULL;
  2361. nvmewqe->bpl_dmabuf = NULL;
  2362. lpfc_sli_release_iocbq(phba, nvmewqe);
  2363. return NULL;
  2364. }
  2365. static struct lpfc_iocbq *
  2366. lpfc_nvmet_prep_fcp_wqe(struct lpfc_hba *phba,
  2367. struct lpfc_async_xchg_ctx *ctxp)
  2368. {
  2369. struct nvmefc_tgt_fcp_req *rsp = &ctxp->hdlrctx.fcp_req;
  2370. struct lpfc_nvmet_tgtport *tgtp;
  2371. struct sli4_sge *sgl;
  2372. struct lpfc_nodelist *ndlp;
  2373. struct lpfc_iocbq *nvmewqe;
  2374. struct scatterlist *sgel;
  2375. union lpfc_wqe128 *wqe;
  2376. struct ulp_bde64 *bde;
  2377. dma_addr_t physaddr;
  2378. int i, cnt, nsegs;
  2379. bool use_pbde = false;
  2380. int xc = 1;
  2381. if (!lpfc_is_link_up(phba)) {
  2382. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2383. "6107 NVMET prep FCP wqe: link err:"
  2384. "NPORT x%x oxid x%x ste %d\n",
  2385. ctxp->sid, ctxp->oxid, ctxp->state);
  2386. return NULL;
  2387. }
  2388. ndlp = lpfc_findnode_did(phba->pport, ctxp->sid);
  2389. if (!ndlp ||
  2390. ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
  2391. (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
  2392. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2393. "6108 NVMET prep FCP wqe: no ndlp: "
  2394. "NPORT x%x oxid x%x ste %d\n",
  2395. ctxp->sid, ctxp->oxid, ctxp->state);
  2396. return NULL;
  2397. }
  2398. if (rsp->sg_cnt > lpfc_tgttemplate.max_sgl_segments) {
  2399. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2400. "6109 NVMET prep FCP wqe: seg cnt err: "
  2401. "NPORT x%x oxid x%x ste %d cnt %d\n",
  2402. ctxp->sid, ctxp->oxid, ctxp->state,
  2403. phba->cfg_nvme_seg_cnt);
  2404. return NULL;
  2405. }
  2406. nsegs = rsp->sg_cnt;
  2407. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2408. nvmewqe = ctxp->wqeq;
  2409. if (nvmewqe == NULL) {
  2410. /* Allocate buffer for command wqe */
  2411. nvmewqe = ctxp->ctxbuf->iocbq;
  2412. if (nvmewqe == NULL) {
  2413. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2414. "6110 NVMET prep FCP wqe: No "
  2415. "WQE: NPORT x%x oxid x%x ste %d\n",
  2416. ctxp->sid, ctxp->oxid, ctxp->state);
  2417. return NULL;
  2418. }
  2419. ctxp->wqeq = nvmewqe;
  2420. xc = 0; /* create new XRI */
  2421. nvmewqe->sli4_lxritag = NO_XRI;
  2422. nvmewqe->sli4_xritag = NO_XRI;
  2423. }
  2424. /* Sanity check */
  2425. if (((ctxp->state == LPFC_NVME_STE_RCV) &&
  2426. (ctxp->entry_cnt == 1)) ||
  2427. (ctxp->state == LPFC_NVME_STE_DATA)) {
  2428. wqe = &nvmewqe->wqe;
  2429. } else {
  2430. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2431. "6111 Wrong state NVMET FCP: %d cnt %d\n",
  2432. ctxp->state, ctxp->entry_cnt);
  2433. return NULL;
  2434. }
  2435. sgl = (struct sli4_sge *)ctxp->ctxbuf->sglq->sgl;
  2436. switch (rsp->op) {
  2437. case NVMET_FCOP_READDATA:
  2438. case NVMET_FCOP_READDATA_RSP:
  2439. /* From the tsend template, initialize words 7 - 11 */
  2440. memcpy(&wqe->words[7],
  2441. &lpfc_tsend_cmd_template.words[7],
  2442. sizeof(uint32_t) * 5);
  2443. /* Words 0 - 2 : The first sg segment */
  2444. sgel = &rsp->sg[0];
  2445. physaddr = sg_dma_address(sgel);
  2446. wqe->fcp_tsend.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
  2447. wqe->fcp_tsend.bde.tus.f.bdeSize = sg_dma_len(sgel);
  2448. wqe->fcp_tsend.bde.addrLow = cpu_to_le32(putPaddrLow(physaddr));
  2449. wqe->fcp_tsend.bde.addrHigh =
  2450. cpu_to_le32(putPaddrHigh(physaddr));
  2451. /* Word 3 */
  2452. wqe->fcp_tsend.payload_offset_len = 0;
  2453. /* Word 4 */
  2454. wqe->fcp_tsend.relative_offset = ctxp->offset;
  2455. /* Word 5 */
  2456. wqe->fcp_tsend.reserved = 0;
  2457. /* Word 6 */
  2458. bf_set(wqe_ctxt_tag, &wqe->fcp_tsend.wqe_com,
  2459. phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
  2460. bf_set(wqe_xri_tag, &wqe->fcp_tsend.wqe_com,
  2461. nvmewqe->sli4_xritag);
  2462. /* Word 7 - set ar later */
  2463. /* Word 8 */
  2464. wqe->fcp_tsend.wqe_com.abort_tag = nvmewqe->iotag;
  2465. /* Word 9 */
  2466. bf_set(wqe_reqtag, &wqe->fcp_tsend.wqe_com, nvmewqe->iotag);
  2467. bf_set(wqe_rcvoxid, &wqe->fcp_tsend.wqe_com, ctxp->oxid);
  2468. /* Word 10 - set wqes later, in template xc=1 */
  2469. if (!xc)
  2470. bf_set(wqe_xc, &wqe->fcp_tsend.wqe_com, 0);
  2471. /* Word 12 */
  2472. wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
  2473. /* Setup 2 SKIP SGEs */
  2474. sgl->addr_hi = 0;
  2475. sgl->addr_lo = 0;
  2476. sgl->word2 = 0;
  2477. bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
  2478. sgl->word2 = cpu_to_le32(sgl->word2);
  2479. sgl->sge_len = 0;
  2480. sgl++;
  2481. sgl->addr_hi = 0;
  2482. sgl->addr_lo = 0;
  2483. sgl->word2 = 0;
  2484. bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
  2485. sgl->word2 = cpu_to_le32(sgl->word2);
  2486. sgl->sge_len = 0;
  2487. sgl++;
  2488. if (rsp->op == NVMET_FCOP_READDATA_RSP) {
  2489. atomic_inc(&tgtp->xmt_fcp_read_rsp);
  2490. /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
  2491. if (rsp->rsplen == LPFC_NVMET_SUCCESS_LEN) {
  2492. if (ndlp->nlp_flag & NLP_SUPPRESS_RSP)
  2493. bf_set(wqe_sup,
  2494. &wqe->fcp_tsend.wqe_com, 1);
  2495. } else {
  2496. bf_set(wqe_wqes, &wqe->fcp_tsend.wqe_com, 1);
  2497. bf_set(wqe_irsp, &wqe->fcp_tsend.wqe_com, 1);
  2498. bf_set(wqe_irsplen, &wqe->fcp_tsend.wqe_com,
  2499. ((rsp->rsplen >> 2) - 1));
  2500. memcpy(&wqe->words[16], rsp->rspaddr,
  2501. rsp->rsplen);
  2502. }
  2503. } else {
  2504. atomic_inc(&tgtp->xmt_fcp_read);
  2505. /* In template ar=1 wqes=0 sup=0 irsp=0 irsplen=0 */
  2506. bf_set(wqe_ar, &wqe->fcp_tsend.wqe_com, 0);
  2507. }
  2508. break;
  2509. case NVMET_FCOP_WRITEDATA:
  2510. /* From the treceive template, initialize words 3 - 11 */
  2511. memcpy(&wqe->words[3],
  2512. &lpfc_treceive_cmd_template.words[3],
  2513. sizeof(uint32_t) * 9);
  2514. /* Words 0 - 2 : First SGE is skipped, set invalid BDE type */
  2515. wqe->fcp_treceive.bde.tus.f.bdeFlags = LPFC_SGE_TYPE_SKIP;
  2516. wqe->fcp_treceive.bde.tus.f.bdeSize = 0;
  2517. wqe->fcp_treceive.bde.addrLow = 0;
  2518. wqe->fcp_treceive.bde.addrHigh = 0;
  2519. /* Word 4 */
  2520. wqe->fcp_treceive.relative_offset = ctxp->offset;
  2521. /* Word 6 */
  2522. bf_set(wqe_ctxt_tag, &wqe->fcp_treceive.wqe_com,
  2523. phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
  2524. bf_set(wqe_xri_tag, &wqe->fcp_treceive.wqe_com,
  2525. nvmewqe->sli4_xritag);
  2526. /* Word 7 */
  2527. /* Word 8 */
  2528. wqe->fcp_treceive.wqe_com.abort_tag = nvmewqe->iotag;
  2529. /* Word 9 */
  2530. bf_set(wqe_reqtag, &wqe->fcp_treceive.wqe_com, nvmewqe->iotag);
  2531. bf_set(wqe_rcvoxid, &wqe->fcp_treceive.wqe_com, ctxp->oxid);
  2532. /* Word 10 - in template xc=1 */
  2533. if (!xc)
  2534. bf_set(wqe_xc, &wqe->fcp_treceive.wqe_com, 0);
  2535. /* Word 11 - check for pbde */
  2536. if (nsegs == 1 && phba->cfg_enable_pbde) {
  2537. use_pbde = true;
  2538. /* Word 11 - PBDE bit already preset by template */
  2539. } else {
  2540. /* Overwrite default template setting */
  2541. bf_set(wqe_pbde, &wqe->fcp_treceive.wqe_com, 0);
  2542. }
  2543. /* Word 12 */
  2544. wqe->fcp_tsend.fcp_data_len = rsp->transfer_length;
  2545. /* Setup 2 SKIP SGEs */
  2546. sgl->addr_hi = 0;
  2547. sgl->addr_lo = 0;
  2548. sgl->word2 = 0;
  2549. bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
  2550. sgl->word2 = cpu_to_le32(sgl->word2);
  2551. sgl->sge_len = 0;
  2552. sgl++;
  2553. sgl->addr_hi = 0;
  2554. sgl->addr_lo = 0;
  2555. sgl->word2 = 0;
  2556. bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_SKIP);
  2557. sgl->word2 = cpu_to_le32(sgl->word2);
  2558. sgl->sge_len = 0;
  2559. sgl++;
  2560. atomic_inc(&tgtp->xmt_fcp_write);
  2561. break;
  2562. case NVMET_FCOP_RSP:
  2563. /* From the treceive template, initialize words 4 - 11 */
  2564. memcpy(&wqe->words[4],
  2565. &lpfc_trsp_cmd_template.words[4],
  2566. sizeof(uint32_t) * 8);
  2567. /* Words 0 - 2 */
  2568. physaddr = rsp->rspdma;
  2569. wqe->fcp_trsp.bde.tus.f.bdeFlags = BUFF_TYPE_BDE_64;
  2570. wqe->fcp_trsp.bde.tus.f.bdeSize = rsp->rsplen;
  2571. wqe->fcp_trsp.bde.addrLow =
  2572. cpu_to_le32(putPaddrLow(physaddr));
  2573. wqe->fcp_trsp.bde.addrHigh =
  2574. cpu_to_le32(putPaddrHigh(physaddr));
  2575. /* Word 3 */
  2576. wqe->fcp_trsp.response_len = rsp->rsplen;
  2577. /* Word 6 */
  2578. bf_set(wqe_ctxt_tag, &wqe->fcp_trsp.wqe_com,
  2579. phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
  2580. bf_set(wqe_xri_tag, &wqe->fcp_trsp.wqe_com,
  2581. nvmewqe->sli4_xritag);
  2582. /* Word 7 */
  2583. /* Word 8 */
  2584. wqe->fcp_trsp.wqe_com.abort_tag = nvmewqe->iotag;
  2585. /* Word 9 */
  2586. bf_set(wqe_reqtag, &wqe->fcp_trsp.wqe_com, nvmewqe->iotag);
  2587. bf_set(wqe_rcvoxid, &wqe->fcp_trsp.wqe_com, ctxp->oxid);
  2588. /* Word 10 */
  2589. if (xc)
  2590. bf_set(wqe_xc, &wqe->fcp_trsp.wqe_com, 1);
  2591. /* Word 11 */
  2592. /* In template wqes=0 irsp=0 irsplen=0 - good response */
  2593. if (rsp->rsplen != LPFC_NVMET_SUCCESS_LEN) {
  2594. /* Bad response - embed it */
  2595. bf_set(wqe_wqes, &wqe->fcp_trsp.wqe_com, 1);
  2596. bf_set(wqe_irsp, &wqe->fcp_trsp.wqe_com, 1);
  2597. bf_set(wqe_irsplen, &wqe->fcp_trsp.wqe_com,
  2598. ((rsp->rsplen >> 2) - 1));
  2599. memcpy(&wqe->words[16], rsp->rspaddr, rsp->rsplen);
  2600. }
  2601. /* Word 12 */
  2602. wqe->fcp_trsp.rsvd_12_15[0] = 0;
  2603. /* Use rspbuf, NOT sg list */
  2604. nsegs = 0;
  2605. sgl->word2 = 0;
  2606. atomic_inc(&tgtp->xmt_fcp_rsp);
  2607. break;
  2608. default:
  2609. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_IOERR,
  2610. "6064 Unknown Rsp Op %d\n",
  2611. rsp->op);
  2612. return NULL;
  2613. }
  2614. nvmewqe->retry = 1;
  2615. nvmewqe->vport = phba->pport;
  2616. nvmewqe->drvrTimeout = (phba->fc_ratov * 3) + LPFC_DRVR_TIMEOUT;
  2617. nvmewqe->ndlp = ndlp;
  2618. for_each_sg(rsp->sg, sgel, nsegs, i) {
  2619. physaddr = sg_dma_address(sgel);
  2620. cnt = sg_dma_len(sgel);
  2621. sgl->addr_hi = putPaddrHigh(physaddr);
  2622. sgl->addr_lo = putPaddrLow(physaddr);
  2623. sgl->word2 = 0;
  2624. bf_set(lpfc_sli4_sge_type, sgl, LPFC_SGE_TYPE_DATA);
  2625. bf_set(lpfc_sli4_sge_offset, sgl, ctxp->offset);
  2626. if ((i+1) == rsp->sg_cnt)
  2627. bf_set(lpfc_sli4_sge_last, sgl, 1);
  2628. sgl->word2 = cpu_to_le32(sgl->word2);
  2629. sgl->sge_len = cpu_to_le32(cnt);
  2630. sgl++;
  2631. ctxp->offset += cnt;
  2632. }
  2633. bde = (struct ulp_bde64 *)&wqe->words[13];
  2634. if (use_pbde) {
  2635. /* decrement sgl ptr backwards once to first data sge */
  2636. sgl--;
  2637. /* Words 13-15 (PBDE) */
  2638. bde->addrLow = sgl->addr_lo;
  2639. bde->addrHigh = sgl->addr_hi;
  2640. bde->tus.f.bdeSize = le32_to_cpu(sgl->sge_len);
  2641. bde->tus.f.bdeFlags = BUFF_TYPE_BDE_64;
  2642. bde->tus.w = cpu_to_le32(bde->tus.w);
  2643. } else {
  2644. memset(bde, 0, sizeof(struct ulp_bde64));
  2645. }
  2646. ctxp->state = LPFC_NVME_STE_DATA;
  2647. ctxp->entry_cnt++;
  2648. return nvmewqe;
  2649. }
  2650. /**
  2651. * lpfc_nvmet_sol_fcp_abort_cmp - Completion handler for ABTS
  2652. * @phba: Pointer to HBA context object.
  2653. * @cmdwqe: Pointer to driver command WQE object.
  2654. * @rspwqe: Pointer to driver response WQE object.
  2655. *
  2656. * The function is called from SLI ring event handler with no
  2657. * lock held. This function is the completion handler for NVME ABTS for FCP cmds
  2658. * The function frees memory resources used for the NVME commands.
  2659. **/
  2660. static void
  2661. lpfc_nvmet_sol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  2662. struct lpfc_iocbq *rspwqe)
  2663. {
  2664. struct lpfc_async_xchg_ctx *ctxp;
  2665. struct lpfc_nvmet_tgtport *tgtp;
  2666. uint32_t result;
  2667. unsigned long flags;
  2668. bool released = false;
  2669. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  2670. ctxp = cmdwqe->context_un.axchg;
  2671. result = wcqe->parameter;
  2672. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2673. if (ctxp->flag & LPFC_NVME_ABORT_OP)
  2674. atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
  2675. spin_lock_irqsave(&ctxp->ctxlock, flags);
  2676. ctxp->state = LPFC_NVME_STE_DONE;
  2677. /* Check if we already received a free context call
  2678. * and we have completed processing an abort situation.
  2679. */
  2680. if ((ctxp->flag & LPFC_NVME_CTX_RLS) &&
  2681. !(ctxp->flag & LPFC_NVME_XBUSY)) {
  2682. spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  2683. list_del_init(&ctxp->list);
  2684. spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  2685. released = true;
  2686. }
  2687. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  2688. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2689. atomic_inc(&tgtp->xmt_abort_rsp);
  2690. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2691. "6165 ABORT cmpl: oxid x%x flg x%x (%d) "
  2692. "WCQE: %08x %08x %08x %08x\n",
  2693. ctxp->oxid, ctxp->flag, released,
  2694. wcqe->word0, wcqe->total_data_placed,
  2695. result, wcqe->word3);
  2696. cmdwqe->rsp_dmabuf = NULL;
  2697. cmdwqe->bpl_dmabuf = NULL;
  2698. /*
  2699. * if transport has released ctx, then can reuse it. Otherwise,
  2700. * will be recycled by transport release call.
  2701. */
  2702. if (released)
  2703. lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
  2704. /* This is the iocbq for the abort, not the command */
  2705. lpfc_sli_release_iocbq(phba, cmdwqe);
  2706. /* Since iaab/iaar are NOT set, there is no work left.
  2707. * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted
  2708. * should have been called already.
  2709. */
  2710. }
  2711. /**
  2712. * lpfc_nvmet_unsol_fcp_abort_cmp - Completion handler for ABTS
  2713. * @phba: Pointer to HBA context object.
  2714. * @cmdwqe: Pointer to driver command WQE object.
  2715. * @rspwqe: Pointer to driver response WQE object.
  2716. *
  2717. * The function is called from SLI ring event handler with no
  2718. * lock held. This function is the completion handler for NVME ABTS for FCP cmds
  2719. * The function frees memory resources used for the NVME commands.
  2720. **/
  2721. static void
  2722. lpfc_nvmet_unsol_fcp_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  2723. struct lpfc_iocbq *rspwqe)
  2724. {
  2725. struct lpfc_async_xchg_ctx *ctxp;
  2726. struct lpfc_nvmet_tgtport *tgtp;
  2727. unsigned long flags;
  2728. uint32_t result;
  2729. bool released = false;
  2730. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  2731. ctxp = cmdwqe->context_un.axchg;
  2732. result = wcqe->parameter;
  2733. if (!ctxp) {
  2734. /* if context is clear, related io alrady complete */
  2735. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2736. "6070 ABTS cmpl: WCQE: %08x %08x %08x %08x\n",
  2737. wcqe->word0, wcqe->total_data_placed,
  2738. result, wcqe->word3);
  2739. return;
  2740. }
  2741. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2742. spin_lock_irqsave(&ctxp->ctxlock, flags);
  2743. if (ctxp->flag & LPFC_NVME_ABORT_OP)
  2744. atomic_inc(&tgtp->xmt_fcp_abort_cmpl);
  2745. /* Sanity check */
  2746. if (ctxp->state != LPFC_NVME_STE_ABORT) {
  2747. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2748. "6112 ABTS Wrong state:%d oxid x%x\n",
  2749. ctxp->state, ctxp->oxid);
  2750. }
  2751. /* Check if we already received a free context call
  2752. * and we have completed processing an abort situation.
  2753. */
  2754. ctxp->state = LPFC_NVME_STE_DONE;
  2755. if ((ctxp->flag & LPFC_NVME_CTX_RLS) &&
  2756. !(ctxp->flag & LPFC_NVME_XBUSY)) {
  2757. spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  2758. list_del_init(&ctxp->list);
  2759. spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  2760. released = true;
  2761. }
  2762. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  2763. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2764. atomic_inc(&tgtp->xmt_abort_rsp);
  2765. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2766. "6316 ABTS cmpl oxid x%x flg x%x (%x) "
  2767. "WCQE: %08x %08x %08x %08x\n",
  2768. ctxp->oxid, ctxp->flag, released,
  2769. wcqe->word0, wcqe->total_data_placed,
  2770. result, wcqe->word3);
  2771. cmdwqe->rsp_dmabuf = NULL;
  2772. cmdwqe->bpl_dmabuf = NULL;
  2773. /*
  2774. * if transport has released ctx, then can reuse it. Otherwise,
  2775. * will be recycled by transport release call.
  2776. */
  2777. if (released)
  2778. lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
  2779. /* Since iaab/iaar are NOT set, there is no work left.
  2780. * For LPFC_NVME_XBUSY, lpfc_sli4_nvmet_xri_aborted
  2781. * should have been called already.
  2782. */
  2783. }
  2784. /**
  2785. * lpfc_nvmet_xmt_ls_abort_cmp - Completion handler for ABTS
  2786. * @phba: Pointer to HBA context object.
  2787. * @cmdwqe: Pointer to driver command WQE object.
  2788. * @rspwqe: Pointer to driver response WQE object.
  2789. *
  2790. * The function is called from SLI ring event handler with no
  2791. * lock held. This function is the completion handler for NVME ABTS for LS cmds
  2792. * The function frees memory resources used for the NVME commands.
  2793. **/
  2794. static void
  2795. lpfc_nvmet_xmt_ls_abort_cmp(struct lpfc_hba *phba, struct lpfc_iocbq *cmdwqe,
  2796. struct lpfc_iocbq *rspwqe)
  2797. {
  2798. struct lpfc_async_xchg_ctx *ctxp;
  2799. struct lpfc_nvmet_tgtport *tgtp;
  2800. uint32_t result;
  2801. struct lpfc_wcqe_complete *wcqe = &rspwqe->wcqe_cmpl;
  2802. ctxp = cmdwqe->context_un.axchg;
  2803. result = wcqe->parameter;
  2804. if (phba->nvmet_support) {
  2805. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2806. atomic_inc(&tgtp->xmt_ls_abort_cmpl);
  2807. }
  2808. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2809. "6083 Abort cmpl: ctx x%px WCQE:%08x %08x %08x %08x\n",
  2810. ctxp, wcqe->word0, wcqe->total_data_placed,
  2811. result, wcqe->word3);
  2812. if (!ctxp) {
  2813. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2814. "6415 NVMET LS Abort No ctx: WCQE: "
  2815. "%08x %08x %08x %08x\n",
  2816. wcqe->word0, wcqe->total_data_placed,
  2817. result, wcqe->word3);
  2818. lpfc_sli_release_iocbq(phba, cmdwqe);
  2819. return;
  2820. }
  2821. if (ctxp->state != LPFC_NVME_STE_LS_ABORT) {
  2822. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2823. "6416 NVMET LS abort cmpl state mismatch: "
  2824. "oxid x%x: %d %d\n",
  2825. ctxp->oxid, ctxp->state, ctxp->entry_cnt);
  2826. }
  2827. cmdwqe->rsp_dmabuf = NULL;
  2828. cmdwqe->bpl_dmabuf = NULL;
  2829. lpfc_sli_release_iocbq(phba, cmdwqe);
  2830. kfree(ctxp);
  2831. }
  2832. static int
  2833. lpfc_nvmet_unsol_issue_abort(struct lpfc_hba *phba,
  2834. struct lpfc_async_xchg_ctx *ctxp,
  2835. uint32_t sid, uint16_t xri)
  2836. {
  2837. struct lpfc_nvmet_tgtport *tgtp = NULL;
  2838. struct lpfc_iocbq *abts_wqeq;
  2839. union lpfc_wqe128 *wqe_abts;
  2840. struct lpfc_nodelist *ndlp;
  2841. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2842. "6067 ABTS: sid %x xri x%x/x%x\n",
  2843. sid, xri, ctxp->wqeq->sli4_xritag);
  2844. if (phba->nvmet_support && phba->targetport)
  2845. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2846. ndlp = lpfc_findnode_did(phba->pport, sid);
  2847. if (!ndlp ||
  2848. ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
  2849. (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
  2850. if (tgtp)
  2851. atomic_inc(&tgtp->xmt_abort_rsp_error);
  2852. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2853. "6134 Drop ABTS - wrong NDLP state x%x.\n",
  2854. (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
  2855. /* No failure to an ABTS request. */
  2856. return 0;
  2857. }
  2858. abts_wqeq = ctxp->wqeq;
  2859. wqe_abts = &abts_wqeq->wqe;
  2860. /*
  2861. * Since we zero the whole WQE, we need to ensure we set the WQE fields
  2862. * that were initialized in lpfc_sli4_nvmet_alloc.
  2863. */
  2864. memset(wqe_abts, 0, sizeof(union lpfc_wqe));
  2865. /* Word 5 */
  2866. bf_set(wqe_dfctl, &wqe_abts->xmit_sequence.wge_ctl, 0);
  2867. bf_set(wqe_ls, &wqe_abts->xmit_sequence.wge_ctl, 1);
  2868. bf_set(wqe_la, &wqe_abts->xmit_sequence.wge_ctl, 0);
  2869. bf_set(wqe_rctl, &wqe_abts->xmit_sequence.wge_ctl, FC_RCTL_BA_ABTS);
  2870. bf_set(wqe_type, &wqe_abts->xmit_sequence.wge_ctl, FC_TYPE_BLS);
  2871. /* Word 6 */
  2872. bf_set(wqe_ctxt_tag, &wqe_abts->xmit_sequence.wqe_com,
  2873. phba->sli4_hba.rpi_ids[ndlp->nlp_rpi]);
  2874. bf_set(wqe_xri_tag, &wqe_abts->xmit_sequence.wqe_com,
  2875. abts_wqeq->sli4_xritag);
  2876. /* Word 7 */
  2877. bf_set(wqe_cmnd, &wqe_abts->xmit_sequence.wqe_com,
  2878. CMD_XMIT_SEQUENCE64_WQE);
  2879. bf_set(wqe_ct, &wqe_abts->xmit_sequence.wqe_com, SLI4_CT_RPI);
  2880. bf_set(wqe_class, &wqe_abts->xmit_sequence.wqe_com, CLASS3);
  2881. bf_set(wqe_pu, &wqe_abts->xmit_sequence.wqe_com, 0);
  2882. /* Word 8 */
  2883. wqe_abts->xmit_sequence.wqe_com.abort_tag = abts_wqeq->iotag;
  2884. /* Word 9 */
  2885. bf_set(wqe_reqtag, &wqe_abts->xmit_sequence.wqe_com, abts_wqeq->iotag);
  2886. /* Needs to be set by caller */
  2887. bf_set(wqe_rcvoxid, &wqe_abts->xmit_sequence.wqe_com, xri);
  2888. /* Word 10 */
  2889. bf_set(wqe_iod, &wqe_abts->xmit_sequence.wqe_com, LPFC_WQE_IOD_WRITE);
  2890. bf_set(wqe_lenloc, &wqe_abts->xmit_sequence.wqe_com,
  2891. LPFC_WQE_LENLOC_WORD12);
  2892. bf_set(wqe_ebde_cnt, &wqe_abts->xmit_sequence.wqe_com, 0);
  2893. bf_set(wqe_qosd, &wqe_abts->xmit_sequence.wqe_com, 0);
  2894. /* Word 11 */
  2895. bf_set(wqe_cqid, &wqe_abts->xmit_sequence.wqe_com,
  2896. LPFC_WQE_CQ_ID_DEFAULT);
  2897. bf_set(wqe_cmd_type, &wqe_abts->xmit_sequence.wqe_com,
  2898. OTHER_COMMAND);
  2899. abts_wqeq->vport = phba->pport;
  2900. abts_wqeq->ndlp = ndlp;
  2901. abts_wqeq->context_un.axchg = ctxp;
  2902. abts_wqeq->bpl_dmabuf = NULL;
  2903. abts_wqeq->num_bdes = 0;
  2904. /* hba_wqidx should already be setup from command we are aborting */
  2905. abts_wqeq->iocb.ulpCommand = CMD_XMIT_SEQUENCE64_CR;
  2906. abts_wqeq->iocb.ulpLe = 1;
  2907. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2908. "6069 Issue ABTS to xri x%x reqtag x%x\n",
  2909. xri, abts_wqeq->iotag);
  2910. return 1;
  2911. }
  2912. static int
  2913. lpfc_nvmet_sol_fcp_issue_abort(struct lpfc_hba *phba,
  2914. struct lpfc_async_xchg_ctx *ctxp,
  2915. uint32_t sid, uint16_t xri)
  2916. {
  2917. struct lpfc_nvmet_tgtport *tgtp;
  2918. struct lpfc_iocbq *abts_wqeq;
  2919. struct lpfc_nodelist *ndlp;
  2920. unsigned long flags;
  2921. bool ia;
  2922. int rc;
  2923. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  2924. if (!ctxp->wqeq) {
  2925. ctxp->wqeq = ctxp->ctxbuf->iocbq;
  2926. ctxp->wqeq->hba_wqidx = 0;
  2927. }
  2928. ndlp = lpfc_findnode_did(phba->pport, sid);
  2929. if (!ndlp ||
  2930. ((ndlp->nlp_state != NLP_STE_UNMAPPED_NODE) &&
  2931. (ndlp->nlp_state != NLP_STE_MAPPED_NODE))) {
  2932. atomic_inc(&tgtp->xmt_abort_rsp_error);
  2933. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2934. "6160 Drop ABORT - wrong NDLP state x%x.\n",
  2935. (ndlp) ? ndlp->nlp_state : NLP_STE_MAX_STATE);
  2936. /* No failure to an ABTS request. */
  2937. spin_lock_irqsave(&ctxp->ctxlock, flags);
  2938. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  2939. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2940. return 0;
  2941. }
  2942. /* Issue ABTS for this WQE based on iotag */
  2943. ctxp->abort_wqeq = lpfc_sli_get_iocbq(phba);
  2944. spin_lock_irqsave(&ctxp->ctxlock, flags);
  2945. if (!ctxp->abort_wqeq) {
  2946. atomic_inc(&tgtp->xmt_abort_rsp_error);
  2947. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2948. "6161 ABORT failed: No wqeqs: "
  2949. "xri: x%x\n", ctxp->oxid);
  2950. /* No failure to an ABTS request. */
  2951. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  2952. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2953. return 0;
  2954. }
  2955. abts_wqeq = ctxp->abort_wqeq;
  2956. ctxp->state = LPFC_NVME_STE_ABORT;
  2957. ia = (ctxp->flag & LPFC_NVME_ABTS_RCV) ? true : false;
  2958. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2959. /* Announce entry to new IO submit field. */
  2960. lpfc_printf_log(phba, KERN_INFO, LOG_NVME_ABTS,
  2961. "6162 ABORT Request to rport DID x%06x "
  2962. "for xri x%x x%x\n",
  2963. ctxp->sid, ctxp->oxid, ctxp->wqeq->sli4_xritag);
  2964. /* If the hba is getting reset, this flag is set. It is
  2965. * cleared when the reset is complete and rings reestablished.
  2966. */
  2967. spin_lock_irqsave(&phba->hbalock, flags);
  2968. /* driver queued commands are in process of being flushed */
  2969. if (phba->hba_flag & HBA_IOQ_FLUSH) {
  2970. spin_unlock_irqrestore(&phba->hbalock, flags);
  2971. atomic_inc(&tgtp->xmt_abort_rsp_error);
  2972. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2973. "6163 Driver in reset cleanup - flushing "
  2974. "NVME Req now. hba_flag x%x oxid x%x\n",
  2975. phba->hba_flag, ctxp->oxid);
  2976. lpfc_sli_release_iocbq(phba, abts_wqeq);
  2977. spin_lock_irqsave(&ctxp->ctxlock, flags);
  2978. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  2979. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2980. return 0;
  2981. }
  2982. /* Outstanding abort is in progress */
  2983. if (abts_wqeq->cmd_flag & LPFC_DRIVER_ABORTED) {
  2984. spin_unlock_irqrestore(&phba->hbalock, flags);
  2985. atomic_inc(&tgtp->xmt_abort_rsp_error);
  2986. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  2987. "6164 Outstanding NVME I/O Abort Request "
  2988. "still pending on oxid x%x\n",
  2989. ctxp->oxid);
  2990. lpfc_sli_release_iocbq(phba, abts_wqeq);
  2991. spin_lock_irqsave(&ctxp->ctxlock, flags);
  2992. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  2993. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  2994. return 0;
  2995. }
  2996. /* Ready - mark outstanding as aborted by driver. */
  2997. abts_wqeq->cmd_flag |= LPFC_DRIVER_ABORTED;
  2998. lpfc_sli_prep_abort_xri(phba, abts_wqeq, ctxp->wqeq->sli4_xritag,
  2999. abts_wqeq->iotag, CLASS3,
  3000. LPFC_WQE_CQ_ID_DEFAULT, ia, true);
  3001. /* ABTS WQE must go to the same WQ as the WQE to be aborted */
  3002. abts_wqeq->hba_wqidx = ctxp->wqeq->hba_wqidx;
  3003. abts_wqeq->cmd_cmpl = lpfc_nvmet_sol_fcp_abort_cmp;
  3004. abts_wqeq->cmd_flag |= LPFC_IO_NVME;
  3005. abts_wqeq->context_un.axchg = ctxp;
  3006. abts_wqeq->vport = phba->pport;
  3007. if (!ctxp->hdwq)
  3008. ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx];
  3009. rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq);
  3010. spin_unlock_irqrestore(&phba->hbalock, flags);
  3011. if (rc == WQE_SUCCESS) {
  3012. atomic_inc(&tgtp->xmt_abort_sol);
  3013. return 0;
  3014. }
  3015. atomic_inc(&tgtp->xmt_abort_rsp_error);
  3016. spin_lock_irqsave(&ctxp->ctxlock, flags);
  3017. ctxp->flag &= ~LPFC_NVME_ABORT_OP;
  3018. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  3019. lpfc_sli_release_iocbq(phba, abts_wqeq);
  3020. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  3021. "6166 Failed ABORT issue_wqe with status x%x "
  3022. "for oxid x%x.\n",
  3023. rc, ctxp->oxid);
  3024. return 1;
  3025. }
  3026. static int
  3027. lpfc_nvmet_unsol_fcp_issue_abort(struct lpfc_hba *phba,
  3028. struct lpfc_async_xchg_ctx *ctxp,
  3029. uint32_t sid, uint16_t xri)
  3030. {
  3031. struct lpfc_nvmet_tgtport *tgtp;
  3032. struct lpfc_iocbq *abts_wqeq;
  3033. unsigned long flags;
  3034. bool released = false;
  3035. int rc;
  3036. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  3037. if (!ctxp->wqeq) {
  3038. ctxp->wqeq = ctxp->ctxbuf->iocbq;
  3039. ctxp->wqeq->hba_wqidx = 0;
  3040. }
  3041. if (ctxp->state == LPFC_NVME_STE_FREE) {
  3042. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  3043. "6417 NVMET ABORT ctx freed %d %d oxid x%x\n",
  3044. ctxp->state, ctxp->entry_cnt, ctxp->oxid);
  3045. rc = WQE_BUSY;
  3046. goto aerr;
  3047. }
  3048. ctxp->state = LPFC_NVME_STE_ABORT;
  3049. ctxp->entry_cnt++;
  3050. rc = lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri);
  3051. if (rc == 0)
  3052. goto aerr;
  3053. spin_lock_irqsave(&phba->hbalock, flags);
  3054. abts_wqeq = ctxp->wqeq;
  3055. abts_wqeq->cmd_cmpl = lpfc_nvmet_unsol_fcp_abort_cmp;
  3056. abts_wqeq->cmd_flag |= LPFC_IO_NVMET;
  3057. if (!ctxp->hdwq)
  3058. ctxp->hdwq = &phba->sli4_hba.hdwq[abts_wqeq->hba_wqidx];
  3059. rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq);
  3060. spin_unlock_irqrestore(&phba->hbalock, flags);
  3061. if (rc == WQE_SUCCESS) {
  3062. return 0;
  3063. }
  3064. aerr:
  3065. spin_lock_irqsave(&ctxp->ctxlock, flags);
  3066. if (ctxp->flag & LPFC_NVME_CTX_RLS) {
  3067. spin_lock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  3068. list_del_init(&ctxp->list);
  3069. spin_unlock(&phba->sli4_hba.abts_nvmet_buf_list_lock);
  3070. released = true;
  3071. }
  3072. ctxp->flag &= ~(LPFC_NVME_ABORT_OP | LPFC_NVME_CTX_RLS);
  3073. spin_unlock_irqrestore(&ctxp->ctxlock, flags);
  3074. atomic_inc(&tgtp->xmt_abort_rsp_error);
  3075. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  3076. "6135 Failed to Issue ABTS for oxid x%x. Status x%x "
  3077. "(%x)\n",
  3078. ctxp->oxid, rc, released);
  3079. if (released)
  3080. lpfc_nvmet_ctxbuf_post(phba, ctxp->ctxbuf);
  3081. return 1;
  3082. }
  3083. /**
  3084. * lpfc_nvme_unsol_ls_issue_abort - issue ABTS on an exchange received
  3085. * via async frame receive where the frame is not handled.
  3086. * @phba: pointer to adapter structure
  3087. * @ctxp: pointer to the asynchronously received received sequence
  3088. * @sid: address of the remote port to send the ABTS to
  3089. * @xri: oxid value to for the ABTS (other side's exchange id).
  3090. **/
  3091. int
  3092. lpfc_nvme_unsol_ls_issue_abort(struct lpfc_hba *phba,
  3093. struct lpfc_async_xchg_ctx *ctxp,
  3094. uint32_t sid, uint16_t xri)
  3095. {
  3096. struct lpfc_nvmet_tgtport *tgtp = NULL;
  3097. struct lpfc_iocbq *abts_wqeq;
  3098. unsigned long flags;
  3099. int rc;
  3100. if ((ctxp->state == LPFC_NVME_STE_LS_RCV && ctxp->entry_cnt == 1) ||
  3101. (ctxp->state == LPFC_NVME_STE_LS_RSP && ctxp->entry_cnt == 2)) {
  3102. ctxp->state = LPFC_NVME_STE_LS_ABORT;
  3103. ctxp->entry_cnt++;
  3104. } else {
  3105. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  3106. "6418 NVMET LS abort state mismatch "
  3107. "IO x%x: %d %d\n",
  3108. ctxp->oxid, ctxp->state, ctxp->entry_cnt);
  3109. ctxp->state = LPFC_NVME_STE_LS_ABORT;
  3110. }
  3111. if (phba->nvmet_support && phba->targetport)
  3112. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  3113. if (!ctxp->wqeq) {
  3114. /* Issue ABTS for this WQE based on iotag */
  3115. ctxp->wqeq = lpfc_sli_get_iocbq(phba);
  3116. if (!ctxp->wqeq) {
  3117. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  3118. "6068 Abort failed: No wqeqs: "
  3119. "xri: x%x\n", xri);
  3120. /* No failure to an ABTS request. */
  3121. kfree(ctxp);
  3122. return 0;
  3123. }
  3124. }
  3125. abts_wqeq = ctxp->wqeq;
  3126. if (lpfc_nvmet_unsol_issue_abort(phba, ctxp, sid, xri) == 0) {
  3127. rc = WQE_BUSY;
  3128. goto out;
  3129. }
  3130. spin_lock_irqsave(&phba->hbalock, flags);
  3131. abts_wqeq->cmd_cmpl = lpfc_nvmet_xmt_ls_abort_cmp;
  3132. abts_wqeq->cmd_flag |= LPFC_IO_NVME_LS;
  3133. rc = lpfc_sli4_issue_wqe(phba, ctxp->hdwq, abts_wqeq);
  3134. spin_unlock_irqrestore(&phba->hbalock, flags);
  3135. if (rc == WQE_SUCCESS) {
  3136. if (tgtp)
  3137. atomic_inc(&tgtp->xmt_abort_unsol);
  3138. return 0;
  3139. }
  3140. out:
  3141. if (tgtp)
  3142. atomic_inc(&tgtp->xmt_abort_rsp_error);
  3143. abts_wqeq->rsp_dmabuf = NULL;
  3144. abts_wqeq->bpl_dmabuf = NULL;
  3145. lpfc_sli_release_iocbq(phba, abts_wqeq);
  3146. lpfc_printf_log(phba, KERN_ERR, LOG_TRACE_EVENT,
  3147. "6056 Failed to Issue ABTS. Status x%x\n", rc);
  3148. return 1;
  3149. }
  3150. /**
  3151. * lpfc_nvmet_invalidate_host
  3152. *
  3153. * @phba: pointer to the driver instance bound to an adapter port.
  3154. * @ndlp: pointer to an lpfc_nodelist type
  3155. *
  3156. * This routine upcalls the nvmet transport to invalidate an NVME
  3157. * host to which this target instance had active connections.
  3158. */
  3159. void
  3160. lpfc_nvmet_invalidate_host(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
  3161. {
  3162. u32 ndlp_has_hh;
  3163. struct lpfc_nvmet_tgtport *tgtp;
  3164. lpfc_printf_log(phba, KERN_INFO,
  3165. LOG_NVME | LOG_NVME_ABTS | LOG_NVME_DISC,
  3166. "6203 Invalidating hosthandle x%px\n",
  3167. ndlp);
  3168. tgtp = (struct lpfc_nvmet_tgtport *)phba->targetport->private;
  3169. atomic_set(&tgtp->state, LPFC_NVMET_INV_HOST_ACTIVE);
  3170. spin_lock_irq(&ndlp->lock);
  3171. ndlp_has_hh = ndlp->fc4_xpt_flags & NLP_XPT_HAS_HH;
  3172. spin_unlock_irq(&ndlp->lock);
  3173. /* Do not invalidate any nodes that do not have a hosthandle.
  3174. * The host_release callbk will cause a node reference
  3175. * count imbalance and a crash.
  3176. */
  3177. if (!ndlp_has_hh) {
  3178. lpfc_printf_log(phba, KERN_INFO,
  3179. LOG_NVME | LOG_NVME_ABTS | LOG_NVME_DISC,
  3180. "6204 Skip invalidate on node x%px DID x%x\n",
  3181. ndlp, ndlp->nlp_DID);
  3182. return;
  3183. }
  3184. #if (IS_ENABLED(CONFIG_NVME_TARGET_FC))
  3185. /* Need to get the nvmet_fc_target_port pointer here.*/
  3186. nvmet_fc_invalidate_host(phba->targetport, ndlp);
  3187. #endif
  3188. }