smcinvoke.c 93 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #define pr_fmt(fmt) "smcinvoke: %s: " fmt, __func__
  7. #include <linux/module.h>
  8. #include <linux/mod_devicetable.h>
  9. #include <linux/device.h>
  10. #include <linux/platform_device.h>
  11. #include <linux/slab.h>
  12. #include <linux/file.h>
  13. #include <linux/fs.h>
  14. #include <linux/anon_inodes.h>
  15. #include <linux/hashtable.h>
  16. #include <linux/cdev.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/dma-buf.h>
  19. #include <linux/delay.h>
  20. #include <linux/kref.h>
  21. #include <linux/signal.h>
  22. #include <linux/msm_ion.h>
  23. #include <linux/mem-buf.h>
  24. #include <linux/of_platform.h>
  25. #include <linux/firmware.h>
  26. #include <linux/qcom_scm.h>
  27. #include <linux/freezer.h>
  28. #include <asm/cacheflush.h>
  29. #include <soc/qcom/qseecomi.h>
  30. #include <linux/qtee_shmbridge.h>
  31. #include <linux/kthread.h>
  32. #include "smcinvoke.h"
  33. #include "smcinvoke_object.h"
  34. #include "IClientEnv.h"
  35. #if IS_ENABLED(CONFIG_QSEECOM_PROXY)
  36. #include <linux/qseecom_kernel.h>
  37. #include "misc/qseecom_priv.h"
  38. #else
  39. #include "misc/qseecom_kernel.h"
  40. #endif
  41. #define CREATE_TRACE_POINTS
  42. #include "trace_smcinvoke.h"
  43. #define SMCINVOKE_DEV "smcinvoke"
  44. #define SMCINVOKE_TZ_ROOT_OBJ 1
  45. #define SMCINVOKE_TZ_OBJ_NULL 0
  46. #define SMCINVOKE_ARGS_ALIGN_SIZE (sizeof(uint64_t))
  47. #define SMCINVOKE_NEXT_AVAILABLE_TXN 0
  48. #define SMCINVOKE_REQ_PLACED 1
  49. #define SMCINVOKE_REQ_PROCESSING 2
  50. #define SMCINVOKE_REQ_PROCESSED 3
  51. #define SMCINVOKE_INCREMENT 1
  52. #define SMCINVOKE_DECREMENT 0
  53. #define SMCINVOKE_OBJ_TYPE_TZ_OBJ 0
  54. #define SMCINVOKE_OBJ_TYPE_SERVER 1
  55. #define SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL 2
  56. #define SMCINVOKE_MEM_MAP_OBJ 0
  57. #define SMCINVOKE_MEM_RGN_OBJ 1
  58. #define SMCINVOKE_MEM_PERM_RW 6
  59. #define SMCINVOKE_SCM_EBUSY_WAIT_MS 30
  60. #define SMCINVOKE_SCM_EBUSY_MAX_RETRY 200
  61. /* TZ defined values - Start */
  62. #define SMCINVOKE_INVOKE_PARAM_ID 0x224
  63. #define SMCINVOKE_CB_RSP_PARAM_ID 0x22
  64. #define SMCINVOKE_INVOKE_CMD_LEGACY 0x32000600
  65. #define SMCINVOKE_INVOKE_CMD 0x32000602
  66. #define SMCINVOKE_CB_RSP_CMD 0x32000601
  67. #define SMCINVOKE_RESULT_INBOUND_REQ_NEEDED 3
  68. /* TZ defined values - End */
  69. /* Asynchronous protocol values */
  70. /* Driver async version is set to match the minimal TZ version that supports async memory object */
  71. #define SMCINVOKE_ASYNC_VERSION (0x00010002)
  72. #define SMCINVOKE_ASYNC_OP_MEMORY_OBJECT (0x00000003)
  73. /*
  74. * This is the state when server FD has been closed but
  75. * TZ still has refs of CBOBjs served by this server
  76. */
  77. #define SMCINVOKE_SERVER_STATE_DEFUNCT 1
  78. #define CBOBJ_MAX_RETRIES 50
  79. #define FOR_ARGS(ndxvar, counts, section) \
  80. for (ndxvar = OBJECT_COUNTS_INDEX_##section(counts); \
  81. ndxvar < (OBJECT_COUNTS_INDEX_##section(counts) \
  82. + OBJECT_COUNTS_NUM_##section(counts)); \
  83. ++ndxvar)
  84. #define TZCB_BUF_OFFSET(tzcb_req) (sizeof(tzcb_req->result) + \
  85. sizeof(struct smcinvoke_msg_hdr) + \
  86. sizeof(union smcinvoke_tz_args) * \
  87. OBJECT_COUNTS_TOTAL(tzcb_req->hdr.counts))
  88. /*
  89. * +ve uhandle : either remote obj or mem obj, decided by f_ops
  90. * -ve uhandle : either Obj NULL or CBObj
  91. * - -1: OBJ NULL
  92. * - < -1: CBObj
  93. */
  94. #define UHANDLE_IS_FD(h) ((h) >= 0)
  95. #define UHANDLE_IS_NULL(h) ((h) == SMCINVOKE_USERSPACE_OBJ_NULL)
  96. #define UHANDLE_IS_CB_OBJ(h) (h < SMCINVOKE_USERSPACE_OBJ_NULL)
  97. #define UHANDLE_NULL (SMCINVOKE_USERSPACE_OBJ_NULL)
  98. /*
  99. * MAKE => create handle for other domain i.e. TZ or userspace
  100. * GET => retrieve obj from incoming handle
  101. */
  102. #define UHANDLE_GET_CB_OBJ(h) (-2-(h))
  103. #define UHANDLE_MAKE_CB_OBJ(o) (-2-(o))
  104. #define UHANDLE_GET_FD(h) (h)
  105. /*
  106. * +ve tzhandle : remote object i.e. owned by TZ
  107. * -ve tzhandle : local object i.e. owned by linux
  108. * --------------------------------------------------
  109. *| 1 (1 bit) | Obj Id (15 bits) | srvr id (16 bits) |
  110. * ---------------------------------------------------
  111. * Server ids are defined below for various local objects
  112. * server id 0 : Kernel Obj
  113. * server id 1 : Memory region Obj
  114. * server id 2 : Memory map Obj
  115. * server id 3-15: Reserverd
  116. * server id 16 & up: Callback Objs
  117. */
  118. #define KRNL_SRVR_ID 0
  119. #define MEM_RGN_SRVR_ID 1
  120. #define MEM_MAP_SRVR_ID 2
  121. #define CBOBJ_SERVER_ID_START 0x10
  122. #define CBOBJ_SERVER_ID_END ((1<<16) - 1)
  123. /* local obj id is represented by 15 bits */
  124. #define MAX_LOCAL_OBJ_ID ((1<<15) - 1)
  125. /* CBOBJs will be served by server id 0x10 onwards */
  126. #define TZHANDLE_GET_SERVER(h) ((uint16_t)((h) & 0xFFFF))
  127. #define TZHANDLE_GET_OBJID(h) (((h) >> 16) & 0x7FFF)
  128. #define TZHANDLE_MAKE_LOCAL(s, o) (((0x8000 | (o)) << 16) | s)
  129. #define SET_BIT(s,b) (s | (1 << b))
  130. #define UNSET_BIT(s,b) (s & (~ (1 << b)))
  131. #define TZHANDLE_IS_NULL(h) ((h) == SMCINVOKE_TZ_OBJ_NULL)
  132. #define TZHANDLE_IS_LOCAL(h) ((h) & 0x80000000)
  133. #define TZHANDLE_IS_REMOTE(h) (!TZHANDLE_IS_NULL(h) && !TZHANDLE_IS_LOCAL(h))
  134. #define TZHANDLE_IS_KERNEL_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  135. TZHANDLE_GET_SERVER(h) == KRNL_SRVR_ID)
  136. #define TZHANDLE_IS_MEM_RGN_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  137. TZHANDLE_GET_SERVER(h) == MEM_RGN_SRVR_ID)
  138. #define TZHANDLE_IS_MEM_MAP_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  139. TZHANDLE_GET_SERVER(h) == MEM_MAP_SRVR_ID)
  140. #define TZHANDLE_IS_MEM_OBJ(h) (TZHANDLE_IS_MEM_RGN_OBJ(h) || \
  141. TZHANDLE_IS_MEM_MAP_OBJ(h))
  142. #define TZHANDLE_IS_CB_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  143. TZHANDLE_GET_SERVER(h) >= CBOBJ_SERVER_ID_START)
  144. #define FILE_IS_REMOTE_OBJ(f) ((f)->f_op && (f)->f_op == &g_smcinvoke_fops)
  145. static DEFINE_MUTEX(g_smcinvoke_lock);
  146. #define NO_LOCK 0
  147. #define TAKE_LOCK 1
  148. #define MUTEX_LOCK(x) { if (x) mutex_lock(&g_smcinvoke_lock); }
  149. #define MUTEX_UNLOCK(x) { if (x) mutex_unlock(&g_smcinvoke_lock); }
  150. #define POST_KT_SLEEP 0
  151. #define POST_KT_WAKEUP 1
  152. #define MAX_CHAR_NAME 50
  153. enum worker_thread_type {
  154. SHMB_WORKER_THREAD = 0,
  155. OBJECT_WORKER_THREAD,
  156. ADCI_WORKER_THREAD,
  157. MAX_THREAD_NUMBER
  158. };
  159. static DEFINE_HASHTABLE(g_cb_servers, 8);
  160. static LIST_HEAD(g_mem_objs);
  161. static uint16_t g_last_cb_server_id = CBOBJ_SERVER_ID_START;
  162. static uint16_t g_last_mem_rgn_id, g_last_mem_map_obj_id;
  163. static size_t g_max_cb_buf_size = SMCINVOKE_TZ_MIN_BUF_SIZE;
  164. static unsigned int cb_reqs_inflight;
  165. static bool legacy_smc_call;
  166. static int invoke_cmd;
  167. static long smcinvoke_ioctl(struct file *, unsigned int, unsigned long);
  168. static int smcinvoke_open(struct inode *, struct file *);
  169. static int smcinvoke_release(struct inode *, struct file *);
  170. static int release_cb_server(uint16_t);
  171. static const struct file_operations g_smcinvoke_fops = {
  172. .owner = THIS_MODULE,
  173. .unlocked_ioctl = smcinvoke_ioctl,
  174. .compat_ioctl = smcinvoke_ioctl,
  175. .open = smcinvoke_open,
  176. .release = smcinvoke_release,
  177. };
  178. static dev_t smcinvoke_device_no;
  179. static struct cdev smcinvoke_cdev;
  180. static struct class *driver_class;
  181. static struct device *class_dev;
  182. static struct platform_device *smcinvoke_pdev;
  183. /* We disable async memory object support by default,
  184. * until we receive the first message from TZ over the
  185. * async channel and can determine TZ async version.
  186. */
  187. static bool mem_obj_async_support = false;
  188. static uint32_t tz_async_version = 0x0;
  189. struct smcinvoke_buf_hdr {
  190. uint32_t offset;
  191. uint32_t size;
  192. };
  193. union smcinvoke_tz_args {
  194. struct smcinvoke_buf_hdr b;
  195. int32_t handle;
  196. };
  197. struct smcinvoke_msg_hdr {
  198. uint32_t tzhandle;
  199. uint32_t op;
  200. uint32_t counts;
  201. };
  202. /* Inbound reqs from TZ */
  203. struct smcinvoke_tzcb_req {
  204. int32_t result;
  205. struct smcinvoke_msg_hdr hdr;
  206. union smcinvoke_tz_args args[0];
  207. };
  208. struct smcinvoke_file_data {
  209. uint32_t context_type;
  210. union {
  211. uint32_t tzhandle;
  212. uint16_t server_id;
  213. };
  214. };
  215. struct smcinvoke_piggyback_msg {
  216. uint32_t version;
  217. uint32_t op;
  218. uint32_t counts;
  219. int32_t objs[0];
  220. };
  221. /* Mapped memory object data
  222. *
  223. * memObjRef Handle reference for the memory object
  224. * mapObjRef Handle reference for the map object
  225. * addr Mapped memory address
  226. * size Size of mapped memory
  227. * perm Access rights for the memory
  228. */
  229. struct smcinvoke_mem_obj_info {
  230. uint32_t memObjRef;
  231. uint32_t mapObjRef;
  232. uint64_t addr;
  233. uint64_t size;
  234. uint32_t perm;
  235. };
  236. /* Memory object info to be written into the async buffer
  237. *
  238. * version Async protocol version
  239. * op Async protocol operation
  240. * count Number of memory objects passed
  241. * mo Mapped memory object data
  242. */
  243. struct smcinvoke_mem_obj_msg {
  244. uint32_t version;
  245. uint32_t op;
  246. uint32_t count;
  247. struct smcinvoke_mem_obj_info mo[];
  248. };
  249. struct smcinvoke_mem_obj_pending_async {
  250. struct smcinvoke_mem_obj *mem_obj;
  251. struct list_head list;
  252. };
  253. /* Data structure to hold request coming from TZ */
  254. struct smcinvoke_cb_txn {
  255. uint32_t txn_id;
  256. int32_t state;
  257. struct smcinvoke_tzcb_req *cb_req;
  258. size_t cb_req_bytes;
  259. struct file **filp_to_release;
  260. struct hlist_node hash;
  261. struct kref ref_cnt;
  262. };
  263. struct smcinvoke_server_info {
  264. uint16_t server_id;
  265. uint16_t state;
  266. uint32_t txn_id;
  267. struct kref ref_cnt;
  268. wait_queue_head_t req_wait_q;
  269. wait_queue_head_t rsp_wait_q;
  270. size_t cb_buf_size;
  271. DECLARE_HASHTABLE(reqs_table, 4);
  272. DECLARE_HASHTABLE(responses_table, 4);
  273. struct hlist_node hash;
  274. struct list_head pending_cbobjs;
  275. uint8_t is_server_suspended;
  276. };
  277. struct smcinvoke_cbobj {
  278. uint16_t cbobj_id;
  279. struct kref ref_cnt;
  280. struct smcinvoke_server_info *server;
  281. struct list_head list;
  282. };
  283. /*
  284. * We require couple of objects, one for mem region & another
  285. * for mapped mem_obj once mem region has been mapped. It is
  286. * possible that TZ can release either independent of other.
  287. */
  288. struct smcinvoke_mem_obj {
  289. /* these ids are objid part of tzhandle */
  290. uint16_t mem_region_id;
  291. uint16_t mem_map_obj_id;
  292. struct dma_buf *dma_buf;
  293. struct dma_buf_attachment *buf_attach;
  294. struct sg_table *sgt;
  295. struct kref mem_regn_ref_cnt;
  296. struct kref mem_map_obj_ref_cnt;
  297. uint64_t p_addr;
  298. size_t p_addr_len;
  299. struct list_head list;
  300. uint64_t shmbridge_handle;
  301. struct smcinvoke_server_info *server;
  302. int32_t mem_obj_user_fd;
  303. };
  304. static LIST_HEAD(g_bridge_postprocess);
  305. DEFINE_MUTEX(bridge_postprocess_lock);
  306. static LIST_HEAD(g_object_postprocess);
  307. DEFINE_MUTEX(object_postprocess_lock);
  308. struct bridge_deregister {
  309. uint64_t shmbridge_handle;
  310. struct dma_buf *dmabuf_to_free;
  311. };
  312. struct object_release {
  313. uint32_t tzhandle;
  314. uint32_t context_type;
  315. };
  316. struct smcinvoke_shmbridge_deregister_pending_list {
  317. struct list_head list;
  318. struct bridge_deregister data;
  319. };
  320. struct smcinvoke_object_release_pending_list {
  321. struct list_head list;
  322. struct object_release data;
  323. };
  324. struct smcinvoke_worker_thread {
  325. enum worker_thread_type type;
  326. atomic_t postprocess_kthread_state;
  327. wait_queue_head_t postprocess_kthread_wq;
  328. struct task_struct *postprocess_kthread_task;
  329. };
  330. static struct smcinvoke_worker_thread smcinvoke[MAX_THREAD_NUMBER];
  331. static const char thread_name[MAX_THREAD_NUMBER][MAX_CHAR_NAME] = {
  332. "smcinvoke_shmbridge_postprocess", "smcinvoke_object_postprocess", "smcinvoke_adci_thread"};
  333. static struct Object adci_rootEnv = Object_NULL;
  334. extern int get_root_obj(struct Object *rootObj);
  335. static int prepare_send_scm_msg(const uint8_t *in_buf, phys_addr_t in_paddr,
  336. size_t in_buf_len,
  337. uint8_t *out_buf, phys_addr_t out_paddr,
  338. size_t out_buf_len,
  339. struct smcinvoke_cmd_req *req,
  340. union smcinvoke_arg *args_buf,
  341. bool *tz_acked, uint32_t context_type,
  342. struct qtee_shm *in_shm, struct qtee_shm *out_shm);
  343. static void process_piggyback_data(void *buf, size_t buf_size);
  344. static void add_mem_obj_info_to_async_side_channel_locked(void *buf, size_t buf_size, struct list_head *l_pending_mem_obj);
  345. static void delete_pending_async_list_locked(struct list_head *l_pending_mem_obj);
  346. static void destroy_cb_server(struct kref *kref)
  347. {
  348. struct smcinvoke_server_info *server = container_of(kref,
  349. struct smcinvoke_server_info, ref_cnt);
  350. if (server) {
  351. hash_del(&server->hash);
  352. kfree(server);
  353. }
  354. }
  355. /*
  356. * A separate find func is reqd mainly for couple of cases:
  357. * next_cb_server_id_locked which checks if server id had been utilized or not.
  358. * - It would be overhead if we do ref_cnt for this case
  359. * smcinvoke_release: which is called when server is closed from userspace.
  360. * - During server creation we init ref count, now put it back
  361. */
  362. static struct smcinvoke_server_info *find_cb_server_locked(uint16_t server_id)
  363. {
  364. struct smcinvoke_server_info *data = NULL;
  365. hash_for_each_possible(g_cb_servers, data, hash, server_id) {
  366. if (data->server_id == server_id)
  367. return data;
  368. }
  369. return NULL;
  370. }
  371. static struct smcinvoke_server_info *get_cb_server_locked(uint16_t server_id)
  372. {
  373. struct smcinvoke_server_info *server = find_cb_server_locked(server_id);
  374. if (server)
  375. kref_get(&server->ref_cnt);
  376. return server;
  377. }
  378. static uint16_t next_cb_server_id_locked(void)
  379. {
  380. if (g_last_cb_server_id == CBOBJ_SERVER_ID_END)
  381. g_last_cb_server_id = CBOBJ_SERVER_ID_START;
  382. while (find_cb_server_locked(++g_last_cb_server_id))
  383. ;
  384. return g_last_cb_server_id;
  385. }
  386. static inline void release_filp(struct file **filp_to_release, size_t arr_len)
  387. {
  388. size_t i = 0;
  389. for (i = 0; i < arr_len; i++) {
  390. if (filp_to_release[i]) {
  391. fput(filp_to_release[i]);
  392. filp_to_release[i] = NULL;
  393. }
  394. }
  395. }
  396. static struct smcinvoke_mem_obj *find_mem_obj_locked(uint16_t mem_obj_id,
  397. bool is_mem_rgn_obj)
  398. {
  399. struct smcinvoke_mem_obj *mem_obj = NULL;
  400. if (list_empty(&g_mem_objs))
  401. return NULL;
  402. list_for_each_entry(mem_obj, &g_mem_objs, list) {
  403. if ((is_mem_rgn_obj &&
  404. (mem_obj->mem_region_id == mem_obj_id)) ||
  405. (!is_mem_rgn_obj &&
  406. (mem_obj->mem_map_obj_id == mem_obj_id)))
  407. return mem_obj;
  408. }
  409. return NULL;
  410. }
  411. static uint32_t next_mem_region_obj_id_locked(void)
  412. {
  413. if (g_last_mem_rgn_id == MAX_LOCAL_OBJ_ID)
  414. g_last_mem_rgn_id = 0;
  415. while (find_mem_obj_locked(++g_last_mem_rgn_id, SMCINVOKE_MEM_RGN_OBJ))
  416. ;
  417. return g_last_mem_rgn_id;
  418. }
  419. static uint32_t next_mem_map_obj_id_locked(void)
  420. {
  421. if (g_last_mem_map_obj_id == MAX_LOCAL_OBJ_ID)
  422. g_last_mem_map_obj_id = 0;
  423. while (find_mem_obj_locked(++g_last_mem_map_obj_id,
  424. SMCINVOKE_MEM_MAP_OBJ))
  425. ;
  426. return g_last_mem_map_obj_id;
  427. }
  428. static void smcinvoke_shmbridge_post_process(void)
  429. {
  430. struct smcinvoke_shmbridge_deregister_pending_list *entry = NULL;
  431. struct list_head *pos;
  432. int ret = 0;
  433. uint64_t handle = 0;
  434. struct dma_buf *dmabuf_to_free = NULL;
  435. do {
  436. mutex_lock(&bridge_postprocess_lock);
  437. if (list_empty(&g_bridge_postprocess)) {
  438. mutex_unlock(&bridge_postprocess_lock);
  439. break;
  440. }
  441. pos = g_bridge_postprocess.next;
  442. entry = list_entry(pos,
  443. struct smcinvoke_shmbridge_deregister_pending_list,
  444. list);
  445. if (entry) {
  446. handle = entry->data.shmbridge_handle;
  447. dmabuf_to_free = entry->data.dmabuf_to_free;
  448. } else {
  449. pr_err("entry is NULL, pos:%#llx\n", (uint64_t)pos);
  450. }
  451. list_del(pos);
  452. kfree_sensitive(entry);
  453. mutex_unlock(&bridge_postprocess_lock);
  454. if (entry) {
  455. do {
  456. ret = qtee_shmbridge_deregister(handle);
  457. if (unlikely(ret)) {
  458. pr_err_ratelimited("SHM failed: ret:%d ptr:0x%x h:%#llx\n",
  459. ret,
  460. dmabuf_to_free,
  461. handle);
  462. } else {
  463. pr_debug("SHM deletion: Handle:%#llx\n",
  464. handle);
  465. dma_buf_put(dmabuf_to_free);
  466. }
  467. } while (-EBUSY == ret);
  468. }
  469. } while (1);
  470. }
  471. static int smcinvoke_release_tz_object(struct qtee_shm *in_shm, struct qtee_shm *out_shm,
  472. uint32_t tzhandle, uint32_t context_type)
  473. {
  474. int ret = 0;
  475. bool release_handles;
  476. uint8_t *in_buf = NULL;
  477. uint8_t *out_buf = NULL;
  478. struct smcinvoke_msg_hdr hdr = {0};
  479. struct smcinvoke_cmd_req req = {0};
  480. in_buf = in_shm->vaddr;
  481. out_buf = out_shm->vaddr;
  482. hdr.tzhandle = tzhandle;
  483. hdr.op = OBJECT_OP_RELEASE;
  484. hdr.counts = 0;
  485. *(struct smcinvoke_msg_hdr *)in_buf = hdr;
  486. ret = prepare_send_scm_msg(in_buf, in_shm->paddr,
  487. SMCINVOKE_TZ_MIN_BUF_SIZE, out_buf, out_shm->paddr,
  488. SMCINVOKE_TZ_MIN_BUF_SIZE, &req, NULL,
  489. &release_handles, context_type, in_shm, out_shm);
  490. process_piggyback_data(out_buf, SMCINVOKE_TZ_MIN_BUF_SIZE);
  491. if (ret) {
  492. pr_err_ratelimited("Failed to release object(0x%x), ret:%d\n",
  493. hdr.tzhandle, ret);
  494. } else {
  495. pr_debug("Released object(0x%x) successfully.\n",
  496. hdr.tzhandle);
  497. }
  498. return ret;
  499. }
  500. static int smcinvoke_object_post_process(void)
  501. {
  502. struct smcinvoke_object_release_pending_list *entry = NULL;
  503. struct list_head *pos;
  504. int ret = 0;
  505. struct qtee_shm in_shm = {0}, out_shm = {0};
  506. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &in_shm);
  507. if (ret) {
  508. ret = -ENOMEM;
  509. pr_err("shmbridge alloc failed for in msg in object release\n");
  510. goto out;
  511. }
  512. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &out_shm);
  513. if (ret) {
  514. ret = -ENOMEM;
  515. pr_err("shmbridge alloc failed for out msg in object release\n");
  516. goto out;
  517. }
  518. do {
  519. mutex_lock(&object_postprocess_lock);
  520. if (list_empty(&g_object_postprocess)) {
  521. mutex_unlock(&object_postprocess_lock);
  522. break;
  523. }
  524. pos = g_object_postprocess.next;
  525. entry = list_entry(pos, struct smcinvoke_object_release_pending_list, list);
  526. list_del(pos);
  527. mutex_unlock(&object_postprocess_lock);
  528. if (entry) {
  529. do {
  530. ret = smcinvoke_release_tz_object(&in_shm, &out_shm,
  531. entry->data.tzhandle, entry->data.context_type);
  532. } while (-EBUSY == ret);
  533. } else {
  534. pr_err("entry is NULL, pos:%#llx\n", (uint64_t)pos);
  535. }
  536. kfree_sensitive(entry);
  537. } while (1);
  538. out:
  539. qtee_shmbridge_free_shm(&in_shm);
  540. qtee_shmbridge_free_shm(&out_shm);
  541. return ret;
  542. }
  543. static void smcinvoke_start_adci_thread(void)
  544. {
  545. int32_t ret = OBJECT_ERROR;
  546. int retry_count = 0;
  547. ret = get_root_obj(&adci_rootEnv);
  548. if (ret) {
  549. pr_err("failed to get rootEnv for ADCI invoke thread. ret = %d\n", ret);
  550. /* Marking it Object_NULL in case of failure scenario in order to avoid
  551. * undefined behavior while relasing garbage adci_rootEnv object. */
  552. adci_rootEnv = Object_NULL;
  553. goto out;
  554. }
  555. /* Invoke call to QTEE which should never return if ADCI is supported */
  556. pr_debug("Invoking adciAccept method in QTEE\n");
  557. do {
  558. ret = IClientEnv_adciAccept(adci_rootEnv);
  559. if (ret == OBJECT_ERROR_BUSY) {
  560. pr_err_ratelimited("Secure side is busy,will retry after 5 ms, retry_count = %d\n",retry_count);
  561. msleep(SMCINVOKE_INTERFACE_BUSY_WAIT_MS);
  562. }
  563. } while ((ret == OBJECT_ERROR_BUSY) && (retry_count++ < SMCINVOKE_INTERFACE_MAX_RETRY));
  564. if (ret == OBJECT_ERROR_INVALID)
  565. pr_err("ADCI feature is not supported on this chipsets, ret = %d\n", ret);
  566. else
  567. pr_debug("Received response from QTEE, ret = %d\n", ret);
  568. out:
  569. /* Control should reach to this point only if ADCI feature is not supported by QTEE
  570. (or) ADCI thread held in QTEE is released. */
  571. Object_ASSIGN_NULL(adci_rootEnv);
  572. }
  573. static void __wakeup_postprocess_kthread(struct smcinvoke_worker_thread *smcinvoke)
  574. {
  575. if (smcinvoke) {
  576. atomic_set(&smcinvoke->postprocess_kthread_state,
  577. POST_KT_WAKEUP);
  578. wake_up_interruptible(&smcinvoke->postprocess_kthread_wq);
  579. } else {
  580. pr_err("Invalid smcinvoke pointer.\n");
  581. }
  582. }
  583. static int smcinvoke_postprocess_kthread_func(void *data)
  584. {
  585. struct smcinvoke_worker_thread *smcinvoke_wrk_trd = data;
  586. static const char *const tag[] = {"shmbridge","object","adci","invalid"};
  587. if (!smcinvoke_wrk_trd) {
  588. pr_err("Bad input.\n");
  589. return -EINVAL;
  590. }
  591. while (!kthread_should_stop()) {
  592. wait_event_interruptible(
  593. smcinvoke_wrk_trd->postprocess_kthread_wq,
  594. kthread_should_stop() ||
  595. (atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state)
  596. == POST_KT_WAKEUP));
  597. switch (smcinvoke_wrk_trd->type) {
  598. case SHMB_WORKER_THREAD:
  599. pr_debug("kthread to %s postprocess is called %d\n",
  600. tag[SHMB_WORKER_THREAD], atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state));
  601. smcinvoke_shmbridge_post_process();
  602. break;
  603. case OBJECT_WORKER_THREAD:
  604. pr_debug("kthread to %s postprocess is called %d\n",
  605. tag[OBJECT_WORKER_THREAD], atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state));
  606. smcinvoke_object_post_process();
  607. break;
  608. case ADCI_WORKER_THREAD:
  609. pr_debug("kthread to %s postprocess is called %d\n",
  610. tag[ADCI_WORKER_THREAD], atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state));
  611. smcinvoke_start_adci_thread();
  612. break;
  613. default:
  614. pr_err("Invalid thread type(%d), do nothing.\n",
  615. (int)smcinvoke_wrk_trd->type);
  616. break;
  617. }
  618. /* For ADCI thread, if control reaches here, that indicates either ADCI
  619. * thread is not supported (or) released by QTEE. Since ADCI thread is
  620. * getting signaled only during the smcinvoke driver initialization,
  621. * there is no point of putting the thread into sleep state again. All the
  622. * required post-processing will be taken care by object and shmbridge threads.
  623. */
  624. if(smcinvoke_wrk_trd->type == ADCI_WORKER_THREAD) {
  625. break;
  626. }
  627. atomic_set(&smcinvoke_wrk_trd->postprocess_kthread_state,
  628. POST_KT_SLEEP);
  629. }
  630. pr_warn("kthread(worker_thread) processed, worker_thread type is %d \n", smcinvoke_wrk_trd->type);
  631. return 0;
  632. }
  633. static int smcinvoke_create_kthreads(void)
  634. {
  635. int i, rc = 0;
  636. const enum worker_thread_type thread_type[MAX_THREAD_NUMBER] = {
  637. SHMB_WORKER_THREAD, OBJECT_WORKER_THREAD, ADCI_WORKER_THREAD};
  638. for (i = 0; i < MAX_THREAD_NUMBER; i++) {
  639. init_waitqueue_head(&smcinvoke[i].postprocess_kthread_wq);
  640. smcinvoke[i].type = thread_type[i];
  641. smcinvoke[i].postprocess_kthread_task = kthread_run(
  642. smcinvoke_postprocess_kthread_func,
  643. &smcinvoke[i], thread_name[i]);
  644. if (IS_ERR(smcinvoke[i].postprocess_kthread_task)) {
  645. rc = PTR_ERR(smcinvoke[i].postprocess_kthread_task);
  646. pr_err("fail to create kthread to postprocess, rc = %x\n",
  647. rc);
  648. return rc;
  649. }
  650. atomic_set(&smcinvoke[i].postprocess_kthread_state,
  651. POST_KT_SLEEP);
  652. }
  653. return rc;
  654. }
  655. static void smcinvoke_destroy_kthreads(void)
  656. {
  657. int i;
  658. int32_t ret = OBJECT_ERROR;
  659. int retry_count = 0;
  660. if (!Object_isNull(adci_rootEnv)) {
  661. pr_debug("Invoking adciShutdown method in QTEE\n");
  662. do {
  663. ret = IClientEnv_adciShutdown(adci_rootEnv);
  664. if (ret == OBJECT_ERROR_BUSY) {
  665. pr_err_ratelimited("Secure side is busy,will retry after 5 ms, retry_count = %d\n",retry_count);
  666. msleep(SMCINVOKE_INTERFACE_BUSY_WAIT_MS);
  667. }
  668. } while ((ret == OBJECT_ERROR_BUSY) && (retry_count++ < SMCINVOKE_INTERFACE_MAX_RETRY));
  669. if (OBJECT_isERROR(ret)) {
  670. pr_err("adciShutdown in QTEE failed with error = %d\n", ret);
  671. }
  672. Object_ASSIGN_NULL(adci_rootEnv);
  673. }
  674. for (i = 0; i < MAX_THREAD_NUMBER; i++) {
  675. kthread_stop(smcinvoke[i].postprocess_kthread_task);
  676. }
  677. }
  678. /* Queue newly created memory object to l_pending_mem_obj list.
  679. * Later, the mapping information for objects in this list will be sent to TZ
  680. * over the async side channel.
  681. *
  682. * No return value as TZ is always able to explicitly ask for this information
  683. * in case this function fails and the memory object is not added to this list.
  684. */
  685. static void queue_mem_obj_pending_async_locked(struct smcinvoke_mem_obj *mem_obj, struct list_head *l_pending_mem_obj)
  686. {
  687. struct smcinvoke_mem_obj_pending_async *t_mem_obj_pending =
  688. kzalloc(sizeof(*t_mem_obj_pending), GFP_KERNEL);
  689. /*
  690. * We are not failing execution in case of a failure here,
  691. * since TZ can always ask for this information explicitly
  692. * if it's not available in the side channel.
  693. */
  694. if (!t_mem_obj_pending) {
  695. pr_err("Unable to allocate memory\n");
  696. return;
  697. }
  698. t_mem_obj_pending->mem_obj = mem_obj;
  699. list_add(&t_mem_obj_pending->list, l_pending_mem_obj);
  700. }
  701. static inline void free_mem_obj_locked(struct smcinvoke_mem_obj *mem_obj)
  702. {
  703. int ret = 0;
  704. struct dma_buf *dmabuf_to_free = mem_obj->dma_buf;
  705. uint64_t shmbridge_handle = mem_obj->shmbridge_handle;
  706. struct smcinvoke_shmbridge_deregister_pending_list *entry = NULL;
  707. list_del(&mem_obj->list);
  708. kfree(mem_obj->server);
  709. kfree(mem_obj);
  710. mem_obj = NULL;
  711. mutex_unlock(&g_smcinvoke_lock);
  712. if (shmbridge_handle)
  713. ret = qtee_shmbridge_deregister(shmbridge_handle);
  714. if (ret) {
  715. pr_err("Error:%d delete bridge failed leaking memory 0x%x\n",
  716. ret, dmabuf_to_free);
  717. if (ret == -EBUSY) {
  718. pr_err("EBUSY: we postpone it 0x%x\n",
  719. dmabuf_to_free);
  720. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  721. if (entry) {
  722. entry->data.shmbridge_handle = shmbridge_handle;
  723. entry->data.dmabuf_to_free = dmabuf_to_free;
  724. mutex_lock(&bridge_postprocess_lock);
  725. list_add_tail(&entry->list, &g_bridge_postprocess);
  726. mutex_unlock(&bridge_postprocess_lock);
  727. pr_debug("SHMBridge list: added a Handle:%#llx\n",
  728. shmbridge_handle);
  729. __wakeup_postprocess_kthread(
  730. &smcinvoke[SHMB_WORKER_THREAD]);
  731. }
  732. }
  733. } else {
  734. dma_buf_put(dmabuf_to_free);
  735. }
  736. mutex_lock(&g_smcinvoke_lock);
  737. }
  738. static void del_mem_regn_obj_locked(struct kref *kref)
  739. {
  740. struct smcinvoke_mem_obj *mem_obj = container_of(kref,
  741. struct smcinvoke_mem_obj, mem_regn_ref_cnt);
  742. /*
  743. * mem_regn obj and mem_map obj are held into mem_obj structure which
  744. * can't be released until both kinds of objs have been released.
  745. * So check whether mem_map iobj has ref 0 and only then release mem_obj
  746. */
  747. if (kref_read(&mem_obj->mem_map_obj_ref_cnt) == 0)
  748. free_mem_obj_locked(mem_obj);
  749. }
  750. static void del_mem_map_obj_locked(struct kref *kref)
  751. {
  752. struct smcinvoke_mem_obj *mem_obj = container_of(kref,
  753. struct smcinvoke_mem_obj, mem_map_obj_ref_cnt);
  754. mem_obj->p_addr_len = 0;
  755. mem_obj->p_addr = 0;
  756. if (mem_obj->sgt)
  757. dma_buf_unmap_attachment(mem_obj->buf_attach,
  758. mem_obj->sgt, DMA_BIDIRECTIONAL);
  759. if (mem_obj->buf_attach)
  760. dma_buf_detach(mem_obj->dma_buf, mem_obj->buf_attach);
  761. /*
  762. * mem_regn obj and mem_map obj are held into mem_obj structure which
  763. * can't be released until both kinds of objs have been released.
  764. * So check if mem_regn obj has ref 0 and only then release mem_obj
  765. */
  766. if (kref_read(&mem_obj->mem_regn_ref_cnt) == 0)
  767. free_mem_obj_locked(mem_obj);
  768. }
  769. static int release_mem_obj_locked(int32_t tzhandle)
  770. {
  771. int is_mem_regn_obj = TZHANDLE_IS_MEM_RGN_OBJ(tzhandle);
  772. struct smcinvoke_mem_obj *mem_obj = find_mem_obj_locked(
  773. TZHANDLE_GET_OBJID(tzhandle), is_mem_regn_obj);
  774. if (!mem_obj) {
  775. pr_err("memory object not found\n");
  776. return OBJECT_ERROR_BADOBJ;
  777. }
  778. if (is_mem_regn_obj)
  779. kref_put(&mem_obj->mem_regn_ref_cnt, del_mem_regn_obj_locked);
  780. else
  781. kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  782. return OBJECT_OK;
  783. }
  784. static void free_pending_cbobj_locked(struct kref *kref)
  785. {
  786. struct smcinvoke_server_info *server = NULL;
  787. struct smcinvoke_cbobj *obj = container_of(kref,
  788. struct smcinvoke_cbobj, ref_cnt);
  789. list_del(&obj->list);
  790. server = obj->server;
  791. kfree(obj);
  792. if (server)
  793. kref_put(&server->ref_cnt, destroy_cb_server);
  794. }
  795. static int get_pending_cbobj_locked(uint16_t srvr_id, int16_t obj_id)
  796. {
  797. int ret = 0;
  798. bool release_server = true;
  799. struct list_head *head = NULL;
  800. struct smcinvoke_cbobj *cbobj = NULL;
  801. struct smcinvoke_cbobj *obj = NULL;
  802. struct smcinvoke_server_info *server = get_cb_server_locked(srvr_id);
  803. if (!server) {
  804. pr_err("%s, server id : %u not found\n", __func__, srvr_id);
  805. return OBJECT_ERROR_BADOBJ;
  806. }
  807. head = &server->pending_cbobjs;
  808. list_for_each_entry(cbobj, head, list)
  809. if (cbobj->cbobj_id == obj_id) {
  810. kref_get(&cbobj->ref_cnt);
  811. goto out;
  812. }
  813. obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  814. if (!obj) {
  815. ret = OBJECT_ERROR_KMEM;
  816. goto out;
  817. }
  818. obj->cbobj_id = obj_id;
  819. kref_init(&obj->ref_cnt);
  820. obj->server = server;
  821. /*
  822. * we are holding server ref in cbobj; we will
  823. * release server ref when cbobj is destroyed
  824. */
  825. release_server = false;
  826. list_add_tail(&obj->list, head);
  827. out:
  828. if (release_server)
  829. kref_put(&server->ref_cnt, destroy_cb_server);
  830. return ret;
  831. }
  832. static int put_pending_cbobj_locked(uint16_t srvr_id, int16_t obj_id)
  833. {
  834. int ret = -EINVAL;
  835. struct smcinvoke_server_info *srvr_info =
  836. get_cb_server_locked(srvr_id);
  837. struct list_head *head = NULL;
  838. struct smcinvoke_cbobj *cbobj = NULL;
  839. if (!srvr_info) {
  840. pr_err("%s, server id : %u not found\n", __func__, srvr_id);
  841. return ret;
  842. }
  843. trace_put_pending_cbobj_locked(srvr_id, obj_id);
  844. head = &srvr_info->pending_cbobjs;
  845. list_for_each_entry(cbobj, head, list)
  846. if (cbobj->cbobj_id == obj_id) {
  847. kref_put(&cbobj->ref_cnt, free_pending_cbobj_locked);
  848. ret = 0;
  849. break;
  850. }
  851. kref_put(&srvr_info->ref_cnt, destroy_cb_server);
  852. return ret;
  853. }
  854. static int release_tzhandle_locked(int32_t tzhandle)
  855. {
  856. if (TZHANDLE_IS_MEM_OBJ(tzhandle))
  857. return release_mem_obj_locked(tzhandle);
  858. else if (TZHANDLE_IS_CB_OBJ(tzhandle))
  859. return put_pending_cbobj_locked(TZHANDLE_GET_SERVER(tzhandle),
  860. TZHANDLE_GET_OBJID(tzhandle));
  861. return OBJECT_ERROR;
  862. }
  863. static void release_tzhandles(const int32_t *tzhandles, size_t len)
  864. {
  865. size_t i;
  866. mutex_lock(&g_smcinvoke_lock);
  867. for (i = 0; i < len; i++)
  868. release_tzhandle_locked(tzhandles[i]);
  869. mutex_unlock(&g_smcinvoke_lock);
  870. }
  871. static void delete_cb_txn_locked(struct kref *kref)
  872. {
  873. struct smcinvoke_cb_txn *cb_txn = container_of(kref,
  874. struct smcinvoke_cb_txn, ref_cnt);
  875. if (OBJECT_OP_METHODID(cb_txn->cb_req->hdr.op) == OBJECT_OP_RELEASE)
  876. release_tzhandle_locked(cb_txn->cb_req->hdr.tzhandle);
  877. kfree(cb_txn->cb_req);
  878. hash_del(&cb_txn->hash);
  879. kfree(cb_txn);
  880. }
  881. static struct smcinvoke_cb_txn *find_cbtxn_locked(
  882. struct smcinvoke_server_info *server,
  883. uint32_t txn_id, int32_t state)
  884. {
  885. int i = 0;
  886. struct smcinvoke_cb_txn *cb_txn = NULL;
  887. /*
  888. * Since HASH_BITS() does not work on pointers, we can't select hash
  889. * table using state and loop over it.
  890. */
  891. if (state == SMCINVOKE_REQ_PLACED) {
  892. /* pick up 1st req */
  893. hash_for_each(server->reqs_table, i, cb_txn, hash) {
  894. kref_get(&cb_txn->ref_cnt);
  895. hash_del(&cb_txn->hash);
  896. return cb_txn;
  897. }
  898. } else if (state == SMCINVOKE_REQ_PROCESSING) {
  899. hash_for_each_possible(
  900. server->responses_table, cb_txn, hash, txn_id) {
  901. if (cb_txn->txn_id == txn_id) {
  902. kref_get(&cb_txn->ref_cnt);
  903. hash_del(&cb_txn->hash);
  904. return cb_txn;
  905. }
  906. }
  907. }
  908. return NULL;
  909. }
  910. /*
  911. * size_add_ saturates at SIZE_MAX. If integer overflow is detected,
  912. * this function would return SIZE_MAX otherwise normal a+b is returned.
  913. */
  914. static inline size_t size_add_(size_t a, size_t b)
  915. {
  916. return (b > (SIZE_MAX - a)) ? SIZE_MAX : a + b;
  917. }
  918. /*
  919. * pad_size is used along with size_align to define a buffer overflow
  920. * protected version of ALIGN
  921. */
  922. static inline size_t pad_size(size_t a, size_t b)
  923. {
  924. return (~a + 1) % b;
  925. }
  926. /*
  927. * size_align saturates at SIZE_MAX. If integer overflow is detected, this
  928. * function would return SIZE_MAX otherwise next aligned size is returned.
  929. */
  930. static inline size_t size_align(size_t a, size_t b)
  931. {
  932. return size_add_(a, pad_size(a, b));
  933. }
  934. static uint16_t get_server_id(int cb_server_fd)
  935. {
  936. uint16_t server_id = 0;
  937. struct smcinvoke_file_data *svr_cxt = NULL;
  938. struct file *tmp_filp = fget(cb_server_fd);
  939. if (!tmp_filp || !FILE_IS_REMOTE_OBJ(tmp_filp))
  940. return server_id;
  941. svr_cxt = tmp_filp->private_data;
  942. if (svr_cxt && svr_cxt->context_type == SMCINVOKE_OBJ_TYPE_SERVER)
  943. server_id = svr_cxt->server_id;
  944. fput(tmp_filp);
  945. return server_id;
  946. }
  947. static bool is_dma_fd(int32_t uhandle, struct dma_buf **dma_buf)
  948. {
  949. *dma_buf = dma_buf_get(uhandle);
  950. return IS_ERR_OR_NULL(*dma_buf) ? false : true;
  951. }
  952. static bool is_remote_obj(int32_t uhandle, struct smcinvoke_file_data **tzobj,
  953. struct file **filp)
  954. {
  955. bool ret = false;
  956. struct file *tmp_filp = fget(uhandle);
  957. if (!tmp_filp)
  958. return ret;
  959. if (FILE_IS_REMOTE_OBJ(tmp_filp)) {
  960. *tzobj = tmp_filp->private_data;
  961. if ((*tzobj)->context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  962. *filp = tmp_filp;
  963. tmp_filp = NULL;
  964. ret = true;
  965. }
  966. }
  967. if (tmp_filp)
  968. fput(tmp_filp);
  969. return ret;
  970. }
  971. static int smcinvoke_create_bridge(struct smcinvoke_mem_obj *mem_obj)
  972. {
  973. int ret = 0;
  974. int tz_perm = PERM_READ|PERM_WRITE;
  975. uint32_t *vmid_list;
  976. uint32_t *perms_list;
  977. uint32_t nelems = 0;
  978. struct dma_buf *dmabuf = mem_obj->dma_buf;
  979. phys_addr_t phys = mem_obj->p_addr;
  980. size_t size = mem_obj->p_addr_len;
  981. if (!qtee_shmbridge_is_enabled())
  982. return 0;
  983. ret = mem_buf_dma_buf_copy_vmperm(dmabuf, (int **)&vmid_list,
  984. (int **)&perms_list, (int *)&nelems);
  985. if (ret) {
  986. pr_err("mem_buf_dma_buf_copy_vmperm failure, err=%d\n", ret);
  987. return ret;
  988. }
  989. if (mem_buf_dma_buf_exclusive_owner(dmabuf))
  990. perms_list[0] = PERM_READ | PERM_WRITE;
  991. ret = qtee_shmbridge_register(phys, size, vmid_list, perms_list, nelems,
  992. tz_perm, &mem_obj->shmbridge_handle);
  993. if (ret) {
  994. pr_err("creation of shm bridge for mem_region_id %d failed ret %d\n",
  995. mem_obj->mem_region_id, ret);
  996. goto exit;
  997. }
  998. trace_smcinvoke_create_bridge(mem_obj->shmbridge_handle, mem_obj->mem_region_id);
  999. exit:
  1000. kfree(perms_list);
  1001. kfree(vmid_list);
  1002. return ret;
  1003. }
  1004. /* Map memory region for a given memory object.
  1005. * Mapping information will be saved as part of the memory object structure.
  1006. */
  1007. static int32_t smcinvoke_map_mem_region_locked(struct smcinvoke_mem_obj* mem_obj)
  1008. {
  1009. int ret = OBJECT_OK;
  1010. struct dma_buf_attachment *buf_attach = NULL;
  1011. struct sg_table *sgt = NULL;
  1012. if (!mem_obj) {
  1013. pr_err("Invalid memory object\n");
  1014. return OBJECT_ERROR_BADOBJ;
  1015. }
  1016. if (!mem_obj->p_addr) {
  1017. kref_init(&mem_obj->mem_map_obj_ref_cnt);
  1018. buf_attach = dma_buf_attach(mem_obj->dma_buf,
  1019. &smcinvoke_pdev->dev);
  1020. if (IS_ERR(buf_attach)) {
  1021. ret = OBJECT_ERROR_KMEM;
  1022. pr_err("dma buf attach failed, ret: %d\n", ret);
  1023. goto out;
  1024. }
  1025. mem_obj->buf_attach = buf_attach;
  1026. sgt = dma_buf_map_attachment(buf_attach, DMA_BIDIRECTIONAL);
  1027. if (IS_ERR(sgt)) {
  1028. pr_err("mapping dma buffers failed, ret: %d\n",
  1029. PTR_ERR(sgt));
  1030. ret = OBJECT_ERROR_KMEM;
  1031. goto out;
  1032. }
  1033. mem_obj->sgt = sgt;
  1034. /* contiguous only => nents=1 */
  1035. if (sgt->nents != 1) {
  1036. ret = OBJECT_ERROR_INVALID;
  1037. pr_err("sg enries are not contigous, ret: %d\n", ret);
  1038. goto out;
  1039. }
  1040. mem_obj->p_addr = sg_dma_address(sgt->sgl);
  1041. mem_obj->p_addr_len = sgt->sgl->length;
  1042. if (!mem_obj->p_addr) {
  1043. ret = OBJECT_ERROR_INVALID;
  1044. pr_err("invalid physical address, ret: %d\n", ret);
  1045. goto out;
  1046. }
  1047. /* Increase reference count as we are feeding the memobj to
  1048. * smcinvoke and unlock the mutex. No need to hold the mutex in
  1049. * case of shmbridge creation.
  1050. */
  1051. kref_get(&mem_obj->mem_map_obj_ref_cnt);
  1052. mutex_unlock(&g_smcinvoke_lock);
  1053. ret = smcinvoke_create_bridge(mem_obj);
  1054. /* Take lock again and decrease the reference count which we
  1055. * increased for shmbridge but before proceeding further we
  1056. * have to check again if the memobj is still valid or not
  1057. * after decreasing the reference.
  1058. */
  1059. mutex_lock(&g_smcinvoke_lock);
  1060. kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  1061. if (ret) {
  1062. ret = OBJECT_ERROR_INVALID;
  1063. pr_err("Unable to create shm bridge, ret: %d\n", ret);
  1064. goto out;
  1065. }
  1066. if (!find_mem_obj_locked(mem_obj->mem_region_id,
  1067. SMCINVOKE_MEM_RGN_OBJ)) {
  1068. mutex_unlock(&g_smcinvoke_lock);
  1069. pr_err("Memory object not found\n");
  1070. return OBJECT_ERROR_BADOBJ;
  1071. }
  1072. mem_obj->mem_map_obj_id = next_mem_map_obj_id_locked();
  1073. }
  1074. out:
  1075. if (ret != OBJECT_OK)
  1076. kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  1077. return ret;
  1078. }
  1079. static int create_mem_obj(struct dma_buf *dma_buf, int32_t *tzhandle,
  1080. struct smcinvoke_mem_obj **mem_obj, int32_t server_id, int32_t user_handle)
  1081. {
  1082. struct smcinvoke_mem_obj *t_mem_obj = NULL;
  1083. struct smcinvoke_server_info *server_i = NULL;
  1084. t_mem_obj = kzalloc(sizeof(struct smcinvoke_mem_obj), GFP_KERNEL);
  1085. if (!t_mem_obj) {
  1086. dma_buf_put(dma_buf);
  1087. return -ENOMEM;
  1088. }
  1089. server_i = kzalloc(sizeof(struct smcinvoke_server_info),GFP_KERNEL);
  1090. if (!server_i) {
  1091. kfree(t_mem_obj);
  1092. dma_buf_put(dma_buf);
  1093. return -ENOMEM;
  1094. }
  1095. kref_init(&t_mem_obj->mem_regn_ref_cnt);
  1096. t_mem_obj->dma_buf = dma_buf;
  1097. mutex_lock(&g_smcinvoke_lock);
  1098. t_mem_obj->mem_region_id = next_mem_region_obj_id_locked();
  1099. server_i->server_id = server_id;
  1100. t_mem_obj->server = server_i;
  1101. t_mem_obj->mem_obj_user_fd = user_handle;
  1102. list_add_tail(&t_mem_obj->list, &g_mem_objs);
  1103. mutex_unlock(&g_smcinvoke_lock);
  1104. *mem_obj = t_mem_obj;
  1105. *tzhandle = TZHANDLE_MAKE_LOCAL(MEM_RGN_SRVR_ID,
  1106. t_mem_obj->mem_region_id);
  1107. return 0;
  1108. }
  1109. /*
  1110. * This function retrieves file pointer corresponding to FD provided. It stores
  1111. * retrieved file pointer until IOCTL call is concluded. Once call is completed,
  1112. * all stored file pointers are released. file pointers are stored to prevent
  1113. * other threads from releasing that FD while IOCTL is in progress.
  1114. */
  1115. static int get_tzhandle_from_uhandle(int32_t uhandle, int32_t server_fd,
  1116. struct file **filp, uint32_t *tzhandle, struct list_head *l_pending_mem_obj)
  1117. {
  1118. int ret = -EBADF;
  1119. uint16_t server_id = 0;
  1120. struct smcinvoke_mem_obj *mem_obj = NULL;
  1121. if (UHANDLE_IS_NULL(uhandle)) {
  1122. *tzhandle = SMCINVOKE_TZ_OBJ_NULL;
  1123. ret = 0;
  1124. } else if (UHANDLE_IS_CB_OBJ(uhandle)) {
  1125. server_id = get_server_id(server_fd);
  1126. if (server_id < CBOBJ_SERVER_ID_START)
  1127. goto out;
  1128. mutex_lock(&g_smcinvoke_lock);
  1129. ret = get_pending_cbobj_locked(server_id,
  1130. UHANDLE_GET_CB_OBJ(uhandle));
  1131. mutex_unlock(&g_smcinvoke_lock);
  1132. if (ret)
  1133. goto out;
  1134. *tzhandle = TZHANDLE_MAKE_LOCAL(server_id,
  1135. UHANDLE_GET_CB_OBJ(uhandle));
  1136. ret = 0;
  1137. } else if (UHANDLE_IS_FD(uhandle)) {
  1138. struct dma_buf *dma_buf = NULL;
  1139. struct smcinvoke_file_data *tzobj = NULL;
  1140. if (is_dma_fd(UHANDLE_GET_FD(uhandle), &dma_buf)) {
  1141. server_id = get_server_id(server_fd);
  1142. ret = create_mem_obj(dma_buf, tzhandle, &mem_obj, server_id, uhandle);
  1143. if (!ret && mem_obj_async_support && l_pending_mem_obj) {
  1144. mutex_lock(&g_smcinvoke_lock);
  1145. /* Map the newly created memory object and add it
  1146. * to l_pending_mem_obj list.
  1147. * Before returning to TZ, add the mapping data
  1148. * to the async side channel so it's available to TZ
  1149. * together with the memory object.
  1150. */
  1151. if (!smcinvoke_map_mem_region_locked(mem_obj)) {
  1152. queue_mem_obj_pending_async_locked(mem_obj, l_pending_mem_obj);
  1153. } else {
  1154. pr_err("Failed to map memory region\n");
  1155. }
  1156. mutex_unlock(&g_smcinvoke_lock);
  1157. }
  1158. } else if (is_remote_obj(UHANDLE_GET_FD(uhandle),
  1159. &tzobj, filp)) {
  1160. *tzhandle = tzobj->tzhandle;
  1161. ret = 0;
  1162. }
  1163. }
  1164. out:
  1165. return ret;
  1166. }
  1167. static int get_fd_for_obj(uint32_t obj_type, uint32_t obj, int32_t *fd)
  1168. {
  1169. int unused_fd = -1, ret = -EINVAL;
  1170. struct file *f = NULL;
  1171. struct smcinvoke_file_data *cxt = NULL;
  1172. cxt = kzalloc(sizeof(*cxt), GFP_KERNEL);
  1173. if (!cxt) {
  1174. ret = -ENOMEM;
  1175. goto out;
  1176. }
  1177. if (obj_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ ||
  1178. obj_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  1179. cxt->context_type = obj_type;
  1180. cxt->tzhandle = obj;
  1181. } else if (obj_type == SMCINVOKE_OBJ_TYPE_SERVER) {
  1182. cxt->context_type = SMCINVOKE_OBJ_TYPE_SERVER;
  1183. cxt->server_id = obj;
  1184. } else {
  1185. goto out;
  1186. }
  1187. unused_fd = get_unused_fd_flags(O_RDWR);
  1188. if (unused_fd < 0)
  1189. goto out;
  1190. if (fd == NULL)
  1191. goto out;
  1192. f = anon_inode_getfile(SMCINVOKE_DEV, &g_smcinvoke_fops, cxt, O_RDWR);
  1193. if (IS_ERR(f))
  1194. goto out;
  1195. *fd = unused_fd;
  1196. fd_install(*fd, f);
  1197. return 0;
  1198. out:
  1199. if (unused_fd >= 0)
  1200. put_unused_fd(unused_fd);
  1201. kfree(cxt);
  1202. return ret;
  1203. }
  1204. static int get_uhandle_from_tzhandle(int32_t tzhandle, int32_t srvr_id,
  1205. int32_t *uhandle, bool lock, uint32_t context_type)
  1206. {
  1207. int ret = -1;
  1208. if (TZHANDLE_IS_NULL(tzhandle)) {
  1209. *uhandle = UHANDLE_NULL;
  1210. ret = 0;
  1211. } else if (TZHANDLE_IS_CB_OBJ(tzhandle)) {
  1212. if (srvr_id != TZHANDLE_GET_SERVER(tzhandle))
  1213. goto out;
  1214. *uhandle = UHANDLE_MAKE_CB_OBJ(TZHANDLE_GET_OBJID(tzhandle));
  1215. MUTEX_LOCK(lock)
  1216. ret = get_pending_cbobj_locked(TZHANDLE_GET_SERVER(tzhandle),
  1217. TZHANDLE_GET_OBJID(tzhandle));
  1218. MUTEX_UNLOCK(lock)
  1219. } else if (TZHANDLE_IS_MEM_RGN_OBJ(tzhandle)) {
  1220. struct smcinvoke_mem_obj *mem_obj = NULL;
  1221. MUTEX_LOCK(lock)
  1222. mem_obj = find_mem_obj_locked(TZHANDLE_GET_OBJID(tzhandle),
  1223. SMCINVOKE_MEM_RGN_OBJ);
  1224. if (mem_obj != NULL) {
  1225. int fd;
  1226. fd = mem_obj->mem_obj_user_fd;
  1227. if (fd < 0)
  1228. goto exit_lock;
  1229. *uhandle = fd;
  1230. ret = 0;
  1231. }
  1232. exit_lock:
  1233. MUTEX_UNLOCK(lock)
  1234. } else if (TZHANDLE_IS_REMOTE(tzhandle)) {
  1235. /* if execution comes here => tzhandle is an unsigned int */
  1236. ret = get_fd_for_obj(context_type,
  1237. (uint32_t)tzhandle, uhandle);
  1238. }
  1239. out:
  1240. return ret;
  1241. }
  1242. static int32_t smcinvoke_release_mem_obj_locked(void *buf, size_t buf_len)
  1243. {
  1244. struct smcinvoke_tzcb_req *msg = buf;
  1245. if (msg->hdr.counts != OBJECT_COUNTS_PACK(0, 0, 0, 0)) {
  1246. pr_err("Invalid object count in %s\n", __func__);
  1247. return OBJECT_ERROR_INVALID;
  1248. }
  1249. trace_release_mem_obj_locked(msg->hdr.tzhandle, buf_len);
  1250. return release_tzhandle_locked(msg->hdr.tzhandle);
  1251. }
  1252. static int32_t smcinvoke_process_map_mem_region_req(void *buf, size_t buf_len)
  1253. {
  1254. int ret = OBJECT_OK;
  1255. struct smcinvoke_tzcb_req *msg = buf;
  1256. struct {
  1257. uint64_t p_addr;
  1258. uint64_t len;
  1259. uint32_t perms;
  1260. } *ob = NULL;
  1261. int32_t *oo = NULL;
  1262. struct smcinvoke_mem_obj *mem_obj = NULL;
  1263. if (msg->hdr.counts != OBJECT_COUNTS_PACK(0, 1, 1, 1) ||
  1264. (buf_len - msg->args[0].b.offset < msg->args[0].b.size)) {
  1265. pr_err("Invalid counts received for mapping mem obj\n");
  1266. return OBJECT_ERROR_INVALID;
  1267. }
  1268. /* args[0] = BO, args[1] = OI, args[2] = OO */
  1269. ob = buf + msg->args[0].b.offset;
  1270. oo = &msg->args[2].handle;
  1271. mutex_lock(&g_smcinvoke_lock);
  1272. mem_obj = find_mem_obj_locked(TZHANDLE_GET_OBJID(msg->args[1].handle),
  1273. SMCINVOKE_MEM_RGN_OBJ);
  1274. if (!mem_obj) {
  1275. mutex_unlock(&g_smcinvoke_lock);
  1276. pr_err("Memory object not found\n");
  1277. return OBJECT_ERROR_BADOBJ;
  1278. }
  1279. if (!mem_obj->p_addr) {
  1280. ret = smcinvoke_map_mem_region_locked(mem_obj);
  1281. } else {
  1282. kref_get(&mem_obj->mem_map_obj_ref_cnt);
  1283. }
  1284. if (!ret) {
  1285. ob->p_addr = mem_obj->p_addr;
  1286. ob->len = mem_obj->p_addr_len;
  1287. ob->perms = SMCINVOKE_MEM_PERM_RW;
  1288. *oo = TZHANDLE_MAKE_LOCAL(MEM_MAP_SRVR_ID, mem_obj->mem_map_obj_id);
  1289. }
  1290. mutex_unlock(&g_smcinvoke_lock);
  1291. return ret;
  1292. }
  1293. static int32_t smcinvoke_sleep(void *buf, size_t buf_len)
  1294. {
  1295. struct smcinvoke_tzcb_req *msg = buf;
  1296. uint32_t sleepTimeMs_val = 0;
  1297. if (msg->hdr.counts != OBJECT_COUNTS_PACK(1, 0, 0, 0) ||
  1298. (buf_len - msg->args[0].b.offset < msg->args[0].b.size)) {
  1299. pr_err("Invalid counts received for sleeping in hlos\n");
  1300. return OBJECT_ERROR_INVALID;
  1301. }
  1302. /* Time in miliseconds is expected from tz */
  1303. sleepTimeMs_val = *((uint32_t *)(buf + msg->args[0].b.offset));
  1304. msleep(sleepTimeMs_val);
  1305. return OBJECT_OK;
  1306. }
  1307. static void process_kernel_obj(void *buf, size_t buf_len)
  1308. {
  1309. struct smcinvoke_tzcb_req *cb_req = buf;
  1310. switch (cb_req->hdr.op) {
  1311. case OBJECT_OP_MAP_REGION:
  1312. pr_err("Received a request to map memory region\n");
  1313. cb_req->result = smcinvoke_process_map_mem_region_req(buf, buf_len);
  1314. break;
  1315. case OBJECT_OP_YIELD:
  1316. cb_req->result = OBJECT_OK;
  1317. break;
  1318. case OBJECT_OP_SLEEP:
  1319. cb_req->result = smcinvoke_sleep(buf, buf_len);
  1320. break;
  1321. default:
  1322. pr_err(" invalid operation for tz kernel object\n");
  1323. cb_req->result = OBJECT_ERROR_INVALID;
  1324. break;
  1325. }
  1326. }
  1327. static void process_mem_obj(void *buf, size_t buf_len)
  1328. {
  1329. struct smcinvoke_tzcb_req *cb_req = buf;
  1330. mutex_lock(&g_smcinvoke_lock);
  1331. cb_req->result = (cb_req->hdr.op == OBJECT_OP_RELEASE) ?
  1332. smcinvoke_release_mem_obj_locked(buf, buf_len) :
  1333. OBJECT_ERROR_INVALID;
  1334. mutex_unlock(&g_smcinvoke_lock);
  1335. }
  1336. static int invoke_cmd_handler(int cmd, phys_addr_t in_paddr, size_t in_buf_len,
  1337. uint8_t *out_buf, phys_addr_t out_paddr,
  1338. size_t out_buf_len, int32_t *result, u64 *response_type,
  1339. unsigned int *data, struct qtee_shm *in_shm,
  1340. struct qtee_shm *out_shm)
  1341. {
  1342. int ret = 0;
  1343. switch (cmd) {
  1344. case SMCINVOKE_INVOKE_CMD_LEGACY:
  1345. qtee_shmbridge_flush_shm_buf(in_shm);
  1346. qtee_shmbridge_flush_shm_buf(out_shm);
  1347. ret = qcom_scm_invoke_smc_legacy(in_paddr, in_buf_len, out_paddr, out_buf_len,
  1348. result, response_type, data);
  1349. qtee_shmbridge_inv_shm_buf(in_shm);
  1350. qtee_shmbridge_inv_shm_buf(out_shm);
  1351. break;
  1352. case SMCINVOKE_INVOKE_CMD:
  1353. ret = qcom_scm_invoke_smc(in_paddr, in_buf_len, out_paddr, out_buf_len,
  1354. result, response_type, data);
  1355. break;
  1356. case SMCINVOKE_CB_RSP_CMD:
  1357. if (legacy_smc_call)
  1358. qtee_shmbridge_flush_shm_buf(out_shm);
  1359. ret = qcom_scm_invoke_callback_response(virt_to_phys(out_buf), out_buf_len,
  1360. result, response_type, data);
  1361. if (legacy_smc_call) {
  1362. qtee_shmbridge_inv_shm_buf(in_shm);
  1363. qtee_shmbridge_inv_shm_buf(out_shm);
  1364. }
  1365. break;
  1366. default:
  1367. ret = -EINVAL;
  1368. break;
  1369. }
  1370. trace_invoke_cmd_handler(cmd, *response_type, *result, ret);
  1371. return ret;
  1372. }
  1373. /*
  1374. * Buf should be aligned to struct smcinvoke_tzcb_req
  1375. */
  1376. static void process_tzcb_req(void *buf, size_t buf_len, struct file **arr_filp)
  1377. {
  1378. /* ret is going to TZ. Provide values from OBJECT_ERROR_<> */
  1379. int ret = OBJECT_ERROR_DEFUNCT;
  1380. int cbobj_retries = 0;
  1381. long timeout_jiff;
  1382. bool wait_interrupted = false;
  1383. struct smcinvoke_cb_txn *cb_txn = NULL;
  1384. struct smcinvoke_tzcb_req *cb_req = NULL, *tmp_cb_req = NULL;
  1385. struct smcinvoke_server_info *srvr_info = NULL;
  1386. struct smcinvoke_mem_obj *mem_obj = NULL;
  1387. uint16_t server_id = 0;
  1388. if (buf_len < sizeof(struct smcinvoke_tzcb_req)) {
  1389. pr_err("smaller buffer length : %u\n", buf_len);
  1390. return;
  1391. }
  1392. cb_req = buf;
  1393. /* check whether it is to be served by kernel or userspace */
  1394. if (TZHANDLE_IS_KERNEL_OBJ(cb_req->hdr.tzhandle)) {
  1395. return process_kernel_obj(buf, buf_len);
  1396. } else if (TZHANDLE_IS_MEM_MAP_OBJ(cb_req->hdr.tzhandle)) {
  1397. /*
  1398. * MEM_MAP memory object is created and owned by kernel,
  1399. * hence its processing(handling deletion) is done in
  1400. * kernel context.
  1401. */
  1402. return process_mem_obj(buf, buf_len);
  1403. } else if(TZHANDLE_IS_MEM_RGN_OBJ(cb_req->hdr.tzhandle)) {
  1404. /*
  1405. * MEM_RGN memory objects are created and owned by userspace,
  1406. * and hence their deletion/handling requires going back to the
  1407. * userspace, similar to that of callback objects. If we enter
  1408. * this 'if' condition, its no-op here, and proceed similar to
  1409. * case of callback objects.
  1410. */
  1411. } else if (!TZHANDLE_IS_CB_OBJ(cb_req->hdr.tzhandle)) {
  1412. pr_err("Request object is not a callback object\n");
  1413. cb_req->result = OBJECT_ERROR_INVALID;
  1414. return;
  1415. }
  1416. /*
  1417. * We need a copy of req that could be sent to server. Otherwise, if
  1418. * someone kills invoke caller, buf would go away and server would be
  1419. * working on already freed buffer, causing a device crash.
  1420. */
  1421. tmp_cb_req = kmemdup(buf, buf_len, GFP_KERNEL);
  1422. if (!tmp_cb_req) {
  1423. /* we need to return error to caller so fill up result */
  1424. cb_req->result = OBJECT_ERROR_KMEM;
  1425. pr_err("failed to create copy of request, set result: %d\n",
  1426. cb_req->result);
  1427. return;
  1428. }
  1429. cb_txn = kzalloc(sizeof(*cb_txn), GFP_KERNEL);
  1430. if (!cb_txn) {
  1431. cb_req->result = OBJECT_ERROR_KMEM;
  1432. pr_err("failed to allocate memory for request, result: %d\n",
  1433. cb_req->result);
  1434. kfree(tmp_cb_req);
  1435. return;
  1436. }
  1437. /* no need for memcpy as we did kmemdup() above */
  1438. cb_req = tmp_cb_req;
  1439. trace_process_tzcb_req_handle(cb_req->hdr.tzhandle, cb_req->hdr.op, cb_req->hdr.counts);
  1440. cb_txn->state = SMCINVOKE_REQ_PLACED;
  1441. cb_txn->cb_req = cb_req;
  1442. cb_txn->cb_req_bytes = buf_len;
  1443. cb_txn->filp_to_release = arr_filp;
  1444. kref_init(&cb_txn->ref_cnt);
  1445. mutex_lock(&g_smcinvoke_lock);
  1446. ++cb_reqs_inflight;
  1447. if(TZHANDLE_IS_MEM_RGN_OBJ(cb_req->hdr.tzhandle)) {
  1448. mem_obj= find_mem_obj_locked(TZHANDLE_GET_OBJID(cb_req->hdr.tzhandle),SMCINVOKE_MEM_RGN_OBJ);
  1449. if(!mem_obj) {
  1450. pr_err("mem obj with tzhandle : %d not found",cb_req->hdr.tzhandle);
  1451. goto out;
  1452. }
  1453. server_id = mem_obj->server->server_id;
  1454. } else {
  1455. server_id = TZHANDLE_GET_SERVER(cb_req->hdr.tzhandle);
  1456. }
  1457. srvr_info = get_cb_server_locked(server_id);
  1458. if (!srvr_info || srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT) {
  1459. /* ret equals Object_ERROR_DEFUNCT, at this point go to out */
  1460. if (!srvr_info)
  1461. pr_err("server is invalid\n");
  1462. else {
  1463. pr_err("server is defunct, state= %d tzhandle = %d\n",
  1464. srvr_info->state, cb_req->hdr.tzhandle);
  1465. }
  1466. mutex_unlock(&g_smcinvoke_lock);
  1467. goto out;
  1468. }
  1469. cb_txn->txn_id = ++srvr_info->txn_id;
  1470. hash_add(srvr_info->reqs_table, &cb_txn->hash, cb_txn->txn_id);
  1471. mutex_unlock(&g_smcinvoke_lock);
  1472. trace_process_tzcb_req_wait(cb_req->hdr.tzhandle, cbobj_retries, cb_txn->txn_id,
  1473. current->pid, current->tgid, srvr_info->state, srvr_info->server_id,
  1474. cb_reqs_inflight);
  1475. /*
  1476. * we need not worry that server_info will be deleted because as long
  1477. * as this CBObj is served by this server, srvr_info will be valid.
  1478. */
  1479. wake_up_interruptible_all(&srvr_info->req_wait_q);
  1480. /* timeout before 1s otherwise tzbusy would come */
  1481. timeout_jiff = msecs_to_jiffies(100);
  1482. while (cbobj_retries < CBOBJ_MAX_RETRIES) {
  1483. if (wait_interrupted) {
  1484. ret = wait_event_timeout(srvr_info->rsp_wait_q,
  1485. (cb_txn->state == SMCINVOKE_REQ_PROCESSED) ||
  1486. (srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT),
  1487. timeout_jiff);
  1488. } else {
  1489. ret = wait_event_interruptible_timeout(srvr_info->rsp_wait_q,
  1490. (cb_txn->state == SMCINVOKE_REQ_PROCESSED) ||
  1491. (srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT),
  1492. timeout_jiff);
  1493. }
  1494. if (ret == 0) {
  1495. if (srvr_info->is_server_suspended == 0) {
  1496. pr_err("CBobj timed out waiting on cbtxn :%d,cb-tzhandle:%d, retry:%d, op:%d counts :%d\n",
  1497. cb_txn->txn_id,cb_req->hdr.tzhandle, cbobj_retries,
  1498. cb_req->hdr.op, cb_req->hdr.counts);
  1499. pr_err("CBobj %d timedout pid %x,tid %x, srvr state=%d, srvr id:%u\n",
  1500. cb_req->hdr.tzhandle, current->pid,
  1501. current->tgid, srvr_info->state,
  1502. srvr_info->server_id);
  1503. }
  1504. } else {
  1505. /* wait_event returned due to a signal */
  1506. if (srvr_info->state != SMCINVOKE_SERVER_STATE_DEFUNCT &&
  1507. cb_txn->state != SMCINVOKE_REQ_PROCESSED) {
  1508. wait_interrupted = true;
  1509. } else {
  1510. break;
  1511. }
  1512. }
  1513. /*
  1514. * If bit corresponding to any accept thread is set, invoke threads
  1515. * should wait infinitely for the accept thread to come back with
  1516. * response.
  1517. */
  1518. if (srvr_info->is_server_suspended > 0) {
  1519. cbobj_retries = 0;
  1520. } else {
  1521. cbobj_retries++;
  1522. }
  1523. }
  1524. out:
  1525. /*
  1526. * we could be here because of either:
  1527. * a. Req is PROCESSED
  1528. * b. Server was killed
  1529. * c. Invoke thread is killed
  1530. * sometime invoke thread and server are part of same process.
  1531. */
  1532. mutex_lock(&g_smcinvoke_lock);
  1533. hash_del(&cb_txn->hash);
  1534. if (ret == 0) {
  1535. pr_err("CBObj timed out! No more retries\n");
  1536. cb_req->result = Object_ERROR_TIMEOUT;
  1537. } else if (ret == -ERESTARTSYS) {
  1538. pr_err("wait event interruped, ret: %d\n", ret);
  1539. cb_req->result = OBJECT_ERROR_ABORT;
  1540. } else {
  1541. if (cb_txn->state == SMCINVOKE_REQ_PROCESSED) {
  1542. /*
  1543. * it is possible that server was killed immediately
  1544. * after CB Req was processed but who cares now!
  1545. */
  1546. } else if (!srvr_info ||
  1547. srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT) {
  1548. cb_req->result = OBJECT_ERROR_DEFUNCT;
  1549. pr_err("server invalid, res: %d\n", cb_req->result);
  1550. } else {
  1551. pr_err("%s: unexpected event happened, ret:%d\n", __func__, ret);
  1552. cb_req->result = OBJECT_ERROR_ABORT;
  1553. }
  1554. }
  1555. --cb_reqs_inflight;
  1556. trace_process_tzcb_req_result(cb_req->result, cb_req->hdr.tzhandle, cb_req->hdr.op,
  1557. cb_req->hdr.counts, cb_reqs_inflight);
  1558. memcpy(buf, cb_req, buf_len);
  1559. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  1560. if (srvr_info)
  1561. kref_put(&srvr_info->ref_cnt, destroy_cb_server);
  1562. mutex_unlock(&g_smcinvoke_lock);
  1563. }
  1564. static int marshal_out_invoke_req(const uint8_t *buf, uint32_t buf_size,
  1565. struct smcinvoke_cmd_req *req,
  1566. union smcinvoke_arg *args_buf,
  1567. uint32_t context_type)
  1568. {
  1569. int ret = -EINVAL, i = 0;
  1570. int32_t temp_fd = UHANDLE_NULL;
  1571. union smcinvoke_tz_args *tz_args = NULL;
  1572. size_t offset = sizeof(struct smcinvoke_msg_hdr) +
  1573. OBJECT_COUNTS_TOTAL(req->counts) *
  1574. sizeof(union smcinvoke_tz_args);
  1575. if (offset > buf_size)
  1576. goto out;
  1577. tz_args = (union smcinvoke_tz_args *)
  1578. (buf + sizeof(struct smcinvoke_msg_hdr));
  1579. tz_args += OBJECT_COUNTS_NUM_BI(req->counts);
  1580. if (args_buf == NULL)
  1581. return 0;
  1582. FOR_ARGS(i, req->counts, BO) {
  1583. args_buf[i].b.size = tz_args->b.size;
  1584. if ((buf_size - tz_args->b.offset < tz_args->b.size) ||
  1585. tz_args->b.offset > buf_size) {
  1586. pr_err("%s: buffer overflow detected\n", __func__);
  1587. goto out;
  1588. }
  1589. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  1590. if (copy_to_user((void __user *)
  1591. (uintptr_t)(args_buf[i].b.addr),
  1592. (uint8_t *)(buf) + tz_args->b.offset,
  1593. tz_args->b.size)) {
  1594. pr_err("Error %d copying ctxt to user\n", ret);
  1595. goto out;
  1596. }
  1597. } else {
  1598. memcpy((uint8_t *)(args_buf[i].b.addr),
  1599. (uint8_t *)(buf) + tz_args->b.offset,
  1600. tz_args->b.size);
  1601. }
  1602. tz_args++;
  1603. }
  1604. tz_args += OBJECT_COUNTS_NUM_OI(req->counts);
  1605. FOR_ARGS(i, req->counts, OO) {
  1606. /*
  1607. * create a new FD and assign to output object's context.
  1608. * We are passing cb_server_fd from output param in case OO
  1609. * is a CBObj. For CBObj, we have to ensure that it is sent
  1610. * to server who serves it and that info comes from USpace.
  1611. */
  1612. temp_fd = UHANDLE_NULL;
  1613. ret = get_uhandle_from_tzhandle(tz_args->handle,
  1614. TZHANDLE_GET_SERVER(tz_args->handle),
  1615. &temp_fd, NO_LOCK, context_type);
  1616. args_buf[i].o.fd = temp_fd;
  1617. if (ret)
  1618. goto out;
  1619. trace_marshal_out_invoke_req(i, tz_args->handle,
  1620. TZHANDLE_GET_SERVER(tz_args->handle), temp_fd);
  1621. tz_args++;
  1622. }
  1623. ret = 0;
  1624. out:
  1625. return ret;
  1626. }
  1627. static bool is_inbound_req(int val)
  1628. {
  1629. return (val == SMCINVOKE_RESULT_INBOUND_REQ_NEEDED ||
  1630. val == QSEOS_RESULT_INCOMPLETE ||
  1631. val == QSEOS_RESULT_BLOCKED_ON_LISTENER);
  1632. }
  1633. static int prepare_send_scm_msg(const uint8_t *in_buf, phys_addr_t in_paddr,
  1634. size_t in_buf_len,
  1635. uint8_t *out_buf, phys_addr_t out_paddr,
  1636. size_t out_buf_len,
  1637. struct smcinvoke_cmd_req *req,
  1638. union smcinvoke_arg *args_buf,
  1639. bool *tz_acked, uint32_t context_type,
  1640. struct qtee_shm *in_shm, struct qtee_shm *out_shm)
  1641. {
  1642. int ret = 0, cmd, retry_count = 0;
  1643. u64 response_type;
  1644. unsigned int data;
  1645. struct file *arr_filp[OBJECT_COUNTS_MAX_OO] = {NULL};
  1646. *tz_acked = false;
  1647. /* buf size should be page aligned */
  1648. if ((in_buf_len % PAGE_SIZE) != 0 || (out_buf_len % PAGE_SIZE) != 0)
  1649. return -EINVAL;
  1650. cmd = invoke_cmd;
  1651. while (1) {
  1652. do {
  1653. ret = invoke_cmd_handler(cmd, in_paddr, in_buf_len, out_buf,
  1654. out_paddr, out_buf_len, &req->result,
  1655. &response_type, &data, in_shm, out_shm);
  1656. if (ret == -EBUSY) {
  1657. pr_err_ratelimited("Secure side is busy,will retry after 30 ms, retry_count = %d\n",retry_count);
  1658. msleep(SMCINVOKE_SCM_EBUSY_WAIT_MS);
  1659. }
  1660. } while ((ret == -EBUSY) &&
  1661. (retry_count++ < SMCINVOKE_SCM_EBUSY_MAX_RETRY));
  1662. if (!ret && !is_inbound_req(response_type)) {
  1663. /* dont marshal if Obj returns an error */
  1664. if (!req->result) {
  1665. if (args_buf != NULL)
  1666. ret = marshal_out_invoke_req(in_buf,
  1667. in_buf_len, req, args_buf,
  1668. context_type);
  1669. }
  1670. *tz_acked = true;
  1671. }
  1672. if (cmd == SMCINVOKE_CB_RSP_CMD)
  1673. release_filp(arr_filp, OBJECT_COUNTS_MAX_OO);
  1674. if (ret || !is_inbound_req(response_type))
  1675. break;
  1676. /* process listener request */
  1677. if (response_type == QSEOS_RESULT_INCOMPLETE ||
  1678. response_type == QSEOS_RESULT_BLOCKED_ON_LISTENER) {
  1679. ret = qseecom_process_listener_from_smcinvoke(
  1680. &req->result, &response_type, &data);
  1681. trace_prepare_send_scm_msg(response_type, req->result);
  1682. if (!req->result &&
  1683. response_type != SMCINVOKE_RESULT_INBOUND_REQ_NEEDED) {
  1684. ret = marshal_out_invoke_req(in_buf,
  1685. in_buf_len, req, args_buf,
  1686. context_type);
  1687. }
  1688. *tz_acked = true;
  1689. }
  1690. /*
  1691. * qseecom does not understand smcinvoke's callback object &&
  1692. * erringly sets ret value as -EINVAL :( We need to handle it.
  1693. */
  1694. if (response_type != SMCINVOKE_RESULT_INBOUND_REQ_NEEDED)
  1695. break;
  1696. if (response_type == SMCINVOKE_RESULT_INBOUND_REQ_NEEDED) {
  1697. trace_status(__func__, "looks like inbnd req reqd");
  1698. process_tzcb_req(out_buf, out_buf_len, arr_filp);
  1699. cmd = SMCINVOKE_CB_RSP_CMD;
  1700. }
  1701. }
  1702. return ret;
  1703. }
  1704. /*
  1705. * SMC expects arguments in following format
  1706. * ---------------------------------------------------------------------------
  1707. * | cxt | op | counts | ptr|size |ptr|size...|ORef|ORef|...| rest of payload |
  1708. * ---------------------------------------------------------------------------
  1709. * cxt: target, op: operation, counts: total arguments
  1710. * offset: offset is from beginning of buffer i.e. cxt
  1711. * size: size is 8 bytes aligned value
  1712. */
  1713. static size_t compute_in_msg_size(const struct smcinvoke_cmd_req *req,
  1714. const union smcinvoke_arg *args_buf)
  1715. {
  1716. uint32_t i = 0;
  1717. size_t total_size = sizeof(struct smcinvoke_msg_hdr) +
  1718. OBJECT_COUNTS_TOTAL(req->counts) *
  1719. sizeof(union smcinvoke_tz_args);
  1720. /* Computed total_size should be 8 bytes aligned from start of buf */
  1721. total_size = ALIGN(total_size, SMCINVOKE_ARGS_ALIGN_SIZE);
  1722. /* each buffer has to be 8 bytes aligned */
  1723. while (i < OBJECT_COUNTS_NUM_buffers(req->counts))
  1724. total_size = size_add_(total_size,
  1725. size_align(args_buf[i++].b.size,
  1726. SMCINVOKE_ARGS_ALIGN_SIZE));
  1727. return PAGE_ALIGN(total_size);
  1728. }
  1729. static int marshal_in_invoke_req(const struct smcinvoke_cmd_req *req,
  1730. const union smcinvoke_arg *args_buf, uint32_t tzhandle,
  1731. uint8_t *buf, size_t buf_size, struct file **arr_filp,
  1732. int32_t *tzhandles_to_release, uint32_t context_type,
  1733. struct list_head *l_pending_mem_obj)
  1734. {
  1735. int ret = -EINVAL, i = 0, j = 0, k = 0;
  1736. const struct smcinvoke_msg_hdr msg_hdr = {
  1737. tzhandle, req->op, req->counts};
  1738. uint32_t offset = sizeof(struct smcinvoke_msg_hdr) +
  1739. sizeof(union smcinvoke_tz_args) *
  1740. OBJECT_COUNTS_TOTAL(req->counts);
  1741. union smcinvoke_tz_args *tz_args = NULL;
  1742. if (buf_size < offset)
  1743. goto out;
  1744. *(struct smcinvoke_msg_hdr *)buf = msg_hdr;
  1745. tz_args = (union smcinvoke_tz_args *)(buf +
  1746. sizeof(struct smcinvoke_msg_hdr));
  1747. if (args_buf == NULL)
  1748. return 0;
  1749. FOR_ARGS(i, req->counts, BI) {
  1750. offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
  1751. if ((offset > buf_size) ||
  1752. (args_buf[i].b.size > (buf_size - offset)))
  1753. goto out;
  1754. tz_args[i].b.offset = offset;
  1755. tz_args[i].b.size = args_buf[i].b.size;
  1756. if (context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  1757. if (copy_from_user(buf + offset,
  1758. (void __user *)(uintptr_t)(args_buf[i].b.addr),
  1759. args_buf[i].b.size))
  1760. goto out;
  1761. } else {
  1762. memcpy(buf + offset, (void *)(args_buf[i].b.addr),
  1763. args_buf[i].b.size);
  1764. }
  1765. offset += args_buf[i].b.size;
  1766. }
  1767. FOR_ARGS(i, req->counts, BO) {
  1768. offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
  1769. if ((offset > buf_size) ||
  1770. (args_buf[i].b.size > (buf_size - offset)))
  1771. goto out;
  1772. tz_args[i].b.offset = offset;
  1773. tz_args[i].b.size = args_buf[i].b.size;
  1774. offset += args_buf[i].b.size;
  1775. }
  1776. FOR_ARGS(i, req->counts, OI) {
  1777. ret = get_tzhandle_from_uhandle(args_buf[i].o.fd,
  1778. args_buf[i].o.cb_server_fd, &arr_filp[j++],
  1779. &(tz_args[i].handle), l_pending_mem_obj);
  1780. if (ret)
  1781. goto out;
  1782. trace_marshal_in_invoke_req(i, args_buf[i].o.fd,
  1783. args_buf[i].o.cb_server_fd, tz_args[i].handle);
  1784. tzhandles_to_release[k++] = tz_args[i].handle;
  1785. }
  1786. ret = 0;
  1787. out:
  1788. return ret;
  1789. }
  1790. static int marshal_in_tzcb_req(const struct smcinvoke_cb_txn *cb_txn,
  1791. struct smcinvoke_accept *user_req, int srvr_id)
  1792. {
  1793. int ret = 0, i = 0;
  1794. int32_t temp_fd = UHANDLE_NULL;
  1795. union smcinvoke_arg tmp_arg;
  1796. struct smcinvoke_tzcb_req *tzcb_req = cb_txn->cb_req;
  1797. union smcinvoke_tz_args *tz_args = tzcb_req->args;
  1798. size_t tzcb_req_len = cb_txn->cb_req_bytes;
  1799. size_t tz_buf_offset = TZCB_BUF_OFFSET(tzcb_req);
  1800. size_t user_req_buf_offset = sizeof(union smcinvoke_arg) *
  1801. OBJECT_COUNTS_TOTAL(tzcb_req->hdr.counts);
  1802. if (tz_buf_offset > tzcb_req_len) {
  1803. ret = -EINVAL;
  1804. goto out;
  1805. }
  1806. user_req->txn_id = cb_txn->txn_id;
  1807. if (get_uhandle_from_tzhandle(tzcb_req->hdr.tzhandle, srvr_id,
  1808. &user_req->cbobj_id, TAKE_LOCK,
  1809. SMCINVOKE_OBJ_TYPE_TZ_OBJ)) {
  1810. ret = -EINVAL;
  1811. goto out;
  1812. }
  1813. user_req->op = tzcb_req->hdr.op;
  1814. user_req->counts = tzcb_req->hdr.counts;
  1815. user_req->argsize = sizeof(union smcinvoke_arg);
  1816. trace_marshal_in_tzcb_req_handle(tzcb_req->hdr.tzhandle, srvr_id,
  1817. user_req->cbobj_id, user_req->op, user_req->counts);
  1818. FOR_ARGS(i, tzcb_req->hdr.counts, BI) {
  1819. user_req_buf_offset = size_align(user_req_buf_offset,
  1820. SMCINVOKE_ARGS_ALIGN_SIZE);
  1821. tmp_arg.b.size = tz_args[i].b.size;
  1822. if ((tz_args[i].b.offset > tzcb_req_len) ||
  1823. (tz_args[i].b.size > tzcb_req_len - tz_args[i].b.offset) ||
  1824. (user_req_buf_offset > user_req->buf_len) ||
  1825. (tmp_arg.b.size >
  1826. user_req->buf_len - user_req_buf_offset)) {
  1827. ret = -EINVAL;
  1828. pr_err("%s: buffer overflow detected\n", __func__);
  1829. goto out;
  1830. }
  1831. tmp_arg.b.addr = user_req->buf_addr + user_req_buf_offset;
  1832. if (copy_to_user(u64_to_user_ptr
  1833. (user_req->buf_addr + i * sizeof(tmp_arg)),
  1834. &tmp_arg, sizeof(tmp_arg)) ||
  1835. copy_to_user(u64_to_user_ptr(tmp_arg.b.addr),
  1836. (uint8_t *)(tzcb_req) + tz_args[i].b.offset,
  1837. tz_args[i].b.size)) {
  1838. ret = -EFAULT;
  1839. goto out;
  1840. }
  1841. user_req_buf_offset += tmp_arg.b.size;
  1842. }
  1843. FOR_ARGS(i, tzcb_req->hdr.counts, BO) {
  1844. user_req_buf_offset = size_align(user_req_buf_offset,
  1845. SMCINVOKE_ARGS_ALIGN_SIZE);
  1846. tmp_arg.b.size = tz_args[i].b.size;
  1847. if ((user_req_buf_offset > user_req->buf_len) ||
  1848. (tmp_arg.b.size >
  1849. user_req->buf_len - user_req_buf_offset)) {
  1850. ret = -EINVAL;
  1851. pr_err("%s: buffer overflow detected\n", __func__);
  1852. goto out;
  1853. }
  1854. tmp_arg.b.addr = user_req->buf_addr + user_req_buf_offset;
  1855. if (copy_to_user(u64_to_user_ptr
  1856. (user_req->buf_addr + i * sizeof(tmp_arg)),
  1857. &tmp_arg, sizeof(tmp_arg))) {
  1858. ret = -EFAULT;
  1859. goto out;
  1860. }
  1861. user_req_buf_offset += tmp_arg.b.size;
  1862. }
  1863. FOR_ARGS(i, tzcb_req->hdr.counts, OI) {
  1864. /*
  1865. * create a new FD and assign to output object's
  1866. * context
  1867. */
  1868. temp_fd = UHANDLE_NULL;
  1869. ret = get_uhandle_from_tzhandle(tz_args[i].handle, srvr_id,
  1870. &temp_fd, TAKE_LOCK, SMCINVOKE_OBJ_TYPE_TZ_OBJ);
  1871. tmp_arg.o.fd = temp_fd;
  1872. if (ret) {
  1873. ret = -EINVAL;
  1874. goto out;
  1875. }
  1876. if (copy_to_user(u64_to_user_ptr
  1877. (user_req->buf_addr + i * sizeof(tmp_arg)),
  1878. &tmp_arg, sizeof(tmp_arg))) {
  1879. ret = -EFAULT;
  1880. goto out;
  1881. }
  1882. trace_marshal_in_tzcb_req_fd(i, tz_args[i].handle, srvr_id, temp_fd);
  1883. }
  1884. out:
  1885. return ret;
  1886. }
  1887. static int marshal_out_tzcb_req(const struct smcinvoke_accept *user_req,
  1888. struct smcinvoke_cb_txn *cb_txn,
  1889. struct file **arr_filp)
  1890. {
  1891. int ret = -EINVAL, i = 0;
  1892. int32_t tzhandles_to_release[OBJECT_COUNTS_MAX_OO] = {0};
  1893. struct smcinvoke_tzcb_req *tzcb_req = cb_txn->cb_req;
  1894. union smcinvoke_tz_args *tz_args = tzcb_req->args;
  1895. size_t tz_buf_offset = TZCB_BUF_OFFSET(tzcb_req);
  1896. LIST_HEAD(l_mem_objs_pending_async); /* Holds new memory objects, to be later sent to TZ */
  1897. uint32_t max_offset = 0;
  1898. uint32_t buffer_size_max_offset = 0;
  1899. void* async_buf_begin;
  1900. size_t async_buf_size;
  1901. uint32_t offset = 0;
  1902. /* We assume 'marshal_in_tzcb_req' increases the ref-count of the CBOBJs.
  1903. * It should be the case for mem-obj as well. However, it does not do that.
  1904. * It is easier to filter out the 'release_tzhandles' for mem-obj here rather
  1905. * than increases its ref-count on 'marshal_in_tzcb_req'. Because, there is no
  1906. * reliable error handling and cleanup in 'marshal_in_tzcb_req'. So if it fails
  1907. * mem-obj may not get released. **/
  1908. if (!TZHANDLE_IS_MEM_OBJ(cb_txn->cb_req->hdr.tzhandle))
  1909. release_tzhandles(&cb_txn->cb_req->hdr.tzhandle, 1);
  1910. tzcb_req->result = user_req->result;
  1911. /* Return without marshaling user args if destination callback invocation was
  1912. unsuccessful. */
  1913. if (tzcb_req->result != 0) {
  1914. ret = 0;
  1915. goto out;
  1916. }
  1917. FOR_ARGS(i, tzcb_req->hdr.counts, BI) {
  1918. /* Find the max offset and the size of the buffer in that offset */
  1919. if (tz_args[i].b.offset > max_offset) {
  1920. max_offset = tz_args[i].b.offset;
  1921. buffer_size_max_offset = tz_args[i].b.size;
  1922. }
  1923. }
  1924. FOR_ARGS(i, tzcb_req->hdr.counts, BO) {
  1925. union smcinvoke_arg tmp_arg;
  1926. if (copy_from_user((uint8_t *)&tmp_arg, u64_to_user_ptr(
  1927. user_req->buf_addr + i * sizeof(union smcinvoke_arg)),
  1928. sizeof(union smcinvoke_arg))) {
  1929. ret = -EFAULT;
  1930. goto out;
  1931. }
  1932. if (tmp_arg.b.size > tz_args[i].b.size)
  1933. goto out;
  1934. if (copy_from_user((uint8_t *)(tzcb_req) + tz_args[i].b.offset,
  1935. u64_to_user_ptr(tmp_arg.b.addr),
  1936. tmp_arg.b.size)) {
  1937. ret = -EFAULT;
  1938. goto out;
  1939. }
  1940. /* Find the max offset and the size of the buffer in that offset */
  1941. if (tz_args[i].b.offset > max_offset) {
  1942. max_offset = tz_args[i].b.offset;
  1943. buffer_size_max_offset = tz_args[i].b.size;
  1944. }
  1945. }
  1946. FOR_ARGS(i, tzcb_req->hdr.counts, OO) {
  1947. union smcinvoke_arg tmp_arg;
  1948. if (copy_from_user((uint8_t *)&tmp_arg, u64_to_user_ptr(
  1949. user_req->buf_addr + i * sizeof(union smcinvoke_arg)),
  1950. sizeof(union smcinvoke_arg))) {
  1951. ret = -EFAULT;
  1952. goto out;
  1953. }
  1954. ret = get_tzhandle_from_uhandle(tmp_arg.o.fd,
  1955. tmp_arg.o.cb_server_fd, &arr_filp[i],
  1956. &(tz_args[i].handle), &l_mem_objs_pending_async);
  1957. if (ret)
  1958. goto out;
  1959. tzhandles_to_release[i] = tz_args[i].handle;
  1960. trace_marshal_out_tzcb_req(i, tmp_arg.o.fd,
  1961. tmp_arg.o.cb_server_fd, tz_args[i].handle);
  1962. }
  1963. ret = 0;
  1964. out:
  1965. FOR_ARGS(i, tzcb_req->hdr.counts, OI) {
  1966. if (TZHANDLE_IS_CB_OBJ(tz_args[i].handle))
  1967. release_tzhandles(&tz_args[i].handle, 1);
  1968. }
  1969. do {
  1970. if (mem_obj_async_support) {
  1971. /* We will be able to add the async information to the buffer beyond the data in the max offset, if exists.
  1972. * If doesn't exist, we can add the async information after the header and the args. */
  1973. offset = (max_offset ? (max_offset + buffer_size_max_offset) : tz_buf_offset);
  1974. offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
  1975. async_buf_begin = (uint8_t *)tzcb_req + offset;
  1976. if (async_buf_begin - (void *)tzcb_req > g_max_cb_buf_size) {
  1977. pr_err("Unable to add memory object info to the async channel\n");
  1978. break;
  1979. } else {
  1980. async_buf_size = g_max_cb_buf_size - (async_buf_begin - (void *)tzcb_req);
  1981. }
  1982. mutex_lock(&g_smcinvoke_lock);
  1983. add_mem_obj_info_to_async_side_channel_locked(async_buf_begin, async_buf_size, &l_mem_objs_pending_async);
  1984. delete_pending_async_list_locked(&l_mem_objs_pending_async);
  1985. mutex_unlock(&g_smcinvoke_lock);
  1986. }
  1987. } while (0);
  1988. if (ret)
  1989. release_tzhandles(tzhandles_to_release, OBJECT_COUNTS_MAX_OO);
  1990. return ret;
  1991. }
  1992. static void set_tz_version (uint32_t tz_version)
  1993. {
  1994. tz_async_version = tz_version;
  1995. /* We enable async memory object support when TZ async
  1996. * version is equal or larger than the driver version.
  1997. * It is expected that if the protocol changes in later
  1998. * TZ versions, TZ will support backward compatibility
  1999. * so this condition should still be valid.
  2000. */
  2001. if (tz_version >= SMCINVOKE_ASYNC_VERSION) {
  2002. mem_obj_async_support = true;
  2003. pr_debug("Enabled asynchronous memory object support\n");
  2004. }
  2005. }
  2006. static void process_piggyback_data(void *buf, size_t buf_size)
  2007. {
  2008. int i;
  2009. struct smcinvoke_tzcb_req req = {0};
  2010. struct smcinvoke_piggyback_msg *msg = buf;
  2011. int32_t *objs = msg->objs;
  2012. for (i = 0; i < msg->counts; i++) {
  2013. req.hdr.op = msg->op;
  2014. req.hdr.counts = 0; /* release op does not require any args */
  2015. req.hdr.tzhandle = objs[i];
  2016. if (tz_async_version == 0)
  2017. set_tz_version(msg->version);
  2018. process_tzcb_req(&req, sizeof(struct smcinvoke_tzcb_req), NULL);
  2019. /* cbobjs_in_flight will be adjusted during CB processing */
  2020. }
  2021. }
  2022. /* Add memory object mapped data to the async side channel, so it's available to TZ
  2023. * together with the memory object.
  2024. *
  2025. * No return value as TZ is always able to explicitly ask for this information
  2026. * in case this function fails.
  2027. */
  2028. static void add_mem_obj_info_to_async_side_channel_locked(void *buf, size_t buf_size, struct list_head *l_pending_mem_obj)
  2029. {
  2030. struct smcinvoke_mem_obj_msg *msg = buf;
  2031. struct smcinvoke_mem_obj_pending_async *mem_obj_pending = NULL;
  2032. size_t header_size = 0;
  2033. size_t mo_size = 0;
  2034. size_t used = 0;
  2035. size_t index = 0;
  2036. if (list_empty(l_pending_mem_obj))
  2037. return;
  2038. header_size = sizeof(struct smcinvoke_mem_obj_msg);
  2039. mo_size = sizeof(struct smcinvoke_mem_obj_info);
  2040. /* Minimal size required is the header data + one mem obj info */
  2041. if (buf_size < header_size + mo_size) {
  2042. pr_err("Unable to add memory object info to async channel\n");
  2043. return;
  2044. }
  2045. msg->version = SMCINVOKE_ASYNC_VERSION;
  2046. msg->op = SMCINVOKE_ASYNC_OP_MEMORY_OBJECT;
  2047. msg->count = 0;
  2048. used = header_size;
  2049. index = 0;
  2050. list_for_each_entry(mem_obj_pending, l_pending_mem_obj, list) {
  2051. if (NULL == mem_obj_pending->mem_obj) {
  2052. pr_err("Memory object is no longer valid\n");
  2053. continue;
  2054. }
  2055. if (used + mo_size > buf_size) {
  2056. pr_err("Not all memory object info was added to the async channel\n");
  2057. break;
  2058. }
  2059. msg->mo[index].memObjRef = TZHANDLE_MAKE_LOCAL(MEM_RGN_SRVR_ID, mem_obj_pending->mem_obj->mem_region_id);
  2060. msg->mo[index].mapObjRef = TZHANDLE_MAKE_LOCAL(MEM_MAP_SRVR_ID, mem_obj_pending->mem_obj->mem_map_obj_id);
  2061. msg->mo[index].addr = mem_obj_pending->mem_obj->p_addr;
  2062. msg->mo[index].size = mem_obj_pending->mem_obj->p_addr_len;
  2063. msg->mo[index].perm = SMCINVOKE_MEM_PERM_RW;
  2064. used += sizeof(msg->mo[index]);
  2065. index++;
  2066. }
  2067. msg->count = index;
  2068. pr_debug("Added %d memory objects to the side channel, total size = %d\n", index, used);
  2069. return;
  2070. }
  2071. /*
  2072. * Delete entire pending async list.
  2073. */
  2074. static void delete_pending_async_list_locked(struct list_head *l_pending_mem_obj)
  2075. {
  2076. struct smcinvoke_mem_obj_pending_async *mem_obj_pending = NULL;
  2077. struct smcinvoke_mem_obj_pending_async *temp = NULL;
  2078. if (list_empty(l_pending_mem_obj))
  2079. return;
  2080. list_for_each_entry_safe(mem_obj_pending, temp, l_pending_mem_obj, list) {
  2081. mem_obj_pending->mem_obj = NULL;
  2082. list_del(&mem_obj_pending->list);
  2083. kfree(mem_obj_pending);
  2084. }
  2085. }
  2086. /*
  2087. * Unmap/release the mapped objects from pending async list.
  2088. */
  2089. static void release_map_obj_pending_async_list_locked(struct list_head *l_pending_mem_obj)
  2090. {
  2091. struct smcinvoke_mem_obj_pending_async *mem_obj_pending = NULL;
  2092. struct smcinvoke_mem_obj_pending_async *temp = NULL;
  2093. if (list_empty(l_pending_mem_obj))
  2094. return;
  2095. list_for_each_entry_safe(mem_obj_pending, temp, l_pending_mem_obj, list) {
  2096. kref_put(&mem_obj_pending->mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  2097. }
  2098. }
  2099. static long process_ack_local_obj(struct file *filp, unsigned int cmd,
  2100. unsigned long arg)
  2101. {
  2102. int ret = -1;
  2103. int32_t local_obj = SMCINVOKE_USERSPACE_OBJ_NULL;
  2104. struct smcinvoke_file_data *filp_data = filp->private_data;
  2105. if (_IOC_SIZE(cmd) != sizeof(int32_t))
  2106. return -EINVAL;
  2107. ret = copy_from_user(&local_obj, (void __user *)(uintptr_t)arg,
  2108. sizeof(int32_t));
  2109. if (ret)
  2110. return -EFAULT;
  2111. mutex_lock(&g_smcinvoke_lock);
  2112. if (UHANDLE_IS_CB_OBJ(local_obj))
  2113. ret = put_pending_cbobj_locked(filp_data->server_id,
  2114. UHANDLE_GET_CB_OBJ(local_obj));
  2115. mutex_unlock(&g_smcinvoke_lock);
  2116. return ret;
  2117. }
  2118. static long process_server_req(struct file *filp, unsigned int cmd,
  2119. unsigned long arg)
  2120. {
  2121. int ret = -1;
  2122. int32_t server_fd = -1;
  2123. struct smcinvoke_server server_req = {0};
  2124. struct smcinvoke_server_info *server_info = NULL;
  2125. if (_IOC_SIZE(cmd) != sizeof(server_req)) {
  2126. pr_err("invalid command size received for server request\n");
  2127. return -EINVAL;
  2128. }
  2129. ret = copy_from_user(&server_req, (void __user *)(uintptr_t)arg,
  2130. sizeof(server_req));
  2131. if (ret) {
  2132. pr_err("copying server request from user failed\n");
  2133. return -EFAULT;
  2134. }
  2135. server_info = kzalloc(sizeof(*server_info), GFP_KERNEL);
  2136. if (!server_info)
  2137. return -ENOMEM;
  2138. kref_init(&server_info->ref_cnt);
  2139. init_waitqueue_head(&server_info->req_wait_q);
  2140. init_waitqueue_head(&server_info->rsp_wait_q);
  2141. server_info->cb_buf_size = server_req.cb_buf_size;
  2142. hash_init(server_info->reqs_table);
  2143. hash_init(server_info->responses_table);
  2144. INIT_LIST_HEAD(&server_info->pending_cbobjs);
  2145. server_info->is_server_suspended = 0;
  2146. mutex_lock(&g_smcinvoke_lock);
  2147. server_info->server_id = next_cb_server_id_locked();
  2148. hash_add(g_cb_servers, &server_info->hash,
  2149. server_info->server_id);
  2150. if (g_max_cb_buf_size < server_req.cb_buf_size)
  2151. g_max_cb_buf_size = server_req.cb_buf_size;
  2152. mutex_unlock(&g_smcinvoke_lock);
  2153. ret = get_fd_for_obj(SMCINVOKE_OBJ_TYPE_SERVER,
  2154. server_info->server_id, &server_fd);
  2155. if (ret)
  2156. release_cb_server(server_info->server_id);
  2157. return server_fd;
  2158. }
  2159. static long process_accept_req(struct file *filp, unsigned int cmd,
  2160. unsigned long arg)
  2161. {
  2162. int ret = -1;
  2163. struct smcinvoke_file_data *server_obj = filp->private_data;
  2164. struct smcinvoke_accept user_args = {0};
  2165. struct smcinvoke_cb_txn *cb_txn = NULL;
  2166. struct smcinvoke_server_info *server_info = NULL;
  2167. if (_IOC_SIZE(cmd) != sizeof(struct smcinvoke_accept)) {
  2168. pr_err("command size invalid for accept request\n");
  2169. return -EINVAL;
  2170. }
  2171. if (copy_from_user(&user_args, (void __user *)arg,
  2172. sizeof(struct smcinvoke_accept))) {
  2173. pr_err("copying accept request from user failed\n");
  2174. return -EFAULT;
  2175. }
  2176. if (user_args.argsize != sizeof(union smcinvoke_arg)) {
  2177. pr_err("arguments size is invalid for accept thread\n");
  2178. return -EINVAL;
  2179. }
  2180. /* ACCEPT is available only on server obj */
  2181. if (server_obj->context_type != SMCINVOKE_OBJ_TYPE_SERVER) {
  2182. pr_err("invalid object type received for accept req\n");
  2183. return -EPERM;
  2184. }
  2185. mutex_lock(&g_smcinvoke_lock);
  2186. server_info = get_cb_server_locked(server_obj->server_id);
  2187. if (!server_info) {
  2188. pr_err("No matching server with server id : %u found\n",
  2189. server_obj->server_id);
  2190. mutex_unlock(&g_smcinvoke_lock);
  2191. return -EINVAL;
  2192. }
  2193. if (server_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT)
  2194. server_info->state = 0;
  2195. server_info->is_server_suspended = UNSET_BIT(server_info->is_server_suspended,
  2196. (current->pid)%DEFAULT_CB_OBJ_THREAD_CNT);
  2197. mutex_unlock(&g_smcinvoke_lock);
  2198. /* First check if it has response otherwise wait for req */
  2199. if (user_args.has_resp) {
  2200. trace_process_accept_req_has_response(current->pid, current->tgid);
  2201. mutex_lock(&g_smcinvoke_lock);
  2202. cb_txn = find_cbtxn_locked(server_info, user_args.txn_id,
  2203. SMCINVOKE_REQ_PROCESSING);
  2204. mutex_unlock(&g_smcinvoke_lock);
  2205. /*
  2206. * cb_txn can be null if userspace provides wrong txn id OR
  2207. * invoke thread died while server was processing cb req.
  2208. * if invoke thread dies, it would remove req from Q. So
  2209. * no matching cb_txn would be on Q and hence NULL cb_txn.
  2210. * In this case, we want this thread to start waiting
  2211. * new cb requests.
  2212. */
  2213. if (!cb_txn) {
  2214. pr_err("%s txn %d either invalid or removed from Q\n",
  2215. __func__, user_args.txn_id);
  2216. goto start_waiting_for_requests;
  2217. }
  2218. ret = marshal_out_tzcb_req(&user_args, cb_txn,
  2219. cb_txn->filp_to_release);
  2220. /*
  2221. * if client did not set error and we get error locally,
  2222. * we return local error to TA
  2223. */
  2224. if (ret && cb_txn->cb_req->result == 0)
  2225. cb_txn->cb_req->result = OBJECT_ERROR_UNAVAIL;
  2226. cb_txn->state = SMCINVOKE_REQ_PROCESSED;
  2227. mutex_lock(&g_smcinvoke_lock);
  2228. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  2229. mutex_unlock(&g_smcinvoke_lock);
  2230. wake_up(&server_info->rsp_wait_q);
  2231. /*
  2232. * if marshal_out fails, we should let userspace release
  2233. * any ref/obj it created for CB processing
  2234. */
  2235. if (ret && OBJECT_COUNTS_NUM_OO(user_args.counts))
  2236. goto out;
  2237. }
  2238. start_waiting_for_requests:
  2239. /*
  2240. * Once response has been delivered, thread will wait for another
  2241. * callback req to process.
  2242. */
  2243. do {
  2244. ret = wait_event_interruptible(server_info->req_wait_q,
  2245. !hash_empty(server_info->reqs_table));
  2246. if (ret) {
  2247. trace_process_accept_req_ret(current->pid, current->tgid, ret);
  2248. /*
  2249. * Ideally, we should destroy server if accept threads
  2250. * are returning due to client being killed or device
  2251. * going down (Shutdown/Reboot) but that would make
  2252. * server_info invalid. Other accept/invoke threads are
  2253. * using server_info and would crash. So dont do that.
  2254. */
  2255. mutex_lock(&g_smcinvoke_lock);
  2256. if(freezing(current)) {
  2257. pr_err_ratelimited("Server id :%d interrupted probaby due to suspend, pid:%d\n",
  2258. server_info->server_id, current->pid);
  2259. /*
  2260. * Each accept thread is identified by bits ranging from
  2261. * 0 to DEFAULT_CBOBJ_THREAD_CNT-1. When an accept thread is
  2262. * interrupted by a signal other than SIGUSR1,SIGKILL,SIGTERM,
  2263. * set the corresponding bit of accept thread, indicating that
  2264. * current accept thread's state to be "suspended"/ or something
  2265. * that needs infinite timeout for invoke thread.
  2266. */
  2267. server_info->is_server_suspended =
  2268. SET_BIT(server_info->is_server_suspended,
  2269. (current->pid)%DEFAULT_CB_OBJ_THREAD_CNT);
  2270. } else {
  2271. pr_err_ratelimited("Setting pid:%d, server id : %d state to defunct\n",
  2272. current->pid, server_info->server_id);
  2273. server_info->state = SMCINVOKE_SERVER_STATE_DEFUNCT;
  2274. }
  2275. mutex_unlock(&g_smcinvoke_lock);
  2276. wake_up_interruptible(&server_info->rsp_wait_q);
  2277. goto out;
  2278. }
  2279. mutex_lock(&g_smcinvoke_lock);
  2280. cb_txn = find_cbtxn_locked(server_info,
  2281. SMCINVOKE_NEXT_AVAILABLE_TXN,
  2282. SMCINVOKE_REQ_PLACED);
  2283. mutex_unlock(&g_smcinvoke_lock);
  2284. if (cb_txn) {
  2285. cb_txn->state = SMCINVOKE_REQ_PROCESSING;
  2286. ret = marshal_in_tzcb_req(cb_txn, &user_args,
  2287. server_obj->server_id);
  2288. if (ret) {
  2289. pr_err("failed to marshal in the callback request\n");
  2290. cb_txn->cb_req->result = OBJECT_ERROR_UNAVAIL;
  2291. cb_txn->state = SMCINVOKE_REQ_PROCESSED;
  2292. mutex_lock(&g_smcinvoke_lock);
  2293. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  2294. mutex_unlock(&g_smcinvoke_lock);
  2295. wake_up_interruptible(&server_info->rsp_wait_q);
  2296. continue;
  2297. }
  2298. mutex_lock(&g_smcinvoke_lock);
  2299. hash_add(server_info->responses_table, &cb_txn->hash,
  2300. cb_txn->txn_id);
  2301. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  2302. mutex_unlock(&g_smcinvoke_lock);
  2303. trace_process_accept_req_placed(current->pid, current->tgid);
  2304. ret = copy_to_user((void __user *)arg, &user_args,
  2305. sizeof(struct smcinvoke_accept));
  2306. }
  2307. } while (!cb_txn);
  2308. out:
  2309. if (server_info)
  2310. kref_put(&server_info->ref_cnt, destroy_cb_server);
  2311. if (ret && ret != -ERESTARTSYS)
  2312. pr_err("accept thread returning with ret: %d\n", ret);
  2313. return ret;
  2314. }
  2315. static long process_invoke_req(struct file *filp, unsigned int cmd,
  2316. unsigned long arg)
  2317. {
  2318. int ret = -1, nr_args = 0;
  2319. struct smcinvoke_cmd_req req = {0};
  2320. void *in_msg = NULL, *out_msg = NULL;
  2321. size_t inmsg_size = 0, outmsg_size = SMCINVOKE_TZ_MIN_BUF_SIZE;
  2322. union smcinvoke_arg *args_buf = NULL;
  2323. struct smcinvoke_file_data *tzobj = filp->private_data;
  2324. struct qtee_shm in_shm = {0}, out_shm = {0};
  2325. LIST_HEAD(l_mem_objs_pending_async); /* Holds new memory objects, to be later sent to TZ */
  2326. /*
  2327. * Hold reference to remote object until invoke op is not
  2328. * completed. Release once invoke is done.
  2329. */
  2330. struct file *filp_to_release[OBJECT_COUNTS_MAX_OO] = {NULL};
  2331. /*
  2332. * If anything goes wrong, release alloted tzhandles for
  2333. * local objs which could be either CBObj or MemObj.
  2334. */
  2335. int32_t tzhandles_to_release[OBJECT_COUNTS_MAX_OO] = {0};
  2336. bool tz_acked = false;
  2337. uint32_t context_type = tzobj->context_type;
  2338. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ &&
  2339. _IOC_SIZE(cmd) != sizeof(req)) {
  2340. pr_err("command size for invoke req is invalid\n");
  2341. return -EINVAL;
  2342. }
  2343. if (context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ &&
  2344. context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  2345. pr_err("invalid context_type %d\n", context_type);
  2346. return -EPERM;
  2347. }
  2348. if (context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  2349. ret = copy_from_user(&req, (void __user *)arg, sizeof(req));
  2350. if (ret) {
  2351. pr_err("copying invoke req failed\n");
  2352. return -EFAULT;
  2353. }
  2354. } else {
  2355. req = *(struct smcinvoke_cmd_req *)arg;
  2356. }
  2357. if (req.argsize != sizeof(union smcinvoke_arg)) {
  2358. pr_err("arguments size for invoke req is invalid\n");
  2359. return -EINVAL;
  2360. }
  2361. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ &&
  2362. tzobj->tzhandle == SMCINVOKE_TZ_ROOT_OBJ &&
  2363. (req.op == IClientEnv_OP_notifyDomainChange ||
  2364. req.op == IClientEnv_OP_registerWithCredentials ||
  2365. req.op == IClientEnv_OP_adciAccept ||
  2366. req.op == IClientEnv_OP_adciShutdown)) {
  2367. pr_err("invalid rootenv op\n");
  2368. return -EINVAL;
  2369. }
  2370. nr_args = OBJECT_COUNTS_NUM_buffers(req.counts) +
  2371. OBJECT_COUNTS_NUM_objects(req.counts);
  2372. if (nr_args) {
  2373. args_buf = kcalloc(nr_args, req.argsize, GFP_KERNEL);
  2374. if (!args_buf)
  2375. return -ENOMEM;
  2376. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  2377. ret = copy_from_user(args_buf,
  2378. u64_to_user_ptr(req.args),
  2379. nr_args * req.argsize);
  2380. if (ret) {
  2381. ret = -EFAULT;
  2382. goto out;
  2383. }
  2384. } else {
  2385. memcpy(args_buf, (void *)(req.args),
  2386. nr_args * req.argsize);
  2387. }
  2388. }
  2389. inmsg_size = compute_in_msg_size(&req, args_buf);
  2390. ret = qtee_shmbridge_allocate_shm(inmsg_size, &in_shm);
  2391. if (ret) {
  2392. ret = -ENOMEM;
  2393. pr_err("shmbridge alloc failed for in msg in invoke req\n");
  2394. goto out;
  2395. }
  2396. in_msg = in_shm.vaddr;
  2397. mutex_lock(&g_smcinvoke_lock);
  2398. outmsg_size = PAGE_ALIGN(g_max_cb_buf_size);
  2399. mutex_unlock(&g_smcinvoke_lock);
  2400. ret = qtee_shmbridge_allocate_shm(outmsg_size, &out_shm);
  2401. if (ret) {
  2402. ret = -ENOMEM;
  2403. pr_err("shmbridge alloc failed for out msg in invoke req\n");
  2404. goto out;
  2405. }
  2406. out_msg = out_shm.vaddr;
  2407. trace_process_invoke_req_tzhandle(tzobj->tzhandle, req.op, req.counts);
  2408. ret = marshal_in_invoke_req(&req, args_buf, tzobj->tzhandle, in_msg,
  2409. inmsg_size, filp_to_release, tzhandles_to_release,
  2410. context_type, &l_mem_objs_pending_async);
  2411. if (ret) {
  2412. pr_err("failed to marshal in invoke req, ret :%d\n", ret);
  2413. goto out;
  2414. }
  2415. if (mem_obj_async_support) {
  2416. mutex_lock(&g_smcinvoke_lock);
  2417. add_mem_obj_info_to_async_side_channel_locked(out_msg, outmsg_size, &l_mem_objs_pending_async);
  2418. delete_pending_async_list_locked(&l_mem_objs_pending_async);
  2419. mutex_unlock(&g_smcinvoke_lock);
  2420. }
  2421. ret = prepare_send_scm_msg(in_msg, in_shm.paddr, inmsg_size,
  2422. out_msg, out_shm.paddr, outmsg_size,
  2423. &req, args_buf, &tz_acked, context_type,
  2424. &in_shm, &out_shm);
  2425. /*
  2426. * If scm_call is success, TZ owns responsibility to release
  2427. * refs for local objs.
  2428. */
  2429. if (!tz_acked) {
  2430. trace_status(__func__, "scm call successful");
  2431. goto out;
  2432. }
  2433. memset(tzhandles_to_release, 0, sizeof(tzhandles_to_release));
  2434. /*
  2435. * if invoke op results in an err, no need to marshal_out and
  2436. * copy args buf to user space
  2437. */
  2438. if (!req.result) {
  2439. /*
  2440. * Dont check ret of marshal_out because there might be a
  2441. * FD for OO which userspace must release even if an error
  2442. * occurs. Releasing FD from user space is much simpler than
  2443. * doing here. ORing of ret is reqd not to miss past error
  2444. */
  2445. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ)
  2446. ret |= copy_to_user(u64_to_user_ptr(req.args),
  2447. args_buf, nr_args * req.argsize);
  2448. else
  2449. memcpy((void *)(req.args), args_buf,
  2450. nr_args * req.argsize);
  2451. }
  2452. /* copy result of invoke op */
  2453. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  2454. ret |= copy_to_user((void __user *)arg, &req, sizeof(req));
  2455. if (ret)
  2456. goto out;
  2457. } else {
  2458. memcpy((void *)arg, (void *)&req, sizeof(req));
  2459. }
  2460. /* Outbuf could be carrying local objs to be released. */
  2461. process_piggyback_data(out_msg, outmsg_size);
  2462. out:
  2463. trace_process_invoke_req_result(ret, req.result, tzobj->tzhandle,
  2464. req.op, req.counts);
  2465. release_filp(filp_to_release, OBJECT_COUNTS_MAX_OO);
  2466. if (ret) {
  2467. release_map_obj_pending_async_list_locked(&l_mem_objs_pending_async);
  2468. release_tzhandles(tzhandles_to_release, OBJECT_COUNTS_MAX_OO);
  2469. }
  2470. qtee_shmbridge_free_shm(&in_shm);
  2471. qtee_shmbridge_free_shm(&out_shm);
  2472. kfree(args_buf);
  2473. if (ret)
  2474. pr_err("invoke thread returning with ret = %d\n", ret);
  2475. return ret;
  2476. }
  2477. static long process_log_info(struct file *filp, unsigned int cmd,
  2478. unsigned long arg)
  2479. {
  2480. int ret = 0;
  2481. char buf[SMCINVOKE_LOG_BUF_SIZE];
  2482. struct smcinvoke_file_data *tzobj = filp->private_data;
  2483. ret = copy_from_user(buf, (void __user *)arg, SMCINVOKE_LOG_BUF_SIZE);
  2484. if (ret) {
  2485. pr_err("logging HLOS info copy failed\n");
  2486. return -EFAULT;
  2487. }
  2488. buf[SMCINVOKE_LOG_BUF_SIZE - 1] = '\0';
  2489. trace_process_log_info(buf, tzobj->context_type, tzobj->tzhandle);
  2490. return ret;
  2491. }
  2492. static long smcinvoke_ioctl(struct file *filp, unsigned int cmd,
  2493. unsigned long arg)
  2494. {
  2495. long ret = 0;
  2496. switch (cmd) {
  2497. case SMCINVOKE_IOCTL_INVOKE_REQ:
  2498. ret = process_invoke_req(filp, cmd, arg);
  2499. break;
  2500. case SMCINVOKE_IOCTL_ACCEPT_REQ:
  2501. ret = process_accept_req(filp, cmd, arg);
  2502. break;
  2503. case SMCINVOKE_IOCTL_SERVER_REQ:
  2504. ret = process_server_req(filp, cmd, arg);
  2505. break;
  2506. case SMCINVOKE_IOCTL_ACK_LOCAL_OBJ:
  2507. ret = process_ack_local_obj(filp, cmd, arg);
  2508. break;
  2509. case SMCINVOKE_IOCTL_LOG:
  2510. ret = process_log_info(filp, cmd, arg);
  2511. break;
  2512. default:
  2513. ret = -ENOIOCTLCMD;
  2514. break;
  2515. }
  2516. trace_smcinvoke_ioctl(cmd, ret);
  2517. return ret;
  2518. }
  2519. int get_root_fd(int *root_fd)
  2520. {
  2521. if (!root_fd)
  2522. return -EINVAL;
  2523. else
  2524. return get_fd_for_obj(SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL,
  2525. SMCINVOKE_TZ_ROOT_OBJ, root_fd);
  2526. }
  2527. int process_invoke_request_from_kernel_client(int fd,
  2528. struct smcinvoke_cmd_req *req)
  2529. {
  2530. struct file *filp = NULL;
  2531. int ret = 0;
  2532. if (!req) {
  2533. pr_err("NULL req\n");
  2534. return -EINVAL;
  2535. }
  2536. filp = fget(fd);
  2537. if (!filp) {
  2538. pr_err("Invalid fd %d\n", fd);
  2539. return -EINVAL;
  2540. }
  2541. ret = process_invoke_req(filp, 0, (uintptr_t)req);
  2542. fput(filp);
  2543. trace_process_invoke_request_from_kernel_client(fd, filp, file_count(filp));
  2544. return ret;
  2545. }
  2546. char *firmware_request_from_smcinvoke(const char *appname, size_t *fw_size, struct qtee_shm *shm)
  2547. {
  2548. int rc = 0;
  2549. const struct firmware *fw_entry = NULL, *fw_entry00 = NULL, *fw_entrylast = NULL;
  2550. char fw_name[MAX_APP_NAME_SIZE] = "\0";
  2551. int num_images = 0, phi = 0;
  2552. unsigned char app_arch = 0;
  2553. u8 *img_data_ptr = NULL;
  2554. size_t bufferOffset = 0, phdr_table_offset = 0;
  2555. size_t *offset = NULL;
  2556. Elf32_Phdr phdr32;
  2557. Elf64_Phdr phdr64;
  2558. struct elf32_hdr *ehdr = NULL;
  2559. struct elf64_hdr *ehdr64 = NULL;
  2560. /* load b00*/
  2561. snprintf(fw_name, sizeof(fw_name), "%s.b00", appname);
  2562. rc = firmware_request_nowarn(&fw_entry00, fw_name, class_dev);
  2563. if (rc) {
  2564. pr_err("Load %s failed, ret:%d\n", fw_name, rc);
  2565. return NULL;
  2566. }
  2567. app_arch = *(unsigned char *)(fw_entry00->data + EI_CLASS);
  2568. /*Get the offsets for split images header*/
  2569. if (app_arch == ELFCLASS32) {
  2570. ehdr = (struct elf32_hdr *)fw_entry00->data;
  2571. num_images = ehdr->e_phnum;
  2572. offset = kcalloc(num_images, sizeof(size_t), GFP_KERNEL);
  2573. if (offset == NULL)
  2574. goto release_fw_entry00;
  2575. phdr_table_offset = (size_t) ehdr->e_phoff;
  2576. for (phi = 1; phi < num_images; ++phi) {
  2577. bufferOffset = phdr_table_offset + phi * sizeof(Elf32_Phdr);
  2578. phdr32 = *(Elf32_Phdr *)(fw_entry00->data + bufferOffset);
  2579. offset[phi] = (size_t)phdr32.p_offset;
  2580. }
  2581. } else if (app_arch == ELFCLASS64) {
  2582. ehdr64 = (struct elf64_hdr *)fw_entry00->data;
  2583. num_images = ehdr64->e_phnum;
  2584. offset = kcalloc(num_images, sizeof(size_t), GFP_KERNEL);
  2585. if (offset == NULL)
  2586. goto release_fw_entry00;
  2587. phdr_table_offset = (size_t) ehdr64->e_phoff;
  2588. for (phi = 1; phi < num_images; ++phi) {
  2589. bufferOffset = phdr_table_offset + phi * sizeof(Elf64_Phdr);
  2590. phdr64 = *(Elf64_Phdr *)(fw_entry00->data + bufferOffset);
  2591. offset[phi] = (size_t)phdr64.p_offset;
  2592. }
  2593. } else {
  2594. pr_err("QSEE %s app, arch %u is not supported\n", appname, app_arch);
  2595. goto release_fw_entry00;
  2596. }
  2597. /*Find the size of last split bin image*/
  2598. snprintf(fw_name, ARRAY_SIZE(fw_name), "%s.b%02d", appname, num_images-1);
  2599. rc = firmware_request_nowarn(&fw_entrylast, fw_name, class_dev);
  2600. if (rc) {
  2601. pr_err("Failed to locate blob %s\n", fw_name);
  2602. goto release_fw_entry00;
  2603. }
  2604. /*Total size of image will be the offset of last image + the size of last split image*/
  2605. *fw_size = fw_entrylast->size + offset[num_images-1];
  2606. /*Allocate memory for the buffer that will hold the split image*/
  2607. rc = qtee_shmbridge_allocate_shm((*fw_size), shm);
  2608. if (rc) {
  2609. pr_err("smbridge alloc failed for size: %zu\n", *fw_size);
  2610. goto release_fw_entrylast;
  2611. }
  2612. img_data_ptr = shm->vaddr;
  2613. /*
  2614. * Copy contents of split bins to the buffer
  2615. */
  2616. memcpy(img_data_ptr, fw_entry00->data, fw_entry00->size);
  2617. for (phi = 1; phi < num_images-1; phi++) {
  2618. snprintf(fw_name, ARRAY_SIZE(fw_name), "%s.b%02d", appname, phi);
  2619. rc = firmware_request_nowarn(&fw_entry, fw_name, class_dev);
  2620. if (rc) {
  2621. pr_err("Failed to locate blob %s\n", fw_name);
  2622. qtee_shmbridge_free_shm(shm);
  2623. img_data_ptr = NULL;
  2624. goto release_fw_entrylast;
  2625. }
  2626. memcpy(img_data_ptr + offset[phi], fw_entry->data, fw_entry->size);
  2627. release_firmware(fw_entry);
  2628. fw_entry = NULL;
  2629. }
  2630. memcpy(img_data_ptr + offset[phi], fw_entrylast->data, fw_entrylast->size);
  2631. release_fw_entrylast:
  2632. release_firmware(fw_entrylast);
  2633. release_fw_entry00:
  2634. release_firmware(fw_entry00);
  2635. kfree(offset);
  2636. return img_data_ptr;
  2637. }
  2638. EXPORT_SYMBOL(firmware_request_from_smcinvoke);
  2639. static int smcinvoke_open(struct inode *nodp, struct file *filp)
  2640. {
  2641. struct smcinvoke_file_data *tzcxt = NULL;
  2642. tzcxt = kzalloc(sizeof(*tzcxt), GFP_KERNEL);
  2643. if (!tzcxt)
  2644. return -ENOMEM;
  2645. tzcxt->tzhandle = SMCINVOKE_TZ_ROOT_OBJ;
  2646. tzcxt->context_type = SMCINVOKE_OBJ_TYPE_TZ_OBJ;
  2647. filp->private_data = tzcxt;
  2648. return 0;
  2649. }
  2650. static int release_cb_server(uint16_t server_id)
  2651. {
  2652. struct smcinvoke_server_info *server = NULL;
  2653. mutex_lock(&g_smcinvoke_lock);
  2654. server = find_cb_server_locked(server_id);
  2655. if (server)
  2656. kref_put(&server->ref_cnt, destroy_cb_server);
  2657. mutex_unlock(&g_smcinvoke_lock);
  2658. return 0;
  2659. }
  2660. int smcinvoke_release_filp(struct file *filp)
  2661. {
  2662. int ret = 0;
  2663. struct smcinvoke_file_data *file_data = filp->private_data;
  2664. uint32_t tzhandle = 0;
  2665. struct smcinvoke_object_release_pending_list *entry = NULL;
  2666. struct qtee_shm in_shm = {0}, out_shm = {0};
  2667. trace_smcinvoke_release_filp(current->files, filp,
  2668. file_count(filp), file_data->context_type);
  2669. if (file_data->context_type == SMCINVOKE_OBJ_TYPE_SERVER) {
  2670. ret = release_cb_server(file_data->server_id);
  2671. goto out;
  2672. }
  2673. tzhandle = file_data->tzhandle;
  2674. /* Root object is special in sense it is indestructible */
  2675. if (!tzhandle || tzhandle == SMCINVOKE_TZ_ROOT_OBJ) {
  2676. if (!tzhandle)
  2677. pr_err("tzhandle not valid in object release\n");
  2678. goto out;
  2679. }
  2680. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &in_shm);
  2681. if (ret) {
  2682. pr_err("shmbridge alloc failed for in msg in object release"
  2683. "with ret %d\n", ret);
  2684. goto out;
  2685. }
  2686. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &out_shm);
  2687. if (ret) {
  2688. pr_err("shmbridge alloc failed for out msg in object release"
  2689. "with ret:%d\n", ret);
  2690. goto out;
  2691. }
  2692. ret = smcinvoke_release_tz_object(&in_shm, &out_shm,
  2693. tzhandle, file_data->context_type);
  2694. if (-EBUSY == ret) {
  2695. pr_debug("failed to release handle in sync adding to list\n");
  2696. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  2697. if (!entry) {
  2698. ret = -ENOMEM;
  2699. goto out;
  2700. }
  2701. ret = 0;
  2702. entry->data.tzhandle = tzhandle;
  2703. entry->data.context_type = file_data->context_type;
  2704. mutex_lock(&object_postprocess_lock);
  2705. list_add_tail(&entry->list, &g_object_postprocess);
  2706. mutex_unlock(&object_postprocess_lock);
  2707. pr_debug("Object release list: added a handle:0x%lx\n", tzhandle);
  2708. __wakeup_postprocess_kthread(&smcinvoke[OBJECT_WORKER_THREAD]);
  2709. }
  2710. out:
  2711. qtee_shmbridge_free_shm(&in_shm);
  2712. qtee_shmbridge_free_shm(&out_shm);
  2713. kfree(filp->private_data);
  2714. filp->private_data = NULL;
  2715. if (ret != 0)
  2716. pr_err ("Object release failed with ret %d\n", ret);
  2717. return ret;
  2718. }
  2719. int smcinvoke_release_from_kernel_client(int fd)
  2720. {
  2721. struct file *filp = NULL;
  2722. /* use fget() to get filp, but this will increase file ref_cnt to 1,
  2723. * then decrease file ref_cnt to 0 with fput().
  2724. */
  2725. filp = fget(fd);
  2726. if (!filp) {
  2727. pr_err("invalid fd %d to release\n", fd);
  2728. return -EINVAL;
  2729. }
  2730. trace_smcinvoke_release_from_kernel_client(current->files, filp,
  2731. file_count(filp));
  2732. /* free filp, notify TZ to release object */
  2733. smcinvoke_release_filp(filp);
  2734. fput(filp);
  2735. return 0;
  2736. }
  2737. static int smcinvoke_release(struct inode *nodp, struct file *filp)
  2738. {
  2739. trace_smcinvoke_release(current->files, filp, file_count(filp),
  2740. filp->private_data);
  2741. if (filp->private_data)
  2742. return smcinvoke_release_filp(filp);
  2743. else
  2744. return 0;
  2745. }
  2746. static int smcinvoke_probe(struct platform_device *pdev)
  2747. {
  2748. unsigned int baseminor = 0;
  2749. unsigned int count = 1;
  2750. int rc = 0;
  2751. rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  2752. if (rc) {
  2753. pr_err("dma_set_mask_and_coherent failed %d\n", rc);
  2754. return rc;
  2755. }
  2756. legacy_smc_call = of_property_read_bool((&pdev->dev)->of_node,
  2757. "qcom,support-legacy_smc");
  2758. invoke_cmd = legacy_smc_call ? SMCINVOKE_INVOKE_CMD_LEGACY : SMCINVOKE_INVOKE_CMD;
  2759. rc = smcinvoke_create_kthreads();
  2760. if (rc) {
  2761. pr_err("smcinvoke_create_kthreads failed %d\n", rc);
  2762. return rc;
  2763. }
  2764. rc = alloc_chrdev_region(&smcinvoke_device_no, baseminor, count,
  2765. SMCINVOKE_DEV);
  2766. if (rc < 0) {
  2767. pr_err("chrdev_region failed %d for %s\n", rc, SMCINVOKE_DEV);
  2768. goto exit_destroy_wkthread;
  2769. }
  2770. driver_class = class_create(THIS_MODULE, SMCINVOKE_DEV);
  2771. if (IS_ERR(driver_class)) {
  2772. rc = -ENOMEM;
  2773. pr_err("class_create failed %d\n", rc);
  2774. goto exit_unreg_chrdev_region;
  2775. }
  2776. class_dev = device_create(driver_class, NULL, smcinvoke_device_no,
  2777. NULL, SMCINVOKE_DEV);
  2778. if (!class_dev) {
  2779. pr_err("class_device_create failed %d\n", rc);
  2780. rc = -ENOMEM;
  2781. goto exit_destroy_class;
  2782. }
  2783. cdev_init(&smcinvoke_cdev, &g_smcinvoke_fops);
  2784. smcinvoke_cdev.owner = THIS_MODULE;
  2785. rc = cdev_add(&smcinvoke_cdev, MKDEV(MAJOR(smcinvoke_device_no), 0),
  2786. count);
  2787. if (rc < 0) {
  2788. pr_err("cdev_add failed %d for %s\n", rc, SMCINVOKE_DEV);
  2789. goto exit_destroy_device;
  2790. }
  2791. smcinvoke_pdev = pdev;
  2792. #if !IS_ENABLED(CONFIG_QSEECOM) && IS_ENABLED(CONFIG_QSEECOM_PROXY)
  2793. /*If the api fails to get the func ops, print the error and continue
  2794. * Do not treat it as fatal*/
  2795. rc = get_qseecom_kernel_fun_ops();
  2796. if (rc) {
  2797. pr_err("failed to get qseecom kernel func ops %d", rc);
  2798. }
  2799. #endif
  2800. __wakeup_postprocess_kthread(&smcinvoke[ADCI_WORKER_THREAD]);
  2801. return 0;
  2802. exit_destroy_device:
  2803. device_destroy(driver_class, smcinvoke_device_no);
  2804. exit_destroy_class:
  2805. class_destroy(driver_class);
  2806. exit_unreg_chrdev_region:
  2807. unregister_chrdev_region(smcinvoke_device_no, count);
  2808. exit_destroy_wkthread:
  2809. smcinvoke_destroy_kthreads();
  2810. return rc;
  2811. }
  2812. static int smcinvoke_remove(struct platform_device *pdev)
  2813. {
  2814. int count = 1;
  2815. smcinvoke_destroy_kthreads();
  2816. cdev_del(&smcinvoke_cdev);
  2817. device_destroy(driver_class, smcinvoke_device_no);
  2818. class_destroy(driver_class);
  2819. unregister_chrdev_region(smcinvoke_device_no, count);
  2820. return 0;
  2821. }
  2822. static int __maybe_unused smcinvoke_suspend(struct platform_device *pdev,
  2823. pm_message_t state)
  2824. {
  2825. int ret = 0;
  2826. mutex_lock(&g_smcinvoke_lock);
  2827. if (cb_reqs_inflight) {
  2828. pr_err("Failed to suspend smcinvoke driver\n");
  2829. ret = -EIO;
  2830. }
  2831. mutex_unlock(&g_smcinvoke_lock);
  2832. return ret;
  2833. }
  2834. static int __maybe_unused smcinvoke_resume(struct platform_device *pdev)
  2835. {
  2836. return 0;
  2837. }
  2838. static const struct of_device_id smcinvoke_match[] = {
  2839. {
  2840. .compatible = "qcom,smcinvoke",
  2841. },
  2842. {},
  2843. };
  2844. static struct platform_driver smcinvoke_plat_driver = {
  2845. .probe = smcinvoke_probe,
  2846. .remove = smcinvoke_remove,
  2847. .suspend = smcinvoke_suspend,
  2848. .resume = smcinvoke_resume,
  2849. .driver = {
  2850. .name = "smcinvoke",
  2851. .of_match_table = smcinvoke_match,
  2852. },
  2853. };
  2854. static int smcinvoke_init(void)
  2855. {
  2856. return platform_driver_register(&smcinvoke_plat_driver);
  2857. }
  2858. static void smcinvoke_exit(void)
  2859. {
  2860. platform_driver_unregister(&smcinvoke_plat_driver);
  2861. }
  2862. module_init(smcinvoke_init);
  2863. module_exit(smcinvoke_exit);
  2864. MODULE_LICENSE("GPL v2");
  2865. MODULE_DESCRIPTION("SMC Invoke driver");
  2866. MODULE_IMPORT_NS(VFS_internal_I_am_really_a_filesystem_and_am_NOT_a_driver);
  2867. MODULE_IMPORT_NS(DMA_BUF);