smcinvoke.c 90 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #define pr_fmt(fmt) "smcinvoke: %s: " fmt, __func__
  7. #include <linux/module.h>
  8. #include <linux/mod_devicetable.h>
  9. #include <linux/device.h>
  10. #include <linux/platform_device.h>
  11. #include <linux/slab.h>
  12. #include <linux/file.h>
  13. #include <linux/fs.h>
  14. #include <linux/anon_inodes.h>
  15. #include <linux/hashtable.h>
  16. #include <linux/cdev.h>
  17. #include <linux/uaccess.h>
  18. #include <linux/dma-buf.h>
  19. #include <linux/delay.h>
  20. #include <linux/kref.h>
  21. #include <linux/signal.h>
  22. #include <linux/msm_ion.h>
  23. #include <linux/mem-buf.h>
  24. #include <linux/of_platform.h>
  25. #include <linux/firmware.h>
  26. #include <linux/qcom_scm.h>
  27. #include <linux/freezer.h>
  28. #include <linux/ratelimit.h>
  29. #include <asm/cacheflush.h>
  30. #include <linux/qtee_shmbridge.h>
  31. #include <linux/kthread.h>
  32. #include "smcinvoke.h"
  33. #include "smcinvoke_object.h"
  34. #include "IClientEnv.h"
  35. #if IS_ENABLED(CONFIG_QSEECOM_PROXY)
  36. #include <linux/qseecom_kernel.h>
  37. #include "misc/qseecom_priv.h"
  38. #else
  39. #include "misc/qseecom_kernel.h"
  40. #endif
  41. #include "misc/qseecomi.h"
  42. #define CREATE_TRACE_POINTS
  43. #include "trace_smcinvoke.h"
  44. #define SMCINVOKE_DEV "smcinvoke"
  45. #define SMCINVOKE_TZ_ROOT_OBJ 1
  46. #define SMCINVOKE_TZ_OBJ_NULL 0
  47. #define SMCINVOKE_ARGS_ALIGN_SIZE (sizeof(uint64_t))
  48. #define SMCINVOKE_NEXT_AVAILABLE_TXN 0
  49. #define SMCINVOKE_REQ_PLACED 1
  50. #define SMCINVOKE_REQ_PROCESSING 2
  51. #define SMCINVOKE_REQ_PROCESSED 3
  52. #define SMCINVOKE_INCREMENT 1
  53. #define SMCINVOKE_DECREMENT 0
  54. #define SMCINVOKE_OBJ_TYPE_TZ_OBJ 0
  55. #define SMCINVOKE_OBJ_TYPE_SERVER 1
  56. #define SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL 2
  57. #define SMCINVOKE_MEM_MAP_OBJ 0
  58. #define SMCINVOKE_MEM_RGN_OBJ 1
  59. #define SMCINVOKE_MEM_PERM_RW 6
  60. #define SMCINVOKE_SCM_EBUSY_WAIT_MS 30
  61. #define SMCINVOKE_SCM_EBUSY_MAX_RETRY 200
  62. #define TZCB_ERR_RATELIMIT_INTERVAL (1*HZ)
  63. #define TZCB_ERR_RATELIMIT_BURST 1
  64. //print tzcb err per sec
  65. #define tzcb_err_ratelimited(fmt, ...) do { \
  66. static DEFINE_RATELIMIT_STATE(_rs, \
  67. TZCB_ERR_RATELIMIT_INTERVAL, \
  68. TZCB_ERR_RATELIMIT_BURST); \
  69. if (__ratelimit(&_rs)) \
  70. pr_err(fmt, ##__VA_ARGS__); \
  71. } while(0)
  72. /* TZ defined values - Start */
  73. #define SMCINVOKE_INVOKE_PARAM_ID 0x224
  74. #define SMCINVOKE_CB_RSP_PARAM_ID 0x22
  75. #define SMCINVOKE_INVOKE_CMD_LEGACY 0x32000600
  76. #define SMCINVOKE_INVOKE_CMD 0x32000602
  77. #define SMCINVOKE_CB_RSP_CMD 0x32000601
  78. #define SMCINVOKE_RESULT_INBOUND_REQ_NEEDED 3
  79. /* TZ defined values - End */
  80. /* Asynchronous protocol values */
  81. /* Driver async version is set to match the minimal TZ version that supports async memory object */
  82. #define SMCINVOKE_ASYNC_VERSION (0x00010002)
  83. #define SMCINVOKE_ASYNC_OP_MEMORY_OBJECT (0x00000003)
  84. /*
  85. * This is the state when server FD has been closed but
  86. * TZ still has refs of CBOBjs served by this server
  87. */
  88. #define SMCINVOKE_SERVER_STATE_DEFUNCT 1
  89. #define CBOBJ_MAX_RETRIES 50
  90. #define FOR_ARGS(ndxvar, counts, section) \
  91. for (ndxvar = OBJECT_COUNTS_INDEX_##section(counts); \
  92. ndxvar < (OBJECT_COUNTS_INDEX_##section(counts) \
  93. + OBJECT_COUNTS_NUM_##section(counts)); \
  94. ++ndxvar)
  95. #define TZCB_BUF_OFFSET(tzcb_req) (sizeof(tzcb_req->result) + \
  96. sizeof(struct smcinvoke_msg_hdr) + \
  97. sizeof(union smcinvoke_tz_args) * \
  98. OBJECT_COUNTS_TOTAL(tzcb_req->hdr.counts))
  99. /*
  100. * +ve uhandle : either remote obj or mem obj, decided by f_ops
  101. * -ve uhandle : either Obj NULL or CBObj
  102. * - -1: OBJ NULL
  103. * - < -1: CBObj
  104. */
  105. #define UHANDLE_IS_FD(h) ((h) >= 0)
  106. #define UHANDLE_IS_NULL(h) ((h) == SMCINVOKE_USERSPACE_OBJ_NULL)
  107. #define UHANDLE_IS_CB_OBJ(h) (h < SMCINVOKE_USERSPACE_OBJ_NULL)
  108. #define UHANDLE_NULL (SMCINVOKE_USERSPACE_OBJ_NULL)
  109. /*
  110. * MAKE => create handle for other domain i.e. TZ or userspace
  111. * GET => retrieve obj from incoming handle
  112. */
  113. #define UHANDLE_GET_CB_OBJ(h) (-2-(h))
  114. #define UHANDLE_MAKE_CB_OBJ(o) (-2-(o))
  115. #define UHANDLE_GET_FD(h) (h)
  116. /*
  117. * +ve tzhandle : remote object i.e. owned by TZ
  118. * -ve tzhandle : local object i.e. owned by linux
  119. * --------------------------------------------------
  120. *| 1 (1 bit) | Obj Id (15 bits) | srvr id (16 bits) |
  121. * ---------------------------------------------------
  122. * Server ids are defined below for various local objects
  123. * server id 0 : Kernel Obj
  124. * server id 1 : Memory region Obj
  125. * server id 2 : Memory map Obj
  126. * server id 3-15: Reserverd
  127. * server id 16 & up: Callback Objs
  128. */
  129. #define KRNL_SRVR_ID 0
  130. #define MEM_RGN_SRVR_ID 1
  131. #define MEM_MAP_SRVR_ID 2
  132. #define CBOBJ_SERVER_ID_START 0x10
  133. #define CBOBJ_SERVER_ID_END ((1<<16) - 1)
  134. /* local obj id is represented by 15 bits */
  135. #define MAX_LOCAL_OBJ_ID ((1<<15) - 1)
  136. /* CBOBJs will be served by server id 0x10 onwards */
  137. #define TZHANDLE_GET_SERVER(h) ((uint16_t)((h) & 0xFFFF))
  138. #define TZHANDLE_GET_OBJID(h) (((h) >> 16) & 0x7FFF)
  139. #define TZHANDLE_MAKE_LOCAL(s, o) (((0x8000 | (o)) << 16) | s)
  140. #define SET_BIT(s,b) (s | (1 << b))
  141. #define UNSET_BIT(s,b) (s & (~ (1 << b)))
  142. #define TZHANDLE_IS_NULL(h) ((h) == SMCINVOKE_TZ_OBJ_NULL)
  143. #define TZHANDLE_IS_LOCAL(h) ((h) & 0x80000000)
  144. #define TZHANDLE_IS_REMOTE(h) (!TZHANDLE_IS_NULL(h) && !TZHANDLE_IS_LOCAL(h))
  145. #define TZHANDLE_IS_KERNEL_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  146. TZHANDLE_GET_SERVER(h) == KRNL_SRVR_ID)
  147. #define TZHANDLE_IS_MEM_RGN_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  148. TZHANDLE_GET_SERVER(h) == MEM_RGN_SRVR_ID)
  149. #define TZHANDLE_IS_MEM_MAP_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  150. TZHANDLE_GET_SERVER(h) == MEM_MAP_SRVR_ID)
  151. #define TZHANDLE_IS_MEM_OBJ(h) (TZHANDLE_IS_MEM_RGN_OBJ(h) || \
  152. TZHANDLE_IS_MEM_MAP_OBJ(h))
  153. #define TZHANDLE_IS_CB_OBJ(h) (TZHANDLE_IS_LOCAL(h) && \
  154. TZHANDLE_GET_SERVER(h) >= CBOBJ_SERVER_ID_START)
  155. #define FILE_IS_REMOTE_OBJ(f) ((f)->f_op && (f)->f_op == &g_smcinvoke_fops)
  156. static DEFINE_MUTEX(g_smcinvoke_lock);
  157. #define NO_LOCK 0
  158. #define TAKE_LOCK 1
  159. #define MUTEX_LOCK(x) { if (x) mutex_lock(&g_smcinvoke_lock); }
  160. #define MUTEX_UNLOCK(x) { if (x) mutex_unlock(&g_smcinvoke_lock); }
  161. #define POST_KT_SLEEP 0
  162. #define POST_KT_WAKEUP 1
  163. #define MAX_CHAR_NAME 50
  164. enum worker_thread_type {
  165. SHMB_WORKER_THREAD = 0,
  166. OBJECT_WORKER_THREAD,
  167. ADCI_WORKER_THREAD,
  168. MAX_THREAD_NUMBER
  169. };
  170. static DEFINE_HASHTABLE(g_cb_servers, 8);
  171. static LIST_HEAD(g_mem_objs);
  172. static uint16_t g_last_cb_server_id = CBOBJ_SERVER_ID_START;
  173. static uint16_t g_last_mem_rgn_id, g_last_mem_map_obj_id;
  174. static size_t g_max_cb_buf_size = SMCINVOKE_TZ_MIN_BUF_SIZE;
  175. static unsigned int cb_reqs_inflight;
  176. static bool legacy_smc_call;
  177. static int invoke_cmd;
  178. static long smcinvoke_ioctl(struct file *, unsigned int, unsigned long);
  179. static int smcinvoke_open(struct inode *, struct file *);
  180. static int smcinvoke_release(struct inode *, struct file *);
  181. static int release_cb_server(uint16_t);
  182. static const struct file_operations g_smcinvoke_fops = {
  183. .owner = THIS_MODULE,
  184. .unlocked_ioctl = smcinvoke_ioctl,
  185. .compat_ioctl = smcinvoke_ioctl,
  186. .open = smcinvoke_open,
  187. .release = smcinvoke_release,
  188. };
  189. static dev_t smcinvoke_device_no;
  190. static struct cdev smcinvoke_cdev;
  191. static struct class *driver_class;
  192. struct device *class_dev;
  193. static struct platform_device *smcinvoke_pdev;
  194. /* We disable async memory object support by default,
  195. * until we receive the first message from TZ over the
  196. * async channel and can determine TZ async version.
  197. */
  198. static bool mem_obj_async_support = false;
  199. static uint32_t tz_async_version = 0x0;
  200. struct smcinvoke_buf_hdr {
  201. uint32_t offset;
  202. uint32_t size;
  203. };
  204. union smcinvoke_tz_args {
  205. struct smcinvoke_buf_hdr b;
  206. int32_t handle;
  207. };
  208. struct smcinvoke_msg_hdr {
  209. uint32_t tzhandle;
  210. uint32_t op;
  211. uint32_t counts;
  212. };
  213. /* Inbound reqs from TZ */
  214. struct smcinvoke_tzcb_req {
  215. int32_t result;
  216. struct smcinvoke_msg_hdr hdr;
  217. union smcinvoke_tz_args args[0];
  218. };
  219. struct smcinvoke_file_data {
  220. uint32_t context_type;
  221. union {
  222. uint32_t tzhandle;
  223. uint16_t server_id;
  224. };
  225. };
  226. struct smcinvoke_piggyback_msg {
  227. uint32_t version;
  228. uint32_t op;
  229. uint32_t counts;
  230. int32_t objs[0];
  231. };
  232. /* Mapped memory object data
  233. *
  234. * memObjRef Handle reference for the memory object
  235. * mapObjRef Handle reference for the map object
  236. * addr Mapped memory address
  237. * size Size of mapped memory
  238. * perm Access rights for the memory
  239. */
  240. struct smcinvoke_mem_obj_info {
  241. uint32_t memObjRef;
  242. uint32_t mapObjRef;
  243. uint64_t addr;
  244. uint64_t size;
  245. uint32_t perm;
  246. };
  247. /* Memory object info to be written into the async buffer
  248. *
  249. * version Async protocol version
  250. * op Async protocol operation
  251. * count Number of memory objects passed
  252. * mo Mapped memory object data
  253. */
  254. struct smcinvoke_mem_obj_msg {
  255. uint32_t version;
  256. uint32_t op;
  257. uint32_t count;
  258. struct smcinvoke_mem_obj_info mo[];
  259. };
  260. struct smcinvoke_mem_obj_pending_async {
  261. struct smcinvoke_mem_obj *mem_obj;
  262. struct list_head list;
  263. };
  264. /* Data structure to hold request coming from TZ */
  265. struct smcinvoke_cb_txn {
  266. uint32_t txn_id;
  267. int32_t state;
  268. struct smcinvoke_tzcb_req *cb_req;
  269. size_t cb_req_bytes;
  270. struct file **filp_to_release;
  271. struct hlist_node hash;
  272. struct kref ref_cnt;
  273. };
  274. struct smcinvoke_server_info {
  275. uint16_t server_id;
  276. uint16_t state;
  277. uint32_t txn_id;
  278. struct kref ref_cnt;
  279. wait_queue_head_t req_wait_q;
  280. wait_queue_head_t rsp_wait_q;
  281. size_t cb_buf_size;
  282. DECLARE_HASHTABLE(reqs_table, 4);
  283. DECLARE_HASHTABLE(responses_table, 4);
  284. struct hlist_node hash;
  285. struct list_head pending_cbobjs;
  286. uint8_t is_server_suspended;
  287. };
  288. struct smcinvoke_cbobj {
  289. uint16_t cbobj_id;
  290. struct kref ref_cnt;
  291. struct smcinvoke_server_info *server;
  292. struct list_head list;
  293. };
  294. /*
  295. * We require couple of objects, one for mem region & another
  296. * for mapped mem_obj once mem region has been mapped. It is
  297. * possible that TZ can release either independent of other.
  298. */
  299. struct smcinvoke_mem_obj {
  300. /* these ids are objid part of tzhandle */
  301. uint16_t mem_region_id;
  302. uint16_t mem_map_obj_id;
  303. struct dma_buf *dma_buf;
  304. struct dma_buf_attachment *buf_attach;
  305. struct sg_table *sgt;
  306. struct kref mem_regn_ref_cnt;
  307. struct kref mem_map_obj_ref_cnt;
  308. uint64_t p_addr;
  309. size_t p_addr_len;
  310. struct list_head list;
  311. uint64_t shmbridge_handle;
  312. struct smcinvoke_server_info *server;
  313. int32_t mem_obj_user_fd;
  314. };
  315. static LIST_HEAD(g_bridge_postprocess);
  316. DEFINE_MUTEX(bridge_postprocess_lock);
  317. static LIST_HEAD(g_object_postprocess);
  318. DEFINE_MUTEX(object_postprocess_lock);
  319. struct bridge_deregister {
  320. uint64_t shmbridge_handle;
  321. struct dma_buf *dmabuf_to_free;
  322. };
  323. struct object_release {
  324. uint32_t tzhandle;
  325. uint32_t context_type;
  326. };
  327. struct smcinvoke_shmbridge_deregister_pending_list {
  328. struct list_head list;
  329. struct bridge_deregister data;
  330. };
  331. struct smcinvoke_object_release_pending_list {
  332. struct list_head list;
  333. struct object_release data;
  334. };
  335. struct smcinvoke_worker_thread {
  336. enum worker_thread_type type;
  337. atomic_t postprocess_kthread_state;
  338. wait_queue_head_t postprocess_kthread_wq;
  339. struct task_struct *postprocess_kthread_task;
  340. };
  341. static struct smcinvoke_worker_thread smcinvoke[MAX_THREAD_NUMBER];
  342. static const char thread_name[MAX_THREAD_NUMBER][MAX_CHAR_NAME] = {
  343. "smcinvoke_shmbridge_postprocess", "smcinvoke_object_postprocess", "smcinvoke_adci_thread"};
  344. static struct Object adci_rootEnv = Object_NULL;
  345. extern int get_root_obj(struct Object *rootObj);
  346. static int prepare_send_scm_msg(const uint8_t *in_buf, phys_addr_t in_paddr,
  347. size_t in_buf_len,
  348. uint8_t *out_buf, phys_addr_t out_paddr,
  349. size_t out_buf_len,
  350. struct smcinvoke_cmd_req *req,
  351. union smcinvoke_arg *args_buf,
  352. bool *tz_acked, uint32_t context_type,
  353. struct qtee_shm *in_shm, struct qtee_shm *out_shm, bool retry);
  354. static void process_piggyback_data(void *buf, size_t buf_size);
  355. static void add_mem_obj_info_to_async_side_channel_locked(void *buf, size_t buf_size, struct list_head *l_pending_mem_obj);
  356. static void delete_pending_async_list_locked(struct list_head *l_pending_mem_obj);
  357. static void destroy_cb_server(struct kref *kref)
  358. {
  359. struct smcinvoke_server_info *server = container_of(kref,
  360. struct smcinvoke_server_info, ref_cnt);
  361. if (server) {
  362. hash_del(&server->hash);
  363. kfree(server);
  364. }
  365. }
  366. /*
  367. * A separate find func is reqd mainly for couple of cases:
  368. * next_cb_server_id_locked which checks if server id had been utilized or not.
  369. * - It would be overhead if we do ref_cnt for this case
  370. * smcinvoke_release: which is called when server is closed from userspace.
  371. * - During server creation we init ref count, now put it back
  372. */
  373. static struct smcinvoke_server_info *find_cb_server_locked(uint16_t server_id)
  374. {
  375. struct smcinvoke_server_info *data = NULL;
  376. hash_for_each_possible(g_cb_servers, data, hash, server_id) {
  377. if (data->server_id == server_id)
  378. return data;
  379. }
  380. return NULL;
  381. }
  382. static struct smcinvoke_server_info *get_cb_server_locked(uint16_t server_id)
  383. {
  384. struct smcinvoke_server_info *server = find_cb_server_locked(server_id);
  385. if (server)
  386. kref_get(&server->ref_cnt);
  387. return server;
  388. }
  389. static uint16_t next_cb_server_id_locked(void)
  390. {
  391. if (g_last_cb_server_id == CBOBJ_SERVER_ID_END)
  392. g_last_cb_server_id = CBOBJ_SERVER_ID_START;
  393. while (find_cb_server_locked(++g_last_cb_server_id))
  394. ;
  395. return g_last_cb_server_id;
  396. }
  397. static inline void release_filp(struct file **filp_to_release, size_t arr_len)
  398. {
  399. size_t i = 0;
  400. for (i = 0; i < arr_len; i++) {
  401. if (filp_to_release[i]) {
  402. fput(filp_to_release[i]);
  403. filp_to_release[i] = NULL;
  404. }
  405. }
  406. }
  407. static struct smcinvoke_mem_obj *find_mem_obj_locked(uint16_t mem_obj_id,
  408. bool is_mem_rgn_obj)
  409. {
  410. struct smcinvoke_mem_obj *mem_obj = NULL;
  411. if (list_empty(&g_mem_objs))
  412. return NULL;
  413. list_for_each_entry(mem_obj, &g_mem_objs, list) {
  414. if ((is_mem_rgn_obj &&
  415. (mem_obj->mem_region_id == mem_obj_id)) ||
  416. (!is_mem_rgn_obj &&
  417. (mem_obj->mem_map_obj_id == mem_obj_id)))
  418. return mem_obj;
  419. }
  420. return NULL;
  421. }
  422. static uint32_t next_mem_region_obj_id_locked(void)
  423. {
  424. if (g_last_mem_rgn_id == MAX_LOCAL_OBJ_ID)
  425. g_last_mem_rgn_id = 0;
  426. while (find_mem_obj_locked(++g_last_mem_rgn_id, SMCINVOKE_MEM_RGN_OBJ))
  427. ;
  428. return g_last_mem_rgn_id;
  429. }
  430. static uint32_t next_mem_map_obj_id_locked(void)
  431. {
  432. if (g_last_mem_map_obj_id == MAX_LOCAL_OBJ_ID)
  433. g_last_mem_map_obj_id = 0;
  434. while (find_mem_obj_locked(++g_last_mem_map_obj_id,
  435. SMCINVOKE_MEM_MAP_OBJ))
  436. ;
  437. return g_last_mem_map_obj_id;
  438. }
  439. static void smcinvoke_shmbridge_post_process(void)
  440. {
  441. struct smcinvoke_shmbridge_deregister_pending_list *entry = NULL;
  442. struct list_head *pos;
  443. int ret = 0;
  444. uint64_t handle = 0;
  445. struct dma_buf *dmabuf_to_free = NULL;
  446. do {
  447. mutex_lock(&bridge_postprocess_lock);
  448. if (list_empty(&g_bridge_postprocess)) {
  449. mutex_unlock(&bridge_postprocess_lock);
  450. break;
  451. }
  452. pos = g_bridge_postprocess.next;
  453. entry = list_entry(pos,
  454. struct smcinvoke_shmbridge_deregister_pending_list,
  455. list);
  456. if (entry) {
  457. handle = entry->data.shmbridge_handle;
  458. dmabuf_to_free = entry->data.dmabuf_to_free;
  459. } else {
  460. pr_err("entry is NULL, pos:%#llx\n", (uint64_t)pos);
  461. }
  462. list_del(pos);
  463. kfree_sensitive(entry);
  464. mutex_unlock(&bridge_postprocess_lock);
  465. if (entry) {
  466. do {
  467. ret = qtee_shmbridge_deregister(handle);
  468. if (unlikely(ret)) {
  469. pr_err_ratelimited("SHM failed: ret:%d ptr:0x%x h:%#llx\n",
  470. ret,
  471. dmabuf_to_free,
  472. handle);
  473. } else {
  474. pr_debug("SHM deletion: Handle:%#llx\n",
  475. handle);
  476. dma_buf_put(dmabuf_to_free);
  477. }
  478. } while (-EBUSY == ret);
  479. }
  480. } while (1);
  481. }
  482. static int smcinvoke_release_tz_object(struct qtee_shm *in_shm, struct qtee_shm *out_shm,
  483. uint32_t tzhandle, uint32_t context_type)
  484. {
  485. int ret = 0;
  486. bool release_handles;
  487. uint8_t *in_buf = NULL;
  488. uint8_t *out_buf = NULL;
  489. struct smcinvoke_msg_hdr hdr = {0};
  490. struct smcinvoke_cmd_req req = {0};
  491. in_buf = in_shm->vaddr;
  492. out_buf = out_shm->vaddr;
  493. hdr.tzhandle = tzhandle;
  494. hdr.op = OBJECT_OP_RELEASE;
  495. hdr.counts = 0;
  496. *(struct smcinvoke_msg_hdr *)in_buf = hdr;
  497. ret = prepare_send_scm_msg(in_buf, in_shm->paddr,
  498. SMCINVOKE_TZ_MIN_BUF_SIZE, out_buf, out_shm->paddr,
  499. SMCINVOKE_TZ_MIN_BUF_SIZE, &req, NULL,
  500. &release_handles, context_type, in_shm, out_shm, false);
  501. process_piggyback_data(out_buf, SMCINVOKE_TZ_MIN_BUF_SIZE);
  502. if (ret) {
  503. pr_err_ratelimited("Failed to release object(0x%x), ret:%d\n",
  504. hdr.tzhandle, ret);
  505. } else {
  506. pr_debug("Released object(0x%x) successfully.\n",
  507. hdr.tzhandle);
  508. }
  509. return ret;
  510. }
  511. static int smcinvoke_object_post_process(void)
  512. {
  513. struct smcinvoke_object_release_pending_list *entry = NULL;
  514. struct list_head *pos;
  515. int ret = 0;
  516. struct qtee_shm in_shm = {0}, out_shm = {0};
  517. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &in_shm);
  518. if (ret) {
  519. ret = -ENOMEM;
  520. pr_err("shmbridge alloc failed for in msg in object release\n");
  521. goto out;
  522. }
  523. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &out_shm);
  524. if (ret) {
  525. ret = -ENOMEM;
  526. pr_err("shmbridge alloc failed for out msg in object release\n");
  527. goto out;
  528. }
  529. do {
  530. mutex_lock(&object_postprocess_lock);
  531. if (list_empty(&g_object_postprocess)) {
  532. mutex_unlock(&object_postprocess_lock);
  533. break;
  534. }
  535. pos = g_object_postprocess.next;
  536. entry = list_entry(pos, struct smcinvoke_object_release_pending_list, list);
  537. list_del(pos);
  538. mutex_unlock(&object_postprocess_lock);
  539. if (entry) {
  540. do {
  541. ret = smcinvoke_release_tz_object(&in_shm, &out_shm,
  542. entry->data.tzhandle, entry->data.context_type);
  543. } while (-EBUSY == ret);
  544. } else {
  545. pr_err("entry is NULL, pos:%#llx\n", (uint64_t)pos);
  546. }
  547. kfree_sensitive(entry);
  548. } while (1);
  549. out:
  550. qtee_shmbridge_free_shm(&in_shm);
  551. qtee_shmbridge_free_shm(&out_shm);
  552. return ret;
  553. }
  554. static void smcinvoke_start_adci_thread(void)
  555. {
  556. int32_t ret = OBJECT_ERROR;
  557. int retry_count = 0;
  558. ret = get_root_obj(&adci_rootEnv);
  559. if (ret) {
  560. pr_err("failed to get rootEnv for ADCI invoke thread. ret = %d\n", ret);
  561. /* Marking it Object_NULL in case of failure scenario in order to avoid
  562. * undefined behavior while relasing garbage adci_rootEnv object. */
  563. adci_rootEnv = Object_NULL;
  564. goto out;
  565. }
  566. /* Invoke call to QTEE which should never return if ADCI is supported */
  567. pr_debug("Invoking adciAccept method in QTEE\n");
  568. do {
  569. ret = IClientEnv_adciAccept(adci_rootEnv);
  570. if (ret == OBJECT_ERROR_BUSY) {
  571. pr_err_ratelimited("Secure side is busy,will retry after 5 ms, retry_count = %d\n",retry_count);
  572. msleep(SMCINVOKE_INTERFACE_BUSY_WAIT_MS);
  573. }
  574. } while ((ret == OBJECT_ERROR_BUSY) && (retry_count++ < SMCINVOKE_INTERFACE_MAX_RETRY));
  575. if (ret == OBJECT_ERROR_INVALID)
  576. pr_err("ADCI feature is not supported on this chipsets, ret = %d\n", ret);
  577. else
  578. pr_debug("Received response from QTEE, ret = %d\n", ret);
  579. out:
  580. /* Control should reach to this point only if ADCI feature is not supported by QTEE
  581. (or) ADCI thread held in QTEE is released. */
  582. Object_ASSIGN_NULL(adci_rootEnv);
  583. }
  584. static void __wakeup_postprocess_kthread(struct smcinvoke_worker_thread *smcinvoke)
  585. {
  586. if (smcinvoke) {
  587. atomic_set(&smcinvoke->postprocess_kthread_state,
  588. POST_KT_WAKEUP);
  589. wake_up_interruptible(&smcinvoke->postprocess_kthread_wq);
  590. } else {
  591. pr_err("Invalid smcinvoke pointer.\n");
  592. }
  593. }
  594. static int smcinvoke_postprocess_kthread_func(void *data)
  595. {
  596. struct smcinvoke_worker_thread *smcinvoke_wrk_trd = data;
  597. static const char *const tag[] = {"shmbridge","object","adci","invalid"};
  598. if (!smcinvoke_wrk_trd) {
  599. pr_err("Bad input.\n");
  600. return -EINVAL;
  601. }
  602. while (!kthread_should_stop()) {
  603. wait_event_interruptible(
  604. smcinvoke_wrk_trd->postprocess_kthread_wq,
  605. kthread_should_stop() ||
  606. (atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state)
  607. == POST_KT_WAKEUP));
  608. switch (smcinvoke_wrk_trd->type) {
  609. case SHMB_WORKER_THREAD:
  610. pr_debug("kthread to %s postprocess is called %d\n",
  611. tag[SHMB_WORKER_THREAD], atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state));
  612. smcinvoke_shmbridge_post_process();
  613. break;
  614. case OBJECT_WORKER_THREAD:
  615. pr_debug("kthread to %s postprocess is called %d\n",
  616. tag[OBJECT_WORKER_THREAD], atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state));
  617. smcinvoke_object_post_process();
  618. break;
  619. case ADCI_WORKER_THREAD:
  620. pr_debug("kthread to %s postprocess is called %d\n",
  621. tag[ADCI_WORKER_THREAD], atomic_read(&smcinvoke_wrk_trd->postprocess_kthread_state));
  622. smcinvoke_start_adci_thread();
  623. break;
  624. default:
  625. pr_err("Invalid thread type(%d), do nothing.\n",
  626. (int)smcinvoke_wrk_trd->type);
  627. break;
  628. }
  629. /* For ADCI thread, if control reaches here, that indicates either ADCI
  630. * thread is not supported (or) released by QTEE. Since ADCI thread is
  631. * getting signaled only during the smcinvoke driver initialization,
  632. * there is no point of putting the thread into sleep state again. All the
  633. * required post-processing will be taken care by object and shmbridge threads.
  634. */
  635. if(smcinvoke_wrk_trd->type == ADCI_WORKER_THREAD) {
  636. break;
  637. }
  638. atomic_set(&smcinvoke_wrk_trd->postprocess_kthread_state,
  639. POST_KT_SLEEP);
  640. }
  641. pr_warn("kthread(worker_thread) processed, worker_thread type is %d \n", smcinvoke_wrk_trd->type);
  642. return 0;
  643. }
  644. static int smcinvoke_create_kthreads(void)
  645. {
  646. int i, rc = 0;
  647. const enum worker_thread_type thread_type[MAX_THREAD_NUMBER] = {
  648. SHMB_WORKER_THREAD, OBJECT_WORKER_THREAD, ADCI_WORKER_THREAD};
  649. for (i = 0; i < MAX_THREAD_NUMBER; i++) {
  650. init_waitqueue_head(&smcinvoke[i].postprocess_kthread_wq);
  651. smcinvoke[i].type = thread_type[i];
  652. smcinvoke[i].postprocess_kthread_task = kthread_run(
  653. smcinvoke_postprocess_kthread_func,
  654. &smcinvoke[i], thread_name[i]);
  655. if (IS_ERR(smcinvoke[i].postprocess_kthread_task)) {
  656. rc = PTR_ERR(smcinvoke[i].postprocess_kthread_task);
  657. pr_err("fail to create kthread to postprocess, rc = %x\n",
  658. rc);
  659. return rc;
  660. }
  661. atomic_set(&smcinvoke[i].postprocess_kthread_state,
  662. POST_KT_SLEEP);
  663. }
  664. return rc;
  665. }
  666. static void smcinvoke_destroy_kthreads(void)
  667. {
  668. int i;
  669. int32_t ret = OBJECT_ERROR;
  670. int retry_count = 0;
  671. if (!Object_isNull(adci_rootEnv)) {
  672. pr_debug("Invoking adciShutdown method in QTEE\n");
  673. do {
  674. ret = IClientEnv_adciShutdown(adci_rootEnv);
  675. if (ret == OBJECT_ERROR_BUSY) {
  676. pr_err_ratelimited("Secure side is busy,will retry after 5 ms, retry_count = %d\n",retry_count);
  677. msleep(SMCINVOKE_INTERFACE_BUSY_WAIT_MS);
  678. }
  679. } while ((ret == OBJECT_ERROR_BUSY) && (retry_count++ < SMCINVOKE_INTERFACE_MAX_RETRY));
  680. if (OBJECT_isERROR(ret)) {
  681. pr_err("adciShutdown in QTEE failed with error = %d\n", ret);
  682. }
  683. Object_ASSIGN_NULL(adci_rootEnv);
  684. }
  685. for (i = 0; i < MAX_THREAD_NUMBER; i++) {
  686. kthread_stop(smcinvoke[i].postprocess_kthread_task);
  687. }
  688. }
  689. /* Queue newly created memory object to l_pending_mem_obj list.
  690. * Later, the mapping information for objects in this list will be sent to TZ
  691. * over the async side channel.
  692. *
  693. * No return value as TZ is always able to explicitly ask for this information
  694. * in case this function fails and the memory object is not added to this list.
  695. */
  696. static void queue_mem_obj_pending_async_locked(struct smcinvoke_mem_obj *mem_obj, struct list_head *l_pending_mem_obj)
  697. {
  698. struct smcinvoke_mem_obj_pending_async *t_mem_obj_pending =
  699. kzalloc(sizeof(*t_mem_obj_pending), GFP_KERNEL);
  700. /*
  701. * We are not failing execution in case of a failure here,
  702. * since TZ can always ask for this information explicitly
  703. * if it's not available in the side channel.
  704. */
  705. if (!t_mem_obj_pending) {
  706. pr_err("Unable to allocate memory\n");
  707. return;
  708. }
  709. t_mem_obj_pending->mem_obj = mem_obj;
  710. list_add(&t_mem_obj_pending->list, l_pending_mem_obj);
  711. }
  712. static inline void free_mem_obj_locked(struct smcinvoke_mem_obj *mem_obj)
  713. {
  714. int ret = 0;
  715. struct dma_buf *dmabuf_to_free = mem_obj->dma_buf;
  716. uint64_t shmbridge_handle = mem_obj->shmbridge_handle;
  717. struct smcinvoke_shmbridge_deregister_pending_list *entry = NULL;
  718. list_del(&mem_obj->list);
  719. kfree(mem_obj->server);
  720. kfree(mem_obj);
  721. mem_obj = NULL;
  722. mutex_unlock(&g_smcinvoke_lock);
  723. if (shmbridge_handle)
  724. ret = qtee_shmbridge_deregister(shmbridge_handle);
  725. if (ret) {
  726. pr_err("Error:%d delete bridge failed leaking memory 0x%x\n",
  727. ret, dmabuf_to_free);
  728. if (ret == -EBUSY) {
  729. pr_err("EBUSY: we postpone it 0x%x\n",
  730. dmabuf_to_free);
  731. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  732. if (entry) {
  733. entry->data.shmbridge_handle = shmbridge_handle;
  734. entry->data.dmabuf_to_free = dmabuf_to_free;
  735. mutex_lock(&bridge_postprocess_lock);
  736. list_add_tail(&entry->list, &g_bridge_postprocess);
  737. mutex_unlock(&bridge_postprocess_lock);
  738. pr_debug("SHMBridge list: added a Handle:%#llx\n",
  739. shmbridge_handle);
  740. __wakeup_postprocess_kthread(
  741. &smcinvoke[SHMB_WORKER_THREAD]);
  742. }
  743. }
  744. } else {
  745. dma_buf_put(dmabuf_to_free);
  746. }
  747. mutex_lock(&g_smcinvoke_lock);
  748. }
  749. static void del_mem_regn_obj_locked(struct kref *kref)
  750. {
  751. struct smcinvoke_mem_obj *mem_obj = container_of(kref,
  752. struct smcinvoke_mem_obj, mem_regn_ref_cnt);
  753. /*
  754. * mem_regn obj and mem_map obj are held into mem_obj structure which
  755. * can't be released until both kinds of objs have been released.
  756. * So check whether mem_map iobj has ref 0 and only then release mem_obj
  757. */
  758. if (kref_read(&mem_obj->mem_map_obj_ref_cnt) == 0)
  759. free_mem_obj_locked(mem_obj);
  760. }
  761. static void del_mem_map_obj_locked(struct kref *kref)
  762. {
  763. struct smcinvoke_mem_obj *mem_obj = container_of(kref,
  764. struct smcinvoke_mem_obj, mem_map_obj_ref_cnt);
  765. mem_obj->p_addr_len = 0;
  766. mem_obj->p_addr = 0;
  767. if (mem_obj->sgt)
  768. dma_buf_unmap_attachment(mem_obj->buf_attach,
  769. mem_obj->sgt, DMA_BIDIRECTIONAL);
  770. if (mem_obj->buf_attach)
  771. dma_buf_detach(mem_obj->dma_buf, mem_obj->buf_attach);
  772. /*
  773. * mem_regn obj and mem_map obj are held into mem_obj structure which
  774. * can't be released until both kinds of objs have been released.
  775. * So check if mem_regn obj has ref 0 and only then release mem_obj
  776. */
  777. if (kref_read(&mem_obj->mem_regn_ref_cnt) == 0)
  778. free_mem_obj_locked(mem_obj);
  779. }
  780. static int release_mem_obj_locked(int32_t tzhandle)
  781. {
  782. int is_mem_regn_obj = TZHANDLE_IS_MEM_RGN_OBJ(tzhandle);
  783. struct smcinvoke_mem_obj *mem_obj = find_mem_obj_locked(
  784. TZHANDLE_GET_OBJID(tzhandle), is_mem_regn_obj);
  785. if (!mem_obj) {
  786. pr_err("memory object not found\n");
  787. return OBJECT_ERROR_BADOBJ;
  788. }
  789. if (is_mem_regn_obj)
  790. kref_put(&mem_obj->mem_regn_ref_cnt, del_mem_regn_obj_locked);
  791. else
  792. kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  793. return OBJECT_OK;
  794. }
  795. static void free_pending_cbobj_locked(struct kref *kref)
  796. {
  797. struct smcinvoke_server_info *server = NULL;
  798. struct smcinvoke_cbobj *obj = container_of(kref,
  799. struct smcinvoke_cbobj, ref_cnt);
  800. list_del(&obj->list);
  801. server = obj->server;
  802. kfree(obj);
  803. if (server)
  804. kref_put(&server->ref_cnt, destroy_cb_server);
  805. }
  806. static int get_pending_cbobj_locked(uint16_t srvr_id, int16_t obj_id)
  807. {
  808. int ret = 0;
  809. bool release_server = true;
  810. struct list_head *head = NULL;
  811. struct smcinvoke_cbobj *cbobj = NULL;
  812. struct smcinvoke_cbobj *obj = NULL;
  813. struct smcinvoke_server_info *server = get_cb_server_locked(srvr_id);
  814. if (!server) {
  815. pr_err("%s, server id : %u not found\n", __func__, srvr_id);
  816. return OBJECT_ERROR_BADOBJ;
  817. }
  818. head = &server->pending_cbobjs;
  819. list_for_each_entry(cbobj, head, list)
  820. if (cbobj->cbobj_id == obj_id) {
  821. kref_get(&cbobj->ref_cnt);
  822. goto out;
  823. }
  824. obj = kzalloc(sizeof(*obj), GFP_KERNEL);
  825. if (!obj) {
  826. ret = OBJECT_ERROR_KMEM;
  827. goto out;
  828. }
  829. obj->cbobj_id = obj_id;
  830. kref_init(&obj->ref_cnt);
  831. obj->server = server;
  832. /*
  833. * we are holding server ref in cbobj; we will
  834. * release server ref when cbobj is destroyed
  835. */
  836. release_server = false;
  837. list_add_tail(&obj->list, head);
  838. out:
  839. if (release_server)
  840. kref_put(&server->ref_cnt, destroy_cb_server);
  841. return ret;
  842. }
  843. static int put_pending_cbobj_locked(uint16_t srvr_id, int16_t obj_id)
  844. {
  845. int ret = -EINVAL;
  846. struct smcinvoke_server_info *srvr_info =
  847. get_cb_server_locked(srvr_id);
  848. struct list_head *head = NULL;
  849. struct smcinvoke_cbobj *cbobj = NULL;
  850. if (!srvr_info) {
  851. pr_err("%s, server id : %u not found\n", __func__, srvr_id);
  852. return ret;
  853. }
  854. trace_put_pending_cbobj_locked(srvr_id, obj_id);
  855. head = &srvr_info->pending_cbobjs;
  856. list_for_each_entry(cbobj, head, list)
  857. if (cbobj->cbobj_id == obj_id) {
  858. kref_put(&cbobj->ref_cnt, free_pending_cbobj_locked);
  859. ret = 0;
  860. break;
  861. }
  862. kref_put(&srvr_info->ref_cnt, destroy_cb_server);
  863. return ret;
  864. }
  865. static int release_tzhandle_locked(int32_t tzhandle)
  866. {
  867. if (TZHANDLE_IS_MEM_OBJ(tzhandle))
  868. return release_mem_obj_locked(tzhandle);
  869. else if (TZHANDLE_IS_CB_OBJ(tzhandle))
  870. return put_pending_cbobj_locked(TZHANDLE_GET_SERVER(tzhandle),
  871. TZHANDLE_GET_OBJID(tzhandle));
  872. return OBJECT_ERROR;
  873. }
  874. static void release_tzhandles(const int32_t *tzhandles, size_t len)
  875. {
  876. size_t i;
  877. mutex_lock(&g_smcinvoke_lock);
  878. for (i = 0; i < len; i++)
  879. release_tzhandle_locked(tzhandles[i]);
  880. mutex_unlock(&g_smcinvoke_lock);
  881. }
  882. static void delete_cb_txn_locked(struct kref *kref)
  883. {
  884. struct smcinvoke_cb_txn *cb_txn = container_of(kref,
  885. struct smcinvoke_cb_txn, ref_cnt);
  886. if (OBJECT_OP_METHODID(cb_txn->cb_req->hdr.op) == OBJECT_OP_RELEASE)
  887. release_tzhandle_locked(cb_txn->cb_req->hdr.tzhandle);
  888. kfree(cb_txn->cb_req);
  889. hash_del(&cb_txn->hash);
  890. kfree(cb_txn);
  891. }
  892. static struct smcinvoke_cb_txn *find_cbtxn_locked(
  893. struct smcinvoke_server_info *server,
  894. uint32_t txn_id, int32_t state)
  895. {
  896. int i = 0;
  897. struct smcinvoke_cb_txn *cb_txn = NULL;
  898. /*
  899. * Since HASH_BITS() does not work on pointers, we can't select hash
  900. * table using state and loop over it.
  901. */
  902. if (state == SMCINVOKE_REQ_PLACED) {
  903. /* pick up 1st req */
  904. hash_for_each(server->reqs_table, i, cb_txn, hash) {
  905. kref_get(&cb_txn->ref_cnt);
  906. hash_del(&cb_txn->hash);
  907. return cb_txn;
  908. }
  909. } else if (state == SMCINVOKE_REQ_PROCESSING) {
  910. hash_for_each_possible(
  911. server->responses_table, cb_txn, hash, txn_id) {
  912. if (cb_txn->txn_id == txn_id) {
  913. kref_get(&cb_txn->ref_cnt);
  914. hash_del(&cb_txn->hash);
  915. return cb_txn;
  916. }
  917. }
  918. }
  919. return NULL;
  920. }
  921. /*
  922. * size_add_ saturates at SIZE_MAX. If integer overflow is detected,
  923. * this function would return SIZE_MAX otherwise normal a+b is returned.
  924. */
  925. static inline size_t size_add_(size_t a, size_t b)
  926. {
  927. return (b > (SIZE_MAX - a)) ? SIZE_MAX : a + b;
  928. }
  929. /*
  930. * pad_size is used along with size_align to define a buffer overflow
  931. * protected version of ALIGN
  932. */
  933. static inline size_t pad_size(size_t a, size_t b)
  934. {
  935. return (~a + 1) % b;
  936. }
  937. /*
  938. * size_align saturates at SIZE_MAX. If integer overflow is detected, this
  939. * function would return SIZE_MAX otherwise next aligned size is returned.
  940. */
  941. static inline size_t size_align(size_t a, size_t b)
  942. {
  943. return size_add_(a, pad_size(a, b));
  944. }
  945. static uint16_t get_server_id(int cb_server_fd)
  946. {
  947. uint16_t server_id = 0;
  948. struct smcinvoke_file_data *svr_cxt = NULL;
  949. struct file *tmp_filp = fget(cb_server_fd);
  950. if (!tmp_filp || !FILE_IS_REMOTE_OBJ(tmp_filp))
  951. return server_id;
  952. svr_cxt = tmp_filp->private_data;
  953. if (svr_cxt && svr_cxt->context_type == SMCINVOKE_OBJ_TYPE_SERVER)
  954. server_id = svr_cxt->server_id;
  955. fput(tmp_filp);
  956. return server_id;
  957. }
  958. static bool is_dma_fd(int32_t uhandle, struct dma_buf **dma_buf)
  959. {
  960. *dma_buf = dma_buf_get(uhandle);
  961. return IS_ERR_OR_NULL(*dma_buf) ? false : true;
  962. }
  963. static bool is_remote_obj(int32_t uhandle, struct smcinvoke_file_data **tzobj,
  964. struct file **filp)
  965. {
  966. bool ret = false;
  967. struct file *tmp_filp = fget(uhandle);
  968. if (!tmp_filp)
  969. return ret;
  970. if (FILE_IS_REMOTE_OBJ(tmp_filp)) {
  971. *tzobj = tmp_filp->private_data;
  972. if ((*tzobj)->context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  973. *filp = tmp_filp;
  974. tmp_filp = NULL;
  975. ret = true;
  976. }
  977. }
  978. if (tmp_filp)
  979. fput(tmp_filp);
  980. return ret;
  981. }
  982. static int smcinvoke_create_bridge(struct smcinvoke_mem_obj *mem_obj)
  983. {
  984. int ret = 0;
  985. int tz_perm = PERM_READ|PERM_WRITE;
  986. uint32_t *vmid_list;
  987. uint32_t *perms_list;
  988. uint32_t nelems = 0;
  989. struct dma_buf *dmabuf = mem_obj->dma_buf;
  990. phys_addr_t phys = mem_obj->p_addr;
  991. size_t size = mem_obj->p_addr_len;
  992. if (!qtee_shmbridge_is_enabled())
  993. return 0;
  994. ret = mem_buf_dma_buf_copy_vmperm(dmabuf, (int **)&vmid_list,
  995. (int **)&perms_list, (int *)&nelems);
  996. if (ret) {
  997. pr_err("mem_buf_dma_buf_copy_vmperm failure, err=%d\n", ret);
  998. return ret;
  999. }
  1000. if (mem_buf_dma_buf_exclusive_owner(dmabuf))
  1001. perms_list[0] = PERM_READ | PERM_WRITE;
  1002. ret = qtee_shmbridge_register(phys, size, vmid_list, perms_list, nelems,
  1003. tz_perm, &mem_obj->shmbridge_handle);
  1004. if (ret) {
  1005. pr_err("creation of shm bridge for mem_region_id %d failed ret %d\n",
  1006. mem_obj->mem_region_id, ret);
  1007. goto exit;
  1008. }
  1009. trace_smcinvoke_create_bridge(mem_obj->shmbridge_handle, mem_obj->mem_region_id);
  1010. exit:
  1011. kfree(perms_list);
  1012. kfree(vmid_list);
  1013. return ret;
  1014. }
  1015. /* Map memory region for a given memory object.
  1016. * Mapping information will be saved as part of the memory object structure.
  1017. */
  1018. static int32_t smcinvoke_map_mem_region_locked(struct smcinvoke_mem_obj* mem_obj)
  1019. {
  1020. int ret = OBJECT_OK;
  1021. struct dma_buf_attachment *buf_attach = NULL;
  1022. struct sg_table *sgt = NULL;
  1023. if (!mem_obj) {
  1024. pr_err("Invalid memory object\n");
  1025. return OBJECT_ERROR_BADOBJ;
  1026. }
  1027. if (!mem_obj->p_addr) {
  1028. kref_init(&mem_obj->mem_map_obj_ref_cnt);
  1029. buf_attach = dma_buf_attach(mem_obj->dma_buf,
  1030. &smcinvoke_pdev->dev);
  1031. if (IS_ERR(buf_attach)) {
  1032. ret = OBJECT_ERROR_KMEM;
  1033. pr_err("dma buf attach failed, ret: %d\n", ret);
  1034. goto out;
  1035. }
  1036. mem_obj->buf_attach = buf_attach;
  1037. sgt = dma_buf_map_attachment(buf_attach, DMA_BIDIRECTIONAL);
  1038. if (IS_ERR(sgt)) {
  1039. pr_err("mapping dma buffers failed, ret: %d\n",
  1040. PTR_ERR(sgt));
  1041. ret = OBJECT_ERROR_KMEM;
  1042. goto out;
  1043. }
  1044. mem_obj->sgt = sgt;
  1045. /* contiguous only => nents=1 */
  1046. if (sgt->nents != 1) {
  1047. ret = OBJECT_ERROR_INVALID;
  1048. pr_err("sg enries are not contigous, ret: %d\n", ret);
  1049. goto out;
  1050. }
  1051. mem_obj->p_addr = sg_dma_address(sgt->sgl);
  1052. mem_obj->p_addr_len = sgt->sgl->length;
  1053. if (!mem_obj->p_addr) {
  1054. ret = OBJECT_ERROR_INVALID;
  1055. pr_err("invalid physical address, ret: %d\n", ret);
  1056. goto out;
  1057. }
  1058. /* Increase reference count as we are feeding the memobj to
  1059. * smcinvoke and unlock the mutex. No need to hold the mutex in
  1060. * case of shmbridge creation.
  1061. */
  1062. kref_get(&mem_obj->mem_map_obj_ref_cnt);
  1063. mutex_unlock(&g_smcinvoke_lock);
  1064. ret = smcinvoke_create_bridge(mem_obj);
  1065. /* Take lock again and decrease the reference count which we
  1066. * increased for shmbridge but before proceeding further we
  1067. * have to check again if the memobj is still valid or not
  1068. * after decreasing the reference.
  1069. */
  1070. mutex_lock(&g_smcinvoke_lock);
  1071. kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  1072. if (ret) {
  1073. ret = OBJECT_ERROR_INVALID;
  1074. pr_err("Unable to create shm bridge, ret: %d\n", ret);
  1075. goto out;
  1076. }
  1077. if (!find_mem_obj_locked(mem_obj->mem_region_id,
  1078. SMCINVOKE_MEM_RGN_OBJ)) {
  1079. mutex_unlock(&g_smcinvoke_lock);
  1080. pr_err("Memory object not found\n");
  1081. return OBJECT_ERROR_BADOBJ;
  1082. }
  1083. mem_obj->mem_map_obj_id = next_mem_map_obj_id_locked();
  1084. }
  1085. out:
  1086. if (ret != OBJECT_OK)
  1087. kref_put(&mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  1088. return ret;
  1089. }
  1090. static int create_mem_obj(struct dma_buf *dma_buf, int32_t *tzhandle,
  1091. struct smcinvoke_mem_obj **mem_obj, int32_t server_id, int32_t user_handle)
  1092. {
  1093. struct smcinvoke_mem_obj *t_mem_obj = NULL;
  1094. struct smcinvoke_server_info *server_i = NULL;
  1095. t_mem_obj = kzalloc(sizeof(struct smcinvoke_mem_obj), GFP_KERNEL);
  1096. if (!t_mem_obj) {
  1097. dma_buf_put(dma_buf);
  1098. return -ENOMEM;
  1099. }
  1100. server_i = kzalloc(sizeof(struct smcinvoke_server_info),GFP_KERNEL);
  1101. if (!server_i) {
  1102. kfree(t_mem_obj);
  1103. dma_buf_put(dma_buf);
  1104. return -ENOMEM;
  1105. }
  1106. kref_init(&t_mem_obj->mem_regn_ref_cnt);
  1107. t_mem_obj->dma_buf = dma_buf;
  1108. mutex_lock(&g_smcinvoke_lock);
  1109. t_mem_obj->mem_region_id = next_mem_region_obj_id_locked();
  1110. server_i->server_id = server_id;
  1111. t_mem_obj->server = server_i;
  1112. t_mem_obj->mem_obj_user_fd = user_handle;
  1113. list_add_tail(&t_mem_obj->list, &g_mem_objs);
  1114. mutex_unlock(&g_smcinvoke_lock);
  1115. *mem_obj = t_mem_obj;
  1116. *tzhandle = TZHANDLE_MAKE_LOCAL(MEM_RGN_SRVR_ID,
  1117. t_mem_obj->mem_region_id);
  1118. return 0;
  1119. }
  1120. /*
  1121. * This function retrieves file pointer corresponding to FD provided. It stores
  1122. * retrieved file pointer until IOCTL call is concluded. Once call is completed,
  1123. * all stored file pointers are released. file pointers are stored to prevent
  1124. * other threads from releasing that FD while IOCTL is in progress.
  1125. */
  1126. static int get_tzhandle_from_uhandle(int32_t uhandle, int32_t server_fd,
  1127. struct file **filp, uint32_t *tzhandle, struct list_head *l_pending_mem_obj)
  1128. {
  1129. int ret = -EBADF;
  1130. uint16_t server_id = 0;
  1131. struct smcinvoke_mem_obj *mem_obj = NULL;
  1132. if (UHANDLE_IS_NULL(uhandle)) {
  1133. *tzhandle = SMCINVOKE_TZ_OBJ_NULL;
  1134. ret = 0;
  1135. } else if (UHANDLE_IS_CB_OBJ(uhandle)) {
  1136. server_id = get_server_id(server_fd);
  1137. if (server_id < CBOBJ_SERVER_ID_START)
  1138. goto out;
  1139. mutex_lock(&g_smcinvoke_lock);
  1140. ret = get_pending_cbobj_locked(server_id,
  1141. UHANDLE_GET_CB_OBJ(uhandle));
  1142. mutex_unlock(&g_smcinvoke_lock);
  1143. if (ret)
  1144. goto out;
  1145. *tzhandle = TZHANDLE_MAKE_LOCAL(server_id,
  1146. UHANDLE_GET_CB_OBJ(uhandle));
  1147. ret = 0;
  1148. } else if (UHANDLE_IS_FD(uhandle)) {
  1149. struct dma_buf *dma_buf = NULL;
  1150. struct smcinvoke_file_data *tzobj = NULL;
  1151. if (is_dma_fd(UHANDLE_GET_FD(uhandle), &dma_buf)) {
  1152. server_id = get_server_id(server_fd);
  1153. ret = create_mem_obj(dma_buf, tzhandle, &mem_obj, server_id, uhandle);
  1154. if (!ret && mem_obj_async_support && l_pending_mem_obj) {
  1155. mutex_lock(&g_smcinvoke_lock);
  1156. /* Map the newly created memory object and add it
  1157. * to l_pending_mem_obj list.
  1158. * Before returning to TZ, add the mapping data
  1159. * to the async side channel so it's available to TZ
  1160. * together with the memory object.
  1161. */
  1162. if (!smcinvoke_map_mem_region_locked(mem_obj)) {
  1163. queue_mem_obj_pending_async_locked(mem_obj, l_pending_mem_obj);
  1164. } else {
  1165. pr_err("Failed to map memory region\n");
  1166. }
  1167. mutex_unlock(&g_smcinvoke_lock);
  1168. }
  1169. } else if (is_remote_obj(UHANDLE_GET_FD(uhandle),
  1170. &tzobj, filp)) {
  1171. *tzhandle = tzobj->tzhandle;
  1172. ret = 0;
  1173. }
  1174. }
  1175. out:
  1176. return ret;
  1177. }
  1178. static int get_fd_for_obj(uint32_t obj_type, uint32_t obj, int32_t *fd)
  1179. {
  1180. int unused_fd = -1, ret = -EINVAL;
  1181. struct file *f = NULL;
  1182. struct smcinvoke_file_data *cxt = NULL;
  1183. cxt = kzalloc(sizeof(*cxt), GFP_KERNEL);
  1184. if (!cxt) {
  1185. ret = -ENOMEM;
  1186. goto out;
  1187. }
  1188. if (obj_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ ||
  1189. obj_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  1190. cxt->context_type = obj_type;
  1191. cxt->tzhandle = obj;
  1192. } else if (obj_type == SMCINVOKE_OBJ_TYPE_SERVER) {
  1193. cxt->context_type = SMCINVOKE_OBJ_TYPE_SERVER;
  1194. cxt->server_id = obj;
  1195. } else {
  1196. goto out;
  1197. }
  1198. unused_fd = get_unused_fd_flags(O_RDWR);
  1199. if (unused_fd < 0)
  1200. goto out;
  1201. if (fd == NULL)
  1202. goto out;
  1203. f = anon_inode_getfile(SMCINVOKE_DEV, &g_smcinvoke_fops, cxt, O_RDWR);
  1204. if (IS_ERR(f))
  1205. goto out;
  1206. *fd = unused_fd;
  1207. fd_install(*fd, f);
  1208. return 0;
  1209. out:
  1210. if (unused_fd >= 0)
  1211. put_unused_fd(unused_fd);
  1212. kfree(cxt);
  1213. return ret;
  1214. }
  1215. static int get_uhandle_from_tzhandle(int32_t tzhandle, int32_t srvr_id,
  1216. int32_t *uhandle, bool lock, uint32_t context_type)
  1217. {
  1218. int ret = -1;
  1219. if (TZHANDLE_IS_NULL(tzhandle)) {
  1220. *uhandle = UHANDLE_NULL;
  1221. ret = 0;
  1222. } else if (TZHANDLE_IS_CB_OBJ(tzhandle)) {
  1223. if (srvr_id != TZHANDLE_GET_SERVER(tzhandle))
  1224. goto out;
  1225. *uhandle = UHANDLE_MAKE_CB_OBJ(TZHANDLE_GET_OBJID(tzhandle));
  1226. MUTEX_LOCK(lock)
  1227. ret = get_pending_cbobj_locked(TZHANDLE_GET_SERVER(tzhandle),
  1228. TZHANDLE_GET_OBJID(tzhandle));
  1229. MUTEX_UNLOCK(lock)
  1230. } else if (TZHANDLE_IS_MEM_RGN_OBJ(tzhandle)) {
  1231. struct smcinvoke_mem_obj *mem_obj = NULL;
  1232. MUTEX_LOCK(lock)
  1233. mem_obj = find_mem_obj_locked(TZHANDLE_GET_OBJID(tzhandle),
  1234. SMCINVOKE_MEM_RGN_OBJ);
  1235. if (mem_obj != NULL) {
  1236. int fd;
  1237. fd = mem_obj->mem_obj_user_fd;
  1238. if (fd < 0)
  1239. goto exit_lock;
  1240. *uhandle = fd;
  1241. ret = 0;
  1242. }
  1243. exit_lock:
  1244. MUTEX_UNLOCK(lock)
  1245. } else if (TZHANDLE_IS_REMOTE(tzhandle)) {
  1246. /* if execution comes here => tzhandle is an unsigned int */
  1247. ret = get_fd_for_obj(context_type,
  1248. (uint32_t)tzhandle, uhandle);
  1249. }
  1250. out:
  1251. return ret;
  1252. }
  1253. static int32_t smcinvoke_release_mem_obj_locked(void *buf, size_t buf_len)
  1254. {
  1255. struct smcinvoke_tzcb_req *msg = buf;
  1256. if (msg->hdr.counts != OBJECT_COUNTS_PACK(0, 0, 0, 0)) {
  1257. pr_err("Invalid object count in %s\n", __func__);
  1258. return OBJECT_ERROR_INVALID;
  1259. }
  1260. trace_release_mem_obj_locked(msg->hdr.tzhandle, buf_len);
  1261. return release_tzhandle_locked(msg->hdr.tzhandle);
  1262. }
  1263. static int32_t smcinvoke_process_map_mem_region_req(void *buf, size_t buf_len)
  1264. {
  1265. int ret = OBJECT_OK;
  1266. struct smcinvoke_tzcb_req *msg = buf;
  1267. struct {
  1268. uint64_t p_addr;
  1269. uint64_t len;
  1270. uint32_t perms;
  1271. } *ob = NULL;
  1272. int32_t *oo = NULL;
  1273. struct smcinvoke_mem_obj *mem_obj = NULL;
  1274. if (msg->hdr.counts != OBJECT_COUNTS_PACK(0, 1, 1, 1) ||
  1275. (buf_len - msg->args[0].b.offset < msg->args[0].b.size)) {
  1276. pr_err("Invalid counts received for mapping mem obj\n");
  1277. return OBJECT_ERROR_INVALID;
  1278. }
  1279. /* args[0] = BO, args[1] = OI, args[2] = OO */
  1280. ob = buf + msg->args[0].b.offset;
  1281. oo = &msg->args[2].handle;
  1282. mutex_lock(&g_smcinvoke_lock);
  1283. mem_obj = find_mem_obj_locked(TZHANDLE_GET_OBJID(msg->args[1].handle),
  1284. SMCINVOKE_MEM_RGN_OBJ);
  1285. if (!mem_obj) {
  1286. mutex_unlock(&g_smcinvoke_lock);
  1287. pr_err("Memory object not found\n");
  1288. return OBJECT_ERROR_BADOBJ;
  1289. }
  1290. if (!mem_obj->p_addr) {
  1291. ret = smcinvoke_map_mem_region_locked(mem_obj);
  1292. } else {
  1293. kref_get(&mem_obj->mem_map_obj_ref_cnt);
  1294. }
  1295. if (!ret) {
  1296. ob->p_addr = mem_obj->p_addr;
  1297. ob->len = mem_obj->p_addr_len;
  1298. ob->perms = SMCINVOKE_MEM_PERM_RW;
  1299. *oo = TZHANDLE_MAKE_LOCAL(MEM_MAP_SRVR_ID, mem_obj->mem_map_obj_id);
  1300. }
  1301. mutex_unlock(&g_smcinvoke_lock);
  1302. return ret;
  1303. }
  1304. static int32_t smcinvoke_sleep(void *buf, size_t buf_len)
  1305. {
  1306. struct smcinvoke_tzcb_req *msg = buf;
  1307. uint32_t sleepTimeMs_val = 0;
  1308. if (msg->hdr.counts != OBJECT_COUNTS_PACK(1, 0, 0, 0) ||
  1309. (buf_len - msg->args[0].b.offset < msg->args[0].b.size)) {
  1310. pr_err("Invalid counts received for sleeping in hlos\n");
  1311. return OBJECT_ERROR_INVALID;
  1312. }
  1313. /* Time in miliseconds is expected from tz */
  1314. sleepTimeMs_val = *((uint32_t *)(buf + msg->args[0].b.offset));
  1315. msleep(sleepTimeMs_val);
  1316. return OBJECT_OK;
  1317. }
  1318. static void process_kernel_obj(void *buf, size_t buf_len)
  1319. {
  1320. struct smcinvoke_tzcb_req *cb_req = buf;
  1321. switch (cb_req->hdr.op) {
  1322. case OBJECT_OP_MAP_REGION:
  1323. if (mem_obj_async_support) {
  1324. /* Mapping requests are not supposed to come
  1325. * from TZ once memory object async support
  1326. * is enabled.
  1327. * If they are still coming, we would like to
  1328. * know about it.
  1329. */
  1330. pr_info("Received a request to map memory region\n");
  1331. }
  1332. cb_req->result = smcinvoke_process_map_mem_region_req(buf, buf_len);
  1333. break;
  1334. case OBJECT_OP_YIELD:
  1335. cb_req->result = OBJECT_OK;
  1336. break;
  1337. case OBJECT_OP_SLEEP:
  1338. cb_req->result = smcinvoke_sleep(buf, buf_len);
  1339. break;
  1340. default:
  1341. pr_err(" invalid operation for tz kernel object\n");
  1342. cb_req->result = OBJECT_ERROR_INVALID;
  1343. break;
  1344. }
  1345. }
  1346. static void process_mem_obj(void *buf, size_t buf_len)
  1347. {
  1348. struct smcinvoke_tzcb_req *cb_req = buf;
  1349. mutex_lock(&g_smcinvoke_lock);
  1350. cb_req->result = (cb_req->hdr.op == OBJECT_OP_RELEASE) ?
  1351. smcinvoke_release_mem_obj_locked(buf, buf_len) :
  1352. OBJECT_ERROR_INVALID;
  1353. mutex_unlock(&g_smcinvoke_lock);
  1354. }
  1355. static int invoke_cmd_handler(int cmd, phys_addr_t in_paddr, size_t in_buf_len,
  1356. uint8_t *out_buf, phys_addr_t out_paddr,
  1357. size_t out_buf_len, int32_t *result, u64 *response_type,
  1358. unsigned int *data, struct qtee_shm *in_shm,
  1359. struct qtee_shm *out_shm)
  1360. {
  1361. int ret = 0;
  1362. switch (cmd) {
  1363. case SMCINVOKE_INVOKE_CMD_LEGACY:
  1364. qtee_shmbridge_flush_shm_buf(in_shm);
  1365. qtee_shmbridge_flush_shm_buf(out_shm);
  1366. ret = qcom_scm_invoke_smc_legacy(in_paddr, in_buf_len, out_paddr, out_buf_len,
  1367. result, response_type, data);
  1368. qtee_shmbridge_inv_shm_buf(in_shm);
  1369. qtee_shmbridge_inv_shm_buf(out_shm);
  1370. break;
  1371. case SMCINVOKE_INVOKE_CMD:
  1372. ret = qcom_scm_invoke_smc(in_paddr, in_buf_len, out_paddr, out_buf_len,
  1373. result, response_type, data);
  1374. break;
  1375. case SMCINVOKE_CB_RSP_CMD:
  1376. if (legacy_smc_call)
  1377. qtee_shmbridge_flush_shm_buf(out_shm);
  1378. ret = qcom_scm_invoke_callback_response(virt_to_phys(out_buf), out_buf_len,
  1379. result, response_type, data);
  1380. if (legacy_smc_call) {
  1381. qtee_shmbridge_inv_shm_buf(in_shm);
  1382. qtee_shmbridge_inv_shm_buf(out_shm);
  1383. }
  1384. break;
  1385. default:
  1386. ret = -EINVAL;
  1387. break;
  1388. }
  1389. trace_invoke_cmd_handler(cmd, *response_type, *result, ret);
  1390. return ret;
  1391. }
  1392. /*
  1393. * Buf should be aligned to struct smcinvoke_tzcb_req
  1394. */
  1395. static void process_tzcb_req(void *buf, size_t buf_len, struct file **arr_filp)
  1396. {
  1397. /* ret is going to TZ. Provide values from OBJECT_ERROR_<> */
  1398. int ret = OBJECT_ERROR_DEFUNCT;
  1399. int cbobj_retries = 0;
  1400. long timeout_jiff;
  1401. bool wait_interrupted = false;
  1402. struct smcinvoke_cb_txn *cb_txn = NULL;
  1403. struct smcinvoke_tzcb_req *cb_req = NULL, *tmp_cb_req = NULL;
  1404. struct smcinvoke_server_info *srvr_info = NULL;
  1405. struct smcinvoke_mem_obj *mem_obj = NULL;
  1406. uint16_t server_id = 0;
  1407. if (buf_len < sizeof(struct smcinvoke_tzcb_req)) {
  1408. pr_err("smaller buffer length : %u\n", buf_len);
  1409. return;
  1410. }
  1411. cb_req = buf;
  1412. /* check whether it is to be served by kernel or userspace */
  1413. if (TZHANDLE_IS_KERNEL_OBJ(cb_req->hdr.tzhandle)) {
  1414. return process_kernel_obj(buf, buf_len);
  1415. } else if (TZHANDLE_IS_MEM_MAP_OBJ(cb_req->hdr.tzhandle)) {
  1416. /*
  1417. * MEM_MAP memory object is created and owned by kernel,
  1418. * hence its processing(handling deletion) is done in
  1419. * kernel context.
  1420. */
  1421. return process_mem_obj(buf, buf_len);
  1422. } else if(TZHANDLE_IS_MEM_RGN_OBJ(cb_req->hdr.tzhandle)) {
  1423. /*
  1424. * MEM_RGN memory objects are created and owned by userspace,
  1425. * and hence their deletion/handling requires going back to the
  1426. * userspace, similar to that of callback objects. If we enter
  1427. * this 'if' condition, its no-op here, and proceed similar to
  1428. * case of callback objects.
  1429. */
  1430. } else if (!TZHANDLE_IS_CB_OBJ(cb_req->hdr.tzhandle)) {
  1431. pr_err("Request object is not a callback object %x\n",
  1432. cb_req->hdr.tzhandle);
  1433. cb_req->result = OBJECT_ERROR_INVALID;
  1434. return;
  1435. }
  1436. /*
  1437. * We need a copy of req that could be sent to server. Otherwise, if
  1438. * someone kills invoke caller, buf would go away and server would be
  1439. * working on already freed buffer, causing a device crash.
  1440. */
  1441. tmp_cb_req = kmemdup(buf, buf_len, GFP_KERNEL);
  1442. if (!tmp_cb_req) {
  1443. /* we need to return error to caller so fill up result */
  1444. cb_req->result = OBJECT_ERROR_KMEM;
  1445. pr_err("failed to create copy of request, set result: %d\n",
  1446. cb_req->result);
  1447. return;
  1448. }
  1449. cb_txn = kzalloc(sizeof(*cb_txn), GFP_KERNEL);
  1450. if (!cb_txn) {
  1451. cb_req->result = OBJECT_ERROR_KMEM;
  1452. pr_err("failed to allocate memory for request, result: %d\n",
  1453. cb_req->result);
  1454. kfree(tmp_cb_req);
  1455. return;
  1456. }
  1457. /* no need for memcpy as we did kmemdup() above */
  1458. cb_req = tmp_cb_req;
  1459. trace_process_tzcb_req_handle(cb_req->hdr.tzhandle, cb_req->hdr.op, cb_req->hdr.counts);
  1460. cb_txn->state = SMCINVOKE_REQ_PLACED;
  1461. cb_txn->cb_req = cb_req;
  1462. cb_txn->cb_req_bytes = buf_len;
  1463. cb_txn->filp_to_release = arr_filp;
  1464. kref_init(&cb_txn->ref_cnt);
  1465. mutex_lock(&g_smcinvoke_lock);
  1466. ++cb_reqs_inflight;
  1467. if(TZHANDLE_IS_MEM_RGN_OBJ(cb_req->hdr.tzhandle)) {
  1468. mem_obj= find_mem_obj_locked(TZHANDLE_GET_OBJID(cb_req->hdr.tzhandle),SMCINVOKE_MEM_RGN_OBJ);
  1469. if(!mem_obj) {
  1470. pr_err("mem obj with tzhandle : %d not found",cb_req->hdr.tzhandle);
  1471. mutex_unlock(&g_smcinvoke_lock);
  1472. goto out;
  1473. }
  1474. server_id = mem_obj->server->server_id;
  1475. } else {
  1476. server_id = TZHANDLE_GET_SERVER(cb_req->hdr.tzhandle);
  1477. }
  1478. srvr_info = get_cb_server_locked(server_id);
  1479. if (!srvr_info || srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT) {
  1480. /* ret equals Object_ERROR_DEFUNCT, at this point go to out */
  1481. if (!srvr_info)
  1482. pr_err("server is invalid\n");
  1483. else {
  1484. pr_err("server is defunct, state= %d tzhandle = %d\n",
  1485. srvr_info->state, cb_req->hdr.tzhandle);
  1486. }
  1487. mutex_unlock(&g_smcinvoke_lock);
  1488. goto out;
  1489. }
  1490. cb_txn->txn_id = ++srvr_info->txn_id;
  1491. hash_add(srvr_info->reqs_table, &cb_txn->hash, cb_txn->txn_id);
  1492. mutex_unlock(&g_smcinvoke_lock);
  1493. trace_process_tzcb_req_wait(cb_req->hdr.tzhandle, cbobj_retries, cb_txn->txn_id,
  1494. current->pid, current->tgid, srvr_info->state, srvr_info->server_id,
  1495. cb_reqs_inflight);
  1496. /*
  1497. * we need not worry that server_info will be deleted because as long
  1498. * as this CBObj is served by this server, srvr_info will be valid.
  1499. */
  1500. wake_up_interruptible_all(&srvr_info->req_wait_q);
  1501. /* timeout before 1s otherwise tzbusy would come */
  1502. timeout_jiff = msecs_to_jiffies(100);
  1503. while (cbobj_retries < CBOBJ_MAX_RETRIES) {
  1504. if (wait_interrupted) {
  1505. ret = wait_event_timeout(srvr_info->rsp_wait_q,
  1506. (cb_txn->state == SMCINVOKE_REQ_PROCESSED) ||
  1507. (srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT),
  1508. timeout_jiff);
  1509. } else {
  1510. ret = wait_event_interruptible_timeout(srvr_info->rsp_wait_q,
  1511. (cb_txn->state == SMCINVOKE_REQ_PROCESSED) ||
  1512. (srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT),
  1513. timeout_jiff);
  1514. }
  1515. if (ret == 0) {
  1516. if (srvr_info->is_server_suspended == 0) {
  1517. tzcb_err_ratelimited("CBobj timed out waiting on cbtxn :%d,cb-tzhandle:%d, retry:%d, op:%d counts :%d\n",
  1518. cb_txn->txn_id,cb_req->hdr.tzhandle, cbobj_retries,
  1519. cb_req->hdr.op, cb_req->hdr.counts);
  1520. tzcb_err_ratelimited("CBobj %d timedout pid %x,tid %x, srvr state=%d, srvr id:%u\n",
  1521. cb_req->hdr.tzhandle, current->pid,
  1522. current->tgid, srvr_info->state,
  1523. srvr_info->server_id);
  1524. }
  1525. } else {
  1526. /* wait_event returned due to a signal */
  1527. if (srvr_info->state != SMCINVOKE_SERVER_STATE_DEFUNCT &&
  1528. cb_txn->state != SMCINVOKE_REQ_PROCESSED) {
  1529. wait_interrupted = true;
  1530. } else {
  1531. break;
  1532. }
  1533. }
  1534. /*
  1535. * If bit corresponding to any accept thread is set, invoke threads
  1536. * should wait infinitely for the accept thread to come back with
  1537. * response.
  1538. */
  1539. if (srvr_info->is_server_suspended > 0) {
  1540. cbobj_retries = 0;
  1541. } else {
  1542. cbobj_retries++;
  1543. }
  1544. }
  1545. out:
  1546. /*
  1547. * we could be here because of either:
  1548. * a. Req is PROCESSED
  1549. * b. Server was killed
  1550. * c. Invoke thread is killed
  1551. * sometime invoke thread and server are part of same process.
  1552. */
  1553. mutex_lock(&g_smcinvoke_lock);
  1554. hash_del(&cb_txn->hash);
  1555. if (ret == 0) {
  1556. pr_err("CBObj timed out! No more retries\n");
  1557. cb_req->result = Object_ERROR_TIMEOUT;
  1558. } else if (ret == -ERESTARTSYS) {
  1559. pr_err("wait event interruped, ret: %d\n", ret);
  1560. cb_req->result = OBJECT_ERROR_ABORT;
  1561. } else {
  1562. if (cb_txn->state == SMCINVOKE_REQ_PROCESSED) {
  1563. /*
  1564. * it is possible that server was killed immediately
  1565. * after CB Req was processed but who cares now!
  1566. */
  1567. } else if (!srvr_info ||
  1568. srvr_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT) {
  1569. cb_req->result = OBJECT_ERROR_DEFUNCT;
  1570. pr_err("server invalid, res: %d\n", cb_req->result);
  1571. } else {
  1572. pr_err("%s: unexpected event happened, ret:%d\n", __func__, ret);
  1573. cb_req->result = OBJECT_ERROR_ABORT;
  1574. }
  1575. }
  1576. --cb_reqs_inflight;
  1577. trace_process_tzcb_req_result(cb_req->result, cb_req->hdr.tzhandle, cb_req->hdr.op,
  1578. cb_req->hdr.counts, cb_reqs_inflight);
  1579. memcpy(buf, cb_req, buf_len);
  1580. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  1581. if (srvr_info)
  1582. kref_put(&srvr_info->ref_cnt, destroy_cb_server);
  1583. mutex_unlock(&g_smcinvoke_lock);
  1584. }
  1585. static int marshal_out_invoke_req(const uint8_t *buf, uint32_t buf_size,
  1586. struct smcinvoke_cmd_req *req,
  1587. union smcinvoke_arg *args_buf,
  1588. uint32_t context_type)
  1589. {
  1590. int ret = -EINVAL, i = 0;
  1591. int32_t temp_fd = UHANDLE_NULL;
  1592. union smcinvoke_tz_args *tz_args = NULL;
  1593. size_t offset = sizeof(struct smcinvoke_msg_hdr) +
  1594. OBJECT_COUNTS_TOTAL(req->counts) *
  1595. sizeof(union smcinvoke_tz_args);
  1596. if (offset > buf_size)
  1597. goto out;
  1598. tz_args = (union smcinvoke_tz_args *)
  1599. (buf + sizeof(struct smcinvoke_msg_hdr));
  1600. tz_args += OBJECT_COUNTS_NUM_BI(req->counts);
  1601. if (args_buf == NULL)
  1602. return 0;
  1603. FOR_ARGS(i, req->counts, BO) {
  1604. args_buf[i].b.size = tz_args->b.size;
  1605. if ((buf_size - tz_args->b.offset < tz_args->b.size) ||
  1606. tz_args->b.offset > buf_size) {
  1607. pr_err("%s: buffer overflow detected\n", __func__);
  1608. goto out;
  1609. }
  1610. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  1611. if (copy_to_user((void __user *)
  1612. (uintptr_t)(args_buf[i].b.addr),
  1613. (uint8_t *)(buf) + tz_args->b.offset,
  1614. tz_args->b.size)) {
  1615. pr_err("Error %d copying ctxt to user\n", ret);
  1616. goto out;
  1617. }
  1618. } else {
  1619. memcpy((uint8_t *)(args_buf[i].b.addr),
  1620. (uint8_t *)(buf) + tz_args->b.offset,
  1621. tz_args->b.size);
  1622. }
  1623. tz_args++;
  1624. }
  1625. tz_args += OBJECT_COUNTS_NUM_OI(req->counts);
  1626. FOR_ARGS(i, req->counts, OO) {
  1627. /*
  1628. * create a new FD and assign to output object's context.
  1629. * We are passing cb_server_fd from output param in case OO
  1630. * is a CBObj. For CBObj, we have to ensure that it is sent
  1631. * to server who serves it and that info comes from USpace.
  1632. */
  1633. temp_fd = UHANDLE_NULL;
  1634. ret = get_uhandle_from_tzhandle(tz_args->handle,
  1635. TZHANDLE_GET_SERVER(tz_args->handle),
  1636. &temp_fd, NO_LOCK, context_type);
  1637. args_buf[i].o.fd = temp_fd;
  1638. if (ret)
  1639. goto out;
  1640. trace_marshal_out_invoke_req(i, tz_args->handle,
  1641. TZHANDLE_GET_SERVER(tz_args->handle), temp_fd);
  1642. tz_args++;
  1643. }
  1644. ret = 0;
  1645. out:
  1646. return ret;
  1647. }
  1648. static bool is_inbound_req(int val)
  1649. {
  1650. return (val == SMCINVOKE_RESULT_INBOUND_REQ_NEEDED ||
  1651. val == QSEOS_RESULT_INCOMPLETE ||
  1652. val == QSEOS_RESULT_BLOCKED_ON_LISTENER);
  1653. }
  1654. static int prepare_send_scm_msg(const uint8_t *in_buf, phys_addr_t in_paddr,
  1655. size_t in_buf_len,
  1656. uint8_t *out_buf, phys_addr_t out_paddr,
  1657. size_t out_buf_len,
  1658. struct smcinvoke_cmd_req *req,
  1659. union smcinvoke_arg *args_buf,
  1660. bool *tz_acked, uint32_t context_type,
  1661. struct qtee_shm *in_shm, struct qtee_shm *out_shm,
  1662. bool retry)
  1663. {
  1664. int ret = 0, cmd, retry_count = 0;
  1665. u64 response_type;
  1666. unsigned int data;
  1667. struct file *arr_filp[OBJECT_COUNTS_MAX_OO] = {NULL};
  1668. *tz_acked = false;
  1669. /* buf size should be page aligned */
  1670. if ((in_buf_len % PAGE_SIZE) != 0 || (out_buf_len % PAGE_SIZE) != 0)
  1671. return -EINVAL;
  1672. cmd = invoke_cmd;
  1673. while (1) {
  1674. do {
  1675. ret = invoke_cmd_handler(cmd, in_paddr, in_buf_len, out_buf,
  1676. out_paddr, out_buf_len, &req->result,
  1677. &response_type, &data, in_shm, out_shm);
  1678. if (ret == -EBUSY) {
  1679. pr_err_ratelimited("Secure side is busy,will retry after 30 ms, retry_count = %d\n",retry_count);
  1680. msleep(SMCINVOKE_SCM_EBUSY_WAIT_MS);
  1681. }
  1682. } while (retry && (ret == -EBUSY) &&
  1683. (retry_count++ < SMCINVOKE_SCM_EBUSY_MAX_RETRY));
  1684. if (!ret && !is_inbound_req(response_type)) {
  1685. /* dont marshal if Obj returns an error */
  1686. if (!req->result) {
  1687. if (args_buf != NULL)
  1688. ret = marshal_out_invoke_req(in_buf,
  1689. in_buf_len, req, args_buf,
  1690. context_type);
  1691. }
  1692. *tz_acked = true;
  1693. }
  1694. if (cmd == SMCINVOKE_CB_RSP_CMD)
  1695. release_filp(arr_filp, OBJECT_COUNTS_MAX_OO);
  1696. if (ret || !is_inbound_req(response_type))
  1697. break;
  1698. /* process listener request */
  1699. if (response_type == QSEOS_RESULT_INCOMPLETE ||
  1700. response_type == QSEOS_RESULT_BLOCKED_ON_LISTENER) {
  1701. ret = qseecom_process_listener_from_smcinvoke(
  1702. &req->result, &response_type, &data);
  1703. trace_prepare_send_scm_msg(response_type, req->result);
  1704. if (!req->result &&
  1705. response_type != SMCINVOKE_RESULT_INBOUND_REQ_NEEDED) {
  1706. ret = marshal_out_invoke_req(in_buf,
  1707. in_buf_len, req, args_buf,
  1708. context_type);
  1709. }
  1710. *tz_acked = true;
  1711. }
  1712. /*
  1713. * qseecom does not understand smcinvoke's callback object &&
  1714. * erringly sets ret value as -EINVAL :( We need to handle it.
  1715. */
  1716. if (response_type != SMCINVOKE_RESULT_INBOUND_REQ_NEEDED)
  1717. break;
  1718. if (response_type == SMCINVOKE_RESULT_INBOUND_REQ_NEEDED) {
  1719. trace_status(__func__, "looks like inbnd req reqd");
  1720. process_tzcb_req(out_buf, out_buf_len, arr_filp);
  1721. cmd = SMCINVOKE_CB_RSP_CMD;
  1722. }
  1723. }
  1724. return ret;
  1725. }
  1726. /*
  1727. * SMC expects arguments in following format
  1728. * ---------------------------------------------------------------------------
  1729. * | cxt | op | counts | ptr|size |ptr|size...|ORef|ORef|...| rest of payload |
  1730. * ---------------------------------------------------------------------------
  1731. * cxt: target, op: operation, counts: total arguments
  1732. * offset: offset is from beginning of buffer i.e. cxt
  1733. * size: size is 8 bytes aligned value
  1734. */
  1735. static size_t compute_in_msg_size(const struct smcinvoke_cmd_req *req,
  1736. const union smcinvoke_arg *args_buf)
  1737. {
  1738. uint32_t i = 0;
  1739. size_t total_size = sizeof(struct smcinvoke_msg_hdr) +
  1740. OBJECT_COUNTS_TOTAL(req->counts) *
  1741. sizeof(union smcinvoke_tz_args);
  1742. /* Computed total_size should be 8 bytes aligned from start of buf */
  1743. total_size = ALIGN(total_size, SMCINVOKE_ARGS_ALIGN_SIZE);
  1744. /* each buffer has to be 8 bytes aligned */
  1745. while (i < OBJECT_COUNTS_NUM_buffers(req->counts))
  1746. total_size = size_add_(total_size,
  1747. size_align(args_buf[i++].b.size,
  1748. SMCINVOKE_ARGS_ALIGN_SIZE));
  1749. return PAGE_ALIGN(total_size);
  1750. }
  1751. static int marshal_in_invoke_req(const struct smcinvoke_cmd_req *req,
  1752. const union smcinvoke_arg *args_buf, uint32_t tzhandle,
  1753. uint8_t *buf, size_t buf_size, struct file **arr_filp,
  1754. int32_t *tzhandles_to_release, uint32_t context_type,
  1755. struct list_head *l_pending_mem_obj)
  1756. {
  1757. int ret = -EINVAL, i = 0, j = 0, k = 0;
  1758. const struct smcinvoke_msg_hdr msg_hdr = {
  1759. tzhandle, req->op, req->counts};
  1760. uint32_t offset = sizeof(struct smcinvoke_msg_hdr) +
  1761. sizeof(union smcinvoke_tz_args) *
  1762. OBJECT_COUNTS_TOTAL(req->counts);
  1763. union smcinvoke_tz_args *tz_args = NULL;
  1764. if (buf_size < offset)
  1765. goto out;
  1766. *(struct smcinvoke_msg_hdr *)buf = msg_hdr;
  1767. tz_args = (union smcinvoke_tz_args *)(buf +
  1768. sizeof(struct smcinvoke_msg_hdr));
  1769. if (args_buf == NULL)
  1770. return 0;
  1771. FOR_ARGS(i, req->counts, BI) {
  1772. offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
  1773. if ((offset > buf_size) ||
  1774. (args_buf[i].b.size > (buf_size - offset)))
  1775. goto out;
  1776. tz_args[i].b.offset = offset;
  1777. tz_args[i].b.size = args_buf[i].b.size;
  1778. if (context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  1779. if (copy_from_user(buf + offset,
  1780. (void __user *)(uintptr_t)(args_buf[i].b.addr),
  1781. args_buf[i].b.size))
  1782. goto out;
  1783. } else {
  1784. memcpy(buf + offset, (void *)(args_buf[i].b.addr),
  1785. args_buf[i].b.size);
  1786. }
  1787. offset += args_buf[i].b.size;
  1788. }
  1789. FOR_ARGS(i, req->counts, BO) {
  1790. offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
  1791. if ((offset > buf_size) ||
  1792. (args_buf[i].b.size > (buf_size - offset)))
  1793. goto out;
  1794. tz_args[i].b.offset = offset;
  1795. tz_args[i].b.size = args_buf[i].b.size;
  1796. offset += args_buf[i].b.size;
  1797. }
  1798. FOR_ARGS(i, req->counts, OI) {
  1799. ret = get_tzhandle_from_uhandle(args_buf[i].o.fd,
  1800. args_buf[i].o.cb_server_fd, &arr_filp[j++],
  1801. &(tz_args[i].handle), l_pending_mem_obj);
  1802. if (ret)
  1803. goto out;
  1804. trace_marshal_in_invoke_req(i, args_buf[i].o.fd,
  1805. args_buf[i].o.cb_server_fd, tz_args[i].handle);
  1806. tzhandles_to_release[k++] = tz_args[i].handle;
  1807. }
  1808. ret = 0;
  1809. out:
  1810. return ret;
  1811. }
  1812. static int marshal_in_tzcb_req(const struct smcinvoke_cb_txn *cb_txn,
  1813. struct smcinvoke_accept *user_req, int srvr_id)
  1814. {
  1815. int ret = 0, i = 0;
  1816. int32_t temp_fd = UHANDLE_NULL;
  1817. union smcinvoke_arg tmp_arg;
  1818. struct smcinvoke_tzcb_req *tzcb_req = cb_txn->cb_req;
  1819. union smcinvoke_tz_args *tz_args = tzcb_req->args;
  1820. size_t tzcb_req_len = cb_txn->cb_req_bytes;
  1821. size_t tz_buf_offset = TZCB_BUF_OFFSET(tzcb_req);
  1822. size_t user_req_buf_offset = sizeof(union smcinvoke_arg) *
  1823. OBJECT_COUNTS_TOTAL(tzcb_req->hdr.counts);
  1824. if (tz_buf_offset > tzcb_req_len) {
  1825. ret = -EINVAL;
  1826. goto out;
  1827. }
  1828. user_req->txn_id = cb_txn->txn_id;
  1829. if (get_uhandle_from_tzhandle(tzcb_req->hdr.tzhandle, srvr_id,
  1830. (int32_t*)(&user_req->cbobj_id), TAKE_LOCK,
  1831. SMCINVOKE_OBJ_TYPE_TZ_OBJ)) {
  1832. ret = -EINVAL;
  1833. goto out;
  1834. }
  1835. user_req->op = tzcb_req->hdr.op;
  1836. user_req->counts = tzcb_req->hdr.counts;
  1837. user_req->argsize = sizeof(union smcinvoke_arg);
  1838. trace_marshal_in_tzcb_req_handle(tzcb_req->hdr.tzhandle, srvr_id,
  1839. user_req->cbobj_id, user_req->op, user_req->counts);
  1840. FOR_ARGS(i, tzcb_req->hdr.counts, BI) {
  1841. user_req_buf_offset = size_align(user_req_buf_offset,
  1842. SMCINVOKE_ARGS_ALIGN_SIZE);
  1843. tmp_arg.b.size = tz_args[i].b.size;
  1844. if ((tz_args[i].b.offset > tzcb_req_len) ||
  1845. (tz_args[i].b.size > tzcb_req_len - tz_args[i].b.offset) ||
  1846. (user_req_buf_offset > user_req->buf_len) ||
  1847. (tmp_arg.b.size >
  1848. user_req->buf_len - user_req_buf_offset)) {
  1849. ret = -EINVAL;
  1850. pr_err("%s: buffer overflow detected\n", __func__);
  1851. goto out;
  1852. }
  1853. tmp_arg.b.addr = user_req->buf_addr + user_req_buf_offset;
  1854. if (copy_to_user(u64_to_user_ptr
  1855. (user_req->buf_addr + i * sizeof(tmp_arg)),
  1856. &tmp_arg, sizeof(tmp_arg)) ||
  1857. copy_to_user(u64_to_user_ptr(tmp_arg.b.addr),
  1858. (uint8_t *)(tzcb_req) + tz_args[i].b.offset,
  1859. tz_args[i].b.size)) {
  1860. ret = -EFAULT;
  1861. goto out;
  1862. }
  1863. user_req_buf_offset += tmp_arg.b.size;
  1864. }
  1865. FOR_ARGS(i, tzcb_req->hdr.counts, BO) {
  1866. user_req_buf_offset = size_align(user_req_buf_offset,
  1867. SMCINVOKE_ARGS_ALIGN_SIZE);
  1868. tmp_arg.b.size = tz_args[i].b.size;
  1869. if ((user_req_buf_offset > user_req->buf_len) ||
  1870. (tmp_arg.b.size >
  1871. user_req->buf_len - user_req_buf_offset)) {
  1872. ret = -EINVAL;
  1873. pr_err("%s: buffer overflow detected\n", __func__);
  1874. goto out;
  1875. }
  1876. tmp_arg.b.addr = user_req->buf_addr + user_req_buf_offset;
  1877. if (copy_to_user(u64_to_user_ptr
  1878. (user_req->buf_addr + i * sizeof(tmp_arg)),
  1879. &tmp_arg, sizeof(tmp_arg))) {
  1880. ret = -EFAULT;
  1881. goto out;
  1882. }
  1883. user_req_buf_offset += tmp_arg.b.size;
  1884. }
  1885. FOR_ARGS(i, tzcb_req->hdr.counts, OI) {
  1886. /*
  1887. * create a new FD and assign to output object's
  1888. * context
  1889. */
  1890. temp_fd = UHANDLE_NULL;
  1891. ret = get_uhandle_from_tzhandle(tz_args[i].handle, srvr_id,
  1892. &temp_fd, TAKE_LOCK, SMCINVOKE_OBJ_TYPE_TZ_OBJ);
  1893. tmp_arg.o.fd = temp_fd;
  1894. if (ret) {
  1895. ret = -EINVAL;
  1896. goto out;
  1897. }
  1898. if (copy_to_user(u64_to_user_ptr
  1899. (user_req->buf_addr + i * sizeof(tmp_arg)),
  1900. &tmp_arg, sizeof(tmp_arg))) {
  1901. ret = -EFAULT;
  1902. goto out;
  1903. }
  1904. trace_marshal_in_tzcb_req_fd(i, tz_args[i].handle, srvr_id, temp_fd);
  1905. }
  1906. out:
  1907. return ret;
  1908. }
  1909. static int marshal_out_tzcb_req(const struct smcinvoke_accept *user_req,
  1910. struct smcinvoke_cb_txn *cb_txn,
  1911. struct file **arr_filp)
  1912. {
  1913. int ret = -EINVAL, i = 0;
  1914. int32_t tzhandles_to_release[OBJECT_COUNTS_MAX_OO] = {0};
  1915. struct smcinvoke_tzcb_req *tzcb_req = cb_txn->cb_req;
  1916. union smcinvoke_tz_args *tz_args = tzcb_req->args;
  1917. size_t tz_buf_offset = TZCB_BUF_OFFSET(tzcb_req);
  1918. LIST_HEAD(l_mem_objs_pending_async); /* Holds new memory objects, to be later sent to TZ */
  1919. uint32_t max_offset = 0;
  1920. uint32_t buffer_size_max_offset = 0;
  1921. void* async_buf_begin;
  1922. size_t async_buf_size;
  1923. uint32_t offset = 0;
  1924. /* We assume 'marshal_in_tzcb_req' increases the ref-count of the CBOBJs.
  1925. * It should be the case for mem-obj as well. However, it does not do that.
  1926. * It is easier to filter out the 'release_tzhandles' for mem-obj here rather
  1927. * than increases its ref-count on 'marshal_in_tzcb_req'. Because, there is no
  1928. * reliable error handling and cleanup in 'marshal_in_tzcb_req'. So if it fails
  1929. * mem-obj may not get released. **/
  1930. if (!TZHANDLE_IS_MEM_OBJ(cb_txn->cb_req->hdr.tzhandle))
  1931. release_tzhandles(&cb_txn->cb_req->hdr.tzhandle, 1);
  1932. tzcb_req->result = user_req->result;
  1933. /* Return without marshaling user args if destination callback invocation was
  1934. unsuccessful. */
  1935. if (tzcb_req->result != 0) {
  1936. ret = 0;
  1937. goto out;
  1938. }
  1939. FOR_ARGS(i, tzcb_req->hdr.counts, BI) {
  1940. /* Find the max offset and the size of the buffer in that offset */
  1941. if (tz_args[i].b.offset > max_offset) {
  1942. max_offset = tz_args[i].b.offset;
  1943. buffer_size_max_offset = tz_args[i].b.size;
  1944. }
  1945. }
  1946. FOR_ARGS(i, tzcb_req->hdr.counts, BO) {
  1947. union smcinvoke_arg tmp_arg;
  1948. if (copy_from_user((uint8_t *)&tmp_arg, u64_to_user_ptr(
  1949. user_req->buf_addr + i * sizeof(union smcinvoke_arg)),
  1950. sizeof(union smcinvoke_arg))) {
  1951. ret = -EFAULT;
  1952. goto out;
  1953. }
  1954. if (tmp_arg.b.size > tz_args[i].b.size)
  1955. goto out;
  1956. if (copy_from_user((uint8_t *)(tzcb_req) + tz_args[i].b.offset,
  1957. u64_to_user_ptr(tmp_arg.b.addr),
  1958. tmp_arg.b.size)) {
  1959. ret = -EFAULT;
  1960. goto out;
  1961. }
  1962. /* Find the max offset and the size of the buffer in that offset */
  1963. if (tz_args[i].b.offset > max_offset) {
  1964. max_offset = tz_args[i].b.offset;
  1965. buffer_size_max_offset = tz_args[i].b.size;
  1966. }
  1967. }
  1968. FOR_ARGS(i, tzcb_req->hdr.counts, OO) {
  1969. union smcinvoke_arg tmp_arg;
  1970. if (copy_from_user((uint8_t *)&tmp_arg, u64_to_user_ptr(
  1971. user_req->buf_addr + i * sizeof(union smcinvoke_arg)),
  1972. sizeof(union smcinvoke_arg))) {
  1973. ret = -EFAULT;
  1974. goto out;
  1975. }
  1976. ret = get_tzhandle_from_uhandle(tmp_arg.o.fd,
  1977. tmp_arg.o.cb_server_fd, &arr_filp[i],
  1978. &(tz_args[i].handle), &l_mem_objs_pending_async);
  1979. if (ret)
  1980. goto out;
  1981. tzhandles_to_release[i] = tz_args[i].handle;
  1982. trace_marshal_out_tzcb_req(i, tmp_arg.o.fd,
  1983. tmp_arg.o.cb_server_fd, tz_args[i].handle);
  1984. }
  1985. ret = 0;
  1986. out:
  1987. FOR_ARGS(i, tzcb_req->hdr.counts, OI) {
  1988. if (TZHANDLE_IS_CB_OBJ(tz_args[i].handle))
  1989. release_tzhandles(&tz_args[i].handle, 1);
  1990. }
  1991. do {
  1992. if (mem_obj_async_support) {
  1993. /* We will be able to add the async information to the buffer beyond the data in the max offset, if exists.
  1994. * If doesn't exist, we can add the async information after the header and the args. */
  1995. offset = (max_offset ? (max_offset + buffer_size_max_offset) : tz_buf_offset);
  1996. offset = size_align(offset, SMCINVOKE_ARGS_ALIGN_SIZE);
  1997. async_buf_begin = (uint8_t *)tzcb_req + offset;
  1998. if (async_buf_begin - (void *)tzcb_req > g_max_cb_buf_size) {
  1999. pr_err("Unable to add memory object info to the async channel\n");
  2000. break;
  2001. } else {
  2002. async_buf_size = g_max_cb_buf_size - (async_buf_begin - (void *)tzcb_req);
  2003. }
  2004. mutex_lock(&g_smcinvoke_lock);
  2005. add_mem_obj_info_to_async_side_channel_locked(async_buf_begin, async_buf_size, &l_mem_objs_pending_async);
  2006. delete_pending_async_list_locked(&l_mem_objs_pending_async);
  2007. mutex_unlock(&g_smcinvoke_lock);
  2008. }
  2009. } while (0);
  2010. if (ret)
  2011. release_tzhandles(tzhandles_to_release, OBJECT_COUNTS_MAX_OO);
  2012. return ret;
  2013. }
  2014. static void set_tz_version (uint32_t tz_version)
  2015. {
  2016. tz_async_version = tz_version;
  2017. /* We enable async memory object support when TZ async
  2018. * version is equal or larger than the driver version.
  2019. * It is expected that if the protocol changes in later
  2020. * TZ versions, TZ will support backward compatibility
  2021. * so this condition should still be valid.
  2022. */
  2023. if (tz_version >= SMCINVOKE_ASYNC_VERSION) {
  2024. mem_obj_async_support = true;
  2025. pr_debug("Enabled asynchronous memory object support\n");
  2026. }
  2027. }
  2028. static void process_piggyback_data(void *buf, size_t buf_size)
  2029. {
  2030. int i;
  2031. struct smcinvoke_tzcb_req req = {0};
  2032. struct smcinvoke_piggyback_msg *msg = buf;
  2033. int32_t *objs = msg->objs;
  2034. if (msg->version == 0) {
  2035. /* QTEE reset the buffer if it is unused. */
  2036. return;
  2037. }
  2038. for (i = 0; i < msg->counts; i++) {
  2039. if (msg->op != OBJECT_OP_RELEASE) {
  2040. /* We only support release handler. */
  2041. break;
  2042. }
  2043. req.hdr.op = msg->op;
  2044. req.hdr.counts = 0; /* release op does not require any args */
  2045. req.hdr.tzhandle = objs[i];
  2046. if (tz_async_version == 0)
  2047. set_tz_version(msg->version);
  2048. process_tzcb_req(&req, sizeof(struct smcinvoke_tzcb_req), NULL);
  2049. /* cbobjs_in_flight will be adjusted during CB processing */
  2050. }
  2051. /* Reset output buffer after processing.*/
  2052. memset(buf, 0, buf_size);
  2053. }
  2054. /* Add memory object mapped data to the async side channel, so it's available to TZ
  2055. * together with the memory object.
  2056. *
  2057. * No return value as TZ is always able to explicitly ask for this information
  2058. * in case this function fails.
  2059. */
  2060. static void add_mem_obj_info_to_async_side_channel_locked(void *buf, size_t buf_size, struct list_head *l_pending_mem_obj)
  2061. {
  2062. struct smcinvoke_mem_obj_msg *msg = buf;
  2063. struct smcinvoke_mem_obj_pending_async *mem_obj_pending = NULL;
  2064. size_t header_size = 0;
  2065. size_t mo_size = 0;
  2066. size_t used = 0;
  2067. size_t index = 0;
  2068. if (list_empty(l_pending_mem_obj))
  2069. return;
  2070. header_size = sizeof(struct smcinvoke_mem_obj_msg);
  2071. mo_size = sizeof(struct smcinvoke_mem_obj_info);
  2072. /* Minimal size required is the header data + one mem obj info */
  2073. if (buf_size < header_size + mo_size) {
  2074. pr_err("Unable to add memory object info to async channel\n");
  2075. return;
  2076. }
  2077. msg->version = SMCINVOKE_ASYNC_VERSION;
  2078. msg->op = SMCINVOKE_ASYNC_OP_MEMORY_OBJECT;
  2079. msg->count = 0;
  2080. used = header_size;
  2081. index = 0;
  2082. list_for_each_entry(mem_obj_pending, l_pending_mem_obj, list) {
  2083. if (NULL == mem_obj_pending->mem_obj) {
  2084. pr_err("Memory object is no longer valid\n");
  2085. continue;
  2086. }
  2087. if (used + mo_size > buf_size) {
  2088. pr_err("Not all memory object info was added to the async channel\n");
  2089. break;
  2090. }
  2091. msg->mo[index].memObjRef = TZHANDLE_MAKE_LOCAL(MEM_RGN_SRVR_ID, mem_obj_pending->mem_obj->mem_region_id);
  2092. msg->mo[index].mapObjRef = TZHANDLE_MAKE_LOCAL(MEM_MAP_SRVR_ID, mem_obj_pending->mem_obj->mem_map_obj_id);
  2093. msg->mo[index].addr = mem_obj_pending->mem_obj->p_addr;
  2094. msg->mo[index].size = mem_obj_pending->mem_obj->p_addr_len;
  2095. msg->mo[index].perm = SMCINVOKE_MEM_PERM_RW;
  2096. used += sizeof(msg->mo[index]);
  2097. index++;
  2098. }
  2099. msg->count = index;
  2100. pr_debug("Added %d memory objects to the side channel, total size = %d\n", index, used);
  2101. return;
  2102. }
  2103. /*
  2104. * Delete entire pending async list.
  2105. */
  2106. static void delete_pending_async_list_locked(struct list_head *l_pending_mem_obj)
  2107. {
  2108. struct smcinvoke_mem_obj_pending_async *mem_obj_pending = NULL;
  2109. struct smcinvoke_mem_obj_pending_async *temp = NULL;
  2110. if (list_empty(l_pending_mem_obj))
  2111. return;
  2112. list_for_each_entry_safe(mem_obj_pending, temp, l_pending_mem_obj, list) {
  2113. mem_obj_pending->mem_obj = NULL;
  2114. list_del(&mem_obj_pending->list);
  2115. kfree(mem_obj_pending);
  2116. }
  2117. }
  2118. /*
  2119. * Unmap/release the mapped objects from pending async list.
  2120. */
  2121. static void release_map_obj_pending_async_list_locked(struct list_head *l_pending_mem_obj)
  2122. {
  2123. struct smcinvoke_mem_obj_pending_async *mem_obj_pending = NULL;
  2124. struct smcinvoke_mem_obj_pending_async *temp = NULL;
  2125. if (list_empty(l_pending_mem_obj))
  2126. return;
  2127. list_for_each_entry_safe(mem_obj_pending, temp, l_pending_mem_obj, list) {
  2128. kref_put(&mem_obj_pending->mem_obj->mem_map_obj_ref_cnt, del_mem_map_obj_locked);
  2129. }
  2130. }
  2131. static long process_ack_local_obj(struct file *filp, unsigned int cmd,
  2132. unsigned long arg)
  2133. {
  2134. int ret = -1;
  2135. int32_t local_obj = SMCINVOKE_USERSPACE_OBJ_NULL;
  2136. struct smcinvoke_file_data *filp_data = filp->private_data;
  2137. if (_IOC_SIZE(cmd) != sizeof(int32_t))
  2138. return -EINVAL;
  2139. ret = copy_from_user(&local_obj, (void __user *)(uintptr_t)arg,
  2140. sizeof(int32_t));
  2141. if (ret)
  2142. return -EFAULT;
  2143. mutex_lock(&g_smcinvoke_lock);
  2144. if (UHANDLE_IS_CB_OBJ(local_obj))
  2145. ret = put_pending_cbobj_locked(filp_data->server_id,
  2146. UHANDLE_GET_CB_OBJ(local_obj));
  2147. mutex_unlock(&g_smcinvoke_lock);
  2148. return ret;
  2149. }
  2150. static long process_server_req(struct file *filp, unsigned int cmd,
  2151. unsigned long arg)
  2152. {
  2153. int ret = -1;
  2154. int32_t server_fd = -1;
  2155. struct smcinvoke_server server_req = {0};
  2156. struct smcinvoke_server_info *server_info = NULL;
  2157. if (_IOC_SIZE(cmd) != sizeof(server_req)) {
  2158. pr_err("invalid command size received for server request\n");
  2159. return -EINVAL;
  2160. }
  2161. ret = copy_from_user(&server_req, (void __user *)(uintptr_t)arg,
  2162. sizeof(server_req));
  2163. if (ret) {
  2164. pr_err("copying server request from user failed\n");
  2165. return -EFAULT;
  2166. }
  2167. server_info = kzalloc(sizeof(*server_info), GFP_KERNEL);
  2168. if (!server_info)
  2169. return -ENOMEM;
  2170. kref_init(&server_info->ref_cnt);
  2171. init_waitqueue_head(&server_info->req_wait_q);
  2172. init_waitqueue_head(&server_info->rsp_wait_q);
  2173. server_info->cb_buf_size = server_req.cb_buf_size;
  2174. hash_init(server_info->reqs_table);
  2175. hash_init(server_info->responses_table);
  2176. INIT_LIST_HEAD(&server_info->pending_cbobjs);
  2177. server_info->is_server_suspended = 0;
  2178. mutex_lock(&g_smcinvoke_lock);
  2179. server_info->server_id = next_cb_server_id_locked();
  2180. hash_add(g_cb_servers, &server_info->hash,
  2181. server_info->server_id);
  2182. if (g_max_cb_buf_size < server_req.cb_buf_size)
  2183. g_max_cb_buf_size = server_req.cb_buf_size;
  2184. mutex_unlock(&g_smcinvoke_lock);
  2185. ret = get_fd_for_obj(SMCINVOKE_OBJ_TYPE_SERVER,
  2186. server_info->server_id, &server_fd);
  2187. if (ret)
  2188. release_cb_server(server_info->server_id);
  2189. return server_fd;
  2190. }
  2191. static long process_accept_req(struct file *filp, unsigned int cmd,
  2192. unsigned long arg)
  2193. {
  2194. int ret = -1;
  2195. struct smcinvoke_file_data *server_obj = filp->private_data;
  2196. struct smcinvoke_accept user_args = {0};
  2197. struct smcinvoke_cb_txn *cb_txn = NULL;
  2198. struct smcinvoke_server_info *server_info = NULL;
  2199. if (_IOC_SIZE(cmd) != sizeof(struct smcinvoke_accept)) {
  2200. pr_err("command size invalid for accept request\n");
  2201. return -EINVAL;
  2202. }
  2203. if (copy_from_user(&user_args, (void __user *)arg,
  2204. sizeof(struct smcinvoke_accept))) {
  2205. pr_err("copying accept request from user failed\n");
  2206. return -EFAULT;
  2207. }
  2208. if (user_args.argsize != sizeof(union smcinvoke_arg)) {
  2209. pr_err("arguments size is invalid for accept thread\n");
  2210. return -EINVAL;
  2211. }
  2212. /* ACCEPT is available only on server obj */
  2213. if (server_obj->context_type != SMCINVOKE_OBJ_TYPE_SERVER) {
  2214. pr_err("invalid object type received for accept req\n");
  2215. return -EPERM;
  2216. }
  2217. mutex_lock(&g_smcinvoke_lock);
  2218. server_info = get_cb_server_locked(server_obj->server_id);
  2219. if (!server_info) {
  2220. pr_err("No matching server with server id : %u found\n",
  2221. server_obj->server_id);
  2222. mutex_unlock(&g_smcinvoke_lock);
  2223. return -EINVAL;
  2224. }
  2225. if (server_info->state == SMCINVOKE_SERVER_STATE_DEFUNCT)
  2226. server_info->state = 0;
  2227. server_info->is_server_suspended = UNSET_BIT(server_info->is_server_suspended,
  2228. (current->pid)%DEFAULT_CB_OBJ_THREAD_CNT);
  2229. mutex_unlock(&g_smcinvoke_lock);
  2230. /* First check if it has response otherwise wait for req */
  2231. if (user_args.has_resp) {
  2232. trace_process_accept_req_has_response(current->pid, current->tgid);
  2233. mutex_lock(&g_smcinvoke_lock);
  2234. cb_txn = find_cbtxn_locked(server_info, user_args.txn_id,
  2235. SMCINVOKE_REQ_PROCESSING);
  2236. mutex_unlock(&g_smcinvoke_lock);
  2237. /*
  2238. * cb_txn can be null if userspace provides wrong txn id OR
  2239. * invoke thread died while server was processing cb req.
  2240. * if invoke thread dies, it would remove req from Q. So
  2241. * no matching cb_txn would be on Q and hence NULL cb_txn.
  2242. * In this case, we want this thread to start waiting
  2243. * new cb requests.
  2244. */
  2245. if (!cb_txn) {
  2246. pr_err_ratelimited("%s txn %d either invalid or removed from Q\n",
  2247. __func__, user_args.txn_id);
  2248. goto start_waiting_for_requests;
  2249. }
  2250. ret = marshal_out_tzcb_req(&user_args, cb_txn,
  2251. cb_txn->filp_to_release);
  2252. /*
  2253. * if client did not set error and we get error locally,
  2254. * we return local error to TA
  2255. */
  2256. if (ret && cb_txn->cb_req->result == 0)
  2257. cb_txn->cb_req->result = OBJECT_ERROR_UNAVAIL;
  2258. cb_txn->state = SMCINVOKE_REQ_PROCESSED;
  2259. mutex_lock(&g_smcinvoke_lock);
  2260. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  2261. mutex_unlock(&g_smcinvoke_lock);
  2262. wake_up(&server_info->rsp_wait_q);
  2263. /*
  2264. * if marshal_out fails, we should let userspace release
  2265. * any ref/obj it created for CB processing
  2266. */
  2267. if (ret && OBJECT_COUNTS_NUM_OO(user_args.counts))
  2268. goto out;
  2269. }
  2270. start_waiting_for_requests:
  2271. /*
  2272. * Once response has been delivered, thread will wait for another
  2273. * callback req to process.
  2274. */
  2275. do {
  2276. ret = wait_event_interruptible(server_info->req_wait_q,
  2277. !hash_empty(server_info->reqs_table));
  2278. if (ret) {
  2279. trace_process_accept_req_ret(current->pid, current->tgid, ret);
  2280. /*
  2281. * Ideally, we should destroy server if accept threads
  2282. * are returning due to client being killed or device
  2283. * going down (Shutdown/Reboot) but that would make
  2284. * server_info invalid. Other accept/invoke threads are
  2285. * using server_info and would crash. So dont do that.
  2286. */
  2287. mutex_lock(&g_smcinvoke_lock);
  2288. if(freezing(current)) {
  2289. pr_err_ratelimited("Server id :%d interrupted probaby due to suspend, pid:%d\n",
  2290. server_info->server_id, current->pid);
  2291. /*
  2292. * Each accept thread is identified by bits ranging from
  2293. * 0 to DEFAULT_CBOBJ_THREAD_CNT-1. When an accept thread is
  2294. * interrupted by a signal other than SIGUSR1,SIGKILL,SIGTERM,
  2295. * set the corresponding bit of accept thread, indicating that
  2296. * current accept thread's state to be "suspended"/ or something
  2297. * that needs infinite timeout for invoke thread.
  2298. */
  2299. server_info->is_server_suspended =
  2300. SET_BIT(server_info->is_server_suspended,
  2301. (current->pid)%DEFAULT_CB_OBJ_THREAD_CNT);
  2302. } else {
  2303. pr_err_ratelimited("Setting pid:%d, server id : %d state to defunct\n",
  2304. current->pid, server_info->server_id);
  2305. server_info->state = SMCINVOKE_SERVER_STATE_DEFUNCT;
  2306. }
  2307. mutex_unlock(&g_smcinvoke_lock);
  2308. wake_up_interruptible(&server_info->rsp_wait_q);
  2309. goto out;
  2310. }
  2311. mutex_lock(&g_smcinvoke_lock);
  2312. cb_txn = find_cbtxn_locked(server_info,
  2313. SMCINVOKE_NEXT_AVAILABLE_TXN,
  2314. SMCINVOKE_REQ_PLACED);
  2315. mutex_unlock(&g_smcinvoke_lock);
  2316. if (cb_txn) {
  2317. cb_txn->state = SMCINVOKE_REQ_PROCESSING;
  2318. ret = marshal_in_tzcb_req(cb_txn, &user_args,
  2319. server_obj->server_id);
  2320. if (ret) {
  2321. pr_err("failed to marshal in the callback request\n");
  2322. cb_txn->cb_req->result = OBJECT_ERROR_UNAVAIL;
  2323. cb_txn->state = SMCINVOKE_REQ_PROCESSED;
  2324. mutex_lock(&g_smcinvoke_lock);
  2325. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  2326. mutex_unlock(&g_smcinvoke_lock);
  2327. wake_up_interruptible(&server_info->rsp_wait_q);
  2328. continue;
  2329. }
  2330. mutex_lock(&g_smcinvoke_lock);
  2331. hash_add(server_info->responses_table, &cb_txn->hash,
  2332. cb_txn->txn_id);
  2333. kref_put(&cb_txn->ref_cnt, delete_cb_txn_locked);
  2334. mutex_unlock(&g_smcinvoke_lock);
  2335. trace_process_accept_req_placed(current->pid, current->tgid);
  2336. ret = copy_to_user((void __user *)arg, &user_args,
  2337. sizeof(struct smcinvoke_accept));
  2338. }
  2339. } while (!cb_txn);
  2340. out:
  2341. if (server_info)
  2342. kref_put(&server_info->ref_cnt, destroy_cb_server);
  2343. if (ret && ret != -ERESTARTSYS)
  2344. pr_err("accept thread returning with ret: %d\n", ret);
  2345. return ret;
  2346. }
  2347. static long process_invoke_req(struct file *filp, unsigned int cmd,
  2348. unsigned long arg)
  2349. {
  2350. int ret = -1, nr_args = 0;
  2351. struct smcinvoke_cmd_req req = {0};
  2352. void *in_msg = NULL, *out_msg = NULL;
  2353. size_t inmsg_size = 0, outmsg_size = SMCINVOKE_TZ_MIN_BUF_SIZE;
  2354. union smcinvoke_arg *args_buf = NULL;
  2355. struct smcinvoke_file_data *tzobj = filp->private_data;
  2356. struct qtee_shm in_shm = {0}, out_shm = {0};
  2357. LIST_HEAD(l_mem_objs_pending_async); /* Holds new memory objects, to be later sent to TZ */
  2358. /*
  2359. * Hold reference to remote object until invoke op is not
  2360. * completed. Release once invoke is done.
  2361. */
  2362. struct file *filp_to_release[OBJECT_COUNTS_MAX_OO] = {NULL};
  2363. /*
  2364. * If anything goes wrong, release alloted tzhandles for
  2365. * local objs which could be either CBObj or MemObj.
  2366. */
  2367. int32_t tzhandles_to_release[OBJECT_COUNTS_MAX_OO] = {0};
  2368. bool tz_acked = false;
  2369. uint32_t context_type = tzobj->context_type;
  2370. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ &&
  2371. _IOC_SIZE(cmd) != sizeof(req)) {
  2372. pr_err("command size for invoke req is invalid\n");
  2373. return -EINVAL;
  2374. }
  2375. if (context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ &&
  2376. context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  2377. pr_err("invalid context_type %d\n", context_type);
  2378. return -EPERM;
  2379. }
  2380. if (context_type != SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL) {
  2381. ret = copy_from_user(&req, (void __user *)arg, sizeof(req));
  2382. if (ret) {
  2383. pr_err("copying invoke req failed\n");
  2384. return -EFAULT;
  2385. }
  2386. } else {
  2387. req = *(struct smcinvoke_cmd_req *)arg;
  2388. }
  2389. if (req.argsize != sizeof(union smcinvoke_arg)) {
  2390. pr_err("arguments size for invoke req is invalid\n");
  2391. return -EINVAL;
  2392. }
  2393. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ &&
  2394. tzobj->tzhandle == SMCINVOKE_TZ_ROOT_OBJ &&
  2395. (req.op == IClientEnv_OP_notifyDomainChange ||
  2396. req.op == IClientEnv_OP_registerWithCredentials ||
  2397. req.op == IClientEnv_OP_adciAccept ||
  2398. req.op == IClientEnv_OP_adciShutdown)) {
  2399. pr_err("invalid rootenv op\n");
  2400. return -EINVAL;
  2401. }
  2402. nr_args = OBJECT_COUNTS_NUM_buffers(req.counts) +
  2403. OBJECT_COUNTS_NUM_objects(req.counts);
  2404. if (nr_args) {
  2405. args_buf = kcalloc(nr_args, req.argsize, GFP_KERNEL);
  2406. if (!args_buf)
  2407. return -ENOMEM;
  2408. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  2409. ret = copy_from_user(args_buf,
  2410. u64_to_user_ptr(req.args),
  2411. nr_args * req.argsize);
  2412. if (ret) {
  2413. ret = -EFAULT;
  2414. goto out;
  2415. }
  2416. } else {
  2417. memcpy(args_buf, (void *)(req.args),
  2418. nr_args * req.argsize);
  2419. }
  2420. }
  2421. inmsg_size = compute_in_msg_size(&req, args_buf);
  2422. ret = qtee_shmbridge_allocate_shm(inmsg_size, &in_shm);
  2423. if (ret) {
  2424. ret = -ENOMEM;
  2425. pr_err("shmbridge alloc failed for in msg in invoke req\n");
  2426. goto out;
  2427. }
  2428. in_msg = in_shm.vaddr;
  2429. mutex_lock(&g_smcinvoke_lock);
  2430. outmsg_size = PAGE_ALIGN(g_max_cb_buf_size);
  2431. mutex_unlock(&g_smcinvoke_lock);
  2432. ret = qtee_shmbridge_allocate_shm(outmsg_size, &out_shm);
  2433. if (ret) {
  2434. ret = -ENOMEM;
  2435. pr_err("shmbridge alloc failed for out msg in invoke req\n");
  2436. goto out;
  2437. }
  2438. out_msg = out_shm.vaddr;
  2439. trace_process_invoke_req_tzhandle(tzobj->tzhandle, req.op, req.counts);
  2440. ret = marshal_in_invoke_req(&req, args_buf, tzobj->tzhandle, in_msg,
  2441. inmsg_size, filp_to_release, tzhandles_to_release,
  2442. context_type, &l_mem_objs_pending_async);
  2443. if (ret) {
  2444. pr_err("failed to marshal in invoke req, ret :%d\n", ret);
  2445. goto out;
  2446. }
  2447. if (mem_obj_async_support) {
  2448. mutex_lock(&g_smcinvoke_lock);
  2449. add_mem_obj_info_to_async_side_channel_locked(out_msg, outmsg_size, &l_mem_objs_pending_async);
  2450. mutex_unlock(&g_smcinvoke_lock);
  2451. }
  2452. ret = prepare_send_scm_msg(in_msg, in_shm.paddr, inmsg_size,
  2453. out_msg, out_shm.paddr, outmsg_size,
  2454. &req, args_buf, &tz_acked, context_type,
  2455. &in_shm, &out_shm, true);
  2456. /*
  2457. * If scm_call is success, TZ owns responsibility to release
  2458. * refs for local objs.
  2459. */
  2460. if (!tz_acked) {
  2461. trace_status(__func__, "scm call successful");
  2462. goto out;
  2463. }
  2464. memset(tzhandles_to_release, 0, sizeof(tzhandles_to_release));
  2465. /*
  2466. * if invoke op results in an err, no need to marshal_out and
  2467. * copy args buf to user space
  2468. */
  2469. if (!req.result) {
  2470. /*
  2471. * Dont check ret of marshal_out because there might be a
  2472. * FD for OO which userspace must release even if an error
  2473. * occurs. Releasing FD from user space is much simpler than
  2474. * doing here. ORing of ret is reqd not to miss past error
  2475. */
  2476. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ)
  2477. ret |= copy_to_user(u64_to_user_ptr(req.args),
  2478. args_buf, nr_args * req.argsize);
  2479. else
  2480. memcpy((void *)(req.args), args_buf,
  2481. nr_args * req.argsize);
  2482. }
  2483. /* copy result of invoke op */
  2484. if (context_type == SMCINVOKE_OBJ_TYPE_TZ_OBJ) {
  2485. ret |= copy_to_user((void __user *)arg, &req, sizeof(req));
  2486. if (ret)
  2487. goto out;
  2488. } else {
  2489. memcpy((void *)arg, (void *)&req, sizeof(req));
  2490. }
  2491. /* Outbuf could be carrying local objs to be released. */
  2492. process_piggyback_data(out_msg, outmsg_size);
  2493. out:
  2494. trace_process_invoke_req_result(ret, req.result, tzobj->tzhandle,
  2495. req.op, req.counts);
  2496. release_filp(filp_to_release, OBJECT_COUNTS_MAX_OO);
  2497. if (ret) {
  2498. release_map_obj_pending_async_list_locked(&l_mem_objs_pending_async);
  2499. release_tzhandles(tzhandles_to_release, OBJECT_COUNTS_MAX_OO);
  2500. }
  2501. delete_pending_async_list_locked(&l_mem_objs_pending_async);
  2502. qtee_shmbridge_free_shm(&in_shm);
  2503. qtee_shmbridge_free_shm(&out_shm);
  2504. kfree(args_buf);
  2505. if (ret)
  2506. pr_err("invoke thread returning with ret = %d\n", ret);
  2507. return ret;
  2508. }
  2509. static long process_log_info(struct file *filp, unsigned int cmd,
  2510. unsigned long arg)
  2511. {
  2512. int ret = 0;
  2513. char buf[SMCINVOKE_LOG_BUF_SIZE];
  2514. struct smcinvoke_file_data *tzobj = filp->private_data;
  2515. ret = copy_from_user(buf, (void __user *)arg, SMCINVOKE_LOG_BUF_SIZE);
  2516. if (ret) {
  2517. pr_err("logging HLOS info copy failed\n");
  2518. return -EFAULT;
  2519. }
  2520. buf[SMCINVOKE_LOG_BUF_SIZE - 1] = '\0';
  2521. trace_process_log_info(buf, tzobj->context_type, tzobj->tzhandle);
  2522. return ret;
  2523. }
  2524. static long smcinvoke_ioctl(struct file *filp, unsigned int cmd,
  2525. unsigned long arg)
  2526. {
  2527. long ret = 0;
  2528. switch (cmd) {
  2529. case SMCINVOKE_IOCTL_INVOKE_REQ:
  2530. ret = process_invoke_req(filp, cmd, arg);
  2531. break;
  2532. case SMCINVOKE_IOCTL_ACCEPT_REQ:
  2533. ret = process_accept_req(filp, cmd, arg);
  2534. break;
  2535. case SMCINVOKE_IOCTL_SERVER_REQ:
  2536. ret = process_server_req(filp, cmd, arg);
  2537. break;
  2538. case SMCINVOKE_IOCTL_ACK_LOCAL_OBJ:
  2539. ret = process_ack_local_obj(filp, cmd, arg);
  2540. break;
  2541. case SMCINVOKE_IOCTL_LOG:
  2542. ret = process_log_info(filp, cmd, arg);
  2543. break;
  2544. default:
  2545. ret = -ENOIOCTLCMD;
  2546. break;
  2547. }
  2548. trace_smcinvoke_ioctl(cmd, ret);
  2549. return ret;
  2550. }
  2551. int get_root_fd(int *root_fd)
  2552. {
  2553. if (!root_fd)
  2554. return -EINVAL;
  2555. else
  2556. return get_fd_for_obj(SMCINVOKE_OBJ_TYPE_TZ_OBJ_FOR_KERNEL,
  2557. SMCINVOKE_TZ_ROOT_OBJ, root_fd);
  2558. }
  2559. int process_invoke_request_from_kernel_client(int fd,
  2560. struct smcinvoke_cmd_req *req)
  2561. {
  2562. struct file *filp = NULL;
  2563. int ret = 0;
  2564. if (!req) {
  2565. pr_err("NULL req\n");
  2566. return -EINVAL;
  2567. }
  2568. filp = fget(fd);
  2569. if (!filp) {
  2570. pr_err("Invalid fd %d\n", fd);
  2571. return -EINVAL;
  2572. }
  2573. ret = process_invoke_req(filp, 0, (uintptr_t)req);
  2574. fput(filp);
  2575. trace_process_invoke_request_from_kernel_client(fd, filp, file_count(filp));
  2576. return ret;
  2577. }
  2578. static int smcinvoke_open(struct inode *nodp, struct file *filp)
  2579. {
  2580. struct smcinvoke_file_data *tzcxt = NULL;
  2581. tzcxt = kzalloc(sizeof(*tzcxt), GFP_KERNEL);
  2582. if (!tzcxt)
  2583. return -ENOMEM;
  2584. tzcxt->tzhandle = SMCINVOKE_TZ_ROOT_OBJ;
  2585. tzcxt->context_type = SMCINVOKE_OBJ_TYPE_TZ_OBJ;
  2586. filp->private_data = tzcxt;
  2587. return 0;
  2588. }
  2589. static int release_cb_server(uint16_t server_id)
  2590. {
  2591. struct smcinvoke_server_info *server = NULL;
  2592. mutex_lock(&g_smcinvoke_lock);
  2593. server = find_cb_server_locked(server_id);
  2594. if (server)
  2595. kref_put(&server->ref_cnt, destroy_cb_server);
  2596. mutex_unlock(&g_smcinvoke_lock);
  2597. return 0;
  2598. }
  2599. int smcinvoke_release_filp(struct file *filp)
  2600. {
  2601. int ret = 0;
  2602. struct smcinvoke_file_data *file_data = filp->private_data;
  2603. uint32_t tzhandle = 0;
  2604. struct smcinvoke_object_release_pending_list *entry = NULL;
  2605. struct qtee_shm in_shm = {0}, out_shm = {0};
  2606. trace_smcinvoke_release_filp(current->files, filp,
  2607. file_count(filp), file_data->context_type);
  2608. if (file_data->context_type == SMCINVOKE_OBJ_TYPE_SERVER) {
  2609. ret = release_cb_server(file_data->server_id);
  2610. goto out;
  2611. }
  2612. tzhandle = file_data->tzhandle;
  2613. /* Root object is special in sense it is indestructible */
  2614. if (!tzhandle || tzhandle == SMCINVOKE_TZ_ROOT_OBJ) {
  2615. if (!tzhandle)
  2616. pr_err("tzhandle not valid in object release\n");
  2617. goto out;
  2618. }
  2619. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &in_shm);
  2620. if (ret) {
  2621. pr_err("shmbridge alloc failed for in msg in object release"
  2622. "with ret %d\n", ret);
  2623. goto out;
  2624. }
  2625. ret = qtee_shmbridge_allocate_shm(SMCINVOKE_TZ_MIN_BUF_SIZE, &out_shm);
  2626. if (ret) {
  2627. pr_err("shmbridge alloc failed for out msg in object release"
  2628. "with ret:%d\n", ret);
  2629. goto out;
  2630. }
  2631. ret = smcinvoke_release_tz_object(&in_shm, &out_shm,
  2632. tzhandle, file_data->context_type);
  2633. if (-EBUSY == ret) {
  2634. pr_debug("failed to release handle in sync adding to list\n");
  2635. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  2636. if (!entry) {
  2637. ret = -ENOMEM;
  2638. goto out;
  2639. }
  2640. ret = 0;
  2641. entry->data.tzhandle = tzhandle;
  2642. entry->data.context_type = file_data->context_type;
  2643. mutex_lock(&object_postprocess_lock);
  2644. list_add_tail(&entry->list, &g_object_postprocess);
  2645. mutex_unlock(&object_postprocess_lock);
  2646. pr_debug("Object release list: added a handle:0x%lx\n", tzhandle);
  2647. __wakeup_postprocess_kthread(&smcinvoke[OBJECT_WORKER_THREAD]);
  2648. }
  2649. out:
  2650. qtee_shmbridge_free_shm(&in_shm);
  2651. qtee_shmbridge_free_shm(&out_shm);
  2652. kfree(filp->private_data);
  2653. filp->private_data = NULL;
  2654. if (ret != 0)
  2655. pr_err ("Object release failed with ret %d\n", ret);
  2656. return ret;
  2657. }
  2658. int smcinvoke_release_from_kernel_client(int fd)
  2659. {
  2660. struct file *filp = NULL;
  2661. /* use fget() to get filp, but this will increase file ref_cnt to 1,
  2662. * then decrease file ref_cnt to 0 with fput().
  2663. */
  2664. filp = fget(fd);
  2665. if (!filp) {
  2666. pr_err("invalid fd %d to release\n", fd);
  2667. return -EINVAL;
  2668. }
  2669. trace_smcinvoke_release_from_kernel_client(current->files, filp,
  2670. file_count(filp));
  2671. /* free filp, notify TZ to release object */
  2672. smcinvoke_release_filp(filp);
  2673. fput(filp);
  2674. return 0;
  2675. }
  2676. static int smcinvoke_release(struct inode *nodp, struct file *filp)
  2677. {
  2678. trace_smcinvoke_release(current->files, filp, file_count(filp),
  2679. filp->private_data);
  2680. if (filp->private_data)
  2681. return smcinvoke_release_filp(filp);
  2682. else
  2683. return 0;
  2684. }
  2685. static int smcinvoke_probe(struct platform_device *pdev)
  2686. {
  2687. unsigned int baseminor = 0;
  2688. unsigned int count = 1;
  2689. int rc = 0;
  2690. rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
  2691. if (rc) {
  2692. pr_err("dma_set_mask_and_coherent failed %d\n", rc);
  2693. return rc;
  2694. }
  2695. legacy_smc_call = of_property_read_bool((&pdev->dev)->of_node,
  2696. "qcom,support-legacy_smc");
  2697. invoke_cmd = legacy_smc_call ? SMCINVOKE_INVOKE_CMD_LEGACY : SMCINVOKE_INVOKE_CMD;
  2698. rc = smcinvoke_create_kthreads();
  2699. if (rc) {
  2700. pr_err("smcinvoke_create_kthreads failed %d\n", rc);
  2701. return rc;
  2702. }
  2703. rc = alloc_chrdev_region(&smcinvoke_device_no, baseminor, count,
  2704. SMCINVOKE_DEV);
  2705. if (rc < 0) {
  2706. pr_err("chrdev_region failed %d for %s\n", rc, SMCINVOKE_DEV);
  2707. goto exit_destroy_wkthread;
  2708. }
  2709. driver_class = class_create(THIS_MODULE, SMCINVOKE_DEV);
  2710. if (IS_ERR(driver_class)) {
  2711. rc = -ENOMEM;
  2712. pr_err("class_create failed %d\n", rc);
  2713. goto exit_unreg_chrdev_region;
  2714. }
  2715. class_dev = device_create(driver_class, NULL, smcinvoke_device_no,
  2716. NULL, SMCINVOKE_DEV);
  2717. if (!class_dev) {
  2718. pr_err("class_device_create failed %d\n", rc);
  2719. rc = -ENOMEM;
  2720. goto exit_destroy_class;
  2721. }
  2722. cdev_init(&smcinvoke_cdev, &g_smcinvoke_fops);
  2723. smcinvoke_cdev.owner = THIS_MODULE;
  2724. rc = cdev_add(&smcinvoke_cdev, MKDEV(MAJOR(smcinvoke_device_no), 0),
  2725. count);
  2726. if (rc < 0) {
  2727. pr_err("cdev_add failed %d for %s\n", rc, SMCINVOKE_DEV);
  2728. goto exit_destroy_device;
  2729. }
  2730. smcinvoke_pdev = pdev;
  2731. #if IS_ENABLED(CONFIG_QSEECOM_COMPAT) && IS_ENABLED(CONFIG_QSEECOM_PROXY)
  2732. /*If the api fails to get the func ops, print the error and continue
  2733. * Do not treat it as fatal*/
  2734. rc = get_qseecom_kernel_fun_ops();
  2735. if (rc) {
  2736. pr_err("failed to get qseecom kernel func ops %d", rc);
  2737. }
  2738. #endif
  2739. __wakeup_postprocess_kthread(&smcinvoke[ADCI_WORKER_THREAD]);
  2740. return 0;
  2741. exit_destroy_device:
  2742. device_destroy(driver_class, smcinvoke_device_no);
  2743. exit_destroy_class:
  2744. class_destroy(driver_class);
  2745. exit_unreg_chrdev_region:
  2746. unregister_chrdev_region(smcinvoke_device_no, count);
  2747. exit_destroy_wkthread:
  2748. smcinvoke_destroy_kthreads();
  2749. return rc;
  2750. }
  2751. static int smcinvoke_remove(struct platform_device *pdev)
  2752. {
  2753. int count = 1;
  2754. smcinvoke_destroy_kthreads();
  2755. cdev_del(&smcinvoke_cdev);
  2756. device_destroy(driver_class, smcinvoke_device_no);
  2757. class_destroy(driver_class);
  2758. unregister_chrdev_region(smcinvoke_device_no, count);
  2759. return 0;
  2760. }
  2761. static int __maybe_unused smcinvoke_suspend(struct platform_device *pdev,
  2762. pm_message_t state)
  2763. {
  2764. int ret = 0;
  2765. mutex_lock(&g_smcinvoke_lock);
  2766. if (cb_reqs_inflight) {
  2767. pr_err("Failed to suspend smcinvoke driver\n");
  2768. ret = -EIO;
  2769. }
  2770. mutex_unlock(&g_smcinvoke_lock);
  2771. return ret;
  2772. }
  2773. static int __maybe_unused smcinvoke_resume(struct platform_device *pdev)
  2774. {
  2775. return 0;
  2776. }
  2777. static const struct of_device_id smcinvoke_match[] = {
  2778. {
  2779. .compatible = "qcom,smcinvoke",
  2780. },
  2781. {},
  2782. };
  2783. static struct platform_driver smcinvoke_plat_driver = {
  2784. .probe = smcinvoke_probe,
  2785. .remove = smcinvoke_remove,
  2786. .suspend = smcinvoke_suspend,
  2787. .resume = smcinvoke_resume,
  2788. .driver = {
  2789. .name = "smcinvoke",
  2790. .of_match_table = smcinvoke_match,
  2791. },
  2792. };
  2793. static int smcinvoke_init(void)
  2794. {
  2795. return platform_driver_register(&smcinvoke_plat_driver);
  2796. }
  2797. static void smcinvoke_exit(void)
  2798. {
  2799. platform_driver_unregister(&smcinvoke_plat_driver);
  2800. }
  2801. module_init(smcinvoke_init);
  2802. module_exit(smcinvoke_exit);
  2803. MODULE_LICENSE("GPL v2");
  2804. MODULE_DESCRIPTION("SMC Invoke driver");
  2805. MODULE_IMPORT_NS(VFS_internal_I_am_really_a_filesystem_and_am_NOT_a_driver);
  2806. MODULE_IMPORT_NS(DMA_BUF);