synx.c 73 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2024, Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #include <linux/atomic.h>
  7. #include <linux/file.h>
  8. #include <linux/fs.h>
  9. #include <linux/module.h>
  10. #include <linux/poll.h>
  11. #include <linux/random.h>
  12. #include <linux/remoteproc/qcom_rproc.h>
  13. #include <linux/slab.h>
  14. #include <linux/sync_file.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/vmalloc.h>
  17. #include "synx_debugfs.h"
  18. #include "synx_private.h"
  19. #include "synx_util.h"
  20. struct synx_device *synx_dev;
  21. static atomic64_t synx_counter = ATOMIC64_INIT(1);
  22. void synx_external_callback(s32 sync_obj, int status, void *data)
  23. {
  24. struct synx_signal_cb *signal_cb = data;
  25. if (IS_ERR_OR_NULL(signal_cb)) {
  26. dprintk(SYNX_ERR,
  27. "invalid payload from external obj %d [%d]\n",
  28. sync_obj, status);
  29. return;
  30. }
  31. signal_cb->status = status;
  32. signal_cb->ext_sync_id = sync_obj;
  33. signal_cb->flag = SYNX_SIGNAL_FROM_CALLBACK;
  34. dprintk(SYNX_DBG,
  35. "external callback from %d on handle %u\n",
  36. sync_obj, signal_cb->handle);
  37. /*
  38. * invoke the handler directly as external callback
  39. * is invoked from separate task.
  40. * avoids creation of separate task again.
  41. */
  42. synx_signal_handler(&signal_cb->cb_dispatch);
  43. }
  44. EXPORT_SYMBOL(synx_external_callback);
  45. bool synx_fence_enable_signaling(struct dma_fence *fence)
  46. {
  47. return true;
  48. }
  49. const char *synx_fence_driver_name(struct dma_fence *fence)
  50. {
  51. return "Global Synx driver";
  52. }
  53. void synx_fence_release(struct dma_fence *fence)
  54. {
  55. /* release the memory allocated during create */
  56. kfree(fence->lock);
  57. kfree(fence);
  58. dprintk(SYNX_MEM, "released backing fence %pK\n", fence);
  59. }
  60. EXPORT_SYMBOL(synx_fence_release);
  61. static struct dma_fence_ops synx_fence_ops = {
  62. .wait = dma_fence_default_wait,
  63. .enable_signaling = synx_fence_enable_signaling,
  64. .get_driver_name = synx_fence_driver_name,
  65. .get_timeline_name = synx_fence_driver_name,
  66. .release = synx_fence_release,
  67. };
  68. static int synx_create_sync_fd(struct dma_fence *fence)
  69. {
  70. int fd;
  71. struct sync_file *sync_file;
  72. if (IS_ERR_OR_NULL(fence))
  73. return -SYNX_INVALID;
  74. fd = get_unused_fd_flags(O_CLOEXEC);
  75. if (fd < 0)
  76. return fd;
  77. sync_file = sync_file_create(fence);
  78. if (IS_ERR_OR_NULL(sync_file)) {
  79. dprintk(SYNX_ERR, "error creating sync file\n");
  80. goto err;
  81. }
  82. fd_install(fd, sync_file->file);
  83. return fd;
  84. err:
  85. put_unused_fd(fd);
  86. return -SYNX_INVALID;
  87. }
  88. void *synx_get_fence(struct synx_session *session,
  89. u32 h_synx)
  90. {
  91. struct synx_client *client;
  92. struct synx_handle_coredata *synx_data;
  93. struct synx_coredata *synx_obj;
  94. struct dma_fence *fence = NULL;
  95. client = synx_get_client(session);
  96. if (IS_ERR_OR_NULL(client))
  97. return NULL;
  98. synx_data = synx_util_acquire_handle(client, h_synx);
  99. synx_obj = synx_util_obtain_object(synx_data);
  100. if (IS_ERR_OR_NULL(synx_obj) ||
  101. IS_ERR_OR_NULL(synx_obj->fence)) {
  102. dprintk(SYNX_ERR,
  103. "[sess :%llu] invalid handle access %u\n",
  104. client->id, h_synx);
  105. goto fail;
  106. }
  107. mutex_lock(&synx_obj->obj_lock);
  108. fence = synx_obj->fence;
  109. /* obtain an additional reference to the fence */
  110. dma_fence_get(fence);
  111. mutex_unlock(&synx_obj->obj_lock);
  112. fail:
  113. synx_util_release_handle(synx_data);
  114. synx_put_client(client);
  115. return fence;
  116. }
  117. EXPORT_SYMBOL(synx_get_fence);
  118. static int synx_native_check_bind(struct synx_client *client,
  119. struct synx_create_params *params)
  120. {
  121. int rc;
  122. u32 h_synx;
  123. struct synx_entry_64 *ext_entry;
  124. struct synx_map_entry *entry;
  125. if (IS_ERR_OR_NULL(params->fence))
  126. return -SYNX_INVALID;
  127. ext_entry = synx_util_retrieve_data(params->fence,
  128. synx_util_map_params_to_type(params->flags));
  129. if (IS_ERR_OR_NULL(ext_entry))
  130. return -SYNX_NOENT;
  131. h_synx = ext_entry->data[0];
  132. synx_util_remove_data(params->fence,
  133. synx_util_map_params_to_type(params->flags));
  134. entry = synx_util_get_map_entry(h_synx);
  135. if (IS_ERR_OR_NULL(entry))
  136. /* possible cleanup, retry to alloc new handle */
  137. return -SYNX_NOENT;
  138. rc = synx_util_init_handle(client, entry->synx_obj,
  139. &h_synx, entry);
  140. if (rc != SYNX_SUCCESS) {
  141. dprintk(SYNX_ERR,
  142. "[sess :%llu] new handle init failed\n",
  143. client->id);
  144. goto fail;
  145. }
  146. *params->h_synx = h_synx;
  147. return SYNX_SUCCESS;
  148. fail:
  149. synx_util_release_map_entry(entry);
  150. return rc;
  151. }
  152. static int synx_native_create_core(struct synx_client *client,
  153. struct synx_create_params *params)
  154. {
  155. int rc;
  156. struct synx_coredata *synx_obj;
  157. struct synx_map_entry *map_entry;
  158. if (IS_ERR_OR_NULL(client) || IS_ERR_OR_NULL(params) ||
  159. IS_ERR_OR_NULL(params->h_synx))
  160. return -SYNX_INVALID;
  161. synx_obj = kzalloc(sizeof(*synx_obj), GFP_KERNEL);
  162. if (IS_ERR_OR_NULL(synx_obj))
  163. return -SYNX_NOMEM;
  164. rc = synx_util_init_coredata(synx_obj, params,
  165. &synx_fence_ops, client->dma_context);
  166. if (rc) {
  167. dprintk(SYNX_ERR,
  168. "[sess :%llu] handle allocation failed\n",
  169. client->id);
  170. kfree(synx_obj);
  171. goto fail;
  172. }
  173. map_entry = synx_util_insert_to_map(synx_obj,
  174. *params->h_synx, 0);
  175. if (IS_ERR_OR_NULL(map_entry)) {
  176. rc = PTR_ERR(map_entry);
  177. synx_util_put_object(synx_obj);
  178. goto fail;
  179. }
  180. rc = synx_util_add_callback(synx_obj, *params->h_synx);
  181. if (rc != SYNX_SUCCESS) {
  182. synx_util_release_map_entry(map_entry);
  183. goto fail;
  184. }
  185. rc = synx_util_init_handle(client, synx_obj,
  186. params->h_synx, map_entry);
  187. if (rc < 0) {
  188. dprintk(SYNX_ERR,
  189. "[sess :%llu] unable to init new handle\n",
  190. client->id);
  191. synx_util_release_map_entry(map_entry);
  192. goto fail;
  193. }
  194. dprintk(SYNX_MEM,
  195. "[sess :%llu] allocated %u, core %pK, fence %pK\n",
  196. client->id, *params->h_synx, synx_obj, synx_obj->fence);
  197. return SYNX_SUCCESS;
  198. fail:
  199. return rc;
  200. }
  201. int synx_create(struct synx_session *session,
  202. struct synx_create_params *params)
  203. {
  204. int rc = -SYNX_NOENT;
  205. struct synx_client *client;
  206. struct synx_external_desc_v2 ext_desc = {0};
  207. if (IS_ERR_OR_NULL(params) || IS_ERR_OR_NULL(params->h_synx) ||
  208. params->flags > SYNX_CREATE_MAX_FLAGS) {
  209. dprintk(SYNX_ERR, "invalid create arguments\n");
  210. return -SYNX_INVALID;
  211. }
  212. if (params->flags & SYNX_CREATE_DMA_FENCE) {
  213. dprintk(SYNX_ERR,
  214. "handle create with native fence not supported\n");
  215. return -SYNX_NOSUPPORT;
  216. }
  217. client = synx_get_client(session);
  218. if (IS_ERR_OR_NULL(client))
  219. return -SYNX_INVALID;
  220. *params->h_synx = 0;
  221. do {
  222. /* create with external fence */
  223. if (!IS_ERR_OR_NULL(params->fence))
  224. rc = synx_native_check_bind(client, params);
  225. if (rc == -SYNX_NOENT) {
  226. rc = synx_native_create_core(client, params);
  227. if (rc == SYNX_SUCCESS &&
  228. !IS_ERR_OR_NULL(params->fence)) {
  229. /* save external fence details */
  230. rc = synx_util_save_data(params->fence,
  231. synx_util_map_params_to_type(params->flags),
  232. *params->h_synx);
  233. if (rc == -SYNX_ALREADY) {
  234. /*
  235. * raced with create on same fence from
  236. * another client. clear the allocated
  237. * handle and retry.
  238. */
  239. synx_native_release_core(client, *params->h_synx);
  240. *params->h_synx = 0;
  241. rc = -SYNX_NOENT;
  242. continue;
  243. } else if (rc != SYNX_SUCCESS) {
  244. dprintk(SYNX_ERR,
  245. "allocating handle failed=%d", rc);
  246. synx_native_release_core(client, *params->h_synx);
  247. break;
  248. }
  249. /* bind with external fence */
  250. ext_desc.id = *((u32 *)params->fence);
  251. ext_desc.type = synx_util_map_params_to_type(params->flags);
  252. rc = synx_bind(session, *params->h_synx, ext_desc);
  253. if (rc != SYNX_SUCCESS) {
  254. dprintk(SYNX_ERR,
  255. "[sess :%llu] bind external fence failed\n",
  256. client->id);
  257. synx_native_release_core(client, *params->h_synx);
  258. goto fail;
  259. }
  260. }
  261. }
  262. if (rc == SYNX_SUCCESS)
  263. dprintk(SYNX_VERB,
  264. "[sess :%llu] handle allocated %u\n",
  265. client->id, *params->h_synx);
  266. break;
  267. } while (true);
  268. fail:
  269. synx_put_client(client);
  270. return rc;
  271. }
  272. EXPORT_SYMBOL(synx_create);
  273. int synx_native_signal_core(struct synx_coredata *synx_obj,
  274. u32 status,
  275. bool cb_signal,
  276. u64 ext_sync_id)
  277. {
  278. int rc = 0;
  279. int ret;
  280. u32 i = 0;
  281. u32 idx = 0;
  282. s32 sync_id;
  283. u32 type;
  284. void *data = NULL;
  285. struct synx_bind_desc bind_descs[SYNX_MAX_NUM_BINDINGS];
  286. struct bind_operations *bind_ops = NULL;
  287. if (IS_ERR_OR_NULL(synx_obj))
  288. return -SYNX_INVALID;
  289. synx_util_callback_dispatch(synx_obj, status);
  290. /*
  291. * signal the external bound sync obj/s even if fence signal fails,
  292. * w/ error signal state (set above) to prevent deadlock
  293. */
  294. if (synx_obj->num_bound_synxs > 0) {
  295. memset(bind_descs, 0,
  296. sizeof(struct synx_bind_desc) * SYNX_MAX_NUM_BINDINGS);
  297. for (i = 0; i < synx_obj->num_bound_synxs; i++) {
  298. /* signal invoked by external sync obj */
  299. if (cb_signal &&
  300. (ext_sync_id ==
  301. synx_obj->bound_synxs[i].external_desc.id)) {
  302. dprintk(SYNX_VERB,
  303. "skipping signaling inbound sync: %llu\n",
  304. ext_sync_id);
  305. type = synx_obj->bound_synxs[i].external_desc.type;
  306. memset(&synx_obj->bound_synxs[i], 0,
  307. sizeof(struct synx_bind_desc));
  308. /* clear the hash table entry */
  309. synx_util_remove_data(&ext_sync_id, type);
  310. continue;
  311. }
  312. memcpy(&bind_descs[idx++],
  313. &synx_obj->bound_synxs[i],
  314. sizeof(struct synx_bind_desc));
  315. /* clear the memory, its been backed up above */
  316. memset(&synx_obj->bound_synxs[i], 0,
  317. sizeof(struct synx_bind_desc));
  318. }
  319. synx_obj->num_bound_synxs = 0;
  320. }
  321. for (i = 0; i < idx; i++) {
  322. sync_id = bind_descs[i].external_desc.id;
  323. data = bind_descs[i].external_data;
  324. type = bind_descs[i].external_desc.type;
  325. bind_ops = synx_util_get_bind_ops(type);
  326. if (IS_ERR_OR_NULL(bind_ops)) {
  327. dprintk(SYNX_ERR,
  328. "invalid bind ops for type: %u\n", type);
  329. kfree(data);
  330. continue;
  331. }
  332. /* clear the hash table entry */
  333. synx_util_remove_data(&sync_id, type);
  334. /*
  335. * we are already signaled, so don't want to
  336. * recursively be signaled
  337. */
  338. ret = bind_ops->deregister_callback(
  339. synx_external_callback, data, sync_id);
  340. if (ret < 0) {
  341. dprintk(SYNX_ERR,
  342. "deregistration fail on %d, type: %u, err=%d\n",
  343. sync_id, type, ret);
  344. continue;
  345. }
  346. dprintk(SYNX_VERB,
  347. "signal external sync: %d, type: %u, status: %u\n",
  348. sync_id, type, status);
  349. /* optional function to enable external signaling */
  350. if (bind_ops->enable_signaling) {
  351. ret = bind_ops->enable_signaling(sync_id);
  352. if (ret < 0)
  353. dprintk(SYNX_ERR,
  354. "enabling fail on %d, type: %u, err=%d\n",
  355. sync_id, type, ret);
  356. }
  357. ret = bind_ops->signal(sync_id, status);
  358. if (ret < 0)
  359. dprintk(SYNX_ERR,
  360. "signaling fail on %d, type: %u, err=%d\n",
  361. sync_id, type, ret);
  362. /*
  363. * release the memory allocated for external data.
  364. * It is safe to release this memory as external cb
  365. * has been already deregistered before this.
  366. */
  367. kfree(data);
  368. }
  369. return rc;
  370. }
  371. int synx_native_signal_fence(struct synx_coredata *synx_obj,
  372. u32 status)
  373. {
  374. int rc = 0;
  375. unsigned long flags;
  376. if (IS_ERR_OR_NULL(synx_obj) || IS_ERR_OR_NULL(synx_obj->fence))
  377. return -SYNX_INVALID;
  378. if (status <= SYNX_STATE_ACTIVE) {
  379. dprintk(SYNX_ERR, "signaling with wrong status: %u\n",
  380. status);
  381. return -SYNX_INVALID;
  382. }
  383. if (synx_util_is_merged_object(synx_obj)) {
  384. dprintk(SYNX_ERR, "signaling a composite handle\n");
  385. return -SYNX_INVALID;
  386. }
  387. if (synx_util_get_object_status(synx_obj) !=
  388. SYNX_STATE_ACTIVE)
  389. return -SYNX_ALREADY;
  390. if (IS_ERR_OR_NULL(synx_obj->signal_cb)) {
  391. dprintk(SYNX_ERR, "signal cb in bad state\n");
  392. return -SYNX_INVALID;
  393. }
  394. /*
  395. * remove registered callback for the fence
  396. * so it does not invoke the signal through callback again
  397. */
  398. if (!dma_fence_remove_callback(synx_obj->fence,
  399. &synx_obj->signal_cb->fence_cb)) {
  400. dprintk(SYNX_ERR, "callback could not be removed\n");
  401. return -SYNX_INVALID;
  402. }
  403. dprintk(SYNX_MEM, "signal cb destroyed %pK\n",
  404. synx_obj->signal_cb);
  405. kfree(synx_obj->signal_cb);
  406. synx_obj->signal_cb = NULL;
  407. /* releasing reference held by signal cb */
  408. synx_util_put_object(synx_obj);
  409. spin_lock_irqsave(synx_obj->fence->lock, flags);
  410. /* check the status again acquiring lock to avoid errors */
  411. if (synx_util_get_object_status_locked(synx_obj) !=
  412. SYNX_STATE_ACTIVE) {
  413. spin_unlock_irqrestore(synx_obj->fence->lock, flags);
  414. return -SYNX_ALREADY;
  415. }
  416. synx_obj->status = status;
  417. if (status >= SYNX_DMA_FENCE_STATE_MAX)
  418. status = SYNX_DMA_FENCE_STATE_MAX - 1;
  419. /* set fence error to model {signal w/ error} */
  420. if (status != SYNX_STATE_SIGNALED_SUCCESS)
  421. dma_fence_set_error(synx_obj->fence, -status);
  422. rc = dma_fence_signal_locked(synx_obj->fence);
  423. if (rc)
  424. dprintk(SYNX_ERR,
  425. "signaling fence %pK failed=%d\n",
  426. synx_obj->fence, rc);
  427. spin_unlock_irqrestore(synx_obj->fence->lock, flags);
  428. return rc;
  429. }
  430. int synx_native_signal_merged_fence(struct synx_coredata *synx_obj, u32 status)
  431. {
  432. int rc = SYNX_SUCCESS;
  433. unsigned long flags;
  434. int i = 0, num_fences = 0;
  435. struct synx_coredata **synx_child_obj = NULL;
  436. rc = synx_get_child_coredata(synx_obj, &synx_child_obj, &num_fences);
  437. if (rc != SYNX_SUCCESS)
  438. return rc;
  439. for(i = 0; i < num_fences; i++)
  440. {
  441. if (IS_ERR_OR_NULL(synx_child_obj[i]) || IS_ERR_OR_NULL(synx_child_obj[i]->fence)) {
  442. dprintk(SYNX_ERR, "Invalid child coredata %d\n", i);
  443. rc = -SYNX_NOENT;
  444. goto fail;
  445. }
  446. mutex_lock(&synx_child_obj[i]->obj_lock);
  447. spin_lock_irqsave(synx_child_obj[i]->fence->lock, flags);
  448. if (synx_util_get_object_status_locked(synx_child_obj[i]) != SYNX_STATE_ACTIVE ||
  449. !synx_util_is_global_object(synx_child_obj[i]))
  450. {
  451. spin_unlock_irqrestore(synx_child_obj[i]->fence->lock, flags);
  452. mutex_unlock(&synx_child_obj[i]->obj_lock);
  453. continue;
  454. }
  455. spin_unlock_irqrestore(synx_child_obj[i]->fence->lock, flags);
  456. status = synx_global_get_status(synx_child_obj[i]->global_idx);
  457. rc = synx_native_signal_fence(synx_child_obj[i], status);
  458. mutex_unlock(&synx_child_obj[i]->obj_lock);
  459. }
  460. fail:
  461. kfree(synx_child_obj);
  462. return rc;
  463. }
  464. u32 synx_get_child_status(struct synx_coredata *synx_obj)
  465. {
  466. u32 h_child = 0, i = 0;
  467. u32 status = SYNX_DMA_FENCE_STATE_MAX - 1, child_status = SYNX_STATE_ACTIVE;
  468. struct dma_fence_array *array = NULL;
  469. struct synx_map_entry *fence_entry = NULL;
  470. struct synx_coredata *synx_child_obj = NULL;
  471. if (!dma_fence_is_array(synx_obj->fence))
  472. return status;
  473. array = to_dma_fence_array(synx_obj->fence);
  474. if (IS_ERR_OR_NULL(array))
  475. goto bail;
  476. for (i = 0; i < array->num_fences; i++) {
  477. h_child = synx_util_get_fence_entry((u64)array->fences[i], 1);
  478. if (h_child == 0)
  479. h_child = synx_util_get_fence_entry((u64)array->fences[i], 0);
  480. if (h_child == 0)
  481. continue;
  482. fence_entry = synx_util_get_map_entry(h_child);
  483. if (IS_ERR_OR_NULL(fence_entry) || IS_ERR_OR_NULL(fence_entry->synx_obj)) {
  484. dprintk(SYNX_ERR, "Invalid handle access %u", h_child);
  485. goto bail;
  486. }
  487. synx_child_obj = fence_entry->synx_obj;
  488. mutex_lock(&synx_child_obj->obj_lock);
  489. if (synx_util_is_global_object(synx_child_obj))
  490. child_status = synx_global_get_status(synx_child_obj->global_idx);
  491. else
  492. child_status = synx_child_obj->status;
  493. mutex_unlock(&synx_child_obj->obj_lock);
  494. synx_util_release_map_entry(fence_entry);
  495. dprintk(SYNX_VERB, "Child handle %u status %d", h_child, child_status);
  496. if (child_status != SYNX_STATE_ACTIVE &&
  497. (status == SYNX_DMA_FENCE_STATE_MAX - 1 ||
  498. (child_status > SYNX_STATE_SIGNALED_SUCCESS &&
  499. child_status <= SYNX_STATE_SIGNALED_MAX)))
  500. status = child_status;
  501. }
  502. bail:
  503. return status;
  504. }
  505. u32 synx_custom_get_status(struct synx_coredata *synx_obj, u32 status)
  506. {
  507. u32 custom_status = status;
  508. u32 parent_global_status =
  509. synx_util_is_global_object(synx_obj) ?
  510. synx_global_get_status(synx_obj->global_idx) : SYNX_STATE_ACTIVE;
  511. if (IS_ERR_OR_NULL(synx_obj))
  512. goto bail;
  513. mutex_lock(&synx_obj->obj_lock);
  514. if (synx_util_is_merged_object(synx_obj)) {
  515. if (parent_global_status == SYNX_STATE_ACTIVE)
  516. synx_obj->status = synx_get_child_status(synx_obj);
  517. else
  518. synx_obj->status = parent_global_status;
  519. }
  520. custom_status = synx_obj->status;
  521. mutex_unlock(&synx_obj->obj_lock);
  522. bail:
  523. return custom_status;
  524. }
  525. void synx_signal_handler(struct work_struct *cb_dispatch)
  526. {
  527. int rc = SYNX_SUCCESS;
  528. u32 idx;
  529. struct synx_signal_cb *signal_cb =
  530. container_of(cb_dispatch, struct synx_signal_cb, cb_dispatch);
  531. struct synx_coredata *synx_obj = signal_cb->synx_obj;
  532. u32 h_synx = signal_cb->handle;
  533. u32 status = signal_cb->status;
  534. if (signal_cb->flag & SYNX_SIGNAL_FROM_FENCE) {
  535. status = synx_custom_get_status(synx_obj, status);
  536. dprintk(SYNX_VERB,
  537. "handle %d will be updated with status %d\n",
  538. h_synx, status);
  539. }
  540. if ((signal_cb->flag & SYNX_SIGNAL_FROM_FENCE) &&
  541. (synx_util_is_global_handle(h_synx) ||
  542. synx_util_is_global_object(synx_obj))) {
  543. idx = (IS_ERR_OR_NULL(synx_obj)) ?
  544. synx_util_global_idx(h_synx) :
  545. synx_obj->global_idx;
  546. if (synx_global_get_status(idx) == SYNX_STATE_ACTIVE) {
  547. rc = synx_global_update_status(idx, status);
  548. if (rc != SYNX_SUCCESS)
  549. dprintk(SYNX_ERR,
  550. "global status update of %u failed=%d\n",
  551. h_synx, rc);
  552. }
  553. /*
  554. * We are decrementing the reference here assuming this code will be
  555. * executed after handle is released. But in case if clients signal
  556. * dma fence in middle of execution sequence, then we will put
  557. * one reference thus deleting the global idx. As of now clients cannot
  558. * signal dma fence.
  559. */
  560. if (IS_ERR_OR_NULL(synx_obj))
  561. synx_global_put_ref(idx);
  562. }
  563. /*
  564. * when invoked from external callback, possible for
  565. * all local clients to have released the handle coredata.
  566. */
  567. if (IS_ERR_OR_NULL(synx_obj)) {
  568. dprintk(SYNX_WARN,
  569. "handle %d has no local clients\n",
  570. h_synx);
  571. dprintk(SYNX_MEM, "signal cb destroyed %pK\n",
  572. signal_cb);
  573. kfree(signal_cb);
  574. return;
  575. }
  576. if (rc != SYNX_SUCCESS) {
  577. dprintk(SYNX_ERR,
  578. "global status update for %u failed=%d\n",
  579. h_synx, rc);
  580. goto fail;
  581. }
  582. mutex_lock(&synx_obj->obj_lock);
  583. if (signal_cb->flag & SYNX_SIGNAL_FROM_IPC &&
  584. synx_util_get_object_status(synx_obj) == SYNX_STATE_ACTIVE) {
  585. if (synx_util_is_merged_object(synx_obj))
  586. rc = synx_native_signal_merged_fence(synx_obj, status);
  587. else
  588. rc = synx_native_signal_fence(synx_obj, status);
  589. }
  590. if (rc != SYNX_SUCCESS) {
  591. mutex_unlock(&synx_obj->obj_lock);
  592. dprintk(SYNX_ERR,
  593. "failed to signal fence %u with err=%d\n",
  594. h_synx, rc);
  595. goto fail;
  596. }
  597. if (rc == SYNX_SUCCESS && synx_util_get_object_status(synx_obj)
  598. != SYNX_STATE_ACTIVE)
  599. rc = synx_native_signal_core(synx_obj, status,
  600. (signal_cb->flag & SYNX_SIGNAL_FROM_CALLBACK) ?
  601. true : false, signal_cb->ext_sync_id);
  602. mutex_unlock(&synx_obj->obj_lock);
  603. if (rc != SYNX_SUCCESS)
  604. dprintk(SYNX_ERR,
  605. "internal signaling %u failed=%d",
  606. h_synx, rc);
  607. fail:
  608. /* release reference held by signal cb */
  609. synx_util_put_object(synx_obj);
  610. dprintk(SYNX_MEM, "signal cb destroyed %pK\n", signal_cb);
  611. kfree(signal_cb);
  612. dprintk(SYNX_VERB, "signal handle %u dispatch complete=%d",
  613. h_synx, rc);
  614. }
  615. /* function would be called from atomic context */
  616. void synx_fence_callback(struct dma_fence *fence,
  617. struct dma_fence_cb *cb)
  618. {
  619. s32 status;
  620. struct synx_signal_cb *signal_cb =
  621. container_of(cb, struct synx_signal_cb, fence_cb);
  622. dprintk(SYNX_DBG,
  623. "callback from external fence %pK for handle %u\n",
  624. fence, signal_cb->handle);
  625. /* other signal_cb members would be set during cb registration */
  626. status = dma_fence_get_status_locked(fence);
  627. /*
  628. * dma_fence_get_status_locked API returns 1 if signaled,
  629. * 0 if ACTIVE,
  630. * and negative error code in case of any failure
  631. */
  632. if (status == 1)
  633. status = SYNX_STATE_SIGNALED_SUCCESS;
  634. else if (status == -SYNX_STATE_SIGNALED_CANCEL)
  635. status = SYNX_STATE_SIGNALED_CANCEL;
  636. else if (status < 0 && status >= -SYNX_STATE_SIGNALED_MAX)
  637. status = SYNX_STATE_SIGNALED_EXTERNAL;
  638. else
  639. status = (u32)-status;
  640. signal_cb->status = status;
  641. INIT_WORK(&signal_cb->cb_dispatch, synx_signal_handler);
  642. queue_work(synx_dev->wq_cb, &signal_cb->cb_dispatch);
  643. }
  644. EXPORT_SYMBOL(synx_fence_callback);
  645. static int synx_signal_offload_job(
  646. struct synx_client *client,
  647. struct synx_coredata *synx_obj,
  648. u32 h_synx, u32 status)
  649. {
  650. int rc = SYNX_SUCCESS;
  651. struct synx_signal_cb *signal_cb;
  652. signal_cb = kzalloc(sizeof(*signal_cb), GFP_ATOMIC);
  653. if (IS_ERR_OR_NULL(signal_cb)) {
  654. rc = -SYNX_NOMEM;
  655. goto fail;
  656. }
  657. /*
  658. * since the signal will be queued to separate thread,
  659. * to ensure the synx coredata pointer remain valid, get
  660. * additional reference, thus avoiding any potential
  661. * use-after-free.
  662. */
  663. synx_util_get_object(synx_obj);
  664. signal_cb->handle = h_synx;
  665. signal_cb->status = status;
  666. signal_cb->synx_obj = synx_obj;
  667. signal_cb->flag = SYNX_SIGNAL_FROM_CLIENT;
  668. dprintk(SYNX_VERB,
  669. "[sess :%llu] signal work queued for %u\n",
  670. client->id, h_synx);
  671. INIT_WORK(&signal_cb->cb_dispatch, synx_signal_handler);
  672. queue_work(synx_dev->wq_cb, &signal_cb->cb_dispatch);
  673. fail:
  674. return rc;
  675. }
  676. int synx_signal(struct synx_session *session, u32 h_synx, u32 status)
  677. {
  678. int rc = SYNX_SUCCESS;
  679. struct synx_client *client;
  680. struct synx_handle_coredata *synx_data = NULL;
  681. struct synx_coredata *synx_obj;
  682. client = synx_get_client(session);
  683. if (IS_ERR_OR_NULL(client))
  684. return -SYNX_INVALID;
  685. if (status <= SYNX_STATE_ACTIVE ||
  686. !(status == SYNX_STATE_SIGNALED_SUCCESS ||
  687. status == SYNX_STATE_SIGNALED_CANCEL ||
  688. status > SYNX_STATE_SIGNALED_MAX)) {
  689. dprintk(SYNX_ERR,
  690. "[sess :%llu] signaling with wrong status: %u\n",
  691. client->id, status);
  692. rc = -SYNX_INVALID;
  693. goto fail;
  694. }
  695. synx_data = synx_util_acquire_handle(client, h_synx);
  696. synx_obj = synx_util_obtain_object(synx_data);
  697. if (IS_ERR_OR_NULL(synx_obj) ||
  698. IS_ERR_OR_NULL(synx_obj->fence)) {
  699. dprintk(SYNX_ERR,
  700. "[sess :%llu] invalid handle access %u\n",
  701. client->id, h_synx);
  702. rc = -SYNX_INVALID;
  703. goto fail;
  704. }
  705. mutex_lock(&synx_obj->obj_lock);
  706. if (synx_util_is_global_handle(h_synx) ||
  707. synx_util_is_global_object(synx_obj))
  708. rc = synx_global_update_status(
  709. synx_obj->global_idx, status);
  710. if (rc != SYNX_SUCCESS) {
  711. mutex_unlock(&synx_obj->obj_lock);
  712. dprintk(SYNX_ERR,
  713. "[sess :%llu] status update %d failed=%d\n",
  714. client->id, h_synx, rc);
  715. goto fail;
  716. }
  717. /*
  718. * offload callback dispatch and external fence
  719. * notification to separate worker thread, if any.
  720. */
  721. if (synx_obj->num_bound_synxs ||
  722. !list_empty(&synx_obj->reg_cbs_list))
  723. rc = synx_signal_offload_job(client, synx_obj,
  724. h_synx, status);
  725. rc = synx_native_signal_fence(synx_obj, status);
  726. if (rc != SYNX_SUCCESS)
  727. dprintk(SYNX_ERR,
  728. "[sess :%llu] signaling %u failed=%d\n",
  729. client->id, h_synx, rc);
  730. mutex_unlock(&synx_obj->obj_lock);
  731. fail:
  732. synx_util_release_handle(synx_data);
  733. synx_put_client(client);
  734. return rc;
  735. }
  736. EXPORT_SYMBOL(synx_signal);
  737. static int synx_match_payload(struct synx_kernel_payload *cb_payload,
  738. struct synx_kernel_payload *payload)
  739. {
  740. int rc = 0;
  741. if (IS_ERR_OR_NULL(cb_payload) || IS_ERR_OR_NULL(payload))
  742. return -SYNX_INVALID;
  743. if ((cb_payload->cb_func == payload->cb_func) &&
  744. (cb_payload->data == payload->data)) {
  745. if (payload->cancel_cb_func) {
  746. cb_payload->cb_func =
  747. payload->cancel_cb_func;
  748. rc = 1;
  749. } else {
  750. rc = 2;
  751. dprintk(SYNX_VERB,
  752. "kernel cb de-registration success\n");
  753. }
  754. }
  755. return rc;
  756. }
  757. /* Timer Callback function. This will be called when timer expires */
  758. void synx_timer_cb(struct timer_list *data)
  759. {
  760. struct synx_client *client;
  761. struct synx_handle_coredata *synx_data;
  762. struct synx_coredata *synx_obj;
  763. struct synx_cb_data *synx_cb = container_of(data, struct synx_cb_data, synx_timer);
  764. client = synx_get_client(synx_cb->session);
  765. if (IS_ERR_OR_NULL(client)) {
  766. dprintk(SYNX_ERR,
  767. "invalid session data 0x%x in cb payload\n",
  768. synx_cb->session);
  769. return;
  770. }
  771. synx_data = synx_util_acquire_handle(client, synx_cb->h_synx);
  772. synx_obj = synx_util_obtain_object(synx_data);
  773. if (IS_ERR_OR_NULL(synx_obj)) {
  774. dprintk(SYNX_ERR,
  775. "[sess :0x%llx] invalid handle access 0x%x\n",
  776. synx_cb->session, synx_cb->h_synx);
  777. return;
  778. }
  779. dprintk(SYNX_VERB,
  780. "Timer expired for synx_cb 0x%x timeout 0x%llx. Deleting the timer.\n",
  781. synx_cb, synx_cb->timeout);
  782. synx_cb->status = SYNX_STATE_TIMEOUT;
  783. del_timer(&synx_cb->synx_timer);
  784. list_del_init(&synx_cb->node);
  785. queue_work(synx_dev->wq_cb, &synx_cb->cb_dispatch);
  786. }
  787. static int synx_start_timer(struct synx_cb_data *synx_cb)
  788. {
  789. int rc = 0;
  790. timer_setup(&synx_cb->synx_timer, synx_timer_cb, 0);
  791. rc = mod_timer(&synx_cb->synx_timer, jiffies + msecs_to_jiffies(synx_cb->timeout));
  792. dprintk(SYNX_VERB,
  793. "Timer started for synx_cb 0x%x timeout 0x%llx\n",
  794. synx_cb, synx_cb->timeout);
  795. return rc;
  796. }
  797. int synx_async_wait(struct synx_session *session,
  798. struct synx_callback_params *params)
  799. {
  800. int rc = 0;
  801. u32 idx;
  802. u32 status;
  803. struct synx_client *client;
  804. struct synx_handle_coredata *synx_data;
  805. struct synx_coredata *synx_obj;
  806. struct synx_cb_data *synx_cb;
  807. struct synx_kernel_payload payload;
  808. if (IS_ERR_OR_NULL(session) || IS_ERR_OR_NULL(params))
  809. return -SYNX_INVALID;
  810. client = synx_get_client(session);
  811. if (IS_ERR_OR_NULL(client))
  812. return -SYNX_INVALID;
  813. synx_data = synx_util_acquire_handle(client, params->h_synx);
  814. synx_obj = synx_util_obtain_object(synx_data);
  815. if (IS_ERR_OR_NULL(synx_obj)) {
  816. dprintk(SYNX_ERR,
  817. "[sess :%llu] invalid handle access %u\n",
  818. client->id, params->h_synx);
  819. rc = -SYNX_INVALID;
  820. goto fail;
  821. }
  822. mutex_lock(&synx_obj->obj_lock);
  823. synx_cb = kzalloc(sizeof(*synx_cb), GFP_ATOMIC);
  824. if (IS_ERR_OR_NULL(synx_cb)) {
  825. rc = -SYNX_NOMEM;
  826. goto release;
  827. }
  828. payload.h_synx = params->h_synx;
  829. payload.cb_func = params->cb_func;
  830. payload.data = params->userdata;
  831. /* allocate a free index from client cb table */
  832. rc = synx_util_alloc_cb_entry(client, &payload, &idx);
  833. if (rc) {
  834. dprintk(SYNX_ERR,
  835. "[sess :%llu] error allocating cb entry\n",
  836. client->id);
  837. kfree(synx_cb);
  838. goto release;
  839. }
  840. if (synx_util_is_global_handle(params->h_synx) ||
  841. synx_util_is_global_object(synx_obj)) {
  842. status = synx_global_test_status_set_wait(
  843. synx_util_global_idx(params->h_synx),
  844. SYNX_CORE_APSS);
  845. if (status != SYNX_STATE_ACTIVE) {
  846. if (synx_util_is_merged_object(synx_obj))
  847. synx_native_signal_merged_fence(synx_obj, status);
  848. else
  849. synx_native_signal_fence(synx_obj, status);
  850. }
  851. }
  852. status = synx_util_get_object_status(synx_obj);
  853. synx_cb->session = session;
  854. synx_cb->idx = idx;
  855. synx_cb->h_synx = params->h_synx;
  856. INIT_WORK(&synx_cb->cb_dispatch, synx_util_cb_dispatch);
  857. /* add callback if object still ACTIVE, dispatch if SIGNALED */
  858. if (status == SYNX_STATE_ACTIVE) {
  859. dprintk(SYNX_VERB,
  860. "[sess :%llu] callback added for handle %u\n",
  861. client->id, params->h_synx);
  862. synx_cb->timeout = params->timeout_ms;
  863. if (params->timeout_ms != SYNX_NO_TIMEOUT) {
  864. rc = synx_start_timer(synx_cb);
  865. if (rc != SYNX_SUCCESS) {
  866. dprintk(SYNX_ERR,
  867. "[sess :%llu] timer start failed - synx_cb: 0x%x, params->timeout_ms: 0x%llx, handle: 0x%x, ret : %d\n",
  868. client->id, synx_cb, params->timeout_ms,
  869. params->h_synx, rc);
  870. goto release;
  871. }
  872. }
  873. list_add(&synx_cb->node, &synx_obj->reg_cbs_list);
  874. } else {
  875. synx_cb->status = status;
  876. dprintk(SYNX_VERB,
  877. "[sess :%llu] callback queued for handle %u\n",
  878. client->id, params->h_synx);
  879. queue_work(synx_dev->wq_cb,
  880. &synx_cb->cb_dispatch);
  881. }
  882. release:
  883. mutex_unlock(&synx_obj->obj_lock);
  884. fail:
  885. synx_util_release_handle(synx_data);
  886. synx_put_client(client);
  887. return rc;
  888. }
  889. EXPORT_SYMBOL(synx_async_wait);
  890. int synx_cancel_async_wait(
  891. struct synx_session *session,
  892. struct synx_callback_params *params)
  893. {
  894. int rc = 0, ret = 0;
  895. u32 status;
  896. bool match_found = false;
  897. struct synx_client *client;
  898. struct synx_handle_coredata *synx_data;
  899. struct synx_coredata *synx_obj;
  900. struct synx_kernel_payload payload;
  901. struct synx_cb_data *synx_cb, *synx_cb_temp;
  902. struct synx_client_cb *cb_payload;
  903. if (IS_ERR_OR_NULL(session) || IS_ERR_OR_NULL(params))
  904. return -SYNX_INVALID;
  905. client = synx_get_client(session);
  906. if (IS_ERR_OR_NULL(client))
  907. return -SYNX_INVALID;
  908. synx_data = synx_util_acquire_handle(client, params->h_synx);
  909. synx_obj = synx_util_obtain_object(synx_data);
  910. if (IS_ERR_OR_NULL(synx_obj)) {
  911. dprintk(SYNX_ERR,
  912. "[sess :%llu] invalid handle access %u\n",
  913. client->id, params->h_synx);
  914. rc = -SYNX_INVALID;
  915. goto fail;
  916. }
  917. mutex_lock(&synx_obj->obj_lock);
  918. if (synx_util_is_external_object(synx_obj)) {
  919. dprintk(SYNX_ERR,
  920. "cannot cancel wait on external fence\n");
  921. goto release;
  922. }
  923. payload.h_synx = params->h_synx;
  924. payload.cb_func = params->cb_func;
  925. payload.data = params->userdata;
  926. payload.cancel_cb_func = params->cancel_cb_func;
  927. status = synx_util_get_object_status(synx_obj);
  928. if (status != SYNX_STATE_ACTIVE) {
  929. dprintk(SYNX_ERR,
  930. "handle %u already signaled or timed out, cannot cancel\n",
  931. params->h_synx);
  932. rc = -SYNX_INVALID;
  933. goto release;
  934. }
  935. status = SYNX_CALLBACK_RESULT_CANCELED;
  936. /* remove all cb payloads mayching the deregister call */
  937. list_for_each_entry_safe(synx_cb, synx_cb_temp,
  938. &synx_obj->reg_cbs_list, node) {
  939. if (synx_cb->session != session) {
  940. continue;
  941. } else if (synx_cb->idx == 0 ||
  942. synx_cb->idx >= SYNX_MAX_OBJS) {
  943. /*
  944. * this should not happen. Even if it does,
  945. * the allocated memory will be cleaned up
  946. * when object is destroyed, preventing any
  947. * memory leaks.
  948. */
  949. dprintk(SYNX_ERR,
  950. "[sess :%llu] invalid callback data\n",
  951. client->id);
  952. continue;
  953. }
  954. cb_payload = &client->cb_table[synx_cb->idx];
  955. ret = synx_match_payload(&cb_payload->kernel_cb, &payload);
  956. if (synx_cb->timeout != SYNX_NO_TIMEOUT) {
  957. dprintk(SYNX_VERB,
  958. "Deleting timer synx_cb 0x%x, timeout 0x%llx\n",
  959. synx_cb, synx_cb->timeout);
  960. del_timer(&synx_cb->synx_timer);
  961. }
  962. switch (ret) {
  963. case 1:
  964. /* queue the cancel cb work */
  965. list_del_init(&synx_cb->node);
  966. synx_cb->status = status;
  967. queue_work(synx_dev->wq_cb,
  968. &synx_cb->cb_dispatch);
  969. match_found = true;
  970. break;
  971. case 2:
  972. /* no cancellation cb */
  973. if (synx_util_clear_cb_entry(client, cb_payload))
  974. dprintk(SYNX_ERR,
  975. "[sess :%llu] error clearing cb %u\n",
  976. client->id, params->h_synx);
  977. list_del_init(&synx_cb->node);
  978. kfree(synx_cb);
  979. match_found = true;
  980. break;
  981. default:
  982. break;
  983. }
  984. }
  985. if (!match_found)
  986. rc = -SYNX_INVALID;
  987. release:
  988. mutex_unlock(&synx_obj->obj_lock);
  989. fail:
  990. synx_util_release_handle(synx_data);
  991. synx_put_client(client);
  992. return rc;
  993. }
  994. EXPORT_SYMBOL(synx_cancel_async_wait);
  995. int synx_merge(struct synx_session *session,
  996. struct synx_merge_params *params)
  997. {
  998. int rc = SYNX_SUCCESS, i, num_signaled = 0;
  999. u32 count = 0, h_child, status = SYNX_STATE_ACTIVE;
  1000. u32 *h_child_list = NULL, *h_child_idx_list = NULL;
  1001. struct synx_client *client;
  1002. struct dma_fence **fences = NULL;
  1003. struct synx_map_entry *map_entry;
  1004. struct synx_coredata *synx_obj, *synx_obj_child;
  1005. struct synx_handle_coredata *synx_data_child;
  1006. if (IS_ERR_OR_NULL(session) || IS_ERR_OR_NULL(params))
  1007. return -SYNX_INVALID;
  1008. if (IS_ERR_OR_NULL(params->h_synxs) ||
  1009. IS_ERR_OR_NULL(params->h_merged_obj)) {
  1010. dprintk(SYNX_ERR, "invalid arguments\n");
  1011. return -SYNX_INVALID;
  1012. }
  1013. client = synx_get_client(session);
  1014. if (IS_ERR_OR_NULL(client))
  1015. return -SYNX_INVALID;
  1016. synx_obj = kzalloc(sizeof(*synx_obj), GFP_KERNEL);
  1017. if (IS_ERR_OR_NULL(synx_obj)) {
  1018. rc = -SYNX_NOMEM;
  1019. goto fail;
  1020. }
  1021. rc = synx_util_validate_merge(client, params->h_synxs,
  1022. params->num_objs, &fences, &count);
  1023. if (rc < 0) {
  1024. dprintk(SYNX_ERR,
  1025. "[sess :%llu] merge validation failed\n",
  1026. client->id);
  1027. rc = -SYNX_INVALID;
  1028. kfree(synx_obj);
  1029. goto fail;
  1030. }
  1031. rc = synx_util_init_group_coredata(synx_obj, fences,
  1032. params, count, client->dma_context);
  1033. if (rc) {
  1034. dprintk(SYNX_ERR,
  1035. "[sess :%llu] error initializing merge handle\n",
  1036. client->id);
  1037. goto clean_up;
  1038. }
  1039. map_entry = synx_util_insert_to_map(synx_obj,
  1040. *params->h_merged_obj, 0);
  1041. if (IS_ERR_OR_NULL(map_entry)) {
  1042. rc = PTR_ERR(map_entry);
  1043. /*
  1044. * dma fence put will take care of removing the references taken
  1045. * on child fences
  1046. */
  1047. dma_fence_put(synx_obj->fence);
  1048. kfree(synx_obj);
  1049. goto fail;
  1050. }
  1051. rc = synx_util_add_callback(synx_obj, *params->h_merged_obj);
  1052. if (rc != SYNX_SUCCESS)
  1053. goto clean_up;
  1054. rc = synx_util_init_handle(client, synx_obj,
  1055. params->h_merged_obj, map_entry);
  1056. if (rc) {
  1057. dprintk(SYNX_ERR,
  1058. "[sess :%llu] unable to init merge handle %u\n",
  1059. client->id, *params->h_merged_obj);
  1060. goto clean_up;
  1061. }
  1062. h_child_list = kzalloc(count*4, GFP_KERNEL);
  1063. if (IS_ERR_OR_NULL(h_child_list)) {
  1064. rc = -SYNX_NOMEM;
  1065. goto clear;
  1066. }
  1067. h_child_idx_list = kzalloc(count*4, GFP_KERNEL);
  1068. if (IS_ERR_OR_NULL(h_child_idx_list)) {
  1069. kfree(h_child_list);
  1070. rc = -SYNX_NOMEM;
  1071. goto clear;
  1072. }
  1073. for (i = 0; i < count; i++) {
  1074. h_child = synx_util_get_fence_entry((u64)fences[i], 1);
  1075. if (!synx_util_is_global_handle(h_child))
  1076. continue;
  1077. h_child_list[num_signaled] = h_child;
  1078. h_child_idx_list[num_signaled++] = synx_util_global_idx(h_child);
  1079. }
  1080. if (params->flags & SYNX_MERGE_GLOBAL_FENCE) {
  1081. rc = synx_global_merge(h_child_idx_list, num_signaled,
  1082. synx_util_global_idx(*params->h_merged_obj));
  1083. if (rc != SYNX_SUCCESS) {
  1084. dprintk(SYNX_ERR, "global merge failed\n");
  1085. kfree(h_child_list);
  1086. kfree(h_child_idx_list);
  1087. goto clear;
  1088. }
  1089. }
  1090. else {
  1091. for(i = 0; i < num_signaled; i++) {
  1092. status = synx_global_test_status_set_wait(synx_util_global_idx(h_child_list[i]), SYNX_CORE_APSS);
  1093. if (status != SYNX_STATE_ACTIVE) {
  1094. synx_data_child = synx_util_acquire_handle(client, h_child_list[i]);
  1095. synx_obj_child = synx_util_obtain_object(synx_data_child);
  1096. if (IS_ERR_OR_NULL(synx_obj_child)) {
  1097. dprintk(SYNX_ERR,
  1098. "[sess :%llu] invalid child handle %u\n",
  1099. client->id, h_child_list[i]);
  1100. continue;
  1101. }
  1102. mutex_lock(&synx_obj_child->obj_lock);
  1103. if (synx_obj->status == SYNX_STATE_ACTIVE)
  1104. rc = synx_native_signal_fence(synx_obj_child, status);
  1105. mutex_unlock(&synx_obj_child->obj_lock);
  1106. if (rc != SYNX_SUCCESS)
  1107. dprintk(SYNX_ERR, "h_synx %u failed with status %d\n", h_child_list[i], rc);
  1108. synx_util_release_handle(synx_data_child);
  1109. }
  1110. }
  1111. }
  1112. dprintk(SYNX_MEM,
  1113. "[sess :%llu] merge allocated %u, core %pK, fence %pK\n",
  1114. client->id, *params->h_merged_obj, synx_obj,
  1115. synx_obj->fence);
  1116. kfree(h_child_list);
  1117. kfree(h_child_idx_list);
  1118. synx_put_client(client);
  1119. return SYNX_SUCCESS;
  1120. clear:
  1121. synx_native_release_core(client, (*params->h_merged_obj));
  1122. synx_put_client(client);
  1123. return rc;
  1124. clean_up:
  1125. /*
  1126. * if map_entry is not created the cleanup of child fences have to be
  1127. * handled manually
  1128. */
  1129. if (IS_ERR_OR_NULL(map_entry)) {
  1130. kfree(synx_obj);
  1131. synx_util_merge_error(client, params->h_synxs, count);
  1132. if (params->num_objs && params->num_objs <= count)
  1133. kfree(fences);
  1134. } else {
  1135. synx_util_release_map_entry(map_entry);
  1136. }
  1137. fail:
  1138. synx_put_client(client);
  1139. return rc;
  1140. }
  1141. EXPORT_SYMBOL(synx_merge);
  1142. int synx_native_release_core(struct synx_client *client,
  1143. u32 h_synx)
  1144. {
  1145. int rc = -SYNX_INVALID;
  1146. struct synx_handle_coredata *curr, *synx_handle = NULL;
  1147. spin_lock_bh(&client->handle_map_lock);
  1148. hash_for_each_possible(client->handle_map,
  1149. curr, node, h_synx) {
  1150. if (curr->key == h_synx &&
  1151. curr->rel_count != 0) {
  1152. curr->rel_count--;
  1153. synx_handle = curr;
  1154. rc = SYNX_SUCCESS;
  1155. break;
  1156. }
  1157. }
  1158. spin_unlock_bh(&client->handle_map_lock);
  1159. /* release the reference obtained at synx creation */
  1160. synx_util_release_handle(synx_handle);
  1161. return rc;
  1162. }
  1163. int synx_release(struct synx_session *session, u32 h_synx)
  1164. {
  1165. int rc = 0;
  1166. struct synx_client *client;
  1167. client = synx_get_client(session);
  1168. if (IS_ERR_OR_NULL(client))
  1169. return -SYNX_INVALID;
  1170. rc = synx_native_release_core(client, h_synx);
  1171. synx_put_client(client);
  1172. return rc;
  1173. }
  1174. EXPORT_SYMBOL(synx_release);
  1175. int synx_wait(struct synx_session *session,
  1176. u32 h_synx, u64 timeout_ms)
  1177. {
  1178. int rc = 0;
  1179. unsigned long timeleft;
  1180. struct synx_client *client;
  1181. struct synx_handle_coredata *synx_data;
  1182. struct synx_coredata *synx_obj;
  1183. client = synx_get_client(session);
  1184. if (IS_ERR_OR_NULL(client))
  1185. return -SYNX_INVALID;
  1186. synx_data = synx_util_acquire_handle(client, h_synx);
  1187. synx_obj = synx_util_obtain_object(synx_data);
  1188. if (IS_ERR_OR_NULL(synx_obj) || IS_ERR_OR_NULL(synx_obj->fence)) {
  1189. dprintk(SYNX_ERR,
  1190. "[sess :%llu] invalid handle access %u\n",
  1191. client->id, h_synx);
  1192. rc = -SYNX_INVALID;
  1193. goto fail;
  1194. }
  1195. if (synx_util_is_global_handle(h_synx)) {
  1196. rc = synx_global_test_status_set_wait(
  1197. synx_util_global_idx(h_synx), SYNX_CORE_APSS);
  1198. if (rc != SYNX_STATE_ACTIVE) {
  1199. mutex_lock(&synx_obj->obj_lock);
  1200. if (synx_util_is_merged_object(synx_obj))
  1201. synx_native_signal_merged_fence(synx_obj, rc);
  1202. else
  1203. synx_native_signal_fence(synx_obj, rc);
  1204. mutex_unlock(&synx_obj->obj_lock);
  1205. }
  1206. }
  1207. timeleft = dma_fence_wait_timeout(synx_obj->fence, (bool) 0,
  1208. msecs_to_jiffies(timeout_ms));
  1209. if (timeleft <= 0) {
  1210. dprintk(SYNX_ERR,
  1211. "[sess :%llu] wait timeout for handle %u\n",
  1212. client->id, h_synx);
  1213. rc = -ETIMEDOUT;
  1214. goto fail;
  1215. }
  1216. mutex_lock(&synx_obj->obj_lock);
  1217. rc = synx_util_get_object_status(synx_obj);
  1218. mutex_unlock(&synx_obj->obj_lock);
  1219. fail:
  1220. synx_util_release_handle(synx_data);
  1221. synx_put_client(client);
  1222. return rc;
  1223. }
  1224. EXPORT_SYMBOL(synx_wait);
  1225. int synx_bind(struct synx_session *session,
  1226. u32 h_synx,
  1227. struct synx_external_desc_v2 external_sync)
  1228. {
  1229. int rc = 0;
  1230. u32 i;
  1231. u32 bound_idx;
  1232. struct synx_client *client;
  1233. struct synx_handle_coredata *synx_data = NULL;
  1234. struct synx_coredata *synx_obj;
  1235. struct synx_signal_cb *data = NULL;
  1236. struct bind_operations *bind_ops = NULL;
  1237. client = synx_get_client(session);
  1238. if (IS_ERR_OR_NULL(client))
  1239. return -SYNX_INVALID;
  1240. synx_data = synx_util_acquire_handle(client, h_synx);
  1241. synx_obj = synx_util_obtain_object(synx_data);
  1242. if (IS_ERR_OR_NULL(synx_obj)) {
  1243. if (rc || synx_data)
  1244. dprintk(SYNX_ERR,
  1245. "[sess :%llu] invalid handle access %u\n",
  1246. client->id, h_synx);
  1247. goto fail;
  1248. }
  1249. bind_ops = synx_util_get_bind_ops(external_sync.type);
  1250. if (IS_ERR_OR_NULL(bind_ops)) {
  1251. dprintk(SYNX_ERR,
  1252. "[sess :%llu] invalid bind ops for %u\n",
  1253. client->id, external_sync.type);
  1254. rc = -SYNX_INVALID;
  1255. goto fail;
  1256. }
  1257. mutex_lock(&synx_obj->obj_lock);
  1258. if (synx_util_is_merged_object(synx_obj)) {
  1259. dprintk(SYNX_ERR,
  1260. "[sess :%llu] cannot bind to composite handle %u\n",
  1261. client->id, h_synx);
  1262. rc = -SYNX_INVALID;
  1263. goto release;
  1264. }
  1265. if (synx_obj->num_bound_synxs >= SYNX_MAX_NUM_BINDINGS) {
  1266. dprintk(SYNX_ERR,
  1267. "[sess :%llu] max bindings reached for handle %u\n",
  1268. client->id, h_synx);
  1269. rc = -SYNX_NOMEM;
  1270. goto release;
  1271. }
  1272. /* don't bind external sync obj if already done */
  1273. for (i = 0; i < synx_obj->num_bound_synxs; i++) {
  1274. if ((external_sync.id ==
  1275. synx_obj->bound_synxs[i].external_desc.id) &&
  1276. (external_sync.type ==
  1277. synx_obj->bound_synxs[i].external_desc.type)){
  1278. dprintk(SYNX_ERR,
  1279. "[sess :%llu] duplicate bind for sync %llu\n",
  1280. client->id, external_sync.id);
  1281. rc = -SYNX_ALREADY;
  1282. goto release;
  1283. }
  1284. }
  1285. data = kzalloc(sizeof(*data), GFP_KERNEL);
  1286. if (IS_ERR_OR_NULL(data)) {
  1287. rc = -SYNX_NOMEM;
  1288. goto release;
  1289. }
  1290. /* get additional reference since passing pointer to cb */
  1291. synx_util_get_object(synx_obj);
  1292. /* data passed to external callback */
  1293. data->handle = h_synx;
  1294. data->synx_obj = synx_obj;
  1295. bound_idx = synx_obj->num_bound_synxs;
  1296. memcpy(&synx_obj->bound_synxs[bound_idx],
  1297. &external_sync, sizeof(struct synx_external_desc_v2));
  1298. synx_obj->bound_synxs[bound_idx].external_data = data;
  1299. synx_obj->num_bound_synxs++;
  1300. mutex_unlock(&synx_obj->obj_lock);
  1301. rc = bind_ops->register_callback(synx_external_callback,
  1302. data, external_sync.id);
  1303. if (rc) {
  1304. dprintk(SYNX_ERR,
  1305. "[sess :%llu] callback reg failed for %llu\n",
  1306. client->id, external_sync.id);
  1307. mutex_lock(&synx_obj->obj_lock);
  1308. memset(&synx_obj->bound_synxs[bound_idx], 0,
  1309. sizeof(struct synx_external_desc_v2));
  1310. synx_obj->num_bound_synxs--;
  1311. mutex_unlock(&synx_obj->obj_lock);
  1312. synx_util_put_object(synx_obj);
  1313. kfree(data);
  1314. goto fail;
  1315. }
  1316. synx_util_release_handle(synx_data);
  1317. dprintk(SYNX_DBG,
  1318. "[sess :%llu] ext sync %llu bound to handle %u\n",
  1319. client->id, external_sync.id, h_synx);
  1320. synx_put_client(client);
  1321. return SYNX_SUCCESS;
  1322. release:
  1323. mutex_unlock(&synx_obj->obj_lock);
  1324. fail:
  1325. synx_util_release_handle(synx_data);
  1326. synx_put_client(client);
  1327. return rc;
  1328. }
  1329. EXPORT_SYMBOL(synx_bind);
  1330. int synx_get_status(struct synx_session *session,
  1331. u32 h_synx)
  1332. {
  1333. int rc = 0, status = 0;
  1334. struct synx_client *client;
  1335. struct synx_handle_coredata *synx_data;
  1336. struct synx_coredata *synx_obj;
  1337. client = synx_get_client(session);
  1338. if (IS_ERR_OR_NULL(client))
  1339. return -SYNX_INVALID;
  1340. synx_data = synx_util_acquire_handle(client, h_synx);
  1341. synx_obj = synx_util_obtain_object(synx_data);
  1342. if (IS_ERR_OR_NULL(synx_obj) ||
  1343. IS_ERR_OR_NULL(synx_obj->fence)) {
  1344. dprintk(SYNX_ERR,
  1345. "[sess :%llu] invalid handle access %u\n",
  1346. client->id, h_synx);
  1347. rc = -SYNX_INVALID;
  1348. goto fail;
  1349. }
  1350. mutex_lock(&synx_obj->obj_lock);
  1351. status = synx_util_get_object_status(synx_obj);
  1352. rc = synx_obj->status;
  1353. mutex_unlock(&synx_obj->obj_lock);
  1354. dprintk(SYNX_VERB,
  1355. "[sess :%llu] handle %u synx coredata status %d and dma fence status %d\n",
  1356. client->id, h_synx, rc, status);
  1357. fail:
  1358. synx_util_release_handle(synx_data);
  1359. synx_put_client(client);
  1360. return rc;
  1361. }
  1362. EXPORT_SYMBOL(synx_get_status);
  1363. static struct synx_map_entry *synx_handle_conversion(
  1364. struct synx_client *client,
  1365. u32 *h_synx, struct synx_map_entry *old_entry)
  1366. {
  1367. int rc;
  1368. struct synx_map_entry *map_entry = NULL;
  1369. struct synx_coredata *synx_obj;
  1370. if (IS_ERR_OR_NULL(old_entry)) {
  1371. old_entry = synx_util_get_map_entry(*h_synx);
  1372. if (IS_ERR_OR_NULL(old_entry)) {
  1373. rc = PTR_ERR(old_entry);
  1374. dprintk(SYNX_ERR,
  1375. "invalid import handle %u err=%d",
  1376. *h_synx, rc);
  1377. return old_entry;
  1378. }
  1379. }
  1380. synx_obj = old_entry->synx_obj;
  1381. BUG_ON(synx_obj == NULL);
  1382. mutex_lock(&synx_obj->obj_lock);
  1383. synx_util_get_object(synx_obj);
  1384. if (synx_obj->global_idx != 0) {
  1385. *h_synx = synx_encode_handle(
  1386. synx_obj->global_idx, SYNX_CORE_APSS, true);
  1387. map_entry = synx_util_get_map_entry(*h_synx);
  1388. if (IS_ERR_OR_NULL(map_entry)) {
  1389. /* raced with release from last global client */
  1390. map_entry = synx_util_insert_to_map(synx_obj,
  1391. *h_synx, 0);
  1392. if (IS_ERR_OR_NULL(map_entry)) {
  1393. rc = PTR_ERR(map_entry);
  1394. dprintk(SYNX_ERR,
  1395. "addition of %u to map failed=%d",
  1396. *h_synx, rc);
  1397. }
  1398. }
  1399. } else {
  1400. synx_obj->map_count++;
  1401. rc = synx_alloc_global_handle(h_synx);
  1402. if (rc == SYNX_SUCCESS) {
  1403. synx_obj->global_idx =
  1404. synx_util_global_idx(*h_synx);
  1405. synx_obj->type |= SYNX_CREATE_GLOBAL_FENCE;
  1406. map_entry = synx_util_insert_to_map(synx_obj,
  1407. *h_synx, 0);
  1408. if (IS_ERR_OR_NULL(map_entry)) {
  1409. rc = PTR_ERR(map_entry);
  1410. synx_global_put_ref(
  1411. synx_util_global_idx(*h_synx));
  1412. dprintk(SYNX_ERR,
  1413. "insertion of %u to map failed=%d",
  1414. *h_synx, rc);
  1415. }
  1416. }
  1417. }
  1418. mutex_unlock(&synx_obj->obj_lock);
  1419. if (IS_ERR_OR_NULL(map_entry))
  1420. synx_util_put_object(synx_obj);
  1421. synx_util_release_map_entry(old_entry);
  1422. return map_entry;
  1423. }
  1424. static int synx_native_import_handle(struct synx_client *client,
  1425. struct synx_import_indv_params *params)
  1426. {
  1427. int rc = SYNX_SUCCESS;
  1428. u32 h_synx, core_id;
  1429. struct synx_map_entry *map_entry, *old_entry;
  1430. struct synx_coredata *synx_obj;
  1431. struct synx_handle_coredata *synx_data = NULL, *curr;
  1432. char name[SYNX_OBJ_NAME_LEN] = {0};
  1433. struct synx_create_params c_params = {0};
  1434. if (IS_ERR_OR_NULL(client) || IS_ERR_OR_NULL(params) ||
  1435. IS_ERR_OR_NULL(params->fence) ||
  1436. IS_ERR_OR_NULL(params->new_h_synx))
  1437. return -SYNX_INVALID;
  1438. h_synx = *((u32 *)params->fence);
  1439. /* check if already mapped to client */
  1440. spin_lock_bh(&client->handle_map_lock);
  1441. hash_for_each_possible(client->handle_map,
  1442. curr, node, h_synx) {
  1443. if (curr->key == h_synx &&
  1444. curr->rel_count != 0 &&
  1445. (synx_util_is_global_handle(h_synx) ||
  1446. params->flags & SYNX_IMPORT_LOCAL_FENCE)) {
  1447. curr->rel_count++;
  1448. kref_get(&curr->refcount);
  1449. synx_data = curr;
  1450. break;
  1451. }
  1452. }
  1453. spin_unlock_bh(&client->handle_map_lock);
  1454. if (synx_data) {
  1455. *params->new_h_synx = h_synx;
  1456. return SYNX_SUCCESS;
  1457. }
  1458. map_entry = synx_util_get_map_entry(h_synx);
  1459. if (IS_ERR_OR_NULL(map_entry)) {
  1460. core_id = (h_synx & SYNX_OBJ_CORE_ID_MASK)
  1461. >> SYNX_HANDLE_INDEX_BITS;
  1462. if (core_id == SYNX_CORE_APSS) {
  1463. dprintk(SYNX_ERR,
  1464. "[sess :%llu] invalid import handle %u\n",
  1465. client->id, h_synx);
  1466. return -SYNX_INVALID;
  1467. } else if (synx_util_is_global_handle(h_synx)) {
  1468. /* import global handle created in another core */
  1469. synx_util_map_import_params_to_create(params, &c_params);
  1470. scnprintf(name, SYNX_OBJ_NAME_LEN, "import-client-%d",
  1471. current->pid);
  1472. c_params.name = name;
  1473. c_params.h_synx = &h_synx;
  1474. rc = synx_native_create_core(client, &c_params);
  1475. if (rc != SYNX_SUCCESS)
  1476. return rc;
  1477. *params->new_h_synx = h_synx;
  1478. return SYNX_SUCCESS;
  1479. }
  1480. dprintk(SYNX_ERR,
  1481. "[sess :%llu] invalid handle %u\n",
  1482. client->id, h_synx);
  1483. return -SYNX_INVALID;
  1484. }
  1485. synx_obj = map_entry->synx_obj;
  1486. BUG_ON(synx_obj == NULL);
  1487. if ((params->flags & SYNX_IMPORT_GLOBAL_FENCE) &&
  1488. !synx_util_is_global_handle(h_synx)) {
  1489. old_entry = map_entry;
  1490. map_entry = synx_handle_conversion(client, &h_synx,
  1491. old_entry);
  1492. }
  1493. if (rc != SYNX_SUCCESS)
  1494. return rc;
  1495. *params->new_h_synx = h_synx;
  1496. rc = synx_util_init_handle(client, map_entry->synx_obj,
  1497. params->new_h_synx, map_entry);
  1498. if (rc != SYNX_SUCCESS) {
  1499. dprintk(SYNX_ERR,
  1500. "[sess :%llu] init of imported handle %u failed=%d\n",
  1501. client->id, h_synx, rc);
  1502. synx_util_release_map_entry(map_entry);
  1503. }
  1504. return rc;
  1505. }
  1506. static int synx_native_import_fence(struct synx_client *client,
  1507. struct synx_import_indv_params *params)
  1508. {
  1509. int rc = SYNX_SUCCESS;
  1510. u32 curr_h_synx;
  1511. u32 global;
  1512. struct synx_create_params c_params = {0};
  1513. char name[SYNX_OBJ_NAME_LEN] = {0};
  1514. struct synx_fence_entry *entry;
  1515. struct synx_map_entry *map_entry = NULL;
  1516. struct synx_handle_coredata *synx_data = NULL, *curr;
  1517. if (IS_ERR_OR_NULL(client) || IS_ERR_OR_NULL(params) ||
  1518. IS_ERR_OR_NULL(params->fence) ||
  1519. IS_ERR_OR_NULL(params->new_h_synx))
  1520. return -SYNX_INVALID;
  1521. global = SYNX_IMPORT_GLOBAL_FENCE & params->flags;
  1522. retry:
  1523. *params->new_h_synx =
  1524. synx_util_get_fence_entry((u64)params->fence, global);
  1525. if (*params->new_h_synx == 0) {
  1526. /* create a new synx obj and add to fence map */
  1527. synx_util_map_import_params_to_create(params, &c_params);
  1528. scnprintf(name, SYNX_OBJ_NAME_LEN, "import-client-%d",
  1529. current->pid);
  1530. c_params.name = name;
  1531. c_params.h_synx = params->new_h_synx;
  1532. c_params.fence = params->fence;
  1533. rc = synx_native_create_core(client, &c_params);
  1534. if (rc != SYNX_SUCCESS)
  1535. return rc;
  1536. curr_h_synx = *params->new_h_synx;
  1537. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  1538. if (IS_ERR_OR_NULL(entry)) {
  1539. rc = -SYNX_NOMEM;
  1540. curr_h_synx = *c_params.h_synx;
  1541. goto fail;
  1542. }
  1543. do {
  1544. entry->key = (u64)params->fence;
  1545. if (global)
  1546. entry->g_handle = *params->new_h_synx;
  1547. else
  1548. entry->l_handle = *params->new_h_synx;
  1549. rc = synx_util_insert_fence_entry(entry,
  1550. params->new_h_synx, global);
  1551. if (rc == SYNX_SUCCESS) {
  1552. dprintk(SYNX_DBG,
  1553. "mapped fence %pK to new handle %u\n",
  1554. params->fence, *params->new_h_synx);
  1555. break;
  1556. } else if (rc == -SYNX_ALREADY) {
  1557. /*
  1558. * release the new handle allocated
  1559. * and use the available handle
  1560. * already mapped instead.
  1561. */
  1562. map_entry = synx_util_get_map_entry(
  1563. *params->new_h_synx);
  1564. if (IS_ERR_OR_NULL(map_entry)) {
  1565. /* race with fence release, need to retry */
  1566. dprintk(SYNX_DBG,
  1567. "re-attempting handle import\n");
  1568. *params->new_h_synx = curr_h_synx;
  1569. continue;
  1570. }
  1571. rc = synx_util_init_handle(client,
  1572. map_entry->synx_obj,
  1573. params->new_h_synx, map_entry);
  1574. dprintk(SYNX_DBG, "mapped fence %pK to handle %u\n",
  1575. params->fence, *params->new_h_synx);
  1576. goto release;
  1577. } else {
  1578. dprintk(SYNX_ERR,
  1579. "importing fence %pK failed, err=%d\n",
  1580. params->fence, rc);
  1581. goto release;
  1582. }
  1583. } while (true);
  1584. } else {
  1585. /* check if already mapped to client */
  1586. spin_lock_bh(&client->handle_map_lock);
  1587. hash_for_each_possible(client->handle_map,
  1588. curr, node, *params->new_h_synx) {
  1589. if (curr->key == *params->new_h_synx &&
  1590. curr->rel_count != 0) {
  1591. curr->rel_count++;
  1592. kref_get(&curr->refcount);
  1593. synx_data = curr;
  1594. break;
  1595. }
  1596. }
  1597. spin_unlock_bh(&client->handle_map_lock);
  1598. if (synx_data) {
  1599. dprintk(SYNX_DBG, "mapped fence %pK to handle %u\n",
  1600. params->fence, *params->new_h_synx);
  1601. return SYNX_SUCCESS;
  1602. }
  1603. if (global && !synx_util_is_global_handle(
  1604. *params->new_h_synx))
  1605. map_entry = synx_handle_conversion(client,
  1606. params->new_h_synx, NULL);
  1607. else
  1608. map_entry = synx_util_get_map_entry(
  1609. *params->new_h_synx);
  1610. if (IS_ERR_OR_NULL(map_entry)) {
  1611. /* race with fence release, need to retry */
  1612. dprintk(SYNX_DBG, "re-attempting handle import\n");
  1613. goto retry;
  1614. }
  1615. rc = synx_util_init_handle(client, map_entry->synx_obj,
  1616. params->new_h_synx, map_entry);
  1617. dprintk(SYNX_DBG, "mapped fence %pK to existing handle %u\n",
  1618. params->fence, *params->new_h_synx);
  1619. }
  1620. return rc;
  1621. release:
  1622. kfree(entry);
  1623. fail:
  1624. synx_native_release_core(client, curr_h_synx);
  1625. return rc;
  1626. }
  1627. static int synx_native_import_indv(struct synx_client *client,
  1628. struct synx_import_indv_params *params)
  1629. {
  1630. int rc = -SYNX_INVALID;
  1631. if (IS_ERR_OR_NULL(params) ||
  1632. IS_ERR_OR_NULL(params->new_h_synx) ||
  1633. IS_ERR_OR_NULL(params->fence)) {
  1634. dprintk(SYNX_ERR, "invalid import arguments\n");
  1635. return -SYNX_INVALID;
  1636. }
  1637. if (likely(params->flags & SYNX_IMPORT_DMA_FENCE))
  1638. rc = synx_native_import_fence(client, params);
  1639. else if (params->flags & SYNX_IMPORT_SYNX_FENCE)
  1640. rc = synx_native_import_handle(client, params);
  1641. dprintk(SYNX_DBG,
  1642. "[sess :%llu] import of fence %pK %s, handle %u\n",
  1643. client->id, params->fence,
  1644. rc ? "failed" : "successful",
  1645. rc ? 0 : *params->new_h_synx);
  1646. return rc;
  1647. }
  1648. static int synx_native_import_arr(struct synx_client *client,
  1649. struct synx_import_arr_params *params)
  1650. {
  1651. u32 i;
  1652. int rc = SYNX_SUCCESS;
  1653. if (IS_ERR_OR_NULL(params) || params->num_fences == 0) {
  1654. dprintk(SYNX_ERR, "invalid import arr arguments\n");
  1655. return -SYNX_INVALID;
  1656. }
  1657. for (i = 0; i < params->num_fences; i++) {
  1658. rc = synx_native_import_indv(client, &params->list[i]);
  1659. if (rc != SYNX_SUCCESS) {
  1660. dprintk(SYNX_ERR,
  1661. "importing fence[%u] %pK failed=%d\n",
  1662. i, params->list[i].fence, rc);
  1663. break;
  1664. }
  1665. }
  1666. if (rc != SYNX_SUCCESS)
  1667. while (i--) {
  1668. /* release the imported handles and cleanup */
  1669. if (synx_native_release_core(client,
  1670. *params->list[i].new_h_synx) != SYNX_SUCCESS)
  1671. dprintk(SYNX_ERR,
  1672. "error cleaning up imported handle[%u] %u\n",
  1673. i, *params->list[i].new_h_synx);
  1674. }
  1675. return rc;
  1676. }
  1677. int synx_import(struct synx_session *session,
  1678. struct synx_import_params *params)
  1679. {
  1680. int rc = 0;
  1681. struct synx_client *client;
  1682. if (IS_ERR_OR_NULL(params)) {
  1683. dprintk(SYNX_ERR, "invalid import arguments\n");
  1684. return -SYNX_INVALID;
  1685. }
  1686. client = synx_get_client(session);
  1687. if (IS_ERR_OR_NULL(client))
  1688. return -SYNX_INVALID;
  1689. /* import fence based on its type */
  1690. if (params->type == SYNX_IMPORT_ARR_PARAMS)
  1691. rc = synx_native_import_arr(client, &params->arr);
  1692. else
  1693. rc = synx_native_import_indv(client, &params->indv);
  1694. synx_put_client(client);
  1695. return rc;
  1696. }
  1697. EXPORT_SYMBOL(synx_import);
  1698. static int synx_handle_create(struct synx_private_ioctl_arg *k_ioctl,
  1699. struct synx_session *session)
  1700. {
  1701. int result;
  1702. int csl_fence;
  1703. struct synx_create_v2 create_info;
  1704. struct synx_create_params params = {0};
  1705. if (k_ioctl->size != sizeof(create_info))
  1706. return -SYNX_INVALID;
  1707. if (copy_from_user(&create_info,
  1708. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1709. k_ioctl->size))
  1710. return -EFAULT;
  1711. params.h_synx = &create_info.synx_obj;
  1712. params.name = create_info.name;
  1713. params.flags = create_info.flags;
  1714. if (create_info.flags & SYNX_CREATE_CSL_FENCE) {
  1715. csl_fence = create_info.desc.id[0];
  1716. params.fence = &csl_fence;
  1717. }
  1718. result = synx_create(session, &params);
  1719. if (!result)
  1720. if (copy_to_user(u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1721. &create_info,
  1722. k_ioctl->size))
  1723. return -EFAULT;
  1724. return result;
  1725. }
  1726. static int synx_handle_getstatus(struct synx_private_ioctl_arg *k_ioctl,
  1727. struct synx_session *session)
  1728. {
  1729. struct synx_signal_v2 signal_info;
  1730. if (k_ioctl->size != sizeof(signal_info))
  1731. return -SYNX_INVALID;
  1732. if (copy_from_user(&signal_info,
  1733. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1734. k_ioctl->size))
  1735. return -EFAULT;
  1736. signal_info.synx_state =
  1737. synx_get_status(session, signal_info.synx_obj);
  1738. if (copy_to_user(u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1739. &signal_info,
  1740. k_ioctl->size))
  1741. return -EFAULT;
  1742. return SYNX_SUCCESS;
  1743. }
  1744. static int synx_handle_import(struct synx_private_ioctl_arg *k_ioctl,
  1745. struct synx_session *session)
  1746. {
  1747. struct synx_import_info import_info;
  1748. struct synx_import_params params = {0};
  1749. int result = SYNX_SUCCESS;
  1750. if (k_ioctl->size != sizeof(import_info))
  1751. return -SYNX_INVALID;
  1752. if (copy_from_user(&import_info,
  1753. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1754. k_ioctl->size))
  1755. return -EFAULT;
  1756. if (import_info.flags & SYNX_IMPORT_DMA_FENCE)
  1757. params.indv.fence =
  1758. sync_file_get_fence(import_info.desc.id[0]);
  1759. else if (import_info.flags & SYNX_IMPORT_SYNX_FENCE)
  1760. params.indv.fence = &import_info.synx_obj;
  1761. params.type = SYNX_IMPORT_INDV_PARAMS;
  1762. params.indv.flags = import_info.flags;
  1763. params.indv.new_h_synx = &import_info.new_synx_obj;
  1764. if (synx_import(session, &params))
  1765. result = -SYNX_INVALID;
  1766. // Fence needs to be put irresepctive of import status
  1767. if (import_info.flags & SYNX_IMPORT_DMA_FENCE)
  1768. dma_fence_put(params.indv.fence);
  1769. if (result != SYNX_SUCCESS)
  1770. return result;
  1771. if (copy_to_user(u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1772. &import_info,
  1773. k_ioctl->size))
  1774. return -EFAULT;
  1775. return result;
  1776. }
  1777. static int synx_handle_import_arr(
  1778. struct synx_private_ioctl_arg *k_ioctl,
  1779. struct synx_session *session)
  1780. {
  1781. int rc = -SYNX_INVALID;
  1782. u32 idx = 0;
  1783. struct synx_client *client;
  1784. struct synx_import_arr_info arr_info;
  1785. struct synx_import_info *arr;
  1786. struct synx_import_indv_params params = {0};
  1787. if (k_ioctl->size != sizeof(arr_info))
  1788. return -SYNX_INVALID;
  1789. if (copy_from_user(&arr_info,
  1790. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1791. k_ioctl->size))
  1792. return -EFAULT;
  1793. arr = kcalloc(arr_info.num_objs,
  1794. sizeof(*arr), GFP_KERNEL);
  1795. if (IS_ERR_OR_NULL(arr))
  1796. return -ENOMEM;
  1797. client = synx_get_client(session);
  1798. if (IS_ERR_OR_NULL(client)) {
  1799. rc = PTR_ERR(client);
  1800. goto clean;
  1801. }
  1802. if (copy_from_user(arr,
  1803. u64_to_user_ptr(arr_info.list),
  1804. sizeof(*arr) * arr_info.num_objs)) {
  1805. rc = -EFAULT;
  1806. goto fail;
  1807. }
  1808. while (idx < arr_info.num_objs) {
  1809. params.new_h_synx = &arr[idx].new_synx_obj;
  1810. params.flags = arr[idx].flags;
  1811. if (arr[idx].flags & SYNX_IMPORT_DMA_FENCE)
  1812. params.fence =
  1813. sync_file_get_fence(arr[idx].desc.id[0]);
  1814. else if (arr[idx].flags & SYNX_IMPORT_SYNX_FENCE)
  1815. params.fence = &arr[idx].synx_obj;
  1816. rc = synx_native_import_indv(client, &params);
  1817. // Fence needs to be put irresepctive of import status
  1818. if (arr[idx].flags & SYNX_IMPORT_DMA_FENCE)
  1819. dma_fence_put(params.fence);
  1820. if (rc != SYNX_SUCCESS)
  1821. break;
  1822. idx++;
  1823. }
  1824. /* release allocated handles in case of failure */
  1825. if (rc != SYNX_SUCCESS) {
  1826. while (idx > 0)
  1827. synx_native_release_core(client,
  1828. arr[--idx].new_synx_obj);
  1829. } else {
  1830. if (copy_to_user(u64_to_user_ptr(arr_info.list),
  1831. arr,
  1832. sizeof(*arr) * arr_info.num_objs)) {
  1833. rc = -EFAULT;
  1834. goto fail;
  1835. }
  1836. }
  1837. fail:
  1838. synx_put_client(client);
  1839. clean:
  1840. kfree(arr);
  1841. return rc;
  1842. }
  1843. static int synx_handle_export(struct synx_private_ioctl_arg *k_ioctl,
  1844. struct synx_session *session)
  1845. {
  1846. return -SYNX_INVALID;
  1847. }
  1848. static int synx_handle_signal(struct synx_private_ioctl_arg *k_ioctl,
  1849. struct synx_session *session)
  1850. {
  1851. struct synx_signal_v2 signal_info;
  1852. if (k_ioctl->size != sizeof(signal_info))
  1853. return -SYNX_INVALID;
  1854. if (copy_from_user(&signal_info,
  1855. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1856. k_ioctl->size))
  1857. return -EFAULT;
  1858. return synx_signal(session, signal_info.synx_obj,
  1859. signal_info.synx_state);
  1860. }
  1861. static int synx_handle_merge(struct synx_private_ioctl_arg *k_ioctl,
  1862. struct synx_session *session)
  1863. {
  1864. u32 *h_synxs;
  1865. int result;
  1866. struct synx_merge_v2 merge_info;
  1867. struct synx_merge_params params = {0};
  1868. if (k_ioctl->size != sizeof(merge_info))
  1869. return -SYNX_INVALID;
  1870. if (copy_from_user(&merge_info,
  1871. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1872. k_ioctl->size))
  1873. return -EFAULT;
  1874. if (merge_info.num_objs >= SYNX_MAX_OBJS)
  1875. return -SYNX_INVALID;
  1876. h_synxs = kcalloc(merge_info.num_objs,
  1877. sizeof(*h_synxs), GFP_KERNEL);
  1878. if (IS_ERR_OR_NULL(h_synxs))
  1879. return -ENOMEM;
  1880. if (copy_from_user(h_synxs,
  1881. u64_to_user_ptr(merge_info.synx_objs),
  1882. sizeof(u32) * merge_info.num_objs)) {
  1883. kfree(h_synxs);
  1884. return -EFAULT;
  1885. }
  1886. params.num_objs = merge_info.num_objs;
  1887. params.h_synxs = h_synxs;
  1888. params.flags = merge_info.flags;
  1889. params.h_merged_obj = &merge_info.merged;
  1890. result = synx_merge(session, &params);
  1891. if (!result)
  1892. if (copy_to_user(u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1893. &merge_info,
  1894. k_ioctl->size)) {
  1895. kfree(h_synxs);
  1896. return -EFAULT;
  1897. }
  1898. kfree(h_synxs);
  1899. return result;
  1900. }
  1901. static int synx_handle_wait(struct synx_private_ioctl_arg *k_ioctl,
  1902. struct synx_session *session)
  1903. {
  1904. struct synx_wait_v2 wait_info;
  1905. if (k_ioctl->size != sizeof(wait_info))
  1906. return -SYNX_INVALID;
  1907. if (copy_from_user(&wait_info,
  1908. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1909. k_ioctl->size))
  1910. return -EFAULT;
  1911. k_ioctl->result = synx_wait(session,
  1912. wait_info.synx_obj, wait_info.timeout_ms);
  1913. return SYNX_SUCCESS;
  1914. }
  1915. static int synx_handle_async_wait(
  1916. struct synx_private_ioctl_arg *k_ioctl,
  1917. struct synx_session *session)
  1918. {
  1919. int rc = 0;
  1920. struct synx_userpayload_info_v2 user_data;
  1921. struct synx_callback_params params = {0};
  1922. if (k_ioctl->size != sizeof(user_data))
  1923. return -SYNX_INVALID;
  1924. if (copy_from_user(&user_data,
  1925. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1926. k_ioctl->size))
  1927. return -EFAULT;
  1928. params.h_synx = user_data.synx_obj;
  1929. params.cb_func = synx_util_default_user_callback;
  1930. params.userdata = (void *)user_data.payload[0];
  1931. params.timeout_ms = user_data.payload[2];
  1932. rc = synx_async_wait(session, &params);
  1933. if (rc)
  1934. dprintk(SYNX_ERR,
  1935. "user cb registration failed for handle %d\n",
  1936. user_data.synx_obj);
  1937. return rc;
  1938. }
  1939. static int synx_handle_cancel_async_wait(
  1940. struct synx_private_ioctl_arg *k_ioctl,
  1941. struct synx_session *session)
  1942. {
  1943. int rc = 0;
  1944. struct synx_userpayload_info_v2 user_data;
  1945. struct synx_callback_params params = {0};
  1946. if (k_ioctl->size != sizeof(user_data))
  1947. return -SYNX_INVALID;
  1948. if (copy_from_user(&user_data,
  1949. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1950. k_ioctl->size))
  1951. return -EFAULT;
  1952. params.h_synx = user_data.synx_obj;
  1953. params.cb_func = synx_util_default_user_callback;
  1954. params.userdata = (void *)user_data.payload[0];
  1955. rc = synx_cancel_async_wait(session, &params);
  1956. if (rc)
  1957. dprintk(SYNX_ERR,
  1958. "user cb deregistration failed for handle %d\n",
  1959. user_data.synx_obj);
  1960. return rc;
  1961. }
  1962. static int synx_handle_bind(struct synx_private_ioctl_arg *k_ioctl,
  1963. struct synx_session *session)
  1964. {
  1965. struct synx_bind_v2 synx_bind_info;
  1966. if (k_ioctl->size != sizeof(synx_bind_info))
  1967. return -SYNX_INVALID;
  1968. if (copy_from_user(&synx_bind_info,
  1969. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1970. k_ioctl->size))
  1971. return -EFAULT;
  1972. k_ioctl->result = synx_bind(session,
  1973. synx_bind_info.synx_obj,
  1974. synx_bind_info.ext_sync_desc);
  1975. return k_ioctl->result;
  1976. }
  1977. static int synx_handle_release(struct synx_private_ioctl_arg *k_ioctl,
  1978. struct synx_session *session)
  1979. {
  1980. struct synx_info release_info;
  1981. if (k_ioctl->size != sizeof(release_info))
  1982. return -SYNX_INVALID;
  1983. if (copy_from_user(&release_info,
  1984. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1985. k_ioctl->size))
  1986. return -EFAULT;
  1987. return synx_release(session, release_info.synx_obj);
  1988. }
  1989. static int synx_handle_get_fence(struct synx_private_ioctl_arg *k_ioctl,
  1990. struct synx_session *session)
  1991. {
  1992. struct synx_fence_fd fence_fd;
  1993. struct dma_fence *fence;
  1994. if (k_ioctl->size != sizeof(fence_fd))
  1995. return -SYNX_INVALID;
  1996. if (copy_from_user(&fence_fd,
  1997. u64_to_user_ptr(k_ioctl->ioctl_ptr),
  1998. k_ioctl->size))
  1999. return -EFAULT;
  2000. fence = synx_get_fence(session, fence_fd.synx_obj);
  2001. fence_fd.fd = synx_create_sync_fd(fence);
  2002. /*
  2003. * release additional reference taken in synx_get_fence.
  2004. * additional reference ensures the fence is valid and
  2005. * does not race with handle/fence release.
  2006. */
  2007. dma_fence_put(fence);
  2008. if (copy_to_user(u64_to_user_ptr(k_ioctl->ioctl_ptr),
  2009. &fence_fd, k_ioctl->size))
  2010. return -EFAULT;
  2011. return SYNX_SUCCESS;
  2012. }
  2013. static long synx_ioctl(struct file *filep,
  2014. unsigned int cmd,
  2015. unsigned long arg)
  2016. {
  2017. s32 rc = 0;
  2018. struct synx_private_ioctl_arg k_ioctl;
  2019. struct synx_session *session = filep->private_data;
  2020. if (cmd != SYNX_PRIVATE_IOCTL_CMD) {
  2021. dprintk(SYNX_ERR, "invalid ioctl cmd\n");
  2022. return -ENOIOCTLCMD;
  2023. }
  2024. if (copy_from_user(&k_ioctl,
  2025. (struct synx_private_ioctl_arg *)arg,
  2026. sizeof(k_ioctl))) {
  2027. dprintk(SYNX_ERR, "invalid ioctl args\n");
  2028. return -EFAULT;
  2029. }
  2030. if (!k_ioctl.ioctl_ptr)
  2031. return -SYNX_INVALID;
  2032. dprintk(SYNX_VERB, "[sess :%llu] Enter cmd %u from pid %d\n",
  2033. ((struct synx_client *)session)->id,
  2034. k_ioctl.id, current->pid);
  2035. switch (k_ioctl.id) {
  2036. case SYNX_CREATE:
  2037. rc = synx_handle_create(&k_ioctl, session);
  2038. break;
  2039. case SYNX_RELEASE:
  2040. rc = synx_handle_release(&k_ioctl, session);
  2041. break;
  2042. case SYNX_REGISTER_PAYLOAD:
  2043. rc = synx_handle_async_wait(&k_ioctl,
  2044. session);
  2045. break;
  2046. case SYNX_DEREGISTER_PAYLOAD:
  2047. rc = synx_handle_cancel_async_wait(&k_ioctl,
  2048. session);
  2049. break;
  2050. case SYNX_SIGNAL:
  2051. rc = synx_handle_signal(&k_ioctl, session);
  2052. break;
  2053. case SYNX_MERGE:
  2054. rc = synx_handle_merge(&k_ioctl, session);
  2055. break;
  2056. case SYNX_WAIT:
  2057. rc = synx_handle_wait(&k_ioctl, session);
  2058. if (copy_to_user((void *)arg,
  2059. &k_ioctl,
  2060. sizeof(k_ioctl))) {
  2061. dprintk(SYNX_ERR, "invalid ioctl args\n");
  2062. rc = -EFAULT;
  2063. }
  2064. break;
  2065. case SYNX_BIND:
  2066. rc = synx_handle_bind(&k_ioctl, session);
  2067. break;
  2068. case SYNX_GETSTATUS:
  2069. rc = synx_handle_getstatus(&k_ioctl, session);
  2070. break;
  2071. case SYNX_IMPORT:
  2072. rc = synx_handle_import(&k_ioctl, session);
  2073. break;
  2074. case SYNX_IMPORT_ARR:
  2075. rc = synx_handle_import_arr(&k_ioctl, session);
  2076. break;
  2077. case SYNX_EXPORT:
  2078. rc = synx_handle_export(&k_ioctl, session);
  2079. break;
  2080. case SYNX_GETFENCE_FD:
  2081. rc = synx_handle_get_fence(&k_ioctl, session);
  2082. break;
  2083. default:
  2084. rc = -SYNX_INVALID;
  2085. }
  2086. dprintk(SYNX_VERB, "[sess :%llu] exit with status %d\n",
  2087. ((struct synx_client *)session)->id, rc);
  2088. return rc;
  2089. }
  2090. static ssize_t synx_read(struct file *filep,
  2091. char __user *buf, size_t size, loff_t *f_pos)
  2092. {
  2093. ssize_t rc = 0;
  2094. struct synx_client *client = NULL;
  2095. struct synx_client_cb *cb;
  2096. struct synx_session *session = filep->private_data;
  2097. struct synx_userpayload_info_v2 data;
  2098. if (size != sizeof(struct synx_userpayload_info_v2)) {
  2099. dprintk(SYNX_ERR, "invalid read size\n");
  2100. return -SYNX_INVALID;
  2101. }
  2102. client = synx_get_client(session);
  2103. if (IS_ERR_OR_NULL(client))
  2104. return -SYNX_INVALID;
  2105. mutex_lock(&client->event_q_lock);
  2106. cb = list_first_entry_or_null(&client->event_q,
  2107. struct synx_client_cb, node);
  2108. if (IS_ERR_OR_NULL(cb)) {
  2109. mutex_unlock(&client->event_q_lock);
  2110. rc = 0;
  2111. goto fail;
  2112. }
  2113. if (cb->idx == 0 || cb->idx >= SYNX_MAX_OBJS) {
  2114. dprintk(SYNX_ERR, "invalid index\n");
  2115. mutex_unlock(&client->event_q_lock);
  2116. rc = -SYNX_INVALID;
  2117. goto fail;
  2118. }
  2119. list_del_init(&cb->node);
  2120. mutex_unlock(&client->event_q_lock);
  2121. rc = size;
  2122. data.synx_obj = cb->kernel_cb.h_synx;
  2123. data.reserved = cb->kernel_cb.status;
  2124. data.payload[0] = (u64)cb->kernel_cb.data;
  2125. if (copy_to_user(buf,
  2126. &data,
  2127. sizeof(struct synx_userpayload_info_v2))) {
  2128. dprintk(SYNX_ERR, "couldn't copy user callback data\n");
  2129. rc = -EFAULT;
  2130. }
  2131. if (synx_util_clear_cb_entry(client, cb))
  2132. dprintk(SYNX_ERR,
  2133. "[sess :%llu] error clearing cb for handle %u\n",
  2134. client->id, data.synx_obj);
  2135. fail:
  2136. synx_put_client(client);
  2137. return rc;
  2138. }
  2139. static unsigned int synx_poll(struct file *filep,
  2140. struct poll_table_struct *poll_table)
  2141. {
  2142. int rc = 0;
  2143. struct synx_client *client;
  2144. struct synx_session *session = filep->private_data;
  2145. client = synx_get_client(session);
  2146. if (IS_ERR_OR_NULL(client)) {
  2147. dprintk(SYNX_ERR, "invalid session in poll\n");
  2148. return SYNX_SUCCESS;
  2149. }
  2150. poll_wait(filep, &client->event_wq, poll_table);
  2151. mutex_lock(&client->event_q_lock);
  2152. if (!list_empty(&client->event_q))
  2153. rc = POLLPRI;
  2154. mutex_unlock(&client->event_q_lock);
  2155. synx_put_client(client);
  2156. return rc;
  2157. }
  2158. struct synx_session *synx_initialize(
  2159. struct synx_initialization_params *params)
  2160. {
  2161. struct synx_client *client;
  2162. if (IS_ERR_OR_NULL(params))
  2163. return ERR_PTR(-SYNX_INVALID);
  2164. client = vzalloc(sizeof(*client));
  2165. if (IS_ERR_OR_NULL(client))
  2166. return ERR_PTR(-SYNX_NOMEM);
  2167. if (params->name)
  2168. strlcpy(client->name, params->name, sizeof(client->name));
  2169. client->active = true;
  2170. client->dma_context = dma_fence_context_alloc(1);
  2171. client->id = atomic64_inc_return(&synx_counter);
  2172. kref_init(&client->refcount);
  2173. spin_lock_init(&client->handle_map_lock);
  2174. mutex_init(&client->event_q_lock);
  2175. INIT_LIST_HEAD(&client->event_q);
  2176. init_waitqueue_head(&client->event_wq);
  2177. /* zero idx not allowed */
  2178. set_bit(0, client->cb_bitmap);
  2179. spin_lock_bh(&synx_dev->native->metadata_map_lock);
  2180. hash_add(synx_dev->native->client_metadata_map,
  2181. &client->node, (u64)client);
  2182. spin_unlock_bh(&synx_dev->native->metadata_map_lock);
  2183. dprintk(SYNX_INFO, "[sess :%llu] session created %s\n",
  2184. client->id, params->name);
  2185. return (struct synx_session *)client;
  2186. }
  2187. EXPORT_SYMBOL(synx_initialize);
  2188. int synx_uninitialize(struct synx_session *session)
  2189. {
  2190. struct synx_client *client = NULL, *curr;
  2191. spin_lock_bh(&synx_dev->native->metadata_map_lock);
  2192. hash_for_each_possible(synx_dev->native->client_metadata_map,
  2193. curr, node, (u64)session) {
  2194. if (curr == (struct synx_client *)session) {
  2195. if (curr->active) {
  2196. curr->active = false;
  2197. client = curr;
  2198. }
  2199. break;
  2200. }
  2201. }
  2202. spin_unlock_bh(&synx_dev->native->metadata_map_lock);
  2203. /* release the reference obtained at synx init */
  2204. synx_put_client(client);
  2205. return SYNX_SUCCESS;
  2206. }
  2207. EXPORT_SYMBOL(synx_uninitialize);
  2208. static int synx_open(struct inode *inode, struct file *filep)
  2209. {
  2210. int rc = 0;
  2211. char name[SYNX_OBJ_NAME_LEN];
  2212. struct synx_initialization_params params = {0};
  2213. dprintk(SYNX_VERB, "Enter pid: %d\n", current->pid);
  2214. scnprintf(name, SYNX_OBJ_NAME_LEN, "umd-client-%d", current->pid);
  2215. params.name = name;
  2216. params.id = SYNX_CLIENT_NATIVE;
  2217. filep->private_data = synx_initialize(&params);
  2218. if (IS_ERR_OR_NULL(filep->private_data)) {
  2219. dprintk(SYNX_ERR, "session allocation failed for pid: %d\n",
  2220. current->pid);
  2221. rc = PTR_ERR(filep->private_data);
  2222. } else {
  2223. dprintk(SYNX_VERB, "allocated new session for pid: %d\n",
  2224. current->pid);
  2225. }
  2226. return rc;
  2227. }
  2228. static int synx_close(struct inode *inode, struct file *filep)
  2229. {
  2230. struct synx_session *session = filep->private_data;
  2231. return synx_uninitialize(session);
  2232. }
  2233. static const struct file_operations synx_fops = {
  2234. .owner = THIS_MODULE,
  2235. .open = synx_open,
  2236. .read = synx_read,
  2237. .release = synx_close,
  2238. .poll = synx_poll,
  2239. .unlocked_ioctl = synx_ioctl,
  2240. #ifdef CONFIG_COMPAT
  2241. .compat_ioctl = synx_ioctl,
  2242. #endif
  2243. };
  2244. int synx_register_ops(
  2245. const struct synx_register_params *params)
  2246. {
  2247. s32 rc = 0;
  2248. struct synx_registered_ops *client_ops;
  2249. if (!synx_dev || !params || !params->name ||
  2250. !synx_util_is_valid_bind_type(params->type) ||
  2251. !params->ops.register_callback ||
  2252. !params->ops.deregister_callback ||
  2253. !params->ops.signal) {
  2254. dprintk(SYNX_ERR, "invalid register params\n");
  2255. return -SYNX_INVALID;
  2256. }
  2257. mutex_lock(&synx_dev->vtbl_lock);
  2258. client_ops = &synx_dev->bind_vtbl[params->type];
  2259. if (!client_ops->valid) {
  2260. client_ops->valid = true;
  2261. memcpy(&client_ops->ops, &params->ops,
  2262. sizeof(client_ops->ops));
  2263. strlcpy(client_ops->name, params->name,
  2264. sizeof(client_ops->name));
  2265. client_ops->type = params->type;
  2266. dprintk(SYNX_INFO,
  2267. "registered bind ops type %u for %s\n",
  2268. params->type, params->name);
  2269. } else {
  2270. dprintk(SYNX_WARN,
  2271. "client already registered for type %u by %s\n",
  2272. client_ops->type, client_ops->name);
  2273. rc = -SYNX_ALREADY;
  2274. }
  2275. mutex_unlock(&synx_dev->vtbl_lock);
  2276. return rc;
  2277. }
  2278. EXPORT_SYMBOL(synx_register_ops);
  2279. int synx_deregister_ops(
  2280. const struct synx_register_params *params)
  2281. {
  2282. struct synx_registered_ops *client_ops;
  2283. if (IS_ERR_OR_NULL(params) || params->name ||
  2284. !synx_util_is_valid_bind_type(params->type)) {
  2285. dprintk(SYNX_ERR, "invalid params\n");
  2286. return -SYNX_INVALID;
  2287. }
  2288. mutex_lock(&synx_dev->vtbl_lock);
  2289. client_ops = &synx_dev->bind_vtbl[params->type];
  2290. memset(client_ops, 0, sizeof(*client_ops));
  2291. dprintk(SYNX_INFO, "deregistered bind ops for %s\n",
  2292. params->name);
  2293. mutex_unlock(&synx_dev->vtbl_lock);
  2294. return SYNX_SUCCESS;
  2295. }
  2296. EXPORT_SYMBOL(synx_deregister_ops);
  2297. void synx_ipc_handler(struct work_struct *cb_dispatch)
  2298. {
  2299. struct synx_signal_cb *signal_cb =
  2300. container_of(cb_dispatch, struct synx_signal_cb, cb_dispatch);
  2301. struct synx_map_entry *map_entry;
  2302. map_entry = synx_util_get_map_entry(signal_cb->handle);
  2303. if (IS_ERR_OR_NULL(map_entry)) {
  2304. dprintk(SYNX_WARN,
  2305. "no clients to notify for %u\n",
  2306. signal_cb->handle);
  2307. dprintk(SYNX_MEM, "signal cb destroyed %pK\n", signal_cb);
  2308. kfree(signal_cb);
  2309. return;
  2310. }
  2311. /* get reference on synx coredata for signal cb */
  2312. synx_util_get_object(map_entry->synx_obj);
  2313. signal_cb->synx_obj = map_entry->synx_obj;
  2314. synx_util_release_map_entry(map_entry);
  2315. synx_signal_handler(&signal_cb->cb_dispatch);
  2316. }
  2317. int synx_ipc_callback(u32 client_id,
  2318. s64 data, void *priv)
  2319. {
  2320. struct synx_signal_cb *signal_cb;
  2321. u32 status = (u32)data;
  2322. u32 handle = (u32)(data >> 32);
  2323. signal_cb = kzalloc(sizeof(*signal_cb), GFP_ATOMIC);
  2324. if (IS_ERR_OR_NULL(signal_cb))
  2325. return -SYNX_NOMEM;
  2326. dprintk(SYNX_DBG,
  2327. "signal notification for %u received with status %u\n",
  2328. handle, status);
  2329. signal_cb->status = status;
  2330. signal_cb->handle = handle;
  2331. signal_cb->flag = SYNX_SIGNAL_FROM_IPC;
  2332. INIT_WORK(&signal_cb->cb_dispatch, synx_ipc_handler);
  2333. queue_work(synx_dev->wq_cb, &signal_cb->cb_dispatch);
  2334. return SYNX_SUCCESS;
  2335. }
  2336. EXPORT_SYMBOL(synx_ipc_callback);
  2337. int synx_recover(enum synx_client_id id)
  2338. {
  2339. u32 core_id;
  2340. core_id = synx_util_map_client_id_to_core(id);
  2341. if (core_id >= SYNX_CORE_MAX) {
  2342. dprintk(SYNX_ERR, "invalid client id %u\n", id);
  2343. return -SYNX_INVALID;
  2344. }
  2345. switch (core_id) {
  2346. case SYNX_CORE_EVA:
  2347. case SYNX_CORE_IRIS:
  2348. case SYNX_CORE_ICP:
  2349. break;
  2350. default:
  2351. dprintk(SYNX_ERR, "recovery not supported on %u\n", id);
  2352. return -SYNX_NOSUPPORT;
  2353. }
  2354. return synx_global_recover(core_id);
  2355. }
  2356. EXPORT_SYMBOL(synx_recover);
  2357. static int synx_local_mem_init(void)
  2358. {
  2359. if (!synx_dev->native)
  2360. return -SYNX_INVALID;
  2361. hash_init(synx_dev->native->client_metadata_map);
  2362. hash_init(synx_dev->native->fence_map);
  2363. hash_init(synx_dev->native->global_map);
  2364. hash_init(synx_dev->native->local_map);
  2365. hash_init(synx_dev->native->csl_fence_map);
  2366. spin_lock_init(&synx_dev->native->metadata_map_lock);
  2367. spin_lock_init(&synx_dev->native->fence_map_lock);
  2368. spin_lock_init(&synx_dev->native->global_map_lock);
  2369. spin_lock_init(&synx_dev->native->local_map_lock);
  2370. spin_lock_init(&synx_dev->native->csl_map_lock);
  2371. /* zero idx not allowed */
  2372. set_bit(0, synx_dev->native->bitmap);
  2373. return 0;
  2374. }
  2375. static int synx_cdsp_restart_notifier(struct notifier_block *nb,
  2376. unsigned long code, void *data)
  2377. {
  2378. struct synx_cdsp_ssr *cdsp_ssr = &synx_dev->cdsp_ssr;
  2379. if (&cdsp_ssr->nb != nb) {
  2380. dprintk(SYNX_ERR, "Invalid SSR Notifier block\n");
  2381. return NOTIFY_BAD;
  2382. }
  2383. switch (code) {
  2384. case QCOM_SSR_BEFORE_SHUTDOWN:
  2385. break;
  2386. case QCOM_SSR_AFTER_SHUTDOWN:
  2387. if (cdsp_ssr->ssrcnt != 0) {
  2388. dprintk(SYNX_INFO, "Cleaning up global memory\n");
  2389. synx_global_recover(SYNX_CORE_NSP);
  2390. }
  2391. break;
  2392. case QCOM_SSR_BEFORE_POWERUP:
  2393. break;
  2394. case QCOM_SSR_AFTER_POWERUP:
  2395. dprintk(SYNX_DBG, "CDSP is up");
  2396. if (cdsp_ssr->ssrcnt == 0)
  2397. cdsp_ssr->ssrcnt++;
  2398. break;
  2399. default:
  2400. dprintk(SYNX_ERR, "Unknown status code for CDSP SSR\n");
  2401. break;
  2402. }
  2403. return NOTIFY_DONE;
  2404. }
  2405. static int __init synx_init(void)
  2406. {
  2407. int rc;
  2408. dprintk(SYNX_INFO, "device initialization start\n");
  2409. synx_dev = kzalloc(sizeof(*synx_dev), GFP_KERNEL);
  2410. if (IS_ERR_OR_NULL(synx_dev))
  2411. return -SYNX_NOMEM;
  2412. rc = alloc_chrdev_region(&synx_dev->dev, 0, 1, SYNX_DEVICE_NAME);
  2413. if (rc < 0) {
  2414. dprintk(SYNX_ERR, "region allocation failed\n");
  2415. goto alloc_fail;
  2416. }
  2417. cdev_init(&synx_dev->cdev, &synx_fops);
  2418. synx_dev->cdev.owner = THIS_MODULE;
  2419. rc = cdev_add(&synx_dev->cdev, synx_dev->dev, 1);
  2420. if (rc < 0) {
  2421. dprintk(SYNX_ERR, "device registation failed\n");
  2422. goto reg_fail;
  2423. }
  2424. synx_dev->class = class_create(THIS_MODULE, SYNX_DEVICE_NAME);
  2425. if (IS_ERR(synx_dev->class)) {
  2426. rc = PTR_ERR(synx_dev->class);
  2427. goto err_class_create;
  2428. }
  2429. device_create(synx_dev->class, NULL, synx_dev->dev,
  2430. NULL, SYNX_DEVICE_NAME);
  2431. synx_dev->wq_cb = alloc_workqueue(SYNX_WQ_CB_NAME,
  2432. WQ_HIGHPRI | WQ_UNBOUND, SYNX_WQ_CB_THREADS);
  2433. synx_dev->wq_cleanup = alloc_workqueue(SYNX_WQ_CLEANUP_NAME,
  2434. WQ_HIGHPRI | WQ_UNBOUND, SYNX_WQ_CLEANUP_THREADS);
  2435. if (!synx_dev->wq_cb || !synx_dev->wq_cleanup) {
  2436. dprintk(SYNX_ERR,
  2437. "high priority work queue creation failed\n");
  2438. rc = -SYNX_INVALID;
  2439. goto fail;
  2440. }
  2441. synx_dev->native = vzalloc(sizeof(*synx_dev->native));
  2442. if (IS_ERR_OR_NULL(synx_dev->native))
  2443. goto fail;
  2444. mutex_init(&synx_dev->vtbl_lock);
  2445. mutex_init(&synx_dev->error_lock);
  2446. INIT_LIST_HEAD(&synx_dev->error_list);
  2447. synx_dev->debugfs_root = synx_init_debugfs_dir(synx_dev);
  2448. rc = synx_global_mem_init();
  2449. if (rc) {
  2450. dprintk(SYNX_ERR, "shared mem init failed, err=%d\n", rc);
  2451. goto err;
  2452. }
  2453. synx_dev->cdsp_ssr.ssrcnt = 0;
  2454. synx_dev->cdsp_ssr.nb.notifier_call = synx_cdsp_restart_notifier;
  2455. synx_dev->cdsp_ssr.handle =
  2456. qcom_register_ssr_notifier("cdsp", &synx_dev->cdsp_ssr.nb);
  2457. if (synx_dev->cdsp_ssr.handle == NULL) {
  2458. dprintk(SYNX_ERR, "SSR registration failed\n");
  2459. goto err;
  2460. }
  2461. ipclite_register_client(synx_ipc_callback, NULL);
  2462. synx_local_mem_init();
  2463. dprintk(SYNX_INFO, "device initialization success\n");
  2464. return 0;
  2465. err:
  2466. vfree(synx_dev->native);
  2467. fail:
  2468. device_destroy(synx_dev->class, synx_dev->dev);
  2469. class_destroy(synx_dev->class);
  2470. err_class_create:
  2471. cdev_del(&synx_dev->cdev);
  2472. reg_fail:
  2473. unregister_chrdev_region(synx_dev->dev, 1);
  2474. alloc_fail:
  2475. kfree(synx_dev);
  2476. synx_dev = NULL;
  2477. return rc;
  2478. }
  2479. static void __exit synx_exit(void)
  2480. {
  2481. struct error_node *err_node, *err_node_tmp;
  2482. flush_workqueue(synx_dev->wq_cb);
  2483. flush_workqueue(synx_dev->wq_cleanup);
  2484. device_destroy(synx_dev->class, synx_dev->dev);
  2485. class_destroy(synx_dev->class);
  2486. cdev_del(&synx_dev->cdev);
  2487. unregister_chrdev_region(synx_dev->dev, 1);
  2488. synx_remove_debugfs_dir(synx_dev);
  2489. /* release uncleared error nodes */
  2490. list_for_each_entry_safe(
  2491. err_node, err_node_tmp,
  2492. &synx_dev->error_list,
  2493. node) {
  2494. list_del(&err_node->node);
  2495. kfree(err_node);
  2496. }
  2497. mutex_destroy(&synx_dev->vtbl_lock);
  2498. mutex_destroy(&synx_dev->error_lock);
  2499. vfree(synx_dev->native);
  2500. kfree(synx_dev);
  2501. }
  2502. module_init(synx_init);
  2503. module_exit(synx_exit);
  2504. MODULE_DESCRIPTION("Global Synx Driver");
  2505. MODULE_LICENSE("GPL v2");