synx_util.c 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #include <linux/slab.h>
  7. #include <linux/random.h>
  8. #include <linux/vmalloc.h>
  9. #include "synx_debugfs.h"
  10. #include "synx_util.h"
  11. extern void synx_external_callback(s32 sync_obj, int status, void *data);
  12. static u32 __fence_state(struct dma_fence *fence, bool locked);
  13. int synx_util_init_coredata(struct synx_coredata *synx_obj,
  14. struct synx_create_params *params,
  15. struct dma_fence_ops *ops,
  16. u64 dma_context)
  17. {
  18. int rc = -SYNX_INVALID;
  19. spinlock_t *fence_lock;
  20. struct dma_fence *fence;
  21. struct synx_fence_entry *entry;
  22. if (IS_ERR_OR_NULL(synx_obj) || IS_ERR_OR_NULL(params) ||
  23. IS_ERR_OR_NULL(ops) || IS_ERR_OR_NULL(params->h_synx))
  24. return -SYNX_INVALID;
  25. if (params->flags & SYNX_CREATE_GLOBAL_FENCE &&
  26. *params->h_synx != 0) {
  27. rc = synx_global_get_ref(
  28. synx_util_global_idx(*params->h_synx));
  29. synx_obj->global_idx = synx_util_global_idx(*params->h_synx);
  30. } else if (params->flags & SYNX_CREATE_GLOBAL_FENCE) {
  31. rc = synx_alloc_global_handle(params->h_synx);
  32. synx_obj->global_idx = synx_util_global_idx(*params->h_synx);
  33. } else {
  34. rc = synx_alloc_local_handle(params->h_synx);
  35. }
  36. if (rc != SYNX_SUCCESS)
  37. return rc;
  38. synx_obj->map_count = 1;
  39. synx_obj->num_bound_synxs = 0;
  40. synx_obj->type |= params->flags;
  41. kref_init(&synx_obj->refcount);
  42. mutex_init(&synx_obj->obj_lock);
  43. INIT_LIST_HEAD(&synx_obj->reg_cbs_list);
  44. if (params->name)
  45. strlcpy(synx_obj->name, params->name, sizeof(synx_obj->name));
  46. if (params->flags & SYNX_CREATE_DMA_FENCE) {
  47. fence = params->fence;
  48. if (IS_ERR_OR_NULL(fence)) {
  49. dprintk(SYNX_ERR, "invalid external fence\n");
  50. goto free;
  51. }
  52. dma_fence_get(fence);
  53. synx_obj->fence = fence;
  54. } else {
  55. /*
  56. * lock and fence memory will be released in fence
  57. * release function
  58. */
  59. fence_lock = kzalloc(sizeof(*fence_lock), GFP_KERNEL);
  60. if (IS_ERR_OR_NULL(fence_lock)) {
  61. rc = -SYNX_NOMEM;
  62. goto free;
  63. }
  64. fence = kzalloc(sizeof(*fence), GFP_KERNEL);
  65. if (IS_ERR_OR_NULL(fence)) {
  66. kfree(fence_lock);
  67. rc = -SYNX_NOMEM;
  68. goto free;
  69. }
  70. spin_lock_init(fence_lock);
  71. dma_fence_init(fence, ops, fence_lock, dma_context, 1);
  72. synx_obj->fence = fence;
  73. synx_util_activate(synx_obj);
  74. dprintk(SYNX_MEM,
  75. "allocated backing fence %pK\n", fence);
  76. entry = kzalloc(sizeof(*entry), GFP_KERNEL);
  77. if (IS_ERR_OR_NULL(entry)) {
  78. rc = -SYNX_NOMEM;
  79. goto clean;
  80. }
  81. entry->key = (u64)fence;
  82. if (params->flags & SYNX_CREATE_GLOBAL_FENCE)
  83. entry->g_handle = *params->h_synx;
  84. else
  85. entry->l_handle = *params->h_synx;
  86. rc = synx_util_insert_fence_entry(entry,
  87. params->h_synx,
  88. params->flags & SYNX_CREATE_GLOBAL_FENCE);
  89. BUG_ON(rc != SYNX_SUCCESS);
  90. }
  91. if (rc != SYNX_SUCCESS)
  92. goto clean;
  93. return SYNX_SUCCESS;
  94. clean:
  95. dma_fence_put(fence);
  96. free:
  97. if (params->flags & SYNX_CREATE_GLOBAL_FENCE)
  98. synx_global_put_ref(
  99. synx_util_global_idx(*params->h_synx));
  100. else
  101. clear_bit(synx_util_global_idx(*params->h_synx),
  102. synx_dev->native->bitmap);
  103. return rc;
  104. }
  105. int synx_util_add_callback(struct synx_coredata *synx_obj,
  106. u32 h_synx)
  107. {
  108. int rc;
  109. struct synx_signal_cb *signal_cb;
  110. if (IS_ERR_OR_NULL(synx_obj))
  111. return -SYNX_INVALID;
  112. signal_cb = kzalloc(sizeof(*signal_cb), GFP_KERNEL);
  113. if (IS_ERR_OR_NULL(signal_cb))
  114. return -SYNX_NOMEM;
  115. signal_cb->handle = h_synx;
  116. signal_cb->flag = SYNX_SIGNAL_FROM_FENCE;
  117. signal_cb->synx_obj = synx_obj;
  118. /* get reference on synx coredata for signal cb */
  119. synx_util_get_object(synx_obj);
  120. /*
  121. * adding callback enables synx framework to
  122. * get notified on signal from clients using
  123. * native dma fence operations.
  124. */
  125. rc = dma_fence_add_callback(synx_obj->fence,
  126. &signal_cb->fence_cb, synx_fence_callback);
  127. if (rc != 0) {
  128. if (rc == -ENOENT) {
  129. if (synx_util_is_global_object(synx_obj)) {
  130. /* signal (if) global handle */
  131. rc = synx_global_update_status(
  132. synx_obj->global_idx,
  133. synx_util_get_object_status(synx_obj));
  134. if (rc != SYNX_SUCCESS)
  135. dprintk(SYNX_ERR,
  136. "status update of %u with fence %pK\n",
  137. synx_obj->global_idx, synx_obj->fence);
  138. } else {
  139. rc = SYNX_SUCCESS;
  140. }
  141. } else {
  142. dprintk(SYNX_ERR,
  143. "error adding callback for %pK err %d\n",
  144. synx_obj->fence, rc);
  145. }
  146. synx_util_put_object(synx_obj);
  147. kfree(signal_cb);
  148. return rc;
  149. }
  150. synx_obj->signal_cb = signal_cb;
  151. dprintk(SYNX_VERB, "added callback %pK to fence %pK\n",
  152. signal_cb, synx_obj->fence);
  153. return SYNX_SUCCESS;
  154. }
  155. int synx_util_init_group_coredata(struct synx_coredata *synx_obj,
  156. struct dma_fence **fences,
  157. struct synx_merge_params *params,
  158. u32 num_objs,
  159. u64 dma_context)
  160. {
  161. int rc;
  162. struct dma_fence_array *array;
  163. if (IS_ERR_OR_NULL(synx_obj))
  164. return -SYNX_INVALID;
  165. if (params->flags & SYNX_MERGE_GLOBAL_FENCE) {
  166. rc = synx_alloc_global_handle(params->h_merged_obj);
  167. synx_obj->global_idx =
  168. synx_util_global_idx(*params->h_merged_obj);
  169. } else {
  170. rc = synx_alloc_local_handle(params->h_merged_obj);
  171. }
  172. if (rc != SYNX_SUCCESS)
  173. return rc;
  174. array = dma_fence_array_create(num_objs, fences,
  175. dma_context, 1, false);
  176. if (IS_ERR_OR_NULL(array))
  177. return -SYNX_INVALID;
  178. synx_obj->fence = &array->base;
  179. synx_obj->map_count = 1;
  180. synx_obj->type = params->flags;
  181. synx_obj->type |= SYNX_CREATE_MERGED_FENCE;
  182. synx_obj->num_bound_synxs = 0;
  183. kref_init(&synx_obj->refcount);
  184. mutex_init(&synx_obj->obj_lock);
  185. INIT_LIST_HEAD(&synx_obj->reg_cbs_list);
  186. synx_util_activate(synx_obj);
  187. return rc;
  188. }
  189. static void synx_util_destroy_coredata(struct kref *kref)
  190. {
  191. int rc;
  192. struct synx_coredata *synx_obj =
  193. container_of(kref, struct synx_coredata, refcount);
  194. if (synx_util_is_global_object(synx_obj)) {
  195. rc = synx_global_clear_subscribed_core(synx_obj->global_idx, SYNX_CORE_APSS);
  196. if (rc)
  197. dprintk(SYNX_ERR, "Failed to clear subscribers");
  198. synx_global_put_ref(synx_obj->global_idx);
  199. }
  200. synx_util_object_destroy(synx_obj);
  201. }
  202. void synx_util_get_object(struct synx_coredata *synx_obj)
  203. {
  204. kref_get(&synx_obj->refcount);
  205. }
  206. void synx_util_put_object(struct synx_coredata *synx_obj)
  207. {
  208. kref_put(&synx_obj->refcount, synx_util_destroy_coredata);
  209. }
  210. int synx_util_cleanup_merged_fence(struct synx_coredata *synx_obj, int status)
  211. {
  212. struct dma_fence_array *array = NULL;
  213. u32 i;
  214. int rc = 0;
  215. if (IS_ERR_OR_NULL(synx_obj) || IS_ERR_OR_NULL(synx_obj->fence))
  216. return -SYNX_INVALID;
  217. if (dma_fence_is_array(synx_obj->fence)) {
  218. array = to_dma_fence_array(synx_obj->fence);
  219. if (IS_ERR_OR_NULL(array))
  220. return -SYNX_INVALID;
  221. for (i = 0; i < array->num_fences; i++) {
  222. if (kref_read(&array->fences[i]->refcount) == 1 &&
  223. __fence_state(array->fences[i], false) == SYNX_STATE_ACTIVE) {
  224. dma_fence_set_error(array->fences[i],
  225. -SYNX_STATE_SIGNALED_CANCEL);
  226. rc = dma_fence_signal(array->fences[i]);
  227. if (rc)
  228. dprintk(SYNX_ERR,
  229. "signaling child fence %pK failed=%d\n",
  230. array->fences[i], rc);
  231. }
  232. dma_fence_put(array->fences[i]);
  233. }
  234. }
  235. return rc;
  236. }
  237. void synx_util_object_destroy(struct synx_coredata *synx_obj)
  238. {
  239. int rc;
  240. u32 i;
  241. s32 sync_id;
  242. u32 type;
  243. unsigned long flags;
  244. struct synx_cb_data *synx_cb, *synx_cb_temp;
  245. struct synx_bind_desc *bind_desc;
  246. struct bind_operations *bind_ops;
  247. struct synx_external_data *data;
  248. /* clear all the undispatched callbacks */
  249. list_for_each_entry_safe(synx_cb,
  250. synx_cb_temp, &synx_obj->reg_cbs_list, node) {
  251. dprintk(SYNX_ERR,
  252. "dipatching un-released callbacks of session %pK\n",
  253. synx_cb->session);
  254. synx_cb->status = SYNX_STATE_SIGNALED_CANCEL;
  255. list_del_init(&synx_cb->node);
  256. queue_work(synx_dev->wq_cb,
  257. &synx_cb->cb_dispatch);
  258. dprintk(SYNX_VERB, "dispatched callback for fence %pKn", synx_obj->fence);
  259. }
  260. for (i = 0; i < synx_obj->num_bound_synxs; i++) {
  261. bind_desc = &synx_obj->bound_synxs[i];
  262. sync_id = bind_desc->external_desc.id;
  263. type = bind_desc->external_desc.type;
  264. data = bind_desc->external_data;
  265. bind_ops = synx_util_get_bind_ops(type);
  266. if (IS_ERR_OR_NULL(bind_ops)) {
  267. dprintk(SYNX_ERR,
  268. "bind ops fail id: %d, type: %u, err: %d\n",
  269. sync_id, type, rc);
  270. continue;
  271. }
  272. /* clear the hash table entry */
  273. synx_util_remove_data(&sync_id, type);
  274. rc = bind_ops->deregister_callback(
  275. synx_external_callback, data, sync_id);
  276. if (rc < 0) {
  277. dprintk(SYNX_ERR,
  278. "de-registration fail id: %d, type: %u, err: %d\n",
  279. sync_id, type, rc);
  280. continue;
  281. }
  282. /*
  283. * release the memory allocated for external data.
  284. * It is safe to release this memory
  285. * only if deregistration is successful.
  286. */
  287. kfree(data);
  288. }
  289. mutex_destroy(&synx_obj->obj_lock);
  290. synx_util_release_fence_entry((u64)synx_obj->fence);
  291. /* dma fence framework expects handles are signaled before release,
  292. * so signal if active handle and has last refcount. Synx handles
  293. * on other cores are still active to carry out usual callflow.
  294. */
  295. if (!IS_ERR_OR_NULL(synx_obj->fence)) {
  296. spin_lock_irqsave(synx_obj->fence->lock, flags);
  297. if (synx_util_is_merged_object(synx_obj) &&
  298. synx_util_get_object_status_locked(synx_obj) == SYNX_STATE_ACTIVE)
  299. rc = synx_util_cleanup_merged_fence(synx_obj, -SYNX_STATE_SIGNALED_CANCEL);
  300. else if (kref_read(&synx_obj->fence->refcount) == 1 &&
  301. (synx_util_get_object_status_locked(synx_obj) ==
  302. SYNX_STATE_ACTIVE)) {
  303. // set fence error to cancel
  304. dma_fence_set_error(synx_obj->fence,
  305. -SYNX_STATE_SIGNALED_CANCEL);
  306. rc = dma_fence_signal_locked(synx_obj->fence);
  307. }
  308. spin_unlock_irqrestore(synx_obj->fence->lock, flags);
  309. if (rc)
  310. dprintk(SYNX_ERR,
  311. "signaling fence %pK failed=%d\n",
  312. synx_obj->fence, rc);
  313. }
  314. dma_fence_put(synx_obj->fence);
  315. kfree(synx_obj);
  316. dprintk(SYNX_MEM, "released synx object %pK\n", synx_obj);
  317. }
  318. long synx_util_get_free_handle(unsigned long *bitmap, unsigned int size)
  319. {
  320. bool bit;
  321. long idx;
  322. do {
  323. idx = find_first_zero_bit(bitmap, size);
  324. if (idx >= size)
  325. break;
  326. bit = test_and_set_bit(idx, bitmap);
  327. } while (bit);
  328. return idx;
  329. }
  330. u32 synx_encode_handle(u32 idx, u32 core_id, bool global_idx)
  331. {
  332. u32 handle = 0;
  333. if (idx >= SYNX_MAX_OBJS)
  334. return 0;
  335. if (global_idx) {
  336. handle = 1;
  337. handle <<= SYNX_HANDLE_CORE_BITS;
  338. }
  339. handle |= core_id;
  340. handle <<= SYNX_HANDLE_INDEX_BITS;
  341. handle |= idx;
  342. return handle;
  343. }
  344. int synx_alloc_global_handle(u32 *new_synx)
  345. {
  346. int rc;
  347. u32 idx;
  348. rc = synx_global_alloc_index(&idx);
  349. if (rc != SYNX_SUCCESS)
  350. return rc;
  351. *new_synx = synx_encode_handle(idx, SYNX_CORE_APSS, true);
  352. dprintk(SYNX_DBG, "allocated global handle %u (0x%x)\n",
  353. *new_synx, *new_synx);
  354. rc = synx_global_init_coredata(*new_synx);
  355. return rc;
  356. }
  357. int synx_alloc_local_handle(u32 *new_synx)
  358. {
  359. u32 idx;
  360. idx = synx_util_get_free_handle(synx_dev->native->bitmap,
  361. SYNX_MAX_OBJS);
  362. if (idx >= SYNX_MAX_OBJS)
  363. return -SYNX_NOMEM;
  364. *new_synx = synx_encode_handle(idx, SYNX_CORE_APSS, false);
  365. dprintk(SYNX_DBG, "allocated local handle %u (0x%x)\n",
  366. *new_synx, *new_synx);
  367. return SYNX_SUCCESS;
  368. }
  369. int synx_util_init_handle(struct synx_client *client,
  370. struct synx_coredata *synx_obj, u32 *new_h_synx,
  371. void *map_entry)
  372. {
  373. int rc = SYNX_SUCCESS;
  374. bool found = false;
  375. struct synx_handle_coredata *synx_data, *curr;
  376. if (IS_ERR_OR_NULL(client) || IS_ERR_OR_NULL(synx_obj) ||
  377. IS_ERR_OR_NULL(new_h_synx) || IS_ERR_OR_NULL(map_entry))
  378. return -SYNX_INVALID;
  379. synx_data = kzalloc(sizeof(*synx_data), GFP_ATOMIC);
  380. if (IS_ERR_OR_NULL(synx_data))
  381. return -SYNX_NOMEM;
  382. synx_data->client = client;
  383. synx_data->synx_obj = synx_obj;
  384. synx_data->key = *new_h_synx;
  385. synx_data->map_entry = map_entry;
  386. kref_init(&synx_data->refcount);
  387. synx_data->rel_count = 1;
  388. spin_lock_bh(&client->handle_map_lock);
  389. hash_for_each_possible(client->handle_map,
  390. curr, node, *new_h_synx) {
  391. if (curr->key == *new_h_synx) {
  392. if (curr->synx_obj != synx_obj) {
  393. rc = -SYNX_INVALID;
  394. dprintk(SYNX_ERR,
  395. "inconsistent data in handle map\n");
  396. } else {
  397. kref_get(&curr->refcount);
  398. curr->rel_count++;
  399. }
  400. found = true;
  401. break;
  402. }
  403. }
  404. if (unlikely(found))
  405. kfree(synx_data);
  406. else
  407. hash_add(client->handle_map,
  408. &synx_data->node, *new_h_synx);
  409. spin_unlock_bh(&client->handle_map_lock);
  410. return rc;
  411. }
  412. int synx_util_activate(struct synx_coredata *synx_obj)
  413. {
  414. if (IS_ERR_OR_NULL(synx_obj))
  415. return -SYNX_INVALID;
  416. /* move synx to ACTIVE state and register cb for merged object */
  417. dma_fence_enable_sw_signaling(synx_obj->fence);
  418. return 0;
  419. }
  420. static u32 synx_util_get_references(struct synx_coredata *synx_obj)
  421. {
  422. u32 count = 0;
  423. u32 i = 0;
  424. struct dma_fence_array *array = NULL;
  425. /* obtain dma fence reference */
  426. if (dma_fence_is_array(synx_obj->fence)) {
  427. array = to_dma_fence_array(synx_obj->fence);
  428. if (IS_ERR_OR_NULL(array))
  429. return 0;
  430. for (i = 0; i < array->num_fences; i++)
  431. dma_fence_get(array->fences[i]);
  432. count = array->num_fences;
  433. } else {
  434. dma_fence_get(synx_obj->fence);
  435. count = 1;
  436. }
  437. return count;
  438. }
  439. static void synx_util_put_references(struct synx_coredata *synx_obj)
  440. {
  441. u32 i = 0;
  442. struct dma_fence_array *array = NULL;
  443. if (dma_fence_is_array(synx_obj->fence)) {
  444. array = to_dma_fence_array(synx_obj->fence);
  445. if (IS_ERR_OR_NULL(array))
  446. return;
  447. for (i = 0; i < array->num_fences; i++)
  448. dma_fence_put(array->fences[i]);
  449. } else {
  450. dma_fence_put(synx_obj->fence);
  451. }
  452. }
  453. static u32 synx_util_add_fence(struct synx_coredata *synx_obj,
  454. struct dma_fence **fences,
  455. u32 idx)
  456. {
  457. struct dma_fence_array *array = NULL;
  458. u32 i = 0;
  459. if (dma_fence_is_array(synx_obj->fence)) {
  460. array = to_dma_fence_array(synx_obj->fence);
  461. if (IS_ERR_OR_NULL(array))
  462. return 0;
  463. for (i = 0; i < array->num_fences; i++)
  464. fences[idx+i] = array->fences[i];
  465. return array->num_fences;
  466. }
  467. fences[idx] = synx_obj->fence;
  468. return 1;
  469. }
  470. static u32 synx_util_remove_duplicates(struct dma_fence **arr, u32 num)
  471. {
  472. int i, j;
  473. u32 wr_idx = 1;
  474. if (IS_ERR_OR_NULL(arr)) {
  475. dprintk(SYNX_ERR, "invalid input array\n");
  476. return 0;
  477. }
  478. for (i = 1; i < num; i++) {
  479. for (j = 0; j < wr_idx ; j++) {
  480. if (arr[i] == arr[j]) {
  481. /* release reference obtained for duplicate */
  482. dprintk(SYNX_DBG,
  483. "releasing duplicate reference\n");
  484. dma_fence_put(arr[i]);
  485. break;
  486. }
  487. }
  488. if (j == wr_idx)
  489. arr[wr_idx++] = arr[i];
  490. }
  491. return wr_idx;
  492. }
  493. s32 synx_util_merge_error(struct synx_client *client,
  494. u32 *h_synxs,
  495. u32 num_objs)
  496. {
  497. u32 i = 0;
  498. struct synx_handle_coredata *synx_data;
  499. struct synx_coredata *synx_obj;
  500. if (IS_ERR_OR_NULL(client) || IS_ERR_OR_NULL(h_synxs))
  501. return -SYNX_INVALID;
  502. for (i = 0; i < num_objs; i++) {
  503. synx_data = synx_util_acquire_handle(client, h_synxs[i]);
  504. synx_obj = synx_util_obtain_object(synx_data);
  505. if (IS_ERR_OR_NULL(synx_obj) ||
  506. IS_ERR_OR_NULL(synx_obj->fence)) {
  507. dprintk(SYNX_ERR,
  508. "[sess :%llu] invalid handle %d in cleanup\n",
  509. client->id, h_synxs[i]);
  510. continue;
  511. }
  512. /* release all references obtained during merge validatation */
  513. synx_util_put_references(synx_obj);
  514. synx_util_release_handle(synx_data);
  515. }
  516. return 0;
  517. }
  518. int synx_util_validate_merge(struct synx_client *client,
  519. u32 *h_synxs,
  520. u32 num_objs,
  521. struct dma_fence ***fence_list,
  522. u32 *fence_cnt)
  523. {
  524. u32 count = 0;
  525. u32 i = 0;
  526. struct synx_handle_coredata **synx_datas;
  527. struct synx_coredata **synx_objs;
  528. struct dma_fence **fences = NULL;
  529. if (num_objs <= 1) {
  530. dprintk(SYNX_ERR, "single handle merge is not allowed\n");
  531. return -SYNX_INVALID;
  532. }
  533. synx_datas = kcalloc(num_objs, sizeof(*synx_datas), GFP_KERNEL);
  534. if (IS_ERR_OR_NULL(synx_datas))
  535. return -SYNX_NOMEM;
  536. synx_objs = kcalloc(num_objs, sizeof(*synx_objs), GFP_KERNEL);
  537. if (IS_ERR_OR_NULL(synx_objs)) {
  538. kfree(synx_datas);
  539. return -SYNX_NOMEM;
  540. }
  541. for (i = 0; i < num_objs; i++) {
  542. synx_datas[i] = synx_util_acquire_handle(client, h_synxs[i]);
  543. synx_objs[i] = synx_util_obtain_object(synx_datas[i]);
  544. if (IS_ERR_OR_NULL(synx_objs[i]) ||
  545. IS_ERR_OR_NULL(synx_objs[i]->fence)) {
  546. dprintk(SYNX_ERR,
  547. "[sess :%llu] invalid handle %d in merge list\n",
  548. client->id, h_synxs[i]);
  549. *fence_cnt = i;
  550. goto error;
  551. }
  552. count += synx_util_get_references(synx_objs[i]);
  553. }
  554. fences = kcalloc(count, sizeof(*fences), GFP_KERNEL);
  555. if (IS_ERR_OR_NULL(fences)) {
  556. *fence_cnt = num_objs;
  557. goto error;
  558. }
  559. /* memory will be released later in the invoking function */
  560. *fence_list = fences;
  561. count = 0;
  562. for (i = 0; i < num_objs; i++) {
  563. count += synx_util_add_fence(synx_objs[i], fences, count);
  564. /* release the reference obtained earlier in the function */
  565. synx_util_release_handle(synx_datas[i]);
  566. }
  567. *fence_cnt = synx_util_remove_duplicates(fences, count);
  568. kfree(synx_objs);
  569. kfree(synx_datas);
  570. return 0;
  571. error:
  572. /* release the reference/s obtained earlier in the function */
  573. for (i = 0; i < *fence_cnt; i++) {
  574. synx_util_put_references(synx_objs[i]);
  575. synx_util_release_handle(synx_datas[i]);
  576. }
  577. *fence_cnt = 0;
  578. kfree(synx_objs);
  579. kfree(synx_datas);
  580. return -SYNX_INVALID;
  581. }
  582. static u32 __fence_state(struct dma_fence *fence, bool locked)
  583. {
  584. s32 status;
  585. u32 state = SYNX_STATE_INVALID;
  586. if (IS_ERR_OR_NULL(fence)) {
  587. dprintk(SYNX_ERR, "invalid fence\n");
  588. return SYNX_STATE_INVALID;
  589. }
  590. if (locked)
  591. status = dma_fence_get_status_locked(fence);
  592. else
  593. status = dma_fence_get_status(fence);
  594. /* convert fence status to synx state */
  595. switch (status) {
  596. case 0:
  597. state = SYNX_STATE_ACTIVE;
  598. break;
  599. case 1:
  600. state = SYNX_STATE_SIGNALED_SUCCESS;
  601. break;
  602. case -SYNX_STATE_SIGNALED_CANCEL:
  603. state = SYNX_STATE_SIGNALED_CANCEL;
  604. break;
  605. case -SYNX_STATE_SIGNALED_EXTERNAL:
  606. state = SYNX_STATE_SIGNALED_EXTERNAL;
  607. break;
  608. case -SYNX_STATE_SIGNALED_ERROR:
  609. state = SYNX_STATE_SIGNALED_ERROR;
  610. break;
  611. default:
  612. state = (u32)(-status);
  613. }
  614. return state;
  615. }
  616. static u32 __fence_group_state(struct dma_fence *fence, bool locked)
  617. {
  618. u32 i = 0;
  619. u32 state = SYNX_STATE_INVALID;
  620. struct dma_fence_array *array = NULL;
  621. u32 intr, actv_cnt, sig_cnt, err_cnt;
  622. if (IS_ERR_OR_NULL(fence)) {
  623. dprintk(SYNX_ERR, "invalid fence\n");
  624. return SYNX_STATE_INVALID;
  625. }
  626. actv_cnt = sig_cnt = err_cnt = 0;
  627. array = to_dma_fence_array(fence);
  628. if (IS_ERR_OR_NULL(array))
  629. return SYNX_STATE_INVALID;
  630. for (i = 0; i < array->num_fences; i++) {
  631. intr = __fence_state(array->fences[i], locked);
  632. switch (intr) {
  633. case SYNX_STATE_ACTIVE:
  634. actv_cnt++;
  635. break;
  636. case SYNX_STATE_SIGNALED_SUCCESS:
  637. sig_cnt++;
  638. break;
  639. default:
  640. err_cnt++;
  641. }
  642. }
  643. dprintk(SYNX_DBG,
  644. "group cnt stats act:%u, sig: %u, err: %u\n",
  645. actv_cnt, sig_cnt, err_cnt);
  646. if (err_cnt)
  647. state = SYNX_STATE_SIGNALED_ERROR;
  648. else if (actv_cnt)
  649. state = SYNX_STATE_ACTIVE;
  650. else if (sig_cnt == array->num_fences)
  651. state = SYNX_STATE_SIGNALED_SUCCESS;
  652. return state;
  653. }
  654. /*
  655. * WARN: Should not hold the fence spinlock when invoking
  656. * this function. Use synx_fence_state_locked instead
  657. */
  658. u32 synx_util_get_object_status(struct synx_coredata *synx_obj)
  659. {
  660. u32 state;
  661. if (IS_ERR_OR_NULL(synx_obj))
  662. return SYNX_STATE_INVALID;
  663. if (synx_util_is_merged_object(synx_obj))
  664. state = __fence_group_state(synx_obj->fence, false);
  665. else
  666. state = __fence_state(synx_obj->fence, false);
  667. return state;
  668. }
  669. /* use this for status check when holding on to metadata spinlock */
  670. u32 synx_util_get_object_status_locked(struct synx_coredata *synx_obj)
  671. {
  672. u32 state;
  673. if (IS_ERR_OR_NULL(synx_obj))
  674. return SYNX_STATE_INVALID;
  675. if (synx_util_is_merged_object(synx_obj))
  676. state = __fence_group_state(synx_obj->fence, true);
  677. else
  678. state = __fence_state(synx_obj->fence, true);
  679. return state;
  680. }
  681. struct synx_handle_coredata *synx_util_acquire_handle(
  682. struct synx_client *client, u32 h_synx)
  683. {
  684. struct synx_handle_coredata *synx_data = NULL;
  685. struct synx_handle_coredata *synx_handle =
  686. ERR_PTR(-SYNX_NOENT);
  687. if (IS_ERR_OR_NULL(client))
  688. return ERR_PTR(-SYNX_INVALID);
  689. spin_lock_bh(&client->handle_map_lock);
  690. hash_for_each_possible(client->handle_map,
  691. synx_data, node, h_synx) {
  692. if (synx_data->key == h_synx &&
  693. synx_data->rel_count != 0) {
  694. kref_get(&synx_data->refcount);
  695. synx_handle = synx_data;
  696. break;
  697. }
  698. }
  699. spin_unlock_bh(&client->handle_map_lock);
  700. return synx_handle;
  701. }
  702. struct synx_map_entry *synx_util_insert_to_map(
  703. struct synx_coredata *synx_obj,
  704. u32 h_synx, u32 flags)
  705. {
  706. struct synx_map_entry *map_entry;
  707. map_entry = kzalloc(sizeof(*map_entry), GFP_KERNEL);
  708. if (IS_ERR_OR_NULL(map_entry))
  709. return ERR_PTR(-SYNX_NOMEM);
  710. kref_init(&map_entry->refcount);
  711. map_entry->synx_obj = synx_obj;
  712. map_entry->flags = flags;
  713. map_entry->key = h_synx;
  714. if (synx_util_is_global_handle(h_synx)) {
  715. spin_lock_bh(&synx_dev->native->global_map_lock);
  716. hash_add(synx_dev->native->global_map,
  717. &map_entry->node, h_synx);
  718. spin_unlock_bh(&synx_dev->native->global_map_lock);
  719. dprintk(SYNX_MEM,
  720. "added handle %u to global map %pK\n",
  721. h_synx, map_entry);
  722. } else {
  723. spin_lock_bh(&synx_dev->native->local_map_lock);
  724. hash_add(synx_dev->native->local_map,
  725. &map_entry->node, h_synx);
  726. spin_unlock_bh(&synx_dev->native->local_map_lock);
  727. dprintk(SYNX_MEM,
  728. "added handle %u to local map %pK\n",
  729. h_synx, map_entry);
  730. }
  731. return map_entry;
  732. }
  733. struct synx_map_entry *synx_util_get_map_entry(u32 h_synx)
  734. {
  735. struct synx_map_entry *curr;
  736. struct synx_map_entry *map_entry = ERR_PTR(-SYNX_NOENT);
  737. if (h_synx == 0)
  738. return ERR_PTR(-SYNX_INVALID);
  739. if (synx_util_is_global_handle(h_synx)) {
  740. spin_lock_bh(&synx_dev->native->global_map_lock);
  741. hash_for_each_possible(synx_dev->native->global_map,
  742. curr, node, h_synx) {
  743. if (curr->key == h_synx) {
  744. kref_get(&curr->refcount);
  745. map_entry = curr;
  746. break;
  747. }
  748. }
  749. spin_unlock_bh(&synx_dev->native->global_map_lock);
  750. } else {
  751. spin_lock_bh(&synx_dev->native->local_map_lock);
  752. hash_for_each_possible(synx_dev->native->local_map,
  753. curr, node, h_synx) {
  754. if (curr->key == h_synx) {
  755. kref_get(&curr->refcount);
  756. map_entry = curr;
  757. break;
  758. }
  759. }
  760. spin_unlock_bh(&synx_dev->native->local_map_lock);
  761. }
  762. /* should we allocate if entry not found? */
  763. return map_entry;
  764. }
  765. static void synx_util_cleanup_fence(
  766. struct synx_coredata *synx_obj)
  767. {
  768. struct synx_signal_cb *signal_cb;
  769. unsigned long flags;
  770. u32 g_status;
  771. u32 f_status;
  772. u32 h_synx = 0;
  773. mutex_lock(&synx_obj->obj_lock);
  774. synx_obj->map_count--;
  775. signal_cb = synx_obj->signal_cb;
  776. f_status = synx_util_get_object_status(synx_obj);
  777. dprintk(SYNX_VERB, "f_status:%u, signal_cb:%p, map:%u, idx:%u\n",
  778. f_status, signal_cb, synx_obj->map_count, synx_obj->global_idx);
  779. if (synx_obj->map_count == 0 &&
  780. (signal_cb != NULL) &&
  781. (synx_obj->global_idx != 0) &&
  782. (f_status == SYNX_STATE_ACTIVE)) {
  783. /*
  784. * no more clients interested for notification
  785. * on handle on local core.
  786. * remove reference held by callback on synx
  787. * coredata structure and update cb (if still
  788. * un-signaled) with global handle idx to
  789. * notify any cross-core clients waiting on
  790. * handle.
  791. */
  792. g_status = synx_global_get_status(synx_obj->global_idx);
  793. if (g_status > SYNX_STATE_ACTIVE) {
  794. dprintk(SYNX_DBG, "signaling fence %pK with status %u\n",
  795. synx_obj->fence, g_status);
  796. synx_native_signal_fence(synx_obj, g_status);
  797. } else {
  798. spin_lock_irqsave(synx_obj->fence->lock, flags);
  799. if (synx_util_get_object_status_locked(synx_obj) ==
  800. SYNX_STATE_ACTIVE) {
  801. signal_cb->synx_obj = NULL;
  802. synx_global_fetch_handle_details(synx_obj->global_idx, &h_synx);
  803. signal_cb->handle = h_synx;
  804. synx_obj->signal_cb = NULL;
  805. /*
  806. * release reference held by signal cb and
  807. * get reference on global index instead.
  808. */
  809. synx_util_put_object(synx_obj);
  810. synx_global_get_ref(synx_obj->global_idx);
  811. }
  812. spin_unlock_irqrestore(synx_obj->fence->lock, flags);
  813. }
  814. } else if (synx_obj->map_count == 0 && signal_cb &&
  815. (f_status == SYNX_STATE_ACTIVE)) {
  816. if (dma_fence_remove_callback(synx_obj->fence,
  817. &signal_cb->fence_cb)) {
  818. kfree(signal_cb);
  819. synx_obj->signal_cb = NULL;
  820. /*
  821. * release reference held by signal cb and
  822. * get reference on global index instead.
  823. */
  824. synx_util_put_object(synx_obj);
  825. dprintk(SYNX_MEM, "signal cb destroyed %pK\n",
  826. synx_obj->signal_cb);
  827. }
  828. }
  829. mutex_unlock(&synx_obj->obj_lock);
  830. }
  831. static void synx_util_destroy_map_entry_worker(
  832. struct work_struct *dispatch)
  833. {
  834. struct synx_map_entry *map_entry =
  835. container_of(dispatch, struct synx_map_entry, dispatch);
  836. struct synx_coredata *synx_obj;
  837. synx_obj = map_entry->synx_obj;
  838. if (!IS_ERR_OR_NULL(synx_obj)) {
  839. synx_util_cleanup_fence(synx_obj);
  840. /* release reference held by map entry */
  841. synx_util_put_object(synx_obj);
  842. }
  843. if (!synx_util_is_global_handle(map_entry->key))
  844. clear_bit(synx_util_global_idx(map_entry->key),
  845. synx_dev->native->bitmap);
  846. dprintk(SYNX_VERB, "map entry for %u destroyed %pK\n",
  847. map_entry->key, map_entry);
  848. kfree(map_entry);
  849. }
  850. static void synx_util_destroy_map_entry(struct kref *kref)
  851. {
  852. struct synx_map_entry *map_entry =
  853. container_of(kref, struct synx_map_entry, refcount);
  854. hash_del(&map_entry->node);
  855. dprintk(SYNX_MEM, "map entry for %u removed %pK\n",
  856. map_entry->key, map_entry);
  857. INIT_WORK(&map_entry->dispatch, synx_util_destroy_map_entry_worker);
  858. queue_work(synx_dev->wq_cleanup, &map_entry->dispatch);
  859. }
  860. void synx_util_release_map_entry(struct synx_map_entry *map_entry)
  861. {
  862. spinlock_t *lock;
  863. if (IS_ERR_OR_NULL(map_entry))
  864. return;
  865. if (synx_util_is_global_handle(map_entry->key))
  866. lock = &synx_dev->native->global_map_lock;
  867. else
  868. lock = &synx_dev->native->local_map_lock;
  869. spin_lock_bh(lock);
  870. kref_put(&map_entry->refcount,
  871. synx_util_destroy_map_entry);
  872. spin_unlock_bh(lock);
  873. }
  874. static void synx_util_destroy_handle_worker(
  875. struct work_struct *dispatch)
  876. {
  877. struct synx_handle_coredata *synx_data =
  878. container_of(dispatch, struct synx_handle_coredata,
  879. dispatch);
  880. synx_util_release_map_entry(synx_data->map_entry);
  881. dprintk(SYNX_VERB, "handle %u destroyed %pK\n",
  882. synx_data->key, synx_data);
  883. kfree(synx_data);
  884. }
  885. static void synx_util_destroy_handle(struct kref *kref)
  886. {
  887. struct synx_handle_coredata *synx_data =
  888. container_of(kref, struct synx_handle_coredata,
  889. refcount);
  890. hash_del(&synx_data->node);
  891. dprintk(SYNX_MEM, "[sess :%llu] handle %u removed %pK\n",
  892. synx_data->client->id, synx_data->key, synx_data);
  893. INIT_WORK(&synx_data->dispatch, synx_util_destroy_handle_worker);
  894. queue_work(synx_dev->wq_cleanup, &synx_data->dispatch);
  895. }
  896. void synx_util_release_handle(struct synx_handle_coredata *synx_data)
  897. {
  898. struct synx_client *client;
  899. if (IS_ERR_OR_NULL(synx_data))
  900. return;
  901. client = synx_data->client;
  902. if (IS_ERR_OR_NULL(client))
  903. return;
  904. spin_lock_bh(&client->handle_map_lock);
  905. kref_put(&synx_data->refcount,
  906. synx_util_destroy_handle);
  907. spin_unlock_bh(&client->handle_map_lock);
  908. }
  909. struct bind_operations *synx_util_get_bind_ops(u32 type)
  910. {
  911. struct synx_registered_ops *client_ops;
  912. if (!synx_util_is_valid_bind_type(type))
  913. return NULL;
  914. mutex_lock(&synx_dev->vtbl_lock);
  915. client_ops = &synx_dev->bind_vtbl[type];
  916. if (!client_ops->valid) {
  917. mutex_unlock(&synx_dev->vtbl_lock);
  918. return NULL;
  919. }
  920. mutex_unlock(&synx_dev->vtbl_lock);
  921. return &client_ops->ops;
  922. }
  923. int synx_util_alloc_cb_entry(struct synx_client *client,
  924. struct synx_kernel_payload *data,
  925. u32 *cb_idx)
  926. {
  927. long idx;
  928. struct synx_client_cb *cb;
  929. if (IS_ERR_OR_NULL(client) || IS_ERR_OR_NULL(data) ||
  930. IS_ERR_OR_NULL(cb_idx))
  931. return -SYNX_INVALID;
  932. idx = synx_util_get_free_handle(client->cb_bitmap, SYNX_MAX_OBJS);
  933. if (idx >= SYNX_MAX_OBJS) {
  934. dprintk(SYNX_ERR,
  935. "[sess :%llu] free cb index not available\n",
  936. client->id);
  937. return -SYNX_NOMEM;
  938. }
  939. cb = &client->cb_table[idx];
  940. memset(cb, 0, sizeof(*cb));
  941. cb->is_valid = true;
  942. cb->client = client;
  943. cb->idx = idx;
  944. memcpy(&cb->kernel_cb, data,
  945. sizeof(cb->kernel_cb));
  946. *cb_idx = idx;
  947. dprintk(SYNX_VERB, "[sess :%llu] allocated cb index %u\n",
  948. client->id, *cb_idx);
  949. return 0;
  950. }
  951. int synx_util_clear_cb_entry(struct synx_client *client,
  952. struct synx_client_cb *cb)
  953. {
  954. int rc = 0;
  955. u32 idx;
  956. if (IS_ERR_OR_NULL(cb))
  957. return -SYNX_INVALID;
  958. idx = cb->idx;
  959. memset(cb, 0, sizeof(*cb));
  960. if (idx && idx < SYNX_MAX_OBJS) {
  961. clear_bit(idx, client->cb_bitmap);
  962. } else {
  963. dprintk(SYNX_ERR, "invalid index\n");
  964. rc = -SYNX_INVALID;
  965. }
  966. return rc;
  967. }
  968. void synx_util_default_user_callback(u32 h_synx,
  969. int status, void *data)
  970. {
  971. struct synx_client_cb *cb = data;
  972. struct synx_client *client = NULL;
  973. if (cb && cb->client) {
  974. client = cb->client;
  975. dprintk(SYNX_VERB,
  976. "[sess :%llu] user cb queued for handle %d\n",
  977. client->id, h_synx);
  978. cb->kernel_cb.status = status;
  979. mutex_lock(&client->event_q_lock);
  980. list_add_tail(&cb->node, &client->event_q);
  981. mutex_unlock(&client->event_q_lock);
  982. wake_up_all(&client->event_wq);
  983. } else {
  984. dprintk(SYNX_ERR, "invalid params\n");
  985. }
  986. }
  987. void synx_util_callback_dispatch(struct synx_coredata *synx_obj, u32 status)
  988. {
  989. struct synx_cb_data *synx_cb, *synx_cb_temp;
  990. if (IS_ERR_OR_NULL(synx_obj)) {
  991. dprintk(SYNX_ERR, "invalid arguments\n");
  992. return;
  993. }
  994. list_for_each_entry_safe(synx_cb,
  995. synx_cb_temp, &synx_obj->reg_cbs_list, node) {
  996. synx_cb->status = status;
  997. list_del_init(&synx_cb->node);
  998. queue_work(synx_dev->wq_cb,
  999. &synx_cb->cb_dispatch);
  1000. dprintk(SYNX_VERB, "dispatched callback\n");
  1001. }
  1002. }
  1003. void synx_util_cb_dispatch(struct work_struct *cb_dispatch)
  1004. {
  1005. struct synx_cb_data *synx_cb =
  1006. container_of(cb_dispatch, struct synx_cb_data, cb_dispatch);
  1007. struct synx_client *client;
  1008. struct synx_client_cb *cb;
  1009. struct synx_kernel_payload payload;
  1010. u32 status;
  1011. client = synx_get_client(synx_cb->session);
  1012. if (IS_ERR_OR_NULL(client)) {
  1013. dprintk(SYNX_ERR,
  1014. "invalid session data %pK in cb payload\n",
  1015. synx_cb->session);
  1016. goto free;
  1017. }
  1018. if (synx_cb->idx == 0 ||
  1019. synx_cb->idx >= SYNX_MAX_OBJS) {
  1020. dprintk(SYNX_ERR,
  1021. "[sess :%llu] invalid cb index %u\n",
  1022. client->id, synx_cb->idx);
  1023. goto fail;
  1024. }
  1025. status = synx_cb->status;
  1026. cb = &client->cb_table[synx_cb->idx];
  1027. if (!cb->is_valid) {
  1028. dprintk(SYNX_ERR, "invalid cb payload\n");
  1029. goto fail;
  1030. }
  1031. memcpy(&payload, &cb->kernel_cb, sizeof(cb->kernel_cb));
  1032. payload.status = status;
  1033. if (payload.cb_func == synx_util_default_user_callback) {
  1034. /*
  1035. * need to send client cb data for default
  1036. * user cb (userspace cb)
  1037. */
  1038. payload.data = cb;
  1039. } else {
  1040. /*
  1041. * clear the cb entry. userspace cb entry
  1042. * will be cleared after data read by the
  1043. * polling thread or when client is destroyed
  1044. */
  1045. if (synx_util_clear_cb_entry(client, cb))
  1046. dprintk(SYNX_ERR,
  1047. "[sess :%llu] error clearing cb entry\n",
  1048. client->id);
  1049. }
  1050. dprintk(SYNX_DBG,
  1051. "callback dispatched for handle %u, status %u, data %pK\n",
  1052. payload.h_synx, payload.status, payload.data);
  1053. /* dispatch kernel callback */
  1054. payload.cb_func(payload.h_synx,
  1055. payload.status, payload.data);
  1056. fail:
  1057. synx_put_client(client);
  1058. free:
  1059. kfree(synx_cb);
  1060. }
  1061. int synx_get_child_coredata(struct synx_coredata *synx_obj, struct synx_coredata ***child_synx_obj, int *num_fences)
  1062. {
  1063. int rc = SYNX_SUCCESS;
  1064. int i = 0, handle_count = 0;
  1065. u32 h_child = 0;
  1066. struct dma_fence_array *array = NULL;
  1067. struct synx_coredata **synx_datas = NULL;
  1068. struct synx_map_entry *fence_entry = NULL;
  1069. if (IS_ERR_OR_NULL(synx_obj) || IS_ERR_OR_NULL(num_fences))
  1070. return -SYNX_INVALID;
  1071. if (dma_fence_is_array(synx_obj->fence)) {
  1072. array = to_dma_fence_array(synx_obj->fence);
  1073. if (IS_ERR_OR_NULL(array))
  1074. return -SYNX_INVALID;
  1075. synx_datas = kcalloc(array->num_fences, sizeof(*synx_datas), GFP_KERNEL);
  1076. if (IS_ERR_OR_NULL(synx_datas))
  1077. return -SYNX_NOMEM;
  1078. for (i = 0; i < array->num_fences; i++) {
  1079. h_child = synx_util_get_fence_entry((u64)array->fences[i], 1);
  1080. fence_entry = synx_util_get_map_entry(h_child);
  1081. if (IS_ERR_OR_NULL(fence_entry) || IS_ERR_OR_NULL(fence_entry->synx_obj))
  1082. {
  1083. dprintk(SYNX_ERR, "Invalid handle access %u", h_child);
  1084. rc = -SYNX_NOENT;
  1085. goto fail;
  1086. }
  1087. synx_datas[handle_count++] = fence_entry->synx_obj;
  1088. synx_util_release_map_entry(fence_entry);
  1089. }
  1090. }
  1091. *child_synx_obj = synx_datas;
  1092. *num_fences = handle_count;
  1093. return rc;
  1094. fail:
  1095. kfree(synx_datas);
  1096. return rc;
  1097. }
  1098. u32 synx_util_get_fence_entry(u64 key, u32 global)
  1099. {
  1100. u32 h_synx = 0;
  1101. struct synx_fence_entry *curr;
  1102. spin_lock_bh(&synx_dev->native->fence_map_lock);
  1103. hash_for_each_possible(synx_dev->native->fence_map,
  1104. curr, node, key) {
  1105. if (curr->key == key) {
  1106. if (global)
  1107. h_synx = curr->g_handle;
  1108. /* return local handle if global not available */
  1109. if (h_synx == 0)
  1110. h_synx = curr->l_handle;
  1111. break;
  1112. }
  1113. }
  1114. spin_unlock_bh(&synx_dev->native->fence_map_lock);
  1115. return h_synx;
  1116. }
  1117. void synx_util_release_fence_entry(u64 key)
  1118. {
  1119. struct synx_fence_entry *entry = NULL, *curr;
  1120. spin_lock_bh(&synx_dev->native->fence_map_lock);
  1121. hash_for_each_possible(synx_dev->native->fence_map,
  1122. curr, node, key) {
  1123. if (curr->key == key) {
  1124. entry = curr;
  1125. break;
  1126. }
  1127. }
  1128. if (entry) {
  1129. hash_del(&entry->node);
  1130. dprintk(SYNX_MEM,
  1131. "released fence entry %pK for fence %pK\n",
  1132. entry, (void *)key);
  1133. kfree(entry);
  1134. }
  1135. spin_unlock_bh(&synx_dev->native->fence_map_lock);
  1136. }
  1137. int synx_util_insert_fence_entry(struct synx_fence_entry *entry,
  1138. u32 *h_synx, u32 global)
  1139. {
  1140. int rc = SYNX_SUCCESS;
  1141. struct synx_fence_entry *curr;
  1142. if (IS_ERR_OR_NULL(entry) || IS_ERR_OR_NULL(h_synx))
  1143. return -SYNX_INVALID;
  1144. spin_lock_bh(&synx_dev->native->fence_map_lock);
  1145. hash_for_each_possible(synx_dev->native->fence_map,
  1146. curr, node, entry->key) {
  1147. /* raced with import from another process on same fence */
  1148. if (curr->key == entry->key) {
  1149. if (global)
  1150. *h_synx = curr->g_handle;
  1151. if (*h_synx == 0 || !global)
  1152. *h_synx = curr->l_handle;
  1153. rc = -SYNX_ALREADY;
  1154. break;
  1155. }
  1156. }
  1157. /* add entry only if its not present in the map */
  1158. if (rc == SYNX_SUCCESS) {
  1159. hash_add(synx_dev->native->fence_map,
  1160. &entry->node, entry->key);
  1161. dprintk(SYNX_MEM,
  1162. "added fence entry %pK for fence %pK\n",
  1163. entry, (void *)entry->key);
  1164. }
  1165. spin_unlock_bh(&synx_dev->native->fence_map_lock);
  1166. return rc;
  1167. }
  1168. struct synx_client *synx_get_client(struct synx_session *session)
  1169. {
  1170. struct synx_client *client = NULL;
  1171. struct synx_client *curr;
  1172. if (IS_ERR_OR_NULL(session))
  1173. return ERR_PTR(-SYNX_INVALID);
  1174. spin_lock_bh(&synx_dev->native->metadata_map_lock);
  1175. hash_for_each_possible(synx_dev->native->client_metadata_map,
  1176. curr, node, (u64)session) {
  1177. if (curr == (struct synx_client *)session) {
  1178. if (curr->active) {
  1179. kref_get(&curr->refcount);
  1180. client = curr;
  1181. }
  1182. break;
  1183. }
  1184. }
  1185. spin_unlock_bh(&synx_dev->native->metadata_map_lock);
  1186. return client;
  1187. }
  1188. static void synx_client_cleanup(struct work_struct *dispatch)
  1189. {
  1190. int i, j;
  1191. struct synx_client *client =
  1192. container_of(dispatch, struct synx_client, dispatch);
  1193. struct synx_handle_coredata *curr;
  1194. struct hlist_node *tmp;
  1195. dprintk(SYNX_INFO, "[sess :%llu] session removed %s\n",
  1196. client->id, client->name);
  1197. /*
  1198. * go over all the remaining synx obj handles
  1199. * un-released from this session and remove them.
  1200. */
  1201. hash_for_each_safe(client->handle_map, i, tmp, curr, node) {
  1202. dprintk(SYNX_WARN,
  1203. "[sess :%llu] un-released handle %u\n",
  1204. client->id, curr->key);
  1205. j = kref_read(&curr->refcount);
  1206. /* release pending reference */
  1207. while (j--)
  1208. kref_put(&curr->refcount, synx_util_destroy_handle);
  1209. }
  1210. mutex_destroy(&client->event_q_lock);
  1211. dprintk(SYNX_VERB, "session %llu [%s] destroyed %pK\n",
  1212. client->id, client->name, client);
  1213. vfree(client);
  1214. }
  1215. static void synx_client_destroy(struct kref *kref)
  1216. {
  1217. struct synx_client *client =
  1218. container_of(kref, struct synx_client, refcount);
  1219. hash_del(&client->node);
  1220. INIT_WORK(&client->dispatch, synx_client_cleanup);
  1221. queue_work(synx_dev->wq_cleanup, &client->dispatch);
  1222. }
  1223. void synx_put_client(struct synx_client *client)
  1224. {
  1225. if (IS_ERR_OR_NULL(client))
  1226. return;
  1227. spin_lock_bh(&synx_dev->native->metadata_map_lock);
  1228. kref_put(&client->refcount, synx_client_destroy);
  1229. spin_unlock_bh(&synx_dev->native->metadata_map_lock);
  1230. }
  1231. void synx_util_generate_timestamp(char *timestamp, size_t size)
  1232. {
  1233. struct timespec64 tv;
  1234. struct tm tm;
  1235. ktime_get_real_ts64(&tv);
  1236. time64_to_tm(tv.tv_sec, 0, &tm);
  1237. snprintf(timestamp, size, "%02d-%02d %02d:%02d:%02d",
  1238. tm.tm_mon + 1, tm.tm_mday, tm.tm_hour,
  1239. tm.tm_min, tm.tm_sec);
  1240. }
  1241. void synx_util_log_error(u32 client_id, u32 h_synx, s32 err)
  1242. {
  1243. struct error_node *err_node;
  1244. if (!synx_dev->debugfs_root)
  1245. return;
  1246. err_node = kzalloc(sizeof(*err_node), GFP_KERNEL);
  1247. if (!err_node)
  1248. return;
  1249. err_node->client_id = client_id;
  1250. err_node->error_code = err;
  1251. err_node->h_synx = h_synx;
  1252. synx_util_generate_timestamp(err_node->timestamp,
  1253. sizeof(err_node->timestamp));
  1254. mutex_lock(&synx_dev->error_lock);
  1255. list_add(&err_node->node,
  1256. &synx_dev->error_list);
  1257. mutex_unlock(&synx_dev->error_lock);
  1258. }
  1259. int synx_util_save_data(void *fence, u32 flags,
  1260. u32 h_synx)
  1261. {
  1262. int rc = SYNX_SUCCESS;
  1263. struct synx_entry_64 *entry, *curr;
  1264. u64 key;
  1265. u32 tbl = synx_util_map_params_to_type(flags);
  1266. switch (tbl) {
  1267. case SYNX_TYPE_CSL:
  1268. key = *(u32 *)fence;
  1269. spin_lock_bh(&synx_dev->native->csl_map_lock);
  1270. /* ensure fence is not already added to map */
  1271. hash_for_each_possible(synx_dev->native->csl_fence_map,
  1272. curr, node, key) {
  1273. if (curr->key == key) {
  1274. rc = -SYNX_ALREADY;
  1275. break;
  1276. }
  1277. }
  1278. if (rc == SYNX_SUCCESS) {
  1279. entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
  1280. if (entry) {
  1281. entry->data[0] = h_synx;
  1282. entry->key = key;
  1283. kref_init(&entry->refcount);
  1284. hash_add(synx_dev->native->csl_fence_map,
  1285. &entry->node, entry->key);
  1286. dprintk(SYNX_MEM, "added csl fence %d to map %pK\n",
  1287. entry->key, entry);
  1288. } else {
  1289. rc = -SYNX_NOMEM;
  1290. }
  1291. }
  1292. spin_unlock_bh(&synx_dev->native->csl_map_lock);
  1293. break;
  1294. default:
  1295. dprintk(SYNX_ERR, "invalid hash table selection\n");
  1296. kfree(entry);
  1297. rc = -SYNX_INVALID;
  1298. }
  1299. return rc;
  1300. }
  1301. struct synx_entry_64 *synx_util_retrieve_data(void *fence,
  1302. u32 type)
  1303. {
  1304. u64 key;
  1305. struct synx_entry_64 *entry = NULL;
  1306. struct synx_entry_64 *curr;
  1307. switch (type) {
  1308. case SYNX_TYPE_CSL:
  1309. key = *(u32 *)fence;
  1310. spin_lock_bh(&synx_dev->native->csl_map_lock);
  1311. hash_for_each_possible(synx_dev->native->csl_fence_map,
  1312. curr, node, key) {
  1313. if (curr->key == key) {
  1314. kref_get(&curr->refcount);
  1315. entry = curr;
  1316. break;
  1317. }
  1318. }
  1319. spin_unlock_bh(&synx_dev->native->csl_map_lock);
  1320. break;
  1321. default:
  1322. dprintk(SYNX_ERR, "invalid hash table selection %u\n",
  1323. type);
  1324. }
  1325. return entry;
  1326. }
  1327. static void synx_util_destroy_data(struct kref *kref)
  1328. {
  1329. struct synx_entry_64 *entry =
  1330. container_of(kref, struct synx_entry_64, refcount);
  1331. hash_del(&entry->node);
  1332. dprintk(SYNX_MEM, "released fence %llu entry %pK\n",
  1333. entry->key, entry);
  1334. kfree(entry);
  1335. }
  1336. void synx_util_remove_data(void *fence,
  1337. u32 type)
  1338. {
  1339. u64 key;
  1340. struct synx_entry_64 *entry = NULL;
  1341. struct synx_entry_64 *curr;
  1342. if (IS_ERR_OR_NULL(fence))
  1343. return;
  1344. switch (type) {
  1345. case SYNX_TYPE_CSL:
  1346. key = *((u32 *)fence);
  1347. spin_lock_bh(&synx_dev->native->csl_map_lock);
  1348. hash_for_each_possible(synx_dev->native->csl_fence_map,
  1349. curr, node, key) {
  1350. if (curr->key == key) {
  1351. entry = curr;
  1352. break;
  1353. }
  1354. }
  1355. if (entry)
  1356. kref_put(&entry->refcount, synx_util_destroy_data);
  1357. spin_unlock_bh(&synx_dev->native->csl_map_lock);
  1358. break;
  1359. default:
  1360. dprintk(SYNX_ERR, "invalid hash table selection %u\n",
  1361. type);
  1362. }
  1363. }
  1364. void synx_util_map_import_params_to_create(
  1365. struct synx_import_indv_params *params,
  1366. struct synx_create_params *c_params)
  1367. {
  1368. if (IS_ERR_OR_NULL(params) || IS_ERR_OR_NULL(c_params))
  1369. return;
  1370. if (params->flags & SYNX_IMPORT_GLOBAL_FENCE)
  1371. c_params->flags |= SYNX_CREATE_GLOBAL_FENCE;
  1372. if (params->flags & SYNX_IMPORT_LOCAL_FENCE)
  1373. c_params->flags |= SYNX_CREATE_LOCAL_FENCE;
  1374. if (params->flags & SYNX_IMPORT_DMA_FENCE)
  1375. c_params->flags |= SYNX_CREATE_DMA_FENCE;
  1376. }
  1377. u32 synx_util_map_client_id_to_core(
  1378. enum synx_client_id id)
  1379. {
  1380. u32 core_id;
  1381. switch (id) {
  1382. case SYNX_CLIENT_NATIVE:
  1383. core_id = SYNX_CORE_APSS; break;
  1384. case SYNX_CLIENT_ICP_CTX0:
  1385. core_id = SYNX_CORE_ICP; break;
  1386. case SYNX_CLIENT_EVA_CTX0:
  1387. core_id = SYNX_CORE_EVA; break;
  1388. case SYNX_CLIENT_VID_CTX0:
  1389. core_id = SYNX_CORE_IRIS; break;
  1390. case SYNX_CLIENT_NSP_CTX0:
  1391. core_id = SYNX_CORE_NSP; break;
  1392. default:
  1393. core_id = SYNX_CORE_MAX;
  1394. }
  1395. return core_id;
  1396. }