cam_sync_dma_fence.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  4. */
  5. #include "cam_sync_dma_fence.h"
  6. /**
  7. * struct cam_dma_fence_row - DMA fence row
  8. */
  9. struct cam_dma_fence_row {
  10. char name[CAM_DMA_FENCE_NAME_LEN];
  11. struct dma_fence *fence;
  12. int32_t fd;
  13. enum cam_dma_fence_state state;
  14. struct dma_fence_cb fence_cb;
  15. int32_t sync_obj;
  16. cam_sync_callback_for_dma_fence sync_cb;
  17. bool cb_registered_for_sync;
  18. bool ext_dma_fence;
  19. bool sync_signal_dma;
  20. };
  21. /**
  22. * struct cam_dma_fence_device - DMA fence device
  23. */
  24. struct cam_dma_fence_device {
  25. uint64_t dma_fence_context;
  26. struct cam_dma_fence_row rows[CAM_DMA_FENCE_MAX_FENCES];
  27. spinlock_t row_spinlocks[CAM_DMA_FENCE_MAX_FENCES];
  28. struct mutex dev_lock;
  29. DECLARE_BITMAP(bitmap, CAM_DMA_FENCE_MAX_FENCES);
  30. };
  31. static atomic64_t g_cam_dma_fence_seq_no;
  32. static struct cam_dma_fence_device *g_cam_dma_fence_dev;
  33. bool __cam_dma_fence_enable_signaling(
  34. struct dma_fence *fence)
  35. {
  36. return true;
  37. }
  38. const char *__cam_dma_fence_get_driver_name(
  39. struct dma_fence *fence)
  40. {
  41. return "Camera DMA fence driver";
  42. }
  43. void __cam_dma_fence_free(struct dma_fence *fence)
  44. {
  45. CAM_DBG(CAM_DMA_FENCE,
  46. "Free memory for dma fence seqno: %llu", fence->seqno);
  47. kfree(fence->lock);
  48. kfree(fence);
  49. }
  50. static struct dma_fence_ops cam_sync_dma_fence_ops = {
  51. .enable_signaling = __cam_dma_fence_enable_signaling,
  52. .get_driver_name = __cam_dma_fence_get_driver_name,
  53. .get_timeline_name = __cam_dma_fence_get_driver_name,
  54. .release = __cam_dma_fence_free,
  55. };
  56. static void __cam_dma_fence_print_table(void)
  57. {
  58. int i;
  59. struct cam_dma_fence_row *row;
  60. struct dma_fence *fence;
  61. for (i = 0; i < CAM_DMA_FENCE_MAX_FENCES; i++) {
  62. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  63. row = &g_cam_dma_fence_dev->rows[i];
  64. fence = row->fence;
  65. CAM_INFO(CAM_DMA_FENCE,
  66. "Idx: %d seqno: %llu name: %s state: %d",
  67. i, fence->seqno, row->name, row->state);
  68. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  69. }
  70. }
  71. static int __cam_dma_fence_find_free_idx(uint32_t *idx)
  72. {
  73. int rc = 0;
  74. *idx = find_first_zero_bit(g_cam_dma_fence_dev->bitmap, CAM_DMA_FENCE_MAX_FENCES);
  75. if (*idx < CAM_DMA_FENCE_MAX_FENCES)
  76. set_bit(*idx, g_cam_dma_fence_dev->bitmap);
  77. else
  78. rc = -ENOMEM;
  79. if (rc) {
  80. CAM_ERR(CAM_DMA_FENCE, "No free idx, printing dma fence table......");
  81. __cam_dma_fence_print_table();
  82. }
  83. return rc;
  84. }
  85. static struct dma_fence *__cam_dma_fence_find_fence_in_table(
  86. int32_t fd, int32_t *idx)
  87. {
  88. int i;
  89. struct dma_fence *fence = NULL;
  90. struct cam_dma_fence_row *row = NULL;
  91. for (i = 0; i < CAM_DMA_FENCE_MAX_FENCES; i++) {
  92. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  93. row = &g_cam_dma_fence_dev->rows[i];
  94. if ((row->state != CAM_DMA_FENCE_STATE_INVALID) && (row->fd == fd)) {
  95. *idx = i;
  96. fence = row->fence;
  97. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  98. break;
  99. }
  100. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  101. }
  102. return fence;
  103. }
  104. static void __cam_dma_fence_init_row(const char *name,
  105. struct dma_fence *dma_fence, int32_t fd, uint32_t idx,
  106. bool ext_dma_fence)
  107. {
  108. struct cam_dma_fence_row *row;
  109. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[idx]);
  110. row = &g_cam_dma_fence_dev->rows[idx];
  111. memset(row, 0, sizeof(*row));
  112. row->fence = dma_fence;
  113. row->fd = fd;
  114. row->state = CAM_DMA_FENCE_STATE_ACTIVE;
  115. row->ext_dma_fence = ext_dma_fence;
  116. strscpy(row->name, name, CAM_DMA_FENCE_NAME_LEN);
  117. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[idx]);
  118. }
  119. void __cam_dma_fence_signal_cb(
  120. struct dma_fence *fence, struct dma_fence_cb *cb)
  121. {
  122. struct cam_dma_fence_signal_sync_obj signal_sync_obj;
  123. struct cam_dma_fence_row *dma_fence_row =
  124. container_of(cb, struct cam_dma_fence_row, fence_cb);
  125. if (dma_fence_row->state == CAM_DMA_FENCE_STATE_INVALID) {
  126. CAM_ERR(CAM_DMA_FENCE, "dma fence seqno: %llu is in invalid state: %d",
  127. fence->seqno, dma_fence_row->state);
  128. return;
  129. }
  130. /* If this dma fence is signaled by sync obj, skip cb */
  131. if (dma_fence_row->sync_signal_dma)
  132. return;
  133. CAM_DBG(CAM_DMA_FENCE, "dma fence seqno: %llu fd: %d signaled, signal sync obj: %d",
  134. fence->seqno, dma_fence_row->fd, dma_fence_row->sync_obj);
  135. if ((dma_fence_row->cb_registered_for_sync) && (dma_fence_row->sync_cb)) {
  136. signal_sync_obj.fd = dma_fence_row->fd;
  137. /*
  138. * Signal is invoked with the fence lock held,
  139. * lock not needed to query status
  140. */
  141. signal_sync_obj.status = dma_fence_get_status_locked(fence);
  142. dma_fence_row->state = CAM_DMA_FENCE_STATE_SIGNALED;
  143. dma_fence_row->sync_cb(dma_fence_row->sync_obj, &signal_sync_obj);
  144. }
  145. }
  146. int cam_dma_fence_get_put_ref(
  147. bool get_or_put, int32_t dma_fence_row_idx)
  148. {
  149. struct dma_fence *dma_fence;
  150. struct cam_dma_fence_row *row;
  151. if ((dma_fence_row_idx < 0) ||
  152. (dma_fence_row_idx >= CAM_DMA_FENCE_MAX_FENCES)) {
  153. CAM_ERR(CAM_DMA_FENCE, "dma fence idx: %d is invalid",
  154. dma_fence_row_idx);
  155. return -EINVAL;
  156. }
  157. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  158. row = &g_cam_dma_fence_dev->rows[dma_fence_row_idx];
  159. if (row->state == CAM_DMA_FENCE_STATE_INVALID) {
  160. CAM_ERR(CAM_DMA_FENCE,
  161. "dma fence at idx: %d is in invalid state: %d",
  162. dma_fence_row_idx, row->state);
  163. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  164. return -EINVAL;
  165. }
  166. dma_fence = row->fence;
  167. if (get_or_put)
  168. dma_fence_get(dma_fence);
  169. else
  170. dma_fence_put(dma_fence);
  171. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  172. CAM_DBG(CAM_DMA_FENCE, "Refcnt: %u after %s for dma fence with seqno: %llu",
  173. kref_read(&dma_fence->refcount), (get_or_put ? "getref" : "putref"),
  174. dma_fence->seqno);
  175. return 0;
  176. }
  177. static struct dma_fence *cam_dma_fence_get_fence_from_sync_file(
  178. int32_t fd, int32_t *dma_fence_row_idx)
  179. {
  180. uint32_t idx;
  181. struct dma_fence *dma_fence = NULL;
  182. dma_fence = sync_file_get_fence(fd);
  183. if (IS_ERR_OR_NULL(dma_fence)) {
  184. CAM_ERR(CAM_DMA_FENCE, "Invalid fd: %d no dma fence found", fd);
  185. return ERR_PTR(-EINVAL);
  186. }
  187. if (__cam_dma_fence_find_free_idx(&idx)) {
  188. CAM_ERR(CAM_DMA_FENCE, "No free idx");
  189. goto end;
  190. }
  191. __cam_dma_fence_init_row(dma_fence->ops->get_driver_name(dma_fence),
  192. dma_fence, fd, idx, true);
  193. *dma_fence_row_idx = idx;
  194. CAM_DBG(CAM_DMA_FENCE,
  195. "External dma fence with fd: %d seqno: %llu ref_cnt: %u updated in tbl",
  196. fd, dma_fence->seqno, kref_read(&dma_fence->refcount));
  197. return dma_fence;
  198. end:
  199. dma_fence_put(dma_fence);
  200. return NULL;
  201. }
  202. struct dma_fence *cam_dma_fence_get_fence_from_fd(
  203. int32_t fd, int32_t *dma_fence_row_idx)
  204. {
  205. struct dma_fence *dma_fence = NULL;
  206. dma_fence = __cam_dma_fence_find_fence_in_table(fd, dma_fence_row_idx);
  207. if (IS_ERR_OR_NULL(dma_fence)) {
  208. CAM_WARN(CAM_DMA_FENCE,
  209. "dma fence with fd: %d is an external fence, querying sync file",
  210. fd);
  211. return cam_dma_fence_get_fence_from_sync_file(fd, dma_fence_row_idx);
  212. }
  213. dma_fence_get(dma_fence);
  214. CAM_DBG(CAM_DMA_FENCE, "dma fence found for fd: %d with seqno: %llu ref_cnt: %u",
  215. fd, dma_fence->seqno, kref_read(&dma_fence->refcount));
  216. return dma_fence;
  217. }
  218. int cam_dma_fence_register_cb(int32_t *sync_obj, int32_t *dma_fence_idx,
  219. cam_sync_callback_for_dma_fence sync_cb)
  220. {
  221. int rc = 0;
  222. int dma_fence_row_idx = 0;
  223. struct cam_dma_fence_row *row = NULL;
  224. struct dma_fence *dma_fence = NULL;
  225. if (!sync_obj || !dma_fence_idx || !sync_cb) {
  226. CAM_ERR(CAM_DMA_FENCE,
  227. "Invalid args sync_obj: %p dma_fence_idx: %p sync_cb: %p",
  228. sync_obj, dma_fence_idx, sync_cb);
  229. return -EINVAL;
  230. }
  231. dma_fence_row_idx = *dma_fence_idx;
  232. if ((dma_fence_row_idx < 0) ||
  233. (dma_fence_row_idx >= CAM_DMA_FENCE_MAX_FENCES)) {
  234. CAM_ERR(CAM_DMA_FENCE, "dma fence idx: %d is invalid",
  235. dma_fence_row_idx);
  236. return -EINVAL;
  237. }
  238. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  239. row = &g_cam_dma_fence_dev->rows[dma_fence_row_idx];
  240. dma_fence = row->fence;
  241. if (row->state != CAM_DMA_FENCE_STATE_ACTIVE) {
  242. CAM_ERR(CAM_DMA_FENCE,
  243. "dma fence at idx: %d fd: %d seqno: %llu is not active, current state: %d",
  244. dma_fence_row_idx, row->fd, dma_fence->seqno, row->state);
  245. rc = -EINVAL;
  246. goto end;
  247. }
  248. /**
  249. * If the cb is already registered, return
  250. * If a fd is closed by userspace without releasing the dma fence, it is
  251. * possible that same fd is returned to a new fence.
  252. */
  253. if (row->cb_registered_for_sync) {
  254. CAM_WARN(CAM_DMA_FENCE,
  255. "dma fence at idx: %d fd: %d seqno: %llu has already registered a cb for sync: %d - same fd for 2 fences?",
  256. dma_fence_row_idx, row->fd, dma_fence->seqno, row->sync_obj);
  257. goto end;
  258. }
  259. rc = dma_fence_add_callback(row->fence, &row->fence_cb,
  260. __cam_dma_fence_signal_cb);
  261. if (rc) {
  262. CAM_ERR(CAM_DMA_FENCE,
  263. "Failed to register cb for dma fence fd: %d seqno: %llu rc: %d",
  264. row->fd, dma_fence->seqno, rc);
  265. goto end;
  266. }
  267. row->cb_registered_for_sync = true;
  268. row->sync_obj = *sync_obj;
  269. row->sync_cb = sync_cb;
  270. CAM_DBG(CAM_DMA_FENCE,
  271. "CB successfully registered for dma fence fd: %d seqno: %llu for sync_obj: %d",
  272. row->fd, dma_fence->seqno, *sync_obj);
  273. end:
  274. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  275. return rc;
  276. }
  277. static int __cam_dma_fence_signal_fence(
  278. struct dma_fence *dma_fence,
  279. int32_t status)
  280. {
  281. bool fence_signaled = false;
  282. fence_signaled = dma_fence_is_signaled(dma_fence);
  283. if (fence_signaled) {
  284. CAM_WARN(CAM_DMA_FENCE,
  285. "dma fence seqno: %llu is already signaled",
  286. dma_fence->seqno);
  287. return 0;
  288. }
  289. if (status)
  290. dma_fence_set_error(dma_fence, status);
  291. return dma_fence_signal(dma_fence);
  292. }
  293. int cam_dma_fence_internal_signal(
  294. int32_t dma_fence_row_idx,
  295. struct cam_dma_fence_signal *signal_dma_fence)
  296. {
  297. int rc = 0;
  298. struct dma_fence *dma_fence = NULL;
  299. struct cam_dma_fence_row *row = NULL;
  300. if ((dma_fence_row_idx < 0) ||
  301. (dma_fence_row_idx >= CAM_DMA_FENCE_MAX_FENCES)) {
  302. CAM_ERR(CAM_DMA_FENCE, "dma fence idx: %d is invalid",
  303. dma_fence_row_idx);
  304. return -EINVAL;
  305. }
  306. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  307. row = &g_cam_dma_fence_dev->rows[dma_fence_row_idx];
  308. /* Ensures sync obj cb is not invoked */
  309. row->sync_signal_dma = true;
  310. dma_fence = row->fence;
  311. if (IS_ERR_OR_NULL(dma_fence)) {
  312. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  313. CAM_ERR(CAM_DMA_FENCE, "DMA fence in row: %d is invalid",
  314. dma_fence_row_idx);
  315. return -EINVAL;
  316. }
  317. if (row->state == CAM_DMA_FENCE_STATE_SIGNALED) {
  318. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  319. CAM_WARN(CAM_DMA_FENCE,
  320. "dma fence fd: %d[seqno: %llu] already in signaled state",
  321. signal_dma_fence->dma_fence_fd, dma_fence->seqno);
  322. return 0;
  323. }
  324. rc = __cam_dma_fence_signal_fence(dma_fence, signal_dma_fence->status);
  325. if (rc)
  326. CAM_WARN(CAM_DMA_FENCE,
  327. "dma fence seqno: %llu fd: %d already signaled rc: %d",
  328. dma_fence->seqno, row->fd, rc);
  329. row->state = CAM_DMA_FENCE_STATE_SIGNALED;
  330. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_fence_row_idx]);
  331. CAM_DBG(CAM_DMA_FENCE,
  332. "dma fence fd: %d[seqno: %llu] signaled with status: %d rc: %d",
  333. signal_dma_fence->dma_fence_fd, dma_fence->seqno,
  334. signal_dma_fence->status, rc);
  335. return rc;
  336. }
  337. int cam_dma_fence_signal_fd(struct cam_dma_fence_signal *signal_dma_fence)
  338. {
  339. int rc = 0;
  340. uint32_t idx;
  341. struct dma_fence *dma_fence = NULL;
  342. struct cam_dma_fence_row *row = NULL;
  343. dma_fence = __cam_dma_fence_find_fence_in_table(
  344. signal_dma_fence->dma_fence_fd, &idx);
  345. if (IS_ERR_OR_NULL(dma_fence)) {
  346. CAM_ERR(CAM_DMA_FENCE, "Failed to find dma fence for fd: %d",
  347. signal_dma_fence->dma_fence_fd);
  348. return -EINVAL;
  349. }
  350. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[idx]);
  351. row = &g_cam_dma_fence_dev->rows[idx];
  352. /*
  353. * Check for invalid state again, there could be a contention
  354. * between signal and release
  355. */
  356. if (row->state == CAM_DMA_FENCE_STATE_INVALID) {
  357. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[idx]);
  358. CAM_ERR(CAM_DMA_FENCE,
  359. "dma fence fd: %d is invalid row_idx: %u, failed to signal",
  360. signal_dma_fence->dma_fence_fd, idx);
  361. return -EINVAL;
  362. }
  363. if (row->state == CAM_DMA_FENCE_STATE_SIGNALED) {
  364. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[idx]);
  365. CAM_WARN(CAM_DMA_FENCE,
  366. "dma fence fd: %d[seqno: %llu] already in signaled state",
  367. signal_dma_fence->dma_fence_fd, dma_fence->seqno);
  368. return 0;
  369. }
  370. rc = __cam_dma_fence_signal_fence(dma_fence, signal_dma_fence->status);
  371. if (rc)
  372. CAM_WARN(CAM_DMA_FENCE,
  373. "dma fence seqno: %llu fd: %d already signaled rc: %d",
  374. dma_fence->seqno, row->fd, rc);
  375. row->state = CAM_DMA_FENCE_STATE_SIGNALED;
  376. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[idx]);
  377. CAM_DBG(CAM_DMA_FENCE,
  378. "dma fence fd: %d[seqno: %llu] signaled with status: %d rc: %d",
  379. signal_dma_fence->dma_fence_fd, dma_fence->seqno,
  380. signal_dma_fence->status, rc);
  381. return rc;
  382. }
  383. static int __cam_dma_fence_get_fd(int32_t *row_idx,
  384. const char *name)
  385. {
  386. int fd = -1;
  387. uint32_t idx;
  388. struct dma_fence *dma_fence = NULL;
  389. spinlock_t *dma_fence_lock = NULL;
  390. struct sync_file *sync_file = NULL;
  391. if (__cam_dma_fence_find_free_idx(&idx))
  392. goto end;
  393. dma_fence_lock = kzalloc(sizeof(spinlock_t), GFP_KERNEL);
  394. if (!dma_fence_lock)
  395. goto free_idx;
  396. dma_fence = kzalloc(sizeof(struct dma_fence), GFP_KERNEL);
  397. if (!dma_fence) {
  398. kfree(dma_fence_lock);
  399. goto free_idx;
  400. }
  401. spin_lock_init(dma_fence_lock);
  402. dma_fence_init(dma_fence, &cam_sync_dma_fence_ops, dma_fence_lock,
  403. g_cam_dma_fence_dev->dma_fence_context,
  404. atomic64_inc_return(&g_cam_dma_fence_seq_no));
  405. fd = get_unused_fd_flags(O_CLOEXEC);
  406. if (fd < 0) {
  407. CAM_ERR(CAM_DMA_FENCE, "failed to get a unused fd: %d", fd);
  408. dma_fence_put(dma_fence);
  409. goto free_idx;
  410. }
  411. sync_file = sync_file_create(dma_fence);
  412. if (!sync_file) {
  413. put_unused_fd(fd);
  414. fd = -1;
  415. dma_fence_put(dma_fence);
  416. goto free_idx;
  417. }
  418. fd_install(fd, sync_file->file);
  419. *row_idx = idx;
  420. __cam_dma_fence_init_row(name, dma_fence, fd, idx, false);
  421. CAM_DBG(CAM_DMA_FENCE, "Created dma fence fd: %d[%s] seqno: %llu row_idx: %u ref_cnt: %u",
  422. fd, name, dma_fence->seqno, idx, kref_read(&dma_fence->refcount));
  423. return fd;
  424. free_idx:
  425. clear_bit(idx, g_cam_dma_fence_dev->bitmap);
  426. end:
  427. return fd;
  428. }
  429. int cam_dma_fence_create_fd(
  430. int32_t *dma_fence_fd, int32_t *dma_fence_row_idx, const char *name)
  431. {
  432. int fd = -1, rc = 0;
  433. if (!dma_fence_fd || !dma_fence_row_idx) {
  434. CAM_ERR(CAM_DMA_FENCE, "Invalid args fd: %pK dma_fence_row_idx: %pK",
  435. dma_fence_fd, dma_fence_row_idx);
  436. return -EINVAL;
  437. }
  438. fd = __cam_dma_fence_get_fd(dma_fence_row_idx, name);
  439. if (fd < 0) {
  440. rc = -EBADFD;
  441. goto end;
  442. }
  443. *dma_fence_fd = fd;
  444. end:
  445. return rc;
  446. }
  447. static int __cam_dma_fence_release(int32_t dma_row_idx)
  448. {
  449. struct dma_fence *dma_fence = NULL;
  450. struct cam_dma_fence_row *row = NULL;
  451. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_row_idx]);
  452. row = &g_cam_dma_fence_dev->rows[dma_row_idx];
  453. dma_fence = row->fence;
  454. if (row->state == CAM_DMA_FENCE_STATE_INVALID) {
  455. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_row_idx]);
  456. CAM_ERR(CAM_DMA_FENCE, "Invalid row index: %u, state: %u",
  457. dma_row_idx, row->state);
  458. return -EINVAL;
  459. }
  460. if (row->state == CAM_DMA_FENCE_STATE_ACTIVE) {
  461. CAM_WARN(CAM_DMA_FENCE,
  462. "Unsignaled fence being released name: %s seqno: %llu fd:%d",
  463. row->name, dma_fence->seqno, row->fd);
  464. __cam_dma_fence_signal_fence(dma_fence, -ECANCELED);
  465. }
  466. CAM_DBG(CAM_DMA_FENCE,
  467. "Releasing dma fence with fd: %d[%s] row_idx: %u current ref_cnt: %u",
  468. row->fd, row->name, dma_row_idx, kref_read(&dma_fence->refcount));
  469. /* putref on dma fence */
  470. dma_fence_put(dma_fence);
  471. /* deinit row */
  472. memset(row, 0, sizeof(struct cam_dma_fence_row));
  473. clear_bit(dma_row_idx, g_cam_dma_fence_dev->bitmap);
  474. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[dma_row_idx]);
  475. return 0;
  476. }
  477. static int __cam_dma_fence_release_fd(int fd)
  478. {
  479. int32_t idx;
  480. struct dma_fence *dma_fence = NULL;
  481. dma_fence = __cam_dma_fence_find_fence_in_table(fd, &idx);
  482. if (IS_ERR_OR_NULL(dma_fence)) {
  483. CAM_ERR(CAM_DMA_FENCE, "Failed to find dma fence for fd: %d", fd);
  484. return -EINVAL;
  485. }
  486. return __cam_dma_fence_release(idx);
  487. }
  488. static int __cam_dma_fence_release_row(
  489. int32_t dma_fence_row_idx)
  490. {
  491. if ((dma_fence_row_idx < 0) ||
  492. (dma_fence_row_idx >= CAM_DMA_FENCE_MAX_FENCES)) {
  493. CAM_ERR(CAM_DMA_FENCE, "dma fence idx: %d is invalid",
  494. dma_fence_row_idx);
  495. return -EINVAL;
  496. }
  497. return __cam_dma_fence_release(dma_fence_row_idx);
  498. }
  499. int cam_dma_fence_release(
  500. struct cam_dma_fence_release_params *release_params)
  501. {
  502. if (release_params->use_row_idx)
  503. return __cam_dma_fence_release_row(release_params->u.dma_row_idx);
  504. else
  505. return __cam_dma_fence_release_fd(release_params->u.dma_fence_fd);
  506. }
  507. void cam_dma_fence_close(void)
  508. {
  509. int i;
  510. struct cam_dma_fence_row *row = NULL;
  511. mutex_lock(&g_cam_dma_fence_dev->dev_lock);
  512. for (i = 0; i < CAM_DMA_FENCE_MAX_FENCES; i++) {
  513. spin_lock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  514. row = &g_cam_dma_fence_dev->rows[i];
  515. if (row->state != CAM_DMA_FENCE_STATE_INVALID) {
  516. CAM_DBG(CAM_DMA_FENCE,
  517. "Releasing dma fence seqno: %llu associated with fd: %d[%s] ref_cnt: %u",
  518. row->fence->seqno, row->fd, row->name,
  519. kref_read(&row->fence->refcount));
  520. /* If registered for cb, remove cb */
  521. if (row->cb_registered_for_sync)
  522. dma_fence_remove_callback(row->fence, &row->fence_cb);
  523. /* Signal and put if the dma fence is created from camera */
  524. if (!row->ext_dma_fence) {
  525. if (row->state != CAM_DMA_FENCE_STATE_SIGNALED)
  526. __cam_dma_fence_signal_fence(row->fence, -EADV);
  527. dma_fence_put(row->fence);
  528. }
  529. memset(row, 0, sizeof(struct cam_dma_fence_row));
  530. clear_bit(i, g_cam_dma_fence_dev->bitmap);
  531. }
  532. spin_unlock_bh(&g_cam_dma_fence_dev->row_spinlocks[i]);
  533. }
  534. mutex_unlock(&g_cam_dma_fence_dev->dev_lock);
  535. CAM_DBG(CAM_DMA_FENCE, "Close on Camera DMA fence driver");
  536. }
  537. void cam_dma_fence_open(void)
  538. {
  539. mutex_lock(&g_cam_dma_fence_dev->dev_lock);
  540. /* DMA fence seqno reset */
  541. atomic64_set(&g_cam_dma_fence_seq_no, 0);
  542. mutex_unlock(&g_cam_dma_fence_dev->dev_lock);
  543. CAM_DBG(CAM_DMA_FENCE, "Camera DMA fence driver opened");
  544. }
  545. int cam_dma_fence_driver_init(void)
  546. {
  547. int i;
  548. g_cam_dma_fence_dev = kzalloc(sizeof(struct cam_dma_fence_device), GFP_KERNEL);
  549. if (!g_cam_dma_fence_dev)
  550. return -ENOMEM;
  551. mutex_init(&g_cam_dma_fence_dev->dev_lock);
  552. for (i = 0; i < CAM_DMA_FENCE_MAX_FENCES; i++)
  553. spin_lock_init(&g_cam_dma_fence_dev->row_spinlocks[i]);
  554. bitmap_zero(g_cam_dma_fence_dev->bitmap, CAM_DMA_FENCE_MAX_FENCES);
  555. g_cam_dma_fence_dev->dma_fence_context = dma_fence_context_alloc(1);
  556. CAM_DBG(CAM_DMA_FENCE, "Camera DMA fence driver initialized");
  557. return 0;
  558. }
  559. void cam_dma_fence_driver_deinit(void)
  560. {
  561. kfree(g_cam_dma_fence_dev);
  562. g_cam_dma_fence_dev = NULL;
  563. CAM_DBG(CAM_DMA_FENCE, "Camera DMA fence driver deinitialized");
  564. }