cam_mem_mgr.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include <linux/debugfs.h>
  12. #include "cam_compat.h"
  13. #include "cam_req_mgr_util.h"
  14. #include "cam_mem_mgr.h"
  15. #include "cam_smmu_api.h"
  16. #include "cam_debug_util.h"
  17. #include "cam_trace.h"
  18. #include "cam_common_util.h"
  19. static struct cam_mem_table tbl;
  20. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  21. static void cam_mem_mgr_print_tbl(void)
  22. {
  23. int i;
  24. uint64_t ms, tmp, hrs, min, sec;
  25. struct timespec64 *ts = NULL;
  26. struct timespec64 current_ts;
  27. ktime_get_real_ts64(&(current_ts));
  28. tmp = current_ts.tv_sec;
  29. ms = (current_ts.tv_nsec) / 1000000;
  30. sec = do_div(tmp, 60);
  31. min = do_div(tmp, 60);
  32. hrs = do_div(tmp, 24);
  33. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  34. hrs, min, sec, ms);
  35. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  36. if (tbl.bufq[i].active) {
  37. ts = &tbl.bufq[i].timestamp;
  38. tmp = ts->tv_sec;
  39. ms = (ts->tv_nsec) / 1000000;
  40. sec = do_div(tmp, 60);
  41. min = do_div(tmp, 60);
  42. hrs = do_div(tmp, 24);
  43. CAM_INFO(CAM_MEM,
  44. "%llu:%llu:%llu:%llu idx %d fd %d size %llu",
  45. hrs, min, sec, ms, i, tbl.bufq[i].fd,
  46. tbl.bufq[i].len);
  47. }
  48. }
  49. }
  50. static int cam_mem_util_get_dma_dir(uint32_t flags)
  51. {
  52. int rc = -EINVAL;
  53. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  54. rc = DMA_TO_DEVICE;
  55. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  56. rc = DMA_FROM_DEVICE;
  57. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  58. rc = DMA_BIDIRECTIONAL;
  59. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  60. rc = DMA_BIDIRECTIONAL;
  61. return rc;
  62. }
  63. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  64. uintptr_t *vaddr,
  65. size_t *len)
  66. {
  67. int i, j, rc;
  68. void *addr;
  69. /*
  70. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  71. * need to be called in pair to avoid stability issue.
  72. */
  73. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  74. if (rc) {
  75. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  76. return rc;
  77. }
  78. /*
  79. * Code could be simplified if ION support of dma_buf_vmap is
  80. * available. This workaround takes the avandaage that ion_alloc
  81. * returns a virtually contiguous memory region, so we just need
  82. * to _kmap each individual page and then only use the virtual
  83. * address returned from the first call to _kmap.
  84. */
  85. for (i = 0; i < PAGE_ALIGN(dmabuf->size) / PAGE_SIZE; i++) {
  86. addr = dma_buf_kmap(dmabuf, i);
  87. if (IS_ERR_OR_NULL(addr)) {
  88. CAM_ERR(CAM_MEM, "kernel map fail");
  89. for (j = 0; j < i; j++)
  90. dma_buf_kunmap(dmabuf,
  91. j,
  92. (void *)(*vaddr + (j * PAGE_SIZE)));
  93. *vaddr = 0;
  94. *len = 0;
  95. rc = -ENOSPC;
  96. goto fail;
  97. }
  98. if (i == 0)
  99. *vaddr = (uint64_t)addr;
  100. }
  101. *len = dmabuf->size;
  102. return 0;
  103. fail:
  104. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  105. return rc;
  106. }
  107. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  108. uint64_t vaddr)
  109. {
  110. int i, rc = 0, page_num;
  111. if (!dmabuf || !vaddr) {
  112. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  113. return -EINVAL;
  114. }
  115. page_num = PAGE_ALIGN(dmabuf->size) / PAGE_SIZE;
  116. for (i = 0; i < page_num; i++) {
  117. dma_buf_kunmap(dmabuf, i,
  118. (void *)(vaddr + (i * PAGE_SIZE)));
  119. }
  120. /*
  121. * dma_buf_begin_cpu_access() and
  122. * dma_buf_end_cpu_access() need to be called in pair
  123. * to avoid stability issue.
  124. */
  125. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  126. if (rc) {
  127. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  128. dmabuf);
  129. return rc;
  130. }
  131. return rc;
  132. }
  133. static int cam_mem_mgr_create_debug_fs(void)
  134. {
  135. tbl.dentry = debugfs_create_dir("camera_memmgr", NULL);
  136. if (!tbl.dentry) {
  137. CAM_ERR(CAM_MEM, "failed to create dentry");
  138. return -ENOMEM;
  139. }
  140. if (!debugfs_create_bool("alloc_profile_enable",
  141. 0644,
  142. tbl.dentry,
  143. &tbl.alloc_profile_enable)) {
  144. CAM_ERR(CAM_MEM,
  145. "failed to create alloc_profile_enable");
  146. goto err;
  147. }
  148. return 0;
  149. err:
  150. debugfs_remove_recursive(tbl.dentry);
  151. return -ENOMEM;
  152. }
  153. int cam_mem_mgr_init(void)
  154. {
  155. int i;
  156. int bitmap_size;
  157. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  158. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  159. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  160. if (!tbl.bitmap)
  161. return -ENOMEM;
  162. tbl.bits = bitmap_size * BITS_PER_BYTE;
  163. bitmap_zero(tbl.bitmap, tbl.bits);
  164. /* We need to reserve slot 0 because 0 is invalid */
  165. set_bit(0, tbl.bitmap);
  166. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  167. tbl.bufq[i].fd = -1;
  168. tbl.bufq[i].buf_handle = -1;
  169. }
  170. mutex_init(&tbl.m_lock);
  171. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  172. cam_mem_mgr_create_debug_fs();
  173. return 0;
  174. }
  175. static int32_t cam_mem_get_slot(void)
  176. {
  177. int32_t idx;
  178. mutex_lock(&tbl.m_lock);
  179. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  180. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  181. mutex_unlock(&tbl.m_lock);
  182. return -ENOMEM;
  183. }
  184. set_bit(idx, tbl.bitmap);
  185. tbl.bufq[idx].active = true;
  186. ktime_get_real_ts64(&(tbl.bufq[idx].timestamp));
  187. mutex_init(&tbl.bufq[idx].q_lock);
  188. mutex_unlock(&tbl.m_lock);
  189. return idx;
  190. }
  191. static void cam_mem_put_slot(int32_t idx)
  192. {
  193. mutex_lock(&tbl.m_lock);
  194. mutex_lock(&tbl.bufq[idx].q_lock);
  195. tbl.bufq[idx].active = false;
  196. tbl.bufq[idx].is_internal = false;
  197. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  198. mutex_unlock(&tbl.bufq[idx].q_lock);
  199. mutex_destroy(&tbl.bufq[idx].q_lock);
  200. clear_bit(idx, tbl.bitmap);
  201. mutex_unlock(&tbl.m_lock);
  202. }
  203. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  204. dma_addr_t *iova_ptr, size_t *len_ptr)
  205. {
  206. int rc = 0, idx;
  207. *len_ptr = 0;
  208. if (!atomic_read(&cam_mem_mgr_state)) {
  209. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  210. return -EINVAL;
  211. }
  212. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  213. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  214. return -ENOENT;
  215. if (!tbl.bufq[idx].active)
  216. return -EAGAIN;
  217. mutex_lock(&tbl.bufq[idx].q_lock);
  218. if (buf_handle != tbl.bufq[idx].buf_handle) {
  219. rc = -EINVAL;
  220. goto handle_mismatch;
  221. }
  222. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  223. rc = cam_smmu_get_stage2_iova(mmu_handle,
  224. tbl.bufq[idx].fd,
  225. iova_ptr,
  226. len_ptr);
  227. else
  228. rc = cam_smmu_get_iova(mmu_handle,
  229. tbl.bufq[idx].fd,
  230. iova_ptr,
  231. len_ptr);
  232. if (rc) {
  233. CAM_ERR(CAM_MEM,
  234. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d",
  235. buf_handle, mmu_handle, tbl.bufq[idx].fd);
  236. goto handle_mismatch;
  237. }
  238. CAM_DBG(CAM_MEM,
  239. "handle:0x%x fd:%d iova_ptr:%pK len_ptr:%llu",
  240. mmu_handle, tbl.bufq[idx].fd, iova_ptr, *len_ptr);
  241. handle_mismatch:
  242. mutex_unlock(&tbl.bufq[idx].q_lock);
  243. return rc;
  244. }
  245. EXPORT_SYMBOL(cam_mem_get_io_buf);
  246. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  247. {
  248. int idx;
  249. if (!atomic_read(&cam_mem_mgr_state)) {
  250. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  251. return -EINVAL;
  252. }
  253. if (!atomic_read(&cam_mem_mgr_state)) {
  254. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  255. return -EINVAL;
  256. }
  257. if (!buf_handle || !vaddr_ptr || !len)
  258. return -EINVAL;
  259. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  260. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  261. return -EINVAL;
  262. if (!tbl.bufq[idx].active)
  263. return -EPERM;
  264. if (buf_handle != tbl.bufq[idx].buf_handle)
  265. return -EINVAL;
  266. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  267. return -EINVAL;
  268. if (tbl.bufq[idx].kmdvaddr) {
  269. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  270. *len = tbl.bufq[idx].len;
  271. } else {
  272. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  273. buf_handle);
  274. return -EINVAL;
  275. }
  276. return 0;
  277. }
  278. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  279. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  280. {
  281. int rc = 0, idx;
  282. uint32_t cache_dir;
  283. unsigned long dmabuf_flag = 0;
  284. if (!atomic_read(&cam_mem_mgr_state)) {
  285. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  286. return -EINVAL;
  287. }
  288. if (!cmd)
  289. return -EINVAL;
  290. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  291. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  292. return -EINVAL;
  293. mutex_lock(&tbl.bufq[idx].q_lock);
  294. if (!tbl.bufq[idx].active) {
  295. rc = -EINVAL;
  296. goto end;
  297. }
  298. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  299. rc = -EINVAL;
  300. goto end;
  301. }
  302. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  303. if (rc) {
  304. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  305. goto end;
  306. }
  307. if (dmabuf_flag & ION_FLAG_CACHED) {
  308. switch (cmd->mem_cache_ops) {
  309. case CAM_MEM_CLEAN_CACHE:
  310. cache_dir = DMA_TO_DEVICE;
  311. break;
  312. case CAM_MEM_INV_CACHE:
  313. cache_dir = DMA_FROM_DEVICE;
  314. break;
  315. case CAM_MEM_CLEAN_INV_CACHE:
  316. cache_dir = DMA_BIDIRECTIONAL;
  317. break;
  318. default:
  319. CAM_ERR(CAM_MEM,
  320. "invalid cache ops :%d", cmd->mem_cache_ops);
  321. rc = -EINVAL;
  322. goto end;
  323. }
  324. } else {
  325. CAM_DBG(CAM_MEM, "BUF is not cached");
  326. goto end;
  327. }
  328. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  329. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  330. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  331. if (rc) {
  332. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  333. goto end;
  334. }
  335. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  336. cache_dir);
  337. if (rc) {
  338. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  339. goto end;
  340. }
  341. end:
  342. mutex_unlock(&tbl.bufq[idx].q_lock);
  343. return rc;
  344. }
  345. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  346. static int cam_mem_util_get_dma_buf(size_t len,
  347. unsigned int heap_id_mask,
  348. unsigned int flags,
  349. struct dma_buf **buf)
  350. {
  351. int rc = 0;
  352. if (!buf) {
  353. CAM_ERR(CAM_MEM, "Invalid params");
  354. return -EINVAL;
  355. }
  356. *buf = ion_alloc(len, heap_id_mask, flags);
  357. if (IS_ERR_OR_NULL(*buf))
  358. return -ENOMEM;
  359. return rc;
  360. }
  361. static int cam_mem_util_get_dma_buf_fd(size_t len,
  362. size_t align,
  363. unsigned int heap_id_mask,
  364. unsigned int flags,
  365. struct dma_buf **buf,
  366. int *fd)
  367. {
  368. struct dma_buf *dmabuf = NULL;
  369. int rc = 0;
  370. struct timespec64 ts1, ts2;
  371. long microsec = 0;
  372. if (!buf || !fd) {
  373. CAM_ERR(CAM_MEM, "Invalid params, buf=%pK, fd=%pK", buf, fd);
  374. return -EINVAL;
  375. }
  376. if (tbl.alloc_profile_enable)
  377. CAM_GET_TIMESTAMP(ts1);
  378. *buf = ion_alloc(len, heap_id_mask, flags);
  379. if (IS_ERR_OR_NULL(*buf))
  380. return -ENOMEM;
  381. *fd = dma_buf_fd(*buf, O_CLOEXEC);
  382. if (*fd < 0) {
  383. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  384. rc = -EINVAL;
  385. goto get_fd_fail;
  386. }
  387. /*
  388. * increment the ref count so that ref count becomes 2 here
  389. * when we close fd, refcount becomes 1 and when we do
  390. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  391. */
  392. dmabuf = dma_buf_get(*fd);
  393. if (IS_ERR_OR_NULL(dmabuf)) {
  394. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d", *fd);
  395. rc = -EINVAL;
  396. }
  397. if (tbl.alloc_profile_enable) {
  398. CAM_GET_TIMESTAMP(ts2);
  399. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  400. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  401. len, microsec);
  402. }
  403. return rc;
  404. get_fd_fail:
  405. dma_buf_put(*buf);
  406. return rc;
  407. }
  408. static int cam_mem_util_ion_alloc(struct cam_mem_mgr_alloc_cmd *cmd,
  409. struct dma_buf **dmabuf,
  410. int *fd)
  411. {
  412. uint32_t heap_id;
  413. uint32_t ion_flag = 0;
  414. int rc;
  415. if ((cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  416. (cmd->flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  417. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  418. ion_flag |=
  419. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  420. } else if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  421. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  422. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  423. } else {
  424. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  425. ION_HEAP(ION_CAMERA_HEAP_ID);
  426. }
  427. if (cmd->flags & CAM_MEM_FLAG_CACHE)
  428. ion_flag |= ION_FLAG_CACHED;
  429. else
  430. ion_flag &= ~ION_FLAG_CACHED;
  431. rc = cam_mem_util_get_dma_buf_fd(cmd->len,
  432. cmd->align,
  433. heap_id,
  434. ion_flag,
  435. dmabuf,
  436. fd);
  437. return rc;
  438. }
  439. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  440. {
  441. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  442. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  443. CAM_MEM_MMU_MAX_HANDLE);
  444. return -EINVAL;
  445. }
  446. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  447. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  448. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  449. return -EINVAL;
  450. }
  451. return 0;
  452. }
  453. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  454. {
  455. if (!cmd->flags) {
  456. CAM_ERR(CAM_MEM, "Invalid flags");
  457. return -EINVAL;
  458. }
  459. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  460. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  461. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  462. return -EINVAL;
  463. }
  464. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  465. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  466. CAM_ERR(CAM_MEM,
  467. "Kernel mapping in secure mode not allowed, flags=0x%x",
  468. cmd->flags);
  469. return -EINVAL;
  470. }
  471. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  472. CAM_ERR(CAM_MEM,
  473. "Shared memory buffers are not allowed to be mapped");
  474. return -EINVAL;
  475. }
  476. return 0;
  477. }
  478. static int cam_mem_util_map_hw_va(uint32_t flags,
  479. int32_t *mmu_hdls,
  480. int32_t num_hdls,
  481. int fd,
  482. dma_addr_t *hw_vaddr,
  483. size_t *len,
  484. enum cam_smmu_region_id region,
  485. bool is_internal)
  486. {
  487. int i;
  488. int rc = -1;
  489. int dir = cam_mem_util_get_dma_dir(flags);
  490. bool dis_delayed_unmap = false;
  491. if (dir < 0) {
  492. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  493. return dir;
  494. }
  495. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  496. dis_delayed_unmap = true;
  497. CAM_DBG(CAM_MEM,
  498. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  499. fd, flags, dir, num_hdls);
  500. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  501. for (i = 0; i < num_hdls; i++) {
  502. rc = cam_smmu_map_stage2_iova(mmu_hdls[i],
  503. fd,
  504. dir,
  505. hw_vaddr,
  506. len);
  507. if (rc < 0) {
  508. CAM_ERR(CAM_MEM,
  509. "Failed to securely map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  510. i, fd, dir, mmu_hdls[i], rc);
  511. goto multi_map_fail;
  512. }
  513. }
  514. } else {
  515. for (i = 0; i < num_hdls; i++) {
  516. rc = cam_smmu_map_user_iova(mmu_hdls[i],
  517. fd,
  518. dis_delayed_unmap,
  519. dir,
  520. (dma_addr_t *)hw_vaddr,
  521. len,
  522. region,
  523. is_internal);
  524. if (rc < 0) {
  525. CAM_ERR(CAM_MEM,
  526. "Failed to map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, region=%d, rc=%d",
  527. i, fd, dir, mmu_hdls[i], region, rc);
  528. goto multi_map_fail;
  529. }
  530. }
  531. }
  532. return rc;
  533. multi_map_fail:
  534. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  535. for (--i; i > 0; i--)
  536. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  537. else
  538. for (--i; i > 0; i--)
  539. cam_smmu_unmap_user_iova(mmu_hdls[i],
  540. fd,
  541. CAM_SMMU_REGION_IO);
  542. return rc;
  543. }
  544. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  545. {
  546. int rc;
  547. int32_t idx;
  548. struct dma_buf *dmabuf = NULL;
  549. int fd = -1;
  550. dma_addr_t hw_vaddr = 0;
  551. size_t len;
  552. uintptr_t kvaddr = 0;
  553. size_t klen;
  554. if (!atomic_read(&cam_mem_mgr_state)) {
  555. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  556. return -EINVAL;
  557. }
  558. if (!cmd) {
  559. CAM_ERR(CAM_MEM, " Invalid argument");
  560. return -EINVAL;
  561. }
  562. len = cmd->len;
  563. rc = cam_mem_util_check_alloc_flags(cmd);
  564. if (rc) {
  565. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  566. cmd->flags, rc);
  567. return rc;
  568. }
  569. rc = cam_mem_util_ion_alloc(cmd,
  570. &dmabuf,
  571. &fd);
  572. if (rc) {
  573. CAM_ERR(CAM_MEM,
  574. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  575. cmd->len, cmd->align, cmd->flags, cmd->num_hdl);
  576. cam_mem_mgr_print_tbl();
  577. return rc;
  578. }
  579. idx = cam_mem_get_slot();
  580. if (idx < 0) {
  581. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  582. rc = -ENOMEM;
  583. goto slot_fail;
  584. }
  585. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  586. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  587. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  588. enum cam_smmu_region_id region;
  589. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  590. region = CAM_SMMU_REGION_IO;
  591. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  592. region = CAM_SMMU_REGION_SHARED;
  593. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  594. region = CAM_SMMU_REGION_SECHEAP;
  595. rc = cam_mem_util_map_hw_va(cmd->flags,
  596. cmd->mmu_hdls,
  597. cmd->num_hdl,
  598. fd,
  599. &hw_vaddr,
  600. &len,
  601. region,
  602. true);
  603. if (rc) {
  604. CAM_ERR(CAM_MEM,
  605. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  606. len, cmd->flags,
  607. fd, region, cmd->num_hdl, rc);
  608. if (rc == -EALREADY) {
  609. if ((size_t)dmabuf->size != len)
  610. rc = -EBADR;
  611. cam_mem_mgr_print_tbl();
  612. }
  613. goto map_hw_fail;
  614. }
  615. }
  616. mutex_lock(&tbl.bufq[idx].q_lock);
  617. tbl.bufq[idx].fd = fd;
  618. tbl.bufq[idx].dma_buf = NULL;
  619. tbl.bufq[idx].flags = cmd->flags;
  620. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  621. tbl.bufq[idx].is_internal = true;
  622. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  623. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  624. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  625. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  626. if (rc) {
  627. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  628. dmabuf, rc);
  629. goto map_kernel_fail;
  630. }
  631. }
  632. tbl.bufq[idx].kmdvaddr = kvaddr;
  633. tbl.bufq[idx].vaddr = hw_vaddr;
  634. tbl.bufq[idx].dma_buf = dmabuf;
  635. tbl.bufq[idx].len = cmd->len;
  636. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  637. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  638. sizeof(int32_t) * cmd->num_hdl);
  639. tbl.bufq[idx].is_imported = false;
  640. mutex_unlock(&tbl.bufq[idx].q_lock);
  641. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  642. cmd->out.fd = tbl.bufq[idx].fd;
  643. cmd->out.vaddr = 0;
  644. CAM_DBG(CAM_MEM,
  645. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  646. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  647. tbl.bufq[idx].len);
  648. return rc;
  649. map_kernel_fail:
  650. mutex_unlock(&tbl.bufq[idx].q_lock);
  651. map_hw_fail:
  652. cam_mem_put_slot(idx);
  653. slot_fail:
  654. dma_buf_put(dmabuf);
  655. return rc;
  656. }
  657. static bool cam_mem_util_is_map_internal(int32_t fd)
  658. {
  659. uint32_t i;
  660. bool is_internal = false;
  661. mutex_lock(&tbl.m_lock);
  662. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  663. if (tbl.bufq[i].fd == fd) {
  664. is_internal = tbl.bufq[i].is_internal;
  665. break;
  666. }
  667. }
  668. mutex_unlock(&tbl.m_lock);
  669. return is_internal;
  670. }
  671. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  672. {
  673. int32_t idx;
  674. int rc;
  675. struct dma_buf *dmabuf;
  676. dma_addr_t hw_vaddr = 0;
  677. size_t len = 0;
  678. bool is_internal = false;
  679. if (!atomic_read(&cam_mem_mgr_state)) {
  680. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  681. return -EINVAL;
  682. }
  683. if (!cmd || (cmd->fd < 0)) {
  684. CAM_ERR(CAM_MEM, "Invalid argument");
  685. return -EINVAL;
  686. }
  687. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  688. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  689. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  690. return -EINVAL;
  691. }
  692. rc = cam_mem_util_check_map_flags(cmd);
  693. if (rc) {
  694. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  695. return rc;
  696. }
  697. dmabuf = dma_buf_get(cmd->fd);
  698. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  699. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  700. return -EINVAL;
  701. }
  702. is_internal = cam_mem_util_is_map_internal(cmd->fd);
  703. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  704. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  705. rc = cam_mem_util_map_hw_va(cmd->flags,
  706. cmd->mmu_hdls,
  707. cmd->num_hdl,
  708. cmd->fd,
  709. &hw_vaddr,
  710. &len,
  711. CAM_SMMU_REGION_IO,
  712. is_internal);
  713. if (rc) {
  714. CAM_ERR(CAM_MEM,
  715. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  716. cmd->flags, cmd->fd, len,
  717. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  718. if (rc == -EALREADY) {
  719. if ((size_t)dmabuf->size != len) {
  720. rc = -EBADR;
  721. cam_mem_mgr_print_tbl();
  722. }
  723. }
  724. goto map_fail;
  725. }
  726. }
  727. idx = cam_mem_get_slot();
  728. if (idx < 0) {
  729. rc = -ENOMEM;
  730. goto map_fail;
  731. }
  732. mutex_lock(&tbl.bufq[idx].q_lock);
  733. tbl.bufq[idx].fd = cmd->fd;
  734. tbl.bufq[idx].dma_buf = NULL;
  735. tbl.bufq[idx].flags = cmd->flags;
  736. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  737. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  738. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  739. tbl.bufq[idx].kmdvaddr = 0;
  740. if (cmd->num_hdl > 0)
  741. tbl.bufq[idx].vaddr = hw_vaddr;
  742. else
  743. tbl.bufq[idx].vaddr = 0;
  744. tbl.bufq[idx].dma_buf = dmabuf;
  745. tbl.bufq[idx].len = len;
  746. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  747. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  748. sizeof(int32_t) * cmd->num_hdl);
  749. tbl.bufq[idx].is_imported = true;
  750. tbl.bufq[idx].is_internal = is_internal;
  751. mutex_unlock(&tbl.bufq[idx].q_lock);
  752. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  753. cmd->out.vaddr = 0;
  754. cmd->out.size = (uint32_t)len;
  755. CAM_DBG(CAM_MEM,
  756. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  757. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  758. tbl.bufq[idx].len);
  759. return rc;
  760. map_fail:
  761. dma_buf_put(dmabuf);
  762. return rc;
  763. }
  764. static int cam_mem_util_unmap_hw_va(int32_t idx,
  765. enum cam_smmu_region_id region,
  766. enum cam_smmu_mapping_client client)
  767. {
  768. int i;
  769. uint32_t flags;
  770. int32_t *mmu_hdls;
  771. int num_hdls;
  772. int fd;
  773. int rc = 0;
  774. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  775. CAM_ERR(CAM_MEM, "Incorrect index");
  776. return -EINVAL;
  777. }
  778. flags = tbl.bufq[idx].flags;
  779. mmu_hdls = tbl.bufq[idx].hdls;
  780. num_hdls = tbl.bufq[idx].num_hdl;
  781. fd = tbl.bufq[idx].fd;
  782. CAM_DBG(CAM_MEM,
  783. "unmap_hw_va : idx=%d, fd=%x, flags=0x%x, num_hdls=%d, client=%d",
  784. idx, fd, flags, num_hdls, client);
  785. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  786. for (i = 0; i < num_hdls; i++) {
  787. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  788. if (rc < 0) {
  789. CAM_ERR(CAM_MEM,
  790. "Failed in secure unmap, i=%d, fd=%d, mmu_hdl=%d, rc=%d",
  791. i, fd, mmu_hdls[i], rc);
  792. goto unmap_end;
  793. }
  794. }
  795. } else {
  796. for (i = 0; i < num_hdls; i++) {
  797. if (client == CAM_SMMU_MAPPING_USER) {
  798. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  799. fd, region);
  800. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  801. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  802. tbl.bufq[idx].dma_buf, region);
  803. } else {
  804. CAM_ERR(CAM_MEM,
  805. "invalid caller for unmapping : %d",
  806. client);
  807. rc = -EINVAL;
  808. }
  809. if (rc < 0) {
  810. CAM_ERR(CAM_MEM,
  811. "Failed in unmap, i=%d, fd=%d, mmu_hdl=%d, region=%d, rc=%d",
  812. i, fd, mmu_hdls[i], region, rc);
  813. goto unmap_end;
  814. }
  815. }
  816. }
  817. return rc;
  818. unmap_end:
  819. CAM_ERR(CAM_MEM, "unmapping failed");
  820. return rc;
  821. }
  822. static void cam_mem_mgr_unmap_active_buf(int idx)
  823. {
  824. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  825. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  826. region = CAM_SMMU_REGION_SHARED;
  827. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  828. region = CAM_SMMU_REGION_IO;
  829. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  830. }
  831. static int cam_mem_mgr_cleanup_table(void)
  832. {
  833. int i;
  834. mutex_lock(&tbl.m_lock);
  835. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  836. if (!tbl.bufq[i].active) {
  837. CAM_DBG(CAM_MEM,
  838. "Buffer inactive at idx=%d, continuing", i);
  839. continue;
  840. } else {
  841. CAM_DBG(CAM_MEM,
  842. "Active buffer at idx=%d, possible leak needs unmapping",
  843. i);
  844. cam_mem_mgr_unmap_active_buf(i);
  845. }
  846. mutex_lock(&tbl.bufq[i].q_lock);
  847. if (tbl.bufq[i].dma_buf) {
  848. dma_buf_put(tbl.bufq[i].dma_buf);
  849. tbl.bufq[i].dma_buf = NULL;
  850. }
  851. tbl.bufq[i].fd = -1;
  852. tbl.bufq[i].flags = 0;
  853. tbl.bufq[i].buf_handle = -1;
  854. tbl.bufq[i].vaddr = 0;
  855. tbl.bufq[i].len = 0;
  856. memset(tbl.bufq[i].hdls, 0,
  857. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  858. tbl.bufq[i].num_hdl = 0;
  859. tbl.bufq[i].dma_buf = NULL;
  860. tbl.bufq[i].active = false;
  861. tbl.bufq[i].is_internal = false;
  862. mutex_unlock(&tbl.bufq[i].q_lock);
  863. mutex_destroy(&tbl.bufq[i].q_lock);
  864. }
  865. bitmap_zero(tbl.bitmap, tbl.bits);
  866. /* We need to reserve slot 0 because 0 is invalid */
  867. set_bit(0, tbl.bitmap);
  868. mutex_unlock(&tbl.m_lock);
  869. return 0;
  870. }
  871. void cam_mem_mgr_deinit(void)
  872. {
  873. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  874. cam_mem_mgr_cleanup_table();
  875. mutex_lock(&tbl.m_lock);
  876. bitmap_zero(tbl.bitmap, tbl.bits);
  877. kfree(tbl.bitmap);
  878. tbl.bitmap = NULL;
  879. mutex_unlock(&tbl.m_lock);
  880. mutex_destroy(&tbl.m_lock);
  881. }
  882. static int cam_mem_util_unmap(int32_t idx,
  883. enum cam_smmu_mapping_client client)
  884. {
  885. int rc = 0;
  886. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  887. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  888. CAM_ERR(CAM_MEM, "Incorrect index");
  889. return -EINVAL;
  890. }
  891. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  892. mutex_lock(&tbl.m_lock);
  893. if ((!tbl.bufq[idx].active) &&
  894. (tbl.bufq[idx].vaddr) == 0) {
  895. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  896. idx);
  897. mutex_unlock(&tbl.m_lock);
  898. return 0;
  899. }
  900. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  901. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  902. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  903. tbl.bufq[idx].kmdvaddr);
  904. if (rc)
  905. CAM_ERR(CAM_MEM,
  906. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  907. tbl.bufq[idx].dma_buf,
  908. (void *) tbl.bufq[idx].kmdvaddr);
  909. }
  910. }
  911. /* SHARED flag gets precedence, all other flags after it */
  912. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  913. region = CAM_SMMU_REGION_SHARED;
  914. } else {
  915. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  916. region = CAM_SMMU_REGION_IO;
  917. }
  918. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  919. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  920. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  921. if (cam_mem_util_unmap_hw_va(idx, region, client))
  922. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  923. tbl.bufq[idx].dma_buf);
  924. if (client == CAM_SMMU_MAPPING_KERNEL)
  925. tbl.bufq[idx].dma_buf = NULL;
  926. }
  927. mutex_lock(&tbl.bufq[idx].q_lock);
  928. tbl.bufq[idx].flags = 0;
  929. tbl.bufq[idx].buf_handle = -1;
  930. tbl.bufq[idx].vaddr = 0;
  931. memset(tbl.bufq[idx].hdls, 0,
  932. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  933. CAM_DBG(CAM_MEM,
  934. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK",
  935. idx, tbl.bufq[idx].fd,
  936. tbl.bufq[idx].is_imported,
  937. tbl.bufq[idx].dma_buf);
  938. if (tbl.bufq[idx].dma_buf)
  939. dma_buf_put(tbl.bufq[idx].dma_buf);
  940. tbl.bufq[idx].fd = -1;
  941. tbl.bufq[idx].dma_buf = NULL;
  942. tbl.bufq[idx].is_imported = false;
  943. tbl.bufq[idx].is_internal = false;
  944. tbl.bufq[idx].len = 0;
  945. tbl.bufq[idx].num_hdl = 0;
  946. tbl.bufq[idx].active = false;
  947. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  948. mutex_unlock(&tbl.bufq[idx].q_lock);
  949. mutex_destroy(&tbl.bufq[idx].q_lock);
  950. clear_bit(idx, tbl.bitmap);
  951. mutex_unlock(&tbl.m_lock);
  952. return rc;
  953. }
  954. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  955. {
  956. int idx;
  957. int rc;
  958. if (!atomic_read(&cam_mem_mgr_state)) {
  959. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  960. return -EINVAL;
  961. }
  962. if (!cmd) {
  963. CAM_ERR(CAM_MEM, "Invalid argument");
  964. return -EINVAL;
  965. }
  966. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  967. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  968. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  969. idx);
  970. return -EINVAL;
  971. }
  972. if (!tbl.bufq[idx].active) {
  973. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  974. return -EINVAL;
  975. }
  976. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  977. CAM_ERR(CAM_MEM,
  978. "Released buf handle %d not matching within table %d, idx=%d",
  979. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  980. return -EINVAL;
  981. }
  982. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  983. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  984. return rc;
  985. }
  986. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  987. struct cam_mem_mgr_memory_desc *out)
  988. {
  989. struct dma_buf *buf = NULL;
  990. int ion_fd = -1;
  991. int rc = 0;
  992. uint32_t heap_id;
  993. int32_t ion_flag = 0;
  994. uintptr_t kvaddr;
  995. dma_addr_t iova = 0;
  996. size_t request_len = 0;
  997. uint32_t mem_handle;
  998. int32_t idx;
  999. int32_t smmu_hdl = 0;
  1000. int32_t num_hdl = 0;
  1001. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1002. if (!atomic_read(&cam_mem_mgr_state)) {
  1003. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1004. return -EINVAL;
  1005. }
  1006. if (!inp || !out) {
  1007. CAM_ERR(CAM_MEM, "Invalid params");
  1008. return -EINVAL;
  1009. }
  1010. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1011. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1012. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1013. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1014. return -EINVAL;
  1015. }
  1016. if (inp->flags & CAM_MEM_FLAG_CACHE)
  1017. ion_flag |= ION_FLAG_CACHED;
  1018. else
  1019. ion_flag &= ~ION_FLAG_CACHED;
  1020. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  1021. ION_HEAP(ION_CAMERA_HEAP_ID);
  1022. rc = cam_mem_util_get_dma_buf(inp->size,
  1023. heap_id,
  1024. ion_flag,
  1025. &buf);
  1026. if (rc) {
  1027. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1028. goto ion_fail;
  1029. } else {
  1030. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1031. }
  1032. /*
  1033. * we are mapping kva always here,
  1034. * update flags so that we do unmap properly
  1035. */
  1036. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1037. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1038. if (rc) {
  1039. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1040. goto map_fail;
  1041. }
  1042. if (!inp->smmu_hdl) {
  1043. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1044. rc = -EINVAL;
  1045. goto smmu_fail;
  1046. }
  1047. /* SHARED flag gets precedence, all other flags after it */
  1048. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1049. region = CAM_SMMU_REGION_SHARED;
  1050. } else {
  1051. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1052. region = CAM_SMMU_REGION_IO;
  1053. }
  1054. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1055. buf,
  1056. CAM_SMMU_MAP_RW,
  1057. &iova,
  1058. &request_len,
  1059. region);
  1060. if (rc < 0) {
  1061. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1062. goto smmu_fail;
  1063. }
  1064. smmu_hdl = inp->smmu_hdl;
  1065. num_hdl = 1;
  1066. idx = cam_mem_get_slot();
  1067. if (idx < 0) {
  1068. rc = -ENOMEM;
  1069. goto slot_fail;
  1070. }
  1071. mutex_lock(&tbl.bufq[idx].q_lock);
  1072. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1073. tbl.bufq[idx].dma_buf = buf;
  1074. tbl.bufq[idx].fd = -1;
  1075. tbl.bufq[idx].flags = inp->flags;
  1076. tbl.bufq[idx].buf_handle = mem_handle;
  1077. tbl.bufq[idx].kmdvaddr = kvaddr;
  1078. tbl.bufq[idx].vaddr = iova;
  1079. tbl.bufq[idx].len = inp->size;
  1080. tbl.bufq[idx].num_hdl = num_hdl;
  1081. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1082. sizeof(int32_t));
  1083. tbl.bufq[idx].is_imported = false;
  1084. mutex_unlock(&tbl.bufq[idx].q_lock);
  1085. out->kva = kvaddr;
  1086. out->iova = (uint32_t)iova;
  1087. out->smmu_hdl = smmu_hdl;
  1088. out->mem_handle = mem_handle;
  1089. out->len = inp->size;
  1090. out->region = region;
  1091. return rc;
  1092. slot_fail:
  1093. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1094. buf, region);
  1095. smmu_fail:
  1096. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1097. map_fail:
  1098. dma_buf_put(buf);
  1099. ion_fail:
  1100. return rc;
  1101. }
  1102. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1103. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1104. {
  1105. int32_t idx;
  1106. int rc;
  1107. if (!atomic_read(&cam_mem_mgr_state)) {
  1108. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1109. return -EINVAL;
  1110. }
  1111. if (!inp) {
  1112. CAM_ERR(CAM_MEM, "Invalid argument");
  1113. return -EINVAL;
  1114. }
  1115. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1116. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1117. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1118. return -EINVAL;
  1119. }
  1120. if (!tbl.bufq[idx].active) {
  1121. if (tbl.bufq[idx].vaddr == 0) {
  1122. CAM_ERR(CAM_MEM, "buffer is released already");
  1123. return 0;
  1124. }
  1125. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1126. return -EINVAL;
  1127. }
  1128. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1129. CAM_ERR(CAM_MEM,
  1130. "Released buf handle not matching within table");
  1131. return -EINVAL;
  1132. }
  1133. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1134. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1135. return rc;
  1136. }
  1137. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1138. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1139. enum cam_smmu_region_id region,
  1140. struct cam_mem_mgr_memory_desc *out)
  1141. {
  1142. struct dma_buf *buf = NULL;
  1143. int rc = 0;
  1144. int ion_fd = -1;
  1145. uint32_t heap_id;
  1146. dma_addr_t iova = 0;
  1147. size_t request_len = 0;
  1148. uint32_t mem_handle;
  1149. int32_t idx;
  1150. int32_t smmu_hdl = 0;
  1151. int32_t num_hdl = 0;
  1152. if (!atomic_read(&cam_mem_mgr_state)) {
  1153. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1154. return -EINVAL;
  1155. }
  1156. if (!inp || !out) {
  1157. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1158. return -EINVAL;
  1159. }
  1160. if (!inp->smmu_hdl) {
  1161. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1162. return -EINVAL;
  1163. }
  1164. if (region != CAM_SMMU_REGION_SECHEAP) {
  1165. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1166. return -EINVAL;
  1167. }
  1168. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  1169. ION_HEAP(ION_CAMERA_HEAP_ID);
  1170. rc = cam_mem_util_get_dma_buf(inp->size,
  1171. heap_id,
  1172. 0,
  1173. &buf);
  1174. if (rc) {
  1175. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1176. goto ion_fail;
  1177. } else {
  1178. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1179. }
  1180. rc = cam_smmu_reserve_sec_heap(inp->smmu_hdl,
  1181. buf,
  1182. &iova,
  1183. &request_len);
  1184. if (rc) {
  1185. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1186. goto smmu_fail;
  1187. }
  1188. smmu_hdl = inp->smmu_hdl;
  1189. num_hdl = 1;
  1190. idx = cam_mem_get_slot();
  1191. if (idx < 0) {
  1192. rc = -ENOMEM;
  1193. goto slot_fail;
  1194. }
  1195. mutex_lock(&tbl.bufq[idx].q_lock);
  1196. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1197. tbl.bufq[idx].fd = -1;
  1198. tbl.bufq[idx].dma_buf = buf;
  1199. tbl.bufq[idx].flags = inp->flags;
  1200. tbl.bufq[idx].buf_handle = mem_handle;
  1201. tbl.bufq[idx].kmdvaddr = 0;
  1202. tbl.bufq[idx].vaddr = iova;
  1203. tbl.bufq[idx].len = request_len;
  1204. tbl.bufq[idx].num_hdl = num_hdl;
  1205. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1206. sizeof(int32_t));
  1207. tbl.bufq[idx].is_imported = false;
  1208. mutex_unlock(&tbl.bufq[idx].q_lock);
  1209. out->kva = 0;
  1210. out->iova = (uint32_t)iova;
  1211. out->smmu_hdl = smmu_hdl;
  1212. out->mem_handle = mem_handle;
  1213. out->len = request_len;
  1214. out->region = region;
  1215. return rc;
  1216. slot_fail:
  1217. cam_smmu_release_sec_heap(smmu_hdl);
  1218. smmu_fail:
  1219. dma_buf_put(buf);
  1220. ion_fail:
  1221. return rc;
  1222. }
  1223. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1224. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1225. {
  1226. int32_t idx;
  1227. int rc;
  1228. int32_t smmu_hdl;
  1229. if (!atomic_read(&cam_mem_mgr_state)) {
  1230. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1231. return -EINVAL;
  1232. }
  1233. if (!inp) {
  1234. CAM_ERR(CAM_MEM, "Invalid argument");
  1235. return -EINVAL;
  1236. }
  1237. if (inp->region != CAM_SMMU_REGION_SECHEAP) {
  1238. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1239. return -EINVAL;
  1240. }
  1241. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1242. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1243. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1244. return -EINVAL;
  1245. }
  1246. if (!tbl.bufq[idx].active) {
  1247. if (tbl.bufq[idx].vaddr == 0) {
  1248. CAM_ERR(CAM_MEM, "buffer is released already");
  1249. return 0;
  1250. }
  1251. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1252. return -EINVAL;
  1253. }
  1254. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1255. CAM_ERR(CAM_MEM,
  1256. "Released buf handle not matching within table");
  1257. return -EINVAL;
  1258. }
  1259. if (tbl.bufq[idx].num_hdl != 1) {
  1260. CAM_ERR(CAM_MEM,
  1261. "Sec heap region should have only one smmu hdl");
  1262. return -ENODEV;
  1263. }
  1264. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1265. sizeof(int32_t));
  1266. if (inp->smmu_hdl != smmu_hdl) {
  1267. CAM_ERR(CAM_MEM,
  1268. "Passed SMMU handle doesn't match with internal hdl");
  1269. return -ENODEV;
  1270. }
  1271. rc = cam_smmu_release_sec_heap(inp->smmu_hdl);
  1272. if (rc) {
  1273. CAM_ERR(CAM_MEM,
  1274. "Sec heap region release failed");
  1275. return -ENODEV;
  1276. }
  1277. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1278. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1279. if (rc)
  1280. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1281. return rc;
  1282. }
  1283. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);