cam_mem_mgr.c 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include <linux/debugfs.h>
  12. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  13. #include <linux/mem-buf.h>
  14. #include <soc/qcom/secure_buffer.h>
  15. #endif
  16. #include "cam_compat.h"
  17. #include "cam_req_mgr_util.h"
  18. #include "cam_mem_mgr.h"
  19. #include "cam_smmu_api.h"
  20. #include "cam_debug_util.h"
  21. #include "cam_trace.h"
  22. #include "cam_common_util.h"
  23. #define CAM_MEM_SHARED_BUFFER_PAD_4K (4 * 1024)
  24. static struct cam_mem_table tbl;
  25. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  26. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  27. static void cam_mem_mgr_put_dma_heaps(void);
  28. static int cam_mem_mgr_get_dma_heaps(void);
  29. #endif
  30. static void cam_mem_mgr_print_tbl(void)
  31. {
  32. int i;
  33. uint64_t ms, tmp, hrs, min, sec;
  34. struct timespec64 *ts = NULL;
  35. struct timespec64 current_ts;
  36. ktime_get_real_ts64(&(current_ts));
  37. tmp = current_ts.tv_sec;
  38. ms = (current_ts.tv_nsec) / 1000000;
  39. sec = do_div(tmp, 60);
  40. min = do_div(tmp, 60);
  41. hrs = do_div(tmp, 24);
  42. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  43. hrs, min, sec, ms);
  44. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  45. if (tbl.bufq[i].active) {
  46. ts = &tbl.bufq[i].timestamp;
  47. tmp = ts->tv_sec;
  48. ms = (ts->tv_nsec) / 1000000;
  49. sec = do_div(tmp, 60);
  50. min = do_div(tmp, 60);
  51. hrs = do_div(tmp, 24);
  52. CAM_INFO(CAM_MEM,
  53. "%llu:%llu:%llu:%llu idx %d fd %d size %llu",
  54. hrs, min, sec, ms, i, tbl.bufq[i].fd,
  55. tbl.bufq[i].len);
  56. }
  57. }
  58. }
  59. static int cam_mem_util_get_dma_dir(uint32_t flags)
  60. {
  61. int rc = -EINVAL;
  62. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  63. rc = DMA_TO_DEVICE;
  64. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  65. rc = DMA_FROM_DEVICE;
  66. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  67. rc = DMA_BIDIRECTIONAL;
  68. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  69. rc = DMA_BIDIRECTIONAL;
  70. return rc;
  71. }
  72. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  73. uintptr_t *vaddr,
  74. size_t *len)
  75. {
  76. int rc = 0;
  77. void *addr;
  78. /*
  79. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  80. * need to be called in pair to avoid stability issue.
  81. */
  82. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  83. if (rc) {
  84. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  85. return rc;
  86. }
  87. addr = dma_buf_vmap(dmabuf);
  88. if (!addr) {
  89. CAM_ERR(CAM_MEM, "kernel map fail");
  90. *vaddr = 0;
  91. *len = 0;
  92. rc = -ENOSPC;
  93. goto fail;
  94. }
  95. *vaddr = (uint64_t)addr;
  96. *len = dmabuf->size;
  97. return 0;
  98. fail:
  99. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  100. return rc;
  101. }
  102. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  103. uint64_t vaddr)
  104. {
  105. int rc = 0;
  106. if (!dmabuf || !vaddr) {
  107. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  108. return -EINVAL;
  109. }
  110. dma_buf_vunmap(dmabuf, (void *)vaddr);
  111. /*
  112. * dma_buf_begin_cpu_access() and
  113. * dma_buf_end_cpu_access() need to be called in pair
  114. * to avoid stability issue.
  115. */
  116. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  117. if (rc) {
  118. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  119. dmabuf);
  120. return rc;
  121. }
  122. return rc;
  123. }
  124. static int cam_mem_mgr_create_debug_fs(void)
  125. {
  126. int rc = 0;
  127. struct dentry *dbgfileptr = NULL;
  128. dbgfileptr = debugfs_create_dir("camera_memmgr", NULL);
  129. if (!dbgfileptr) {
  130. CAM_ERR(CAM_MEM,"DebugFS could not create directory!");
  131. rc = -ENOENT;
  132. goto end;
  133. }
  134. /* Store parent inode for cleanup in caller */
  135. tbl.dentry = dbgfileptr;
  136. dbgfileptr = debugfs_create_bool("alloc_profile_enable", 0644,
  137. tbl.dentry, &tbl.alloc_profile_enable);
  138. if (IS_ERR(dbgfileptr)) {
  139. if (PTR_ERR(dbgfileptr) == -ENODEV)
  140. CAM_WARN(CAM_MEM, "DebugFS not enabled in kernel!");
  141. else
  142. rc = PTR_ERR(dbgfileptr);
  143. }
  144. end:
  145. return rc;
  146. }
  147. int cam_mem_mgr_init(void)
  148. {
  149. int i;
  150. int bitmap_size;
  151. int rc = 0;
  152. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  153. if (cam_smmu_need_force_alloc_cached(&tbl.force_cache_allocs)) {
  154. CAM_ERR(CAM_MEM, "Error in getting force cache alloc flag");
  155. return -EINVAL;
  156. }
  157. tbl.need_shared_buffer_padding = cam_smmu_need_shared_buffer_padding();
  158. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  159. rc = cam_mem_mgr_get_dma_heaps();
  160. if (rc) {
  161. CAM_ERR(CAM_MEM, "Failed in getting dma heaps rc=%d", rc);
  162. return rc;
  163. }
  164. #endif
  165. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  166. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  167. if (!tbl.bitmap) {
  168. rc = -ENOMEM;
  169. goto put_heaps;
  170. }
  171. tbl.bits = bitmap_size * BITS_PER_BYTE;
  172. bitmap_zero(tbl.bitmap, tbl.bits);
  173. /* We need to reserve slot 0 because 0 is invalid */
  174. set_bit(0, tbl.bitmap);
  175. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  176. tbl.bufq[i].fd = -1;
  177. tbl.bufq[i].buf_handle = -1;
  178. }
  179. mutex_init(&tbl.m_lock);
  180. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  181. cam_mem_mgr_create_debug_fs();
  182. return 0;
  183. put_heaps:
  184. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  185. cam_mem_mgr_put_dma_heaps();
  186. #endif
  187. return rc;
  188. }
  189. static int32_t cam_mem_get_slot(void)
  190. {
  191. int32_t idx;
  192. mutex_lock(&tbl.m_lock);
  193. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  194. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  195. mutex_unlock(&tbl.m_lock);
  196. return -ENOMEM;
  197. }
  198. set_bit(idx, tbl.bitmap);
  199. tbl.bufq[idx].active = true;
  200. ktime_get_real_ts64(&(tbl.bufq[idx].timestamp));
  201. mutex_init(&tbl.bufq[idx].q_lock);
  202. mutex_unlock(&tbl.m_lock);
  203. return idx;
  204. }
  205. static void cam_mem_put_slot(int32_t idx)
  206. {
  207. mutex_lock(&tbl.m_lock);
  208. mutex_lock(&tbl.bufq[idx].q_lock);
  209. tbl.bufq[idx].active = false;
  210. tbl.bufq[idx].is_internal = false;
  211. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  212. mutex_unlock(&tbl.bufq[idx].q_lock);
  213. mutex_destroy(&tbl.bufq[idx].q_lock);
  214. clear_bit(idx, tbl.bitmap);
  215. mutex_unlock(&tbl.m_lock);
  216. }
  217. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  218. dma_addr_t *iova_ptr, size_t *len_ptr, uint32_t *flags)
  219. {
  220. int rc = 0, idx;
  221. *len_ptr = 0;
  222. if (!atomic_read(&cam_mem_mgr_state)) {
  223. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  224. return -EINVAL;
  225. }
  226. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  227. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  228. return -ENOENT;
  229. if (!tbl.bufq[idx].active) {
  230. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  231. idx);
  232. return -EAGAIN;
  233. }
  234. mutex_lock(&tbl.bufq[idx].q_lock);
  235. if (buf_handle != tbl.bufq[idx].buf_handle) {
  236. rc = -EINVAL;
  237. goto handle_mismatch;
  238. }
  239. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  240. rc = cam_smmu_get_stage2_iova(mmu_handle, tbl.bufq[idx].fd,
  241. iova_ptr, len_ptr);
  242. else
  243. rc = cam_smmu_get_iova(mmu_handle, tbl.bufq[idx].fd,
  244. iova_ptr, len_ptr);
  245. if (rc) {
  246. CAM_ERR(CAM_MEM,
  247. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d",
  248. buf_handle, mmu_handle, tbl.bufq[idx].fd);
  249. goto handle_mismatch;
  250. }
  251. if (flags)
  252. *flags = tbl.bufq[idx].flags;
  253. CAM_DBG(CAM_MEM,
  254. "handle:0x%x fd:%d iova_ptr:0x%llx len_ptr:%llu",
  255. mmu_handle, tbl.bufq[idx].fd, iova_ptr, *len_ptr);
  256. handle_mismatch:
  257. mutex_unlock(&tbl.bufq[idx].q_lock);
  258. return rc;
  259. }
  260. EXPORT_SYMBOL(cam_mem_get_io_buf);
  261. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  262. {
  263. int idx;
  264. if (!atomic_read(&cam_mem_mgr_state)) {
  265. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  266. return -EINVAL;
  267. }
  268. if (!buf_handle || !vaddr_ptr || !len)
  269. return -EINVAL;
  270. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  271. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  272. return -EINVAL;
  273. if (!tbl.bufq[idx].active) {
  274. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  275. idx);
  276. return -EPERM;
  277. }
  278. if (buf_handle != tbl.bufq[idx].buf_handle)
  279. return -EINVAL;
  280. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  281. return -EINVAL;
  282. if (tbl.bufq[idx].kmdvaddr) {
  283. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  284. *len = tbl.bufq[idx].len;
  285. } else {
  286. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  287. buf_handle);
  288. return -EINVAL;
  289. }
  290. return 0;
  291. }
  292. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  293. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  294. {
  295. int rc = 0, idx;
  296. uint32_t cache_dir;
  297. unsigned long dmabuf_flag = 0;
  298. if (!atomic_read(&cam_mem_mgr_state)) {
  299. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  300. return -EINVAL;
  301. }
  302. if (!cmd)
  303. return -EINVAL;
  304. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  305. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  306. return -EINVAL;
  307. mutex_lock(&tbl.bufq[idx].q_lock);
  308. if (!tbl.bufq[idx].active) {
  309. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  310. idx);
  311. rc = -EINVAL;
  312. goto end;
  313. }
  314. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  315. rc = -EINVAL;
  316. goto end;
  317. }
  318. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  319. if (rc) {
  320. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  321. goto end;
  322. }
  323. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  324. CAM_DBG(CAM_MEM, "Calling dmap buf APIs for cache operations");
  325. cache_dir = DMA_BIDIRECTIONAL;
  326. #else
  327. if (dmabuf_flag & ION_FLAG_CACHED) {
  328. switch (cmd->mem_cache_ops) {
  329. case CAM_MEM_CLEAN_CACHE:
  330. cache_dir = DMA_TO_DEVICE;
  331. break;
  332. case CAM_MEM_INV_CACHE:
  333. cache_dir = DMA_FROM_DEVICE;
  334. break;
  335. case CAM_MEM_CLEAN_INV_CACHE:
  336. cache_dir = DMA_BIDIRECTIONAL;
  337. break;
  338. default:
  339. CAM_ERR(CAM_MEM,
  340. "invalid cache ops :%d", cmd->mem_cache_ops);
  341. rc = -EINVAL;
  342. goto end;
  343. }
  344. } else {
  345. CAM_DBG(CAM_MEM, "BUF is not cached");
  346. goto end;
  347. }
  348. #endif
  349. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  350. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  351. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  352. if (rc) {
  353. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  354. goto end;
  355. }
  356. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  357. cache_dir);
  358. if (rc) {
  359. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  360. goto end;
  361. }
  362. end:
  363. mutex_unlock(&tbl.bufq[idx].q_lock);
  364. return rc;
  365. }
  366. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  367. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  368. #define CAM_MAX_VMIDS 4
  369. static void cam_mem_mgr_put_dma_heaps(void)
  370. {
  371. CAM_DBG(CAM_MEM, "Releasing DMA Buf heaps usage");
  372. }
  373. static int cam_mem_mgr_get_dma_heaps(void)
  374. {
  375. int rc = 0;
  376. tbl.system_heap = NULL;
  377. tbl.system_uncached_heap = NULL;
  378. tbl.camera_heap = NULL;
  379. tbl.camera_uncached_heap = NULL;
  380. tbl.secure_display_heap = NULL;
  381. tbl.system_heap = dma_heap_find("qcom,system");
  382. if (IS_ERR_OR_NULL(tbl.system_heap)) {
  383. rc = PTR_ERR(tbl.system_heap);
  384. CAM_ERR(CAM_MEM, "qcom system heap not found, rc=%d", rc);
  385. tbl.system_heap = NULL;
  386. goto put_heaps;
  387. }
  388. tbl.system_uncached_heap = dma_heap_find("qcom,system-uncached");
  389. if (IS_ERR_OR_NULL(tbl.system_uncached_heap)) {
  390. if (tbl.force_cache_allocs) {
  391. /* optional, we anyway do not use uncached */
  392. CAM_DBG(CAM_MEM,
  393. "qcom system-uncached heap not found, err=%d",
  394. PTR_ERR(tbl.system_uncached_heap));
  395. tbl.system_uncached_heap = NULL;
  396. } else {
  397. /* fatal, must need uncached heaps */
  398. rc = PTR_ERR(tbl.system_uncached_heap);
  399. CAM_ERR(CAM_MEM,
  400. "qcom system-uncached heap not found, rc=%d",
  401. rc);
  402. tbl.system_uncached_heap = NULL;
  403. goto put_heaps;
  404. }
  405. }
  406. tbl.secure_display_heap = dma_heap_find("qcom,display");
  407. if (IS_ERR_OR_NULL(tbl.secure_display_heap)) {
  408. rc = PTR_ERR(tbl.secure_display_heap);
  409. CAM_ERR(CAM_MEM, "qcom,display heap not found, rc=%d",
  410. rc);
  411. tbl.secure_display_heap = NULL;
  412. goto put_heaps;
  413. }
  414. tbl.camera_heap = dma_heap_find("qcom,camera");
  415. if (IS_ERR_OR_NULL(tbl.camera_heap)) {
  416. /* optional heap, not a fatal error */
  417. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  418. PTR_ERR(tbl.camera_heap));
  419. tbl.camera_heap = NULL;
  420. }
  421. tbl.camera_uncached_heap = dma_heap_find("qcom,camera-uncached");
  422. if (IS_ERR_OR_NULL(tbl.camera_uncached_heap)) {
  423. /* optional heap, not a fatal error */
  424. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  425. PTR_ERR(tbl.camera_uncached_heap));
  426. tbl.camera_uncached_heap = NULL;
  427. }
  428. CAM_INFO(CAM_MEM,
  429. "Heaps : system=%pK, system_uncached=%pK, camera=%pK, camera-uncached=%pK, secure_display=%pK",
  430. tbl.system_heap, tbl.system_uncached_heap,
  431. tbl.camera_heap, tbl.camera_uncached_heap,
  432. tbl.secure_display_heap);
  433. return 0;
  434. put_heaps:
  435. cam_mem_mgr_put_dma_heaps();
  436. return rc;
  437. }
  438. static int cam_mem_util_get_dma_buf(size_t len,
  439. unsigned int cam_flags,
  440. struct dma_buf **buf)
  441. {
  442. int rc = 0;
  443. struct dma_heap *heap;
  444. struct dma_heap *try_heap = NULL;
  445. struct timespec64 ts1, ts2;
  446. long microsec = 0;
  447. bool use_cached_heap = false;
  448. struct mem_buf_lend_kernel_arg arg;
  449. int vmids[CAM_MAX_VMIDS];
  450. int perms[CAM_MAX_VMIDS];
  451. int num_vmids = 0;
  452. if (!buf) {
  453. CAM_ERR(CAM_MEM, "Invalid params");
  454. return -EINVAL;
  455. }
  456. if (tbl.alloc_profile_enable)
  457. CAM_GET_TIMESTAMP(ts1);
  458. if ((cam_flags & CAM_MEM_FLAG_CACHE) ||
  459. (tbl.force_cache_allocs &&
  460. (!(cam_flags & CAM_MEM_FLAG_PROTECTED_MODE)))) {
  461. CAM_DBG(CAM_MEM,
  462. "Using CACHED heap, cam_flags=0x%x, force_cache_allocs=%d",
  463. cam_flags, tbl.force_cache_allocs);
  464. use_cached_heap = true;
  465. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  466. use_cached_heap = true;
  467. CAM_DBG(CAM_MEM,
  468. "Using CACHED heap for secure, cam_flags=0x%x, force_cache_allocs=%d",
  469. cam_flags, tbl.force_cache_allocs);
  470. } else {
  471. use_cached_heap = false;
  472. CAM_ERR(CAM_MEM,
  473. "Using UNCACHED heap not supported, cam_flags=0x%x, force_cache_allocs=%d",
  474. cam_flags, tbl.force_cache_allocs);
  475. /*
  476. * Need a better handling based on whether dma-buf-heaps support
  477. * uncached heaps or not. For now, assume not supported.
  478. */
  479. return -EINVAL;
  480. }
  481. if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  482. heap = tbl.secure_display_heap;
  483. vmids[num_vmids] = VMID_CP_CAMERA;
  484. perms[num_vmids] = PERM_READ | PERM_WRITE;
  485. num_vmids++;
  486. if (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT) {
  487. CAM_DBG(CAM_MEM, "Secure mode CDSP flags");
  488. vmids[num_vmids] = VMID_CP_CDSP;
  489. perms[num_vmids] = PERM_READ | PERM_WRITE;
  490. num_vmids++;
  491. }
  492. } else if (use_cached_heap) {
  493. try_heap = tbl.camera_heap;
  494. heap = tbl.system_heap;
  495. } else {
  496. try_heap = tbl.camera_uncached_heap;
  497. heap = tbl.system_uncached_heap;
  498. }
  499. CAM_DBG(CAM_MEM, "Using heaps : try=%pK, heap=%pK", try_heap, heap);
  500. *buf = NULL;
  501. if (!try_heap && !heap) {
  502. CAM_ERR(CAM_MEM,
  503. "No heap available for allocation, cant allocate");
  504. return -EINVAL;
  505. }
  506. if (try_heap) {
  507. *buf = dma_heap_buffer_alloc(try_heap, len, O_RDWR, 0);
  508. if (IS_ERR(*buf)) {
  509. CAM_WARN(CAM_MEM,
  510. "Failed in allocating from try heap, heap=%pK, len=%zu, err=%d",
  511. try_heap, len, PTR_ERR(*buf));
  512. *buf = NULL;
  513. }
  514. }
  515. if (*buf == NULL) {
  516. *buf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0);
  517. if (IS_ERR(*buf)) {
  518. rc = PTR_ERR(*buf);
  519. CAM_ERR(CAM_MEM,
  520. "Failed in allocating from heap, heap=%pK, len=%zu, err=%d",
  521. heap, len, rc);
  522. *buf = NULL;
  523. return rc;
  524. }
  525. }
  526. if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  527. if (num_vmids >= CAM_MAX_VMIDS) {
  528. CAM_ERR(CAM_MEM, "Insufficient array size for vmids %d", num_vmids);
  529. rc = -EINVAL;
  530. goto end;
  531. }
  532. arg.nr_acl_entries = num_vmids;
  533. arg.vmids = vmids;
  534. arg.perms = perms;
  535. rc = mem_buf_lend(*buf, &arg);
  536. if (rc) {
  537. CAM_ERR(CAM_MEM,
  538. "Failed in buf lend rc=%d, buf=%pK, vmids [0]=0x%x, [1]=0x%x, [2]=0x%x",
  539. rc, *buf, vmids[0], vmids[1], vmids[2]);
  540. goto end;
  541. }
  542. }
  543. CAM_DBG(CAM_MEM, "Allocate success, len=%zu, *buf=%pK", len, *buf);
  544. if (tbl.alloc_profile_enable) {
  545. CAM_GET_TIMESTAMP(ts2);
  546. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  547. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  548. len, microsec);
  549. }
  550. return rc;
  551. end:
  552. dma_buf_put(*buf);
  553. return rc;
  554. }
  555. #else
  556. static int cam_mem_util_get_dma_buf(size_t len,
  557. unsigned int cam_flags,
  558. struct dma_buf **buf)
  559. {
  560. int rc = 0;
  561. unsigned int heap_id;
  562. int32_t ion_flag = 0;
  563. struct timespec64 ts1, ts2;
  564. long microsec = 0;
  565. if (!buf) {
  566. CAM_ERR(CAM_MEM, "Invalid params");
  567. return -EINVAL;
  568. }
  569. if (tbl.alloc_profile_enable)
  570. CAM_GET_TIMESTAMP(ts1);
  571. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  572. (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  573. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  574. ion_flag |=
  575. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  576. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  577. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  578. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  579. } else {
  580. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  581. ION_HEAP(ION_CAMERA_HEAP_ID);
  582. }
  583. if (cam_flags & CAM_MEM_FLAG_CACHE)
  584. ion_flag |= ION_FLAG_CACHED;
  585. else
  586. ion_flag &= ~ION_FLAG_CACHED;
  587. if (tbl.force_cache_allocs && (!(ion_flag & ION_FLAG_SECURE)))
  588. ion_flag |= ION_FLAG_CACHED;
  589. *buf = ion_alloc(len, heap_id, ion_flag);
  590. if (IS_ERR_OR_NULL(*buf))
  591. return -ENOMEM;
  592. if (tbl.alloc_profile_enable) {
  593. CAM_GET_TIMESTAMP(ts2);
  594. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  595. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  596. len, microsec);
  597. }
  598. return rc;
  599. }
  600. #endif
  601. static int cam_mem_util_buffer_alloc(size_t len, uint32_t flags,
  602. struct dma_buf **dmabuf,
  603. int *fd)
  604. {
  605. int rc;
  606. struct dma_buf *temp_dmabuf = NULL;
  607. rc = cam_mem_util_get_dma_buf(len, flags, dmabuf);
  608. if (rc) {
  609. CAM_ERR(CAM_MEM,
  610. "Error allocating dma buf : len=%llu, flags=0x%x",
  611. len, flags);
  612. return rc;
  613. }
  614. *fd = dma_buf_fd(*dmabuf, O_CLOEXEC);
  615. if (*fd < 0) {
  616. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  617. rc = -EINVAL;
  618. goto put_buf;
  619. }
  620. CAM_DBG(CAM_MEM, "Alloc success : len=%zu, *dmabuf=%pK, fd=%d",
  621. len, *dmabuf, *fd);
  622. /*
  623. * increment the ref count so that ref count becomes 2 here
  624. * when we close fd, refcount becomes 1 and when we do
  625. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  626. */
  627. temp_dmabuf = dma_buf_get(*fd);
  628. if (IS_ERR_OR_NULL(temp_dmabuf)) {
  629. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d", *fd);
  630. rc = -EINVAL;
  631. goto put_buf;
  632. }
  633. return rc;
  634. put_buf:
  635. dma_buf_put(*dmabuf);
  636. return rc;
  637. }
  638. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  639. {
  640. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  641. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  642. CAM_MEM_MMU_MAX_HANDLE);
  643. return -EINVAL;
  644. }
  645. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  646. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  647. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  648. return -EINVAL;
  649. }
  650. return 0;
  651. }
  652. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  653. {
  654. if (!cmd->flags) {
  655. CAM_ERR(CAM_MEM, "Invalid flags");
  656. return -EINVAL;
  657. }
  658. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  659. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  660. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  661. return -EINVAL;
  662. }
  663. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  664. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  665. CAM_ERR(CAM_MEM,
  666. "Kernel mapping in secure mode not allowed, flags=0x%x",
  667. cmd->flags);
  668. return -EINVAL;
  669. }
  670. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  671. CAM_ERR(CAM_MEM,
  672. "Shared memory buffers are not allowed to be mapped");
  673. return -EINVAL;
  674. }
  675. return 0;
  676. }
  677. static int cam_mem_util_map_hw_va(uint32_t flags,
  678. int32_t *mmu_hdls,
  679. int32_t num_hdls,
  680. int fd,
  681. dma_addr_t *hw_vaddr,
  682. size_t *len,
  683. enum cam_smmu_region_id region,
  684. bool is_internal)
  685. {
  686. int i;
  687. int rc = -1;
  688. int dir = cam_mem_util_get_dma_dir(flags);
  689. bool dis_delayed_unmap = false;
  690. if (dir < 0) {
  691. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  692. return dir;
  693. }
  694. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  695. dis_delayed_unmap = true;
  696. CAM_DBG(CAM_MEM,
  697. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  698. fd, flags, dir, num_hdls);
  699. for (i = 0; i < num_hdls; i++) {
  700. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  701. rc = cam_smmu_map_stage2_iova(mmu_hdls[i], fd, dir, hw_vaddr, len);
  702. else
  703. rc = cam_smmu_map_user_iova(mmu_hdls[i], fd, dis_delayed_unmap, dir,
  704. hw_vaddr, len, region, is_internal);
  705. if (rc) {
  706. CAM_ERR(CAM_MEM,
  707. "Failed %s map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  708. (flags & CAM_MEM_FLAG_PROTECTED_MODE) ? "" : "secured",
  709. i, fd, dir, mmu_hdls[i], rc);
  710. goto multi_map_fail;
  711. }
  712. }
  713. return rc;
  714. multi_map_fail:
  715. for (--i; i>= 0; i--) {
  716. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  717. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  718. else
  719. cam_smmu_unmap_user_iova(mmu_hdls[i], fd, CAM_SMMU_REGION_IO);
  720. }
  721. return rc;
  722. }
  723. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  724. {
  725. int rc;
  726. int32_t idx;
  727. struct dma_buf *dmabuf = NULL;
  728. int fd = -1;
  729. dma_addr_t hw_vaddr = 0;
  730. size_t len;
  731. uintptr_t kvaddr = 0;
  732. size_t klen;
  733. if (!atomic_read(&cam_mem_mgr_state)) {
  734. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  735. return -EINVAL;
  736. }
  737. if (!cmd) {
  738. CAM_ERR(CAM_MEM, " Invalid argument");
  739. return -EINVAL;
  740. }
  741. len = cmd->len;
  742. if (tbl.need_shared_buffer_padding &&
  743. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)) {
  744. len += CAM_MEM_SHARED_BUFFER_PAD_4K;
  745. CAM_DBG(CAM_MEM, "Pad 4k size, actual %llu, allocating %zu",
  746. cmd->len, len);
  747. }
  748. rc = cam_mem_util_check_alloc_flags(cmd);
  749. if (rc) {
  750. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  751. cmd->flags, rc);
  752. return rc;
  753. }
  754. rc = cam_mem_util_buffer_alloc(len, cmd->flags, &dmabuf, &fd);
  755. if (rc) {
  756. CAM_ERR(CAM_MEM,
  757. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  758. len, cmd->align, cmd->flags, cmd->num_hdl);
  759. cam_mem_mgr_print_tbl();
  760. return rc;
  761. }
  762. if (!dmabuf) {
  763. CAM_ERR(CAM_MEM,
  764. "Ion Alloc return NULL dmabuf! fd=%d, len=%d", fd, len);
  765. cam_mem_mgr_print_tbl();
  766. return rc;
  767. }
  768. idx = cam_mem_get_slot();
  769. if (idx < 0) {
  770. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  771. rc = -ENOMEM;
  772. goto slot_fail;
  773. }
  774. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  775. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  776. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  777. enum cam_smmu_region_id region;
  778. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  779. region = CAM_SMMU_REGION_IO;
  780. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  781. (cam_smmu_is_expanded_memory() && cmd->flags & CAM_MEM_FLAG_CMD_BUF_TYPE))
  782. region = CAM_SMMU_REGION_SHARED;
  783. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  784. region = CAM_SMMU_REGION_IO;
  785. rc = cam_mem_util_map_hw_va(cmd->flags,
  786. cmd->mmu_hdls,
  787. cmd->num_hdl,
  788. fd,
  789. &hw_vaddr,
  790. &len,
  791. region,
  792. true);
  793. if (rc) {
  794. CAM_ERR(CAM_MEM,
  795. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  796. len, cmd->flags,
  797. fd, region, cmd->num_hdl, rc);
  798. if (rc == -EALREADY) {
  799. if ((size_t)dmabuf->size != len)
  800. rc = -EBADR;
  801. cam_mem_mgr_print_tbl();
  802. }
  803. goto map_hw_fail;
  804. }
  805. }
  806. mutex_lock(&tbl.bufq[idx].q_lock);
  807. tbl.bufq[idx].fd = fd;
  808. tbl.bufq[idx].dma_buf = NULL;
  809. tbl.bufq[idx].flags = cmd->flags;
  810. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  811. tbl.bufq[idx].is_internal = true;
  812. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  813. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  814. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  815. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  816. if (rc) {
  817. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  818. dmabuf, rc);
  819. goto map_kernel_fail;
  820. }
  821. }
  822. if (cmd->flags & CAM_MEM_FLAG_KMD_DEBUG_FLAG)
  823. tbl.dbg_buf_idx = idx;
  824. tbl.bufq[idx].kmdvaddr = kvaddr;
  825. tbl.bufq[idx].vaddr = hw_vaddr;
  826. tbl.bufq[idx].dma_buf = dmabuf;
  827. tbl.bufq[idx].len = len;
  828. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  829. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  830. sizeof(int32_t) * cmd->num_hdl);
  831. tbl.bufq[idx].is_imported = false;
  832. mutex_unlock(&tbl.bufq[idx].q_lock);
  833. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  834. cmd->out.fd = tbl.bufq[idx].fd;
  835. cmd->out.vaddr = 0;
  836. CAM_DBG(CAM_MEM,
  837. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  838. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  839. tbl.bufq[idx].len);
  840. return rc;
  841. map_kernel_fail:
  842. mutex_unlock(&tbl.bufq[idx].q_lock);
  843. map_hw_fail:
  844. cam_mem_put_slot(idx);
  845. slot_fail:
  846. dma_buf_put(dmabuf);
  847. return rc;
  848. }
  849. static bool cam_mem_util_is_map_internal(int32_t fd)
  850. {
  851. uint32_t i;
  852. bool is_internal = false;
  853. mutex_lock(&tbl.m_lock);
  854. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  855. if (tbl.bufq[i].fd == fd) {
  856. is_internal = tbl.bufq[i].is_internal;
  857. break;
  858. }
  859. }
  860. mutex_unlock(&tbl.m_lock);
  861. return is_internal;
  862. }
  863. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  864. {
  865. int32_t idx;
  866. int rc;
  867. struct dma_buf *dmabuf;
  868. dma_addr_t hw_vaddr = 0;
  869. size_t len = 0;
  870. bool is_internal = false;
  871. if (!atomic_read(&cam_mem_mgr_state)) {
  872. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  873. return -EINVAL;
  874. }
  875. if (!cmd || (cmd->fd < 0)) {
  876. CAM_ERR(CAM_MEM, "Invalid argument");
  877. return -EINVAL;
  878. }
  879. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  880. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  881. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  882. return -EINVAL;
  883. }
  884. rc = cam_mem_util_check_map_flags(cmd);
  885. if (rc) {
  886. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  887. return rc;
  888. }
  889. dmabuf = dma_buf_get(cmd->fd);
  890. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  891. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  892. return -EINVAL;
  893. }
  894. is_internal = cam_mem_util_is_map_internal(cmd->fd);
  895. idx = cam_mem_get_slot();
  896. if (idx < 0) {
  897. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d, fd=%d",
  898. idx, cmd->fd);
  899. rc = -ENOMEM;
  900. goto slot_fail;
  901. }
  902. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  903. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  904. rc = cam_mem_util_map_hw_va(cmd->flags,
  905. cmd->mmu_hdls,
  906. cmd->num_hdl,
  907. cmd->fd,
  908. &hw_vaddr,
  909. &len,
  910. CAM_SMMU_REGION_IO,
  911. is_internal);
  912. if (rc) {
  913. CAM_ERR(CAM_MEM,
  914. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  915. cmd->flags, cmd->fd, len,
  916. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  917. if (rc == -EALREADY) {
  918. if ((size_t)dmabuf->size != len) {
  919. rc = -EBADR;
  920. cam_mem_mgr_print_tbl();
  921. }
  922. }
  923. goto map_fail;
  924. }
  925. }
  926. mutex_lock(&tbl.bufq[idx].q_lock);
  927. tbl.bufq[idx].fd = cmd->fd;
  928. tbl.bufq[idx].dma_buf = NULL;
  929. tbl.bufq[idx].flags = cmd->flags;
  930. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  931. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  932. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  933. tbl.bufq[idx].kmdvaddr = 0;
  934. if (cmd->num_hdl > 0)
  935. tbl.bufq[idx].vaddr = hw_vaddr;
  936. else
  937. tbl.bufq[idx].vaddr = 0;
  938. tbl.bufq[idx].dma_buf = dmabuf;
  939. tbl.bufq[idx].len = len;
  940. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  941. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  942. sizeof(int32_t) * cmd->num_hdl);
  943. tbl.bufq[idx].is_imported = true;
  944. tbl.bufq[idx].is_internal = is_internal;
  945. mutex_unlock(&tbl.bufq[idx].q_lock);
  946. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  947. cmd->out.vaddr = 0;
  948. cmd->out.size = (uint32_t)len;
  949. CAM_DBG(CAM_MEM,
  950. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  951. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  952. tbl.bufq[idx].len);
  953. return rc;
  954. map_fail:
  955. cam_mem_put_slot(idx);
  956. slot_fail:
  957. dma_buf_put(dmabuf);
  958. return rc;
  959. }
  960. static int cam_mem_util_unmap_hw_va(int32_t idx,
  961. enum cam_smmu_region_id region,
  962. enum cam_smmu_mapping_client client)
  963. {
  964. int i;
  965. uint32_t flags;
  966. int32_t *mmu_hdls;
  967. int num_hdls;
  968. int fd;
  969. int rc = 0;
  970. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  971. CAM_ERR(CAM_MEM, "Incorrect index");
  972. return -EINVAL;
  973. }
  974. flags = tbl.bufq[idx].flags;
  975. mmu_hdls = tbl.bufq[idx].hdls;
  976. num_hdls = tbl.bufq[idx].num_hdl;
  977. fd = tbl.bufq[idx].fd;
  978. CAM_DBG(CAM_MEM,
  979. "unmap_hw_va : idx=%d, fd=%x, flags=0x%x, num_hdls=%d, client=%d",
  980. idx, fd, flags, num_hdls, client);
  981. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  982. for (i = 0; i < num_hdls; i++) {
  983. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  984. if (rc < 0) {
  985. CAM_ERR(CAM_MEM,
  986. "Failed in secure unmap, i=%d, fd=%d, mmu_hdl=%d, rc=%d",
  987. i, fd, mmu_hdls[i], rc);
  988. goto unmap_end;
  989. }
  990. }
  991. } else {
  992. for (i = 0; i < num_hdls; i++) {
  993. if (client == CAM_SMMU_MAPPING_USER) {
  994. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  995. fd, region);
  996. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  997. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  998. tbl.bufq[idx].dma_buf, region);
  999. } else {
  1000. CAM_ERR(CAM_MEM,
  1001. "invalid caller for unmapping : %d",
  1002. client);
  1003. rc = -EINVAL;
  1004. }
  1005. if (rc < 0) {
  1006. CAM_ERR(CAM_MEM,
  1007. "Failed in unmap, i=%d, fd=%d, mmu_hdl=%d, region=%d, rc=%d",
  1008. i, fd, mmu_hdls[i], region, rc);
  1009. goto unmap_end;
  1010. }
  1011. }
  1012. }
  1013. return rc;
  1014. unmap_end:
  1015. CAM_ERR(CAM_MEM, "unmapping failed");
  1016. return rc;
  1017. }
  1018. static void cam_mem_mgr_unmap_active_buf(int idx)
  1019. {
  1020. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1021. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  1022. region = CAM_SMMU_REGION_SHARED;
  1023. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1024. region = CAM_SMMU_REGION_IO;
  1025. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  1026. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS)
  1027. cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1028. tbl.bufq[idx].kmdvaddr);
  1029. }
  1030. static int cam_mem_mgr_cleanup_table(void)
  1031. {
  1032. int i;
  1033. mutex_lock(&tbl.m_lock);
  1034. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  1035. if (!tbl.bufq[i].active) {
  1036. CAM_DBG(CAM_MEM,
  1037. "Buffer inactive at idx=%d, continuing", i);
  1038. continue;
  1039. } else {
  1040. CAM_DBG(CAM_MEM,
  1041. "Active buffer at idx=%d, possible leak needs unmapping",
  1042. i);
  1043. cam_mem_mgr_unmap_active_buf(i);
  1044. }
  1045. mutex_lock(&tbl.bufq[i].q_lock);
  1046. if (tbl.bufq[i].dma_buf) {
  1047. dma_buf_put(tbl.bufq[i].dma_buf);
  1048. tbl.bufq[i].dma_buf = NULL;
  1049. }
  1050. tbl.bufq[i].fd = -1;
  1051. tbl.bufq[i].flags = 0;
  1052. tbl.bufq[i].buf_handle = -1;
  1053. tbl.bufq[i].vaddr = 0;
  1054. tbl.bufq[i].len = 0;
  1055. memset(tbl.bufq[i].hdls, 0,
  1056. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  1057. tbl.bufq[i].num_hdl = 0;
  1058. tbl.bufq[i].dma_buf = NULL;
  1059. tbl.bufq[i].active = false;
  1060. tbl.bufq[i].is_internal = false;
  1061. mutex_unlock(&tbl.bufq[i].q_lock);
  1062. mutex_destroy(&tbl.bufq[i].q_lock);
  1063. }
  1064. bitmap_zero(tbl.bitmap, tbl.bits);
  1065. /* We need to reserve slot 0 because 0 is invalid */
  1066. set_bit(0, tbl.bitmap);
  1067. mutex_unlock(&tbl.m_lock);
  1068. return 0;
  1069. }
  1070. void cam_mem_mgr_deinit(void)
  1071. {
  1072. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  1073. cam_mem_mgr_cleanup_table();
  1074. debugfs_remove_recursive(tbl.dentry);
  1075. mutex_lock(&tbl.m_lock);
  1076. bitmap_zero(tbl.bitmap, tbl.bits);
  1077. kfree(tbl.bitmap);
  1078. tbl.bitmap = NULL;
  1079. tbl.dbg_buf_idx = -1;
  1080. mutex_unlock(&tbl.m_lock);
  1081. mutex_destroy(&tbl.m_lock);
  1082. }
  1083. static int cam_mem_util_unmap(int32_t idx,
  1084. enum cam_smmu_mapping_client client)
  1085. {
  1086. int rc = 0;
  1087. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1088. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1089. CAM_ERR(CAM_MEM, "Incorrect index");
  1090. return -EINVAL;
  1091. }
  1092. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  1093. mutex_lock(&tbl.m_lock);
  1094. if ((!tbl.bufq[idx].active) &&
  1095. (tbl.bufq[idx].vaddr) == 0) {
  1096. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  1097. idx);
  1098. mutex_unlock(&tbl.m_lock);
  1099. return 0;
  1100. }
  1101. /* Deactivate the buffer queue to prevent multiple unmap */
  1102. mutex_lock(&tbl.bufq[idx].q_lock);
  1103. tbl.bufq[idx].active = false;
  1104. tbl.bufq[idx].vaddr = 0;
  1105. mutex_unlock(&tbl.bufq[idx].q_lock);
  1106. mutex_unlock(&tbl.m_lock);
  1107. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1108. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  1109. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1110. tbl.bufq[idx].kmdvaddr);
  1111. if (rc)
  1112. CAM_ERR(CAM_MEM,
  1113. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  1114. tbl.bufq[idx].dma_buf,
  1115. (void *) tbl.bufq[idx].kmdvaddr);
  1116. }
  1117. }
  1118. /* SHARED flag gets precedence, all other flags after it */
  1119. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1120. region = CAM_SMMU_REGION_SHARED;
  1121. } else {
  1122. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1123. region = CAM_SMMU_REGION_IO;
  1124. }
  1125. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1126. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  1127. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1128. if (cam_mem_util_unmap_hw_va(idx, region, client))
  1129. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  1130. tbl.bufq[idx].dma_buf);
  1131. if (client == CAM_SMMU_MAPPING_KERNEL)
  1132. tbl.bufq[idx].dma_buf = NULL;
  1133. }
  1134. mutex_lock(&tbl.m_lock);
  1135. mutex_lock(&tbl.bufq[idx].q_lock);
  1136. tbl.bufq[idx].flags = 0;
  1137. tbl.bufq[idx].buf_handle = -1;
  1138. memset(tbl.bufq[idx].hdls, 0,
  1139. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  1140. CAM_DBG(CAM_MEM,
  1141. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK",
  1142. idx, tbl.bufq[idx].fd,
  1143. tbl.bufq[idx].is_imported,
  1144. tbl.bufq[idx].dma_buf);
  1145. if (tbl.bufq[idx].dma_buf)
  1146. dma_buf_put(tbl.bufq[idx].dma_buf);
  1147. tbl.bufq[idx].fd = -1;
  1148. tbl.bufq[idx].dma_buf = NULL;
  1149. tbl.bufq[idx].is_imported = false;
  1150. tbl.bufq[idx].is_internal = false;
  1151. tbl.bufq[idx].len = 0;
  1152. tbl.bufq[idx].num_hdl = 0;
  1153. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  1154. mutex_unlock(&tbl.bufq[idx].q_lock);
  1155. mutex_destroy(&tbl.bufq[idx].q_lock);
  1156. clear_bit(idx, tbl.bitmap);
  1157. mutex_unlock(&tbl.m_lock);
  1158. return rc;
  1159. }
  1160. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  1161. {
  1162. int idx;
  1163. int rc;
  1164. if (!atomic_read(&cam_mem_mgr_state)) {
  1165. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1166. return -EINVAL;
  1167. }
  1168. if (!cmd) {
  1169. CAM_ERR(CAM_MEM, "Invalid argument");
  1170. return -EINVAL;
  1171. }
  1172. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  1173. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1174. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  1175. idx);
  1176. return -EINVAL;
  1177. }
  1178. if (!tbl.bufq[idx].active) {
  1179. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1180. return -EINVAL;
  1181. }
  1182. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  1183. CAM_ERR(CAM_MEM,
  1184. "Released buf handle %d not matching within table %d, idx=%d",
  1185. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  1186. return -EINVAL;
  1187. }
  1188. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  1189. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  1190. return rc;
  1191. }
  1192. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  1193. struct cam_mem_mgr_memory_desc *out)
  1194. {
  1195. struct dma_buf *buf = NULL;
  1196. int ion_fd = -1;
  1197. int rc = 0;
  1198. uintptr_t kvaddr;
  1199. dma_addr_t iova = 0;
  1200. size_t request_len = 0;
  1201. uint32_t mem_handle;
  1202. int32_t idx;
  1203. int32_t smmu_hdl = 0;
  1204. int32_t num_hdl = 0;
  1205. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1206. if (!atomic_read(&cam_mem_mgr_state)) {
  1207. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1208. return -EINVAL;
  1209. }
  1210. if (!inp || !out) {
  1211. CAM_ERR(CAM_MEM, "Invalid params");
  1212. return -EINVAL;
  1213. }
  1214. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1215. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1216. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1217. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1218. return -EINVAL;
  1219. }
  1220. rc = cam_mem_util_get_dma_buf(inp->size, inp->flags, &buf);
  1221. if (rc) {
  1222. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1223. goto ion_fail;
  1224. } else if (!buf) {
  1225. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1226. goto ion_fail;
  1227. } else {
  1228. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1229. }
  1230. /*
  1231. * we are mapping kva always here,
  1232. * update flags so that we do unmap properly
  1233. */
  1234. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1235. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1236. if (rc) {
  1237. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1238. goto map_fail;
  1239. }
  1240. if (!inp->smmu_hdl) {
  1241. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1242. rc = -EINVAL;
  1243. goto smmu_fail;
  1244. }
  1245. /* SHARED flag gets precedence, all other flags after it */
  1246. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1247. region = CAM_SMMU_REGION_SHARED;
  1248. } else {
  1249. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1250. region = CAM_SMMU_REGION_IO;
  1251. }
  1252. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1253. buf,
  1254. CAM_SMMU_MAP_RW,
  1255. &iova,
  1256. &request_len,
  1257. region);
  1258. if (rc < 0) {
  1259. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1260. goto smmu_fail;
  1261. }
  1262. smmu_hdl = inp->smmu_hdl;
  1263. num_hdl = 1;
  1264. idx = cam_mem_get_slot();
  1265. if (idx < 0) {
  1266. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1267. rc = -ENOMEM;
  1268. goto slot_fail;
  1269. }
  1270. mutex_lock(&tbl.bufq[idx].q_lock);
  1271. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1272. tbl.bufq[idx].dma_buf = buf;
  1273. tbl.bufq[idx].fd = -1;
  1274. tbl.bufq[idx].flags = inp->flags;
  1275. tbl.bufq[idx].buf_handle = mem_handle;
  1276. tbl.bufq[idx].kmdvaddr = kvaddr;
  1277. tbl.bufq[idx].vaddr = iova;
  1278. tbl.bufq[idx].len = inp->size;
  1279. tbl.bufq[idx].num_hdl = num_hdl;
  1280. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1281. sizeof(int32_t));
  1282. tbl.bufq[idx].is_imported = false;
  1283. mutex_unlock(&tbl.bufq[idx].q_lock);
  1284. out->kva = kvaddr;
  1285. out->iova = (uint32_t)iova;
  1286. out->smmu_hdl = smmu_hdl;
  1287. out->mem_handle = mem_handle;
  1288. out->len = inp->size;
  1289. out->region = region;
  1290. return rc;
  1291. slot_fail:
  1292. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1293. buf, region);
  1294. smmu_fail:
  1295. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1296. map_fail:
  1297. dma_buf_put(buf);
  1298. ion_fail:
  1299. return rc;
  1300. }
  1301. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1302. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1303. {
  1304. int32_t idx;
  1305. int rc;
  1306. if (!atomic_read(&cam_mem_mgr_state)) {
  1307. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1308. return -EINVAL;
  1309. }
  1310. if (!inp) {
  1311. CAM_ERR(CAM_MEM, "Invalid argument");
  1312. return -EINVAL;
  1313. }
  1314. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1315. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1316. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1317. return -EINVAL;
  1318. }
  1319. if (!tbl.bufq[idx].active) {
  1320. if (tbl.bufq[idx].vaddr == 0) {
  1321. CAM_ERR(CAM_MEM, "buffer is released already");
  1322. return 0;
  1323. }
  1324. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1325. return -EINVAL;
  1326. }
  1327. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1328. CAM_ERR(CAM_MEM,
  1329. "Released buf handle not matching within table");
  1330. return -EINVAL;
  1331. }
  1332. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1333. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1334. return rc;
  1335. }
  1336. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1337. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1338. enum cam_smmu_region_id region,
  1339. struct cam_mem_mgr_memory_desc *out)
  1340. {
  1341. struct dma_buf *buf = NULL;
  1342. int rc = 0;
  1343. int ion_fd = -1;
  1344. dma_addr_t iova = 0;
  1345. size_t request_len = 0;
  1346. uint32_t mem_handle;
  1347. int32_t idx;
  1348. int32_t smmu_hdl = 0;
  1349. int32_t num_hdl = 0;
  1350. uintptr_t kvaddr = 0;
  1351. if (!atomic_read(&cam_mem_mgr_state)) {
  1352. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1353. return -EINVAL;
  1354. }
  1355. if (!inp || !out) {
  1356. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1357. return -EINVAL;
  1358. }
  1359. if (!inp->smmu_hdl) {
  1360. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1361. return -EINVAL;
  1362. }
  1363. if ((region != CAM_SMMU_REGION_SECHEAP) &&
  1364. (region != CAM_SMMU_REGION_FWUNCACHED)) {
  1365. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1366. return -EINVAL;
  1367. }
  1368. rc = cam_mem_util_get_dma_buf(inp->size, 0, &buf);
  1369. if (rc) {
  1370. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1371. goto ion_fail;
  1372. } else if (!buf) {
  1373. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1374. goto ion_fail;
  1375. } else {
  1376. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1377. }
  1378. if (inp->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1379. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1380. if (rc) {
  1381. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1382. goto kmap_fail;
  1383. }
  1384. }
  1385. rc = cam_smmu_reserve_buf_region(region,
  1386. inp->smmu_hdl, buf, &iova, &request_len);
  1387. if (rc) {
  1388. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1389. goto smmu_fail;
  1390. }
  1391. smmu_hdl = inp->smmu_hdl;
  1392. num_hdl = 1;
  1393. idx = cam_mem_get_slot();
  1394. if (idx < 0) {
  1395. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1396. rc = -ENOMEM;
  1397. goto slot_fail;
  1398. }
  1399. mutex_lock(&tbl.bufq[idx].q_lock);
  1400. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1401. tbl.bufq[idx].fd = -1;
  1402. tbl.bufq[idx].dma_buf = buf;
  1403. tbl.bufq[idx].flags = inp->flags;
  1404. tbl.bufq[idx].buf_handle = mem_handle;
  1405. tbl.bufq[idx].kmdvaddr = kvaddr;
  1406. tbl.bufq[idx].vaddr = iova;
  1407. tbl.bufq[idx].len = request_len;
  1408. tbl.bufq[idx].num_hdl = num_hdl;
  1409. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1410. sizeof(int32_t));
  1411. tbl.bufq[idx].is_imported = false;
  1412. mutex_unlock(&tbl.bufq[idx].q_lock);
  1413. out->kva = kvaddr;
  1414. out->iova = (uint32_t)iova;
  1415. out->smmu_hdl = smmu_hdl;
  1416. out->mem_handle = mem_handle;
  1417. out->len = request_len;
  1418. out->region = region;
  1419. return rc;
  1420. slot_fail:
  1421. cam_smmu_release_buf_region(region, smmu_hdl);
  1422. smmu_fail:
  1423. if (region == CAM_SMMU_REGION_FWUNCACHED)
  1424. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1425. kmap_fail:
  1426. dma_buf_put(buf);
  1427. ion_fail:
  1428. return rc;
  1429. }
  1430. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1431. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1432. {
  1433. int32_t idx;
  1434. int rc;
  1435. int32_t smmu_hdl;
  1436. if (!atomic_read(&cam_mem_mgr_state)) {
  1437. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1438. return -EINVAL;
  1439. }
  1440. if (!inp) {
  1441. CAM_ERR(CAM_MEM, "Invalid argument");
  1442. return -EINVAL;
  1443. }
  1444. if ((inp->region != CAM_SMMU_REGION_SECHEAP) &&
  1445. (inp->region != CAM_SMMU_REGION_FWUNCACHED)) {
  1446. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1447. return -EINVAL;
  1448. }
  1449. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1450. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1451. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1452. return -EINVAL;
  1453. }
  1454. if (!tbl.bufq[idx].active) {
  1455. if (tbl.bufq[idx].vaddr == 0) {
  1456. CAM_ERR(CAM_MEM, "buffer is released already");
  1457. return 0;
  1458. }
  1459. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1460. return -EINVAL;
  1461. }
  1462. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1463. CAM_ERR(CAM_MEM,
  1464. "Released buf handle not matching within table");
  1465. return -EINVAL;
  1466. }
  1467. if (tbl.bufq[idx].num_hdl != 1) {
  1468. CAM_ERR(CAM_MEM,
  1469. "Sec heap region should have only one smmu hdl");
  1470. return -ENODEV;
  1471. }
  1472. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1473. sizeof(int32_t));
  1474. if (inp->smmu_hdl != smmu_hdl) {
  1475. CAM_ERR(CAM_MEM,
  1476. "Passed SMMU handle doesn't match with internal hdl");
  1477. return -ENODEV;
  1478. }
  1479. rc = cam_smmu_release_buf_region(inp->region, inp->smmu_hdl);
  1480. if (rc) {
  1481. CAM_ERR(CAM_MEM,
  1482. "Sec heap region release failed");
  1483. return -ENODEV;
  1484. }
  1485. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1486. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1487. if (rc)
  1488. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1489. return rc;
  1490. }
  1491. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);
  1492. #ifndef CONFIG_CAM_PRESIL
  1493. struct dma_buf * cam_mem_mgr_get_dma_buf(int fd)
  1494. {
  1495. return NULL;
  1496. }
  1497. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1498. {
  1499. return 0;
  1500. }
  1501. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1502. {
  1503. return 0;
  1504. }
  1505. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle,
  1506. uint32_t buf_size,
  1507. uint32_t offset,
  1508. int32_t iommu_hdl)
  1509. {
  1510. return 0;
  1511. }
  1512. #endif