cam_mem_mgr.c 42 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include <linux/debugfs.h>
  12. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  13. #include <linux/mem-buf.h>
  14. #include <soc/qcom/secure_buffer.h>
  15. #endif
  16. #include "cam_compat.h"
  17. #include "cam_req_mgr_util.h"
  18. #include "cam_mem_mgr.h"
  19. #include "cam_smmu_api.h"
  20. #include "cam_debug_util.h"
  21. #include "cam_trace.h"
  22. #include "cam_common_util.h"
  23. #define CAM_MEM_SHARED_BUFFER_PAD_4K (4 * 1024)
  24. static struct cam_mem_table tbl;
  25. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  26. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  27. static void cam_mem_mgr_put_dma_heaps(void);
  28. static int cam_mem_mgr_get_dma_heaps(void);
  29. #endif
  30. static void cam_mem_mgr_print_tbl(void)
  31. {
  32. int i;
  33. uint64_t ms, tmp, hrs, min, sec;
  34. struct timespec64 *ts = NULL;
  35. struct timespec64 current_ts;
  36. ktime_get_real_ts64(&(current_ts));
  37. tmp = current_ts.tv_sec;
  38. ms = (current_ts.tv_nsec) / 1000000;
  39. sec = do_div(tmp, 60);
  40. min = do_div(tmp, 60);
  41. hrs = do_div(tmp, 24);
  42. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  43. hrs, min, sec, ms);
  44. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  45. if (tbl.bufq[i].active) {
  46. ts = &tbl.bufq[i].timestamp;
  47. tmp = ts->tv_sec;
  48. ms = (ts->tv_nsec) / 1000000;
  49. sec = do_div(tmp, 60);
  50. min = do_div(tmp, 60);
  51. hrs = do_div(tmp, 24);
  52. CAM_INFO(CAM_MEM,
  53. "%llu:%llu:%llu:%llu idx %d fd %d size %llu",
  54. hrs, min, sec, ms, i, tbl.bufq[i].fd,
  55. tbl.bufq[i].len);
  56. }
  57. }
  58. }
  59. static int cam_mem_util_get_dma_dir(uint32_t flags)
  60. {
  61. int rc = -EINVAL;
  62. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  63. rc = DMA_TO_DEVICE;
  64. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  65. rc = DMA_FROM_DEVICE;
  66. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  67. rc = DMA_BIDIRECTIONAL;
  68. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  69. rc = DMA_BIDIRECTIONAL;
  70. return rc;
  71. }
  72. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  73. uintptr_t *vaddr,
  74. size_t *len)
  75. {
  76. int rc = 0;
  77. void *addr;
  78. /*
  79. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  80. * need to be called in pair to avoid stability issue.
  81. */
  82. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  83. if (rc) {
  84. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  85. return rc;
  86. }
  87. addr = dma_buf_vmap(dmabuf);
  88. if (!addr) {
  89. CAM_ERR(CAM_MEM, "kernel map fail");
  90. *vaddr = 0;
  91. *len = 0;
  92. rc = -ENOSPC;
  93. goto fail;
  94. }
  95. *vaddr = (uint64_t)addr;
  96. *len = dmabuf->size;
  97. return 0;
  98. fail:
  99. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  100. return rc;
  101. }
  102. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  103. uint64_t vaddr)
  104. {
  105. int rc = 0;
  106. if (!dmabuf || !vaddr) {
  107. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  108. return -EINVAL;
  109. }
  110. dma_buf_vunmap(dmabuf, (void *)vaddr);
  111. /*
  112. * dma_buf_begin_cpu_access() and
  113. * dma_buf_end_cpu_access() need to be called in pair
  114. * to avoid stability issue.
  115. */
  116. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  117. if (rc) {
  118. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  119. dmabuf);
  120. return rc;
  121. }
  122. return rc;
  123. }
  124. static int cam_mem_mgr_create_debug_fs(void)
  125. {
  126. int rc = 0;
  127. struct dentry *dbgfileptr = NULL;
  128. dbgfileptr = debugfs_create_dir("camera_memmgr", NULL);
  129. if (!dbgfileptr) {
  130. CAM_ERR(CAM_MEM,"DebugFS could not create directory!");
  131. rc = -ENOENT;
  132. goto end;
  133. }
  134. /* Store parent inode for cleanup in caller */
  135. tbl.dentry = dbgfileptr;
  136. dbgfileptr = debugfs_create_bool("alloc_profile_enable", 0644,
  137. tbl.dentry, &tbl.alloc_profile_enable);
  138. if (IS_ERR(dbgfileptr)) {
  139. if (PTR_ERR(dbgfileptr) == -ENODEV)
  140. CAM_WARN(CAM_MEM, "DebugFS not enabled in kernel!");
  141. else
  142. rc = PTR_ERR(dbgfileptr);
  143. }
  144. end:
  145. return rc;
  146. }
  147. int cam_mem_mgr_init(void)
  148. {
  149. int i;
  150. int bitmap_size;
  151. int rc = 0;
  152. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  153. if (cam_smmu_need_force_alloc_cached(&tbl.force_cache_allocs)) {
  154. CAM_ERR(CAM_MEM, "Error in getting force cache alloc flag");
  155. return -EINVAL;
  156. }
  157. tbl.need_shared_buffer_padding = cam_smmu_need_shared_buffer_padding();
  158. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  159. rc = cam_mem_mgr_get_dma_heaps();
  160. if (rc) {
  161. CAM_ERR(CAM_MEM, "Failed in getting dma heaps rc=%d", rc);
  162. return rc;
  163. }
  164. #endif
  165. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  166. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  167. if (!tbl.bitmap) {
  168. rc = -ENOMEM;
  169. goto put_heaps;
  170. }
  171. tbl.bits = bitmap_size * BITS_PER_BYTE;
  172. bitmap_zero(tbl.bitmap, tbl.bits);
  173. /* We need to reserve slot 0 because 0 is invalid */
  174. set_bit(0, tbl.bitmap);
  175. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  176. tbl.bufq[i].fd = -1;
  177. tbl.bufq[i].buf_handle = -1;
  178. }
  179. mutex_init(&tbl.m_lock);
  180. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  181. cam_mem_mgr_create_debug_fs();
  182. return 0;
  183. put_heaps:
  184. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  185. cam_mem_mgr_put_dma_heaps();
  186. #endif
  187. return rc;
  188. }
  189. static int32_t cam_mem_get_slot(void)
  190. {
  191. int32_t idx;
  192. mutex_lock(&tbl.m_lock);
  193. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  194. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  195. mutex_unlock(&tbl.m_lock);
  196. return -ENOMEM;
  197. }
  198. set_bit(idx, tbl.bitmap);
  199. tbl.bufq[idx].active = true;
  200. ktime_get_real_ts64(&(tbl.bufq[idx].timestamp));
  201. mutex_init(&tbl.bufq[idx].q_lock);
  202. mutex_unlock(&tbl.m_lock);
  203. return idx;
  204. }
  205. static void cam_mem_put_slot(int32_t idx)
  206. {
  207. mutex_lock(&tbl.m_lock);
  208. mutex_lock(&tbl.bufq[idx].q_lock);
  209. tbl.bufq[idx].active = false;
  210. tbl.bufq[idx].is_internal = false;
  211. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  212. mutex_unlock(&tbl.bufq[idx].q_lock);
  213. mutex_destroy(&tbl.bufq[idx].q_lock);
  214. clear_bit(idx, tbl.bitmap);
  215. mutex_unlock(&tbl.m_lock);
  216. }
  217. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  218. dma_addr_t *iova_ptr, size_t *len_ptr)
  219. {
  220. int rc = 0, idx;
  221. *len_ptr = 0;
  222. if (!atomic_read(&cam_mem_mgr_state)) {
  223. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  224. return -EINVAL;
  225. }
  226. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  227. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  228. return -ENOENT;
  229. if (!tbl.bufq[idx].active) {
  230. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  231. idx);
  232. return -EAGAIN;
  233. }
  234. mutex_lock(&tbl.bufq[idx].q_lock);
  235. if (buf_handle != tbl.bufq[idx].buf_handle) {
  236. rc = -EINVAL;
  237. goto handle_mismatch;
  238. }
  239. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  240. rc = cam_smmu_get_stage2_iova(mmu_handle,
  241. tbl.bufq[idx].fd,
  242. iova_ptr,
  243. len_ptr);
  244. else
  245. rc = cam_smmu_get_iova(mmu_handle,
  246. tbl.bufq[idx].fd,
  247. iova_ptr,
  248. len_ptr);
  249. if (rc) {
  250. CAM_ERR(CAM_MEM,
  251. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d",
  252. buf_handle, mmu_handle, tbl.bufq[idx].fd);
  253. goto handle_mismatch;
  254. }
  255. CAM_DBG(CAM_MEM,
  256. "handle:0x%x fd:%d iova_ptr:0x%llx len_ptr:%llu",
  257. mmu_handle, tbl.bufq[idx].fd, iova_ptr, *len_ptr);
  258. handle_mismatch:
  259. mutex_unlock(&tbl.bufq[idx].q_lock);
  260. return rc;
  261. }
  262. EXPORT_SYMBOL(cam_mem_get_io_buf);
  263. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  264. {
  265. int idx;
  266. if (!atomic_read(&cam_mem_mgr_state)) {
  267. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  268. return -EINVAL;
  269. }
  270. if (!atomic_read(&cam_mem_mgr_state)) {
  271. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  272. return -EINVAL;
  273. }
  274. if (!buf_handle || !vaddr_ptr || !len)
  275. return -EINVAL;
  276. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  277. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  278. return -EINVAL;
  279. if (!tbl.bufq[idx].active) {
  280. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  281. idx);
  282. return -EPERM;
  283. }
  284. if (buf_handle != tbl.bufq[idx].buf_handle)
  285. return -EINVAL;
  286. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  287. return -EINVAL;
  288. if (tbl.bufq[idx].kmdvaddr) {
  289. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  290. *len = tbl.bufq[idx].len;
  291. } else {
  292. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  293. buf_handle);
  294. return -EINVAL;
  295. }
  296. return 0;
  297. }
  298. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  299. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  300. {
  301. int rc = 0, idx;
  302. uint32_t cache_dir;
  303. unsigned long dmabuf_flag = 0;
  304. if (!atomic_read(&cam_mem_mgr_state)) {
  305. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  306. return -EINVAL;
  307. }
  308. if (!cmd)
  309. return -EINVAL;
  310. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  311. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  312. return -EINVAL;
  313. mutex_lock(&tbl.bufq[idx].q_lock);
  314. if (!tbl.bufq[idx].active) {
  315. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  316. idx);
  317. rc = -EINVAL;
  318. goto end;
  319. }
  320. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  321. rc = -EINVAL;
  322. goto end;
  323. }
  324. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  325. if (rc) {
  326. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  327. goto end;
  328. }
  329. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  330. CAM_DBG(CAM_MEM, "Calling dmap buf APIs for cache operations");
  331. cache_dir = DMA_BIDIRECTIONAL;
  332. #else
  333. if (dmabuf_flag & ION_FLAG_CACHED) {
  334. switch (cmd->mem_cache_ops) {
  335. case CAM_MEM_CLEAN_CACHE:
  336. cache_dir = DMA_TO_DEVICE;
  337. break;
  338. case CAM_MEM_INV_CACHE:
  339. cache_dir = DMA_FROM_DEVICE;
  340. break;
  341. case CAM_MEM_CLEAN_INV_CACHE:
  342. cache_dir = DMA_BIDIRECTIONAL;
  343. break;
  344. default:
  345. CAM_ERR(CAM_MEM,
  346. "invalid cache ops :%d", cmd->mem_cache_ops);
  347. rc = -EINVAL;
  348. goto end;
  349. }
  350. } else {
  351. CAM_DBG(CAM_MEM, "BUF is not cached");
  352. goto end;
  353. }
  354. #endif
  355. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  356. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  357. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  358. if (rc) {
  359. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  360. goto end;
  361. }
  362. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  363. cache_dir);
  364. if (rc) {
  365. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  366. goto end;
  367. }
  368. end:
  369. mutex_unlock(&tbl.bufq[idx].q_lock);
  370. return rc;
  371. }
  372. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  373. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  374. #define CAM_MAX_VMIDS 4
  375. static void cam_mem_mgr_put_dma_heaps(void)
  376. {
  377. CAM_DBG(CAM_MEM, "Releasing DMA Buf heaps usage");
  378. }
  379. static int cam_mem_mgr_get_dma_heaps(void)
  380. {
  381. int rc = 0;
  382. tbl.system_heap = NULL;
  383. tbl.system_uncached_heap = NULL;
  384. tbl.camera_heap = NULL;
  385. tbl.camera_uncached_heap = NULL;
  386. tbl.secure_display_heap = NULL;
  387. tbl.system_heap = dma_heap_find("qcom,system");
  388. if (IS_ERR_OR_NULL(tbl.system_heap)) {
  389. rc = PTR_ERR(tbl.system_heap);
  390. CAM_ERR(CAM_MEM, "qcom system heap not found, rc=%d", rc);
  391. tbl.system_heap = NULL;
  392. goto put_heaps;
  393. }
  394. tbl.system_uncached_heap = dma_heap_find("qcom,system-uncached");
  395. if (IS_ERR_OR_NULL(tbl.system_uncached_heap)) {
  396. if (tbl.force_cache_allocs) {
  397. /* optional, we anyway do not use uncached */
  398. CAM_DBG(CAM_MEM,
  399. "qcom system-uncached heap not found, err=%d",
  400. PTR_ERR(tbl.system_uncached_heap));
  401. tbl.system_uncached_heap = NULL;
  402. } else {
  403. /* fatal, must need uncached heaps */
  404. rc = PTR_ERR(tbl.system_uncached_heap);
  405. CAM_ERR(CAM_MEM,
  406. "qcom system-uncached heap not found, rc=%d",
  407. rc);
  408. tbl.system_uncached_heap = NULL;
  409. goto put_heaps;
  410. }
  411. }
  412. tbl.secure_display_heap = dma_heap_find("qcom,display");
  413. if (IS_ERR_OR_NULL(tbl.secure_display_heap)) {
  414. rc = PTR_ERR(tbl.secure_display_heap);
  415. CAM_ERR(CAM_MEM, "qcom,display heap not found, rc=%d",
  416. rc);
  417. tbl.secure_display_heap = NULL;
  418. goto put_heaps;
  419. }
  420. tbl.camera_heap = dma_heap_find("qcom,camera");
  421. if (IS_ERR_OR_NULL(tbl.camera_heap)) {
  422. /* optional heap, not a fatal error */
  423. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  424. PTR_ERR(tbl.camera_heap));
  425. tbl.camera_heap = NULL;
  426. }
  427. tbl.camera_uncached_heap = dma_heap_find("qcom,camera-uncached");
  428. if (IS_ERR_OR_NULL(tbl.camera_uncached_heap)) {
  429. /* optional heap, not a fatal error */
  430. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  431. PTR_ERR(tbl.camera_uncached_heap));
  432. tbl.camera_uncached_heap = NULL;
  433. }
  434. CAM_INFO(CAM_MEM,
  435. "Heaps : system=%pK, system_uncached=%pK, camera=%pK, camera-uncached=%pK, secure_display=%pK",
  436. tbl.system_heap, tbl.system_uncached_heap,
  437. tbl.camera_heap, tbl.camera_uncached_heap,
  438. tbl.secure_display_heap);
  439. return 0;
  440. put_heaps:
  441. cam_mem_mgr_put_dma_heaps();
  442. return rc;
  443. }
  444. static int cam_mem_util_get_dma_buf(size_t len,
  445. unsigned int cam_flags,
  446. struct dma_buf **buf)
  447. {
  448. int rc = 0;
  449. struct dma_heap *heap;
  450. struct dma_heap *try_heap = NULL;
  451. struct timespec64 ts1, ts2;
  452. long microsec = 0;
  453. bool use_cached_heap = false;
  454. struct mem_buf_lend_kernel_arg arg;
  455. int vmids[CAM_MAX_VMIDS];
  456. int perms[CAM_MAX_VMIDS];
  457. int num_vmids = 0;
  458. if (!buf) {
  459. CAM_ERR(CAM_MEM, "Invalid params");
  460. return -EINVAL;
  461. }
  462. if (tbl.alloc_profile_enable)
  463. CAM_GET_TIMESTAMP(ts1);
  464. if ((cam_flags & CAM_MEM_FLAG_CACHE) ||
  465. (tbl.force_cache_allocs &&
  466. (!(cam_flags & CAM_MEM_FLAG_PROTECTED_MODE)))) {
  467. CAM_DBG(CAM_MEM,
  468. "Using CACHED heap, cam_flags=0x%x, force_cache_allocs=%d",
  469. cam_flags, tbl.force_cache_allocs);
  470. use_cached_heap = true;
  471. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  472. use_cached_heap = true;
  473. CAM_DBG(CAM_MEM,
  474. "Using CACHED heap for secure, cam_flags=0x%x, force_cache_allocs=%d",
  475. cam_flags, tbl.force_cache_allocs);
  476. } else {
  477. use_cached_heap = false;
  478. CAM_ERR(CAM_MEM,
  479. "Using UNCACHED heap not supported, cam_flags=0x%x, force_cache_allocs=%d",
  480. cam_flags, tbl.force_cache_allocs);
  481. /*
  482. * Need a better handling based on whether dma-buf-heaps support
  483. * uncached heaps or not. For now, assume not supported.
  484. */
  485. return -EINVAL;
  486. }
  487. if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  488. heap = tbl.secure_display_heap;
  489. vmids[num_vmids] = VMID_CP_CAMERA;
  490. perms[num_vmids] = PERM_READ | PERM_WRITE;
  491. num_vmids++;
  492. if (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT) {
  493. CAM_DBG(CAM_MEM, "Secure mode CDSP flags");
  494. vmids[num_vmids] = VMID_CP_CDSP;
  495. perms[num_vmids] = PERM_READ | PERM_WRITE;
  496. num_vmids++;
  497. }
  498. } else if (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL) {
  499. heap = tbl.secure_display_heap;
  500. vmids[num_vmids] = VMID_CP_NON_PIXEL;
  501. perms[num_vmids] = PERM_READ | PERM_WRITE;
  502. num_vmids++;
  503. } else if (use_cached_heap) {
  504. try_heap = tbl.camera_heap;
  505. heap = tbl.system_heap;
  506. } else {
  507. try_heap = tbl.camera_uncached_heap;
  508. heap = tbl.system_uncached_heap;
  509. }
  510. CAM_DBG(CAM_MEM, "Using heaps : try=%pK, heap=%pK", try_heap, heap);
  511. *buf = NULL;
  512. if (!try_heap && !heap) {
  513. CAM_ERR(CAM_MEM,
  514. "No heap available for allocation, cant allocate");
  515. return -EINVAL;
  516. }
  517. if (try_heap) {
  518. *buf = dma_heap_buffer_alloc(try_heap, len, O_RDWR, 0);
  519. if (IS_ERR(*buf)) {
  520. CAM_WARN(CAM_MEM,
  521. "Failed in allocating from try heap, heap=%pK, len=%zu, err=%d",
  522. try_heap, len, PTR_ERR(*buf));
  523. *buf = NULL;
  524. }
  525. }
  526. if (*buf == NULL) {
  527. *buf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0);
  528. if (IS_ERR(*buf)) {
  529. rc = PTR_ERR(*buf);
  530. CAM_ERR(CAM_MEM,
  531. "Failed in allocating from heap, heap=%pK, len=%zu, err=%d",
  532. heap, len, rc);
  533. *buf = NULL;
  534. return rc;
  535. }
  536. }
  537. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) ||
  538. (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL)) {
  539. if (num_vmids >= CAM_MAX_VMIDS) {
  540. CAM_ERR(CAM_MEM, "Insufficient array size for vmids %d", num_vmids);
  541. rc = -EINVAL;
  542. goto end;
  543. }
  544. arg.nr_acl_entries = num_vmids;
  545. arg.vmids = vmids;
  546. arg.perms = perms;
  547. rc = mem_buf_lend(*buf, &arg);
  548. if (rc) {
  549. CAM_ERR(CAM_MEM,
  550. "Failed in buf lend rc=%d, buf=%pK, vmids [0]=0x%x, [1]=0x%x, [2]=0x%x",
  551. rc, *buf, vmids[0], vmids[1], vmids[2]);
  552. goto end;
  553. }
  554. }
  555. CAM_DBG(CAM_MEM, "Allocate success, len=%zu, *buf=%pK", len, *buf);
  556. if (tbl.alloc_profile_enable) {
  557. CAM_GET_TIMESTAMP(ts2);
  558. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  559. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  560. len, microsec);
  561. }
  562. return rc;
  563. end:
  564. dma_buf_put(*buf);
  565. return rc;
  566. }
  567. #else
  568. static int cam_mem_util_get_dma_buf(size_t len,
  569. unsigned int cam_flags,
  570. struct dma_buf **buf)
  571. {
  572. int rc = 0;
  573. unsigned int heap_id;
  574. int32_t ion_flag = 0;
  575. struct timespec64 ts1, ts2;
  576. long microsec = 0;
  577. if (!buf) {
  578. CAM_ERR(CAM_MEM, "Invalid params");
  579. return -EINVAL;
  580. }
  581. if (tbl.alloc_profile_enable)
  582. CAM_GET_TIMESTAMP(ts1);
  583. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  584. (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  585. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  586. ion_flag |=
  587. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  588. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  589. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  590. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  591. } else {
  592. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  593. ION_HEAP(ION_CAMERA_HEAP_ID);
  594. }
  595. if (cam_flags & CAM_MEM_FLAG_CACHE)
  596. ion_flag |= ION_FLAG_CACHED;
  597. else
  598. ion_flag &= ~ION_FLAG_CACHED;
  599. if (tbl.force_cache_allocs && (!(ion_flag & ION_FLAG_SECURE)))
  600. ion_flag |= ION_FLAG_CACHED;
  601. *buf = ion_alloc(len, heap_id, ion_flag);
  602. if (IS_ERR_OR_NULL(*buf))
  603. return -ENOMEM;
  604. if (tbl.alloc_profile_enable) {
  605. CAM_GET_TIMESTAMP(ts2);
  606. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  607. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  608. len, microsec);
  609. }
  610. return rc;
  611. }
  612. #endif
  613. static int cam_mem_util_buffer_alloc(size_t len, uint32_t flags,
  614. struct dma_buf **dmabuf,
  615. int *fd)
  616. {
  617. int rc;
  618. struct dma_buf *temp_dmabuf = NULL;
  619. rc = cam_mem_util_get_dma_buf(len, flags, dmabuf);
  620. if (rc) {
  621. CAM_ERR(CAM_MEM,
  622. "Error allocating dma buf : len=%llu, flags=0x%x",
  623. len, flags);
  624. return rc;
  625. }
  626. *fd = dma_buf_fd(*dmabuf, O_CLOEXEC);
  627. if (*fd < 0) {
  628. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  629. rc = -EINVAL;
  630. goto put_buf;
  631. }
  632. CAM_DBG(CAM_MEM, "Alloc success : len=%zu, *dmabuf=%pK, fd=%d",
  633. len, *dmabuf, *fd);
  634. /*
  635. * increment the ref count so that ref count becomes 2 here
  636. * when we close fd, refcount becomes 1 and when we do
  637. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  638. */
  639. temp_dmabuf = dma_buf_get(*fd);
  640. if (IS_ERR_OR_NULL(temp_dmabuf)) {
  641. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d", *fd);
  642. rc = -EINVAL;
  643. goto put_buf;
  644. }
  645. return rc;
  646. put_buf:
  647. dma_buf_put(*dmabuf);
  648. return rc;
  649. }
  650. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  651. {
  652. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  653. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  654. CAM_MEM_MMU_MAX_HANDLE);
  655. return -EINVAL;
  656. }
  657. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  658. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  659. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  660. return -EINVAL;
  661. }
  662. if ((cmd->flags & CAM_MEM_FLAG_EVA_NOPIXEL) &&
  663. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE ||
  664. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS)){
  665. CAM_ERR(CAM_MEM,
  666. "Kernel mapping and secure mode not allowed in no pixel mode");
  667. return -EINVAL;
  668. }
  669. return 0;
  670. }
  671. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  672. {
  673. if (!cmd->flags) {
  674. CAM_ERR(CAM_MEM, "Invalid flags");
  675. return -EINVAL;
  676. }
  677. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  678. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  679. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  680. return -EINVAL;
  681. }
  682. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  683. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  684. CAM_ERR(CAM_MEM,
  685. "Kernel mapping in secure mode not allowed, flags=0x%x",
  686. cmd->flags);
  687. return -EINVAL;
  688. }
  689. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  690. CAM_ERR(CAM_MEM,
  691. "Shared memory buffers are not allowed to be mapped");
  692. return -EINVAL;
  693. }
  694. return 0;
  695. }
  696. static int cam_mem_util_map_hw_va(uint32_t flags,
  697. int32_t *mmu_hdls,
  698. int32_t num_hdls,
  699. int fd,
  700. dma_addr_t *hw_vaddr,
  701. size_t *len,
  702. enum cam_smmu_region_id region,
  703. bool is_internal)
  704. {
  705. int i;
  706. int rc = -1;
  707. int dir = cam_mem_util_get_dma_dir(flags);
  708. bool dis_delayed_unmap = false;
  709. if (dir < 0) {
  710. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  711. return dir;
  712. }
  713. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  714. dis_delayed_unmap = true;
  715. CAM_DBG(CAM_MEM,
  716. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  717. fd, flags, dir, num_hdls);
  718. for (i = 0; i < num_hdls; i++) {
  719. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  720. rc = cam_smmu_map_stage2_iova(mmu_hdls[i], fd, dir, hw_vaddr, len);
  721. else
  722. rc = cam_smmu_map_user_iova(mmu_hdls[i], fd, dis_delayed_unmap, dir,
  723. hw_vaddr, len, region, is_internal);
  724. if (rc) {
  725. CAM_ERR(CAM_MEM,
  726. "Failed %s map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  727. (flags & CAM_MEM_FLAG_PROTECTED_MODE) ? "" : "secured",
  728. i, fd, dir, mmu_hdls[i], rc);
  729. goto multi_map_fail;
  730. }
  731. }
  732. return rc;
  733. multi_map_fail:
  734. for (--i; i>= 0; i--) {
  735. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  736. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  737. else
  738. cam_smmu_unmap_user_iova(mmu_hdls[i], fd, CAM_SMMU_REGION_IO);
  739. }
  740. return rc;
  741. }
  742. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  743. {
  744. int rc;
  745. int32_t idx;
  746. struct dma_buf *dmabuf = NULL;
  747. int fd = -1;
  748. dma_addr_t hw_vaddr = 0;
  749. size_t len;
  750. uintptr_t kvaddr = 0;
  751. size_t klen;
  752. if (!atomic_read(&cam_mem_mgr_state)) {
  753. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  754. return -EINVAL;
  755. }
  756. if (!cmd) {
  757. CAM_ERR(CAM_MEM, " Invalid argument");
  758. return -EINVAL;
  759. }
  760. len = cmd->len;
  761. if (tbl.need_shared_buffer_padding &&
  762. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)) {
  763. len += CAM_MEM_SHARED_BUFFER_PAD_4K;
  764. CAM_DBG(CAM_MEM, "Pad 4k size, actual %llu, allocating %zu",
  765. cmd->len, len);
  766. }
  767. rc = cam_mem_util_check_alloc_flags(cmd);
  768. if (rc) {
  769. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  770. cmd->flags, rc);
  771. return rc;
  772. }
  773. rc = cam_mem_util_buffer_alloc(len, cmd->flags, &dmabuf, &fd);
  774. if (rc) {
  775. CAM_ERR(CAM_MEM,
  776. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  777. len, cmd->align, cmd->flags, cmd->num_hdl);
  778. cam_mem_mgr_print_tbl();
  779. return rc;
  780. }
  781. if (!dmabuf) {
  782. CAM_ERR(CAM_MEM,
  783. "Ion Alloc return NULL dmabuf! fd=%d, len=%d", fd, len);
  784. cam_mem_mgr_print_tbl();
  785. return rc;
  786. }
  787. idx = cam_mem_get_slot();
  788. if (idx < 0) {
  789. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  790. rc = -ENOMEM;
  791. goto slot_fail;
  792. }
  793. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  794. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  795. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  796. enum cam_smmu_region_id region;
  797. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  798. region = CAM_SMMU_REGION_IO;
  799. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  800. region = CAM_SMMU_REGION_SHARED;
  801. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  802. region = CAM_SMMU_REGION_IO;
  803. rc = cam_mem_util_map_hw_va(cmd->flags,
  804. cmd->mmu_hdls,
  805. cmd->num_hdl,
  806. fd,
  807. &hw_vaddr,
  808. &len,
  809. region,
  810. true);
  811. if (rc) {
  812. CAM_ERR(CAM_MEM,
  813. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  814. len, cmd->flags,
  815. fd, region, cmd->num_hdl, rc);
  816. if (rc == -EALREADY) {
  817. if ((size_t)dmabuf->size != len)
  818. rc = -EBADR;
  819. cam_mem_mgr_print_tbl();
  820. }
  821. goto map_hw_fail;
  822. }
  823. }
  824. mutex_lock(&tbl.bufq[idx].q_lock);
  825. tbl.bufq[idx].fd = fd;
  826. tbl.bufq[idx].dma_buf = NULL;
  827. tbl.bufq[idx].flags = cmd->flags;
  828. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  829. tbl.bufq[idx].is_internal = true;
  830. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  831. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  832. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  833. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  834. if (rc) {
  835. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  836. dmabuf, rc);
  837. goto map_kernel_fail;
  838. }
  839. }
  840. if (cmd->flags & CAM_MEM_FLAG_KMD_DEBUG_FLAG)
  841. tbl.dbg_buf_idx = idx;
  842. tbl.bufq[idx].kmdvaddr = kvaddr;
  843. tbl.bufq[idx].vaddr = hw_vaddr;
  844. tbl.bufq[idx].dma_buf = dmabuf;
  845. tbl.bufq[idx].len = len;
  846. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  847. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  848. sizeof(int32_t) * cmd->num_hdl);
  849. tbl.bufq[idx].is_imported = false;
  850. mutex_unlock(&tbl.bufq[idx].q_lock);
  851. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  852. cmd->out.fd = tbl.bufq[idx].fd;
  853. cmd->out.vaddr = 0;
  854. CAM_DBG(CAM_MEM,
  855. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  856. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  857. tbl.bufq[idx].len);
  858. return rc;
  859. map_kernel_fail:
  860. mutex_unlock(&tbl.bufq[idx].q_lock);
  861. map_hw_fail:
  862. cam_mem_put_slot(idx);
  863. slot_fail:
  864. dma_buf_put(dmabuf);
  865. return rc;
  866. }
  867. static bool cam_mem_util_is_map_internal(int32_t fd)
  868. {
  869. uint32_t i;
  870. bool is_internal = false;
  871. mutex_lock(&tbl.m_lock);
  872. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  873. if (tbl.bufq[i].fd == fd) {
  874. is_internal = tbl.bufq[i].is_internal;
  875. break;
  876. }
  877. }
  878. mutex_unlock(&tbl.m_lock);
  879. return is_internal;
  880. }
  881. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  882. {
  883. int32_t idx;
  884. int rc;
  885. struct dma_buf *dmabuf;
  886. dma_addr_t hw_vaddr = 0;
  887. size_t len = 0;
  888. bool is_internal = false;
  889. if (!atomic_read(&cam_mem_mgr_state)) {
  890. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  891. return -EINVAL;
  892. }
  893. if (!cmd || (cmd->fd < 0)) {
  894. CAM_ERR(CAM_MEM, "Invalid argument");
  895. return -EINVAL;
  896. }
  897. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  898. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  899. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  900. return -EINVAL;
  901. }
  902. rc = cam_mem_util_check_map_flags(cmd);
  903. if (rc) {
  904. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  905. return rc;
  906. }
  907. dmabuf = dma_buf_get(cmd->fd);
  908. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  909. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  910. return -EINVAL;
  911. }
  912. is_internal = cam_mem_util_is_map_internal(cmd->fd);
  913. idx = cam_mem_get_slot();
  914. if (idx < 0) {
  915. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d, fd=%d",
  916. idx, cmd->fd);
  917. rc = -ENOMEM;
  918. goto slot_fail;
  919. }
  920. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  921. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  922. rc = cam_mem_util_map_hw_va(cmd->flags,
  923. cmd->mmu_hdls,
  924. cmd->num_hdl,
  925. cmd->fd,
  926. &hw_vaddr,
  927. &len,
  928. CAM_SMMU_REGION_IO,
  929. is_internal);
  930. if (rc) {
  931. CAM_ERR(CAM_MEM,
  932. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  933. cmd->flags, cmd->fd, len,
  934. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  935. if (rc == -EALREADY) {
  936. if ((size_t)dmabuf->size != len) {
  937. rc = -EBADR;
  938. cam_mem_mgr_print_tbl();
  939. }
  940. }
  941. goto map_fail;
  942. }
  943. }
  944. mutex_lock(&tbl.bufq[idx].q_lock);
  945. tbl.bufq[idx].fd = cmd->fd;
  946. tbl.bufq[idx].dma_buf = NULL;
  947. tbl.bufq[idx].flags = cmd->flags;
  948. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  949. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  950. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  951. tbl.bufq[idx].kmdvaddr = 0;
  952. if (cmd->num_hdl > 0)
  953. tbl.bufq[idx].vaddr = hw_vaddr;
  954. else
  955. tbl.bufq[idx].vaddr = 0;
  956. tbl.bufq[idx].dma_buf = dmabuf;
  957. tbl.bufq[idx].len = len;
  958. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  959. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  960. sizeof(int32_t) * cmd->num_hdl);
  961. tbl.bufq[idx].is_imported = true;
  962. tbl.bufq[idx].is_internal = is_internal;
  963. mutex_unlock(&tbl.bufq[idx].q_lock);
  964. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  965. cmd->out.vaddr = 0;
  966. cmd->out.size = (uint32_t)len;
  967. CAM_DBG(CAM_MEM,
  968. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  969. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  970. tbl.bufq[idx].len);
  971. return rc;
  972. map_fail:
  973. cam_mem_put_slot(idx);
  974. slot_fail:
  975. dma_buf_put(dmabuf);
  976. return rc;
  977. }
  978. static int cam_mem_util_unmap_hw_va(int32_t idx,
  979. enum cam_smmu_region_id region,
  980. enum cam_smmu_mapping_client client)
  981. {
  982. int i;
  983. uint32_t flags;
  984. int32_t *mmu_hdls;
  985. int num_hdls;
  986. int fd;
  987. int rc = 0;
  988. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  989. CAM_ERR(CAM_MEM, "Incorrect index");
  990. return -EINVAL;
  991. }
  992. flags = tbl.bufq[idx].flags;
  993. mmu_hdls = tbl.bufq[idx].hdls;
  994. num_hdls = tbl.bufq[idx].num_hdl;
  995. fd = tbl.bufq[idx].fd;
  996. CAM_DBG(CAM_MEM,
  997. "unmap_hw_va : idx=%d, fd=%x, flags=0x%x, num_hdls=%d, client=%d",
  998. idx, fd, flags, num_hdls, client);
  999. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  1000. for (i = 0; i < num_hdls; i++) {
  1001. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  1002. if (rc < 0) {
  1003. CAM_ERR(CAM_MEM,
  1004. "Failed in secure unmap, i=%d, fd=%d, mmu_hdl=%d, rc=%d",
  1005. i, fd, mmu_hdls[i], rc);
  1006. goto unmap_end;
  1007. }
  1008. }
  1009. } else {
  1010. for (i = 0; i < num_hdls; i++) {
  1011. if (client == CAM_SMMU_MAPPING_USER) {
  1012. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  1013. fd, region);
  1014. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  1015. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  1016. tbl.bufq[idx].dma_buf, region);
  1017. } else {
  1018. CAM_ERR(CAM_MEM,
  1019. "invalid caller for unmapping : %d",
  1020. client);
  1021. rc = -EINVAL;
  1022. }
  1023. if (rc < 0) {
  1024. CAM_ERR(CAM_MEM,
  1025. "Failed in unmap, i=%d, fd=%d, mmu_hdl=%d, region=%d, rc=%d",
  1026. i, fd, mmu_hdls[i], region, rc);
  1027. goto unmap_end;
  1028. }
  1029. }
  1030. }
  1031. return rc;
  1032. unmap_end:
  1033. CAM_ERR(CAM_MEM, "unmapping failed");
  1034. return rc;
  1035. }
  1036. static void cam_mem_mgr_unmap_active_buf(int idx)
  1037. {
  1038. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1039. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  1040. region = CAM_SMMU_REGION_SHARED;
  1041. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1042. region = CAM_SMMU_REGION_IO;
  1043. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  1044. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS)
  1045. cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1046. tbl.bufq[idx].kmdvaddr);
  1047. }
  1048. static int cam_mem_mgr_cleanup_table(void)
  1049. {
  1050. int i;
  1051. mutex_lock(&tbl.m_lock);
  1052. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  1053. if (!tbl.bufq[i].active) {
  1054. CAM_DBG(CAM_MEM,
  1055. "Buffer inactive at idx=%d, continuing", i);
  1056. continue;
  1057. } else {
  1058. CAM_DBG(CAM_MEM,
  1059. "Active buffer at idx=%d, possible leak needs unmapping",
  1060. i);
  1061. cam_mem_mgr_unmap_active_buf(i);
  1062. }
  1063. mutex_lock(&tbl.bufq[i].q_lock);
  1064. if (tbl.bufq[i].dma_buf) {
  1065. dma_buf_put(tbl.bufq[i].dma_buf);
  1066. tbl.bufq[i].dma_buf = NULL;
  1067. }
  1068. tbl.bufq[i].fd = -1;
  1069. tbl.bufq[i].flags = 0;
  1070. tbl.bufq[i].buf_handle = -1;
  1071. tbl.bufq[i].vaddr = 0;
  1072. tbl.bufq[i].len = 0;
  1073. memset(tbl.bufq[i].hdls, 0,
  1074. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  1075. tbl.bufq[i].num_hdl = 0;
  1076. tbl.bufq[i].dma_buf = NULL;
  1077. tbl.bufq[i].active = false;
  1078. tbl.bufq[i].is_internal = false;
  1079. mutex_unlock(&tbl.bufq[i].q_lock);
  1080. mutex_destroy(&tbl.bufq[i].q_lock);
  1081. }
  1082. bitmap_zero(tbl.bitmap, tbl.bits);
  1083. /* We need to reserve slot 0 because 0 is invalid */
  1084. set_bit(0, tbl.bitmap);
  1085. mutex_unlock(&tbl.m_lock);
  1086. return 0;
  1087. }
  1088. void cam_mem_mgr_deinit(void)
  1089. {
  1090. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  1091. cam_mem_mgr_cleanup_table();
  1092. debugfs_remove_recursive(tbl.dentry);
  1093. mutex_lock(&tbl.m_lock);
  1094. bitmap_zero(tbl.bitmap, tbl.bits);
  1095. kfree(tbl.bitmap);
  1096. tbl.bitmap = NULL;
  1097. tbl.dbg_buf_idx = -1;
  1098. mutex_unlock(&tbl.m_lock);
  1099. mutex_destroy(&tbl.m_lock);
  1100. }
  1101. static int cam_mem_util_unmap(int32_t idx,
  1102. enum cam_smmu_mapping_client client)
  1103. {
  1104. int rc = 0;
  1105. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1106. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1107. CAM_ERR(CAM_MEM, "Incorrect index");
  1108. return -EINVAL;
  1109. }
  1110. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  1111. mutex_lock(&tbl.m_lock);
  1112. if ((!tbl.bufq[idx].active) &&
  1113. (tbl.bufq[idx].vaddr) == 0) {
  1114. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  1115. idx);
  1116. mutex_unlock(&tbl.m_lock);
  1117. return 0;
  1118. }
  1119. /* Deactivate the buffer queue to prevent multiple unmap */
  1120. mutex_lock(&tbl.bufq[idx].q_lock);
  1121. tbl.bufq[idx].active = false;
  1122. tbl.bufq[idx].vaddr = 0;
  1123. mutex_unlock(&tbl.bufq[idx].q_lock);
  1124. mutex_unlock(&tbl.m_lock);
  1125. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1126. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  1127. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1128. tbl.bufq[idx].kmdvaddr);
  1129. if (rc)
  1130. CAM_ERR(CAM_MEM,
  1131. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  1132. tbl.bufq[idx].dma_buf,
  1133. (void *) tbl.bufq[idx].kmdvaddr);
  1134. }
  1135. }
  1136. /* SHARED flag gets precedence, all other flags after it */
  1137. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1138. region = CAM_SMMU_REGION_SHARED;
  1139. } else {
  1140. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1141. region = CAM_SMMU_REGION_IO;
  1142. }
  1143. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1144. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  1145. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1146. if (cam_mem_util_unmap_hw_va(idx, region, client))
  1147. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  1148. tbl.bufq[idx].dma_buf);
  1149. if (client == CAM_SMMU_MAPPING_KERNEL)
  1150. tbl.bufq[idx].dma_buf = NULL;
  1151. }
  1152. mutex_lock(&tbl.m_lock);
  1153. mutex_lock(&tbl.bufq[idx].q_lock);
  1154. tbl.bufq[idx].flags = 0;
  1155. tbl.bufq[idx].buf_handle = -1;
  1156. memset(tbl.bufq[idx].hdls, 0,
  1157. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  1158. CAM_DBG(CAM_MEM,
  1159. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK",
  1160. idx, tbl.bufq[idx].fd,
  1161. tbl.bufq[idx].is_imported,
  1162. tbl.bufq[idx].dma_buf);
  1163. if (tbl.bufq[idx].dma_buf)
  1164. dma_buf_put(tbl.bufq[idx].dma_buf);
  1165. tbl.bufq[idx].fd = -1;
  1166. tbl.bufq[idx].dma_buf = NULL;
  1167. tbl.bufq[idx].is_imported = false;
  1168. tbl.bufq[idx].is_internal = false;
  1169. tbl.bufq[idx].len = 0;
  1170. tbl.bufq[idx].num_hdl = 0;
  1171. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  1172. mutex_unlock(&tbl.bufq[idx].q_lock);
  1173. mutex_destroy(&tbl.bufq[idx].q_lock);
  1174. clear_bit(idx, tbl.bitmap);
  1175. mutex_unlock(&tbl.m_lock);
  1176. return rc;
  1177. }
  1178. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  1179. {
  1180. int idx;
  1181. int rc;
  1182. if (!atomic_read(&cam_mem_mgr_state)) {
  1183. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1184. return -EINVAL;
  1185. }
  1186. if (!cmd) {
  1187. CAM_ERR(CAM_MEM, "Invalid argument");
  1188. return -EINVAL;
  1189. }
  1190. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  1191. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1192. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  1193. idx);
  1194. return -EINVAL;
  1195. }
  1196. if (!tbl.bufq[idx].active) {
  1197. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1198. return -EINVAL;
  1199. }
  1200. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  1201. CAM_ERR(CAM_MEM,
  1202. "Released buf handle %d not matching within table %d, idx=%d",
  1203. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  1204. return -EINVAL;
  1205. }
  1206. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  1207. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  1208. return rc;
  1209. }
  1210. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  1211. struct cam_mem_mgr_memory_desc *out)
  1212. {
  1213. struct dma_buf *buf = NULL;
  1214. int ion_fd = -1;
  1215. int rc = 0;
  1216. uintptr_t kvaddr;
  1217. dma_addr_t iova = 0;
  1218. size_t request_len = 0;
  1219. uint32_t mem_handle;
  1220. int32_t idx;
  1221. int32_t smmu_hdl = 0;
  1222. int32_t num_hdl = 0;
  1223. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1224. if (!atomic_read(&cam_mem_mgr_state)) {
  1225. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1226. return -EINVAL;
  1227. }
  1228. if (!inp || !out) {
  1229. CAM_ERR(CAM_MEM, "Invalid params");
  1230. return -EINVAL;
  1231. }
  1232. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1233. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1234. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1235. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1236. return -EINVAL;
  1237. }
  1238. rc = cam_mem_util_get_dma_buf(inp->size, inp->flags, &buf);
  1239. if (rc) {
  1240. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1241. goto ion_fail;
  1242. } else if (!buf) {
  1243. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1244. goto ion_fail;
  1245. } else {
  1246. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1247. }
  1248. /*
  1249. * we are mapping kva always here,
  1250. * update flags so that we do unmap properly
  1251. */
  1252. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1253. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1254. if (rc) {
  1255. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1256. goto map_fail;
  1257. }
  1258. if (!inp->smmu_hdl) {
  1259. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1260. rc = -EINVAL;
  1261. goto smmu_fail;
  1262. }
  1263. /* SHARED flag gets precedence, all other flags after it */
  1264. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1265. region = CAM_SMMU_REGION_SHARED;
  1266. } else {
  1267. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1268. region = CAM_SMMU_REGION_IO;
  1269. }
  1270. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1271. buf,
  1272. CAM_SMMU_MAP_RW,
  1273. &iova,
  1274. &request_len,
  1275. region);
  1276. if (rc < 0) {
  1277. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1278. goto smmu_fail;
  1279. }
  1280. smmu_hdl = inp->smmu_hdl;
  1281. num_hdl = 1;
  1282. idx = cam_mem_get_slot();
  1283. if (idx < 0) {
  1284. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1285. rc = -ENOMEM;
  1286. goto slot_fail;
  1287. }
  1288. mutex_lock(&tbl.bufq[idx].q_lock);
  1289. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1290. tbl.bufq[idx].dma_buf = buf;
  1291. tbl.bufq[idx].fd = -1;
  1292. tbl.bufq[idx].flags = inp->flags;
  1293. tbl.bufq[idx].buf_handle = mem_handle;
  1294. tbl.bufq[idx].kmdvaddr = kvaddr;
  1295. tbl.bufq[idx].vaddr = iova;
  1296. tbl.bufq[idx].len = inp->size;
  1297. tbl.bufq[idx].num_hdl = num_hdl;
  1298. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1299. sizeof(int32_t));
  1300. tbl.bufq[idx].is_imported = false;
  1301. mutex_unlock(&tbl.bufq[idx].q_lock);
  1302. out->kva = kvaddr;
  1303. out->iova = (uint32_t)iova;
  1304. out->smmu_hdl = smmu_hdl;
  1305. out->mem_handle = mem_handle;
  1306. out->len = inp->size;
  1307. out->region = region;
  1308. return rc;
  1309. slot_fail:
  1310. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1311. buf, region);
  1312. smmu_fail:
  1313. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1314. map_fail:
  1315. dma_buf_put(buf);
  1316. ion_fail:
  1317. return rc;
  1318. }
  1319. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1320. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1321. {
  1322. int32_t idx;
  1323. int rc;
  1324. if (!atomic_read(&cam_mem_mgr_state)) {
  1325. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1326. return -EINVAL;
  1327. }
  1328. if (!inp) {
  1329. CAM_ERR(CAM_MEM, "Invalid argument");
  1330. return -EINVAL;
  1331. }
  1332. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1333. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1334. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1335. return -EINVAL;
  1336. }
  1337. if (!tbl.bufq[idx].active) {
  1338. if (tbl.bufq[idx].vaddr == 0) {
  1339. CAM_ERR(CAM_MEM, "buffer is released already");
  1340. return 0;
  1341. }
  1342. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1343. return -EINVAL;
  1344. }
  1345. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1346. CAM_ERR(CAM_MEM,
  1347. "Released buf handle not matching within table");
  1348. return -EINVAL;
  1349. }
  1350. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1351. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1352. return rc;
  1353. }
  1354. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1355. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1356. enum cam_smmu_region_id region,
  1357. struct cam_mem_mgr_memory_desc *out)
  1358. {
  1359. struct dma_buf *buf = NULL;
  1360. int rc = 0;
  1361. int ion_fd = -1;
  1362. dma_addr_t iova = 0;
  1363. size_t request_len = 0;
  1364. uint32_t mem_handle;
  1365. int32_t idx;
  1366. int32_t smmu_hdl = 0;
  1367. int32_t num_hdl = 0;
  1368. uintptr_t kvaddr = 0;
  1369. if (!atomic_read(&cam_mem_mgr_state)) {
  1370. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1371. return -EINVAL;
  1372. }
  1373. if (!inp || !out) {
  1374. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1375. return -EINVAL;
  1376. }
  1377. if (!inp->smmu_hdl) {
  1378. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1379. return -EINVAL;
  1380. }
  1381. if ((region != CAM_SMMU_REGION_SECHEAP) &&
  1382. (region != CAM_SMMU_REGION_FWUNCACHED)) {
  1383. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1384. return -EINVAL;
  1385. }
  1386. rc = cam_mem_util_get_dma_buf(inp->size, 0, &buf);
  1387. if (rc) {
  1388. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1389. goto ion_fail;
  1390. } else if (!buf) {
  1391. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1392. goto ion_fail;
  1393. } else {
  1394. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1395. }
  1396. if (inp->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1397. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1398. if (rc) {
  1399. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1400. goto kmap_fail;
  1401. }
  1402. }
  1403. rc = cam_smmu_reserve_buf_region(region,
  1404. inp->smmu_hdl, buf, &iova, &request_len);
  1405. if (rc) {
  1406. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1407. goto smmu_fail;
  1408. }
  1409. smmu_hdl = inp->smmu_hdl;
  1410. num_hdl = 1;
  1411. idx = cam_mem_get_slot();
  1412. if (idx < 0) {
  1413. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1414. rc = -ENOMEM;
  1415. goto slot_fail;
  1416. }
  1417. mutex_lock(&tbl.bufq[idx].q_lock);
  1418. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1419. tbl.bufq[idx].fd = -1;
  1420. tbl.bufq[idx].dma_buf = buf;
  1421. tbl.bufq[idx].flags = inp->flags;
  1422. tbl.bufq[idx].buf_handle = mem_handle;
  1423. tbl.bufq[idx].kmdvaddr = kvaddr;
  1424. tbl.bufq[idx].vaddr = iova;
  1425. tbl.bufq[idx].len = request_len;
  1426. tbl.bufq[idx].num_hdl = num_hdl;
  1427. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1428. sizeof(int32_t));
  1429. tbl.bufq[idx].is_imported = false;
  1430. mutex_unlock(&tbl.bufq[idx].q_lock);
  1431. out->kva = kvaddr;
  1432. out->iova = (uint32_t)iova;
  1433. out->smmu_hdl = smmu_hdl;
  1434. out->mem_handle = mem_handle;
  1435. out->len = request_len;
  1436. out->region = region;
  1437. return rc;
  1438. slot_fail:
  1439. cam_smmu_release_buf_region(region, smmu_hdl);
  1440. smmu_fail:
  1441. if (region == CAM_SMMU_REGION_FWUNCACHED)
  1442. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1443. kmap_fail:
  1444. dma_buf_put(buf);
  1445. ion_fail:
  1446. return rc;
  1447. }
  1448. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1449. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1450. {
  1451. int32_t idx;
  1452. int rc;
  1453. int32_t smmu_hdl;
  1454. if (!atomic_read(&cam_mem_mgr_state)) {
  1455. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1456. return -EINVAL;
  1457. }
  1458. if (!inp) {
  1459. CAM_ERR(CAM_MEM, "Invalid argument");
  1460. return -EINVAL;
  1461. }
  1462. if ((inp->region != CAM_SMMU_REGION_SECHEAP) &&
  1463. (inp->region != CAM_SMMU_REGION_FWUNCACHED)) {
  1464. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1465. return -EINVAL;
  1466. }
  1467. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1468. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1469. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1470. return -EINVAL;
  1471. }
  1472. if (!tbl.bufq[idx].active) {
  1473. if (tbl.bufq[idx].vaddr == 0) {
  1474. CAM_ERR(CAM_MEM, "buffer is released already");
  1475. return 0;
  1476. }
  1477. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1478. return -EINVAL;
  1479. }
  1480. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1481. CAM_ERR(CAM_MEM,
  1482. "Released buf handle not matching within table");
  1483. return -EINVAL;
  1484. }
  1485. if (tbl.bufq[idx].num_hdl != 1) {
  1486. CAM_ERR(CAM_MEM,
  1487. "Sec heap region should have only one smmu hdl");
  1488. return -ENODEV;
  1489. }
  1490. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1491. sizeof(int32_t));
  1492. if (inp->smmu_hdl != smmu_hdl) {
  1493. CAM_ERR(CAM_MEM,
  1494. "Passed SMMU handle doesn't match with internal hdl");
  1495. return -ENODEV;
  1496. }
  1497. rc = cam_smmu_release_buf_region(inp->region, inp->smmu_hdl);
  1498. if (rc) {
  1499. CAM_ERR(CAM_MEM,
  1500. "Sec heap region release failed");
  1501. return -ENODEV;
  1502. }
  1503. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1504. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1505. if (rc)
  1506. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1507. return rc;
  1508. }
  1509. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);
  1510. #ifndef CONFIG_CAM_PRESIL
  1511. struct dma_buf * cam_mem_mgr_get_dma_buf(int fd)
  1512. {
  1513. return NULL;
  1514. }
  1515. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1516. {
  1517. return 0;
  1518. }
  1519. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1520. {
  1521. return 0;
  1522. }
  1523. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle,
  1524. uint32_t buf_size,
  1525. uint32_t offset,
  1526. int32_t iommu_hdl)
  1527. {
  1528. return 0;
  1529. }
  1530. #endif