cam_mem_mgr.c 57 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #include <linux/module.h>
  7. #include <linux/types.h>
  8. #include <linux/mutex.h>
  9. #include <linux/slab.h>
  10. #include <linux/dma-buf.h>
  11. #include <linux/version.h>
  12. #include <linux/debugfs.h>
  13. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  14. #include <linux/mem-buf.h>
  15. #include <soc/qcom/secure_buffer.h>
  16. #endif
  17. #include "cam_compat.h"
  18. #include "cam_req_mgr_util.h"
  19. #include "cam_mem_mgr.h"
  20. #include "cam_smmu_api.h"
  21. #include "cam_debug_util.h"
  22. #include "cam_trace.h"
  23. #include "cam_common_util.h"
  24. #include "cam_presil_hw_access.h"
  25. #include "cam_compat.h"
  26. #define CAM_MEM_SHARED_BUFFER_PAD_4K (4 * 1024)
  27. static struct cam_mem_table tbl;
  28. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  29. /* cam_mem_mgr_debug - global struct to keep track of debug settings for mem mgr
  30. *
  31. * @dentry : Directory entry to the mem mgr root folder
  32. * @alloc_profile_enable : Whether to enable alloc profiling
  33. * @override_cpu_access_dir : Override cpu access direction to BIDIRECTIONAL
  34. */
  35. static struct {
  36. struct dentry *dentry;
  37. bool alloc_profile_enable;
  38. bool override_cpu_access_dir;
  39. } g_cam_mem_mgr_debug;
  40. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  41. static void cam_mem_mgr_put_dma_heaps(void);
  42. static int cam_mem_mgr_get_dma_heaps(void);
  43. #endif
  44. #ifdef CONFIG_CAM_PRESIL
  45. static inline void cam_mem_mgr_reset_presil_params(int idx)
  46. {
  47. tbl.bufq[idx].presil_params.fd_for_umd_daemon = -1;
  48. tbl.bufq[idx].presil_params.refcount = 0;
  49. }
  50. #else
  51. static inline void cam_mem_mgr_reset_presil_params(int idx)
  52. {
  53. return;
  54. }
  55. #endif
  56. static unsigned long cam_mem_mgr_mini_dump_cb(void *dst, unsigned long len)
  57. {
  58. struct cam_mem_table_mini_dump *md;
  59. if (!dst) {
  60. CAM_ERR(CAM_MEM, "Invalid params");
  61. return 0;
  62. }
  63. if (len < sizeof(*md)) {
  64. CAM_ERR(CAM_MEM, "Insufficient length %u", len);
  65. return 0;
  66. }
  67. md = (struct cam_mem_table_mini_dump *)dst;
  68. memcpy(md->bufq, tbl.bufq, CAM_MEM_BUFQ_MAX * sizeof(struct cam_mem_buf_queue));
  69. md->dbg_buf_idx = tbl.dbg_buf_idx;
  70. md->alloc_profile_enable = g_cam_mem_mgr_debug.alloc_profile_enable;
  71. md->force_cache_allocs = tbl.force_cache_allocs;
  72. md->need_shared_buffer_padding = tbl.need_shared_buffer_padding;
  73. return sizeof(*md);
  74. }
  75. static void cam_mem_mgr_print_tbl(void)
  76. {
  77. int i;
  78. uint64_t ms, hrs, min, sec;
  79. struct timespec64 current_ts;
  80. CAM_GET_TIMESTAMP(current_ts);
  81. CAM_CONVERT_TIMESTAMP_FORMAT(current_ts, hrs, min, sec, ms);
  82. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  83. hrs, min, sec, ms);
  84. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  85. if (tbl.bufq[i].active) {
  86. CAM_CONVERT_TIMESTAMP_FORMAT((tbl.bufq[i].timestamp), hrs, min, sec, ms);
  87. CAM_INFO(CAM_MEM,
  88. "%llu:%llu:%llu:%llu idx %d fd %d i_ino %lu size %llu",
  89. hrs, min, sec, ms, i, tbl.bufq[i].fd, tbl.bufq[i].i_ino,
  90. tbl.bufq[i].len);
  91. }
  92. }
  93. }
  94. static int cam_mem_util_get_dma_dir(uint32_t flags)
  95. {
  96. int rc = -EINVAL;
  97. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  98. rc = DMA_TO_DEVICE;
  99. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  100. rc = DMA_FROM_DEVICE;
  101. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  102. rc = DMA_BIDIRECTIONAL;
  103. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  104. rc = DMA_BIDIRECTIONAL;
  105. return rc;
  106. }
  107. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf, uintptr_t *vaddr, size_t *len)
  108. {
  109. int rc = 0;
  110. /*
  111. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  112. * need to be called in pair to avoid stability issue.
  113. */
  114. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  115. if (rc) {
  116. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  117. return rc;
  118. }
  119. rc = cam_compat_util_get_dmabuf_va(dmabuf, vaddr);
  120. if (rc) {
  121. CAM_ERR(CAM_MEM, "kernel vmap failed: rc = %d", rc);
  122. *len = 0;
  123. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  124. }
  125. else {
  126. *len = dmabuf->size;
  127. CAM_DBG(CAM_MEM, "vaddr = %llu, len = %zu", *vaddr, *len);
  128. }
  129. return rc;
  130. }
  131. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  132. uint64_t vaddr)
  133. {
  134. int rc = 0;
  135. if (!dmabuf || !vaddr) {
  136. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  137. return -EINVAL;
  138. }
  139. cam_compat_util_put_dmabuf_va(dmabuf, (void *)vaddr);
  140. /*
  141. * dma_buf_begin_cpu_access() and
  142. * dma_buf_end_cpu_access() need to be called in pair
  143. * to avoid stability issue.
  144. */
  145. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  146. if (rc) {
  147. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  148. dmabuf);
  149. return rc;
  150. }
  151. return rc;
  152. }
  153. static int cam_mem_mgr_create_debug_fs(void)
  154. {
  155. int rc = 0;
  156. struct dentry *dbgfileptr = NULL;
  157. if (!cam_debugfs_available() || g_cam_mem_mgr_debug.dentry)
  158. return 0;
  159. rc = cam_debugfs_create_subdir("memmgr", &dbgfileptr);
  160. if (rc) {
  161. CAM_ERR(CAM_MEM, "DebugFS could not create directory!");
  162. rc = -ENOENT;
  163. goto end;
  164. }
  165. g_cam_mem_mgr_debug.dentry = dbgfileptr;
  166. debugfs_create_bool("alloc_profile_enable", 0644, g_cam_mem_mgr_debug.dentry,
  167. &g_cam_mem_mgr_debug.alloc_profile_enable);
  168. debugfs_create_bool("override_cpu_access_dir", 0644, g_cam_mem_mgr_debug.dentry,
  169. &g_cam_mem_mgr_debug.override_cpu_access_dir);
  170. end:
  171. return rc;
  172. }
  173. int cam_mem_mgr_init(void)
  174. {
  175. int i;
  176. int bitmap_size;
  177. int rc = 0;
  178. if (atomic_read(&cam_mem_mgr_state))
  179. return 0;
  180. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  181. if (cam_smmu_need_force_alloc_cached(&tbl.force_cache_allocs)) {
  182. CAM_ERR(CAM_MEM, "Error in getting force cache alloc flag");
  183. return -EINVAL;
  184. }
  185. tbl.need_shared_buffer_padding = cam_smmu_need_shared_buffer_padding();
  186. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  187. rc = cam_mem_mgr_get_dma_heaps();
  188. if (rc) {
  189. CAM_ERR(CAM_MEM, "Failed in getting dma heaps rc=%d", rc);
  190. return rc;
  191. }
  192. #endif
  193. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  194. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  195. if (!tbl.bitmap) {
  196. rc = -ENOMEM;
  197. goto put_heaps;
  198. }
  199. tbl.bits = bitmap_size * BITS_PER_BYTE;
  200. bitmap_zero(tbl.bitmap, tbl.bits);
  201. /* We need to reserve slot 0 because 0 is invalid */
  202. set_bit(0, tbl.bitmap);
  203. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  204. tbl.bufq[i].fd = -1;
  205. tbl.bufq[i].buf_handle = -1;
  206. cam_mem_mgr_reset_presil_params(i);
  207. }
  208. mutex_init(&tbl.m_lock);
  209. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  210. cam_mem_mgr_create_debug_fs();
  211. cam_common_register_mini_dump_cb(cam_mem_mgr_mini_dump_cb,
  212. "cam_mem");
  213. return 0;
  214. put_heaps:
  215. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  216. cam_mem_mgr_put_dma_heaps();
  217. #endif
  218. return rc;
  219. }
  220. static int32_t cam_mem_get_slot(void)
  221. {
  222. int32_t idx;
  223. mutex_lock(&tbl.m_lock);
  224. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  225. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  226. mutex_unlock(&tbl.m_lock);
  227. return -ENOMEM;
  228. }
  229. set_bit(idx, tbl.bitmap);
  230. tbl.bufq[idx].active = true;
  231. CAM_GET_TIMESTAMP((tbl.bufq[idx].timestamp));
  232. mutex_init(&tbl.bufq[idx].q_lock);
  233. mutex_unlock(&tbl.m_lock);
  234. return idx;
  235. }
  236. static void cam_mem_put_slot(int32_t idx)
  237. {
  238. mutex_lock(&tbl.m_lock);
  239. mutex_lock(&tbl.bufq[idx].q_lock);
  240. tbl.bufq[idx].active = false;
  241. tbl.bufq[idx].is_internal = false;
  242. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  243. mutex_unlock(&tbl.bufq[idx].q_lock);
  244. mutex_destroy(&tbl.bufq[idx].q_lock);
  245. clear_bit(idx, tbl.bitmap);
  246. mutex_unlock(&tbl.m_lock);
  247. }
  248. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  249. dma_addr_t *iova_ptr, size_t *len_ptr, uint32_t *flags)
  250. {
  251. int rc = 0, idx;
  252. *len_ptr = 0;
  253. if (!atomic_read(&cam_mem_mgr_state)) {
  254. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  255. return -EINVAL;
  256. }
  257. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  258. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  259. return -ENOENT;
  260. if (!tbl.bufq[idx].active) {
  261. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  262. idx);
  263. return -EAGAIN;
  264. }
  265. mutex_lock(&tbl.bufq[idx].q_lock);
  266. if (buf_handle != tbl.bufq[idx].buf_handle) {
  267. rc = -EINVAL;
  268. goto handle_mismatch;
  269. }
  270. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  271. rc = cam_smmu_get_stage2_iova(mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].dma_buf,
  272. iova_ptr, len_ptr);
  273. else
  274. rc = cam_smmu_get_iova(mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].dma_buf,
  275. iova_ptr, len_ptr);
  276. if (rc) {
  277. CAM_ERR(CAM_MEM,
  278. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d i_ino:%lu",
  279. buf_handle, mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].i_ino);
  280. goto handle_mismatch;
  281. }
  282. if (flags)
  283. *flags = tbl.bufq[idx].flags;
  284. CAM_DBG(CAM_MEM,
  285. "handle:0x%x fd:%d i_ino:%lu iova_ptr:0x%lx len_ptr:%lu",
  286. mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].i_ino, *iova_ptr, *len_ptr);
  287. handle_mismatch:
  288. mutex_unlock(&tbl.bufq[idx].q_lock);
  289. return rc;
  290. }
  291. EXPORT_SYMBOL(cam_mem_get_io_buf);
  292. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  293. {
  294. int idx;
  295. if (!atomic_read(&cam_mem_mgr_state)) {
  296. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  297. return -EINVAL;
  298. }
  299. if (!buf_handle || !vaddr_ptr || !len)
  300. return -EINVAL;
  301. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  302. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  303. return -EINVAL;
  304. if (!tbl.bufq[idx].active) {
  305. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  306. idx);
  307. return -EPERM;
  308. }
  309. if (buf_handle != tbl.bufq[idx].buf_handle)
  310. return -EINVAL;
  311. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  312. return -EINVAL;
  313. if (tbl.bufq[idx].kmdvaddr) {
  314. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  315. *len = tbl.bufq[idx].len;
  316. } else {
  317. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  318. buf_handle);
  319. return -EINVAL;
  320. }
  321. return 0;
  322. }
  323. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  324. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  325. {
  326. int rc = 0, idx;
  327. uint32_t cache_dir;
  328. unsigned long dmabuf_flag = 0;
  329. if (!atomic_read(&cam_mem_mgr_state)) {
  330. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  331. return -EINVAL;
  332. }
  333. if (!cmd)
  334. return -EINVAL;
  335. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  336. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  337. return -EINVAL;
  338. mutex_lock(&tbl.m_lock);
  339. if (!test_bit(idx, tbl.bitmap)) {
  340. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  341. idx);
  342. mutex_unlock(&tbl.m_lock);
  343. return -EINVAL;
  344. }
  345. mutex_lock(&tbl.bufq[idx].q_lock);
  346. mutex_unlock(&tbl.m_lock);
  347. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  348. rc = -EINVAL;
  349. goto end;
  350. }
  351. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  352. if (rc) {
  353. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  354. goto end;
  355. }
  356. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  357. CAM_DBG(CAM_MEM, "Calling dmap buf APIs for cache operations");
  358. cache_dir = DMA_BIDIRECTIONAL;
  359. #else
  360. if (dmabuf_flag & ION_FLAG_CACHED) {
  361. switch (cmd->mem_cache_ops) {
  362. case CAM_MEM_CLEAN_CACHE:
  363. cache_dir = DMA_TO_DEVICE;
  364. break;
  365. case CAM_MEM_INV_CACHE:
  366. cache_dir = DMA_FROM_DEVICE;
  367. break;
  368. case CAM_MEM_CLEAN_INV_CACHE:
  369. cache_dir = DMA_BIDIRECTIONAL;
  370. break;
  371. default:
  372. CAM_ERR(CAM_MEM,
  373. "invalid cache ops :%d", cmd->mem_cache_ops);
  374. rc = -EINVAL;
  375. goto end;
  376. }
  377. } else {
  378. CAM_DBG(CAM_MEM, "BUF is not cached");
  379. goto end;
  380. }
  381. #endif
  382. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  383. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  384. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  385. if (rc) {
  386. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  387. goto end;
  388. }
  389. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  390. cache_dir);
  391. if (rc) {
  392. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  393. goto end;
  394. }
  395. end:
  396. mutex_unlock(&tbl.bufq[idx].q_lock);
  397. return rc;
  398. }
  399. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  400. int cam_mem_mgr_cpu_access_op(struct cam_mem_cpu_access_op *cmd)
  401. {
  402. int rc = 0, idx;
  403. uint32_t direction;
  404. if (!atomic_read(&cam_mem_mgr_state)) {
  405. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  406. return -EINVAL;
  407. }
  408. if (!cmd) {
  409. CAM_ERR(CAM_MEM, "Invalid cmd");
  410. return -EINVAL;
  411. }
  412. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  413. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  414. CAM_ERR(CAM_MEM, "Invalid idx=%d, buf_handle 0x%x, access=0x%x",
  415. idx, cmd->buf_handle, cmd->access);
  416. return -EINVAL;
  417. }
  418. mutex_lock(&tbl.m_lock);
  419. if (!test_bit(idx, tbl.bitmap)) {
  420. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already freed/unmapped", idx);
  421. mutex_unlock(&tbl.m_lock);
  422. return -EINVAL;
  423. }
  424. mutex_lock(&tbl.bufq[idx].q_lock);
  425. mutex_unlock(&tbl.m_lock);
  426. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  427. CAM_ERR(CAM_MEM,
  428. "Buffer at idx=%d is different incoming handle 0x%x, actual handle 0x%x",
  429. idx, cmd->buf_handle, tbl.bufq[idx].buf_handle);
  430. rc = -EINVAL;
  431. goto end;
  432. }
  433. CAM_DBG(CAM_MEM, "buf_handle=0x%x, access=0x%x, access_type=0x%x, override_access=%d",
  434. cmd->buf_handle, cmd->access, cmd->access_type,
  435. g_cam_mem_mgr_debug.override_cpu_access_dir);
  436. if (cmd->access_type & CAM_MEM_CPU_ACCESS_READ &&
  437. cmd->access_type & CAM_MEM_CPU_ACCESS_WRITE) {
  438. direction = DMA_BIDIRECTIONAL;
  439. } else if (cmd->access_type & CAM_MEM_CPU_ACCESS_READ) {
  440. direction = DMA_FROM_DEVICE;
  441. } else if (cmd->access_type & CAM_MEM_CPU_ACCESS_WRITE) {
  442. direction = DMA_TO_DEVICE;
  443. } else {
  444. direction = DMA_BIDIRECTIONAL;
  445. CAM_WARN(CAM_MEM,
  446. "Invalid access type buf_handle=0x%x, access=0x%x, access_type=0x%x",
  447. cmd->buf_handle, cmd->access, cmd->access_type);
  448. }
  449. if (g_cam_mem_mgr_debug.override_cpu_access_dir)
  450. direction = DMA_BIDIRECTIONAL;
  451. if (cmd->access & CAM_MEM_BEGIN_CPU_ACCESS) {
  452. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf, direction);
  453. if (rc) {
  454. CAM_ERR(CAM_MEM,
  455. "dma begin cpu access failed rc=%d, buf_handle=0x%x, access=0x%x, access_type=0x%x",
  456. rc, cmd->buf_handle, cmd->access, cmd->access_type);
  457. goto end;
  458. }
  459. }
  460. if (cmd->access & CAM_MEM_END_CPU_ACCESS) {
  461. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf, direction);
  462. if (rc) {
  463. CAM_ERR(CAM_MEM,
  464. "dma end cpu access failed rc=%d, buf_handle=0x%x, access=0x%x, access_type=0x%x",
  465. rc, cmd->buf_handle, cmd->access, cmd->access_type);
  466. goto end;
  467. }
  468. }
  469. end:
  470. mutex_unlock(&tbl.bufq[idx].q_lock);
  471. return rc;
  472. }
  473. EXPORT_SYMBOL(cam_mem_mgr_cpu_access_op);
  474. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  475. #define CAM_MAX_VMIDS 4
  476. static void cam_mem_mgr_put_dma_heaps(void)
  477. {
  478. CAM_DBG(CAM_MEM, "Releasing DMA Buf heaps usage");
  479. }
  480. static int cam_mem_mgr_get_dma_heaps(void)
  481. {
  482. int rc = 0;
  483. tbl.system_heap = NULL;
  484. tbl.system_uncached_heap = NULL;
  485. tbl.camera_heap = NULL;
  486. tbl.camera_uncached_heap = NULL;
  487. tbl.secure_display_heap = NULL;
  488. tbl.ubwc_p_heap = NULL;
  489. tbl.system_heap = dma_heap_find("qcom,system");
  490. if (IS_ERR_OR_NULL(tbl.system_heap)) {
  491. rc = PTR_ERR(tbl.system_heap);
  492. CAM_ERR(CAM_MEM, "qcom system heap not found, rc=%d", rc);
  493. tbl.system_heap = NULL;
  494. goto put_heaps;
  495. }
  496. tbl.system_uncached_heap = dma_heap_find("qcom,system-uncached");
  497. if (IS_ERR_OR_NULL(tbl.system_uncached_heap)) {
  498. if (tbl.force_cache_allocs) {
  499. /* optional, we anyway do not use uncached */
  500. CAM_DBG(CAM_MEM,
  501. "qcom system-uncached heap not found, err=%d",
  502. PTR_ERR(tbl.system_uncached_heap));
  503. tbl.system_uncached_heap = NULL;
  504. } else {
  505. /* fatal, must need uncached heaps */
  506. rc = PTR_ERR(tbl.system_uncached_heap);
  507. CAM_ERR(CAM_MEM,
  508. "qcom system-uncached heap not found, rc=%d",
  509. rc);
  510. tbl.system_uncached_heap = NULL;
  511. goto put_heaps;
  512. }
  513. }
  514. tbl.ubwc_p_heap = dma_heap_find("qcom,ubwcp");
  515. if (IS_ERR_OR_NULL(tbl.ubwc_p_heap)) {
  516. CAM_DBG(CAM_MEM, "qcom ubwcp heap not found, err=%d", PTR_ERR(tbl.ubwc_p_heap));
  517. tbl.ubwc_p_heap = NULL;
  518. }
  519. tbl.secure_display_heap = dma_heap_find("qcom,display");
  520. if (IS_ERR_OR_NULL(tbl.secure_display_heap)) {
  521. rc = PTR_ERR(tbl.secure_display_heap);
  522. CAM_ERR(CAM_MEM, "qcom,display heap not found, rc=%d",
  523. rc);
  524. tbl.secure_display_heap = NULL;
  525. goto put_heaps;
  526. }
  527. tbl.camera_heap = dma_heap_find("qcom,camera");
  528. if (IS_ERR_OR_NULL(tbl.camera_heap)) {
  529. /* optional heap, not a fatal error */
  530. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  531. PTR_ERR(tbl.camera_heap));
  532. tbl.camera_heap = NULL;
  533. }
  534. tbl.camera_uncached_heap = dma_heap_find("qcom,camera-uncached");
  535. if (IS_ERR_OR_NULL(tbl.camera_uncached_heap)) {
  536. /* optional heap, not a fatal error */
  537. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  538. PTR_ERR(tbl.camera_uncached_heap));
  539. tbl.camera_uncached_heap = NULL;
  540. }
  541. CAM_INFO(CAM_MEM,
  542. "Heaps : system=%pK, system_uncached=%pK, camera=%pK, camera-uncached=%pK, secure_display=%pK, ubwc_p_heap=%pK",
  543. tbl.system_heap, tbl.system_uncached_heap,
  544. tbl.camera_heap, tbl.camera_uncached_heap,
  545. tbl.secure_display_heap, tbl.ubwc_p_heap);
  546. return 0;
  547. put_heaps:
  548. cam_mem_mgr_put_dma_heaps();
  549. return rc;
  550. }
  551. bool cam_mem_mgr_ubwc_p_heap_supported(void)
  552. {
  553. if (tbl.ubwc_p_heap)
  554. return true;
  555. return false;
  556. }
  557. static int cam_mem_util_get_dma_buf(size_t len,
  558. unsigned int cam_flags,
  559. struct dma_buf **buf,
  560. unsigned long *i_ino)
  561. {
  562. int rc = 0;
  563. struct dma_heap *heap;
  564. struct dma_heap *try_heap = NULL;
  565. struct timespec64 ts1, ts2;
  566. long microsec = 0;
  567. bool use_cached_heap = false;
  568. struct mem_buf_lend_kernel_arg arg;
  569. int vmids[CAM_MAX_VMIDS];
  570. int perms[CAM_MAX_VMIDS];
  571. int num_vmids = 0;
  572. if (!buf) {
  573. CAM_ERR(CAM_MEM, "Invalid params");
  574. return -EINVAL;
  575. }
  576. if (g_cam_mem_mgr_debug.alloc_profile_enable)
  577. CAM_GET_TIMESTAMP(ts1);
  578. if ((cam_flags & CAM_MEM_FLAG_CACHE) ||
  579. (tbl.force_cache_allocs &&
  580. (!(cam_flags & CAM_MEM_FLAG_PROTECTED_MODE)))) {
  581. CAM_DBG(CAM_MEM,
  582. "Using CACHED heap, cam_flags=0x%x, force_cache_allocs=%d",
  583. cam_flags, tbl.force_cache_allocs);
  584. use_cached_heap = true;
  585. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  586. use_cached_heap = true;
  587. CAM_DBG(CAM_MEM,
  588. "Using CACHED heap for secure, cam_flags=0x%x, force_cache_allocs=%d",
  589. cam_flags, tbl.force_cache_allocs);
  590. } else {
  591. use_cached_heap = false;
  592. CAM_ERR(CAM_MEM,
  593. "Using UNCACHED heap not supported, cam_flags=0x%x, force_cache_allocs=%d",
  594. cam_flags, tbl.force_cache_allocs);
  595. /*
  596. * Need a better handling based on whether dma-buf-heaps support
  597. * uncached heaps or not. For now, assume not supported.
  598. */
  599. return -EINVAL;
  600. }
  601. if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  602. heap = tbl.secure_display_heap;
  603. vmids[num_vmids] = VMID_CP_CAMERA;
  604. perms[num_vmids] = PERM_READ | PERM_WRITE;
  605. num_vmids++;
  606. if (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT) {
  607. CAM_DBG(CAM_MEM, "Secure mode CDSP flags");
  608. vmids[num_vmids] = VMID_CP_CDSP;
  609. perms[num_vmids] = PERM_READ | PERM_WRITE;
  610. num_vmids++;
  611. }
  612. } else if (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL) {
  613. heap = tbl.secure_display_heap;
  614. vmids[num_vmids] = VMID_CP_NON_PIXEL;
  615. perms[num_vmids] = PERM_READ | PERM_WRITE;
  616. num_vmids++;
  617. } else if (cam_flags & CAM_MEM_FLAG_UBWC_P_HEAP) {
  618. if (!tbl.ubwc_p_heap) {
  619. CAM_ERR(CAM_MEM, "ubwc-p heap is not available, can't allocate");
  620. return -EINVAL;
  621. }
  622. heap = tbl.ubwc_p_heap;
  623. CAM_DBG(CAM_MEM, "Allocating from ubwc-p heap, size=%d, flags=0x%x",
  624. len, cam_flags);
  625. } else if (use_cached_heap) {
  626. try_heap = tbl.camera_heap;
  627. heap = tbl.system_heap;
  628. } else {
  629. try_heap = tbl.camera_uncached_heap;
  630. heap = tbl.system_uncached_heap;
  631. }
  632. CAM_DBG(CAM_MEM, "Using heaps : try=%pK, heap=%pK", try_heap, heap);
  633. *buf = NULL;
  634. if (!try_heap && !heap) {
  635. CAM_ERR(CAM_MEM,
  636. "No heap available for allocation, cant allocate");
  637. return -EINVAL;
  638. }
  639. if (try_heap) {
  640. *buf = dma_heap_buffer_alloc(try_heap, len, O_RDWR, 0);
  641. if (IS_ERR(*buf)) {
  642. CAM_WARN(CAM_MEM,
  643. "Failed in allocating from try heap, heap=%pK, len=%zu, err=%d",
  644. try_heap, len, PTR_ERR(*buf));
  645. *buf = NULL;
  646. }
  647. }
  648. if (*buf == NULL) {
  649. *buf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0);
  650. if (IS_ERR(*buf)) {
  651. rc = PTR_ERR(*buf);
  652. CAM_ERR(CAM_MEM,
  653. "Failed in allocating from heap, heap=%pK, len=%zu, err=%d",
  654. heap, len, rc);
  655. *buf = NULL;
  656. return rc;
  657. }
  658. }
  659. *i_ino = file_inode((*buf)->file)->i_ino;
  660. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) ||
  661. (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL)) {
  662. if (num_vmids >= CAM_MAX_VMIDS) {
  663. CAM_ERR(CAM_MEM, "Insufficient array size for vmids %d", num_vmids);
  664. rc = -EINVAL;
  665. goto end;
  666. }
  667. arg.nr_acl_entries = num_vmids;
  668. arg.vmids = vmids;
  669. arg.perms = perms;
  670. rc = mem_buf_lend(*buf, &arg);
  671. if (rc) {
  672. CAM_ERR(CAM_MEM,
  673. "Failed in buf lend rc=%d, buf=%pK, vmids [0]=0x%x, [1]=0x%x, [2]=0x%x",
  674. rc, *buf, vmids[0], vmids[1], vmids[2]);
  675. goto end;
  676. }
  677. }
  678. CAM_DBG(CAM_MEM, "Allocate success, len=%zu, *buf=%pK, i_ino=%lu", len, *buf, *i_ino);
  679. if (g_cam_mem_mgr_debug.alloc_profile_enable) {
  680. CAM_GET_TIMESTAMP(ts2);
  681. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  682. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  683. len, microsec);
  684. }
  685. return rc;
  686. end:
  687. dma_buf_put(*buf);
  688. return rc;
  689. }
  690. #else
  691. bool cam_mem_mgr_ubwc_p_heap_supported(void)
  692. {
  693. return false;
  694. }
  695. static int cam_mem_util_get_dma_buf(size_t len,
  696. unsigned int cam_flags,
  697. struct dma_buf **buf,
  698. unsigned long *i_ino)
  699. {
  700. int rc = 0;
  701. unsigned int heap_id;
  702. int32_t ion_flag = 0;
  703. struct timespec64 ts1, ts2;
  704. long microsec = 0;
  705. if (!buf) {
  706. CAM_ERR(CAM_MEM, "Invalid params");
  707. return -EINVAL;
  708. }
  709. if (cam_flags & CAM_MEM_FLAG_UBWC_P_HEAP) {
  710. CAM_ERR(CAM_MEM, "ubwcp heap not supported");
  711. return -EINVAL;
  712. }
  713. if (g_cam_mem_mgr_debug.alloc_profile_enable)
  714. CAM_GET_TIMESTAMP(ts1);
  715. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  716. (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  717. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  718. ion_flag |=
  719. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  720. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  721. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  722. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  723. } else {
  724. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  725. ION_HEAP(ION_CAMERA_HEAP_ID);
  726. }
  727. if (cam_flags & CAM_MEM_FLAG_CACHE)
  728. ion_flag |= ION_FLAG_CACHED;
  729. else
  730. ion_flag &= ~ION_FLAG_CACHED;
  731. if (tbl.force_cache_allocs && (!(ion_flag & ION_FLAG_SECURE)))
  732. ion_flag |= ION_FLAG_CACHED;
  733. *buf = ion_alloc(len, heap_id, ion_flag);
  734. if (IS_ERR_OR_NULL(*buf))
  735. return -ENOMEM;
  736. *i_ino = file_inode((*buf)->file)->i_ino;
  737. if (g_cam_mem_mgr_debug.alloc_profile_enable) {
  738. CAM_GET_TIMESTAMP(ts2);
  739. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  740. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  741. len, microsec);
  742. }
  743. return rc;
  744. }
  745. #endif
  746. static int cam_mem_util_buffer_alloc(size_t len, uint32_t flags,
  747. struct dma_buf **dmabuf,
  748. int *fd,
  749. unsigned long *i_ino)
  750. {
  751. int rc;
  752. rc = cam_mem_util_get_dma_buf(len, flags, dmabuf, i_ino);
  753. if (rc) {
  754. CAM_ERR(CAM_MEM,
  755. "Error allocating dma buf : len=%llu, flags=0x%x",
  756. len, flags);
  757. return rc;
  758. }
  759. /*
  760. * increment the ref count so that ref count becomes 2 here
  761. * when we close fd, refcount becomes 1 and when we do
  762. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  763. */
  764. get_dma_buf(*dmabuf);
  765. *fd = dma_buf_fd(*dmabuf, O_CLOEXEC);
  766. if (*fd < 0) {
  767. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  768. rc = -EINVAL;
  769. goto put_buf;
  770. }
  771. CAM_DBG(CAM_MEM, "Alloc success : len=%zu, *dmabuf=%pK, fd=%d, i_ino=%lu",
  772. len, *dmabuf, *fd, *i_ino);
  773. return rc;
  774. put_buf:
  775. dma_buf_put(*dmabuf);
  776. return rc;
  777. }
  778. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd_v2 *cmd)
  779. {
  780. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  781. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  782. CAM_MEM_MMU_MAX_HANDLE);
  783. return -EINVAL;
  784. }
  785. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  786. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  787. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  788. return -EINVAL;
  789. }
  790. if ((cmd->flags & CAM_MEM_FLAG_EVA_NOPIXEL) &&
  791. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE ||
  792. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS)) {
  793. CAM_ERR(CAM_MEM,
  794. "Kernel mapping and secure mode not allowed in no pixel mode");
  795. return -EINVAL;
  796. }
  797. if (cmd->flags & CAM_MEM_FLAG_UBWC_P_HEAP &&
  798. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE ||
  799. cmd->flags & CAM_MEM_FLAG_EVA_NOPIXEL ||
  800. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS ||
  801. cmd->flags & CAM_MEM_FLAG_CMD_BUF_TYPE ||
  802. cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  803. cmd->flags & CAM_MEM_FLAG_HW_AND_CDM_OR_SHARED)) {
  804. CAM_ERR(CAM_MEM,
  805. "UBWC-P buffer not supported with this combinatation of flags 0x%x",
  806. cmd->flags);
  807. return -EINVAL;
  808. }
  809. return 0;
  810. }
  811. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd_v2 *cmd)
  812. {
  813. if (!cmd->flags) {
  814. CAM_ERR(CAM_MEM, "Invalid flags");
  815. return -EINVAL;
  816. }
  817. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  818. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  819. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  820. return -EINVAL;
  821. }
  822. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  823. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  824. CAM_ERR(CAM_MEM,
  825. "Kernel mapping in secure mode not allowed, flags=0x%x",
  826. cmd->flags);
  827. return -EINVAL;
  828. }
  829. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  830. CAM_ERR(CAM_MEM,
  831. "Shared memory buffers are not allowed to be mapped");
  832. return -EINVAL;
  833. }
  834. return 0;
  835. }
  836. static int cam_mem_util_map_hw_va(uint32_t flags,
  837. int32_t *mmu_hdls,
  838. int32_t num_hdls,
  839. int fd,
  840. struct dma_buf *dmabuf,
  841. dma_addr_t *hw_vaddr,
  842. size_t *len,
  843. enum cam_smmu_region_id region,
  844. bool is_internal)
  845. {
  846. int i;
  847. int rc = -1;
  848. int dir = cam_mem_util_get_dma_dir(flags);
  849. bool dis_delayed_unmap = false;
  850. if (dir < 0) {
  851. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  852. return dir;
  853. }
  854. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  855. dis_delayed_unmap = true;
  856. CAM_DBG(CAM_MEM,
  857. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  858. fd, flags, dir, num_hdls);
  859. for (i = 0; i < num_hdls; i++) {
  860. /* If 36-bit enabled, check for ICP cmd buffers and map them within the shared region */
  861. if (cam_smmu_is_expanded_memory() &&
  862. cam_smmu_supports_shared_region(mmu_hdls[i]) &&
  863. ((flags & CAM_MEM_FLAG_CMD_BUF_TYPE) ||
  864. (flags & CAM_MEM_FLAG_HW_AND_CDM_OR_SHARED)))
  865. region = CAM_SMMU_REGION_SHARED;
  866. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  867. rc = cam_smmu_map_stage2_iova(mmu_hdls[i], fd, dmabuf, dir, hw_vaddr, len);
  868. else
  869. rc = cam_smmu_map_user_iova(mmu_hdls[i], fd, dmabuf, dis_delayed_unmap, dir,
  870. hw_vaddr, len, region, is_internal);
  871. if (rc) {
  872. CAM_ERR(CAM_MEM,
  873. "Failed %s map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  874. (flags & CAM_MEM_FLAG_PROTECTED_MODE) ? "" : "secured",
  875. i, fd, dir, mmu_hdls[i], rc);
  876. goto multi_map_fail;
  877. }
  878. }
  879. return rc;
  880. multi_map_fail:
  881. for (--i; i>= 0; i--) {
  882. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  883. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd, dmabuf);
  884. else
  885. cam_smmu_unmap_user_iova(mmu_hdls[i], fd, dmabuf, CAM_SMMU_REGION_IO);
  886. }
  887. return rc;
  888. }
  889. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd_v2 *cmd)
  890. {
  891. int rc;
  892. int32_t idx;
  893. struct dma_buf *dmabuf = NULL;
  894. int fd = -1;
  895. dma_addr_t hw_vaddr = 0;
  896. size_t len;
  897. uintptr_t kvaddr = 0;
  898. size_t klen;
  899. unsigned long i_ino = 0;
  900. if (!atomic_read(&cam_mem_mgr_state)) {
  901. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  902. return -EINVAL;
  903. }
  904. if (!cmd) {
  905. CAM_ERR(CAM_MEM, " Invalid argument");
  906. return -EINVAL;
  907. }
  908. len = cmd->len;
  909. if (tbl.need_shared_buffer_padding &&
  910. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)) {
  911. len += CAM_MEM_SHARED_BUFFER_PAD_4K;
  912. CAM_DBG(CAM_MEM, "Pad 4k size, actual %llu, allocating %zu",
  913. cmd->len, len);
  914. }
  915. rc = cam_mem_util_check_alloc_flags(cmd);
  916. if (rc) {
  917. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  918. cmd->flags, rc);
  919. return rc;
  920. }
  921. rc = cam_mem_util_buffer_alloc(len, cmd->flags, &dmabuf, &fd, &i_ino);
  922. if (rc) {
  923. CAM_ERR(CAM_MEM,
  924. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  925. len, cmd->align, cmd->flags, cmd->num_hdl);
  926. cam_mem_mgr_print_tbl();
  927. return rc;
  928. }
  929. if (!dmabuf) {
  930. CAM_ERR(CAM_MEM,
  931. "Ion Alloc return NULL dmabuf! fd=%d, i_ino=%lu, len=%d", fd, i_ino, len);
  932. cam_mem_mgr_print_tbl();
  933. return rc;
  934. }
  935. idx = cam_mem_get_slot();
  936. if (idx < 0) {
  937. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  938. rc = -ENOMEM;
  939. goto slot_fail;
  940. }
  941. if (cam_dma_buf_set_name(dmabuf, cmd->buf_name))
  942. CAM_ERR(CAM_MEM, "set dma buffer name(%s) failed", cmd->buf_name);
  943. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  944. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  945. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  946. enum cam_smmu_region_id region;
  947. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  948. region = CAM_SMMU_REGION_IO;
  949. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  950. region = CAM_SMMU_REGION_SHARED;
  951. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  952. region = CAM_SMMU_REGION_IO;
  953. rc = cam_mem_util_map_hw_va(cmd->flags,
  954. cmd->mmu_hdls,
  955. cmd->num_hdl,
  956. fd,
  957. dmabuf,
  958. &hw_vaddr,
  959. &len,
  960. region,
  961. true);
  962. if (rc) {
  963. CAM_ERR(CAM_MEM,
  964. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  965. len, cmd->flags,
  966. fd, region, cmd->num_hdl, rc);
  967. if (rc == -EALREADY) {
  968. if ((size_t)dmabuf->size != len)
  969. rc = -EBADR;
  970. cam_mem_mgr_print_tbl();
  971. }
  972. goto map_hw_fail;
  973. }
  974. }
  975. mutex_lock(&tbl.bufq[idx].q_lock);
  976. tbl.bufq[idx].fd = fd;
  977. tbl.bufq[idx].i_ino = i_ino;
  978. tbl.bufq[idx].dma_buf = NULL;
  979. tbl.bufq[idx].flags = cmd->flags;
  980. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  981. tbl.bufq[idx].is_internal = true;
  982. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  983. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  984. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  985. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  986. if (rc) {
  987. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  988. dmabuf, rc);
  989. goto map_kernel_fail;
  990. }
  991. }
  992. if (cmd->flags & CAM_MEM_FLAG_KMD_DEBUG_FLAG)
  993. tbl.dbg_buf_idx = idx;
  994. tbl.bufq[idx].kmdvaddr = kvaddr;
  995. tbl.bufq[idx].vaddr = hw_vaddr;
  996. tbl.bufq[idx].dma_buf = dmabuf;
  997. tbl.bufq[idx].len = len;
  998. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  999. cam_mem_mgr_reset_presil_params(idx);
  1000. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  1001. sizeof(int32_t) * cmd->num_hdl);
  1002. tbl.bufq[idx].is_imported = false;
  1003. mutex_unlock(&tbl.bufq[idx].q_lock);
  1004. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  1005. cmd->out.fd = tbl.bufq[idx].fd;
  1006. cmd->out.vaddr = 0;
  1007. CAM_DBG(CAM_MEM,
  1008. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu, i_ino=%lu, name:%s",
  1009. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  1010. tbl.bufq[idx].len, tbl.bufq[idx].i_ino, cmd->buf_name);
  1011. return rc;
  1012. map_kernel_fail:
  1013. mutex_unlock(&tbl.bufq[idx].q_lock);
  1014. map_hw_fail:
  1015. cam_mem_put_slot(idx);
  1016. slot_fail:
  1017. dma_buf_put(dmabuf);
  1018. return rc;
  1019. }
  1020. static bool cam_mem_util_is_map_internal(int32_t fd, unsigned i_ino)
  1021. {
  1022. uint32_t i;
  1023. bool is_internal = false;
  1024. mutex_lock(&tbl.m_lock);
  1025. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  1026. if ((tbl.bufq[i].fd == fd) && (tbl.bufq[i].i_ino == i_ino)) {
  1027. is_internal = tbl.bufq[i].is_internal;
  1028. break;
  1029. }
  1030. }
  1031. mutex_unlock(&tbl.m_lock);
  1032. return is_internal;
  1033. }
  1034. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd_v2 *cmd)
  1035. {
  1036. int32_t idx;
  1037. int rc;
  1038. struct dma_buf *dmabuf;
  1039. dma_addr_t hw_vaddr = 0;
  1040. size_t len = 0;
  1041. bool is_internal = false;
  1042. unsigned long i_ino;
  1043. if (!atomic_read(&cam_mem_mgr_state)) {
  1044. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1045. return -EINVAL;
  1046. }
  1047. if (!cmd || (cmd->fd < 0)) {
  1048. CAM_ERR(CAM_MEM, "Invalid argument");
  1049. return -EINVAL;
  1050. }
  1051. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  1052. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  1053. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  1054. return -EINVAL;
  1055. }
  1056. rc = cam_mem_util_check_map_flags(cmd);
  1057. if (rc) {
  1058. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  1059. return rc;
  1060. }
  1061. dmabuf = dma_buf_get(cmd->fd);
  1062. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  1063. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  1064. return -EINVAL;
  1065. }
  1066. i_ino = file_inode(dmabuf->file)->i_ino;
  1067. is_internal = cam_mem_util_is_map_internal(cmd->fd, i_ino);
  1068. idx = cam_mem_get_slot();
  1069. if (idx < 0) {
  1070. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d, fd=%d",
  1071. idx, cmd->fd);
  1072. rc = -ENOMEM;
  1073. goto slot_fail;
  1074. }
  1075. if (cam_dma_buf_set_name(dmabuf, cmd->buf_name))
  1076. CAM_ERR(CAM_MEM, "set dma buffer name(%s) failed", cmd->buf_name);
  1077. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1078. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1079. rc = cam_mem_util_map_hw_va(cmd->flags,
  1080. cmd->mmu_hdls,
  1081. cmd->num_hdl,
  1082. cmd->fd,
  1083. dmabuf,
  1084. &hw_vaddr,
  1085. &len,
  1086. CAM_SMMU_REGION_IO,
  1087. is_internal);
  1088. if (rc) {
  1089. CAM_ERR(CAM_MEM,
  1090. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  1091. cmd->flags, cmd->fd, len,
  1092. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  1093. if (rc == -EALREADY) {
  1094. if ((size_t)dmabuf->size != len) {
  1095. rc = -EBADR;
  1096. cam_mem_mgr_print_tbl();
  1097. }
  1098. }
  1099. goto map_fail;
  1100. }
  1101. }
  1102. mutex_lock(&tbl.bufq[idx].q_lock);
  1103. tbl.bufq[idx].fd = cmd->fd;
  1104. tbl.bufq[idx].i_ino = i_ino;
  1105. tbl.bufq[idx].dma_buf = NULL;
  1106. tbl.bufq[idx].flags = cmd->flags;
  1107. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  1108. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  1109. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  1110. tbl.bufq[idx].kmdvaddr = 0;
  1111. if (cmd->num_hdl > 0)
  1112. tbl.bufq[idx].vaddr = hw_vaddr;
  1113. else
  1114. tbl.bufq[idx].vaddr = 0;
  1115. tbl.bufq[idx].dma_buf = dmabuf;
  1116. tbl.bufq[idx].len = len;
  1117. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  1118. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  1119. sizeof(int32_t) * cmd->num_hdl);
  1120. tbl.bufq[idx].is_imported = true;
  1121. tbl.bufq[idx].is_internal = is_internal;
  1122. mutex_unlock(&tbl.bufq[idx].q_lock);
  1123. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  1124. cmd->out.vaddr = 0;
  1125. cmd->out.size = (uint32_t)len;
  1126. CAM_DBG(CAM_MEM,
  1127. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu, i_ino=%lu, name:%s",
  1128. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  1129. tbl.bufq[idx].len, tbl.bufq[idx].i_ino, cmd->buf_name);
  1130. return rc;
  1131. map_fail:
  1132. cam_mem_put_slot(idx);
  1133. slot_fail:
  1134. dma_buf_put(dmabuf);
  1135. return rc;
  1136. }
  1137. static int cam_mem_util_unmap_hw_va(int32_t idx,
  1138. enum cam_smmu_region_id region,
  1139. enum cam_smmu_mapping_client client)
  1140. {
  1141. int i;
  1142. uint32_t flags;
  1143. int32_t *mmu_hdls;
  1144. int num_hdls;
  1145. int fd;
  1146. struct dma_buf *dma_buf;
  1147. unsigned long i_ino;
  1148. int rc = 0;
  1149. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1150. CAM_ERR(CAM_MEM, "Incorrect index");
  1151. return -EINVAL;
  1152. }
  1153. flags = tbl.bufq[idx].flags;
  1154. mmu_hdls = tbl.bufq[idx].hdls;
  1155. num_hdls = tbl.bufq[idx].num_hdl;
  1156. fd = tbl.bufq[idx].fd;
  1157. dma_buf = tbl.bufq[idx].dma_buf;
  1158. i_ino = tbl.bufq[idx].i_ino;
  1159. CAM_DBG(CAM_MEM,
  1160. "unmap_hw_va : idx=%d, fd=%x, i_ino=%lu flags=0x%x, num_hdls=%d, client=%d",
  1161. idx, fd, i_ino, flags, num_hdls, client);
  1162. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  1163. for (i = 0; i < num_hdls; i++) {
  1164. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd, dma_buf);
  1165. if (rc < 0) {
  1166. CAM_ERR(CAM_MEM,
  1167. "Failed in secure unmap, i=%d, fd=%d, i_ino=%lu, mmu_hdl=%d, rc=%d",
  1168. i, fd, i_ino, mmu_hdls[i], rc);
  1169. goto unmap_end;
  1170. }
  1171. }
  1172. } else {
  1173. for (i = 0; i < num_hdls; i++) {
  1174. if (client == CAM_SMMU_MAPPING_USER) {
  1175. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  1176. fd, dma_buf, region);
  1177. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  1178. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  1179. tbl.bufq[idx].dma_buf, region);
  1180. } else {
  1181. CAM_ERR(CAM_MEM,
  1182. "invalid caller for unmapping : %d",
  1183. client);
  1184. rc = -EINVAL;
  1185. }
  1186. if (rc < 0) {
  1187. CAM_ERR(CAM_MEM,
  1188. "Failed in unmap, i=%d, fd=%d, i_ino=%lu, mmu_hdl=%d, region=%d, rc=%d",
  1189. i, fd, i_ino, mmu_hdls[i], region, rc);
  1190. goto unmap_end;
  1191. }
  1192. }
  1193. }
  1194. return rc;
  1195. unmap_end:
  1196. CAM_ERR(CAM_MEM, "unmapping failed");
  1197. return rc;
  1198. }
  1199. static void cam_mem_mgr_unmap_active_buf(int idx)
  1200. {
  1201. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1202. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  1203. region = CAM_SMMU_REGION_SHARED;
  1204. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1205. region = CAM_SMMU_REGION_IO;
  1206. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  1207. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS)
  1208. cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1209. tbl.bufq[idx].kmdvaddr);
  1210. }
  1211. static int cam_mem_mgr_cleanup_table(void)
  1212. {
  1213. int i;
  1214. mutex_lock(&tbl.m_lock);
  1215. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  1216. if (!tbl.bufq[i].active) {
  1217. CAM_DBG(CAM_MEM,
  1218. "Buffer inactive at idx=%d, continuing", i);
  1219. continue;
  1220. } else {
  1221. CAM_DBG(CAM_MEM,
  1222. "Active buffer at idx=%d, possible leak needs unmapping",
  1223. i);
  1224. cam_mem_mgr_unmap_active_buf(i);
  1225. }
  1226. mutex_lock(&tbl.bufq[i].q_lock);
  1227. if (tbl.bufq[i].dma_buf) {
  1228. dma_buf_put(tbl.bufq[i].dma_buf);
  1229. tbl.bufq[i].dma_buf = NULL;
  1230. }
  1231. tbl.bufq[i].fd = -1;
  1232. tbl.bufq[i].i_ino = 0;
  1233. tbl.bufq[i].flags = 0;
  1234. tbl.bufq[i].buf_handle = -1;
  1235. tbl.bufq[i].vaddr = 0;
  1236. tbl.bufq[i].len = 0;
  1237. memset(tbl.bufq[i].hdls, 0,
  1238. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  1239. tbl.bufq[i].num_hdl = 0;
  1240. tbl.bufq[i].dma_buf = NULL;
  1241. tbl.bufq[i].active = false;
  1242. tbl.bufq[i].is_internal = false;
  1243. cam_mem_mgr_reset_presil_params(i);
  1244. mutex_unlock(&tbl.bufq[i].q_lock);
  1245. mutex_destroy(&tbl.bufq[i].q_lock);
  1246. }
  1247. bitmap_zero(tbl.bitmap, tbl.bits);
  1248. /* We need to reserve slot 0 because 0 is invalid */
  1249. set_bit(0, tbl.bitmap);
  1250. mutex_unlock(&tbl.m_lock);
  1251. return 0;
  1252. }
  1253. void cam_mem_mgr_deinit(void)
  1254. {
  1255. if (!atomic_read(&cam_mem_mgr_state))
  1256. return;
  1257. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  1258. cam_mem_mgr_cleanup_table();
  1259. mutex_lock(&tbl.m_lock);
  1260. bitmap_zero(tbl.bitmap, tbl.bits);
  1261. kfree(tbl.bitmap);
  1262. tbl.bitmap = NULL;
  1263. tbl.dbg_buf_idx = -1;
  1264. mutex_unlock(&tbl.m_lock);
  1265. mutex_destroy(&tbl.m_lock);
  1266. }
  1267. static int cam_mem_util_unmap(int32_t idx,
  1268. enum cam_smmu_mapping_client client)
  1269. {
  1270. int rc = 0;
  1271. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1272. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1273. CAM_ERR(CAM_MEM, "Incorrect index");
  1274. return -EINVAL;
  1275. }
  1276. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  1277. mutex_lock(&tbl.m_lock);
  1278. if ((!tbl.bufq[idx].active) &&
  1279. (tbl.bufq[idx].vaddr) == 0) {
  1280. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  1281. idx);
  1282. mutex_unlock(&tbl.m_lock);
  1283. return 0;
  1284. }
  1285. /* Deactivate the buffer queue to prevent multiple unmap */
  1286. mutex_lock(&tbl.bufq[idx].q_lock);
  1287. tbl.bufq[idx].active = false;
  1288. tbl.bufq[idx].vaddr = 0;
  1289. mutex_unlock(&tbl.bufq[idx].q_lock);
  1290. mutex_unlock(&tbl.m_lock);
  1291. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1292. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  1293. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1294. tbl.bufq[idx].kmdvaddr);
  1295. if (rc)
  1296. CAM_ERR(CAM_MEM,
  1297. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  1298. tbl.bufq[idx].dma_buf,
  1299. (void *) tbl.bufq[idx].kmdvaddr);
  1300. }
  1301. }
  1302. /* SHARED flag gets precedence, all other flags after it */
  1303. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1304. region = CAM_SMMU_REGION_SHARED;
  1305. } else {
  1306. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1307. region = CAM_SMMU_REGION_IO;
  1308. }
  1309. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1310. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  1311. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1312. if (cam_mem_util_unmap_hw_va(idx, region, client))
  1313. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  1314. tbl.bufq[idx].dma_buf);
  1315. }
  1316. mutex_lock(&tbl.m_lock);
  1317. mutex_lock(&tbl.bufq[idx].q_lock);
  1318. tbl.bufq[idx].flags = 0;
  1319. tbl.bufq[idx].buf_handle = -1;
  1320. memset(tbl.bufq[idx].hdls, 0,
  1321. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  1322. CAM_DBG(CAM_MEM,
  1323. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK, i_ino %lu",
  1324. idx, tbl.bufq[idx].fd, tbl.bufq[idx].is_imported, tbl.bufq[idx].dma_buf,
  1325. tbl.bufq[idx].i_ino);
  1326. if (tbl.bufq[idx].dma_buf)
  1327. dma_buf_put(tbl.bufq[idx].dma_buf);
  1328. tbl.bufq[idx].fd = -1;
  1329. tbl.bufq[idx].i_ino = 0;
  1330. tbl.bufq[idx].dma_buf = NULL;
  1331. tbl.bufq[idx].is_imported = false;
  1332. tbl.bufq[idx].is_internal = false;
  1333. tbl.bufq[idx].len = 0;
  1334. tbl.bufq[idx].num_hdl = 0;
  1335. cam_mem_mgr_reset_presil_params(idx);
  1336. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  1337. mutex_unlock(&tbl.bufq[idx].q_lock);
  1338. mutex_destroy(&tbl.bufq[idx].q_lock);
  1339. clear_bit(idx, tbl.bitmap);
  1340. mutex_unlock(&tbl.m_lock);
  1341. return rc;
  1342. }
  1343. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  1344. {
  1345. int idx;
  1346. int rc;
  1347. if (!atomic_read(&cam_mem_mgr_state)) {
  1348. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1349. return -EINVAL;
  1350. }
  1351. if (!cmd) {
  1352. CAM_ERR(CAM_MEM, "Invalid argument");
  1353. return -EINVAL;
  1354. }
  1355. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  1356. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1357. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  1358. idx);
  1359. return -EINVAL;
  1360. }
  1361. if (!tbl.bufq[idx].active) {
  1362. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1363. return -EINVAL;
  1364. }
  1365. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  1366. CAM_ERR(CAM_MEM,
  1367. "Released buf handle %d not matching within table %d, idx=%d",
  1368. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  1369. return -EINVAL;
  1370. }
  1371. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  1372. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  1373. return rc;
  1374. }
  1375. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  1376. struct cam_mem_mgr_memory_desc *out)
  1377. {
  1378. struct dma_buf *buf = NULL;
  1379. int ion_fd = -1;
  1380. int rc = 0;
  1381. uintptr_t kvaddr;
  1382. dma_addr_t iova = 0;
  1383. size_t request_len = 0;
  1384. uint32_t mem_handle;
  1385. int32_t idx;
  1386. int32_t smmu_hdl = 0;
  1387. int32_t num_hdl = 0;
  1388. unsigned long i_ino = 0;
  1389. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1390. if (!atomic_read(&cam_mem_mgr_state)) {
  1391. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1392. return -EINVAL;
  1393. }
  1394. if (!inp || !out) {
  1395. CAM_ERR(CAM_MEM, "Invalid params");
  1396. return -EINVAL;
  1397. }
  1398. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1399. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1400. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1401. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1402. return -EINVAL;
  1403. }
  1404. rc = cam_mem_util_get_dma_buf(inp->size, inp->flags, &buf, &i_ino);
  1405. if (rc) {
  1406. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1407. goto ion_fail;
  1408. } else if (!buf) {
  1409. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1410. goto ion_fail;
  1411. } else {
  1412. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1413. }
  1414. /*
  1415. * we are mapping kva always here,
  1416. * update flags so that we do unmap properly
  1417. */
  1418. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1419. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1420. if (rc) {
  1421. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1422. goto map_fail;
  1423. }
  1424. if (!inp->smmu_hdl) {
  1425. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1426. rc = -EINVAL;
  1427. goto smmu_fail;
  1428. }
  1429. /* SHARED flag gets precedence, all other flags after it */
  1430. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1431. region = CAM_SMMU_REGION_SHARED;
  1432. } else {
  1433. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1434. region = CAM_SMMU_REGION_IO;
  1435. }
  1436. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1437. buf,
  1438. CAM_SMMU_MAP_RW,
  1439. &iova,
  1440. &request_len,
  1441. region);
  1442. if (rc < 0) {
  1443. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1444. goto smmu_fail;
  1445. }
  1446. smmu_hdl = inp->smmu_hdl;
  1447. num_hdl = 1;
  1448. idx = cam_mem_get_slot();
  1449. if (idx < 0) {
  1450. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1451. rc = -ENOMEM;
  1452. goto slot_fail;
  1453. }
  1454. mutex_lock(&tbl.bufq[idx].q_lock);
  1455. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1456. tbl.bufq[idx].dma_buf = buf;
  1457. tbl.bufq[idx].fd = -1;
  1458. tbl.bufq[idx].i_ino = i_ino;
  1459. tbl.bufq[idx].flags = inp->flags;
  1460. tbl.bufq[idx].buf_handle = mem_handle;
  1461. tbl.bufq[idx].kmdvaddr = kvaddr;
  1462. tbl.bufq[idx].vaddr = iova;
  1463. tbl.bufq[idx].len = inp->size;
  1464. tbl.bufq[idx].num_hdl = num_hdl;
  1465. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1466. sizeof(int32_t));
  1467. tbl.bufq[idx].is_imported = false;
  1468. mutex_unlock(&tbl.bufq[idx].q_lock);
  1469. out->kva = kvaddr;
  1470. out->iova = (uint32_t)iova;
  1471. out->smmu_hdl = smmu_hdl;
  1472. out->mem_handle = mem_handle;
  1473. out->len = inp->size;
  1474. out->region = region;
  1475. CAM_DBG(CAM_MEM, "idx=%d, dmabuf=%pK, i_ino=%lu, flags=0x%x, mem_handle=0x%x",
  1476. idx, buf, i_ino, inp->flags, mem_handle);
  1477. return rc;
  1478. slot_fail:
  1479. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1480. buf, region);
  1481. smmu_fail:
  1482. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1483. map_fail:
  1484. dma_buf_put(buf);
  1485. ion_fail:
  1486. return rc;
  1487. }
  1488. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1489. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1490. {
  1491. int32_t idx;
  1492. int rc;
  1493. if (!atomic_read(&cam_mem_mgr_state)) {
  1494. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1495. return -EINVAL;
  1496. }
  1497. if (!inp) {
  1498. CAM_ERR(CAM_MEM, "Invalid argument");
  1499. return -EINVAL;
  1500. }
  1501. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1502. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1503. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1504. return -EINVAL;
  1505. }
  1506. if (!tbl.bufq[idx].active) {
  1507. if (tbl.bufq[idx].vaddr == 0) {
  1508. CAM_ERR(CAM_MEM, "buffer is released already");
  1509. return 0;
  1510. }
  1511. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1512. return -EINVAL;
  1513. }
  1514. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1515. CAM_ERR(CAM_MEM,
  1516. "Released buf handle not matching within table");
  1517. return -EINVAL;
  1518. }
  1519. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1520. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1521. return rc;
  1522. }
  1523. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1524. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1525. enum cam_smmu_region_id region,
  1526. struct cam_mem_mgr_memory_desc *out)
  1527. {
  1528. struct dma_buf *buf = NULL;
  1529. int rc = 0;
  1530. int ion_fd = -1;
  1531. dma_addr_t iova = 0;
  1532. size_t request_len = 0;
  1533. uint32_t mem_handle;
  1534. int32_t idx;
  1535. int32_t smmu_hdl = 0;
  1536. int32_t num_hdl = 0;
  1537. uintptr_t kvaddr = 0;
  1538. unsigned long i_ino = 0;
  1539. if (!atomic_read(&cam_mem_mgr_state)) {
  1540. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1541. return -EINVAL;
  1542. }
  1543. if (!inp || !out) {
  1544. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1545. return -EINVAL;
  1546. }
  1547. if (!inp->smmu_hdl) {
  1548. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1549. return -EINVAL;
  1550. }
  1551. if ((region != CAM_SMMU_REGION_SECHEAP) &&
  1552. (region != CAM_SMMU_REGION_FWUNCACHED)) {
  1553. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1554. return -EINVAL;
  1555. }
  1556. rc = cam_mem_util_get_dma_buf(inp->size, 0, &buf, &i_ino);
  1557. if (rc) {
  1558. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1559. goto ion_fail;
  1560. } else if (!buf) {
  1561. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1562. goto ion_fail;
  1563. } else {
  1564. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1565. }
  1566. if (inp->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1567. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1568. if (rc) {
  1569. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1570. goto kmap_fail;
  1571. }
  1572. }
  1573. rc = cam_smmu_reserve_buf_region(region,
  1574. inp->smmu_hdl, buf, &iova, &request_len);
  1575. if (rc) {
  1576. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1577. goto smmu_fail;
  1578. }
  1579. smmu_hdl = inp->smmu_hdl;
  1580. num_hdl = 1;
  1581. idx = cam_mem_get_slot();
  1582. if (idx < 0) {
  1583. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1584. rc = -ENOMEM;
  1585. goto slot_fail;
  1586. }
  1587. mutex_lock(&tbl.bufq[idx].q_lock);
  1588. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1589. tbl.bufq[idx].fd = -1;
  1590. tbl.bufq[idx].i_ino = i_ino;
  1591. tbl.bufq[idx].dma_buf = buf;
  1592. tbl.bufq[idx].flags = inp->flags;
  1593. tbl.bufq[idx].buf_handle = mem_handle;
  1594. tbl.bufq[idx].kmdvaddr = kvaddr;
  1595. tbl.bufq[idx].vaddr = iova;
  1596. tbl.bufq[idx].len = request_len;
  1597. tbl.bufq[idx].num_hdl = num_hdl;
  1598. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1599. sizeof(int32_t));
  1600. tbl.bufq[idx].is_imported = false;
  1601. mutex_unlock(&tbl.bufq[idx].q_lock);
  1602. out->kva = kvaddr;
  1603. out->iova = (uint32_t)iova;
  1604. out->smmu_hdl = smmu_hdl;
  1605. out->mem_handle = mem_handle;
  1606. out->len = request_len;
  1607. out->region = region;
  1608. return rc;
  1609. slot_fail:
  1610. cam_smmu_release_buf_region(region, smmu_hdl);
  1611. smmu_fail:
  1612. if (region == CAM_SMMU_REGION_FWUNCACHED)
  1613. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1614. kmap_fail:
  1615. dma_buf_put(buf);
  1616. ion_fail:
  1617. return rc;
  1618. }
  1619. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1620. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1621. {
  1622. int32_t idx;
  1623. int rc;
  1624. int32_t smmu_hdl;
  1625. if (!atomic_read(&cam_mem_mgr_state)) {
  1626. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1627. return -EINVAL;
  1628. }
  1629. if (!inp) {
  1630. CAM_ERR(CAM_MEM, "Invalid argument");
  1631. return -EINVAL;
  1632. }
  1633. if ((inp->region != CAM_SMMU_REGION_SECHEAP) &&
  1634. (inp->region != CAM_SMMU_REGION_FWUNCACHED)) {
  1635. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1636. return -EINVAL;
  1637. }
  1638. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1639. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1640. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1641. return -EINVAL;
  1642. }
  1643. if (!tbl.bufq[idx].active) {
  1644. if (tbl.bufq[idx].vaddr == 0) {
  1645. CAM_ERR(CAM_MEM, "buffer is released already");
  1646. return 0;
  1647. }
  1648. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1649. return -EINVAL;
  1650. }
  1651. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1652. CAM_ERR(CAM_MEM,
  1653. "Released buf handle not matching within table");
  1654. return -EINVAL;
  1655. }
  1656. if (tbl.bufq[idx].num_hdl != 1) {
  1657. CAM_ERR(CAM_MEM,
  1658. "Sec heap region should have only one smmu hdl");
  1659. return -ENODEV;
  1660. }
  1661. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1662. sizeof(int32_t));
  1663. if (inp->smmu_hdl != smmu_hdl) {
  1664. CAM_ERR(CAM_MEM,
  1665. "Passed SMMU handle doesn't match with internal hdl");
  1666. return -ENODEV;
  1667. }
  1668. rc = cam_smmu_release_buf_region(inp->region, inp->smmu_hdl);
  1669. if (rc) {
  1670. CAM_ERR(CAM_MEM,
  1671. "Sec heap region release failed");
  1672. return -ENODEV;
  1673. }
  1674. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1675. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1676. if (rc)
  1677. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1678. return rc;
  1679. }
  1680. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);
  1681. #ifdef CONFIG_CAM_PRESIL
  1682. struct dma_buf *cam_mem_mgr_get_dma_buf(int fd)
  1683. {
  1684. struct dma_buf *dmabuf = NULL;
  1685. dmabuf = dma_buf_get(fd);
  1686. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  1687. CAM_ERR(CAM_MEM, "Failed to import dma_buf for fd");
  1688. return NULL;
  1689. }
  1690. CAM_INFO(CAM_PRESIL, "Received DMA Buf* %pK", dmabuf);
  1691. return dmabuf;
  1692. }
  1693. int cam_mem_mgr_put_dmabuf_from_fd(uint64_t input_dmabuf)
  1694. {
  1695. struct dma_buf *dmabuf = (struct dma_buf *)(uint64_t)input_dmabuf;
  1696. int idx = 0;
  1697. CAM_INFO(CAM_PRESIL, "Received dma_buf :%pK", dmabuf);
  1698. if (!dmabuf) {
  1699. CAM_ERR(CAM_PRESIL, "NULL to import dma_buf fd");
  1700. return -EINVAL;
  1701. }
  1702. for (idx = 0; idx < CAM_MEM_BUFQ_MAX; idx++) {
  1703. if ((tbl.bufq[idx].dma_buf != NULL) && (tbl.bufq[idx].dma_buf == dmabuf)) {
  1704. if (tbl.bufq[idx].presil_params.refcount)
  1705. tbl.bufq[idx].presil_params.refcount--;
  1706. else
  1707. CAM_ERR(CAM_PRESIL, "Unbalanced dmabuf put: %pK", dmabuf);
  1708. if (!tbl.bufq[idx].presil_params.refcount) {
  1709. dma_buf_put(dmabuf);
  1710. cam_mem_mgr_reset_presil_params(idx);
  1711. CAM_DBG(CAM_PRESIL, "Done dma_buf_put for %pK", dmabuf);
  1712. }
  1713. }
  1714. }
  1715. return 0;
  1716. }
  1717. int cam_mem_mgr_get_fd_from_dmabuf(uint64_t input_dmabuf)
  1718. {
  1719. int fd_for_dmabuf = -1;
  1720. struct dma_buf *dmabuf = (struct dma_buf *)(uint64_t)input_dmabuf;
  1721. int idx = 0;
  1722. CAM_DBG(CAM_PRESIL, "Received dma_buf :%pK", dmabuf);
  1723. if (!dmabuf) {
  1724. CAM_ERR(CAM_PRESIL, "NULL to import dma_buf fd");
  1725. return -EINVAL;
  1726. }
  1727. for (idx = 0; idx < CAM_MEM_BUFQ_MAX; idx++) {
  1728. if ((tbl.bufq[idx].dma_buf != NULL) && (tbl.bufq[idx].dma_buf == dmabuf)) {
  1729. CAM_DBG(CAM_PRESIL,
  1730. "Found entry for request from Presil UMD Daemon at %d, dmabuf %pK fd_for_umd_daemon %d refcount: %d",
  1731. idx, tbl.bufq[idx].dma_buf,
  1732. tbl.bufq[idx].presil_params.fd_for_umd_daemon,
  1733. tbl.bufq[idx].presil_params.refcount);
  1734. if (tbl.bufq[idx].presil_params.fd_for_umd_daemon < 0) {
  1735. fd_for_dmabuf = dma_buf_fd(dmabuf, O_CLOEXEC);
  1736. if (fd_for_dmabuf < 0) {
  1737. CAM_ERR(CAM_PRESIL, "get fd fail, fd_for_dmabuf=%d",
  1738. fd_for_dmabuf);
  1739. return -EINVAL;
  1740. }
  1741. tbl.bufq[idx].presil_params.fd_for_umd_daemon = fd_for_dmabuf;
  1742. CAM_INFO(CAM_PRESIL,
  1743. "Received generated idx %d fd_for_dmabuf Buf* %lld", idx,
  1744. fd_for_dmabuf);
  1745. } else {
  1746. fd_for_dmabuf = tbl.bufq[idx].presil_params.fd_for_umd_daemon;
  1747. CAM_INFO(CAM_PRESIL,
  1748. "Received existing at idx %d fd_for_dmabuf Buf* %lld", idx,
  1749. fd_for_dmabuf);
  1750. }
  1751. tbl.bufq[idx].presil_params.refcount++;
  1752. } else {
  1753. CAM_DBG(CAM_MEM,
  1754. "Not found dmabuf at idx=%d, dma_buf %pK handle 0x%0x active %d ",
  1755. idx, tbl.bufq[idx].dma_buf, tbl.bufq[idx].buf_handle,
  1756. tbl.bufq[idx].active);
  1757. }
  1758. }
  1759. return (int)fd_for_dmabuf;
  1760. }
  1761. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1762. {
  1763. int rc = 0;
  1764. /* Sending Presil IO Buf to PC side ( as iova start address indicates) */
  1765. uint64_t io_buf_addr;
  1766. size_t io_buf_size;
  1767. int i, j, fd = -1, idx = 0;
  1768. uint8_t *iova_ptr = NULL;
  1769. uint64_t dmabuf = 0;
  1770. bool is_mapped_in_cb = false;
  1771. CAM_DBG(CAM_PRESIL, "buf handle 0x%0x", buf_handle);
  1772. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  1773. for (i = 0; i < tbl.bufq[idx].num_hdl; i++) {
  1774. if (tbl.bufq[idx].hdls[i] == iommu_hdl)
  1775. is_mapped_in_cb = true;
  1776. }
  1777. if (!is_mapped_in_cb) {
  1778. for (j = 0; j < CAM_MEM_BUFQ_MAX; j++) {
  1779. if (tbl.bufq[j].i_ino == tbl.bufq[idx].i_ino) {
  1780. for (i = 0; i < tbl.bufq[j].num_hdl; i++) {
  1781. if (tbl.bufq[j].hdls[i] == iommu_hdl)
  1782. is_mapped_in_cb = true;
  1783. }
  1784. }
  1785. }
  1786. if (!is_mapped_in_cb) {
  1787. CAM_DBG(CAM_PRESIL,
  1788. "Still Could not find idx=%d, FD %d buf_handle 0x%0x",
  1789. idx, GET_FD_FROM_HANDLE(buf_handle), buf_handle);
  1790. /*
  1791. * Okay to return 0, since this function also gets called for buffers that
  1792. * are shared only between umd/kmd, these may not be mapped with smmu
  1793. */
  1794. return 0;
  1795. }
  1796. }
  1797. if ((tbl.bufq[idx].buf_handle != 0) && (tbl.bufq[idx].active) &&
  1798. (tbl.bufq[idx].buf_handle == buf_handle)) {
  1799. CAM_DBG(CAM_PRESIL,
  1800. "Found dmabuf in bufq idx %d, FD %d handle 0x%0x dmabuf %pK",
  1801. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].dma_buf);
  1802. dmabuf = (uint64_t)tbl.bufq[idx].dma_buf;
  1803. fd = tbl.bufq[idx].fd;
  1804. } else {
  1805. CAM_ERR(CAM_PRESIL,
  1806. "Could not find dmabuf Invalid Mem idx=%d, FD %d handle 0x%0x active %d",
  1807. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].active);
  1808. return -EINVAL;
  1809. }
  1810. rc = cam_mem_get_io_buf(buf_handle, iommu_hdl, &io_buf_addr, &io_buf_size, NULL);
  1811. if (rc || NULL == (void *)io_buf_addr) {
  1812. CAM_DBG(CAM_PRESIL, "Invalid ioaddr : 0x%x, fd = %d, dmabuf = %pK",
  1813. io_buf_addr, fd, dmabuf);
  1814. return -EINVAL;
  1815. }
  1816. iova_ptr = (uint8_t *)io_buf_addr;
  1817. CAM_INFO(CAM_PRESIL, "Sending buffer with ioaddr : 0x%x, fd = %d, dmabuf = %pK",
  1818. io_buf_addr, fd, dmabuf);
  1819. rc = cam_presil_send_buffer(dmabuf, 0, 0, (uint32_t)io_buf_size, (uint64_t)iova_ptr);
  1820. return rc;
  1821. }
  1822. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1823. {
  1824. int idx = 0;
  1825. int rc = 0;
  1826. int32_t fd_already_sent[128];
  1827. int fd_already_sent_count = 0;
  1828. int fd_already_index = 0;
  1829. int fd_already_sent_found = 0;
  1830. memset(&fd_already_sent, 0x0, sizeof(fd_already_sent));
  1831. for (idx = 0; idx < CAM_MEM_BUFQ_MAX; idx++) {
  1832. if ((tbl.bufq[idx].buf_handle != 0) && (tbl.bufq[idx].active)) {
  1833. CAM_DBG(CAM_PRESIL, "Sending %d, FD %d handle 0x%0x", idx, tbl.bufq[idx].fd,
  1834. tbl.bufq[idx].buf_handle);
  1835. fd_already_sent_found = 0;
  1836. for (fd_already_index = 0; fd_already_index < fd_already_sent_count;
  1837. fd_already_index++) {
  1838. if (fd_already_sent[fd_already_index] == tbl.bufq[idx].fd) {
  1839. fd_already_sent_found = 1;
  1840. CAM_DBG(CAM_PRESIL,
  1841. "fd_already_sent %d, FD %d handle 0x%0x flags=0x%0x",
  1842. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle,
  1843. tbl.bufq[idx].flags);
  1844. }
  1845. }
  1846. if (fd_already_sent_found)
  1847. continue;
  1848. CAM_DBG(CAM_PRESIL, "Sending %d, FD %d handle 0x%0x flags=0x%0x", idx,
  1849. tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].flags);
  1850. rc = cam_mem_mgr_send_buffer_to_presil(iommu_hdl, tbl.bufq[idx].buf_handle);
  1851. fd_already_sent[fd_already_sent_count++] = tbl.bufq[idx].fd;
  1852. } else {
  1853. CAM_DBG(CAM_PRESIL, "Invalid Mem idx=%d, FD %d handle 0x%0x active %d",
  1854. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle,
  1855. tbl.bufq[idx].active);
  1856. }
  1857. }
  1858. return rc;
  1859. }
  1860. EXPORT_SYMBOL(cam_mem_mgr_send_all_buffers_to_presil);
  1861. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle, uint32_t buf_size,
  1862. uint32_t offset, int32_t iommu_hdl)
  1863. {
  1864. int rc = 0;
  1865. /* Receive output buffer from Presil IO Buf to PC side (as iova start address indicates) */
  1866. uint64_t io_buf_addr;
  1867. size_t io_buf_size;
  1868. uint64_t dmabuf = 0;
  1869. int fd = 0;
  1870. uint8_t *iova_ptr = NULL;
  1871. int idx = 0;
  1872. CAM_DBG(CAM_PRESIL, "buf handle 0x%0x ", buf_handle);
  1873. rc = cam_mem_get_io_buf(buf_handle, iommu_hdl, &io_buf_addr, &io_buf_size, NULL);
  1874. if (rc) {
  1875. CAM_ERR(CAM_PRESIL, "Unable to get IOVA for buffer buf_hdl: 0x%0x iommu_hdl: 0x%0x",
  1876. buf_handle, iommu_hdl);
  1877. return -EINVAL;
  1878. }
  1879. iova_ptr = (uint8_t *)io_buf_addr;
  1880. iova_ptr += offset; // correct target address to start writing buffer to.
  1881. if (!buf_size) {
  1882. buf_size = io_buf_size;
  1883. CAM_DBG(CAM_PRESIL, "Updated buf_size from Zero to 0x%0x", buf_size);
  1884. }
  1885. fd = GET_FD_FROM_HANDLE(buf_handle);
  1886. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  1887. if ((tbl.bufq[idx].buf_handle != 0) && (tbl.bufq[idx].active) &&
  1888. (tbl.bufq[idx].buf_handle == buf_handle)) {
  1889. CAM_DBG(CAM_PRESIL, "Found dmabuf in bufq idx %d, FD %d handle 0x%0x dmabuf %pK",
  1890. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].dma_buf);
  1891. dmabuf = (uint64_t)tbl.bufq[idx].dma_buf;
  1892. } else {
  1893. CAM_ERR(CAM_PRESIL,
  1894. "Could not find dmabuf Invalid Mem idx=%d, FD %d handle 0x%0x active %d ",
  1895. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].active);
  1896. }
  1897. CAM_DBG(CAM_PRESIL,
  1898. "Retrieving buffer with ioaddr : 0x%x, offset = %d, size = %d, fd = %d, dmabuf = %pK",
  1899. io_buf_addr, offset, buf_size, fd, dmabuf);
  1900. rc = cam_presil_retrieve_buffer(dmabuf, 0, 0, (uint32_t)buf_size, (uint64_t)io_buf_addr);
  1901. CAM_INFO(CAM_PRESIL,
  1902. "Retrieved buffer with ioaddr : 0x%x, offset = %d, size = %d, fd = %d, dmabuf = %pK",
  1903. io_buf_addr, 0, buf_size, fd, dmabuf);
  1904. return rc;
  1905. }
  1906. #else /* ifdef CONFIG_CAM_PRESIL */
  1907. struct dma_buf * cam_mem_mgr_get_dma_buf(int fd)
  1908. {
  1909. return NULL;
  1910. }
  1911. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1912. {
  1913. return 0;
  1914. }
  1915. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1916. {
  1917. return 0;
  1918. }
  1919. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle,
  1920. uint32_t buf_size,
  1921. uint32_t offset,
  1922. int32_t iommu_hdl)
  1923. {
  1924. return 0;
  1925. }
  1926. #endif /* ifdef CONFIG_CAM_PRESIL */