cam_mem_mgr.c 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include <linux/debugfs.h>
  12. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  13. #include <linux/mem-buf.h>
  14. #include <soc/qcom/secure_buffer.h>
  15. #endif
  16. #include "cam_compat.h"
  17. #include "cam_req_mgr_util.h"
  18. #include "cam_mem_mgr.h"
  19. #include "cam_smmu_api.h"
  20. #include "cam_debug_util.h"
  21. #include "cam_trace.h"
  22. #include "cam_common_util.h"
  23. #define CAM_MEM_SHARED_BUFFER_PAD_4K (4 * 1024)
  24. static struct cam_mem_table tbl;
  25. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  26. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  27. static void cam_mem_mgr_put_dma_heaps(void);
  28. static int cam_mem_mgr_get_dma_heaps(void);
  29. #endif
  30. static void cam_mem_mgr_print_tbl(void)
  31. {
  32. int i;
  33. uint64_t ms, hrs, min, sec;
  34. struct timespec64 current_ts;
  35. CAM_GET_TIMESTAMP(current_ts);
  36. CAM_CONVERT_TIMESTAMP_FORMAT(current_ts, hrs, min, sec, ms);
  37. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  38. hrs, min, sec, ms);
  39. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  40. if (tbl.bufq[i].active) {
  41. CAM_CONVERT_TIMESTAMP_FORMAT((tbl.bufq[i].timestamp), hrs, min, sec, ms);
  42. CAM_INFO(CAM_MEM,
  43. "%llu:%llu:%llu:%llu idx %d fd %d i_ino %lu size %llu",
  44. hrs, min, sec, ms, i, tbl.bufq[i].fd, tbl.bufq[i].i_ino,
  45. tbl.bufq[i].len);
  46. }
  47. }
  48. }
  49. static int cam_mem_util_get_dma_dir(uint32_t flags)
  50. {
  51. int rc = -EINVAL;
  52. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  53. rc = DMA_TO_DEVICE;
  54. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  55. rc = DMA_FROM_DEVICE;
  56. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  57. rc = DMA_BIDIRECTIONAL;
  58. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  59. rc = DMA_BIDIRECTIONAL;
  60. return rc;
  61. }
  62. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  63. uintptr_t *vaddr,
  64. size_t *len)
  65. {
  66. int rc = 0;
  67. void *addr;
  68. /*
  69. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  70. * need to be called in pair to avoid stability issue.
  71. */
  72. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  73. if (rc) {
  74. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  75. return rc;
  76. }
  77. addr = dma_buf_vmap(dmabuf);
  78. if (!addr) {
  79. CAM_ERR(CAM_MEM, "kernel map fail");
  80. *vaddr = 0;
  81. *len = 0;
  82. rc = -ENOSPC;
  83. goto fail;
  84. }
  85. *vaddr = (uint64_t)addr;
  86. *len = dmabuf->size;
  87. return 0;
  88. fail:
  89. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  90. return rc;
  91. }
  92. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  93. uint64_t vaddr)
  94. {
  95. int rc = 0;
  96. if (!dmabuf || !vaddr) {
  97. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  98. return -EINVAL;
  99. }
  100. dma_buf_vunmap(dmabuf, (void *)vaddr);
  101. /*
  102. * dma_buf_begin_cpu_access() and
  103. * dma_buf_end_cpu_access() need to be called in pair
  104. * to avoid stability issue.
  105. */
  106. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  107. if (rc) {
  108. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  109. dmabuf);
  110. return rc;
  111. }
  112. return rc;
  113. }
  114. static int cam_mem_mgr_create_debug_fs(void)
  115. {
  116. int rc = 0;
  117. struct dentry *dbgfileptr = NULL;
  118. dbgfileptr = debugfs_create_dir("camera_memmgr", NULL);
  119. if (!dbgfileptr) {
  120. CAM_ERR(CAM_MEM,"DebugFS could not create directory!");
  121. rc = -ENOENT;
  122. goto end;
  123. }
  124. /* Store parent inode for cleanup in caller */
  125. tbl.dentry = dbgfileptr;
  126. dbgfileptr = debugfs_create_bool("alloc_profile_enable", 0644,
  127. tbl.dentry, &tbl.alloc_profile_enable);
  128. if (IS_ERR(dbgfileptr)) {
  129. if (PTR_ERR(dbgfileptr) == -ENODEV)
  130. CAM_WARN(CAM_MEM, "DebugFS not enabled in kernel!");
  131. else
  132. rc = PTR_ERR(dbgfileptr);
  133. }
  134. end:
  135. return rc;
  136. }
  137. int cam_mem_mgr_init(void)
  138. {
  139. int i;
  140. int bitmap_size;
  141. int rc = 0;
  142. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  143. if (cam_smmu_need_force_alloc_cached(&tbl.force_cache_allocs)) {
  144. CAM_ERR(CAM_MEM, "Error in getting force cache alloc flag");
  145. return -EINVAL;
  146. }
  147. tbl.need_shared_buffer_padding = cam_smmu_need_shared_buffer_padding();
  148. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  149. rc = cam_mem_mgr_get_dma_heaps();
  150. if (rc) {
  151. CAM_ERR(CAM_MEM, "Failed in getting dma heaps rc=%d", rc);
  152. return rc;
  153. }
  154. #endif
  155. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  156. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  157. if (!tbl.bitmap) {
  158. rc = -ENOMEM;
  159. goto put_heaps;
  160. }
  161. tbl.bits = bitmap_size * BITS_PER_BYTE;
  162. bitmap_zero(tbl.bitmap, tbl.bits);
  163. /* We need to reserve slot 0 because 0 is invalid */
  164. set_bit(0, tbl.bitmap);
  165. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  166. tbl.bufq[i].fd = -1;
  167. tbl.bufq[i].buf_handle = -1;
  168. }
  169. mutex_init(&tbl.m_lock);
  170. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  171. cam_mem_mgr_create_debug_fs();
  172. return 0;
  173. put_heaps:
  174. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  175. cam_mem_mgr_put_dma_heaps();
  176. #endif
  177. return rc;
  178. }
  179. static int32_t cam_mem_get_slot(void)
  180. {
  181. int32_t idx;
  182. mutex_lock(&tbl.m_lock);
  183. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  184. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  185. mutex_unlock(&tbl.m_lock);
  186. return -ENOMEM;
  187. }
  188. set_bit(idx, tbl.bitmap);
  189. tbl.bufq[idx].active = true;
  190. CAM_GET_TIMESTAMP((tbl.bufq[idx].timestamp));
  191. mutex_init(&tbl.bufq[idx].q_lock);
  192. mutex_unlock(&tbl.m_lock);
  193. return idx;
  194. }
  195. static void cam_mem_put_slot(int32_t idx)
  196. {
  197. mutex_lock(&tbl.m_lock);
  198. mutex_lock(&tbl.bufq[idx].q_lock);
  199. tbl.bufq[idx].active = false;
  200. tbl.bufq[idx].is_internal = false;
  201. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  202. mutex_unlock(&tbl.bufq[idx].q_lock);
  203. mutex_destroy(&tbl.bufq[idx].q_lock);
  204. clear_bit(idx, tbl.bitmap);
  205. mutex_unlock(&tbl.m_lock);
  206. }
  207. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  208. dma_addr_t *iova_ptr, size_t *len_ptr, uint32_t *flags)
  209. {
  210. int rc = 0, idx;
  211. *len_ptr = 0;
  212. if (!atomic_read(&cam_mem_mgr_state)) {
  213. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  214. return -EINVAL;
  215. }
  216. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  217. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  218. return -ENOENT;
  219. if (!tbl.bufq[idx].active) {
  220. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  221. idx);
  222. return -EAGAIN;
  223. }
  224. mutex_lock(&tbl.bufq[idx].q_lock);
  225. if (buf_handle != tbl.bufq[idx].buf_handle) {
  226. rc = -EINVAL;
  227. goto handle_mismatch;
  228. }
  229. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  230. rc = cam_smmu_get_stage2_iova(mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].dma_buf,
  231. iova_ptr, len_ptr);
  232. else
  233. rc = cam_smmu_get_iova(mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].dma_buf,
  234. iova_ptr, len_ptr);
  235. if (rc) {
  236. CAM_ERR(CAM_MEM,
  237. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d i_ino:%lu",
  238. buf_handle, mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].i_ino);
  239. goto handle_mismatch;
  240. }
  241. if (flags)
  242. *flags = tbl.bufq[idx].flags;
  243. CAM_DBG(CAM_MEM,
  244. "handle:0x%x fd:%d i_ino:%lu iova_ptr:0x%llx len_ptr:%llu",
  245. mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].i_ino, iova_ptr, *len_ptr);
  246. handle_mismatch:
  247. mutex_unlock(&tbl.bufq[idx].q_lock);
  248. return rc;
  249. }
  250. EXPORT_SYMBOL(cam_mem_get_io_buf);
  251. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  252. {
  253. int idx;
  254. if (!atomic_read(&cam_mem_mgr_state)) {
  255. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  256. return -EINVAL;
  257. }
  258. if (!buf_handle || !vaddr_ptr || !len)
  259. return -EINVAL;
  260. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  261. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  262. return -EINVAL;
  263. if (!tbl.bufq[idx].active) {
  264. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  265. idx);
  266. return -EPERM;
  267. }
  268. if (buf_handle != tbl.bufq[idx].buf_handle)
  269. return -EINVAL;
  270. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  271. return -EINVAL;
  272. if (tbl.bufq[idx].kmdvaddr) {
  273. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  274. *len = tbl.bufq[idx].len;
  275. } else {
  276. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  277. buf_handle);
  278. return -EINVAL;
  279. }
  280. return 0;
  281. }
  282. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  283. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  284. {
  285. int rc = 0, idx;
  286. uint32_t cache_dir;
  287. unsigned long dmabuf_flag = 0;
  288. if (!atomic_read(&cam_mem_mgr_state)) {
  289. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  290. return -EINVAL;
  291. }
  292. if (!cmd)
  293. return -EINVAL;
  294. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  295. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  296. return -EINVAL;
  297. mutex_lock(&tbl.bufq[idx].q_lock);
  298. if (!tbl.bufq[idx].active) {
  299. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  300. idx);
  301. rc = -EINVAL;
  302. goto end;
  303. }
  304. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  305. rc = -EINVAL;
  306. goto end;
  307. }
  308. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  309. if (rc) {
  310. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  311. goto end;
  312. }
  313. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  314. CAM_DBG(CAM_MEM, "Calling dmap buf APIs for cache operations");
  315. cache_dir = DMA_BIDIRECTIONAL;
  316. #else
  317. if (dmabuf_flag & ION_FLAG_CACHED) {
  318. switch (cmd->mem_cache_ops) {
  319. case CAM_MEM_CLEAN_CACHE:
  320. cache_dir = DMA_TO_DEVICE;
  321. break;
  322. case CAM_MEM_INV_CACHE:
  323. cache_dir = DMA_FROM_DEVICE;
  324. break;
  325. case CAM_MEM_CLEAN_INV_CACHE:
  326. cache_dir = DMA_BIDIRECTIONAL;
  327. break;
  328. default:
  329. CAM_ERR(CAM_MEM,
  330. "invalid cache ops :%d", cmd->mem_cache_ops);
  331. rc = -EINVAL;
  332. goto end;
  333. }
  334. } else {
  335. CAM_DBG(CAM_MEM, "BUF is not cached");
  336. goto end;
  337. }
  338. #endif
  339. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  340. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  341. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  342. if (rc) {
  343. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  344. goto end;
  345. }
  346. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  347. cache_dir);
  348. if (rc) {
  349. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  350. goto end;
  351. }
  352. end:
  353. mutex_unlock(&tbl.bufq[idx].q_lock);
  354. return rc;
  355. }
  356. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  357. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  358. #define CAM_MAX_VMIDS 4
  359. static void cam_mem_mgr_put_dma_heaps(void)
  360. {
  361. CAM_DBG(CAM_MEM, "Releasing DMA Buf heaps usage");
  362. }
  363. static int cam_mem_mgr_get_dma_heaps(void)
  364. {
  365. int rc = 0;
  366. tbl.system_heap = NULL;
  367. tbl.system_uncached_heap = NULL;
  368. tbl.camera_heap = NULL;
  369. tbl.camera_uncached_heap = NULL;
  370. tbl.secure_display_heap = NULL;
  371. tbl.system_heap = dma_heap_find("qcom,system");
  372. if (IS_ERR_OR_NULL(tbl.system_heap)) {
  373. rc = PTR_ERR(tbl.system_heap);
  374. CAM_ERR(CAM_MEM, "qcom system heap not found, rc=%d", rc);
  375. tbl.system_heap = NULL;
  376. goto put_heaps;
  377. }
  378. tbl.system_uncached_heap = dma_heap_find("qcom,system-uncached");
  379. if (IS_ERR_OR_NULL(tbl.system_uncached_heap)) {
  380. if (tbl.force_cache_allocs) {
  381. /* optional, we anyway do not use uncached */
  382. CAM_DBG(CAM_MEM,
  383. "qcom system-uncached heap not found, err=%d",
  384. PTR_ERR(tbl.system_uncached_heap));
  385. tbl.system_uncached_heap = NULL;
  386. } else {
  387. /* fatal, must need uncached heaps */
  388. rc = PTR_ERR(tbl.system_uncached_heap);
  389. CAM_ERR(CAM_MEM,
  390. "qcom system-uncached heap not found, rc=%d",
  391. rc);
  392. tbl.system_uncached_heap = NULL;
  393. goto put_heaps;
  394. }
  395. }
  396. tbl.secure_display_heap = dma_heap_find("qcom,display");
  397. if (IS_ERR_OR_NULL(tbl.secure_display_heap)) {
  398. rc = PTR_ERR(tbl.secure_display_heap);
  399. CAM_ERR(CAM_MEM, "qcom,display heap not found, rc=%d",
  400. rc);
  401. tbl.secure_display_heap = NULL;
  402. goto put_heaps;
  403. }
  404. tbl.camera_heap = dma_heap_find("qcom,camera");
  405. if (IS_ERR_OR_NULL(tbl.camera_heap)) {
  406. /* optional heap, not a fatal error */
  407. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  408. PTR_ERR(tbl.camera_heap));
  409. tbl.camera_heap = NULL;
  410. }
  411. tbl.camera_uncached_heap = dma_heap_find("qcom,camera-uncached");
  412. if (IS_ERR_OR_NULL(tbl.camera_uncached_heap)) {
  413. /* optional heap, not a fatal error */
  414. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  415. PTR_ERR(tbl.camera_uncached_heap));
  416. tbl.camera_uncached_heap = NULL;
  417. }
  418. CAM_INFO(CAM_MEM,
  419. "Heaps : system=%pK, system_uncached=%pK, camera=%pK, camera-uncached=%pK, secure_display=%pK",
  420. tbl.system_heap, tbl.system_uncached_heap,
  421. tbl.camera_heap, tbl.camera_uncached_heap,
  422. tbl.secure_display_heap);
  423. return 0;
  424. put_heaps:
  425. cam_mem_mgr_put_dma_heaps();
  426. return rc;
  427. }
  428. static int cam_mem_util_get_dma_buf(size_t len,
  429. unsigned int cam_flags,
  430. struct dma_buf **buf,
  431. unsigned long *i_ino)
  432. {
  433. int rc = 0;
  434. struct dma_heap *heap;
  435. struct dma_heap *try_heap = NULL;
  436. struct timespec64 ts1, ts2;
  437. long microsec = 0;
  438. bool use_cached_heap = false;
  439. struct mem_buf_lend_kernel_arg arg;
  440. int vmids[CAM_MAX_VMIDS];
  441. int perms[CAM_MAX_VMIDS];
  442. int num_vmids = 0;
  443. if (!buf) {
  444. CAM_ERR(CAM_MEM, "Invalid params");
  445. return -EINVAL;
  446. }
  447. if (tbl.alloc_profile_enable)
  448. CAM_GET_TIMESTAMP(ts1);
  449. if ((cam_flags & CAM_MEM_FLAG_CACHE) ||
  450. (tbl.force_cache_allocs &&
  451. (!(cam_flags & CAM_MEM_FLAG_PROTECTED_MODE)))) {
  452. CAM_DBG(CAM_MEM,
  453. "Using CACHED heap, cam_flags=0x%x, force_cache_allocs=%d",
  454. cam_flags, tbl.force_cache_allocs);
  455. use_cached_heap = true;
  456. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  457. use_cached_heap = true;
  458. CAM_DBG(CAM_MEM,
  459. "Using CACHED heap for secure, cam_flags=0x%x, force_cache_allocs=%d",
  460. cam_flags, tbl.force_cache_allocs);
  461. } else {
  462. use_cached_heap = false;
  463. CAM_ERR(CAM_MEM,
  464. "Using UNCACHED heap not supported, cam_flags=0x%x, force_cache_allocs=%d",
  465. cam_flags, tbl.force_cache_allocs);
  466. /*
  467. * Need a better handling based on whether dma-buf-heaps support
  468. * uncached heaps or not. For now, assume not supported.
  469. */
  470. return -EINVAL;
  471. }
  472. if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  473. heap = tbl.secure_display_heap;
  474. vmids[num_vmids] = VMID_CP_CAMERA;
  475. perms[num_vmids] = PERM_READ | PERM_WRITE;
  476. num_vmids++;
  477. if (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT) {
  478. CAM_DBG(CAM_MEM, "Secure mode CDSP flags");
  479. vmids[num_vmids] = VMID_CP_CDSP;
  480. perms[num_vmids] = PERM_READ | PERM_WRITE;
  481. num_vmids++;
  482. }
  483. } else if (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL) {
  484. heap = tbl.secure_display_heap;
  485. vmids[num_vmids] = VMID_CP_NON_PIXEL;
  486. perms[num_vmids] = PERM_READ | PERM_WRITE;
  487. num_vmids++;
  488. } else if (use_cached_heap) {
  489. try_heap = tbl.camera_heap;
  490. heap = tbl.system_heap;
  491. } else {
  492. try_heap = tbl.camera_uncached_heap;
  493. heap = tbl.system_uncached_heap;
  494. }
  495. CAM_DBG(CAM_MEM, "Using heaps : try=%pK, heap=%pK", try_heap, heap);
  496. *buf = NULL;
  497. if (!try_heap && !heap) {
  498. CAM_ERR(CAM_MEM,
  499. "No heap available for allocation, cant allocate");
  500. return -EINVAL;
  501. }
  502. if (try_heap) {
  503. *buf = dma_heap_buffer_alloc(try_heap, len, O_RDWR, 0);
  504. if (IS_ERR(*buf)) {
  505. CAM_WARN(CAM_MEM,
  506. "Failed in allocating from try heap, heap=%pK, len=%zu, err=%d",
  507. try_heap, len, PTR_ERR(*buf));
  508. *buf = NULL;
  509. }
  510. }
  511. if (*buf == NULL) {
  512. *buf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0);
  513. if (IS_ERR(*buf)) {
  514. rc = PTR_ERR(*buf);
  515. CAM_ERR(CAM_MEM,
  516. "Failed in allocating from heap, heap=%pK, len=%zu, err=%d",
  517. heap, len, rc);
  518. *buf = NULL;
  519. return rc;
  520. }
  521. }
  522. *i_ino = file_inode((*buf)->file)->i_ino;
  523. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) ||
  524. (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL)) {
  525. if (num_vmids >= CAM_MAX_VMIDS) {
  526. CAM_ERR(CAM_MEM, "Insufficient array size for vmids %d", num_vmids);
  527. rc = -EINVAL;
  528. goto end;
  529. }
  530. arg.nr_acl_entries = num_vmids;
  531. arg.vmids = vmids;
  532. arg.perms = perms;
  533. rc = mem_buf_lend(*buf, &arg);
  534. if (rc) {
  535. CAM_ERR(CAM_MEM,
  536. "Failed in buf lend rc=%d, buf=%pK, vmids [0]=0x%x, [1]=0x%x, [2]=0x%x",
  537. rc, *buf, vmids[0], vmids[1], vmids[2]);
  538. goto end;
  539. }
  540. }
  541. CAM_DBG(CAM_MEM, "Allocate success, len=%zu, *buf=%pK, i_ino=%lu", len, *buf, *i_ino);
  542. if (tbl.alloc_profile_enable) {
  543. CAM_GET_TIMESTAMP(ts2);
  544. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  545. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  546. len, microsec);
  547. }
  548. return rc;
  549. end:
  550. dma_buf_put(*buf);
  551. return rc;
  552. }
  553. #else
  554. static int cam_mem_util_get_dma_buf(size_t len,
  555. unsigned int cam_flags,
  556. struct dma_buf **buf,
  557. unsigned long *i_ino)
  558. {
  559. int rc = 0;
  560. unsigned int heap_id;
  561. int32_t ion_flag = 0;
  562. struct timespec64 ts1, ts2;
  563. long microsec = 0;
  564. if (!buf) {
  565. CAM_ERR(CAM_MEM, "Invalid params");
  566. return -EINVAL;
  567. }
  568. if (tbl.alloc_profile_enable)
  569. CAM_GET_TIMESTAMP(ts1);
  570. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  571. (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  572. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  573. ion_flag |=
  574. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  575. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  576. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  577. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  578. } else {
  579. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  580. ION_HEAP(ION_CAMERA_HEAP_ID);
  581. }
  582. if (cam_flags & CAM_MEM_FLAG_CACHE)
  583. ion_flag |= ION_FLAG_CACHED;
  584. else
  585. ion_flag &= ~ION_FLAG_CACHED;
  586. if (tbl.force_cache_allocs && (!(ion_flag & ION_FLAG_SECURE)))
  587. ion_flag |= ION_FLAG_CACHED;
  588. *buf = ion_alloc(len, heap_id, ion_flag);
  589. if (IS_ERR_OR_NULL(*buf))
  590. return -ENOMEM;
  591. *i_ino = file_inode((*buf)->file)->i_ino;
  592. if (tbl.alloc_profile_enable) {
  593. CAM_GET_TIMESTAMP(ts2);
  594. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  595. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  596. len, microsec);
  597. }
  598. return rc;
  599. }
  600. #endif
  601. static int cam_mem_util_buffer_alloc(size_t len, uint32_t flags,
  602. struct dma_buf **dmabuf,
  603. int *fd,
  604. unsigned long *i_ino)
  605. {
  606. int rc;
  607. struct dma_buf *temp_dmabuf = NULL;
  608. rc = cam_mem_util_get_dma_buf(len, flags, dmabuf, i_ino);
  609. if (rc) {
  610. CAM_ERR(CAM_MEM,
  611. "Error allocating dma buf : len=%llu, flags=0x%x",
  612. len, flags);
  613. return rc;
  614. }
  615. *fd = dma_buf_fd(*dmabuf, O_CLOEXEC);
  616. if (*fd < 0) {
  617. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  618. rc = -EINVAL;
  619. goto put_buf;
  620. }
  621. CAM_DBG(CAM_MEM, "Alloc success : len=%zu, *dmabuf=%pK, fd=%d, i_ino=%lu",
  622. len, *dmabuf, *fd, *i_ino);
  623. /*
  624. * increment the ref count so that ref count becomes 2 here
  625. * when we close fd, refcount becomes 1 and when we do
  626. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  627. */
  628. temp_dmabuf = dma_buf_get(*fd);
  629. if (IS_ERR_OR_NULL(temp_dmabuf)) {
  630. rc = PTR_ERR(temp_dmabuf);
  631. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d, i_ino=%lu, rc=%d", *fd, *i_ino, rc);
  632. goto put_buf;
  633. }
  634. return rc;
  635. put_buf:
  636. dma_buf_put(*dmabuf);
  637. return rc;
  638. }
  639. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  640. {
  641. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  642. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  643. CAM_MEM_MMU_MAX_HANDLE);
  644. return -EINVAL;
  645. }
  646. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  647. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  648. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  649. return -EINVAL;
  650. }
  651. if ((cmd->flags & CAM_MEM_FLAG_EVA_NOPIXEL) &&
  652. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE ||
  653. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS)){
  654. CAM_ERR(CAM_MEM,
  655. "Kernel mapping and secure mode not allowed in no pixel mode");
  656. return -EINVAL;
  657. }
  658. return 0;
  659. }
  660. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  661. {
  662. if (!cmd->flags) {
  663. CAM_ERR(CAM_MEM, "Invalid flags");
  664. return -EINVAL;
  665. }
  666. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  667. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  668. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  669. return -EINVAL;
  670. }
  671. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  672. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  673. CAM_ERR(CAM_MEM,
  674. "Kernel mapping in secure mode not allowed, flags=0x%x",
  675. cmd->flags);
  676. return -EINVAL;
  677. }
  678. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  679. CAM_ERR(CAM_MEM,
  680. "Shared memory buffers are not allowed to be mapped");
  681. return -EINVAL;
  682. }
  683. return 0;
  684. }
  685. static int cam_mem_util_map_hw_va(uint32_t flags,
  686. int32_t *mmu_hdls,
  687. int32_t num_hdls,
  688. int fd,
  689. struct dma_buf *dmabuf,
  690. dma_addr_t *hw_vaddr,
  691. size_t *len,
  692. enum cam_smmu_region_id region,
  693. bool is_internal)
  694. {
  695. int i;
  696. int rc = -1;
  697. int dir = cam_mem_util_get_dma_dir(flags);
  698. bool dis_delayed_unmap = false;
  699. if (dir < 0) {
  700. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  701. return dir;
  702. }
  703. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  704. dis_delayed_unmap = true;
  705. CAM_DBG(CAM_MEM,
  706. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  707. fd, flags, dir, num_hdls);
  708. for (i = 0; i < num_hdls; i++) {
  709. /* If 36-bit enabled, check for ICP cmd buffers and map them within the shared region */
  710. if (cam_smmu_is_expanded_memory() &&
  711. cam_smmu_supports_shared_region(mmu_hdls[i]) &&
  712. (flags & CAM_MEM_FLAG_CMD_BUF_TYPE))
  713. region = CAM_SMMU_REGION_SHARED;
  714. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  715. rc = cam_smmu_map_stage2_iova(mmu_hdls[i], fd, dmabuf, dir, hw_vaddr, len);
  716. else
  717. rc = cam_smmu_map_user_iova(mmu_hdls[i], fd, dmabuf, dis_delayed_unmap, dir,
  718. hw_vaddr, len, region, is_internal);
  719. if (rc) {
  720. CAM_ERR(CAM_MEM,
  721. "Failed %s map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  722. (flags & CAM_MEM_FLAG_PROTECTED_MODE) ? "" : "secured",
  723. i, fd, dir, mmu_hdls[i], rc);
  724. goto multi_map_fail;
  725. }
  726. }
  727. return rc;
  728. multi_map_fail:
  729. for (--i; i>= 0; i--) {
  730. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  731. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd, dmabuf);
  732. else
  733. cam_smmu_unmap_user_iova(mmu_hdls[i], fd, dmabuf, CAM_SMMU_REGION_IO);
  734. }
  735. return rc;
  736. }
  737. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  738. {
  739. int rc;
  740. int32_t idx;
  741. struct dma_buf *dmabuf = NULL;
  742. int fd = -1;
  743. dma_addr_t hw_vaddr = 0;
  744. size_t len;
  745. uintptr_t kvaddr = 0;
  746. size_t klen;
  747. unsigned long i_ino = 0;
  748. if (!atomic_read(&cam_mem_mgr_state)) {
  749. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  750. return -EINVAL;
  751. }
  752. if (!cmd) {
  753. CAM_ERR(CAM_MEM, " Invalid argument");
  754. return -EINVAL;
  755. }
  756. len = cmd->len;
  757. if (tbl.need_shared_buffer_padding &&
  758. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)) {
  759. len += CAM_MEM_SHARED_BUFFER_PAD_4K;
  760. CAM_DBG(CAM_MEM, "Pad 4k size, actual %llu, allocating %zu",
  761. cmd->len, len);
  762. }
  763. rc = cam_mem_util_check_alloc_flags(cmd);
  764. if (rc) {
  765. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  766. cmd->flags, rc);
  767. return rc;
  768. }
  769. rc = cam_mem_util_buffer_alloc(len, cmd->flags, &dmabuf, &fd, &i_ino);
  770. if (rc) {
  771. CAM_ERR(CAM_MEM,
  772. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  773. len, cmd->align, cmd->flags, cmd->num_hdl);
  774. cam_mem_mgr_print_tbl();
  775. return rc;
  776. }
  777. if (!dmabuf) {
  778. CAM_ERR(CAM_MEM,
  779. "Ion Alloc return NULL dmabuf! fd=%d, i_ino=%lu, len=%d", fd, i_ino, len);
  780. cam_mem_mgr_print_tbl();
  781. return rc;
  782. }
  783. idx = cam_mem_get_slot();
  784. if (idx < 0) {
  785. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  786. rc = -ENOMEM;
  787. goto slot_fail;
  788. }
  789. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  790. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  791. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  792. enum cam_smmu_region_id region;
  793. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  794. region = CAM_SMMU_REGION_IO;
  795. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  796. region = CAM_SMMU_REGION_SHARED;
  797. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  798. region = CAM_SMMU_REGION_IO;
  799. rc = cam_mem_util_map_hw_va(cmd->flags,
  800. cmd->mmu_hdls,
  801. cmd->num_hdl,
  802. fd,
  803. dmabuf,
  804. &hw_vaddr,
  805. &len,
  806. region,
  807. true);
  808. if (rc) {
  809. CAM_ERR(CAM_MEM,
  810. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  811. len, cmd->flags,
  812. fd, region, cmd->num_hdl, rc);
  813. if (rc == -EALREADY) {
  814. if ((size_t)dmabuf->size != len)
  815. rc = -EBADR;
  816. cam_mem_mgr_print_tbl();
  817. }
  818. goto map_hw_fail;
  819. }
  820. }
  821. mutex_lock(&tbl.bufq[idx].q_lock);
  822. tbl.bufq[idx].fd = fd;
  823. tbl.bufq[idx].i_ino = i_ino;
  824. tbl.bufq[idx].dma_buf = NULL;
  825. tbl.bufq[idx].flags = cmd->flags;
  826. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  827. tbl.bufq[idx].is_internal = true;
  828. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  829. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  830. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  831. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  832. if (rc) {
  833. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  834. dmabuf, rc);
  835. goto map_kernel_fail;
  836. }
  837. }
  838. if (cmd->flags & CAM_MEM_FLAG_KMD_DEBUG_FLAG)
  839. tbl.dbg_buf_idx = idx;
  840. tbl.bufq[idx].kmdvaddr = kvaddr;
  841. tbl.bufq[idx].vaddr = hw_vaddr;
  842. tbl.bufq[idx].dma_buf = dmabuf;
  843. tbl.bufq[idx].len = len;
  844. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  845. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  846. sizeof(int32_t) * cmd->num_hdl);
  847. tbl.bufq[idx].is_imported = false;
  848. mutex_unlock(&tbl.bufq[idx].q_lock);
  849. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  850. cmd->out.fd = tbl.bufq[idx].fd;
  851. cmd->out.vaddr = 0;
  852. CAM_DBG(CAM_MEM,
  853. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu, i_ino=%lu",
  854. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  855. tbl.bufq[idx].len, tbl.bufq[idx].i_ino);
  856. return rc;
  857. map_kernel_fail:
  858. mutex_unlock(&tbl.bufq[idx].q_lock);
  859. map_hw_fail:
  860. cam_mem_put_slot(idx);
  861. slot_fail:
  862. dma_buf_put(dmabuf);
  863. return rc;
  864. }
  865. static bool cam_mem_util_is_map_internal(int32_t fd, unsigned i_ino)
  866. {
  867. uint32_t i;
  868. bool is_internal = false;
  869. mutex_lock(&tbl.m_lock);
  870. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  871. if ((tbl.bufq[i].fd == fd) && (tbl.bufq[i].i_ino == i_ino)) {
  872. is_internal = tbl.bufq[i].is_internal;
  873. break;
  874. }
  875. }
  876. mutex_unlock(&tbl.m_lock);
  877. return is_internal;
  878. }
  879. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  880. {
  881. int32_t idx;
  882. int rc;
  883. struct dma_buf *dmabuf;
  884. dma_addr_t hw_vaddr = 0;
  885. size_t len = 0;
  886. bool is_internal = false;
  887. unsigned long i_ino;
  888. if (!atomic_read(&cam_mem_mgr_state)) {
  889. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  890. return -EINVAL;
  891. }
  892. if (!cmd || (cmd->fd < 0)) {
  893. CAM_ERR(CAM_MEM, "Invalid argument");
  894. return -EINVAL;
  895. }
  896. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  897. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  898. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  899. return -EINVAL;
  900. }
  901. rc = cam_mem_util_check_map_flags(cmd);
  902. if (rc) {
  903. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  904. return rc;
  905. }
  906. dmabuf = dma_buf_get(cmd->fd);
  907. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  908. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  909. return -EINVAL;
  910. }
  911. i_ino = file_inode(dmabuf->file)->i_ino;
  912. is_internal = cam_mem_util_is_map_internal(cmd->fd, i_ino);
  913. idx = cam_mem_get_slot();
  914. if (idx < 0) {
  915. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d, fd=%d",
  916. idx, cmd->fd);
  917. rc = -ENOMEM;
  918. goto slot_fail;
  919. }
  920. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  921. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  922. rc = cam_mem_util_map_hw_va(cmd->flags,
  923. cmd->mmu_hdls,
  924. cmd->num_hdl,
  925. cmd->fd,
  926. dmabuf,
  927. &hw_vaddr,
  928. &len,
  929. CAM_SMMU_REGION_IO,
  930. is_internal);
  931. if (rc) {
  932. CAM_ERR(CAM_MEM,
  933. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  934. cmd->flags, cmd->fd, len,
  935. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  936. if (rc == -EALREADY) {
  937. if ((size_t)dmabuf->size != len) {
  938. rc = -EBADR;
  939. cam_mem_mgr_print_tbl();
  940. }
  941. }
  942. goto map_fail;
  943. }
  944. }
  945. mutex_lock(&tbl.bufq[idx].q_lock);
  946. tbl.bufq[idx].fd = cmd->fd;
  947. tbl.bufq[idx].i_ino = i_ino;
  948. tbl.bufq[idx].dma_buf = NULL;
  949. tbl.bufq[idx].flags = cmd->flags;
  950. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  951. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  952. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  953. tbl.bufq[idx].kmdvaddr = 0;
  954. if (cmd->num_hdl > 0)
  955. tbl.bufq[idx].vaddr = hw_vaddr;
  956. else
  957. tbl.bufq[idx].vaddr = 0;
  958. tbl.bufq[idx].dma_buf = dmabuf;
  959. tbl.bufq[idx].len = len;
  960. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  961. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  962. sizeof(int32_t) * cmd->num_hdl);
  963. tbl.bufq[idx].is_imported = true;
  964. tbl.bufq[idx].is_internal = is_internal;
  965. mutex_unlock(&tbl.bufq[idx].q_lock);
  966. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  967. cmd->out.vaddr = 0;
  968. cmd->out.size = (uint32_t)len;
  969. CAM_DBG(CAM_MEM,
  970. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu, i_ino=%lu",
  971. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  972. tbl.bufq[idx].len, tbl.bufq[idx].i_ino);
  973. return rc;
  974. map_fail:
  975. cam_mem_put_slot(idx);
  976. slot_fail:
  977. dma_buf_put(dmabuf);
  978. return rc;
  979. }
  980. static int cam_mem_util_unmap_hw_va(int32_t idx,
  981. enum cam_smmu_region_id region,
  982. enum cam_smmu_mapping_client client)
  983. {
  984. int i;
  985. uint32_t flags;
  986. int32_t *mmu_hdls;
  987. int num_hdls;
  988. int fd;
  989. struct dma_buf *dma_buf;
  990. unsigned long i_ino;
  991. int rc = 0;
  992. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  993. CAM_ERR(CAM_MEM, "Incorrect index");
  994. return -EINVAL;
  995. }
  996. flags = tbl.bufq[idx].flags;
  997. mmu_hdls = tbl.bufq[idx].hdls;
  998. num_hdls = tbl.bufq[idx].num_hdl;
  999. fd = tbl.bufq[idx].fd;
  1000. dma_buf = tbl.bufq[idx].dma_buf;
  1001. i_ino = tbl.bufq[idx].i_ino;
  1002. CAM_DBG(CAM_MEM,
  1003. "unmap_hw_va : idx=%d, fd=%x, i_ino=%lu flags=0x%x, num_hdls=%d, client=%d",
  1004. idx, fd, i_ino, flags, num_hdls, client);
  1005. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  1006. for (i = 0; i < num_hdls; i++) {
  1007. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd, dma_buf);
  1008. if (rc < 0) {
  1009. CAM_ERR(CAM_MEM,
  1010. "Failed in secure unmap, i=%d, fd=%d, i_ino=%lu, mmu_hdl=%d, rc=%d",
  1011. i, fd, i_ino, mmu_hdls[i], rc);
  1012. goto unmap_end;
  1013. }
  1014. }
  1015. } else {
  1016. for (i = 0; i < num_hdls; i++) {
  1017. if (client == CAM_SMMU_MAPPING_USER) {
  1018. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  1019. fd, dma_buf, region);
  1020. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  1021. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  1022. tbl.bufq[idx].dma_buf, region);
  1023. } else {
  1024. CAM_ERR(CAM_MEM,
  1025. "invalid caller for unmapping : %d",
  1026. client);
  1027. rc = -EINVAL;
  1028. }
  1029. if (rc < 0) {
  1030. CAM_ERR(CAM_MEM,
  1031. "Failed in unmap, i=%d, fd=%d, i_ino=%lu, mmu_hdl=%d, region=%d, rc=%d",
  1032. i, fd, i_ino, mmu_hdls[i], region, rc);
  1033. goto unmap_end;
  1034. }
  1035. }
  1036. }
  1037. return rc;
  1038. unmap_end:
  1039. CAM_ERR(CAM_MEM, "unmapping failed");
  1040. return rc;
  1041. }
  1042. static void cam_mem_mgr_unmap_active_buf(int idx)
  1043. {
  1044. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1045. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  1046. region = CAM_SMMU_REGION_SHARED;
  1047. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1048. region = CAM_SMMU_REGION_IO;
  1049. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  1050. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS)
  1051. cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1052. tbl.bufq[idx].kmdvaddr);
  1053. }
  1054. static int cam_mem_mgr_cleanup_table(void)
  1055. {
  1056. int i;
  1057. mutex_lock(&tbl.m_lock);
  1058. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  1059. if (!tbl.bufq[i].active) {
  1060. CAM_DBG(CAM_MEM,
  1061. "Buffer inactive at idx=%d, continuing", i);
  1062. continue;
  1063. } else {
  1064. CAM_DBG(CAM_MEM,
  1065. "Active buffer at idx=%d, possible leak needs unmapping",
  1066. i);
  1067. cam_mem_mgr_unmap_active_buf(i);
  1068. }
  1069. mutex_lock(&tbl.bufq[i].q_lock);
  1070. if (tbl.bufq[i].dma_buf) {
  1071. dma_buf_put(tbl.bufq[i].dma_buf);
  1072. tbl.bufq[i].dma_buf = NULL;
  1073. }
  1074. tbl.bufq[i].fd = -1;
  1075. tbl.bufq[i].i_ino = 0;
  1076. tbl.bufq[i].flags = 0;
  1077. tbl.bufq[i].buf_handle = -1;
  1078. tbl.bufq[i].vaddr = 0;
  1079. tbl.bufq[i].len = 0;
  1080. memset(tbl.bufq[i].hdls, 0,
  1081. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  1082. tbl.bufq[i].num_hdl = 0;
  1083. tbl.bufq[i].dma_buf = NULL;
  1084. tbl.bufq[i].active = false;
  1085. tbl.bufq[i].is_internal = false;
  1086. mutex_unlock(&tbl.bufq[i].q_lock);
  1087. mutex_destroy(&tbl.bufq[i].q_lock);
  1088. }
  1089. bitmap_zero(tbl.bitmap, tbl.bits);
  1090. /* We need to reserve slot 0 because 0 is invalid */
  1091. set_bit(0, tbl.bitmap);
  1092. mutex_unlock(&tbl.m_lock);
  1093. return 0;
  1094. }
  1095. void cam_mem_mgr_deinit(void)
  1096. {
  1097. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  1098. cam_mem_mgr_cleanup_table();
  1099. debugfs_remove_recursive(tbl.dentry);
  1100. mutex_lock(&tbl.m_lock);
  1101. bitmap_zero(tbl.bitmap, tbl.bits);
  1102. kfree(tbl.bitmap);
  1103. tbl.bitmap = NULL;
  1104. tbl.dbg_buf_idx = -1;
  1105. mutex_unlock(&tbl.m_lock);
  1106. mutex_destroy(&tbl.m_lock);
  1107. }
  1108. static int cam_mem_util_unmap(int32_t idx,
  1109. enum cam_smmu_mapping_client client)
  1110. {
  1111. int rc = 0;
  1112. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1113. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1114. CAM_ERR(CAM_MEM, "Incorrect index");
  1115. return -EINVAL;
  1116. }
  1117. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  1118. mutex_lock(&tbl.m_lock);
  1119. if ((!tbl.bufq[idx].active) &&
  1120. (tbl.bufq[idx].vaddr) == 0) {
  1121. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  1122. idx);
  1123. mutex_unlock(&tbl.m_lock);
  1124. return 0;
  1125. }
  1126. /* Deactivate the buffer queue to prevent multiple unmap */
  1127. mutex_lock(&tbl.bufq[idx].q_lock);
  1128. tbl.bufq[idx].active = false;
  1129. tbl.bufq[idx].vaddr = 0;
  1130. mutex_unlock(&tbl.bufq[idx].q_lock);
  1131. mutex_unlock(&tbl.m_lock);
  1132. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1133. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  1134. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1135. tbl.bufq[idx].kmdvaddr);
  1136. if (rc)
  1137. CAM_ERR(CAM_MEM,
  1138. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  1139. tbl.bufq[idx].dma_buf,
  1140. (void *) tbl.bufq[idx].kmdvaddr);
  1141. }
  1142. }
  1143. /* SHARED flag gets precedence, all other flags after it */
  1144. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1145. region = CAM_SMMU_REGION_SHARED;
  1146. } else {
  1147. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1148. region = CAM_SMMU_REGION_IO;
  1149. }
  1150. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1151. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  1152. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1153. if (cam_mem_util_unmap_hw_va(idx, region, client))
  1154. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  1155. tbl.bufq[idx].dma_buf);
  1156. /*
  1157. * Workaround as smmu driver doing put_buf without get_buf for kernel mappings
  1158. * Setting NULL here so that we dont call dma_buf_pt again below
  1159. */
  1160. if (client == CAM_SMMU_MAPPING_KERNEL)
  1161. tbl.bufq[idx].dma_buf = NULL;
  1162. }
  1163. mutex_lock(&tbl.m_lock);
  1164. mutex_lock(&tbl.bufq[idx].q_lock);
  1165. tbl.bufq[idx].flags = 0;
  1166. tbl.bufq[idx].buf_handle = -1;
  1167. memset(tbl.bufq[idx].hdls, 0,
  1168. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  1169. CAM_DBG(CAM_MEM,
  1170. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK, i_ino %lu",
  1171. idx, tbl.bufq[idx].fd, tbl.bufq[idx].is_imported, tbl.bufq[idx].dma_buf,
  1172. tbl.bufq[idx].i_ino);
  1173. if (tbl.bufq[idx].dma_buf)
  1174. dma_buf_put(tbl.bufq[idx].dma_buf);
  1175. tbl.bufq[idx].fd = -1;
  1176. tbl.bufq[idx].i_ino = 0;
  1177. tbl.bufq[idx].dma_buf = NULL;
  1178. tbl.bufq[idx].is_imported = false;
  1179. tbl.bufq[idx].is_internal = false;
  1180. tbl.bufq[idx].len = 0;
  1181. tbl.bufq[idx].num_hdl = 0;
  1182. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  1183. mutex_unlock(&tbl.bufq[idx].q_lock);
  1184. mutex_destroy(&tbl.bufq[idx].q_lock);
  1185. clear_bit(idx, tbl.bitmap);
  1186. mutex_unlock(&tbl.m_lock);
  1187. return rc;
  1188. }
  1189. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  1190. {
  1191. int idx;
  1192. int rc;
  1193. if (!atomic_read(&cam_mem_mgr_state)) {
  1194. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1195. return -EINVAL;
  1196. }
  1197. if (!cmd) {
  1198. CAM_ERR(CAM_MEM, "Invalid argument");
  1199. return -EINVAL;
  1200. }
  1201. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  1202. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1203. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  1204. idx);
  1205. return -EINVAL;
  1206. }
  1207. if (!tbl.bufq[idx].active) {
  1208. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1209. return -EINVAL;
  1210. }
  1211. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  1212. CAM_ERR(CAM_MEM,
  1213. "Released buf handle %d not matching within table %d, idx=%d",
  1214. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  1215. return -EINVAL;
  1216. }
  1217. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  1218. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  1219. return rc;
  1220. }
  1221. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  1222. struct cam_mem_mgr_memory_desc *out)
  1223. {
  1224. struct dma_buf *buf = NULL;
  1225. int ion_fd = -1;
  1226. int rc = 0;
  1227. uintptr_t kvaddr;
  1228. dma_addr_t iova = 0;
  1229. size_t request_len = 0;
  1230. uint32_t mem_handle;
  1231. int32_t idx;
  1232. int32_t smmu_hdl = 0;
  1233. int32_t num_hdl = 0;
  1234. unsigned long i_ino = 0;
  1235. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1236. if (!atomic_read(&cam_mem_mgr_state)) {
  1237. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1238. return -EINVAL;
  1239. }
  1240. if (!inp || !out) {
  1241. CAM_ERR(CAM_MEM, "Invalid params");
  1242. return -EINVAL;
  1243. }
  1244. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1245. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1246. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1247. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1248. return -EINVAL;
  1249. }
  1250. rc = cam_mem_util_get_dma_buf(inp->size, inp->flags, &buf, &i_ino);
  1251. if (rc) {
  1252. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1253. goto ion_fail;
  1254. } else if (!buf) {
  1255. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1256. goto ion_fail;
  1257. } else {
  1258. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1259. }
  1260. /*
  1261. * we are mapping kva always here,
  1262. * update flags so that we do unmap properly
  1263. */
  1264. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1265. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1266. if (rc) {
  1267. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1268. goto map_fail;
  1269. }
  1270. if (!inp->smmu_hdl) {
  1271. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1272. rc = -EINVAL;
  1273. goto smmu_fail;
  1274. }
  1275. /* SHARED flag gets precedence, all other flags after it */
  1276. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1277. region = CAM_SMMU_REGION_SHARED;
  1278. } else {
  1279. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1280. region = CAM_SMMU_REGION_IO;
  1281. }
  1282. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1283. buf,
  1284. CAM_SMMU_MAP_RW,
  1285. &iova,
  1286. &request_len,
  1287. region);
  1288. if (rc < 0) {
  1289. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1290. goto smmu_fail;
  1291. }
  1292. smmu_hdl = inp->smmu_hdl;
  1293. num_hdl = 1;
  1294. idx = cam_mem_get_slot();
  1295. if (idx < 0) {
  1296. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1297. rc = -ENOMEM;
  1298. goto slot_fail;
  1299. }
  1300. mutex_lock(&tbl.bufq[idx].q_lock);
  1301. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1302. tbl.bufq[idx].dma_buf = buf;
  1303. tbl.bufq[idx].fd = -1;
  1304. tbl.bufq[idx].i_ino = i_ino;
  1305. tbl.bufq[idx].flags = inp->flags;
  1306. tbl.bufq[idx].buf_handle = mem_handle;
  1307. tbl.bufq[idx].kmdvaddr = kvaddr;
  1308. tbl.bufq[idx].vaddr = iova;
  1309. tbl.bufq[idx].len = inp->size;
  1310. tbl.bufq[idx].num_hdl = num_hdl;
  1311. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1312. sizeof(int32_t));
  1313. tbl.bufq[idx].is_imported = false;
  1314. mutex_unlock(&tbl.bufq[idx].q_lock);
  1315. out->kva = kvaddr;
  1316. out->iova = (uint32_t)iova;
  1317. out->smmu_hdl = smmu_hdl;
  1318. out->mem_handle = mem_handle;
  1319. out->len = inp->size;
  1320. out->region = region;
  1321. CAM_DBG(CAM_MEM, "idx=%d, dmabuf=%pK, i_ino=%lu, flags=0x%x, mem_handle=0x%x",
  1322. idx, buf, i_ino, inp->flags, mem_handle);
  1323. return rc;
  1324. slot_fail:
  1325. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1326. buf, region);
  1327. smmu_fail:
  1328. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1329. map_fail:
  1330. dma_buf_put(buf);
  1331. ion_fail:
  1332. return rc;
  1333. }
  1334. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1335. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1336. {
  1337. int32_t idx;
  1338. int rc;
  1339. if (!atomic_read(&cam_mem_mgr_state)) {
  1340. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1341. return -EINVAL;
  1342. }
  1343. if (!inp) {
  1344. CAM_ERR(CAM_MEM, "Invalid argument");
  1345. return -EINVAL;
  1346. }
  1347. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1348. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1349. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1350. return -EINVAL;
  1351. }
  1352. if (!tbl.bufq[idx].active) {
  1353. if (tbl.bufq[idx].vaddr == 0) {
  1354. CAM_ERR(CAM_MEM, "buffer is released already");
  1355. return 0;
  1356. }
  1357. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1358. return -EINVAL;
  1359. }
  1360. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1361. CAM_ERR(CAM_MEM,
  1362. "Released buf handle not matching within table");
  1363. return -EINVAL;
  1364. }
  1365. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1366. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1367. return rc;
  1368. }
  1369. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1370. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1371. enum cam_smmu_region_id region,
  1372. struct cam_mem_mgr_memory_desc *out)
  1373. {
  1374. struct dma_buf *buf = NULL;
  1375. int rc = 0;
  1376. int ion_fd = -1;
  1377. dma_addr_t iova = 0;
  1378. size_t request_len = 0;
  1379. uint32_t mem_handle;
  1380. int32_t idx;
  1381. int32_t smmu_hdl = 0;
  1382. int32_t num_hdl = 0;
  1383. uintptr_t kvaddr = 0;
  1384. unsigned long i_ino = 0;
  1385. if (!atomic_read(&cam_mem_mgr_state)) {
  1386. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1387. return -EINVAL;
  1388. }
  1389. if (!inp || !out) {
  1390. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1391. return -EINVAL;
  1392. }
  1393. if (!inp->smmu_hdl) {
  1394. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1395. return -EINVAL;
  1396. }
  1397. if ((region != CAM_SMMU_REGION_SECHEAP) &&
  1398. (region != CAM_SMMU_REGION_FWUNCACHED)) {
  1399. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1400. return -EINVAL;
  1401. }
  1402. rc = cam_mem_util_get_dma_buf(inp->size, 0, &buf, &i_ino);
  1403. if (rc) {
  1404. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1405. goto ion_fail;
  1406. } else if (!buf) {
  1407. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1408. goto ion_fail;
  1409. } else {
  1410. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1411. }
  1412. if (inp->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1413. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1414. if (rc) {
  1415. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1416. goto kmap_fail;
  1417. }
  1418. }
  1419. rc = cam_smmu_reserve_buf_region(region,
  1420. inp->smmu_hdl, buf, &iova, &request_len);
  1421. if (rc) {
  1422. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1423. goto smmu_fail;
  1424. }
  1425. smmu_hdl = inp->smmu_hdl;
  1426. num_hdl = 1;
  1427. idx = cam_mem_get_slot();
  1428. if (idx < 0) {
  1429. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1430. rc = -ENOMEM;
  1431. goto slot_fail;
  1432. }
  1433. mutex_lock(&tbl.bufq[idx].q_lock);
  1434. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1435. tbl.bufq[idx].fd = -1;
  1436. tbl.bufq[idx].i_ino = i_ino;
  1437. tbl.bufq[idx].dma_buf = buf;
  1438. tbl.bufq[idx].flags = inp->flags;
  1439. tbl.bufq[idx].buf_handle = mem_handle;
  1440. tbl.bufq[idx].kmdvaddr = kvaddr;
  1441. tbl.bufq[idx].vaddr = iova;
  1442. tbl.bufq[idx].len = request_len;
  1443. tbl.bufq[idx].num_hdl = num_hdl;
  1444. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1445. sizeof(int32_t));
  1446. tbl.bufq[idx].is_imported = false;
  1447. mutex_unlock(&tbl.bufq[idx].q_lock);
  1448. out->kva = kvaddr;
  1449. out->iova = (uint32_t)iova;
  1450. out->smmu_hdl = smmu_hdl;
  1451. out->mem_handle = mem_handle;
  1452. out->len = request_len;
  1453. out->region = region;
  1454. return rc;
  1455. slot_fail:
  1456. cam_smmu_release_buf_region(region, smmu_hdl);
  1457. smmu_fail:
  1458. if (region == CAM_SMMU_REGION_FWUNCACHED)
  1459. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1460. kmap_fail:
  1461. dma_buf_put(buf);
  1462. ion_fail:
  1463. return rc;
  1464. }
  1465. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1466. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1467. {
  1468. int32_t idx;
  1469. int rc;
  1470. int32_t smmu_hdl;
  1471. if (!atomic_read(&cam_mem_mgr_state)) {
  1472. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1473. return -EINVAL;
  1474. }
  1475. if (!inp) {
  1476. CAM_ERR(CAM_MEM, "Invalid argument");
  1477. return -EINVAL;
  1478. }
  1479. if ((inp->region != CAM_SMMU_REGION_SECHEAP) &&
  1480. (inp->region != CAM_SMMU_REGION_FWUNCACHED)) {
  1481. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1482. return -EINVAL;
  1483. }
  1484. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1485. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1486. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1487. return -EINVAL;
  1488. }
  1489. if (!tbl.bufq[idx].active) {
  1490. if (tbl.bufq[idx].vaddr == 0) {
  1491. CAM_ERR(CAM_MEM, "buffer is released already");
  1492. return 0;
  1493. }
  1494. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1495. return -EINVAL;
  1496. }
  1497. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1498. CAM_ERR(CAM_MEM,
  1499. "Released buf handle not matching within table");
  1500. return -EINVAL;
  1501. }
  1502. if (tbl.bufq[idx].num_hdl != 1) {
  1503. CAM_ERR(CAM_MEM,
  1504. "Sec heap region should have only one smmu hdl");
  1505. return -ENODEV;
  1506. }
  1507. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1508. sizeof(int32_t));
  1509. if (inp->smmu_hdl != smmu_hdl) {
  1510. CAM_ERR(CAM_MEM,
  1511. "Passed SMMU handle doesn't match with internal hdl");
  1512. return -ENODEV;
  1513. }
  1514. rc = cam_smmu_release_buf_region(inp->region, inp->smmu_hdl);
  1515. if (rc) {
  1516. CAM_ERR(CAM_MEM,
  1517. "Sec heap region release failed");
  1518. return -ENODEV;
  1519. }
  1520. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1521. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1522. if (rc)
  1523. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1524. return rc;
  1525. }
  1526. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);
  1527. #ifndef CONFIG_CAM_PRESIL
  1528. struct dma_buf * cam_mem_mgr_get_dma_buf(int fd)
  1529. {
  1530. return NULL;
  1531. }
  1532. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1533. {
  1534. return 0;
  1535. }
  1536. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1537. {
  1538. return 0;
  1539. }
  1540. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle,
  1541. uint32_t buf_size,
  1542. uint32_t offset,
  1543. int32_t iommu_hdl)
  1544. {
  1545. return 0;
  1546. }
  1547. #endif