cam_mem_mgr.c 32 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include <linux/debugfs.h>
  12. #include "cam_compat.h"
  13. #include "cam_req_mgr_util.h"
  14. #include "cam_mem_mgr.h"
  15. #include "cam_smmu_api.h"
  16. #include "cam_debug_util.h"
  17. #include "cam_trace.h"
  18. #include "cam_common_util.h"
  19. static struct cam_mem_table tbl;
  20. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  21. static int cam_mem_util_get_dma_dir(uint32_t flags)
  22. {
  23. int rc = -EINVAL;
  24. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  25. rc = DMA_TO_DEVICE;
  26. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  27. rc = DMA_FROM_DEVICE;
  28. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  29. rc = DMA_BIDIRECTIONAL;
  30. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  31. rc = DMA_BIDIRECTIONAL;
  32. return rc;
  33. }
  34. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  35. uintptr_t *vaddr,
  36. size_t *len)
  37. {
  38. int i, j, rc;
  39. void *addr;
  40. /*
  41. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  42. * need to be called in pair to avoid stability issue.
  43. */
  44. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  45. if (rc) {
  46. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  47. return rc;
  48. }
  49. /*
  50. * Code could be simplified if ION support of dma_buf_vmap is
  51. * available. This workaround takes the avandaage that ion_alloc
  52. * returns a virtually contiguous memory region, so we just need
  53. * to _kmap each individual page and then only use the virtual
  54. * address returned from the first call to _kmap.
  55. */
  56. for (i = 0; i < PAGE_ALIGN(dmabuf->size) / PAGE_SIZE; i++) {
  57. addr = dma_buf_kmap(dmabuf, i);
  58. if (IS_ERR_OR_NULL(addr)) {
  59. CAM_ERR(CAM_MEM, "kernel map fail");
  60. for (j = 0; j < i; j++)
  61. dma_buf_kunmap(dmabuf,
  62. j,
  63. (void *)(*vaddr + (j * PAGE_SIZE)));
  64. *vaddr = 0;
  65. *len = 0;
  66. rc = -ENOSPC;
  67. goto fail;
  68. }
  69. if (i == 0)
  70. *vaddr = (uint64_t)addr;
  71. }
  72. *len = dmabuf->size;
  73. return 0;
  74. fail:
  75. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  76. return rc;
  77. }
  78. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  79. uint64_t vaddr)
  80. {
  81. int i, rc = 0, page_num;
  82. if (!dmabuf || !vaddr) {
  83. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  84. return -EINVAL;
  85. }
  86. page_num = PAGE_ALIGN(dmabuf->size) / PAGE_SIZE;
  87. for (i = 0; i < page_num; i++) {
  88. dma_buf_kunmap(dmabuf, i,
  89. (void *)(vaddr + (i * PAGE_SIZE)));
  90. }
  91. /*
  92. * dma_buf_begin_cpu_access() and
  93. * dma_buf_end_cpu_access() need to be called in pair
  94. * to avoid stability issue.
  95. */
  96. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  97. if (rc) {
  98. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  99. dmabuf);
  100. return rc;
  101. }
  102. return rc;
  103. }
  104. static int cam_mem_mgr_create_debug_fs(void)
  105. {
  106. tbl.dentry = debugfs_create_dir("camera_memmgr", NULL);
  107. if (!tbl.dentry) {
  108. CAM_ERR(CAM_MEM, "failed to create dentry");
  109. return -ENOMEM;
  110. }
  111. if (!debugfs_create_bool("alloc_profile_enable",
  112. 0644,
  113. tbl.dentry,
  114. &tbl.alloc_profile_enable)) {
  115. CAM_ERR(CAM_MEM,
  116. "failed to create alloc_profile_enable");
  117. goto err;
  118. }
  119. return 0;
  120. err:
  121. debugfs_remove_recursive(tbl.dentry);
  122. return -ENOMEM;
  123. }
  124. int cam_mem_mgr_init(void)
  125. {
  126. int i;
  127. int bitmap_size;
  128. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  129. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  130. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  131. if (!tbl.bitmap)
  132. return -ENOMEM;
  133. tbl.bits = bitmap_size * BITS_PER_BYTE;
  134. bitmap_zero(tbl.bitmap, tbl.bits);
  135. /* We need to reserve slot 0 because 0 is invalid */
  136. set_bit(0, tbl.bitmap);
  137. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  138. tbl.bufq[i].fd = -1;
  139. tbl.bufq[i].buf_handle = -1;
  140. }
  141. mutex_init(&tbl.m_lock);
  142. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  143. cam_mem_mgr_create_debug_fs();
  144. return 0;
  145. }
  146. static int32_t cam_mem_get_slot(void)
  147. {
  148. int32_t idx;
  149. mutex_lock(&tbl.m_lock);
  150. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  151. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  152. mutex_unlock(&tbl.m_lock);
  153. return -ENOMEM;
  154. }
  155. set_bit(idx, tbl.bitmap);
  156. tbl.bufq[idx].active = true;
  157. mutex_init(&tbl.bufq[idx].q_lock);
  158. mutex_unlock(&tbl.m_lock);
  159. return idx;
  160. }
  161. static void cam_mem_put_slot(int32_t idx)
  162. {
  163. mutex_lock(&tbl.m_lock);
  164. mutex_lock(&tbl.bufq[idx].q_lock);
  165. tbl.bufq[idx].active = false;
  166. mutex_unlock(&tbl.bufq[idx].q_lock);
  167. mutex_destroy(&tbl.bufq[idx].q_lock);
  168. clear_bit(idx, tbl.bitmap);
  169. mutex_unlock(&tbl.m_lock);
  170. }
  171. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  172. dma_addr_t *iova_ptr, size_t *len_ptr)
  173. {
  174. int rc = 0, idx;
  175. *len_ptr = 0;
  176. if (!atomic_read(&cam_mem_mgr_state)) {
  177. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  178. return -EINVAL;
  179. }
  180. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  181. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  182. return -ENOENT;
  183. if (!tbl.bufq[idx].active)
  184. return -EAGAIN;
  185. mutex_lock(&tbl.bufq[idx].q_lock);
  186. if (buf_handle != tbl.bufq[idx].buf_handle) {
  187. rc = -EINVAL;
  188. goto handle_mismatch;
  189. }
  190. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  191. rc = cam_smmu_get_stage2_iova(mmu_handle,
  192. tbl.bufq[idx].fd,
  193. iova_ptr,
  194. len_ptr);
  195. else
  196. rc = cam_smmu_get_iova(mmu_handle,
  197. tbl.bufq[idx].fd,
  198. iova_ptr,
  199. len_ptr);
  200. if (rc) {
  201. CAM_ERR(CAM_MEM,
  202. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d",
  203. buf_handle, mmu_handle, tbl.bufq[idx].fd);
  204. goto handle_mismatch;
  205. }
  206. CAM_DBG(CAM_MEM,
  207. "handle:0x%x fd:%d iova_ptr:%pK len_ptr:%llu",
  208. mmu_handle, tbl.bufq[idx].fd, iova_ptr, *len_ptr);
  209. handle_mismatch:
  210. mutex_unlock(&tbl.bufq[idx].q_lock);
  211. return rc;
  212. }
  213. EXPORT_SYMBOL(cam_mem_get_io_buf);
  214. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  215. {
  216. int idx;
  217. if (!atomic_read(&cam_mem_mgr_state)) {
  218. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  219. return -EINVAL;
  220. }
  221. if (!atomic_read(&cam_mem_mgr_state)) {
  222. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  223. return -EINVAL;
  224. }
  225. if (!buf_handle || !vaddr_ptr || !len)
  226. return -EINVAL;
  227. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  228. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  229. return -EINVAL;
  230. if (!tbl.bufq[idx].active)
  231. return -EPERM;
  232. if (buf_handle != tbl.bufq[idx].buf_handle)
  233. return -EINVAL;
  234. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  235. return -EINVAL;
  236. if (tbl.bufq[idx].kmdvaddr) {
  237. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  238. *len = tbl.bufq[idx].len;
  239. } else {
  240. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  241. buf_handle);
  242. return -EINVAL;
  243. }
  244. return 0;
  245. }
  246. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  247. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  248. {
  249. int rc = 0, idx;
  250. uint32_t cache_dir;
  251. unsigned long dmabuf_flag = 0;
  252. if (!atomic_read(&cam_mem_mgr_state)) {
  253. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  254. return -EINVAL;
  255. }
  256. if (!cmd)
  257. return -EINVAL;
  258. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  259. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  260. return -EINVAL;
  261. mutex_lock(&tbl.bufq[idx].q_lock);
  262. if (!tbl.bufq[idx].active) {
  263. rc = -EINVAL;
  264. goto end;
  265. }
  266. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  267. rc = -EINVAL;
  268. goto end;
  269. }
  270. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  271. if (rc) {
  272. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  273. goto end;
  274. }
  275. if (dmabuf_flag & ION_FLAG_CACHED) {
  276. switch (cmd->mem_cache_ops) {
  277. case CAM_MEM_CLEAN_CACHE:
  278. cache_dir = DMA_TO_DEVICE;
  279. break;
  280. case CAM_MEM_INV_CACHE:
  281. cache_dir = DMA_FROM_DEVICE;
  282. break;
  283. case CAM_MEM_CLEAN_INV_CACHE:
  284. cache_dir = DMA_BIDIRECTIONAL;
  285. break;
  286. default:
  287. CAM_ERR(CAM_MEM,
  288. "invalid cache ops :%d", cmd->mem_cache_ops);
  289. rc = -EINVAL;
  290. goto end;
  291. }
  292. } else {
  293. CAM_DBG(CAM_MEM, "BUF is not cached");
  294. goto end;
  295. }
  296. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  297. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  298. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  299. if (rc) {
  300. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  301. goto end;
  302. }
  303. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  304. cache_dir);
  305. if (rc) {
  306. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  307. goto end;
  308. }
  309. end:
  310. mutex_unlock(&tbl.bufq[idx].q_lock);
  311. return rc;
  312. }
  313. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  314. static int cam_mem_util_get_dma_buf(size_t len,
  315. unsigned int heap_id_mask,
  316. unsigned int flags,
  317. struct dma_buf **buf)
  318. {
  319. int rc = 0;
  320. if (!buf) {
  321. CAM_ERR(CAM_MEM, "Invalid params");
  322. return -EINVAL;
  323. }
  324. *buf = ion_alloc(len, heap_id_mask, flags);
  325. if (IS_ERR_OR_NULL(*buf))
  326. return -ENOMEM;
  327. return rc;
  328. }
  329. static int cam_mem_util_get_dma_buf_fd(size_t len,
  330. size_t align,
  331. unsigned int heap_id_mask,
  332. unsigned int flags,
  333. struct dma_buf **buf,
  334. int *fd)
  335. {
  336. struct dma_buf *dmabuf = NULL;
  337. int rc = 0;
  338. struct timespec64 ts1, ts2;
  339. long microsec = 0;
  340. if (!buf || !fd) {
  341. CAM_ERR(CAM_MEM, "Invalid params, buf=%pK, fd=%pK", buf, fd);
  342. return -EINVAL;
  343. }
  344. if (tbl.alloc_profile_enable)
  345. CAM_GET_TIMESTAMP(ts1);
  346. *buf = ion_alloc(len, heap_id_mask, flags);
  347. if (IS_ERR_OR_NULL(*buf))
  348. return -ENOMEM;
  349. *fd = dma_buf_fd(*buf, O_CLOEXEC);
  350. if (*fd < 0) {
  351. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  352. rc = -EINVAL;
  353. goto get_fd_fail;
  354. }
  355. /*
  356. * increment the ref count so that ref count becomes 2 here
  357. * when we close fd, refcount becomes 1 and when we do
  358. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  359. */
  360. dmabuf = dma_buf_get(*fd);
  361. if (IS_ERR_OR_NULL(dmabuf)) {
  362. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d", *fd);
  363. rc = -EINVAL;
  364. }
  365. if (tbl.alloc_profile_enable) {
  366. CAM_GET_TIMESTAMP(ts2);
  367. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  368. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  369. len, microsec);
  370. }
  371. return rc;
  372. get_fd_fail:
  373. dma_buf_put(*buf);
  374. return rc;
  375. }
  376. static int cam_mem_util_ion_alloc(struct cam_mem_mgr_alloc_cmd *cmd,
  377. struct dma_buf **dmabuf,
  378. int *fd)
  379. {
  380. uint32_t heap_id;
  381. uint32_t ion_flag = 0;
  382. int rc;
  383. if ((cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  384. (cmd->flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  385. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  386. ion_flag |=
  387. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  388. } else if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  389. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  390. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  391. } else {
  392. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  393. ION_HEAP(ION_CAMERA_HEAP_ID);
  394. }
  395. if (cmd->flags & CAM_MEM_FLAG_CACHE)
  396. ion_flag |= ION_FLAG_CACHED;
  397. else
  398. ion_flag &= ~ION_FLAG_CACHED;
  399. rc = cam_mem_util_get_dma_buf_fd(cmd->len,
  400. cmd->align,
  401. heap_id,
  402. ion_flag,
  403. dmabuf,
  404. fd);
  405. return rc;
  406. }
  407. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  408. {
  409. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  410. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  411. CAM_MEM_MMU_MAX_HANDLE);
  412. return -EINVAL;
  413. }
  414. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  415. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  416. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  417. return -EINVAL;
  418. }
  419. return 0;
  420. }
  421. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  422. {
  423. if (!cmd->flags) {
  424. CAM_ERR(CAM_MEM, "Invalid flags");
  425. return -EINVAL;
  426. }
  427. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  428. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  429. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  430. return -EINVAL;
  431. }
  432. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  433. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  434. CAM_ERR(CAM_MEM,
  435. "Kernel mapping in secure mode not allowed, flags=0x%x",
  436. cmd->flags);
  437. return -EINVAL;
  438. }
  439. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  440. CAM_ERR(CAM_MEM,
  441. "Shared memory buffers are not allowed to be mapped");
  442. return -EINVAL;
  443. }
  444. return 0;
  445. }
  446. static int cam_mem_util_map_hw_va(uint32_t flags,
  447. int32_t *mmu_hdls,
  448. int32_t num_hdls,
  449. int fd,
  450. dma_addr_t *hw_vaddr,
  451. size_t *len,
  452. enum cam_smmu_region_id region)
  453. {
  454. int i;
  455. int rc = -1;
  456. int dir = cam_mem_util_get_dma_dir(flags);
  457. bool dis_delayed_unmap = false;
  458. if (dir < 0) {
  459. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  460. return dir;
  461. }
  462. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  463. dis_delayed_unmap = true;
  464. CAM_DBG(CAM_MEM,
  465. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  466. fd, flags, dir, num_hdls);
  467. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  468. for (i = 0; i < num_hdls; i++) {
  469. rc = cam_smmu_map_stage2_iova(mmu_hdls[i],
  470. fd,
  471. dir,
  472. hw_vaddr,
  473. len);
  474. if (rc < 0) {
  475. CAM_ERR(CAM_MEM,
  476. "Failed to securely map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  477. i, fd, dir, mmu_hdls[i], rc);
  478. goto multi_map_fail;
  479. }
  480. }
  481. } else {
  482. for (i = 0; i < num_hdls; i++) {
  483. rc = cam_smmu_map_user_iova(mmu_hdls[i],
  484. fd,
  485. dis_delayed_unmap,
  486. dir,
  487. (dma_addr_t *)hw_vaddr,
  488. len,
  489. region);
  490. if (rc < 0) {
  491. CAM_ERR(CAM_MEM,
  492. "Failed to map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, region=%d, rc=%d",
  493. i, fd, dir, mmu_hdls[i], region, rc);
  494. goto multi_map_fail;
  495. }
  496. }
  497. }
  498. return rc;
  499. multi_map_fail:
  500. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  501. for (--i; i > 0; i--)
  502. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  503. else
  504. for (--i; i > 0; i--)
  505. cam_smmu_unmap_user_iova(mmu_hdls[i],
  506. fd,
  507. CAM_SMMU_REGION_IO);
  508. return rc;
  509. }
  510. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  511. {
  512. int rc;
  513. int32_t idx;
  514. struct dma_buf *dmabuf = NULL;
  515. int fd = -1;
  516. dma_addr_t hw_vaddr = 0;
  517. size_t len;
  518. uintptr_t kvaddr = 0;
  519. size_t klen;
  520. if (!atomic_read(&cam_mem_mgr_state)) {
  521. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  522. return -EINVAL;
  523. }
  524. if (!cmd) {
  525. CAM_ERR(CAM_MEM, " Invalid argument");
  526. return -EINVAL;
  527. }
  528. len = cmd->len;
  529. rc = cam_mem_util_check_alloc_flags(cmd);
  530. if (rc) {
  531. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  532. cmd->flags, rc);
  533. return rc;
  534. }
  535. rc = cam_mem_util_ion_alloc(cmd,
  536. &dmabuf,
  537. &fd);
  538. if (rc) {
  539. CAM_ERR(CAM_MEM,
  540. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  541. cmd->len, cmd->align, cmd->flags, cmd->num_hdl);
  542. return rc;
  543. }
  544. idx = cam_mem_get_slot();
  545. if (idx < 0) {
  546. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  547. rc = -ENOMEM;
  548. goto slot_fail;
  549. }
  550. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  551. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  552. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  553. enum cam_smmu_region_id region;
  554. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  555. region = CAM_SMMU_REGION_IO;
  556. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  557. region = CAM_SMMU_REGION_SHARED;
  558. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  559. region = CAM_SMMU_REGION_SECHEAP;
  560. rc = cam_mem_util_map_hw_va(cmd->flags,
  561. cmd->mmu_hdls,
  562. cmd->num_hdl,
  563. fd,
  564. &hw_vaddr,
  565. &len,
  566. region);
  567. if (rc) {
  568. CAM_ERR(CAM_MEM,
  569. "Failed in map_hw_va, len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  570. cmd->len, cmd->flags, fd, region,
  571. cmd->num_hdl, rc);
  572. goto map_hw_fail;
  573. }
  574. }
  575. mutex_lock(&tbl.bufq[idx].q_lock);
  576. tbl.bufq[idx].fd = fd;
  577. tbl.bufq[idx].dma_buf = NULL;
  578. tbl.bufq[idx].flags = cmd->flags;
  579. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  580. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  581. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  582. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  583. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  584. if (rc) {
  585. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  586. dmabuf, rc);
  587. goto map_kernel_fail;
  588. }
  589. }
  590. tbl.bufq[idx].kmdvaddr = kvaddr;
  591. tbl.bufq[idx].vaddr = hw_vaddr;
  592. tbl.bufq[idx].dma_buf = dmabuf;
  593. tbl.bufq[idx].len = cmd->len;
  594. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  595. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  596. sizeof(int32_t) * cmd->num_hdl);
  597. tbl.bufq[idx].is_imported = false;
  598. mutex_unlock(&tbl.bufq[idx].q_lock);
  599. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  600. cmd->out.fd = tbl.bufq[idx].fd;
  601. cmd->out.vaddr = 0;
  602. CAM_DBG(CAM_MEM,
  603. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  604. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  605. tbl.bufq[idx].len);
  606. return rc;
  607. map_kernel_fail:
  608. mutex_unlock(&tbl.bufq[idx].q_lock);
  609. map_hw_fail:
  610. cam_mem_put_slot(idx);
  611. slot_fail:
  612. dma_buf_put(dmabuf);
  613. return rc;
  614. }
  615. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  616. {
  617. int32_t idx;
  618. int rc;
  619. struct dma_buf *dmabuf;
  620. dma_addr_t hw_vaddr = 0;
  621. size_t len = 0;
  622. if (!atomic_read(&cam_mem_mgr_state)) {
  623. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  624. return -EINVAL;
  625. }
  626. if (!cmd || (cmd->fd < 0)) {
  627. CAM_ERR(CAM_MEM, "Invalid argument");
  628. return -EINVAL;
  629. }
  630. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  631. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  632. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  633. return -EINVAL;
  634. }
  635. rc = cam_mem_util_check_map_flags(cmd);
  636. if (rc) {
  637. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  638. return rc;
  639. }
  640. dmabuf = dma_buf_get(cmd->fd);
  641. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  642. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  643. return -EINVAL;
  644. }
  645. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  646. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  647. rc = cam_mem_util_map_hw_va(cmd->flags,
  648. cmd->mmu_hdls,
  649. cmd->num_hdl,
  650. cmd->fd,
  651. &hw_vaddr,
  652. &len,
  653. CAM_SMMU_REGION_IO);
  654. if (rc) {
  655. CAM_ERR(CAM_MEM,
  656. "Failed in map_hw_va, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  657. cmd->flags, cmd->fd, CAM_SMMU_REGION_IO,
  658. cmd->num_hdl, rc);
  659. goto map_fail;
  660. }
  661. }
  662. idx = cam_mem_get_slot();
  663. if (idx < 0) {
  664. rc = -ENOMEM;
  665. goto map_fail;
  666. }
  667. mutex_lock(&tbl.bufq[idx].q_lock);
  668. tbl.bufq[idx].fd = cmd->fd;
  669. tbl.bufq[idx].dma_buf = NULL;
  670. tbl.bufq[idx].flags = cmd->flags;
  671. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  672. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  673. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  674. tbl.bufq[idx].kmdvaddr = 0;
  675. if (cmd->num_hdl > 0)
  676. tbl.bufq[idx].vaddr = hw_vaddr;
  677. else
  678. tbl.bufq[idx].vaddr = 0;
  679. tbl.bufq[idx].dma_buf = dmabuf;
  680. tbl.bufq[idx].len = len;
  681. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  682. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  683. sizeof(int32_t) * cmd->num_hdl);
  684. tbl.bufq[idx].is_imported = true;
  685. mutex_unlock(&tbl.bufq[idx].q_lock);
  686. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  687. cmd->out.vaddr = 0;
  688. CAM_DBG(CAM_MEM,
  689. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  690. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  691. tbl.bufq[idx].len);
  692. return rc;
  693. map_fail:
  694. dma_buf_put(dmabuf);
  695. return rc;
  696. }
  697. static int cam_mem_util_unmap_hw_va(int32_t idx,
  698. enum cam_smmu_region_id region,
  699. enum cam_smmu_mapping_client client)
  700. {
  701. int i;
  702. uint32_t flags;
  703. int32_t *mmu_hdls;
  704. int num_hdls;
  705. int fd;
  706. int rc = 0;
  707. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  708. CAM_ERR(CAM_MEM, "Incorrect index");
  709. return -EINVAL;
  710. }
  711. flags = tbl.bufq[idx].flags;
  712. mmu_hdls = tbl.bufq[idx].hdls;
  713. num_hdls = tbl.bufq[idx].num_hdl;
  714. fd = tbl.bufq[idx].fd;
  715. CAM_DBG(CAM_MEM,
  716. "unmap_hw_va : idx=%d, fd=%x, flags=0x%x, num_hdls=%d, client=%d",
  717. idx, fd, flags, num_hdls, client);
  718. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  719. for (i = 0; i < num_hdls; i++) {
  720. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  721. if (rc < 0) {
  722. CAM_ERR(CAM_MEM,
  723. "Failed in secure unmap, i=%d, fd=%d, mmu_hdl=%d, rc=%d",
  724. i, fd, mmu_hdls[i], rc);
  725. goto unmap_end;
  726. }
  727. }
  728. } else {
  729. for (i = 0; i < num_hdls; i++) {
  730. if (client == CAM_SMMU_MAPPING_USER) {
  731. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  732. fd, region);
  733. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  734. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  735. tbl.bufq[idx].dma_buf, region);
  736. } else {
  737. CAM_ERR(CAM_MEM,
  738. "invalid caller for unmapping : %d",
  739. client);
  740. rc = -EINVAL;
  741. }
  742. if (rc < 0) {
  743. CAM_ERR(CAM_MEM,
  744. "Failed in unmap, i=%d, fd=%d, mmu_hdl=%d, region=%d, rc=%d",
  745. i, fd, mmu_hdls[i], region, rc);
  746. goto unmap_end;
  747. }
  748. }
  749. }
  750. return rc;
  751. unmap_end:
  752. CAM_ERR(CAM_MEM, "unmapping failed");
  753. return rc;
  754. }
  755. static void cam_mem_mgr_unmap_active_buf(int idx)
  756. {
  757. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  758. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  759. region = CAM_SMMU_REGION_SHARED;
  760. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  761. region = CAM_SMMU_REGION_IO;
  762. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  763. }
  764. static int cam_mem_mgr_cleanup_table(void)
  765. {
  766. int i;
  767. mutex_lock(&tbl.m_lock);
  768. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  769. if (!tbl.bufq[i].active) {
  770. CAM_DBG(CAM_MEM,
  771. "Buffer inactive at idx=%d, continuing", i);
  772. continue;
  773. } else {
  774. CAM_DBG(CAM_MEM,
  775. "Active buffer at idx=%d, possible leak needs unmapping",
  776. i);
  777. cam_mem_mgr_unmap_active_buf(i);
  778. }
  779. mutex_lock(&tbl.bufq[i].q_lock);
  780. if (tbl.bufq[i].dma_buf) {
  781. dma_buf_put(tbl.bufq[i].dma_buf);
  782. tbl.bufq[i].dma_buf = NULL;
  783. }
  784. tbl.bufq[i].fd = -1;
  785. tbl.bufq[i].flags = 0;
  786. tbl.bufq[i].buf_handle = -1;
  787. tbl.bufq[i].vaddr = 0;
  788. tbl.bufq[i].len = 0;
  789. memset(tbl.bufq[i].hdls, 0,
  790. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  791. tbl.bufq[i].num_hdl = 0;
  792. tbl.bufq[i].dma_buf = NULL;
  793. tbl.bufq[i].active = false;
  794. mutex_unlock(&tbl.bufq[i].q_lock);
  795. mutex_destroy(&tbl.bufq[i].q_lock);
  796. }
  797. bitmap_zero(tbl.bitmap, tbl.bits);
  798. /* We need to reserve slot 0 because 0 is invalid */
  799. set_bit(0, tbl.bitmap);
  800. mutex_unlock(&tbl.m_lock);
  801. return 0;
  802. }
  803. void cam_mem_mgr_deinit(void)
  804. {
  805. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  806. cam_mem_mgr_cleanup_table();
  807. mutex_lock(&tbl.m_lock);
  808. bitmap_zero(tbl.bitmap, tbl.bits);
  809. kfree(tbl.bitmap);
  810. tbl.bitmap = NULL;
  811. mutex_unlock(&tbl.m_lock);
  812. mutex_destroy(&tbl.m_lock);
  813. }
  814. static int cam_mem_util_unmap(int32_t idx,
  815. enum cam_smmu_mapping_client client)
  816. {
  817. int rc = 0;
  818. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  819. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  820. CAM_ERR(CAM_MEM, "Incorrect index");
  821. return -EINVAL;
  822. }
  823. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  824. mutex_lock(&tbl.m_lock);
  825. if ((!tbl.bufq[idx].active) &&
  826. (tbl.bufq[idx].vaddr) == 0) {
  827. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  828. idx);
  829. mutex_unlock(&tbl.m_lock);
  830. return 0;
  831. }
  832. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  833. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  834. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  835. tbl.bufq[idx].kmdvaddr);
  836. if (rc)
  837. CAM_ERR(CAM_MEM,
  838. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  839. tbl.bufq[idx].dma_buf,
  840. (void *) tbl.bufq[idx].kmdvaddr);
  841. }
  842. }
  843. /* SHARED flag gets precedence, all other flags after it */
  844. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  845. region = CAM_SMMU_REGION_SHARED;
  846. } else {
  847. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  848. region = CAM_SMMU_REGION_IO;
  849. }
  850. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  851. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  852. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  853. if (cam_mem_util_unmap_hw_va(idx, region, client))
  854. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  855. tbl.bufq[idx].dma_buf);
  856. if (client == CAM_SMMU_MAPPING_KERNEL)
  857. tbl.bufq[idx].dma_buf = NULL;
  858. }
  859. mutex_lock(&tbl.bufq[idx].q_lock);
  860. tbl.bufq[idx].flags = 0;
  861. tbl.bufq[idx].buf_handle = -1;
  862. tbl.bufq[idx].vaddr = 0;
  863. memset(tbl.bufq[idx].hdls, 0,
  864. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  865. CAM_DBG(CAM_MEM,
  866. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK",
  867. idx, tbl.bufq[idx].fd,
  868. tbl.bufq[idx].is_imported,
  869. tbl.bufq[idx].dma_buf);
  870. if (tbl.bufq[idx].dma_buf)
  871. dma_buf_put(tbl.bufq[idx].dma_buf);
  872. tbl.bufq[idx].fd = -1;
  873. tbl.bufq[idx].dma_buf = NULL;
  874. tbl.bufq[idx].is_imported = false;
  875. tbl.bufq[idx].len = 0;
  876. tbl.bufq[idx].num_hdl = 0;
  877. tbl.bufq[idx].active = false;
  878. mutex_unlock(&tbl.bufq[idx].q_lock);
  879. mutex_destroy(&tbl.bufq[idx].q_lock);
  880. clear_bit(idx, tbl.bitmap);
  881. mutex_unlock(&tbl.m_lock);
  882. return rc;
  883. }
  884. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  885. {
  886. int idx;
  887. int rc;
  888. if (!atomic_read(&cam_mem_mgr_state)) {
  889. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  890. return -EINVAL;
  891. }
  892. if (!cmd) {
  893. CAM_ERR(CAM_MEM, "Invalid argument");
  894. return -EINVAL;
  895. }
  896. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  897. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  898. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  899. idx);
  900. return -EINVAL;
  901. }
  902. if (!tbl.bufq[idx].active) {
  903. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  904. return -EINVAL;
  905. }
  906. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  907. CAM_ERR(CAM_MEM,
  908. "Released buf handle %d not matching within table %d, idx=%d",
  909. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  910. return -EINVAL;
  911. }
  912. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  913. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  914. return rc;
  915. }
  916. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  917. struct cam_mem_mgr_memory_desc *out)
  918. {
  919. struct dma_buf *buf = NULL;
  920. int ion_fd = -1;
  921. int rc = 0;
  922. uint32_t heap_id;
  923. int32_t ion_flag = 0;
  924. uintptr_t kvaddr;
  925. dma_addr_t iova = 0;
  926. size_t request_len = 0;
  927. uint32_t mem_handle;
  928. int32_t idx;
  929. int32_t smmu_hdl = 0;
  930. int32_t num_hdl = 0;
  931. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  932. if (!atomic_read(&cam_mem_mgr_state)) {
  933. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  934. return -EINVAL;
  935. }
  936. if (!inp || !out) {
  937. CAM_ERR(CAM_MEM, "Invalid params");
  938. return -EINVAL;
  939. }
  940. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  941. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  942. inp->flags & CAM_MEM_FLAG_CACHE)) {
  943. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  944. return -EINVAL;
  945. }
  946. if (inp->flags & CAM_MEM_FLAG_CACHE)
  947. ion_flag |= ION_FLAG_CACHED;
  948. else
  949. ion_flag &= ~ION_FLAG_CACHED;
  950. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  951. ION_HEAP(ION_CAMERA_HEAP_ID);
  952. rc = cam_mem_util_get_dma_buf(inp->size,
  953. heap_id,
  954. ion_flag,
  955. &buf);
  956. if (rc) {
  957. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  958. goto ion_fail;
  959. } else {
  960. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  961. }
  962. /*
  963. * we are mapping kva always here,
  964. * update flags so that we do unmap properly
  965. */
  966. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  967. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  968. if (rc) {
  969. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  970. goto map_fail;
  971. }
  972. if (!inp->smmu_hdl) {
  973. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  974. rc = -EINVAL;
  975. goto smmu_fail;
  976. }
  977. /* SHARED flag gets precedence, all other flags after it */
  978. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  979. region = CAM_SMMU_REGION_SHARED;
  980. } else {
  981. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  982. region = CAM_SMMU_REGION_IO;
  983. }
  984. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  985. buf,
  986. CAM_SMMU_MAP_RW,
  987. &iova,
  988. &request_len,
  989. region);
  990. if (rc < 0) {
  991. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  992. goto smmu_fail;
  993. }
  994. smmu_hdl = inp->smmu_hdl;
  995. num_hdl = 1;
  996. idx = cam_mem_get_slot();
  997. if (idx < 0) {
  998. rc = -ENOMEM;
  999. goto slot_fail;
  1000. }
  1001. mutex_lock(&tbl.bufq[idx].q_lock);
  1002. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1003. tbl.bufq[idx].dma_buf = buf;
  1004. tbl.bufq[idx].fd = -1;
  1005. tbl.bufq[idx].flags = inp->flags;
  1006. tbl.bufq[idx].buf_handle = mem_handle;
  1007. tbl.bufq[idx].kmdvaddr = kvaddr;
  1008. tbl.bufq[idx].vaddr = iova;
  1009. tbl.bufq[idx].len = inp->size;
  1010. tbl.bufq[idx].num_hdl = num_hdl;
  1011. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1012. sizeof(int32_t));
  1013. tbl.bufq[idx].is_imported = false;
  1014. mutex_unlock(&tbl.bufq[idx].q_lock);
  1015. out->kva = kvaddr;
  1016. out->iova = (uint32_t)iova;
  1017. out->smmu_hdl = smmu_hdl;
  1018. out->mem_handle = mem_handle;
  1019. out->len = inp->size;
  1020. out->region = region;
  1021. return rc;
  1022. slot_fail:
  1023. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1024. buf, region);
  1025. smmu_fail:
  1026. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1027. map_fail:
  1028. dma_buf_put(buf);
  1029. ion_fail:
  1030. return rc;
  1031. }
  1032. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1033. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1034. {
  1035. int32_t idx;
  1036. int rc;
  1037. if (!atomic_read(&cam_mem_mgr_state)) {
  1038. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1039. return -EINVAL;
  1040. }
  1041. if (!inp) {
  1042. CAM_ERR(CAM_MEM, "Invalid argument");
  1043. return -EINVAL;
  1044. }
  1045. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1046. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1047. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1048. return -EINVAL;
  1049. }
  1050. if (!tbl.bufq[idx].active) {
  1051. if (tbl.bufq[idx].vaddr == 0) {
  1052. CAM_ERR(CAM_MEM, "buffer is released already");
  1053. return 0;
  1054. }
  1055. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1056. return -EINVAL;
  1057. }
  1058. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1059. CAM_ERR(CAM_MEM,
  1060. "Released buf handle not matching within table");
  1061. return -EINVAL;
  1062. }
  1063. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1064. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1065. return rc;
  1066. }
  1067. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1068. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1069. enum cam_smmu_region_id region,
  1070. struct cam_mem_mgr_memory_desc *out)
  1071. {
  1072. struct dma_buf *buf = NULL;
  1073. int rc = 0;
  1074. int ion_fd = -1;
  1075. uint32_t heap_id;
  1076. dma_addr_t iova = 0;
  1077. size_t request_len = 0;
  1078. uint32_t mem_handle;
  1079. int32_t idx;
  1080. int32_t smmu_hdl = 0;
  1081. int32_t num_hdl = 0;
  1082. if (!atomic_read(&cam_mem_mgr_state)) {
  1083. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1084. return -EINVAL;
  1085. }
  1086. if (!inp || !out) {
  1087. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1088. return -EINVAL;
  1089. }
  1090. if (!inp->smmu_hdl) {
  1091. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1092. return -EINVAL;
  1093. }
  1094. if (region != CAM_SMMU_REGION_SECHEAP) {
  1095. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1096. return -EINVAL;
  1097. }
  1098. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  1099. ION_HEAP(ION_CAMERA_HEAP_ID);
  1100. rc = cam_mem_util_get_dma_buf(inp->size,
  1101. heap_id,
  1102. 0,
  1103. &buf);
  1104. if (rc) {
  1105. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1106. goto ion_fail;
  1107. } else {
  1108. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1109. }
  1110. rc = cam_smmu_reserve_sec_heap(inp->smmu_hdl,
  1111. buf,
  1112. &iova,
  1113. &request_len);
  1114. if (rc) {
  1115. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1116. goto smmu_fail;
  1117. }
  1118. smmu_hdl = inp->smmu_hdl;
  1119. num_hdl = 1;
  1120. idx = cam_mem_get_slot();
  1121. if (idx < 0) {
  1122. rc = -ENOMEM;
  1123. goto slot_fail;
  1124. }
  1125. mutex_lock(&tbl.bufq[idx].q_lock);
  1126. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1127. tbl.bufq[idx].fd = -1;
  1128. tbl.bufq[idx].dma_buf = buf;
  1129. tbl.bufq[idx].flags = inp->flags;
  1130. tbl.bufq[idx].buf_handle = mem_handle;
  1131. tbl.bufq[idx].kmdvaddr = 0;
  1132. tbl.bufq[idx].vaddr = iova;
  1133. tbl.bufq[idx].len = request_len;
  1134. tbl.bufq[idx].num_hdl = num_hdl;
  1135. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1136. sizeof(int32_t));
  1137. tbl.bufq[idx].is_imported = false;
  1138. mutex_unlock(&tbl.bufq[idx].q_lock);
  1139. out->kva = 0;
  1140. out->iova = (uint32_t)iova;
  1141. out->smmu_hdl = smmu_hdl;
  1142. out->mem_handle = mem_handle;
  1143. out->len = request_len;
  1144. out->region = region;
  1145. return rc;
  1146. slot_fail:
  1147. cam_smmu_release_sec_heap(smmu_hdl);
  1148. smmu_fail:
  1149. dma_buf_put(buf);
  1150. ion_fail:
  1151. return rc;
  1152. }
  1153. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1154. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1155. {
  1156. int32_t idx;
  1157. int rc;
  1158. int32_t smmu_hdl;
  1159. if (!atomic_read(&cam_mem_mgr_state)) {
  1160. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1161. return -EINVAL;
  1162. }
  1163. if (!inp) {
  1164. CAM_ERR(CAM_MEM, "Invalid argument");
  1165. return -EINVAL;
  1166. }
  1167. if (inp->region != CAM_SMMU_REGION_SECHEAP) {
  1168. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1169. return -EINVAL;
  1170. }
  1171. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1172. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1173. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1174. return -EINVAL;
  1175. }
  1176. if (!tbl.bufq[idx].active) {
  1177. if (tbl.bufq[idx].vaddr == 0) {
  1178. CAM_ERR(CAM_MEM, "buffer is released already");
  1179. return 0;
  1180. }
  1181. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1182. return -EINVAL;
  1183. }
  1184. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1185. CAM_ERR(CAM_MEM,
  1186. "Released buf handle not matching within table");
  1187. return -EINVAL;
  1188. }
  1189. if (tbl.bufq[idx].num_hdl != 1) {
  1190. CAM_ERR(CAM_MEM,
  1191. "Sec heap region should have only one smmu hdl");
  1192. return -ENODEV;
  1193. }
  1194. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1195. sizeof(int32_t));
  1196. if (inp->smmu_hdl != smmu_hdl) {
  1197. CAM_ERR(CAM_MEM,
  1198. "Passed SMMU handle doesn't match with internal hdl");
  1199. return -ENODEV;
  1200. }
  1201. rc = cam_smmu_release_sec_heap(inp->smmu_hdl);
  1202. if (rc) {
  1203. CAM_ERR(CAM_MEM,
  1204. "Sec heap region release failed");
  1205. return -ENODEV;
  1206. }
  1207. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1208. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1209. if (rc)
  1210. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1211. return rc;
  1212. }
  1213. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);