cam_mem_mgr.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2019, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include "cam_compat.h"
  12. #include "cam_req_mgr_util.h"
  13. #include "cam_mem_mgr.h"
  14. #include "cam_smmu_api.h"
  15. #include "cam_debug_util.h"
  16. static struct cam_mem_table tbl;
  17. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  18. static int cam_mem_util_get_dma_dir(uint32_t flags)
  19. {
  20. int rc = -EINVAL;
  21. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  22. rc = DMA_TO_DEVICE;
  23. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  24. rc = DMA_FROM_DEVICE;
  25. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  26. rc = DMA_BIDIRECTIONAL;
  27. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  28. rc = DMA_BIDIRECTIONAL;
  29. return rc;
  30. }
  31. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  32. uintptr_t *vaddr,
  33. size_t *len)
  34. {
  35. int i, j, rc;
  36. void *addr;
  37. /*
  38. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  39. * need to be called in pair to avoid stability issue.
  40. */
  41. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  42. if (rc) {
  43. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  44. return rc;
  45. }
  46. /*
  47. * Code could be simplified if ION support of dma_buf_vmap is
  48. * available. This workaround takes the avandaage that ion_alloc
  49. * returns a virtually contiguous memory region, so we just need
  50. * to _kmap each individual page and then only use the virtual
  51. * address returned from the first call to _kmap.
  52. */
  53. for (i = 0; i < PAGE_ALIGN(dmabuf->size) / PAGE_SIZE; i++) {
  54. addr = dma_buf_kmap(dmabuf, i);
  55. if (IS_ERR_OR_NULL(addr)) {
  56. CAM_ERR(CAM_MEM, "kernel map fail");
  57. for (j = 0; j < i; j++)
  58. dma_buf_kunmap(dmabuf,
  59. j,
  60. (void *)(*vaddr + (j * PAGE_SIZE)));
  61. *vaddr = 0;
  62. *len = 0;
  63. rc = -ENOSPC;
  64. goto fail;
  65. }
  66. if (i == 0)
  67. *vaddr = (uint64_t)addr;
  68. }
  69. *len = dmabuf->size;
  70. return 0;
  71. fail:
  72. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  73. return rc;
  74. }
  75. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  76. uint64_t vaddr)
  77. {
  78. int i, rc = 0, page_num;
  79. if (!dmabuf || !vaddr) {
  80. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  81. return -EINVAL;
  82. }
  83. page_num = PAGE_ALIGN(dmabuf->size) / PAGE_SIZE;
  84. for (i = 0; i < page_num; i++) {
  85. dma_buf_kunmap(dmabuf, i,
  86. (void *)(vaddr + (i * PAGE_SIZE)));
  87. }
  88. /*
  89. * dma_buf_begin_cpu_access() and
  90. * dma_buf_end_cpu_access() need to be called in pair
  91. * to avoid stability issue.
  92. */
  93. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  94. if (rc) {
  95. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  96. dmabuf);
  97. return rc;
  98. }
  99. return rc;
  100. }
  101. int cam_mem_mgr_init(void)
  102. {
  103. int i;
  104. int bitmap_size;
  105. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  106. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  107. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  108. if (!tbl.bitmap)
  109. return -ENOMEM;
  110. tbl.bits = bitmap_size * BITS_PER_BYTE;
  111. bitmap_zero(tbl.bitmap, tbl.bits);
  112. /* We need to reserve slot 0 because 0 is invalid */
  113. set_bit(0, tbl.bitmap);
  114. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  115. tbl.bufq[i].fd = -1;
  116. tbl.bufq[i].buf_handle = -1;
  117. }
  118. mutex_init(&tbl.m_lock);
  119. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  120. return 0;
  121. }
  122. static int32_t cam_mem_get_slot(void)
  123. {
  124. int32_t idx;
  125. mutex_lock(&tbl.m_lock);
  126. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  127. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  128. mutex_unlock(&tbl.m_lock);
  129. return -ENOMEM;
  130. }
  131. set_bit(idx, tbl.bitmap);
  132. tbl.bufq[idx].active = true;
  133. mutex_init(&tbl.bufq[idx].q_lock);
  134. mutex_unlock(&tbl.m_lock);
  135. return idx;
  136. }
  137. static void cam_mem_put_slot(int32_t idx)
  138. {
  139. mutex_lock(&tbl.m_lock);
  140. mutex_lock(&tbl.bufq[idx].q_lock);
  141. tbl.bufq[idx].active = false;
  142. mutex_unlock(&tbl.bufq[idx].q_lock);
  143. mutex_destroy(&tbl.bufq[idx].q_lock);
  144. clear_bit(idx, tbl.bitmap);
  145. mutex_unlock(&tbl.m_lock);
  146. }
  147. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  148. dma_addr_t *iova_ptr, size_t *len_ptr)
  149. {
  150. int rc = 0, idx;
  151. *len_ptr = 0;
  152. if (!atomic_read(&cam_mem_mgr_state)) {
  153. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  154. return -EINVAL;
  155. }
  156. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  157. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  158. return -ENOENT;
  159. if (!tbl.bufq[idx].active)
  160. return -EAGAIN;
  161. mutex_lock(&tbl.bufq[idx].q_lock);
  162. if (buf_handle != tbl.bufq[idx].buf_handle) {
  163. rc = -EINVAL;
  164. goto handle_mismatch;
  165. }
  166. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  167. rc = cam_smmu_get_stage2_iova(mmu_handle,
  168. tbl.bufq[idx].fd,
  169. iova_ptr,
  170. len_ptr);
  171. else
  172. rc = cam_smmu_get_iova(mmu_handle,
  173. tbl.bufq[idx].fd,
  174. iova_ptr,
  175. len_ptr);
  176. if (rc) {
  177. CAM_ERR(CAM_MEM,
  178. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d",
  179. buf_handle, mmu_handle, tbl.bufq[idx].fd);
  180. goto handle_mismatch;
  181. }
  182. CAM_DBG(CAM_MEM,
  183. "handle:0x%x fd:%d iova_ptr:%pK len_ptr:%llu",
  184. mmu_handle, tbl.bufq[idx].fd, iova_ptr, *len_ptr);
  185. handle_mismatch:
  186. mutex_unlock(&tbl.bufq[idx].q_lock);
  187. return rc;
  188. }
  189. EXPORT_SYMBOL(cam_mem_get_io_buf);
  190. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  191. {
  192. int idx;
  193. if (!atomic_read(&cam_mem_mgr_state)) {
  194. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  195. return -EINVAL;
  196. }
  197. if (!atomic_read(&cam_mem_mgr_state)) {
  198. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  199. return -EINVAL;
  200. }
  201. if (!buf_handle || !vaddr_ptr || !len)
  202. return -EINVAL;
  203. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  204. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  205. return -EINVAL;
  206. if (!tbl.bufq[idx].active)
  207. return -EPERM;
  208. if (buf_handle != tbl.bufq[idx].buf_handle)
  209. return -EINVAL;
  210. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  211. return -EINVAL;
  212. if (tbl.bufq[idx].kmdvaddr) {
  213. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  214. *len = tbl.bufq[idx].len;
  215. } else {
  216. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  217. buf_handle);
  218. return -EINVAL;
  219. }
  220. return 0;
  221. }
  222. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  223. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  224. {
  225. int rc = 0, idx;
  226. uint32_t cache_dir;
  227. unsigned long dmabuf_flag = 0;
  228. if (!atomic_read(&cam_mem_mgr_state)) {
  229. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  230. return -EINVAL;
  231. }
  232. if (!cmd)
  233. return -EINVAL;
  234. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  235. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  236. return -EINVAL;
  237. mutex_lock(&tbl.bufq[idx].q_lock);
  238. if (!tbl.bufq[idx].active) {
  239. rc = -EINVAL;
  240. goto end;
  241. }
  242. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  243. rc = -EINVAL;
  244. goto end;
  245. }
  246. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  247. if (rc) {
  248. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  249. goto end;
  250. }
  251. if (dmabuf_flag & ION_FLAG_CACHED) {
  252. switch (cmd->mem_cache_ops) {
  253. case CAM_MEM_CLEAN_CACHE:
  254. cache_dir = DMA_TO_DEVICE;
  255. break;
  256. case CAM_MEM_INV_CACHE:
  257. cache_dir = DMA_FROM_DEVICE;
  258. break;
  259. case CAM_MEM_CLEAN_INV_CACHE:
  260. cache_dir = DMA_BIDIRECTIONAL;
  261. break;
  262. default:
  263. CAM_ERR(CAM_MEM,
  264. "invalid cache ops :%d", cmd->mem_cache_ops);
  265. rc = -EINVAL;
  266. goto end;
  267. }
  268. } else {
  269. CAM_DBG(CAM_MEM, "BUF is not cached");
  270. goto end;
  271. }
  272. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  273. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  274. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  275. if (rc) {
  276. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  277. goto end;
  278. }
  279. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  280. cache_dir);
  281. if (rc) {
  282. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  283. goto end;
  284. }
  285. end:
  286. mutex_unlock(&tbl.bufq[idx].q_lock);
  287. return rc;
  288. }
  289. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  290. static int cam_mem_util_get_dma_buf(size_t len,
  291. unsigned int heap_id_mask,
  292. unsigned int flags,
  293. struct dma_buf **buf)
  294. {
  295. int rc = 0;
  296. if (!buf) {
  297. CAM_ERR(CAM_MEM, "Invalid params");
  298. return -EINVAL;
  299. }
  300. *buf = ion_alloc(len, heap_id_mask, flags);
  301. if (IS_ERR_OR_NULL(*buf))
  302. return -ENOMEM;
  303. return rc;
  304. }
  305. static int cam_mem_util_get_dma_buf_fd(size_t len,
  306. size_t align,
  307. unsigned int heap_id_mask,
  308. unsigned int flags,
  309. struct dma_buf **buf,
  310. int *fd)
  311. {
  312. struct dma_buf *dmabuf = NULL;
  313. int rc = 0;
  314. if (!buf || !fd) {
  315. CAM_ERR(CAM_MEM, "Invalid params, buf=%pK, fd=%pK", buf, fd);
  316. return -EINVAL;
  317. }
  318. *buf = ion_alloc(len, heap_id_mask, flags);
  319. if (IS_ERR_OR_NULL(*buf))
  320. return -ENOMEM;
  321. *fd = dma_buf_fd(*buf, O_CLOEXEC);
  322. if (*fd < 0) {
  323. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  324. rc = -EINVAL;
  325. goto get_fd_fail;
  326. }
  327. /*
  328. * increment the ref count so that ref count becomes 2 here
  329. * when we close fd, refcount becomes 1 and when we do
  330. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  331. */
  332. dmabuf = dma_buf_get(*fd);
  333. if (IS_ERR_OR_NULL(dmabuf)) {
  334. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d", *fd);
  335. rc = -EINVAL;
  336. }
  337. return rc;
  338. get_fd_fail:
  339. dma_buf_put(*buf);
  340. return rc;
  341. }
  342. static int cam_mem_util_ion_alloc(struct cam_mem_mgr_alloc_cmd *cmd,
  343. struct dma_buf **dmabuf,
  344. int *fd)
  345. {
  346. uint32_t heap_id;
  347. uint32_t ion_flag = 0;
  348. int rc;
  349. if ((cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  350. (cmd->flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  351. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  352. ion_flag |=
  353. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  354. } else if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  355. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  356. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  357. } else {
  358. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  359. ION_HEAP(ION_CAMERA_HEAP_ID);
  360. }
  361. if (cmd->flags & CAM_MEM_FLAG_CACHE)
  362. ion_flag |= ION_FLAG_CACHED;
  363. else
  364. ion_flag &= ~ION_FLAG_CACHED;
  365. rc = cam_mem_util_get_dma_buf_fd(cmd->len,
  366. cmd->align,
  367. heap_id,
  368. ion_flag,
  369. dmabuf,
  370. fd);
  371. return rc;
  372. }
  373. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  374. {
  375. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  376. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  377. CAM_MEM_MMU_MAX_HANDLE);
  378. return -EINVAL;
  379. }
  380. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  381. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  382. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  383. return -EINVAL;
  384. }
  385. return 0;
  386. }
  387. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  388. {
  389. if (!cmd->flags) {
  390. CAM_ERR(CAM_MEM, "Invalid flags");
  391. return -EINVAL;
  392. }
  393. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  394. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  395. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  396. return -EINVAL;
  397. }
  398. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  399. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  400. CAM_ERR(CAM_MEM,
  401. "Kernel mapping in secure mode not allowed, flags=0x%x",
  402. cmd->flags);
  403. return -EINVAL;
  404. }
  405. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  406. CAM_ERR(CAM_MEM,
  407. "Shared memory buffers are not allowed to be mapped");
  408. return -EINVAL;
  409. }
  410. return 0;
  411. }
  412. static int cam_mem_util_map_hw_va(uint32_t flags,
  413. int32_t *mmu_hdls,
  414. int32_t num_hdls,
  415. int fd,
  416. dma_addr_t *hw_vaddr,
  417. size_t *len,
  418. enum cam_smmu_region_id region)
  419. {
  420. int i;
  421. int rc = -1;
  422. int dir = cam_mem_util_get_dma_dir(flags);
  423. bool dis_delayed_unmap = false;
  424. if (dir < 0) {
  425. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  426. return dir;
  427. }
  428. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  429. dis_delayed_unmap = true;
  430. CAM_DBG(CAM_MEM,
  431. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  432. fd, flags, dir, num_hdls);
  433. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  434. for (i = 0; i < num_hdls; i++) {
  435. rc = cam_smmu_map_stage2_iova(mmu_hdls[i],
  436. fd,
  437. dir,
  438. hw_vaddr,
  439. len);
  440. if (rc < 0) {
  441. CAM_ERR(CAM_MEM,
  442. "Failed to securely map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  443. i, fd, dir, mmu_hdls[i], rc);
  444. goto multi_map_fail;
  445. }
  446. }
  447. } else {
  448. for (i = 0; i < num_hdls; i++) {
  449. rc = cam_smmu_map_user_iova(mmu_hdls[i],
  450. fd,
  451. dis_delayed_unmap,
  452. dir,
  453. (dma_addr_t *)hw_vaddr,
  454. len,
  455. region);
  456. if (rc < 0) {
  457. CAM_ERR(CAM_MEM,
  458. "Failed to map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, region=%d, rc=%d",
  459. i, fd, dir, mmu_hdls[i], region, rc);
  460. goto multi_map_fail;
  461. }
  462. }
  463. }
  464. return rc;
  465. multi_map_fail:
  466. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  467. for (--i; i > 0; i--)
  468. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  469. else
  470. for (--i; i > 0; i--)
  471. cam_smmu_unmap_user_iova(mmu_hdls[i],
  472. fd,
  473. CAM_SMMU_REGION_IO);
  474. return rc;
  475. }
  476. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  477. {
  478. int rc;
  479. int32_t idx;
  480. struct dma_buf *dmabuf = NULL;
  481. int fd = -1;
  482. dma_addr_t hw_vaddr = 0;
  483. size_t len;
  484. uintptr_t kvaddr = 0;
  485. size_t klen;
  486. if (!atomic_read(&cam_mem_mgr_state)) {
  487. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  488. return -EINVAL;
  489. }
  490. if (!cmd) {
  491. CAM_ERR(CAM_MEM, " Invalid argument");
  492. return -EINVAL;
  493. }
  494. len = cmd->len;
  495. rc = cam_mem_util_check_alloc_flags(cmd);
  496. if (rc) {
  497. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  498. cmd->flags, rc);
  499. return rc;
  500. }
  501. rc = cam_mem_util_ion_alloc(cmd,
  502. &dmabuf,
  503. &fd);
  504. if (rc) {
  505. CAM_ERR(CAM_MEM,
  506. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  507. cmd->len, cmd->align, cmd->flags, cmd->num_hdl);
  508. return rc;
  509. }
  510. idx = cam_mem_get_slot();
  511. if (idx < 0) {
  512. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  513. rc = -ENOMEM;
  514. goto slot_fail;
  515. }
  516. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  517. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  518. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  519. enum cam_smmu_region_id region;
  520. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  521. region = CAM_SMMU_REGION_IO;
  522. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  523. region = CAM_SMMU_REGION_SHARED;
  524. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  525. region = CAM_SMMU_REGION_SECHEAP;
  526. rc = cam_mem_util_map_hw_va(cmd->flags,
  527. cmd->mmu_hdls,
  528. cmd->num_hdl,
  529. fd,
  530. &hw_vaddr,
  531. &len,
  532. region);
  533. if (rc) {
  534. CAM_ERR(CAM_MEM,
  535. "Failed in map_hw_va, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  536. cmd->flags, fd, region, cmd->num_hdl, rc);
  537. goto map_hw_fail;
  538. }
  539. }
  540. mutex_lock(&tbl.bufq[idx].q_lock);
  541. tbl.bufq[idx].fd = fd;
  542. tbl.bufq[idx].dma_buf = NULL;
  543. tbl.bufq[idx].flags = cmd->flags;
  544. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  545. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  546. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  547. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  548. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  549. if (rc) {
  550. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  551. dmabuf, rc);
  552. goto map_kernel_fail;
  553. }
  554. }
  555. tbl.bufq[idx].kmdvaddr = kvaddr;
  556. tbl.bufq[idx].vaddr = hw_vaddr;
  557. tbl.bufq[idx].dma_buf = dmabuf;
  558. tbl.bufq[idx].len = cmd->len;
  559. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  560. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  561. sizeof(int32_t) * cmd->num_hdl);
  562. tbl.bufq[idx].is_imported = false;
  563. mutex_unlock(&tbl.bufq[idx].q_lock);
  564. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  565. cmd->out.fd = tbl.bufq[idx].fd;
  566. cmd->out.vaddr = 0;
  567. CAM_DBG(CAM_MEM,
  568. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  569. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  570. tbl.bufq[idx].len);
  571. return rc;
  572. map_kernel_fail:
  573. mutex_unlock(&tbl.bufq[idx].q_lock);
  574. map_hw_fail:
  575. cam_mem_put_slot(idx);
  576. slot_fail:
  577. dma_buf_put(dmabuf);
  578. return rc;
  579. }
  580. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  581. {
  582. int32_t idx;
  583. int rc;
  584. struct dma_buf *dmabuf;
  585. dma_addr_t hw_vaddr = 0;
  586. size_t len = 0;
  587. if (!atomic_read(&cam_mem_mgr_state)) {
  588. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  589. return -EINVAL;
  590. }
  591. if (!cmd || (cmd->fd < 0)) {
  592. CAM_ERR(CAM_MEM, "Invalid argument");
  593. return -EINVAL;
  594. }
  595. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  596. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  597. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  598. return -EINVAL;
  599. }
  600. rc = cam_mem_util_check_map_flags(cmd);
  601. if (rc) {
  602. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  603. return rc;
  604. }
  605. dmabuf = dma_buf_get(cmd->fd);
  606. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  607. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  608. return -EINVAL;
  609. }
  610. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  611. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  612. rc = cam_mem_util_map_hw_va(cmd->flags,
  613. cmd->mmu_hdls,
  614. cmd->num_hdl,
  615. cmd->fd,
  616. &hw_vaddr,
  617. &len,
  618. CAM_SMMU_REGION_IO);
  619. if (rc) {
  620. CAM_ERR(CAM_MEM,
  621. "Failed in map_hw_va, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  622. cmd->flags, cmd->fd, CAM_SMMU_REGION_IO,
  623. cmd->num_hdl, rc);
  624. goto map_fail;
  625. }
  626. }
  627. idx = cam_mem_get_slot();
  628. if (idx < 0) {
  629. rc = -ENOMEM;
  630. goto map_fail;
  631. }
  632. mutex_lock(&tbl.bufq[idx].q_lock);
  633. tbl.bufq[idx].fd = cmd->fd;
  634. tbl.bufq[idx].dma_buf = NULL;
  635. tbl.bufq[idx].flags = cmd->flags;
  636. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  637. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  638. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  639. tbl.bufq[idx].kmdvaddr = 0;
  640. if (cmd->num_hdl > 0)
  641. tbl.bufq[idx].vaddr = hw_vaddr;
  642. else
  643. tbl.bufq[idx].vaddr = 0;
  644. tbl.bufq[idx].dma_buf = dmabuf;
  645. tbl.bufq[idx].len = len;
  646. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  647. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  648. sizeof(int32_t) * cmd->num_hdl);
  649. tbl.bufq[idx].is_imported = true;
  650. mutex_unlock(&tbl.bufq[idx].q_lock);
  651. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  652. cmd->out.vaddr = 0;
  653. CAM_DBG(CAM_MEM,
  654. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  655. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  656. tbl.bufq[idx].len);
  657. return rc;
  658. map_fail:
  659. dma_buf_put(dmabuf);
  660. return rc;
  661. }
  662. static int cam_mem_util_unmap_hw_va(int32_t idx,
  663. enum cam_smmu_region_id region,
  664. enum cam_smmu_mapping_client client)
  665. {
  666. int i;
  667. uint32_t flags;
  668. int32_t *mmu_hdls;
  669. int num_hdls;
  670. int fd;
  671. int rc = 0;
  672. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  673. CAM_ERR(CAM_MEM, "Incorrect index");
  674. return -EINVAL;
  675. }
  676. flags = tbl.bufq[idx].flags;
  677. mmu_hdls = tbl.bufq[idx].hdls;
  678. num_hdls = tbl.bufq[idx].num_hdl;
  679. fd = tbl.bufq[idx].fd;
  680. CAM_DBG(CAM_MEM,
  681. "unmap_hw_va : idx=%d, fd=%x, flags=0x%x, num_hdls=%d, client=%d",
  682. idx, fd, flags, num_hdls, client);
  683. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  684. for (i = 0; i < num_hdls; i++) {
  685. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  686. if (rc < 0) {
  687. CAM_ERR(CAM_MEM,
  688. "Failed in secure unmap, i=%d, fd=%d, mmu_hdl=%d, rc=%d",
  689. i, fd, mmu_hdls[i], rc);
  690. goto unmap_end;
  691. }
  692. }
  693. } else {
  694. for (i = 0; i < num_hdls; i++) {
  695. if (client == CAM_SMMU_MAPPING_USER) {
  696. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  697. fd, region);
  698. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  699. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  700. tbl.bufq[idx].dma_buf, region);
  701. } else {
  702. CAM_ERR(CAM_MEM,
  703. "invalid caller for unmapping : %d",
  704. client);
  705. rc = -EINVAL;
  706. }
  707. if (rc < 0) {
  708. CAM_ERR(CAM_MEM,
  709. "Failed in unmap, i=%d, fd=%d, mmu_hdl=%d, region=%d, rc=%d",
  710. i, fd, mmu_hdls[i], region, rc);
  711. goto unmap_end;
  712. }
  713. }
  714. }
  715. return rc;
  716. unmap_end:
  717. CAM_ERR(CAM_MEM, "unmapping failed");
  718. return rc;
  719. }
  720. static void cam_mem_mgr_unmap_active_buf(int idx)
  721. {
  722. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  723. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  724. region = CAM_SMMU_REGION_SHARED;
  725. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  726. region = CAM_SMMU_REGION_IO;
  727. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  728. }
  729. static int cam_mem_mgr_cleanup_table(void)
  730. {
  731. int i;
  732. mutex_lock(&tbl.m_lock);
  733. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  734. if (!tbl.bufq[i].active) {
  735. CAM_DBG(CAM_MEM,
  736. "Buffer inactive at idx=%d, continuing", i);
  737. continue;
  738. } else {
  739. CAM_DBG(CAM_MEM,
  740. "Active buffer at idx=%d, possible leak needs unmapping",
  741. i);
  742. cam_mem_mgr_unmap_active_buf(i);
  743. }
  744. mutex_lock(&tbl.bufq[i].q_lock);
  745. if (tbl.bufq[i].dma_buf) {
  746. dma_buf_put(tbl.bufq[i].dma_buf);
  747. tbl.bufq[i].dma_buf = NULL;
  748. }
  749. tbl.bufq[i].fd = -1;
  750. tbl.bufq[i].flags = 0;
  751. tbl.bufq[i].buf_handle = -1;
  752. tbl.bufq[i].vaddr = 0;
  753. tbl.bufq[i].len = 0;
  754. memset(tbl.bufq[i].hdls, 0,
  755. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  756. tbl.bufq[i].num_hdl = 0;
  757. tbl.bufq[i].dma_buf = NULL;
  758. tbl.bufq[i].active = false;
  759. mutex_unlock(&tbl.bufq[i].q_lock);
  760. mutex_destroy(&tbl.bufq[i].q_lock);
  761. }
  762. bitmap_zero(tbl.bitmap, tbl.bits);
  763. /* We need to reserve slot 0 because 0 is invalid */
  764. set_bit(0, tbl.bitmap);
  765. mutex_unlock(&tbl.m_lock);
  766. return 0;
  767. }
  768. void cam_mem_mgr_deinit(void)
  769. {
  770. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  771. cam_mem_mgr_cleanup_table();
  772. mutex_lock(&tbl.m_lock);
  773. bitmap_zero(tbl.bitmap, tbl.bits);
  774. kfree(tbl.bitmap);
  775. tbl.bitmap = NULL;
  776. mutex_unlock(&tbl.m_lock);
  777. mutex_destroy(&tbl.m_lock);
  778. }
  779. static int cam_mem_util_unmap(int32_t idx,
  780. enum cam_smmu_mapping_client client)
  781. {
  782. int rc = 0;
  783. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  784. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  785. CAM_ERR(CAM_MEM, "Incorrect index");
  786. return -EINVAL;
  787. }
  788. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  789. mutex_lock(&tbl.m_lock);
  790. if ((!tbl.bufq[idx].active) &&
  791. (tbl.bufq[idx].vaddr) == 0) {
  792. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  793. idx);
  794. mutex_unlock(&tbl.m_lock);
  795. return 0;
  796. }
  797. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  798. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  799. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  800. tbl.bufq[idx].kmdvaddr);
  801. if (rc)
  802. CAM_ERR(CAM_MEM,
  803. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  804. tbl.bufq[idx].dma_buf,
  805. (void *) tbl.bufq[idx].kmdvaddr);
  806. }
  807. }
  808. /* SHARED flag gets precedence, all other flags after it */
  809. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  810. region = CAM_SMMU_REGION_SHARED;
  811. } else {
  812. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  813. region = CAM_SMMU_REGION_IO;
  814. }
  815. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  816. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  817. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  818. if (cam_mem_util_unmap_hw_va(idx, region, client))
  819. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  820. tbl.bufq[idx].dma_buf);
  821. if (client == CAM_SMMU_MAPPING_KERNEL)
  822. tbl.bufq[idx].dma_buf = NULL;
  823. }
  824. mutex_lock(&tbl.bufq[idx].q_lock);
  825. tbl.bufq[idx].flags = 0;
  826. tbl.bufq[idx].buf_handle = -1;
  827. tbl.bufq[idx].vaddr = 0;
  828. memset(tbl.bufq[idx].hdls, 0,
  829. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  830. CAM_DBG(CAM_MEM,
  831. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK",
  832. idx, tbl.bufq[idx].fd,
  833. tbl.bufq[idx].is_imported,
  834. tbl.bufq[idx].dma_buf);
  835. if (tbl.bufq[idx].dma_buf)
  836. dma_buf_put(tbl.bufq[idx].dma_buf);
  837. tbl.bufq[idx].fd = -1;
  838. tbl.bufq[idx].dma_buf = NULL;
  839. tbl.bufq[idx].is_imported = false;
  840. tbl.bufq[idx].len = 0;
  841. tbl.bufq[idx].num_hdl = 0;
  842. tbl.bufq[idx].active = false;
  843. mutex_unlock(&tbl.bufq[idx].q_lock);
  844. mutex_destroy(&tbl.bufq[idx].q_lock);
  845. clear_bit(idx, tbl.bitmap);
  846. mutex_unlock(&tbl.m_lock);
  847. return rc;
  848. }
  849. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  850. {
  851. int idx;
  852. int rc;
  853. if (!atomic_read(&cam_mem_mgr_state)) {
  854. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  855. return -EINVAL;
  856. }
  857. if (!cmd) {
  858. CAM_ERR(CAM_MEM, "Invalid argument");
  859. return -EINVAL;
  860. }
  861. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  862. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  863. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  864. idx);
  865. return -EINVAL;
  866. }
  867. if (!tbl.bufq[idx].active) {
  868. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  869. return -EINVAL;
  870. }
  871. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  872. CAM_ERR(CAM_MEM,
  873. "Released buf handle %d not matching within table %d, idx=%d",
  874. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  875. return -EINVAL;
  876. }
  877. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  878. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  879. return rc;
  880. }
  881. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  882. struct cam_mem_mgr_memory_desc *out)
  883. {
  884. struct dma_buf *buf = NULL;
  885. int ion_fd = -1;
  886. int rc = 0;
  887. uint32_t heap_id;
  888. int32_t ion_flag = 0;
  889. uintptr_t kvaddr;
  890. dma_addr_t iova = 0;
  891. size_t request_len = 0;
  892. uint32_t mem_handle;
  893. int32_t idx;
  894. int32_t smmu_hdl = 0;
  895. int32_t num_hdl = 0;
  896. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  897. if (!atomic_read(&cam_mem_mgr_state)) {
  898. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  899. return -EINVAL;
  900. }
  901. if (!inp || !out) {
  902. CAM_ERR(CAM_MEM, "Invalid params");
  903. return -EINVAL;
  904. }
  905. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  906. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  907. inp->flags & CAM_MEM_FLAG_CACHE)) {
  908. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  909. return -EINVAL;
  910. }
  911. if (inp->flags & CAM_MEM_FLAG_CACHE)
  912. ion_flag |= ION_FLAG_CACHED;
  913. else
  914. ion_flag &= ~ION_FLAG_CACHED;
  915. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  916. ION_HEAP(ION_CAMERA_HEAP_ID);
  917. rc = cam_mem_util_get_dma_buf(inp->size,
  918. heap_id,
  919. ion_flag,
  920. &buf);
  921. if (rc) {
  922. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  923. goto ion_fail;
  924. } else {
  925. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  926. }
  927. /*
  928. * we are mapping kva always here,
  929. * update flags so that we do unmap properly
  930. */
  931. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  932. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  933. if (rc) {
  934. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  935. goto map_fail;
  936. }
  937. if (!inp->smmu_hdl) {
  938. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  939. rc = -EINVAL;
  940. goto smmu_fail;
  941. }
  942. /* SHARED flag gets precedence, all other flags after it */
  943. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  944. region = CAM_SMMU_REGION_SHARED;
  945. } else {
  946. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  947. region = CAM_SMMU_REGION_IO;
  948. }
  949. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  950. buf,
  951. CAM_SMMU_MAP_RW,
  952. &iova,
  953. &request_len,
  954. region);
  955. if (rc < 0) {
  956. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  957. goto smmu_fail;
  958. }
  959. smmu_hdl = inp->smmu_hdl;
  960. num_hdl = 1;
  961. idx = cam_mem_get_slot();
  962. if (idx < 0) {
  963. rc = -ENOMEM;
  964. goto slot_fail;
  965. }
  966. mutex_lock(&tbl.bufq[idx].q_lock);
  967. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  968. tbl.bufq[idx].dma_buf = buf;
  969. tbl.bufq[idx].fd = -1;
  970. tbl.bufq[idx].flags = inp->flags;
  971. tbl.bufq[idx].buf_handle = mem_handle;
  972. tbl.bufq[idx].kmdvaddr = kvaddr;
  973. tbl.bufq[idx].vaddr = iova;
  974. tbl.bufq[idx].len = inp->size;
  975. tbl.bufq[idx].num_hdl = num_hdl;
  976. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  977. sizeof(int32_t));
  978. tbl.bufq[idx].is_imported = false;
  979. mutex_unlock(&tbl.bufq[idx].q_lock);
  980. out->kva = kvaddr;
  981. out->iova = (uint32_t)iova;
  982. out->smmu_hdl = smmu_hdl;
  983. out->mem_handle = mem_handle;
  984. out->len = inp->size;
  985. out->region = region;
  986. return rc;
  987. slot_fail:
  988. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  989. buf, region);
  990. smmu_fail:
  991. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  992. map_fail:
  993. dma_buf_put(buf);
  994. ion_fail:
  995. return rc;
  996. }
  997. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  998. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  999. {
  1000. int32_t idx;
  1001. int rc;
  1002. if (!atomic_read(&cam_mem_mgr_state)) {
  1003. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1004. return -EINVAL;
  1005. }
  1006. if (!inp) {
  1007. CAM_ERR(CAM_MEM, "Invalid argument");
  1008. return -EINVAL;
  1009. }
  1010. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1011. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1012. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1013. return -EINVAL;
  1014. }
  1015. if (!tbl.bufq[idx].active) {
  1016. if (tbl.bufq[idx].vaddr == 0) {
  1017. CAM_ERR(CAM_MEM, "buffer is released already");
  1018. return 0;
  1019. }
  1020. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1021. return -EINVAL;
  1022. }
  1023. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1024. CAM_ERR(CAM_MEM,
  1025. "Released buf handle not matching within table");
  1026. return -EINVAL;
  1027. }
  1028. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1029. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1030. return rc;
  1031. }
  1032. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1033. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1034. enum cam_smmu_region_id region,
  1035. struct cam_mem_mgr_memory_desc *out)
  1036. {
  1037. struct dma_buf *buf = NULL;
  1038. int rc = 0;
  1039. int ion_fd = -1;
  1040. uint32_t heap_id;
  1041. dma_addr_t iova = 0;
  1042. size_t request_len = 0;
  1043. uint32_t mem_handle;
  1044. int32_t idx;
  1045. int32_t smmu_hdl = 0;
  1046. int32_t num_hdl = 0;
  1047. if (!atomic_read(&cam_mem_mgr_state)) {
  1048. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1049. return -EINVAL;
  1050. }
  1051. if (!inp || !out) {
  1052. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1053. return -EINVAL;
  1054. }
  1055. if (!inp->smmu_hdl) {
  1056. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1057. return -EINVAL;
  1058. }
  1059. if (region != CAM_SMMU_REGION_SECHEAP) {
  1060. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1061. return -EINVAL;
  1062. }
  1063. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  1064. ION_HEAP(ION_CAMERA_HEAP_ID);
  1065. rc = cam_mem_util_get_dma_buf(inp->size,
  1066. heap_id,
  1067. 0,
  1068. &buf);
  1069. if (rc) {
  1070. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1071. goto ion_fail;
  1072. } else {
  1073. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1074. }
  1075. rc = cam_smmu_reserve_sec_heap(inp->smmu_hdl,
  1076. buf,
  1077. &iova,
  1078. &request_len);
  1079. if (rc) {
  1080. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1081. goto smmu_fail;
  1082. }
  1083. smmu_hdl = inp->smmu_hdl;
  1084. num_hdl = 1;
  1085. idx = cam_mem_get_slot();
  1086. if (idx < 0) {
  1087. rc = -ENOMEM;
  1088. goto slot_fail;
  1089. }
  1090. mutex_lock(&tbl.bufq[idx].q_lock);
  1091. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1092. tbl.bufq[idx].fd = -1;
  1093. tbl.bufq[idx].dma_buf = buf;
  1094. tbl.bufq[idx].flags = inp->flags;
  1095. tbl.bufq[idx].buf_handle = mem_handle;
  1096. tbl.bufq[idx].kmdvaddr = 0;
  1097. tbl.bufq[idx].vaddr = iova;
  1098. tbl.bufq[idx].len = request_len;
  1099. tbl.bufq[idx].num_hdl = num_hdl;
  1100. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1101. sizeof(int32_t));
  1102. tbl.bufq[idx].is_imported = false;
  1103. mutex_unlock(&tbl.bufq[idx].q_lock);
  1104. out->kva = 0;
  1105. out->iova = (uint32_t)iova;
  1106. out->smmu_hdl = smmu_hdl;
  1107. out->mem_handle = mem_handle;
  1108. out->len = request_len;
  1109. out->region = region;
  1110. return rc;
  1111. slot_fail:
  1112. cam_smmu_release_sec_heap(smmu_hdl);
  1113. smmu_fail:
  1114. dma_buf_put(buf);
  1115. ion_fail:
  1116. return rc;
  1117. }
  1118. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1119. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1120. {
  1121. int32_t idx;
  1122. int rc;
  1123. int32_t smmu_hdl;
  1124. if (!atomic_read(&cam_mem_mgr_state)) {
  1125. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1126. return -EINVAL;
  1127. }
  1128. if (!inp) {
  1129. CAM_ERR(CAM_MEM, "Invalid argument");
  1130. return -EINVAL;
  1131. }
  1132. if (inp->region != CAM_SMMU_REGION_SECHEAP) {
  1133. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1134. return -EINVAL;
  1135. }
  1136. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1137. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1138. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1139. return -EINVAL;
  1140. }
  1141. if (!tbl.bufq[idx].active) {
  1142. if (tbl.bufq[idx].vaddr == 0) {
  1143. CAM_ERR(CAM_MEM, "buffer is released already");
  1144. return 0;
  1145. }
  1146. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1147. return -EINVAL;
  1148. }
  1149. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1150. CAM_ERR(CAM_MEM,
  1151. "Released buf handle not matching within table");
  1152. return -EINVAL;
  1153. }
  1154. if (tbl.bufq[idx].num_hdl != 1) {
  1155. CAM_ERR(CAM_MEM,
  1156. "Sec heap region should have only one smmu hdl");
  1157. return -ENODEV;
  1158. }
  1159. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1160. sizeof(int32_t));
  1161. if (inp->smmu_hdl != smmu_hdl) {
  1162. CAM_ERR(CAM_MEM,
  1163. "Passed SMMU handle doesn't match with internal hdl");
  1164. return -ENODEV;
  1165. }
  1166. rc = cam_smmu_release_sec_heap(inp->smmu_hdl);
  1167. if (rc) {
  1168. CAM_ERR(CAM_MEM,
  1169. "Sec heap region release failed");
  1170. return -ENODEV;
  1171. }
  1172. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1173. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1174. if (rc)
  1175. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1176. return rc;
  1177. }
  1178. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);