cam_mem_mgr.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2020, The Linux Foundation. All rights reserved.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/types.h>
  7. #include <linux/mutex.h>
  8. #include <linux/slab.h>
  9. #include <linux/dma-buf.h>
  10. #include <linux/version.h>
  11. #include <linux/debugfs.h>
  12. #include "cam_compat.h"
  13. #include "cam_req_mgr_util.h"
  14. #include "cam_mem_mgr.h"
  15. #include "cam_smmu_api.h"
  16. #include "cam_debug_util.h"
  17. #include "cam_trace.h"
  18. #include "cam_common_util.h"
  19. static struct cam_mem_table tbl;
  20. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  21. static void cam_mem_mgr_print_tbl(void)
  22. {
  23. int i;
  24. uint64_t ms, tmp, hrs, min, sec;
  25. struct timespec64 *ts = NULL;
  26. struct timespec64 current_ts;
  27. ktime_get_real_ts64(&(current_ts));
  28. tmp = current_ts.tv_sec;
  29. ms = (current_ts.tv_nsec) / 1000000;
  30. sec = do_div(tmp, 60);
  31. min = do_div(tmp, 60);
  32. hrs = do_div(tmp, 24);
  33. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  34. hrs, min, sec, ms);
  35. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  36. if (tbl.bufq[i].active) {
  37. ts = &tbl.bufq[i].timestamp;
  38. tmp = ts->tv_sec;
  39. ms = (ts->tv_nsec) / 1000000;
  40. sec = do_div(tmp, 60);
  41. min = do_div(tmp, 60);
  42. hrs = do_div(tmp, 24);
  43. CAM_INFO(CAM_MEM,
  44. "%llu:%llu:%llu:%llu idx %d fd %d size %llu",
  45. hrs, min, sec, ms, i, tbl.bufq[i].fd,
  46. tbl.bufq[i].len);
  47. }
  48. }
  49. }
  50. static int cam_mem_util_get_dma_dir(uint32_t flags)
  51. {
  52. int rc = -EINVAL;
  53. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  54. rc = DMA_TO_DEVICE;
  55. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  56. rc = DMA_FROM_DEVICE;
  57. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  58. rc = DMA_BIDIRECTIONAL;
  59. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  60. rc = DMA_BIDIRECTIONAL;
  61. return rc;
  62. }
  63. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf,
  64. uintptr_t *vaddr,
  65. size_t *len)
  66. {
  67. int rc = 0;
  68. void *addr;
  69. /*
  70. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  71. * need to be called in pair to avoid stability issue.
  72. */
  73. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  74. if (rc) {
  75. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  76. return rc;
  77. }
  78. addr = dma_buf_vmap(dmabuf);
  79. if (!addr) {
  80. CAM_ERR(CAM_MEM, "kernel map fail");
  81. *vaddr = 0;
  82. *len = 0;
  83. rc = -ENOSPC;
  84. goto fail;
  85. }
  86. *vaddr = (uint64_t)addr;
  87. *len = dmabuf->size;
  88. return 0;
  89. fail:
  90. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  91. return rc;
  92. }
  93. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  94. uint64_t vaddr)
  95. {
  96. int rc = 0;
  97. if (!dmabuf || !vaddr) {
  98. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  99. return -EINVAL;
  100. }
  101. dma_buf_vunmap(dmabuf, (void *)vaddr);
  102. /*
  103. * dma_buf_begin_cpu_access() and
  104. * dma_buf_end_cpu_access() need to be called in pair
  105. * to avoid stability issue.
  106. */
  107. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  108. if (rc) {
  109. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  110. dmabuf);
  111. return rc;
  112. }
  113. return rc;
  114. }
  115. static int cam_mem_mgr_create_debug_fs(void)
  116. {
  117. int rc = 0;
  118. struct dentry *dbgfileptr = NULL;
  119. dbgfileptr = debugfs_create_dir("camera_memmgr", NULL);
  120. if (!dbgfileptr) {
  121. CAM_ERR(CAM_MEM,"DebugFS could not create directory!");
  122. rc = -ENOENT;
  123. goto end;
  124. }
  125. /* Store parent inode for cleanup in caller */
  126. tbl.dentry = dbgfileptr;
  127. dbgfileptr = debugfs_create_bool("alloc_profile_enable", 0644,
  128. tbl.dentry, &tbl.alloc_profile_enable);
  129. if (IS_ERR(dbgfileptr)) {
  130. if (PTR_ERR(dbgfileptr) == -ENODEV)
  131. CAM_WARN(CAM_MEM, "DebugFS not enabled in kernel!");
  132. else
  133. rc = PTR_ERR(dbgfileptr);
  134. }
  135. end:
  136. return rc;
  137. }
  138. int cam_mem_mgr_init(void)
  139. {
  140. int i;
  141. int bitmap_size;
  142. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  143. if (cam_smmu_need_force_alloc_cached(&tbl.force_cache_allocs)) {
  144. CAM_ERR(CAM_MEM, "Error in getting force cache alloc flag");
  145. return -EINVAL;
  146. }
  147. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  148. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  149. if (!tbl.bitmap)
  150. return -ENOMEM;
  151. tbl.bits = bitmap_size * BITS_PER_BYTE;
  152. bitmap_zero(tbl.bitmap, tbl.bits);
  153. /* We need to reserve slot 0 because 0 is invalid */
  154. set_bit(0, tbl.bitmap);
  155. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  156. tbl.bufq[i].fd = -1;
  157. tbl.bufq[i].buf_handle = -1;
  158. }
  159. mutex_init(&tbl.m_lock);
  160. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  161. cam_mem_mgr_create_debug_fs();
  162. return 0;
  163. }
  164. static int32_t cam_mem_get_slot(void)
  165. {
  166. int32_t idx;
  167. mutex_lock(&tbl.m_lock);
  168. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  169. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  170. mutex_unlock(&tbl.m_lock);
  171. return -ENOMEM;
  172. }
  173. set_bit(idx, tbl.bitmap);
  174. tbl.bufq[idx].active = true;
  175. ktime_get_real_ts64(&(tbl.bufq[idx].timestamp));
  176. mutex_init(&tbl.bufq[idx].q_lock);
  177. mutex_unlock(&tbl.m_lock);
  178. return idx;
  179. }
  180. static void cam_mem_put_slot(int32_t idx)
  181. {
  182. mutex_lock(&tbl.m_lock);
  183. mutex_lock(&tbl.bufq[idx].q_lock);
  184. tbl.bufq[idx].active = false;
  185. tbl.bufq[idx].is_internal = false;
  186. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  187. mutex_unlock(&tbl.bufq[idx].q_lock);
  188. mutex_destroy(&tbl.bufq[idx].q_lock);
  189. clear_bit(idx, tbl.bitmap);
  190. mutex_unlock(&tbl.m_lock);
  191. }
  192. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  193. dma_addr_t *iova_ptr, size_t *len_ptr)
  194. {
  195. int rc = 0, idx;
  196. *len_ptr = 0;
  197. if (!atomic_read(&cam_mem_mgr_state)) {
  198. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  199. return -EINVAL;
  200. }
  201. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  202. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  203. return -ENOENT;
  204. if (!tbl.bufq[idx].active) {
  205. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  206. idx);
  207. return -EAGAIN;
  208. }
  209. mutex_lock(&tbl.bufq[idx].q_lock);
  210. if (buf_handle != tbl.bufq[idx].buf_handle) {
  211. rc = -EINVAL;
  212. goto handle_mismatch;
  213. }
  214. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  215. rc = cam_smmu_get_stage2_iova(mmu_handle,
  216. tbl.bufq[idx].fd,
  217. iova_ptr,
  218. len_ptr);
  219. else
  220. rc = cam_smmu_get_iova(mmu_handle,
  221. tbl.bufq[idx].fd,
  222. iova_ptr,
  223. len_ptr);
  224. if (rc) {
  225. CAM_ERR(CAM_MEM,
  226. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d",
  227. buf_handle, mmu_handle, tbl.bufq[idx].fd);
  228. goto handle_mismatch;
  229. }
  230. CAM_DBG(CAM_MEM,
  231. "handle:0x%x fd:%d iova_ptr:%pK len_ptr:%llu",
  232. mmu_handle, tbl.bufq[idx].fd, iova_ptr, *len_ptr);
  233. handle_mismatch:
  234. mutex_unlock(&tbl.bufq[idx].q_lock);
  235. return rc;
  236. }
  237. EXPORT_SYMBOL(cam_mem_get_io_buf);
  238. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  239. {
  240. int idx;
  241. if (!atomic_read(&cam_mem_mgr_state)) {
  242. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  243. return -EINVAL;
  244. }
  245. if (!atomic_read(&cam_mem_mgr_state)) {
  246. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  247. return -EINVAL;
  248. }
  249. if (!buf_handle || !vaddr_ptr || !len)
  250. return -EINVAL;
  251. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  252. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  253. return -EINVAL;
  254. if (!tbl.bufq[idx].active) {
  255. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  256. idx);
  257. return -EPERM;
  258. }
  259. if (buf_handle != tbl.bufq[idx].buf_handle)
  260. return -EINVAL;
  261. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  262. return -EINVAL;
  263. if (tbl.bufq[idx].kmdvaddr) {
  264. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  265. *len = tbl.bufq[idx].len;
  266. } else {
  267. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  268. buf_handle);
  269. return -EINVAL;
  270. }
  271. return 0;
  272. }
  273. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  274. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  275. {
  276. int rc = 0, idx;
  277. uint32_t cache_dir;
  278. unsigned long dmabuf_flag = 0;
  279. if (!atomic_read(&cam_mem_mgr_state)) {
  280. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  281. return -EINVAL;
  282. }
  283. if (!cmd)
  284. return -EINVAL;
  285. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  286. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  287. return -EINVAL;
  288. mutex_lock(&tbl.bufq[idx].q_lock);
  289. if (!tbl.bufq[idx].active) {
  290. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  291. idx);
  292. rc = -EINVAL;
  293. goto end;
  294. }
  295. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  296. rc = -EINVAL;
  297. goto end;
  298. }
  299. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  300. if (rc) {
  301. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  302. goto end;
  303. }
  304. if (dmabuf_flag & ION_FLAG_CACHED) {
  305. switch (cmd->mem_cache_ops) {
  306. case CAM_MEM_CLEAN_CACHE:
  307. cache_dir = DMA_TO_DEVICE;
  308. break;
  309. case CAM_MEM_INV_CACHE:
  310. cache_dir = DMA_FROM_DEVICE;
  311. break;
  312. case CAM_MEM_CLEAN_INV_CACHE:
  313. cache_dir = DMA_BIDIRECTIONAL;
  314. break;
  315. default:
  316. CAM_ERR(CAM_MEM,
  317. "invalid cache ops :%d", cmd->mem_cache_ops);
  318. rc = -EINVAL;
  319. goto end;
  320. }
  321. } else {
  322. CAM_DBG(CAM_MEM, "BUF is not cached");
  323. goto end;
  324. }
  325. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  326. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  327. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  328. if (rc) {
  329. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  330. goto end;
  331. }
  332. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  333. cache_dir);
  334. if (rc) {
  335. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  336. goto end;
  337. }
  338. end:
  339. mutex_unlock(&tbl.bufq[idx].q_lock);
  340. return rc;
  341. }
  342. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  343. static int cam_mem_util_get_dma_buf(size_t len,
  344. unsigned int heap_id_mask,
  345. unsigned int flags,
  346. struct dma_buf **buf)
  347. {
  348. int rc = 0;
  349. if (!buf) {
  350. CAM_ERR(CAM_MEM, "Invalid params");
  351. return -EINVAL;
  352. }
  353. if (tbl.force_cache_allocs && (!(flags & ION_FLAG_SECURE)))
  354. flags |= ION_FLAG_CACHED;
  355. *buf = ion_alloc(len, heap_id_mask, flags);
  356. if (IS_ERR_OR_NULL(*buf))
  357. return -ENOMEM;
  358. return rc;
  359. }
  360. static int cam_mem_util_get_dma_buf_fd(size_t len,
  361. size_t align,
  362. unsigned int heap_id_mask,
  363. unsigned int flags,
  364. struct dma_buf **buf,
  365. int *fd)
  366. {
  367. struct dma_buf *dmabuf = NULL;
  368. int rc = 0;
  369. struct timespec64 ts1, ts2;
  370. long microsec = 0;
  371. if (!buf || !fd) {
  372. CAM_ERR(CAM_MEM, "Invalid params, buf=%pK, fd=%pK", buf, fd);
  373. return -EINVAL;
  374. }
  375. if (tbl.alloc_profile_enable)
  376. CAM_GET_TIMESTAMP(ts1);
  377. if (tbl.force_cache_allocs && (!(flags & ION_FLAG_SECURE)))
  378. flags |= ION_FLAG_CACHED;
  379. *buf = ion_alloc(len, heap_id_mask, flags);
  380. if (IS_ERR_OR_NULL(*buf))
  381. return -ENOMEM;
  382. *fd = dma_buf_fd(*buf, O_CLOEXEC);
  383. if (*fd < 0) {
  384. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  385. rc = -EINVAL;
  386. goto get_fd_fail;
  387. }
  388. /*
  389. * increment the ref count so that ref count becomes 2 here
  390. * when we close fd, refcount becomes 1 and when we do
  391. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  392. */
  393. dmabuf = dma_buf_get(*fd);
  394. if (IS_ERR_OR_NULL(dmabuf)) {
  395. CAM_ERR(CAM_MEM, "dma_buf_get failed, *fd=%d", *fd);
  396. rc = -EINVAL;
  397. }
  398. if (tbl.alloc_profile_enable) {
  399. CAM_GET_TIMESTAMP(ts2);
  400. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  401. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  402. len, microsec);
  403. }
  404. return rc;
  405. get_fd_fail:
  406. dma_buf_put(*buf);
  407. return rc;
  408. }
  409. static int cam_mem_util_ion_alloc(struct cam_mem_mgr_alloc_cmd *cmd,
  410. struct dma_buf **dmabuf,
  411. int *fd)
  412. {
  413. uint32_t heap_id;
  414. uint32_t ion_flag = 0;
  415. int rc;
  416. if ((cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  417. (cmd->flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  418. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  419. ion_flag |=
  420. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  421. } else if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  422. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  423. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  424. } else {
  425. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  426. ION_HEAP(ION_CAMERA_HEAP_ID);
  427. }
  428. if (cmd->flags & CAM_MEM_FLAG_CACHE)
  429. ion_flag |= ION_FLAG_CACHED;
  430. else
  431. ion_flag &= ~ION_FLAG_CACHED;
  432. rc = cam_mem_util_get_dma_buf_fd(cmd->len,
  433. cmd->align,
  434. heap_id,
  435. ion_flag,
  436. dmabuf,
  437. fd);
  438. return rc;
  439. }
  440. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd *cmd)
  441. {
  442. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  443. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  444. CAM_MEM_MMU_MAX_HANDLE);
  445. return -EINVAL;
  446. }
  447. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  448. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  449. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  450. return -EINVAL;
  451. }
  452. return 0;
  453. }
  454. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd *cmd)
  455. {
  456. if (!cmd->flags) {
  457. CAM_ERR(CAM_MEM, "Invalid flags");
  458. return -EINVAL;
  459. }
  460. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  461. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  462. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  463. return -EINVAL;
  464. }
  465. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  466. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  467. CAM_ERR(CAM_MEM,
  468. "Kernel mapping in secure mode not allowed, flags=0x%x",
  469. cmd->flags);
  470. return -EINVAL;
  471. }
  472. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  473. CAM_ERR(CAM_MEM,
  474. "Shared memory buffers are not allowed to be mapped");
  475. return -EINVAL;
  476. }
  477. return 0;
  478. }
  479. static int cam_mem_util_map_hw_va(uint32_t flags,
  480. int32_t *mmu_hdls,
  481. int32_t num_hdls,
  482. int fd,
  483. dma_addr_t *hw_vaddr,
  484. size_t *len,
  485. enum cam_smmu_region_id region,
  486. bool is_internal)
  487. {
  488. int i;
  489. int rc = -1;
  490. int dir = cam_mem_util_get_dma_dir(flags);
  491. bool dis_delayed_unmap = false;
  492. if (dir < 0) {
  493. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  494. return dir;
  495. }
  496. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  497. dis_delayed_unmap = true;
  498. CAM_DBG(CAM_MEM,
  499. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  500. fd, flags, dir, num_hdls);
  501. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  502. for (i = 0; i < num_hdls; i++) {
  503. rc = cam_smmu_map_stage2_iova(mmu_hdls[i],
  504. fd,
  505. dir,
  506. hw_vaddr,
  507. len);
  508. if (rc < 0) {
  509. CAM_ERR(CAM_MEM,
  510. "Failed to securely map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  511. i, fd, dir, mmu_hdls[i], rc);
  512. goto multi_map_fail;
  513. }
  514. }
  515. } else {
  516. for (i = 0; i < num_hdls; i++) {
  517. rc = cam_smmu_map_user_iova(mmu_hdls[i],
  518. fd,
  519. dis_delayed_unmap,
  520. dir,
  521. (dma_addr_t *)hw_vaddr,
  522. len,
  523. region,
  524. is_internal);
  525. if (rc < 0) {
  526. CAM_ERR(CAM_MEM,
  527. "Failed to map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, region=%d, rc=%d",
  528. i, fd, dir, mmu_hdls[i], region, rc);
  529. goto multi_map_fail;
  530. }
  531. }
  532. }
  533. return rc;
  534. multi_map_fail:
  535. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  536. for (--i; i >= 0; i--)
  537. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  538. else
  539. for (--i; i >= 0; i--)
  540. cam_smmu_unmap_user_iova(mmu_hdls[i],
  541. fd,
  542. CAM_SMMU_REGION_IO);
  543. return rc;
  544. }
  545. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd *cmd)
  546. {
  547. int rc;
  548. int32_t idx;
  549. struct dma_buf *dmabuf = NULL;
  550. int fd = -1;
  551. dma_addr_t hw_vaddr = 0;
  552. size_t len;
  553. uintptr_t kvaddr = 0;
  554. size_t klen;
  555. if (!atomic_read(&cam_mem_mgr_state)) {
  556. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  557. return -EINVAL;
  558. }
  559. if (!cmd) {
  560. CAM_ERR(CAM_MEM, " Invalid argument");
  561. return -EINVAL;
  562. }
  563. len = cmd->len;
  564. rc = cam_mem_util_check_alloc_flags(cmd);
  565. if (rc) {
  566. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  567. cmd->flags, rc);
  568. return rc;
  569. }
  570. rc = cam_mem_util_ion_alloc(cmd,
  571. &dmabuf,
  572. &fd);
  573. if (rc) {
  574. CAM_ERR(CAM_MEM,
  575. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  576. cmd->len, cmd->align, cmd->flags, cmd->num_hdl);
  577. cam_mem_mgr_print_tbl();
  578. return rc;
  579. }
  580. idx = cam_mem_get_slot();
  581. if (idx < 0) {
  582. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  583. rc = -ENOMEM;
  584. goto slot_fail;
  585. }
  586. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  587. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  588. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  589. enum cam_smmu_region_id region;
  590. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  591. region = CAM_SMMU_REGION_IO;
  592. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  593. region = CAM_SMMU_REGION_SHARED;
  594. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  595. region = CAM_SMMU_REGION_SECHEAP;
  596. rc = cam_mem_util_map_hw_va(cmd->flags,
  597. cmd->mmu_hdls,
  598. cmd->num_hdl,
  599. fd,
  600. &hw_vaddr,
  601. &len,
  602. region,
  603. true);
  604. if (rc) {
  605. CAM_ERR(CAM_MEM,
  606. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  607. len, cmd->flags,
  608. fd, region, cmd->num_hdl, rc);
  609. if (rc == -EALREADY) {
  610. if ((size_t)dmabuf->size != len)
  611. rc = -EBADR;
  612. cam_mem_mgr_print_tbl();
  613. }
  614. goto map_hw_fail;
  615. }
  616. }
  617. mutex_lock(&tbl.bufq[idx].q_lock);
  618. tbl.bufq[idx].fd = fd;
  619. tbl.bufq[idx].dma_buf = NULL;
  620. tbl.bufq[idx].flags = cmd->flags;
  621. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  622. tbl.bufq[idx].is_internal = true;
  623. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  624. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  625. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  626. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  627. if (rc) {
  628. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  629. dmabuf, rc);
  630. goto map_kernel_fail;
  631. }
  632. }
  633. if (cmd->flags & CAM_MEM_FLAG_KMD_DEBUG_FLAG)
  634. tbl.dbg_buf_idx = idx;
  635. tbl.bufq[idx].kmdvaddr = kvaddr;
  636. tbl.bufq[idx].vaddr = hw_vaddr;
  637. tbl.bufq[idx].dma_buf = dmabuf;
  638. tbl.bufq[idx].len = cmd->len;
  639. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  640. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  641. sizeof(int32_t) * cmd->num_hdl);
  642. tbl.bufq[idx].is_imported = false;
  643. mutex_unlock(&tbl.bufq[idx].q_lock);
  644. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  645. cmd->out.fd = tbl.bufq[idx].fd;
  646. cmd->out.vaddr = 0;
  647. CAM_DBG(CAM_MEM,
  648. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  649. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  650. tbl.bufq[idx].len);
  651. return rc;
  652. map_kernel_fail:
  653. mutex_unlock(&tbl.bufq[idx].q_lock);
  654. map_hw_fail:
  655. cam_mem_put_slot(idx);
  656. slot_fail:
  657. dma_buf_put(dmabuf);
  658. return rc;
  659. }
  660. static bool cam_mem_util_is_map_internal(int32_t fd)
  661. {
  662. uint32_t i;
  663. bool is_internal = false;
  664. mutex_lock(&tbl.m_lock);
  665. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  666. if (tbl.bufq[i].fd == fd) {
  667. is_internal = tbl.bufq[i].is_internal;
  668. break;
  669. }
  670. }
  671. mutex_unlock(&tbl.m_lock);
  672. return is_internal;
  673. }
  674. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd *cmd)
  675. {
  676. int32_t idx;
  677. int rc;
  678. struct dma_buf *dmabuf;
  679. dma_addr_t hw_vaddr = 0;
  680. size_t len = 0;
  681. bool is_internal = false;
  682. if (!atomic_read(&cam_mem_mgr_state)) {
  683. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  684. return -EINVAL;
  685. }
  686. if (!cmd || (cmd->fd < 0)) {
  687. CAM_ERR(CAM_MEM, "Invalid argument");
  688. return -EINVAL;
  689. }
  690. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  691. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  692. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  693. return -EINVAL;
  694. }
  695. rc = cam_mem_util_check_map_flags(cmd);
  696. if (rc) {
  697. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  698. return rc;
  699. }
  700. dmabuf = dma_buf_get(cmd->fd);
  701. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  702. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  703. return -EINVAL;
  704. }
  705. is_internal = cam_mem_util_is_map_internal(cmd->fd);
  706. idx = cam_mem_get_slot();
  707. if (idx < 0) {
  708. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d, fd=%d",
  709. idx, cmd->fd);
  710. rc = -ENOMEM;
  711. goto slot_fail;
  712. }
  713. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  714. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  715. rc = cam_mem_util_map_hw_va(cmd->flags,
  716. cmd->mmu_hdls,
  717. cmd->num_hdl,
  718. cmd->fd,
  719. &hw_vaddr,
  720. &len,
  721. CAM_SMMU_REGION_IO,
  722. is_internal);
  723. if (rc) {
  724. CAM_ERR(CAM_MEM,
  725. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  726. cmd->flags, cmd->fd, len,
  727. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  728. if (rc == -EALREADY) {
  729. if ((size_t)dmabuf->size != len) {
  730. rc = -EBADR;
  731. cam_mem_mgr_print_tbl();
  732. }
  733. }
  734. goto map_fail;
  735. }
  736. }
  737. mutex_lock(&tbl.bufq[idx].q_lock);
  738. tbl.bufq[idx].fd = cmd->fd;
  739. tbl.bufq[idx].dma_buf = NULL;
  740. tbl.bufq[idx].flags = cmd->flags;
  741. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  742. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  743. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  744. tbl.bufq[idx].kmdvaddr = 0;
  745. if (cmd->num_hdl > 0)
  746. tbl.bufq[idx].vaddr = hw_vaddr;
  747. else
  748. tbl.bufq[idx].vaddr = 0;
  749. tbl.bufq[idx].dma_buf = dmabuf;
  750. tbl.bufq[idx].len = len;
  751. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  752. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  753. sizeof(int32_t) * cmd->num_hdl);
  754. tbl.bufq[idx].is_imported = true;
  755. tbl.bufq[idx].is_internal = is_internal;
  756. mutex_unlock(&tbl.bufq[idx].q_lock);
  757. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  758. cmd->out.vaddr = 0;
  759. cmd->out.size = (uint32_t)len;
  760. CAM_DBG(CAM_MEM,
  761. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu",
  762. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  763. tbl.bufq[idx].len);
  764. return rc;
  765. map_fail:
  766. cam_mem_put_slot(idx);
  767. slot_fail:
  768. dma_buf_put(dmabuf);
  769. return rc;
  770. }
  771. static int cam_mem_util_unmap_hw_va(int32_t idx,
  772. enum cam_smmu_region_id region,
  773. enum cam_smmu_mapping_client client)
  774. {
  775. int i;
  776. uint32_t flags;
  777. int32_t *mmu_hdls;
  778. int num_hdls;
  779. int fd;
  780. int rc = 0;
  781. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  782. CAM_ERR(CAM_MEM, "Incorrect index");
  783. return -EINVAL;
  784. }
  785. flags = tbl.bufq[idx].flags;
  786. mmu_hdls = tbl.bufq[idx].hdls;
  787. num_hdls = tbl.bufq[idx].num_hdl;
  788. fd = tbl.bufq[idx].fd;
  789. CAM_DBG(CAM_MEM,
  790. "unmap_hw_va : idx=%d, fd=%x, flags=0x%x, num_hdls=%d, client=%d",
  791. idx, fd, flags, num_hdls, client);
  792. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  793. for (i = 0; i < num_hdls; i++) {
  794. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd);
  795. if (rc < 0) {
  796. CAM_ERR(CAM_MEM,
  797. "Failed in secure unmap, i=%d, fd=%d, mmu_hdl=%d, rc=%d",
  798. i, fd, mmu_hdls[i], rc);
  799. goto unmap_end;
  800. }
  801. }
  802. } else {
  803. for (i = 0; i < num_hdls; i++) {
  804. if (client == CAM_SMMU_MAPPING_USER) {
  805. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  806. fd, region);
  807. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  808. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  809. tbl.bufq[idx].dma_buf, region);
  810. } else {
  811. CAM_ERR(CAM_MEM,
  812. "invalid caller for unmapping : %d",
  813. client);
  814. rc = -EINVAL;
  815. }
  816. if (rc < 0) {
  817. CAM_ERR(CAM_MEM,
  818. "Failed in unmap, i=%d, fd=%d, mmu_hdl=%d, region=%d, rc=%d",
  819. i, fd, mmu_hdls[i], region, rc);
  820. goto unmap_end;
  821. }
  822. }
  823. }
  824. return rc;
  825. unmap_end:
  826. CAM_ERR(CAM_MEM, "unmapping failed");
  827. return rc;
  828. }
  829. static void cam_mem_mgr_unmap_active_buf(int idx)
  830. {
  831. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  832. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  833. region = CAM_SMMU_REGION_SHARED;
  834. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  835. region = CAM_SMMU_REGION_IO;
  836. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  837. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS)
  838. cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  839. tbl.bufq[idx].kmdvaddr);
  840. }
  841. static int cam_mem_mgr_cleanup_table(void)
  842. {
  843. int i;
  844. mutex_lock(&tbl.m_lock);
  845. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  846. if (!tbl.bufq[i].active) {
  847. CAM_DBG(CAM_MEM,
  848. "Buffer inactive at idx=%d, continuing", i);
  849. continue;
  850. } else {
  851. CAM_DBG(CAM_MEM,
  852. "Active buffer at idx=%d, possible leak needs unmapping",
  853. i);
  854. cam_mem_mgr_unmap_active_buf(i);
  855. }
  856. mutex_lock(&tbl.bufq[i].q_lock);
  857. if (tbl.bufq[i].dma_buf) {
  858. dma_buf_put(tbl.bufq[i].dma_buf);
  859. tbl.bufq[i].dma_buf = NULL;
  860. }
  861. tbl.bufq[i].fd = -1;
  862. tbl.bufq[i].flags = 0;
  863. tbl.bufq[i].buf_handle = -1;
  864. tbl.bufq[i].vaddr = 0;
  865. tbl.bufq[i].len = 0;
  866. memset(tbl.bufq[i].hdls, 0,
  867. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  868. tbl.bufq[i].num_hdl = 0;
  869. tbl.bufq[i].dma_buf = NULL;
  870. tbl.bufq[i].active = false;
  871. tbl.bufq[i].is_internal = false;
  872. mutex_unlock(&tbl.bufq[i].q_lock);
  873. mutex_destroy(&tbl.bufq[i].q_lock);
  874. }
  875. bitmap_zero(tbl.bitmap, tbl.bits);
  876. /* We need to reserve slot 0 because 0 is invalid */
  877. set_bit(0, tbl.bitmap);
  878. mutex_unlock(&tbl.m_lock);
  879. return 0;
  880. }
  881. void cam_mem_mgr_deinit(void)
  882. {
  883. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  884. cam_mem_mgr_cleanup_table();
  885. debugfs_remove_recursive(tbl.dentry);
  886. mutex_lock(&tbl.m_lock);
  887. bitmap_zero(tbl.bitmap, tbl.bits);
  888. kfree(tbl.bitmap);
  889. tbl.bitmap = NULL;
  890. tbl.dbg_buf_idx = -1;
  891. mutex_unlock(&tbl.m_lock);
  892. mutex_destroy(&tbl.m_lock);
  893. }
  894. static int cam_mem_util_unmap(int32_t idx,
  895. enum cam_smmu_mapping_client client)
  896. {
  897. int rc = 0;
  898. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  899. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  900. CAM_ERR(CAM_MEM, "Incorrect index");
  901. return -EINVAL;
  902. }
  903. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  904. mutex_lock(&tbl.m_lock);
  905. if ((!tbl.bufq[idx].active) &&
  906. (tbl.bufq[idx].vaddr) == 0) {
  907. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  908. idx);
  909. mutex_unlock(&tbl.m_lock);
  910. return 0;
  911. }
  912. /* Deactivate the buffer queue to prevent multiple unmap */
  913. mutex_lock(&tbl.bufq[idx].q_lock);
  914. tbl.bufq[idx].active = false;
  915. tbl.bufq[idx].vaddr = 0;
  916. mutex_unlock(&tbl.bufq[idx].q_lock);
  917. mutex_unlock(&tbl.m_lock);
  918. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  919. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  920. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  921. tbl.bufq[idx].kmdvaddr);
  922. if (rc)
  923. CAM_ERR(CAM_MEM,
  924. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  925. tbl.bufq[idx].dma_buf,
  926. (void *) tbl.bufq[idx].kmdvaddr);
  927. }
  928. }
  929. /* SHARED flag gets precedence, all other flags after it */
  930. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  931. region = CAM_SMMU_REGION_SHARED;
  932. } else {
  933. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  934. region = CAM_SMMU_REGION_IO;
  935. }
  936. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  937. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  938. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  939. if (cam_mem_util_unmap_hw_va(idx, region, client))
  940. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  941. tbl.bufq[idx].dma_buf);
  942. if (client == CAM_SMMU_MAPPING_KERNEL)
  943. tbl.bufq[idx].dma_buf = NULL;
  944. }
  945. mutex_lock(&tbl.m_lock);
  946. mutex_lock(&tbl.bufq[idx].q_lock);
  947. tbl.bufq[idx].flags = 0;
  948. tbl.bufq[idx].buf_handle = -1;
  949. memset(tbl.bufq[idx].hdls, 0,
  950. sizeof(int32_t) * CAM_MEM_MMU_MAX_HANDLE);
  951. CAM_DBG(CAM_MEM,
  952. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK",
  953. idx, tbl.bufq[idx].fd,
  954. tbl.bufq[idx].is_imported,
  955. tbl.bufq[idx].dma_buf);
  956. if (tbl.bufq[idx].dma_buf)
  957. dma_buf_put(tbl.bufq[idx].dma_buf);
  958. tbl.bufq[idx].fd = -1;
  959. tbl.bufq[idx].dma_buf = NULL;
  960. tbl.bufq[idx].is_imported = false;
  961. tbl.bufq[idx].is_internal = false;
  962. tbl.bufq[idx].len = 0;
  963. tbl.bufq[idx].num_hdl = 0;
  964. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  965. mutex_unlock(&tbl.bufq[idx].q_lock);
  966. mutex_destroy(&tbl.bufq[idx].q_lock);
  967. clear_bit(idx, tbl.bitmap);
  968. mutex_unlock(&tbl.m_lock);
  969. return rc;
  970. }
  971. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  972. {
  973. int idx;
  974. int rc;
  975. if (!atomic_read(&cam_mem_mgr_state)) {
  976. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  977. return -EINVAL;
  978. }
  979. if (!cmd) {
  980. CAM_ERR(CAM_MEM, "Invalid argument");
  981. return -EINVAL;
  982. }
  983. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  984. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  985. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  986. idx);
  987. return -EINVAL;
  988. }
  989. if (!tbl.bufq[idx].active) {
  990. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  991. return -EINVAL;
  992. }
  993. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  994. CAM_ERR(CAM_MEM,
  995. "Released buf handle %d not matching within table %d, idx=%d",
  996. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  997. return -EINVAL;
  998. }
  999. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  1000. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  1001. return rc;
  1002. }
  1003. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  1004. struct cam_mem_mgr_memory_desc *out)
  1005. {
  1006. struct dma_buf *buf = NULL;
  1007. int ion_fd = -1;
  1008. int rc = 0;
  1009. uint32_t heap_id;
  1010. int32_t ion_flag = 0;
  1011. uintptr_t kvaddr;
  1012. dma_addr_t iova = 0;
  1013. size_t request_len = 0;
  1014. uint32_t mem_handle;
  1015. int32_t idx;
  1016. int32_t smmu_hdl = 0;
  1017. int32_t num_hdl = 0;
  1018. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1019. if (!atomic_read(&cam_mem_mgr_state)) {
  1020. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1021. return -EINVAL;
  1022. }
  1023. if (!inp || !out) {
  1024. CAM_ERR(CAM_MEM, "Invalid params");
  1025. return -EINVAL;
  1026. }
  1027. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1028. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1029. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1030. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1031. return -EINVAL;
  1032. }
  1033. if (inp->flags & CAM_MEM_FLAG_CACHE)
  1034. ion_flag |= ION_FLAG_CACHED;
  1035. else
  1036. ion_flag &= ~ION_FLAG_CACHED;
  1037. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  1038. ION_HEAP(ION_CAMERA_HEAP_ID);
  1039. rc = cam_mem_util_get_dma_buf(inp->size,
  1040. heap_id,
  1041. ion_flag,
  1042. &buf);
  1043. if (rc) {
  1044. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1045. goto ion_fail;
  1046. } else {
  1047. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1048. }
  1049. /*
  1050. * we are mapping kva always here,
  1051. * update flags so that we do unmap properly
  1052. */
  1053. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1054. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1055. if (rc) {
  1056. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1057. goto map_fail;
  1058. }
  1059. if (!inp->smmu_hdl) {
  1060. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1061. rc = -EINVAL;
  1062. goto smmu_fail;
  1063. }
  1064. /* SHARED flag gets precedence, all other flags after it */
  1065. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1066. region = CAM_SMMU_REGION_SHARED;
  1067. } else {
  1068. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1069. region = CAM_SMMU_REGION_IO;
  1070. }
  1071. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1072. buf,
  1073. CAM_SMMU_MAP_RW,
  1074. &iova,
  1075. &request_len,
  1076. region);
  1077. if (rc < 0) {
  1078. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1079. goto smmu_fail;
  1080. }
  1081. smmu_hdl = inp->smmu_hdl;
  1082. num_hdl = 1;
  1083. idx = cam_mem_get_slot();
  1084. if (idx < 0) {
  1085. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1086. rc = -ENOMEM;
  1087. goto slot_fail;
  1088. }
  1089. mutex_lock(&tbl.bufq[idx].q_lock);
  1090. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1091. tbl.bufq[idx].dma_buf = buf;
  1092. tbl.bufq[idx].fd = -1;
  1093. tbl.bufq[idx].flags = inp->flags;
  1094. tbl.bufq[idx].buf_handle = mem_handle;
  1095. tbl.bufq[idx].kmdvaddr = kvaddr;
  1096. tbl.bufq[idx].vaddr = iova;
  1097. tbl.bufq[idx].len = inp->size;
  1098. tbl.bufq[idx].num_hdl = num_hdl;
  1099. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1100. sizeof(int32_t));
  1101. tbl.bufq[idx].is_imported = false;
  1102. mutex_unlock(&tbl.bufq[idx].q_lock);
  1103. out->kva = kvaddr;
  1104. out->iova = (uint32_t)iova;
  1105. out->smmu_hdl = smmu_hdl;
  1106. out->mem_handle = mem_handle;
  1107. out->len = inp->size;
  1108. out->region = region;
  1109. return rc;
  1110. slot_fail:
  1111. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1112. buf, region);
  1113. smmu_fail:
  1114. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1115. map_fail:
  1116. dma_buf_put(buf);
  1117. ion_fail:
  1118. return rc;
  1119. }
  1120. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1121. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1122. {
  1123. int32_t idx;
  1124. int rc;
  1125. if (!atomic_read(&cam_mem_mgr_state)) {
  1126. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1127. return -EINVAL;
  1128. }
  1129. if (!inp) {
  1130. CAM_ERR(CAM_MEM, "Invalid argument");
  1131. return -EINVAL;
  1132. }
  1133. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1134. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1135. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1136. return -EINVAL;
  1137. }
  1138. if (!tbl.bufq[idx].active) {
  1139. if (tbl.bufq[idx].vaddr == 0) {
  1140. CAM_ERR(CAM_MEM, "buffer is released already");
  1141. return 0;
  1142. }
  1143. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1144. return -EINVAL;
  1145. }
  1146. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1147. CAM_ERR(CAM_MEM,
  1148. "Released buf handle not matching within table");
  1149. return -EINVAL;
  1150. }
  1151. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1152. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1153. return rc;
  1154. }
  1155. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1156. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1157. enum cam_smmu_region_id region,
  1158. struct cam_mem_mgr_memory_desc *out)
  1159. {
  1160. struct dma_buf *buf = NULL;
  1161. int rc = 0;
  1162. int ion_fd = -1;
  1163. uint32_t heap_id;
  1164. dma_addr_t iova = 0;
  1165. size_t request_len = 0;
  1166. uint32_t mem_handle;
  1167. int32_t idx;
  1168. int32_t smmu_hdl = 0;
  1169. int32_t num_hdl = 0;
  1170. if (!atomic_read(&cam_mem_mgr_state)) {
  1171. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1172. return -EINVAL;
  1173. }
  1174. if (!inp || !out) {
  1175. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1176. return -EINVAL;
  1177. }
  1178. if (!inp->smmu_hdl) {
  1179. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1180. return -EINVAL;
  1181. }
  1182. if (region != CAM_SMMU_REGION_SECHEAP) {
  1183. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1184. return -EINVAL;
  1185. }
  1186. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  1187. ION_HEAP(ION_CAMERA_HEAP_ID);
  1188. rc = cam_mem_util_get_dma_buf(inp->size,
  1189. heap_id,
  1190. 0,
  1191. &buf);
  1192. if (rc) {
  1193. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1194. goto ion_fail;
  1195. } else {
  1196. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1197. }
  1198. rc = cam_smmu_reserve_sec_heap(inp->smmu_hdl,
  1199. buf,
  1200. &iova,
  1201. &request_len);
  1202. if (rc) {
  1203. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1204. goto smmu_fail;
  1205. }
  1206. smmu_hdl = inp->smmu_hdl;
  1207. num_hdl = 1;
  1208. idx = cam_mem_get_slot();
  1209. if (idx < 0) {
  1210. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1211. rc = -ENOMEM;
  1212. goto slot_fail;
  1213. }
  1214. mutex_lock(&tbl.bufq[idx].q_lock);
  1215. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1216. tbl.bufq[idx].fd = -1;
  1217. tbl.bufq[idx].dma_buf = buf;
  1218. tbl.bufq[idx].flags = inp->flags;
  1219. tbl.bufq[idx].buf_handle = mem_handle;
  1220. tbl.bufq[idx].kmdvaddr = 0;
  1221. tbl.bufq[idx].vaddr = iova;
  1222. tbl.bufq[idx].len = request_len;
  1223. tbl.bufq[idx].num_hdl = num_hdl;
  1224. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1225. sizeof(int32_t));
  1226. tbl.bufq[idx].is_imported = false;
  1227. mutex_unlock(&tbl.bufq[idx].q_lock);
  1228. out->kva = 0;
  1229. out->iova = (uint32_t)iova;
  1230. out->smmu_hdl = smmu_hdl;
  1231. out->mem_handle = mem_handle;
  1232. out->len = request_len;
  1233. out->region = region;
  1234. return rc;
  1235. slot_fail:
  1236. cam_smmu_release_sec_heap(smmu_hdl);
  1237. smmu_fail:
  1238. dma_buf_put(buf);
  1239. ion_fail:
  1240. return rc;
  1241. }
  1242. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1243. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1244. {
  1245. int32_t idx;
  1246. int rc;
  1247. int32_t smmu_hdl;
  1248. if (!atomic_read(&cam_mem_mgr_state)) {
  1249. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1250. return -EINVAL;
  1251. }
  1252. if (!inp) {
  1253. CAM_ERR(CAM_MEM, "Invalid argument");
  1254. return -EINVAL;
  1255. }
  1256. if (inp->region != CAM_SMMU_REGION_SECHEAP) {
  1257. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1258. return -EINVAL;
  1259. }
  1260. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1261. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1262. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1263. return -EINVAL;
  1264. }
  1265. if (!tbl.bufq[idx].active) {
  1266. if (tbl.bufq[idx].vaddr == 0) {
  1267. CAM_ERR(CAM_MEM, "buffer is released already");
  1268. return 0;
  1269. }
  1270. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1271. return -EINVAL;
  1272. }
  1273. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1274. CAM_ERR(CAM_MEM,
  1275. "Released buf handle not matching within table");
  1276. return -EINVAL;
  1277. }
  1278. if (tbl.bufq[idx].num_hdl != 1) {
  1279. CAM_ERR(CAM_MEM,
  1280. "Sec heap region should have only one smmu hdl");
  1281. return -ENODEV;
  1282. }
  1283. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1284. sizeof(int32_t));
  1285. if (inp->smmu_hdl != smmu_hdl) {
  1286. CAM_ERR(CAM_MEM,
  1287. "Passed SMMU handle doesn't match with internal hdl");
  1288. return -ENODEV;
  1289. }
  1290. rc = cam_smmu_release_sec_heap(inp->smmu_hdl);
  1291. if (rc) {
  1292. CAM_ERR(CAM_MEM,
  1293. "Sec heap region release failed");
  1294. return -ENODEV;
  1295. }
  1296. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1297. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1298. if (rc)
  1299. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1300. return rc;
  1301. }
  1302. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);