cam_mem_mgr.c 59 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2016-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #include <linux/module.h>
  7. #include <linux/types.h>
  8. #include <linux/mutex.h>
  9. #include <linux/slab.h>
  10. #include <linux/dma-buf.h>
  11. #include <linux/version.h>
  12. #include <linux/debugfs.h>
  13. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  14. #include <linux/mem-buf.h>
  15. #include <soc/qcom/secure_buffer.h>
  16. #endif
  17. #include "cam_compat.h"
  18. #include "cam_req_mgr_util.h"
  19. #include "cam_mem_mgr.h"
  20. #include "cam_smmu_api.h"
  21. #include "cam_debug_util.h"
  22. #include "cam_trace.h"
  23. #include "cam_common_util.h"
  24. #include "cam_presil_hw_access.h"
  25. #include "cam_compat.h"
  26. #define CAM_MEM_SHARED_BUFFER_PAD_4K (4 * 1024)
  27. static struct cam_mem_table tbl;
  28. static atomic_t cam_mem_mgr_state = ATOMIC_INIT(CAM_MEM_MGR_UNINITIALIZED);
  29. /* Number of words for dumping req state info */
  30. #define CAM_MEM_MGR_DUMP_BUF_NUM_WORDS 29
  31. /* cam_mem_mgr_debug - global struct to keep track of debug settings for mem mgr
  32. *
  33. * @dentry : Directory entry to the mem mgr root folder
  34. * @alloc_profile_enable : Whether to enable alloc profiling
  35. * @override_cpu_access_dir : Override cpu access direction to BIDIRECTIONAL
  36. */
  37. static struct {
  38. struct dentry *dentry;
  39. bool alloc_profile_enable;
  40. bool override_cpu_access_dir;
  41. } g_cam_mem_mgr_debug;
  42. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  43. static void cam_mem_mgr_put_dma_heaps(void);
  44. static int cam_mem_mgr_get_dma_heaps(void);
  45. #endif
  46. #ifdef CONFIG_CAM_PRESIL
  47. static inline void cam_mem_mgr_reset_presil_params(int idx)
  48. {
  49. tbl.bufq[idx].presil_params.fd_for_umd_daemon = -1;
  50. tbl.bufq[idx].presil_params.refcount = 0;
  51. }
  52. #else
  53. static inline void cam_mem_mgr_reset_presil_params(int idx)
  54. {
  55. return;
  56. }
  57. #endif
  58. static unsigned long cam_mem_mgr_mini_dump_cb(void *dst, unsigned long len,
  59. void *priv_data)
  60. {
  61. struct cam_mem_table_mini_dump *md;
  62. if (!dst) {
  63. CAM_ERR(CAM_MEM, "Invalid params");
  64. return 0;
  65. }
  66. if (len < sizeof(*md)) {
  67. CAM_ERR(CAM_MEM, "Insufficient length %u", len);
  68. return 0;
  69. }
  70. md = (struct cam_mem_table_mini_dump *)dst;
  71. memcpy(md->bufq, tbl.bufq, CAM_MEM_BUFQ_MAX * sizeof(struct cam_mem_buf_queue));
  72. md->dbg_buf_idx = tbl.dbg_buf_idx;
  73. md->alloc_profile_enable = g_cam_mem_mgr_debug.alloc_profile_enable;
  74. md->force_cache_allocs = tbl.force_cache_allocs;
  75. md->need_shared_buffer_padding = tbl.need_shared_buffer_padding;
  76. return sizeof(*md);
  77. }
  78. static void cam_mem_mgr_print_tbl(void)
  79. {
  80. int i;
  81. uint64_t ms, hrs, min, sec;
  82. struct timespec64 current_ts;
  83. CAM_GET_TIMESTAMP(current_ts);
  84. CAM_CONVERT_TIMESTAMP_FORMAT(current_ts, hrs, min, sec, ms);
  85. CAM_INFO(CAM_MEM, "***%llu:%llu:%llu:%llu Mem mgr table dump***",
  86. hrs, min, sec, ms);
  87. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  88. if (tbl.bufq[i].active) {
  89. CAM_CONVERT_TIMESTAMP_FORMAT((tbl.bufq[i].timestamp), hrs, min, sec, ms);
  90. CAM_INFO(CAM_MEM,
  91. "%llu:%llu:%llu:%llu idx %d fd %d i_ino %lu size %llu",
  92. hrs, min, sec, ms, i, tbl.bufq[i].fd, tbl.bufq[i].i_ino,
  93. tbl.bufq[i].len);
  94. }
  95. }
  96. }
  97. static int cam_mem_util_get_dma_dir(uint32_t flags)
  98. {
  99. int rc = -EINVAL;
  100. if (flags & CAM_MEM_FLAG_HW_READ_ONLY)
  101. rc = DMA_TO_DEVICE;
  102. else if (flags & CAM_MEM_FLAG_HW_WRITE_ONLY)
  103. rc = DMA_FROM_DEVICE;
  104. else if (flags & CAM_MEM_FLAG_HW_READ_WRITE)
  105. rc = DMA_BIDIRECTIONAL;
  106. else if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  107. rc = DMA_BIDIRECTIONAL;
  108. return rc;
  109. }
  110. static int cam_mem_util_map_cpu_va(struct dma_buf *dmabuf, uintptr_t *vaddr, size_t *len)
  111. {
  112. int rc = 0;
  113. /*
  114. * dma_buf_begin_cpu_access() and dma_buf_end_cpu_access()
  115. * need to be called in pair to avoid stability issue.
  116. */
  117. rc = dma_buf_begin_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  118. if (rc) {
  119. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  120. return rc;
  121. }
  122. rc = cam_compat_util_get_dmabuf_va(dmabuf, vaddr);
  123. if (rc) {
  124. CAM_ERR(CAM_MEM, "kernel vmap failed: rc = %d", rc);
  125. *len = 0;
  126. dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  127. }
  128. else {
  129. *len = dmabuf->size;
  130. CAM_DBG(CAM_MEM, "vaddr = %llu, len = %zu", *vaddr, *len);
  131. }
  132. return rc;
  133. }
  134. static int cam_mem_util_unmap_cpu_va(struct dma_buf *dmabuf,
  135. uint64_t vaddr)
  136. {
  137. int rc = 0;
  138. if (!dmabuf || !vaddr) {
  139. CAM_ERR(CAM_MEM, "Invalid input args %pK %llX", dmabuf, vaddr);
  140. return -EINVAL;
  141. }
  142. cam_compat_util_put_dmabuf_va(dmabuf, (void *)vaddr);
  143. /*
  144. * dma_buf_begin_cpu_access() and
  145. * dma_buf_end_cpu_access() need to be called in pair
  146. * to avoid stability issue.
  147. */
  148. rc = dma_buf_end_cpu_access(dmabuf, DMA_BIDIRECTIONAL);
  149. if (rc) {
  150. CAM_ERR(CAM_MEM, "Failed in end cpu access, dmabuf=%pK",
  151. dmabuf);
  152. return rc;
  153. }
  154. return rc;
  155. }
  156. static int cam_mem_mgr_create_debug_fs(void)
  157. {
  158. int rc = 0;
  159. struct dentry *dbgfileptr = NULL;
  160. if (!cam_debugfs_available() || g_cam_mem_mgr_debug.dentry)
  161. return 0;
  162. rc = cam_debugfs_create_subdir("memmgr", &dbgfileptr);
  163. if (rc) {
  164. CAM_ERR(CAM_MEM, "DebugFS could not create directory!");
  165. rc = -ENOENT;
  166. goto end;
  167. }
  168. g_cam_mem_mgr_debug.dentry = dbgfileptr;
  169. debugfs_create_bool("alloc_profile_enable", 0644, g_cam_mem_mgr_debug.dentry,
  170. &g_cam_mem_mgr_debug.alloc_profile_enable);
  171. debugfs_create_bool("override_cpu_access_dir", 0644, g_cam_mem_mgr_debug.dentry,
  172. &g_cam_mem_mgr_debug.override_cpu_access_dir);
  173. end:
  174. return rc;
  175. }
  176. int cam_mem_mgr_init(void)
  177. {
  178. int i;
  179. int bitmap_size;
  180. int rc = 0;
  181. if (atomic_read(&cam_mem_mgr_state))
  182. return 0;
  183. memset(tbl.bufq, 0, sizeof(tbl.bufq));
  184. if (cam_smmu_need_force_alloc_cached(&tbl.force_cache_allocs)) {
  185. CAM_ERR(CAM_MEM, "Error in getting force cache alloc flag");
  186. return -EINVAL;
  187. }
  188. tbl.need_shared_buffer_padding = cam_smmu_need_shared_buffer_padding();
  189. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  190. rc = cam_mem_mgr_get_dma_heaps();
  191. if (rc) {
  192. CAM_ERR(CAM_MEM, "Failed in getting dma heaps rc=%d", rc);
  193. return rc;
  194. }
  195. #endif
  196. bitmap_size = BITS_TO_LONGS(CAM_MEM_BUFQ_MAX) * sizeof(long);
  197. tbl.bitmap = kzalloc(bitmap_size, GFP_KERNEL);
  198. if (!tbl.bitmap) {
  199. rc = -ENOMEM;
  200. goto put_heaps;
  201. }
  202. tbl.bits = bitmap_size * BITS_PER_BYTE;
  203. bitmap_zero(tbl.bitmap, tbl.bits);
  204. /* We need to reserve slot 0 because 0 is invalid */
  205. set_bit(0, tbl.bitmap);
  206. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  207. tbl.bufq[i].fd = -1;
  208. tbl.bufq[i].buf_handle = -1;
  209. cam_mem_mgr_reset_presil_params(i);
  210. }
  211. mutex_init(&tbl.m_lock);
  212. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_INITIALIZED);
  213. cam_mem_mgr_create_debug_fs();
  214. cam_common_register_mini_dump_cb(cam_mem_mgr_mini_dump_cb,
  215. "cam_mem", NULL);
  216. return 0;
  217. put_heaps:
  218. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  219. cam_mem_mgr_put_dma_heaps();
  220. #endif
  221. return rc;
  222. }
  223. static int32_t cam_mem_get_slot(void)
  224. {
  225. int32_t idx;
  226. mutex_lock(&tbl.m_lock);
  227. idx = find_first_zero_bit(tbl.bitmap, tbl.bits);
  228. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  229. mutex_unlock(&tbl.m_lock);
  230. return -ENOMEM;
  231. }
  232. set_bit(idx, tbl.bitmap);
  233. tbl.bufq[idx].active = true;
  234. CAM_GET_TIMESTAMP((tbl.bufq[idx].timestamp));
  235. mutex_init(&tbl.bufq[idx].q_lock);
  236. mutex_unlock(&tbl.m_lock);
  237. return idx;
  238. }
  239. static void cam_mem_put_slot(int32_t idx)
  240. {
  241. mutex_lock(&tbl.m_lock);
  242. mutex_lock(&tbl.bufq[idx].q_lock);
  243. tbl.bufq[idx].active = false;
  244. tbl.bufq[idx].is_internal = false;
  245. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  246. mutex_unlock(&tbl.bufq[idx].q_lock);
  247. mutex_destroy(&tbl.bufq[idx].q_lock);
  248. clear_bit(idx, tbl.bitmap);
  249. mutex_unlock(&tbl.m_lock);
  250. }
  251. int cam_mem_get_io_buf(int32_t buf_handle, int32_t mmu_handle,
  252. dma_addr_t *iova_ptr, size_t *len_ptr, uint32_t *flags)
  253. {
  254. int rc = 0, idx;
  255. *len_ptr = 0;
  256. if (!atomic_read(&cam_mem_mgr_state)) {
  257. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  258. return -EINVAL;
  259. }
  260. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  261. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  262. return -ENOENT;
  263. if (!tbl.bufq[idx].active) {
  264. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  265. idx);
  266. return -EAGAIN;
  267. }
  268. mutex_lock(&tbl.bufq[idx].q_lock);
  269. if (buf_handle != tbl.bufq[idx].buf_handle) {
  270. rc = -EINVAL;
  271. goto handle_mismatch;
  272. }
  273. if (CAM_MEM_MGR_IS_SECURE_HDL(buf_handle))
  274. rc = cam_smmu_get_stage2_iova(mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].dma_buf,
  275. iova_ptr, len_ptr);
  276. else
  277. rc = cam_smmu_get_iova(mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].dma_buf,
  278. iova_ptr, len_ptr);
  279. if (rc) {
  280. CAM_ERR(CAM_MEM,
  281. "fail to map buf_hdl:0x%x, mmu_hdl: 0x%x for fd:%d i_ino:%lu",
  282. buf_handle, mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].i_ino);
  283. goto handle_mismatch;
  284. }
  285. if (flags)
  286. *flags = tbl.bufq[idx].flags;
  287. CAM_DBG(CAM_MEM,
  288. "handle:0x%x fd:%d i_ino:%lu iova_ptr:0x%lx len_ptr:%lu",
  289. mmu_handle, tbl.bufq[idx].fd, tbl.bufq[idx].i_ino, *iova_ptr, *len_ptr);
  290. handle_mismatch:
  291. mutex_unlock(&tbl.bufq[idx].q_lock);
  292. return rc;
  293. }
  294. EXPORT_SYMBOL(cam_mem_get_io_buf);
  295. int cam_mem_get_cpu_buf(int32_t buf_handle, uintptr_t *vaddr_ptr, size_t *len)
  296. {
  297. int idx;
  298. if (!atomic_read(&cam_mem_mgr_state)) {
  299. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  300. return -EINVAL;
  301. }
  302. if (!buf_handle || !vaddr_ptr || !len)
  303. return -EINVAL;
  304. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  305. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  306. return -EINVAL;
  307. if (!tbl.bufq[idx].active) {
  308. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  309. idx);
  310. return -EPERM;
  311. }
  312. if (buf_handle != tbl.bufq[idx].buf_handle)
  313. return -EINVAL;
  314. if (!(tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS))
  315. return -EINVAL;
  316. if (tbl.bufq[idx].kmdvaddr) {
  317. *vaddr_ptr = tbl.bufq[idx].kmdvaddr;
  318. *len = tbl.bufq[idx].len;
  319. } else {
  320. CAM_ERR(CAM_MEM, "No KMD access was requested for 0x%x handle",
  321. buf_handle);
  322. return -EINVAL;
  323. }
  324. return 0;
  325. }
  326. EXPORT_SYMBOL(cam_mem_get_cpu_buf);
  327. int cam_mem_mgr_cache_ops(struct cam_mem_cache_ops_cmd *cmd)
  328. {
  329. int rc = 0, idx;
  330. uint32_t cache_dir;
  331. unsigned long dmabuf_flag = 0;
  332. if (!atomic_read(&cam_mem_mgr_state)) {
  333. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  334. return -EINVAL;
  335. }
  336. if (!cmd)
  337. return -EINVAL;
  338. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  339. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0)
  340. return -EINVAL;
  341. mutex_lock(&tbl.m_lock);
  342. if (!test_bit(idx, tbl.bitmap)) {
  343. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  344. idx);
  345. mutex_unlock(&tbl.m_lock);
  346. return -EINVAL;
  347. }
  348. mutex_lock(&tbl.bufq[idx].q_lock);
  349. mutex_unlock(&tbl.m_lock);
  350. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  351. rc = -EINVAL;
  352. goto end;
  353. }
  354. rc = dma_buf_get_flags(tbl.bufq[idx].dma_buf, &dmabuf_flag);
  355. if (rc) {
  356. CAM_ERR(CAM_MEM, "cache get flags failed %d", rc);
  357. goto end;
  358. }
  359. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  360. CAM_DBG(CAM_MEM, "Calling dmap buf APIs for cache operations");
  361. cache_dir = DMA_BIDIRECTIONAL;
  362. #else
  363. if (dmabuf_flag & ION_FLAG_CACHED) {
  364. switch (cmd->mem_cache_ops) {
  365. case CAM_MEM_CLEAN_CACHE:
  366. cache_dir = DMA_TO_DEVICE;
  367. break;
  368. case CAM_MEM_INV_CACHE:
  369. cache_dir = DMA_FROM_DEVICE;
  370. break;
  371. case CAM_MEM_CLEAN_INV_CACHE:
  372. cache_dir = DMA_BIDIRECTIONAL;
  373. break;
  374. default:
  375. CAM_ERR(CAM_MEM,
  376. "invalid cache ops :%d", cmd->mem_cache_ops);
  377. rc = -EINVAL;
  378. goto end;
  379. }
  380. } else {
  381. CAM_DBG(CAM_MEM, "BUF is not cached");
  382. goto end;
  383. }
  384. #endif
  385. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf,
  386. (cmd->mem_cache_ops == CAM_MEM_CLEAN_INV_CACHE) ?
  387. DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
  388. if (rc) {
  389. CAM_ERR(CAM_MEM, "dma begin access failed rc=%d", rc);
  390. goto end;
  391. }
  392. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf,
  393. cache_dir);
  394. if (rc) {
  395. CAM_ERR(CAM_MEM, "dma end access failed rc=%d", rc);
  396. goto end;
  397. }
  398. end:
  399. mutex_unlock(&tbl.bufq[idx].q_lock);
  400. return rc;
  401. }
  402. EXPORT_SYMBOL(cam_mem_mgr_cache_ops);
  403. int cam_mem_mgr_cpu_access_op(struct cam_mem_cpu_access_op *cmd)
  404. {
  405. int rc = 0, idx;
  406. uint32_t direction;
  407. if (!atomic_read(&cam_mem_mgr_state)) {
  408. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  409. return -EINVAL;
  410. }
  411. if (!cmd) {
  412. CAM_ERR(CAM_MEM, "Invalid cmd");
  413. return -EINVAL;
  414. }
  415. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  416. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  417. CAM_ERR(CAM_MEM, "Invalid idx=%d, buf_handle 0x%x, access=0x%x",
  418. idx, cmd->buf_handle, cmd->access);
  419. return -EINVAL;
  420. }
  421. mutex_lock(&tbl.m_lock);
  422. if (!test_bit(idx, tbl.bitmap)) {
  423. CAM_ERR(CAM_MEM, "Buffer at idx=%d is already freed/unmapped", idx);
  424. mutex_unlock(&tbl.m_lock);
  425. return -EINVAL;
  426. }
  427. mutex_lock(&tbl.bufq[idx].q_lock);
  428. mutex_unlock(&tbl.m_lock);
  429. if (cmd->buf_handle != tbl.bufq[idx].buf_handle) {
  430. CAM_ERR(CAM_MEM,
  431. "Buffer at idx=%d is different incoming handle 0x%x, actual handle 0x%x",
  432. idx, cmd->buf_handle, tbl.bufq[idx].buf_handle);
  433. rc = -EINVAL;
  434. goto end;
  435. }
  436. CAM_DBG(CAM_MEM, "buf_handle=0x%x, access=0x%x, access_type=0x%x, override_access=%d",
  437. cmd->buf_handle, cmd->access, cmd->access_type,
  438. g_cam_mem_mgr_debug.override_cpu_access_dir);
  439. if (cmd->access_type & CAM_MEM_CPU_ACCESS_READ &&
  440. cmd->access_type & CAM_MEM_CPU_ACCESS_WRITE) {
  441. direction = DMA_BIDIRECTIONAL;
  442. } else if (cmd->access_type & CAM_MEM_CPU_ACCESS_READ) {
  443. direction = DMA_FROM_DEVICE;
  444. } else if (cmd->access_type & CAM_MEM_CPU_ACCESS_WRITE) {
  445. direction = DMA_TO_DEVICE;
  446. } else {
  447. direction = DMA_BIDIRECTIONAL;
  448. CAM_WARN(CAM_MEM,
  449. "Invalid access type buf_handle=0x%x, access=0x%x, access_type=0x%x",
  450. cmd->buf_handle, cmd->access, cmd->access_type);
  451. }
  452. if (g_cam_mem_mgr_debug.override_cpu_access_dir)
  453. direction = DMA_BIDIRECTIONAL;
  454. if (cmd->access & CAM_MEM_BEGIN_CPU_ACCESS) {
  455. rc = dma_buf_begin_cpu_access(tbl.bufq[idx].dma_buf, direction);
  456. if (rc) {
  457. CAM_ERR(CAM_MEM,
  458. "dma begin cpu access failed rc=%d, buf_handle=0x%x, access=0x%x, access_type=0x%x",
  459. rc, cmd->buf_handle, cmd->access, cmd->access_type);
  460. goto end;
  461. }
  462. }
  463. if (cmd->access & CAM_MEM_END_CPU_ACCESS) {
  464. rc = dma_buf_end_cpu_access(tbl.bufq[idx].dma_buf, direction);
  465. if (rc) {
  466. CAM_ERR(CAM_MEM,
  467. "dma end cpu access failed rc=%d, buf_handle=0x%x, access=0x%x, access_type=0x%x",
  468. rc, cmd->buf_handle, cmd->access, cmd->access_type);
  469. goto end;
  470. }
  471. }
  472. end:
  473. mutex_unlock(&tbl.bufq[idx].q_lock);
  474. return rc;
  475. }
  476. EXPORT_SYMBOL(cam_mem_mgr_cpu_access_op);
  477. #if IS_REACHABLE(CONFIG_DMABUF_HEAPS)
  478. #define CAM_MAX_VMIDS 4
  479. static void cam_mem_mgr_put_dma_heaps(void)
  480. {
  481. CAM_DBG(CAM_MEM, "Releasing DMA Buf heaps usage");
  482. }
  483. static int cam_mem_mgr_get_dma_heaps(void)
  484. {
  485. int rc = 0;
  486. tbl.system_heap = NULL;
  487. tbl.system_uncached_heap = NULL;
  488. tbl.camera_heap = NULL;
  489. tbl.camera_uncached_heap = NULL;
  490. tbl.secure_display_heap = NULL;
  491. tbl.ubwc_p_heap = NULL;
  492. tbl.system_heap = dma_heap_find("qcom,system");
  493. if (IS_ERR_OR_NULL(tbl.system_heap)) {
  494. rc = PTR_ERR(tbl.system_heap);
  495. CAM_ERR(CAM_MEM, "qcom system heap not found, rc=%d", rc);
  496. tbl.system_heap = NULL;
  497. goto put_heaps;
  498. }
  499. tbl.system_uncached_heap = dma_heap_find("qcom,system-uncached");
  500. if (IS_ERR_OR_NULL(tbl.system_uncached_heap)) {
  501. if (tbl.force_cache_allocs) {
  502. /* optional, we anyway do not use uncached */
  503. CAM_DBG(CAM_MEM,
  504. "qcom system-uncached heap not found, err=%d",
  505. PTR_ERR(tbl.system_uncached_heap));
  506. tbl.system_uncached_heap = NULL;
  507. } else {
  508. /* fatal, must need uncached heaps */
  509. rc = PTR_ERR(tbl.system_uncached_heap);
  510. CAM_ERR(CAM_MEM,
  511. "qcom system-uncached heap not found, rc=%d",
  512. rc);
  513. tbl.system_uncached_heap = NULL;
  514. goto put_heaps;
  515. }
  516. }
  517. tbl.ubwc_p_heap = dma_heap_find("qcom,ubwcp");
  518. if (IS_ERR_OR_NULL(tbl.ubwc_p_heap)) {
  519. CAM_DBG(CAM_MEM, "qcom ubwcp heap not found, err=%d", PTR_ERR(tbl.ubwc_p_heap));
  520. tbl.ubwc_p_heap = NULL;
  521. }
  522. tbl.secure_display_heap = dma_heap_find("qcom,display");
  523. if (IS_ERR_OR_NULL(tbl.secure_display_heap)) {
  524. rc = PTR_ERR(tbl.secure_display_heap);
  525. CAM_ERR(CAM_MEM, "qcom,display heap not found, rc=%d",
  526. rc);
  527. tbl.secure_display_heap = NULL;
  528. goto put_heaps;
  529. }
  530. tbl.camera_heap = dma_heap_find("qcom,camera");
  531. if (IS_ERR_OR_NULL(tbl.camera_heap)) {
  532. /* optional heap, not a fatal error */
  533. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  534. PTR_ERR(tbl.camera_heap));
  535. tbl.camera_heap = NULL;
  536. }
  537. tbl.camera_uncached_heap = dma_heap_find("qcom,camera-uncached");
  538. if (IS_ERR_OR_NULL(tbl.camera_uncached_heap)) {
  539. /* optional heap, not a fatal error */
  540. CAM_DBG(CAM_MEM, "qcom camera heap not found, err=%d",
  541. PTR_ERR(tbl.camera_uncached_heap));
  542. tbl.camera_uncached_heap = NULL;
  543. }
  544. CAM_INFO(CAM_MEM,
  545. "Heaps : system=%pK, system_uncached=%pK, camera=%pK, camera-uncached=%pK, secure_display=%pK, ubwc_p_heap=%pK",
  546. tbl.system_heap, tbl.system_uncached_heap,
  547. tbl.camera_heap, tbl.camera_uncached_heap,
  548. tbl.secure_display_heap, tbl.ubwc_p_heap);
  549. return 0;
  550. put_heaps:
  551. cam_mem_mgr_put_dma_heaps();
  552. return rc;
  553. }
  554. bool cam_mem_mgr_ubwc_p_heap_supported(void)
  555. {
  556. if (tbl.ubwc_p_heap)
  557. return true;
  558. return false;
  559. }
  560. static int cam_mem_util_get_dma_buf(size_t len,
  561. unsigned int cam_flags,
  562. struct dma_buf **buf,
  563. unsigned long *i_ino)
  564. {
  565. int rc = 0;
  566. struct dma_heap *heap;
  567. struct dma_heap *try_heap = NULL;
  568. struct timespec64 ts1, ts2;
  569. long microsec = 0;
  570. bool use_cached_heap = false;
  571. struct mem_buf_lend_kernel_arg arg;
  572. int vmids[CAM_MAX_VMIDS];
  573. int perms[CAM_MAX_VMIDS];
  574. int num_vmids = 0;
  575. if (!buf) {
  576. CAM_ERR(CAM_MEM, "Invalid params");
  577. return -EINVAL;
  578. }
  579. if (g_cam_mem_mgr_debug.alloc_profile_enable)
  580. CAM_GET_TIMESTAMP(ts1);
  581. if ((cam_flags & CAM_MEM_FLAG_CACHE) ||
  582. (tbl.force_cache_allocs &&
  583. (!(cam_flags & CAM_MEM_FLAG_PROTECTED_MODE)))) {
  584. CAM_DBG(CAM_MEM,
  585. "Using CACHED heap, cam_flags=0x%x, force_cache_allocs=%d",
  586. cam_flags, tbl.force_cache_allocs);
  587. use_cached_heap = true;
  588. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  589. use_cached_heap = true;
  590. CAM_DBG(CAM_MEM,
  591. "Using CACHED heap for secure, cam_flags=0x%x, force_cache_allocs=%d",
  592. cam_flags, tbl.force_cache_allocs);
  593. } else {
  594. use_cached_heap = false;
  595. if (!tbl.system_uncached_heap) {
  596. CAM_ERR(CAM_MEM,
  597. "Using UNCACHED heap not supported, cam_flags=0x%x, force_cache_allocs=%d",
  598. cam_flags, tbl.force_cache_allocs);
  599. return -EINVAL;
  600. }
  601. }
  602. if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  603. heap = tbl.secure_display_heap;
  604. vmids[num_vmids] = VMID_CP_CAMERA;
  605. perms[num_vmids] = PERM_READ | PERM_WRITE;
  606. num_vmids++;
  607. if (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT) {
  608. CAM_DBG(CAM_MEM, "Secure mode CDSP flags");
  609. vmids[num_vmids] = VMID_CP_CDSP;
  610. perms[num_vmids] = PERM_READ | PERM_WRITE;
  611. num_vmids++;
  612. }
  613. } else if (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL) {
  614. heap = tbl.secure_display_heap;
  615. vmids[num_vmids] = VMID_CP_NON_PIXEL;
  616. perms[num_vmids] = PERM_READ | PERM_WRITE;
  617. num_vmids++;
  618. } else if (cam_flags & CAM_MEM_FLAG_UBWC_P_HEAP) {
  619. if (!tbl.ubwc_p_heap) {
  620. CAM_ERR(CAM_MEM, "ubwc-p heap is not available, can't allocate");
  621. return -EINVAL;
  622. }
  623. heap = tbl.ubwc_p_heap;
  624. CAM_DBG(CAM_MEM, "Allocating from ubwc-p heap, size=%d, flags=0x%x",
  625. len, cam_flags);
  626. } else if (use_cached_heap) {
  627. try_heap = tbl.camera_heap;
  628. heap = tbl.system_heap;
  629. } else {
  630. try_heap = tbl.camera_uncached_heap;
  631. heap = tbl.system_uncached_heap;
  632. }
  633. CAM_DBG(CAM_MEM, "Using heaps : try=%pK, heap=%pK", try_heap, heap);
  634. *buf = NULL;
  635. if (!try_heap && !heap) {
  636. CAM_ERR(CAM_MEM,
  637. "No heap available for allocation, cant allocate");
  638. return -EINVAL;
  639. }
  640. if (try_heap) {
  641. *buf = dma_heap_buffer_alloc(try_heap, len, O_RDWR, 0);
  642. if (IS_ERR(*buf)) {
  643. CAM_WARN(CAM_MEM,
  644. "Failed in allocating from try heap, heap=%pK, len=%zu, err=%d",
  645. try_heap, len, PTR_ERR(*buf));
  646. *buf = NULL;
  647. }
  648. }
  649. if (*buf == NULL) {
  650. *buf = dma_heap_buffer_alloc(heap, len, O_RDWR, 0);
  651. if (IS_ERR(*buf)) {
  652. rc = PTR_ERR(*buf);
  653. CAM_ERR(CAM_MEM,
  654. "Failed in allocating from heap, heap=%pK, len=%zu, err=%d",
  655. heap, len, rc);
  656. *buf = NULL;
  657. return rc;
  658. }
  659. }
  660. *i_ino = file_inode((*buf)->file)->i_ino;
  661. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) ||
  662. (cam_flags & CAM_MEM_FLAG_EVA_NOPIXEL)) {
  663. if (num_vmids >= CAM_MAX_VMIDS) {
  664. CAM_ERR(CAM_MEM, "Insufficient array size for vmids %d", num_vmids);
  665. rc = -EINVAL;
  666. goto end;
  667. }
  668. arg.nr_acl_entries = num_vmids;
  669. arg.vmids = vmids;
  670. arg.perms = perms;
  671. rc = mem_buf_lend(*buf, &arg);
  672. if (rc) {
  673. CAM_ERR(CAM_MEM,
  674. "Failed in buf lend rc=%d, buf=%pK, vmids [0]=0x%x, [1]=0x%x, [2]=0x%x",
  675. rc, *buf, vmids[0], vmids[1], vmids[2]);
  676. goto end;
  677. }
  678. }
  679. CAM_DBG(CAM_MEM, "Allocate success, len=%zu, *buf=%pK, i_ino=%lu", len, *buf, *i_ino);
  680. if (g_cam_mem_mgr_debug.alloc_profile_enable) {
  681. CAM_GET_TIMESTAMP(ts2);
  682. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  683. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  684. len, microsec);
  685. }
  686. return rc;
  687. end:
  688. dma_buf_put(*buf);
  689. return rc;
  690. }
  691. #else
  692. bool cam_mem_mgr_ubwc_p_heap_supported(void)
  693. {
  694. return false;
  695. }
  696. static int cam_mem_util_get_dma_buf(size_t len,
  697. unsigned int cam_flags,
  698. struct dma_buf **buf,
  699. unsigned long *i_ino)
  700. {
  701. int rc = 0;
  702. unsigned int heap_id;
  703. int32_t ion_flag = 0;
  704. struct timespec64 ts1, ts2;
  705. long microsec = 0;
  706. if (!buf) {
  707. CAM_ERR(CAM_MEM, "Invalid params");
  708. return -EINVAL;
  709. }
  710. if (cam_flags & CAM_MEM_FLAG_UBWC_P_HEAP) {
  711. CAM_ERR(CAM_MEM, "ubwcp heap not supported");
  712. return -EINVAL;
  713. }
  714. if (g_cam_mem_mgr_debug.alloc_profile_enable)
  715. CAM_GET_TIMESTAMP(ts1);
  716. if ((cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) &&
  717. (cam_flags & CAM_MEM_FLAG_CDSP_OUTPUT)) {
  718. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  719. ion_flag |=
  720. ION_FLAG_SECURE | ION_FLAG_CP_CAMERA | ION_FLAG_CP_CDSP;
  721. } else if (cam_flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  722. heap_id = ION_HEAP(ION_SECURE_DISPLAY_HEAP_ID);
  723. ion_flag |= ION_FLAG_SECURE | ION_FLAG_CP_CAMERA;
  724. } else {
  725. heap_id = ION_HEAP(ION_SYSTEM_HEAP_ID) |
  726. ION_HEAP(ION_CAMERA_HEAP_ID);
  727. }
  728. if (cam_flags & CAM_MEM_FLAG_CACHE)
  729. ion_flag |= ION_FLAG_CACHED;
  730. else
  731. ion_flag &= ~ION_FLAG_CACHED;
  732. if (tbl.force_cache_allocs && (!(ion_flag & ION_FLAG_SECURE)))
  733. ion_flag |= ION_FLAG_CACHED;
  734. *buf = ion_alloc(len, heap_id, ion_flag);
  735. if (IS_ERR_OR_NULL(*buf))
  736. return -ENOMEM;
  737. *i_ino = file_inode((*buf)->file)->i_ino;
  738. if (g_cam_mem_mgr_debug.alloc_profile_enable) {
  739. CAM_GET_TIMESTAMP(ts2);
  740. CAM_GET_TIMESTAMP_DIFF_IN_MICRO(ts1, ts2, microsec);
  741. trace_cam_log_event("IONAllocProfile", "size and time in micro",
  742. len, microsec);
  743. }
  744. return rc;
  745. }
  746. #endif
  747. static int cam_mem_util_buffer_alloc(size_t len, uint32_t flags,
  748. struct dma_buf **dmabuf,
  749. int *fd,
  750. unsigned long *i_ino)
  751. {
  752. int rc;
  753. rc = cam_mem_util_get_dma_buf(len, flags, dmabuf, i_ino);
  754. if (rc) {
  755. CAM_ERR(CAM_MEM,
  756. "Error allocating dma buf : len=%llu, flags=0x%x",
  757. len, flags);
  758. return rc;
  759. }
  760. /*
  761. * increment the ref count so that ref count becomes 2 here
  762. * when we close fd, refcount becomes 1 and when we do
  763. * dmap_put_buf, ref count becomes 0 and memory will be freed.
  764. */
  765. get_dma_buf(*dmabuf);
  766. *fd = dma_buf_fd(*dmabuf, O_CLOEXEC);
  767. if (*fd < 0) {
  768. CAM_ERR(CAM_MEM, "get fd fail, *fd=%d", *fd);
  769. rc = -EINVAL;
  770. goto put_buf;
  771. }
  772. CAM_DBG(CAM_MEM, "Alloc success : len=%zu, *dmabuf=%pK, fd=%d, i_ino=%lu",
  773. len, *dmabuf, *fd, *i_ino);
  774. return rc;
  775. put_buf:
  776. dma_buf_put(*dmabuf);
  777. return rc;
  778. }
  779. static int cam_mem_util_check_alloc_flags(struct cam_mem_mgr_alloc_cmd_v2 *cmd)
  780. {
  781. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  782. CAM_ERR(CAM_MEM, "Num of mmu hdl exceeded maximum(%d)",
  783. CAM_MEM_MMU_MAX_HANDLE);
  784. return -EINVAL;
  785. }
  786. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  787. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  788. CAM_ERR(CAM_MEM, "Kernel mapping in secure mode not allowed");
  789. return -EINVAL;
  790. }
  791. if ((cmd->flags & CAM_MEM_FLAG_EVA_NOPIXEL) &&
  792. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE ||
  793. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS)) {
  794. CAM_ERR(CAM_MEM,
  795. "Kernel mapping and secure mode not allowed in no pixel mode");
  796. return -EINVAL;
  797. }
  798. if (cmd->flags & CAM_MEM_FLAG_UBWC_P_HEAP &&
  799. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE ||
  800. cmd->flags & CAM_MEM_FLAG_EVA_NOPIXEL ||
  801. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS ||
  802. cmd->flags & CAM_MEM_FLAG_CMD_BUF_TYPE ||
  803. cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  804. cmd->flags & CAM_MEM_FLAG_HW_AND_CDM_OR_SHARED)) {
  805. CAM_ERR(CAM_MEM,
  806. "UBWC-P buffer not supported with this combinatation of flags 0x%x",
  807. cmd->flags);
  808. return -EINVAL;
  809. }
  810. return 0;
  811. }
  812. static int cam_mem_util_check_map_flags(struct cam_mem_mgr_map_cmd_v2 *cmd)
  813. {
  814. if (!cmd->flags) {
  815. CAM_ERR(CAM_MEM, "Invalid flags");
  816. return -EINVAL;
  817. }
  818. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  819. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  820. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  821. return -EINVAL;
  822. }
  823. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE &&
  824. cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  825. CAM_ERR(CAM_MEM,
  826. "Kernel mapping in secure mode not allowed, flags=0x%x",
  827. cmd->flags);
  828. return -EINVAL;
  829. }
  830. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  831. CAM_ERR(CAM_MEM,
  832. "Shared memory buffers are not allowed to be mapped");
  833. return -EINVAL;
  834. }
  835. return 0;
  836. }
  837. static int cam_mem_util_map_hw_va(uint32_t flags,
  838. int32_t *mmu_hdls,
  839. int32_t num_hdls,
  840. int fd,
  841. struct dma_buf *dmabuf,
  842. dma_addr_t *hw_vaddr,
  843. size_t *len,
  844. enum cam_smmu_region_id region,
  845. bool is_internal)
  846. {
  847. int i;
  848. int rc = -1;
  849. int dir = cam_mem_util_get_dma_dir(flags);
  850. bool dis_delayed_unmap = false;
  851. if (dir < 0) {
  852. CAM_ERR(CAM_MEM, "fail to map DMA direction, dir=%d", dir);
  853. return dir;
  854. }
  855. if (flags & CAM_MEM_FLAG_DISABLE_DELAYED_UNMAP)
  856. dis_delayed_unmap = true;
  857. CAM_DBG(CAM_MEM,
  858. "map_hw_va : fd = %d, flags = 0x%x, dir=%d, num_hdls=%d",
  859. fd, flags, dir, num_hdls);
  860. for (i = 0; i < num_hdls; i++) {
  861. /* If 36-bit enabled, check for ICP cmd buffers and map them within the shared region */
  862. if (cam_smmu_is_expanded_memory() &&
  863. cam_smmu_supports_shared_region(mmu_hdls[i]) &&
  864. ((flags & CAM_MEM_FLAG_CMD_BUF_TYPE) ||
  865. (flags & CAM_MEM_FLAG_HW_AND_CDM_OR_SHARED)))
  866. region = CAM_SMMU_REGION_SHARED;
  867. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  868. rc = cam_smmu_map_stage2_iova(mmu_hdls[i], fd, dmabuf, dir, hw_vaddr, len);
  869. else
  870. rc = cam_smmu_map_user_iova(mmu_hdls[i], fd, dmabuf, dis_delayed_unmap, dir,
  871. hw_vaddr, len, region, is_internal);
  872. if (rc) {
  873. CAM_ERR(CAM_MEM,
  874. "Failed %s map to smmu, i=%d, fd=%d, dir=%d, mmu_hdl=%d, rc=%d",
  875. (flags & CAM_MEM_FLAG_PROTECTED_MODE) ? "" : "secured",
  876. i, fd, dir, mmu_hdls[i], rc);
  877. goto multi_map_fail;
  878. }
  879. }
  880. return rc;
  881. multi_map_fail:
  882. for (--i; i>= 0; i--) {
  883. if (flags & CAM_MEM_FLAG_PROTECTED_MODE)
  884. cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd, dmabuf);
  885. else
  886. cam_smmu_unmap_user_iova(mmu_hdls[i], fd, dmabuf, CAM_SMMU_REGION_IO);
  887. }
  888. return rc;
  889. }
  890. int cam_mem_mgr_alloc_and_map(struct cam_mem_mgr_alloc_cmd_v2 *cmd)
  891. {
  892. int rc;
  893. int32_t idx;
  894. struct dma_buf *dmabuf = NULL;
  895. int fd = -1;
  896. dma_addr_t hw_vaddr = 0;
  897. size_t len;
  898. uintptr_t kvaddr = 0;
  899. size_t klen;
  900. unsigned long i_ino = 0;
  901. if (!atomic_read(&cam_mem_mgr_state)) {
  902. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  903. return -EINVAL;
  904. }
  905. if (!cmd) {
  906. CAM_ERR(CAM_MEM, " Invalid argument");
  907. return -EINVAL;
  908. }
  909. len = cmd->len;
  910. if (tbl.need_shared_buffer_padding &&
  911. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)) {
  912. len += CAM_MEM_SHARED_BUFFER_PAD_4K;
  913. CAM_DBG(CAM_MEM, "Pad 4k size, actual %llu, allocating %zu",
  914. cmd->len, len);
  915. }
  916. rc = cam_mem_util_check_alloc_flags(cmd);
  917. if (rc) {
  918. CAM_ERR(CAM_MEM, "Invalid flags: flags = 0x%X, rc=%d",
  919. cmd->flags, rc);
  920. return rc;
  921. }
  922. rc = cam_mem_util_buffer_alloc(len, cmd->flags, &dmabuf, &fd, &i_ino);
  923. if (rc) {
  924. CAM_ERR(CAM_MEM,
  925. "Ion Alloc failed, len=%llu, align=%llu, flags=0x%x, num_hdl=%d",
  926. len, cmd->align, cmd->flags, cmd->num_hdl);
  927. cam_mem_mgr_print_tbl();
  928. return rc;
  929. }
  930. if (!dmabuf) {
  931. CAM_ERR(CAM_MEM,
  932. "Ion Alloc return NULL dmabuf! fd=%d, i_ino=%lu, len=%d", fd, i_ino, len);
  933. cam_mem_mgr_print_tbl();
  934. return rc;
  935. }
  936. idx = cam_mem_get_slot();
  937. if (idx < 0) {
  938. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  939. rc = -ENOMEM;
  940. goto slot_fail;
  941. }
  942. if (cam_dma_buf_set_name(dmabuf, cmd->buf_name))
  943. CAM_ERR(CAM_MEM, "set dma buffer name(%s) failed", cmd->buf_name);
  944. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  945. (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  946. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  947. enum cam_smmu_region_id region;
  948. if (cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  949. region = CAM_SMMU_REGION_IO;
  950. if (cmd->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  951. region = CAM_SMMU_REGION_SHARED;
  952. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  953. region = CAM_SMMU_REGION_IO;
  954. rc = cam_mem_util_map_hw_va(cmd->flags,
  955. cmd->mmu_hdls,
  956. cmd->num_hdl,
  957. fd,
  958. dmabuf,
  959. &hw_vaddr,
  960. &len,
  961. region,
  962. true);
  963. if (rc) {
  964. CAM_ERR(CAM_MEM,
  965. "Failed in map_hw_va len=%llu, flags=0x%x, fd=%d, region=%d, num_hdl=%d, rc=%d",
  966. len, cmd->flags,
  967. fd, region, cmd->num_hdl, rc);
  968. if (rc == -EALREADY) {
  969. if ((size_t)dmabuf->size != len)
  970. rc = -EBADR;
  971. cam_mem_mgr_print_tbl();
  972. }
  973. goto map_hw_fail;
  974. }
  975. }
  976. mutex_lock(&tbl.bufq[idx].q_lock);
  977. tbl.bufq[idx].fd = fd;
  978. tbl.bufq[idx].i_ino = i_ino;
  979. tbl.bufq[idx].dma_buf = NULL;
  980. tbl.bufq[idx].flags = cmd->flags;
  981. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, fd);
  982. tbl.bufq[idx].is_internal = true;
  983. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  984. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  985. if (cmd->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  986. rc = cam_mem_util_map_cpu_va(dmabuf, &kvaddr, &klen);
  987. if (rc) {
  988. CAM_ERR(CAM_MEM, "dmabuf: %pK mapping failed: %d",
  989. dmabuf, rc);
  990. goto map_kernel_fail;
  991. }
  992. }
  993. if (cmd->flags & CAM_MEM_FLAG_KMD_DEBUG_FLAG)
  994. tbl.dbg_buf_idx = idx;
  995. tbl.bufq[idx].kmdvaddr = kvaddr;
  996. tbl.bufq[idx].vaddr = hw_vaddr;
  997. tbl.bufq[idx].dma_buf = dmabuf;
  998. tbl.bufq[idx].len = len;
  999. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  1000. cam_mem_mgr_reset_presil_params(idx);
  1001. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  1002. sizeof(int32_t) * cmd->num_hdl);
  1003. tbl.bufq[idx].is_imported = false;
  1004. mutex_unlock(&tbl.bufq[idx].q_lock);
  1005. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  1006. cmd->out.fd = tbl.bufq[idx].fd;
  1007. cmd->out.vaddr = 0;
  1008. CAM_DBG(CAM_MEM,
  1009. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu, i_ino=%lu, name:%s",
  1010. cmd->out.fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  1011. tbl.bufq[idx].len, tbl.bufq[idx].i_ino, cmd->buf_name);
  1012. return rc;
  1013. map_kernel_fail:
  1014. mutex_unlock(&tbl.bufq[idx].q_lock);
  1015. map_hw_fail:
  1016. cam_mem_put_slot(idx);
  1017. slot_fail:
  1018. dma_buf_put(dmabuf);
  1019. return rc;
  1020. }
  1021. static bool cam_mem_util_is_map_internal(int32_t fd, unsigned i_ino)
  1022. {
  1023. uint32_t i;
  1024. bool is_internal = false;
  1025. mutex_lock(&tbl.m_lock);
  1026. for_each_set_bit(i, tbl.bitmap, tbl.bits) {
  1027. if ((tbl.bufq[i].fd == fd) && (tbl.bufq[i].i_ino == i_ino)) {
  1028. is_internal = tbl.bufq[i].is_internal;
  1029. break;
  1030. }
  1031. }
  1032. mutex_unlock(&tbl.m_lock);
  1033. return is_internal;
  1034. }
  1035. int cam_mem_mgr_map(struct cam_mem_mgr_map_cmd_v2 *cmd)
  1036. {
  1037. int32_t idx;
  1038. int rc;
  1039. struct dma_buf *dmabuf;
  1040. dma_addr_t hw_vaddr = 0;
  1041. size_t len = 0;
  1042. bool is_internal = false;
  1043. unsigned long i_ino;
  1044. if (!atomic_read(&cam_mem_mgr_state)) {
  1045. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1046. return -EINVAL;
  1047. }
  1048. if (!cmd || (cmd->fd < 0)) {
  1049. CAM_ERR(CAM_MEM, "Invalid argument");
  1050. return -EINVAL;
  1051. }
  1052. if (cmd->num_hdl > CAM_MEM_MMU_MAX_HANDLE) {
  1053. CAM_ERR(CAM_MEM, "Num of mmu hdl %d exceeded maximum(%d)",
  1054. cmd->num_hdl, CAM_MEM_MMU_MAX_HANDLE);
  1055. return -EINVAL;
  1056. }
  1057. rc = cam_mem_util_check_map_flags(cmd);
  1058. if (rc) {
  1059. CAM_ERR(CAM_MEM, "Invalid flags: flags = %X", cmd->flags);
  1060. return rc;
  1061. }
  1062. dmabuf = dma_buf_get(cmd->fd);
  1063. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  1064. CAM_ERR(CAM_MEM, "Failed to import dma_buf fd");
  1065. return -EINVAL;
  1066. }
  1067. i_ino = file_inode(dmabuf->file)->i_ino;
  1068. is_internal = cam_mem_util_is_map_internal(cmd->fd, i_ino);
  1069. idx = cam_mem_get_slot();
  1070. if (idx < 0) {
  1071. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d, fd=%d",
  1072. idx, cmd->fd);
  1073. rc = -ENOMEM;
  1074. goto slot_fail;
  1075. }
  1076. if (cam_dma_buf_set_name(dmabuf, cmd->buf_name))
  1077. CAM_DBG(CAM_MEM, "Dma buffer (%s) busy", cmd->buf_name);
  1078. if ((cmd->flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1079. (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1080. rc = cam_mem_util_map_hw_va(cmd->flags,
  1081. cmd->mmu_hdls,
  1082. cmd->num_hdl,
  1083. cmd->fd,
  1084. dmabuf,
  1085. &hw_vaddr,
  1086. &len,
  1087. CAM_SMMU_REGION_IO,
  1088. is_internal);
  1089. if (rc) {
  1090. CAM_ERR(CAM_MEM,
  1091. "Failed in map_hw_va, flags=0x%x, fd=%d, len=%llu, region=%d, num_hdl=%d, rc=%d",
  1092. cmd->flags, cmd->fd, len,
  1093. CAM_SMMU_REGION_IO, cmd->num_hdl, rc);
  1094. if (rc == -EALREADY) {
  1095. if ((size_t)dmabuf->size != len) {
  1096. rc = -EBADR;
  1097. cam_mem_mgr_print_tbl();
  1098. }
  1099. }
  1100. goto map_fail;
  1101. }
  1102. }
  1103. mutex_lock(&tbl.bufq[idx].q_lock);
  1104. tbl.bufq[idx].fd = cmd->fd;
  1105. tbl.bufq[idx].i_ino = i_ino;
  1106. tbl.bufq[idx].dma_buf = NULL;
  1107. tbl.bufq[idx].flags = cmd->flags;
  1108. tbl.bufq[idx].buf_handle = GET_MEM_HANDLE(idx, cmd->fd);
  1109. if (cmd->flags & CAM_MEM_FLAG_PROTECTED_MODE)
  1110. CAM_MEM_MGR_SET_SECURE_HDL(tbl.bufq[idx].buf_handle, true);
  1111. tbl.bufq[idx].kmdvaddr = 0;
  1112. if (cmd->num_hdl > 0)
  1113. tbl.bufq[idx].vaddr = hw_vaddr;
  1114. else
  1115. tbl.bufq[idx].vaddr = 0;
  1116. tbl.bufq[idx].dma_buf = dmabuf;
  1117. tbl.bufq[idx].len = len;
  1118. tbl.bufq[idx].num_hdl = cmd->num_hdl;
  1119. memcpy(tbl.bufq[idx].hdls, cmd->mmu_hdls,
  1120. sizeof(int32_t) * cmd->num_hdl);
  1121. tbl.bufq[idx].is_imported = true;
  1122. tbl.bufq[idx].is_internal = is_internal;
  1123. mutex_unlock(&tbl.bufq[idx].q_lock);
  1124. cmd->out.buf_handle = tbl.bufq[idx].buf_handle;
  1125. cmd->out.vaddr = 0;
  1126. cmd->out.size = (uint32_t)len;
  1127. CAM_DBG(CAM_MEM,
  1128. "fd=%d, flags=0x%x, num_hdl=%d, idx=%d, buf handle=%x, len=%zu, i_ino=%lu, name:%s",
  1129. cmd->fd, cmd->flags, cmd->num_hdl, idx, cmd->out.buf_handle,
  1130. tbl.bufq[idx].len, tbl.bufq[idx].i_ino, cmd->buf_name);
  1131. return rc;
  1132. map_fail:
  1133. cam_mem_put_slot(idx);
  1134. slot_fail:
  1135. dma_buf_put(dmabuf);
  1136. return rc;
  1137. }
  1138. static int cam_mem_util_unmap_hw_va(int32_t idx,
  1139. enum cam_smmu_region_id region,
  1140. enum cam_smmu_mapping_client client)
  1141. {
  1142. int i;
  1143. uint32_t flags;
  1144. int32_t *mmu_hdls;
  1145. int num_hdls;
  1146. int fd;
  1147. struct dma_buf *dma_buf;
  1148. unsigned long i_ino;
  1149. int rc = 0;
  1150. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1151. CAM_ERR(CAM_MEM, "Incorrect index");
  1152. return -EINVAL;
  1153. }
  1154. flags = tbl.bufq[idx].flags;
  1155. mmu_hdls = tbl.bufq[idx].hdls;
  1156. num_hdls = tbl.bufq[idx].num_hdl;
  1157. fd = tbl.bufq[idx].fd;
  1158. dma_buf = tbl.bufq[idx].dma_buf;
  1159. i_ino = tbl.bufq[idx].i_ino;
  1160. CAM_DBG(CAM_MEM,
  1161. "unmap_hw_va : idx=%d, fd=%x, i_ino=%lu flags=0x%x, num_hdls=%d, client=%d",
  1162. idx, fd, i_ino, flags, num_hdls, client);
  1163. if (flags & CAM_MEM_FLAG_PROTECTED_MODE) {
  1164. for (i = 0; i < num_hdls; i++) {
  1165. rc = cam_smmu_unmap_stage2_iova(mmu_hdls[i], fd, dma_buf);
  1166. if (rc < 0) {
  1167. CAM_ERR(CAM_MEM,
  1168. "Failed in secure unmap, i=%d, fd=%d, i_ino=%lu, mmu_hdl=%d, rc=%d",
  1169. i, fd, i_ino, mmu_hdls[i], rc);
  1170. goto unmap_end;
  1171. }
  1172. }
  1173. } else {
  1174. for (i = 0; i < num_hdls; i++) {
  1175. if (client == CAM_SMMU_MAPPING_USER) {
  1176. rc = cam_smmu_unmap_user_iova(mmu_hdls[i],
  1177. fd, dma_buf, region);
  1178. } else if (client == CAM_SMMU_MAPPING_KERNEL) {
  1179. rc = cam_smmu_unmap_kernel_iova(mmu_hdls[i],
  1180. tbl.bufq[idx].dma_buf, region);
  1181. } else {
  1182. CAM_ERR(CAM_MEM,
  1183. "invalid caller for unmapping : %d",
  1184. client);
  1185. rc = -EINVAL;
  1186. }
  1187. if (rc < 0) {
  1188. CAM_ERR(CAM_MEM,
  1189. "Failed in unmap, i=%d, fd=%d, i_ino=%lu, mmu_hdl=%d, region=%d, rc=%d",
  1190. i, fd, i_ino, mmu_hdls[i], region, rc);
  1191. goto unmap_end;
  1192. }
  1193. }
  1194. }
  1195. return rc;
  1196. unmap_end:
  1197. CAM_ERR(CAM_MEM, "unmapping failed");
  1198. return rc;
  1199. }
  1200. static void cam_mem_mgr_unmap_active_buf(int idx)
  1201. {
  1202. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1203. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS)
  1204. region = CAM_SMMU_REGION_SHARED;
  1205. else if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1206. region = CAM_SMMU_REGION_IO;
  1207. cam_mem_util_unmap_hw_va(idx, region, CAM_SMMU_MAPPING_USER);
  1208. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS)
  1209. cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1210. tbl.bufq[idx].kmdvaddr);
  1211. }
  1212. static int cam_mem_mgr_cleanup_table(void)
  1213. {
  1214. int i;
  1215. mutex_lock(&tbl.m_lock);
  1216. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  1217. if (!tbl.bufq[i].active) {
  1218. CAM_DBG(CAM_MEM,
  1219. "Buffer inactive at idx=%d, continuing", i);
  1220. continue;
  1221. } else {
  1222. CAM_DBG(CAM_MEM,
  1223. "Active buffer at idx=%d, possible leak needs unmapping",
  1224. i);
  1225. cam_mem_mgr_unmap_active_buf(i);
  1226. }
  1227. mutex_lock(&tbl.bufq[i].q_lock);
  1228. if (tbl.bufq[i].dma_buf) {
  1229. dma_buf_put(tbl.bufq[i].dma_buf);
  1230. tbl.bufq[i].dma_buf = NULL;
  1231. }
  1232. tbl.bufq[i].fd = -1;
  1233. tbl.bufq[i].i_ino = 0;
  1234. tbl.bufq[i].flags = 0;
  1235. tbl.bufq[i].buf_handle = -1;
  1236. tbl.bufq[i].vaddr = 0;
  1237. tbl.bufq[i].len = 0;
  1238. memset(tbl.bufq[i].hdls, 0,
  1239. sizeof(int32_t) * tbl.bufq[i].num_hdl);
  1240. tbl.bufq[i].num_hdl = 0;
  1241. tbl.bufq[i].dma_buf = NULL;
  1242. tbl.bufq[i].active = false;
  1243. tbl.bufq[i].is_internal = false;
  1244. cam_mem_mgr_reset_presil_params(i);
  1245. mutex_unlock(&tbl.bufq[i].q_lock);
  1246. mutex_destroy(&tbl.bufq[i].q_lock);
  1247. }
  1248. bitmap_zero(tbl.bitmap, tbl.bits);
  1249. /* We need to reserve slot 0 because 0 is invalid */
  1250. set_bit(0, tbl.bitmap);
  1251. mutex_unlock(&tbl.m_lock);
  1252. return 0;
  1253. }
  1254. void cam_mem_mgr_deinit(void)
  1255. {
  1256. if (!atomic_read(&cam_mem_mgr_state))
  1257. return;
  1258. atomic_set(&cam_mem_mgr_state, CAM_MEM_MGR_UNINITIALIZED);
  1259. cam_mem_mgr_cleanup_table();
  1260. mutex_lock(&tbl.m_lock);
  1261. bitmap_zero(tbl.bitmap, tbl.bits);
  1262. kfree(tbl.bitmap);
  1263. tbl.bitmap = NULL;
  1264. tbl.dbg_buf_idx = -1;
  1265. mutex_unlock(&tbl.m_lock);
  1266. mutex_destroy(&tbl.m_lock);
  1267. }
  1268. static int cam_mem_util_unmap(int32_t idx,
  1269. enum cam_smmu_mapping_client client)
  1270. {
  1271. int rc = 0;
  1272. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1273. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1274. CAM_ERR(CAM_MEM, "Incorrect index");
  1275. return -EINVAL;
  1276. }
  1277. CAM_DBG(CAM_MEM, "Flags = %X idx %d", tbl.bufq[idx].flags, idx);
  1278. mutex_lock(&tbl.m_lock);
  1279. if ((!tbl.bufq[idx].active) &&
  1280. (tbl.bufq[idx].vaddr) == 0) {
  1281. CAM_WARN(CAM_MEM, "Buffer at idx=%d is already unmapped,",
  1282. idx);
  1283. mutex_unlock(&tbl.m_lock);
  1284. return 0;
  1285. }
  1286. /* Deactivate the buffer queue to prevent multiple unmap */
  1287. mutex_lock(&tbl.bufq[idx].q_lock);
  1288. tbl.bufq[idx].active = false;
  1289. tbl.bufq[idx].vaddr = 0;
  1290. mutex_unlock(&tbl.bufq[idx].q_lock);
  1291. mutex_unlock(&tbl.m_lock);
  1292. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1293. if (tbl.bufq[idx].dma_buf && tbl.bufq[idx].kmdvaddr) {
  1294. rc = cam_mem_util_unmap_cpu_va(tbl.bufq[idx].dma_buf,
  1295. tbl.bufq[idx].kmdvaddr);
  1296. if (rc)
  1297. CAM_ERR(CAM_MEM,
  1298. "Failed, dmabuf=%pK, kmdvaddr=%pK",
  1299. tbl.bufq[idx].dma_buf,
  1300. (void *) tbl.bufq[idx].kmdvaddr);
  1301. }
  1302. }
  1303. /* SHARED flag gets precedence, all other flags after it */
  1304. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1305. region = CAM_SMMU_REGION_SHARED;
  1306. } else {
  1307. if (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1308. region = CAM_SMMU_REGION_IO;
  1309. }
  1310. if ((tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_READ_WRITE) ||
  1311. (tbl.bufq[idx].flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) ||
  1312. (tbl.bufq[idx].flags & CAM_MEM_FLAG_PROTECTED_MODE)) {
  1313. if (cam_mem_util_unmap_hw_va(idx, region, client))
  1314. CAM_ERR(CAM_MEM, "Failed, dmabuf=%pK",
  1315. tbl.bufq[idx].dma_buf);
  1316. }
  1317. mutex_lock(&tbl.m_lock);
  1318. mutex_lock(&tbl.bufq[idx].q_lock);
  1319. tbl.bufq[idx].flags = 0;
  1320. tbl.bufq[idx].buf_handle = -1;
  1321. memset(tbl.bufq[idx].hdls, 0,
  1322. sizeof(int32_t) * tbl.bufq[idx].num_hdl);
  1323. CAM_DBG(CAM_MEM,
  1324. "Ion buf at idx = %d freeing fd = %d, imported %d, dma_buf %pK, i_ino %lu",
  1325. idx, tbl.bufq[idx].fd, tbl.bufq[idx].is_imported, tbl.bufq[idx].dma_buf,
  1326. tbl.bufq[idx].i_ino);
  1327. if (tbl.bufq[idx].dma_buf)
  1328. dma_buf_put(tbl.bufq[idx].dma_buf);
  1329. tbl.bufq[idx].fd = -1;
  1330. tbl.bufq[idx].i_ino = 0;
  1331. tbl.bufq[idx].dma_buf = NULL;
  1332. tbl.bufq[idx].is_imported = false;
  1333. tbl.bufq[idx].is_internal = false;
  1334. tbl.bufq[idx].len = 0;
  1335. tbl.bufq[idx].num_hdl = 0;
  1336. cam_mem_mgr_reset_presil_params(idx);
  1337. memset(&tbl.bufq[idx].timestamp, 0, sizeof(struct timespec64));
  1338. mutex_unlock(&tbl.bufq[idx].q_lock);
  1339. mutex_destroy(&tbl.bufq[idx].q_lock);
  1340. clear_bit(idx, tbl.bitmap);
  1341. mutex_unlock(&tbl.m_lock);
  1342. return rc;
  1343. }
  1344. int cam_mem_mgr_release(struct cam_mem_mgr_release_cmd *cmd)
  1345. {
  1346. int idx;
  1347. int rc;
  1348. if (!atomic_read(&cam_mem_mgr_state)) {
  1349. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1350. return -EINVAL;
  1351. }
  1352. if (!cmd) {
  1353. CAM_ERR(CAM_MEM, "Invalid argument");
  1354. return -EINVAL;
  1355. }
  1356. idx = CAM_MEM_MGR_GET_HDL_IDX(cmd->buf_handle);
  1357. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1358. CAM_ERR(CAM_MEM, "Incorrect index %d extracted from mem handle",
  1359. idx);
  1360. return -EINVAL;
  1361. }
  1362. if (!tbl.bufq[idx].active) {
  1363. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1364. return -EINVAL;
  1365. }
  1366. if (tbl.bufq[idx].buf_handle != cmd->buf_handle) {
  1367. CAM_ERR(CAM_MEM,
  1368. "Released buf handle %d not matching within table %d, idx=%d",
  1369. cmd->buf_handle, tbl.bufq[idx].buf_handle, idx);
  1370. return -EINVAL;
  1371. }
  1372. CAM_DBG(CAM_MEM, "Releasing hdl = %x, idx = %d", cmd->buf_handle, idx);
  1373. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_USER);
  1374. return rc;
  1375. }
  1376. int cam_mem_mgr_request_mem(struct cam_mem_mgr_request_desc *inp,
  1377. struct cam_mem_mgr_memory_desc *out)
  1378. {
  1379. struct dma_buf *buf = NULL;
  1380. int ion_fd = -1;
  1381. int rc = 0;
  1382. uintptr_t kvaddr;
  1383. dma_addr_t iova = 0;
  1384. size_t request_len = 0;
  1385. uint32_t mem_handle;
  1386. int32_t idx;
  1387. int32_t smmu_hdl = 0;
  1388. int32_t num_hdl = 0;
  1389. unsigned long i_ino = 0;
  1390. enum cam_smmu_region_id region = CAM_SMMU_REGION_SHARED;
  1391. if (!atomic_read(&cam_mem_mgr_state)) {
  1392. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1393. return -EINVAL;
  1394. }
  1395. if (!inp || !out) {
  1396. CAM_ERR(CAM_MEM, "Invalid params");
  1397. return -EINVAL;
  1398. }
  1399. if (!(inp->flags & CAM_MEM_FLAG_HW_READ_WRITE ||
  1400. inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS ||
  1401. inp->flags & CAM_MEM_FLAG_CACHE)) {
  1402. CAM_ERR(CAM_MEM, "Invalid flags for request mem");
  1403. return -EINVAL;
  1404. }
  1405. rc = cam_mem_util_get_dma_buf(inp->size, inp->flags, &buf, &i_ino);
  1406. if (rc) {
  1407. CAM_ERR(CAM_MEM, "ION alloc failed for shared buffer");
  1408. goto ion_fail;
  1409. } else if (!buf) {
  1410. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1411. goto ion_fail;
  1412. } else {
  1413. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1414. }
  1415. /*
  1416. * we are mapping kva always here,
  1417. * update flags so that we do unmap properly
  1418. */
  1419. inp->flags |= CAM_MEM_FLAG_KMD_ACCESS;
  1420. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1421. if (rc) {
  1422. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1423. goto map_fail;
  1424. }
  1425. if (!inp->smmu_hdl) {
  1426. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1427. rc = -EINVAL;
  1428. goto smmu_fail;
  1429. }
  1430. /* SHARED flag gets precedence, all other flags after it */
  1431. if (inp->flags & CAM_MEM_FLAG_HW_SHARED_ACCESS) {
  1432. region = CAM_SMMU_REGION_SHARED;
  1433. } else {
  1434. if (inp->flags & CAM_MEM_FLAG_HW_READ_WRITE)
  1435. region = CAM_SMMU_REGION_IO;
  1436. }
  1437. rc = cam_smmu_map_kernel_iova(inp->smmu_hdl,
  1438. buf,
  1439. CAM_SMMU_MAP_RW,
  1440. &iova,
  1441. &request_len,
  1442. region);
  1443. if (rc < 0) {
  1444. CAM_ERR(CAM_MEM, "SMMU mapping failed");
  1445. goto smmu_fail;
  1446. }
  1447. smmu_hdl = inp->smmu_hdl;
  1448. num_hdl = 1;
  1449. idx = cam_mem_get_slot();
  1450. if (idx < 0) {
  1451. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1452. rc = -ENOMEM;
  1453. goto slot_fail;
  1454. }
  1455. mutex_lock(&tbl.bufq[idx].q_lock);
  1456. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1457. tbl.bufq[idx].dma_buf = buf;
  1458. tbl.bufq[idx].fd = -1;
  1459. tbl.bufq[idx].i_ino = i_ino;
  1460. tbl.bufq[idx].flags = inp->flags;
  1461. tbl.bufq[idx].buf_handle = mem_handle;
  1462. tbl.bufq[idx].kmdvaddr = kvaddr;
  1463. tbl.bufq[idx].vaddr = iova;
  1464. tbl.bufq[idx].len = inp->size;
  1465. tbl.bufq[idx].num_hdl = num_hdl;
  1466. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1467. sizeof(int32_t));
  1468. tbl.bufq[idx].is_imported = false;
  1469. mutex_unlock(&tbl.bufq[idx].q_lock);
  1470. out->kva = kvaddr;
  1471. out->iova = (uint32_t)iova;
  1472. out->smmu_hdl = smmu_hdl;
  1473. out->mem_handle = mem_handle;
  1474. out->len = inp->size;
  1475. out->region = region;
  1476. CAM_DBG(CAM_MEM, "idx=%d, dmabuf=%pK, i_ino=%lu, flags=0x%x, mem_handle=0x%x",
  1477. idx, buf, i_ino, inp->flags, mem_handle);
  1478. return rc;
  1479. slot_fail:
  1480. cam_smmu_unmap_kernel_iova(inp->smmu_hdl,
  1481. buf, region);
  1482. smmu_fail:
  1483. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1484. map_fail:
  1485. dma_buf_put(buf);
  1486. ion_fail:
  1487. return rc;
  1488. }
  1489. EXPORT_SYMBOL(cam_mem_mgr_request_mem);
  1490. int cam_mem_mgr_release_mem(struct cam_mem_mgr_memory_desc *inp)
  1491. {
  1492. int32_t idx;
  1493. int rc;
  1494. if (!atomic_read(&cam_mem_mgr_state)) {
  1495. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1496. return -EINVAL;
  1497. }
  1498. if (!inp) {
  1499. CAM_ERR(CAM_MEM, "Invalid argument");
  1500. return -EINVAL;
  1501. }
  1502. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1503. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1504. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1505. return -EINVAL;
  1506. }
  1507. if (!tbl.bufq[idx].active) {
  1508. if (tbl.bufq[idx].vaddr == 0) {
  1509. CAM_ERR(CAM_MEM, "buffer is released already");
  1510. return 0;
  1511. }
  1512. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1513. return -EINVAL;
  1514. }
  1515. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1516. CAM_ERR(CAM_MEM,
  1517. "Released buf handle not matching within table");
  1518. return -EINVAL;
  1519. }
  1520. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1521. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1522. return rc;
  1523. }
  1524. EXPORT_SYMBOL(cam_mem_mgr_release_mem);
  1525. int cam_mem_mgr_reserve_memory_region(struct cam_mem_mgr_request_desc *inp,
  1526. enum cam_smmu_region_id region,
  1527. struct cam_mem_mgr_memory_desc *out)
  1528. {
  1529. struct dma_buf *buf = NULL;
  1530. int rc = 0;
  1531. int ion_fd = -1;
  1532. dma_addr_t iova = 0;
  1533. size_t request_len = 0;
  1534. uint32_t mem_handle;
  1535. int32_t idx;
  1536. int32_t smmu_hdl = 0;
  1537. int32_t num_hdl = 0;
  1538. uintptr_t kvaddr = 0;
  1539. unsigned long i_ino = 0;
  1540. if (!atomic_read(&cam_mem_mgr_state)) {
  1541. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1542. return -EINVAL;
  1543. }
  1544. if (!inp || !out) {
  1545. CAM_ERR(CAM_MEM, "Invalid param(s)");
  1546. return -EINVAL;
  1547. }
  1548. if (!inp->smmu_hdl) {
  1549. CAM_ERR(CAM_MEM, "Invalid SMMU handle");
  1550. return -EINVAL;
  1551. }
  1552. if ((region != CAM_SMMU_REGION_SECHEAP) &&
  1553. (region != CAM_SMMU_REGION_FWUNCACHED)) {
  1554. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1555. return -EINVAL;
  1556. }
  1557. rc = cam_mem_util_get_dma_buf(inp->size, 0, &buf, &i_ino);
  1558. if (rc) {
  1559. CAM_ERR(CAM_MEM, "ION alloc failed for sec heap buffer");
  1560. goto ion_fail;
  1561. } else if (!buf) {
  1562. CAM_ERR(CAM_MEM, "ION alloc returned NULL buffer");
  1563. goto ion_fail;
  1564. } else {
  1565. CAM_DBG(CAM_MEM, "Got dma_buf = %pK", buf);
  1566. }
  1567. if (inp->flags & CAM_MEM_FLAG_KMD_ACCESS) {
  1568. rc = cam_mem_util_map_cpu_va(buf, &kvaddr, &request_len);
  1569. if (rc) {
  1570. CAM_ERR(CAM_MEM, "Failed to get kernel vaddr");
  1571. goto kmap_fail;
  1572. }
  1573. }
  1574. rc = cam_smmu_reserve_buf_region(region,
  1575. inp->smmu_hdl, buf, &iova, &request_len);
  1576. if (rc) {
  1577. CAM_ERR(CAM_MEM, "Reserving secondary heap failed");
  1578. goto smmu_fail;
  1579. }
  1580. smmu_hdl = inp->smmu_hdl;
  1581. num_hdl = 1;
  1582. idx = cam_mem_get_slot();
  1583. if (idx < 0) {
  1584. CAM_ERR(CAM_MEM, "Failed in getting mem slot, idx=%d", idx);
  1585. rc = -ENOMEM;
  1586. goto slot_fail;
  1587. }
  1588. mutex_lock(&tbl.bufq[idx].q_lock);
  1589. mem_handle = GET_MEM_HANDLE(idx, ion_fd);
  1590. tbl.bufq[idx].fd = -1;
  1591. tbl.bufq[idx].i_ino = i_ino;
  1592. tbl.bufq[idx].dma_buf = buf;
  1593. tbl.bufq[idx].flags = inp->flags;
  1594. tbl.bufq[idx].buf_handle = mem_handle;
  1595. tbl.bufq[idx].kmdvaddr = kvaddr;
  1596. tbl.bufq[idx].vaddr = iova;
  1597. tbl.bufq[idx].len = request_len;
  1598. tbl.bufq[idx].num_hdl = num_hdl;
  1599. memcpy(tbl.bufq[idx].hdls, &smmu_hdl,
  1600. sizeof(int32_t));
  1601. tbl.bufq[idx].is_imported = false;
  1602. mutex_unlock(&tbl.bufq[idx].q_lock);
  1603. out->kva = kvaddr;
  1604. out->iova = (uint32_t)iova;
  1605. out->smmu_hdl = smmu_hdl;
  1606. out->mem_handle = mem_handle;
  1607. out->len = request_len;
  1608. out->region = region;
  1609. return rc;
  1610. slot_fail:
  1611. cam_smmu_release_buf_region(region, smmu_hdl);
  1612. smmu_fail:
  1613. if (region == CAM_SMMU_REGION_FWUNCACHED)
  1614. cam_mem_util_unmap_cpu_va(buf, kvaddr);
  1615. kmap_fail:
  1616. dma_buf_put(buf);
  1617. ion_fail:
  1618. return rc;
  1619. }
  1620. EXPORT_SYMBOL(cam_mem_mgr_reserve_memory_region);
  1621. static void *cam_mem_mgr_user_dump_buf(
  1622. void *dump_struct, uint8_t *addr_ptr)
  1623. {
  1624. struct cam_mem_buf_queue *buf = NULL;
  1625. uint64_t *addr;
  1626. int i = 0;
  1627. buf = (struct cam_mem_buf_queue *)dump_struct;
  1628. addr = (uint64_t *)addr_ptr;
  1629. *addr++ = buf->timestamp.tv_sec;
  1630. *addr++ = buf->timestamp.tv_nsec / NSEC_PER_USEC;
  1631. *addr++ = buf->fd;
  1632. *addr++ = buf->i_ino;
  1633. *addr++ = buf->buf_handle;
  1634. *addr++ = buf->len;
  1635. *addr++ = buf->align;
  1636. *addr++ = buf->flags;
  1637. *addr++ = buf->vaddr;
  1638. *addr++ = buf->kmdvaddr;
  1639. *addr++ = buf->is_imported;
  1640. *addr++ = buf->is_internal;
  1641. *addr++ = buf->num_hdl;
  1642. for (i = 0; i < buf->num_hdl; i++)
  1643. *addr++ = buf->hdls[i];
  1644. return addr;
  1645. }
  1646. int cam_mem_mgr_dump_user(struct cam_dump_req_cmd *dump_req)
  1647. {
  1648. int rc = 0;
  1649. int i;
  1650. struct cam_common_hw_dump_args dump_args;
  1651. size_t buf_len;
  1652. size_t remain_len;
  1653. uint32_t min_len;
  1654. uintptr_t cpu_addr;
  1655. rc = cam_mem_get_cpu_buf(dump_req->buf_handle,
  1656. &cpu_addr, &buf_len);
  1657. if (rc) {
  1658. CAM_ERR(CAM_MEM, "Invalid handle %u rc %d",
  1659. dump_req->buf_handle, rc);
  1660. return rc;
  1661. }
  1662. if (buf_len <= dump_req->offset) {
  1663. CAM_WARN(CAM_MEM, "Dump buffer overshoot len %zu offset %zu",
  1664. buf_len, dump_req->offset);
  1665. return -ENOSPC;
  1666. }
  1667. remain_len = buf_len - dump_req->offset;
  1668. min_len =
  1669. (CAM_MEM_BUFQ_MAX *
  1670. (CAM_MEM_MGR_DUMP_BUF_NUM_WORDS * sizeof(uint64_t) +
  1671. sizeof(struct cam_common_hw_dump_header)));
  1672. if (remain_len < min_len) {
  1673. CAM_WARN(CAM_MEM, "Dump buffer exhaust remain %zu min %u",
  1674. remain_len, min_len);
  1675. return -ENOSPC;
  1676. }
  1677. dump_args.req_id = dump_req->issue_req_id;
  1678. dump_args.cpu_addr = cpu_addr;
  1679. dump_args.buf_len = buf_len;
  1680. dump_args.offset = dump_req->offset;
  1681. dump_args.ctxt_to_hw_map = NULL;
  1682. mutex_lock(&tbl.m_lock);
  1683. for (i = 1; i < CAM_MEM_BUFQ_MAX; i++) {
  1684. if (tbl.bufq[i].active) {
  1685. mutex_lock(&tbl.bufq[i].q_lock);
  1686. rc = cam_common_user_dump_helper(&dump_args,
  1687. cam_mem_mgr_user_dump_buf,
  1688. &tbl.bufq[i],
  1689. sizeof(uint64_t), "MEM_MGR_BUF.%d:", i);
  1690. if (rc) {
  1691. CAM_ERR(CAM_CRM,
  1692. "Dump state info failed, rc: %d",
  1693. rc);
  1694. return rc;
  1695. }
  1696. mutex_unlock(&tbl.bufq[i].q_lock);
  1697. }
  1698. }
  1699. mutex_unlock(&tbl.m_lock);
  1700. dump_req->offset = dump_args.offset;
  1701. return rc;
  1702. }
  1703. int cam_mem_mgr_free_memory_region(struct cam_mem_mgr_memory_desc *inp)
  1704. {
  1705. int32_t idx;
  1706. int rc;
  1707. int32_t smmu_hdl;
  1708. if (!atomic_read(&cam_mem_mgr_state)) {
  1709. CAM_ERR(CAM_MEM, "failed. mem_mgr not initialized");
  1710. return -EINVAL;
  1711. }
  1712. if (!inp) {
  1713. CAM_ERR(CAM_MEM, "Invalid argument");
  1714. return -EINVAL;
  1715. }
  1716. if ((inp->region != CAM_SMMU_REGION_SECHEAP) &&
  1717. (inp->region != CAM_SMMU_REGION_FWUNCACHED)) {
  1718. CAM_ERR(CAM_MEM, "Only secondary heap supported");
  1719. return -EINVAL;
  1720. }
  1721. idx = CAM_MEM_MGR_GET_HDL_IDX(inp->mem_handle);
  1722. if (idx >= CAM_MEM_BUFQ_MAX || idx <= 0) {
  1723. CAM_ERR(CAM_MEM, "Incorrect index extracted from mem handle");
  1724. return -EINVAL;
  1725. }
  1726. if (!tbl.bufq[idx].active) {
  1727. if (tbl.bufq[idx].vaddr == 0) {
  1728. CAM_ERR(CAM_MEM, "buffer is released already");
  1729. return 0;
  1730. }
  1731. CAM_ERR(CAM_MEM, "Released buffer state should be active");
  1732. return -EINVAL;
  1733. }
  1734. if (tbl.bufq[idx].buf_handle != inp->mem_handle) {
  1735. CAM_ERR(CAM_MEM,
  1736. "Released buf handle not matching within table");
  1737. return -EINVAL;
  1738. }
  1739. if (tbl.bufq[idx].num_hdl != 1) {
  1740. CAM_ERR(CAM_MEM,
  1741. "Sec heap region should have only one smmu hdl");
  1742. return -ENODEV;
  1743. }
  1744. memcpy(&smmu_hdl, tbl.bufq[idx].hdls,
  1745. sizeof(int32_t));
  1746. if (inp->smmu_hdl != smmu_hdl) {
  1747. CAM_ERR(CAM_MEM,
  1748. "Passed SMMU handle doesn't match with internal hdl");
  1749. return -ENODEV;
  1750. }
  1751. rc = cam_smmu_release_buf_region(inp->region, inp->smmu_hdl);
  1752. if (rc) {
  1753. CAM_ERR(CAM_MEM,
  1754. "Sec heap region release failed");
  1755. return -ENODEV;
  1756. }
  1757. CAM_DBG(CAM_MEM, "Releasing hdl = %X", inp->mem_handle);
  1758. rc = cam_mem_util_unmap(idx, CAM_SMMU_MAPPING_KERNEL);
  1759. if (rc)
  1760. CAM_ERR(CAM_MEM, "unmapping secondary heap failed");
  1761. return rc;
  1762. }
  1763. EXPORT_SYMBOL(cam_mem_mgr_free_memory_region);
  1764. #ifdef CONFIG_CAM_PRESIL
  1765. struct dma_buf *cam_mem_mgr_get_dma_buf(int fd)
  1766. {
  1767. struct dma_buf *dmabuf = NULL;
  1768. dmabuf = dma_buf_get(fd);
  1769. if (IS_ERR_OR_NULL((void *)(dmabuf))) {
  1770. CAM_ERR(CAM_MEM, "Failed to import dma_buf for fd");
  1771. return NULL;
  1772. }
  1773. CAM_INFO(CAM_PRESIL, "Received DMA Buf* %pK", dmabuf);
  1774. return dmabuf;
  1775. }
  1776. int cam_mem_mgr_put_dmabuf_from_fd(uint64_t input_dmabuf)
  1777. {
  1778. struct dma_buf *dmabuf = (struct dma_buf *)(uint64_t)input_dmabuf;
  1779. int idx = 0;
  1780. CAM_INFO(CAM_PRESIL, "Received dma_buf :%pK", dmabuf);
  1781. if (!dmabuf) {
  1782. CAM_ERR(CAM_PRESIL, "NULL to import dma_buf fd");
  1783. return -EINVAL;
  1784. }
  1785. for (idx = 0; idx < CAM_MEM_BUFQ_MAX; idx++) {
  1786. if ((tbl.bufq[idx].dma_buf != NULL) && (tbl.bufq[idx].dma_buf == dmabuf)) {
  1787. if (tbl.bufq[idx].presil_params.refcount)
  1788. tbl.bufq[idx].presil_params.refcount--;
  1789. else
  1790. CAM_ERR(CAM_PRESIL, "Unbalanced dmabuf put: %pK", dmabuf);
  1791. if (!tbl.bufq[idx].presil_params.refcount) {
  1792. dma_buf_put(dmabuf);
  1793. cam_mem_mgr_reset_presil_params(idx);
  1794. CAM_DBG(CAM_PRESIL, "Done dma_buf_put for %pK", dmabuf);
  1795. }
  1796. }
  1797. }
  1798. return 0;
  1799. }
  1800. int cam_mem_mgr_get_fd_from_dmabuf(uint64_t input_dmabuf)
  1801. {
  1802. int fd_for_dmabuf = -1;
  1803. struct dma_buf *dmabuf = (struct dma_buf *)(uint64_t)input_dmabuf;
  1804. int idx = 0;
  1805. CAM_DBG(CAM_PRESIL, "Received dma_buf :%pK", dmabuf);
  1806. if (!dmabuf) {
  1807. CAM_ERR(CAM_PRESIL, "NULL to import dma_buf fd");
  1808. return -EINVAL;
  1809. }
  1810. for (idx = 0; idx < CAM_MEM_BUFQ_MAX; idx++) {
  1811. if ((tbl.bufq[idx].dma_buf != NULL) && (tbl.bufq[idx].dma_buf == dmabuf)) {
  1812. CAM_DBG(CAM_PRESIL,
  1813. "Found entry for request from Presil UMD Daemon at %d, dmabuf %pK fd_for_umd_daemon %d refcount: %d",
  1814. idx, tbl.bufq[idx].dma_buf,
  1815. tbl.bufq[idx].presil_params.fd_for_umd_daemon,
  1816. tbl.bufq[idx].presil_params.refcount);
  1817. if (tbl.bufq[idx].presil_params.fd_for_umd_daemon < 0) {
  1818. fd_for_dmabuf = dma_buf_fd(dmabuf, O_CLOEXEC);
  1819. if (fd_for_dmabuf < 0) {
  1820. CAM_ERR(CAM_PRESIL, "get fd fail, fd_for_dmabuf=%d",
  1821. fd_for_dmabuf);
  1822. return -EINVAL;
  1823. }
  1824. tbl.bufq[idx].presil_params.fd_for_umd_daemon = fd_for_dmabuf;
  1825. CAM_INFO(CAM_PRESIL,
  1826. "Received generated idx %d fd_for_dmabuf Buf* %lld", idx,
  1827. fd_for_dmabuf);
  1828. } else {
  1829. fd_for_dmabuf = tbl.bufq[idx].presil_params.fd_for_umd_daemon;
  1830. CAM_INFO(CAM_PRESIL,
  1831. "Received existing at idx %d fd_for_dmabuf Buf* %lld", idx,
  1832. fd_for_dmabuf);
  1833. }
  1834. tbl.bufq[idx].presil_params.refcount++;
  1835. } else {
  1836. CAM_DBG(CAM_MEM,
  1837. "Not found dmabuf at idx=%d, dma_buf %pK handle 0x%0x active %d ",
  1838. idx, tbl.bufq[idx].dma_buf, tbl.bufq[idx].buf_handle,
  1839. tbl.bufq[idx].active);
  1840. }
  1841. }
  1842. return (int)fd_for_dmabuf;
  1843. }
  1844. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1845. {
  1846. int rc = 0;
  1847. /* Sending Presil IO Buf to PC side ( as iova start address indicates) */
  1848. uint64_t io_buf_addr;
  1849. size_t io_buf_size;
  1850. int i, j, fd = -1, idx = 0;
  1851. uint8_t *iova_ptr = NULL;
  1852. uint64_t dmabuf = 0;
  1853. bool is_mapped_in_cb = false;
  1854. CAM_DBG(CAM_PRESIL, "buf handle 0x%0x", buf_handle);
  1855. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  1856. for (i = 0; i < tbl.bufq[idx].num_hdl; i++) {
  1857. if (tbl.bufq[idx].hdls[i] == iommu_hdl)
  1858. is_mapped_in_cb = true;
  1859. }
  1860. if (!is_mapped_in_cb) {
  1861. for (j = 0; j < CAM_MEM_BUFQ_MAX; j++) {
  1862. if (tbl.bufq[j].i_ino == tbl.bufq[idx].i_ino) {
  1863. for (i = 0; i < tbl.bufq[j].num_hdl; i++) {
  1864. if (tbl.bufq[j].hdls[i] == iommu_hdl)
  1865. is_mapped_in_cb = true;
  1866. }
  1867. }
  1868. }
  1869. if (!is_mapped_in_cb) {
  1870. CAM_DBG(CAM_PRESIL,
  1871. "Still Could not find idx=%d, FD %d buf_handle 0x%0x",
  1872. idx, GET_FD_FROM_HANDLE(buf_handle), buf_handle);
  1873. /*
  1874. * Okay to return 0, since this function also gets called for buffers that
  1875. * are shared only between umd/kmd, these may not be mapped with smmu
  1876. */
  1877. return 0;
  1878. }
  1879. }
  1880. if ((tbl.bufq[idx].buf_handle != 0) && (tbl.bufq[idx].active) &&
  1881. (tbl.bufq[idx].buf_handle == buf_handle)) {
  1882. CAM_DBG(CAM_PRESIL,
  1883. "Found dmabuf in bufq idx %d, FD %d handle 0x%0x dmabuf %pK",
  1884. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].dma_buf);
  1885. dmabuf = (uint64_t)tbl.bufq[idx].dma_buf;
  1886. fd = tbl.bufq[idx].fd;
  1887. } else {
  1888. CAM_ERR(CAM_PRESIL,
  1889. "Could not find dmabuf Invalid Mem idx=%d, FD %d handle 0x%0x active %d",
  1890. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].active);
  1891. return -EINVAL;
  1892. }
  1893. rc = cam_mem_get_io_buf(buf_handle, iommu_hdl, &io_buf_addr, &io_buf_size, NULL);
  1894. if (rc || NULL == (void *)io_buf_addr) {
  1895. CAM_DBG(CAM_PRESIL, "Invalid ioaddr : 0x%x, fd = %d, dmabuf = %pK",
  1896. io_buf_addr, fd, dmabuf);
  1897. return -EINVAL;
  1898. }
  1899. iova_ptr = (uint8_t *)io_buf_addr;
  1900. CAM_INFO(CAM_PRESIL, "Sending buffer with ioaddr : 0x%x, fd = %d, dmabuf = %pK",
  1901. io_buf_addr, fd, dmabuf);
  1902. rc = cam_presil_send_buffer(dmabuf, 0, 0, (uint32_t)io_buf_size, (uint64_t)iova_ptr);
  1903. return rc;
  1904. }
  1905. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1906. {
  1907. int idx = 0;
  1908. int rc = 0;
  1909. int32_t fd_already_sent[128];
  1910. int fd_already_sent_count = 0;
  1911. int fd_already_index = 0;
  1912. int fd_already_sent_found = 0;
  1913. memset(&fd_already_sent, 0x0, sizeof(fd_already_sent));
  1914. for (idx = 0; idx < CAM_MEM_BUFQ_MAX; idx++) {
  1915. if ((tbl.bufq[idx].buf_handle != 0) && (tbl.bufq[idx].active)) {
  1916. CAM_DBG(CAM_PRESIL, "Sending %d, FD %d handle 0x%0x", idx, tbl.bufq[idx].fd,
  1917. tbl.bufq[idx].buf_handle);
  1918. fd_already_sent_found = 0;
  1919. for (fd_already_index = 0; fd_already_index < fd_already_sent_count;
  1920. fd_already_index++) {
  1921. if (fd_already_sent[fd_already_index] == tbl.bufq[idx].fd) {
  1922. fd_already_sent_found = 1;
  1923. CAM_DBG(CAM_PRESIL,
  1924. "fd_already_sent %d, FD %d handle 0x%0x flags=0x%0x",
  1925. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle,
  1926. tbl.bufq[idx].flags);
  1927. }
  1928. }
  1929. if (fd_already_sent_found)
  1930. continue;
  1931. CAM_DBG(CAM_PRESIL, "Sending %d, FD %d handle 0x%0x flags=0x%0x", idx,
  1932. tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].flags);
  1933. rc = cam_mem_mgr_send_buffer_to_presil(iommu_hdl, tbl.bufq[idx].buf_handle);
  1934. fd_already_sent[fd_already_sent_count++] = tbl.bufq[idx].fd;
  1935. } else {
  1936. CAM_DBG(CAM_PRESIL, "Invalid Mem idx=%d, FD %d handle 0x%0x active %d",
  1937. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle,
  1938. tbl.bufq[idx].active);
  1939. }
  1940. }
  1941. return rc;
  1942. }
  1943. EXPORT_SYMBOL(cam_mem_mgr_send_all_buffers_to_presil);
  1944. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle, uint32_t buf_size,
  1945. uint32_t offset, int32_t iommu_hdl)
  1946. {
  1947. int rc = 0;
  1948. /* Receive output buffer from Presil IO Buf to PC side (as iova start address indicates) */
  1949. uint64_t io_buf_addr;
  1950. size_t io_buf_size;
  1951. uint64_t dmabuf = 0;
  1952. int fd = 0;
  1953. uint8_t *iova_ptr = NULL;
  1954. int idx = 0;
  1955. CAM_DBG(CAM_PRESIL, "buf handle 0x%0x ", buf_handle);
  1956. rc = cam_mem_get_io_buf(buf_handle, iommu_hdl, &io_buf_addr, &io_buf_size, NULL);
  1957. if (rc) {
  1958. CAM_ERR(CAM_PRESIL, "Unable to get IOVA for buffer buf_hdl: 0x%0x iommu_hdl: 0x%0x",
  1959. buf_handle, iommu_hdl);
  1960. return -EINVAL;
  1961. }
  1962. iova_ptr = (uint8_t *)io_buf_addr;
  1963. iova_ptr += offset; // correct target address to start writing buffer to.
  1964. if (!buf_size) {
  1965. buf_size = io_buf_size;
  1966. CAM_DBG(CAM_PRESIL, "Updated buf_size from Zero to 0x%0x", buf_size);
  1967. }
  1968. fd = GET_FD_FROM_HANDLE(buf_handle);
  1969. idx = CAM_MEM_MGR_GET_HDL_IDX(buf_handle);
  1970. if ((tbl.bufq[idx].buf_handle != 0) && (tbl.bufq[idx].active) &&
  1971. (tbl.bufq[idx].buf_handle == buf_handle)) {
  1972. CAM_DBG(CAM_PRESIL, "Found dmabuf in bufq idx %d, FD %d handle 0x%0x dmabuf %pK",
  1973. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].dma_buf);
  1974. dmabuf = (uint64_t)tbl.bufq[idx].dma_buf;
  1975. } else {
  1976. CAM_ERR(CAM_PRESIL,
  1977. "Could not find dmabuf Invalid Mem idx=%d, FD %d handle 0x%0x active %d ",
  1978. idx, tbl.bufq[idx].fd, tbl.bufq[idx].buf_handle, tbl.bufq[idx].active);
  1979. }
  1980. CAM_DBG(CAM_PRESIL,
  1981. "Retrieving buffer with ioaddr : 0x%x, offset = %d, size = %d, fd = %d, dmabuf = %pK",
  1982. io_buf_addr, offset, buf_size, fd, dmabuf);
  1983. rc = cam_presil_retrieve_buffer(dmabuf, 0, 0, (uint32_t)buf_size, (uint64_t)io_buf_addr);
  1984. CAM_INFO(CAM_PRESIL,
  1985. "Retrieved buffer with ioaddr : 0x%x, offset = %d, size = %d, fd = %d, dmabuf = %pK",
  1986. io_buf_addr, 0, buf_size, fd, dmabuf);
  1987. return rc;
  1988. }
  1989. #else /* ifdef CONFIG_CAM_PRESIL */
  1990. struct dma_buf * cam_mem_mgr_get_dma_buf(int fd)
  1991. {
  1992. return NULL;
  1993. }
  1994. int cam_mem_mgr_send_all_buffers_to_presil(int32_t iommu_hdl)
  1995. {
  1996. return 0;
  1997. }
  1998. int cam_mem_mgr_send_buffer_to_presil(int32_t iommu_hdl, int32_t buf_handle)
  1999. {
  2000. return 0;
  2001. }
  2002. int cam_mem_mgr_retrieve_buffer_from_presil(int32_t buf_handle,
  2003. uint32_t buf_size,
  2004. uint32_t offset,
  2005. int32_t iommu_hdl)
  2006. {
  2007. return 0;
  2008. }
  2009. #endif /* ifdef CONFIG_CAM_PRESIL */