msm_gem.c 32 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372
  1. /*
  2. * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved.
  3. * Copyright (C) 2013 Red Hat
  4. * Author: Rob Clark <[email protected]>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published by
  8. * the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful, but WITHOUT
  11. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  13. * more details.
  14. *
  15. * You should have received a copy of the GNU General Public License along with
  16. * this program. If not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #include <linux/spinlock.h>
  19. #include <linux/shmem_fs.h>
  20. #include <linux/dma-buf.h>
  21. #include <linux/pfn_t.h>
  22. #include <linux/ion.h>
  23. #include "msm_drv.h"
  24. #include "msm_gem.h"
  25. #include "msm_mmu.h"
  26. #include "sde_dbg.h"
  27. static void msm_gem_vunmap_locked(struct drm_gem_object *obj);
  28. static dma_addr_t physaddr(struct drm_gem_object *obj)
  29. {
  30. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  31. struct msm_drm_private *priv = obj->dev->dev_private;
  32. return (((dma_addr_t)msm_obj->vram_node->start) << PAGE_SHIFT) +
  33. priv->vram.paddr;
  34. }
  35. static bool use_pages(struct drm_gem_object *obj)
  36. {
  37. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  38. return !msm_obj->vram_node;
  39. }
  40. /* allocate pages from VRAM carveout, used when no IOMMU: */
  41. static struct page **get_pages_vram(struct drm_gem_object *obj, int npages)
  42. {
  43. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  44. struct msm_drm_private *priv = obj->dev->dev_private;
  45. dma_addr_t paddr;
  46. struct page **p;
  47. int ret, i;
  48. p = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
  49. if (!p)
  50. return ERR_PTR(-ENOMEM);
  51. spin_lock(&priv->vram.lock);
  52. ret = drm_mm_insert_node(&priv->vram.mm, msm_obj->vram_node, npages);
  53. spin_unlock(&priv->vram.lock);
  54. if (ret) {
  55. kvfree(p);
  56. return ERR_PTR(ret);
  57. }
  58. paddr = physaddr(obj);
  59. for (i = 0; i < npages; i++) {
  60. p[i] = phys_to_page(paddr);
  61. paddr += PAGE_SIZE;
  62. }
  63. return p;
  64. }
  65. static struct page **get_pages(struct drm_gem_object *obj)
  66. {
  67. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  68. struct device *aspace_dev;
  69. if (obj->import_attach)
  70. return msm_obj->pages;
  71. if (!msm_obj->pages) {
  72. struct drm_device *dev = obj->dev;
  73. struct page **p;
  74. int npages = obj->size >> PAGE_SHIFT;
  75. if (use_pages(obj))
  76. p = drm_gem_get_pages(obj);
  77. else
  78. p = get_pages_vram(obj, npages);
  79. if (IS_ERR(p)) {
  80. dev_err(dev->dev, "could not get pages: %ld\n",
  81. PTR_ERR(p));
  82. return p;
  83. }
  84. msm_obj->pages = p;
  85. msm_obj->sgt = drm_prime_pages_to_sg(p, npages);
  86. if (IS_ERR(msm_obj->sgt)) {
  87. void *ptr = ERR_CAST(msm_obj->sgt);
  88. dev_err(dev->dev, "failed to allocate sgt\n");
  89. msm_obj->sgt = NULL;
  90. return ptr;
  91. }
  92. /* For non-cached buffers, ensure the new pages are clean
  93. * because display controller, GPU, etc. are not coherent:
  94. */
  95. if (msm_obj->flags & (MSM_BO_WC|MSM_BO_UNCACHED)) {
  96. aspace_dev = msm_gem_get_aspace_device(msm_obj->aspace);
  97. dma_map_sg(aspace_dev, msm_obj->sgt->sgl,
  98. msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
  99. }
  100. }
  101. return msm_obj->pages;
  102. }
  103. static void put_pages_vram(struct drm_gem_object *obj)
  104. {
  105. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  106. struct msm_drm_private *priv = obj->dev->dev_private;
  107. spin_lock(&priv->vram.lock);
  108. drm_mm_remove_node(msm_obj->vram_node);
  109. spin_unlock(&priv->vram.lock);
  110. kvfree(msm_obj->pages);
  111. }
  112. static void put_pages(struct drm_gem_object *obj)
  113. {
  114. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  115. if (msm_obj->pages) {
  116. if (msm_obj->sgt) {
  117. sg_free_table(msm_obj->sgt);
  118. kfree(msm_obj->sgt);
  119. }
  120. if (use_pages(obj))
  121. drm_gem_put_pages(obj, msm_obj->pages, true, false);
  122. else
  123. put_pages_vram(obj);
  124. msm_obj->pages = NULL;
  125. }
  126. }
  127. struct page **msm_gem_get_pages(struct drm_gem_object *obj)
  128. {
  129. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  130. struct page **p;
  131. mutex_lock(&msm_obj->lock);
  132. if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
  133. mutex_unlock(&msm_obj->lock);
  134. return ERR_PTR(-EBUSY);
  135. }
  136. p = get_pages(obj);
  137. mutex_unlock(&msm_obj->lock);
  138. return p;
  139. }
  140. void msm_gem_put_pages(struct drm_gem_object *obj)
  141. {
  142. /* when we start tracking the pin count, then do something here */
  143. }
  144. void msm_gem_sync(struct drm_gem_object *obj)
  145. {
  146. struct msm_gem_object *msm_obj;
  147. struct device *aspace_dev;
  148. if (!obj)
  149. return;
  150. msm_obj = to_msm_bo(obj);
  151. /*
  152. * dma_sync_sg_for_device synchronises a single contiguous or
  153. * scatter/gather mapping for the CPU and device.
  154. */
  155. aspace_dev = msm_gem_get_aspace_device(msm_obj->aspace);
  156. dma_sync_sg_for_device(aspace_dev, msm_obj->sgt->sgl,
  157. msm_obj->sgt->nents, DMA_BIDIRECTIONAL);
  158. }
  159. int msm_gem_mmap_obj(struct drm_gem_object *obj,
  160. struct vm_area_struct *vma)
  161. {
  162. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  163. vma->vm_flags &= ~VM_PFNMAP;
  164. vma->vm_flags |= VM_MIXEDMAP;
  165. if (msm_obj->flags & MSM_BO_WC) {
  166. vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
  167. } else if (msm_obj->flags & MSM_BO_UNCACHED) {
  168. vma->vm_page_prot = pgprot_noncached(vm_get_page_prot(vma->vm_flags));
  169. } else {
  170. /*
  171. * Shunt off cached objs to shmem file so they have their own
  172. * address_space (so unmap_mapping_range does what we want,
  173. * in particular in the case of mmap'd dmabufs)
  174. */
  175. fput(vma->vm_file);
  176. get_file(obj->filp);
  177. vma->vm_pgoff = 0;
  178. vma->vm_file = obj->filp;
  179. vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
  180. }
  181. return 0;
  182. }
  183. int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
  184. {
  185. int ret;
  186. ret = drm_gem_mmap(filp, vma);
  187. if (ret) {
  188. DBG("mmap failed: %d", ret);
  189. return ret;
  190. }
  191. return msm_gem_mmap_obj(vma->vm_private_data, vma);
  192. }
  193. vm_fault_t msm_gem_fault(struct vm_fault *vmf)
  194. {
  195. struct vm_area_struct *vma = vmf->vma;
  196. struct drm_gem_object *obj = vma->vm_private_data;
  197. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  198. struct page **pages;
  199. unsigned long pfn;
  200. pgoff_t pgoff;
  201. int err;
  202. vm_fault_t ret;
  203. /*
  204. * vm_ops.open/drm_gem_mmap_obj and close get and put
  205. * a reference on obj. So, we dont need to hold one here.
  206. */
  207. err = mutex_lock_interruptible(&msm_obj->lock);
  208. if (err) {
  209. ret = VM_FAULT_NOPAGE;
  210. goto out;
  211. }
  212. if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) {
  213. mutex_unlock(&msm_obj->lock);
  214. return VM_FAULT_SIGBUS;
  215. }
  216. /* make sure we have pages attached now */
  217. pages = get_pages(obj);
  218. if (IS_ERR(pages)) {
  219. ret = vmf_error(PTR_ERR(pages));
  220. goto out_unlock;
  221. }
  222. /* We don't use vmf->pgoff since that has the fake offset: */
  223. pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
  224. pfn = page_to_pfn(pages[pgoff]);
  225. VERB("Inserting %pK pfn %lx, pa %lx", (void *)vmf->address,
  226. pfn, pfn << PAGE_SHIFT);
  227. ret = vmf_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV));
  228. out_unlock:
  229. mutex_unlock(&msm_obj->lock);
  230. out:
  231. return ret;
  232. }
  233. /** get mmap offset */
  234. static uint64_t mmap_offset(struct drm_gem_object *obj)
  235. {
  236. struct drm_device *dev = obj->dev;
  237. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  238. int ret;
  239. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  240. /* Make it mmapable */
  241. ret = drm_gem_create_mmap_offset(obj);
  242. if (ret) {
  243. dev_err(dev->dev, "could not allocate mmap offset\n");
  244. return 0;
  245. }
  246. return drm_vma_node_offset_addr(&obj->vma_node);
  247. }
  248. uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
  249. {
  250. uint64_t offset;
  251. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  252. mutex_lock(&msm_obj->lock);
  253. offset = mmap_offset(obj);
  254. mutex_unlock(&msm_obj->lock);
  255. return offset;
  256. }
  257. dma_addr_t msm_gem_get_dma_addr(struct drm_gem_object *obj)
  258. {
  259. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  260. struct sg_table *sgt;
  261. if (!msm_obj->sgt) {
  262. sgt = dma_buf_map_attachment(obj->import_attach,
  263. DMA_BIDIRECTIONAL);
  264. if (IS_ERR_OR_NULL(sgt)) {
  265. DRM_ERROR("dma_buf_map_attachment failure, err=%ld\n",
  266. PTR_ERR(sgt));
  267. return 0;
  268. }
  269. msm_obj->sgt = sgt;
  270. }
  271. return sg_phys(msm_obj->sgt->sgl);
  272. }
  273. static struct msm_gem_vma *add_vma(struct drm_gem_object *obj,
  274. struct msm_gem_address_space *aspace)
  275. {
  276. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  277. struct msm_gem_vma *vma;
  278. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  279. vma = kzalloc(sizeof(*vma), GFP_KERNEL);
  280. if (!vma)
  281. return ERR_PTR(-ENOMEM);
  282. vma->aspace = aspace;
  283. msm_obj->aspace = aspace;
  284. list_add_tail(&vma->list, &msm_obj->vmas);
  285. return vma;
  286. }
  287. static struct msm_gem_vma *lookup_vma(struct drm_gem_object *obj,
  288. struct msm_gem_address_space *aspace)
  289. {
  290. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  291. struct msm_gem_vma *vma;
  292. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  293. list_for_each_entry(vma, &msm_obj->vmas, list) {
  294. if (vma->aspace == aspace)
  295. return vma;
  296. }
  297. return NULL;
  298. }
  299. static void del_vma(struct msm_gem_vma *vma)
  300. {
  301. if (!vma)
  302. return;
  303. list_del(&vma->list);
  304. kfree(vma);
  305. }
  306. /* Called with msm_obj->lock locked */
  307. static void
  308. put_iova(struct drm_gem_object *obj)
  309. {
  310. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  311. struct msm_gem_vma *vma, *tmp;
  312. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  313. list_for_each_entry_safe(vma, tmp, &msm_obj->vmas, list) {
  314. msm_gem_unmap_vma(vma->aspace, vma, msm_obj->sgt,
  315. msm_obj->flags);
  316. /*
  317. * put_iova removes the domain connected to the obj which makes
  318. * the aspace inaccessible. Store the aspace, as it is used to
  319. * update the active_list during gem_free_obj and gem_purge.
  320. */
  321. msm_obj->aspace = vma->aspace;
  322. del_vma(vma);
  323. }
  324. }
  325. /* get iova, taking a reference. Should have a matching put */
  326. static int msm_gem_get_iova_locked(struct drm_gem_object *obj,
  327. struct msm_gem_address_space *aspace, uint64_t *iova)
  328. {
  329. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  330. struct msm_gem_vma *vma;
  331. int ret = 0;
  332. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  333. vma = lookup_vma(obj, aspace);
  334. if (!vma) {
  335. struct page **pages;
  336. struct device *dev;
  337. struct dma_buf *dmabuf;
  338. bool reattach = false;
  339. dev = msm_gem_get_aspace_device(aspace);
  340. if (dev && obj->import_attach &&
  341. (dev != obj->import_attach->dev)) {
  342. dmabuf = obj->import_attach->dmabuf;
  343. DRM_DEBUG("detach nsec-dev:%pK attach sec-dev:%pK\n",
  344. obj->import_attach->dev, dev);
  345. SDE_EVT32(obj->import_attach->dev, dev, msm_obj->sgt);
  346. if (msm_obj->sgt)
  347. dma_buf_unmap_attachment(obj->import_attach,
  348. msm_obj->sgt, DMA_BIDIRECTIONAL);
  349. dma_buf_detach(dmabuf, obj->import_attach);
  350. obj->import_attach = dma_buf_attach(dmabuf, dev);
  351. if (IS_ERR(obj->import_attach)) {
  352. DRM_ERROR("dma_buf_attach failure, err=%ld\n",
  353. PTR_ERR(obj->import_attach));
  354. goto unlock;
  355. }
  356. reattach = true;
  357. }
  358. /* perform delayed import for buffers without existing sgt */
  359. if (((msm_obj->flags & MSM_BO_EXTBUF) && !(msm_obj->sgt))
  360. || reattach) {
  361. ret = msm_gem_delayed_import(obj);
  362. if (ret) {
  363. DRM_ERROR("delayed dma-buf import failed %d\n",
  364. ret);
  365. goto unlock;
  366. }
  367. }
  368. vma = add_vma(obj, aspace);
  369. if (IS_ERR(vma)) {
  370. ret = PTR_ERR(vma);
  371. goto unlock;
  372. }
  373. pages = get_pages(obj);
  374. if (IS_ERR(pages)) {
  375. ret = PTR_ERR(pages);
  376. goto fail;
  377. }
  378. ret = msm_gem_map_vma(aspace, vma, msm_obj->sgt,
  379. obj->size >> PAGE_SHIFT,
  380. msm_obj->flags);
  381. if (ret)
  382. goto fail;
  383. }
  384. *iova = vma->iova;
  385. if (aspace && !msm_obj->in_active_list) {
  386. mutex_lock(&aspace->list_lock);
  387. msm_gem_add_obj_to_aspace_active_list(aspace, obj);
  388. mutex_unlock(&aspace->list_lock);
  389. }
  390. mutex_unlock(&msm_obj->lock);
  391. return 0;
  392. fail:
  393. del_vma(vma);
  394. unlock:
  395. mutex_unlock(&msm_obj->lock);
  396. return ret;
  397. }
  398. static int msm_gem_pin_iova(struct drm_gem_object *obj,
  399. struct msm_gem_address_space *aspace)
  400. {
  401. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  402. struct msm_gem_vma *vma;
  403. struct page **pages;
  404. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  405. if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED))
  406. return -EBUSY;
  407. vma = lookup_vma(obj, aspace);
  408. if (WARN_ON(!vma))
  409. return -EINVAL;
  410. pages = get_pages(obj);
  411. if (IS_ERR(pages))
  412. return PTR_ERR(pages);
  413. return msm_gem_map_vma(aspace, vma, msm_obj->sgt,
  414. obj->size >> PAGE_SHIFT, msm_obj->flags);
  415. }
  416. /* get iova and pin it. Should have a matching put */
  417. int msm_gem_get_and_pin_iova(struct drm_gem_object *obj,
  418. struct msm_gem_address_space *aspace, uint64_t *iova)
  419. {
  420. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  421. u64 local;
  422. int ret;
  423. mutex_lock(&msm_obj->lock);
  424. ret = msm_gem_get_iova_locked(obj, aspace, &local);
  425. if (!ret)
  426. ret = msm_gem_pin_iova(obj, aspace);
  427. if (!ret)
  428. *iova = local;
  429. mutex_unlock(&msm_obj->lock);
  430. return ret;
  431. }
  432. int msm_gem_get_iova(struct drm_gem_object *obj,
  433. struct msm_gem_address_space *aspace, uint64_t *iova)
  434. {
  435. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  436. int ret;
  437. mutex_lock(&msm_obj->lock);
  438. ret = msm_gem_get_iova_locked(obj, aspace, iova);
  439. mutex_unlock(&msm_obj->lock);
  440. return ret;
  441. }
  442. /* get iova without taking a reference, used in places where you have
  443. * already done a 'msm_gem_get_iova()'.
  444. */
  445. uint64_t msm_gem_iova(struct drm_gem_object *obj,
  446. struct msm_gem_address_space *aspace)
  447. {
  448. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  449. struct msm_gem_vma *vma;
  450. mutex_lock(&msm_obj->lock);
  451. vma = lookup_vma(obj, aspace);
  452. mutex_unlock(&msm_obj->lock);
  453. WARN_ON(!vma);
  454. return vma ? vma->iova : 0;
  455. }
  456. /*
  457. * Unpin a iova by updating the reference counts. The memory isn't actually
  458. * purged until something else (shrinker, mm_notifier, destroy, etc) decides
  459. * to get rid of it
  460. */
  461. void msm_gem_unpin_iova(struct drm_gem_object *obj,
  462. struct msm_gem_address_space *aspace)
  463. {
  464. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  465. struct msm_gem_vma *vma;
  466. mutex_lock(&msm_obj->lock);
  467. vma = lookup_vma(obj, aspace);
  468. if (!WARN_ON(!vma))
  469. msm_gem_unmap_vma(vma->aspace, vma, msm_obj->sgt,
  470. msm_obj->flags);
  471. mutex_unlock(&msm_obj->lock);
  472. }
  473. void msm_gem_put_iova(struct drm_gem_object *obj,
  474. struct msm_gem_address_space *aspace)
  475. {
  476. // XXX TODO ..
  477. // NOTE: probably don't need a _locked() version.. we wouldn't
  478. // normally unmap here, but instead just mark that it could be
  479. // unmapped (if the iova refcnt drops to zero), but then later
  480. // if another _get_iova_locked() fails we can start unmapping
  481. // things that are no longer needed..
  482. }
  483. void msm_gem_aspace_domain_attach_detach_update(
  484. struct msm_gem_address_space *aspace,
  485. bool is_detach)
  486. {
  487. struct msm_gem_object *msm_obj;
  488. struct drm_gem_object *obj;
  489. struct aspace_client *aclient;
  490. int ret;
  491. uint64_t iova;
  492. if (!aspace)
  493. return;
  494. mutex_lock(&aspace->list_lock);
  495. if (is_detach) {
  496. /* Indicate to clients domain is getting detached */
  497. list_for_each_entry(aclient, &aspace->clients, list) {
  498. if (aclient->cb)
  499. aclient->cb(aclient->cb_data,
  500. is_detach);
  501. }
  502. /**
  503. * Unmap active buffers,
  504. * typically clients should do this when the callback is called,
  505. * but this needs to be done for the buffers which are not
  506. * attached to any planes.
  507. */
  508. list_for_each_entry(msm_obj, &aspace->active_list, iova_list) {
  509. obj = &msm_obj->base;
  510. if (obj->import_attach) {
  511. mutex_lock(&msm_obj->lock);
  512. put_iova(obj);
  513. mutex_unlock(&msm_obj->lock);
  514. }
  515. }
  516. } else {
  517. /* map active buffers */
  518. list_for_each_entry(msm_obj, &aspace->active_list, iova_list) {
  519. obj = &msm_obj->base;
  520. ret = msm_gem_get_iova(obj, aspace, &iova);
  521. if (ret) {
  522. mutex_unlock(&aspace->list_lock);
  523. return;
  524. }
  525. }
  526. /* Indicate to clients domain is attached */
  527. list_for_each_entry(aclient, &aspace->clients, list) {
  528. if (aclient->cb)
  529. aclient->cb(aclient->cb_data,
  530. is_detach);
  531. }
  532. }
  533. mutex_unlock(&aspace->list_lock);
  534. }
  535. int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
  536. struct drm_mode_create_dumb *args)
  537. {
  538. args->pitch = align_pitch(args->width, args->bpp);
  539. args->size = PAGE_ALIGN(args->pitch * args->height);
  540. return msm_gem_new_handle(dev, file, args->size,
  541. MSM_BO_SCANOUT | MSM_BO_WC, &args->handle, "dumb");
  542. }
  543. int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
  544. uint32_t handle, uint64_t *offset)
  545. {
  546. struct drm_gem_object *obj;
  547. int ret = 0;
  548. /* GEM does all our handle to object mapping */
  549. obj = drm_gem_object_lookup(file, handle);
  550. if (obj == NULL) {
  551. ret = -ENOENT;
  552. goto fail;
  553. }
  554. *offset = msm_gem_mmap_offset(obj);
  555. drm_gem_object_put_unlocked(obj);
  556. fail:
  557. return ret;
  558. }
  559. static void *get_vaddr(struct drm_gem_object *obj, unsigned madv)
  560. {
  561. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  562. int ret = 0;
  563. mutex_lock(&msm_obj->lock);
  564. if (WARN_ON(msm_obj->madv > madv)) {
  565. dev_err(obj->dev->dev, "Invalid madv state: %u vs %u\n",
  566. msm_obj->madv, madv);
  567. mutex_unlock(&msm_obj->lock);
  568. return ERR_PTR(-EBUSY);
  569. }
  570. /* increment vmap_count *before* vmap() call, so shrinker can
  571. * check vmap_count (is_vunmapable()) outside of msm_obj->lock.
  572. * This guarantees that we won't try to msm_gem_vunmap() this
  573. * same object from within the vmap() call (while we already
  574. * hold msm_obj->lock)
  575. */
  576. msm_obj->vmap_count++;
  577. if (!msm_obj->vaddr) {
  578. struct page **pages = get_pages(obj);
  579. if (IS_ERR(pages)) {
  580. ret = PTR_ERR(pages);
  581. goto fail;
  582. }
  583. if (obj->import_attach) {
  584. ret = dma_buf_begin_cpu_access(
  585. obj->import_attach->dmabuf, DMA_BIDIRECTIONAL);
  586. if (ret)
  587. goto fail;
  588. msm_obj->vaddr =
  589. dma_buf_vmap(obj->import_attach->dmabuf);
  590. } else {
  591. msm_obj->vaddr = vmap(pages, obj->size >> PAGE_SHIFT,
  592. VM_MAP, pgprot_writecombine(PAGE_KERNEL));
  593. }
  594. if (msm_obj->vaddr == NULL) {
  595. ret = -ENOMEM;
  596. goto fail;
  597. }
  598. }
  599. mutex_unlock(&msm_obj->lock);
  600. return msm_obj->vaddr;
  601. fail:
  602. msm_obj->vmap_count--;
  603. mutex_unlock(&msm_obj->lock);
  604. return ERR_PTR(ret);
  605. }
  606. void *msm_gem_get_vaddr(struct drm_gem_object *obj)
  607. {
  608. return get_vaddr(obj, MSM_MADV_WILLNEED);
  609. }
  610. /*
  611. * Don't use this! It is for the very special case of dumping
  612. * submits from GPU hangs or faults, were the bo may already
  613. * be MSM_MADV_DONTNEED, but we know the buffer is still on the
  614. * active list.
  615. */
  616. void *msm_gem_get_vaddr_active(struct drm_gem_object *obj)
  617. {
  618. return get_vaddr(obj, __MSM_MADV_PURGED);
  619. }
  620. void msm_gem_put_vaddr(struct drm_gem_object *obj)
  621. {
  622. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  623. mutex_lock(&msm_obj->lock);
  624. WARN_ON(msm_obj->vmap_count < 1);
  625. msm_obj->vmap_count--;
  626. mutex_unlock(&msm_obj->lock);
  627. }
  628. /* Update madvise status, returns true if not purged, else
  629. * false or -errno.
  630. */
  631. int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
  632. {
  633. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  634. mutex_lock(&msm_obj->lock);
  635. WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
  636. if (msm_obj->madv != __MSM_MADV_PURGED)
  637. msm_obj->madv = madv;
  638. madv = msm_obj->madv;
  639. mutex_unlock(&msm_obj->lock);
  640. return (madv != __MSM_MADV_PURGED);
  641. }
  642. void msm_gem_purge(struct drm_gem_object *obj, enum msm_gem_lock subclass)
  643. {
  644. struct drm_device *dev = obj->dev;
  645. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  646. WARN_ON(!mutex_is_locked(&dev->struct_mutex));
  647. WARN_ON(!is_purgeable(msm_obj));
  648. WARN_ON(obj->import_attach);
  649. mutex_lock_nested(&msm_obj->lock, subclass);
  650. put_iova(obj);
  651. if (msm_obj->aspace) {
  652. mutex_lock(&msm_obj->aspace->list_lock);
  653. msm_gem_remove_obj_from_aspace_active_list(msm_obj->aspace,
  654. obj);
  655. mutex_unlock(&msm_obj->aspace->list_lock);
  656. }
  657. msm_gem_vunmap_locked(obj);
  658. put_pages(obj);
  659. msm_obj->madv = __MSM_MADV_PURGED;
  660. drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
  661. drm_gem_free_mmap_offset(obj);
  662. /* Our goal here is to return as much of the memory as
  663. * is possible back to the system as we are called from OOM.
  664. * To do this we must instruct the shmfs to drop all of its
  665. * backing pages, *now*.
  666. */
  667. shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1);
  668. invalidate_mapping_pages(file_inode(obj->filp)->i_mapping,
  669. 0, (loff_t)-1);
  670. mutex_unlock(&msm_obj->lock);
  671. }
  672. static void msm_gem_vunmap_locked(struct drm_gem_object *obj)
  673. {
  674. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  675. WARN_ON(!mutex_is_locked(&msm_obj->lock));
  676. if (!msm_obj->vaddr || WARN_ON(!is_vunmapable(msm_obj)))
  677. return;
  678. if (obj->import_attach) {
  679. dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
  680. dma_buf_end_cpu_access(obj->import_attach->dmabuf,
  681. DMA_BIDIRECTIONAL);
  682. } else {
  683. vunmap(msm_obj->vaddr);
  684. }
  685. msm_obj->vaddr = NULL;
  686. }
  687. void msm_gem_vunmap(struct drm_gem_object *obj, enum msm_gem_lock subclass)
  688. {
  689. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  690. mutex_lock_nested(&msm_obj->lock, subclass);
  691. msm_gem_vunmap_locked(obj);
  692. mutex_unlock(&msm_obj->lock);
  693. }
  694. int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op, ktime_t *timeout)
  695. {
  696. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  697. bool write = !!(op & MSM_PREP_WRITE);
  698. unsigned long remain =
  699. op & MSM_PREP_NOSYNC ? 0 : timeout_to_jiffies(timeout);
  700. long ret;
  701. ret = dma_resv_wait_timeout_rcu(msm_obj->resv, write,
  702. true, remain);
  703. if (ret == 0)
  704. return remain == 0 ? -EBUSY : -ETIMEDOUT;
  705. else if (ret < 0)
  706. return ret;
  707. /* TODO cache maintenance */
  708. return 0;
  709. }
  710. int msm_gem_cpu_fini(struct drm_gem_object *obj)
  711. {
  712. /* TODO cache maintenance */
  713. return 0;
  714. }
  715. #ifdef CONFIG_DEBUG_FS
  716. static void describe_fence(struct dma_fence *fence, const char *type,
  717. struct seq_file *m)
  718. {
  719. if (!dma_fence_is_signaled(fence))
  720. seq_printf(m, "\t%9s: %s %s seq %llu\n", type,
  721. fence->ops->get_driver_name(fence),
  722. fence->ops->get_timeline_name(fence),
  723. fence->seqno);
  724. }
  725. void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
  726. {
  727. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  728. struct dma_resv *robj = msm_obj->resv;
  729. struct dma_resv_list *fobj;
  730. struct dma_fence *fence;
  731. struct msm_gem_vma *vma;
  732. uint64_t off = drm_vma_node_start(&obj->vma_node);
  733. const char *madv;
  734. mutex_lock(&msm_obj->lock);
  735. switch (msm_obj->madv) {
  736. case __MSM_MADV_PURGED:
  737. madv = " purged";
  738. break;
  739. case MSM_MADV_DONTNEED:
  740. madv = " purgeable";
  741. break;
  742. case MSM_MADV_WILLNEED:
  743. default:
  744. madv = "";
  745. break;
  746. }
  747. seq_printf(m, "%08x: %c %2d (%2d) %08llx %pK\t",
  748. msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
  749. obj->name, kref_read(&obj->refcount),
  750. off, msm_obj->vaddr);
  751. seq_printf(m, " %08zu %9s %-32s\n", obj->size, madv, msm_obj->name);
  752. if (!list_empty(&msm_obj->vmas)) {
  753. seq_puts(m, " vmas:");
  754. list_for_each_entry(vma, &msm_obj->vmas, list)
  755. seq_printf(m, " [%s: %08llx,%s,inuse=%d]", vma->aspace->name,
  756. vma->iova, vma->mapped ? "mapped" : "unmapped",
  757. vma->inuse);
  758. seq_puts(m, "\n");
  759. }
  760. rcu_read_lock();
  761. fobj = rcu_dereference(robj->fence);
  762. if (fobj) {
  763. unsigned int i, shared_count = fobj->shared_count;
  764. for (i = 0; i < shared_count; i++) {
  765. fence = rcu_dereference(fobj->shared[i]);
  766. describe_fence(fence, "Shared", m);
  767. }
  768. }
  769. fence = rcu_dereference(robj->fence_excl);
  770. if (fence)
  771. describe_fence(fence, "Exclusive", m);
  772. rcu_read_unlock();
  773. mutex_unlock(&msm_obj->lock);
  774. }
  775. void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
  776. {
  777. struct msm_gem_object *msm_obj;
  778. int count = 0;
  779. size_t size = 0;
  780. seq_puts(m, " flags id ref offset kaddr size madv name\n");
  781. list_for_each_entry(msm_obj, list, mm_list) {
  782. struct drm_gem_object *obj = &msm_obj->base;
  783. seq_puts(m, " ");
  784. msm_gem_describe(obj, m);
  785. count++;
  786. size += obj->size;
  787. }
  788. seq_printf(m, "Total %d objects, %zu bytes\n", count, size);
  789. }
  790. #endif
  791. /* don't call directly! Use drm_gem_object_put() and friends */
  792. void msm_gem_free_object(struct drm_gem_object *obj)
  793. {
  794. struct drm_device *dev = obj->dev;
  795. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  796. WARN_ON(!mutex_is_locked(&dev->struct_mutex));
  797. /* object should not be on active list: */
  798. WARN_ON(is_active(msm_obj));
  799. list_del(&msm_obj->mm_list);
  800. mutex_lock(&msm_obj->lock);
  801. put_iova(obj);
  802. if (msm_obj->aspace) {
  803. mutex_lock(&msm_obj->aspace->list_lock);
  804. msm_gem_remove_obj_from_aspace_active_list(msm_obj->aspace,
  805. obj);
  806. mutex_unlock(&msm_obj->aspace->list_lock);
  807. }
  808. if (obj->import_attach) {
  809. if (msm_obj->vaddr)
  810. dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
  811. /* Don't drop the pages for imported dmabuf, as they are not
  812. * ours, just free the array we allocated:
  813. */
  814. if (msm_obj->pages)
  815. kvfree(msm_obj->pages);
  816. drm_prime_gem_destroy(obj, msm_obj->sgt);
  817. } else {
  818. msm_gem_vunmap_locked(obj);
  819. put_pages(obj);
  820. }
  821. if (msm_obj->resv == &msm_obj->_resv)
  822. dma_resv_fini(msm_obj->resv);
  823. drm_gem_object_release(obj);
  824. mutex_unlock(&msm_obj->lock);
  825. kfree(msm_obj);
  826. }
  827. /* convenience method to construct a GEM buffer object, and userspace handle */
  828. int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
  829. uint32_t size, uint32_t flags, uint32_t *handle,
  830. char *name)
  831. {
  832. struct drm_gem_object *obj;
  833. int ret;
  834. obj = msm_gem_new(dev, size, flags);
  835. if (IS_ERR(obj))
  836. return PTR_ERR(obj);
  837. if (name)
  838. msm_gem_object_set_name(obj, "%s", name);
  839. ret = drm_gem_handle_create(file, obj, handle);
  840. /* drop reference from allocate - handle holds it now */
  841. drm_gem_object_put_unlocked(obj);
  842. return ret;
  843. }
  844. static int msm_gem_new_impl(struct drm_device *dev,
  845. uint32_t size, uint32_t flags,
  846. struct dma_resv *resv,
  847. struct drm_gem_object **obj,
  848. bool struct_mutex_locked)
  849. {
  850. struct msm_drm_private *priv = dev->dev_private;
  851. struct msm_gem_object *msm_obj;
  852. switch (flags & MSM_BO_CACHE_MASK) {
  853. case MSM_BO_UNCACHED:
  854. case MSM_BO_CACHED:
  855. case MSM_BO_WC:
  856. break;
  857. default:
  858. dev_err(dev->dev, "invalid cache flag: %x\n",
  859. (flags & MSM_BO_CACHE_MASK));
  860. return -EINVAL;
  861. }
  862. msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);
  863. if (!msm_obj)
  864. return -ENOMEM;
  865. mutex_init(&msm_obj->lock);
  866. msm_obj->flags = flags;
  867. msm_obj->madv = MSM_MADV_WILLNEED;
  868. if (resv) {
  869. msm_obj->resv = resv;
  870. } else {
  871. msm_obj->resv = &msm_obj->_resv;
  872. dma_resv_init(msm_obj->resv);
  873. }
  874. INIT_LIST_HEAD(&msm_obj->submit_entry);
  875. INIT_LIST_HEAD(&msm_obj->vmas);
  876. INIT_LIST_HEAD(&msm_obj->iova_list);
  877. msm_obj->aspace = NULL;
  878. msm_obj->in_active_list = false;
  879. if (struct_mutex_locked) {
  880. WARN_ON(!mutex_is_locked(&dev->struct_mutex));
  881. list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
  882. } else {
  883. mutex_lock(&dev->struct_mutex);
  884. list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
  885. mutex_unlock(&dev->struct_mutex);
  886. }
  887. *obj = &msm_obj->base;
  888. return 0;
  889. }
  890. static struct drm_gem_object *_msm_gem_new(struct drm_device *dev,
  891. uint32_t size, uint32_t flags, bool struct_mutex_locked)
  892. {
  893. struct msm_drm_private *priv = dev->dev_private;
  894. struct drm_gem_object *obj = NULL;
  895. bool use_vram = false;
  896. int ret;
  897. size = PAGE_ALIGN(size);
  898. if (!iommu_present(&platform_bus_type))
  899. use_vram = true;
  900. else if ((flags & (MSM_BO_STOLEN | MSM_BO_SCANOUT)) && priv->vram.size)
  901. use_vram = true;
  902. if (WARN_ON(use_vram && !priv->vram.size))
  903. return ERR_PTR(-EINVAL);
  904. /* Disallow zero sized objects as they make the underlying
  905. * infrastructure grumpy
  906. */
  907. if (size == 0)
  908. return ERR_PTR(-EINVAL);
  909. ret = msm_gem_new_impl(dev, size, flags, NULL, &obj, struct_mutex_locked);
  910. if (ret)
  911. goto fail;
  912. if (use_vram) {
  913. struct msm_gem_vma *vma;
  914. struct page **pages;
  915. struct msm_gem_object *msm_obj = to_msm_bo(obj);
  916. mutex_lock(&msm_obj->lock);
  917. vma = add_vma(obj, NULL);
  918. mutex_unlock(&msm_obj->lock);
  919. if (IS_ERR(vma)) {
  920. ret = PTR_ERR(vma);
  921. goto fail;
  922. }
  923. to_msm_bo(obj)->vram_node = &vma->node;
  924. drm_gem_private_object_init(dev, obj, size);
  925. pages = get_pages(obj);
  926. if (IS_ERR(pages)) {
  927. ret = PTR_ERR(pages);
  928. goto fail;
  929. }
  930. vma->iova = physaddr(obj);
  931. } else {
  932. ret = drm_gem_object_init(dev, obj, size);
  933. if (ret)
  934. goto fail;
  935. }
  936. return obj;
  937. fail:
  938. drm_gem_object_put_unlocked(obj);
  939. return ERR_PTR(ret);
  940. }
  941. struct drm_gem_object *msm_gem_new_locked(struct drm_device *dev,
  942. uint32_t size, uint32_t flags)
  943. {
  944. return _msm_gem_new(dev, size, flags, true);
  945. }
  946. struct drm_gem_object *msm_gem_new(struct drm_device *dev,
  947. uint32_t size, uint32_t flags)
  948. {
  949. return _msm_gem_new(dev, size, flags, false);
  950. }
  951. int msm_gem_delayed_import(struct drm_gem_object *obj)
  952. {
  953. struct dma_buf_attachment *attach;
  954. struct sg_table *sgt;
  955. struct msm_gem_object *msm_obj;
  956. int ret = 0;
  957. if (!obj) {
  958. DRM_ERROR("NULL drm gem object\n");
  959. return -EINVAL;
  960. }
  961. msm_obj = to_msm_bo(obj);
  962. if (!obj->import_attach) {
  963. DRM_ERROR("NULL dma_buf_attachment in drm gem object\n");
  964. return -EINVAL;
  965. }
  966. attach = obj->import_attach;
  967. attach->dma_map_attrs |= DMA_ATTR_DELAYED_UNMAP;
  968. if (msm_obj->flags & MSM_BO_SKIPSYNC)
  969. attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  970. if (msm_obj->flags & MSM_BO_KEEPATTRS)
  971. attach->dma_map_attrs |=
  972. DMA_ATTR_IOMMU_USE_UPSTREAM_HINT;
  973. /*
  974. * dma_buf_map_attachment will call dma_map_sg for ion buffer
  975. * mapping, and iova will get mapped when the function returns.
  976. */
  977. sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
  978. if (IS_ERR(sgt)) {
  979. ret = PTR_ERR(sgt);
  980. DRM_ERROR("dma_buf_map_attachment failure, err=%d\n",
  981. ret);
  982. goto fail_import;
  983. }
  984. msm_obj->sgt = sgt;
  985. msm_obj->pages = NULL;
  986. fail_import:
  987. return ret;
  988. }
  989. struct drm_gem_object *msm_gem_import(struct drm_device *dev,
  990. struct dma_buf *dmabuf, struct sg_table *sgt)
  991. {
  992. struct msm_gem_object *msm_obj;
  993. struct drm_gem_object *obj = NULL;
  994. uint32_t size;
  995. int ret;
  996. unsigned long flags = 0;
  997. /* if we don't have IOMMU, don't bother pretending we can import: */
  998. if (!iommu_present(&platform_bus_type)) {
  999. dev_err(dev->dev, "cannot import without IOMMU\n");
  1000. return ERR_PTR(-EINVAL);
  1001. }
  1002. size = PAGE_ALIGN(dmabuf->size);
  1003. ret = msm_gem_new_impl(dev, size, MSM_BO_WC, dmabuf->resv, &obj,
  1004. false);
  1005. if (ret)
  1006. goto fail;
  1007. drm_gem_private_object_init(dev, obj, size);
  1008. msm_obj = to_msm_bo(obj);
  1009. mutex_lock(&msm_obj->lock);
  1010. msm_obj->sgt = sgt;
  1011. msm_obj->pages = NULL;
  1012. /*
  1013. * 1) If sg table is NULL, user should call msm_gem_delayed_import
  1014. * to add back the sg table to the drm gem object.
  1015. *
  1016. * 2) Add buffer flag unconditionally for all import cases.
  1017. * # Cached buffer will be attached immediately hence sgt will
  1018. * be available upon gem obj creation.
  1019. * # Un-cached buffer will follow delayed attach hence sgt
  1020. * will be NULL upon gem obj creation.
  1021. */
  1022. msm_obj->flags |= MSM_BO_EXTBUF;
  1023. /*
  1024. * For all uncached buffers, there is no need to perform cache
  1025. * maintenance on dma map/unmap time.
  1026. */
  1027. ret = dma_buf_get_flags(dmabuf, &flags);
  1028. if (ret) {
  1029. DRM_ERROR("dma_buf_get_flags failure, err=%d\n", ret);
  1030. } else if ((flags & ION_FLAG_CACHED) == 0) {
  1031. DRM_DEBUG("Buffer is uncached type\n");
  1032. msm_obj->flags |= MSM_BO_SKIPSYNC;
  1033. }
  1034. mutex_unlock(&msm_obj->lock);
  1035. return obj;
  1036. fail:
  1037. drm_gem_object_put_unlocked(obj);
  1038. return ERR_PTR(ret);
  1039. }
  1040. static void *_msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
  1041. uint32_t flags, struct msm_gem_address_space *aspace,
  1042. struct drm_gem_object **bo, uint64_t *iova, bool locked)
  1043. {
  1044. void *vaddr;
  1045. struct drm_gem_object *obj = _msm_gem_new(dev, size, flags, locked);
  1046. int ret;
  1047. if (IS_ERR(obj))
  1048. return ERR_CAST(obj);
  1049. if (iova) {
  1050. ret = msm_gem_get_iova(obj, aspace, iova);
  1051. if (ret)
  1052. goto err;
  1053. }
  1054. vaddr = msm_gem_get_vaddr(obj);
  1055. if (IS_ERR(vaddr)) {
  1056. msm_gem_put_iova(obj, aspace);
  1057. ret = PTR_ERR(vaddr);
  1058. goto err;
  1059. }
  1060. if (bo)
  1061. *bo = obj;
  1062. return vaddr;
  1063. err:
  1064. if (locked)
  1065. drm_gem_object_put(obj);
  1066. else
  1067. drm_gem_object_put_unlocked(obj);
  1068. return ERR_PTR(ret);
  1069. }
  1070. void *msm_gem_kernel_new(struct drm_device *dev, uint32_t size,
  1071. uint32_t flags, struct msm_gem_address_space *aspace,
  1072. struct drm_gem_object **bo, uint64_t *iova)
  1073. {
  1074. return _msm_gem_kernel_new(dev, size, flags, aspace, bo, iova, false);
  1075. }
  1076. void *msm_gem_kernel_new_locked(struct drm_device *dev, uint32_t size,
  1077. uint32_t flags, struct msm_gem_address_space *aspace,
  1078. struct drm_gem_object **bo, uint64_t *iova)
  1079. {
  1080. return _msm_gem_kernel_new(dev, size, flags, aspace, bo, iova, true);
  1081. }
  1082. void msm_gem_kernel_put(struct drm_gem_object *bo,
  1083. struct msm_gem_address_space *aspace, bool locked)
  1084. {
  1085. if (IS_ERR_OR_NULL(bo))
  1086. return;
  1087. msm_gem_put_vaddr(bo);
  1088. msm_gem_unpin_iova(bo, aspace);
  1089. if (locked)
  1090. drm_gem_object_put(bo);
  1091. else
  1092. drm_gem_object_put_unlocked(bo);
  1093. }
  1094. void msm_gem_object_set_name(struct drm_gem_object *bo, const char *fmt, ...)
  1095. {
  1096. struct msm_gem_object *msm_obj = to_msm_bo(bo);
  1097. va_list ap;
  1098. if (!fmt)
  1099. return;
  1100. va_start(ap, fmt);
  1101. vsnprintf(msm_obj->name, sizeof(msm_obj->name), fmt, ap);
  1102. va_end(ap);
  1103. }