omap-iommu.c 41 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * omap iommu: tlb and pagetable primitives
  4. *
  5. * Copyright (C) 2008-2010 Nokia Corporation
  6. * Copyright (C) 2013-2017 Texas Instruments Incorporated - https://www.ti.com/
  7. *
  8. * Written by Hiroshi DOYU <[email protected]>,
  9. * Paul Mundt and Toshihiro Kobayashi
  10. */
  11. #include <linux/dma-mapping.h>
  12. #include <linux/err.h>
  13. #include <linux/slab.h>
  14. #include <linux/interrupt.h>
  15. #include <linux/ioport.h>
  16. #include <linux/platform_device.h>
  17. #include <linux/iommu.h>
  18. #include <linux/omap-iommu.h>
  19. #include <linux/mutex.h>
  20. #include <linux/spinlock.h>
  21. #include <linux/io.h>
  22. #include <linux/pm_runtime.h>
  23. #include <linux/of.h>
  24. #include <linux/of_irq.h>
  25. #include <linux/of_platform.h>
  26. #include <linux/regmap.h>
  27. #include <linux/mfd/syscon.h>
  28. #include <linux/platform_data/iommu-omap.h>
  29. #include "omap-iopgtable.h"
  30. #include "omap-iommu.h"
  31. static const struct iommu_ops omap_iommu_ops;
  32. #define to_iommu(dev) ((struct omap_iommu *)dev_get_drvdata(dev))
  33. /* bitmap of the page sizes currently supported */
  34. #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M)
  35. #define MMU_LOCK_BASE_SHIFT 10
  36. #define MMU_LOCK_BASE_MASK (0x1f << MMU_LOCK_BASE_SHIFT)
  37. #define MMU_LOCK_BASE(x) \
  38. ((x & MMU_LOCK_BASE_MASK) >> MMU_LOCK_BASE_SHIFT)
  39. #define MMU_LOCK_VICT_SHIFT 4
  40. #define MMU_LOCK_VICT_MASK (0x1f << MMU_LOCK_VICT_SHIFT)
  41. #define MMU_LOCK_VICT(x) \
  42. ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT)
  43. static struct platform_driver omap_iommu_driver;
  44. static struct kmem_cache *iopte_cachep;
  45. /**
  46. * to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
  47. * @dom: generic iommu domain handle
  48. **/
  49. static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom)
  50. {
  51. return container_of(dom, struct omap_iommu_domain, domain);
  52. }
  53. /**
  54. * omap_iommu_save_ctx - Save registers for pm off-mode support
  55. * @dev: client device
  56. *
  57. * This should be treated as an deprecated API. It is preserved only
  58. * to maintain existing functionality for OMAP3 ISP driver.
  59. **/
  60. void omap_iommu_save_ctx(struct device *dev)
  61. {
  62. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  63. struct omap_iommu *obj;
  64. u32 *p;
  65. int i;
  66. if (!arch_data)
  67. return;
  68. while (arch_data->iommu_dev) {
  69. obj = arch_data->iommu_dev;
  70. p = obj->ctx;
  71. for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
  72. p[i] = iommu_read_reg(obj, i * sizeof(u32));
  73. dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i,
  74. p[i]);
  75. }
  76. arch_data++;
  77. }
  78. }
  79. EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
  80. /**
  81. * omap_iommu_restore_ctx - Restore registers for pm off-mode support
  82. * @dev: client device
  83. *
  84. * This should be treated as an deprecated API. It is preserved only
  85. * to maintain existing functionality for OMAP3 ISP driver.
  86. **/
  87. void omap_iommu_restore_ctx(struct device *dev)
  88. {
  89. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  90. struct omap_iommu *obj;
  91. u32 *p;
  92. int i;
  93. if (!arch_data)
  94. return;
  95. while (arch_data->iommu_dev) {
  96. obj = arch_data->iommu_dev;
  97. p = obj->ctx;
  98. for (i = 0; i < (MMU_REG_SIZE / sizeof(u32)); i++) {
  99. iommu_write_reg(obj, p[i], i * sizeof(u32));
  100. dev_dbg(obj->dev, "%s\t[%02d] %08x\n", __func__, i,
  101. p[i]);
  102. }
  103. arch_data++;
  104. }
  105. }
  106. EXPORT_SYMBOL_GPL(omap_iommu_restore_ctx);
  107. static void dra7_cfg_dspsys_mmu(struct omap_iommu *obj, bool enable)
  108. {
  109. u32 val, mask;
  110. if (!obj->syscfg)
  111. return;
  112. mask = (1 << (obj->id * DSP_SYS_MMU_CONFIG_EN_SHIFT));
  113. val = enable ? mask : 0;
  114. regmap_update_bits(obj->syscfg, DSP_SYS_MMU_CONFIG, mask, val);
  115. }
  116. static void __iommu_set_twl(struct omap_iommu *obj, bool on)
  117. {
  118. u32 l = iommu_read_reg(obj, MMU_CNTL);
  119. if (on)
  120. iommu_write_reg(obj, MMU_IRQ_TWL_MASK, MMU_IRQENABLE);
  121. else
  122. iommu_write_reg(obj, MMU_IRQ_TLB_MISS_MASK, MMU_IRQENABLE);
  123. l &= ~MMU_CNTL_MASK;
  124. if (on)
  125. l |= (MMU_CNTL_MMU_EN | MMU_CNTL_TWL_EN);
  126. else
  127. l |= (MMU_CNTL_MMU_EN);
  128. iommu_write_reg(obj, l, MMU_CNTL);
  129. }
  130. static int omap2_iommu_enable(struct omap_iommu *obj)
  131. {
  132. u32 l, pa;
  133. if (!obj->iopgd || !IS_ALIGNED((unsigned long)obj->iopgd, SZ_16K))
  134. return -EINVAL;
  135. pa = virt_to_phys(obj->iopgd);
  136. if (!IS_ALIGNED(pa, SZ_16K))
  137. return -EINVAL;
  138. l = iommu_read_reg(obj, MMU_REVISION);
  139. dev_info(obj->dev, "%s: version %d.%d\n", obj->name,
  140. (l >> 4) & 0xf, l & 0xf);
  141. iommu_write_reg(obj, pa, MMU_TTB);
  142. dra7_cfg_dspsys_mmu(obj, true);
  143. if (obj->has_bus_err_back)
  144. iommu_write_reg(obj, MMU_GP_REG_BUS_ERR_BACK_EN, MMU_GP_REG);
  145. __iommu_set_twl(obj, true);
  146. return 0;
  147. }
  148. static void omap2_iommu_disable(struct omap_iommu *obj)
  149. {
  150. u32 l = iommu_read_reg(obj, MMU_CNTL);
  151. l &= ~MMU_CNTL_MASK;
  152. iommu_write_reg(obj, l, MMU_CNTL);
  153. dra7_cfg_dspsys_mmu(obj, false);
  154. dev_dbg(obj->dev, "%s is shutting down\n", obj->name);
  155. }
  156. static int iommu_enable(struct omap_iommu *obj)
  157. {
  158. int ret;
  159. ret = pm_runtime_get_sync(obj->dev);
  160. if (ret < 0)
  161. pm_runtime_put_noidle(obj->dev);
  162. return ret < 0 ? ret : 0;
  163. }
  164. static void iommu_disable(struct omap_iommu *obj)
  165. {
  166. pm_runtime_put_sync(obj->dev);
  167. }
  168. /*
  169. * TLB operations
  170. */
  171. static u32 iotlb_cr_to_virt(struct cr_regs *cr)
  172. {
  173. u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK;
  174. u32 mask = get_cam_va_mask(cr->cam & page_size);
  175. return cr->cam & mask;
  176. }
  177. static u32 get_iopte_attr(struct iotlb_entry *e)
  178. {
  179. u32 attr;
  180. attr = e->mixed << 5;
  181. attr |= e->endian;
  182. attr |= e->elsz >> 3;
  183. attr <<= (((e->pgsz == MMU_CAM_PGSZ_4K) ||
  184. (e->pgsz == MMU_CAM_PGSZ_64K)) ? 0 : 6);
  185. return attr;
  186. }
  187. static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da)
  188. {
  189. u32 status, fault_addr;
  190. status = iommu_read_reg(obj, MMU_IRQSTATUS);
  191. status &= MMU_IRQ_MASK;
  192. if (!status) {
  193. *da = 0;
  194. return 0;
  195. }
  196. fault_addr = iommu_read_reg(obj, MMU_FAULT_AD);
  197. *da = fault_addr;
  198. iommu_write_reg(obj, status, MMU_IRQSTATUS);
  199. return status;
  200. }
  201. void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l)
  202. {
  203. u32 val;
  204. val = iommu_read_reg(obj, MMU_LOCK);
  205. l->base = MMU_LOCK_BASE(val);
  206. l->vict = MMU_LOCK_VICT(val);
  207. }
  208. void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l)
  209. {
  210. u32 val;
  211. val = (l->base << MMU_LOCK_BASE_SHIFT);
  212. val |= (l->vict << MMU_LOCK_VICT_SHIFT);
  213. iommu_write_reg(obj, val, MMU_LOCK);
  214. }
  215. static void iotlb_read_cr(struct omap_iommu *obj, struct cr_regs *cr)
  216. {
  217. cr->cam = iommu_read_reg(obj, MMU_READ_CAM);
  218. cr->ram = iommu_read_reg(obj, MMU_READ_RAM);
  219. }
  220. static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr)
  221. {
  222. iommu_write_reg(obj, cr->cam | MMU_CAM_V, MMU_CAM);
  223. iommu_write_reg(obj, cr->ram, MMU_RAM);
  224. iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
  225. iommu_write_reg(obj, 1, MMU_LD_TLB);
  226. }
  227. /* only used in iotlb iteration for-loop */
  228. struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n)
  229. {
  230. struct cr_regs cr;
  231. struct iotlb_lock l;
  232. iotlb_lock_get(obj, &l);
  233. l.vict = n;
  234. iotlb_lock_set(obj, &l);
  235. iotlb_read_cr(obj, &cr);
  236. return cr;
  237. }
  238. #ifdef PREFETCH_IOTLB
  239. static struct cr_regs *iotlb_alloc_cr(struct omap_iommu *obj,
  240. struct iotlb_entry *e)
  241. {
  242. struct cr_regs *cr;
  243. if (!e)
  244. return NULL;
  245. if (e->da & ~(get_cam_va_mask(e->pgsz))) {
  246. dev_err(obj->dev, "%s:\twrong alignment: %08x\n", __func__,
  247. e->da);
  248. return ERR_PTR(-EINVAL);
  249. }
  250. cr = kmalloc(sizeof(*cr), GFP_KERNEL);
  251. if (!cr)
  252. return ERR_PTR(-ENOMEM);
  253. cr->cam = (e->da & MMU_CAM_VATAG_MASK) | e->prsvd | e->pgsz | e->valid;
  254. cr->ram = e->pa | e->endian | e->elsz | e->mixed;
  255. return cr;
  256. }
  257. /**
  258. * load_iotlb_entry - Set an iommu tlb entry
  259. * @obj: target iommu
  260. * @e: an iommu tlb entry info
  261. **/
  262. static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  263. {
  264. int err = 0;
  265. struct iotlb_lock l;
  266. struct cr_regs *cr;
  267. if (!obj || !obj->nr_tlb_entries || !e)
  268. return -EINVAL;
  269. pm_runtime_get_sync(obj->dev);
  270. iotlb_lock_get(obj, &l);
  271. if (l.base == obj->nr_tlb_entries) {
  272. dev_warn(obj->dev, "%s: preserve entries full\n", __func__);
  273. err = -EBUSY;
  274. goto out;
  275. }
  276. if (!e->prsvd) {
  277. int i;
  278. struct cr_regs tmp;
  279. for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, tmp)
  280. if (!iotlb_cr_valid(&tmp))
  281. break;
  282. if (i == obj->nr_tlb_entries) {
  283. dev_dbg(obj->dev, "%s: full: no entry\n", __func__);
  284. err = -EBUSY;
  285. goto out;
  286. }
  287. iotlb_lock_get(obj, &l);
  288. } else {
  289. l.vict = l.base;
  290. iotlb_lock_set(obj, &l);
  291. }
  292. cr = iotlb_alloc_cr(obj, e);
  293. if (IS_ERR(cr)) {
  294. pm_runtime_put_sync(obj->dev);
  295. return PTR_ERR(cr);
  296. }
  297. iotlb_load_cr(obj, cr);
  298. kfree(cr);
  299. if (e->prsvd)
  300. l.base++;
  301. /* increment victim for next tlb load */
  302. if (++l.vict == obj->nr_tlb_entries)
  303. l.vict = l.base;
  304. iotlb_lock_set(obj, &l);
  305. out:
  306. pm_runtime_put_sync(obj->dev);
  307. return err;
  308. }
  309. #else /* !PREFETCH_IOTLB */
  310. static int load_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  311. {
  312. return 0;
  313. }
  314. #endif /* !PREFETCH_IOTLB */
  315. static int prefetch_iotlb_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  316. {
  317. return load_iotlb_entry(obj, e);
  318. }
  319. /**
  320. * flush_iotlb_page - Clear an iommu tlb entry
  321. * @obj: target iommu
  322. * @da: iommu device virtual address
  323. *
  324. * Clear an iommu tlb entry which includes 'da' address.
  325. **/
  326. static void flush_iotlb_page(struct omap_iommu *obj, u32 da)
  327. {
  328. int i;
  329. struct cr_regs cr;
  330. pm_runtime_get_sync(obj->dev);
  331. for_each_iotlb_cr(obj, obj->nr_tlb_entries, i, cr) {
  332. u32 start;
  333. size_t bytes;
  334. if (!iotlb_cr_valid(&cr))
  335. continue;
  336. start = iotlb_cr_to_virt(&cr);
  337. bytes = iopgsz_to_bytes(cr.cam & 3);
  338. if ((start <= da) && (da < start + bytes)) {
  339. dev_dbg(obj->dev, "%s: %08x<=%08x(%zx)\n",
  340. __func__, start, da, bytes);
  341. iotlb_load_cr(obj, &cr);
  342. iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY);
  343. break;
  344. }
  345. }
  346. pm_runtime_put_sync(obj->dev);
  347. if (i == obj->nr_tlb_entries)
  348. dev_dbg(obj->dev, "%s: no page for %08x\n", __func__, da);
  349. }
  350. /**
  351. * flush_iotlb_all - Clear all iommu tlb entries
  352. * @obj: target iommu
  353. **/
  354. static void flush_iotlb_all(struct omap_iommu *obj)
  355. {
  356. struct iotlb_lock l;
  357. pm_runtime_get_sync(obj->dev);
  358. l.base = 0;
  359. l.vict = 0;
  360. iotlb_lock_set(obj, &l);
  361. iommu_write_reg(obj, 1, MMU_GFLUSH);
  362. pm_runtime_put_sync(obj->dev);
  363. }
  364. /*
  365. * H/W pagetable operations
  366. */
  367. static void flush_iopte_range(struct device *dev, dma_addr_t dma,
  368. unsigned long offset, int num_entries)
  369. {
  370. size_t size = num_entries * sizeof(u32);
  371. dma_sync_single_range_for_device(dev, dma, offset, size, DMA_TO_DEVICE);
  372. }
  373. static void iopte_free(struct omap_iommu *obj, u32 *iopte, bool dma_valid)
  374. {
  375. dma_addr_t pt_dma;
  376. /* Note: freed iopte's must be clean ready for re-use */
  377. if (iopte) {
  378. if (dma_valid) {
  379. pt_dma = virt_to_phys(iopte);
  380. dma_unmap_single(obj->dev, pt_dma, IOPTE_TABLE_SIZE,
  381. DMA_TO_DEVICE);
  382. }
  383. kmem_cache_free(iopte_cachep, iopte);
  384. }
  385. }
  386. static u32 *iopte_alloc(struct omap_iommu *obj, u32 *iopgd,
  387. dma_addr_t *pt_dma, u32 da)
  388. {
  389. u32 *iopte;
  390. unsigned long offset = iopgd_index(da) * sizeof(da);
  391. /* a table has already existed */
  392. if (*iopgd)
  393. goto pte_ready;
  394. /*
  395. * do the allocation outside the page table lock
  396. */
  397. spin_unlock(&obj->page_table_lock);
  398. iopte = kmem_cache_zalloc(iopte_cachep, GFP_KERNEL);
  399. spin_lock(&obj->page_table_lock);
  400. if (!*iopgd) {
  401. if (!iopte)
  402. return ERR_PTR(-ENOMEM);
  403. *pt_dma = dma_map_single(obj->dev, iopte, IOPTE_TABLE_SIZE,
  404. DMA_TO_DEVICE);
  405. if (dma_mapping_error(obj->dev, *pt_dma)) {
  406. dev_err(obj->dev, "DMA map error for L2 table\n");
  407. iopte_free(obj, iopte, false);
  408. return ERR_PTR(-ENOMEM);
  409. }
  410. /*
  411. * we rely on dma address and the physical address to be
  412. * the same for mapping the L2 table
  413. */
  414. if (WARN_ON(*pt_dma != virt_to_phys(iopte))) {
  415. dev_err(obj->dev, "DMA translation error for L2 table\n");
  416. dma_unmap_single(obj->dev, *pt_dma, IOPTE_TABLE_SIZE,
  417. DMA_TO_DEVICE);
  418. iopte_free(obj, iopte, false);
  419. return ERR_PTR(-ENOMEM);
  420. }
  421. *iopgd = virt_to_phys(iopte) | IOPGD_TABLE;
  422. flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
  423. dev_vdbg(obj->dev, "%s: a new pte:%p\n", __func__, iopte);
  424. } else {
  425. /* We raced, free the reduniovant table */
  426. iopte_free(obj, iopte, false);
  427. }
  428. pte_ready:
  429. iopte = iopte_offset(iopgd, da);
  430. *pt_dma = iopgd_page_paddr(iopgd);
  431. dev_vdbg(obj->dev,
  432. "%s: da:%08x pgd:%p *pgd:%08x pte:%p *pte:%08x\n",
  433. __func__, da, iopgd, *iopgd, iopte, *iopte);
  434. return iopte;
  435. }
  436. static int iopgd_alloc_section(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  437. {
  438. u32 *iopgd = iopgd_offset(obj, da);
  439. unsigned long offset = iopgd_index(da) * sizeof(da);
  440. if ((da | pa) & ~IOSECTION_MASK) {
  441. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  442. __func__, da, pa, IOSECTION_SIZE);
  443. return -EINVAL;
  444. }
  445. *iopgd = (pa & IOSECTION_MASK) | prot | IOPGD_SECTION;
  446. flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
  447. return 0;
  448. }
  449. static int iopgd_alloc_super(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  450. {
  451. u32 *iopgd = iopgd_offset(obj, da);
  452. unsigned long offset = iopgd_index(da) * sizeof(da);
  453. int i;
  454. if ((da | pa) & ~IOSUPER_MASK) {
  455. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  456. __func__, da, pa, IOSUPER_SIZE);
  457. return -EINVAL;
  458. }
  459. for (i = 0; i < 16; i++)
  460. *(iopgd + i) = (pa & IOSUPER_MASK) | prot | IOPGD_SUPER;
  461. flush_iopte_range(obj->dev, obj->pd_dma, offset, 16);
  462. return 0;
  463. }
  464. static int iopte_alloc_page(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  465. {
  466. u32 *iopgd = iopgd_offset(obj, da);
  467. dma_addr_t pt_dma;
  468. u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
  469. unsigned long offset = iopte_index(da) * sizeof(da);
  470. if (IS_ERR(iopte))
  471. return PTR_ERR(iopte);
  472. *iopte = (pa & IOPAGE_MASK) | prot | IOPTE_SMALL;
  473. flush_iopte_range(obj->dev, pt_dma, offset, 1);
  474. dev_vdbg(obj->dev, "%s: da:%08x pa:%08x pte:%p *pte:%08x\n",
  475. __func__, da, pa, iopte, *iopte);
  476. return 0;
  477. }
  478. static int iopte_alloc_large(struct omap_iommu *obj, u32 da, u32 pa, u32 prot)
  479. {
  480. u32 *iopgd = iopgd_offset(obj, da);
  481. dma_addr_t pt_dma;
  482. u32 *iopte = iopte_alloc(obj, iopgd, &pt_dma, da);
  483. unsigned long offset = iopte_index(da) * sizeof(da);
  484. int i;
  485. if ((da | pa) & ~IOLARGE_MASK) {
  486. dev_err(obj->dev, "%s: %08x:%08x should aligned on %08lx\n",
  487. __func__, da, pa, IOLARGE_SIZE);
  488. return -EINVAL;
  489. }
  490. if (IS_ERR(iopte))
  491. return PTR_ERR(iopte);
  492. for (i = 0; i < 16; i++)
  493. *(iopte + i) = (pa & IOLARGE_MASK) | prot | IOPTE_LARGE;
  494. flush_iopte_range(obj->dev, pt_dma, offset, 16);
  495. return 0;
  496. }
  497. static int
  498. iopgtable_store_entry_core(struct omap_iommu *obj, struct iotlb_entry *e)
  499. {
  500. int (*fn)(struct omap_iommu *, u32, u32, u32);
  501. u32 prot;
  502. int err;
  503. if (!obj || !e)
  504. return -EINVAL;
  505. switch (e->pgsz) {
  506. case MMU_CAM_PGSZ_16M:
  507. fn = iopgd_alloc_super;
  508. break;
  509. case MMU_CAM_PGSZ_1M:
  510. fn = iopgd_alloc_section;
  511. break;
  512. case MMU_CAM_PGSZ_64K:
  513. fn = iopte_alloc_large;
  514. break;
  515. case MMU_CAM_PGSZ_4K:
  516. fn = iopte_alloc_page;
  517. break;
  518. default:
  519. fn = NULL;
  520. break;
  521. }
  522. if (WARN_ON(!fn))
  523. return -EINVAL;
  524. prot = get_iopte_attr(e);
  525. spin_lock(&obj->page_table_lock);
  526. err = fn(obj, e->da, e->pa, prot);
  527. spin_unlock(&obj->page_table_lock);
  528. return err;
  529. }
  530. /**
  531. * omap_iopgtable_store_entry - Make an iommu pte entry
  532. * @obj: target iommu
  533. * @e: an iommu tlb entry info
  534. **/
  535. static int
  536. omap_iopgtable_store_entry(struct omap_iommu *obj, struct iotlb_entry *e)
  537. {
  538. int err;
  539. flush_iotlb_page(obj, e->da);
  540. err = iopgtable_store_entry_core(obj, e);
  541. if (!err)
  542. prefetch_iotlb_entry(obj, e);
  543. return err;
  544. }
  545. /**
  546. * iopgtable_lookup_entry - Lookup an iommu pte entry
  547. * @obj: target iommu
  548. * @da: iommu device virtual address
  549. * @ppgd: iommu pgd entry pointer to be returned
  550. * @ppte: iommu pte entry pointer to be returned
  551. **/
  552. static void
  553. iopgtable_lookup_entry(struct omap_iommu *obj, u32 da, u32 **ppgd, u32 **ppte)
  554. {
  555. u32 *iopgd, *iopte = NULL;
  556. iopgd = iopgd_offset(obj, da);
  557. if (!*iopgd)
  558. goto out;
  559. if (iopgd_is_table(*iopgd))
  560. iopte = iopte_offset(iopgd, da);
  561. out:
  562. *ppgd = iopgd;
  563. *ppte = iopte;
  564. }
  565. static size_t iopgtable_clear_entry_core(struct omap_iommu *obj, u32 da)
  566. {
  567. size_t bytes;
  568. u32 *iopgd = iopgd_offset(obj, da);
  569. int nent = 1;
  570. dma_addr_t pt_dma;
  571. unsigned long pd_offset = iopgd_index(da) * sizeof(da);
  572. unsigned long pt_offset = iopte_index(da) * sizeof(da);
  573. if (!*iopgd)
  574. return 0;
  575. if (iopgd_is_table(*iopgd)) {
  576. int i;
  577. u32 *iopte = iopte_offset(iopgd, da);
  578. bytes = IOPTE_SIZE;
  579. if (*iopte & IOPTE_LARGE) {
  580. nent *= 16;
  581. /* rewind to the 1st entry */
  582. iopte = iopte_offset(iopgd, (da & IOLARGE_MASK));
  583. }
  584. bytes *= nent;
  585. memset(iopte, 0, nent * sizeof(*iopte));
  586. pt_dma = iopgd_page_paddr(iopgd);
  587. flush_iopte_range(obj->dev, pt_dma, pt_offset, nent);
  588. /*
  589. * do table walk to check if this table is necessary or not
  590. */
  591. iopte = iopte_offset(iopgd, 0);
  592. for (i = 0; i < PTRS_PER_IOPTE; i++)
  593. if (iopte[i])
  594. goto out;
  595. iopte_free(obj, iopte, true);
  596. nent = 1; /* for the next L1 entry */
  597. } else {
  598. bytes = IOPGD_SIZE;
  599. if ((*iopgd & IOPGD_SUPER) == IOPGD_SUPER) {
  600. nent *= 16;
  601. /* rewind to the 1st entry */
  602. iopgd = iopgd_offset(obj, (da & IOSUPER_MASK));
  603. }
  604. bytes *= nent;
  605. }
  606. memset(iopgd, 0, nent * sizeof(*iopgd));
  607. flush_iopte_range(obj->dev, obj->pd_dma, pd_offset, nent);
  608. out:
  609. return bytes;
  610. }
  611. /**
  612. * iopgtable_clear_entry - Remove an iommu pte entry
  613. * @obj: target iommu
  614. * @da: iommu device virtual address
  615. **/
  616. static size_t iopgtable_clear_entry(struct omap_iommu *obj, u32 da)
  617. {
  618. size_t bytes;
  619. spin_lock(&obj->page_table_lock);
  620. bytes = iopgtable_clear_entry_core(obj, da);
  621. flush_iotlb_page(obj, da);
  622. spin_unlock(&obj->page_table_lock);
  623. return bytes;
  624. }
  625. static void iopgtable_clear_entry_all(struct omap_iommu *obj)
  626. {
  627. unsigned long offset;
  628. int i;
  629. spin_lock(&obj->page_table_lock);
  630. for (i = 0; i < PTRS_PER_IOPGD; i++) {
  631. u32 da;
  632. u32 *iopgd;
  633. da = i << IOPGD_SHIFT;
  634. iopgd = iopgd_offset(obj, da);
  635. offset = iopgd_index(da) * sizeof(da);
  636. if (!*iopgd)
  637. continue;
  638. if (iopgd_is_table(*iopgd))
  639. iopte_free(obj, iopte_offset(iopgd, 0), true);
  640. *iopgd = 0;
  641. flush_iopte_range(obj->dev, obj->pd_dma, offset, 1);
  642. }
  643. flush_iotlb_all(obj);
  644. spin_unlock(&obj->page_table_lock);
  645. }
  646. /*
  647. * Device IOMMU generic operations
  648. */
  649. static irqreturn_t iommu_fault_handler(int irq, void *data)
  650. {
  651. u32 da, errs;
  652. u32 *iopgd, *iopte;
  653. struct omap_iommu *obj = data;
  654. struct iommu_domain *domain = obj->domain;
  655. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  656. if (!omap_domain->dev)
  657. return IRQ_NONE;
  658. errs = iommu_report_fault(obj, &da);
  659. if (errs == 0)
  660. return IRQ_HANDLED;
  661. /* Fault callback or TLB/PTE Dynamic loading */
  662. if (!report_iommu_fault(domain, obj->dev, da, 0))
  663. return IRQ_HANDLED;
  664. iommu_write_reg(obj, 0, MMU_IRQENABLE);
  665. iopgd = iopgd_offset(obj, da);
  666. if (!iopgd_is_table(*iopgd)) {
  667. dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n",
  668. obj->name, errs, da, iopgd, *iopgd);
  669. return IRQ_NONE;
  670. }
  671. iopte = iopte_offset(iopgd, da);
  672. dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n",
  673. obj->name, errs, da, iopgd, *iopgd, iopte, *iopte);
  674. return IRQ_NONE;
  675. }
  676. /**
  677. * omap_iommu_attach() - attach iommu device to an iommu domain
  678. * @obj: target omap iommu device
  679. * @iopgd: page table
  680. **/
  681. static int omap_iommu_attach(struct omap_iommu *obj, u32 *iopgd)
  682. {
  683. int err;
  684. spin_lock(&obj->iommu_lock);
  685. obj->pd_dma = dma_map_single(obj->dev, iopgd, IOPGD_TABLE_SIZE,
  686. DMA_TO_DEVICE);
  687. if (dma_mapping_error(obj->dev, obj->pd_dma)) {
  688. dev_err(obj->dev, "DMA map error for L1 table\n");
  689. err = -ENOMEM;
  690. goto out_err;
  691. }
  692. obj->iopgd = iopgd;
  693. err = iommu_enable(obj);
  694. if (err)
  695. goto out_err;
  696. flush_iotlb_all(obj);
  697. spin_unlock(&obj->iommu_lock);
  698. dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
  699. return 0;
  700. out_err:
  701. spin_unlock(&obj->iommu_lock);
  702. return err;
  703. }
  704. /**
  705. * omap_iommu_detach - release iommu device
  706. * @obj: target iommu
  707. **/
  708. static void omap_iommu_detach(struct omap_iommu *obj)
  709. {
  710. if (!obj || IS_ERR(obj))
  711. return;
  712. spin_lock(&obj->iommu_lock);
  713. dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
  714. DMA_TO_DEVICE);
  715. obj->pd_dma = 0;
  716. obj->iopgd = NULL;
  717. iommu_disable(obj);
  718. spin_unlock(&obj->iommu_lock);
  719. dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
  720. }
  721. static void omap_iommu_save_tlb_entries(struct omap_iommu *obj)
  722. {
  723. struct iotlb_lock lock;
  724. struct cr_regs cr;
  725. struct cr_regs *tmp;
  726. int i;
  727. /* check if there are any locked tlbs to save */
  728. iotlb_lock_get(obj, &lock);
  729. obj->num_cr_ctx = lock.base;
  730. if (!obj->num_cr_ctx)
  731. return;
  732. tmp = obj->cr_ctx;
  733. for_each_iotlb_cr(obj, obj->num_cr_ctx, i, cr)
  734. * tmp++ = cr;
  735. }
  736. static void omap_iommu_restore_tlb_entries(struct omap_iommu *obj)
  737. {
  738. struct iotlb_lock l;
  739. struct cr_regs *tmp;
  740. int i;
  741. /* no locked tlbs to restore */
  742. if (!obj->num_cr_ctx)
  743. return;
  744. l.base = 0;
  745. tmp = obj->cr_ctx;
  746. for (i = 0; i < obj->num_cr_ctx; i++, tmp++) {
  747. l.vict = i;
  748. iotlb_lock_set(obj, &l);
  749. iotlb_load_cr(obj, tmp);
  750. }
  751. l.base = obj->num_cr_ctx;
  752. l.vict = i;
  753. iotlb_lock_set(obj, &l);
  754. }
  755. /**
  756. * omap_iommu_domain_deactivate - deactivate attached iommu devices
  757. * @domain: iommu domain attached to the target iommu device
  758. *
  759. * This API allows the client devices of IOMMU devices to suspend
  760. * the IOMMUs they control at runtime, after they are idled and
  761. * suspended all activity. System Suspend will leverage the PM
  762. * driver late callbacks.
  763. **/
  764. int omap_iommu_domain_deactivate(struct iommu_domain *domain)
  765. {
  766. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  767. struct omap_iommu_device *iommu;
  768. struct omap_iommu *oiommu;
  769. int i;
  770. if (!omap_domain->dev)
  771. return 0;
  772. iommu = omap_domain->iommus;
  773. iommu += (omap_domain->num_iommus - 1);
  774. for (i = 0; i < omap_domain->num_iommus; i++, iommu--) {
  775. oiommu = iommu->iommu_dev;
  776. pm_runtime_put_sync(oiommu->dev);
  777. }
  778. return 0;
  779. }
  780. EXPORT_SYMBOL_GPL(omap_iommu_domain_deactivate);
  781. /**
  782. * omap_iommu_domain_activate - activate attached iommu devices
  783. * @domain: iommu domain attached to the target iommu device
  784. *
  785. * This API allows the client devices of IOMMU devices to resume the
  786. * IOMMUs they control at runtime, before they can resume operations.
  787. * System Resume will leverage the PM driver late callbacks.
  788. **/
  789. int omap_iommu_domain_activate(struct iommu_domain *domain)
  790. {
  791. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  792. struct omap_iommu_device *iommu;
  793. struct omap_iommu *oiommu;
  794. int i;
  795. if (!omap_domain->dev)
  796. return 0;
  797. iommu = omap_domain->iommus;
  798. for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
  799. oiommu = iommu->iommu_dev;
  800. pm_runtime_get_sync(oiommu->dev);
  801. }
  802. return 0;
  803. }
  804. EXPORT_SYMBOL_GPL(omap_iommu_domain_activate);
  805. /**
  806. * omap_iommu_runtime_suspend - disable an iommu device
  807. * @dev: iommu device
  808. *
  809. * This function performs all that is necessary to disable an
  810. * IOMMU device, either during final detachment from a client
  811. * device, or during system/runtime suspend of the device. This
  812. * includes programming all the appropriate IOMMU registers, and
  813. * managing the associated omap_hwmod's state and the device's
  814. * reset line. This function also saves the context of any
  815. * locked TLBs if suspending.
  816. **/
  817. static __maybe_unused int omap_iommu_runtime_suspend(struct device *dev)
  818. {
  819. struct platform_device *pdev = to_platform_device(dev);
  820. struct iommu_platform_data *pdata = dev_get_platdata(dev);
  821. struct omap_iommu *obj = to_iommu(dev);
  822. int ret;
  823. /* save the TLBs only during suspend, and not for power down */
  824. if (obj->domain && obj->iopgd)
  825. omap_iommu_save_tlb_entries(obj);
  826. omap2_iommu_disable(obj);
  827. if (pdata && pdata->device_idle)
  828. pdata->device_idle(pdev);
  829. if (pdata && pdata->assert_reset)
  830. pdata->assert_reset(pdev, pdata->reset_name);
  831. if (pdata && pdata->set_pwrdm_constraint) {
  832. ret = pdata->set_pwrdm_constraint(pdev, false, &obj->pwrst);
  833. if (ret) {
  834. dev_warn(obj->dev, "pwrdm_constraint failed to be reset, status = %d\n",
  835. ret);
  836. }
  837. }
  838. return 0;
  839. }
  840. /**
  841. * omap_iommu_runtime_resume - enable an iommu device
  842. * @dev: iommu device
  843. *
  844. * This function performs all that is necessary to enable an
  845. * IOMMU device, either during initial attachment to a client
  846. * device, or during system/runtime resume of the device. This
  847. * includes programming all the appropriate IOMMU registers, and
  848. * managing the associated omap_hwmod's state and the device's
  849. * reset line. The function also restores any locked TLBs if
  850. * resuming after a suspend.
  851. **/
  852. static __maybe_unused int omap_iommu_runtime_resume(struct device *dev)
  853. {
  854. struct platform_device *pdev = to_platform_device(dev);
  855. struct iommu_platform_data *pdata = dev_get_platdata(dev);
  856. struct omap_iommu *obj = to_iommu(dev);
  857. int ret = 0;
  858. if (pdata && pdata->set_pwrdm_constraint) {
  859. ret = pdata->set_pwrdm_constraint(pdev, true, &obj->pwrst);
  860. if (ret) {
  861. dev_warn(obj->dev, "pwrdm_constraint failed to be set, status = %d\n",
  862. ret);
  863. }
  864. }
  865. if (pdata && pdata->deassert_reset) {
  866. ret = pdata->deassert_reset(pdev, pdata->reset_name);
  867. if (ret) {
  868. dev_err(dev, "deassert_reset failed: %d\n", ret);
  869. return ret;
  870. }
  871. }
  872. if (pdata && pdata->device_enable)
  873. pdata->device_enable(pdev);
  874. /* restore the TLBs only during resume, and not for power up */
  875. if (obj->domain)
  876. omap_iommu_restore_tlb_entries(obj);
  877. ret = omap2_iommu_enable(obj);
  878. return ret;
  879. }
  880. /**
  881. * omap_iommu_prepare - prepare() dev_pm_ops implementation
  882. * @dev: iommu device
  883. *
  884. * This function performs the necessary checks to determine if the IOMMU
  885. * device needs suspending or not. The function checks if the runtime_pm
  886. * status of the device is suspended, and returns 1 in that case. This
  887. * results in the PM core to skip invoking any of the Sleep PM callbacks
  888. * (suspend, suspend_late, resume, resume_early etc).
  889. */
  890. static int omap_iommu_prepare(struct device *dev)
  891. {
  892. if (pm_runtime_status_suspended(dev))
  893. return 1;
  894. return 0;
  895. }
  896. static bool omap_iommu_can_register(struct platform_device *pdev)
  897. {
  898. struct device_node *np = pdev->dev.of_node;
  899. if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
  900. return true;
  901. /*
  902. * restrict IOMMU core registration only for processor-port MDMA MMUs
  903. * on DRA7 DSPs
  904. */
  905. if ((!strcmp(dev_name(&pdev->dev), "40d01000.mmu")) ||
  906. (!strcmp(dev_name(&pdev->dev), "41501000.mmu")))
  907. return true;
  908. return false;
  909. }
  910. static int omap_iommu_dra7_get_dsp_system_cfg(struct platform_device *pdev,
  911. struct omap_iommu *obj)
  912. {
  913. struct device_node *np = pdev->dev.of_node;
  914. int ret;
  915. if (!of_device_is_compatible(np, "ti,dra7-dsp-iommu"))
  916. return 0;
  917. if (!of_property_read_bool(np, "ti,syscon-mmuconfig")) {
  918. dev_err(&pdev->dev, "ti,syscon-mmuconfig property is missing\n");
  919. return -EINVAL;
  920. }
  921. obj->syscfg =
  922. syscon_regmap_lookup_by_phandle(np, "ti,syscon-mmuconfig");
  923. if (IS_ERR(obj->syscfg)) {
  924. /* can fail with -EPROBE_DEFER */
  925. ret = PTR_ERR(obj->syscfg);
  926. return ret;
  927. }
  928. if (of_property_read_u32_index(np, "ti,syscon-mmuconfig", 1,
  929. &obj->id)) {
  930. dev_err(&pdev->dev, "couldn't get the IOMMU instance id within subsystem\n");
  931. return -EINVAL;
  932. }
  933. if (obj->id != 0 && obj->id != 1) {
  934. dev_err(&pdev->dev, "invalid IOMMU instance id\n");
  935. return -EINVAL;
  936. }
  937. return 0;
  938. }
  939. /*
  940. * OMAP Device MMU(IOMMU) detection
  941. */
  942. static int omap_iommu_probe(struct platform_device *pdev)
  943. {
  944. int err = -ENODEV;
  945. int irq;
  946. struct omap_iommu *obj;
  947. struct resource *res;
  948. struct device_node *of = pdev->dev.of_node;
  949. if (!of) {
  950. pr_err("%s: only DT-based devices are supported\n", __func__);
  951. return -ENODEV;
  952. }
  953. obj = devm_kzalloc(&pdev->dev, sizeof(*obj) + MMU_REG_SIZE, GFP_KERNEL);
  954. if (!obj)
  955. return -ENOMEM;
  956. /*
  957. * self-manage the ordering dependencies between omap_device_enable/idle
  958. * and omap_device_assert/deassert_hardreset API
  959. */
  960. if (pdev->dev.pm_domain) {
  961. dev_dbg(&pdev->dev, "device pm_domain is being reset\n");
  962. pdev->dev.pm_domain = NULL;
  963. }
  964. obj->name = dev_name(&pdev->dev);
  965. obj->nr_tlb_entries = 32;
  966. err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries);
  967. if (err && err != -EINVAL)
  968. return err;
  969. if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8)
  970. return -EINVAL;
  971. if (of_find_property(of, "ti,iommu-bus-err-back", NULL))
  972. obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN;
  973. obj->dev = &pdev->dev;
  974. obj->ctx = (void *)obj + sizeof(*obj);
  975. obj->cr_ctx = devm_kzalloc(&pdev->dev,
  976. sizeof(*obj->cr_ctx) * obj->nr_tlb_entries,
  977. GFP_KERNEL);
  978. if (!obj->cr_ctx)
  979. return -ENOMEM;
  980. spin_lock_init(&obj->iommu_lock);
  981. spin_lock_init(&obj->page_table_lock);
  982. res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  983. obj->regbase = devm_ioremap_resource(obj->dev, res);
  984. if (IS_ERR(obj->regbase))
  985. return PTR_ERR(obj->regbase);
  986. err = omap_iommu_dra7_get_dsp_system_cfg(pdev, obj);
  987. if (err)
  988. return err;
  989. irq = platform_get_irq(pdev, 0);
  990. if (irq < 0)
  991. return -ENODEV;
  992. err = devm_request_irq(obj->dev, irq, iommu_fault_handler, IRQF_SHARED,
  993. dev_name(obj->dev), obj);
  994. if (err < 0)
  995. return err;
  996. platform_set_drvdata(pdev, obj);
  997. if (omap_iommu_can_register(pdev)) {
  998. obj->group = iommu_group_alloc();
  999. if (IS_ERR(obj->group))
  1000. return PTR_ERR(obj->group);
  1001. err = iommu_device_sysfs_add(&obj->iommu, obj->dev, NULL,
  1002. obj->name);
  1003. if (err)
  1004. goto out_group;
  1005. err = iommu_device_register(&obj->iommu, &omap_iommu_ops, &pdev->dev);
  1006. if (err)
  1007. goto out_sysfs;
  1008. }
  1009. pm_runtime_enable(obj->dev);
  1010. omap_iommu_debugfs_add(obj);
  1011. dev_info(&pdev->dev, "%s registered\n", obj->name);
  1012. /* Re-probe bus to probe device attached to this IOMMU */
  1013. bus_iommu_probe(&platform_bus_type);
  1014. return 0;
  1015. out_sysfs:
  1016. iommu_device_sysfs_remove(&obj->iommu);
  1017. out_group:
  1018. iommu_group_put(obj->group);
  1019. return err;
  1020. }
  1021. static int omap_iommu_remove(struct platform_device *pdev)
  1022. {
  1023. struct omap_iommu *obj = platform_get_drvdata(pdev);
  1024. if (obj->group) {
  1025. iommu_group_put(obj->group);
  1026. obj->group = NULL;
  1027. iommu_device_sysfs_remove(&obj->iommu);
  1028. iommu_device_unregister(&obj->iommu);
  1029. }
  1030. omap_iommu_debugfs_remove(obj);
  1031. pm_runtime_disable(obj->dev);
  1032. dev_info(&pdev->dev, "%s removed\n", obj->name);
  1033. return 0;
  1034. }
  1035. static const struct dev_pm_ops omap_iommu_pm_ops = {
  1036. .prepare = omap_iommu_prepare,
  1037. SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
  1038. pm_runtime_force_resume)
  1039. SET_RUNTIME_PM_OPS(omap_iommu_runtime_suspend,
  1040. omap_iommu_runtime_resume, NULL)
  1041. };
  1042. static const struct of_device_id omap_iommu_of_match[] = {
  1043. { .compatible = "ti,omap2-iommu" },
  1044. { .compatible = "ti,omap4-iommu" },
  1045. { .compatible = "ti,dra7-iommu" },
  1046. { .compatible = "ti,dra7-dsp-iommu" },
  1047. {},
  1048. };
  1049. static struct platform_driver omap_iommu_driver = {
  1050. .probe = omap_iommu_probe,
  1051. .remove = omap_iommu_remove,
  1052. .driver = {
  1053. .name = "omap-iommu",
  1054. .pm = &omap_iommu_pm_ops,
  1055. .of_match_table = of_match_ptr(omap_iommu_of_match),
  1056. },
  1057. };
  1058. static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
  1059. {
  1060. memset(e, 0, sizeof(*e));
  1061. e->da = da;
  1062. e->pa = pa;
  1063. e->valid = MMU_CAM_V;
  1064. e->pgsz = pgsz;
  1065. e->endian = MMU_RAM_ENDIAN_LITTLE;
  1066. e->elsz = MMU_RAM_ELSZ_8;
  1067. e->mixed = 0;
  1068. return iopgsz_to_bytes(e->pgsz);
  1069. }
  1070. static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
  1071. phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
  1072. {
  1073. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1074. struct device *dev = omap_domain->dev;
  1075. struct omap_iommu_device *iommu;
  1076. struct omap_iommu *oiommu;
  1077. struct iotlb_entry e;
  1078. int omap_pgsz;
  1079. u32 ret = -EINVAL;
  1080. int i;
  1081. omap_pgsz = bytes_to_iopgsz(bytes);
  1082. if (omap_pgsz < 0) {
  1083. dev_err(dev, "invalid size to map: %zu\n", bytes);
  1084. return -EINVAL;
  1085. }
  1086. dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%zx\n", da, &pa, bytes);
  1087. iotlb_init_entry(&e, da, pa, omap_pgsz);
  1088. iommu = omap_domain->iommus;
  1089. for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
  1090. oiommu = iommu->iommu_dev;
  1091. ret = omap_iopgtable_store_entry(oiommu, &e);
  1092. if (ret) {
  1093. dev_err(dev, "omap_iopgtable_store_entry failed: %d\n",
  1094. ret);
  1095. break;
  1096. }
  1097. }
  1098. if (ret) {
  1099. while (i--) {
  1100. iommu--;
  1101. oiommu = iommu->iommu_dev;
  1102. iopgtable_clear_entry(oiommu, da);
  1103. }
  1104. }
  1105. return ret;
  1106. }
  1107. static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
  1108. size_t size, struct iommu_iotlb_gather *gather)
  1109. {
  1110. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1111. struct device *dev = omap_domain->dev;
  1112. struct omap_iommu_device *iommu;
  1113. struct omap_iommu *oiommu;
  1114. bool error = false;
  1115. size_t bytes = 0;
  1116. int i;
  1117. dev_dbg(dev, "unmapping da 0x%lx size %zu\n", da, size);
  1118. iommu = omap_domain->iommus;
  1119. for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
  1120. oiommu = iommu->iommu_dev;
  1121. bytes = iopgtable_clear_entry(oiommu, da);
  1122. if (!bytes)
  1123. error = true;
  1124. }
  1125. /*
  1126. * simplify return - we are only checking if any of the iommus
  1127. * reported an error, but not if all of them are unmapping the
  1128. * same number of entries. This should not occur due to the
  1129. * mirror programming.
  1130. */
  1131. return error ? 0 : bytes;
  1132. }
  1133. static int omap_iommu_count(struct device *dev)
  1134. {
  1135. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1136. int count = 0;
  1137. while (arch_data->iommu_dev) {
  1138. count++;
  1139. arch_data++;
  1140. }
  1141. return count;
  1142. }
  1143. /* caller should call cleanup if this function fails */
  1144. static int omap_iommu_attach_init(struct device *dev,
  1145. struct omap_iommu_domain *odomain)
  1146. {
  1147. struct omap_iommu_device *iommu;
  1148. int i;
  1149. odomain->num_iommus = omap_iommu_count(dev);
  1150. if (!odomain->num_iommus)
  1151. return -EINVAL;
  1152. odomain->iommus = kcalloc(odomain->num_iommus, sizeof(*iommu),
  1153. GFP_ATOMIC);
  1154. if (!odomain->iommus)
  1155. return -ENOMEM;
  1156. iommu = odomain->iommus;
  1157. for (i = 0; i < odomain->num_iommus; i++, iommu++) {
  1158. iommu->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_ATOMIC);
  1159. if (!iommu->pgtable)
  1160. return -ENOMEM;
  1161. /*
  1162. * should never fail, but please keep this around to ensure
  1163. * we keep the hardware happy
  1164. */
  1165. if (WARN_ON(!IS_ALIGNED((long)iommu->pgtable,
  1166. IOPGD_TABLE_SIZE)))
  1167. return -EINVAL;
  1168. }
  1169. return 0;
  1170. }
  1171. static void omap_iommu_detach_fini(struct omap_iommu_domain *odomain)
  1172. {
  1173. int i;
  1174. struct omap_iommu_device *iommu = odomain->iommus;
  1175. for (i = 0; iommu && i < odomain->num_iommus; i++, iommu++)
  1176. kfree(iommu->pgtable);
  1177. kfree(odomain->iommus);
  1178. odomain->num_iommus = 0;
  1179. odomain->iommus = NULL;
  1180. }
  1181. static int
  1182. omap_iommu_attach_dev(struct iommu_domain *domain, struct device *dev)
  1183. {
  1184. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1185. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1186. struct omap_iommu_device *iommu;
  1187. struct omap_iommu *oiommu;
  1188. int ret = 0;
  1189. int i;
  1190. if (!arch_data || !arch_data->iommu_dev) {
  1191. dev_err(dev, "device doesn't have an associated iommu\n");
  1192. return -EINVAL;
  1193. }
  1194. spin_lock(&omap_domain->lock);
  1195. /* only a single client device can be attached to a domain */
  1196. if (omap_domain->dev) {
  1197. dev_err(dev, "iommu domain is already attached\n");
  1198. ret = -EBUSY;
  1199. goto out;
  1200. }
  1201. ret = omap_iommu_attach_init(dev, omap_domain);
  1202. if (ret) {
  1203. dev_err(dev, "failed to allocate required iommu data %d\n",
  1204. ret);
  1205. goto init_fail;
  1206. }
  1207. iommu = omap_domain->iommus;
  1208. for (i = 0; i < omap_domain->num_iommus; i++, iommu++, arch_data++) {
  1209. /* configure and enable the omap iommu */
  1210. oiommu = arch_data->iommu_dev;
  1211. ret = omap_iommu_attach(oiommu, iommu->pgtable);
  1212. if (ret) {
  1213. dev_err(dev, "can't get omap iommu: %d\n", ret);
  1214. goto attach_fail;
  1215. }
  1216. oiommu->domain = domain;
  1217. iommu->iommu_dev = oiommu;
  1218. }
  1219. omap_domain->dev = dev;
  1220. goto out;
  1221. attach_fail:
  1222. while (i--) {
  1223. iommu--;
  1224. arch_data--;
  1225. oiommu = iommu->iommu_dev;
  1226. omap_iommu_detach(oiommu);
  1227. iommu->iommu_dev = NULL;
  1228. oiommu->domain = NULL;
  1229. }
  1230. init_fail:
  1231. omap_iommu_detach_fini(omap_domain);
  1232. out:
  1233. spin_unlock(&omap_domain->lock);
  1234. return ret;
  1235. }
  1236. static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain,
  1237. struct device *dev)
  1238. {
  1239. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1240. struct omap_iommu_device *iommu = omap_domain->iommus;
  1241. struct omap_iommu *oiommu;
  1242. int i;
  1243. if (!omap_domain->dev) {
  1244. dev_err(dev, "domain has no attached device\n");
  1245. return;
  1246. }
  1247. /* only a single device is supported per domain for now */
  1248. if (omap_domain->dev != dev) {
  1249. dev_err(dev, "invalid attached device\n");
  1250. return;
  1251. }
  1252. /*
  1253. * cleanup in the reverse order of attachment - this addresses
  1254. * any h/w dependencies between multiple instances, if any
  1255. */
  1256. iommu += (omap_domain->num_iommus - 1);
  1257. arch_data += (omap_domain->num_iommus - 1);
  1258. for (i = 0; i < omap_domain->num_iommus; i++, iommu--, arch_data--) {
  1259. oiommu = iommu->iommu_dev;
  1260. iopgtable_clear_entry_all(oiommu);
  1261. omap_iommu_detach(oiommu);
  1262. iommu->iommu_dev = NULL;
  1263. oiommu->domain = NULL;
  1264. }
  1265. omap_iommu_detach_fini(omap_domain);
  1266. omap_domain->dev = NULL;
  1267. }
  1268. static void omap_iommu_detach_dev(struct iommu_domain *domain,
  1269. struct device *dev)
  1270. {
  1271. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1272. spin_lock(&omap_domain->lock);
  1273. _omap_iommu_detach_dev(omap_domain, dev);
  1274. spin_unlock(&omap_domain->lock);
  1275. }
  1276. static struct iommu_domain *omap_iommu_domain_alloc(unsigned type)
  1277. {
  1278. struct omap_iommu_domain *omap_domain;
  1279. if (type != IOMMU_DOMAIN_UNMANAGED)
  1280. return NULL;
  1281. omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL);
  1282. if (!omap_domain)
  1283. return NULL;
  1284. spin_lock_init(&omap_domain->lock);
  1285. omap_domain->domain.geometry.aperture_start = 0;
  1286. omap_domain->domain.geometry.aperture_end = (1ULL << 32) - 1;
  1287. omap_domain->domain.geometry.force_aperture = true;
  1288. return &omap_domain->domain;
  1289. }
  1290. static void omap_iommu_domain_free(struct iommu_domain *domain)
  1291. {
  1292. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1293. /*
  1294. * An iommu device is still attached
  1295. * (currently, only one device can be attached) ?
  1296. */
  1297. if (omap_domain->dev)
  1298. _omap_iommu_detach_dev(omap_domain, omap_domain->dev);
  1299. kfree(omap_domain);
  1300. }
  1301. static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
  1302. dma_addr_t da)
  1303. {
  1304. struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
  1305. struct omap_iommu_device *iommu = omap_domain->iommus;
  1306. struct omap_iommu *oiommu = iommu->iommu_dev;
  1307. struct device *dev = oiommu->dev;
  1308. u32 *pgd, *pte;
  1309. phys_addr_t ret = 0;
  1310. /*
  1311. * all the iommus within the domain will have identical programming,
  1312. * so perform the lookup using just the first iommu
  1313. */
  1314. iopgtable_lookup_entry(oiommu, da, &pgd, &pte);
  1315. if (pte) {
  1316. if (iopte_is_small(*pte))
  1317. ret = omap_iommu_translate(*pte, da, IOPTE_MASK);
  1318. else if (iopte_is_large(*pte))
  1319. ret = omap_iommu_translate(*pte, da, IOLARGE_MASK);
  1320. else
  1321. dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte,
  1322. (unsigned long long)da);
  1323. } else {
  1324. if (iopgd_is_section(*pgd))
  1325. ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK);
  1326. else if (iopgd_is_super(*pgd))
  1327. ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK);
  1328. else
  1329. dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd,
  1330. (unsigned long long)da);
  1331. }
  1332. return ret;
  1333. }
  1334. static struct iommu_device *omap_iommu_probe_device(struct device *dev)
  1335. {
  1336. struct omap_iommu_arch_data *arch_data, *tmp;
  1337. struct platform_device *pdev;
  1338. struct omap_iommu *oiommu;
  1339. struct device_node *np;
  1340. int num_iommus, i;
  1341. /*
  1342. * Allocate the per-device iommu structure for DT-based devices.
  1343. *
  1344. * TODO: Simplify this when removing non-DT support completely from the
  1345. * IOMMU users.
  1346. */
  1347. if (!dev->of_node)
  1348. return ERR_PTR(-ENODEV);
  1349. /*
  1350. * retrieve the count of IOMMU nodes using phandle size as element size
  1351. * since #iommu-cells = 0 for OMAP
  1352. */
  1353. num_iommus = of_property_count_elems_of_size(dev->of_node, "iommus",
  1354. sizeof(phandle));
  1355. if (num_iommus < 0)
  1356. return ERR_PTR(-ENODEV);
  1357. arch_data = kcalloc(num_iommus + 1, sizeof(*arch_data), GFP_KERNEL);
  1358. if (!arch_data)
  1359. return ERR_PTR(-ENOMEM);
  1360. for (i = 0, tmp = arch_data; i < num_iommus; i++, tmp++) {
  1361. np = of_parse_phandle(dev->of_node, "iommus", i);
  1362. if (!np) {
  1363. kfree(arch_data);
  1364. return ERR_PTR(-EINVAL);
  1365. }
  1366. pdev = of_find_device_by_node(np);
  1367. if (!pdev) {
  1368. of_node_put(np);
  1369. kfree(arch_data);
  1370. return ERR_PTR(-ENODEV);
  1371. }
  1372. oiommu = platform_get_drvdata(pdev);
  1373. if (!oiommu) {
  1374. of_node_put(np);
  1375. kfree(arch_data);
  1376. return ERR_PTR(-EINVAL);
  1377. }
  1378. tmp->iommu_dev = oiommu;
  1379. tmp->dev = &pdev->dev;
  1380. of_node_put(np);
  1381. }
  1382. dev_iommu_priv_set(dev, arch_data);
  1383. /*
  1384. * use the first IOMMU alone for the sysfs device linking.
  1385. * TODO: Evaluate if a single iommu_group needs to be
  1386. * maintained for both IOMMUs
  1387. */
  1388. oiommu = arch_data->iommu_dev;
  1389. return &oiommu->iommu;
  1390. }
  1391. static void omap_iommu_release_device(struct device *dev)
  1392. {
  1393. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1394. if (!dev->of_node || !arch_data)
  1395. return;
  1396. dev_iommu_priv_set(dev, NULL);
  1397. kfree(arch_data);
  1398. }
  1399. static struct iommu_group *omap_iommu_device_group(struct device *dev)
  1400. {
  1401. struct omap_iommu_arch_data *arch_data = dev_iommu_priv_get(dev);
  1402. struct iommu_group *group = ERR_PTR(-EINVAL);
  1403. if (!arch_data)
  1404. return ERR_PTR(-ENODEV);
  1405. if (arch_data->iommu_dev)
  1406. group = iommu_group_ref_get(arch_data->iommu_dev->group);
  1407. return group;
  1408. }
  1409. static const struct iommu_ops omap_iommu_ops = {
  1410. .domain_alloc = omap_iommu_domain_alloc,
  1411. .probe_device = omap_iommu_probe_device,
  1412. .release_device = omap_iommu_release_device,
  1413. .device_group = omap_iommu_device_group,
  1414. .pgsize_bitmap = OMAP_IOMMU_PGSIZES,
  1415. .default_domain_ops = &(const struct iommu_domain_ops) {
  1416. .attach_dev = omap_iommu_attach_dev,
  1417. .detach_dev = omap_iommu_detach_dev,
  1418. .map = omap_iommu_map,
  1419. .unmap = omap_iommu_unmap,
  1420. .iova_to_phys = omap_iommu_iova_to_phys,
  1421. .free = omap_iommu_domain_free,
  1422. }
  1423. };
  1424. static int __init omap_iommu_init(void)
  1425. {
  1426. struct kmem_cache *p;
  1427. const slab_flags_t flags = SLAB_HWCACHE_ALIGN;
  1428. size_t align = 1 << 10; /* L2 pagetable alignement */
  1429. struct device_node *np;
  1430. int ret;
  1431. np = of_find_matching_node(NULL, omap_iommu_of_match);
  1432. if (!np)
  1433. return 0;
  1434. of_node_put(np);
  1435. p = kmem_cache_create("iopte_cache", IOPTE_TABLE_SIZE, align, flags,
  1436. NULL);
  1437. if (!p)
  1438. return -ENOMEM;
  1439. iopte_cachep = p;
  1440. omap_iommu_debugfs_init();
  1441. ret = platform_driver_register(&omap_iommu_driver);
  1442. if (ret) {
  1443. pr_err("%s: failed to register driver\n", __func__);
  1444. goto fail_driver;
  1445. }
  1446. return 0;
  1447. fail_driver:
  1448. kmem_cache_destroy(iopte_cachep);
  1449. return ret;
  1450. }
  1451. subsys_initcall(omap_iommu_init);
  1452. /* must be ready before omap3isp is probed */