msm_smmu.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618
  1. /*
  2. * Copyright (c) 2015-2021, The Linux Foundation. All rights reserved.
  3. * Copyright (C) 2013 Red Hat
  4. * Author: Rob Clark <[email protected]>
  5. *
  6. * This program is free software; you can redistribute it and/or modify it
  7. * under the terms of the GNU General Public License version 2 as published by
  8. * the Free Software Foundation.
  9. *
  10. * This program is distributed in the hope that it will be useful, but WITHOUT
  11. * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12. * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  13. * more details.
  14. *
  15. * You should have received a copy of the GNU General Public License along with
  16. * this program. If not, see <http://www.gnu.org/licenses/>.
  17. */
  18. #include <linux/module.h>
  19. #include <linux/of_platform.h>
  20. #include <linux/pm_runtime.h>
  21. #include <linux/qcom-dma-mapping.h>
  22. #include <linux/msm_dma_iommu_mapping.h>
  23. #include <linux/dma-mapping.h>
  24. #include <linux/qcom-iommu-util.h>
  25. #include <soc/qcom/secure_buffer.h>
  26. #include "msm_drv.h"
  27. #include "msm_gem.h"
  28. #include "msm_mmu.h"
  29. #include "sde_dbg.h"
  30. struct msm_smmu_client {
  31. struct device *dev;
  32. const char *compat;
  33. struct iommu_domain *domain;
  34. const struct dma_map_ops *dma_ops;
  35. bool domain_attached;
  36. bool secure;
  37. struct list_head smmu_list;
  38. };
  39. struct msm_smmu {
  40. struct msm_mmu base;
  41. struct device *client_dev;
  42. struct msm_smmu_client *client;
  43. };
  44. struct msm_smmu_domain {
  45. const char *label;
  46. bool secure;
  47. };
  48. #define to_msm_smmu(x) container_of(x, struct msm_smmu, base)
  49. #define msm_smmu_to_client(smmu) (smmu->client)
  50. /* Serialization lock for smmu_list */
  51. static DEFINE_MUTEX(smmu_list_lock);
  52. /* List of all smmu devices installed */
  53. static LIST_HEAD(sde_smmu_list);
  54. static int msm_smmu_attach(struct msm_mmu *mmu, const char * const *names,
  55. int cnt)
  56. {
  57. struct msm_smmu *smmu = to_msm_smmu(mmu);
  58. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  59. int rc = 0;
  60. if (!client) {
  61. pr_err("undefined smmu client\n");
  62. return -EINVAL;
  63. }
  64. /* domain attach only once */
  65. if (client->domain_attached)
  66. return 0;
  67. if (client->dma_ops) {
  68. set_dma_ops(client->dev, client->dma_ops);
  69. client->dma_ops = NULL;
  70. dev_dbg(client->dev, "iommu domain ops restored\n");
  71. }
  72. rc = qcom_iommu_sid_switch(client->dev, SID_ACQUIRE);
  73. if (rc) {
  74. dev_err(client->dev, "iommu sid switch failed (%d)\n", rc);
  75. return rc;
  76. }
  77. client->domain_attached = true;
  78. dev_dbg(client->dev, "iommu domain attached\n");
  79. return 0;
  80. }
  81. static void msm_smmu_detach(struct msm_mmu *mmu, const char * const *names,
  82. int cnt)
  83. {
  84. struct msm_smmu *smmu = to_msm_smmu(mmu);
  85. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  86. int rc;
  87. if (!client) {
  88. pr_err("undefined smmu client\n");
  89. return;
  90. }
  91. if (!client->domain_attached)
  92. return;
  93. pm_runtime_get_sync(mmu->dev);
  94. msm_dma_unmap_all_for_dev(client->dev);
  95. rc = qcom_iommu_sid_switch(client->dev, SID_RELEASE);
  96. if (rc)
  97. DRM_ERROR("iommu sid switch failed (%d)\n", rc);
  98. client->dma_ops = get_dma_ops(client->dev);
  99. if (client->dma_ops) {
  100. set_dma_ops(client->dev, NULL);
  101. dev_dbg(client->dev, "iommu domain ops removed\n");
  102. }
  103. pm_runtime_put_sync(mmu->dev);
  104. client->domain_attached = false;
  105. dev_dbg(client->dev, "iommu domain detached\n");
  106. }
  107. #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
  108. static int msm_enable_smmu_translations(struct msm_mmu *mmu)
  109. {
  110. struct msm_smmu *smmu = to_msm_smmu(mmu);
  111. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  112. int ret = 0;
  113. if (!client || !client->domain)
  114. return -ENODEV;
  115. ret = qcom_iommu_enable_s1_translation(client->domain);
  116. if (ret)
  117. DRM_ERROR("enable iommu s1 translations failed:%d\n", ret);
  118. return ret;
  119. }
  120. #else
  121. static int msm_smmu_set_attribute(struct msm_mmu *mmu,
  122. enum iommu_attr attr, void *data)
  123. {
  124. struct msm_smmu *smmu = to_msm_smmu(mmu);
  125. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  126. int ret = 0;
  127. if (!client || !client->domain)
  128. return -ENODEV;
  129. ret = iommu_domain_set_attr(client->domain, attr, data);
  130. if (ret)
  131. DRM_ERROR("set domain attribute failed:%d\n", ret);
  132. return ret;
  133. }
  134. #endif
  135. static int msm_smmu_one_to_one_unmap(struct msm_mmu *mmu,
  136. uint32_t dest_address, uint32_t size)
  137. {
  138. struct msm_smmu *smmu = to_msm_smmu(mmu);
  139. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  140. int ret = 0;
  141. if (!client || !client->domain)
  142. return -ENODEV;
  143. ret = iommu_unmap(client->domain, dest_address, size);
  144. if (ret != size)
  145. pr_err("smmu unmap failed\n");
  146. return 0;
  147. }
  148. static int msm_smmu_one_to_one_map(struct msm_mmu *mmu, uint32_t iova,
  149. uint32_t dest_address, uint32_t size, int prot)
  150. {
  151. struct msm_smmu *smmu = to_msm_smmu(mmu);
  152. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  153. int ret = 0;
  154. if (!client || !client->domain)
  155. return -ENODEV;
  156. ret = iommu_map(client->domain, dest_address, dest_address,
  157. size, prot);
  158. if (ret)
  159. pr_err("smmu map failed\n");
  160. return ret;
  161. }
  162. static int msm_smmu_map(struct msm_mmu *mmu, uint64_t iova,
  163. struct sg_table *sgt, unsigned int len, int prot)
  164. {
  165. struct msm_smmu *smmu = to_msm_smmu(mmu);
  166. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  167. size_t ret = 0;
  168. if (sgt && sgt->sgl) {
  169. ret = iommu_map_sg(client->domain, iova, sgt->sgl,
  170. sgt->orig_nents, prot);
  171. WARN_ON((int)ret < 0);
  172. DRM_DEBUG("%pad/0x%x/0x%x/\n", &sgt->sgl->dma_address,
  173. sgt->sgl->dma_length, prot);
  174. SDE_EVT32(sgt->sgl->dma_address, sgt->sgl->dma_length, prot);
  175. }
  176. return (ret == len) ? 0 : -EINVAL;
  177. }
  178. static int msm_smmu_unmap(struct msm_mmu *mmu, uint64_t iova,
  179. struct sg_table *sgt, unsigned int len)
  180. {
  181. struct msm_smmu *smmu = to_msm_smmu(mmu);
  182. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  183. pm_runtime_get_sync(mmu->dev);
  184. iommu_unmap(client->domain, iova, len);
  185. pm_runtime_put_sync(mmu->dev);
  186. return 0;
  187. }
  188. static void msm_smmu_destroy(struct msm_mmu *mmu)
  189. {
  190. struct msm_smmu *smmu = to_msm_smmu(mmu);
  191. struct platform_device *pdev = to_platform_device(smmu->client_dev);
  192. if (smmu->client_dev)
  193. platform_device_unregister(pdev);
  194. kfree(smmu);
  195. }
  196. struct device *msm_smmu_get_dev(struct msm_mmu *mmu)
  197. {
  198. struct msm_smmu *smmu = to_msm_smmu(mmu);
  199. return smmu->client_dev;
  200. }
  201. static int msm_smmu_map_dma_buf(struct msm_mmu *mmu, struct sg_table *sgt,
  202. int dir, u32 flags)
  203. {
  204. struct msm_smmu *smmu = to_msm_smmu(mmu);
  205. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  206. unsigned long attrs = 0x0;
  207. int ret;
  208. if (!sgt || !client) {
  209. DRM_ERROR("sg table is invalid\n");
  210. return -ENOMEM;
  211. }
  212. /*
  213. * For import buffer type, dma_map_sg_attrs is called during
  214. * dma_buf_map_attachment and is not required to call again
  215. */
  216. if (!(flags & MSM_BO_EXTBUF)) {
  217. ret = dma_map_sg_attrs(client->dev, sgt->sgl, sgt->nents, dir,
  218. attrs);
  219. if (!ret) {
  220. DRM_ERROR("dma map sg failed\n");
  221. return -ENOMEM;
  222. }
  223. }
  224. if (sgt && sgt->sgl) {
  225. DRM_DEBUG("%pad/0x%x/0x%x/0x%lx\n",
  226. &sgt->sgl->dma_address, sgt->sgl->dma_length,
  227. dir, attrs);
  228. SDE_EVT32(sgt->sgl->dma_address, sgt->sgl->dma_length,
  229. dir, attrs, client->secure, flags);
  230. }
  231. return 0;
  232. }
  233. static void msm_smmu_unmap_dma_buf(struct msm_mmu *mmu, struct sg_table *sgt,
  234. int dir, u32 flags)
  235. {
  236. struct msm_smmu *smmu = to_msm_smmu(mmu);
  237. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  238. if (!sgt || !client) {
  239. DRM_ERROR("sg table is invalid\n");
  240. return;
  241. }
  242. if (sgt->sgl) {
  243. DRM_DEBUG("%pad/0x%x/0x%x\n",
  244. &sgt->sgl->dma_address, sgt->sgl->dma_length,
  245. dir);
  246. SDE_EVT32(sgt->sgl->dma_address, sgt->sgl->dma_length,
  247. dir, client->secure, flags);
  248. }
  249. if (!(flags & MSM_BO_EXTBUF))
  250. dma_unmap_sg(client->dev, sgt->sgl, sgt->nents, dir);
  251. }
  252. static bool msm_smmu_is_domain_secure(struct msm_mmu *mmu)
  253. {
  254. struct msm_smmu *smmu = to_msm_smmu(mmu);
  255. struct msm_smmu_client *client = msm_smmu_to_client(smmu);
  256. return client->secure;
  257. }
  258. static const struct msm_mmu_funcs funcs = {
  259. .attach = msm_smmu_attach,
  260. .detach = msm_smmu_detach,
  261. .map = msm_smmu_map,
  262. .unmap = msm_smmu_unmap,
  263. .map_dma_buf = msm_smmu_map_dma_buf,
  264. .unmap_dma_buf = msm_smmu_unmap_dma_buf,
  265. .destroy = msm_smmu_destroy,
  266. .is_domain_secure = msm_smmu_is_domain_secure,
  267. #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0))
  268. .enable_smmu_translations = msm_enable_smmu_translations,
  269. #else
  270. .set_attribute = msm_smmu_set_attribute,
  271. #endif
  272. .one_to_one_map = msm_smmu_one_to_one_map,
  273. .one_to_one_unmap = msm_smmu_one_to_one_unmap,
  274. .get_dev = msm_smmu_get_dev,
  275. };
  276. static struct msm_smmu_domain msm_smmu_domains[MSM_SMMU_DOMAIN_MAX] = {
  277. [MSM_SMMU_DOMAIN_UNSECURE] = {
  278. .label = "mdp_ns",
  279. .secure = false,
  280. },
  281. [MSM_SMMU_DOMAIN_SECURE] = {
  282. .label = "mdp_s",
  283. .secure = true,
  284. },
  285. [MSM_SMMU_DOMAIN_NRT_UNSECURE] = {
  286. .label = "rot_ns",
  287. .secure = false,
  288. },
  289. [MSM_SMMU_DOMAIN_NRT_SECURE] = {
  290. .label = "rot_s",
  291. .secure = true,
  292. },
  293. };
  294. static const struct of_device_id msm_smmu_dt_match[] = {
  295. { .compatible = "qcom,smmu_sde_unsec",
  296. .data = &msm_smmu_domains[MSM_SMMU_DOMAIN_UNSECURE] },
  297. { .compatible = "qcom,smmu_sde_sec",
  298. .data = &msm_smmu_domains[MSM_SMMU_DOMAIN_SECURE] },
  299. { .compatible = "qcom,smmu_sde_nrt_unsec",
  300. .data = &msm_smmu_domains[MSM_SMMU_DOMAIN_NRT_UNSECURE] },
  301. { .compatible = "qcom,smmu_sde_nrt_sec",
  302. .data = &msm_smmu_domains[MSM_SMMU_DOMAIN_NRT_SECURE] },
  303. {}
  304. };
  305. MODULE_DEVICE_TABLE(of, msm_smmu_dt_match);
  306. static struct msm_smmu_client *msm_smmu_get_smmu(const char *compat)
  307. {
  308. struct msm_smmu_client *curr = NULL;
  309. bool found = false;
  310. if (!compat) {
  311. pr_err("invalid param\n");
  312. return ERR_PTR(-EINVAL);
  313. }
  314. mutex_lock(&smmu_list_lock);
  315. list_for_each_entry(curr, &sde_smmu_list, smmu_list) {
  316. if (of_compat_cmp(compat, curr->compat, strlen(compat)) == 0) {
  317. DRM_DEBUG("found msm_smmu_client for %s\n", compat);
  318. found = true;
  319. break;
  320. }
  321. }
  322. mutex_unlock(&smmu_list_lock);
  323. if (!found)
  324. return ERR_PTR(-ENODEV);
  325. return curr;
  326. }
  327. static struct device *msm_smmu_device_add(struct device *dev,
  328. enum msm_mmu_domain_type domain,
  329. struct msm_smmu *smmu)
  330. {
  331. int i;
  332. const char *compat = NULL;
  333. for (i = 0; i < ARRAY_SIZE(msm_smmu_dt_match); i++) {
  334. if (msm_smmu_dt_match[i].data == &msm_smmu_domains[domain]) {
  335. compat = msm_smmu_dt_match[i].compatible;
  336. break;
  337. }
  338. }
  339. if (!compat) {
  340. DRM_DEBUG("unable to find matching domain for %d\n", domain);
  341. return ERR_PTR(-ENOENT);
  342. }
  343. DRM_DEBUG("found domain %d compat: %s\n", domain, compat);
  344. smmu->client = msm_smmu_get_smmu(compat);
  345. if (IS_ERR_OR_NULL(smmu->client)) {
  346. DRM_ERROR("unable to find domain %d compat: %s\n", domain,
  347. compat);
  348. return ERR_PTR(-ENODEV);
  349. }
  350. return smmu->client->dev;
  351. }
  352. struct msm_mmu *msm_smmu_new(struct device *dev,
  353. enum msm_mmu_domain_type domain)
  354. {
  355. struct msm_smmu *smmu;
  356. struct device *client_dev;
  357. smmu = kzalloc(sizeof(*smmu), GFP_KERNEL);
  358. if (!smmu)
  359. return ERR_PTR(-ENOMEM);
  360. client_dev = msm_smmu_device_add(dev, domain, smmu);
  361. if (IS_ERR_OR_NULL(client_dev)) {
  362. kfree(smmu);
  363. return (void *)client_dev ? : ERR_PTR(-ENODEV);
  364. }
  365. smmu->client_dev = client_dev;
  366. msm_mmu_init(&smmu->base, dev, &funcs);
  367. return &smmu->base;
  368. }
  369. static int msm_smmu_fault_handler(struct iommu_domain *domain,
  370. struct device *dev, unsigned long iova,
  371. int flags, void *token)
  372. {
  373. struct msm_smmu_client *client;
  374. int rc = -EINVAL;
  375. if (!token) {
  376. DRM_ERROR("Error: token is NULL\n");
  377. return -EINVAL;
  378. }
  379. client = (struct msm_smmu_client *)token;
  380. /* see iommu.h for fault flags definition */
  381. SDE_EVT32(iova, flags);
  382. DRM_ERROR("trigger dump, iova=0x%08lx, flags=0x%x\n", iova, flags);
  383. DRM_ERROR("SMMU device:%s", client->dev ? client->dev->kobj.name : "");
  384. /*
  385. * return -ENOSYS to allow smmu driver to dump out useful
  386. * debug info.
  387. */
  388. return rc;
  389. }
  390. /**
  391. * msm_smmu_bind - bind smmu device with controlling device
  392. * @dev: Pointer to base of platform device
  393. * @master: Pointer to container of drm device
  394. * @data: Pointer to private data
  395. * Returns: Zero on success
  396. */
  397. static int msm_smmu_bind(struct device *dev, struct device *master, void *data)
  398. {
  399. return 0;
  400. }
  401. /**
  402. * msm_smmu_unbind - unbind msm_smmu from controlling device
  403. * @dev: Pointer to base of platform device
  404. * @master: Pointer to container of drm device
  405. * @data: Pointer to private data
  406. */
  407. static void msm_smmu_unbind(struct device *dev,
  408. struct device *master, void *data)
  409. {
  410. }
  411. static const struct component_ops msm_smmu_comp_ops = {
  412. .bind = msm_smmu_bind,
  413. .unbind = msm_smmu_unbind,
  414. };
  415. /**
  416. * msm_smmu_probe()
  417. * @pdev: platform device
  418. *
  419. * Each smmu context acts as a separate device and the context banks are
  420. * configured with a VA range.
  421. * Registers the clks as each context bank has its own clks, for which voting
  422. * has to be done everytime before using that context bank.
  423. */
  424. static int msm_smmu_probe(struct platform_device *pdev)
  425. {
  426. const struct of_device_id *match;
  427. struct msm_smmu_client *client;
  428. const struct msm_smmu_domain *domain;
  429. int ret;
  430. match = of_match_device(msm_smmu_dt_match, &pdev->dev);
  431. if (!match || !match->data) {
  432. dev_err(&pdev->dev, "probe failed as match data is invalid\n");
  433. return -EINVAL;
  434. }
  435. domain = match->data;
  436. if (!domain) {
  437. dev_err(&pdev->dev, "no matching device found\n");
  438. return -EINVAL;
  439. }
  440. DRM_INFO("probing device %s\n", match->compatible);
  441. client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
  442. if (!client)
  443. return -ENOMEM;
  444. client->dev = &pdev->dev;
  445. client->domain = iommu_get_domain_for_dev(client->dev);
  446. if (!client->domain) {
  447. dev_err(&pdev->dev, "iommu get domain for dev failed\n");
  448. return -EINVAL;
  449. }
  450. client->compat = match->compatible;
  451. client->secure = domain->secure;
  452. client->domain_attached = true;
  453. if (!client->dev->dma_parms)
  454. client->dev->dma_parms = devm_kzalloc(client->dev,
  455. sizeof(*client->dev->dma_parms), GFP_KERNEL);
  456. dma_set_max_seg_size(client->dev, DMA_BIT_MASK(32));
  457. dma_set_seg_boundary(client->dev, (unsigned long)DMA_BIT_MASK(64));
  458. iommu_set_fault_handler(client->domain,
  459. msm_smmu_fault_handler, (void *)client);
  460. DRM_INFO("Created domain %s, secure=%d\n",
  461. domain->label, domain->secure);
  462. platform_set_drvdata(pdev, client);
  463. mutex_lock(&smmu_list_lock);
  464. list_add(&client->smmu_list, &sde_smmu_list);
  465. mutex_unlock(&smmu_list_lock);
  466. ret = component_add(&pdev->dev, &msm_smmu_comp_ops);
  467. if (ret)
  468. pr_err("component add failed\n");
  469. return ret;
  470. }
  471. static int msm_smmu_remove(struct platform_device *pdev)
  472. {
  473. struct msm_smmu_client *client;
  474. struct msm_smmu_client *curr, *next;
  475. client = platform_get_drvdata(pdev);
  476. client->domain_attached = false;
  477. mutex_lock(&smmu_list_lock);
  478. list_for_each_entry_safe(curr, next, &sde_smmu_list, smmu_list) {
  479. if (curr == client) {
  480. list_del(&client->smmu_list);
  481. break;
  482. }
  483. }
  484. mutex_unlock(&smmu_list_lock);
  485. return 0;
  486. }
  487. static struct platform_driver msm_smmu_driver = {
  488. .probe = msm_smmu_probe,
  489. .remove = msm_smmu_remove,
  490. .driver = {
  491. .name = "msmdrm_smmu",
  492. .of_match_table = msm_smmu_dt_match,
  493. .suppress_bind_attrs = true,
  494. },
  495. };
  496. int __init msm_smmu_driver_init(void)
  497. {
  498. int ret;
  499. ret = platform_driver_register(&msm_smmu_driver);
  500. if (ret)
  501. pr_err("mdss_smmu_register_driver() failed!\n");
  502. return ret;
  503. }
  504. void __exit msm_smmu_driver_cleanup(void)
  505. {
  506. platform_driver_unregister(&msm_smmu_driver);
  507. }
  508. MODULE_LICENSE("GPL v2");
  509. MODULE_DESCRIPTION("MSM SMMU driver");