From 3b6916b4d499de947cbcfeada39bf9aacffdf87d Mon Sep 17 00:00:00 2001 From: Patrick Daly Date: Tue, 22 Jun 2021 12:05:36 -0700 Subject: [PATCH] ANDROID: iommu/io-pgtable-arm: Add IOMMU_CACHE_ICACHE_OCACHE_NWA Allow io-coherent devices to use a inner writeback read/write allocate, outer writeback read allocate, no-write allocate cache policy. The outer cache policy affects the behavior of a system cache, at least on qcom boards which have one. The rational follows that of IOMMU_SYS_CACHE_ONLY_NWA. Certain gpu usecases perform better when using a no-write allocate policy. Rename the IOMMU_SYS_CACHE_* flags to better reflect that they are not exclusive with IOMMU_CACHE. Bug: 191811876 Change-Id: Ic91616a148f39fead008a5b87a54ffd781fee734 Signed-off-by: Patrick Daly --- drivers/iommu/dma-iommu.c | 4 ++-- drivers/iommu/io-pgtable-arm.c | 38 ++++++++++++++++++++-------------- include/linux/iommu.h | 17 +++++++-------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 27f74c12bd02..7bbccb680c2b 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -434,9 +434,9 @@ static int dma_info_to_prot(enum dma_data_direction dir, bool coherent, if (attrs & DMA_ATTR_PRIVILEGED) prot |= IOMMU_PRIV; if (attrs & DMA_ATTR_SYS_CACHE_ONLY) - prot |= IOMMU_SYS_CACHE_ONLY; + prot |= IOMMU_SYS_CACHE; if (attrs & DMA_ATTR_SYS_CACHE_ONLY_NWA) - prot |= IOMMU_SYS_CACHE_ONLY_NWA; + prot |= IOMMU_SYS_CACHE_NWA; switch (dir) { case DMA_BIDIRECTIONAL: diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 0d7bb004e587..ec31966b4f6f 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -112,18 +112,20 @@ #define ARM_LPAE_VTCR_PS_SHIFT 16 #define ARM_LPAE_VTCR_PS_MASK 0x7 -#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) -#define ARM_LPAE_MAIR_ATTR_MASK 0xff -#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04ULL -#define ARM_LPAE_MAIR_ATTR_NC 0x44ULL -#define ARM_LPAE_MAIR_ATTR_INC_OWBRANWA 0xe4ULL -#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4ULL -#define ARM_LPAE_MAIR_ATTR_WBRWA 0xffULL -#define ARM_LPAE_MAIR_ATTR_IDX_NC 0 -#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 -#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 -#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3 -#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA 4 +#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define ARM_LPAE_MAIR_ATTR_MASK 0xff +#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04ULL +#define ARM_LPAE_MAIR_ATTR_NC 0x44ULL +#define ARM_LPAE_MAIR_ATTR_INC_OWBRANWA 0xe4ULL +#define ARM_LPAE_MAIR_ATTR_IWBRWA_OWBRANWA 0xefULL +#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4ULL +#define ARM_LPAE_MAIR_ATTR_WBRWA 0xffULL +#define ARM_LPAE_MAIR_ATTR_IDX_NC 0 +#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 +#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 +#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3 +#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA 4 +#define ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA 5 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0) #define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2) @@ -435,13 +437,17 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, if (prot & IOMMU_MMIO) pte |= (ARM_LPAE_MAIR_ATTR_IDX_DEV << ARM_LPAE_PTE_ATTRINDX_SHIFT); + else if ((prot & IOMMU_CACHE) && (prot & IOMMU_SYS_CACHE_NWA)) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + /* IOMMU_CACHE + IOMMU_SYS_CACHE equivalent to IOMMU_CACHE */ else if (prot & IOMMU_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); - else if (prot & IOMMU_SYS_CACHE_ONLY) + else if (prot & IOMMU_SYS_CACHE) pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE << ARM_LPAE_PTE_ATTRINDX_SHIFT); - else if (prot & IOMMU_SYS_CACHE_ONLY_NWA) + else if (prot & IOMMU_SYS_CACHE_NWA) pte |= (ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA << ARM_LPAE_PTE_ATTRINDX_SHIFT); } @@ -904,7 +910,9 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) (ARM_LPAE_MAIR_ATTR_INC_OWBRWA << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE)) | (ARM_LPAE_MAIR_ATTR_INC_OWBRANWA - << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA)); + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE_NWA)) | + (ARM_LPAE_MAIR_ATTR_IWBRWA_OWBRANWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_ICACHE_OCACHE_NWA)); cfg->arm_lpae_s1_cfg.mair = reg; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 8417233a11a5..f7f6ada34836 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -32,18 +32,17 @@ */ #define IOMMU_PRIV (1 << 5) /* - * Non-coherent masters can use this page protection flag to set cacheable - * memory attributes for only a transparent outer level of cache, also known as - * the last-level or system cache. + * Allow caching in a transparent outer level of cache, also known as + * the last-level or system cache, with a read/write allocation policy. + * Does not depend on IOMMU_CACHE. Incompatible with IOMMU_SYS_CACHE_NWA. */ -#define IOMMU_SYS_CACHE_ONLY (1 << 6) +#define IOMMU_SYS_CACHE (1 << 6) /* - * Non-coherent masters can use this page protection flag to set cacheable - * memory attributes with a no write allocation cache policy for only a - * transparent outer level of cache, also known as the last-level or system - * cache. + * Allow caching in a transparent outer level of cache, also known as + * the last-level or system cache, with a read allocation policy. + * Does not depend on IOMMU_CACHE. Incompatible with IOMMU_SYS_CACHE. */ -#define IOMMU_SYS_CACHE_ONLY_NWA (1 << 7) +#define IOMMU_SYS_CACHE_NWA (1 << 7) struct iommu_ops; struct iommu_group;