xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branches 'arm/omap', 'arm/exynos', 'arm/smmu', 'arm/mediatek', 'arm/qcom', 'arm/renesas', 'x86/amd', 'x86/vt-d' and 'core' into next

Browse Source
This commit is contained in:
Joerg Roedel
2019-09-11 12:39:19 +02:00
parent f74c2bb987 96088a203a 7991eb39ee 097a7df2e3 4c00889341 8758553791 3623002f0f 3d70889532 1f76249cc3 2896ba40d0
commit e95adb9add
63 changed files with 3873 additions and 1494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
drivers/iommu/Kconfig
View File

@@ -182,6 +182,7 @@ config INTEL_IOMMU
select IOMMU_IOVA
select NEED_DMA_MAP_STATE
select DMAR_TABLE
select SWIOTLB
help
DMA remapping (DMAR) devices support enables independent address
translations for Direct Memory Access (DMA) from devices.

5
drivers/iommu/Makefile
View File

@@ -10,13 +10,14 @@ obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o
obj-$(CONFIG_IOMMU_IOVA) += iova.o
obj-$(CONFIG_OF_IOMMU) += of_iommu.o
obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o
obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o
obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o
obj-$(CONFIG_ARM_SMMU) += arm-smmu.o
obj-$(CONFIG_ARM_SMMU) += arm-smmu.o arm-smmu-impl.o
obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o
obj-$(CONFIG_DMAR_TABLE) += dmar.o
obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o
obj-$(CONFIG_INTEL_IOMMU) += intel-trace.o
obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel-iommu-debugfs.o
obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o
obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o

106
drivers/iommu/amd_iommu.c
View File

@@ -436,7 +436,7 @@ static int iommu_init_device(struct device *dev)
* invalid address), we ignore the capability for the device so
* it'll be forced to go into translation mode.
*/
if ((iommu_pass_through || !amd_iommu_force_isolation) &&
if ((iommu_default_passthrough() || !amd_iommu_force_isolation) &&
dev_is_pci(dev) && pci_iommuv2_capable(to_pci_dev(dev))) {
struct amd_iommu *iommu;
@@ -2256,7 +2256,7 @@ static int amd_iommu_add_device(struct device *dev)
BUG_ON(!dev_data);
if (iommu_pass_through || dev_data->iommu_v2)
if (dev_data->iommu_v2)
iommu_request_dm_for_dev(dev);
/* Domains are initialized for this device - have a look what we ended up with */
@@ -2577,7 +2577,9 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
bus_addr = address + s->dma_address + (j << PAGE_SHIFT);
phys_addr = (sg_phys(s) & PAGE_MASK) + (j << PAGE_SHIFT);
ret = iommu_map_page(domain, bus_addr, phys_addr, PAGE_SIZE, prot, GFP_ATOMIC);
ret = iommu_map_page(domain, bus_addr, phys_addr,
PAGE_SIZE, prot,
GFP_ATOMIC | __GFP_NOWARN);
if (ret)
goto out_unmap;
@@ -2835,7 +2837,7 @@ int __init amd_iommu_init_api(void)
int __init amd_iommu_init_dma_ops(void)
{
swiotlb = (iommu_pass_through || sme_me_mask) ? 1 : 0;
swiotlb = (iommu_default_passthrough() || sme_me_mask) ? 1 : 0;
iommu_detected = 1;
if (amd_iommu_unmap_flush)
@@ -3085,7 +3087,8 @@ static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
}
static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
size_t page_size)
size_t page_size,
struct iommu_iotlb_gather *gather)
{
struct protection_domain *domain = to_pdomain(dom);
size_t unmap_size;
@@ -3226,9 +3229,10 @@ static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
domain_flush_complete(dom);
}
static void amd_iommu_iotlb_range_add(struct iommu_domain *domain,
unsigned long iova, size_t size)
static void amd_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
amd_iommu_flush_iotlb_all(domain);
}
const struct iommu_ops amd_iommu_ops = {
@@ -3249,8 +3253,7 @@ const struct iommu_ops amd_iommu_ops = {
.is_attach_deferred = amd_iommu_is_attach_deferred,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
.flush_iotlb_all = amd_iommu_flush_iotlb_all,
.iotlb_range_add = amd_iommu_iotlb_range_add,
.iotlb_sync = amd_iommu_flush_iotlb_all,
.iotlb_sync = amd_iommu_iotlb_sync,
};
/*****************************************************************************
@@ -4343,13 +4346,62 @@ static const struct irq_domain_ops amd_ir_domain_ops = {
.deactivate = irq_remapping_deactivate,
};
int amd_iommu_activate_guest_mode(void *data)
{
struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
!entry || entry->lo.fields_vapic.guest_mode)
return 0;
entry->lo.val = 0;
entry->hi.val = 0;
entry->lo.fields_vapic.guest_mode = 1;
entry->lo.fields_vapic.ga_log_intr = 1;
entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr;
entry->hi.fields.vector = ir_data->ga_vector;
entry->lo.fields_vapic.ga_tag = ir_data->ga_tag;
return modify_irte_ga(ir_data->irq_2_irte.devid,
ir_data->irq_2_irte.index, entry, NULL);
}
EXPORT_SYMBOL(amd_iommu_activate_guest_mode);
int amd_iommu_deactivate_guest_mode(void *data)
{
struct amd_ir_data *ir_data = (struct amd_ir_data *)data;
struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
struct irq_cfg *cfg = ir_data->cfg;
if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) ||
!entry || !entry->lo.fields_vapic.guest_mode)
return 0;
entry->lo.val = 0;
entry->hi.val = 0;
entry->lo.fields_remap.dm = apic->irq_dest_mode;
entry->lo.fields_remap.int_type = apic->irq_delivery_mode;
entry->hi.fields.vector = cfg->vector;
entry->lo.fields_remap.destination =
APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
entry->hi.fields.destination =
APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
return modify_irte_ga(ir_data->irq_2_irte.devid,
ir_data->irq_2_irte.index, entry, NULL);
}
EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode);
static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
{
int ret;
struct amd_iommu *iommu;
struct amd_iommu_pi_data *pi_data = vcpu_info;
struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data;
struct amd_ir_data *ir_data = data->chip_data;
struct irte_ga *irte = (struct irte_ga *) ir_data->entry;
struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid);
@@ -4360,6 +4412,7 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
if (!dev_data || !dev_data->use_vapic)
return 0;
ir_data->cfg = irqd_cfg(data);
pi_data->ir_data = ir_data;
/* Note:
@@ -4378,37 +4431,24 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *vcpu_info)
pi_data->prev_ga_tag = ir_data->cached_ga_tag;
if (pi_data->is_guest_mode) {
/* Setting */
irte->hi.fields.ga_root_ptr = (pi_data->base >> 12);
irte->hi.fields.vector = vcpu_pi_info->vector;
irte->lo.fields_vapic.ga_log_intr = 1;
irte->lo.fields_vapic.guest_mode = 1;
irte->lo.fields_vapic.ga_tag = pi_data->ga_tag;
ir_data->cached_ga_tag = pi_data->ga_tag;
ir_data->ga_root_ptr = (pi_data->base >> 12);
ir_data->ga_vector = vcpu_pi_info->vector;
ir_data->ga_tag = pi_data->ga_tag;
ret = amd_iommu_activate_guest_mode(ir_data);
if (!ret)
ir_data->cached_ga_tag = pi_data->ga_tag;
} else {
/* Un-Setting */
struct irq_cfg *cfg = irqd_cfg(data);
irte->hi.val = 0;
irte->lo.val = 0;
irte->hi.fields.vector = cfg->vector;
irte->lo.fields_remap.guest_mode = 0;
irte->lo.fields_remap.destination =
APICID_TO_IRTE_DEST_LO(cfg->dest_apicid);
irte->hi.fields.destination =
APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
irte->lo.fields_remap.int_type = apic->irq_delivery_mode;
irte->lo.fields_remap.dm = apic->irq_dest_mode;
ret = amd_iommu_deactivate_guest_mode(ir_data);
/*
* This communicates the ga_tag back to the caller
* so that it can do all the necessary clean up.
*/
ir_data->cached_ga_tag = 0;
if (!ret)
ir_data->cached_ga_tag = 0;
}
return modify_irte_ga(irte_info->devid, irte_info->index, irte, ir_data);
return ret;
}

14
drivers/iommu/amd_iommu.h Normal file
View File

@@ -0,0 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef AMD_IOMMU_H
#define AMD_IOMMU_H
int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line);
#ifdef CONFIG_DMI
void amd_iommu_apply_ivrs_quirks(void);
#else
static void amd_iommu_apply_ivrs_quirks(void) { }
#endif
#endif

5
drivers/iommu/amd_iommu_init.c
View File

@@ -32,6 +32,7 @@
#include <asm/irq_remapping.h>
#include <linux/crash_dump.h>
#include "amd_iommu.h"
#include "amd_iommu_proto.h"
#include "amd_iommu_types.h"
#include "irq_remapping.h"
@@ -1002,7 +1003,7 @@ static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
set_iommu_for_device(iommu, devid);
}
static int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line)
int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line)
{
struct devid_map *entry;
struct list_head *list;
@@ -1153,6 +1154,8 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
if (ret)
return ret;
amd_iommu_apply_ivrs_quirks();
/*
* First save the recommended feature enable bits from ACPI
*/

92
drivers/iommu/amd_iommu_quirks.c Normal file
View File

@@ -0,0 +1,92 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Quirks for AMD IOMMU
*
* Copyright (C) 2019 Kai-Heng Feng <kai.heng.feng@canonical.com>
*/
#ifdef CONFIG_DMI
#include <linux/dmi.h>
#include "amd_iommu.h"
#define IVHD_SPECIAL_IOAPIC 1
struct ivrs_quirk_entry {
u8 id;
u16 devid;
};
enum {
DELL_INSPIRON_7375 = 0,
DELL_LATITUDE_5495,
LENOVO_IDEAPAD_330S_15ARR,
};
static const struct ivrs_quirk_entry ivrs_ioapic_quirks[][3] __initconst = {
/* ivrs_ioapic[4]=00:14.0 ivrs_ioapic[5]=00:00.2 */
[DELL_INSPIRON_7375] = {
{ .id = 4, .devid = 0xa0 },
{ .id = 5, .devid = 0x2 },
{}
},
/* ivrs_ioapic[4]=00:14.0 */
[DELL_LATITUDE_5495] = {
{ .id = 4, .devid = 0xa0 },
{}
},
/* ivrs_ioapic[32]=00:14.0 */
[LENOVO_IDEAPAD_330S_15ARR] = {
{ .id = 32, .devid = 0xa0 },
{}
},
{}
};
static int __init ivrs_ioapic_quirk_cb(const struct dmi_system_id *d)
{
const struct ivrs_quirk_entry *i;
for (i = d->driver_data; i->id != 0 && i->devid != 0; i++)
add_special_device(IVHD_SPECIAL_IOAPIC, i->id, (u16 *)&i->devid, 0);
return 0;
}
static const struct dmi_system_id ivrs_quirks[] __initconst = {
{
.callback = ivrs_ioapic_quirk_cb,
.ident = "Dell Inspiron 7375",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7375"),
},
.driver_data = (void *)&ivrs_ioapic_quirks[DELL_INSPIRON_7375],
},
{
.callback = ivrs_ioapic_quirk_cb,
.ident = "Dell Latitude 5495",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
DMI_MATCH(DMI_PRODUCT_NAME, "Latitude 5495"),
},
.driver_data = (void *)&ivrs_ioapic_quirks[DELL_LATITUDE_5495],
},
{
.callback = ivrs_ioapic_quirk_cb,
.ident = "Lenovo ideapad 330S-15ARR",
.matches = {
DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
DMI_MATCH(DMI_PRODUCT_NAME, "81FB"),
},
.driver_data = (void *)&ivrs_ioapic_quirks[LENOVO_IDEAPAD_330S_15ARR],
},
{}
};
void __init amd_iommu_apply_ivrs_quirks(void)
{
dmi_check_system(ivrs_quirks);
}
#endif

9
drivers/iommu/amd_iommu_types.h
View File

@@ -873,6 +873,15 @@ struct amd_ir_data {
struct msi_msg msi_entry;
void *entry; /* Pointer to union irte or struct irte_ga */
void *ref; /* Pointer to the actual irte */
/**
* Store information for activate/de-activate
* Guest virtual APIC mode during runtime.
*/
struct irq_cfg *cfg;
int ga_vector;
int ga_root_ptr;
int ga_tag;
};
struct amd_irte_ops {

174
drivers/iommu/arm-smmu-impl.c Normal file
View File

@@ -0,0 +1,174 @@
// SPDX-License-Identifier: GPL-2.0-only
// Miscellaneous Arm SMMU implementation and integration quirks
// Copyright (C) 2019 Arm Limited
#define pr_fmt(fmt) "arm-smmu: " fmt
#include <linux/bitfield.h>
#include <linux/of.h>
#include "arm-smmu.h"
static int arm_smmu_gr0_ns(int offset)
{
switch(offset) {
case ARM_SMMU_GR0_sCR0:
case ARM_SMMU_GR0_sACR:
case ARM_SMMU_GR0_sGFSR:
case ARM_SMMU_GR0_sGFSYNR0:
case ARM_SMMU_GR0_sGFSYNR1:
case ARM_SMMU_GR0_sGFSYNR2:
return offset + 0x400;
default:
return offset;
}
}
static u32 arm_smmu_read_ns(struct arm_smmu_device *smmu, int page,
int offset)
{
if (page == ARM_SMMU_GR0)
offset = arm_smmu_gr0_ns(offset);
return readl_relaxed(arm_smmu_page(smmu, page) + offset);
}
static void arm_smmu_write_ns(struct arm_smmu_device *smmu, int page,
int offset, u32 val)
{
if (page == ARM_SMMU_GR0)
offset = arm_smmu_gr0_ns(offset);
writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
}
/* Since we don't care for sGFAR, we can do without 64-bit accessors */
static const struct arm_smmu_impl calxeda_impl = {
.read_reg = arm_smmu_read_ns,
.write_reg = arm_smmu_write_ns,
};
struct cavium_smmu {
struct arm_smmu_device smmu;
u32 id_base;
};
static int cavium_cfg_probe(struct arm_smmu_device *smmu)
{
static atomic_t context_count = ATOMIC_INIT(0);
struct cavium_smmu *cs = container_of(smmu, struct cavium_smmu, smmu);
/*
* Cavium CN88xx erratum #27704.
* Ensure ASID and VMID allocation is unique across all SMMUs in
* the system.
*/
cs->id_base = atomic_fetch_add(smmu->num_context_banks, &context_count);
dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
return 0;
}
static int cavium_init_context(struct arm_smmu_domain *smmu_domain)
{
struct cavium_smmu *cs = container_of(smmu_domain->smmu,
struct cavium_smmu, smmu);
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
smmu_domain->cfg.vmid += cs->id_base;
else
smmu_domain->cfg.asid += cs->id_base;
return 0;
}
static const struct arm_smmu_impl cavium_impl = {
.cfg_probe = cavium_cfg_probe,
.init_context = cavium_init_context,
};
static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smmu)
{
struct cavium_smmu *cs;
cs = devm_kzalloc(smmu->dev, sizeof(*cs), GFP_KERNEL);
if (!cs)
return ERR_PTR(-ENOMEM);
cs->smmu = *smmu;
cs->smmu.impl = &cavium_impl;
devm_kfree(smmu->dev, smmu);
return &cs->smmu;
}
#define ARM_MMU500_ACTLR_CPRE (1 << 1)
#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
#define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10)
#define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
static int arm_mmu500_reset(struct arm_smmu_device *smmu)
{
u32 reg, major;
int i;
/*
* On MMU-500 r2p0 onwards we need to clear ACR.CACHE_LOCK before
* writes to the context bank ACTLRs will stick. And we just hope that
* Secure has also cleared SACR.CACHE_LOCK for this to take effect...
*/
reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
major = FIELD_GET(ID7_MAJOR, reg);
reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
if (major >= 2)
reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
/*
* Allow unmatched Stream IDs to allocate bypass
* TLB entries for reduced latency.
*/
reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
/*
* Disable MMU-500's not-particularly-beneficial next-page
* prefetcher for the sake of errata #841119 and #826419.
*/
for (i = 0; i < smmu->num_context_banks; ++i) {
reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
reg &= ~ARM_MMU500_ACTLR_CPRE;
arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
}
return 0;
}
static const struct arm_smmu_impl arm_mmu500_impl = {
.reset = arm_mmu500_reset,
};
struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
{
/*
* We will inevitably have to combine model-specific implementation
* quirks with platform-specific integration quirks, but everything
* we currently support happens to work out as straightforward
* mutually-exclusive assignments.
*/
switch (smmu->model) {
case ARM_MMU500:
smmu->impl = &arm_mmu500_impl;
break;
case CAVIUM_SMMUV2:
return cavium_smmu_impl_init(smmu);
default:
break;
}
if (of_property_read_bool(smmu->dev->of_node,
"calxeda,smmu-secure-config-access"))
smmu->impl = &calxeda_impl;
return smmu;
}

210
drivers/iommu/arm-smmu-regs.h
View File

@@ -1,210 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* IOMMU API for ARM architected SMMU implementations.
*
* Copyright (C) 2013 ARM Limited
*
* Author: Will Deacon <will.deacon@arm.com>
*/
#ifndef _ARM_SMMU_REGS_H
#define _ARM_SMMU_REGS_H
/* Configuration registers */
#define ARM_SMMU_GR0_sCR0 0x0
#define sCR0_CLIENTPD (1 << 0)
#define sCR0_GFRE (1 << 1)
#define sCR0_GFIE (1 << 2)
#define sCR0_EXIDENABLE (1 << 3)
#define sCR0_GCFGFRE (1 << 4)
#define sCR0_GCFGFIE (1 << 5)
#define sCR0_USFCFG (1 << 10)
#define sCR0_VMIDPNE (1 << 11)
#define sCR0_PTM (1 << 12)
#define sCR0_FB (1 << 13)
#define sCR0_VMID16EN (1 << 31)
#define sCR0_BSU_SHIFT 14
#define sCR0_BSU_MASK 0x3
/* Auxiliary Configuration register */
#define ARM_SMMU_GR0_sACR 0x10
/* Identification registers */
#define ARM_SMMU_GR0_ID0 0x20
#define ARM_SMMU_GR0_ID1 0x24
#define ARM_SMMU_GR0_ID2 0x28
#define ARM_SMMU_GR0_ID3 0x2c
#define ARM_SMMU_GR0_ID4 0x30
#define ARM_SMMU_GR0_ID5 0x34
#define ARM_SMMU_GR0_ID6 0x38
#define ARM_SMMU_GR0_ID7 0x3c
#define ARM_SMMU_GR0_sGFSR 0x48
#define ARM_SMMU_GR0_sGFSYNR0 0x50
#define ARM_SMMU_GR0_sGFSYNR1 0x54
#define ARM_SMMU_GR0_sGFSYNR2 0x58
#define ID0_S1TS (1 << 30)
#define ID0_S2TS (1 << 29)
#define ID0_NTS (1 << 28)
#define ID0_SMS (1 << 27)
#define ID0_ATOSNS (1 << 26)
#define ID0_PTFS_NO_AARCH32 (1 << 25)
#define ID0_PTFS_NO_AARCH32S (1 << 24)
#define ID0_CTTW (1 << 14)
#define ID0_NUMIRPT_SHIFT 16
#define ID0_NUMIRPT_MASK 0xff
#define ID0_NUMSIDB_SHIFT 9
#define ID0_NUMSIDB_MASK 0xf
#define ID0_EXIDS (1 << 8)
#define ID0_NUMSMRG_SHIFT 0
#define ID0_NUMSMRG_MASK 0xff
#define ID1_PAGESIZE (1 << 31)
#define ID1_NUMPAGENDXB_SHIFT 28
#define ID1_NUMPAGENDXB_MASK 7
#define ID1_NUMS2CB_SHIFT 16
#define ID1_NUMS2CB_MASK 0xff
#define ID1_NUMCB_SHIFT 0
#define ID1_NUMCB_MASK 0xff
#define ID2_OAS_SHIFT 4
#define ID2_OAS_MASK 0xf
#define ID2_IAS_SHIFT 0
#define ID2_IAS_MASK 0xf
#define ID2_UBS_SHIFT 8
#define ID2_UBS_MASK 0xf
#define ID2_PTFS_4K (1 << 12)
#define ID2_PTFS_16K (1 << 13)
#define ID2_PTFS_64K (1 << 14)
#define ID2_VMID16 (1 << 15)
#define ID7_MAJOR_SHIFT 4
#define ID7_MAJOR_MASK 0xf
/* Global TLB invalidation */
#define ARM_SMMU_GR0_TLBIVMID 0x64
#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
#define ARM_SMMU_GR0_TLBIALLH 0x6c
#define ARM_SMMU_GR0_sTLBGSYNC 0x70
#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
#define sTLBGSTATUS_GSACTIVE (1 << 0)
/* Stream mapping registers */
#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
#define SMR_VALID (1 << 31)
#define SMR_MASK_SHIFT 16
#define SMR_ID_SHIFT 0
#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
#define S2CR_CBNDX_SHIFT 0
#define S2CR_CBNDX_MASK 0xff
#define S2CR_EXIDVALID (1 << 10)
#define S2CR_TYPE_SHIFT 16
#define S2CR_TYPE_MASK 0x3
enum arm_smmu_s2cr_type {
S2CR_TYPE_TRANS,
S2CR_TYPE_BYPASS,
S2CR_TYPE_FAULT,
};
#define S2CR_PRIVCFG_SHIFT 24
#define S2CR_PRIVCFG_MASK 0x3
enum arm_smmu_s2cr_privcfg {
S2CR_PRIVCFG_DEFAULT,
S2CR_PRIVCFG_DIPAN,
S2CR_PRIVCFG_UNPRIV,
S2CR_PRIVCFG_PRIV,
};
/* Context bank attribute registers */
#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
#define CBAR_VMID_SHIFT 0
#define CBAR_VMID_MASK 0xff
#define CBAR_S1_BPSHCFG_SHIFT 8
#define CBAR_S1_BPSHCFG_MASK 3
#define CBAR_S1_BPSHCFG_NSH 3
#define CBAR_S1_MEMATTR_SHIFT 12
#define CBAR_S1_MEMATTR_MASK 0xf
#define CBAR_S1_MEMATTR_WB 0xf
#define CBAR_TYPE_SHIFT 16
#define CBAR_TYPE_MASK 0x3
#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT)
#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT)
#define CBAR_IRPTNDX_SHIFT 24
#define CBAR_IRPTNDX_MASK 0xff
#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
#define CBA2R_RW64_32BIT (0 << 0)
#define CBA2R_RW64_64BIT (1 << 0)
#define CBA2R_VMID_SHIFT 16
#define CBA2R_VMID_MASK 0xffff
#define ARM_SMMU_CB_SCTLR 0x0
#define ARM_SMMU_CB_ACTLR 0x4
#define ARM_SMMU_CB_RESUME 0x8
#define ARM_SMMU_CB_TTBCR2 0x10
#define ARM_SMMU_CB_TTBR0 0x20
#define ARM_SMMU_CB_TTBR1 0x28
#define ARM_SMMU_CB_TTBCR 0x30
#define ARM_SMMU_CB_CONTEXTIDR 0x34
#define ARM_SMMU_CB_S1_MAIR0 0x38
#define ARM_SMMU_CB_S1_MAIR1 0x3c
#define ARM_SMMU_CB_PAR 0x50
#define ARM_SMMU_CB_FSR 0x58
#define ARM_SMMU_CB_FAR 0x60
#define ARM_SMMU_CB_FSYNR0 0x68
#define ARM_SMMU_CB_S1_TLBIVA 0x600
#define ARM_SMMU_CB_S1_TLBIASID 0x610
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
#define ARM_SMMU_CB_TLBSYNC 0x7f0
#define ARM_SMMU_CB_TLBSTATUS 0x7f4
#define ARM_SMMU_CB_ATS1PR 0x800
#define ARM_SMMU_CB_ATSR 0x8f0
#define SCTLR_S1_ASIDPNE (1 << 12)
#define SCTLR_CFCFG (1 << 7)
#define SCTLR_CFIE (1 << 6)
#define SCTLR_CFRE (1 << 5)
#define SCTLR_E (1 << 4)
#define SCTLR_AFE (1 << 2)
#define SCTLR_TRE (1 << 1)
#define SCTLR_M (1 << 0)
#define CB_PAR_F (1 << 0)
#define ATSR_ACTIVE (1 << 0)
#define RESUME_RETRY (0 << 0)
#define RESUME_TERMINATE (1 << 0)
#define TTBCR2_SEP_SHIFT 15
#define TTBCR2_SEP_UPSTREAM (0x7 << TTBCR2_SEP_SHIFT)
#define TTBCR2_AS (1 << 4)
#define TTBRn_ASID_SHIFT 48
#define FSR_MULTI (1 << 31)
#define FSR_SS (1 << 30)
#define FSR_UUT (1 << 8)
#define FSR_ASF (1 << 7)
#define FSR_TLBLKF (1 << 6)
#define FSR_TLBMCF (1 << 5)
#define FSR_EF (1 << 4)
#define FSR_PF (1 << 3)
#define FSR_AFF (1 << 2)
#define FSR_TF (1 << 1)
#define FSR_IGN (FSR_AFF | FSR_ASF | \
FSR_TLBMCF | FSR_TLBLKF)
#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
#define FSYNR0_WNR (1 << 4)
#endif /* _ARM_SMMU_REGS_H */

1018
drivers/iommu/arm-smmu-v3.c
View File

File diff suppressed because it is too large Load Diff

664
drivers/iommu/arm-smmu.c
View File

File diff suppressed because it is too large Load Diff

402
drivers/iommu/arm-smmu.h Normal file
View File

@@ -0,0 +1,402 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* IOMMU API for ARM architected SMMU implementations.
*
* Copyright (C) 2013 ARM Limited
*
* Author: Will Deacon <will.deacon@arm.com>
*/
#ifndef _ARM_SMMU_H
#define _ARM_SMMU_H
#include <linux/atomic.h>
#include <linux/bits.h>
#include <linux/clk.h>
#include <linux/device.h>
#include <linux/io-64-nonatomic-hi-lo.h>
#include <linux/io-pgtable.h>
#include <linux/iommu.h>
#include <linux/mutex.h>
#include <linux/spinlock.h>
#include <linux/types.h>
/* Configuration registers */
#define ARM_SMMU_GR0_sCR0 0x0
#define sCR0_VMID16EN BIT(31)
#define sCR0_BSU GENMASK(15, 14)
#define sCR0_FB BIT(13)
#define sCR0_PTM BIT(12)
#define sCR0_VMIDPNE BIT(11)
#define sCR0_USFCFG BIT(10)
#define sCR0_GCFGFIE BIT(5)
#define sCR0_GCFGFRE BIT(4)
#define sCR0_EXIDENABLE BIT(3)
#define sCR0_GFIE BIT(2)
#define sCR0_GFRE BIT(1)
#define sCR0_CLIENTPD BIT(0)
/* Auxiliary Configuration register */
#define ARM_SMMU_GR0_sACR 0x10
/* Identification registers */
#define ARM_SMMU_GR0_ID0 0x20
#define ID0_S1TS BIT(30)
#define ID0_S2TS BIT(29)
#define ID0_NTS BIT(28)
#define ID0_SMS BIT(27)
#define ID0_ATOSNS BIT(26)
#define ID0_PTFS_NO_AARCH32 BIT(25)
#define ID0_PTFS_NO_AARCH32S BIT(24)
#define ID0_NUMIRPT GENMASK(23, 16)
#define ID0_CTTW BIT(14)
#define ID0_NUMSIDB GENMASK(12, 9)
#define ID0_EXIDS BIT(8)
#define ID0_NUMSMRG GENMASK(7, 0)
#define ARM_SMMU_GR0_ID1 0x24
#define ID1_PAGESIZE BIT(31)
#define ID1_NUMPAGENDXB GENMASK(30, 28)
#define ID1_NUMS2CB GENMASK(23, 16)
#define ID1_NUMCB GENMASK(7, 0)
#define ARM_SMMU_GR0_ID2 0x28
#define ID2_VMID16 BIT(15)
#define ID2_PTFS_64K BIT(14)
#define ID2_PTFS_16K BIT(13)
#define ID2_PTFS_4K BIT(12)
#define ID2_UBS GENMASK(11, 8)
#define ID2_OAS GENMASK(7, 4)
#define ID2_IAS GENMASK(3, 0)
#define ARM_SMMU_GR0_ID3 0x2c
#define ARM_SMMU_GR0_ID4 0x30
#define ARM_SMMU_GR0_ID5 0x34
#define ARM_SMMU_GR0_ID6 0x38
#define ARM_SMMU_GR0_ID7 0x3c
#define ID7_MAJOR GENMASK(7, 4)
#define ID7_MINOR GENMASK(3, 0)
#define ARM_SMMU_GR0_sGFSR 0x48
#define ARM_SMMU_GR0_sGFSYNR0 0x50
#define ARM_SMMU_GR0_sGFSYNR1 0x54
#define ARM_SMMU_GR0_sGFSYNR2 0x58
/* Global TLB invalidation */
#define ARM_SMMU_GR0_TLBIVMID 0x64
#define ARM_SMMU_GR0_TLBIALLNSNH 0x68
#define ARM_SMMU_GR0_TLBIALLH 0x6c
#define ARM_SMMU_GR0_sTLBGSYNC 0x70
#define ARM_SMMU_GR0_sTLBGSTATUS 0x74
#define sTLBGSTATUS_GSACTIVE BIT(0)
/* Stream mapping registers */
#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2))
#define SMR_VALID BIT(31)
#define SMR_MASK GENMASK(31, 16)
#define SMR_ID GENMASK(15, 0)
#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2))
#define S2CR_PRIVCFG GENMASK(25, 24)
enum arm_smmu_s2cr_privcfg {
S2CR_PRIVCFG_DEFAULT,
S2CR_PRIVCFG_DIPAN,
S2CR_PRIVCFG_UNPRIV,
S2CR_PRIVCFG_PRIV,
};
#define S2CR_TYPE GENMASK(17, 16)
enum arm_smmu_s2cr_type {
S2CR_TYPE_TRANS,
S2CR_TYPE_BYPASS,
S2CR_TYPE_FAULT,
};
#define S2CR_EXIDVALID BIT(10)
#define S2CR_CBNDX GENMASK(7, 0)
/* Context bank attribute registers */
#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2))
#define CBAR_IRPTNDX GENMASK(31, 24)
#define CBAR_TYPE GENMASK(17, 16)
enum arm_smmu_cbar_type {
CBAR_TYPE_S2_TRANS,
CBAR_TYPE_S1_TRANS_S2_BYPASS,
CBAR_TYPE_S1_TRANS_S2_FAULT,
CBAR_TYPE_S1_TRANS_S2_TRANS,
};
#define CBAR_S1_MEMATTR GENMASK(15, 12)
#define CBAR_S1_MEMATTR_WB 0xf
#define CBAR_S1_BPSHCFG GENMASK(9, 8)
#define CBAR_S1_BPSHCFG_NSH 3
#define CBAR_VMID GENMASK(7, 0)
#define ARM_SMMU_GR1_CBFRSYNRA(n) (0x400 + ((n) << 2))
#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2))
#define CBA2R_VMID16 GENMASK(31, 16)
#define CBA2R_VA64 BIT(0)
#define ARM_SMMU_CB_SCTLR 0x0
#define SCTLR_S1_ASIDPNE BIT(12)
#define SCTLR_CFCFG BIT(7)
#define SCTLR_CFIE BIT(6)
#define SCTLR_CFRE BIT(5)
#define SCTLR_E BIT(4)
#define SCTLR_AFE BIT(2)
#define SCTLR_TRE BIT(1)
#define SCTLR_M BIT(0)
#define ARM_SMMU_CB_ACTLR 0x4
#define ARM_SMMU_CB_RESUME 0x8
#define RESUME_TERMINATE BIT(0)
#define ARM_SMMU_CB_TCR2 0x10
#define TCR2_SEP GENMASK(17, 15)
#define TCR2_SEP_UPSTREAM 0x7
#define TCR2_AS BIT(4)
#define ARM_SMMU_CB_TTBR0 0x20
#define ARM_SMMU_CB_TTBR1 0x28
#define TTBRn_ASID GENMASK_ULL(63, 48)
#define ARM_SMMU_CB_TCR 0x30
#define ARM_SMMU_CB_CONTEXTIDR 0x34
#define ARM_SMMU_CB_S1_MAIR0 0x38
#define ARM_SMMU_CB_S1_MAIR1 0x3c
#define ARM_SMMU_CB_PAR 0x50
#define CB_PAR_F BIT(0)
#define ARM_SMMU_CB_FSR 0x58
#define FSR_MULTI BIT(31)
#define FSR_SS BIT(30)
#define FSR_UUT BIT(8)
#define FSR_ASF BIT(7)
#define FSR_TLBLKF BIT(6)
#define FSR_TLBMCF BIT(5)
#define FSR_EF BIT(4)
#define FSR_PF BIT(3)
#define FSR_AFF BIT(2)
#define FSR_TF BIT(1)
#define FSR_IGN (FSR_AFF | FSR_ASF | \
FSR_TLBMCF | FSR_TLBLKF)
#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \
FSR_EF | FSR_PF | FSR_TF | FSR_IGN)
#define ARM_SMMU_CB_FAR 0x60
#define ARM_SMMU_CB_FSYNR0 0x68
#define FSYNR0_WNR BIT(4)
#define ARM_SMMU_CB_S1_TLBIVA 0x600
#define ARM_SMMU_CB_S1_TLBIASID 0x610
#define ARM_SMMU_CB_S1_TLBIVAL 0x620
#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630
#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638
#define ARM_SMMU_CB_TLBSYNC 0x7f0
#define ARM_SMMU_CB_TLBSTATUS 0x7f4
#define ARM_SMMU_CB_ATS1PR 0x800
#define ARM_SMMU_CB_ATSR 0x8f0
#define ATSR_ACTIVE BIT(0)
/* Maximum number of context banks per SMMU */
#define ARM_SMMU_MAX_CBS 128
/* Shared driver definitions */
enum arm_smmu_arch_version {
ARM_SMMU_V1,
ARM_SMMU_V1_64K,
ARM_SMMU_V2,
};
enum arm_smmu_implementation {
GENERIC_SMMU,
ARM_MMU500,
CAVIUM_SMMUV2,
QCOM_SMMUV2,
};
struct arm_smmu_device {
struct device *dev;
void __iomem *base;
unsigned int numpage;
unsigned int pgshift;
#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0)
#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1)
#define ARM_SMMU_FEAT_TRANS_S1 (1 << 2)
#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3)
#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4)
#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5)
#define ARM_SMMU_FEAT_VMID16 (1 << 6)
#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7)
#define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8)
#define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9)
#define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10)
#define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11)
#define ARM_SMMU_FEAT_EXIDS (1 << 12)
u32 features;
enum arm_smmu_arch_version version;
enum arm_smmu_implementation model;
const struct arm_smmu_impl *impl;
u32 num_context_banks;
u32 num_s2_context_banks;
DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS);
struct arm_smmu_cb *cbs;
atomic_t irptndx;
u32 num_mapping_groups;
u16 streamid_mask;
u16 smr_mask_mask;
struct arm_smmu_smr *smrs;
struct arm_smmu_s2cr *s2crs;
struct mutex stream_map_mutex;
unsigned long va_size;
unsigned long ipa_size;
unsigned long pa_size;
unsigned long pgsize_bitmap;
u32 num_global_irqs;
u32 num_context_irqs;
unsigned int *irqs;
struct clk_bulk_data *clks;
int num_clks;
spinlock_t global_sync_lock;
/* IOMMU core code handle */
struct iommu_device iommu;
};
enum arm_smmu_context_fmt {
ARM_SMMU_CTX_FMT_NONE,
ARM_SMMU_CTX_FMT_AARCH64,
ARM_SMMU_CTX_FMT_AARCH32_L,
ARM_SMMU_CTX_FMT_AARCH32_S,
};
struct arm_smmu_cfg {
u8 cbndx;
u8 irptndx;
union {
u16 asid;
u16 vmid;
};
enum arm_smmu_cbar_type cbar;
enum arm_smmu_context_fmt fmt;
};
#define INVALID_IRPTNDX 0xff
enum arm_smmu_domain_stage {
ARM_SMMU_DOMAIN_S1 = 0,
ARM_SMMU_DOMAIN_S2,
ARM_SMMU_DOMAIN_NESTED,
ARM_SMMU_DOMAIN_BYPASS,
};
struct arm_smmu_flush_ops {
struct iommu_flush_ops tlb;
void (*tlb_inv_range)(unsigned long iova, size_t size, size_t granule,
bool leaf, void *cookie);
void (*tlb_sync)(void *cookie);
};
struct arm_smmu_domain {
struct arm_smmu_device *smmu;
struct io_pgtable_ops *pgtbl_ops;
const struct arm_smmu_flush_ops *flush_ops;
struct arm_smmu_cfg cfg;
enum arm_smmu_domain_stage stage;
bool non_strict;
struct mutex init_mutex; /* Protects smmu pointer */
spinlock_t cb_lock; /* Serialises ATS1* ops and TLB syncs */
struct iommu_domain domain;
};
/* Implementation details, yay! */
struct arm_smmu_impl {
u32 (*read_reg)(struct arm_smmu_device *smmu, int page, int offset);
void (*write_reg)(struct arm_smmu_device *smmu, int page, int offset,
u32 val);
u64 (*read_reg64)(struct arm_smmu_device *smmu, int page, int offset);
void (*write_reg64)(struct arm_smmu_device *smmu, int page, int offset,
u64 val);
int (*cfg_probe)(struct arm_smmu_device *smmu);
int (*reset)(struct arm_smmu_device *smmu);
int (*init_context)(struct arm_smmu_domain *smmu_domain);
};
static inline void __iomem *arm_smmu_page(struct arm_smmu_device *smmu, int n)
{
return smmu->base + (n << smmu->pgshift);
}
static inline u32 arm_smmu_readl(struct arm_smmu_device *smmu, int page, int offset)
{
if (smmu->impl && unlikely(smmu->impl->read_reg))
return smmu->impl->read_reg(smmu, page, offset);
return readl_relaxed(arm_smmu_page(smmu, page) + offset);
}
static inline void arm_smmu_writel(struct arm_smmu_device *smmu, int page,
int offset, u32 val)
{
if (smmu->impl && unlikely(smmu->impl->write_reg))
smmu->impl->write_reg(smmu, page, offset, val);
else
writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
}
static inline u64 arm_smmu_readq(struct arm_smmu_device *smmu, int page, int offset)
{
if (smmu->impl && unlikely(smmu->impl->read_reg64))
return smmu->impl->read_reg64(smmu, page, offset);
return readq_relaxed(arm_smmu_page(smmu, page) + offset);
}
static inline void arm_smmu_writeq(struct arm_smmu_device *smmu, int page,
int offset, u64 val)
{
if (smmu->impl && unlikely(smmu->impl->write_reg64))
smmu->impl->write_reg64(smmu, page, offset, val);
else
writeq_relaxed(val, arm_smmu_page(smmu, page) + offset);
}
#define ARM_SMMU_GR0 0
#define ARM_SMMU_GR1 1
#define ARM_SMMU_CB(s, n) ((s)->numpage + (n))
#define arm_smmu_gr0_read(s, o) \
arm_smmu_readl((s), ARM_SMMU_GR0, (o))
#define arm_smmu_gr0_write(s, o, v) \
arm_smmu_writel((s), ARM_SMMU_GR0, (o), (v))
#define arm_smmu_gr1_read(s, o) \
arm_smmu_readl((s), ARM_SMMU_GR1, (o))
#define arm_smmu_gr1_write(s, o, v) \
arm_smmu_writel((s), ARM_SMMU_GR1, (o), (v))
#define arm_smmu_cb_read(s, n, o) \
arm_smmu_readl((s), ARM_SMMU_CB((s), (n)), (o))
#define arm_smmu_cb_write(s, n, o, v) \
arm_smmu_writel((s), ARM_SMMU_CB((s), (n)), (o), (v))
#define arm_smmu_cb_readq(s, n, o) \
arm_smmu_readq((s), ARM_SMMU_CB((s), (n)), (o))
#define arm_smmu_cb_writeq(s, n, o, v) \
arm_smmu_writeq((s), ARM_SMMU_CB((s), (n)), (o), (v))
struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu);
#endif /* _ARM_SMMU_H */

13
drivers/iommu/dma-iommu.c
View File

@@ -303,13 +303,15 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
u64 size, struct device *dev)
{
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
unsigned long order, base_pfn;
struct iova_domain *iovad;
int attr;
if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
return -EINVAL;
iovad = &cookie->iovad;
/* Use the smallest supported page size for IOVA granularity */
order = __ffs(domain->pgsize_bitmap);
base_pfn = max_t(unsigned long, 1, base >> order);
@@ -444,13 +446,18 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
struct iommu_dma_cookie *cookie = domain->iova_cookie;
struct iova_domain *iovad = &cookie->iovad;
size_t iova_off = iova_offset(iovad, dma_addr);
struct iommu_iotlb_gather iotlb_gather;
size_t unmapped;
dma_addr -= iova_off;
size = iova_align(iovad, size + iova_off);
iommu_iotlb_gather_init(&iotlb_gather);
unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
WARN_ON(unmapped != size);
WARN_ON(iommu_unmap_fast(domain, dma_addr, size) != size);
if (!cookie->fq_domain)
iommu_tlb_sync(domain);
iommu_tlb_sync(domain, &iotlb_gather);
iommu_dma_free_iova(cookie, dma_addr, size);
}

77
drivers/iommu/dmar.c
View File

@@ -1519,6 +1519,64 @@ static const char *dma_remap_fault_reasons[] =
"PCE for translation request specifies blocking",
};
static const char * const dma_remap_sm_fault_reasons[] = {
"SM: Invalid Root Table Address",
"SM: TTM 0 for request with PASID",
"SM: TTM 0 for page group request",
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
"SM: Error attempting to access Root Entry",
"SM: Present bit in Root Entry is clear",
"SM: Non-zero reserved field set in Root Entry",
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
"SM: Error attempting to access Context Entry",
"SM: Present bit in Context Entry is clear",
"SM: Non-zero reserved field set in the Context Entry",
"SM: Invalid Context Entry",
"SM: DTE field in Context Entry is clear",
"SM: PASID Enable field in Context Entry is clear",
"SM: PASID is larger than the max in Context Entry",
"SM: PRE field in Context-Entry is clear",
"SM: RID_PASID field error in Context-Entry",
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
"SM: Error attempting to access the PASID Directory Entry",
"SM: Present bit in Directory Entry is clear",
"SM: Non-zero reserved field set in PASID Directory Entry",
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
"SM: Error attempting to access PASID Table Entry",
"SM: Present bit in PASID Table Entry is clear",
"SM: Non-zero reserved field set in PASID Table Entry",
"SM: Invalid Scalable-Mode PASID Table Entry",
"SM: ERE field is clear in PASID Table Entry",
"SM: SRE field is clear in PASID Table Entry",
"Unknown", "Unknown",/* 0x5E-0x5F */
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
"SM: Error attempting to access first-level paging entry",
"SM: Present bit in first-level paging entry is clear",
"SM: Non-zero reserved field set in first-level paging entry",
"SM: Error attempting to access FL-PML4 entry",
"SM: First-level entry address beyond MGAW in Nested translation",
"SM: Read permission error in FL-PML4 entry in Nested translation",
"SM: Read permission error in first-level paging entry in Nested translation",
"SM: Write permission error in first-level paging entry in Nested translation",
"SM: Error attempting to access second-level paging entry",
"SM: Read/Write permission error in second-level paging entry",
"SM: Non-zero reserved field set in second-level paging entry",
"SM: Invalid second-level page table pointer",
"SM: A/D bit update needed in second-level entry when set up in no snoop",
"Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
"SM: Address in first-level translation is not canonical",
"SM: U/S set 0 for first-level translation with user privilege",
"SM: No execute permission for request with PASID and ER=1",
"SM: Address beyond the DMA hardware max",
"SM: Second-level entry address beyond the max",
"SM: No write permission for Write/AtomicOp request",
"SM: No read permission for Read/AtomicOp request",
"SM: Invalid address-interrupt address",
"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
"SM: A/D bit update needed in first-level entry when set up in no snoop",
};
static const char *irq_remap_fault_reasons[] =
{
"Detected reserved fields in the decoded interrupt-remapped request",
@@ -1536,6 +1594,10 @@ static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
ARRAY_SIZE(irq_remap_fault_reasons))) {
*fault_type = INTR_REMAP;
return irq_remap_fault_reasons[fault_reason - 0x20];
} else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
*fault_type = DMA_REMAP;
return dma_remap_sm_fault_reasons[fault_reason - 0x30];
} else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
*fault_type = DMA_REMAP;
return dma_remap_fault_reasons[fault_reason];
@@ -1611,7 +1673,8 @@ void dmar_msi_read(int irq, struct msi_msg *msg)
}
static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
u8 fault_reason, u16 source_id, unsigned long long addr)
u8 fault_reason, int pasid, u16 source_id,
unsigned long long addr)
{
const char *reason;
int fault_type;
@@ -1624,10 +1687,11 @@ static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
PCI_FUNC(source_id & 0xFF), addr >> 48,
fault_reason, reason);
else
pr_err("[%s] Request device [%02x:%02x.%d] fault addr %llx [fault reason %02d] %s\n",
pr_err("[%s] Request device [%02x:%02x.%d] PASID %x fault addr %llx [fault reason %02d] %s\n",
type ? "DMA Read" : "DMA Write",
source_id >> 8, PCI_SLOT(source_id & 0xFF),
PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
PCI_FUNC(source_id & 0xFF), pasid, addr,
fault_reason, reason);
return 0;
}
@@ -1659,8 +1723,9 @@ irqreturn_t dmar_fault(int irq, void *dev_id)
u8 fault_reason;
u16 source_id;
u64 guest_addr;
int type;
int type, pasid;
u32 data;
bool pasid_present;
/* highest 32 bits */
data = readl(iommu->reg + reg +
@@ -1672,10 +1737,12 @@ irqreturn_t dmar_fault(int irq, void *dev_id)
fault_reason = dma_frcd_fault_reason(data);
type = dma_frcd_type(data);
pasid = dma_frcd_pasid_value(data);
data = readl(iommu->reg + reg +
fault_index * PRIMARY_FAULT_REG_LEN + 8);
source_id = dma_frcd_source_id(data);
pasid_present = dma_frcd_pasid_present(data);
guest_addr = dmar_readq(iommu->reg + reg +
fault_index * PRIMARY_FAULT_REG_LEN);
guest_addr = dma_frcd_page_addr(guest_addr);
@@ -1688,7 +1755,9 @@ irqreturn_t dmar_fault(int irq, void *dev_id)
raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
if (!ratelimited)
/* Using pasid -1 if pasid is not present */
dmar_fault_do_one(iommu, type, fault_reason,
pasid_present ? pasid : -1,
source_id, guest_addr);
fault_index++;

9
drivers/iommu/exynos-iommu.c
View File

@@ -566,7 +566,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
static const struct iommu_ops exynos_iommu_ops;
static int __init exynos_sysmmu_probe(struct platform_device *pdev)
static int exynos_sysmmu_probe(struct platform_device *pdev)
{
int irq, ret;
struct device *dev = &pdev->dev;
@@ -583,10 +583,8 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev)
return PTR_ERR(data->sfrbase);
irq = platform_get_irq(pdev, 0);
if (irq <= 0) {
dev_err(dev, "Unable to find IRQ resource\n");
if (irq <= 0)
return irq;
}
ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0,
dev_name(dev), data);
@@ -1130,7 +1128,8 @@ static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain
}
static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain,
unsigned long l_iova, size_t size)
unsigned long l_iova, size_t size,
struct iommu_iotlb_gather *gather)
{
struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
sysmmu_iova_t iova = (sysmmu_iova_t)l_iova;

359
drivers/iommu/intel-iommu.c
View File

@@ -41,9 +41,11 @@
#include <linux/dma-direct.h>
#include <linux/crash_dump.h>
#include <linux/numa.h>
#include <linux/swiotlb.h>
#include <asm/irq_remapping.h>
#include <asm/cacheflush.h>
#include <asm/iommu.h>
#include <trace/events/intel_iommu.h>
#include "irq_remapping.h"
#include "intel-pasid.h"
@@ -346,6 +348,8 @@ static int domain_detach_iommu(struct dmar_domain *domain,
static bool device_is_rmrr_locked(struct device *dev);
static int intel_iommu_attach_device(struct iommu_domain *domain,
struct device *dev);
static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova);
#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
int dmar_disabled = 0;
@@ -362,6 +366,7 @@ static int dmar_forcedac;
static int intel_iommu_strict;
static int intel_iommu_superpage = 1;
static int iommu_identity_mapping;
static int intel_no_bounce;
#define IDENTMAP_ALL 1
#define IDENTMAP_GFX 2
@@ -375,6 +380,9 @@ EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
static DEFINE_SPINLOCK(device_domain_lock);
static LIST_HEAD(device_domain_list);
#define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
to_pci_dev(d)->untrusted)
/*
* Iterate over elements in device_domain_list and call the specified
* callback @fn against each element.
@@ -457,6 +465,9 @@ static int __init intel_iommu_setup(char *str)
printk(KERN_INFO
"Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
intel_iommu_tboot_noforce = 1;
} else if (!strncmp(str, "nobounce", 8)) {
pr_info("Intel-IOMMU: No bounce buffer. This could expose security risks of DMA attacks\n");
intel_no_bounce = 1;
}
str += strcspn(str, ",");
@@ -3296,7 +3307,7 @@ static int __init init_dmars(void)
iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
}
if (iommu_pass_through)
if (iommu_default_passthrough())
iommu_identity_mapping |= IDENTMAP_ALL;
#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
@@ -3534,6 +3545,9 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
start_paddr += paddr & ~PAGE_MASK;
trace_map_single(dev, start_paddr, paddr, size << VTD_PAGE_SHIFT);
return start_paddr;
error:
@@ -3589,10 +3603,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
if (dev_is_pci(dev))
pdev = to_pci_dev(dev);
dev_dbg(dev, "Device unmapping: pfn %lx-%lx\n", start_pfn, last_pfn);
freelist = domain_unmap(domain, start_pfn, last_pfn);
if (intel_iommu_strict || (pdev && pdev->untrusted) ||
!has_iova_flush_queue(&domain->iovad)) {
iommu_flush_iotlb_psi(iommu, domain, start_pfn,
@@ -3608,6 +3619,8 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
* cpu used up by the iotlb flush operation...
*/
}
trace_unmap_single(dev, dev_addr, size);
}
static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
@@ -3698,6 +3711,8 @@ static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
}
intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
trace_unmap_sg(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
}
static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
@@ -3754,6 +3769,9 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele
return 0;
}
trace_map_sg(dev, iova_pfn << PAGE_SHIFT,
sg_phys(sglist), size << VTD_PAGE_SHIFT);
return nelems;
}
@@ -3769,6 +3787,252 @@ static const struct dma_map_ops intel_dma_ops = {
.dma_supported = dma_direct_supported,
};
static void
bounce_sync_single(struct device *dev, dma_addr_t addr, size_t size,
enum dma_data_direction dir, enum dma_sync_target target)
{
struct dmar_domain *domain;
phys_addr_t tlb_addr;
domain = find_domain(dev);
if (WARN_ON(!domain))
return;
tlb_addr = intel_iommu_iova_to_phys(&domain->domain, addr);
if (is_swiotlb_buffer(tlb_addr))
swiotlb_tbl_sync_single(dev, tlb_addr, size, dir, target);
}
static dma_addr_t
bounce_map_single(struct device *dev, phys_addr_t paddr, size_t size,
enum dma_data_direction dir, unsigned long attrs,
u64 dma_mask)
{
size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
struct dmar_domain *domain;
struct intel_iommu *iommu;
unsigned long iova_pfn;
unsigned long nrpages;
phys_addr_t tlb_addr;
int prot = 0;
int ret;
domain = find_domain(dev);
if (WARN_ON(dir == DMA_NONE || !domain))
return DMA_MAPPING_ERROR;
iommu = domain_get_iommu(domain);
if (WARN_ON(!iommu))
return DMA_MAPPING_ERROR;
nrpages = aligned_nrpages(0, size);
iova_pfn = intel_alloc_iova(dev, domain,
dma_to_mm_pfn(nrpages), dma_mask);
if (!iova_pfn)
return DMA_MAPPING_ERROR;
/*
* Check if DMAR supports zero-length reads on write only
* mappings..
*/
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL ||
!cap_zlr(iommu->cap))
prot |= DMA_PTE_READ;
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
/*
* If both the physical buffer start address and size are
* page aligned, we don't need to use a bounce page.
*/
if (!IS_ALIGNED(paddr | size, VTD_PAGE_SIZE)) {
tlb_addr = swiotlb_tbl_map_single(dev,
__phys_to_dma(dev, io_tlb_start),
paddr, size, aligned_size, dir, attrs);
if (tlb_addr == DMA_MAPPING_ERROR) {
goto swiotlb_error;
} else {
/* Cleanup the padding area. */
void *padding_start = phys_to_virt(tlb_addr);
size_t padding_size = aligned_size;
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_TO_DEVICE ||
dir == DMA_BIDIRECTIONAL)) {
padding_start += size;
padding_size -= size;
}
memset(padding_start, 0, padding_size);
}
} else {
tlb_addr = paddr;
}
ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
tlb_addr >> VTD_PAGE_SHIFT, nrpages, prot);
if (ret)
goto mapping_error;
trace_bounce_map_single(dev, iova_pfn << PAGE_SHIFT, paddr, size);
return (phys_addr_t)iova_pfn << PAGE_SHIFT;
mapping_error:
if (is_swiotlb_buffer(tlb_addr))
swiotlb_tbl_unmap_single(dev, tlb_addr, size,
aligned_size, dir, attrs);
swiotlb_error:
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
dev_err(dev, "Device bounce map: %zx@%llx dir %d --- failed\n",
size, (unsigned long long)paddr, dir);
return DMA_MAPPING_ERROR;
}
static void
bounce_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
struct dmar_domain *domain;
phys_addr_t tlb_addr;
domain = find_domain(dev);
if (WARN_ON(!domain))
return;
tlb_addr = intel_iommu_iova_to_phys(&domain->domain, dev_addr);
if (WARN_ON(!tlb_addr))
return;
intel_unmap(dev, dev_addr, size);
if (is_swiotlb_buffer(tlb_addr))
swiotlb_tbl_unmap_single(dev, tlb_addr, size,
aligned_size, dir, attrs);
trace_bounce_unmap_single(dev, dev_addr, size);
}
static dma_addr_t
bounce_map_page(struct device *dev, struct page *page, unsigned long offset,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
return bounce_map_single(dev, page_to_phys(page) + offset,
size, dir, attrs, *dev->dma_mask);
}
static dma_addr_t
bounce_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
return bounce_map_single(dev, phys_addr, size,
dir, attrs, *dev->dma_mask);
}
static void
bounce_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
bounce_unmap_single(dev, dev_addr, size, dir, attrs);
}
static void
bounce_unmap_resource(struct device *dev, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
bounce_unmap_single(dev, dev_addr, size, dir, attrs);
}
static void
bounce_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i)
bounce_unmap_page(dev, sg->dma_address,
sg_dma_len(sg), dir, attrs);
}
static int
bounce_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, unsigned long attrs)
{
int i;
struct scatterlist *sg;
for_each_sg(sglist, sg, nelems, i) {
sg->dma_address = bounce_map_page(dev, sg_page(sg),
sg->offset, sg->length,
dir, attrs);
if (sg->dma_address == DMA_MAPPING_ERROR)
goto out_unmap;
sg_dma_len(sg) = sg->length;
}
return nelems;
out_unmap:
bounce_unmap_sg(dev, sglist, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
return 0;
}
static void
bounce_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir)
{
bounce_sync_single(dev, addr, size, dir, SYNC_FOR_CPU);
}
static void
bounce_sync_single_for_device(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir)
{
bounce_sync_single(dev, addr, size, dir, SYNC_FOR_DEVICE);
}
static void
bounce_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i)
bounce_sync_single(dev, sg_dma_address(sg),
sg_dma_len(sg), dir, SYNC_FOR_CPU);
}
static void
bounce_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i)
bounce_sync_single(dev, sg_dma_address(sg),
sg_dma_len(sg), dir, SYNC_FOR_DEVICE);
}
static const struct dma_map_ops bounce_dma_ops = {
.alloc = intel_alloc_coherent,
.free = intel_free_coherent,
.map_sg = bounce_map_sg,
.unmap_sg = bounce_unmap_sg,
.map_page = bounce_map_page,
.unmap_page = bounce_unmap_page,
.sync_single_for_cpu = bounce_sync_single_for_cpu,
.sync_single_for_device = bounce_sync_single_for_device,
.sync_sg_for_cpu = bounce_sync_sg_for_cpu,
.sync_sg_for_device = bounce_sync_sg_for_device,
.map_resource = bounce_map_resource,
.unmap_resource = bounce_unmap_resource,
.dma_supported = dma_direct_supported,
};
static inline int iommu_domain_cache_init(void)
{
int ret = 0;
@@ -4569,22 +4833,20 @@ const struct attribute_group *intel_iommu_groups[] = {
NULL,
};
static int __init platform_optin_force_iommu(void)
static inline bool has_untrusted_dev(void)
{
struct pci_dev *pdev = NULL;
bool has_untrusted_dev = false;
if (!dmar_platform_optin() || no_platform_optin)
return 0;
for_each_pci_dev(pdev)
if (pdev->untrusted)
return true;
for_each_pci_dev(pdev) {
if (pdev->untrusted) {
has_untrusted_dev = true;
break;
}
}
return false;
}
if (!has_untrusted_dev)
static int __init platform_optin_force_iommu(void)
{
if (!dmar_platform_optin() || no_platform_optin || !has_untrusted_dev())
return 0;
if (no_iommu || dmar_disabled)
@@ -4598,9 +4860,6 @@ static int __init platform_optin_force_iommu(void)
iommu_identity_mapping |= IDENTMAP_ALL;
dmar_disabled = 0;
#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
swiotlb = 0;
#endif
no_iommu = 0;
return 1;
@@ -4740,7 +4999,14 @@ int __init intel_iommu_init(void)
up_write(&dmar_global_lock);
#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
swiotlb = 0;
/*
* If the system has no untrusted device or the user has decided
* to disable the bounce page mechanisms, we don't need swiotlb.
* Mark this and the pre-allocated bounce pages will be released
* later.
*/
if (!has_untrusted_dev() || intel_no_bounce)
swiotlb = 0;
#endif
dma_ops = &intel_dma_ops;
@@ -5204,7 +5470,8 @@ static int intel_iommu_map(struct iommu_domain *domain,
}
static size_t intel_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
unsigned long iova, size_t size,
struct iommu_iotlb_gather *gather)
{
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
struct page *freelist = NULL;
@@ -5360,6 +5627,11 @@ static int intel_iommu_add_device(struct device *dev)
}
}
if (device_needs_bounce(dev)) {
dev_info(dev, "Use Intel IOMMU bounce page dma_ops\n");
set_dma_ops(dev, &bounce_dma_ops);
}
return 0;
}
@@ -5377,6 +5649,9 @@ static void intel_iommu_remove_device(struct device *dev)
iommu_group_remove_device(dev);
iommu_device_unlink(&iommu->iommu, dev);
if (device_needs_bounce(dev))
set_dma_ops(dev, NULL);
}
static void intel_iommu_get_resv_regions(struct device *device,
@@ -5690,20 +5965,46 @@ const struct iommu_ops intel_iommu_ops = {
.pgsize_bitmap = INTEL_IOMMU_PGSIZES,
};
static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
static void quirk_iommu_igfx(struct pci_dev *dev)
{
/* G4x/GM45 integrated gfx dmar support is totally busted. */
pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
dmar_map_gfx = 0;
}
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
/* G4x/GM45 integrated gfx dmar support is totally busted. */
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
/* Broadwell igfx malfunctions with dmar */
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
static void quirk_iommu_rwbf(struct pci_dev *dev)
{

14
drivers/iommu/intel-trace.c Normal file
View File

@@ -0,0 +1,14 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Intel IOMMU trace support
*
* Copyright (C) 2019 Intel Corporation
*
* Author: Lu Baolu <baolu.lu@linux.intel.com>
*/
#include <linux/string.h>
#include <linux/types.h>
#define CREATE_TRACE_POINTS
#include <trace/events/intel_iommu.h>

6
drivers/iommu/intel_irq_remapping.c
View File

@@ -376,13 +376,13 @@ static int set_msi_sid_cb(struct pci_dev *pdev, u16 alias, void *opaque)
{
struct set_msi_sid_data *data = opaque;
if (data->count == 0 || PCI_BUS_NUM(alias) == PCI_BUS_NUM(data->alias))
data->busmatch_count++;
data->pdev = pdev;
data->alias = alias;
data->count++;
if (PCI_BUS_NUM(alias) == pdev->bus->number)
data->busmatch_count++;
return 0;
}

145
drivers/iommu/io-pgtable-arm-v7s.c
View File

@@ -112,7 +112,9 @@
#define ARM_V7S_TEX_MASK 0x7
#define ARM_V7S_ATTR_TEX(val) (((val) & ARM_V7S_TEX_MASK) << ARM_V7S_TEX_SHIFT)
#define ARM_V7S_ATTR_MTK_4GB BIT(9) /* MTK extend it for 4GB mode */
/* MediaTek extend the two bits for PA 32bit/33bit */
#define ARM_V7S_ATTR_MTK_PA_BIT32 BIT(9)
#define ARM_V7S_ATTR_MTK_PA_BIT33 BIT(4)
/* *well, except for TEX on level 2 large pages, of course :( */
#define ARM_V7S_CONT_PAGE_TEX_SHIFT 6
@@ -169,18 +171,62 @@ struct arm_v7s_io_pgtable {
spinlock_t split_lock;
};
static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
static dma_addr_t __arm_v7s_dma_addr(void *pages)
{
return (dma_addr_t)virt_to_phys(pages);
}
static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl)
static bool arm_v7s_is_mtk_enabled(struct io_pgtable_cfg *cfg)
{
return IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT) &&
(cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT);
}
static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl,
struct io_pgtable_cfg *cfg)
{
arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl);
if (!arm_v7s_is_mtk_enabled(cfg))
return pte;
if (paddr & BIT_ULL(32))
pte |= ARM_V7S_ATTR_MTK_PA_BIT32;
if (paddr & BIT_ULL(33))
pte |= ARM_V7S_ATTR_MTK_PA_BIT33;
return pte;
}
static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl,
struct io_pgtable_cfg *cfg)
{
arm_v7s_iopte mask;
phys_addr_t paddr;
if (ARM_V7S_PTE_IS_TABLE(pte, lvl))
pte &= ARM_V7S_TABLE_MASK;
mask = ARM_V7S_TABLE_MASK;
else if (arm_v7s_pte_is_cont(pte, lvl))
mask = ARM_V7S_LVL_MASK(lvl) * ARM_V7S_CONT_PAGES;
else
pte &= ARM_V7S_LVL_MASK(lvl);
return phys_to_virt(pte);
mask = ARM_V7S_LVL_MASK(lvl);
paddr = pte & mask;
if (!arm_v7s_is_mtk_enabled(cfg))
return paddr;
if (pte & ARM_V7S_ATTR_MTK_PA_BIT32)
paddr |= BIT_ULL(32);
if (pte & ARM_V7S_ATTR_MTK_PA_BIT33)
paddr |= BIT_ULL(33);
return paddr;
}
static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl,
struct arm_v7s_io_pgtable *data)
{
return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));
}
static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
@@ -295,9 +341,6 @@ static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl,
if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS))
pte |= ARM_V7S_ATTR_NS_SECTION;
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_4GB)
pte |= ARM_V7S_ATTR_MTK_4GB;
return pte;
}
@@ -362,7 +405,8 @@ static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)
return false;
}
static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *, unsigned long,
static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *,
struct iommu_iotlb_gather *, unsigned long,
size_t, int, arm_v7s_iopte *);
static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data,
@@ -383,7 +427,7 @@ static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data,
size_t sz = ARM_V7S_BLOCK_SIZE(lvl);
tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl);
if (WARN_ON(__arm_v7s_unmap(data, iova + i * sz,
if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz,
sz, lvl, tblp) != sz))
return -EINVAL;
} else if (ptep[i]) {
@@ -396,7 +440,7 @@ static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data,
if (num_entries > 1)
pte = arm_v7s_pte_to_cont(pte, lvl);
pte |= paddr & ARM_V7S_LVL_MASK(lvl);
pte |= paddr_to_iopte(paddr, lvl, cfg);
__arm_v7s_set_pte(ptep, pte, num_entries, cfg);
return 0;
@@ -462,7 +506,7 @@ static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova,
}
if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) {
cptep = iopte_deref(pte, lvl);
cptep = iopte_deref(pte, lvl, data);
} else if (pte) {
/* We require an unmap first */
WARN_ON(!selftest_running);
@@ -484,7 +528,8 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova,
if (!(prot & (IOMMU_READ | IOMMU_WRITE)))
return 0;
if (WARN_ON(upper_32_bits(iova) || upper_32_bits(paddr)))
if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) ||
paddr >= (1ULL << data->iop.cfg.oas)))
return -ERANGE;
ret = __arm_v7s_map(data, iova, paddr, size, prot, 1, data->pgd);
@@ -493,9 +538,8 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova,
* a chance for anything to kick off a table walk for the new iova.
*/
if (iop->cfg.quirks & IO_PGTABLE_QUIRK_TLBI_ON_MAP) {
io_pgtable_tlb_add_flush(iop, iova, size,
ARM_V7S_BLOCK_SIZE(2), false);
io_pgtable_tlb_sync(iop);
io_pgtable_tlb_flush_walk(iop, iova, size,
ARM_V7S_BLOCK_SIZE(2));
} else {
wmb();
}
@@ -512,7 +556,8 @@ static void arm_v7s_free_pgtable(struct io_pgtable *iop)
arm_v7s_iopte pte = data->pgd[i];
if (ARM_V7S_PTE_IS_TABLE(pte, 1))
__arm_v7s_free_table(iopte_deref(pte, 1), 2, data);
__arm_v7s_free_table(iopte_deref(pte, 1, data),
2, data);
}
__arm_v7s_free_table(data->pgd, 1, data);
kmem_cache_destroy(data->l2_tables);
@@ -541,12 +586,12 @@ static arm_v7s_iopte arm_v7s_split_cont(struct arm_v7s_io_pgtable *data,
__arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg);
size *= ARM_V7S_CONT_PAGES;
io_pgtable_tlb_add_flush(iop, iova, size, size, true);
io_pgtable_tlb_sync(iop);
io_pgtable_tlb_flush_leaf(iop, iova, size, size);
return pte;
}
static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,
struct iommu_iotlb_gather *gather,
unsigned long iova, size_t size,
arm_v7s_iopte blk_pte,
arm_v7s_iopte *ptep)
@@ -582,16 +627,16 @@ static size_t arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data,
if (!ARM_V7S_PTE_IS_TABLE(pte, 1))
return 0;
tablep = iopte_deref(pte, 1);
return __arm_v7s_unmap(data, iova, size, 2, tablep);
tablep = iopte_deref(pte, 1, data);
return __arm_v7s_unmap(data, gather, iova, size, 2, tablep);
}
io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);
io_pgtable_tlb_sync(&data->iop);
io_pgtable_tlb_add_page(&data->iop, gather, iova, size);
return size;
}
static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
struct iommu_iotlb_gather *gather,
unsigned long iova, size_t size, int lvl,
arm_v7s_iopte *ptep)
{
@@ -638,10 +683,9 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
for (i = 0; i < num_entries; i++) {
if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) {
/* Also flush any partial walks */
io_pgtable_tlb_add_flush(iop, iova, blk_size,
ARM_V7S_BLOCK_SIZE(lvl + 1), false);
io_pgtable_tlb_sync(iop);
ptep = iopte_deref(pte[i], lvl);
io_pgtable_tlb_flush_walk(iop, iova, blk_size,
ARM_V7S_BLOCK_SIZE(lvl + 1));
ptep = iopte_deref(pte[i], lvl, data);
__arm_v7s_free_table(ptep, lvl + 1, data);
} else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
/*
@@ -651,8 +695,7 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
*/
smp_wmb();
} else {
io_pgtable_tlb_add_flush(iop, iova, blk_size,
blk_size, true);
io_pgtable_tlb_add_page(iop, gather, iova, blk_size);
}
iova += blk_size;
}
@@ -662,23 +705,24 @@ static size_t __arm_v7s_unmap(struct arm_v7s_io_pgtable *data,
* Insert a table at the next level to map the old region,
* minus the part we want to unmap
*/
return arm_v7s_split_blk_unmap(data, iova, size, pte[0], ptep);
return arm_v7s_split_blk_unmap(data, gather, iova, size, pte[0],
ptep);
}
/* Keep on walkin' */
ptep = iopte_deref(pte[0], lvl);
return __arm_v7s_unmap(data, iova, size, lvl + 1, ptep);
ptep = iopte_deref(pte[0], lvl, data);
return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep);
}
static size_t arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops);
if (WARN_ON(upper_32_bits(iova)))
return 0;
return __arm_v7s_unmap(data, iova, size, 1, data->pgd);
return __arm_v7s_unmap(data, gather, iova, size, 1, data->pgd);
}
static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops,
@@ -692,7 +736,7 @@ static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops,
do {
ptep += ARM_V7S_LVL_IDX(iova, ++lvl);
pte = READ_ONCE(*ptep);
ptep = iopte_deref(pte, lvl);
ptep = iopte_deref(pte, lvl, data);
} while (ARM_V7S_PTE_IS_TABLE(pte, lvl));
if (!ARM_V7S_PTE_IS_VALID(pte))
@@ -701,7 +745,7 @@ static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops,
mask = ARM_V7S_LVL_MASK(lvl);
if (arm_v7s_pte_is_cont(pte, lvl))
mask *= ARM_V7S_CONT_PAGES;
return (pte & mask) | (iova & ~mask);
return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);
}
static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
@@ -709,18 +753,21 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg,
{
struct arm_v7s_io_pgtable *data;
if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS)
if (cfg->ias > ARM_V7S_ADDR_BITS)
return NULL;
if (cfg->oas > (arm_v7s_is_mtk_enabled(cfg) ? 34 : ARM_V7S_ADDR_BITS))
return NULL;
if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS |
IO_PGTABLE_QUIRK_NO_PERMS |
IO_PGTABLE_QUIRK_TLBI_ON_MAP |
IO_PGTABLE_QUIRK_ARM_MTK_4GB |
IO_PGTABLE_QUIRK_ARM_MTK_EXT |
IO_PGTABLE_QUIRK_NON_STRICT))
return NULL;
/* If ARM_MTK_4GB is enabled, the NO_PERMS is also expected. */
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_4GB &&
if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_MTK_EXT &&
!(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS))
return NULL;
@@ -806,22 +853,24 @@ static void dummy_tlb_flush_all(void *cookie)
WARN_ON(cookie != cfg_cookie);
}
static void dummy_tlb_add_flush(unsigned long iova, size_t size,
size_t granule, bool leaf, void *cookie)
static void dummy_tlb_flush(unsigned long iova, size_t size, size_t granule,
void *cookie)
{
WARN_ON(cookie != cfg_cookie);
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
}
static void dummy_tlb_sync(void *cookie)
static void dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
unsigned long iova, size_t granule, void *cookie)
{
WARN_ON(cookie != cfg_cookie);
dummy_tlb_flush(iova, granule, granule, cookie);
}
static const struct iommu_gather_ops dummy_tlb_ops = {
static const struct iommu_flush_ops dummy_tlb_ops = {
.tlb_flush_all = dummy_tlb_flush_all,
.tlb_add_flush = dummy_tlb_add_flush,
.tlb_sync = dummy_tlb_sync,
.tlb_flush_walk = dummy_tlb_flush,
.tlb_flush_leaf = dummy_tlb_flush,
.tlb_add_page = dummy_tlb_add_page,
};
#define __FAIL(ops) ({ \
@@ -896,7 +945,7 @@ static int __init arm_v7s_do_selftests(void)
size = 1UL << __ffs(cfg.pgsize_bitmap);
while (i < loopnr) {
iova_start = i * SZ_16M;
if (ops->unmap(ops, iova_start + size, size) != size)
if (ops->unmap(ops, iova_start + size, size, NULL) != size)
return __FAIL(ops);
/* Remap of partial unmap */
@@ -914,7 +963,7 @@ static int __init arm_v7s_do_selftests(void)
for_each_set_bit(i, &cfg.pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << i;
if (ops->unmap(ops, iova, size) != size)
if (ops->unmap(ops, iova, size, NULL) != size)
return __FAIL(ops);
if (ops->iova_to_phys(ops, iova + 42))

48
drivers/iommu/io-pgtable-arm.c
View File

@@ -12,7 +12,6 @@
#include <linux/atomic.h>
#include <linux/bitops.h>
#include <linux/io-pgtable.h>
#include <linux/iommu.h>
#include <linux/kernel.h>
#include <linux/sizes.h>
#include <linux/slab.h>
@@ -290,6 +289,7 @@ static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte,
}
static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
struct iommu_iotlb_gather *gather,
unsigned long iova, size_t size, int lvl,
arm_lpae_iopte *ptep);
@@ -335,8 +335,10 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz))
if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) {
WARN_ON(1);
return -EINVAL;
}
}
__arm_lpae_init_pte(data, paddr, prot, lvl, ptep);
@@ -537,6 +539,7 @@ static void arm_lpae_free_pgtable(struct io_pgtable *iop)
}
static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
struct iommu_iotlb_gather *gather,
unsigned long iova, size_t size,
arm_lpae_iopte blk_pte, int lvl,
arm_lpae_iopte *ptep)
@@ -582,15 +585,15 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
tablep = iopte_deref(pte, data);
} else if (unmap_idx >= 0) {
io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);
io_pgtable_tlb_sync(&data->iop);
io_pgtable_tlb_add_page(&data->iop, gather, iova, size);
return size;
}
return __arm_lpae_unmap(data, iova, size, lvl, tablep);
return __arm_lpae_unmap(data, gather, iova, size, lvl, tablep);
}
static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
struct iommu_iotlb_gather *gather,
unsigned long iova, size_t size, int lvl,
arm_lpae_iopte *ptep)
{
@@ -612,9 +615,8 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
if (!iopte_leaf(pte, lvl, iop->fmt)) {
/* Also flush any partial walks */
io_pgtable_tlb_add_flush(iop, iova, size,
ARM_LPAE_GRANULE(data), false);
io_pgtable_tlb_sync(iop);
io_pgtable_tlb_flush_walk(iop, iova, size,
ARM_LPAE_GRANULE(data));
ptep = iopte_deref(pte, data);
__arm_lpae_free_pgtable(data, lvl + 1, ptep);
} else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
@@ -625,7 +627,7 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
*/
smp_wmb();
} else {
io_pgtable_tlb_add_flush(iop, iova, size, size, true);
io_pgtable_tlb_add_page(iop, gather, iova, size);
}
return size;
@@ -634,17 +636,17 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
* Insert a table at the next level to map the old region,
* minus the part we want to unmap
*/
return arm_lpae_split_blk_unmap(data, iova, size, pte,
return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,
lvl + 1, ptep);
}
/* Keep on walkin' */
ptep = iopte_deref(pte, data);
return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep);
return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep);
}
static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
arm_lpae_iopte *ptep = data->pgd;
@@ -653,7 +655,7 @@ static size_t arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova,
if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias)))
return 0;
return __arm_lpae_unmap(data, iova, size, lvl, ptep);
return __arm_lpae_unmap(data, gather, iova, size, lvl, ptep);
}
static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops,
@@ -1070,22 +1072,24 @@ static void dummy_tlb_flush_all(void *cookie)
WARN_ON(cookie != cfg_cookie);
}
static void dummy_tlb_add_flush(unsigned long iova, size_t size,
size_t granule, bool leaf, void *cookie)
static void dummy_tlb_flush(unsigned long iova, size_t size, size_t granule,
void *cookie)
{
WARN_ON(cookie != cfg_cookie);
WARN_ON(!(size & cfg_cookie->pgsize_bitmap));
}
static void dummy_tlb_sync(void *cookie)
static void dummy_tlb_add_page(struct iommu_iotlb_gather *gather,
unsigned long iova, size_t granule, void *cookie)
{
WARN_ON(cookie != cfg_cookie);
dummy_tlb_flush(iova, granule, granule, cookie);
}
static const struct iommu_gather_ops dummy_tlb_ops __initconst = {
static const struct iommu_flush_ops dummy_tlb_ops __initconst = {
.tlb_flush_all = dummy_tlb_flush_all,
.tlb_add_flush = dummy_tlb_add_flush,
.tlb_sync = dummy_tlb_sync,
.tlb_flush_walk = dummy_tlb_flush,
.tlb_flush_leaf = dummy_tlb_flush,
.tlb_add_page = dummy_tlb_add_page,
};
static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops)
@@ -1168,7 +1172,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
/* Partial unmap */
size = 1UL << __ffs(cfg->pgsize_bitmap);
if (ops->unmap(ops, SZ_1G + size, size) != size)
if (ops->unmap(ops, SZ_1G + size, size, NULL) != size)
return __FAIL(ops, i);
/* Remap of partial unmap */
@@ -1183,7 +1187,7 @@ static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg)
for_each_set_bit(j, &cfg->pgsize_bitmap, BITS_PER_LONG) {
size = 1UL << j;
if (ops->unmap(ops, iova, size) != size)
if (ops->unmap(ops, iova, size, NULL) != size)
return __FAIL(ops, i);
if (ops->iova_to_phys(ops, iova + 42))

217
drivers/iommu/iommu.c
View File

@@ -26,12 +26,10 @@
static struct kset *iommu_group_kset;
static DEFINE_IDA(iommu_group_ida);
#ifdef CONFIG_IOMMU_DEFAULT_PASSTHROUGH
static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
#else
static unsigned int iommu_def_domain_type = IOMMU_DOMAIN_DMA;
#endif
static unsigned int iommu_def_domain_type __read_mostly;
static bool iommu_dma_strict __read_mostly = true;
static u32 iommu_cmd_line __read_mostly;
struct iommu_group {
struct kobject kobj;
@@ -68,6 +66,18 @@ static const char * const iommu_group_resv_type_string[] = {
[IOMMU_RESV_SW_MSI] = "msi",
};
#define IOMMU_CMD_LINE_DMA_API BIT(0)
static void iommu_set_cmd_line_dma_api(void)
{
iommu_cmd_line |= IOMMU_CMD_LINE_DMA_API;
}
static bool iommu_cmd_line_dma_api(void)
{
return !!(iommu_cmd_line & IOMMU_CMD_LINE_DMA_API);
}
#define IOMMU_GROUP_ATTR(_name, _mode, _show, _store) \
struct iommu_group_attribute iommu_group_attr_##_name = \
__ATTR(_name, _mode, _show, _store)
@@ -80,12 +90,55 @@ struct iommu_group_attribute iommu_group_attr_##_name = \
static LIST_HEAD(iommu_device_list);
static DEFINE_SPINLOCK(iommu_device_lock);
/*
* Use a function instead of an array here because the domain-type is a
* bit-field, so an array would waste memory.
*/
static const char *iommu_domain_type_str(unsigned int t)
{
switch (t) {
case IOMMU_DOMAIN_BLOCKED:
return "Blocked";
case IOMMU_DOMAIN_IDENTITY:
return "Passthrough";
case IOMMU_DOMAIN_UNMANAGED:
return "Unmanaged";
case IOMMU_DOMAIN_DMA:
return "Translated";
default:
return "Unknown";
}
}
static int __init iommu_subsys_init(void)
{
bool cmd_line = iommu_cmd_line_dma_api();
if (!cmd_line) {
if (IS_ENABLED(CONFIG_IOMMU_DEFAULT_PASSTHROUGH))
iommu_set_default_passthrough(false);
else
iommu_set_default_translated(false);
if (iommu_default_passthrough() && mem_encrypt_active()) {
pr_info("Memory encryption detected - Disabling default IOMMU Passthrough\n");
iommu_set_default_translated(false);
}
}
pr_info("Default domain type: %s %s\n",
iommu_domain_type_str(iommu_def_domain_type),
cmd_line ? "(set via kernel command line)" : "");
return 0;
}
subsys_initcall(iommu_subsys_init);
int iommu_device_register(struct iommu_device *iommu)
{
spin_lock(&iommu_device_lock);
list_add_tail(&iommu->list, &iommu_device_list);
spin_unlock(&iommu_device_lock);
return 0;
}
@@ -165,7 +218,11 @@ static int __init iommu_set_def_domain_type(char *str)
if (ret)
return ret;
iommu_def_domain_type = pt ? IOMMU_DOMAIN_IDENTITY : IOMMU_DOMAIN_DMA;
if (pt)
iommu_set_default_passthrough(true);
else
iommu_set_default_translated(true);
return 0;
}
early_param("iommu.passthrough", iommu_set_def_domain_type);
@@ -229,60 +286,58 @@ static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)
* @new: new region to insert
* @regions: list of regions
*
* The new element is sorted by address with respect to the other
* regions of the same type. In case it overlaps with another
* region of the same type, regions are merged. In case it
* overlaps with another region of different type, regions are
* not merged.
* Elements are sorted by start address and overlapping segments
* of the same type are merged.
*/
static int iommu_insert_resv_region(struct iommu_resv_region *new,
struct list_head *regions)
int iommu_insert_resv_region(struct iommu_resv_region *new,
struct list_head *regions)
{
struct iommu_resv_region *region;
phys_addr_t start = new->start;
phys_addr_t end = new->start + new->length - 1;
struct list_head *pos = regions->next;
struct iommu_resv_region *iter, *tmp, *nr, *top;
LIST_HEAD(stack);
while (pos != regions) {
struct iommu_resv_region *entry =
list_entry(pos, struct iommu_resv_region, list);
phys_addr_t a = entry->start;
phys_addr_t b = entry->start + entry->length - 1;
int type = entry->type;
if (end < a) {
goto insert;
} else if (start > b) {
pos = pos->next;
} else if ((start >= a) && (end <= b)) {
if (new->type == type)
return 0;
else
pos = pos->next;
} else {
if (new->type == type) {
phys_addr_t new_start = min(a, start);
phys_addr_t new_end = max(b, end);
int ret;
list_del(&entry->list);
entry->start = new_start;
entry->length = new_end - new_start + 1;
ret = iommu_insert_resv_region(entry, regions);
kfree(entry);
return ret;
} else {
pos = pos->next;
}
}
}
insert:
region = iommu_alloc_resv_region(new->start, new->length,
new->prot, new->type);
if (!region)
nr = iommu_alloc_resv_region(new->start, new->length,
new->prot, new->type);
if (!nr)
return -ENOMEM;
list_add_tail(&region->list, pos);
/* First add the new element based on start address sorting */
list_for_each_entry(iter, regions, list) {
if (nr->start < iter->start ||
(nr->start == iter->start && nr->type <= iter->type))
break;
}
list_add_tail(&nr->list, &iter->list);
/* Merge overlapping segments of type nr->type in @regions, if any */
list_for_each_entry_safe(iter, tmp, regions, list) {
phys_addr_t top_end, iter_end = iter->start + iter->length - 1;
/* no merge needed on elements of different types than @nr */
if (iter->type != nr->type) {
list_move_tail(&iter->list, &stack);
continue;
}
/* look for the last stack element of same type as @iter */
list_for_each_entry_reverse(top, &stack, list)
if (top->type == iter->type)
goto check_overlap;
list_move_tail(&iter->list, &stack);
continue;
check_overlap:
top_end = top->start + top->length - 1;
if (iter->start > top_end + 1) {
list_move_tail(&iter->list, &stack);
} else {
top->length = max(top_end, iter_end) - top->start + 1;
list_del(&iter->list);
kfree(iter);
}
}
list_splice(&stack, regions);
return 0;
}
@@ -1862,7 +1917,7 @@ EXPORT_SYMBOL_GPL(iommu_map);
static size_t __iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size,
bool sync)
struct iommu_iotlb_gather *iotlb_gather)
{
const struct iommu_ops *ops = domain->ops;
size_t unmapped_page, unmapped = 0;
@@ -1899,13 +1954,10 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
while (unmapped < size) {
size_t pgsize = iommu_pgsize(domain, iova, size - unmapped);
unmapped_page = ops->unmap(domain, iova, pgsize);
unmapped_page = ops->unmap(domain, iova, pgsize, iotlb_gather);
if (!unmapped_page)
break;
if (sync && ops->iotlb_range_add)
ops->iotlb_range_add(domain, iova, pgsize);
pr_debug("unmapped: iova 0x%lx size 0x%zx\n",
iova, unmapped_page);
@@ -1913,9 +1965,6 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
unmapped += unmapped_page;
}
if (sync && ops->iotlb_sync)
ops->iotlb_sync(domain);
trace_unmap(orig_iova, size, unmapped);
return unmapped;
}
@@ -1923,14 +1972,22 @@ static size_t __iommu_unmap(struct iommu_domain *domain,
size_t iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
{
return __iommu_unmap(domain, iova, size, true);
struct iommu_iotlb_gather iotlb_gather;
size_t ret;
iommu_iotlb_gather_init(&iotlb_gather);
ret = __iommu_unmap(domain, iova, size, &iotlb_gather);
iommu_tlb_sync(domain, &iotlb_gather);
return ret;
}
EXPORT_SYMBOL_GPL(iommu_unmap);
size_t iommu_unmap_fast(struct iommu_domain *domain,
unsigned long iova, size_t size)
unsigned long iova, size_t size,
struct iommu_iotlb_gather *iotlb_gather)
{
return __iommu_unmap(domain, iova, size, false);
return __iommu_unmap(domain, iova, size, iotlb_gather);
}
EXPORT_SYMBOL_GPL(iommu_unmap_fast);
@@ -2143,7 +2200,6 @@ request_default_domain_for_dev(struct device *dev, unsigned long type)
mutex_lock(&group->mutex);
/* Check if the default domain is already direct mapped */
ret = 0;
if (group->default_domain && group->default_domain->type == type)
goto out;
@@ -2153,7 +2209,6 @@ request_default_domain_for_dev(struct device *dev, unsigned long type)
if (iommu_group_device_count(group) != 1)
goto out;
/* Allocate a direct mapped domain */
ret = -ENOMEM;
domain = __iommu_domain_alloc(dev->bus, type);
if (!domain)
@@ -2168,7 +2223,7 @@ request_default_domain_for_dev(struct device *dev, unsigned long type)
iommu_group_create_direct_mappings(group, dev);
/* Make the direct mapped domain the default for this group */
/* Make the domain the default for this group */
if (group->default_domain)
iommu_domain_free(group->default_domain);
group->default_domain = domain;
@@ -2196,6 +2251,28 @@ int iommu_request_dma_domain_for_dev(struct device *dev)
return request_default_domain_for_dev(dev, IOMMU_DOMAIN_DMA);
}
void iommu_set_default_passthrough(bool cmd_line)
{
if (cmd_line)
iommu_set_cmd_line_dma_api();
iommu_def_domain_type = IOMMU_DOMAIN_IDENTITY;
}
void iommu_set_default_translated(bool cmd_line)
{
if (cmd_line)
iommu_set_cmd_line_dma_api();
iommu_def_domain_type = IOMMU_DOMAIN_DMA;
}
bool iommu_default_passthrough(void)
{
return iommu_def_domain_type == IOMMU_DOMAIN_IDENTITY;
}
EXPORT_SYMBOL_GPL(iommu_default_passthrough);
const struct iommu_ops *iommu_ops_from_fwnode(struct fwnode_handle *fwnode)
{
const struct iommu_ops *ops = NULL;

4
drivers/iommu/iova.c
View File

@@ -577,7 +577,9 @@ void queue_iova(struct iova_domain *iovad,
spin_unlock_irqrestore(&fq->lock, flags);
if (atomic_cmpxchg(&iovad->fq_timer_on, 0, 1) == 0)
/* Avoid false sharing as much as possible. */
if (!atomic_read(&iovad->fq_timer_on) &&
!atomic_cmpxchg(&iovad->fq_timer_on, 0, 1))
mod_timer(&iovad->fq_timer,
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
}

106
drivers/iommu/ipmmu-vmsa.c
View File

@@ -49,6 +49,7 @@ struct ipmmu_features {
bool setup_imbuscr;
bool twobit_imttbcr_sl0;
bool reserved_context;
bool cache_snoop;
};
struct ipmmu_vmsa_device {
@@ -115,45 +116,44 @@ static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev)
#define IMTTBCR 0x0008
#define IMTTBCR_EAE (1 << 31)
#define IMTTBCR_PMB (1 << 30)
#define IMTTBCR_SH1_NON_SHAREABLE (0 << 28)
#define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28)
#define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28)
#define IMTTBCR_SH1_MASK (3 << 28)
#define IMTTBCR_ORGN1_NC (0 << 26)
#define IMTTBCR_ORGN1_WB_WA (1 << 26)
#define IMTTBCR_ORGN1_WT (2 << 26)
#define IMTTBCR_ORGN1_WB (3 << 26)
#define IMTTBCR_ORGN1_MASK (3 << 26)
#define IMTTBCR_IRGN1_NC (0 << 24)
#define IMTTBCR_IRGN1_WB_WA (1 << 24)
#define IMTTBCR_IRGN1_WT (2 << 24)
#define IMTTBCR_IRGN1_WB (3 << 24)
#define IMTTBCR_IRGN1_MASK (3 << 24)
#define IMTTBCR_SH1_NON_SHAREABLE (0 << 28) /* R-Car Gen2 only */
#define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28) /* R-Car Gen2 only */
#define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28) /* R-Car Gen2 only */
#define IMTTBCR_SH1_MASK (3 << 28) /* R-Car Gen2 only */
#define IMTTBCR_ORGN1_NC (0 << 26) /* R-Car Gen2 only */
#define IMTTBCR_ORGN1_WB_WA (1 << 26) /* R-Car Gen2 only */
#define IMTTBCR_ORGN1_WT (2 << 26) /* R-Car Gen2 only */
#define IMTTBCR_ORGN1_WB (3 << 26) /* R-Car Gen2 only */
#define IMTTBCR_ORGN1_MASK (3 << 26) /* R-Car Gen2 only */
#define IMTTBCR_IRGN1_NC (0 << 24) /* R-Car Gen2 only */
#define IMTTBCR_IRGN1_WB_WA (1 << 24) /* R-Car Gen2 only */
#define IMTTBCR_IRGN1_WT (2 << 24) /* R-Car Gen2 only */
#define IMTTBCR_IRGN1_WB (3 << 24) /* R-Car Gen2 only */
#define IMTTBCR_IRGN1_MASK (3 << 24) /* R-Car Gen2 only */
#define IMTTBCR_TSZ1_MASK (7 << 16)
#define IMTTBCR_TSZ1_SHIFT 16
#define IMTTBCR_SH0_NON_SHAREABLE (0 << 12)
#define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12)
#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
#define IMTTBCR_SH0_MASK (3 << 12)
#define IMTTBCR_ORGN0_NC (0 << 10)
#define IMTTBCR_ORGN0_WB_WA (1 << 10)
#define IMTTBCR_ORGN0_WT (2 << 10)
#define IMTTBCR_ORGN0_WB (3 << 10)
#define IMTTBCR_ORGN0_MASK (3 << 10)
#define IMTTBCR_IRGN0_NC (0 << 8)
#define IMTTBCR_IRGN0_WB_WA (1 << 8)
#define IMTTBCR_IRGN0_WT (2 << 8)
#define IMTTBCR_IRGN0_WB (3 << 8)
#define IMTTBCR_IRGN0_MASK (3 << 8)
#define IMTTBCR_SH0_NON_SHAREABLE (0 << 12) /* R-Car Gen2 only */
#define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12) /* R-Car Gen2 only */
#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */
#define IMTTBCR_SH0_MASK (3 << 12) /* R-Car Gen2 only */
#define IMTTBCR_ORGN0_NC (0 << 10) /* R-Car Gen2 only */
#define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */
#define IMTTBCR_ORGN0_WT (2 << 10) /* R-Car Gen2 only */
#define IMTTBCR_ORGN0_WB (3 << 10) /* R-Car Gen2 only */
#define IMTTBCR_ORGN0_MASK (3 << 10) /* R-Car Gen2 only */
#define IMTTBCR_IRGN0_NC (0 << 8) /* R-Car Gen2 only */
#define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */
#define IMTTBCR_IRGN0_WT (2 << 8) /* R-Car Gen2 only */
#define IMTTBCR_IRGN0_WB (3 << 8) /* R-Car Gen2 only */
#define IMTTBCR_IRGN0_MASK (3 << 8) /* R-Car Gen2 only */
#define IMTTBCR_SL0_TWOBIT_LVL_3 (0 << 6) /* R-Car Gen3 only */
#define IMTTBCR_SL0_TWOBIT_LVL_2 (1 << 6) /* R-Car Gen3 only */
#define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */
#define IMTTBCR_SL0_LVL_2 (0 << 4)
#define IMTTBCR_SL0_LVL_1 (1 << 4)
#define IMTTBCR_TSZ0_MASK (7 << 0)
#define IMTTBCR_TSZ0_SHIFT O
#define IMTTBCR_SL0_TWOBIT_LVL_3 (0 << 6)
#define IMTTBCR_SL0_TWOBIT_LVL_2 (1 << 6)
#define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6)
#define IMBUSCR 0x000c
#define IMBUSCR_DVM (1 << 2)
#define IMBUSCR_BUSSEL_SYS (0 << 0)
@@ -361,16 +361,16 @@ static void ipmmu_tlb_flush_all(void *cookie)
ipmmu_tlb_invalidate(domain);
}
static void ipmmu_tlb_add_flush(unsigned long iova, size_t size,
size_t granule, bool leaf, void *cookie)
static void ipmmu_tlb_flush(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
/* The hardware doesn't support selective TLB flush. */
ipmmu_tlb_flush_all(cookie);
}
static const struct iommu_gather_ops ipmmu_gather_ops = {
static const struct iommu_flush_ops ipmmu_flush_ops = {
.tlb_flush_all = ipmmu_tlb_flush_all,
.tlb_add_flush = ipmmu_tlb_add_flush,
.tlb_sync = ipmmu_tlb_flush_all,
.tlb_flush_walk = ipmmu_tlb_flush,
.tlb_flush_leaf = ipmmu_tlb_flush,
};
/* -----------------------------------------------------------------------------
@@ -422,17 +422,19 @@ static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain)
/*
* TTBCR
* We use long descriptors with inner-shareable WBWA tables and allocate
* the whole 32-bit VA space to TTBR0.
* We use long descriptors and allocate the whole 32-bit VA space to
* TTBR0.
*/
if (domain->mmu->features->twobit_imttbcr_sl0)
tmp = IMTTBCR_SL0_TWOBIT_LVL_1;
else
tmp = IMTTBCR_SL0_LVL_1;
ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE |
IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
IMTTBCR_IRGN0_WB_WA | tmp);
if (domain->mmu->features->cache_snoop)
tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
IMTTBCR_IRGN0_WB_WA;
ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp);
/* MAIR0 */
ipmmu_ctx_write_root(domain, IMMAIR0,
@@ -480,7 +482,7 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K;
domain->cfg.ias = 32;
domain->cfg.oas = 40;
domain->cfg.tlb = &ipmmu_gather_ops;
domain->cfg.tlb = &ipmmu_flush_ops;
domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32);
domain->io_domain.geometry.force_aperture = true;
/*
@@ -733,14 +735,14 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
}
static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
return domain->iop->unmap(domain->iop, iova, size);
return domain->iop->unmap(domain->iop, iova, size, gather);
}
static void ipmmu_iotlb_sync(struct iommu_domain *io_domain)
static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
{
struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
@@ -748,6 +750,12 @@ static void ipmmu_iotlb_sync(struct iommu_domain *io_domain)
ipmmu_tlb_flush_all(domain);
}
static void ipmmu_iotlb_sync(struct iommu_domain *io_domain,
struct iommu_iotlb_gather *gather)
{
ipmmu_flush_iotlb_all(io_domain);
}
static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
dma_addr_t iova)
{
@@ -957,7 +965,7 @@ static const struct iommu_ops ipmmu_ops = {
.detach_dev = ipmmu_detach_device,
.map = ipmmu_map,
.unmap = ipmmu_unmap,
.flush_iotlb_all = ipmmu_iotlb_sync,
.flush_iotlb_all = ipmmu_flush_iotlb_all,
.iotlb_sync = ipmmu_iotlb_sync,
.iova_to_phys = ipmmu_iova_to_phys,
.add_device = ipmmu_add_device,
@@ -988,6 +996,7 @@ static const struct ipmmu_features ipmmu_features_default = {
.setup_imbuscr = true,
.twobit_imttbcr_sl0 = false,
.reserved_context = false,
.cache_snoop = true,
};
static const struct ipmmu_features ipmmu_features_rcar_gen3 = {
@@ -998,6 +1007,7 @@ static const struct ipmmu_features ipmmu_features_rcar_gen3 = {
.setup_imbuscr = false,
.twobit_imttbcr_sl0 = true,
.reserved_context = true,
.cache_snoop = false,
};
static const struct of_device_id ipmmu_of_ids[] = {

43
drivers/iommu/msm_iommu.c
View File

@@ -168,20 +168,29 @@ fail:
return;
}
static void __flush_iotlb_sync(void *cookie)
static void __flush_iotlb_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
/*
* Nothing is needed here, the barrier to guarantee
* completion of the tlb sync operation is implicitly
* taken care when the iommu client does a writel before
* kick starting the other master.
*/
__flush_iotlb_range(iova, size, granule, false, cookie);
}
static const struct iommu_gather_ops msm_iommu_gather_ops = {
static void __flush_iotlb_leaf(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
__flush_iotlb_range(iova, size, granule, true, cookie);
}
static void __flush_iotlb_page(struct iommu_iotlb_gather *gather,
unsigned long iova, size_t granule, void *cookie)
{
__flush_iotlb_range(iova, granule, granule, true, cookie);
}
static const struct iommu_flush_ops msm_iommu_flush_ops = {
.tlb_flush_all = __flush_iotlb,
.tlb_add_flush = __flush_iotlb_range,
.tlb_sync = __flush_iotlb_sync,
.tlb_flush_walk = __flush_iotlb_walk,
.tlb_flush_leaf = __flush_iotlb_leaf,
.tlb_add_page = __flush_iotlb_page,
};
static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end)
@@ -345,7 +354,7 @@ static int msm_iommu_domain_config(struct msm_priv *priv)
.pgsize_bitmap = msm_iommu_ops.pgsize_bitmap,
.ias = 32,
.oas = 32,
.tlb = &msm_iommu_gather_ops,
.tlb = &msm_iommu_flush_ops,
.iommu_dev = priv->dev,
};
@@ -509,13 +518,13 @@ static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t len)
size_t len, struct iommu_iotlb_gather *gather)
{
struct msm_priv *priv = to_msm_priv(domain);
unsigned long flags;
spin_lock_irqsave(&priv->pgtlock, flags);
len = priv->iop->unmap(priv->iop, iova, len);
len = priv->iop->unmap(priv->iop, iova, len, gather);
spin_unlock_irqrestore(&priv->pgtlock, flags);
return len;
@@ -691,6 +700,13 @@ static struct iommu_ops msm_iommu_ops = {
.detach_dev = msm_iommu_detach_dev,
.map = msm_iommu_map,
.unmap = msm_iommu_unmap,
/*
* Nothing is needed here, the barrier to guarantee
* completion of the tlb sync operation is implicitly
* taken care when the iommu client does a writel before
* kick starting the other master.
*/
.iotlb_sync = NULL,
.iova_to_phys = msm_iommu_iova_to_phys,
.add_device = msm_iommu_add_device,
.remove_device = msm_iommu_remove_device,
@@ -750,7 +766,6 @@ static int msm_iommu_probe(struct platform_device *pdev)
iommu->irq = platform_get_irq(pdev, 0);
if (iommu->irq < 0) {
dev_err(iommu->dev, "could not get iommu irq\n");
ret = -ENODEV;
goto fail;
}

213
drivers/iommu/mtk_iommu.c
View File

@@ -28,6 +28,7 @@
#include "mtk_iommu.h"
#define REG_MMU_PT_BASE_ADDR 0x000
#define MMU_PT_ADDR_MASK GENMASK(31, 7)
#define REG_MMU_INVALIDATE 0x020
#define F_ALL_INVLD 0x2
@@ -44,12 +45,9 @@
#define REG_MMU_DCM_DIS 0x050
#define REG_MMU_CTRL_REG 0x110
#define F_MMU_TF_PROT_TO_PROGRAM_ADDR (2 << 4)
#define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4)
#define F_MMU_TF_PROTECT_SEL_SHIFT(data) \
((data)->m4u_plat == M4U_MT2712 ? 4 : 5)
/* It's named by F_MMU_TF_PROT_SEL in mt2712. */
#define F_MMU_TF_PROTECT_SEL(prot, data) \
(((prot) & 0x3) << F_MMU_TF_PROTECT_SEL_SHIFT(data))
#define F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173 (2 << 5)
#define REG_MMU_IVRP_PADDR 0x114
@@ -66,26 +64,32 @@
#define F_INT_CLR_BIT BIT(12)
#define REG_MMU_INT_MAIN_CONTROL 0x124
#define F_INT_TRANSLATION_FAULT BIT(0)
#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1)
#define F_INT_INVALID_PA_FAULT BIT(2)
#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3)
#define F_INT_TLB_MISS_FAULT BIT(4)
#define F_INT_MISS_TRANSACTION_FIFO_FAULT BIT(5)
#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT BIT(6)
/* mmu0 | mmu1 */
#define F_INT_TRANSLATION_FAULT (BIT(0) | BIT(7))
#define F_INT_MAIN_MULTI_HIT_FAULT (BIT(1) | BIT(8))
#define F_INT_INVALID_PA_FAULT (BIT(2) | BIT(9))
#define F_INT_ENTRY_REPLACEMENT_FAULT (BIT(3) | BIT(10))
#define F_INT_TLB_MISS_FAULT (BIT(4) | BIT(11))
#define F_INT_MISS_TRANSACTION_FIFO_FAULT (BIT(5) | BIT(12))
#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT (BIT(6) | BIT(13))
#define REG_MMU_CPE_DONE 0x12C
#define REG_MMU_FAULT_ST1 0x134
#define F_REG_MMU0_FAULT_MASK GENMASK(6, 0)
#define F_REG_MMU1_FAULT_MASK GENMASK(13, 7)
#define REG_MMU_FAULT_VA 0x13c
#define REG_MMU0_FAULT_VA 0x13c
#define F_MMU_FAULT_VA_WRITE_BIT BIT(1)
#define F_MMU_FAULT_VA_LAYER_BIT BIT(0)
#define REG_MMU_INVLD_PA 0x140
#define REG_MMU_INT_ID 0x150
#define F_MMU0_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
#define F_MMU0_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
#define REG_MMU0_INVLD_PA 0x140
#define REG_MMU1_FAULT_VA 0x144
#define REG_MMU1_INVLD_PA 0x148
#define REG_MMU0_INT_ID 0x150
#define REG_MMU1_INT_ID 0x154
#define F_MMU_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7)
#define F_MMU_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f)
#define MTK_PROTECT_PA_ALIGN 128
@@ -107,6 +111,30 @@ struct mtk_iommu_domain {
static const struct iommu_ops mtk_iommu_ops;
/*
* In M4U 4GB mode, the physical address is remapped as below:
*
* CPU Physical address:
* ====================
*
* 0 1G 2G 3G 4G 5G
* |---A---|---B---|---C---|---D---|---E---|
* +--I/O--+------------Memory-------------+
*
* IOMMU output physical address:
* =============================
*
* 4G 5G 6G 7G 8G
* |---E---|---B---|---C---|---D---|
* +------------Memory-------------+
*
* The Region 'A'(I/O) can NOT be mapped by M4U; For Region 'B'/'C'/'D', the
* bit32 of the CPU physical address always is needed to set, and for Region
* 'E', the CPU physical address keep as is.
* Additionally, The iommu consumers always use the CPU phyiscal address.
*/
#define MTK_IOMMU_4GB_MODE_REMAP_BASE 0x140000000UL
static LIST_HEAD(m4ulist); /* List all the M4U HWs */
#define for_each_m4u(data) list_for_each_entry(data, &m4ulist, list)
@@ -188,10 +216,32 @@ static void mtk_iommu_tlb_sync(void *cookie)
}
}
static const struct iommu_gather_ops mtk_iommu_gather_ops = {
static void mtk_iommu_tlb_flush_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
mtk_iommu_tlb_add_flush_nosync(iova, size, granule, false, cookie);
mtk_iommu_tlb_sync(cookie);
}
static void mtk_iommu_tlb_flush_leaf(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
mtk_iommu_tlb_add_flush_nosync(iova, size, granule, true, cookie);
mtk_iommu_tlb_sync(cookie);
}
static void mtk_iommu_tlb_flush_page_nosync(struct iommu_iotlb_gather *gather,
unsigned long iova, size_t granule,
void *cookie)
{
mtk_iommu_tlb_add_flush_nosync(iova, granule, granule, true, cookie);
}
static const struct iommu_flush_ops mtk_iommu_flush_ops = {
.tlb_flush_all = mtk_iommu_tlb_flush_all,
.tlb_add_flush = mtk_iommu_tlb_add_flush_nosync,
.tlb_sync = mtk_iommu_tlb_sync,
.tlb_flush_walk = mtk_iommu_tlb_flush_walk,
.tlb_flush_leaf = mtk_iommu_tlb_flush_leaf,
.tlb_add_page = mtk_iommu_tlb_flush_page_nosync,
};
static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
@@ -204,13 +254,21 @@ static irqreturn_t mtk_iommu_isr(int irq, void *dev_id)
/* Read error info from registers */
int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1);
fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);
if (int_state & F_REG_MMU0_FAULT_MASK) {
regval = readl_relaxed(data->base + REG_MMU0_INT_ID);
fault_iova = readl_relaxed(data->base + REG_MMU0_FAULT_VA);
fault_pa = readl_relaxed(data->base + REG_MMU0_INVLD_PA);
} else {
regval = readl_relaxed(data->base + REG_MMU1_INT_ID);
fault_iova = readl_relaxed(data->base + REG_MMU1_FAULT_VA);
fault_pa = readl_relaxed(data->base + REG_MMU1_INVLD_PA);
}
layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT;
write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT;
fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);
regval = readl_relaxed(data->base + REG_MMU_INT_ID);
fault_larb = F_MMU0_INT_ID_LARB_ID(regval);
fault_port = F_MMU0_INT_ID_PORT_ID(regval);
fault_larb = F_MMU_INT_ID_LARB_ID(regval);
fault_port = F_MMU_INT_ID_PORT_ID(regval);
fault_larb = data->plat_data->larbid_remap[fault_larb];
if (report_iommu_fault(&dom->domain, data->dev, fault_iova,
write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) {
@@ -242,7 +300,7 @@ static void mtk_iommu_config(struct mtk_iommu_data *data,
for (i = 0; i < fwspec->num_ids; ++i) {
larbid = MTK_M4U_TO_LARB(fwspec->ids[i]);
portid = MTK_M4U_TO_PORT(fwspec->ids[i]);
larb_mmu = &data->smi_imu.larb_imu[larbid];
larb_mmu = &data->larb_imu[larbid];
dev_dbg(dev, "%s iommu port: %d\n",
enable ? "enable" : "disable", portid);
@@ -263,17 +321,15 @@ static int mtk_iommu_domain_finalise(struct mtk_iommu_domain *dom)
dom->cfg = (struct io_pgtable_cfg) {
.quirks = IO_PGTABLE_QUIRK_ARM_NS |
IO_PGTABLE_QUIRK_NO_PERMS |
IO_PGTABLE_QUIRK_TLBI_ON_MAP,
IO_PGTABLE_QUIRK_TLBI_ON_MAP |
IO_PGTABLE_QUIRK_ARM_MTK_EXT,
.pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap,
.ias = 32,
.oas = 32,
.tlb = &mtk_iommu_gather_ops,
.oas = 34,
.tlb = &mtk_iommu_flush_ops,
.iommu_dev = data->dev,
};
if (data->enable_4GB)
dom->cfg.quirks |= IO_PGTABLE_QUIRK_ARM_MTK_4GB;
dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data);
if (!dom->iop) {
dev_err(data->dev, "Failed to alloc io pgtable\n");
@@ -336,7 +392,7 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain,
/* Update the pgtable base address register of the M4U HW */
if (!data->m4u_dom) {
data->m4u_dom = dom;
writel(dom->cfg.arm_v7s_cfg.ttbr[0],
writel(dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,
data->base + REG_MMU_PT_BASE_ADDR);
}
@@ -359,32 +415,43 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();
unsigned long flags;
int ret;
/* The "4GB mode" M4U physically can not use the lower remap of Dram. */
if (data->enable_4GB)
paddr |= BIT_ULL(32);
spin_lock_irqsave(&dom->pgtlock, flags);
ret = dom->iop->map(dom->iop, iova, paddr & DMA_BIT_MASK(32),
size, prot);
ret = dom->iop->map(dom->iop, iova, paddr, size, prot);
spin_unlock_irqrestore(&dom->pgtlock, flags);
return ret;
}
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
unsigned long iova, size_t size,
struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
unsigned long flags;
size_t unmapsz;
spin_lock_irqsave(&dom->pgtlock, flags);
unmapsz = dom->iop->unmap(dom->iop, iova, size);
unmapsz = dom->iop->unmap(dom->iop, iova, size, gather);
spin_unlock_irqrestore(&dom->pgtlock, flags);
return unmapsz;
}
static void mtk_iommu_iotlb_sync(struct iommu_domain *domain)
static void mtk_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data());
}
static void mtk_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
mtk_iommu_tlb_sync(mtk_iommu_get_m4u_data());
}
@@ -401,8 +468,8 @@ static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain,
pa = dom->iop->iova_to_phys(dom->iop, iova);
spin_unlock_irqrestore(&dom->pgtlock, flags);
if (data->enable_4GB)
pa |= BIT_ULL(32);
if (data->enable_4GB && pa >= MTK_IOMMU_4GB_MODE_REMAP_BASE)
pa &= ~BIT_ULL(32);
return pa;
}
@@ -490,7 +557,7 @@ static const struct iommu_ops mtk_iommu_ops = {
.detach_dev = mtk_iommu_detach_device,
.map = mtk_iommu_map,
.unmap = mtk_iommu_unmap,
.flush_iotlb_all = mtk_iommu_iotlb_sync,
.flush_iotlb_all = mtk_iommu_flush_iotlb_all,
.iotlb_sync = mtk_iommu_iotlb_sync,
.iova_to_phys = mtk_iommu_iova_to_phys,
.add_device = mtk_iommu_add_device,
@@ -511,9 +578,11 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
return ret;
}
regval = F_MMU_TF_PROTECT_SEL(2, data);
if (data->m4u_plat == M4U_MT8173)
regval |= F_MMU_PREFETCH_RT_REPLACE_MOD;
if (data->plat_data->m4u_plat == M4U_MT8173)
regval = F_MMU_PREFETCH_RT_REPLACE_MOD |
F_MMU_TF_PROT_TO_PROGRAM_ADDR_MT8173;
else
regval = F_MMU_TF_PROT_TO_PROGRAM_ADDR;
writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);
regval = F_L2_MULIT_HIT_EN |
@@ -533,14 +602,14 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
F_INT_PRETETCH_TRANSATION_FIFO_FAULT;
writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL);
if (data->m4u_plat == M4U_MT8173)
if (data->plat_data->m4u_plat == M4U_MT8173)
regval = (data->protect_base >> 1) | (data->enable_4GB << 31);
else
regval = lower_32_bits(data->protect_base) |
upper_32_bits(data->protect_base);
writel_relaxed(regval, data->base + REG_MMU_IVRP_PADDR);
if (data->enable_4GB && data->m4u_plat != M4U_MT8173) {
if (data->enable_4GB && data->plat_data->has_vld_pa_rng) {
/*
* If 4GB mode is enabled, the validate PA range is from
* 0x1_0000_0000 to 0x1_ffff_ffff. here record bit[32:30].
@@ -550,8 +619,7 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data)
}
writel_relaxed(0, data->base + REG_MMU_DCM_DIS);
/* It's MISC control register whose default value is ok except mt8173.*/
if (data->m4u_plat == M4U_MT8173)
if (data->plat_data->reset_axi)
writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE);
if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0,
@@ -584,7 +652,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (!data)
return -ENOMEM;
data->dev = dev;
data->m4u_plat = (enum mtk_iommu_plat)of_device_get_match_data(dev);
data->plat_data = of_device_get_match_data(dev);
/* Protect memory. HW will access here while translation fault.*/
protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL);
@@ -594,6 +662,8 @@ static int mtk_iommu_probe(struct platform_device *pdev)
/* Whether the current dram is over 4GB */
data->enable_4GB = !!(max_pfn > (BIT_ULL(32) >> PAGE_SHIFT));
if (!data->plat_data->has_4gb_mode)
data->enable_4GB = false;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
data->base = devm_ioremap_resource(dev, res);
@@ -605,15 +675,16 @@ static int mtk_iommu_probe(struct platform_device *pdev)
if (data->irq < 0)
return data->irq;
data->bclk = devm_clk_get(dev, "bclk");
if (IS_ERR(data->bclk))
return PTR_ERR(data->bclk);
if (data->plat_data->has_bclk) {
data->bclk = devm_clk_get(dev, "bclk");
if (IS_ERR(data->bclk))
return PTR_ERR(data->bclk);
}
larb_nr = of_count_phandle_with_args(dev->of_node,
"mediatek,larbs", NULL);
if (larb_nr < 0)
return larb_nr;
data->smi_imu.larb_nr = larb_nr;
for (i = 0; i < larb_nr; i++) {
struct device_node *larbnode;
@@ -638,7 +709,7 @@ static int mtk_iommu_probe(struct platform_device *pdev)
of_node_put(larbnode);
return -EPROBE_DEFER;
}
data->smi_imu.larb_imu[id].dev = &plarbdev->dev;
data->larb_imu[id].dev = &plarbdev->dev;
component_match_add_release(dev, &match, release_of,
compare_of, larbnode);
@@ -699,6 +770,7 @@ static int __maybe_unused mtk_iommu_suspend(struct device *dev)
reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0);
reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL);
reg->ivrp_paddr = readl_relaxed(base + REG_MMU_IVRP_PADDR);
reg->vld_pa_rng = readl_relaxed(base + REG_MMU_VLD_PA_RNG);
clk_disable_unprepare(data->bclk);
return 0;
}
@@ -707,6 +779,7 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
{
struct mtk_iommu_data *data = dev_get_drvdata(dev);
struct mtk_iommu_suspend_reg *reg = &data->reg;
struct mtk_iommu_domain *m4u_dom = data->m4u_dom;
void __iomem *base = data->base;
int ret;
@@ -722,8 +795,9 @@ static int __maybe_unused mtk_iommu_resume(struct device *dev)
writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0);
writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL);
writel_relaxed(reg->ivrp_paddr, base + REG_MMU_IVRP_PADDR);
if (data->m4u_dom)
writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0],
writel_relaxed(reg->vld_pa_rng, base + REG_MMU_VLD_PA_RNG);
if (m4u_dom)
writel(m4u_dom->cfg.arm_v7s_cfg.ttbr[0] & MMU_PT_ADDR_MASK,
base + REG_MMU_PT_BASE_ADDR);
return 0;
}
@@ -732,9 +806,32 @@ static const struct dev_pm_ops mtk_iommu_pm_ops = {
SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume)
};
static const struct mtk_iommu_plat_data mt2712_data = {
.m4u_plat = M4U_MT2712,
.has_4gb_mode = true,
.has_bclk = true,
.has_vld_pa_rng = true,
.larbid_remap = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
};
static const struct mtk_iommu_plat_data mt8173_data = {
.m4u_plat = M4U_MT8173,
.has_4gb_mode = true,
.has_bclk = true,
.reset_axi = true,
.larbid_remap = {0, 1, 2, 3, 4, 5}, /* Linear mapping. */
};
static const struct mtk_iommu_plat_data mt8183_data = {
.m4u_plat = M4U_MT8183,
.reset_axi = true,
.larbid_remap = {0, 4, 5, 6, 7, 2, 3, 1},
};
static const struct of_device_id mtk_iommu_of_ids[] = {
{ .compatible = "mediatek,mt2712-m4u", .data = (void *)M4U_MT2712},
{ .compatible = "mediatek,mt8173-m4u", .data = (void *)M4U_MT8173},
{ .compatible = "mediatek,mt2712-m4u", .data = &mt2712_data},
{ .compatible = "mediatek,mt8173-m4u", .data = &mt8173_data},
{ .compatible = "mediatek,mt8183-m4u", .data = &mt8183_data},
{}
};

21
drivers/iommu/mtk_iommu.h
View File

@@ -24,12 +24,25 @@ struct mtk_iommu_suspend_reg {
u32 int_control0;
u32 int_main_control;
u32 ivrp_paddr;
u32 vld_pa_rng;
};
enum mtk_iommu_plat {
M4U_MT2701,
M4U_MT2712,
M4U_MT8173,
M4U_MT8183,
};
struct mtk_iommu_plat_data {
enum mtk_iommu_plat m4u_plat;
bool has_4gb_mode;
/* HW will use the EMI clock if there isn't the "bclk". */
bool has_bclk;
bool has_vld_pa_rng;
bool reset_axi;
unsigned char larbid_remap[MTK_LARB_NR_MAX];
};
struct mtk_iommu_domain;
@@ -43,14 +56,14 @@ struct mtk_iommu_data {
struct mtk_iommu_suspend_reg reg;
struct mtk_iommu_domain *m4u_dom;
struct iommu_group *m4u_group;
struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */
bool enable_4GB;
bool tlb_flush_active;
struct iommu_device iommu;
enum mtk_iommu_plat m4u_plat;
const struct mtk_iommu_plat_data *plat_data;
struct list_head list;
struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX];
};
static inline int compare_of(struct device *dev, void *data)
@@ -67,14 +80,14 @@ static inline int mtk_iommu_bind(struct device *dev)
{
struct mtk_iommu_data *data = dev_get_drvdata(dev);
return component_bind_all(dev, &data->smi_imu);
return component_bind_all(dev, &data->larb_imu);
}
static inline void mtk_iommu_unbind(struct device *dev)
{
struct mtk_iommu_data *data = dev_get_drvdata(dev);
component_unbind_all(dev, &data->smi_imu);
component_unbind_all(dev, &data->larb_imu);
}
#endif

9
drivers/iommu/mtk_iommu_v1.c
View File

@@ -206,7 +206,7 @@ static void mtk_iommu_config(struct mtk_iommu_data *data,
for (i = 0; i < fwspec->num_ids; ++i) {
larbid = mt2701_m4u_to_larb(fwspec->ids[i]);
portid = mt2701_m4u_to_port(fwspec->ids[i]);
larb_mmu = &data->smi_imu.larb_imu[larbid];
larb_mmu = &data->larb_imu[larbid];
dev_dbg(dev, "%s iommu port: %d\n",
enable ? "enable" : "disable", portid);
@@ -324,7 +324,8 @@ static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t mtk_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
unsigned long iova, size_t size,
struct iommu_iotlb_gather *gather)
{
struct mtk_iommu_domain *dom = to_mtk_domain(domain);
unsigned long flags;
@@ -610,14 +611,12 @@ static int mtk_iommu_probe(struct platform_device *pdev)
}
}
data->smi_imu.larb_imu[larb_nr].dev = &plarbdev->dev;
data->larb_imu[larb_nr].dev = &plarbdev->dev;
component_match_add_release(dev, &match, release_of,
compare_of, larb_spec.np);
larb_nr++;
}
data->smi_imu.larb_nr = larb_nr;
platform_set_drvdata(pdev, data);
ret = mtk_iommu_hw_init(data);

324
drivers/iommu/omap-iommu.c
View File

@@ -35,6 +35,15 @@
static const struct iommu_ops omap_iommu_ops;
struct orphan_dev {
struct device *dev;
struct list_head node;
};
static LIST_HEAD(orphan_dev_list);
static DEFINE_SPINLOCK(orphan_lock);
#define to_iommu(dev) ((struct omap_iommu *)dev_get_drvdata(dev))
/* bitmap of the page sizes currently supported */
@@ -53,6 +62,8 @@ static const struct iommu_ops omap_iommu_ops;
static struct platform_driver omap_iommu_driver;
static struct kmem_cache *iopte_cachep;
static int _omap_iommu_add_device(struct device *dev);
/**
* to_omap_domain - Get struct omap_iommu_domain from generic iommu_domain
* @dom: generic iommu domain handle
@@ -65,6 +76,9 @@ static struct omap_iommu_domain *to_omap_domain(struct iommu_domain *dom)
/**
* omap_iommu_save_ctx - Save registers for pm off-mode support
* @dev: client device
*
* This should be treated as an deprecated API. It is preserved only
* to maintain existing functionality for OMAP3 ISP driver.
**/
void omap_iommu_save_ctx(struct device *dev)
{
@@ -92,6 +106,9 @@ EXPORT_SYMBOL_GPL(omap_iommu_save_ctx);
/**
* omap_iommu_restore_ctx - Restore registers for pm off-mode support
* @dev: client device
*
* This should be treated as an deprecated API. It is preserved only
* to maintain existing functionality for OMAP3 ISP driver.
**/
void omap_iommu_restore_ctx(struct device *dev)
{
@@ -186,36 +203,18 @@ static void omap2_iommu_disable(struct omap_iommu *obj)
static int iommu_enable(struct omap_iommu *obj)
{
int err;
struct platform_device *pdev = to_platform_device(obj->dev);
struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
int ret;
if (pdata && pdata->deassert_reset) {
err = pdata->deassert_reset(pdev, pdata->reset_name);
if (err) {
dev_err(obj->dev, "deassert_reset failed: %d\n", err);
return err;
}
}
ret = pm_runtime_get_sync(obj->dev);
if (ret < 0)
pm_runtime_put_noidle(obj->dev);
pm_runtime_get_sync(obj->dev);
err = omap2_iommu_enable(obj);
return err;
return ret < 0 ? ret : 0;
}
static void iommu_disable(struct omap_iommu *obj)
{
struct platform_device *pdev = to_platform_device(obj->dev);
struct iommu_platform_data *pdata = dev_get_platdata(&pdev->dev);
omap2_iommu_disable(obj);
pm_runtime_put_sync(obj->dev);
if (pdata && pdata->assert_reset)
pdata->assert_reset(pdev, pdata->reset_name);
}
/*
@@ -901,15 +900,219 @@ static void omap_iommu_detach(struct omap_iommu *obj)
dma_unmap_single(obj->dev, obj->pd_dma, IOPGD_TABLE_SIZE,
DMA_TO_DEVICE);
iommu_disable(obj);
obj->pd_dma = 0;
obj->iopgd = NULL;
iommu_disable(obj);
spin_unlock(&obj->iommu_lock);
dev_dbg(obj->dev, "%s: %s\n", __func__, obj->name);
}
static void omap_iommu_save_tlb_entries(struct omap_iommu *obj)
{
struct iotlb_lock lock;
struct cr_regs cr;
struct cr_regs *tmp;
int i;
/* check if there are any locked tlbs to save */
iotlb_lock_get(obj, &lock);
obj->num_cr_ctx = lock.base;
if (!obj->num_cr_ctx)
return;
tmp = obj->cr_ctx;
for_each_iotlb_cr(obj, obj->num_cr_ctx, i, cr)
* tmp++ = cr;
}
static void omap_iommu_restore_tlb_entries(struct omap_iommu *obj)
{
struct iotlb_lock l;
struct cr_regs *tmp;
int i;
/* no locked tlbs to restore */
if (!obj->num_cr_ctx)
return;
l.base = 0;
tmp = obj->cr_ctx;
for (i = 0; i < obj->num_cr_ctx; i++, tmp++) {
l.vict = i;
iotlb_lock_set(obj, &l);
iotlb_load_cr(obj, tmp);
}
l.base = obj->num_cr_ctx;
l.vict = i;
iotlb_lock_set(obj, &l);
}
/**
* omap_iommu_domain_deactivate - deactivate attached iommu devices
* @domain: iommu domain attached to the target iommu device
*
* This API allows the client devices of IOMMU devices to suspend
* the IOMMUs they control at runtime, after they are idled and
* suspended all activity. System Suspend will leverage the PM
* driver late callbacks.
**/
int omap_iommu_domain_deactivate(struct iommu_domain *domain)
{
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct omap_iommu_device *iommu;
struct omap_iommu *oiommu;
int i;
if (!omap_domain->dev)
return 0;
iommu = omap_domain->iommus;
iommu += (omap_domain->num_iommus - 1);
for (i = 0; i < omap_domain->num_iommus; i++, iommu--) {
oiommu = iommu->iommu_dev;
pm_runtime_put_sync(oiommu->dev);
}
return 0;
}
EXPORT_SYMBOL_GPL(omap_iommu_domain_deactivate);
/**
* omap_iommu_domain_activate - activate attached iommu devices
* @domain: iommu domain attached to the target iommu device
*
* This API allows the client devices of IOMMU devices to resume the
* IOMMUs they control at runtime, before they can resume operations.
* System Resume will leverage the PM driver late callbacks.
**/
int omap_iommu_domain_activate(struct iommu_domain *domain)
{
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct omap_iommu_device *iommu;
struct omap_iommu *oiommu;
int i;
if (!omap_domain->dev)
return 0;
iommu = omap_domain->iommus;
for (i = 0; i < omap_domain->num_iommus; i++, iommu++) {
oiommu = iommu->iommu_dev;
pm_runtime_get_sync(oiommu->dev);
}
return 0;
}
EXPORT_SYMBOL_GPL(omap_iommu_domain_activate);
/**
* omap_iommu_runtime_suspend - disable an iommu device
* @dev: iommu device
*
* This function performs all that is necessary to disable an
* IOMMU device, either during final detachment from a client
* device, or during system/runtime suspend of the device. This
* includes programming all the appropriate IOMMU registers, and
* managing the associated omap_hwmod's state and the device's
* reset line. This function also saves the context of any
* locked TLBs if suspending.
**/
static __maybe_unused int omap_iommu_runtime_suspend(struct device *dev)
{
struct platform_device *pdev = to_platform_device(dev);
struct iommu_platform_data *pdata = dev_get_platdata(dev);
struct omap_iommu *obj = to_iommu(dev);
int ret;
/* save the TLBs only during suspend, and not for power down */
if (obj->domain && obj->iopgd)
omap_iommu_save_tlb_entries(obj);
omap2_iommu_disable(obj);
if (pdata && pdata->device_idle)
pdata->device_idle(pdev);
if (pdata && pdata->assert_reset)
pdata->assert_reset(pdev, pdata->reset_name);
if (pdata && pdata->set_pwrdm_constraint) {
ret = pdata->set_pwrdm_constraint(pdev, false, &obj->pwrst);
if (ret) {
dev_warn(obj->dev, "pwrdm_constraint failed to be reset, status = %d\n",
ret);
}
}
return 0;
}
/**
* omap_iommu_runtime_resume - enable an iommu device
* @dev: iommu device
*
* This function performs all that is necessary to enable an
* IOMMU device, either during initial attachment to a client
* device, or during system/runtime resume of the device. This
* includes programming all the appropriate IOMMU registers, and
* managing the associated omap_hwmod's state and the device's
* reset line. The function also restores any locked TLBs if
* resuming after a suspend.
**/
static __maybe_unused int omap_iommu_runtime_resume(struct device *dev)
{
struct platform_device *pdev = to_platform_device(dev);
struct iommu_platform_data *pdata = dev_get_platdata(dev);
struct omap_iommu *obj = to_iommu(dev);
int ret = 0;
if (pdata && pdata->set_pwrdm_constraint) {
ret = pdata->set_pwrdm_constraint(pdev, true, &obj->pwrst);
if (ret) {
dev_warn(obj->dev, "pwrdm_constraint failed to be set, status = %d\n",
ret);
}
}
if (pdata && pdata->deassert_reset) {
ret = pdata->deassert_reset(pdev, pdata->reset_name);
if (ret) {
dev_err(dev, "deassert_reset failed: %d\n", ret);
return ret;
}
}
if (pdata && pdata->device_enable)
pdata->device_enable(pdev);
/* restore the TLBs only during resume, and not for power up */
if (obj->domain)
omap_iommu_restore_tlb_entries(obj);
ret = omap2_iommu_enable(obj);
return ret;
}
/**
* omap_iommu_suspend_prepare - prepare() dev_pm_ops implementation
* @dev: iommu device
*
* This function performs the necessary checks to determine if the IOMMU
* device needs suspending or not. The function checks if the runtime_pm
* status of the device is suspended, and returns 1 in that case. This
* results in the PM core to skip invoking any of the Sleep PM callbacks
* (suspend, suspend_late, resume, resume_early etc).
*/
static int omap_iommu_prepare(struct device *dev)
{
if (pm_runtime_status_suspended(dev))
return 1;
return 0;
}
static bool omap_iommu_can_register(struct platform_device *pdev)
{
struct device_node *np = pdev->dev.of_node;
@@ -974,6 +1177,7 @@ static int omap_iommu_probe(struct platform_device *pdev)
struct omap_iommu *obj;
struct resource *res;
struct device_node *of = pdev->dev.of_node;
struct orphan_dev *orphan_dev, *tmp;
if (!of) {
pr_err("%s: only DT-based devices are supported\n", __func__);
@@ -984,6 +1188,15 @@ static int omap_iommu_probe(struct platform_device *pdev)
if (!obj)
return -ENOMEM;
/*
* self-manage the ordering dependencies between omap_device_enable/idle
* and omap_device_assert/deassert_hardreset API
*/
if (pdev->dev.pm_domain) {
dev_dbg(&pdev->dev, "device pm_domain is being reset\n");
pdev->dev.pm_domain = NULL;
}
obj->name = dev_name(&pdev->dev);
obj->nr_tlb_entries = 32;
err = of_property_read_u32(of, "ti,#tlb-entries", &obj->nr_tlb_entries);
@@ -996,6 +1209,11 @@ static int omap_iommu_probe(struct platform_device *pdev)
obj->dev = &pdev->dev;
obj->ctx = (void *)obj + sizeof(*obj);
obj->cr_ctx = devm_kzalloc(&pdev->dev,
sizeof(*obj->cr_ctx) * obj->nr_tlb_entries,
GFP_KERNEL);
if (!obj->cr_ctx)
return -ENOMEM;
spin_lock_init(&obj->iommu_lock);
spin_lock_init(&obj->page_table_lock);
@@ -1036,13 +1254,20 @@ static int omap_iommu_probe(struct platform_device *pdev)
goto out_sysfs;
}
pm_runtime_irq_safe(obj->dev);
pm_runtime_enable(obj->dev);
omap_iommu_debugfs_add(obj);
dev_info(&pdev->dev, "%s registered\n", obj->name);
list_for_each_entry_safe(orphan_dev, tmp, &orphan_dev_list, node) {
err = _omap_iommu_add_device(orphan_dev->dev);
if (!err) {
list_del(&orphan_dev->node);
kfree(orphan_dev);
}
}
return 0;
out_sysfs:
@@ -1072,6 +1297,14 @@ static int omap_iommu_remove(struct platform_device *pdev)
return 0;
}
static const struct dev_pm_ops omap_iommu_pm_ops = {
.prepare = omap_iommu_prepare,
SET_LATE_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
pm_runtime_force_resume)
SET_RUNTIME_PM_OPS(omap_iommu_runtime_suspend,
omap_iommu_runtime_resume, NULL)
};
static const struct of_device_id omap_iommu_of_match[] = {
{ .compatible = "ti,omap2-iommu" },
{ .compatible = "ti,omap4-iommu" },
@@ -1085,6 +1318,7 @@ static struct platform_driver omap_iommu_driver = {
.remove = omap_iommu_remove,
.driver = {
.name = "omap-iommu",
.pm = &omap_iommu_pm_ops,
.of_match_table = of_match_ptr(omap_iommu_of_match),
},
};
@@ -1149,7 +1383,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
}
static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
struct device *dev = omap_domain->dev;
@@ -1423,7 +1657,7 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain,
return ret;
}
static int omap_iommu_add_device(struct device *dev)
static int _omap_iommu_add_device(struct device *dev)
{
struct omap_iommu_arch_data *arch_data, *tmp;
struct omap_iommu *oiommu;
@@ -1432,6 +1666,8 @@ static int omap_iommu_add_device(struct device *dev)
struct platform_device *pdev;
int num_iommus, i;
int ret;
struct orphan_dev *orphan_dev;
unsigned long flags;
/*
* Allocate the archdata iommu structure for DT-based devices.
@@ -1463,10 +1699,26 @@ static int omap_iommu_add_device(struct device *dev)
}
pdev = of_find_device_by_node(np);
if (WARN_ON(!pdev)) {
if (!pdev) {
of_node_put(np);
kfree(arch_data);
return -EINVAL;
spin_lock_irqsave(&orphan_lock, flags);
list_for_each_entry(orphan_dev, &orphan_dev_list,
node) {
if (orphan_dev->dev == dev)
break;
}
spin_unlock_irqrestore(&orphan_lock, flags);
if (orphan_dev && orphan_dev->dev == dev)
return -EPROBE_DEFER;
orphan_dev = kzalloc(sizeof(*orphan_dev), GFP_KERNEL);
orphan_dev->dev = dev;
spin_lock_irqsave(&orphan_lock, flags);
list_add(&orphan_dev->node, &orphan_dev_list);
spin_unlock_irqrestore(&orphan_lock, flags);
return -EPROBE_DEFER;
}
oiommu = platform_get_drvdata(pdev);
@@ -1477,6 +1729,7 @@ static int omap_iommu_add_device(struct device *dev)
}
tmp->iommu_dev = oiommu;
tmp->dev = &pdev->dev;
of_node_put(np);
}
@@ -1511,6 +1764,17 @@ static int omap_iommu_add_device(struct device *dev)
return 0;
}
static int omap_iommu_add_device(struct device *dev)
{
int ret;
ret = _omap_iommu_add_device(dev);
if (ret == -EPROBE_DEFER)
return 0;
return ret;
}
static void omap_iommu_remove_device(struct device *dev)
{
struct omap_iommu_arch_data *arch_data = dev->archdata.iommu;
@@ -1554,7 +1818,7 @@ static const struct iommu_ops omap_iommu_ops = {
static int __init omap_iommu_init(void)
{
struct kmem_cache *p;
const unsigned long flags = SLAB_HWCACHE_ALIGN;
const slab_flags_t flags = SLAB_HWCACHE_ALIGN;
size_t align = 1 << 10; /* L2 pagetable alignement */
struct device_node *np;
int ret;

9
drivers/iommu/omap-iommu.h
View File

@@ -73,16 +73,22 @@ struct omap_iommu {
void *ctx; /* iommu context: registres saved area */
struct cr_regs *cr_ctx;
u32 num_cr_ctx;
int has_bus_err_back;
u32 id;
struct iommu_device iommu;
struct iommu_group *group;
u8 pwrst;
};
/**
* struct omap_iommu_arch_data - omap iommu private data
* @iommu_dev: handle of the iommu device
* @iommu_dev: handle of the OMAP iommu device
* @dev: handle of the iommu device
*
* This is an omap iommu private data object, which binds an iommu user
* to its iommu device. This object should be placed at the iommu user's
@@ -91,6 +97,7 @@ struct omap_iommu {
*/
struct omap_iommu_arch_data {
struct omap_iommu *iommu_dev;
struct device *dev;
};
struct cr_regs {

72
drivers/iommu/qcom_iommu.c
View File

@@ -7,6 +7,7 @@
*/
#include <linux/atomic.h>
#include <linux/bitfield.h>
#include <linux/clk.h>
#include <linux/delay.h>
#include <linux/dma-iommu.h>
@@ -32,7 +33,7 @@
#include <linux/slab.h>
#include <linux/spinlock.h>
#include "arm-smmu-regs.h"
#include "arm-smmu.h"
#define SMMU_INTR_SEL_NS 0x2000
@@ -155,7 +156,7 @@ static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
struct qcom_iommu_ctx *ctx = to_ctx(fwspec, fwspec->ids[i]);
size_t s = size;
iova &= ~12UL;
iova = (iova >> 12) << 12;
iova |= ctx->asid;
do {
iommu_writel(ctx, reg, iova);
@@ -164,10 +165,32 @@ static void qcom_iommu_tlb_inv_range_nosync(unsigned long iova, size_t size,
}
}
static const struct iommu_gather_ops qcom_gather_ops = {
static void qcom_iommu_tlb_flush_walk(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
qcom_iommu_tlb_inv_range_nosync(iova, size, granule, false, cookie);
qcom_iommu_tlb_sync(cookie);
}
static void qcom_iommu_tlb_flush_leaf(unsigned long iova, size_t size,
size_t granule, void *cookie)
{
qcom_iommu_tlb_inv_range_nosync(iova, size, granule, true, cookie);
qcom_iommu_tlb_sync(cookie);
}
static void qcom_iommu_tlb_add_page(struct iommu_iotlb_gather *gather,
unsigned long iova, size_t granule,
void *cookie)
{
qcom_iommu_tlb_inv_range_nosync(iova, granule, granule, true, cookie);
}
static const struct iommu_flush_ops qcom_flush_ops = {
.tlb_flush_all = qcom_iommu_tlb_inv_context,
.tlb_add_flush = qcom_iommu_tlb_inv_range_nosync,
.tlb_sync = qcom_iommu_tlb_sync,
.tlb_flush_walk = qcom_iommu_tlb_flush_walk,
.tlb_flush_leaf = qcom_iommu_tlb_flush_leaf,
.tlb_add_page = qcom_iommu_tlb_add_page,
};
static irqreturn_t qcom_iommu_fault(int irq, void *dev)
@@ -215,7 +238,7 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain,
.pgsize_bitmap = qcom_iommu_ops.pgsize_bitmap,
.ias = 32,
.oas = 40,
.tlb = &qcom_gather_ops,
.tlb = &qcom_flush_ops,
.iommu_dev = qcom_iommu->dev,
};
@@ -247,16 +270,16 @@ static int qcom_iommu_init_domain(struct iommu_domain *domain,
/* TTBRs */
iommu_writeq(ctx, ARM_SMMU_CB_TTBR0,
pgtbl_cfg.arm_lpae_s1_cfg.ttbr[0] |
((u64)ctx->asid << TTBRn_ASID_SHIFT));
FIELD_PREP(TTBRn_ASID, ctx->asid));
iommu_writeq(ctx, ARM_SMMU_CB_TTBR1,
pgtbl_cfg.arm_lpae_s1_cfg.ttbr[1] |
((u64)ctx->asid << TTBRn_ASID_SHIFT));
FIELD_PREP(TTBRn_ASID, ctx->asid));
/* TTBCR */
iommu_writel(ctx, ARM_SMMU_CB_TTBCR2,
/* TCR */
iommu_writel(ctx, ARM_SMMU_CB_TCR2,
(pgtbl_cfg.arm_lpae_s1_cfg.tcr >> 32) |
TTBCR2_SEP_UPSTREAM);
iommu_writel(ctx, ARM_SMMU_CB_TTBCR,
FIELD_PREP(TCR2_SEP, TCR2_SEP_UPSTREAM));
iommu_writel(ctx, ARM_SMMU_CB_TCR,
pgtbl_cfg.arm_lpae_s1_cfg.tcr);
/* MAIRs (stage-1 only) */
@@ -417,7 +440,7 @@ static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
size_t ret;
unsigned long flags;
@@ -434,14 +457,14 @@ static size_t qcom_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
*/
pm_runtime_get_sync(qcom_domain->iommu->dev);
spin_lock_irqsave(&qcom_domain->pgtbl_lock, flags);
ret = ops->unmap(ops, iova, size);
ret = ops->unmap(ops, iova, size, gather);
spin_unlock_irqrestore(&qcom_domain->pgtbl_lock, flags);
pm_runtime_put_sync(qcom_domain->iommu->dev);
return ret;
}
static void qcom_iommu_iotlb_sync(struct iommu_domain *domain)
static void qcom_iommu_flush_iotlb_all(struct iommu_domain *domain)
{
struct qcom_iommu_domain *qcom_domain = to_qcom_iommu_domain(domain);
struct io_pgtable *pgtable = container_of(qcom_domain->pgtbl_ops,
@@ -454,6 +477,12 @@ static void qcom_iommu_iotlb_sync(struct iommu_domain *domain)
pm_runtime_put_sync(qcom_domain->iommu->dev);
}
static void qcom_iommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
qcom_iommu_flush_iotlb_all(domain);
}
static phys_addr_t qcom_iommu_iova_to_phys(struct iommu_domain *domain,
dma_addr_t iova)
{
@@ -581,7 +610,7 @@ static const struct iommu_ops qcom_iommu_ops = {
.detach_dev = qcom_iommu_detach_dev,
.map = qcom_iommu_map,
.unmap = qcom_iommu_unmap,
.flush_iotlb_all = qcom_iommu_iotlb_sync,
.flush_iotlb_all = qcom_iommu_flush_iotlb_all,
.iotlb_sync = qcom_iommu_iotlb_sync,
.iova_to_phys = qcom_iommu_iova_to_phys,
.add_device = qcom_iommu_add_device,
@@ -696,10 +725,8 @@ static int qcom_iommu_ctx_probe(struct platform_device *pdev)
return PTR_ERR(ctx->base);
irq = platform_get_irq(pdev, 0);
if (irq < 0) {
dev_err(dev, "failed to get irq\n");
if (irq < 0)
return -ENODEV;
}
/* clear IRQs before registering fault handler, just in case the
* boot-loader left us a surprise:
@@ -775,7 +802,7 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
struct qcom_iommu_dev *qcom_iommu;
struct device *dev = &pdev->dev;
struct resource *res;
int ret, sz, max_asid = 0;
int ret, max_asid = 0;
/* find the max asid (which is 1:1 to ctx bank idx), so we know how
* many child ctx devices we have:
@@ -783,9 +810,8 @@ static int qcom_iommu_device_probe(struct platform_device *pdev)
for_each_child_of_node(dev->of_node, child)
max_asid = max(max_asid, get_asid(child));
sz = sizeof(*qcom_iommu) + (max_asid * sizeof(qcom_iommu->ctxs[0]));
qcom_iommu = devm_kzalloc(dev, sz, GFP_KERNEL);
qcom_iommu = devm_kzalloc(dev, struct_size(qcom_iommu, ctxs, max_asid),
GFP_KERNEL);
if (!qcom_iommu)
return -ENOMEM;
qcom_iommu->num_ctxs = max_asid;

2
drivers/iommu/rockchip-iommu.c
View File

@@ -794,7 +794,7 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
}
static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
unsigned long flags;

3
drivers/iommu/s390-iommu.c
View File

@@ -314,7 +314,8 @@ static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain,
}
static size_t s390_iommu_unmap(struct iommu_domain *domain,
unsigned long iova, size_t size)
unsigned long iova, size_t size,
struct iommu_iotlb_gather *gather)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
int flags = ZPCI_PTE_INVALID;

12
drivers/iommu/tegra-gart.c
View File

@@ -207,7 +207,7 @@ static inline int __gart_iommu_unmap(struct gart_device *gart,
}
static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t bytes)
size_t bytes, struct iommu_iotlb_gather *gather)
{
struct gart_device *gart = gart_handle;
int err;
@@ -273,11 +273,17 @@ static int gart_iommu_of_xlate(struct device *dev,
return 0;
}
static void gart_iommu_sync(struct iommu_domain *domain)
static void gart_iommu_sync_map(struct iommu_domain *domain)
{
FLUSH_GART_REGS(gart_handle);
}
static void gart_iommu_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
gart_iommu_sync_map(domain);
}
static const struct iommu_ops gart_iommu_ops = {
.capable = gart_iommu_capable,
.domain_alloc = gart_iommu_domain_alloc,
@@ -292,7 +298,7 @@ static const struct iommu_ops gart_iommu_ops = {
.iova_to_phys = gart_iommu_iova_to_phys,
.pgsize_bitmap = GART_IOMMU_PGSIZES,
.of_xlate = gart_iommu_of_xlate,
.iotlb_sync_map = gart_iommu_sync,
.iotlb_sync_map = gart_iommu_sync_map,
.iotlb_sync = gart_iommu_sync,
};

2
drivers/iommu/tegra-smmu.c
View File

@@ -680,7 +680,7 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
dma_addr_t pte_dma;

5
drivers/iommu/virtio-iommu.c
View File

@@ -751,7 +751,7 @@ static int viommu_map(struct iommu_domain *domain, unsigned long iova,
}
static size_t viommu_unmap(struct iommu_domain *domain, unsigned long iova,
size_t size)
size_t size, struct iommu_iotlb_gather *gather)
{
int ret = 0;
size_t unmapped;
@@ -797,7 +797,8 @@ static phys_addr_t viommu_iova_to_phys(struct iommu_domain *domain,
return paddr;
}
static void viommu_iotlb_sync(struct iommu_domain *domain)
static void viommu_iotlb_sync(struct iommu_domain *domain,
struct iommu_iotlb_gather *gather)
{
struct viommu_domain *vdomain = to_viommu_domain(domain);