xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

iommu/arm-smmu: Defer TLB maintenance until after a buffer is unmapped

Browse Source 
TLB maintenance is currently performed while a buffer is being unmapped.
For large buffers, doing so is suboptimal, as opposed to invalidating
the entire TLB for a particular context after the buffer has been
unmapped.

Thus, defer TLB maintenance until a buffer is unmapped, and
iommu_iotlb_sync() is invoked. This shows a significant amount
of improvement in the latency incurred by unmapping a buffer.

Without this patch, we observe the following latencies:

(average over 10 iterations)
    size           iommu_map      iommu_unmap
      4K            1.265 us       651.619 us
     64K            7.666 us       678.968 us
      1M           90.979 us      1152.072 us
      2M          179.885 us      2303.020 us
     12M         1082.140 us      5537.349 us
     24M         2159.463 us      9415.588 us
     32M         2878.609 us     12001.406 us

    size        iommu_map_sg      iommu_unmap
      4K            1.088 us       647.921 us
     64K            7.208 us       680.312 us
      1M          103.505 us      1153.520 us
      2M          200.885 us      2302.593 us
     12M         1159.146 us      5534.989 us
     24M         2300.744 us      9411.614 us
     32M         3057.343 us     12000.468 us

While applying this patch yields the following latencies:

(average over 10 iterations)
    size           iommu_map      iommu_unmap
      4K            1.172 us         5.218 us
     64K            6.229 us         9.338 us
      1M           91.812 us        77.828 us
      2M          179.500 us       154.156 us
     12M         1077.927 us       154.572 us
     24M         2159.630 us       157.453 us
     32M         2883.953 us       157.921 us

    size        iommu_map_sg      iommu_unmap
      4K            1.041 us         5.005 us
     64K            6.781 us         9.364 us
      1M          102.390 us        79.515 us
      2M          200.328 us       152.270 us
     12M         1161.000 us       154.515 us
     24M         2304.369 us       157.822 us
     32M         3059.416 us       160.734 us.

Change-Id: I7aecf559746eb65d2543ce9b16ad12492eb70fa1
Signed-off-by: Isaac J. Manjarres <isaacm@codeaurora.org>
This commit is contained in:
Isaac J. Manjarres
2021-02-12 18:08:52 -08:00
parent 890731edea
commit c38ad44557
2 changed files with 1 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drivers/iommu/arm/arm-smmu/arm-smmu.c
View File

@@ -2423,7 +2423,7 @@ static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
arm_smmu_rpm_get(smmu); arm_smmu_rpm_get(smmu);
if (smmu->version == ARM_SMMU_V2 || if (smmu->version == ARM_SMMU_V2 ||
smmu_domain->stage == ARM_SMMU_DOMAIN_S1) smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
arm_smmu_tlb_sync_context(smmu_domain); arm_smmu_tlb_inv_context_s1(smmu_domain);
else else
arm_smmu_tlb_sync_global(smmu); arm_smmu_tlb_sync_global(smmu);
arm_smmu_rpm_put(smmu); arm_smmu_rpm_put(smmu);

12
drivers/iommu/qcom-io-pgtable-arm.c
View File

@@ -624,7 +624,6 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
tablep = iopte_deref(pte, data); tablep = iopte_deref(pte, data);
} else if (unmap_idx >= 0) { } else if (unmap_idx >= 0) {
io_pgtable_tlb_add_page(&data->iop, gather, iova, size);
return size; return size;
} }
@@ -653,9 +652,6 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
__arm_lpae_set_pte(ptep, 0, &iop->cfg); __arm_lpae_set_pte(ptep, 0, &iop->cfg);
if (!iopte_leaf(pte, lvl, iop->fmt)) { if (!iopte_leaf(pte, lvl, iop->fmt)) {
/* Also flush any partial walks */
io_pgtable_tlb_flush_walk(iop, iova, size,
ARM_LPAE_GRANULE(data));
ptep = iopte_deref(pte, data); ptep = iopte_deref(pte, data);
__arm_lpae_free_pgtable(data, lvl + 1, ptep); __arm_lpae_free_pgtable(data, lvl + 1, ptep);
} else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) { } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
@@ -665,8 +661,6 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
* has observed it before the TLBIALL can be issued. * has observed it before the TLBIALL can be issued.
*/ */
smp_wmb(); smp_wmb();
} else {
io_pgtable_tlb_add_page(iop, gather, iova, size);
} }
return size; return size;
@@ -679,15 +673,9 @@ static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
iopte_tblcnt_sub(ptep, 1); iopte_tblcnt_sub(ptep, 1);
if (!iopte_tblcnt(*ptep)) { if (!iopte_tblcnt(*ptep)) {
size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
/* no valid mappings left under this table. free it. */ /* no valid mappings left under this table. free it. */
__arm_lpae_set_pte(ptep, 0, &iop->cfg); __arm_lpae_set_pte(ptep, 0, &iop->cfg);
io_pgtable_tlb_flush_walk(iop, ALIGN_DOWN(iova, blk_size),
blk_size, ARM_LPAE_GRANULE(data));
__arm_lpae_free_pgtable(data, lvl + 1, table); __arm_lpae_free_pgtable(data, lvl + 1, table);
} else {
io_pgtable_tlb_add_page(iop, gather, iova, size);
} }
return size; return size;