From 4e3e9b6997b24383264031198bf8905b3746221e Mon Sep 17 00:00:00 2001 From: Tirumalesh Chalamarla Date: Tue, 23 Feb 2016 10:19:00 -0800 Subject: iommu/arm-smmu: Add support for 16 bit VMID This patch adds support for 16-bit VMIDs on implementations of SMMUv2 that support it. Signed-off-by: Tirumalesh Chalamarla [will: commit messsage and comments] Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 2409e3b..25e884a 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -94,6 +94,7 @@ #define sCR0_VMIDPNE (1 << 11) #define sCR0_PTM (1 << 12) #define sCR0_FB (1 << 13) +#define sCR0_VMID16EN (1 << 31) #define sCR0_BSU_SHIFT 14 #define sCR0_BSU_MASK 0x3 @@ -141,6 +142,7 @@ #define ID2_PTFS_4K (1 << 12) #define ID2_PTFS_16K (1 << 13) #define ID2_PTFS_64K (1 << 14) +#define ID2_VMID16 (1 << 15) /* Global TLB invalidation */ #define ARM_SMMU_GR0_TLBIVMID 0x64 @@ -193,6 +195,8 @@ #define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) #define CBA2R_RW64_32BIT (0 << 0) #define CBA2R_RW64_64BIT (1 << 0) +#define CBA2R_VMID_SHIFT 16 +#define CBA2R_VMID_MASK 0xffff /* Translation context bank */ #define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1)) @@ -305,6 +309,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) +#define ARM_SMMU_FEAT_VMID16 (1 << 6) u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) @@ -734,16 +739,15 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); if (smmu->version > ARM_SMMU_V1) { - /* - * CBA2R. - * *Must* be initialised before CBAR thanks to VMID16 - * architectural oversight affected some implementations. - */ #ifdef CONFIG_64BIT reg = CBA2R_RW64_64BIT; #else reg = CBA2R_RW64_32BIT; #endif + /* 16-bit VMIDs live in CBA2R */ + if (smmu->features & ARM_SMMU_FEAT_VMID16) + reg |= ARM_SMMU_CB_VMID(cfg) << CBA2R_VMID_SHIFT; + writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); } @@ -759,7 +763,8 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, if (stage1) { reg |= (CBAR_S1_BPSHCFG_NSH << CBAR_S1_BPSHCFG_SHIFT) | (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); - } else { + } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { + /* 8-bit VMIDs live in CBAR */ reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT; } writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); @@ -1529,6 +1534,9 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Don't upgrade barriers */ reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); + if (smmu->features & ARM_SMMU_FEAT_VMID16) + reg |= sCR0_VMID16EN; + /* Push the button */ __arm_smmu_tlb_sync(smmu); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); @@ -1679,6 +1687,9 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); smmu->pa_size = size; + if (id & ID2_VMID16) + smmu->features |= ARM_SMMU_FEAT_VMID16; + /* * What the page table walker can address actually depends on which * descriptor format is in use, but since a) we don't know that yet, -- cgit v0.10.2 From 1bd37a6835bef0ecd2138cb42f9088fd890f9939 Mon Sep 17 00:00:00 2001 From: Tirumalesh Chalamarla Date: Fri, 4 Mar 2016 13:56:09 -0800 Subject: iommu/arm-smmu: Workaround for ThunderX erratum #27704 Due to erratum #27704, the CN88xx SMMUv2 implementation supports only shared ASID and VMID numberspaces. This patch ensures that ASID and VMIDs are unique across all SMMU instances on affected Cavium systems. Signed-off-by: Tirumalesh Chalamarla Signed-off-by: Akula Geethasowjanya [will: commit message, comments and formatting] Signed-off-by: Will Deacon diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt index ba4b6ac..806f91c 100644 --- a/Documentation/arm64/silicon-errata.txt +++ b/Documentation/arm64/silicon-errata.txt @@ -57,3 +57,4 @@ stable kernels. | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | | Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 | | Cavium | ThunderX Core | #27456 | CAVIUM_ERRATUM_27456 | +| Cavium | ThunderX SMMUv2 | #27704 | N/A | diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt index 7180745..19fe6f2 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt @@ -16,6 +16,7 @@ conditions. "arm,mmu-400" "arm,mmu-401" "arm,mmu-500" + "cavium,smmu-v2" depending on the particular implementation and/or the version of the architecture implemented. diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 25e884a..e933679 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -334,6 +334,8 @@ struct arm_smmu_device { struct list_head list; struct rb_root masters; + + u32 cavium_id_base; /* Specific to Cavium */ }; struct arm_smmu_cfg { @@ -343,8 +345,8 @@ struct arm_smmu_cfg { }; #define INVALID_IRPTNDX 0xff -#define ARM_SMMU_CB_ASID(cfg) ((cfg)->cbndx) -#define ARM_SMMU_CB_VMID(cfg) ((cfg)->cbndx + 1) +#define ARM_SMMU_CB_ASID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx) +#define ARM_SMMU_CB_VMID(smmu, cfg) ((u16)(smmu)->cavium_id_base + (cfg)->cbndx + 1) enum arm_smmu_domain_stage { ARM_SMMU_DOMAIN_S1 = 0, @@ -372,6 +374,8 @@ struct arm_smmu_option_prop { const char *prop; }; +static atomic_t cavium_smmu_context_count = ATOMIC_INIT(0); + static struct arm_smmu_option_prop arm_smmu_options[] = { { ARM_SMMU_OPT_SECURE_CFG_ACCESS, "calxeda,smmu-secure-config-access" }, { 0, NULL}, @@ -583,11 +587,11 @@ static void arm_smmu_tlb_inv_context(void *cookie) if (stage1) { base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); - writel_relaxed(ARM_SMMU_CB_ASID(cfg), + writel_relaxed(ARM_SMMU_CB_ASID(smmu, cfg), base + ARM_SMMU_CB_S1_TLBIASID); } else { base = ARM_SMMU_GR0(smmu); - writel_relaxed(ARM_SMMU_CB_VMID(cfg), + writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), base + ARM_SMMU_GR0_TLBIVMID); } @@ -609,7 +613,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { iova &= ~12UL; - iova |= ARM_SMMU_CB_ASID(cfg); + iova |= ARM_SMMU_CB_ASID(smmu, cfg); do { writel_relaxed(iova, reg); iova += granule; @@ -617,7 +621,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, #ifdef CONFIG_64BIT } else { iova >>= 12; - iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48; + iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48; do { writeq_relaxed(iova, reg); iova += granule >> 12; @@ -637,7 +641,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, #endif } else { reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; - writel_relaxed(ARM_SMMU_CB_VMID(cfg), reg); + writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg); } } @@ -746,7 +750,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, #endif /* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) - reg |= ARM_SMMU_CB_VMID(cfg) << CBA2R_VMID_SHIFT; + reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT; writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); } @@ -765,7 +769,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { /* 8-bit VMIDs live in CBAR */ - reg |= ARM_SMMU_CB_VMID(cfg) << CBAR_VMID_SHIFT; + reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBAR_VMID_SHIFT; } writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(cfg->cbndx)); @@ -773,11 +777,11 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, if (stage1) { reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; - reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT; + reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0); reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; - reg64 |= ((u64)ARM_SMMU_CB_ASID(cfg)) << TTBRn_ASID_SHIFT; + reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1); } else { reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; @@ -1737,6 +1741,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 }, { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 }, { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 }, + { .compatible = "cavium,smmu-v2", .data = (void *)ARM_SMMU_V2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); @@ -1847,6 +1852,18 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } + /* + * Cavium CN88xx erratum #27704. + * Ensure ASID and VMID allocation is unique across all SMMUs in + * the system. + */ + if (of_device_is_compatible(dev->of_node, "cavium,smmu-v2")) { + smmu->cavium_id_base = + atomic_add_return(smmu->num_context_banks, + &cavium_smmu_context_count); + smmu->cavium_id_base -= smmu->num_context_banks; + } + INIT_LIST_HEAD(&smmu->list); spin_lock(&arm_smmu_devices_lock); list_add(&smmu->list, &arm_smmu_devices); -- cgit v0.10.2 From 67b65a3fb8e658d00ad1bb06e341f09b1f93a25c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:12:57 +0100 Subject: iommu/arm-smmu: Differentiate specific implementations As the inevitable reality of implementation-specific errata workarounds begin to accrue alongside our integration quirk handling, it's about time the driver had a decent way of keeping track. Extend the per-SMMU data so we can identify specific implementations in an efficient and firmware-agnostic manner. Acked-by: Tirumalesh Chalamarla Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index e933679..2d5f357 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -278,6 +278,10 @@ enum arm_smmu_arch_version { ARM_SMMU_V2, }; +enum arm_smmu_implementation { + GENERIC_SMMU, +}; + struct arm_smmu_smr { u8 idx; u16 mask; @@ -315,6 +319,7 @@ struct arm_smmu_device { #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) u32 options; enum arm_smmu_arch_version version; + enum arm_smmu_implementation model; u32 num_context_banks; u32 num_s2_context_banks; @@ -1735,13 +1740,24 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return 0; } +struct arm_smmu_match_data { + enum arm_smmu_arch_version version; + enum arm_smmu_implementation model; +}; + +#define ARM_SMMU_MATCH_DATA(name, ver, imp) \ +static struct arm_smmu_match_data name = { .version = ver, .model = imp } + +ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); + static const struct of_device_id arm_smmu_of_match[] = { - { .compatible = "arm,smmu-v1", .data = (void *)ARM_SMMU_V1 }, - { .compatible = "arm,smmu-v2", .data = (void *)ARM_SMMU_V2 }, - { .compatible = "arm,mmu-400", .data = (void *)ARM_SMMU_V1 }, - { .compatible = "arm,mmu-401", .data = (void *)ARM_SMMU_V1 }, - { .compatible = "arm,mmu-500", .data = (void *)ARM_SMMU_V2 }, - { .compatible = "cavium,smmu-v2", .data = (void *)ARM_SMMU_V2 }, + { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, + { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, + { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, + { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, + { .compatible = "arm,mmu-500", .data = &smmu_generic_v2 }, + { .compatible = "cavium,smmu-v2", .data = &smmu_generic_v2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); @@ -1749,6 +1765,7 @@ MODULE_DEVICE_TABLE(of, arm_smmu_of_match); static int arm_smmu_device_dt_probe(struct platform_device *pdev) { const struct of_device_id *of_id; + const struct arm_smmu_match_data *data; struct resource *res; struct arm_smmu_device *smmu; struct device *dev = &pdev->dev; @@ -1764,7 +1781,9 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) smmu->dev = dev; of_id = of_match_node(arm_smmu_of_match, dev->of_node); - smmu->version = (enum arm_smmu_arch_version)of_id->data; + data = of_id->data; + smmu->version = data->version; + smmu->model = data->model; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); smmu->base = devm_ioremap_resource(dev, res); -- cgit v0.10.2 From e086d912d4d78781652669618e7fb01a4d466703 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:12:58 +0100 Subject: iommu/arm-smmu: Convert ThunderX workaround to new method With a framework for implementation-specific funtionality in place, the currently-FDT-dependent ThunderX workaround gets to be the first user. Acked-by: Tirumalesh Chalamarla Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 2d5f357..d8bc20a 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -280,6 +280,7 @@ enum arm_smmu_arch_version { enum arm_smmu_implementation { GENERIC_SMMU, + CAVIUM_SMMUV2, }; struct arm_smmu_smr { @@ -1686,6 +1687,17 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) } dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", smmu->num_context_banks, smmu->num_s2_context_banks); + /* + * Cavium CN88xx erratum #27704. + * Ensure ASID and VMID allocation is unique across all SMMUs in + * the system. + */ + if (smmu->model == CAVIUM_SMMUV2) { + smmu->cavium_id_base = + atomic_add_return(smmu->num_context_banks, + &cavium_smmu_context_count); + smmu->cavium_id_base -= smmu->num_context_banks; + } /* ID2 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); @@ -1750,6 +1762,7 @@ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, @@ -1757,7 +1770,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-500", .data = &smmu_generic_v2 }, - { .compatible = "cavium,smmu-v2", .data = &smmu_generic_v2 }, + { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, }; MODULE_DEVICE_TABLE(of, arm_smmu_of_match); @@ -1871,18 +1884,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } } - /* - * Cavium CN88xx erratum #27704. - * Ensure ASID and VMID allocation is unique across all SMMUs in - * the system. - */ - if (of_device_is_compatible(dev->of_node, "cavium,smmu-v2")) { - smmu->cavium_id_base = - atomic_add_return(smmu->num_context_banks, - &cavium_smmu_context_count); - smmu->cavium_id_base -= smmu->num_context_banks; - } - INIT_LIST_HEAD(&smmu->list); spin_lock(&arm_smmu_devices_lock); list_add(&smmu->list, &arm_smmu_devices); -- cgit v0.10.2 From f0cfffc48cac516e37711786227f6808491913a5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:12:59 +0100 Subject: iommu/arm-smmu: Work around MMU-500 prefetch errata MMU-500 erratum #841119 is tickled by a particular set of circumstances interacting with the next-page prefetcher. Since said prefetcher is quite dumb and actually detrimental to performance in some cases (by causing unwanted TLB evictions for non-sequential access patterns), we lose very little by turning it off, and what we gain is a guarantee that the erratum is never hit. As a bonus, the same workaround will also prevent erratum #826419 once v7 short descriptor support is implemented. CC: Catalin Marinas CC: Will Deacon Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/Documentation/arm64/silicon-errata.txt b/Documentation/arm64/silicon-errata.txt index 806f91c..c6938e5 100644 --- a/Documentation/arm64/silicon-errata.txt +++ b/Documentation/arm64/silicon-errata.txt @@ -53,6 +53,7 @@ stable kernels. | ARM | Cortex-A57 | #832075 | ARM64_ERRATUM_832075 | | ARM | Cortex-A57 | #852523 | N/A | | ARM | Cortex-A57 | #834220 | ARM64_ERRATUM_834220 | +| ARM | MMU-500 | #841119,#826419 | N/A | | | | | | | Cavium | ThunderX ITS | #22375, #24313 | CAVIUM_ERRATUM_22375 | | Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 | diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index d8bc20a..085fc8d 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -203,6 +203,7 @@ #define ARM_SMMU_CB(smmu, n) ((n) * (1 << (smmu)->pgshift)) #define ARM_SMMU_CB_SCTLR 0x0 +#define ARM_SMMU_CB_ACTLR 0x4 #define ARM_SMMU_CB_RESUME 0x8 #define ARM_SMMU_CB_TTBCR2 0x10 #define ARM_SMMU_CB_TTBR0 0x20 @@ -234,6 +235,8 @@ #define SCTLR_M (1 << 0) #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE) +#define ARM_MMU500_ACTLR_CPRE (1 << 1) + #define CB_PAR_F (1 << 0) #define ATSR_ACTIVE (1 << 0) @@ -280,6 +283,7 @@ enum arm_smmu_arch_version { enum arm_smmu_implementation { GENERIC_SMMU, + ARM_MMU500, CAVIUM_SMMUV2, }; @@ -1517,6 +1521,15 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i); writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); writel_relaxed(FSR_FAULT, cb_base + ARM_SMMU_CB_FSR); + /* + * Disable MMU-500's not-particularly-beneficial next-page + * prefetcher for the sake of errata #841119 and #826419. + */ + if (smmu->model == ARM_MMU500) { + reg = readl_relaxed(cb_base + ARM_SMMU_CB_ACTLR); + reg &= ~ARM_MMU500_ACTLR_CPRE; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_ACTLR); + } } /* Invalidate the TLB, just in case */ @@ -1762,6 +1775,7 @@ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); static const struct of_device_id arm_smmu_of_match[] = { @@ -1769,7 +1783,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, - { .compatible = "arm,mmu-500", .data = &smmu_generic_v2 }, + { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, }; -- cgit v0.10.2 From e511267bc25e18926826e7cccdf7872bcbb4776a Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Apr 2016 11:38:20 +0100 Subject: io-64-nonatomic: Add relaxed accessor variants Whilst commit 9439eb3ab9d1 ("asm-generic: io: implement relaxed accessor macros as conditional wrappers") makes the *_relaxed forms of I/O accessors universally available to drivers, in cases where writeq() is implemented via the io-64-nonatomic helpers, writeq_relaxed() will end up falling back to writel() regardless of whether writel_relaxed() is available (identically for s/write/read/). Add corresponding relaxed forms of the nonatomic helpers to delegate to the equivalent 32-bit accessors as appropriate. We also need to fix io.h to avoid defining default relaxed variants if the basic accessors themselves don't exist. CC: Christoph Hellwig CC: Darren Hart CC: Hitoshi Mitake Acked-by: Arnd Bergmann Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index eed3bbe..002b81f 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -191,7 +191,7 @@ static inline void writeq(u64 value, volatile void __iomem *addr) #define readl_relaxed readl #endif -#ifndef readq_relaxed +#if defined(readq) && !defined(readq_relaxed) #define readq_relaxed readq #endif @@ -207,7 +207,7 @@ static inline void writeq(u64 value, volatile void __iomem *addr) #define writel_relaxed writel #endif -#ifndef writeq_relaxed +#if defined(writeq) && !defined(writeq_relaxed) #define writeq_relaxed writeq #endif diff --git a/include/linux/io-64-nonatomic-hi-lo.h b/include/linux/io-64-nonatomic-hi-lo.h index 11d7e84..defcc46 100644 --- a/include/linux/io-64-nonatomic-hi-lo.h +++ b/include/linux/io-64-nonatomic-hi-lo.h @@ -21,6 +21,23 @@ static inline void hi_lo_writeq(__u64 val, volatile void __iomem *addr) writel(val, addr); } +static inline __u64 hi_lo_readq_relaxed(const volatile void __iomem *addr) +{ + const volatile u32 __iomem *p = addr; + u32 low, high; + + high = readl_relaxed(p + 1); + low = readl_relaxed(p); + + return low + ((u64)high << 32); +} + +static inline void hi_lo_writeq_relaxed(__u64 val, volatile void __iomem *addr) +{ + writel_relaxed(val >> 32, addr + 4); + writel_relaxed(val, addr); +} + #ifndef readq #define readq hi_lo_readq #endif @@ -29,4 +46,12 @@ static inline void hi_lo_writeq(__u64 val, volatile void __iomem *addr) #define writeq hi_lo_writeq #endif +#ifndef readq_relaxed +#define readq_relaxed hi_lo_readq_relaxed +#endif + +#ifndef writeq_relaxed +#define writeq_relaxed hi_lo_writeq_relaxed +#endif + #endif /* _LINUX_IO_64_NONATOMIC_HI_LO_H_ */ diff --git a/include/linux/io-64-nonatomic-lo-hi.h b/include/linux/io-64-nonatomic-lo-hi.h index 1a4315f..084461a 100644 --- a/include/linux/io-64-nonatomic-lo-hi.h +++ b/include/linux/io-64-nonatomic-lo-hi.h @@ -21,6 +21,23 @@ static inline void lo_hi_writeq(__u64 val, volatile void __iomem *addr) writel(val >> 32, addr + 4); } +static inline __u64 lo_hi_readq_relaxed(const volatile void __iomem *addr) +{ + const volatile u32 __iomem *p = addr; + u32 low, high; + + low = readl_relaxed(p); + high = readl_relaxed(p + 1); + + return low + ((u64)high << 32); +} + +static inline void lo_hi_writeq_relaxed(__u64 val, volatile void __iomem *addr) +{ + writel_relaxed(val, addr); + writel_relaxed(val >> 32, addr + 4); +} + #ifndef readq #define readq lo_hi_readq #endif @@ -29,4 +46,12 @@ static inline void lo_hi_writeq(__u64 val, volatile void __iomem *addr) #define writeq lo_hi_writeq #endif +#ifndef readq_relaxed +#define readq_relaxed lo_hi_readq_relaxed +#endif + +#ifndef writeq_relaxed +#define writeq_relaxed lo_hi_writeq_relaxed +#endif + #endif /* _LINUX_IO_64_NONATOMIC_LO_HI_H_ */ -- cgit v0.10.2 From f9a05f05b12a42f2c52f3d3b8cc71fe2a6e60bce Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:13:01 +0100 Subject: iommu/arm-smmu: Tidy up 64-bit/atomic I/O accesses With {read,write}q_relaxed now able to fall back to the common nonatomic-hi-lo helper, make use of that so that we don't have to open-code our own. In the process, also convert the other remaining split accesses, and repurpose the custom accessor to smooth out the couple of troublesome instances where we really want to avoid nonatomic writes (and a 64-bit access is unnecessary in the 32-bit context formats we would use on a 32-bit CPU). This paves the way for getting rid of some of the assumptions currently baked into the driver which make it really awkward to use 32-bit context formats with SMMUv2 under a 64-bit kernel. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 085fc8d..acff332 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -71,16 +72,15 @@ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ ? 0x400 : 0)) +/* + * Some 64-bit registers only make sense to write atomically, but in such + * cases all the data relevant to AArch32 formats lies within the lower word, + * therefore this actually makes more sense than it might first appear. + */ #ifdef CONFIG_64BIT -#define smmu_writeq writeq_relaxed +#define smmu_write_atomic_lq writeq_relaxed #else -#define smmu_writeq(reg64, addr) \ - do { \ - u64 __val = (reg64); \ - void __iomem *__addr = (addr); \ - writel_relaxed(__val >> 32, __addr + 4); \ - writel_relaxed(__val, __addr); \ - } while (0) +#define smmu_write_atomic_lq writel_relaxed #endif /* Configuration registers */ @@ -211,11 +211,9 @@ #define ARM_SMMU_CB_TTBCR 0x30 #define ARM_SMMU_CB_S1_MAIR0 0x38 #define ARM_SMMU_CB_S1_MAIR1 0x3c -#define ARM_SMMU_CB_PAR_LO 0x50 -#define ARM_SMMU_CB_PAR_HI 0x54 +#define ARM_SMMU_CB_PAR 0x50 #define ARM_SMMU_CB_FSR 0x58 -#define ARM_SMMU_CB_FAR_LO 0x60 -#define ARM_SMMU_CB_FAR_HI 0x64 +#define ARM_SMMU_CB_FAR 0x60 #define ARM_SMMU_CB_FSYNR0 0x68 #define ARM_SMMU_CB_S1_TLBIVA 0x600 #define ARM_SMMU_CB_S1_TLBIASID 0x610 @@ -645,7 +643,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, ARM_SMMU_CB_S2_TLBIIPAS2; iova >>= 12; do { - writeq_relaxed(iova, reg); + smmu_write_atomic_lq(iova, reg); iova += granule >> 12; } while (size -= granule); #endif @@ -664,7 +662,7 @@ static struct iommu_gather_ops arm_smmu_gather_ops = { static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { int flags, ret; - u32 fsr, far, fsynr, resume; + u32 fsr, fsynr, resume; unsigned long iova; struct iommu_domain *domain = dev; struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); @@ -686,13 +684,7 @@ static irqreturn_t arm_smmu_context_fault(int irq, void *dev) fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; - far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO); - iova = far; -#ifdef CONFIG_64BIT - far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI); - iova |= ((unsigned long)far << 32); -#endif - + iova = readq_relaxed(cb_base + ARM_SMMU_CB_FAR); if (!report_iommu_fault(domain, smmu->dev, iova, flags)) { ret = IRQ_HANDLED; resume = RESUME_RETRY; @@ -788,14 +780,14 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; - smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0); + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); reg64 = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; reg64 |= ((u64)ARM_SMMU_CB_ASID(smmu, cfg)) << TTBRn_ASID_SHIFT; - smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR1); + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR1); } else { reg64 = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; - smmu_writeq(reg64, cb_base + ARM_SMMU_CB_TTBR0); + writeq_relaxed(reg64, cb_base + ARM_SMMU_CB_TTBR0); } /* TTBCR */ @@ -1263,8 +1255,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, /* ATS1 registers can only be written atomically */ va = iova & ~0xfffUL; if (smmu->version == ARM_SMMU_V2) - smmu_writeq(va, cb_base + ARM_SMMU_CB_ATS1PR); - else + smmu_write_atomic_lq(va, cb_base + ARM_SMMU_CB_ATS1PR); + else /* Register is only 32-bit in v1 */ writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, @@ -1275,9 +1267,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, return ops->iova_to_phys(ops, iova); } - phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO); - phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32; - + phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR); if (phys & CB_PAR_F) { dev_err(dev, "translation fault!\n"); dev_err(dev, "PAR = 0x%llx\n", phys); -- cgit v0.10.2 From 7602b8710645da48b2937f05fa41d627a0e73dad Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Thu, 28 Apr 2016 17:12:09 +0100 Subject: iommu/arm-smmu: Decouple context format from kernel config The way the driver currently forces an AArch32 or AArch64 context format based on the kernel config and SMMU architecture version is suboptimal, in that it makes it very hard to support oddball mix-and-match cases like the SMMUv1 64KB supplement, or situations where the reduced table depth of an AArch32 short descriptor context may be desirable under an AArch64 kernel. It also only happens to work on current implementations which do support all the relevant formats. Introduce an explicit notion of context format, so we can manage that independently and get rid of the inflexible #ifdeffery. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index acff332..f2ded69 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -117,6 +117,8 @@ #define ID0_NTS (1 << 28) #define ID0_SMS (1 << 27) #define ID0_ATOSNS (1 << 26) +#define ID0_PTFS_NO_AARCH32 (1 << 25) +#define ID0_PTFS_NO_AARCH32S (1 << 24) #define ID0_CTTW (1 << 14) #define ID0_NUMIRPT_SHIFT 16 #define ID0_NUMIRPT_MASK 0xff @@ -317,6 +319,11 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) #define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) #define ARM_SMMU_FEAT_VMID16 (1 << 6) +#define ARM_SMMU_FEAT_FMT_AARCH64_4K (1 << 7) +#define ARM_SMMU_FEAT_FMT_AARCH64_16K (1 << 8) +#define ARM_SMMU_FEAT_FMT_AARCH64_64K (1 << 9) +#define ARM_SMMU_FEAT_FMT_AARCH32_L (1 << 10) +#define ARM_SMMU_FEAT_FMT_AARCH32_S (1 << 11) u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) @@ -346,10 +353,18 @@ struct arm_smmu_device { u32 cavium_id_base; /* Specific to Cavium */ }; +enum arm_smmu_context_fmt { + ARM_SMMU_CTX_FMT_NONE, + ARM_SMMU_CTX_FMT_AARCH64, + ARM_SMMU_CTX_FMT_AARCH32_L, + ARM_SMMU_CTX_FMT_AARCH32_S, +}; + struct arm_smmu_cfg { u8 cbndx; u8 irptndx; u32 cbar; + enum arm_smmu_context_fmt fmt; }; #define INVALID_IRPTNDX 0xff @@ -619,14 +634,13 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; - if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { + if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { iova &= ~12UL; iova |= ARM_SMMU_CB_ASID(smmu, cfg); do { writel_relaxed(iova, reg); iova += granule; } while (size -= granule); -#ifdef CONFIG_64BIT } else { iova >>= 12; iova |= (u64)ARM_SMMU_CB_ASID(smmu, cfg) << 48; @@ -634,9 +648,7 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, writeq_relaxed(iova, reg); iova += granule >> 12; } while (size -= granule); -#endif } -#ifdef CONFIG_64BIT } else if (smmu->version == ARM_SMMU_V2) { reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : @@ -646,7 +658,6 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, smmu_write_atomic_lq(iova, reg); iova += granule >> 12; } while (size -= granule); -#endif } else { reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; writel_relaxed(ARM_SMMU_CB_VMID(smmu, cfg), reg); @@ -745,11 +756,10 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); if (smmu->version > ARM_SMMU_V1) { -#ifdef CONFIG_64BIT - reg = CBA2R_RW64_64BIT; -#else - reg = CBA2R_RW64_32BIT; -#endif + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) + reg = CBA2R_RW64_64BIT; + else + reg = CBA2R_RW64_32BIT; /* 16-bit VMIDs live in CBA2R */ if (smmu->features & ARM_SMMU_FEAT_VMID16) reg |= ARM_SMMU_CB_VMID(smmu, cfg) << CBA2R_VMID_SHIFT; @@ -860,16 +870,40 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) smmu_domain->stage = ARM_SMMU_DOMAIN_S1; + /* + * Choosing a suitable context format is even more fiddly. Until we + * grow some way for the caller to express a preference, and/or move + * the decision into the io-pgtable code where it arguably belongs, + * just aim for the closest thing to the rest of the system, and hope + * that the hardware isn't esoteric enough that we can't assume AArch64 + * support to be a superset of AArch32 support... + */ + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) + cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; + if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && + (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | + ARM_SMMU_FEAT_FMT_AARCH64_16K | + ARM_SMMU_FEAT_FMT_AARCH64_4K))) + cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; + + if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { + ret = -EINVAL; + goto out_unlock; + } + switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; start = smmu->num_s2_context_banks; ias = smmu->va_size; oas = smmu->ipa_size; - if (IS_ENABLED(CONFIG_64BIT)) + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S1; - else + } else { fmt = ARM_32_LPAE_S1; + ias = min(ias, 32UL); + oas = min(oas, 40UL); + } break; case ARM_SMMU_DOMAIN_NESTED: /* @@ -881,10 +915,13 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, start = 0; ias = smmu->ipa_size; oas = smmu->pa_size; - if (IS_ENABLED(CONFIG_64BIT)) + if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { fmt = ARM_64_LPAE_S2; - else + } else { fmt = ARM_32_LPAE_S2; + ias = min(ias, 40UL); + oas = min(oas, 40UL); + } break; default: ret = -EINVAL; @@ -1670,6 +1707,12 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) ID0_NUMSIDB_MASK; } + if (smmu->version < ARM_SMMU_V2 || !(id & ID0_PTFS_NO_AARCH32)) { + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; + if (!(id & ID0_PTFS_NO_AARCH32S)) + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; + } + /* ID1 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; @@ -1725,22 +1768,29 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) if (smmu->version == ARM_SMMU_V1) { smmu->va_size = smmu->ipa_size; - size = SZ_4K | SZ_2M | SZ_1G; } else { size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; smmu->va_size = arm_smmu_id_size_to_bits(size); -#ifndef CONFIG_64BIT - smmu->va_size = min(32UL, smmu->va_size); -#endif - size = 0; if (id & ID2_PTFS_4K) - size |= SZ_4K | SZ_2M | SZ_1G; + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; if (id & ID2_PTFS_16K) - size |= SZ_16K | SZ_32M; + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; if (id & ID2_PTFS_64K) - size |= SZ_64K | SZ_512M; + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } + /* Now we've corralled the various formats, what'll it do? */ + size = 0; + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) + size |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; + if (smmu->features & + (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) + size |= SZ_4K | SZ_2M | SZ_1G; + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) + size |= SZ_16K | SZ_32M; + if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) + size |= SZ_64K | SZ_512M; + arm_smmu_ops.pgsize_bitmap &= size; dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size); -- cgit v0.10.2 From b7862e3559f9ab4aaa258dcb846986601a7ca0b8 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 13 Apr 2016 18:13:03 +0100 Subject: iommu/arm-smmu: Support SMMUv1 64KB supplement The 64KB Translation Granule Supplement to the SMMUv1 architecture allows an SMMUv1 implementation to support 64KB pages for stage 2 translations, using a constrained VMSAv8 descriptor format limited to 40-bit addresses. Now that we can freely mix and match context formats, we can actually handle having 4KB pages via an AArch32 context but 64KB pages via an AArch64 context, so plumb it in. It is assumed that any implementations will have hardware capabilities matching the format constraints, thus obviating the need for excessive sanity-checking; this is the case for MMU-401, the only ARM Ltd. implementation. CC: Eric Auger Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index f2ded69..72dea31 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -277,7 +277,8 @@ MODULE_PARM_DESC(disable_bypass, "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); enum arm_smmu_arch_version { - ARM_SMMU_V1 = 1, + ARM_SMMU_V1, + ARM_SMMU_V1_64K, ARM_SMMU_V2, }; @@ -769,7 +770,7 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, /* CBAR */ reg = cfg->cbar; - if (smmu->version == ARM_SMMU_V1) + if (smmu->version < ARM_SMMU_V2) reg |= cfg->irptndx << CBAR_IRPTNDX_SHIFT; /* @@ -934,7 +935,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, goto out_unlock; cfg->cbndx = ret; - if (smmu->version == ARM_SMMU_V1) { + if (smmu->version < ARM_SMMU_V2) { cfg->irptndx = atomic_inc_return(&smmu->irptndx); cfg->irptndx %= smmu->num_context_irqs; } else { @@ -1619,7 +1620,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) bool cttw_dt, cttw_reg; dev_notice(smmu->dev, "probing hardware configuration...\n"); - dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); + dev_notice(smmu->dev, "SMMUv%d with:\n", + smmu->version == ARM_SMMU_V2 ? 2 : 1); /* ID0 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); @@ -1651,7 +1653,8 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return -ENODEV; } - if ((id & ID0_S1TS) && ((smmu->version == 1) || !(id & ID0_ATOSNS))) { + if ((id & ID0_S1TS) && + ((smmu->version < ARM_SMMU_V2) || !(id & ID0_ATOSNS))) { smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; dev_notice(smmu->dev, "\taddress translation ops\n"); } @@ -1766,8 +1769,10 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "failed to set DMA mask for table walker\n"); - if (smmu->version == ARM_SMMU_V1) { + if (smmu->version < ARM_SMMU_V2) { smmu->va_size = smmu->ipa_size; + if (smmu->version == ARM_SMMU_V1_64K) + smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; } else { size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; smmu->va_size = arm_smmu_id_size_to_bits(size); @@ -1815,6 +1820,7 @@ static struct arm_smmu_match_data name = { .version = ver, .model = imp } ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); +ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); @@ -1822,7 +1828,7 @@ static const struct of_device_id arm_smmu_of_match[] = { { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, - { .compatible = "arm,mmu-401", .data = &smmu_generic_v1 }, + { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, { }, @@ -1916,7 +1922,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) parse_driver_options(smmu); - if (smmu->version > ARM_SMMU_V1 && + if (smmu->version == ARM_SMMU_V2 && smmu->num_context_banks != smmu->num_context_irqs) { dev_err(dev, "found only %d context interrupt(s) but %d required\n", -- cgit v0.10.2 From 3ca3712a42f9e632eb41da94ca4eab4f1fb06fcb Mon Sep 17 00:00:00 2001 From: Peng Fan Date: Tue, 3 May 2016 21:50:30 +0800 Subject: iommu/arm-smmu: Clear cache lock bit of ACR According MMU-500r2 TRM, section 3.7.1 Auxiliary Control registers, You can modify ACTLR only when the ACR.CACHE_LOCK bit is 0. So before clearing ARM_MMU500_ACTLR_CPRE of each context bank, need clear CACHE_LOCK bit of ACR register first. Since CACHE_LOCK bit is only present in MMU-500r2 onwards, need to check the major number of IDR7. Reviewed-by: Robin Murphy Signed-off-by: Peng Fan Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 72dea31..2fb9c33 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -98,6 +98,9 @@ #define sCR0_BSU_SHIFT 14 #define sCR0_BSU_MASK 0x3 +/* Auxiliary Configuration register */ +#define ARM_SMMU_GR0_sACR 0x10 + /* Identification registers */ #define ARM_SMMU_GR0_ID0 0x20 #define ARM_SMMU_GR0_ID1 0x24 @@ -146,6 +149,9 @@ #define ID2_PTFS_64K (1 << 14) #define ID2_VMID16 (1 << 15) +#define ID7_MAJOR_SHIFT 4 +#define ID7_MAJOR_MASK 0xf + /* Global TLB invalidation */ #define ARM_SMMU_GR0_TLBIVMID 0x64 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68 @@ -237,6 +243,8 @@ #define ARM_MMU500_ACTLR_CPRE (1 << 1) +#define ARM_MMU500_ACR_CACHE_LOCK (1 << 26) + #define CB_PAR_F (1 << 0) #define ATSR_ACTIVE (1 << 0) @@ -1531,7 +1539,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) void __iomem *gr0_base = ARM_SMMU_GR0(smmu); void __iomem *cb_base; int i = 0; - u32 reg; + u32 reg, major; /* clear global FSR */ reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); @@ -1544,6 +1552,19 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i)); } + /* + * Before clearing ARM_MMU500_ACTLR_CPRE, need to + * clear CACHE_LOCK bit of ACR first. And, CACHE_LOCK + * bit is only present in MMU-500r2 onwards. + */ + reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID7); + major = (reg >> ID7_MAJOR_SHIFT) & ID7_MAJOR_MASK; + if ((smmu->model == ARM_MMU500) && (major >= 2)) { + reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sACR); + reg &= ~ARM_MMU500_ACR_CACHE_LOCK; + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_sACR); + } + /* Make sure all context banks are disabled and clear CB_FSR */ for (i = 0; i < smmu->num_context_banks; ++i) { cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, i); -- cgit v0.10.2