From ccd6385dfbb7b2f2e6670b5cfc55bb7ec0aa3839 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 15 Jul 2015 11:55:18 +0100 Subject: iommu/arm-smmu: Fix enabling of PRIQ interrupt When an ARM SMMUv3 instance supports PRI, the driver registers an interrupt handler, but fails to enable the generation of such interrupt at the SMMU level. This patches simply moves the enable flags to a variable that gets updated by the PRI handling code before being written to the SMMU register. Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index da902ba..5d2cbda 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -118,6 +118,7 @@ #define ARM_SMMU_IRQ_CTRL 0x50 #define IRQ_CTRL_EVTQ_IRQEN (1 << 2) +#define IRQ_CTRL_PRIQ_IRQEN (1 << 1) #define IRQ_CTRL_GERROR_IRQEN (1 << 0) #define ARM_SMMU_IRQ_CTRLACK 0x54 @@ -2198,6 +2199,7 @@ static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) { int ret, irq; + u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; /* Disable IRQs first */ ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, @@ -2252,13 +2254,13 @@ static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) if (IS_ERR_VALUE(ret)) dev_warn(smmu->dev, "failed to enable priq irq\n"); + else + irqen_flags |= IRQ_CTRL_PRIQ_IRQEN; } } /* Enable interrupt generation on the SMMU */ - ret = arm_smmu_write_reg_sync(smmu, - IRQ_CTRL_EVTQ_IRQEN | - IRQ_CTRL_GERROR_IRQEN, + ret = arm_smmu_write_reg_sync(smmu, irqen_flags, ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); if (ret) dev_warn(smmu->dev, "failed to enable irqs\n"); -- cgit v0.10.2 From ec11d63c677bbba15e65a35f5ba06c1d6eba4dbe Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 15 Jul 2015 11:55:19 +0100 Subject: iommu/arm-smmu: Fix MSI memory attributes to match specification The MSI memory attributes in the SMMUv3 driver are from an older revision of the spec, which doesn't match the current implementations. Out with the old, in with the new. Signed-off-by: Marc Zyngier Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 5d2cbda..c2c1ad8 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -174,14 +174,14 @@ #define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc /* Common MSI config fields */ -#define MSI_CFG0_SH_SHIFT 60 -#define MSI_CFG0_SH_NSH (0UL << MSI_CFG0_SH_SHIFT) -#define MSI_CFG0_SH_OSH (2UL << MSI_CFG0_SH_SHIFT) -#define MSI_CFG0_SH_ISH (3UL << MSI_CFG0_SH_SHIFT) -#define MSI_CFG0_MEMATTR_SHIFT 56 -#define MSI_CFG0_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG0_MEMATTR_SHIFT) #define MSI_CFG0_ADDR_SHIFT 2 #define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL +#define MSI_CFG2_SH_SHIFT 4 +#define MSI_CFG2_SH_NSH (0UL << MSI_CFG2_SH_SHIFT) +#define MSI_CFG2_SH_OSH (2UL << MSI_CFG2_SH_SHIFT) +#define MSI_CFG2_SH_ISH (3UL << MSI_CFG2_SH_SHIFT) +#define MSI_CFG2_MEMATTR_SHIFT 0 +#define MSI_CFG2_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG2_MEMATTR_SHIFT) #define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1)) #define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift)) -- cgit v0.10.2 From 28c8b4045b18b013e05656b493ce9a57cbf1f09a Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 16 Jul 2015 17:50:12 +0100 Subject: iommu/arm-smmu: Limit 2-level strtab allocation for small SID sizes If the StreamIDs in a system can all be resolved by a single level-2 stream table (i.e. SIDSIZE < SPLIT), then we currently get our maths wrong and allocate the largest strtab we support, thanks to unsigned overflow in our calculation. This patch fixes the issue by checking the SIDSIZE explicitly when calculating the size of our first-level stream table. Reported-by: Matt Evans Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index c2c1ad8..4f09337 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2054,9 +2054,17 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) int ret; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - /* Calculate the L1 size, capped to the SIDSIZE */ - size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); - size = min(size, smmu->sid_bits - STRTAB_SPLIT); + /* + * If we can resolve everything with a single L2 table, then we + * just need a single L1 descriptor. Otherwise, calculate the L1 + * size, capped to the SIDSIZE. + */ + if (smmu->sid_bits < STRTAB_SPLIT) { + size = 0; + } else { + size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); + size = min(size, smmu->sid_bits - STRTAB_SPLIT); + } cfg->num_l1_ents = 1 << size; size += STRTAB_SPLIT; -- cgit v0.10.2 From bae2c2d421cdea9dd8d62425eef99e389584cdb3 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:05 +0100 Subject: iommu/arm-smmu: Sort out coherency Currently, we detect whether the SMMU has coherent page table walk capability from the IDR0.CTTW field, and base our cache maintenance decisions on that. In preparation for fixing the bogus DMA API usage, however, we need to ensure that the DMA API agrees about this, which necessitates deferring to the dma-coherent property in the device tree for the final say. As an added bonus, since systems exist where an external CTTW signal has been tied off incorrectly at integration, allowing DT to override it offers a neat workaround for coherency issues with such SMMUs. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu.txt b/Documentation/devicetree/bindings/iommu/arm,smmu.txt index 0676050..7180745 100644 --- a/Documentation/devicetree/bindings/iommu/arm,smmu.txt +++ b/Documentation/devicetree/bindings/iommu/arm,smmu.txt @@ -43,6 +43,12 @@ conditions. ** System MMU optional properties: +- dma-coherent : Present if page table walks made by the SMMU are + cache coherent with the CPU. + + NOTE: this only applies to the SMMU itself, not + masters connected upstream of the SMMU. + - calxeda,smmu-secure-config-access : Enable proper handling of buggy implementations that always use secure access to SMMU configuration registers. In this case non-secure diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 4cd0c29..0583ed2 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -1532,6 +1533,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) unsigned long size; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); u32 id; + bool cttw_dt, cttw_reg; dev_notice(smmu->dev, "probing hardware configuration...\n"); dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); @@ -1571,10 +1573,22 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) dev_notice(smmu->dev, "\taddress translation ops\n"); } - if (id & ID0_CTTW) { + /* + * In order for DMA API calls to work properly, we must defer to what + * the DT says about coherency, regardless of what the hardware claims. + * Fortunately, this also opens up a workaround for systems where the + * ID register value has ended up configured incorrectly. + */ + cttw_dt = of_dma_is_coherent(smmu->dev->of_node); + cttw_reg = !!(id & ID0_CTTW); + if (cttw_dt) smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; - dev_notice(smmu->dev, "\tcoherent table walk\n"); - } + if (cttw_dt || cttw_reg) + dev_notice(smmu->dev, "\t%scoherent table walk\n", + cttw_dt ? "" : "non-"); + if (cttw_dt != cttw_reg) + dev_notice(smmu->dev, + "\t(IDR0.CTTW overridden by dma-coherent property)\n"); if (id & ID0_SMS) { u32 smr, sid, mask; -- cgit v0.10.2 From a73622a70a31c846d9d2a971c78fd1ffab88afcd Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:23 -0500 Subject: Documentation: dt: Add #iommu-cells info to OMAP iommu bindings The OMAP IOMMU bindings is updated to reflect the required #iommu-cells property. Signed-off-by: Suman Anna Reviewed-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt b/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt index 42531dc..8696999 100644 --- a/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt +++ b/Documentation/devicetree/bindings/iommu/ti,omap-iommu.txt @@ -8,6 +8,11 @@ Required properties: - ti,hwmods : Name of the hwmod associated with the IOMMU instance - reg : Address space for the configuration registers - interrupts : Interrupt specifier for the IOMMU instance +- #iommu-cells : Should be 0. OMAP IOMMUs are all "single-master" devices, + and needs no additional data in the pargs specifier. Please + also refer to the generic bindings document for more info + on this property, + Documentation/devicetree/bindings/iommu/iommu.txt Optional properties: - ti,#tlb-entries : Number of entries in the translation look-aside buffer. @@ -18,6 +23,7 @@ Optional properties: Example: /* OMAP3 ISP MMU */ mmu_isp: mmu@480bd400 { + #iommu-cells = <0>; compatible = "ti,omap2-iommu"; reg = <0x480bd400 0x80>; interrupts = <24>; -- cgit v0.10.2 From 0cdbf727167a2fcc9ba2aaea98e2a76124ba072e Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:24 -0500 Subject: iommu/omap: Remove all module references The OMAP IOMMU driver has been adapted to the IOMMU framework for a while now, and it does not support being built as a module anymore. So, remove all the module references from the OMAP IOMMU driver. While at it, also relocate a comment around the subsys_initcall to avoid a checkpatch strict warning about using a blank line after function/struct/union/enum declarations. Signed-off-by: Suman Anna Reviewed-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index a22c33d..eeecfc4 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -12,7 +12,6 @@ */ #include -#include #include #include #include @@ -1089,7 +1088,6 @@ static const struct of_device_id omap_iommu_of_match[] = { { .compatible = "ti,dra7-iommu" }, {}, }; -MODULE_DEVICE_TABLE(of, omap_iommu_of_match); static struct platform_driver omap_iommu_driver = { .probe = omap_iommu_probe, @@ -1405,20 +1403,5 @@ static int __init omap_iommu_init(void) return platform_driver_register(&omap_iommu_driver); } -/* must be ready before omap3isp is probed */ subsys_initcall(omap_iommu_init); - -static void __exit omap_iommu_exit(void) -{ - kmem_cache_destroy(iopte_cachep); - - platform_driver_unregister(&omap_iommu_driver); - - omap_iommu_debugfs_exit(); -} -module_exit(omap_iommu_exit); - -MODULE_DESCRIPTION("omap iommu: tlb and pagetable primitives"); -MODULE_ALIAS("platform:omap-iommu"); -MODULE_AUTHOR("Hiroshi DOYU, Paul Mundt and Toshihiro Kobayashi"); -MODULE_LICENSE("GPL v2"); +/* must be ready before omap3isp is probed */ -- cgit v0.10.2 From 69c2c196328e73d3091dd0be89ab4b0c2af4b210 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:25 -0500 Subject: iommu/omap: Move debugfs functions to omap-iommu-debug.c The main OMAP IOMMU driver file has some helper functions used by the OMAP IOMMU debugfs functionality, and there is already a dedicated source file omap-iommu-debug.c dealing with these debugfs routines. Move all these functions to the omap-iommu-debug.c file, so that all the debugfs related routines are in one place. The move required exposing some new functions and moving some definitions to the internal omap-iommu.h header file. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index f3d20a2..b4b96db 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -29,6 +30,59 @@ static inline bool is_omap_iommu_detached(struct omap_iommu *obj) return !obj->domain; } +#define pr_reg(name) \ + do { \ + ssize_t bytes; \ + const char *str = "%20s: %08x\n"; \ + const int maxcol = 32; \ + bytes = snprintf(p, maxcol, str, __stringify(name), \ + iommu_read_reg(obj, MMU_##name)); \ + p += bytes; \ + len -= bytes; \ + if (len < maxcol) \ + goto out; \ + } while (0) + +static ssize_t +omap2_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len) +{ + char *p = buf; + + pr_reg(REVISION); + pr_reg(IRQSTATUS); + pr_reg(IRQENABLE); + pr_reg(WALKING_ST); + pr_reg(CNTL); + pr_reg(FAULT_AD); + pr_reg(TTB); + pr_reg(LOCK); + pr_reg(LD_TLB); + pr_reg(CAM); + pr_reg(RAM); + pr_reg(GFLUSH); + pr_reg(FLUSH_ENTRY); + pr_reg(READ_CAM); + pr_reg(READ_RAM); + pr_reg(EMU_FAULT_AD); +out: + return p - buf; +} + +static ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, + ssize_t bytes) +{ + if (!obj || !buf) + return -EINVAL; + + pm_runtime_get_sync(obj->dev); + + bytes = omap2_iommu_dump_ctx(obj, buf, bytes); + + pm_runtime_put_sync(obj->dev); + + return bytes; +} + static ssize_t debug_read_regs(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { @@ -55,6 +109,63 @@ static ssize_t debug_read_regs(struct file *file, char __user *userbuf, return bytes; } +static int +__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num) +{ + int i; + struct iotlb_lock saved; + struct cr_regs tmp; + struct cr_regs *p = crs; + + pm_runtime_get_sync(obj->dev); + iotlb_lock_get(obj, &saved); + + for_each_iotlb_cr(obj, num, i, tmp) { + if (!iotlb_cr_valid(&tmp)) + continue; + *p++ = tmp; + } + + iotlb_lock_set(obj, &saved); + pm_runtime_put_sync(obj->dev); + + return p - crs; +} + +static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, + char *buf) +{ + char *p = buf; + + /* FIXME: Need more detail analysis of cam/ram */ + p += sprintf(p, "%08x %08x %01x\n", cr->cam, cr->ram, + (cr->cam & MMU_CAM_P) ? 1 : 0); + + return p - buf; +} + +static size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, + ssize_t bytes) +{ + int i, num; + struct cr_regs *cr; + char *p = buf; + + num = bytes / sizeof(*cr); + num = min(obj->nr_tlb_entries, num); + + cr = kcalloc(num, sizeof(*cr), GFP_KERNEL); + if (!cr) + return 0; + + num = __dump_tlb_entries(obj, cr, num); + for (i = 0; i < num; i++) + p += iotlb_dump_cr(obj, cr + i, p); + kfree(cr); + + return p - buf; +} + static ssize_t debug_read_tlb(struct file *file, char __user *userbuf, size_t count, loff_t *ppos) { diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index eeecfc4..0fc00f3 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -37,11 +37,6 @@ #define to_iommu(dev) \ ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) -#define for_each_iotlb_cr(obj, n, __i, cr) \ - for (__i = 0; \ - (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ - __i++) - /* bitmap of the page sizes currently supported */ #define OMAP_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) @@ -71,11 +66,6 @@ struct omap_iommu_domain { #define MMU_LOCK_VICT(x) \ ((x & MMU_LOCK_VICT_MASK) >> MMU_LOCK_VICT_SHIFT) -struct iotlb_lock { - short base; - short vict; -}; - static struct platform_driver omap_iommu_driver; static struct kmem_cache *iopte_cachep; @@ -212,14 +202,6 @@ static void iommu_disable(struct omap_iommu *obj) /* * TLB operations */ -static inline int iotlb_cr_valid(struct cr_regs *cr) -{ - if (!cr) - return -EINVAL; - - return cr->cam & MMU_CAM_V; -} - static u32 iotlb_cr_to_virt(struct cr_regs *cr) { u32 page_size = cr->cam & MMU_CAM_PGSZ_MASK; @@ -259,7 +241,7 @@ static u32 iommu_report_fault(struct omap_iommu *obj, u32 *da) return status; } -static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) +void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) { u32 val; @@ -267,10 +249,9 @@ static void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l) l->base = MMU_LOCK_BASE(val); l->vict = MMU_LOCK_VICT(val); - } -static void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) +void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l) { u32 val; @@ -296,7 +277,7 @@ static void iotlb_load_cr(struct omap_iommu *obj, struct cr_regs *cr) } /* only used in iotlb iteration for-loop */ -static struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) +struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n) { struct cr_regs cr; struct iotlb_lock l; @@ -467,129 +448,6 @@ static void flush_iotlb_all(struct omap_iommu *obj) pm_runtime_put_sync(obj->dev); } -#ifdef CONFIG_OMAP_IOMMU_DEBUG - -#define pr_reg(name) \ - do { \ - ssize_t bytes; \ - const char *str = "%20s: %08x\n"; \ - const int maxcol = 32; \ - bytes = snprintf(p, maxcol, str, __stringify(name), \ - iommu_read_reg(obj, MMU_##name)); \ - p += bytes; \ - len -= bytes; \ - if (len < maxcol) \ - goto out; \ - } while (0) - -static ssize_t -omap2_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len) -{ - char *p = buf; - - pr_reg(REVISION); - pr_reg(IRQSTATUS); - pr_reg(IRQENABLE); - pr_reg(WALKING_ST); - pr_reg(CNTL); - pr_reg(FAULT_AD); - pr_reg(TTB); - pr_reg(LOCK); - pr_reg(LD_TLB); - pr_reg(CAM); - pr_reg(RAM); - pr_reg(GFLUSH); - pr_reg(FLUSH_ENTRY); - pr_reg(READ_CAM); - pr_reg(READ_RAM); - pr_reg(EMU_FAULT_AD); -out: - return p - buf; -} - -ssize_t omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t bytes) -{ - if (!obj || !buf) - return -EINVAL; - - pm_runtime_get_sync(obj->dev); - - bytes = omap2_iommu_dump_ctx(obj, buf, bytes); - - pm_runtime_put_sync(obj->dev); - - return bytes; -} - -static int -__dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num) -{ - int i; - struct iotlb_lock saved; - struct cr_regs tmp; - struct cr_regs *p = crs; - - pm_runtime_get_sync(obj->dev); - iotlb_lock_get(obj, &saved); - - for_each_iotlb_cr(obj, num, i, tmp) { - if (!iotlb_cr_valid(&tmp)) - continue; - *p++ = tmp; - } - - iotlb_lock_set(obj, &saved); - pm_runtime_put_sync(obj->dev); - - return p - crs; -} - -/** - * iotlb_dump_cr - Dump an iommu tlb entry into buf - * @obj: target iommu - * @cr: contents of cam and ram register - * @buf: output buffer - **/ -static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, - char *buf) -{ - char *p = buf; - - /* FIXME: Need more detail analysis of cam/ram */ - p += sprintf(p, "%08x %08x %01x\n", cr->cam, cr->ram, - (cr->cam & MMU_CAM_P) ? 1 : 0); - - return p - buf; -} - -/** - * omap_dump_tlb_entries - dump cr arrays to given buffer - * @obj: target iommu - * @buf: output buffer - **/ -size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t bytes) -{ - int i, num; - struct cr_regs *cr; - char *p = buf; - - num = bytes / sizeof(*cr); - num = min(obj->nr_tlb_entries, num); - - cr = kcalloc(num, sizeof(*cr), GFP_KERNEL); - if (!cr) - return 0; - - num = __dump_tlb_entries(obj, cr, num); - for (i = 0; i < num; i++) - p += iotlb_dump_cr(obj, cr + i, p); - kfree(cr); - - return p - buf; -} - -#endif /* CONFIG_OMAP_IOMMU_DEBUG */ - /* * H/W pagetable operations */ diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h index d736630..b6cc90b 100644 --- a/drivers/iommu/omap-iommu.h +++ b/drivers/iommu/omap-iommu.h @@ -13,6 +13,11 @@ #ifndef _OMAP_IOMMU_H #define _OMAP_IOMMU_H +#define for_each_iotlb_cr(obj, n, __i, cr) \ + for (__i = 0; \ + (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ + __i++) + struct iotlb_entry { u32 da; u32 pa; @@ -65,6 +70,11 @@ struct cr_regs { }; }; +struct iotlb_lock { + short base; + short vict; +}; + /** * dev_to_omap_iommu() - retrieves an omap iommu object from a user device * @dev: iommu client device @@ -190,12 +200,12 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev) /* * global functions */ -#ifdef CONFIG_OMAP_IOMMU_DEBUG -extern ssize_t -omap_iommu_dump_ctx(struct omap_iommu *obj, char *buf, ssize_t len); -extern size_t -omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, ssize_t len); +struct cr_regs __iotlb_read_cr(struct omap_iommu *obj, int n); +void iotlb_lock_get(struct omap_iommu *obj, struct iotlb_lock *l); +void iotlb_lock_set(struct omap_iommu *obj, struct iotlb_lock *l); + +#ifdef CONFIG_OMAP_IOMMU_DEBUG void omap_iommu_debugfs_init(void); void omap_iommu_debugfs_exit(void); @@ -222,4 +232,12 @@ static inline void iommu_write_reg(struct omap_iommu *obj, u32 val, size_t offs) __raw_writel(val, obj->regbase + offs); } +static inline int iotlb_cr_valid(struct cr_regs *cr) +{ + if (!cr) + return -EINVAL; + + return cr->cam & MMU_CAM_V; +} + #endif /* _OMAP_IOMMU_H */ -- cgit v0.10.2 From ad8e29a0804494bff5f0059df3805423ed2020b8 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:26 -0500 Subject: iommu/omap: Protect omap-iopgtable.h against double inclusion Protect the omap-pgtable.h header against double inclusion in source code by using the standard include guard mechanism. Signed-off-by: Suman Anna Reviewed-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h index f891683..bfde540 100644 --- a/drivers/iommu/omap-iopgtable.h +++ b/drivers/iommu/omap-iopgtable.h @@ -10,6 +10,9 @@ * published by the Free Software Foundation. */ +#ifndef _OMAP_IOPGTABLE_H +#define _OMAP_IOPGTABLE_H + /* * "L2 table" address mask and size definitions. */ @@ -93,3 +96,5 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask) /* to find an entry in the second-level page table. */ #define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1)) #define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da)) + +#endif /* _OMAP_IOPGTABLE_H */ -- cgit v0.10.2 From dc308f9f92b084a25989fd2002fac06cbf4a73d4 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:27 -0500 Subject: iommu/omap: Remove unused union fields There are couple of unions defined in the structures iotlb_entry and cr_regs. There are no usage/references to some of these union fields in the code, so clean them up and simplify the structures. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h index b6cc90b..5b98408 100644 --- a/drivers/iommu/omap-iommu.h +++ b/drivers/iommu/omap-iommu.h @@ -22,12 +22,7 @@ struct iotlb_entry { u32 da; u32 pa; u32 pgsz, prsvd, valid; - union { - u16 ap; - struct { - u32 endian, elsz, mixed; - }; - }; + u32 endian, elsz, mixed; }; struct omap_iommu { @@ -54,20 +49,8 @@ struct omap_iommu { }; struct cr_regs { - union { - struct { - u16 cam_l; - u16 cam_h; - }; - u32 cam; - }; - union { - struct { - u16 ram_l; - u16 ram_h; - }; - u32 ram; - }; + u32 cam; + u32 ram; }; struct iotlb_lock { -- cgit v0.10.2 From 5b39a37abc007542995506ad0d8e4c3991e6970a Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:28 -0500 Subject: iommu/omap: Remove trailing semi-colon from a macro Remove the trailing semi-colon in the DEBUG_FOPS_RO macro definition. This fixes the checking warning, "WARNING: macros should not use a trailing semicolon" Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index b4b96db..e9f116f 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -265,7 +265,7 @@ static int debug_read_pagetable(struct seq_file *s, void *data) .open = simple_open, \ .read = debug_read_##name, \ .llseek = generic_file_llseek, \ - }; + } DEBUG_FOPS_RO(regs); DEBUG_FOPS_RO(tlb); -- cgit v0.10.2 From 99ee98d6ac964f1a2412d9fe08e577aa4f13905d Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:29 -0500 Subject: iommu/omap: Remove unnecessary error traces on alloc failures Fix couple of checkpatch warnings of the type, "WARNING: Possible unnecessary 'out of memory' message" Signed-off-by: Suman Anna Reviewed-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 0fc00f3..4328d98 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1093,16 +1093,12 @@ static struct iommu_domain *omap_iommu_domain_alloc(unsigned type) return NULL; omap_domain = kzalloc(sizeof(*omap_domain), GFP_KERNEL); - if (!omap_domain) { - pr_err("kzalloc failed\n"); + if (!omap_domain) goto out; - } omap_domain->pgtable = kzalloc(IOPGD_TABLE_SIZE, GFP_KERNEL); - if (!omap_domain->pgtable) { - pr_err("kzalloc failed\n"); + if (!omap_domain->pgtable) goto fail_nomem; - } /* * should never fail, but please keep this around to ensure -- cgit v0.10.2 From 5ff98fa68c88d7babf96b7df7c713aaf2ed6558a Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:30 -0500 Subject: iommu/omap: Use BIT(x) macros in omap-iopgtable.h Switch to using the BIT(x) macros in omap-iopgtable.h where possible. This eliminates the following checkpatch check warning: "CHECK: Prefer using the BIT macro" A couple of macros that used zero bit shifting are defined directly to avoid the above warning on one of the macros. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h index bfde540..01a3152 100644 --- a/drivers/iommu/omap-iopgtable.h +++ b/drivers/iommu/omap-iopgtable.h @@ -13,25 +13,27 @@ #ifndef _OMAP_IOPGTABLE_H #define _OMAP_IOPGTABLE_H +#include + /* * "L2 table" address mask and size definitions. */ #define IOPGD_SHIFT 20 -#define IOPGD_SIZE (1UL << IOPGD_SHIFT) +#define IOPGD_SIZE BIT(IOPGD_SHIFT) #define IOPGD_MASK (~(IOPGD_SIZE - 1)) /* * "section" address mask and size definitions. */ #define IOSECTION_SHIFT 20 -#define IOSECTION_SIZE (1UL << IOSECTION_SHIFT) +#define IOSECTION_SIZE BIT(IOSECTION_SHIFT) #define IOSECTION_MASK (~(IOSECTION_SIZE - 1)) /* * "supersection" address mask and size definitions. */ #define IOSUPER_SHIFT 24 -#define IOSUPER_SIZE (1UL << IOSUPER_SHIFT) +#define IOSUPER_SIZE BIT(IOSUPER_SHIFT) #define IOSUPER_MASK (~(IOSUPER_SIZE - 1)) #define PTRS_PER_IOPGD (1UL << (32 - IOPGD_SHIFT)) @@ -41,14 +43,14 @@ * "small page" address mask and size definitions. */ #define IOPTE_SHIFT 12 -#define IOPTE_SIZE (1UL << IOPTE_SHIFT) +#define IOPTE_SIZE BIT(IOPTE_SHIFT) #define IOPTE_MASK (~(IOPTE_SIZE - 1)) /* * "large page" address mask and size definitions. */ #define IOLARGE_SHIFT 16 -#define IOLARGE_SIZE (1UL << IOLARGE_SHIFT) +#define IOLARGE_SIZE BIT(IOLARGE_SHIFT) #define IOLARGE_MASK (~(IOLARGE_SIZE - 1)) #define PTRS_PER_IOPTE (1UL << (IOPGD_SHIFT - IOPTE_SHIFT)) @@ -72,16 +74,16 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask) /* * some descriptor attributes. */ -#define IOPGD_TABLE (1 << 0) -#define IOPGD_SECTION (2 << 0) -#define IOPGD_SUPER (1 << 18 | 2 << 0) +#define IOPGD_TABLE (1) +#define IOPGD_SECTION (2) +#define IOPGD_SUPER (BIT(18) | IOPGD_SECTION) #define iopgd_is_table(x) (((x) & 3) == IOPGD_TABLE) #define iopgd_is_section(x) (((x) & (1 << 18 | 3)) == IOPGD_SECTION) #define iopgd_is_super(x) (((x) & (1 << 18 | 3)) == IOPGD_SUPER) -#define IOPTE_SMALL (2 << 0) -#define IOPTE_LARGE (1 << 0) +#define IOPTE_SMALL (2) +#define IOPTE_LARGE (1) #define iopte_is_small(x) (((x) & 2) == IOPTE_SMALL) #define iopte_is_large(x) (((x) & 3) == IOPTE_LARGE) -- cgit v0.10.2 From eb642a3f5afdb13aa2b7ba0bda314b0d2b62165d Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:31 -0500 Subject: iommu/omap: Use BIT(x) macros in omap-iommu.h Switch to using the BIT(x) macros in omap-iommu.h where possible. This eliminates the following checkpatch check warning: "CHECK: Prefer using the BIT macro" A couple of the warnings were ignored for better readability of the bit-shift for the different values. Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu.h b/drivers/iommu/omap-iommu.h index 5b98408..a656df2 100644 --- a/drivers/iommu/omap-iommu.h +++ b/drivers/iommu/omap-iommu.h @@ -13,6 +13,8 @@ #ifndef _OMAP_IOMMU_H #define _OMAP_IOMMU_H +#include + #define for_each_iotlb_cr(obj, n, __i, cr) \ for (__i = 0; \ (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ @@ -96,11 +98,11 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev) * MMU Register bit definitions */ /* IRQSTATUS & IRQENABLE */ -#define MMU_IRQ_MULTIHITFAULT (1 << 4) -#define MMU_IRQ_TABLEWALKFAULT (1 << 3) -#define MMU_IRQ_EMUMISS (1 << 2) -#define MMU_IRQ_TRANSLATIONFAULT (1 << 1) -#define MMU_IRQ_TLBMISS (1 << 0) +#define MMU_IRQ_MULTIHITFAULT BIT(4) +#define MMU_IRQ_TABLEWALKFAULT BIT(3) +#define MMU_IRQ_EMUMISS BIT(2) +#define MMU_IRQ_TRANSLATIONFAULT BIT(1) +#define MMU_IRQ_TLBMISS BIT(0) #define __MMU_IRQ_FAULT \ (MMU_IRQ_MULTIHITFAULT | MMU_IRQ_EMUMISS | MMU_IRQ_TRANSLATIONFAULT) @@ -112,16 +114,16 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev) /* MMU_CNTL */ #define MMU_CNTL_SHIFT 1 #define MMU_CNTL_MASK (7 << MMU_CNTL_SHIFT) -#define MMU_CNTL_EML_TLB (1 << 3) -#define MMU_CNTL_TWL_EN (1 << 2) -#define MMU_CNTL_MMU_EN (1 << 1) +#define MMU_CNTL_EML_TLB BIT(3) +#define MMU_CNTL_TWL_EN BIT(2) +#define MMU_CNTL_MMU_EN BIT(1) /* CAM */ #define MMU_CAM_VATAG_SHIFT 12 #define MMU_CAM_VATAG_MASK \ ((~0UL >> MMU_CAM_VATAG_SHIFT) << MMU_CAM_VATAG_SHIFT) -#define MMU_CAM_P (1 << 3) -#define MMU_CAM_V (1 << 2) +#define MMU_CAM_P BIT(3) +#define MMU_CAM_V BIT(2) #define MMU_CAM_PGSZ_MASK 3 #define MMU_CAM_PGSZ_1M (0 << 0) #define MMU_CAM_PGSZ_64K (1 << 0) @@ -134,9 +136,9 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev) ((~0UL >> MMU_RAM_PADDR_SHIFT) << MMU_RAM_PADDR_SHIFT) #define MMU_RAM_ENDIAN_SHIFT 9 -#define MMU_RAM_ENDIAN_MASK (1 << MMU_RAM_ENDIAN_SHIFT) +#define MMU_RAM_ENDIAN_MASK BIT(MMU_RAM_ENDIAN_SHIFT) #define MMU_RAM_ENDIAN_LITTLE (0 << MMU_RAM_ENDIAN_SHIFT) -#define MMU_RAM_ENDIAN_BIG (1 << MMU_RAM_ENDIAN_SHIFT) +#define MMU_RAM_ENDIAN_BIG BIT(MMU_RAM_ENDIAN_SHIFT) #define MMU_RAM_ELSZ_SHIFT 7 #define MMU_RAM_ELSZ_MASK (3 << MMU_RAM_ELSZ_SHIFT) @@ -145,7 +147,7 @@ static inline struct omap_iommu *dev_to_omap_iommu(struct device *dev) #define MMU_RAM_ELSZ_32 (2 << MMU_RAM_ELSZ_SHIFT) #define MMU_RAM_ELSZ_NONE (3 << MMU_RAM_ELSZ_SHIFT) #define MMU_RAM_MIXED_SHIFT 6 -#define MMU_RAM_MIXED_MASK (1 << MMU_RAM_MIXED_SHIFT) +#define MMU_RAM_MIXED_MASK BIT(MMU_RAM_MIXED_SHIFT) #define MMU_RAM_MIXED MMU_RAM_MIXED_MASK #define MMU_GP_REG_BUS_ERR_BACK_EN 0x1 -- cgit v0.10.2 From 5835b6a64ce39434d5cc9857769c73982d488b42 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 20 Jul 2015 17:33:32 -0500 Subject: iommu/omap: Align code with open parenthesis Fix all the occurrences of the following check warning generated with the checkpatch --strict option: "CHECK: Alignment should match open parenthesis" Signed-off-by: Suman Anna Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 4328d98..36d0033 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -787,14 +787,14 @@ static irqreturn_t iommu_fault_handler(int irq, void *data) if (!iopgd_is_table(*iopgd)) { dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", - obj->name, errs, da, iopgd, *iopgd); + obj->name, errs, da, iopgd, *iopgd); return IRQ_NONE; } iopte = iopte_offset(iopgd, da); dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", - obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); + obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); return IRQ_NONE; } @@ -820,9 +820,8 @@ static struct omap_iommu *omap_iommu_attach(const char *name, u32 *iopgd) struct device *dev; struct omap_iommu *obj; - dev = driver_find_device(&omap_iommu_driver.driver, NULL, - (void *)name, - device_match_by_alias); + dev = driver_find_device(&omap_iommu_driver.driver, NULL, (void *)name, + device_match_by_alias); if (!dev) return ERR_PTR(-ENODEV); @@ -977,7 +976,7 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) } static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, - phys_addr_t pa, size_t bytes, int prot) + phys_addr_t pa, size_t bytes, int prot) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); struct omap_iommu *oiommu = omap_domain->iommu_dev; @@ -1004,7 +1003,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, } static size_t omap_iommu_unmap(struct iommu_domain *domain, unsigned long da, - size_t size) + size_t size) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); struct omap_iommu *oiommu = omap_domain->iommu_dev; @@ -1055,7 +1054,7 @@ out: } static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, - struct device *dev) + struct device *dev) { struct omap_iommu *oiommu = dev_to_omap_iommu(dev); struct omap_iommu_arch_data *arch_data = dev->archdata.iommu; @@ -1076,7 +1075,7 @@ static void _omap_iommu_detach_dev(struct omap_iommu_domain *omap_domain, } static void omap_iommu_detach_dev(struct iommu_domain *domain, - struct device *dev) + struct device *dev) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); @@ -1137,7 +1136,7 @@ static void omap_iommu_domain_free(struct iommu_domain *domain) } static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, - dma_addr_t da) + dma_addr_t da) { struct omap_iommu_domain *omap_domain = to_omap_domain(domain); struct omap_iommu *oiommu = omap_domain->iommu_dev; @@ -1154,7 +1153,7 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); else dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, - (unsigned long long)da); + (unsigned long long)da); } else { if (iopgd_is_section(*pgd)) ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); @@ -1162,7 +1161,7 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); else dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, - (unsigned long long)da); + (unsigned long long)da); } return ret; -- cgit v0.10.2 From 7b0ce727bf7ac5240a433109f53bf78788f9159b Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 22 Jul 2015 18:47:00 +0100 Subject: of: iommu: Silence misleading warning Printing "IOMMU is currently not supported for PCI" for every PCI device probed on a DT-based system proves to be both irritatingly noisy and confusing to users who have misinterpreted it to mean they can no longer use VFIO device assignment. Since configuring DMA masks for PCI devices via of_dma_configure() has not in fact changed anything with regard to IOMMUs there really is nothing to warn about here; shut it up. Signed-off-by: Robin Murphy Acked-by: Marc Zyngier Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 43429ab..60ba238 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -141,10 +141,12 @@ struct iommu_ops *of_iommu_configure(struct device *dev, struct iommu_ops *ops = NULL; int idx = 0; - if (dev_is_pci(dev)) { - dev_err(dev, "IOMMU is currently not supported for PCI\n"); + /* + * We can't do much for PCI devices without knowing how + * device IDs are wired up from the PCI bus to the IOMMU. + */ + if (dev_is_pci(dev)) return NULL; - } /* * We don't currently walk up the tree looking for a parent IOMMU. -- cgit v0.10.2 From 2439d4aa9247f4c94351d0cf7d75c16146785eb8 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Fri, 24 Jul 2015 16:27:57 -0700 Subject: iommu/vt-d: Avoid format string leaks into iommu_device_create This makes sure it won't be possible to accidentally leak format strings into iommu device names. Current name allocations are safe, but this makes the "%s" explicit. Signed-off-by: Kees Cook Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index c9db04d..8757f8d 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1068,7 +1068,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) if (intel_iommu_enabled) iommu->iommu_dev = iommu_device_create(NULL, iommu, intel_iommu_groups, - iommu->name); + "%s", iommu->name); return 0; diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0649b94..0be23c5 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4533,7 +4533,7 @@ int __init intel_iommu_init(void) for_each_active_iommu(iommu, drhd) iommu->iommu_dev = iommu_device_create(NULL, iommu, intel_iommu_groups, - iommu->name); + "%s", iommu->name); bus_set_iommu(&pci_bus_type, &intel_iommu_ops); bus_register_notifier(&pci_bus_type, &device_nb); -- cgit v0.10.2 From 50690762cfe32abadbaa5b22bebe3855e5b8ede8 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 30 Jul 2015 12:54:01 -0400 Subject: iommu/vt-d: Fix leaked ioremap mapping iommu_load_old_irte() appears to leak the old_irte mapping after use. Cc: Joerg Roedel Signed-off-by: Dan Williams Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index f15692a..27cdfa8 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -426,6 +426,8 @@ static int iommu_load_old_irte(struct intel_iommu *iommu) bitmap_set(iommu->ir_table->bitmap, i, 1); } + iounmap(old_ir_table); + return 0; } -- cgit v0.10.2 From 2238c0827a9bfa8d517e3175110ed603fb7b9537 Mon Sep 17 00:00:00 2001 From: Alex Williamson Date: Tue, 14 Jul 2015 15:24:53 -0600 Subject: iommu/vt-d: Report domain usage in sysfs Debugging domain ID leakage typically requires long running tests in order to exhaust the domain ID space or kernel instrumentation to track the setting and clearing of bits. A couple trivial intel-iommu specific sysfs extensions make it much easier to expose the IOMMU capabilities and current usage. Signed-off-by: Alex Williamson Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0be23c5..013cbc2 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4449,11 +4449,32 @@ static ssize_t intel_iommu_show_ecap(struct device *dev, } static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL); +static ssize_t intel_iommu_show_ndoms(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct intel_iommu *iommu = dev_get_drvdata(dev); + return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap)); +} +static DEVICE_ATTR(domains_supported, S_IRUGO, intel_iommu_show_ndoms, NULL); + +static ssize_t intel_iommu_show_ndoms_used(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct intel_iommu *iommu = dev_get_drvdata(dev); + return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids, + cap_ndoms(iommu->cap))); +} +static DEVICE_ATTR(domains_used, S_IRUGO, intel_iommu_show_ndoms_used, NULL); + static struct attribute *intel_iommu_attrs[] = { &dev_attr_version.attr, &dev_attr_address.attr, &dev_attr_cap.attr, &dev_attr_ecap.attr, + &dev_attr_domains_supported.attr, + &dev_attr_domains_used.attr, NULL, }; -- cgit v0.10.2 From e203db293863fa15b4b1917d4398fb5bd63c4e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Salva=20Peir=C3=B3?= Date: Thu, 23 Jul 2015 14:26:19 +0200 Subject: iommu/omap: Fix debug_read_tlb() to use seq_printf() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The debug_read_tlb() uses the sprintf() functions directly on the buffer allocated by buf = kmalloc(count), without taking into account the size of the buffer, with the consequence corrupting the heap, depending on the count requested by the user. The patch fixes the issue replacing sprintf() by seq_printf(). Signed-off-by: Salva Peiró Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu-debug.c b/drivers/iommu/omap-iommu-debug.c index e9f116f..0717aa9 100644 --- a/drivers/iommu/omap-iommu-debug.c +++ b/drivers/iommu/omap-iommu-debug.c @@ -133,26 +133,18 @@ __dump_tlb_entries(struct omap_iommu *obj, struct cr_regs *crs, int num) } static ssize_t iotlb_dump_cr(struct omap_iommu *obj, struct cr_regs *cr, - char *buf) + struct seq_file *s) { - char *p = buf; - - /* FIXME: Need more detail analysis of cam/ram */ - p += sprintf(p, "%08x %08x %01x\n", cr->cam, cr->ram, - (cr->cam & MMU_CAM_P) ? 1 : 0); - - return p - buf; + return seq_printf(s, "%08x %08x %01x\n", cr->cam, cr->ram, + (cr->cam & MMU_CAM_P) ? 1 : 0); } -static size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, - ssize_t bytes) +static size_t omap_dump_tlb_entries(struct omap_iommu *obj, struct seq_file *s) { int i, num; struct cr_regs *cr; - char *p = buf; - num = bytes / sizeof(*cr); - num = min(obj->nr_tlb_entries, num); + num = obj->nr_tlb_entries; cr = kcalloc(num, sizeof(*cr), GFP_KERNEL); if (!cr) @@ -160,40 +152,28 @@ static size_t omap_dump_tlb_entries(struct omap_iommu *obj, char *buf, num = __dump_tlb_entries(obj, cr, num); for (i = 0; i < num; i++) - p += iotlb_dump_cr(obj, cr + i, p); + iotlb_dump_cr(obj, cr + i, s); kfree(cr); - return p - buf; + return 0; } -static ssize_t debug_read_tlb(struct file *file, char __user *userbuf, - size_t count, loff_t *ppos) +static int debug_read_tlb(struct seq_file *s, void *data) { - struct omap_iommu *obj = file->private_data; - char *p, *buf; - ssize_t bytes, rest; + struct omap_iommu *obj = s->private; if (is_omap_iommu_detached(obj)) return -EPERM; - buf = kmalloc(count, GFP_KERNEL); - if (!buf) - return -ENOMEM; - p = buf; - mutex_lock(&iommu_debug_lock); - p += sprintf(p, "%8s %8s\n", "cam:", "ram:"); - p += sprintf(p, "-----------------------------------------\n"); - rest = count - (p - buf); - p += omap_dump_tlb_entries(obj, p, rest); - - bytes = simple_read_from_buffer(userbuf, count, ppos, buf, p - buf); + seq_printf(s, "%8s %8s\n", "cam:", "ram:"); + seq_puts(s, "-----------------------------------------\n"); + omap_dump_tlb_entries(obj, s); mutex_unlock(&iommu_debug_lock); - kfree(buf); - return bytes; + return 0; } static void dump_ioptable(struct seq_file *s) @@ -268,7 +248,7 @@ static int debug_read_pagetable(struct seq_file *s, void *data) } DEBUG_FOPS_RO(regs); -DEBUG_FOPS_RO(tlb); +DEBUG_SEQ_FOPS_RO(tlb); DEBUG_SEQ_FOPS_RO(pagetable); #define __DEBUG_ADD_FILE(attr, mode) \ -- cgit v0.10.2 From 85430968ae72650a63f77f05a29d5c56e41581db Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 3 Aug 2015 10:35:40 +0100 Subject: iommu/arm-smmu: Treat unknown OAS as 48-bit A late change to the SMMUv3 architecture ensures that the OAS field will be monotonically increasing, so we can assume that an unknown OAS is at least 48-bit and use that, rather than fail the device probe. Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4f09337..e51646a 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -2550,12 +2550,12 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) case IDR5_OAS_44_BIT: smmu->oas = 44; break; + default: + dev_info(smmu->dev, + "unknown output address size. Truncating to 48-bit\n"); + /* Fallthrough */ case IDR5_OAS_48_BIT: smmu->oas = 48; - break; - default: - dev_err(smmu->dev, "unknown output address size!\n"); - return -ENXIO; } /* Set the DMA mask for our table walker */ -- cgit v0.10.2 From f8d5496131554f61b0fd931fa046f0233fe2aac2 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:04 +0100 Subject: iommu/io-pgtable-arm: Allow appropriate DMA API use Currently, users of the LPAE page table code are (ab)using dma_map_page() as a means to flush page table updates for non-coherent IOMMUs. Since from the CPU's point of view, creating IOMMU page tables *is* passing DMA buffers to a device (the IOMMU's page table walker), there's little reason not to use the DMA API correctly. Allow IOMMU drivers to opt into DMA API operations for page table allocation and updates by providing their appropriate device pointer. The expectation is that an LPAE IOMMU should have a full view of system memory, so use streaming mappings to avoid unnecessary pressure on ZONE_DMA, and treat any DMA translation as a warning sign. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index f1fb1d3..d77a848 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -23,7 +23,8 @@ config IOMMU_IO_PGTABLE config IOMMU_IO_PGTABLE_LPAE bool "ARMv7/v8 Long Descriptor Format" select IOMMU_IO_PGTABLE - depends on ARM || ARM64 || COMPILE_TEST + # SWIOTLB guarantees a dma_to_phys() implementation + depends on ARM || ARM64 || (COMPILE_TEST && SWIOTLB) help Enable support for the ARM long descriptor pagetable format. This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 4e46021..28cca8a 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -200,12 +200,76 @@ typedef u64 arm_lpae_iopte; static bool selftest_running = false; +static dma_addr_t __arm_lpae_dma_addr(struct device *dev, void *pages) +{ + return phys_to_dma(dev, virt_to_phys(pages)); +} + +static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg) +{ + struct device *dev = cfg->iommu_dev; + dma_addr_t dma; + void *pages = alloc_pages_exact(size, gfp | __GFP_ZERO); + + if (!pages) + return NULL; + + if (dev) { + dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + goto out_free; + /* + * We depend on the IOMMU being able to work with any physical + * address directly, so if the DMA layer suggests it can't by + * giving us back some translation, that bodes very badly... + */ + if (dma != __arm_lpae_dma_addr(dev, pages)) + goto out_unmap; + } + + return pages; + +out_unmap: + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); +out_free: + free_pages_exact(pages, size); + return NULL; +} + +static void __arm_lpae_free_pages(void *pages, size_t size, + struct io_pgtable_cfg *cfg) +{ + struct device *dev = cfg->iommu_dev; + + if (dev) + dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages), + size, DMA_TO_DEVICE); + free_pages_exact(pages, size); +} + +static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, + struct io_pgtable_cfg *cfg, void *cookie) +{ + struct device *dev = cfg->iommu_dev; + + *ptep = pte; + + if (dev) + dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep), + sizeof(pte), DMA_TO_DEVICE); + else if (cfg->tlb->flush_pgtable) + cfg->tlb->flush_pgtable(ptep, sizeof(pte), cookie); +} + static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte pte = prot; + struct io_pgtable_cfg *cfg = &data->iop.cfg; /* We require an unmap first */ if (iopte_leaf(*ptep, lvl)) { @@ -213,7 +277,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, return -EEXIST; } - if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NS; if (lvl == ARM_LPAE_MAX_LEVELS - 1) @@ -224,8 +288,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; pte |= pfn_to_iopte(paddr >> data->pg_shift, data); - *ptep = pte; - data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), data->iop.cookie); + __arm_lpae_set_pte(ptep, pte, cfg, data->iop.cookie); return 0; } @@ -236,12 +299,13 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, arm_lpae_iopte *cptep, pte; void *cookie = data->iop.cookie; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + struct io_pgtable_cfg *cfg = &data->iop.cfg; /* Find our entry at the current level */ ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); /* If we can install a leaf entry at this level, then do so */ - if (size == block_size && (size & data->iop.cfg.pgsize_bitmap)) + if (size == block_size && (size & cfg->pgsize_bitmap)) return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); /* We can't allocate tables at the final level */ @@ -251,18 +315,15 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = *ptep; if (!pte) { - cptep = alloc_pages_exact(1UL << data->pg_shift, - GFP_ATOMIC | __GFP_ZERO); + cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift, + GFP_ATOMIC, cfg); if (!cptep) return -ENOMEM; - data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift, - cookie); pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; - if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; - *ptep = pte; - data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, pte, cfg, cookie); } else { cptep = iopte_deref(pte, data); } @@ -347,7 +408,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); } - free_pages_exact(start, table_size); + __arm_lpae_free_pages(start, table_size, &data->iop.cfg); } static void arm_lpae_free_pgtable(struct io_pgtable *iop) @@ -366,8 +427,8 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, unsigned long blk_start, blk_end; phys_addr_t blk_paddr; arm_lpae_iopte table = 0; + struct io_pgtable_cfg *cfg = &data->iop.cfg; void *cookie = data->iop.cookie; - const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; blk_start = iova & ~(blk_size - 1); blk_end = blk_start + blk_size; @@ -393,10 +454,9 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, } } - *ptep = table; - tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, table, cfg, cookie); iova &= ~(blk_size - 1); - tlb->tlb_add_flush(iova, blk_size, true, cookie); + cfg->tlb->tlb_add_flush(iova, blk_size, true, cookie); return size; } @@ -418,13 +478,12 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* If the size matches this level, we're in the right place */ if (size == blk_size) { - *ptep = 0; - tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + __arm_lpae_set_pte(ptep, 0, &data->iop.cfg, cookie); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ tlb->tlb_add_flush(iova, size, false, cookie); - tlb->tlb_sync(data->iop.cookie); + tlb->tlb_sync(cookie); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); } else { @@ -640,11 +699,12 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s1_cfg.mair[1] = 0; /* Looking good; allocate a pgd */ - data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); if (!data->pgd) goto out_free_data; - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + if (cfg->tlb->flush_pgtable) + cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); @@ -728,11 +788,12 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_lpae_s2_cfg.vtcr = reg; /* Allocate pgd pages */ - data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); + data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL, cfg); if (!data->pgd) goto out_free_data; - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + if (cfg->tlb->flush_pgtable) + cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); /* VTTBR */ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 10e32f6..c69529c 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -41,6 +41,8 @@ struct iommu_gather_ops { * @ias: Input address (iova) size, in bits. * @oas: Output address (paddr) size, in bits. * @tlb: TLB management callbacks for this set of tables. + * @iommu_dev: The device representing the DMA configuration for the + * page table walker. */ struct io_pgtable_cfg { #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */ @@ -49,6 +51,7 @@ struct io_pgtable_cfg { unsigned int ias; unsigned int oas; const struct iommu_gather_ops *tlb; + struct device *iommu_dev; /* Low-level data specific to the table format */ union { -- cgit v0.10.2 From 2df7a25ce4a79092946330ac4b7a2fbb5944d1d6 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:06 +0100 Subject: iommu/arm-smmu: Clean up DMA API usage With the correct DMA API calls now integrated into the io-pgtable code, let that handle the flushing of non-coherent page table updates. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 0583ed2..5770ab9 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -611,24 +611,13 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; - struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; - - /* Ensure new page tables are visible to the hardware walker */ - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { + /* + * Ensure new page tables are visible to a coherent hardware walker. + * The page table code deals with flushing for the non-coherent case. + */ + if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) dsb(ishst); - } else { - /* - * If the SMMU can't walk tables in the CPU caches, treat them - * like non-coherent DMA since we need to flush the new entries - * all the way out to memory. There's no possibility of - * recursion here as the SMMU table walker will not be wired - * through another SMMU. - */ - dma_map_page(smmu->dev, virt_to_page(addr), offset, size, - DMA_TO_DEVICE); - } } static struct iommu_gather_ops arm_smmu_gather_ops = { @@ -899,6 +888,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, + .iommu_dev = smmu->dev, }; smmu_domain->smmu = smmu; -- cgit v0.10.2 From bdc6d973473f32891a8518c51b210ce7daaa10ac Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:07 +0100 Subject: iommu/arm-smmu: Clean up DMA API usage With the correct DMA API calls now integrated into the io-pgtable code, let that handle the flushing of non-coherent page table updates. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index e51646a..54c68d1 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1334,23 +1334,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; - struct arm_smmu_device *smmu = smmu_domain->smmu; - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; - if (smmu->features & ARM_SMMU_FEAT_COHERENCY) { + /* The page table code handles flushing in the non-coherent case */ + if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENCY) dsb(ishst); - } else { - dma_addr_t dma_addr; - struct device *dev = smmu->dev; - - dma_addr = dma_map_page(dev, virt_to_page(addr), offset, size, - DMA_TO_DEVICE); - - if (dma_mapping_error(dev, dma_addr)) - dev_err(dev, "failed to flush pgtable at %p\n", addr); - else - dma_unmap_page(dev, dma_addr, size, DMA_TO_DEVICE); - } } static struct iommu_gather_ops arm_smmu_gather_ops = { @@ -1532,6 +1519,7 @@ static int arm_smmu_domain_finalise(struct iommu_domain *domain) .ias = ias, .oas = oas, .tlb = &arm_smmu_gather_ops, + .iommu_dev = smmu->dev, }; pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); -- cgit v0.10.2 From ff2ed96dde3b30d8f1b2ab0d9b164140f2278e6e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:08 +0100 Subject: iommu/ipmmu-vmsa: Clean up DMA API usage With the correct DMA API calls now integrated into the io-pgtable code, let that handle the flushing of non-coherent page table updates. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 1a67c53..8cf605f 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -283,24 +283,10 @@ static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, bool leaf, /* The hardware doesn't support selective TLB flush. */ } -static void ipmmu_flush_pgtable(void *ptr, size_t size, void *cookie) -{ - unsigned long offset = (unsigned long)ptr & ~PAGE_MASK; - struct ipmmu_vmsa_domain *domain = cookie; - - /* - * TODO: Add support for coherent walk through CCI with DVM and remove - * cache handling. - */ - dma_map_page(domain->mmu->dev, virt_to_page(ptr), offset, size, - DMA_TO_DEVICE); -} - static struct iommu_gather_ops ipmmu_gather_ops = { .tlb_flush_all = ipmmu_tlb_flush_all, .tlb_add_flush = ipmmu_tlb_add_flush, .tlb_sync = ipmmu_tlb_flush_all, - .flush_pgtable = ipmmu_flush_pgtable, }; /* ----------------------------------------------------------------------------- @@ -327,6 +313,11 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) domain->cfg.ias = 32; domain->cfg.oas = 40; domain->cfg.tlb = &ipmmu_gather_ops; + /* + * TODO: Add support for coherent walk through CCI with DVM and remove + * cache handling. For now, delegate it to the io-pgtable code. + */ + domain->cfg.iommu_dev = domain->mmu->dev; domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, domain); -- cgit v0.10.2 From 87a91b15d691d6f4aa0a5baffb5767bbc6e4a8c4 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:09 +0100 Subject: iommu/io-pgtable-arm: Centralise sync points With all current users now opted in to DMA API operations, make the iommu_dev pointer mandatory, rendering the flush_pgtable callback redundant for cache maintenance. However, since the DMA calls could be nops in the case of a coherent IOMMU, we still need to ensure the page table updates are fully synchronised against a subsequent page table walk. In the unmap path, the TLB sync will usually need to do this anyway, so just cement that requirement; in the map path which may consist solely of cacheable memory writes (in the coherent case), insert an appropriate barrier at the end of the operation, and obviate the need to call flush_pgtable on every individual update for synchronisation. Signed-off-by: Robin Murphy [will: slight clarification to tlb_sync comment] Signed-off-by: Will Deacon diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 28cca8a..0617687 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -26,6 +26,8 @@ #include #include +#include + #include "io-pgtable.h" #define ARM_LPAE_MAX_ADDR_BITS 48 @@ -215,7 +217,7 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, if (!pages) return NULL; - if (dev) { + if (!selftest_running) { dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) goto out_free; @@ -243,24 +245,22 @@ static void __arm_lpae_free_pages(void *pages, size_t size, { struct device *dev = cfg->iommu_dev; - if (dev) + if (!selftest_running) dma_unmap_single(dev, __arm_lpae_dma_addr(dev, pages), size, DMA_TO_DEVICE); free_pages_exact(pages, size); } static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, - struct io_pgtable_cfg *cfg, void *cookie) + struct io_pgtable_cfg *cfg) { struct device *dev = cfg->iommu_dev; *ptep = pte; - if (dev) + if (!selftest_running) dma_sync_single_for_device(dev, __arm_lpae_dma_addr(dev, ptep), sizeof(pte), DMA_TO_DEVICE); - else if (cfg->tlb->flush_pgtable) - cfg->tlb->flush_pgtable(ptep, sizeof(pte), cookie); } static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, @@ -288,7 +288,7 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; pte |= pfn_to_iopte(paddr >> data->pg_shift, data); - __arm_lpae_set_pte(ptep, pte, cfg, data->iop.cookie); + __arm_lpae_set_pte(ptep, pte, cfg); return 0; } @@ -297,7 +297,6 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, int lvl, arm_lpae_iopte *ptep) { arm_lpae_iopte *cptep, pte; - void *cookie = data->iop.cookie; size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); struct io_pgtable_cfg *cfg = &data->iop.cfg; @@ -323,7 +322,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) pte |= ARM_LPAE_PTE_NSTABLE; - __arm_lpae_set_pte(ptep, pte, cfg, cookie); + __arm_lpae_set_pte(ptep, pte, cfg); } else { cptep = iopte_deref(pte, data); } @@ -370,7 +369,7 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, { struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); arm_lpae_iopte *ptep = data->pgd; - int lvl = ARM_LPAE_START_LVL(data); + int ret, lvl = ARM_LPAE_START_LVL(data); arm_lpae_iopte prot; /* If no access, then nothing to do */ @@ -378,7 +377,14 @@ static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, return 0; prot = arm_lpae_prot_to_pte(data, iommu_prot); - return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); + ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + return ret; } static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, @@ -428,7 +434,6 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, phys_addr_t blk_paddr; arm_lpae_iopte table = 0; struct io_pgtable_cfg *cfg = &data->iop.cfg; - void *cookie = data->iop.cookie; blk_start = iova & ~(blk_size - 1); blk_end = blk_start + blk_size; @@ -454,9 +459,9 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, } } - __arm_lpae_set_pte(ptep, table, cfg, cookie); + __arm_lpae_set_pte(ptep, table, cfg); iova &= ~(blk_size - 1); - cfg->tlb->tlb_add_flush(iova, blk_size, true, cookie); + cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie); return size; } @@ -478,7 +483,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* If the size matches this level, we're in the right place */ if (size == blk_size) { - __arm_lpae_set_pte(ptep, 0, &data->iop.cfg, cookie); + __arm_lpae_set_pte(ptep, 0, &data->iop.cfg); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ @@ -703,8 +708,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) if (!data->pgd) goto out_free_data; - if (cfg->tlb->flush_pgtable) - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); /* TTBRs */ cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); @@ -792,8 +797,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) if (!data->pgd) goto out_free_data; - if (cfg->tlb->flush_pgtable) - cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); /* VTTBR */ cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index c69529c..e8fadb0 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -17,7 +17,9 @@ enum io_pgtable_fmt { * * @tlb_flush_all: Synchronously invalidate the entire TLB context. * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. - * @tlb_sync: Ensure any queue TLB invalidation has taken effect. + * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and + * any corresponding page table updates are visible to the + * IOMMU. * @flush_pgtable: Ensure page table updates are visible to the IOMMU. * * Note that these can all be called in atomic context and must therefore -- cgit v0.10.2 From 4103d662cbd0c045d7a44a18c82172220478b20c Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:10 +0100 Subject: iommu/arm-smmu: Remove arm_smmu_flush_pgtable() With the io-pgtable code now enforcing its own appropriate sync points, the vestigial flush_pgtable callback becomes entirely redundant, so remove it altogether. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 5770ab9..48a39df 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -608,23 +608,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, } } -static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) -{ - struct arm_smmu_domain *smmu_domain = cookie; - - /* - * Ensure new page tables are visible to a coherent hardware walker. - * The page table code deals with flushing for the non-coherent case. - */ - if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) - dsb(ishst); -} - static struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, - .flush_pgtable = arm_smmu_flush_pgtable, }; static irqreturn_t arm_smmu_context_fault(int irq, void *dev) -- cgit v0.10.2 From 857c88ca62f1e2594e1e760ef9a45ec1961f2a53 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:11 +0100 Subject: iommu/arm-smmu: Remove arm_smmu_flush_pgtable() With the io-pgtable code now enforcing its own appropriate sync points, the vestigial flush_pgtable callback becomes entirely redundant, so remove it altogether. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 54c68d1..dafaf59 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1331,20 +1331,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, arm_smmu_cmdq_issue_cmd(smmu, &cmd); } -static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) -{ - struct arm_smmu_domain *smmu_domain = cookie; - - /* The page table code handles flushing in the non-coherent case */ - if (smmu_domain->smmu->features & ARM_SMMU_FEAT_COHERENCY) - dsb(ishst); -} - static struct iommu_gather_ops arm_smmu_gather_ops = { .tlb_flush_all = arm_smmu_tlb_inv_context, .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, .tlb_sync = arm_smmu_tlb_sync, - .flush_pgtable = arm_smmu_flush_pgtable, }; /* IOMMU API */ -- cgit v0.10.2 From f5b831907da3e64bfb0288089a5c07124266b1a5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 29 Jul 2015 19:46:12 +0100 Subject: iommu/io-pgtable: Remove flush_pgtable callback With the users fully converted to DMA API operations, it's dead, Jim. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 0617687..e4bc2b2 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -884,16 +884,10 @@ static void dummy_tlb_sync(void *cookie) WARN_ON(cookie != cfg_cookie); } -static void dummy_flush_pgtable(void *ptr, size_t size, void *cookie) -{ - WARN_ON(cookie != cfg_cookie); -} - static struct iommu_gather_ops dummy_tlb_ops __initdata = { .tlb_flush_all = dummy_tlb_flush_all, .tlb_add_flush = dummy_tlb_add_flush, .tlb_sync = dummy_tlb_sync, - .flush_pgtable = dummy_flush_pgtable, }; static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index e8fadb0..48538a3 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -20,7 +20,6 @@ enum io_pgtable_fmt { * @tlb_sync: Ensure any queued TLB invalidation has taken effect, and * any corresponding page table updates are visible to the * IOMMU. - * @flush_pgtable: Ensure page table updates are visible to the IOMMU. * * Note that these can all be called in atomic context and must therefore * not block. @@ -30,7 +29,6 @@ struct iommu_gather_ops { void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf, void *cookie); void (*tlb_sync)(void *cookie); - void (*flush_pgtable)(void *ptr, size_t size, void *cookie); }; /** -- cgit v0.10.2 From c0e8a6c8033e205835fa5e67db7ab4589d2491b2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 09:39:46 +0200 Subject: iommu/vt-d: Keep track of per-iommu domain ids Instead of searching in the domain array for already allocated domain ids, keep track of them explicitly. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 013cbc2..bf92cd1 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -378,6 +378,11 @@ struct dmar_domain { DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED); /* bitmap of iommus this domain uses*/ + u16 iommu_did[DMAR_UNITS_SUPPORTED]; + /* Domain ids per IOMMU. Use u16 since + * domain ids are 16 bit wide according + * to VT-d spec, section 9.3 */ + struct list_head devices; /* all devices' list */ struct iova_domain iovad; /* iova's that belong to this domain */ @@ -1543,11 +1548,13 @@ static int iommu_init_domains(struct intel_iommu *iommu) } /* - * if Caching mode is set, then invalid translations are tagged - * with domainid 0. Hence we need to pre-allocate it. + * If Caching mode is set, then invalid translations are tagged + * with domain-id 0, hence we need to pre-allocate it. We also + * use domain-id 0 as a marker for non-allocated domain-id, so + * make sure it is not used for a real domain. */ - if (cap_caching_mode(iommu->cap)) - set_bit(0, iommu->domain_ids); + set_bit(0, iommu->domain_ids); + return 0; } @@ -1560,9 +1567,10 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) { /* * Domain id 0 is reserved for invalid translation - * if hardware supports caching mode. + * if hardware supports caching mode and used as + * a non-allocated marker. */ - if (cap_caching_mode(iommu->cap) && i == 0) + if (i == 0) continue; domain = iommu->domains[i]; @@ -1624,6 +1632,7 @@ static int __iommu_attach_domain(struct dmar_domain *domain, if (num < ndomains) { set_bit(num, iommu->domain_ids); iommu->domains[num] = domain; + domain->iommu_did[iommu->seq_id] = num; } else { num = -ENOSPC; } @@ -1650,12 +1659,10 @@ static int iommu_attach_vm_domain(struct dmar_domain *domain, struct intel_iommu *iommu) { int num; - unsigned long ndomains; - ndomains = cap_ndoms(iommu->cap); - for_each_set_bit(num, iommu->domain_ids, ndomains) - if (iommu->domains[num] == domain) - return num; + num = domain->iommu_did[iommu->seq_id]; + if (num) + return num; return __iommu_attach_domain(domain, iommu); } @@ -1664,22 +1671,18 @@ static void iommu_detach_domain(struct dmar_domain *domain, struct intel_iommu *iommu) { unsigned long flags; - int num, ndomains; + int num; spin_lock_irqsave(&iommu->lock, flags); - if (domain_type_is_vm_or_si(domain)) { - ndomains = cap_ndoms(iommu->cap); - for_each_set_bit(num, iommu->domain_ids, ndomains) { - if (iommu->domains[num] == domain) { - clear_bit(num, iommu->domain_ids); - iommu->domains[num] = NULL; - break; - } - } - } else { - clear_bit(domain->id, iommu->domain_ids); - iommu->domains[domain->id] = NULL; - } + + num = domain->iommu_did[iommu->seq_id]; + + if (num == 0) + return; + + clear_bit(num, iommu->domain_ids); + iommu->domains[num] = NULL; + spin_unlock_irqrestore(&iommu->lock, flags); } @@ -1708,6 +1711,7 @@ static int domain_detach_iommu(struct dmar_domain *domain, if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) { count = --domain->iommu_count; domain_update_iommu_cap(domain); + domain->iommu_did[iommu->seq_id] = 0; } spin_unlock_irqrestore(&domain->iommu_lock, flags); -- cgit v0.10.2 From 9452d5bfe5c3df6befb89835d2c44920e03bd390 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 10:00:56 +0200 Subject: iommu/vt-d: Add access functions for iommu->domains This makes it easier to change the layout of the data structure later. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index bf92cd1..1c2d612 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -569,6 +569,17 @@ __setup("intel_iommu=", intel_iommu_setup); static struct kmem_cache *iommu_domain_cache; static struct kmem_cache *iommu_devinfo_cache; +static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did) +{ + return iommu->domains[did]; +} + +static void set_iommu_domain(struct intel_iommu *iommu, u16 did, + struct dmar_domain *domain) +{ + iommu->domains[did] = domain; +} + static inline void *alloc_pgtable_page(int node) { struct page *page; @@ -1463,7 +1474,8 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, * flush. However, device IOTLB doesn't need to be flushed in this case. */ if (!cap_caching_mode(iommu->cap) || !map) - iommu_flush_dev_iotlb(iommu->domains[did], addr, mask); + iommu_flush_dev_iotlb(get_iommu_domain(iommu, did), + addr, mask); } static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu) @@ -1573,7 +1585,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) if (i == 0) continue; - domain = iommu->domains[i]; + domain = get_iommu_domain(iommu, i); clear_bit(i, iommu->domain_ids); if (domain_detach_iommu(domain, iommu) == 0 && !domain_type_is_vm(domain)) @@ -1631,7 +1643,7 @@ static int __iommu_attach_domain(struct dmar_domain *domain, num = find_first_zero_bit(iommu->domain_ids, ndomains); if (num < ndomains) { set_bit(num, iommu->domain_ids); - iommu->domains[num] = domain; + set_iommu_domain(iommu, num, domain); domain->iommu_did[iommu->seq_id] = num; } else { num = -ENOSPC; @@ -1681,7 +1693,7 @@ static void iommu_detach_domain(struct dmar_domain *domain, return; clear_bit(num, iommu->domain_ids); - iommu->domains[num] = NULL; + set_iommu_domain(iommu, num, NULL); spin_unlock_irqrestore(&iommu->lock, flags); } @@ -4852,7 +4864,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, */ ndomains = cap_ndoms(iommu->cap); for_each_set_bit(num, iommu->domain_ids, ndomains) { - if (iommu->domains[num] == dmar_domain) + if (get_iommu_domain(iommu, num) == dmar_domain) iommu_flush_iotlb_psi(iommu, num, start_pfn, npages, !freelist, 0); } -- cgit v0.10.2 From 8bf478163e69e42973c7070179a11815139e5bf0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 10:41:21 +0200 Subject: iommu/vt-d: Split up iommu->domains array This array is indexed by the domain-id and contains the pointers to the domains attached to this iommu. Modern systems support 65536 domain ids, so that this array has a size of 512kb, per iommu. This is a huge waste of space, as the array is usually sparsely populated. This patch makes the array two-dimensional and allocates the memory for the domain pointers on-demand. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1c2d612..90ab4b0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -571,13 +571,32 @@ static struct kmem_cache *iommu_devinfo_cache; static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did) { - return iommu->domains[did]; + struct dmar_domain **domains; + int idx = did >> 8; + + domains = iommu->domains[idx]; + if (!domains) + return NULL; + + return domains[did & 0xff]; } static void set_iommu_domain(struct intel_iommu *iommu, u16 did, struct dmar_domain *domain) { - iommu->domains[did] = domain; + struct dmar_domain **domains; + int idx = did >> 8; + + if (!iommu->domains[idx]) { + size_t size = 256 * sizeof(struct dmar_domain *); + iommu->domains[idx] = kzalloc(size, GFP_ATOMIC); + } + + domains = iommu->domains[idx]; + if (WARN_ON(!domains)) + return; + else + domains[did & 0xff] = domain; } static inline void *alloc_pgtable_page(int node) @@ -1530,35 +1549,43 @@ static void iommu_disable_translation(struct intel_iommu *iommu) static int iommu_init_domains(struct intel_iommu *iommu) { - unsigned long ndomains; - unsigned long nlongs; + u32 ndomains, nlongs; + size_t size; ndomains = cap_ndoms(iommu->cap); - pr_debug("%s: Number of Domains supported <%ld>\n", + pr_debug("%s: Number of Domains supported <%d>\n", iommu->name, ndomains); nlongs = BITS_TO_LONGS(ndomains); spin_lock_init(&iommu->lock); - /* TBD: there might be 64K domains, - * consider other allocation for future chip - */ iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); if (!iommu->domain_ids) { pr_err("%s: Allocating domain id array failed\n", iommu->name); return -ENOMEM; } - iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *), - GFP_KERNEL); - if (!iommu->domains) { + + size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **); + iommu->domains = kzalloc(size, GFP_KERNEL); + + if (iommu->domains) { + size = 256 * sizeof(struct dmar_domain *); + iommu->domains[0] = kzalloc(size, GFP_KERNEL); + } + + if (!iommu->domains || !iommu->domains[0]) { pr_err("%s: Allocating domain array failed\n", iommu->name); kfree(iommu->domain_ids); + kfree(iommu->domains); iommu->domain_ids = NULL; + iommu->domains = NULL; return -ENOMEM; } + + /* * If Caching mode is set, then invalid translations are tagged * with domain-id 0, hence we need to pre-allocate it. We also @@ -1600,6 +1627,11 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) static void free_dmar_iommu(struct intel_iommu *iommu) { if ((iommu->domains) && (iommu->domain_ids)) { + int elems = (cap_ndoms(iommu->cap) >> 8) + 1; + int i; + + for (i = 0; i < elems; i++) + kfree(iommu->domains[i]); kfree(iommu->domains); kfree(iommu->domain_ids); iommu->domains = NULL; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index d9a366d..6240063 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -344,7 +344,7 @@ struct intel_iommu { #ifdef CONFIG_INTEL_IOMMU unsigned long *domain_ids; /* bitmap of domains */ - struct dmar_domain **domains; /* ptr to domains */ + struct dmar_domain ***domains; /* ptr to domains */ spinlock_t lock; /* protect context, domain ids */ struct root_entry *root_entry; /* virtual address */ -- cgit v0.10.2 From e2411427f7d3ddcf8d5f35d5ab0a397180deac3a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 11:18:21 +0200 Subject: iommu/vt-d: Get rid of iommu_attach_vm_domain() The special case for VM domains is not needed, as other domains could be attached to the iommu in the same way. So get rid of this special case. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 90ab4b0..0a07b44 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1671,8 +1671,13 @@ static int __iommu_attach_domain(struct dmar_domain *domain, int num; unsigned long ndomains; + num = domain->iommu_did[iommu->seq_id]; + if (num) + return num; + ndomains = cap_ndoms(iommu->cap); - num = find_first_zero_bit(iommu->domain_ids, ndomains); + num = find_first_zero_bit(iommu->domain_ids, ndomains); + if (num < ndomains) { set_bit(num, iommu->domain_ids); set_iommu_domain(iommu, num, domain); @@ -1681,6 +1686,9 @@ static int __iommu_attach_domain(struct dmar_domain *domain, num = -ENOSPC; } + if (num < 0) + pr_err("%s: No free domain ids\n", iommu->name); + return num; } @@ -1693,24 +1701,10 @@ static int iommu_attach_domain(struct dmar_domain *domain, spin_lock_irqsave(&iommu->lock, flags); num = __iommu_attach_domain(domain, iommu); spin_unlock_irqrestore(&iommu->lock, flags); - if (num < 0) - pr_err("%s: No free domain ids\n", iommu->name); return num; } -static int iommu_attach_vm_domain(struct dmar_domain *domain, - struct intel_iommu *iommu) -{ - int num; - - num = domain->iommu_did[iommu->seq_id]; - if (num) - return num; - - return __iommu_attach_domain(domain, iommu); -} - static void iommu_detach_domain(struct dmar_domain *domain, struct intel_iommu *iommu) { @@ -1947,7 +1941,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, if (domain_type_is_vm_or_si(domain)) { if (domain_type_is_vm(domain)) { - id = iommu_attach_vm_domain(domain, iommu); + id = __iommu_attach_domain(domain, iommu); if (id < 0) { spin_unlock_irqrestore(&iommu->lock, flags); pr_err("%s: No free domain ids\n", iommu->name); -- cgit v0.10.2 From 28ccce0d954a1cf3baba335bf12581357112fb35 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 14:45:31 +0200 Subject: iommu/vt-d: Calculate translation in domain_context_mapping_one There is no reason to pass the translation type through multiple layers. It can also be determined in the domain_context_mapping_one function directly. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0a07b44..ca6ca3d 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -364,7 +364,8 @@ static inline int first_pte_in_page(struct dma_pte *pte) static struct dmar_domain *si_domain; static int hw_pass_through = 1; -/* domain represents a virtual machine, more than one devices +/* + * Domain represents a virtual machine, more than one devices * across iommus may be owned in one domain, e.g. kvm guest. */ #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0) @@ -640,6 +641,11 @@ static inline int domain_type_is_vm(struct dmar_domain *domain) return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE; } +static inline int domain_type_is_si(struct dmar_domain *domain) +{ + return domain->flags & DOMAIN_FLAG_STATIC_IDENTITY; +} + static inline int domain_type_is_vm_or_si(struct dmar_domain *domain) { return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE | @@ -1907,21 +1913,23 @@ static void domain_exit(struct dmar_domain *domain) static int domain_context_mapping_one(struct dmar_domain *domain, struct intel_iommu *iommu, - u8 bus, u8 devfn, int translation) + u8 bus, u8 devfn) { + int translation = CONTEXT_TT_MULTI_LEVEL; + struct device_domain_info *info = NULL; struct context_entry *context; unsigned long flags; struct dma_pte *pgd; int id; int agaw; - struct device_domain_info *info = NULL; + + if (hw_pass_through && domain_type_is_si(domain)) + translation = CONTEXT_TT_PASS_THROUGH; pr_debug("Set context mapping for %02x:%02x.%d\n", bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); BUG_ON(!domain->pgd); - BUG_ON(translation != CONTEXT_TT_PASS_THROUGH && - translation != CONTEXT_TT_MULTI_LEVEL); spin_lock_irqsave(&iommu->lock, flags); context = iommu_context_addr(iommu, bus, devfn, 1); @@ -2013,7 +2021,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain, struct domain_context_mapping_data { struct dmar_domain *domain; struct intel_iommu *iommu; - int translation; }; static int domain_context_mapping_cb(struct pci_dev *pdev, @@ -2022,13 +2029,11 @@ static int domain_context_mapping_cb(struct pci_dev *pdev, struct domain_context_mapping_data *data = opaque; return domain_context_mapping_one(data->domain, data->iommu, - PCI_BUS_NUM(alias), alias & 0xff, - data->translation); + PCI_BUS_NUM(alias), alias & 0xff); } static int -domain_context_mapping(struct dmar_domain *domain, struct device *dev, - int translation) +domain_context_mapping(struct dmar_domain *domain, struct device *dev) { struct intel_iommu *iommu; u8 bus, devfn; @@ -2039,12 +2044,10 @@ domain_context_mapping(struct dmar_domain *domain, struct device *dev, return -ENODEV; if (!dev_is_pci(dev)) - return domain_context_mapping_one(domain, iommu, bus, devfn, - translation); + return domain_context_mapping_one(domain, iommu, bus, devfn); data.domain = domain; data.iommu = iommu; - data.translation = translation; return pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_mapping_cb, &data); @@ -2511,7 +2514,7 @@ static int iommu_prepare_identity_map(struct device *dev, goto error; /* context entry init */ - ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL); + ret = domain_context_mapping(domain, dev); if (ret) goto error; @@ -2624,8 +2627,7 @@ static int identity_mapping(struct device *dev) return 0; } -static int domain_add_dev_info(struct dmar_domain *domain, - struct device *dev, int translation) +static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) { struct dmar_domain *ndomain; struct intel_iommu *iommu; @@ -2640,7 +2642,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, if (ndomain != domain) return -EBUSY; - ret = domain_context_mapping(domain, dev, translation); + ret = domain_context_mapping(domain, dev); if (ret) { domain_remove_one_dev_info(domain, dev); return ret; @@ -2785,9 +2787,7 @@ static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw if (!iommu_should_identity_map(dev, 1)) return 0; - ret = domain_add_dev_info(si_domain, dev, - hw ? CONTEXT_TT_PASS_THROUGH : - CONTEXT_TT_MULTI_LEVEL); + ret = domain_add_dev_info(si_domain, dev); if (!ret) pr_info("%s identity mapping for device %s\n", hw ? "Hardware" : "Software", dev_name(dev)); @@ -3314,7 +3314,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) /* make sure context mapping is ok */ if (unlikely(!domain_context_mapped(dev))) { - ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL); + ret = domain_context_mapping(domain, dev); if (ret) { pr_err("Domain context map for %s failed\n", dev_name(dev)); @@ -3369,10 +3369,7 @@ static int iommu_no_mapping(struct device *dev) */ if (iommu_should_identity_map(dev, 0)) { int ret; - ret = domain_add_dev_info(si_domain, dev, - hw_pass_through ? - CONTEXT_TT_PASS_THROUGH : - CONTEXT_TT_MULTI_LEVEL); + ret = domain_add_dev_info(si_domain, dev); if (!ret) { pr_info("64bit %s uses identity mapping\n", dev_name(dev)); @@ -4810,7 +4807,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, dmar_domain->agaw--; } - return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL); + return domain_add_dev_info(dmar_domain, dev); } static void intel_iommu_detach_device(struct iommu_domain *domain, -- cgit v0.10.2 From de24e55395698e29f2a0582ae1899fa0001f829a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 14:53:04 +0200 Subject: iommu/vt-d: Simplify domain_context_mapping_one Get rid of the special cases for VM domains vs. non-VM domains and simplify the code further to just handle the hardware passthrough vs. page-table case. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ca6ca3d..5d4261ff6 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1942,52 +1942,44 @@ static int domain_context_mapping_one(struct dmar_domain *domain, return 0; } - context_clear_entry(context); - - id = domain->id; pgd = domain->pgd; - if (domain_type_is_vm_or_si(domain)) { - if (domain_type_is_vm(domain)) { - id = __iommu_attach_domain(domain, iommu); - if (id < 0) { - spin_unlock_irqrestore(&iommu->lock, flags); - pr_err("%s: No free domain ids\n", iommu->name); - return -EFAULT; - } - } - - /* Skip top levels of page tables for - * iommu which has less agaw than default. - * Unnecessary for PT mode. - */ - if (translation != CONTEXT_TT_PASS_THROUGH) { - for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) { - pgd = phys_to_virt(dma_pte_addr(pgd)); - if (!dma_pte_present(pgd)) { - spin_unlock_irqrestore(&iommu->lock, flags); - return -ENOMEM; - } - } - } + id = __iommu_attach_domain(domain, iommu); + if (id < 0) { + spin_unlock_irqrestore(&iommu->lock, flags); + pr_err("%s: No free domain ids\n", iommu->name); + return -EFAULT; } + context_clear_entry(context); context_set_domain_id(context, id); + /* + * Skip top levels of page tables for iommu which has less agaw + * than default. Unnecessary for PT mode. + */ if (translation != CONTEXT_TT_PASS_THROUGH) { + for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) { + pgd = phys_to_virt(dma_pte_addr(pgd)); + if (!dma_pte_present(pgd)) { + spin_unlock_irqrestore(&iommu->lock, flags); + return -ENOMEM; + } + } + info = iommu_support_dev_iotlb(domain, iommu, bus, devfn); translation = info ? CONTEXT_TT_DEV_IOTLB : CONTEXT_TT_MULTI_LEVEL; - } - /* - * In pass through mode, AW must be programmed to indicate the largest - * AGAW value supported by hardware. And ASR is ignored by hardware. - */ - if (unlikely(translation == CONTEXT_TT_PASS_THROUGH)) - context_set_address_width(context, iommu->msagaw); - else { + context_set_address_root(context, virt_to_phys(pgd)); context_set_address_width(context, iommu->agaw); + } else { + /* + * In pass through mode, AW must be programmed to + * indicate the largest AGAW value supported by + * hardware. And ASR is ignored by hardware. + */ + context_set_address_width(context, iommu->msagaw); } context_set_translation_type(context, translation); -- cgit v0.10.2 From a1ddcbe9301023928f877b675a40914427928f2a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 15:20:32 +0200 Subject: iommu/vt-d: Pass dmar_domain directly into iommu_flush_iotlb_psi This function can figure out the domain-id to use itself from the iommu_did array. This is more reliable over different domain types and brings us one step further to remove the domain->id field. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5d4261ff6..380b4e2 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1471,11 +1471,14 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, spin_unlock_irqrestore(&device_domain_lock, flags); } -static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did, - unsigned long pfn, unsigned int pages, int ih, int map) +static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, + struct dmar_domain *domain, + unsigned long pfn, unsigned int pages, + int ih, int map) { unsigned int mask = ilog2(__roundup_pow_of_two(pages)); uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; + u16 did = domain->iommu_did[iommu->seq_id]; BUG_ON(pages == 0); @@ -3422,7 +3425,9 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, /* it's a non-present to present mapping. Only flush if caching mode */ if (cap_caching_mode(iommu->cap)) - iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1); + iommu_flush_iotlb_psi(iommu, domain, + mm_to_dma_pfn(iova->pfn_lo), + size, 0, 1); else iommu_flush_write_buffer(iommu); @@ -3473,7 +3478,7 @@ static void flush_unmaps(void) /* On real hardware multiple invalidations are expensive */ if (cap_caching_mode(iommu->cap)) - iommu_flush_iotlb_psi(iommu, domain->id, + iommu_flush_iotlb_psi(iommu, domain, iova->pfn_lo, iova_size(iova), !deferred_flush[i].freelist[j], 0); else { @@ -3557,7 +3562,7 @@ static void intel_unmap(struct device *dev, dma_addr_t dev_addr) freelist = domain_unmap(domain, start_pfn, last_pfn); if (intel_iommu_strict) { - iommu_flush_iotlb_psi(iommu, domain->id, start_pfn, + iommu_flush_iotlb_psi(iommu, domain, start_pfn, last_pfn - start_pfn + 1, !freelist, 0); /* free iova */ __free_iova(&domain->iovad, iova); @@ -3715,7 +3720,7 @@ static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nele /* it's a non-present to present mapping. Only flush if caching mode */ if (cap_caching_mode(iommu->cap)) - iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1); + iommu_flush_iotlb_psi(iommu, domain, start_vpfn, size, 0, 1); else iommu_flush_write_buffer(iommu); @@ -4421,7 +4426,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, rcu_read_lock(); for_each_active_iommu(iommu, drhd) - iommu_flush_iotlb_psi(iommu, si_domain->id, + iommu_flush_iotlb_psi(iommu, si_domain, iova->pfn_lo, iova_size(iova), !freelist, 0); rcu_read_unlock(); @@ -4872,17 +4877,18 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, npages = last_pfn - start_pfn + 1; for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) { - iommu = g_iommus[iommu_id]; - - /* - * find bit position of dmar_domain - */ - ndomains = cap_ndoms(iommu->cap); - for_each_set_bit(num, iommu->domain_ids, ndomains) { - if (get_iommu_domain(iommu, num) == dmar_domain) - iommu_flush_iotlb_psi(iommu, num, start_pfn, - npages, !freelist, 0); - } + iommu = g_iommus[iommu_id]; + + /* + * find bit position of dmar_domain + */ + ndomains = cap_ndoms(iommu->cap); + for_each_set_bit(num, iommu->domain_ids, ndomains) { + if (get_iommu_domain(iommu, num) == dmar_domain) + iommu_flush_iotlb_psi(iommu, dmar_domain, + start_pfn, npages, + !freelist, 0); + } } -- cgit v0.10.2 From 0dc7971594aad73b50722878ea7175055a4fdfcd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 15:40:06 +0200 Subject: iommu/vt-d: Don't pre-allocate domain ids for si_domain There is no reason for this special handling of the si_domain. The per-iommu domain-id can be allocated on-demand like for any other domain. So remove the pre-allocation code. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 380b4e2..2a64c3f 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2558,37 +2558,18 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width); static int __init si_domain_init(int hw) { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; int nid, ret = 0; - bool first = true; si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY); if (!si_domain) return -EFAULT; - for_each_active_iommu(iommu, drhd) { - ret = iommu_attach_domain(si_domain, iommu); - if (ret < 0) { - domain_exit(si_domain); - return -EFAULT; - } else if (first) { - si_domain->id = ret; - first = false; - } else if (si_domain->id != ret) { - domain_exit(si_domain); - return -EFAULT; - } - domain_attach_iommu(si_domain, iommu); - } - if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { domain_exit(si_domain); return -EFAULT; } - pr_debug("Identity mapping domain is domain %d\n", - si_domain->id); + pr_debug("Identity mapping domain allocated\n"); if (hw) return 0; @@ -4197,13 +4178,6 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); iommu_enable_translation(iommu); - if (si_domain) { - ret = iommu_attach_domain(si_domain, iommu); - if (ret < 0 || si_domain->id != ret) - goto disable_iommu; - domain_attach_iommu(si_domain, iommu); - } - iommu_disable_protect_mem_regions(iommu); return 0; -- cgit v0.10.2 From af1089ce388b2d14c8331b96567b7e3b7eb5f35b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 15:45:19 +0200 Subject: iommu/vt-d: Kill dmar_domain->id This field is now obsolete because all places use the per-iommu domain-ids. Kill the remaining uses of this field and remove it. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 2a64c3f..91f0c3d 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -374,7 +374,6 @@ static int hw_pass_through = 1; #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1) struct dmar_domain { - int id; /* domain id */ int nid; /* node id */ DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED); /* bitmap of iommus this domain uses*/ @@ -1655,8 +1654,6 @@ static void free_dmar_iommu(struct intel_iommu *iommu) static struct dmar_domain *alloc_domain(int flags) { - /* domain id for virtual machine, it won't be set in context */ - static atomic_t vm_domid = ATOMIC_INIT(0); struct dmar_domain *domain; domain = alloc_domain_mem(); @@ -1668,8 +1665,6 @@ static struct dmar_domain *alloc_domain(int flags) domain->flags = flags; spin_lock_init(&domain->iommu_lock); INIT_LIST_HEAD(&domain->devices); - if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE) - domain->id = atomic_inc_return(&vm_domid); return domain; } @@ -2392,8 +2387,7 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) domain = alloc_domain(0); if (!domain) return NULL; - domain->id = iommu_attach_domain(domain, iommu); - if (domain->id < 0) { + if (iommu_attach_domain(domain, iommu) < 0) { free_domain_mem(domain); return NULL; } @@ -2446,8 +2440,7 @@ static int iommu_domain_identity_map(struct dmar_domain *domain, return -ENOMEM; } - pr_debug("Mapping reserved region %llx-%llx for domain %d\n", - start, end, domain->id); + pr_debug("Mapping reserved region %llx-%llx\n", start, end); /* * RMRR range might have overlap with physical memory range, * clear it first -- cgit v0.10.2 From 29a27719abaa4d74aed928803c1aa9437bbdde89 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 17:17:12 +0200 Subject: iommu/vt-d: Replace iommu_bmp with a refcount This replaces the dmar_domain->iommu_bmp with a similar reference count array. This allows us to keep track of how many devices behind each iommu are attached to the domain. This is necessary for further simplifications and optimizations to the iommu<->domain attachment code. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 91f0c3d..a5aa957 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -373,10 +373,16 @@ static int hw_pass_through = 1; /* si_domain contains mulitple devices */ #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1) +#define for_each_domain_iommu(idx, domain) \ + for (idx = 0; idx < g_num_of_iommus; idx++) \ + if (domain->iommu_refcnt[idx]) + struct dmar_domain { int nid; /* node id */ - DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED); - /* bitmap of iommus this domain uses*/ + + unsigned iommu_refcnt[DMAR_UNITS_SUPPORTED]; + /* Refcount of devices per iommu */ + u16 iommu_did[DMAR_UNITS_SUPPORTED]; /* Domain ids per IOMMU. Use u16 since @@ -699,7 +705,9 @@ static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain) /* si_domain and vm domain should not get here. */ BUG_ON(domain_type_is_vm_or_si(domain)); - iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus); + for_each_domain_iommu(iommu_id, domain) + break; + if (iommu_id < 0 || iommu_id >= g_num_of_iommus) return NULL; @@ -715,7 +723,7 @@ static void domain_update_iommu_coherency(struct dmar_domain *domain) domain->iommu_coherency = 1; - for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) { + for_each_domain_iommu(i, domain) { found = true; if (!ecap_coherent(g_iommus[i]->ecap)) { domain->iommu_coherency = 0; @@ -1607,25 +1615,26 @@ static int iommu_init_domains(struct intel_iommu *iommu) static void disable_dmar_iommu(struct intel_iommu *iommu) { - struct dmar_domain *domain; - int i; + struct device_domain_info *info, *tmp; - if ((iommu->domains) && (iommu->domain_ids)) { - for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) { - /* - * Domain id 0 is reserved for invalid translation - * if hardware supports caching mode and used as - * a non-allocated marker. - */ - if (i == 0) - continue; + if (!iommu->domains || !iommu->domain_ids) + return; - domain = get_iommu_domain(iommu, i); - clear_bit(i, iommu->domain_ids); - if (domain_detach_iommu(domain, iommu) == 0 && - !domain_type_is_vm(domain)) - domain_exit(domain); - } + list_for_each_entry_safe(info, tmp, &device_domain_list, global) { + struct dmar_domain *domain; + + if (info->iommu != iommu) + continue; + + if (!info->dev || !info->domain) + continue; + + domain = info->domain; + + domain_remove_one_dev_info(domain, info->dev); + + if (!domain_type_is_vm_or_si(domain)) + domain_exit(domain); } if (iommu->gcmd & DMA_GCMD_TE) @@ -1734,10 +1743,10 @@ static void domain_attach_iommu(struct dmar_domain *domain, unsigned long flags; spin_lock_irqsave(&domain->iommu_lock, flags); - if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) { - domain->iommu_count++; - if (domain->iommu_count == 1) - domain->nid = iommu->node; + domain->iommu_refcnt[iommu->seq_id] += 1; + domain->iommu_count += 1; + if (domain->iommu_refcnt[iommu->seq_id] == 1) { + domain->nid = iommu->node; domain_update_iommu_cap(domain); } spin_unlock_irqrestore(&domain->iommu_lock, flags); @@ -1750,8 +1759,9 @@ static int domain_detach_iommu(struct dmar_domain *domain, int count = INT_MAX; spin_lock_irqsave(&domain->iommu_lock, flags); - if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) { - count = --domain->iommu_count; + domain->iommu_refcnt[iommu->seq_id] -= 1; + count = --domain->iommu_count; + if (domain->iommu_refcnt[iommu->seq_id] == 0) { domain_update_iommu_cap(domain); domain->iommu_did[iommu->seq_id] = 0; } @@ -1876,9 +1886,8 @@ static int domain_init(struct dmar_domain *domain, int guest_width) static void domain_exit(struct dmar_domain *domain) { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; struct page *freelist = NULL; + int i; /* Domain 0 is reserved, so dont process it */ if (!domain) @@ -1898,10 +1907,8 @@ static void domain_exit(struct dmar_domain *domain) /* clear attached or cached domains */ rcu_read_lock(); - for_each_active_iommu(iommu, drhd) - if (domain_type_is_vm(domain) || - test_bit(iommu->seq_id, domain->iommu_bmp)) - iommu_detach_domain(domain, iommu); + for_each_domain_iommu(i, domain) + iommu_detach_domain(domain, g_iommus[i]); rcu_read_unlock(); dma_free_pagelist(freelist); @@ -4633,9 +4640,10 @@ static void domain_remove_one_dev_info(struct dmar_domain *domain, continue; } - /* if there is no other devices under the same iommu - * owned by this domain, clear this iommu in iommu_bmp - * update iommu count and coherency + /* + * If there is no other devices under the same iommu owned by + * this domain, clear this iommu in iommu_refcnt update iommu + * count and coherency. */ if (info->iommu == iommu) found = true; @@ -4843,7 +4851,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, npages = last_pfn - start_pfn + 1; - for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) { + for_each_domain_iommu(iommu_id, dmar_domain) { iommu = g_iommus[iommu_id]; /* -- cgit v0.10.2 From 42e8c186b595a32918933b3dec445f0bf0f486f6 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 15:50:02 +0200 Subject: iommu/vt-d: Simplify io/tlb flushing in intel_iommu_unmap We don't need to do an expensive search for domain-ids anymore, as we keep track of per-iommu domain-ids. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a5aa957..57c115c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4834,7 +4834,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, struct intel_iommu *iommu; unsigned long start_pfn, last_pfn; unsigned int npages; - int iommu_id, num, ndomains, level = 0; + int iommu_id, level = 0; /* Cope with horrid API which requires us to unmap more than the size argument if it happens to be a large-page mapping. */ @@ -4854,17 +4854,8 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, for_each_domain_iommu(iommu_id, dmar_domain) { iommu = g_iommus[iommu_id]; - /* - * find bit position of dmar_domain - */ - ndomains = cap_ndoms(iommu->cap); - for_each_set_bit(num, iommu->domain_ids, ndomains) { - if (get_iommu_domain(iommu, num) == dmar_domain) - iommu_flush_iotlb_psi(iommu, dmar_domain, - start_pfn, npages, - !freelist, 0); - } - + iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain, + start_pfn, npages, !freelist, 0); } dma_free_pagelist(freelist); -- cgit v0.10.2 From b608ac3b6d54c38d6cf0eb91547f0f960633eb2d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 18:19:08 +0200 Subject: iommu/vt-d: Simplify domain_remove_one_dev_info() Simplify this function as much as possible with the new iommu_refcnt field. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 57c115c..d241353 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4610,52 +4610,34 @@ static void iommu_detach_dependent_devices(struct intel_iommu *iommu, static void domain_remove_one_dev_info(struct dmar_domain *domain, struct device *dev) { - struct device_domain_info *info, *tmp; + struct device_domain_info *info; struct intel_iommu *iommu; unsigned long flags; - bool found = false; u8 bus, devfn; iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) return; - spin_lock_irqsave(&device_domain_lock, flags); - list_for_each_entry_safe(info, tmp, &domain->devices, link) { - if (info->iommu == iommu && info->bus == bus && - info->devfn == devfn) { - unlink_domain_info(info); - spin_unlock_irqrestore(&device_domain_lock, flags); - - iommu_disable_dev_iotlb(info); - iommu_detach_dev(iommu, info->bus, info->devfn); - iommu_detach_dependent_devices(iommu, dev); - free_devinfo_mem(info); - - spin_lock_irqsave(&device_domain_lock, flags); - - if (found) - break; - else - continue; - } + info = dev->archdata.iommu; - /* - * If there is no other devices under the same iommu owned by - * this domain, clear this iommu in iommu_refcnt update iommu - * count and coherency. - */ - if (info->iommu == iommu) - found = true; - } + if (WARN_ON(!info)) + return; + spin_lock_irqsave(&device_domain_lock, flags); + unlink_domain_info(info); spin_unlock_irqrestore(&device_domain_lock, flags); - if (found == 0) { - domain_detach_iommu(domain, iommu); - if (!domain_type_is_vm_or_si(domain)) - iommu_detach_domain(domain, iommu); - } + iommu_disable_dev_iotlb(info); + iommu_detach_dev(iommu, info->bus, info->devfn); + iommu_detach_dependent_devices(iommu, dev); + free_devinfo_mem(info); + domain_detach_iommu(domain, iommu); + + spin_lock_irqsave(&domain->iommu_lock, flags); + if (!domain->iommu_refcnt[iommu->seq_id]) + iommu_detach_domain(domain, iommu); + spin_unlock_irqrestore(&domain->iommu_lock, flags); } static int md_domain_init(struct dmar_domain *domain, int guest_width) -- cgit v0.10.2 From 76f45fe35c7a54e6fe5539660db2c8cfb23a2972 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 21 Jul 2015 18:25:11 +0200 Subject: iommu/vt-d: Simplify domain_remove_dev_info() Just call domain_remove_one_dev_info() for all devices in the domain instead of reimplementing the functionality. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d241353..c674aa1 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2253,25 +2253,9 @@ static inline void unlink_domain_info(struct device_domain_info *info) static void domain_remove_dev_info(struct dmar_domain *domain) { struct device_domain_info *info, *tmp; - unsigned long flags; - - spin_lock_irqsave(&device_domain_lock, flags); - list_for_each_entry_safe(info, tmp, &domain->devices, link) { - unlink_domain_info(info); - spin_unlock_irqrestore(&device_domain_lock, flags); - - iommu_disable_dev_iotlb(info); - iommu_detach_dev(info->iommu, info->bus, info->devfn); - - if (domain_type_is_vm(domain)) { - iommu_detach_dependent_devices(info->iommu, info->dev); - domain_detach_iommu(domain, info->iommu); - } - free_devinfo_mem(info); - spin_lock_irqsave(&device_domain_lock, flags); - } - spin_unlock_irqrestore(&device_domain_lock, flags); + list_for_each_entry_safe(info, tmp, &domain->devices, link) + domain_remove_one_dev_info(domain, info->dev); } /* -- cgit v0.10.2 From cc4e2575cc96b1aac910f56e1d7ef45d219b40b2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 10:04:36 +0200 Subject: iommu/vt-d: Move context-mapping into dmar_insert_dev_info Do the context-mapping of devices from a single place in the call-path and clean up the other call-sites. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index c674aa1..a5ac99c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2327,6 +2327,12 @@ static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu, dev->archdata.iommu = info; spin_unlock_irqrestore(&device_domain_lock, flags); + if (dev && domain_context_mapping(domain, dev)) { + pr_err("Domain context map for %s failed\n", dev_name(dev)); + domain_remove_one_dev_info(domain, dev); + return NULL; + } + return domain; } @@ -2339,11 +2345,11 @@ static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque) /* domain is initialized */ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) { + struct device_domain_info *info = NULL; struct dmar_domain *domain, *tmp; struct intel_iommu *iommu; - struct device_domain_info *info; - u16 dma_alias; unsigned long flags; + u16 dma_alias; u8 bus, devfn; domain = find_domain(dev); @@ -2492,11 +2498,6 @@ static int iommu_prepare_identity_map(struct device *dev, if (ret) goto error; - /* context entry init */ - ret = domain_context_mapping(domain, dev); - if (ret) - goto error; - return 0; error: @@ -2592,7 +2593,6 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) struct dmar_domain *ndomain; struct intel_iommu *iommu; u8 bus, devfn; - int ret; iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) @@ -2602,12 +2602,6 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) if (ndomain != domain) return -EBUSY; - ret = domain_context_mapping(domain, dev); - if (ret) { - domain_remove_one_dev_info(domain, dev); - return ret; - } - return 0; } @@ -3263,7 +3257,6 @@ static struct iova *intel_alloc_iova(struct device *dev, static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) { struct dmar_domain *domain; - int ret; domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); if (!domain) { @@ -3272,16 +3265,6 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) return NULL; } - /* make sure context mapping is ok */ - if (unlikely(!domain_context_mapped(dev))) { - ret = domain_context_mapping(domain, dev); - if (ret) { - pr_err("Domain context map for %s failed\n", - dev_name(dev)); - return NULL; - } - } - return domain; } -- cgit v0.10.2 From 5db31569e9503654477b504de7161d01f85f7261 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 12:40:43 +0200 Subject: iommu/vt-d: Rename dmar_insert_dev_info() Rename this function to dmar_insert_one_dev_info() to match the name better with its counter part function domain_remove_one_dev_info(). Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a5ac99c..4a7fc0a 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2286,10 +2286,10 @@ dmar_search_domain_by_dev_info(int segment, int bus, int devfn) return NULL; } -static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu, - int bus, int devfn, - struct device *dev, - struct dmar_domain *domain) +static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, + int bus, int devfn, + struct device *dev, + struct dmar_domain *domain) { struct dmar_domain *found = NULL; struct device_domain_info *info; @@ -2396,8 +2396,8 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) /* register PCI DMA alias device */ if (dev_is_pci(dev)) { - tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias), - dma_alias & 0xff, NULL, domain); + tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias), + dma_alias & 0xff, NULL, domain); if (!tmp || tmp != domain) { domain_exit(domain); @@ -2409,7 +2409,7 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) } found_domain: - tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain); + tmp = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain); if (!tmp || tmp != domain) { domain_exit(domain); @@ -2598,7 +2598,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) if (!iommu) return -ENODEV; - ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain); + ndomain = dmar_insert_one_dev_info(iommu, bus, devfn, dev, domain); if (ndomain != domain) return -EBUSY; -- cgit v0.10.2 From e6de0f8dfcd0395efee874db97536531555d91af Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 16:30:36 +0200 Subject: iommu/vt-d: Rename domain_remove_one_dev_info() Rename the function to dmar_remove_one_dev_info to match is name better with its dmar_insert_one_dev_info counterpart. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 4a7fc0a..faa95d1 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -472,8 +472,8 @@ static long list_size; static void domain_exit(struct dmar_domain *domain); static void domain_remove_dev_info(struct dmar_domain *domain); -static void domain_remove_one_dev_info(struct dmar_domain *domain, - struct device *dev); +static void dmar_remove_one_dev_info(struct dmar_domain *domain, + struct device *dev); static void iommu_detach_dependent_devices(struct intel_iommu *iommu, struct device *dev); static int domain_detach_iommu(struct dmar_domain *domain, @@ -1631,7 +1631,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) domain = info->domain; - domain_remove_one_dev_info(domain, info->dev); + dmar_remove_one_dev_info(domain, info->dev); if (!domain_type_is_vm_or_si(domain)) domain_exit(domain); @@ -2255,7 +2255,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain) struct device_domain_info *info, *tmp; list_for_each_entry_safe(info, tmp, &domain->devices, link) - domain_remove_one_dev_info(domain, info->dev); + dmar_remove_one_dev_info(domain, info->dev); } /* @@ -2329,7 +2329,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, if (dev && domain_context_mapping(domain, dev)) { pr_err("Domain context map for %s failed\n", dev_name(dev)); - domain_remove_one_dev_info(domain, dev); + dmar_remove_one_dev_info(domain, dev); return NULL; } @@ -3300,7 +3300,7 @@ static int iommu_no_mapping(struct device *dev) * 32 bit DMA is removed from si_domain and fall back * to non-identity mapping. */ - domain_remove_one_dev_info(si_domain, dev); + dmar_remove_one_dev_info(si_domain, dev); pr_info("32bit %s uses non-identity mapping\n", dev_name(dev)); return 0; @@ -4307,7 +4307,7 @@ static int device_notifier(struct notifier_block *nb, return 0; down_read(&dmar_global_lock); - domain_remove_one_dev_info(domain, dev); + dmar_remove_one_dev_info(domain, dev); if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices)) domain_exit(domain); up_read(&dmar_global_lock); @@ -4574,8 +4574,8 @@ static void iommu_detach_dependent_devices(struct intel_iommu *iommu, pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu); } -static void domain_remove_one_dev_info(struct dmar_domain *domain, - struct device *dev) +static void dmar_remove_one_dev_info(struct dmar_domain *domain, + struct device *dev) { struct device_domain_info *info; struct intel_iommu *iommu; @@ -4686,7 +4686,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, old_domain = find_domain(dev); if (old_domain) { if (domain_type_is_vm_or_si(dmar_domain)) - domain_remove_one_dev_info(old_domain, dev); + dmar_remove_one_dev_info(old_domain, dev); else domain_remove_dev_info(old_domain); @@ -4734,7 +4734,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, static void intel_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - domain_remove_one_dev_info(to_dmar_domain(domain), dev); + dmar_remove_one_dev_info(to_dmar_domain(domain), dev); } static int intel_iommu_map(struct iommu_domain *domain, -- cgit v0.10.2 From 2452d9db1218fdb1c29afb921838c323987f5799 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 23 Jul 2015 16:20:14 +0200 Subject: iommu/vt-d: Rename iommu_detach_dependent_devices() Rename this function and the ones further down its call-chain to domain_context_clear_*. In particular this means: iommu_detach_dependent_devices -> domain_context_clear iommu_detach_dev_cb -> domain_context_clear_one_cb iommu_detach_dev -> domain_context_clear_one These names match a lot better with its domain_context_mapping counterparts. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index faa95d1..62c27ef 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -474,8 +474,8 @@ static void domain_exit(struct dmar_domain *domain); static void domain_remove_dev_info(struct dmar_domain *domain); static void dmar_remove_one_dev_info(struct dmar_domain *domain, struct device *dev); -static void iommu_detach_dependent_devices(struct intel_iommu *iommu, - struct device *dev); +static void domain_context_clear(struct intel_iommu *iommu, + struct device *dev); static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu); @@ -2230,7 +2230,7 @@ static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long i return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot); } -static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn) +static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn) { if (!iommu) return; @@ -4551,11 +4551,11 @@ out_free_dmar: return ret; } -static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque) +static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque) { struct intel_iommu *iommu = opaque; - iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff); + domain_context_clear_one(iommu, PCI_BUS_NUM(alias), alias & 0xff); return 0; } @@ -4565,13 +4565,12 @@ static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque) * devices, unbinding the driver from any one of them will possibly leave * the others unable to operate. */ -static void iommu_detach_dependent_devices(struct intel_iommu *iommu, - struct device *dev) +static void domain_context_clear(struct intel_iommu *iommu, struct device *dev) { if (!iommu || !dev || !dev_is_pci(dev)) return; - pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu); + pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu); } static void dmar_remove_one_dev_info(struct dmar_domain *domain, @@ -4596,8 +4595,7 @@ static void dmar_remove_one_dev_info(struct dmar_domain *domain, spin_unlock_irqrestore(&device_domain_lock, flags); iommu_disable_dev_iotlb(info); - iommu_detach_dev(iommu, info->bus, info->devfn); - iommu_detach_dependent_devices(iommu, dev); + domain_context_clear(iommu, dev); free_devinfo_mem(info); domain_detach_iommu(domain, iommu); -- cgit v0.10.2 From dc534b25d11f42e6b4caa5b1918f549d9c0e9d4d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 12:44:02 +0200 Subject: iommu/vt-d: Pass an iommu pointer to domain_init() This allows to do domain->iommu attachment after domain_init has run. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 62c27ef..de5384e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1832,9 +1832,9 @@ static inline int guestwidth_to_adjustwidth(int gaw) return agaw; } -static int domain_init(struct dmar_domain *domain, int guest_width) +static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu, + int guest_width) { - struct intel_iommu *iommu; int adjust_width, agaw; unsigned long sagaw; @@ -1843,7 +1843,6 @@ static int domain_init(struct dmar_domain *domain, int guest_width) domain_reserve_special_ranges(domain); /* calculate AGAW */ - iommu = domain_get_iommu(domain); if (guest_width > cap_mgaw(iommu->cap)) guest_width = cap_mgaw(iommu->cap); domain->gaw = guest_width; @@ -2389,7 +2388,7 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) return NULL; } domain_attach_iommu(domain, iommu); - if (domain_init(domain, gaw)) { + if (domain_init(domain, iommu, gaw)) { domain_exit(domain); return NULL; } -- cgit v0.10.2 From c6c2cebd665933216785246a1d15b4112fa74bbf Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 13:11:53 +0200 Subject: iommu/vt-d: Establish domain<->iommu link in dmar_insert_one_dev_info This makes domain attachment more synchronous with domain deattachment. The domain<->iommu link is released in dmar_remove_one_dev_info. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index de5384e..46359bd 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1919,14 +1919,16 @@ static int domain_context_mapping_one(struct dmar_domain *domain, struct intel_iommu *iommu, u8 bus, u8 devfn) { + u16 did = domain->iommu_did[iommu->seq_id]; int translation = CONTEXT_TT_MULTI_LEVEL; struct device_domain_info *info = NULL; struct context_entry *context; unsigned long flags; struct dma_pte *pgd; - int id; int agaw; + WARN_ON(did == 0); + if (hw_pass_through && domain_type_is_si(domain)) translation = CONTEXT_TT_PASS_THROUGH; @@ -1948,15 +1950,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain, pgd = domain->pgd; - id = __iommu_attach_domain(domain, iommu); - if (id < 0) { - spin_unlock_irqrestore(&iommu->lock, flags); - pr_err("%s: No free domain ids\n", iommu->name); - return -EFAULT; - } - context_clear_entry(context); - context_set_domain_id(context, id); + context_set_domain_id(context, did); /* * Skip top levels of page tables for iommu which has less agaw @@ -2002,15 +1997,13 @@ static int domain_context_mapping_one(struct dmar_domain *domain, (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, DMA_CCMD_DEVICE_INVL); - iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH); + iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH); } else { iommu_flush_write_buffer(iommu); } iommu_enable_dev_iotlb(info); spin_unlock_irqrestore(&iommu->lock, flags); - domain_attach_iommu(domain, iommu); - return 0; } @@ -2320,6 +2313,12 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, return found; } + if (iommu_attach_domain(domain, iommu) < 0) { + spin_unlock_irqrestore(&device_domain_lock, flags); + return NULL; + } + domain_attach_iommu(domain, iommu); + list_add(&info->link, &domain->devices); list_add(&info->global, &device_domain_list); if (dev) @@ -2383,11 +2382,6 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) domain = alloc_domain(0); if (!domain) return NULL; - if (iommu_attach_domain(domain, iommu) < 0) { - free_domain_mem(domain); - return NULL; - } - domain_attach_iommu(domain, iommu); if (domain_init(domain, iommu, gaw)) { domain_exit(domain); return NULL; -- cgit v0.10.2 From d160aca5276d093fc68d6ff48888586c90309d03 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 11:52:53 +0200 Subject: iommu/vt-d: Unify domain->iommu attach/detachment Move the code to attach/detach domains to iommus and vice verce into a single function to make sure there are no dangling references. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 46359bd..1cb7a3e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1678,90 +1678,64 @@ static struct dmar_domain *alloc_domain(int flags) return domain; } -static int __iommu_attach_domain(struct dmar_domain *domain, - struct intel_iommu *iommu) -{ - int num; - unsigned long ndomains; - - num = domain->iommu_did[iommu->seq_id]; - if (num) - return num; - - ndomains = cap_ndoms(iommu->cap); - num = find_first_zero_bit(iommu->domain_ids, ndomains); - - if (num < ndomains) { - set_bit(num, iommu->domain_ids); - set_iommu_domain(iommu, num, domain); - domain->iommu_did[iommu->seq_id] = num; - } else { - num = -ENOSPC; - } - - if (num < 0) - pr_err("%s: No free domain ids\n", iommu->name); - - return num; -} - -static int iommu_attach_domain(struct dmar_domain *domain, +/* Must be called with iommu->lock */ +static int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) { - int num; - unsigned long flags; - - spin_lock_irqsave(&iommu->lock, flags); - num = __iommu_attach_domain(domain, iommu); - spin_unlock_irqrestore(&iommu->lock, flags); - - return num; -} - -static void iommu_detach_domain(struct dmar_domain *domain, - struct intel_iommu *iommu) -{ + unsigned long ndomains; unsigned long flags; - int num; - - spin_lock_irqsave(&iommu->lock, flags); + int ret, num; - num = domain->iommu_did[iommu->seq_id]; - - if (num == 0) - return; - - clear_bit(num, iommu->domain_ids); - set_iommu_domain(iommu, num, NULL); - - spin_unlock_irqrestore(&iommu->lock, flags); -} - -static void domain_attach_iommu(struct dmar_domain *domain, - struct intel_iommu *iommu) -{ - unsigned long flags; + assert_spin_locked(&iommu->lock); spin_lock_irqsave(&domain->iommu_lock, flags); + domain->iommu_refcnt[iommu->seq_id] += 1; domain->iommu_count += 1; if (domain->iommu_refcnt[iommu->seq_id] == 1) { - domain->nid = iommu->node; + ndomains = cap_ndoms(iommu->cap); + num = find_first_zero_bit(iommu->domain_ids, ndomains); + + if (num >= ndomains) { + pr_err("%s: No free domain ids\n", iommu->name); + domain->iommu_refcnt[iommu->seq_id] -= 1; + domain->iommu_count -= 1; + ret = -ENOSPC; + goto out_unlock; + } + + set_bit(num, iommu->domain_ids); + set_iommu_domain(iommu, num, domain); + + domain->iommu_did[iommu->seq_id] = num; + domain->nid = iommu->node; + domain_update_iommu_cap(domain); } + + ret = 0; +out_unlock: spin_unlock_irqrestore(&domain->iommu_lock, flags); + + return ret; } static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) { + int num, count = INT_MAX; unsigned long flags; - int count = INT_MAX; + + assert_spin_locked(&iommu->lock); spin_lock_irqsave(&domain->iommu_lock, flags); domain->iommu_refcnt[iommu->seq_id] -= 1; count = --domain->iommu_count; if (domain->iommu_refcnt[iommu->seq_id] == 0) { + num = domain->iommu_did[iommu->seq_id]; + clear_bit(num, iommu->domain_ids); + set_iommu_domain(iommu, num, NULL); + domain_update_iommu_cap(domain); domain->iommu_did[iommu->seq_id] = 0; } @@ -1886,7 +1860,6 @@ static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu, static void domain_exit(struct dmar_domain *domain) { struct page *freelist = NULL; - int i; /* Domain 0 is reserved, so dont process it */ if (!domain) @@ -1896,20 +1869,16 @@ static void domain_exit(struct dmar_domain *domain) if (!intel_iommu_strict) flush_unmaps_timeout(0); - /* remove associated devices */ + /* Remove associated devices and clear attached or cached domains */ + rcu_read_lock(); domain_remove_dev_info(domain); + rcu_read_unlock(); /* destroy iovas */ put_iova_domain(&domain->iovad); freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); - /* clear attached or cached domains */ - rcu_read_lock(); - for_each_domain_iommu(i, domain) - iommu_detach_domain(domain, g_iommus[i]); - rcu_read_unlock(); - dma_free_pagelist(freelist); free_domain_mem(domain); @@ -2286,6 +2255,7 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, struct dmar_domain *found = NULL; struct device_domain_info *info; unsigned long flags; + int ret; info = alloc_devinfo_mem(); if (!info) @@ -2313,11 +2283,14 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, return found; } - if (iommu_attach_domain(domain, iommu) < 0) { + spin_lock(&iommu->lock); + ret = domain_attach_iommu(domain, iommu); + spin_unlock(&iommu->lock); + + if (ret) { spin_unlock_irqrestore(&device_domain_lock, flags); return NULL; } - domain_attach_iommu(domain, iommu); list_add(&info->link, &domain->devices); list_add(&info->global, &device_domain_list); @@ -4590,12 +4563,10 @@ static void dmar_remove_one_dev_info(struct dmar_domain *domain, iommu_disable_dev_iotlb(info); domain_context_clear(iommu, dev); free_devinfo_mem(info); - domain_detach_iommu(domain, iommu); - spin_lock_irqsave(&domain->iommu_lock, flags); - if (!domain->iommu_refcnt[iommu->seq_id]) - iommu_detach_domain(domain, iommu); - spin_unlock_irqrestore(&domain->iommu_lock, flags); + spin_lock_irqsave(&iommu->lock, flags); + domain_detach_iommu(domain, iommu); + spin_unlock_irqrestore(&iommu->lock, flags); } static int md_domain_init(struct dmar_domain *domain, int guest_width) @@ -4676,10 +4647,12 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, old_domain = find_domain(dev); if (old_domain) { + rcu_read_lock(); if (domain_type_is_vm_or_si(dmar_domain)) dmar_remove_one_dev_info(old_domain, dev); else domain_remove_dev_info(old_domain); + rcu_read_unlock(); if (!domain_type_is_vm_or_si(old_domain) && list_empty(&old_domain->devices)) -- cgit v0.10.2 From de7e888646466e6c32cdd41124c0164cfed4abcb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 11:58:07 +0200 Subject: iommu/vt-d: Only call domain_remove_one_dev_info to detach old domain There is no need to make a difference here between VM and non-VM domains, so simplify this code here. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1cb7a3e..c8d9bef 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4648,10 +4648,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, old_domain = find_domain(dev); if (old_domain) { rcu_read_lock(); - if (domain_type_is_vm_or_si(dmar_domain)) - dmar_remove_one_dev_info(old_domain, dev); - else - domain_remove_dev_info(old_domain); + dmar_remove_one_dev_info(old_domain, dev); rcu_read_unlock(); if (!domain_type_is_vm_or_si(old_domain) && -- cgit v0.10.2 From 55d940430ab91b89ff5fc7240555544d86475783 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 16:50:40 +0200 Subject: iommu/vt-d: Get rid of domain->iommu_lock When this lock is held the device_domain_lock is also required to make sure the device_domain_info does not vanish while in use. So this lock can be removed as it gives no additional protection. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index c8d9bef..0f258f0 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -406,7 +406,6 @@ struct dmar_domain { int iommu_superpage;/* Level of superpages supported: 0 == 4KiB (no superpages), 1 == 2MiB, 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */ - spinlock_t iommu_lock; /* protect iommu set in domain */ u64 max_addr; /* maximum mapped address */ struct iommu_domain domain; /* generic domain data structure for @@ -476,6 +475,8 @@ static void dmar_remove_one_dev_info(struct dmar_domain *domain, struct device *dev); static void domain_context_clear(struct intel_iommu *iommu, struct device *dev); +static void __dmar_remove_one_dev_info(struct dmar_domain *domain, + struct device *dev); static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu); @@ -1404,24 +1405,23 @@ iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu, u8 bus, u8 devfn) { bool found = false; - unsigned long flags; struct device_domain_info *info; struct pci_dev *pdev; + assert_spin_locked(&device_domain_lock); + if (!ecap_dev_iotlb_support(iommu->ecap)) return NULL; if (!iommu->qi) return NULL; - spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry(info, &domain->devices, link) if (info->iommu == iommu && info->bus == bus && info->devfn == devfn) { found = true; break; } - spin_unlock_irqrestore(&device_domain_lock, flags); if (!found || !info->dev || !dev_is_pci(info->dev)) return NULL; @@ -1616,10 +1616,12 @@ static int iommu_init_domains(struct intel_iommu *iommu) static void disable_dmar_iommu(struct intel_iommu *iommu) { struct device_domain_info *info, *tmp; + unsigned long flags; if (!iommu->domains || !iommu->domain_ids) return; + spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry_safe(info, tmp, &device_domain_list, global) { struct dmar_domain *domain; @@ -1636,6 +1638,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) if (!domain_type_is_vm_or_si(domain)) domain_exit(domain); } + spin_unlock_irqrestore(&device_domain_lock, flags); if (iommu->gcmd & DMA_GCMD_TE) iommu_disable_translation(iommu); @@ -1672,7 +1675,6 @@ static struct dmar_domain *alloc_domain(int flags) memset(domain, 0, sizeof(*domain)); domain->nid = -1; domain->flags = flags; - spin_lock_init(&domain->iommu_lock); INIT_LIST_HEAD(&domain->devices); return domain; @@ -1683,13 +1685,11 @@ static int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) { unsigned long ndomains; - unsigned long flags; - int ret, num; + int num; + assert_spin_locked(&device_domain_lock); assert_spin_locked(&iommu->lock); - spin_lock_irqsave(&domain->iommu_lock, flags); - domain->iommu_refcnt[iommu->seq_id] += 1; domain->iommu_count += 1; if (domain->iommu_refcnt[iommu->seq_id] == 1) { @@ -1700,8 +1700,7 @@ static int domain_attach_iommu(struct dmar_domain *domain, pr_err("%s: No free domain ids\n", iommu->name); domain->iommu_refcnt[iommu->seq_id] -= 1; domain->iommu_count -= 1; - ret = -ENOSPC; - goto out_unlock; + return -ENOSPC; } set_bit(num, iommu->domain_ids); @@ -1713,22 +1712,17 @@ static int domain_attach_iommu(struct dmar_domain *domain, domain_update_iommu_cap(domain); } - ret = 0; -out_unlock: - spin_unlock_irqrestore(&domain->iommu_lock, flags); - - return ret; + return 0; } static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu) { int num, count = INT_MAX; - unsigned long flags; + assert_spin_locked(&device_domain_lock); assert_spin_locked(&iommu->lock); - spin_lock_irqsave(&domain->iommu_lock, flags); domain->iommu_refcnt[iommu->seq_id] -= 1; count = --domain->iommu_count; if (domain->iommu_refcnt[iommu->seq_id] == 0) { @@ -1739,7 +1733,6 @@ static int domain_detach_iommu(struct dmar_domain *domain, domain_update_iommu_cap(domain); domain->iommu_did[iommu->seq_id] = 0; } - spin_unlock_irqrestore(&domain->iommu_lock, flags); return count; } @@ -1894,7 +1887,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, struct context_entry *context; unsigned long flags; struct dma_pte *pgd; - int agaw; + int ret, agaw; WARN_ON(did == 0); @@ -1906,16 +1899,17 @@ static int domain_context_mapping_one(struct dmar_domain *domain, BUG_ON(!domain->pgd); - spin_lock_irqsave(&iommu->lock, flags); + spin_lock_irqsave(&device_domain_lock, flags); + spin_lock(&iommu->lock); + + ret = -ENOMEM; context = iommu_context_addr(iommu, bus, devfn, 1); - spin_unlock_irqrestore(&iommu->lock, flags); if (!context) - return -ENOMEM; - spin_lock_irqsave(&iommu->lock, flags); - if (context_present(context)) { - spin_unlock_irqrestore(&iommu->lock, flags); - return 0; - } + goto out_unlock; + + ret = 0; + if (context_present(context)) + goto out_unlock; pgd = domain->pgd; @@ -1928,11 +1922,10 @@ static int domain_context_mapping_one(struct dmar_domain *domain, */ if (translation != CONTEXT_TT_PASS_THROUGH) { for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) { + ret = -ENOMEM; pgd = phys_to_virt(dma_pte_addr(pgd)); - if (!dma_pte_present(pgd)) { - spin_unlock_irqrestore(&iommu->lock, flags); - return -ENOMEM; - } + if (!dma_pte_present(pgd)) + goto out_unlock; } info = iommu_support_dev_iotlb(domain, iommu, bus, devfn); @@ -1971,7 +1964,12 @@ static int domain_context_mapping_one(struct dmar_domain *domain, iommu_flush_write_buffer(iommu); } iommu_enable_dev_iotlb(info); - spin_unlock_irqrestore(&iommu->lock, flags); + + ret = 0; + +out_unlock: + spin_unlock(&iommu->lock); + spin_unlock_irqrestore(&device_domain_lock, flags); return 0; } @@ -2214,9 +2212,12 @@ static inline void unlink_domain_info(struct device_domain_info *info) static void domain_remove_dev_info(struct dmar_domain *domain) { struct device_domain_info *info, *tmp; + unsigned long flags; + spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry_safe(info, tmp, &domain->devices, link) - dmar_remove_one_dev_info(domain, info->dev); + __dmar_remove_one_dev_info(domain, info->dev); + spin_unlock_irqrestore(&device_domain_lock, flags); } /* @@ -4539,14 +4540,16 @@ static void domain_context_clear(struct intel_iommu *iommu, struct device *dev) pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu); } -static void dmar_remove_one_dev_info(struct dmar_domain *domain, - struct device *dev) +static void __dmar_remove_one_dev_info(struct dmar_domain *domain, + struct device *dev) { struct device_domain_info *info; struct intel_iommu *iommu; unsigned long flags; u8 bus, devfn; + assert_spin_locked(&device_domain_lock); + iommu = device_to_iommu(dev, &bus, &devfn); if (!iommu) return; @@ -4556,9 +4559,7 @@ static void dmar_remove_one_dev_info(struct dmar_domain *domain, if (WARN_ON(!info)) return; - spin_lock_irqsave(&device_domain_lock, flags); unlink_domain_info(info); - spin_unlock_irqrestore(&device_domain_lock, flags); iommu_disable_dev_iotlb(info); domain_context_clear(iommu, dev); @@ -4569,6 +4570,16 @@ static void dmar_remove_one_dev_info(struct dmar_domain *domain, spin_unlock_irqrestore(&iommu->lock, flags); } +static void dmar_remove_one_dev_info(struct dmar_domain *domain, + struct device *dev) +{ + unsigned long flags; + + spin_lock_irqsave(&device_domain_lock, flags); + __dmar_remove_one_dev_info(domain, dev); + spin_unlock_irqrestore(&device_domain_lock, flags); +} + static int md_domain_init(struct dmar_domain *domain, int guest_width) { int adjust_width; -- cgit v0.10.2 From 2309bd793ead6d5e4ace611502aa87b3202856ca Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 22 Jul 2015 17:29:47 +0200 Subject: iommu/vt-d: Remove dmar_global_lock from device_notifier The code in the locked section does not touch anything protected by the dmar_global_lock. Remove it from there. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0f258f0..d55ef9d 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4273,11 +4273,9 @@ static int device_notifier(struct notifier_block *nb, if (!domain) return 0; - down_read(&dmar_global_lock); dmar_remove_one_dev_info(domain, dev); if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices)) domain_exit(domain); - up_read(&dmar_global_lock); return 0; } -- cgit v0.10.2 From 127c761598f7fbe7ffe6650cdc491eb57c5aaecd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 23 Jul 2015 17:44:46 +0200 Subject: iommu/vt-d: Pass device_domain_info to __dmar_remove_one_dev_info This struct contains all necessary information for the function already. Also handle the info->dev == NULL case while at it. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d55ef9d..18e4442 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -473,10 +473,9 @@ static void domain_exit(struct dmar_domain *domain); static void domain_remove_dev_info(struct dmar_domain *domain); static void dmar_remove_one_dev_info(struct dmar_domain *domain, struct device *dev); +static void __dmar_remove_one_dev_info(struct device_domain_info *info); static void domain_context_clear(struct intel_iommu *iommu, struct device *dev); -static void __dmar_remove_one_dev_info(struct dmar_domain *domain, - struct device *dev); static int domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu); @@ -2216,7 +2215,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain) spin_lock_irqsave(&device_domain_lock, flags); list_for_each_entry_safe(info, tmp, &domain->devices, link) - __dmar_remove_one_dev_info(domain, info->dev); + __dmar_remove_one_dev_info(info); spin_unlock_irqrestore(&device_domain_lock, flags); } @@ -4538,43 +4537,41 @@ static void domain_context_clear(struct intel_iommu *iommu, struct device *dev) pci_for_each_dma_alias(to_pci_dev(dev), &domain_context_clear_one_cb, iommu); } -static void __dmar_remove_one_dev_info(struct dmar_domain *domain, - struct device *dev) +static void __dmar_remove_one_dev_info(struct device_domain_info *info) { - struct device_domain_info *info; struct intel_iommu *iommu; unsigned long flags; - u8 bus, devfn; assert_spin_locked(&device_domain_lock); - iommu = device_to_iommu(dev, &bus, &devfn); - if (!iommu) + if (WARN_ON(!info)) return; - info = dev->archdata.iommu; + iommu = info->iommu; - if (WARN_ON(!info)) - return; + if (info->dev) { + iommu_disable_dev_iotlb(info); + domain_context_clear(iommu, info->dev); + } unlink_domain_info(info); - iommu_disable_dev_iotlb(info); - domain_context_clear(iommu, dev); - free_devinfo_mem(info); - spin_lock_irqsave(&iommu->lock, flags); - domain_detach_iommu(domain, iommu); + domain_detach_iommu(info->domain, iommu); spin_unlock_irqrestore(&iommu->lock, flags); + + free_devinfo_mem(info); } static void dmar_remove_one_dev_info(struct dmar_domain *domain, struct device *dev) { + struct device_domain_info *info; unsigned long flags; spin_lock_irqsave(&device_domain_lock, flags); - __dmar_remove_one_dev_info(domain, dev); + info = dev->archdata.iommu; + __dmar_remove_one_dev_info(info); spin_unlock_irqrestore(&device_domain_lock, flags); } -- cgit v0.10.2 From 08a7f456a759e971caf0cc13987a963de2b0ae7c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 23 Jul 2015 18:09:11 +0200 Subject: iommu/vt-d: Only insert alias dev_info if there is an alias For devices without an PCI alias there will be two device_domain_info structures added. Prevent that by checking if the alias is different from the device. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 18e4442..6e61b3e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2319,8 +2319,8 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) struct device_domain_info *info = NULL; struct dmar_domain *domain, *tmp; struct intel_iommu *iommu; + u16 req_id, dma_alias; unsigned long flags; - u16 dma_alias; u8 bus, devfn; domain = find_domain(dev); @@ -2331,6 +2331,8 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) if (!iommu) return NULL; + req_id = ((u16)bus << 8) | devfn; + if (dev_is_pci(dev)) { struct pci_dev *pdev = to_pci_dev(dev); @@ -2361,7 +2363,7 @@ static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw) } /* register PCI DMA alias device */ - if (dev_is_pci(dev)) { + if (req_id != dma_alias && dev_is_pci(dev)) { tmp = dmar_insert_one_dev_info(iommu, PCI_BUS_NUM(dma_alias), dma_alias & 0xff, NULL, domain); -- cgit v0.10.2 From f303e50766298feac17c8715e29ecd14b2c12680 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 23 Jul 2015 18:37:13 +0200 Subject: iommu/vt-d: Avoid duplicate device_domain_info structures When a 'struct device_domain_info' is created as an alias for another device, this struct will not be re-used when the real device is encountered. Fix that to avoid duplicate device_domain_info structures being added. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 6e61b3e..8834765 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2270,12 +2270,16 @@ static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, spin_lock_irqsave(&device_domain_lock, flags); if (dev) found = find_domain(dev); - else { + + if (!found) { struct device_domain_info *info2; info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn); - if (info2) - found = info2->domain; + if (info2) { + found = info2->domain; + info2->dev = dev; + } } + if (found) { spin_unlock_irqrestore(&device_domain_lock, flags); free_devinfo_mem(info); -- cgit v0.10.2 From 9113785c3e918187b6b0c084c60e0344a2f1685c Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:00 +0100 Subject: iommu/tegra-smmu: Fix iova_to_phys() method iova_to_phys() has several problems: (a) iova_to_phys() is supposed to return 0 if there is no entry present for the iova. (b) if as_get_pte() fails, we oops the kernel by dereferencing a NULL pointer. Really, we should not even be trying to allocate a page table at all, but should only be returning the presence of the 2nd level page table. This will be fixed in a subsequent patch. Treat both of these conditions as "no mapping" conditions. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index c1f2e52..0833549 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -592,6 +592,9 @@ static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, u32 *pte; pte = as_get_pte(as, iova, &page); + if (!pte || !*pte) + return 0; + pfn = *pte & as->smmu->pfn_mask; return PFN_PHYS(pfn); -- cgit v0.10.2 From b98e34f0c6f1c4ac7af41afecc4a26f5f2ebe68d Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:05 +0100 Subject: iommu/tegra-smmu: Fix unmap() method The Tegra SMMU unmap path has several problems: 1. as_pte_put() can perform a write-after-free 2. tegra_smmu_unmap() can perform cache maintanence on a page we have just freed. 3. when a page table is unmapped, there is no CPU cache maintanence of the write clearing the page directory entry, nor is there any maintanence of the IOMMU to ensure that it sees the page table has gone. Fix this by getting rid of as_pte_put(), and instead coding the PTE unmap separately from the PDE unmap, placing the PDE unmap after the PTE unmap has been completed. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 0833549..a7a7645 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -509,29 +509,35 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, return &pt[pte]; } -static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova) +static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) { + struct tegra_smmu *smmu = as->smmu; u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; - u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; u32 *count = page_address(as->count); - u32 *pd = page_address(as->pd), *pt; + u32 *pd = page_address(as->pd); struct page *page; - page = pfn_to_page(pd[pde] & as->smmu->pfn_mask); - pt = page_address(page); + page = pfn_to_page(pd[pde] & smmu->pfn_mask); /* * When no entries in this page table are used anymore, return the * memory page to the system. */ - if (pt[pte] != 0) { - if (--count[pde] == 0) { - ClearPageReserved(page); - __free_page(page); - pd[pde] = 0; - } + if (--count[pde] == 0) { + unsigned int offset = pde * sizeof(*pd); - pt[pte] = 0; + /* Clear the page directory entry first */ + pd[pde] = 0; + + /* Flush the page directory entry */ + smmu->soc->ops->flush_dcache(as->pd, offset, sizeof(*pd)); + smmu_flush_ptc(smmu, as->pd, offset); + smmu_flush_tlb_section(smmu, as->id, iova); + smmu_flush(smmu); + + /* Finally, free the page */ + ClearPageReserved(page); + __free_page(page); } } @@ -569,17 +575,20 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, u32 *pte; pte = as_get_pte(as, iova, &page); - if (!pte) + if (!pte || !*pte) return 0; + *pte = 0; + offset = offset_in_page(pte); - as_put_pte(as, iova); smmu->soc->ops->flush_dcache(page, offset, 4); smmu_flush_ptc(smmu, page, offset); smmu_flush_tlb_group(smmu, as->id, iova); smmu_flush(smmu); + tegra_smmu_pte_put_use(as, iova); + return size; } -- cgit v0.10.2 From 8482ee5ea1097445f6498ee522965f5311667763 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:10 +0100 Subject: iommu/tegra-smmu: Factor out common PTE setting Factor out the common PTE setting code into a separate function. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index a7a7645..53d0f15 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -541,12 +541,24 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) } } +static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, + u32 *pte, struct page *pte_page, u32 val) +{ + struct tegra_smmu *smmu = as->smmu; + unsigned long offset = offset_in_page(pte); + + *pte = val; + + smmu->soc->ops->flush_dcache(pte_page, offset, 4); + smmu_flush_ptc(smmu, pte_page, offset); + smmu_flush_tlb_group(smmu, as->id, iova); + smmu_flush(smmu); +} + static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct tegra_smmu *smmu = as->smmu; - unsigned long offset; struct page *page; u32 *pte; @@ -554,13 +566,8 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, if (!pte) return -ENOMEM; - *pte = __phys_to_pfn(paddr) | SMMU_PTE_ATTR; - offset = offset_in_page(pte); - - smmu->soc->ops->flush_dcache(page, offset, 4); - smmu_flush_ptc(smmu, page, offset); - smmu_flush_tlb_group(smmu, as->id, iova); - smmu_flush(smmu); + tegra_smmu_set_pte(as, iova, pte, page, + __phys_to_pfn(paddr) | SMMU_PTE_ATTR); return 0; } @@ -569,8 +576,6 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct tegra_smmu *smmu = as->smmu; - unsigned long offset; struct page *page; u32 *pte; @@ -578,15 +583,7 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, if (!pte || !*pte) return 0; - *pte = 0; - - offset = offset_in_page(pte); - - smmu->soc->ops->flush_dcache(page, offset, 4); - smmu_flush_ptc(smmu, page, offset); - smmu_flush_tlb_group(smmu, as->id, iova); - smmu_flush(smmu); - + tegra_smmu_set_pte(as, iova, pte, page, 0); tegra_smmu_pte_put_use(as, iova); return size; -- cgit v0.10.2 From 34d35f8cbe51bf93faf3214ee5b5d6f8ae7df4c1 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:16 +0100 Subject: iommu/tegra-smmu: Add iova_pd_index() and iova_pt_index() helpers Add a pair of helpers to get the page directory and page table indexes. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 53d0f15..4c4bc79 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -134,6 +134,16 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) #define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \ SMMU_PTE_NONSECURE) +static unsigned int iova_pd_index(unsigned long iova) +{ + return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); +} + +static unsigned int iova_pt_index(unsigned long iova) +{ + return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); +} + static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, unsigned long offset) { @@ -469,8 +479,8 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, struct page **pagep) { u32 *pd = page_address(as->pd), *pt, *count; - u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; - u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff; + unsigned int pde = iova_pd_index(iova); + unsigned int pte = iova_pt_index(iova); struct tegra_smmu *smmu = as->smmu; struct page *page; unsigned int i; @@ -512,7 +522,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) { struct tegra_smmu *smmu = as->smmu; - u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff; + unsigned int pde = iova_pd_index(iova); u32 *count = page_address(as->count); u32 *pd = page_address(as->pd); struct page *page; -- cgit v0.10.2 From 0b42c7c1132f331fba263f0d2ca23544770584b7 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:21 +0100 Subject: iommu/tegra-smmu: Fix page table lookup in unmap/iova_to_phys methods Fix the page table lookup in the unmap and iova_to_phys methods. Neither of these methods should allocate a page table; a missing page table should be treated the same as no mapping present. More importantly, using as_get_pte() for an IOVA corresponding with a non-present page table entry increments the use-count for the page table, on the assumption that the caller of as_get_pte() is going to setup a mapping. This is an incorrect assumption. Fix both of these bugs by providing a separate helper which only looks up the page table, but never allocates it. This is akin to pte_offset() for CPU page tables. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 4c4bc79..bbff5b64 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -475,12 +475,36 @@ static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *de } } +static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) +{ + u32 *pt = page_address(pt_page); + + return pt + iova_pt_index(iova); +} + +static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, + struct page **pagep) +{ + unsigned int pd_index = iova_pd_index(iova); + struct page *pt_page; + u32 *pd; + + pd = page_address(as->pd); + + if (!pd[pd_index]) + return NULL; + + pt_page = pfn_to_page(pd[pd_index] & as->smmu->pfn_mask); + *pagep = pt_page; + + return tegra_smmu_pte_offset(pt_page, iova); +} + static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, struct page **pagep) { u32 *pd = page_address(as->pd), *pt, *count; unsigned int pde = iova_pd_index(iova); - unsigned int pte = iova_pt_index(iova); struct tegra_smmu *smmu = as->smmu; struct page *page; unsigned int i; @@ -506,17 +530,18 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, smmu_flush(smmu); } else { page = pfn_to_page(pd[pde] & smmu->pfn_mask); - pt = page_address(page); } *pagep = page; + pt = page_address(page); + /* Keep track of entries in this page table. */ count = page_address(as->count); - if (pt[pte] == 0) + if (pt[iova_pt_index(iova)] == 0) count[pde]++; - return &pt[pte]; + return tegra_smmu_pte_offset(page, iova); } static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) @@ -586,14 +611,14 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct page *page; + struct page *pte_page; u32 *pte; - pte = as_get_pte(as, iova, &page); + pte = tegra_smmu_pte_lookup(as, iova, &pte_page); if (!pte || !*pte) return 0; - tegra_smmu_set_pte(as, iova, pte, page, 0); + tegra_smmu_set_pte(as, iova, pte, pte_page, 0); tegra_smmu_pte_put_use(as, iova); return size; @@ -603,11 +628,11 @@ static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct page *page; + struct page *pte_page; unsigned long pfn; u32 *pte; - pte = as_get_pte(as, iova, &page); + pte = tegra_smmu_pte_lookup(as, iova, &pte_page); if (!pte || !*pte) return 0; -- cgit v0.10.2 From 853520fa96511e4a49942d2cba34a329528c7e41 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:26 +0100 Subject: iommu/tegra-smmu: Store struct page pointer for page tables Store the struct page pointer for the second level page tables, rather than working back from the page directory entry. This is necessary as we want to eliminate the use of physical addresses used with arch-private functions, switching instead to use the streaming DMA API. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index bbff5b64..8ec5ac4 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -41,6 +41,7 @@ struct tegra_smmu_as { struct tegra_smmu *smmu; unsigned int use_count; struct page *count; + struct page **pts; struct page *pd; unsigned id; u32 attr; @@ -271,6 +272,14 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) return NULL; } + as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); + if (!as->pts) { + __free_page(as->count); + __free_page(as->pd); + kfree(as); + return NULL; + } + /* clear PDEs */ pd = page_address(as->pd); SetPageReserved(as->pd); @@ -487,14 +496,11 @@ static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, { unsigned int pd_index = iova_pd_index(iova); struct page *pt_page; - u32 *pd; - pd = page_address(as->pd); - - if (!pd[pd_index]) + pt_page = as->pts[pd_index]; + if (!pt_page) return NULL; - pt_page = pfn_to_page(pd[pd_index] & as->smmu->pfn_mask); *pagep = pt_page; return tegra_smmu_pte_offset(pt_page, iova); @@ -509,7 +515,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, struct page *page; unsigned int i; - if (pd[pde] == 0) { + if (!as->pts[pde]) { page = alloc_page(GFP_KERNEL | __GFP_DMA); if (!page) return NULL; @@ -520,6 +526,8 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, for (i = 0; i < SMMU_NUM_PTE; i++) pt[i] = 0; + as->pts[pde] = page; + smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT); pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); @@ -529,7 +537,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); } else { - page = pfn_to_page(pd[pde] & smmu->pfn_mask); + page = as->pts[pde]; } *pagep = page; @@ -550,9 +558,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) unsigned int pde = iova_pd_index(iova); u32 *count = page_address(as->count); u32 *pd = page_address(as->pd); - struct page *page; - - page = pfn_to_page(pd[pde] & smmu->pfn_mask); + struct page *page = as->pts[pde]; /* * When no entries in this page table are used anymore, return the @@ -573,6 +579,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) /* Finally, free the page */ ClearPageReserved(page); __free_page(page); + as->pts[pde] = NULL; } } -- cgit v0.10.2 From 32924c76b0cbc67aa4cf0741f7bc6c37f097aaf3 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:31 +0100 Subject: iommu/tegra-smmu: Use kcalloc() to allocate counter array Use kcalloc() to allocate the use-counter array for the page directory entries/page tables. Using kcalloc() allows us to be provided with zero-initialised memory from the allocators, rather than initialising it ourselves. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 8ec5ac4..d649b06 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -40,7 +40,7 @@ struct tegra_smmu_as { struct iommu_domain domain; struct tegra_smmu *smmu; unsigned int use_count; - struct page *count; + u32 *count; struct page **pts; struct page *pd; unsigned id; @@ -265,7 +265,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) return NULL; } - as->count = alloc_page(GFP_KERNEL); + as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); if (!as->count) { __free_page(as->pd); kfree(as); @@ -274,7 +274,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); if (!as->pts) { - __free_page(as->count); + kfree(as->count); __free_page(as->pd); kfree(as); return NULL; @@ -287,13 +287,6 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) for (i = 0; i < SMMU_NUM_PDE; i++) pd[i] = 0; - /* clear PDE usage counters */ - pd = page_address(as->count); - SetPageReserved(as->count); - - for (i = 0; i < SMMU_NUM_PDE; i++) - pd[i] = 0; - /* setup aperture */ as->domain.geometry.aperture_start = 0; as->domain.geometry.aperture_end = 0xffffffff; @@ -509,7 +502,7 @@ static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, struct page **pagep) { - u32 *pd = page_address(as->pd), *pt, *count; + u32 *pd = page_address(as->pd), *pt; unsigned int pde = iova_pd_index(iova); struct tegra_smmu *smmu = as->smmu; struct page *page; @@ -545,9 +538,8 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, pt = page_address(page); /* Keep track of entries in this page table. */ - count = page_address(as->count); if (pt[iova_pt_index(iova)] == 0) - count[pde]++; + as->count[pde]++; return tegra_smmu_pte_offset(page, iova); } @@ -556,7 +548,6 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) { struct tegra_smmu *smmu = as->smmu; unsigned int pde = iova_pd_index(iova); - u32 *count = page_address(as->count); u32 *pd = page_address(as->pd); struct page *page = as->pts[pde]; @@ -564,7 +555,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) * When no entries in this page table are used anymore, return the * memory page to the system. */ - if (--count[pde] == 0) { + if (--as->count[pde] == 0) { unsigned int offset = pde * sizeof(*pd); /* Clear the page directory entry first */ -- cgit v0.10.2 From 4b3c7d10765403ab19628fb7d530b8ce1c50b81d Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:36 +0100 Subject: iommu/tegra-smmu: Move flush_dcache to tegra-smmu.c Drivers should not be using __cpuc_* functions nor outer_cache_flush() directly. This change partly cleans up tegra-smmu.c. The only difference between cache handling of the tegra variants is Denver, which omits the call to outer_cache_flush(). This is due to Denver being an ARM64 CPU, and the ARM64 architecture does not provide this function. (This, in itself, is a good reason why these should not be used.) Signed-off-by: Russell King [treding@nvidia.com: fix build failure on 64-bit ARM] Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index d649b06..42b13c0 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -16,6 +16,8 @@ #include #include +#include + #include #include @@ -145,6 +147,24 @@ static unsigned int iova_pt_index(unsigned long iova) return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); } +static void smmu_flush_dcache(struct page *page, unsigned long offset, + size_t size) +{ +#ifdef CONFIG_ARM + phys_addr_t phys = page_to_phys(page) + offset; +#endif + void *virt = page_address(page) + offset; + +#ifdef CONFIG_ARM + __cpuc_flush_dcache_area(virt, size); + outer_flush_range(phys, phys + size); +#endif + +#ifdef CONFIG_ARM64 + __flush_dcache_area(virt, size); +#endif +} + static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, unsigned long offset) { @@ -392,7 +412,7 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, if (err < 0) return err; - smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD); + smmu_flush_dcache(as->pd, 0, SMMU_SIZE_PD); smmu_flush_ptc(smmu, as->pd, 0); smmu_flush_tlb_asid(smmu, as->id); @@ -521,11 +541,11 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, as->pts[pde] = page; - smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT); + smmu_flush_dcache(page, 0, SMMU_SIZE_PT); pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); - smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4); + smmu_flush_dcache(as->pd, pde << 2, 4); smmu_flush_ptc(smmu, as->pd, pde << 2); smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); @@ -562,7 +582,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) pd[pde] = 0; /* Flush the page directory entry */ - smmu->soc->ops->flush_dcache(as->pd, offset, sizeof(*pd)); + smmu_flush_dcache(as->pd, offset, sizeof(*pd)); smmu_flush_ptc(smmu, as->pd, offset); smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); @@ -582,7 +602,7 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, *pte = val; - smmu->soc->ops->flush_dcache(pte_page, offset, 4); + smmu_flush_dcache(pte_page, offset, 4); smmu_flush_ptc(smmu, pte_page, offset); smmu_flush_tlb_group(smmu, as->id, iova); smmu_flush(smmu); diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c index 9f57958..7122f39 100644 --- a/drivers/memory/tegra/tegra114.c +++ b/drivers/memory/tegra/tegra114.c @@ -9,8 +9,6 @@ #include #include -#include - #include #include "mc.h" @@ -914,20 +912,6 @@ static const struct tegra_smmu_swgroup tegra114_swgroups[] = { { .name = "tsec", .swgroup = TEGRA_SWGROUP_TSEC, .reg = 0x294 }, }; -static void tegra114_flush_dcache(struct page *page, unsigned long offset, - size_t size) -{ - phys_addr_t phys = page_to_phys(page) + offset; - void *virt = page_address(page) + offset; - - __cpuc_flush_dcache_area(virt, size); - outer_flush_range(phys, phys + size); -} - -static const struct tegra_smmu_ops tegra114_smmu_ops = { - .flush_dcache = tegra114_flush_dcache, -}; - static const struct tegra_smmu_soc tegra114_smmu_soc = { .clients = tegra114_mc_clients, .num_clients = ARRAY_SIZE(tegra114_mc_clients), @@ -936,7 +920,6 @@ static const struct tegra_smmu_soc tegra114_smmu_soc = { .supports_round_robin_arbitration = false, .supports_request_limit = false, .num_asids = 4, - .ops = &tegra114_smmu_ops, }; const struct tegra_mc_soc tegra114_mc_soc = { diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c index 966e155..ebda632 100644 --- a/drivers/memory/tegra/tegra124.c +++ b/drivers/memory/tegra/tegra124.c @@ -9,8 +9,6 @@ #include #include -#include - #include #include "mc.h" @@ -1002,20 +1000,6 @@ static const struct tegra_smmu_swgroup tegra124_swgroups[] = { }; #ifdef CONFIG_ARCH_TEGRA_124_SOC -static void tegra124_flush_dcache(struct page *page, unsigned long offset, - size_t size) -{ - phys_addr_t phys = page_to_phys(page) + offset; - void *virt = page_address(page) + offset; - - __cpuc_flush_dcache_area(virt, size); - outer_flush_range(phys, phys + size); -} - -static const struct tegra_smmu_ops tegra124_smmu_ops = { - .flush_dcache = tegra124_flush_dcache, -}; - static const struct tegra_smmu_soc tegra124_smmu_soc = { .clients = tegra124_mc_clients, .num_clients = ARRAY_SIZE(tegra124_mc_clients), @@ -1024,7 +1008,6 @@ static const struct tegra_smmu_soc tegra124_smmu_soc = { .supports_round_robin_arbitration = true, .supports_request_limit = true, .num_asids = 128, - .ops = &tegra124_smmu_ops, }; const struct tegra_mc_soc tegra124_mc_soc = { @@ -1039,18 +1022,6 @@ const struct tegra_mc_soc tegra124_mc_soc = { #endif /* CONFIG_ARCH_TEGRA_124_SOC */ #ifdef CONFIG_ARCH_TEGRA_132_SOC -static void tegra132_flush_dcache(struct page *page, unsigned long offset, - size_t size) -{ - void *virt = page_address(page) + offset; - - __flush_dcache_area(virt, size); -} - -static const struct tegra_smmu_ops tegra132_smmu_ops = { - .flush_dcache = tegra132_flush_dcache, -}; - static const struct tegra_smmu_soc tegra132_smmu_soc = { .clients = tegra124_mc_clients, .num_clients = ARRAY_SIZE(tegra124_mc_clients), @@ -1059,7 +1030,6 @@ static const struct tegra_smmu_soc tegra132_smmu_soc = { .supports_round_robin_arbitration = true, .supports_request_limit = true, .num_asids = 128, - .ops = &tegra132_smmu_ops, }; const struct tegra_mc_soc tegra132_mc_soc = { diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c index 1abcd8f..3cb30b6 100644 --- a/drivers/memory/tegra/tegra30.c +++ b/drivers/memory/tegra/tegra30.c @@ -9,8 +9,6 @@ #include #include -#include - #include #include "mc.h" @@ -936,20 +934,6 @@ static const struct tegra_smmu_swgroup tegra30_swgroups[] = { { .name = "isp", .swgroup = TEGRA_SWGROUP_ISP, .reg = 0x258 }, }; -static void tegra30_flush_dcache(struct page *page, unsigned long offset, - size_t size) -{ - phys_addr_t phys = page_to_phys(page) + offset; - void *virt = page_address(page) + offset; - - __cpuc_flush_dcache_area(virt, size); - outer_flush_range(phys, phys + size); -} - -static const struct tegra_smmu_ops tegra30_smmu_ops = { - .flush_dcache = tegra30_flush_dcache, -}; - static const struct tegra_smmu_soc tegra30_smmu_soc = { .clients = tegra30_mc_clients, .num_clients = ARRAY_SIZE(tegra30_mc_clients), @@ -958,7 +942,6 @@ static const struct tegra_smmu_soc tegra30_smmu_soc = { .supports_round_robin_arbitration = false, .supports_request_limit = false, .num_asids = 4, - .ops = &tegra30_smmu_ops, }; const struct tegra_mc_soc tegra30_mc_soc = { diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h index 1ab2813..d6c3190 100644 --- a/include/soc/tegra/mc.h +++ b/include/soc/tegra/mc.h @@ -51,11 +51,6 @@ struct tegra_smmu_swgroup { unsigned int reg; }; -struct tegra_smmu_ops { - void (*flush_dcache)(struct page *page, unsigned long offset, - size_t size); -}; - struct tegra_smmu_soc { const struct tegra_mc_client *clients; unsigned int num_clients; @@ -67,8 +62,6 @@ struct tegra_smmu_soc { bool supports_request_limit; unsigned int num_asids; - - const struct tegra_smmu_ops *ops; }; struct tegra_mc; -- cgit v0.10.2 From b8fe03827b192a23d04e99c40d72e6b938fa6576 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:41 +0100 Subject: iommu/tegra-smmu: Split smmu_flush_ptc() smmu_flush_ptc() is used in two modes: one is to flush an individual entry, the other is to flush all entries. We know at the call site which we require. Split the function into smmu_flush_ptc_all() and smmu_flush_ptc(). Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 42b13c0..5c775b7 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -165,29 +165,29 @@ static void smmu_flush_dcache(struct page *page, unsigned long offset, #endif } +static void smmu_flush_ptc_all(struct tegra_smmu *smmu) +{ + smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); +} + static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, unsigned long offset) { - phys_addr_t phys = page ? page_to_phys(page) : 0; + phys_addr_t phys = page_to_phys(page); u32 value; - if (page) { - offset &= ~(smmu->mc->soc->atom_size - 1); + offset &= ~(smmu->mc->soc->atom_size - 1); - if (smmu->mc->soc->num_address_bits > 32) { + if (smmu->mc->soc->num_address_bits > 32) { #ifdef CONFIG_PHYS_ADDR_T_64BIT - value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; + value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; #else - value = 0; + value = 0; #endif - smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); - } - - value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR; - } else { - value = SMMU_PTC_FLUSH_TYPE_ALL; + smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); } + value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR; smmu_writel(smmu, value, SMMU_PTC_FLUSH); } @@ -894,7 +894,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, smmu_writel(smmu, value, SMMU_TLB_CONFIG); - smmu_flush_ptc(smmu, NULL, 0); + smmu_flush_ptc_all(smmu); smmu_flush_tlb(smmu); smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); smmu_flush(smmu); -- cgit v0.10.2 From d62c7a886c2bc9f9258164814245dc0678b9a52e Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:46 +0100 Subject: iommu/tegra-smmu: smmu_flush_ptc() wants device addresses Pass smmu_flush_ptc() the device address rather than struct page pointer. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 5c775b7..f420d87 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -170,10 +170,9 @@ static void smmu_flush_ptc_all(struct tegra_smmu *smmu) smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); } -static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page, +static inline void smmu_flush_ptc(struct tegra_smmu *smmu, phys_addr_t phys, unsigned long offset) { - phys_addr_t phys = page_to_phys(page); u32 value; offset &= ~(smmu->mc->soc->atom_size - 1); @@ -413,7 +412,7 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, return err; smmu_flush_dcache(as->pd, 0, SMMU_SIZE_PD); - smmu_flush_ptc(smmu, as->pd, 0); + smmu_flush_ptc(smmu, page_to_phys(as->pd), 0); smmu_flush_tlb_asid(smmu, as->id); smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); @@ -546,7 +545,7 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); smmu_flush_dcache(as->pd, pde << 2, 4); - smmu_flush_ptc(smmu, as->pd, pde << 2); + smmu_flush_ptc(smmu, page_to_phys(as->pd), pde << 2); smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); } else { @@ -583,7 +582,7 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) /* Flush the page directory entry */ smmu_flush_dcache(as->pd, offset, sizeof(*pd)); - smmu_flush_ptc(smmu, as->pd, offset); + smmu_flush_ptc(smmu, page_to_phys(as->pd), offset); smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); @@ -603,7 +602,7 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, *pte = val; smmu_flush_dcache(pte_page, offset, 4); - smmu_flush_ptc(smmu, pte_page, offset); + smmu_flush_ptc(smmu, page_to_phys(pte_page), offset); smmu_flush_tlb_group(smmu, as->id, iova); smmu_flush(smmu); } -- cgit v0.10.2 From e3c971960fd41fc55235ba05b95e053355cb0e73 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:52 +0100 Subject: iommu/tegra-smmu: Convert to use DMA API Use the DMA API instead of calling architecture internal functions in the Tegra SMMU driver. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index f420d87..43b69c8 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -16,8 +16,6 @@ #include #include -#include - #include #include @@ -45,6 +43,7 @@ struct tegra_smmu_as { u32 *count; struct page **pts; struct page *pd; + dma_addr_t pd_dma; unsigned id; u32 attr; }; @@ -82,9 +81,9 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) #define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) #define SMMU_PTB_DATA 0x020 -#define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr)) +#define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr)) -#define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr)) +#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr)) #define SMMU_TLB_FLUSH 0x030 #define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) @@ -147,22 +146,15 @@ static unsigned int iova_pt_index(unsigned long iova) return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); } -static void smmu_flush_dcache(struct page *page, unsigned long offset, - size_t size) +static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr) { -#ifdef CONFIG_ARM - phys_addr_t phys = page_to_phys(page) + offset; -#endif - void *virt = page_address(page) + offset; - -#ifdef CONFIG_ARM - __cpuc_flush_dcache_area(virt, size); - outer_flush_range(phys, phys + size); -#endif + addr >>= 12; + return (addr & smmu->pfn_mask) == addr; +} -#ifdef CONFIG_ARM64 - __flush_dcache_area(virt, size); -#endif +static dma_addr_t smmu_pde_to_dma(u32 pde) +{ + return pde << 12; } static void smmu_flush_ptc_all(struct tegra_smmu *smmu) @@ -170,7 +162,7 @@ static void smmu_flush_ptc_all(struct tegra_smmu *smmu) smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); } -static inline void smmu_flush_ptc(struct tegra_smmu *smmu, phys_addr_t phys, +static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma, unsigned long offset) { u32 value; @@ -178,15 +170,15 @@ static inline void smmu_flush_ptc(struct tegra_smmu *smmu, phys_addr_t phys, offset &= ~(smmu->mc->soc->atom_size - 1); if (smmu->mc->soc->num_address_bits > 32) { -#ifdef CONFIG_PHYS_ADDR_T_64BIT - value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK; +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT + value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK; #else value = 0; #endif smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); } - value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR; + value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR; smmu_writel(smmu, value, SMMU_PTC_FLUSH); } @@ -407,16 +399,26 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, return 0; } + as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, + DMA_TO_DEVICE); + if (dma_mapping_error(smmu->dev, as->pd_dma)) + return -ENOMEM; + + /* We can't handle 64-bit DMA addresses */ + if (!smmu_dma_addr_valid(smmu, as->pd_dma)) { + err = -ENOMEM; + goto err_unmap; + } + err = tegra_smmu_alloc_asid(smmu, &as->id); if (err < 0) - return err; + goto err_unmap; - smmu_flush_dcache(as->pd, 0, SMMU_SIZE_PD); - smmu_flush_ptc(smmu, page_to_phys(as->pd), 0); + smmu_flush_ptc(smmu, as->pd_dma, 0); smmu_flush_tlb_asid(smmu, as->id); smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); - value = SMMU_PTB_DATA_VALUE(as->pd, as->attr); + value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr); smmu_writel(smmu, value, SMMU_PTB_DATA); smmu_flush(smmu); @@ -424,6 +426,10 @@ static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, as->use_count++; return 0; + +err_unmap: + dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); + return err; } static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, @@ -433,6 +439,9 @@ static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, return; tegra_smmu_free_asid(smmu, as->id); + + dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); + as->smmu = NULL; } @@ -504,63 +513,81 @@ static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) } static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, - struct page **pagep) + dma_addr_t *dmap) { unsigned int pd_index = iova_pd_index(iova); struct page *pt_page; + u32 *pd; pt_page = as->pts[pd_index]; if (!pt_page) return NULL; - *pagep = pt_page; + pd = page_address(as->pd); + *dmap = smmu_pde_to_dma(pd[pd_index]); return tegra_smmu_pte_offset(pt_page, iova); } static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, - struct page **pagep) + dma_addr_t *dmap) { u32 *pd = page_address(as->pd), *pt; unsigned int pde = iova_pd_index(iova); struct tegra_smmu *smmu = as->smmu; - struct page *page; unsigned int i; if (!as->pts[pde]) { + struct page *page; + dma_addr_t dma; + page = alloc_page(GFP_KERNEL | __GFP_DMA); if (!page) return NULL; pt = page_address(page); - SetPageReserved(page); for (i = 0; i < SMMU_NUM_PTE; i++) pt[i] = 0; + dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, + DMA_TO_DEVICE); + if (dma_mapping_error(smmu->dev, dma)) { + __free_page(page); + return NULL; + } + + if (!smmu_dma_addr_valid(smmu, dma)) { + dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, + DMA_TO_DEVICE); + __free_page(page); + return NULL; + } + as->pts[pde] = page; - smmu_flush_dcache(page, 0, SMMU_SIZE_PT); + SetPageReserved(page); - pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT); + pd[pde] = SMMU_MK_PDE(dma, SMMU_PDE_ATTR | SMMU_PDE_NEXT); - smmu_flush_dcache(as->pd, pde << 2, 4); - smmu_flush_ptc(smmu, page_to_phys(as->pd), pde << 2); + dma_sync_single_range_for_device(smmu->dev, as->pd_dma, + pde << 2, 4, DMA_TO_DEVICE); + smmu_flush_ptc(smmu, as->pd_dma, pde << 2); smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); + + *dmap = dma; } else { - page = as->pts[pde]; + *dmap = smmu_pde_to_dma(pd[pde]); } - *pagep = page; - - pt = page_address(page); + pt = tegra_smmu_pte_offset(as->pts[pde], iova); /* Keep track of entries in this page table. */ - if (pt[iova_pt_index(iova)] == 0) + if (*pt == 0) as->count[pde]++; - return tegra_smmu_pte_offset(page, iova); + return pt; } static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) @@ -576,17 +603,20 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) */ if (--as->count[pde] == 0) { unsigned int offset = pde * sizeof(*pd); + dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]); /* Clear the page directory entry first */ pd[pde] = 0; /* Flush the page directory entry */ - smmu_flush_dcache(as->pd, offset, sizeof(*pd)); - smmu_flush_ptc(smmu, page_to_phys(as->pd), offset); + dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, + sizeof(*pd), DMA_TO_DEVICE); + smmu_flush_ptc(smmu, as->pd_dma, offset); smmu_flush_tlb_section(smmu, as->id, iova); smmu_flush(smmu); /* Finally, free the page */ + dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); ClearPageReserved(page); __free_page(page); as->pts[pde] = NULL; @@ -594,15 +624,16 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) } static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, - u32 *pte, struct page *pte_page, u32 val) + u32 *pte, dma_addr_t pte_dma, u32 val) { struct tegra_smmu *smmu = as->smmu; unsigned long offset = offset_in_page(pte); *pte = val; - smmu_flush_dcache(pte_page, offset, 4); - smmu_flush_ptc(smmu, page_to_phys(pte_page), offset); + dma_sync_single_range_for_device(smmu->dev, pte_dma, offset, + 4, DMA_TO_DEVICE); + smmu_flush_ptc(smmu, pte_dma, offset); smmu_flush_tlb_group(smmu, as->id, iova); smmu_flush(smmu); } @@ -611,14 +642,14 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct page *page; + dma_addr_t pte_dma; u32 *pte; - pte = as_get_pte(as, iova, &page); + pte = as_get_pte(as, iova, &pte_dma); if (!pte) return -ENOMEM; - tegra_smmu_set_pte(as, iova, pte, page, + tegra_smmu_set_pte(as, iova, pte, pte_dma, __phys_to_pfn(paddr) | SMMU_PTE_ATTR); return 0; @@ -628,14 +659,14 @@ static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct page *pte_page; + dma_addr_t pte_dma; u32 *pte; - pte = tegra_smmu_pte_lookup(as, iova, &pte_page); + pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); if (!pte || !*pte) return 0; - tegra_smmu_set_pte(as, iova, pte, pte_page, 0); + tegra_smmu_set_pte(as, iova, pte, pte_dma, 0); tegra_smmu_pte_put_use(as, iova); return size; @@ -645,11 +676,11 @@ static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct tegra_smmu_as *as = to_smmu_as(domain); - struct page *pte_page; unsigned long pfn; + dma_addr_t pte_dma; u32 *pte; - pte = tegra_smmu_pte_lookup(as, iova, &pte_page); + pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); if (!pte || !*pte) return 0; -- cgit v0.10.2 From 05a65f06f69fa6c487c2933f2971d9ec4e33eb0d Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:29:57 +0100 Subject: iommu/tegra-smmu: Remove PageReserved manipulation Remove the unnecessary manipulation of the PageReserved flags in the Tegra SMMU driver. None of this is required as the page(s) remain private to the SMMU driver. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 43b69c8..eb9f606 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -293,7 +293,6 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) /* clear PDEs */ pd = page_address(as->pd); - SetPageReserved(as->pd); for (i = 0; i < SMMU_NUM_PDE; i++) pd[i] = 0; @@ -311,7 +310,6 @@ static void tegra_smmu_domain_free(struct iommu_domain *domain) struct tegra_smmu_as *as = to_smmu_as(domain); /* TODO: free page directory and page tables */ - ClearPageReserved(as->pd); kfree(as); } @@ -566,8 +564,6 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, as->pts[pde] = page; - SetPageReserved(page); - pd[pde] = SMMU_MK_PDE(dma, SMMU_PDE_ATTR | SMMU_PDE_NEXT); dma_sync_single_range_for_device(smmu->dev, as->pd_dma, @@ -617,7 +613,6 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) /* Finally, free the page */ dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); - ClearPageReserved(page); __free_page(page); as->pts[pde] = NULL; } -- cgit v0.10.2 From 707917cbc6ac0c0ea968b5eb635722ea84808286 Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:30:02 +0100 Subject: iommu/tegra-smmu: Use __GFP_ZERO to allocate zeroed pages Rather than explicitly zeroing pages allocated via alloc_page(), add __GFP_ZERO to the gfp mask to ask the allocator for zeroed pages. Signed-off-by: Russell King Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index eb9f606..27d31f6 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -258,8 +258,6 @@ static bool tegra_smmu_capable(enum iommu_cap cap) static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) { struct tegra_smmu_as *as; - unsigned int i; - uint32_t *pd; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; @@ -270,7 +268,7 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; - as->pd = alloc_page(GFP_KERNEL | __GFP_DMA); + as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); if (!as->pd) { kfree(as); return NULL; @@ -291,12 +289,6 @@ static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) return NULL; } - /* clear PDEs */ - pd = page_address(as->pd); - - for (i = 0; i < SMMU_NUM_PDE; i++) - pd[i] = 0; - /* setup aperture */ as->domain.geometry.aperture_start = 0; as->domain.geometry.aperture_end = 0xffffffff; @@ -533,21 +525,15 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, u32 *pd = page_address(as->pd), *pt; unsigned int pde = iova_pd_index(iova); struct tegra_smmu *smmu = as->smmu; - unsigned int i; if (!as->pts[pde]) { struct page *page; dma_addr_t dma; - page = alloc_page(GFP_KERNEL | __GFP_DMA); + page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); if (!page) return NULL; - pt = page_address(page); - - for (i = 0; i < SMMU_NUM_PTE; i++) - pt[i] = 0; - dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, DMA_TO_DEVICE); if (dma_mapping_error(smmu->dev, dma)) { -- cgit v0.10.2 From 7ffc6f066eb73b07a0ef7c94d05107aef271ac21 Mon Sep 17 00:00:00 2001 From: Russell King Date: Thu, 6 Aug 2015 14:56:39 +0200 Subject: iommu/tegra-smmu: Extract tegra_smmu_pte_get_use() Extract the use count reference accounting into a separate function and separate it from allocating the PTE. Signed-off-by: Russell King [treding@nvidia.com: extract and write commit message] Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 27d31f6..74ad1f4 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -522,7 +522,7 @@ static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, dma_addr_t *dmap) { - u32 *pd = page_address(as->pd), *pt; + u32 *pd = page_address(as->pd); unsigned int pde = iova_pd_index(iova); struct tegra_smmu *smmu = as->smmu; @@ -563,13 +563,14 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, *dmap = smmu_pde_to_dma(pd[pde]); } - pt = tegra_smmu_pte_offset(as->pts[pde], iova); + return tegra_smmu_pte_offset(as->pts[pde], iova); +} - /* Keep track of entries in this page table. */ - if (*pt == 0) - as->count[pde]++; +static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) +{ + unsigned int pd_index = iova_pd_index(iova); - return pt; + as->count[pd_index]++; } static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) @@ -630,6 +631,10 @@ static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, if (!pte) return -ENOMEM; + /* If we aren't overwriting a pre-existing entry, increment use */ + if (*pte == 0) + tegra_smmu_pte_get_use(as, iova); + tegra_smmu_set_pte(as, iova, pte, pte_dma, __phys_to_pfn(paddr) | SMMU_PTE_ATTR); -- cgit v0.10.2 From 4080e99b8341f81c4ed1e17d8ef44d171c473a1b Mon Sep 17 00:00:00 2001 From: Russell King Date: Mon, 27 Jul 2015 13:30:12 +0100 Subject: iommu/tegra-smmu: Factor out tegra_smmu_set_pde() This code is used both when creating a new page directory entry and when tearing it down, with only the PDE value changing between both cases. Factor the code out so that it can be reused. Signed-off-by: Russell King [treding@nvidia.com: make commit message more accurate] Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 74ad1f4..2f1481a 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -495,6 +495,27 @@ static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *de } } +static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova, + u32 value) +{ + unsigned int pd_index = iova_pd_index(iova); + struct tegra_smmu *smmu = as->smmu; + u32 *pd = page_address(as->pd); + unsigned long offset = pd_index * sizeof(*pd); + + /* Set the page directory entry first */ + pd[pd_index] = value; + + /* The flush the page directory entry from caches */ + dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, + sizeof(*pd), DMA_TO_DEVICE); + + /* And flush the iommu */ + smmu_flush_ptc(smmu, as->pd_dma, offset); + smmu_flush_tlb_section(smmu, as->id, iova); + smmu_flush(smmu); +} + static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) { u32 *pt = page_address(pt_page); @@ -522,7 +543,6 @@ static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, dma_addr_t *dmap) { - u32 *pd = page_address(as->pd); unsigned int pde = iova_pd_index(iova); struct tegra_smmu *smmu = as->smmu; @@ -550,16 +570,13 @@ static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, as->pts[pde] = page; - pd[pde] = SMMU_MK_PDE(dma, SMMU_PDE_ATTR | SMMU_PDE_NEXT); - - dma_sync_single_range_for_device(smmu->dev, as->pd_dma, - pde << 2, 4, DMA_TO_DEVICE); - smmu_flush_ptc(smmu, as->pd_dma, pde << 2); - smmu_flush_tlb_section(smmu, as->id, iova); - smmu_flush(smmu); + tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | + SMMU_PDE_NEXT)); *dmap = dma; } else { + u32 *pd = page_address(as->pd); + *dmap = smmu_pde_to_dma(pd[pde]); } @@ -575,9 +592,7 @@ static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) { - struct tegra_smmu *smmu = as->smmu; unsigned int pde = iova_pd_index(iova); - u32 *pd = page_address(as->pd); struct page *page = as->pts[pde]; /* @@ -585,20 +600,12 @@ static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) * memory page to the system. */ if (--as->count[pde] == 0) { - unsigned int offset = pde * sizeof(*pd); + struct tegra_smmu *smmu = as->smmu; + u32 *pd = page_address(as->pd); dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]); - /* Clear the page directory entry first */ - pd[pde] = 0; - - /* Flush the page directory entry */ - dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, - sizeof(*pd), DMA_TO_DEVICE); - smmu_flush_ptc(smmu, as->pd_dma, offset); - smmu_flush_tlb_section(smmu, as->id, iova); - smmu_flush(smmu); + tegra_smmu_set_pde(as, iova, 0); - /* Finally, free the page */ dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); __free_page(page); as->pts[pde] = NULL; -- cgit v0.10.2 From 11cec15bf3fb498206ef63b1fa26c27689e02d0e Mon Sep 17 00:00:00 2001 From: Thierry Reding Date: Thu, 6 Aug 2015 14:20:31 +0200 Subject: iommu/tegra-smmu: Parameterize number of TLB lines The number of TLB lines was increased from 16 on Tegra30 to 32 on Tegra114 and later. Parameterize the value so that the initial default can be set accordingly. On Tegra30, initializing the value to 32 would effectively disable the TLB and hence cause massive latencies for memory accesses translated through the SMMU. This is especially noticeable for isochronuous clients such as display, whose FIFOs would continuously underrun. Fixes: 891846516317 ("memory: Add NVIDIA Tegra memory controller support") Signed-off-by: Thierry Reding diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 2f1481a..9305964 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -27,6 +27,7 @@ struct tegra_smmu { const struct tegra_smmu_soc *soc; unsigned long pfn_mask; + unsigned long tlb_mask; unsigned long *asids; struct mutex lock; @@ -70,7 +71,8 @@ static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) #define SMMU_TLB_CONFIG 0x14 #define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) #define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) -#define SMMU_TLB_CONFIG_ACTIVE_LINES(x) ((x) & 0x3f) +#define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \ + ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask) #define SMMU_PTC_CONFIG 0x18 #define SMMU_PTC_CONFIG_ENABLE (1 << 29) @@ -901,6 +903,9 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, smmu->pfn_mask = BIT_MASK(mc->soc->num_address_bits - PAGE_SHIFT) - 1; dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", mc->soc->num_address_bits, smmu->pfn_mask); + smmu->tlb_mask = (smmu->soc->num_tlb_lines << 1) - 1; + dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines, + smmu->tlb_mask); value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); @@ -910,7 +915,7 @@ struct tegra_smmu *tegra_smmu_probe(struct device *dev, smmu_writel(smmu, value, SMMU_PTC_CONFIG); value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | - SMMU_TLB_CONFIG_ACTIVE_LINES(0x20); + SMMU_TLB_CONFIG_ACTIVE_LINES(smmu); if (soc->supports_round_robin_arbitration) value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c index 7122f39..8053f70 100644 --- a/drivers/memory/tegra/tegra114.c +++ b/drivers/memory/tegra/tegra114.c @@ -919,6 +919,7 @@ static const struct tegra_smmu_soc tegra114_smmu_soc = { .num_swgroups = ARRAY_SIZE(tegra114_swgroups), .supports_round_robin_arbitration = false, .supports_request_limit = false, + .num_tlb_lines = 32, .num_asids = 4, }; diff --git a/drivers/memory/tegra/tegra124.c b/drivers/memory/tegra/tegra124.c index ebda632..7d734be 100644 --- a/drivers/memory/tegra/tegra124.c +++ b/drivers/memory/tegra/tegra124.c @@ -1029,6 +1029,7 @@ static const struct tegra_smmu_soc tegra132_smmu_soc = { .num_swgroups = ARRAY_SIZE(tegra124_swgroups), .supports_round_robin_arbitration = true, .supports_request_limit = true, + .num_tlb_lines = 32, .num_asids = 128, }; diff --git a/drivers/memory/tegra/tegra30.c b/drivers/memory/tegra/tegra30.c index 3cb30b6..7e0694d 100644 --- a/drivers/memory/tegra/tegra30.c +++ b/drivers/memory/tegra/tegra30.c @@ -941,6 +941,7 @@ static const struct tegra_smmu_soc tegra30_smmu_soc = { .num_swgroups = ARRAY_SIZE(tegra30_swgroups), .supports_round_robin_arbitration = false, .supports_request_limit = false, + .num_tlb_lines = 16, .num_asids = 4, }; diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h index d6c3190..8cb3a7e 100644 --- a/include/soc/tegra/mc.h +++ b/include/soc/tegra/mc.h @@ -61,6 +61,7 @@ struct tegra_smmu_soc { bool supports_round_robin_arbitration; bool supports_request_limit; + unsigned int num_tlb_lines; unsigned int num_asids; }; -- cgit v0.10.2 From a130e69f28ba1d180242b581a15d09f06dad9227 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:07:25 +0200 Subject: iommu/amd: Simplify allocation in irq_remapping_alloc() Allocate the irq data only in the loop. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 658ee39..a158579 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3947,11 +3947,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, if (ret < 0) return ret; - ret = -ENOMEM; - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - goto out_free_parent; - if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { if (get_irq_table(devid, true)) index = info->ioapic_pin; @@ -3962,7 +3957,6 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, } if (index < 0) { pr_warn("Failed to allocate IRTE\n"); - kfree(data); goto out_free_parent; } @@ -3974,17 +3968,18 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq, goto out_free_data; } - if (i > 0) { - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) - goto out_free_data; - } + ret = -ENOMEM; + data = kzalloc(sizeof(*data), GFP_KERNEL); + if (!data) + goto out_free_data; + irq_data->hwirq = (devid << 16) + i; irq_data->chip_data = data; irq_data->chip = &amd_ir_chip; irq_remapping_prepare_irte(data, cfg, info, devid, index, i); irq_set_status_flags(virq + i, IRQ_MOVE_PCNTXT); } + return 0; out_free_data: -- cgit v0.10.2 From 4160cd9e5ef9ac9f1c9c429e1606bf08a56c8a49 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:31:48 +0200 Subject: iommu/amd: Make a symbol static Symbol is only used in that file and can be static. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index a24495e..5ef347a 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -154,7 +154,7 @@ bool amd_iommu_iotlb_sup __read_mostly = true; u32 amd_iommu_max_pasid __read_mostly = ~0; bool amd_iommu_v2_present __read_mostly; -bool amd_iommu_pc_present __read_mostly; +static bool amd_iommu_pc_present __read_mostly; bool amd_iommu_force_isolation __read_mostly; -- cgit v0.10.2 From 23d3a98c13ee0ffe2647121fac7533282643e6f1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:15:13 +0200 Subject: iommu/amd: Use BUG_ON instead of if () BUG() Found by a coccicheck script. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index a158579..f82060e7 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1835,8 +1835,8 @@ static void free_gcr3_table(struct protection_domain *domain) free_gcr3_tbl_level2(domain->gcr3_tbl); else if (domain->glx == 1) free_gcr3_tbl_level1(domain->gcr3_tbl); - else if (domain->glx != 0) - BUG(); + else + BUG_ON(domain->glx != 0); free_page((unsigned long)domain->gcr3_tbl); } diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index f7b875b..1131664 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -356,8 +356,8 @@ static void free_pasid_states(struct device_state *dev_state) free_pasid_states_level2(dev_state->states); else if (dev_state->pasid_levels == 1) free_pasid_states_level1(dev_state->states); - else if (dev_state->pasid_levels != 0) - BUG(); + else + BUG_ON(dev_state->pasid_levels != 0); free_page((unsigned long)dev_state->states); } -- cgit v0.10.2 From 30e93761fbf706c0f8a6f7d1abc1b0ddbeea208c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:13:17 +0200 Subject: iommu/vt-d: Return false instead of 0 in irq_remapping_cap() The function return type is bool, so return false instead of 0. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 2d99930..913455a 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -84,7 +84,7 @@ void set_irq_remapping_broken(void) bool irq_remapping_cap(enum irq_remap_cap cap) { if (!remap_ops || disable_irq_post) - return 0; + return false; return (remap_ops->capability & (1 << cap)); } -- cgit v0.10.2 From dc02e46e8d0234eed9f6e42f50763b406c380bc4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:15:13 +0200 Subject: iommu/vt-d: Use BUG_ON instead of if () BUG() Found by a coccicheck script. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 8834765..2a7e017 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4760,8 +4760,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, /* Cope with horrid API which requires us to unmap more than the size argument if it happens to be a large-page mapping. */ - if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level)) - BUG(); + BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level)); if (size < VTD_PAGE_SIZE << level_to_offset_bits(level)) size = VTD_PAGE_SIZE << level_to_offset_bits(level); -- cgit v0.10.2 From b690420a406256c83ef2c7e96466052e5cab7676 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:32:18 +0200 Subject: iommu/vt-d: Make two functions static These functions are only used in that file and can be static. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 2a7e017..93f16aa 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1210,9 +1210,9 @@ next: /* We can't just free the pages because the IOMMU may still be walking the page tables, and may have cached the intermediate levels. The pages can only be freed after the IOTLB flush has been done. */ -struct page *domain_unmap(struct dmar_domain *domain, - unsigned long start_pfn, - unsigned long last_pfn) +static struct page *domain_unmap(struct dmar_domain *domain, + unsigned long start_pfn, + unsigned long last_pfn) { struct page *freelist = NULL; @@ -1236,7 +1236,7 @@ struct page *domain_unmap(struct dmar_domain *domain, return freelist; } -void dma_free_pagelist(struct page *freelist) +static void dma_free_pagelist(struct page *freelist) { struct page *pg; -- cgit v0.10.2 From 543c8dcf1d3762c6fe372acf78eedc8898709106 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:56:59 +0200 Subject: iommu/vt-d: Access iomem correctly This fixes wrong accesses to iomem introduced by the kdump fixing code. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 93f16aa..a85077d 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2799,15 +2799,18 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu) } static int copy_context_table(struct intel_iommu *iommu, - struct root_entry *old_re, + struct root_entry __iomem *old_re, struct context_entry **tbl, int bus, bool ext) { - struct context_entry *old_ce = NULL, *new_ce = NULL, ce; int tbl_idx, pos = 0, idx, devfn, ret = 0, did; + struct context_entry __iomem *old_ce = NULL; + struct context_entry *new_ce = NULL, ce; + struct root_entry re; phys_addr_t old_ce_phys; tbl_idx = ext ? bus * 2 : bus; + memcpy_fromio(&re, old_re, sizeof(re)); for (devfn = 0; devfn < 256; devfn++) { /* First calculate the correct index */ @@ -2827,9 +2830,9 @@ static int copy_context_table(struct intel_iommu *iommu, ret = 0; if (devfn < 0x80) - old_ce_phys = root_entry_lctp(old_re); + old_ce_phys = root_entry_lctp(&re); else - old_ce_phys = root_entry_uctp(old_re); + old_ce_phys = root_entry_uctp(&re); if (!old_ce_phys) { if (ext && devfn == 0) { @@ -2854,7 +2857,7 @@ static int copy_context_table(struct intel_iommu *iommu, } /* Now copy the context entry */ - ce = old_ce[idx]; + memcpy_fromio(&ce, old_ce + idx, sizeof(ce)); if (!__context_present(&ce)) continue; @@ -2898,8 +2901,8 @@ out: static int copy_translation_tables(struct intel_iommu *iommu) { + struct root_entry __iomem *old_rt; struct context_entry **ctxt_tbls; - struct root_entry *old_rt; phys_addr_t old_rt_phys; int ctxt_table_entries; unsigned long flags; diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 27cdfa8..9ec4e0d 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -384,7 +384,7 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev) static int iommu_load_old_irte(struct intel_iommu *iommu) { - struct irte *old_ir_table; + struct irte __iomem *old_ir_table; phys_addr_t irt_phys; unsigned int i; size_t size; @@ -413,7 +413,7 @@ static int iommu_load_old_irte(struct intel_iommu *iommu) return -ENOMEM; /* Copy data over */ - memcpy(iommu->ir_table->base, old_ir_table, size); + memcpy_fromio(iommu->ir_table->base, old_ir_table, size); __iommu_flush_cache(iommu, iommu->ir_table->base, size); -- cgit v0.10.2 From 6e6cfbc859481b2af7282170ff732fa5e035d842 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 11:15:13 +0200 Subject: iommu/msm: Use BUG_ON instead of if () BUG() Found by a coccicheck script. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 15a2063..e321fa5 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -106,8 +106,8 @@ static int __flush_iotlb(struct iommu_domain *domain) #endif list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { - if (!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent) - BUG(); + + BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent); iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); BUG_ON(!iommu_drvdata); -- cgit v0.10.2 From 2e169bb3ccc20fea5e6f59d1abcea249c6598163 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 13 Aug 2015 12:01:10 +0200 Subject: iommu/io-pgtable-arm: Move init-fn declarations to io-pgtable.h Avoid extern declarations in c files. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 6436fe2..6f2e319 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -24,11 +24,6 @@ #include "io-pgtable.h" -extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; -extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; - static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 48538a3..ac9e234 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -143,4 +143,9 @@ struct io_pgtable_init_fns { void (*free)(struct io_pgtable *iop); }; +extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; + #endif /* __IO_PGTABLE_H */ -- cgit v0.10.2 From cf27ec930be906e142c752f9161197d69ca534d7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Tue, 11 Aug 2015 16:48:32 +0100 Subject: iommu/io-pgtable-arm: Unmap and free table when overwriting with block When installing a block mapping, we unconditionally overwrite a non-leaf PTE if we find one. However, this can cause a problem if the following sequence of events occur: (1) iommu_map called for a 4k (i.e. PAGE_SIZE) mapping at some address - We initialise the page table all the way down to a leaf entry - No TLB maintenance is required, because we're going from invalid to valid. (2) iommu_unmap is called on the mapping installed in (1) - We walk the page table to the final (leaf) entry and zero it - We only changed a valid leaf entry, so we invalidate leaf-only (3) iommu_map is called on the same address as (1), but this time for a 2MB (i.e. BLOCK_SIZE) mapping) - We walk the page table down to the penultimate level, where we find a table entry - We overwrite the table entry with a block mapping and return without any TLB maintenance and without freeing the memory used by the now-orphaned table. This last step can lead to a walk-cache caching the overwritten table entry, causing unexpected faults when the new mapping is accessed by a device. One way to fix this would be to collapse the page table when freeing the last page at a given level, but this would require expensive iteration on every map call. Instead, this patch detects the case when we are overwriting a table entry and explicitly unmaps the table first, which takes care of both freeing and TLB invalidation. Cc: Reported-by: Brian Starkey Tested-by: Brian Starkey Signed-off-by: Will Deacon Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index e4bc2b2..73c0748 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -263,6 +263,10 @@ static void __arm_lpae_set_pte(arm_lpae_iopte *ptep, arm_lpae_iopte pte, sizeof(pte), DMA_TO_DEVICE); } +static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_lpae_iopte *ptep); + static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, @@ -271,10 +275,21 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, arm_lpae_iopte pte = prot; struct io_pgtable_cfg *cfg = &data->iop.cfg; - /* We require an unmap first */ if (iopte_leaf(*ptep, lvl)) { + /* We require an unmap first */ WARN_ON(!selftest_running); return -EEXIST; + } else if (iopte_type(*ptep, lvl) == ARM_LPAE_PTE_TYPE_TABLE) { + /* + * We need to unmap and free the old table before + * overwriting it with a block entry. + */ + arm_lpae_iopte *tblp; + size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data); + + tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data); + if (WARN_ON(__arm_lpae_unmap(data, iova, sz, lvl, tblp) != sz)) + return -EINVAL; } if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) -- cgit v0.10.2 From 57fb907da89977640ef183556a621336c1348fa0 Mon Sep 17 00:00:00 2001 From: Emil Medve Date: Wed, 25 Mar 2015 00:28:48 -0500 Subject: iommu/fsl: Really fix init section(s) content '0f1fb99 iommu/fsl: Fix section mismatch' was intended to address the modpost warning and the potential crash. Crash which is actually easy to trigger with a 'unbind' followed by a 'bind' sequence. The fix is wrong as fsl_of_pamu_driver.driver gets added by bus_add_driver() to a couple of klist(s) which become invalid/corrupted as soon as the init sections are freed. Depending on when/how the init sections storage is reused various/random errors and crashes will happen 'cd70d46 iommu/fsl: Various cleanups' contains annotations that go further down the wrong path laid by '0f1fb99 iommu/fsl: Fix section mismatch' Now remove all the incorrect annotations from the above mentioned patches (not exactly a revert) and those previously existing in the code, This fixes the modpost warning(s), the unbind/bind sequence crashes and the random errors/crashes Fixes: 0f1fb99b62ce ("iommu/fsl: Fix section mismatch") Fixes: cd70d4659ff3 ("iommu/fsl: Various cleanups") Signed-off-by: Emil Medve Acked-by: Varun Sethi Cc: stable@vger.kernel.org Tested-by: Madalin Bucur Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index abeedc9..2570f2a2 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -41,7 +41,6 @@ struct pamu_isr_data { static struct paace *ppaact; static struct paace *spaact; -static struct ome *omt __initdata; /* * Table for matching compatible strings, for device tree @@ -50,7 +49,7 @@ static struct ome *omt __initdata; * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" * string would be used. */ -static const struct of_device_id guts_device_ids[] __initconst = { +static const struct of_device_id guts_device_ids[] = { { .compatible = "fsl,qoriq-device-config-1.0", }, { .compatible = "fsl,qoriq-device-config-2.0", }, {} @@ -599,7 +598,7 @@ found_cpu_node: * Memory accesses to QMAN and BMAN private memory need not be coherent, so * clear the PAACE entry coherency attribute for them. */ -static void __init setup_qbman_paace(struct paace *ppaace, int paace_type) +static void setup_qbman_paace(struct paace *ppaace, int paace_type) { switch (paace_type) { case QMAN_PAACE: @@ -629,7 +628,7 @@ static void __init setup_qbman_paace(struct paace *ppaace, int paace_type) * this table to translate device transaction to appropriate corenet * transaction. */ -static void __init setup_omt(struct ome *omt) +static void setup_omt(struct ome *omt) { struct ome *ome; @@ -666,7 +665,7 @@ static void __init setup_omt(struct ome *omt) * Get the maximum number of PAACT table entries * and subwindows supported by PAMU */ -static void __init get_pamu_cap_values(unsigned long pamu_reg_base) +static void get_pamu_cap_values(unsigned long pamu_reg_base) { u32 pc_val; @@ -676,9 +675,9 @@ static void __init get_pamu_cap_values(unsigned long pamu_reg_base) } /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ -static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, - phys_addr_t ppaact_phys, phys_addr_t spaact_phys, - phys_addr_t omt_phys) +static int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, + phys_addr_t ppaact_phys, phys_addr_t spaact_phys, + phys_addr_t omt_phys) { u32 *pc; struct pamu_mmap_regs *pamu_regs; @@ -720,7 +719,7 @@ static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu } /* Enable all device LIODNS */ -static void __init setup_liodns(void) +static void setup_liodns(void) { int i, len; struct paace *ppaace; @@ -846,7 +845,7 @@ struct ccsr_law { /* * Create a coherence subdomain for a given memory block. */ -static int __init create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) +static int create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) { struct device_node *np; const __be32 *iprop; @@ -988,7 +987,7 @@ error: static const struct { u32 svr; u32 port_id; -} port_id_map[] __initconst = { +} port_id_map[] = { {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ @@ -1006,7 +1005,7 @@ static const struct { #define SVR_SECURITY 0x80000 /* The Security (E) bit */ -static int __init fsl_pamu_probe(struct platform_device *pdev) +static int fsl_pamu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; void __iomem *pamu_regs = NULL; @@ -1022,6 +1021,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) int irq; phys_addr_t ppaact_phys; phys_addr_t spaact_phys; + struct ome *omt; phys_addr_t omt_phys; size_t mem_size = 0; unsigned int order = 0; @@ -1200,7 +1200,7 @@ error: return ret; } -static struct platform_driver fsl_of_pamu_driver __initdata = { +static struct platform_driver fsl_of_pamu_driver = { .driver = { .name = "fsl-of-pamu", }, -- cgit v0.10.2 From ca30475698696af3a03f6eaee16472ae09d42269 Mon Sep 17 00:00:00 2001 From: "Xiao, Nan" Date: Mon, 24 Aug 2015 06:22:42 +0000 Subject: x86/vt-d: Fix documentation of DRHD According to "Intel Virtualization Technology for Directed I/O" specification, DRHD stands for "DMA Remapping Hardware Unit Definition" , not "DMA Engine Reporting Structure". Signed-off-by: Nan Xiao Signed-off-by: Joerg Roedel diff --git a/Documentation/Intel-IOMMU.txt b/Documentation/Intel-IOMMU.txt index cf9431d..7b57fc0 100644 --- a/Documentation/Intel-IOMMU.txt +++ b/Documentation/Intel-IOMMU.txt @@ -10,7 +10,7 @@ This guide gives a quick cheat sheet for some basic understanding. Some Keywords DMAR - DMA remapping -DRHD - DMA Engine Reporting Structure +DRHD - DMA Remapping Hardware Unit Definition RMRR - Reserved memory Region Reporting Structure ZLR - Zero length reads from PCI devices IOVA - IO Virtual address. -- cgit v0.10.2 From 4df4eab168c1c4058603be55a3169d4a45779cc0 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 25 Aug 2015 10:54:28 +0200 Subject: iommu/vt-d: Really use upper context table when necessary There is a bug in iommu_context_addr() which will always use the lower context table, even when the upper context table needs to be used. Fix this issue. Fixes: 03ecc32c5274 ("iommu/vt-d: support extended root and context entries") Reported-by: Xiao, Nan Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a85077d..63daf1b 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -803,6 +803,7 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu struct context_entry *context; u64 *entry; + entry = &root->lo; if (ecs_enabled(iommu)) { if (devfn >= 0x80) { devfn -= 0x80; @@ -810,7 +811,6 @@ static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu } devfn *= 2; } - entry = &root->lo; if (*entry & 1) context = phys_to_virt(*entry & VTD_PAGE_MASK); else { -- cgit v0.10.2