From d760e3e0f1afb96e0b813384258cea0afcf1b4d4 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 17 Mar 2014 20:31:32 -0500 Subject: iommu/omap: Correct init value of iotlb_entry valid field The iotlb_entry field values are used directly in omap2_alloc_cr, a function used in preparing the MMU_CAM and MMU_RAM registers. The iotlb_entry.valid value is being set incorrectly to 1 at the moment, and this would result in overriding the PAGESIZE bit field of the MMU_CAM register if prefetching of the entries were to be supported. The bug has not caused any MMU faults due to incorrect size programming so far as the prefetching is disabled by default. Fix this by using the correct init value for the iotlb_entry.valid field. Signed-off-by: Suman Anna Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 7fcbfc4..64d935b 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1044,7 +1044,7 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, e->da = da; e->pa = pa; - e->valid = 1; + e->valid = MMU_CAM_V; /* FIXME: add OMAP1 support */ e->pgsz = flags & MMU_CAM_PGSZ_MASK; e->endian = flags & MMU_RAM_ENDIAN_MASK; -- cgit v0.10.2 From 2ac6133bf6280350105b3181bbed31fb183b9734 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 17 Mar 2014 20:31:33 -0500 Subject: iommu/omap: Remove omap_iommu_domain_has_cap() function The current OMAP IOMMU ops for .domain_has_cap is a stub, and the iommu core already returns a value of 0 if the domain doesn't have a .domain_has_cap ops plugged in. So, clean up this stub function. Signed-off-by: Suman Anna Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 64d935b..7ce136d 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1248,12 +1248,6 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, return ret; } -static int omap_iommu_domain_has_cap(struct iommu_domain *domain, - unsigned long cap) -{ - return 0; -} - static int omap_iommu_add_device(struct device *dev) { struct omap_iommu_arch_data *arch_data; @@ -1305,7 +1299,6 @@ static struct iommu_ops omap_iommu_ops = { .map = omap_iommu_map, .unmap = omap_iommu_unmap, .iova_to_phys = omap_iommu_iova_to_phys, - .domain_has_cap = omap_iommu_domain_has_cap, .add_device = omap_iommu_add_device, .remove_device = omap_iommu_remove_device, .pgsize_bitmap = OMAP_IOMMU_PGSIZES, -- cgit v0.10.2 From 5acc97db94321343f42866f2da90c3b02095c374 Mon Sep 17 00:00:00 2001 From: Suman Anna Date: Mon, 17 Mar 2014 20:31:34 -0500 Subject: iommu/omap: Move to_iommu definition from omap-iopgtable.h The to_iommu definition is used only locally to the omap-iommu.c source file, and it has nothing to do with the page attributes defined in omap-iopgtable.h. So, move the definition out of omap-iopgtable.h header file. Signed-off-by: Suman Anna Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 7ce136d..78f32d1 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -34,6 +34,9 @@ #include "omap-iopgtable.h" #include "omap-iommu.h" +#define to_iommu(dev) \ + ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) + #define for_each_iotlb_cr(obj, n, __i, cr) \ for (__i = 0; \ (__i < (n)) && (cr = __iotlb_read_cr((obj), __i), true); \ diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h index b6f9a51..f891683 100644 --- a/drivers/iommu/omap-iopgtable.h +++ b/drivers/iommu/omap-iopgtable.h @@ -93,6 +93,3 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask) /* to find an entry in the second-level page table. */ #define iopte_index(da) (((da) >> IOPTE_SHIFT) & (PTRS_PER_IOPTE - 1)) #define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da)) - -#define to_iommu(dev) \ - (platform_get_drvdata(to_platform_device(dev))) -- cgit v0.10.2 From f7129a0e4d5835b7851208d35984bc80116f1ed0 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Fri, 7 Mar 2014 23:47:03 +0100 Subject: iommu/omap: Fix 'no page for' debug message in flush_iotlb_page() The flush_iotlb_page() function prints a debug message when no corresponding page was found in the TLB. That condition is incorrectly checked and always resolves to true, given that the for_each_iotlb_cr() loop is never interrupted and always reaches obj->nr_tlb_entries. Given that we can't have two TLB entries for the same VA, break from the loop when a match is found. Signed-off-by: Laurent Pinchart Acked-by: Suman Anna diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 78f32d1..b5787f1 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -394,6 +394,7 @@ static void flush_iotlb_page(struct omap_iommu *obj, u32 da) __func__, start, da, bytes); iotlb_load_cr(obj, &cr); iommu_write_reg(obj, 1, MMU_FLUSH_ENTRY); + break; } } pm_runtime_put_sync(obj->dev); -- cgit v0.10.2 From 67b779d28d6e8afdd79a70423324273018114cad Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sat, 8 Mar 2014 00:50:39 +0100 Subject: iommu/omap: Remove comment about supporting single page mappings only The IOMMU core breaks out mappings into pages already, there's no need to support mapping multiple pages in one go. Signed-off-by: Laurent Pinchart Reviewed-by: Sakari Ailus diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index b5787f1..31cebf2 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1068,7 +1068,6 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, int omap_pgsz; u32 ret, flags; - /* we only support mapping a single iommu page for now */ omap_pgsz = bytes_to_iopgsz(bytes); if (omap_pgsz < 0) { dev_err(dev, "invalid size to map: %d\n", bytes); -- cgit v0.10.2 From 286f600bc890347f7ec7bd50d33210d53a9095a3 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sat, 8 Mar 2014 00:44:38 +0100 Subject: iommu/omap: Fix map protection value handling The prot flags passed to the IOMMU map handler are defined in include/linux/iommu.h as IOMMU_(READ|WRITE|CACHE|EXEC). However, the driver expects to receive MMU_RAM_* OMAP-specific flags. This causes IOMMU flags being interpreted as page sizes, leading to failures. Hardcode the OMAP mapping parameters to little-endian, 8-bits and non-mixed page attributes. Furthermore, as the OMAP IOMMU doesn't support read-only or write-only mappings, ignore the prot value. Signed-off-by: Laurent Pinchart Reviewed-by: Sakari Ailus Acked-by: Suman Anna diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 31cebf2..895af06 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1041,8 +1041,7 @@ static void iopte_cachep_ctor(void *iopte) clean_dcache_area(iopte, IOPTE_TABLE_SIZE); } -static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, - u32 flags) +static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz) { memset(e, 0, sizeof(*e)); @@ -1050,10 +1049,10 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, e->pa = pa; e->valid = MMU_CAM_V; /* FIXME: add OMAP1 support */ - e->pgsz = flags & MMU_CAM_PGSZ_MASK; - e->endian = flags & MMU_RAM_ENDIAN_MASK; - e->elsz = flags & MMU_RAM_ELSZ_MASK; - e->mixed = flags & MMU_RAM_MIXED_MASK; + e->pgsz = pgsz; + e->endian = MMU_RAM_ENDIAN_LITTLE; + e->elsz = MMU_RAM_ELSZ_8; + e->mixed = 0; return iopgsz_to_bytes(e->pgsz); } @@ -1066,7 +1065,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, struct device *dev = oiommu->dev; struct iotlb_entry e; int omap_pgsz; - u32 ret, flags; + u32 ret; omap_pgsz = bytes_to_iopgsz(bytes); if (omap_pgsz < 0) { @@ -1076,9 +1075,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); - flags = omap_pgsz | prot; - - iotlb_init_entry(&e, da, pa, flags); + iotlb_init_entry(&e, da, pa, omap_pgsz); ret = omap_iopgtable_store_entry(oiommu, &e); if (ret) -- cgit v0.10.2 From ac1ef1fed3d2c1cd4902ecb92876cb239c97a870 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Fri, 21 Feb 2014 16:54:53 +0100 Subject: iommu/shmobile: Don't ignore the ipmmu_iommu_init() return value The function can fail, don't ignore its error value. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c index e3bc2e1..6ba6110 100644 --- a/drivers/iommu/shmobile-ipmmu.c +++ b/drivers/iommu/shmobile-ipmmu.c @@ -117,8 +117,7 @@ static int ipmmu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, ipmmu); ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */ ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */ - ipmmu_iommu_init(ipmmu); - return 0; + return ipmmu_iommu_init(ipmmu); } static struct platform_driver ipmmu_driver = { -- cgit v0.10.2 From b11762f8431b3a5cf1c156876e79efab7d7f80c4 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Fri, 21 Feb 2014 16:54:53 +0100 Subject: iommu/shmobile: Use devm_ioremap_resource() Replace the devm_ioremap_nocache() call with devm_ioremap_resource(). This simplifies error checking. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c index 6ba6110..bd97ade 100644 --- a/drivers/iommu/shmobile-ipmmu.c +++ b/drivers/iommu/shmobile-ipmmu.c @@ -94,11 +94,6 @@ static int ipmmu_probe(struct platform_device *pdev) struct resource *res; struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_err(&pdev->dev, "cannot get platform resources\n"); - return -ENOENT; - } ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL); if (!ipmmu) { dev_err(&pdev->dev, "cannot allocate device data\n"); @@ -106,12 +101,12 @@ static int ipmmu_probe(struct platform_device *pdev) } spin_lock_init(&ipmmu->flush_lock); ipmmu->dev = &pdev->dev; - ipmmu->ipmmu_base = devm_ioremap_nocache(&pdev->dev, res->start, - resource_size(res)); - if (!ipmmu->ipmmu_base) { - dev_err(&pdev->dev, "ioremap_nocache failed\n"); - return -ENOMEM; - } + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(ipmmu->ipmmu_base)) + return PTR_ERR(ipmmu->ipmmu_base); + ipmmu->dev_names = pdata->dev_names; ipmmu->num_dev_names = pdata->num_dev_names; platform_set_drvdata(pdev, ipmmu); -- cgit v0.10.2 From c44308413b92a555d1501a6c0db9489e288bb81b Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 13 Mar 2014 11:46:57 +0000 Subject: iommu/arm-smmu: allow 42-bit stage-1 output size with 64K pages The output size of stage-1 is currently limited by the input size of stage-2, which is further limited by VA_BITS since we make use of the standard pgd_alloc functions for creating page tables. This patch ensures that we use VA_BITS instead of a hardcoded '39' for the stage-1 output size limit. Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 647c3c7..afcb0e3 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1804,7 +1804,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) * allocation (PTRS_PER_PGD). */ #ifdef CONFIG_64BIT - smmu->s1_output_size = min(39UL, size); + smmu->s1_output_size = min((unsigned long)VA_BITS, size); #else smmu->s1_output_size = min(32UL, size); #endif -- cgit v0.10.2 From 6069d23ce0e568909d847f28af0257b085e8e77a Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Fri, 18 Apr 2014 10:20:48 +0800 Subject: iommu/arm-smmu: fix incorrect use of S2CR_TYPE_SHIFT There is already S2CR_TYPE_SHIFT in S2CR_TYPE_TRANS macro, so drop the second shift. Note that, since S2CR_TYPE_SHIFT is 0x0, there is no functional change introduced by this patch. Signed-off-by: Kefeng Wang Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index afcb0e3..1599354 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1167,7 +1167,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, for (i = 0; i < master->num_streamids; ++i) { u32 idx, s2cr; idx = master->smrs ? master->smrs[i].idx : master->streamids[i]; - s2cr = (S2CR_TYPE_TRANS << S2CR_TYPE_SHIFT) | + s2cr = S2CR_TYPE_TRANS | (smmu_domain->root_cfg.cbndx << S2CR_CBNDX_SHIFT); writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); } -- cgit v0.10.2 From 7222e8db2d506197ee183de0f9b76b3ad97e8c18 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:46 +0530 Subject: iommu/exynos: Fix build errors Commit 25e9d28d92 (ARM: EXYNOS: remove system mmu initialization from exynos tree) removed arch/arm/mach-exynos/mach/sysmmu.h header without removing remaining use of it from exynos-iommu driver, thus causing a compilation error. This patch fixes the error by removing respective include line from exynos-iommu.c. Use of __pa and __va macro is changed to virt_to_phys and phys_to_virt which are recommended in driver code. printk formatting of physical address is also fixed to %pa. Also System MMU driver is changed to control only a single instance of System MMU at a time. Since a single instance of System MMU has only a single clock descriptor for its clock gating, single address range for control registers, there is no need to obtain two or more clock descriptors and ioremaped region. CC: Tomasz Figa Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 0740189..8d7c3f9 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -29,8 +29,6 @@ #include #include -#include - /* We does not consider super section mapping (16MB) */ #define SECT_ORDER 20 #define LPAGE_ORDER 16 @@ -108,7 +106,8 @@ static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova) static unsigned long *page_entry(unsigned long *sent, unsigned long iova) { - return (unsigned long *)__va(lv2table_base(sent)) + lv2ent_offset(iova); + return (unsigned long *)phys_to_virt( + lv2table_base(sent)) + lv2ent_offset(iova); } enum exynos_sysmmu_inttype { @@ -132,7 +131,7 @@ enum exynos_sysmmu_inttype { * translated. This is 0 if @itype is SYSMMU_BUSERROR. */ typedef int (*sysmmu_fault_handler_t)(enum exynos_sysmmu_inttype itype, - unsigned long pgtable_base, unsigned long fault_addr); + phys_addr_t pgtable_base, unsigned long fault_addr); static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = { REG_PAGE_FAULT_ADDR, @@ -170,14 +169,13 @@ struct sysmmu_drvdata { struct device *sysmmu; /* System MMU's device descriptor */ struct device *dev; /* Owner of system MMU */ char *dbgname; - int nsfrs; - void __iomem **sfrbases; - struct clk *clk[2]; + void __iomem *sfrbase; + struct clk *clk; int activations; rwlock_t lock; struct iommu_domain *domain; sysmmu_fault_handler_t fault_handler; - unsigned long pgtable; + phys_addr_t pgtable; }; static bool set_sysmmu_active(struct sysmmu_drvdata *data) @@ -266,17 +264,17 @@ void exynos_sysmmu_set_fault_handler(struct device *dev, } static int default_fault_handler(enum exynos_sysmmu_inttype itype, - unsigned long pgtable_base, unsigned long fault_addr) + phys_addr_t pgtable_base, unsigned long fault_addr) { unsigned long *ent; if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) itype = SYSMMU_FAULT_UNKNOWN; - pr_err("%s occurred at 0x%lx(Page table base: 0x%lx)\n", - sysmmu_fault_name[itype], fault_addr, pgtable_base); + pr_err("%s occurred at 0x%lx(Page table base: %pa)\n", + sysmmu_fault_name[itype], fault_addr, &pgtable_base); - ent = section_entry(__va(pgtable_base), fault_addr); + ent = section_entry(phys_to_virt(pgtable_base), fault_addr); pr_err("\tLv1 entry: 0x%lx\n", *ent); if (lv1ent_page(ent)) { @@ -295,56 +293,39 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { /* SYSMMU is in blocked when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; - struct resource *irqres; - struct platform_device *pdev; enum exynos_sysmmu_inttype itype; unsigned long addr = -1; - - int i, ret = -ENOSYS; + int ret = -ENOSYS; read_lock(&data->lock); WARN_ON(!is_sysmmu_active(data)); - pdev = to_platform_device(data->sysmmu); - for (i = 0; i < (pdev->num_resources / 2); i++) { - irqres = platform_get_resource(pdev, IORESOURCE_IRQ, i); - if (irqres && ((int)irqres->start == irq)) - break; - } - - if (i == pdev->num_resources) { + itype = (enum exynos_sysmmu_inttype) + __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); + if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) itype = SYSMMU_FAULT_UNKNOWN; - } else { - itype = (enum exynos_sysmmu_inttype) - __ffs(__raw_readl(data->sfrbases[i] + REG_INT_STATUS)); - if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) - itype = SYSMMU_FAULT_UNKNOWN; - else - addr = __raw_readl( - data->sfrbases[i] + fault_reg_offset[itype]); - } + else + addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]); if (data->domain) - ret = report_iommu_fault(data->domain, data->dev, - addr, itype); + ret = report_iommu_fault(data->domain, data->dev, addr, itype); if ((ret == -ENOSYS) && data->fault_handler) { unsigned long base = data->pgtable; if (itype != SYSMMU_FAULT_UNKNOWN) - base = __raw_readl( - data->sfrbases[i] + REG_PT_BASE_ADDR); + base = __raw_readl(data->sfrbase + REG_PT_BASE_ADDR); ret = data->fault_handler(itype, base, addr); } if (!ret && (itype != SYSMMU_FAULT_UNKNOWN)) - __raw_writel(1 << itype, data->sfrbases[i] + REG_INT_CLEAR); + __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); else dev_dbg(data->sysmmu, "(%s) %s is not handled.\n", data->dbgname, sysmmu_fault_name[itype]); if (itype != SYSMMU_FAULT_UNKNOWN) - sysmmu_unblock(data->sfrbases[i]); + sysmmu_unblock(data->sfrbase); read_unlock(&data->lock); @@ -355,20 +336,16 @@ static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) { unsigned long flags; bool disabled = false; - int i; write_lock_irqsave(&data->lock, flags); if (!set_sysmmu_inactive(data)) goto finish; - for (i = 0; i < data->nsfrs; i++) - __raw_writel(CTRL_DISABLE, data->sfrbases[i] + REG_MMU_CTRL); + __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); - if (data->clk[1]) - clk_disable(data->clk[1]); - if (data->clk[0]) - clk_disable(data->clk[0]); + if (!IS_ERR(data->clk)) + clk_disable(data->clk); disabled = true; data->pgtable = 0; @@ -394,7 +371,7 @@ finish: static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, unsigned long pgtable, struct iommu_domain *domain) { - int i, ret = 0; + int ret = 0; unsigned long flags; write_lock_irqsave(&data->lock, flags); @@ -411,27 +388,22 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, goto finish; } - if (data->clk[0]) - clk_enable(data->clk[0]); - if (data->clk[1]) - clk_enable(data->clk[1]); + if (!IS_ERR(data->clk)) + clk_enable(data->clk); data->pgtable = pgtable; - for (i = 0; i < data->nsfrs; i++) { - __sysmmu_set_ptbase(data->sfrbases[i], pgtable); - - if ((readl(data->sfrbases[i] + REG_MMU_VERSION) >> 28) == 3) { - /* System MMU version is 3.x */ - __raw_writel((1 << 12) | (2 << 28), - data->sfrbases[i] + REG_MMU_CFG); - __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 0); - __sysmmu_set_prefbuf(data->sfrbases[i], 0, -1, 1); - } - - __raw_writel(CTRL_ENABLE, data->sfrbases[i] + REG_MMU_CTRL); + __sysmmu_set_ptbase(data->sfrbase, pgtable); + if ((readl(data->sfrbase + REG_MMU_VERSION) >> 28) == 3) { + /* System MMU version is 3.x */ + __raw_writel((1 << 12) | (2 << 28), + data->sfrbase + REG_MMU_CFG); + __sysmmu_set_prefbuf(data->sfrbase, 0, -1, 0); + __sysmmu_set_prefbuf(data->sfrbase, 0, -1, 1); } + __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + data->domain = domain; dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname); @@ -458,7 +430,7 @@ int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable) if (WARN_ON(ret < 0)) { pm_runtime_put(data->sysmmu); dev_err(data->sysmmu, - "(%s) Already enabled with page table %#lx\n", + "(%s) Already enabled with page table %#x\n", data->dbgname, data->pgtable); } else { data->dev = dev; @@ -486,13 +458,10 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova) read_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { - int i; - for (i = 0; i < data->nsfrs; i++) { - if (sysmmu_block(data->sfrbases[i])) { - __sysmmu_tlb_invalidate_entry( - data->sfrbases[i], iova); - sysmmu_unblock(data->sfrbases[i]); - } + if (sysmmu_block(data->sfrbase)) { + __sysmmu_tlb_invalidate_entry( + data->sfrbase, iova); + sysmmu_unblock(data->sfrbase); } } else { dev_dbg(data->sysmmu, @@ -511,12 +480,9 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) read_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { - int i; - for (i = 0; i < data->nsfrs; i++) { - if (sysmmu_block(data->sfrbases[i])) { - __sysmmu_tlb_invalidate(data->sfrbases[i]); - sysmmu_unblock(data->sfrbases[i]); - } + if (sysmmu_block(data->sfrbase)) { + __sysmmu_tlb_invalidate(data->sfrbase); + sysmmu_unblock(data->sfrbase); } } else { dev_dbg(data->sysmmu, @@ -529,11 +495,10 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) static int exynos_sysmmu_probe(struct platform_device *pdev) { - int i, ret; - struct device *dev; + int ret; + struct device *dev = &pdev->dev; struct sysmmu_drvdata *data; - - dev = &pdev->dev; + struct resource *res; data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { @@ -542,82 +507,37 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) goto err_alloc; } - ret = dev_set_drvdata(dev, data); - if (ret) { - dev_dbg(dev, "Unabled to initialize driver data\n"); + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_dbg(dev, "Unable to find IOMEM region\n"); + ret = -ENOENT; goto err_init; } - data->nsfrs = pdev->num_resources / 2; - data->sfrbases = kmalloc(sizeof(*data->sfrbases) * data->nsfrs, - GFP_KERNEL); - if (data->sfrbases == NULL) { - dev_dbg(dev, "Not enough memory\n"); - ret = -ENOMEM; - goto err_init; + data->sfrbase = ioremap(res->start, resource_size(res)); + if (!data->sfrbase) { + dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n", res->start); + ret = -ENOENT; + goto err_res; } - for (i = 0; i < data->nsfrs; i++) { - struct resource *res; - res = platform_get_resource(pdev, IORESOURCE_MEM, i); - if (!res) { - dev_dbg(dev, "Unable to find IOMEM region\n"); - ret = -ENOENT; - goto err_res; - } - - data->sfrbases[i] = ioremap(res->start, resource_size(res)); - if (!data->sfrbases[i]) { - dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n", - res->start); - ret = -ENOENT; - goto err_res; - } + ret = platform_get_irq(pdev, 0); + if (ret <= 0) { + dev_dbg(dev, "Unable to find IRQ resource\n"); + goto err_irq; } - for (i = 0; i < data->nsfrs; i++) { - ret = platform_get_irq(pdev, i); - if (ret <= 0) { - dev_dbg(dev, "Unable to find IRQ resource\n"); - goto err_irq; - } - - ret = request_irq(ret, exynos_sysmmu_irq, 0, - dev_name(dev), data); - if (ret) { - dev_dbg(dev, "Unabled to register interrupt handler\n"); - goto err_irq; - } + ret = request_irq(ret, exynos_sysmmu_irq, 0, + dev_name(dev), data); + if (ret) { + dev_dbg(dev, "Unabled to register interrupt handler\n"); + goto err_irq; } if (dev_get_platdata(dev)) { - char *deli, *beg; - struct sysmmu_platform_data *platdata = dev_get_platdata(dev); - - beg = platdata->clockname; - - for (deli = beg; (*deli != '\0') && (*deli != ','); deli++) - /* NOTHING */; - - if (*deli == '\0') - deli = NULL; - else - *deli = '\0'; - - data->clk[0] = clk_get(dev, beg); - if (IS_ERR(data->clk[0])) { - data->clk[0] = NULL; + data->clk = clk_get(dev, "sysmmu"); + if (IS_ERR(data->clk)) dev_dbg(dev, "No clock descriptor registered\n"); - } - - if (data->clk[0] && deli) { - *deli = ','; - data->clk[1] = clk_get(dev, deli + 1); - if (IS_ERR(data->clk[1])) - data->clk[1] = NULL; - } - - data->dbgname = platdata->dbgname; } data->sysmmu = dev; @@ -626,22 +546,17 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) __set_fault_handler(data, &default_fault_handler); + platform_set_drvdata(pdev, data); + if (dev->parent) pm_runtime_enable(dev); dev_dbg(dev, "(%s) Initialized\n", data->dbgname); return 0; err_irq: - while (i-- > 0) { - int irq; - - irq = platform_get_irq(pdev, i); - free_irq(irq, data); - } + free_irq(platform_get_irq(pdev, 0), data); err_res: - while (data->nsfrs-- > 0) - iounmap(data->sfrbases[data->nsfrs]); - kfree(data->sfrbases); + iounmap(data->sfrbase); err_init: kfree(data); err_alloc: @@ -722,7 +637,7 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain) for (i = 0; i < NUM_LV1ENTRIES; i++) if (lv1ent_page(priv->pgtable + i)) - kfree(__va(lv2table_base(priv->pgtable + i))); + kfree(phys_to_virt(lv2table_base(priv->pgtable + i))); free_pages((unsigned long)priv->pgtable, 2); free_pages((unsigned long)priv->lv2entcnt, 1); @@ -735,6 +650,7 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, { struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); struct exynos_iommu_domain *priv = domain->priv; + phys_addr_t pagetable = virt_to_phys(priv->pgtable); unsigned long flags; int ret; @@ -746,7 +662,7 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, spin_lock_irqsave(&priv->lock, flags); - ret = __exynos_sysmmu_enable(data, __pa(priv->pgtable), domain); + ret = __exynos_sysmmu_enable(data, pagetable, domain); if (ret == 0) { /* 'data->node' must not be appeared in priv->clients */ @@ -758,17 +674,15 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, spin_unlock_irqrestore(&priv->lock, flags); if (ret < 0) { - dev_err(dev, "%s: Failed to attach IOMMU with pgtable %#lx\n", - __func__, __pa(priv->pgtable)); + dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n", + __func__, &pagetable); pm_runtime_put(data->sysmmu); - } else if (ret > 0) { - dev_dbg(dev, "%s: IOMMU with pgtable 0x%lx already attached\n", - __func__, __pa(priv->pgtable)); - } else { - dev_dbg(dev, "%s: Attached new IOMMU with pgtable 0x%lx\n", - __func__, __pa(priv->pgtable)); + return ret; } + dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n", + __func__, &pagetable, (ret == 0) ? "" : ", again"); + return ret; } @@ -778,6 +692,7 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); struct exynos_iommu_domain *priv = domain->priv; struct list_head *pos; + phys_addr_t pagetable = virt_to_phys(priv->pgtable); unsigned long flags; bool found = false; @@ -794,13 +709,13 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, goto finish; if (__exynos_sysmmu_disable(data)) { - dev_dbg(dev, "%s: Detached IOMMU with pgtable %#lx\n", - __func__, __pa(priv->pgtable)); + dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", + __func__, &pagetable); list_del_init(&data->node); } else { - dev_dbg(dev, "%s: Detaching IOMMU with pgtable %#lx delayed", - __func__, __pa(priv->pgtable)); + dev_dbg(dev, "%s: Detaching IOMMU with pgtable %pa delayed", + __func__, &pagetable); } finish: @@ -821,7 +736,7 @@ static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, if (!pent) return NULL; - *sent = mk_lv1ent_page(__pa(pent)); + *sent = mk_lv1ent_page(virt_to_phys(pent)); *pgcounter = NUM_LV2ENTRIES; pgtable_flush(pent, pent + NUM_LV2ENTRIES); pgtable_flush(sent, sent + 1); -- cgit v0.10.2 From 61128f08fc110fc27cd50f5a8ca57c6a8c3bcddc Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:47 +0530 Subject: iommu/exynos: Change error handling when page table update is failed This patch changes not to panic on any error when updating page table. Instead prints error messages with callstack. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 8d7c3f9..aec7fd7 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -728,13 +728,18 @@ finish: static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, short *pgcounter) { + if (lv1ent_section(sent)) { + WARN(1, "Trying mapping on %#08lx mapped with 1MiB page", iova); + return ERR_PTR(-EADDRINUSE); + } + if (lv1ent_fault(sent)) { unsigned long *pent; pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC); BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1)); if (!pent) - return NULL; + return ERR_PTR(-ENOMEM); *sent = mk_lv1ent_page(virt_to_phys(pent)); *pgcounter = NUM_LV2ENTRIES; @@ -745,14 +750,21 @@ static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, return page_entry(sent, iova); } -static int lv1set_section(unsigned long *sent, phys_addr_t paddr, short *pgcnt) +static int lv1set_section(unsigned long *sent, unsigned long iova, + phys_addr_t paddr, short *pgcnt) { - if (lv1ent_section(sent)) + if (lv1ent_section(sent)) { + WARN(1, "Trying mapping on 1MiB@%#08lx that is mapped", + iova); return -EADDRINUSE; + } if (lv1ent_page(sent)) { - if (*pgcnt != NUM_LV2ENTRIES) + if (*pgcnt != NUM_LV2ENTRIES) { + WARN(1, "Trying mapping on 1MiB@%#08lx that is mapped", + iova); return -EADDRINUSE; + } kfree(page_entry(sent, 0)); @@ -770,8 +782,10 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size, short *pgcnt) { if (size == SPAGE_SIZE) { - if (!lv2ent_fault(pent)) + if (!lv2ent_fault(pent)) { + WARN(1, "Trying mapping on 4KiB where mapping exists"); return -EADDRINUSE; + } *pent = mk_lv2ent_spage(paddr); pgtable_flush(pent, pent + 1); @@ -780,7 +794,10 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size, int i; for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { if (!lv2ent_fault(pent)) { - memset(pent, 0, sizeof(*pent) * i); + WARN(1, + "Trying mapping on 64KiB where mapping exists"); + if (i > 0) + memset(pent - i, 0, sizeof(*pent) * i); return -EADDRINUSE; } @@ -808,7 +825,7 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, entry = section_entry(priv->pgtable, iova); if (size == SECT_SIZE) { - ret = lv1set_section(entry, paddr, + ret = lv1set_section(entry, iova, paddr, &priv->lv2entcnt[lv1ent_offset(iova)]); } else { unsigned long *pent; @@ -816,17 +833,16 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, pent = alloc_lv2entry(entry, iova, &priv->lv2entcnt[lv1ent_offset(iova)]); - if (!pent) - ret = -ENOMEM; + if (IS_ERR(pent)) + ret = PTR_ERR(pent); else ret = lv2set_page(pent, paddr, size, &priv->lv2entcnt[lv1ent_offset(iova)]); } - if (ret) { + if (ret) pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n", __func__, iova, size); - } spin_unlock_irqrestore(&priv->pgtablelock, flags); @@ -840,6 +856,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, struct sysmmu_drvdata *data; unsigned long flags; unsigned long *ent; + size_t err_pgsize; BUG_ON(priv->pgtable == NULL); @@ -848,7 +865,10 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, ent = section_entry(priv->pgtable, iova); if (lv1ent_section(ent)) { - BUG_ON(size < SECT_SIZE); + if (size < SECT_SIZE) { + err_pgsize = SECT_SIZE; + goto err; + } *ent = 0; pgtable_flush(ent, ent + 1); @@ -879,7 +899,10 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, } /* lv1ent_large(ent) == true here */ - BUG_ON(size < LPAGE_SIZE); + if (size < LPAGE_SIZE) { + err_pgsize = LPAGE_SIZE; + goto err; + } memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); @@ -893,8 +916,15 @@ done: sysmmu_tlb_invalidate_entry(data->dev, iova); spin_unlock_irqrestore(&priv->lock, flags); - return size; +err: + spin_unlock_irqrestore(&priv->pgtablelock, flags); + + WARN(1, + "%s: Failed due to size(%#x) @ %#08lx is smaller than page size %#x\n", + __func__, size, iova, err_pgsize); + + return 0; } static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, -- cgit v0.10.2 From 734c3c732ca91adbe705f2a289509627810231a3 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:48 +0530 Subject: iommu/exynos: Allocate lv2 page table from own slab Since kmalloc() does not guarantee that the allignment of 1KiB when it allocates 1KiB, it is required to allocate lv2 page table from own slab that guarantees alignment of 1KiB Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index aec7fd7..4ff4b0b 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -99,6 +99,8 @@ #define REG_PB1_SADDR 0x054 #define REG_PB1_EADDR 0x058 +static struct kmem_cache *lv2table_kmem_cache; + static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova) { return pgtable + lv1ent_offset(iova); @@ -637,7 +639,8 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain) for (i = 0; i < NUM_LV1ENTRIES; i++) if (lv1ent_page(priv->pgtable + i)) - kfree(phys_to_virt(lv2table_base(priv->pgtable + i))); + kmem_cache_free(lv2table_kmem_cache, + phys_to_virt(lv2table_base(priv->pgtable + i))); free_pages((unsigned long)priv->pgtable, 2); free_pages((unsigned long)priv->lv2entcnt, 1); @@ -736,7 +739,7 @@ static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, if (lv1ent_fault(sent)) { unsigned long *pent; - pent = kzalloc(LV2TABLE_SIZE, GFP_ATOMIC); + pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1)); if (!pent) return ERR_PTR(-ENOMEM); @@ -766,8 +769,7 @@ static int lv1set_section(unsigned long *sent, unsigned long iova, return -EADDRINUSE; } - kfree(page_entry(sent, 0)); - + kmem_cache_free(lv2table_kmem_cache, page_entry(sent, 0)); *pgcnt = 0; } @@ -970,11 +972,31 @@ static int __init exynos_iommu_init(void) { int ret; + lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", + LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); + if (!lv2table_kmem_cache) { + pr_err("%s: Failed to create kmem cache\n", __func__); + return -ENOMEM; + } + ret = platform_driver_register(&exynos_sysmmu_driver); + if (ret) { + pr_err("%s: Failed to register driver\n", __func__); + goto err_reg_driver; + } - if (ret == 0) - bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); + ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); + if (ret) { + pr_err("%s: Failed to register exynos-iommu driver.\n", + __func__); + goto err_set_iommu; + } + return 0; +err_set_iommu: + platform_driver_unregister(&exynos_sysmmu_driver); +err_reg_driver: + kmem_cache_destroy(lv2table_kmem_cache); return ret; } subsys_initcall(exynos_iommu_init); -- cgit v0.10.2 From 3ad6b7f3a41ba6bf4503be442802072ca9dc8336 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:49 +0530 Subject: iommu/exynos: Fix L2TLB invalidation L2TLB is 8-way set-associative TLB with 512 entries. The number of sets is 64. A single 4KB(small page) translation information is cached only to a set whose index is the same with the lower 6 bits of the page frame number. A single 64KB(large page) translation information can be cached to any 16 sets whose top two bits of their indices are the same with the bit [5:4] of the page frame number. A single 1MB(section) or larger translation information can be cached to any set in the TLB. It is required to invalidate entire sets that may cache the target translation information to guarantee that the L2TLB has no stale data. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 4ff4b0b..06fc70e 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -226,9 +226,14 @@ static void __sysmmu_tlb_invalidate(void __iomem *sfrbase) } static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, - unsigned long iova) + unsigned long iova, unsigned int num_inv) { - __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY); + unsigned int i; + for (i = 0; i < num_inv; i++) { + __raw_writel((iova & SPAGE_MASK) | 1, + sfrbase + REG_MMU_FLUSH_ENTRY); + iova += SPAGE_SIZE; + } } static void __sysmmu_set_ptbase(void __iomem *sfrbase, @@ -452,7 +457,8 @@ static bool exynos_sysmmu_disable(struct device *dev) return disabled; } -static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova) +static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, + size_t size) { unsigned long flags; struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); @@ -460,9 +466,25 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova) read_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { + unsigned int maj; + unsigned int num_inv = 1; + maj = __raw_readl(data->sfrbase + REG_MMU_VERSION); + /* + * L2TLB invalidation required + * 4KB page: 1 invalidation + * 64KB page: 16 invalidation + * 1MB page: 64 invalidation + * because it is set-associative TLB + * with 8-way and 64 sets. + * 1MB page can be cached in one of all sets. + * 64KB page can be one of 16 consecutive sets. + */ + if ((maj >> 28) == 2) /* major version number */ + num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); + if (sysmmu_block(data->sfrbase)) { __sysmmu_tlb_invalidate_entry( - data->sfrbase, iova); + data->sfrbase, iova, num_inv); sysmmu_unblock(data->sfrbase); } } else { @@ -915,7 +937,7 @@ done: spin_lock_irqsave(&priv->lock, flags); list_for_each_entry(data, &priv->clients, node) - sysmmu_tlb_invalidate_entry(data->dev, iova); + sysmmu_tlb_invalidate_entry(data->dev, iova, size); spin_unlock_irqrestore(&priv->lock, flags); return size; -- cgit v0.10.2 From 8f8fcf970e5866150af98233ae7ebf25ffe8cba7 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:50 +0530 Subject: iommu/exynos: Remove prefetch buffer setting Prefetch buffer is a cache of System MMU 3.x and caches a block of page table entries to make effect of larger page with small pages. However, how to control prefetch buffers and the specifications of prefetch buffers different from minor versions of System MMU v3. Prefetch buffers must be controled with care because there are some restrictions in H/W design. The interface and implementation to initiate prefetch buffers will be prepared later. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 06fc70e..4fc31fc 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -245,13 +245,6 @@ static void __sysmmu_set_ptbase(void __iomem *sfrbase, __sysmmu_tlb_invalidate(sfrbase); } -static void __sysmmu_set_prefbuf(void __iomem *sfrbase, unsigned long base, - unsigned long size, int idx) -{ - __raw_writel(base, sfrbase + REG_PB0_SADDR + idx * 8); - __raw_writel(size - 1 + base, sfrbase + REG_PB0_EADDR + idx * 8); -} - static void __set_fault_handler(struct sysmmu_drvdata *data, sysmmu_fault_handler_t handler) { @@ -401,13 +394,6 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, data->pgtable = pgtable; __sysmmu_set_ptbase(data->sfrbase, pgtable); - if ((readl(data->sfrbase + REG_MMU_VERSION) >> 28) == 3) { - /* System MMU version is 3.x */ - __raw_writel((1 << 12) | (2 << 28), - data->sfrbase + REG_MMU_CFG); - __sysmmu_set_prefbuf(data->sfrbase, 0, -1, 0); - __sysmmu_set_prefbuf(data->sfrbase, 0, -1, 1); - } __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); -- cgit v0.10.2 From 6cb47ed7397e0881afede05161ccaf5b70f95414 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:51 +0530 Subject: iommu/exynos: Add missing cache flush for removed page table entries This commit adds cache flush for removed small and large page entries in exynos_iommu_unmap(). Missing cache flush of removed page table entries can cause missing page fault interrupt when a master IP accesses an unmapped area. Reviewed-by: Tomasz Figa Tested-by: Grant Grundler Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 4fc31fc..6915235 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -904,6 +904,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, if (lv2ent_small(ent)) { *ent = 0; size = SPAGE_SIZE; + pgtable_flush(ent, ent + 1); priv->lv2entcnt[lv1ent_offset(iova)] += 1; goto done; } @@ -915,6 +916,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, } memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); + pgtable_flush(ent, ent + SPAGES_PER_LPAGE); size = LPAGE_SIZE; priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; -- cgit v0.10.2 From f4723ec1723c0925c4191f5f33192ec09c6164f8 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:52 +0530 Subject: iommu/exynos: Always enable runtime PM Checking if the probing device has a parent device was just to discover if the probing device is involved in a power domain when the power domain controlled by Samsung's custom implementation. Since generic IO power domain is applied, it is required to remove the condition to see if the probing device has a parent device. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 6915235..ef771a2 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -558,8 +558,7 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); - if (dev->parent) - pm_runtime_enable(dev); + pm_runtime_enable(dev); dev_dbg(dev, "(%s) Initialized\n", data->dbgname); return 0; -- cgit v0.10.2 From e5cf63c3025c2cb7e1db78658827f6ab2a7a039f Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:53 +0530 Subject: iommu/exynos: Remove dbgname from drvdata of a System MMU This patch removes dbgname member from sysmmu_drvdata structure. Kernel message for debugging already has the name of a single System MMU node. It also removes some compilation warnings. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index ef771a2..be7a7b9 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -170,7 +170,6 @@ struct sysmmu_drvdata { struct list_head node; /* entry of exynos_iommu_domain.clients */ struct device *sysmmu; /* System MMU's device descriptor */ struct device *dev; /* Owner of system MMU */ - char *dbgname; void __iomem *sfrbase; struct clk *clk; int activations; @@ -321,8 +320,8 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) if (!ret && (itype != SYSMMU_FAULT_UNKNOWN)) __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); else - dev_dbg(data->sysmmu, "(%s) %s is not handled.\n", - data->dbgname, sysmmu_fault_name[itype]); + dev_dbg(data->sysmmu, "%s is not handled.\n", + sysmmu_fault_name[itype]); if (itype != SYSMMU_FAULT_UNKNOWN) sysmmu_unblock(data->sfrbase); @@ -354,10 +353,10 @@ finish: write_unlock_irqrestore(&data->lock, flags); if (disabled) - dev_dbg(data->sysmmu, "(%s) Disabled\n", data->dbgname); + dev_dbg(data->sysmmu, "Disabled\n"); else - dev_dbg(data->sysmmu, "(%s) %d times left to be disabled\n", - data->dbgname, data->activations); + dev_dbg(data->sysmmu, "%d times left to be disabled\n", + data->activations); return disabled; } @@ -384,7 +383,7 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, ret = 1; } - dev_dbg(data->sysmmu, "(%s) Already enabled\n", data->dbgname); + dev_dbg(data->sysmmu, "Already enabled\n"); goto finish; } @@ -399,7 +398,7 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, data->domain = domain; - dev_dbg(data->sysmmu, "(%s) Enabled\n", data->dbgname); + dev_dbg(data->sysmmu, "Enabled\n"); finish: write_unlock_irqrestore(&data->lock, flags); @@ -415,16 +414,15 @@ int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable) ret = pm_runtime_get_sync(data->sysmmu); if (ret < 0) { - dev_dbg(data->sysmmu, "(%s) Failed to enable\n", data->dbgname); + dev_dbg(data->sysmmu, "Failed to enable\n"); return ret; } ret = __exynos_sysmmu_enable(data, pgtable, NULL); if (WARN_ON(ret < 0)) { pm_runtime_put(data->sysmmu); - dev_err(data->sysmmu, - "(%s) Already enabled with page table %#x\n", - data->dbgname, data->pgtable); + dev_err(data->sysmmu, "Already enabled with page table %#x\n", + data->pgtable); } else { data->dev = dev; } @@ -474,9 +472,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, sysmmu_unblock(data->sfrbase); } } else { - dev_dbg(data->sysmmu, - "(%s) Disabled. Skipping invalidating TLB.\n", - data->dbgname); + dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); } read_unlock_irqrestore(&data->lock, flags); @@ -495,9 +491,7 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) sysmmu_unblock(data->sfrbase); } } else { - dev_dbg(data->sysmmu, - "(%s) Disabled. Skipping invalidating TLB.\n", - data->dbgname); + dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); } read_unlock_irqrestore(&data->lock, flags); @@ -560,7 +554,7 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) pm_runtime_enable(dev); - dev_dbg(dev, "(%s) Initialized\n", data->dbgname); + dev_dbg(dev, "Initialized\n"); return 0; err_irq: free_irq(platform_get_irq(pdev, 0), data); -- cgit v0.10.2 From 46c16d1e4c11ffa0481b1dd6841e0bcabcf278e7 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:54 +0530 Subject: iommu/exynos: Use managed device helper functions This patch uses managed device helper functions in the probe(). Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index be7a7b9..c86e374 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -343,8 +343,7 @@ static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); - if (!IS_ERR(data->clk)) - clk_disable(data->clk); + clk_disable(data->clk); disabled = true; data->pgtable = 0; @@ -387,8 +386,7 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, goto finish; } - if (!IS_ERR(data->clk)) - clk_enable(data->clk); + clk_enable(data->clk); data->pgtable = pgtable; @@ -499,49 +497,43 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) static int exynos_sysmmu_probe(struct platform_device *pdev) { - int ret; + int irq, ret; struct device *dev = &pdev->dev; struct sysmmu_drvdata *data; struct resource *res; - data = kzalloc(sizeof(*data), GFP_KERNEL); - if (!data) { - dev_dbg(dev, "Not enough memory\n"); - ret = -ENOMEM; - goto err_alloc; - } + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (!res) { - dev_dbg(dev, "Unable to find IOMEM region\n"); - ret = -ENOENT; - goto err_init; - } + data->sfrbase = devm_ioremap_resource(dev, res); + if (IS_ERR(data->sfrbase)) + return PTR_ERR(data->sfrbase); - data->sfrbase = ioremap(res->start, resource_size(res)); - if (!data->sfrbase) { - dev_dbg(dev, "Unable to map IOMEM @ PA:%#x\n", res->start); - ret = -ENOENT; - goto err_res; - } - - ret = platform_get_irq(pdev, 0); - if (ret <= 0) { + irq = platform_get_irq(pdev, 0); + if (irq <= 0) { dev_dbg(dev, "Unable to find IRQ resource\n"); - goto err_irq; + return irq; } - ret = request_irq(ret, exynos_sysmmu_irq, 0, + ret = devm_request_irq(dev, irq, exynos_sysmmu_irq, 0, dev_name(dev), data); if (ret) { - dev_dbg(dev, "Unabled to register interrupt handler\n"); - goto err_irq; + dev_err(dev, "Unabled to register handler of irq %d\n", irq); + return ret; } - if (dev_get_platdata(dev)) { - data->clk = clk_get(dev, "sysmmu"); - if (IS_ERR(data->clk)) - dev_dbg(dev, "No clock descriptor registered\n"); + data->clk = devm_clk_get(dev, "sysmmu"); + if (IS_ERR(data->clk)) { + dev_err(dev, "Failed to get clock!\n"); + return PTR_ERR(data->clk); + } else { + ret = clk_prepare(data->clk); + if (ret) { + dev_err(dev, "Failed to prepare clk\n"); + return ret; + } } data->sysmmu = dev; @@ -554,17 +546,7 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) pm_runtime_enable(dev); - dev_dbg(dev, "Initialized\n"); return 0; -err_irq: - free_irq(platform_get_irq(pdev, 0), data); -err_res: - iounmap(data->sfrbase); -err_init: - kfree(data); -err_alloc: - dev_err(dev, "Failed to initialize\n"); - return ret; } static struct platform_driver exynos_sysmmu_driver = { -- cgit v0.10.2 From 7060587052e0370ea1b7a41c84d5ad364be16f51 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:55 +0530 Subject: iommu/exynos: Gating clocks of master H/W This patch gates clocks of master H/W as well as clocks of System MMU if master clocks are specified. Some Exynos SoCs (i.e. GScalers in Exynos5250) have dependencies in the gating clocks of master H/W and its System MMU. If a H/W is the case, accessing control registers of System MMU is prohibited unless both of the gating clocks of System MMU and its master H/W. CC: Tomasz Figa Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index c86e374..5af5c5c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -172,6 +172,7 @@ struct sysmmu_drvdata { struct device *dev; /* Owner of system MMU */ void __iomem *sfrbase; struct clk *clk; + struct clk *clk_master; int activations; rwlock_t lock; struct iommu_domain *domain; @@ -300,6 +301,8 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) WARN_ON(!is_sysmmu_active(data)); + if (!IS_ERR(data->clk_master)) + clk_enable(data->clk_master); itype = (enum exynos_sysmmu_inttype) __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) @@ -326,6 +329,9 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) if (itype != SYSMMU_FAULT_UNKNOWN) sysmmu_unblock(data->sfrbase); + if (!IS_ERR(data->clk_master)) + clk_disable(data->clk_master); + read_unlock(&data->lock); return IRQ_HANDLED; @@ -341,9 +347,14 @@ static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) if (!set_sysmmu_inactive(data)) goto finish; + if (!IS_ERR(data->clk_master)) + clk_enable(data->clk_master); + __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); clk_disable(data->clk); + if (!IS_ERR(data->clk_master)) + clk_disable(data->clk_master); disabled = true; data->pgtable = 0; @@ -386,14 +397,19 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, goto finish; } - clk_enable(data->clk); - data->pgtable = pgtable; + if (!IS_ERR(data->clk_master)) + clk_enable(data->clk_master); + clk_enable(data->clk); + __sysmmu_set_ptbase(data->sfrbase, pgtable); __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + if (!IS_ERR(data->clk_master)) + clk_disable(data->clk_master); + data->domain = domain; dev_dbg(data->sysmmu, "Enabled\n"); @@ -450,6 +466,10 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, if (is_sysmmu_active(data)) { unsigned int maj; unsigned int num_inv = 1; + + if (!IS_ERR(data->clk_master)) + clk_enable(data->clk_master); + maj = __raw_readl(data->sfrbase + REG_MMU_VERSION); /* * L2TLB invalidation required @@ -469,6 +489,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, data->sfrbase, iova, num_inv); sysmmu_unblock(data->sfrbase); } + if (!IS_ERR(data->clk_master)) + clk_disable(data->clk_master); } else { dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); } @@ -484,10 +506,14 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) read_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { + if (!IS_ERR(data->clk_master)) + clk_enable(data->clk_master); if (sysmmu_block(data->sfrbase)) { __sysmmu_tlb_invalidate(data->sfrbase); sysmmu_unblock(data->sfrbase); } + if (!IS_ERR(data->clk_master)) + clk_disable(data->clk_master); } else { dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); } @@ -536,6 +562,16 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) } } + data->clk_master = devm_clk_get(dev, "master"); + if (!IS_ERR(data->clk_master)) { + ret = clk_prepare(data->clk_master); + if (ret) { + clk_unprepare(data->clk); + dev_err(dev, "Failed to prepare master's clk\n"); + return ret; + } + } + data->sysmmu = dev; rwlock_init(&data->lock); INIT_LIST_HEAD(&data->node); -- cgit v0.10.2 From 1fab7fa7230fa0422ca81cea7d1bfbf3f9b0d3f9 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:56 +0530 Subject: iommu/exynos: Remove custom fault handler This commit removes custom fault handler. The device drivers that need to register fault handler can register with iommu_set_fault_handler(). CC: Grant Grundler Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 5af5c5c..c1be65f 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -125,16 +125,6 @@ enum exynos_sysmmu_inttype { SYSMMU_FAULTS_NUM }; -/* - * @itype: type of fault. - * @pgtable_base: the physical address of page table base. This is 0 if @itype - * is SYSMMU_BUSERROR. - * @fault_addr: the device (virtual) address that the System MMU tried to - * translated. This is 0 if @itype is SYSMMU_BUSERROR. - */ -typedef int (*sysmmu_fault_handler_t)(enum exynos_sysmmu_inttype itype, - phys_addr_t pgtable_base, unsigned long fault_addr); - static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = { REG_PAGE_FAULT_ADDR, REG_AR_FAULT_ADDR, @@ -176,7 +166,6 @@ struct sysmmu_drvdata { int activations; rwlock_t lock; struct iommu_domain *domain; - sysmmu_fault_handler_t fault_handler; phys_addr_t pgtable; }; @@ -245,34 +234,17 @@ static void __sysmmu_set_ptbase(void __iomem *sfrbase, __sysmmu_tlb_invalidate(sfrbase); } -static void __set_fault_handler(struct sysmmu_drvdata *data, - sysmmu_fault_handler_t handler) -{ - unsigned long flags; - - write_lock_irqsave(&data->lock, flags); - data->fault_handler = handler; - write_unlock_irqrestore(&data->lock, flags); -} - -void exynos_sysmmu_set_fault_handler(struct device *dev, - sysmmu_fault_handler_t handler) -{ - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); - - __set_fault_handler(data, handler); -} - -static int default_fault_handler(enum exynos_sysmmu_inttype itype, - phys_addr_t pgtable_base, unsigned long fault_addr) +static void show_fault_information(const char *name, + enum exynos_sysmmu_inttype itype, + phys_addr_t pgtable_base, unsigned long fault_addr) { unsigned long *ent; if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) itype = SYSMMU_FAULT_UNKNOWN; - pr_err("%s occurred at 0x%lx(Page table base: %pa)\n", - sysmmu_fault_name[itype], fault_addr, &pgtable_base); + pr_err("%s occurred at %#lx by %s(Page table base: %pa)\n", + sysmmu_fault_name[itype], fault_addr, name, &pgtable_base); ent = section_entry(phys_to_virt(pgtable_base), fault_addr); pr_err("\tLv1 entry: 0x%lx\n", *ent); @@ -281,12 +253,6 @@ static int default_fault_handler(enum exynos_sysmmu_inttype itype, ent = page_entry(ent, fault_addr); pr_err("\t Lv2 entry: 0x%lx\n", *ent); } - - pr_err("Generating Kernel OOPS... because it is unrecoverable.\n"); - - BUG(); - - return 0; } static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) @@ -310,24 +276,28 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) else addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]); - if (data->domain) - ret = report_iommu_fault(data->domain, data->dev, addr, itype); - - if ((ret == -ENOSYS) && data->fault_handler) { - unsigned long base = data->pgtable; - if (itype != SYSMMU_FAULT_UNKNOWN) - base = __raw_readl(data->sfrbase + REG_PT_BASE_ADDR); - ret = data->fault_handler(itype, base, addr); + if (itype == SYSMMU_FAULT_UNKNOWN) { + pr_err("%s: Fault is not occurred by System MMU '%s'!\n", + __func__, dev_name(data->sysmmu)); + pr_err("%s: Please check if IRQ is correctly configured.\n", + __func__); + BUG(); + } else { + unsigned long base = + __raw_readl(data->sfrbase + REG_PT_BASE_ADDR); + show_fault_information(dev_name(data->sysmmu), + itype, base, addr); + if (data->domain) + ret = report_iommu_fault(data->domain, + data->dev, addr, itype); } - if (!ret && (itype != SYSMMU_FAULT_UNKNOWN)) - __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); - else - dev_dbg(data->sysmmu, "%s is not handled.\n", - sysmmu_fault_name[itype]); + /* fault is not recovered by fault handler */ + BUG_ON(ret != 0); - if (itype != SYSMMU_FAULT_UNKNOWN) - sysmmu_unblock(data->sfrbase); + __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); + + sysmmu_unblock(data->sfrbase); if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); @@ -576,8 +546,6 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) rwlock_init(&data->lock); INIT_LIST_HEAD(&data->node); - __set_fault_handler(data, &default_fault_handler); - platform_set_drvdata(pdev, data); pm_runtime_enable(dev); -- cgit v0.10.2 From 9d4e7a24d77a05fb5c4e4121051a8d80501c74d3 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:57 +0530 Subject: iommu/exynos: Change rwlock to spinlock Since acquiring read_lock is not more frequent than write_lock, it is not beneficial to use rwlock, this commit changes rwlock to spinlock. Reviewed-by: Grant Grundler Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index c1be65f..d89ad5f 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -164,7 +164,7 @@ struct sysmmu_drvdata { struct clk *clk; struct clk *clk_master; int activations; - rwlock_t lock; + spinlock_t lock; struct iommu_domain *domain; phys_addr_t pgtable; }; @@ -263,12 +263,13 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) unsigned long addr = -1; int ret = -ENOSYS; - read_lock(&data->lock); - WARN_ON(!is_sysmmu_active(data)); + spin_lock(&data->lock); + if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); + itype = (enum exynos_sysmmu_inttype) __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) @@ -302,7 +303,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); - read_unlock(&data->lock); + spin_unlock(&data->lock); return IRQ_HANDLED; } @@ -312,7 +313,7 @@ static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) unsigned long flags; bool disabled = false; - write_lock_irqsave(&data->lock, flags); + spin_lock_irqsave(&data->lock, flags); if (!set_sysmmu_inactive(data)) goto finish; @@ -330,7 +331,7 @@ static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) data->pgtable = 0; data->domain = NULL; finish: - write_unlock_irqrestore(&data->lock, flags); + spin_unlock_irqrestore(&data->lock, flags); if (disabled) dev_dbg(data->sysmmu, "Disabled\n"); @@ -353,7 +354,7 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, int ret = 0; unsigned long flags; - write_lock_irqsave(&data->lock, flags); + spin_lock_irqsave(&data->lock, flags); if (!set_sysmmu_active(data)) { if (WARN_ON(pgtable != data->pgtable)) { @@ -384,7 +385,7 @@ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, dev_dbg(data->sysmmu, "Enabled\n"); finish: - write_unlock_irqrestore(&data->lock, flags); + spin_unlock_irqrestore(&data->lock, flags); return ret; } @@ -431,7 +432,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, unsigned long flags; struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); - read_lock_irqsave(&data->lock, flags); + spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { unsigned int maj; @@ -465,7 +466,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); } - read_unlock_irqrestore(&data->lock, flags); + spin_unlock_irqrestore(&data->lock, flags); } void exynos_sysmmu_tlb_invalidate(struct device *dev) @@ -473,7 +474,7 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) unsigned long flags; struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); - read_lock_irqsave(&data->lock, flags); + spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { if (!IS_ERR(data->clk_master)) @@ -488,7 +489,7 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); } - read_unlock_irqrestore(&data->lock, flags); + spin_unlock_irqrestore(&data->lock, flags); } static int exynos_sysmmu_probe(struct platform_device *pdev) @@ -543,7 +544,7 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) } data->sysmmu = dev; - rwlock_init(&data->lock); + spin_lock_init(&data->lock); INIT_LIST_HEAD(&data->node); platform_set_drvdata(pdev, data); -- cgit v0.10.2 From d09d78fc986be8355928256d1b86b713588999d7 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:44:58 +0530 Subject: iommu/exynos: Use exynos-iommu specific typedef This commit introduces sysmmu_pte_t for page table entries and sysmmu_iova_t vor I/O virtual address that is manipulated by exynos-iommu driver. The purpose of the typedef is to remove dependencies to the driver code from the change of CPU architecture from 32 bit to 64 bit. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index d89ad5f..3291619 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -29,6 +29,9 @@ #include #include +typedef u32 sysmmu_iova_t; +typedef u32 sysmmu_pte_t; + /* We does not consider super section mapping (16MB) */ #define SECT_ORDER 20 #define LPAGE_ORDER 16 @@ -50,20 +53,32 @@ #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) +static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size) +{ + return iova & (size - 1); +} + #define section_phys(sent) (*(sent) & SECT_MASK) -#define section_offs(iova) ((iova) & 0xFFFFF) +#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE) #define lpage_phys(pent) (*(pent) & LPAGE_MASK) -#define lpage_offs(iova) ((iova) & 0xFFFF) +#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE) #define spage_phys(pent) (*(pent) & SPAGE_MASK) -#define spage_offs(iova) ((iova) & 0xFFF) - -#define lv1ent_offset(iova) ((iova) >> SECT_ORDER) -#define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER) +#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE) #define NUM_LV1ENTRIES 4096 -#define NUM_LV2ENTRIES 256 +#define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) -#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long)) +static u32 lv1ent_offset(sysmmu_iova_t iova) +{ + return iova >> SECT_ORDER; +} + +static u32 lv2ent_offset(sysmmu_iova_t iova) +{ + return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); +} + +#define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) @@ -101,14 +116,14 @@ static struct kmem_cache *lv2table_kmem_cache; -static unsigned long *section_entry(unsigned long *pgtable, unsigned long iova) +static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) { return pgtable + lv1ent_offset(iova); } -static unsigned long *page_entry(unsigned long *sent, unsigned long iova) +static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) { - return (unsigned long *)phys_to_virt( + return (sysmmu_pte_t *)phys_to_virt( lv2table_base(sent)) + lv2ent_offset(iova); } @@ -150,7 +165,7 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { struct exynos_iommu_domain { struct list_head clients; /* list of sysmmu_drvdata.node */ - unsigned long *pgtable; /* lv1 page table, 16KB */ + sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ short *lv2entcnt; /* free lv2 entry counter for each section */ spinlock_t lock; /* lock for this structure */ spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ @@ -215,7 +230,7 @@ static void __sysmmu_tlb_invalidate(void __iomem *sfrbase) } static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, - unsigned long iova, unsigned int num_inv) + sysmmu_iova_t iova, unsigned int num_inv) { unsigned int i; for (i = 0; i < num_inv; i++) { @@ -226,7 +241,7 @@ static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, } static void __sysmmu_set_ptbase(void __iomem *sfrbase, - unsigned long pgd) + phys_addr_t pgd) { __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */ __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR); @@ -236,22 +251,22 @@ static void __sysmmu_set_ptbase(void __iomem *sfrbase, static void show_fault_information(const char *name, enum exynos_sysmmu_inttype itype, - phys_addr_t pgtable_base, unsigned long fault_addr) + phys_addr_t pgtable_base, sysmmu_iova_t fault_addr) { - unsigned long *ent; + sysmmu_pte_t *ent; if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) itype = SYSMMU_FAULT_UNKNOWN; - pr_err("%s occurred at %#lx by %s(Page table base: %pa)\n", + pr_err("%s occurred at %#x by %s(Page table base: %pa)\n", sysmmu_fault_name[itype], fault_addr, name, &pgtable_base); ent = section_entry(phys_to_virt(pgtable_base), fault_addr); - pr_err("\tLv1 entry: 0x%lx\n", *ent); + pr_err("\tLv1 entry: %#x\n", *ent); if (lv1ent_page(ent)) { ent = page_entry(ent, fault_addr); - pr_err("\t Lv2 entry: 0x%lx\n", *ent); + pr_err("\t Lv2 entry: %#x\n", *ent); } } @@ -260,7 +275,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) /* SYSMMU is in blocked when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; enum exynos_sysmmu_inttype itype; - unsigned long addr = -1; + sysmmu_iova_t addr = -1; int ret = -ENOSYS; WARN_ON(!is_sysmmu_active(data)); @@ -284,7 +299,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) __func__); BUG(); } else { - unsigned long base = + unsigned int base = __raw_readl(data->sfrbase + REG_PT_BASE_ADDR); show_fault_information(dev_name(data->sysmmu), itype, base, addr); @@ -349,7 +364,7 @@ finish: * enabled before. */ static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, - unsigned long pgtable, struct iommu_domain *domain) + phys_addr_t pgtable, struct iommu_domain *domain) { int ret = 0; unsigned long flags; @@ -390,7 +405,7 @@ finish: return ret; } -int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable) +int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable) { struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); int ret; @@ -426,7 +441,7 @@ static bool exynos_sysmmu_disable(struct device *dev) return disabled; } -static void sysmmu_tlb_invalidate_entry(struct device *dev, unsigned long iova, +static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, size_t size) { unsigned long flags; @@ -577,7 +592,7 @@ static int exynos_iommu_domain_init(struct iommu_domain *domain) if (!priv) return -ENOMEM; - priv->pgtable = (unsigned long *)__get_free_pages( + priv->pgtable = (sysmmu_pte_t *)__get_free_pages( GFP_KERNEL | __GFP_ZERO, 2); if (!priv->pgtable) goto err_pgtable; @@ -716,19 +731,19 @@ finish: pm_runtime_put(data->sysmmu); } -static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, +static sysmmu_pte_t *alloc_lv2entry(sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { if (lv1ent_section(sent)) { - WARN(1, "Trying mapping on %#08lx mapped with 1MiB page", iova); + WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova); return ERR_PTR(-EADDRINUSE); } if (lv1ent_fault(sent)) { - unsigned long *pent; + sysmmu_pte_t *pent; pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); - BUG_ON((unsigned long)pent & (LV2TABLE_SIZE - 1)); + BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1)); if (!pent) return ERR_PTR(-ENOMEM); @@ -741,18 +756,18 @@ static unsigned long *alloc_lv2entry(unsigned long *sent, unsigned long iova, return page_entry(sent, iova); } -static int lv1set_section(unsigned long *sent, unsigned long iova, +static int lv1set_section(sysmmu_pte_t *sent, sysmmu_iova_t iova, phys_addr_t paddr, short *pgcnt) { if (lv1ent_section(sent)) { - WARN(1, "Trying mapping on 1MiB@%#08lx that is mapped", + WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", iova); return -EADDRINUSE; } if (lv1ent_page(sent)) { if (*pgcnt != NUM_LV2ENTRIES) { - WARN(1, "Trying mapping on 1MiB@%#08lx that is mapped", + WARN(1, "Trying mapping on 1MiB@%#08x that is mapped", iova); return -EADDRINUSE; } @@ -768,7 +783,7 @@ static int lv1set_section(unsigned long *sent, unsigned long iova, return 0; } -static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size, +static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, short *pgcnt) { if (size == SPAGE_SIZE) { @@ -800,11 +815,12 @@ static int lv2set_page(unsigned long *pent, phys_addr_t paddr, size_t size, return 0; } -static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, +static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, phys_addr_t paddr, size_t size, int prot) { struct exynos_iommu_domain *priv = domain->priv; - unsigned long *entry; + sysmmu_pte_t *entry; + sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; unsigned long flags; int ret = -ENOMEM; @@ -818,7 +834,7 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, ret = lv1set_section(entry, iova, paddr, &priv->lv2entcnt[lv1ent_offset(iova)]); } else { - unsigned long *pent; + sysmmu_pte_t *pent; pent = alloc_lv2entry(entry, iova, &priv->lv2entcnt[lv1ent_offset(iova)]); @@ -831,7 +847,7 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, } if (ret) - pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n", + pr_debug("%s: Failed to map iova %#x/%#zx bytes\n", __func__, iova, size); spin_unlock_irqrestore(&priv->pgtablelock, flags); @@ -840,13 +856,14 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long iova, } static size_t exynos_iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size) + unsigned long l_iova, size_t size) { struct exynos_iommu_domain *priv = domain->priv; struct sysmmu_drvdata *data; - unsigned long flags; - unsigned long *ent; + sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; + sysmmu_pte_t *ent; size_t err_pgsize; + unsigned long flags; BUG_ON(priv->pgtable == NULL); @@ -913,7 +930,7 @@ err: spin_unlock_irqrestore(&priv->pgtablelock, flags); WARN(1, - "%s: Failed due to size(%#x) @ %#08lx is smaller than page size %#x\n", + "%s: Failed due to size(%#zx) @ %#x is smaller than page size %#zx\n", __func__, size, iova, err_pgsize); return 0; @@ -923,7 +940,7 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { struct exynos_iommu_domain *priv = domain->priv; - unsigned long *entry; + sysmmu_pte_t *entry; unsigned long flags; phys_addr_t phys = 0; -- cgit v0.10.2 From bf4a1c920286076609779c2c70c5a2bf69169fc7 Mon Sep 17 00:00:00 2001 From: Antonios Motakis Date: Mon, 12 May 2014 11:44:59 +0530 Subject: iommu/exynos: Add devices attached to the System MMU to an IOMMU group Patch written by Antonios Motakis : IOMMU groups are expected by certain users of the IOMMU API, e.g. VFIO. Since each device is behind its own System MMU, we can allocate a new IOMMU group for each device. Reviewed-by: Cho KyongHo Signed-off-by: Antonios Motakis Signed-off-by: Shaik Ameeer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 3291619..d18dc37 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -964,6 +964,32 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, return phys; } +static int exynos_iommu_add_device(struct device *dev) +{ + struct iommu_group *group; + int ret; + + group = iommu_group_get(dev); + + if (!group) { + group = iommu_group_alloc(); + if (IS_ERR(group)) { + dev_err(dev, "Failed to allocate IOMMU group\n"); + return PTR_ERR(group); + } + } + + ret = iommu_group_add_device(group, dev); + iommu_group_put(group); + + return ret; +} + +static void exynos_iommu_remove_device(struct device *dev) +{ + iommu_group_remove_device(dev); +} + static struct iommu_ops exynos_iommu_ops = { .domain_init = &exynos_iommu_domain_init, .domain_destroy = &exynos_iommu_domain_destroy, @@ -972,6 +998,8 @@ static struct iommu_ops exynos_iommu_ops = { .map = &exynos_iommu_map, .unmap = &exynos_iommu_unmap, .iova_to_phys = &exynos_iommu_iova_to_phys, + .add_device = &exynos_iommu_add_device, + .remove_device = &exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, }; -- cgit v0.10.2 From 0bf4e54dbebff8aa4b69057e88431ba8b48d3d19 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:45:00 +0530 Subject: iommu/exynos: Enhanced error messages Some redundant error message is removed and some error messages are changed to error level from debug level. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index d18dc37..7188b47 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -525,7 +525,7 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) irq = platform_get_irq(pdev, 0); if (irq <= 0) { - dev_dbg(dev, "Unable to find IRQ resource\n"); + dev_err(dev, "Unable to find IRQ resource\n"); return irq; } @@ -787,10 +787,8 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, short *pgcnt) { if (size == SPAGE_SIZE) { - if (!lv2ent_fault(pent)) { - WARN(1, "Trying mapping on 4KiB where mapping exists"); + if (WARN_ON(!lv2ent_fault(pent))) return -EADDRINUSE; - } *pent = mk_lv2ent_spage(paddr); pgtable_flush(pent, pent + 1); @@ -798,9 +796,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, } else { /* size == LPAGE_SIZE */ int i; for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { - if (!lv2ent_fault(pent)) { - WARN(1, - "Trying mapping on 64KiB where mapping exists"); + if (WARN_ON(!lv2ent_fault(pent))) { if (i > 0) memset(pent - i, 0, sizeof(*pent) * i); return -EADDRINUSE; @@ -847,8 +843,8 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, } if (ret) - pr_debug("%s: Failed to map iova %#x/%#zx bytes\n", - __func__, iova, size); + pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n", + __func__, ret, size, iova); spin_unlock_irqrestore(&priv->pgtablelock, flags); @@ -872,7 +868,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, ent = section_entry(priv->pgtable, iova); if (lv1ent_section(ent)) { - if (size < SECT_SIZE) { + if (WARN_ON(size < SECT_SIZE)) { err_pgsize = SECT_SIZE; goto err; } @@ -907,7 +903,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, } /* lv1ent_large(ent) == true here */ - if (size < LPAGE_SIZE) { + if (WARN_ON(size < LPAGE_SIZE)) { err_pgsize = LPAGE_SIZE; goto err; } @@ -929,9 +925,8 @@ done: err: spin_unlock_irqrestore(&priv->pgtablelock, flags); - WARN(1, - "%s: Failed due to size(%#zx) @ %#x is smaller than page size %#zx\n", - __func__, size, iova, err_pgsize); + pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n", + __func__, size, iova, err_pgsize); return 0; } -- cgit v0.10.2 From 93e268dca82eafb59157bfb7eb70e97fe3564412 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:45:01 +0530 Subject: documentation: iommu: Add binding document of Exynos System MMU This patch adds a description of the device tree binding for the Samsung Exynos System MMU. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt new file mode 100644 index 0000000..15b2a2b --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt @@ -0,0 +1,65 @@ +Samsung Exynos IOMMU H/W, System MMU (System Memory Management Unit) + +Samsung's Exynos architecture contains System MMUs that enables scattered +physical memory chunks visible as a contiguous region to DMA-capable peripheral +devices like MFC, FIMC, FIMD, GScaler, FIMC-IS and so forth. + +System MMU is an IOMMU and supports identical translation table format to +ARMv7 translation tables with minimum set of page properties including access +permissions, shareability and security protection. In addition, System MMU has +another capabilities like L2 TLB or block-fetch buffers to minimize translation +latency. + +System MMUs are in many to one relation with peripheral devices, i.e. single +peripheral device might have multiple System MMUs (usually one for each bus +master), but one System MMU can handle transactions from only one peripheral +device. The relation between a System MMU and the peripheral device needs to be +defined in device node of the peripheral device. + +MFC in all Exynos SoCs and FIMD, M2M Scalers and G2D in Exynos5420 has 2 System +MMUs. +* MFC has one System MMU on its left and right bus. +* FIMD in Exynos5420 has one System MMU for window 0 and 4, the other system MMU + for window 1, 2 and 3. +* M2M Scalers and G2D in Exynos5420 has one System MMU on the read channel and + the other System MMU on the write channel. +The drivers must consider how to handle those System MMUs. One of the idea is +to implement child devices or sub-devices which are the client devices of the +System MMU. + +Required properties: +- compatible: Should be "samsung,exynos-sysmmu" +- reg: A tuple of base address and size of System MMU registers. +- interrupt-parent: The phandle of the interrupt controller of System MMU +- interrupts: An interrupt specifier for interrupt signal of System MMU, + according to the format defined by a particular interrupt + controller. +- clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock. + Optional "master" if the clock to the System MMU is gated by + another gate clock other than "sysmmu". + Exynos4 SoCs, there needs no "master" clock. + Exynos5 SoCs, some System MMUs must have "master" clocks. +- clocks: Required if the System MMU is needed to gate its clock. +- samsung,power-domain: Required if the System MMU is needed to gate its power. + Please refer to the following document: + Documentation/devicetree/bindings/arm/exynos/power_domain.txt + +Examples: + gsc_0: gsc@13e00000 { + compatible = "samsung,exynos5-gsc"; + reg = <0x13e00000 0x1000>; + interrupts = <0 85 0>; + samsung,power-domain = <&pd_gsc>; + clocks = <&clock CLK_GSCL0>; + clock-names = "gscl"; + }; + + sysmmu_gsc0: sysmmu@13E80000 { + compatible = "samsung,exynos-sysmmu"; + reg = <0x13E80000 0x1000>; + interrupt-parent = <&combiner>; + interrupts = <2 0>; + clock-names = "sysmmu", "master"; + clocks = <&clock CLK_SMMU_GSCL0>, <&clock CLK_GSCL0>; + samsung,power-domain = <&pd_gsc>; + }; -- cgit v0.10.2 From 6b21a5db36427d54a94091fc26b5890c3d6a8af3 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:45:02 +0530 Subject: iommu/exynos: Support for device tree This commit adds device tree support for System MMU. Also, system mmu handling is improved. Previously, an IOMMU domain is bound to a System MMU which is not correct. This patch binds an IOMMU domain with the master device of a System MMU. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 7188b47..b937490 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -114,6 +114,8 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define REG_PB1_SADDR 0x054 #define REG_PB1_EADDR 0x058 +#define has_sysmmu(dev) (dev->archdata.iommu != NULL) + static struct kmem_cache *lv2table_kmem_cache; static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) @@ -163,6 +165,16 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { "UNKNOWN FAULT" }; +/* attached to dev.archdata.iommu of the master device */ +struct exynos_iommu_owner { + struct list_head client; /* entry of exynos_iommu_domain.clients */ + struct device *dev; + struct device *sysmmu; + struct iommu_domain *domain; + void *vmm_data; /* IO virtual memory manager's data */ + spinlock_t lock; /* Lock to preserve consistency of System MMU */ +}; + struct exynos_iommu_domain { struct list_head clients; /* list of sysmmu_drvdata.node */ sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ @@ -172,9 +184,8 @@ struct exynos_iommu_domain { }; struct sysmmu_drvdata { - struct list_head node; /* entry of exynos_iommu_domain.clients */ struct device *sysmmu; /* System MMU's device descriptor */ - struct device *dev; /* Owner of system MMU */ + struct device *master; /* Owner of system MMU */ void __iomem *sfrbase; struct clk *clk; struct clk *clk_master; @@ -243,7 +254,6 @@ static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, static void __sysmmu_set_ptbase(void __iomem *sfrbase, phys_addr_t pgd) { - __raw_writel(0x1, sfrbase + REG_MMU_CFG); /* 16KB LV1, LRU */ __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR); __sysmmu_tlb_invalidate(sfrbase); @@ -305,7 +315,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) itype, base, addr); if (data->domain) ret = report_iommu_fault(data->domain, - data->dev, addr, itype); + data->master, addr, itype); } /* fault is not recovered by fault handler */ @@ -323,120 +333,152 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) return IRQ_HANDLED; } -static bool __exynos_sysmmu_disable(struct sysmmu_drvdata *data) +static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) { - unsigned long flags; - bool disabled = false; - - spin_lock_irqsave(&data->lock, flags); - - if (!set_sysmmu_inactive(data)) - goto finish; - if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); + __raw_writel(0, data->sfrbase + REG_MMU_CFG); clk_disable(data->clk); if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); - - disabled = true; - data->pgtable = 0; - data->domain = NULL; -finish: - spin_unlock_irqrestore(&data->lock, flags); - - if (disabled) - dev_dbg(data->sysmmu, "Disabled\n"); - else - dev_dbg(data->sysmmu, "%d times left to be disabled\n", - data->activations); - - return disabled; } -/* __exynos_sysmmu_enable: Enables System MMU - * - * returns -error if an error occurred and System MMU is not enabled, - * 0 if the System MMU has been just enabled and 1 if System MMU was already - * enabled before. - */ -static int __exynos_sysmmu_enable(struct sysmmu_drvdata *data, - phys_addr_t pgtable, struct iommu_domain *domain) +static bool __sysmmu_disable(struct sysmmu_drvdata *data) { - int ret = 0; + bool disabled; unsigned long flags; spin_lock_irqsave(&data->lock, flags); - if (!set_sysmmu_active(data)) { - if (WARN_ON(pgtable != data->pgtable)) { - ret = -EBUSY; - set_sysmmu_inactive(data); - } else { - ret = 1; - } + disabled = set_sysmmu_inactive(data); + + if (disabled) { + data->pgtable = 0; + data->domain = NULL; + + __sysmmu_disable_nocount(data); - dev_dbg(data->sysmmu, "Already enabled\n"); - goto finish; + dev_dbg(data->sysmmu, "Disabled\n"); + } else { + dev_dbg(data->sysmmu, "%d times left to disable\n", + data->activations); } - data->pgtable = pgtable; + spin_unlock_irqrestore(&data->lock, flags); + + return disabled; +} +static void __sysmmu_init_config(struct sysmmu_drvdata *data) +{ + unsigned int cfg = 0; + + __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); +} + +static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) +{ if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); clk_enable(data->clk); - __sysmmu_set_ptbase(data->sfrbase, pgtable); + __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + + __sysmmu_init_config(data); + + __sysmmu_set_ptbase(data->sfrbase, data->pgtable); __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); +} - data->domain = domain; +static int __sysmmu_enable(struct sysmmu_drvdata *data, + phys_addr_t pgtable, struct iommu_domain *domain) +{ + int ret = 0; + unsigned long flags; + + spin_lock_irqsave(&data->lock, flags); + if (set_sysmmu_active(data)) { + data->pgtable = pgtable; + data->domain = domain; + + __sysmmu_enable_nocount(data); + + dev_dbg(data->sysmmu, "Enabled\n"); + } else { + ret = (pgtable == data->pgtable) ? 1 : -EBUSY; + + dev_dbg(data->sysmmu, "already enabled\n"); + } + + if (WARN_ON(ret < 0)) + set_sysmmu_inactive(data); /* decrement count */ - dev_dbg(data->sysmmu, "Enabled\n"); -finish: spin_unlock_irqrestore(&data->lock, flags); return ret; } -int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable) +/* __exynos_sysmmu_enable: Enables System MMU + * + * returns -error if an error occurred and System MMU is not enabled, + * 0 if the System MMU has been just enabled and 1 if System MMU was already + * enabled before. + */ +static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable, + struct iommu_domain *domain) { - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); - int ret; + int ret = 0; + unsigned long flags; + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct sysmmu_drvdata *data; - BUG_ON(!memblock_is_memory(pgtable)); + BUG_ON(!has_sysmmu(dev)); - ret = pm_runtime_get_sync(data->sysmmu); - if (ret < 0) { - dev_dbg(data->sysmmu, "Failed to enable\n"); - return ret; - } + spin_lock_irqsave(&owner->lock, flags); - ret = __exynos_sysmmu_enable(data, pgtable, NULL); - if (WARN_ON(ret < 0)) { - pm_runtime_put(data->sysmmu); - dev_err(data->sysmmu, "Already enabled with page table %#x\n", - data->pgtable); - } else { - data->dev = dev; - } + data = dev_get_drvdata(owner->sysmmu); + + ret = __sysmmu_enable(data, pgtable, domain); + if (ret >= 0) + data->master = dev; + + spin_unlock_irqrestore(&owner->lock, flags); return ret; } +int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable) +{ + BUG_ON(!memblock_is_memory(pgtable)); + + return __exynos_sysmmu_enable(dev, pgtable, NULL); +} + static bool exynos_sysmmu_disable(struct device *dev) { - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); - bool disabled; + unsigned long flags; + bool disabled = true; + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct sysmmu_drvdata *data; + + BUG_ON(!has_sysmmu(dev)); - disabled = __exynos_sysmmu_disable(data); - pm_runtime_put(data->sysmmu); + spin_lock_irqsave(&owner->lock, flags); + + data = dev_get_drvdata(owner->sysmmu); + + disabled = __sysmmu_disable(data); + if (disabled) + data->master = NULL; + + spin_unlock_irqrestore(&owner->lock, flags); return disabled; } @@ -444,11 +486,13 @@ static bool exynos_sysmmu_disable(struct device *dev) static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, size_t size) { + struct exynos_iommu_owner *owner = dev->archdata.iommu; unsigned long flags; - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + struct sysmmu_drvdata *data; - spin_lock_irqsave(&data->lock, flags); + data = dev_get_drvdata(owner->sysmmu); + spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { unsigned int maj; unsigned int num_inv = 1; @@ -478,19 +522,21 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); } else { - dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); + dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n", + iova); } - spin_unlock_irqrestore(&data->lock, flags); } void exynos_sysmmu_tlb_invalidate(struct device *dev) { + struct exynos_iommu_owner *owner = dev->archdata.iommu; unsigned long flags; - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + struct sysmmu_drvdata *data; - spin_lock_irqsave(&data->lock, flags); + data = dev_get_drvdata(owner->sysmmu); + spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); @@ -501,13 +547,12 @@ void exynos_sysmmu_tlb_invalidate(struct device *dev) if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); } else { - dev_dbg(data->sysmmu, "Disabled. Skipping invalidating TLB.\n"); + dev_dbg(dev, "disabled. Skipping TLB invalidation\n"); } - spin_unlock_irqrestore(&data->lock, flags); } -static int exynos_sysmmu_probe(struct platform_device *pdev) +static int __init exynos_sysmmu_probe(struct platform_device *pdev) { int irq, ret; struct device *dev = &pdev->dev; @@ -560,7 +605,6 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) data->sysmmu = dev; spin_lock_init(&data->lock); - INIT_LIST_HEAD(&data->node); platform_set_drvdata(pdev, data); @@ -569,11 +613,17 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) return 0; } -static struct platform_driver exynos_sysmmu_driver = { - .probe = exynos_sysmmu_probe, - .driver = { +static const struct of_device_id sysmmu_of_match[] __initconst = { + { .compatible = "samsung,exynos-sysmmu", }, + { }, +}; + +static struct platform_driver exynos_sysmmu_driver __refdata = { + .probe = exynos_sysmmu_probe, + .driver = { .owner = THIS_MODULE, .name = "exynos-sysmmu", + .of_match_table = sysmmu_of_match, } }; @@ -625,7 +675,7 @@ err_pgtable: static void exynos_iommu_domain_destroy(struct iommu_domain *domain) { struct exynos_iommu_domain *priv = domain->priv; - struct sysmmu_drvdata *data; + struct exynos_iommu_owner *owner; unsigned long flags; int i; @@ -633,11 +683,14 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain) spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry(data, &priv->clients, node) { - while (!exynos_sysmmu_disable(data->dev)) + list_for_each_entry(owner, &priv->clients, client) { + while (!exynos_sysmmu_disable(owner->dev)) ; /* until System MMU is actually disabled */ } + while (!list_empty(&priv->clients)) + list_del_init(priv->clients.next); + spin_unlock_irqrestore(&priv->lock, flags); for (i = 0; i < NUM_LV1ENTRIES; i++) @@ -654,27 +707,18 @@ static void exynos_iommu_domain_destroy(struct iommu_domain *domain) static int exynos_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + struct exynos_iommu_owner *owner = dev->archdata.iommu; struct exynos_iommu_domain *priv = domain->priv; phys_addr_t pagetable = virt_to_phys(priv->pgtable); unsigned long flags; int ret; - ret = pm_runtime_get_sync(data->sysmmu); - if (ret < 0) - return ret; - - ret = 0; - spin_lock_irqsave(&priv->lock, flags); - ret = __exynos_sysmmu_enable(data, pagetable, domain); - + ret = __exynos_sysmmu_enable(dev, pagetable, domain); if (ret == 0) { - /* 'data->node' must not be appeared in priv->clients */ - BUG_ON(!list_empty(&data->node)); - data->dev = dev; - list_add_tail(&data->node, &priv->clients); + list_add_tail(&owner->client, &priv->clients); + owner->domain = domain; } spin_unlock_irqrestore(&priv->lock, flags); @@ -682,7 +726,6 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, if (ret < 0) { dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n", __func__, &pagetable); - pm_runtime_put(data->sysmmu); return ret; } @@ -695,40 +738,30 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, static void exynos_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); + struct exynos_iommu_owner *owner; struct exynos_iommu_domain *priv = domain->priv; - struct list_head *pos; phys_addr_t pagetable = virt_to_phys(priv->pgtable); unsigned long flags; - bool found = false; spin_lock_irqsave(&priv->lock, flags); - list_for_each(pos, &priv->clients) { - if (list_entry(pos, struct sysmmu_drvdata, node) == data) { - found = true; + list_for_each_entry(owner, &priv->clients, client) { + if (owner == dev->archdata.iommu) { + if (exynos_sysmmu_disable(dev)) { + list_del_init(&owner->client); + owner->domain = NULL; + } break; } } - if (!found) - goto finish; + spin_unlock_irqrestore(&priv->lock, flags); - if (__exynos_sysmmu_disable(data)) { + if (owner == dev->archdata.iommu) dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__, &pagetable); - list_del_init(&data->node); - - } else { - dev_dbg(dev, "%s: Detaching IOMMU with pgtable %pa delayed", - __func__, &pagetable); - } - -finish: - spin_unlock_irqrestore(&priv->lock, flags); - - if (found) - pm_runtime_put(data->sysmmu); + else + dev_err(dev, "%s: No IOMMU is attached\n", __func__); } static sysmmu_pte_t *alloc_lv2entry(sysmmu_pte_t *sent, sysmmu_iova_t iova, @@ -855,7 +888,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, unsigned long l_iova, size_t size) { struct exynos_iommu_domain *priv = domain->priv; - struct sysmmu_drvdata *data; + struct exynos_iommu_owner *owner; sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; sysmmu_pte_t *ent; size_t err_pgsize; @@ -917,8 +950,8 @@ done: spin_unlock_irqrestore(&priv->pgtablelock, flags); spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry(data, &priv->clients, node) - sysmmu_tlb_invalidate_entry(data->dev, iova, size); + list_for_each_entry(owner, &priv->clients, client) + sysmmu_tlb_invalidate_entry(owner->dev, iova, size); spin_unlock_irqrestore(&priv->lock, flags); return size; -- cgit v0.10.2 From eeb5184bb7a149b539fe725ee8e9cd89b4bf2840 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:45:03 +0530 Subject: iommu/exynos: Turn on useful configuration options This turns on FLPD_CACHE, ACGEN and SYSSEL. FLPD_CACHE is a cache of 1st level page table entries that contains the address of a 2nd level page table to reduce latency of page table walking. ACGEN is architectural clock gating that gates clocks by System MMU itself if it is not active. Note that ACGEN is different from clock gating by the CPU. ACGEN just gates clocks to the internal logic of System MMU while clock gating by the CPU gates clocks to the System MMU. SYSSEL selects System MMU version in some Exynos SoCs. Some Exynos SoCs have an option to select System MMU versions exclusively because the SoCs adopts new System MMU version experimentally. This also always selects LRU as TLB replacement policy. Selecting TLB replacement policy is deprecated from System MMU 3.2. TLB in System MMU 3.3 has single TLB replacement policy, LRU. The bit of MMU_CFG selecting TLB replacement policy is remained as reserved. QoS value of page table walking is set to 15 (highst value). System MMU 3.3 can inherit QoS value of page table walking from its master H/W's transaction. This new feature is enabled by default and QoS value written to MMU_CFG is ignored. This patch also adds simplifies the sysmmu version checking by introducing some macros. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index b937490..26fb4d7 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -93,6 +93,13 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CTRL_BLOCK 0x7 #define CTRL_DISABLE 0x0 +#define CFG_LRU 0x1 +#define CFG_QOS(n) ((n & 0xF) << 7) +#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */ +#define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ +#define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ +#define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ + #define REG_MMU_CTRL 0x000 #define REG_MMU_CFG 0x004 #define REG_MMU_STATUS 0x008 @@ -109,6 +116,12 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define REG_MMU_VERSION 0x034 +#define MMU_MAJ_VER(val) ((val) >> 7) +#define MMU_MIN_VER(val) ((val) & 0x7F) +#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ + +#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) + #define REG_PB0_SADDR 0x04C #define REG_PB0_EADDR 0x050 #define REG_PB1_SADDR 0x054 @@ -219,6 +232,11 @@ static void sysmmu_unblock(void __iomem *sfrbase) __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); } +static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data) +{ + return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); +} + static bool sysmmu_block(void __iomem *sfrbase) { int i = 120; @@ -374,7 +392,21 @@ static bool __sysmmu_disable(struct sysmmu_drvdata *data) static void __sysmmu_init_config(struct sysmmu_drvdata *data) { - unsigned int cfg = 0; + unsigned int cfg = CFG_LRU | CFG_QOS(15); + unsigned int ver; + + ver = __raw_sysmmu_version(data); + if (MMU_MAJ_VER(ver) == 3) { + if (MMU_MIN_VER(ver) >= 2) { + cfg |= CFG_FLPDCACHE; + if (MMU_MIN_VER(ver) == 3) { + cfg |= CFG_ACGEN; + cfg &= ~CFG_LRU; + } else { + cfg |= CFG_SYSSEL; + } + } + } __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); } @@ -494,13 +526,11 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { - unsigned int maj; unsigned int num_inv = 1; if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); - maj = __raw_readl(data->sfrbase + REG_MMU_VERSION); /* * L2TLB invalidation required * 4KB page: 1 invalidation @@ -511,7 +541,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, * 1MB page can be cached in one of all sets. * 64KB page can be one of 16 consecutive sets. */ - if ((maj >> 28) == 2) /* major version number */ + if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2) num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); if (sysmmu_block(data->sfrbase)) { -- cgit v0.10.2 From 66a7ed84b345d676d7f60f0d397705ccb1c5c7e2 Mon Sep 17 00:00:00 2001 From: Cho KyongHo Date: Mon, 12 May 2014 11:45:04 +0530 Subject: iommu/exynos: Apply workaround of caching fault page table entries This patch contains 2 workaround for the System MMU v3.x. System MMU v3.2 and v3.3 has FLPD cache that caches first level page table entries to reduce page table walking latency. However, the FLPD cache is filled with a first level page table entry even though it is not accessed by a master H/W because System MMU v3.3 speculatively prefetches page table entries that may be accessed in the near future by the master H/W. The prefetched FLPD cache entries are not invalidated by iommu_unmap() because iommu_unmap() only unmaps and invalidates the page table entries that is mapped. Because exynos-iommu driver discards a second level page table when it needs to be replaced with another second level page table or a first level page table entry with 1MB mapping, It is required to invalidate FLPD cache that may contain the first level page table entry that points to the second level page table. Another workaround of System MMU v3.3 is initializing the first level page table entries with the second level page table which is filled with all zeros. This prevents System MMU prefetches 'fault' first level page table entry which may lead page fault on access to 16MiB wide. System MMU 3.x fetches consecutive page table entries by a page table walking to maximize bus utilization and to minimize TLB miss panelty. Unfortunately, functional problem is raised with the fetching behavior because it fetches 'fault' page table entries that specifies no translation information and that a valid translation information will be written to in the near future. The logic in the System MMU generates page fault with the cached fault entries that is no longer coherent with the page table which is updated. There is another workaround that must be implemented by I/O virtual memory manager: any two consecutive I/O virtual memory area must have a hole between the two that is larger than or equal to 128KiB. Also, next I/O virtual memory area must be started from the next 128KiB boundary. 0 128K 256K 384K 512K |-------------|---------------|-----------------|----------------| |area1---------------->|.........hole...........|<--- area2 ----- The constraint is depicted above. The size is selected by the calculation followed: - System MMU can fetch consecutive 64 page table entries at once 64 * 4KiB = 256KiB. This is the size between 128K ~ 384K of the above picture. This style of fetching is 'block fetch'. It fetches the page table entries predefined consecutive page table entries including the entry that is the reason of the page table walking. - System MMU can prefetch upto consecutive 32 page table entries. This is the size between 256K ~ 384K. Signed-off-by: Cho KyongHo Signed-off-by: Shaik Ameer Basha Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 26fb4d7..82aecd0 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -45,8 +45,12 @@ typedef u32 sysmmu_pte_t; #define LPAGE_MASK (~(LPAGE_SIZE - 1)) #define SPAGE_MASK (~(SPAGE_SIZE - 1)) -#define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) -#define lv1ent_page(sent) ((*(sent) & 3) == 1) +#define lv1ent_fault(sent) ((*(sent) == ZERO_LV2LINK) || \ + ((*(sent) & 3) == 0) || ((*(sent) & 3) == 3)) +#define lv1ent_zero(sent) (*(sent) == ZERO_LV2LINK) +#define lv1ent_page_zero(sent) ((*(sent) & 3) == 1) +#define lv1ent_page(sent) ((*(sent) != ZERO_LV2LINK) && \ + ((*(sent) & 3) == 1)) #define lv1ent_section(sent) ((*(sent) & 3) == 2) #define lv2ent_fault(pent) ((*(pent) & 3) == 0) @@ -130,6 +134,8 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define has_sysmmu(dev) (dev->archdata.iommu != NULL) static struct kmem_cache *lv2table_kmem_cache; +static sysmmu_pte_t *zero_lv2_table; +#define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table)) static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) { @@ -515,6 +521,32 @@ static bool exynos_sysmmu_disable(struct device *dev) return disabled; } +static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, + sysmmu_iova_t iova) +{ + if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3)) + __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); +} + +static void sysmmu_tlb_invalidate_flpdcache(struct device *dev, + sysmmu_iova_t iova) +{ + unsigned long flags; + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu); + + if (!IS_ERR(data->clk_master)) + clk_enable(data->clk_master); + + spin_lock_irqsave(&data->lock, flags); + if (is_sysmmu_active(data)) + __sysmmu_tlb_invalidate_flpdcache(data, iova); + spin_unlock_irqrestore(&data->lock, flags); + + if (!IS_ERR(data->clk_master)) + clk_disable(data->clk_master); +} + static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, size_t size) { @@ -667,21 +699,32 @@ static inline void pgtable_flush(void *vastart, void *vaend) static int exynos_iommu_domain_init(struct iommu_domain *domain) { struct exynos_iommu_domain *priv; + int i; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - priv->pgtable = (sysmmu_pte_t *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, 2); + priv->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); if (!priv->pgtable) goto err_pgtable; - priv->lv2entcnt = (short *)__get_free_pages( - GFP_KERNEL | __GFP_ZERO, 1); + priv->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); if (!priv->lv2entcnt) goto err_counter; + /* w/a of System MMU v3.3 to prevent caching 1MiB mapping */ + for (i = 0; i < NUM_LV1ENTRIES; i += 8) { + priv->pgtable[i + 0] = ZERO_LV2LINK; + priv->pgtable[i + 1] = ZERO_LV2LINK; + priv->pgtable[i + 2] = ZERO_LV2LINK; + priv->pgtable[i + 3] = ZERO_LV2LINK; + priv->pgtable[i + 4] = ZERO_LV2LINK; + priv->pgtable[i + 5] = ZERO_LV2LINK; + priv->pgtable[i + 6] = ZERO_LV2LINK; + priv->pgtable[i + 7] = ZERO_LV2LINK; + } + pgtable_flush(priv->pgtable, priv->pgtable + NUM_LV1ENTRIES); spin_lock_init(&priv->lock); @@ -794,8 +837,8 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, dev_err(dev, "%s: No IOMMU is attached\n", __func__); } -static sysmmu_pte_t *alloc_lv2entry(sysmmu_pte_t *sent, sysmmu_iova_t iova, - short *pgcounter) +static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, + sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { if (lv1ent_section(sent)) { WARN(1, "Trying mapping on %#08x mapped with 1MiB page", iova); @@ -804,6 +847,7 @@ static sysmmu_pte_t *alloc_lv2entry(sysmmu_pte_t *sent, sysmmu_iova_t iova, if (lv1ent_fault(sent)) { sysmmu_pte_t *pent; + bool need_flush_flpd_cache = lv1ent_zero(sent); pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1)); @@ -814,12 +858,39 @@ static sysmmu_pte_t *alloc_lv2entry(sysmmu_pte_t *sent, sysmmu_iova_t iova, *pgcounter = NUM_LV2ENTRIES; pgtable_flush(pent, pent + NUM_LV2ENTRIES); pgtable_flush(sent, sent + 1); + + /* + * If pretched SLPD is a fault SLPD in zero_l2_table, FLPD cache + * may caches the address of zero_l2_table. This function + * replaces the zero_l2_table with new L2 page table to write + * valid mappings. + * Accessing the valid area may cause page fault since FLPD + * cache may still caches zero_l2_table for the valid area + * instead of new L2 page table that have the mapping + * information of the valid area + * Thus any replacement of zero_l2_table with other valid L2 + * page table must involve FLPD cache invalidation for System + * MMU v3.3. + * FLPD cache invalidation is performed with TLB invalidation + * by VPN without blocking. It is safe to invalidate TLB without + * blocking because the target address of TLB invalidation is + * not currently mapped. + */ + if (need_flush_flpd_cache) { + struct exynos_iommu_owner *owner; + spin_lock(&priv->lock); + list_for_each_entry(owner, &priv->clients, client) + sysmmu_tlb_invalidate_flpdcache( + owner->dev, iova); + spin_unlock(&priv->lock); + } } return page_entry(sent, iova); } -static int lv1set_section(sysmmu_pte_t *sent, sysmmu_iova_t iova, +static int lv1set_section(struct exynos_iommu_domain *priv, + sysmmu_pte_t *sent, sysmmu_iova_t iova, phys_addr_t paddr, short *pgcnt) { if (lv1ent_section(sent)) { @@ -843,6 +914,18 @@ static int lv1set_section(sysmmu_pte_t *sent, sysmmu_iova_t iova, pgtable_flush(sent, sent + 1); + spin_lock(&priv->lock); + if (lv1ent_page_zero(sent)) { + struct exynos_iommu_owner *owner; + /* + * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD + * entry by speculative prefetch of SLPD which has no mapping. + */ + list_for_each_entry(owner, &priv->clients, client) + sysmmu_tlb_invalidate_flpdcache(owner->dev, iova); + } + spin_unlock(&priv->lock); + return 0; } @@ -874,6 +957,32 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, return 0; } +/* + * *CAUTION* to the I/O virtual memory managers that support exynos-iommu: + * + * System MMU v3.x have an advanced logic to improve address translation + * performance with caching more page table entries by a page table walk. + * However, the logic has a bug that caching fault page table entries and System + * MMU reports page fault if the cached fault entry is hit even though the fault + * entry is updated to a valid entry after the entry is cached. + * To prevent caching fault page table entries which may be updated to valid + * entries later, the virtual memory manager should care about the w/a about the + * problem. The followings describe w/a. + * + * Any two consecutive I/O virtual address regions must have a hole of 128KiB + * in maximum to prevent misbehavior of System MMU 3.x. (w/a of h/w bug) + * + * Precisely, any start address of I/O virtual region must be aligned by + * the following sizes for System MMU v3.1 and v3.2. + * System MMU v3.1: 128KiB + * System MMU v3.2: 256KiB + * + * Because System MMU v3.3 caches page table entries more aggressively, it needs + * more w/a. + * - Any two consecutive I/O virtual regions must be have a hole of larger size + * than or equal size to 128KiB. + * - Start address of an I/O virtual region must be aligned by 128KiB. + */ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, phys_addr_t paddr, size_t size, int prot) { @@ -890,12 +999,12 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, entry = section_entry(priv->pgtable, iova); if (size == SECT_SIZE) { - ret = lv1set_section(entry, iova, paddr, + ret = lv1set_section(priv, entry, iova, paddr, &priv->lv2entcnt[lv1ent_offset(iova)]); } else { sysmmu_pte_t *pent; - pent = alloc_lv2entry(entry, iova, + pent = alloc_lv2entry(priv, entry, iova, &priv->lv2entcnt[lv1ent_offset(iova)]); if (IS_ERR(pent)) @@ -914,11 +1023,24 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, return ret; } +static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv, + sysmmu_iova_t iova, size_t size) +{ + struct exynos_iommu_owner *owner; + unsigned long flags; + + spin_lock_irqsave(&priv->lock, flags); + + list_for_each_entry(owner, &priv->clients, client) + sysmmu_tlb_invalidate_entry(owner->dev, iova, size); + + spin_unlock_irqrestore(&priv->lock, flags); +} + static size_t exynos_iommu_unmap(struct iommu_domain *domain, unsigned long l_iova, size_t size) { struct exynos_iommu_domain *priv = domain->priv; - struct exynos_iommu_owner *owner; sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; sysmmu_pte_t *ent; size_t err_pgsize; @@ -936,7 +1058,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, goto err; } - *ent = 0; + *ent = ZERO_LV2LINK; /* w/a for h/w bug in Sysmem MMU v3.3 */ pgtable_flush(ent, ent + 1); size = SECT_SIZE; goto done; @@ -979,10 +1101,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, done: spin_unlock_irqrestore(&priv->pgtablelock, flags); - spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_entry(owner->dev, iova, size); - spin_unlock_irqrestore(&priv->lock, flags); + exynos_iommu_tlb_invalidate_entry(priv, iova, size); return size; err: @@ -1078,6 +1197,14 @@ static int __init exynos_iommu_init(void) goto err_reg_driver; } + zero_lv2_table = kmem_cache_zalloc(lv2table_kmem_cache, GFP_KERNEL); + if (zero_lv2_table == NULL) { + pr_err("%s: Failed to allocate zero level2 page table\n", + __func__); + ret = -ENOMEM; + goto err_zero_lv2; + } + ret = bus_set_iommu(&platform_bus_type, &exynos_iommu_ops); if (ret) { pr_err("%s: Failed to register exynos-iommu driver.\n", @@ -1087,6 +1214,8 @@ static int __init exynos_iommu_init(void) return 0; err_set_iommu: + kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); +err_zero_lv2: platform_driver_unregister(&exynos_sysmmu_driver); err_reg_driver: kmem_cache_destroy(lv2table_kmem_cache); -- cgit v0.10.2 From df15e1de9c7f18289f40c9cf1f6bfa5fd662b1dd Mon Sep 17 00:00:00 2001 From: Shaik Ameer Basha Date: Wed, 14 May 2014 11:23:33 +0530 Subject: documentation/iommu: Add note on existing DT binding status The current dt binding for Exynos System MMU can be changed, if found incompatible with the support for "Generic IOMMU Binding". This patch adds a note to the binding documentation stating the same. Signed-off-by: Shaik Ameer Basha Acked-by: Arnd Bergmann Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt index 15b2a2b..6fa4c73 100644 --- a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt +++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt @@ -27,6 +27,11 @@ The drivers must consider how to handle those System MMUs. One of the idea is to implement child devices or sub-devices which are the client devices of the System MMU. +Note: +The current DT binding for the Exynos System MMU is incomplete. +The following properties can be removed or changed, if found incompatible with +the "Generic IOMMU Binding" support for attaching devices to the IOMMU. + Required properties: - compatible: Should be "samsung,exynos-sysmmu" - reg: A tuple of base address and size of System MMU registers. -- cgit v0.10.2 From ba5fa6f652151882ee630c70373365cf5c043df7 Mon Sep 17 00:00:00 2001 From: Bjorn Helgaas Date: Thu, 8 May 2014 14:49:14 -0600 Subject: iommu/exynos: Remove unnecessary "&" from function pointers Remove unnecessary "&" from function pointers in exynos_iommu_ops. Signed-off-by: Bjorn Helgaas Acked-by: Arnd Bergmann Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 82aecd0..09f69b1 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1168,15 +1168,15 @@ static void exynos_iommu_remove_device(struct device *dev) } static struct iommu_ops exynos_iommu_ops = { - .domain_init = &exynos_iommu_domain_init, - .domain_destroy = &exynos_iommu_domain_destroy, - .attach_dev = &exynos_iommu_attach_device, - .detach_dev = &exynos_iommu_detach_device, - .map = &exynos_iommu_map, - .unmap = &exynos_iommu_unmap, - .iova_to_phys = &exynos_iommu_iova_to_phys, - .add_device = &exynos_iommu_add_device, - .remove_device = &exynos_iommu_remove_device, + .domain_init = exynos_iommu_domain_init, + .domain_destroy = exynos_iommu_domain_destroy, + .attach_dev = exynos_iommu_attach_device, + .detach_dev = exynos_iommu_detach_device, + .map = exynos_iommu_map, + .unmap = exynos_iommu_unmap, + .iova_to_phys = exynos_iommu_iova_to_phys, + .add_device = exynos_iommu_add_device, + .remove_device = exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, }; -- cgit v0.10.2 From d25a2a16f0889de4a1cd8639896f35dc9465f6f5 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 2 Apr 2014 12:47:37 +0200 Subject: iommu: Add driver for Renesas VMSA-compatible IPMMU Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index df56e4c..a22b537 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -272,6 +272,18 @@ config SHMOBILE_IOMMU_L1SIZE default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB +config IPMMU_VMSA + bool "Renesas VMSA-compatible IPMMU" + depends on ARM_LPAE + depends on ARCH_SHMOBILE || COMPILE_TEST + select IOMMU_API + select ARM_DMA_USE_IOMMU + help + Support for the Renesas VMSA-compatible IPMMU Renesas found in the + R-Mobile APE6 and R-Car H2/M2 SoCs. + + If unsure, say N. + config SPAPR_TCE_IOMMU bool "sPAPR TCE IOMMU Support" depends on PPC_POWERNV || PPC_PSERIES diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 5d58bf1..8893bad 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -7,6 +7,7 @@ obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm-smmu.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o +obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu2.o diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c new file mode 100644 index 0000000..b084530 --- /dev/null +++ b/drivers/iommu/ipmmu-vmsa.c @@ -0,0 +1,1070 @@ +/* + * IPMMU VMSA + * + * Copyright (C) 2014 Renesas Electronics Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct ipmmu_vmsa_device { + struct device *dev; + void __iomem *base; + struct list_head list; + + const struct ipmmu_vmsa_platform_data *pdata; + unsigned int num_utlbs; + + struct dma_iommu_mapping *mapping; +}; + +struct ipmmu_vmsa_domain { + struct ipmmu_vmsa_device *mmu; + struct iommu_domain *io_domain; + + unsigned int context_id; + spinlock_t lock; /* Protects mappings */ + pgd_t *pgd; +}; + +static DEFINE_SPINLOCK(ipmmu_devices_lock); +static LIST_HEAD(ipmmu_devices); + +#define TLB_LOOP_TIMEOUT 100 /* 100us */ + +/* ----------------------------------------------------------------------------- + * Registers Definition + */ + +#define IM_CTX_SIZE 0x40 + +#define IMCTR 0x0000 +#define IMCTR_TRE (1 << 17) +#define IMCTR_AFE (1 << 16) +#define IMCTR_RTSEL_MASK (3 << 4) +#define IMCTR_RTSEL_SHIFT 4 +#define IMCTR_TREN (1 << 3) +#define IMCTR_INTEN (1 << 2) +#define IMCTR_FLUSH (1 << 1) +#define IMCTR_MMUEN (1 << 0) + +#define IMCAAR 0x0004 + +#define IMTTBCR 0x0008 +#define IMTTBCR_EAE (1 << 31) +#define IMTTBCR_PMB (1 << 30) +#define IMTTBCR_SH1_NON_SHAREABLE (0 << 28) +#define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28) +#define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28) +#define IMTTBCR_SH1_MASK (3 << 28) +#define IMTTBCR_ORGN1_NC (0 << 26) +#define IMTTBCR_ORGN1_WB_WA (1 << 26) +#define IMTTBCR_ORGN1_WT (2 << 26) +#define IMTTBCR_ORGN1_WB (3 << 26) +#define IMTTBCR_ORGN1_MASK (3 << 26) +#define IMTTBCR_IRGN1_NC (0 << 24) +#define IMTTBCR_IRGN1_WB_WA (1 << 24) +#define IMTTBCR_IRGN1_WT (2 << 24) +#define IMTTBCR_IRGN1_WB (3 << 24) +#define IMTTBCR_IRGN1_MASK (3 << 24) +#define IMTTBCR_TSZ1_MASK (7 << 16) +#define IMTTBCR_TSZ1_SHIFT 16 +#define IMTTBCR_SH0_NON_SHAREABLE (0 << 12) +#define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12) +#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) +#define IMTTBCR_SH0_MASK (3 << 12) +#define IMTTBCR_ORGN0_NC (0 << 10) +#define IMTTBCR_ORGN0_WB_WA (1 << 10) +#define IMTTBCR_ORGN0_WT (2 << 10) +#define IMTTBCR_ORGN0_WB (3 << 10) +#define IMTTBCR_ORGN0_MASK (3 << 10) +#define IMTTBCR_IRGN0_NC (0 << 8) +#define IMTTBCR_IRGN0_WB_WA (1 << 8) +#define IMTTBCR_IRGN0_WT (2 << 8) +#define IMTTBCR_IRGN0_WB (3 << 8) +#define IMTTBCR_IRGN0_MASK (3 << 8) +#define IMTTBCR_SL0_LVL_2 (0 << 4) +#define IMTTBCR_SL0_LVL_1 (1 << 4) +#define IMTTBCR_TSZ0_MASK (7 << 0) +#define IMTTBCR_TSZ0_SHIFT O + +#define IMBUSCR 0x000c +#define IMBUSCR_DVM (1 << 2) +#define IMBUSCR_BUSSEL_SYS (0 << 0) +#define IMBUSCR_BUSSEL_CCI (1 << 0) +#define IMBUSCR_BUSSEL_IMCAAR (2 << 0) +#define IMBUSCR_BUSSEL_CCI_IMCAAR (3 << 0) +#define IMBUSCR_BUSSEL_MASK (3 << 0) + +#define IMTTLBR0 0x0010 +#define IMTTUBR0 0x0014 +#define IMTTLBR1 0x0018 +#define IMTTUBR1 0x001c + +#define IMSTR 0x0020 +#define IMSTR_ERRLVL_MASK (3 << 12) +#define IMSTR_ERRLVL_SHIFT 12 +#define IMSTR_ERRCODE_TLB_FORMAT (1 << 8) +#define IMSTR_ERRCODE_ACCESS_PERM (4 << 8) +#define IMSTR_ERRCODE_SECURE_ACCESS (5 << 8) +#define IMSTR_ERRCODE_MASK (7 << 8) +#define IMSTR_MHIT (1 << 4) +#define IMSTR_ABORT (1 << 2) +#define IMSTR_PF (1 << 1) +#define IMSTR_TF (1 << 0) + +#define IMMAIR0 0x0028 +#define IMMAIR1 0x002c +#define IMMAIR_ATTR_MASK 0xff +#define IMMAIR_ATTR_DEVICE 0x04 +#define IMMAIR_ATTR_NC 0x44 +#define IMMAIR_ATTR_WBRWA 0xff +#define IMMAIR_ATTR_SHIFT(n) ((n) << 3) +#define IMMAIR_ATTR_IDX_NC 0 +#define IMMAIR_ATTR_IDX_WBRWA 1 +#define IMMAIR_ATTR_IDX_DEV 2 + +#define IMEAR 0x0030 + +#define IMPCTR 0x0200 +#define IMPSTR 0x0208 +#define IMPEAR 0x020c +#define IMPMBA(n) (0x0280 + ((n) * 4)) +#define IMPMBD(n) (0x02c0 + ((n) * 4)) + +#define IMUCTR(n) (0x0300 + ((n) * 16)) +#define IMUCTR_FIXADDEN (1 << 31) +#define IMUCTR_FIXADD_MASK (0xff << 16) +#define IMUCTR_FIXADD_SHIFT 16 +#define IMUCTR_TTSEL_MMU(n) ((n) << 4) +#define IMUCTR_TTSEL_PMB (8 << 4) +#define IMUCTR_TTSEL_MASK (15 << 4) +#define IMUCTR_FLUSH (1 << 1) +#define IMUCTR_MMUEN (1 << 0) + +#define IMUASID(n) (0x0308 + ((n) * 16)) +#define IMUASID_ASID8_MASK (0xff << 8) +#define IMUASID_ASID8_SHIFT 8 +#define IMUASID_ASID0_MASK (0xff << 0) +#define IMUASID_ASID0_SHIFT 0 + +/* ----------------------------------------------------------------------------- + * Page Table Bits + */ + +/* + * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access, + * Long-descriptor format" that the NStable bit being set in a table descriptor + * will result in the NStable and NS bits of all child entries being ignored and + * considered as being set. The IPMMU seems not to comply with this, as it + * generates a secure access page fault if any of the NStable and NS bits isn't + * set when running in non-secure mode. + */ +#ifndef PMD_NSTABLE +#define PMD_NSTABLE (_AT(pmdval_t, 1) << 63) +#endif + +#define ARM_VMSA_PTE_XN (((pteval_t)3) << 53) +#define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52) +#define ARM_VMSA_PTE_AF (((pteval_t)1) << 10) +#define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8) +#define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8) +#define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8) +#define ARM_VMSA_PTE_NS (((pteval_t)1) << 5) +#define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0) + +/* Stage-1 PTE */ +#define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6) +#define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6) +#define ARM_VMSA_PTE_ATTRINDX_SHIFT 2 +#define ARM_VMSA_PTE_nG (((pteval_t)1) << 11) + +/* Stage-2 PTE */ +#define ARM_VMSA_PTE_HAP_FAULT (((pteval_t)0) << 6) +#define ARM_VMSA_PTE_HAP_READ (((pteval_t)1) << 6) +#define ARM_VMSA_PTE_HAP_WRITE (((pteval_t)2) << 6) +#define ARM_VMSA_PTE_MEMATTR_OIWB (((pteval_t)0xf) << 2) +#define ARM_VMSA_PTE_MEMATTR_NC (((pteval_t)0x5) << 2) +#define ARM_VMSA_PTE_MEMATTR_DEV (((pteval_t)0x1) << 2) + +/* ----------------------------------------------------------------------------- + * Read/Write Access + */ + +static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset) +{ + return ioread32(mmu->base + offset); +} + +static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset, + u32 data) +{ + iowrite32(data, mmu->base + offset); +} + +static u32 ipmmu_ctx_read(struct ipmmu_vmsa_domain *domain, unsigned int reg) +{ + return ipmmu_read(domain->mmu, domain->context_id * IM_CTX_SIZE + reg); +} + +static void ipmmu_ctx_write(struct ipmmu_vmsa_domain *domain, unsigned int reg, + u32 data) +{ + ipmmu_write(domain->mmu, domain->context_id * IM_CTX_SIZE + reg, data); +} + +/* ----------------------------------------------------------------------------- + * TLB and microTLB Management + */ + +/* Wait for any pending TLB invalidations to complete */ +static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain) +{ + unsigned int count = 0; + + while (ipmmu_ctx_read(domain, IMCTR) & IMCTR_FLUSH) { + cpu_relax(); + if (++count == TLB_LOOP_TIMEOUT) { + dev_err_ratelimited(domain->mmu->dev, + "TLB sync timed out -- MMU may be deadlocked\n"); + return; + } + udelay(1); + } +} + +static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) +{ + u32 reg; + + reg = ipmmu_ctx_read(domain, IMCTR); + reg |= IMCTR_FLUSH; + ipmmu_ctx_write(domain, IMCTR, reg); + + ipmmu_tlb_sync(domain); +} + +/* + * Enable MMU translation for the microTLB. + */ +static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, + const struct ipmmu_vmsa_master *master) +{ + struct ipmmu_vmsa_device *mmu = domain->mmu; + + /* TODO: What should we set the ASID to ? */ + ipmmu_write(mmu, IMUASID(master->utlb), 0); + /* TODO: Do we need to flush the microTLB ? */ + ipmmu_write(mmu, IMUCTR(master->utlb), + IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | + IMUCTR_MMUEN); +} + +/* + * Disable MMU translation for the microTLB. + */ +static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, + const struct ipmmu_vmsa_master *master) +{ + struct ipmmu_vmsa_device *mmu = domain->mmu; + + ipmmu_write(mmu, IMUCTR(master->utlb), 0); +} + +static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr, + size_t size) +{ + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + + /* + * TODO: Add support for coherent walk through CCI with DVM and remove + * cache handling. + */ + dma_map_page(mmu->dev, virt_to_page(addr), offset, size, DMA_TO_DEVICE); +} + +/* ----------------------------------------------------------------------------- + * Domain/Context Management + */ + +static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) +{ + phys_addr_t ttbr; + u32 reg; + + /* + * TODO: When adding support for multiple contexts, find an unused + * context. + */ + domain->context_id = 0; + + /* TTBR0 */ + ipmmu_flush_pgtable(domain->mmu, domain->pgd, + PTRS_PER_PGD * sizeof(*domain->pgd)); + ttbr = __pa(domain->pgd); + ipmmu_ctx_write(domain, IMTTLBR0, ttbr); + ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32); + + /* + * TTBCR + * We use long descriptors with inner-shareable WBWA tables and allocate + * the whole 32-bit VA space to TTBR0. + */ + ipmmu_ctx_write(domain, IMTTBCR, IMTTBCR_EAE | + IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | + IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); + + /* + * MAIR0 + * We need three attributes only, non-cacheable, write-back read/write + * allocate and device memory. + */ + reg = (IMMAIR_ATTR_NC << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC)) + | (IMMAIR_ATTR_WBRWA << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA)) + | (IMMAIR_ATTR_DEVICE << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV)); + ipmmu_ctx_write(domain, IMMAIR0, reg); + + /* IMBUSCR */ + ipmmu_ctx_write(domain, IMBUSCR, + ipmmu_ctx_read(domain, IMBUSCR) & + ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK)); + + /* + * IMSTR + * Clear all interrupt flags. + */ + ipmmu_ctx_write(domain, IMSTR, ipmmu_ctx_read(domain, IMSTR)); + + /* + * IMCTR + * Enable the MMU and interrupt generation. The long-descriptor + * translation table format doesn't use TEX remapping. Don't enable AF + * software management as we have no use for it. Flush the TLB as + * required when modifying the context registers. + */ + ipmmu_ctx_write(domain, IMCTR, IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN); + + return 0; +} + +static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain) +{ + /* + * Disable the context. Flush the TLB as required when modifying the + * context registers. + * + * TODO: Is TLB flush really needed ? + */ + ipmmu_ctx_write(domain, IMCTR, IMCTR_FLUSH); + ipmmu_tlb_sync(domain); +} + +/* ----------------------------------------------------------------------------- + * Fault Handling + */ + +static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain) +{ + const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF; + struct ipmmu_vmsa_device *mmu = domain->mmu; + u32 status; + u32 iova; + + status = ipmmu_ctx_read(domain, IMSTR); + if (!(status & err_mask)) + return IRQ_NONE; + + iova = ipmmu_ctx_read(domain, IMEAR); + + /* + * Clear the error status flags. Unlike traditional interrupt flag + * registers that must be cleared by writing 1, this status register + * seems to require 0. The error address register must be read before, + * otherwise its value will be 0. + */ + ipmmu_ctx_write(domain, IMSTR, 0); + + /* Log fatal errors. */ + if (status & IMSTR_MHIT) + dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%08x\n", + iova); + if (status & IMSTR_ABORT) + dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%08x\n", + iova); + + if (!(status & (IMSTR_PF | IMSTR_TF))) + return IRQ_NONE; + + /* + * Try to handle page faults and translation faults. + * + * TODO: We need to look up the faulty device based on the I/O VA. Use + * the IOMMU device for now. + */ + if (!report_iommu_fault(domain->io_domain, mmu->dev, iova, 0)) + return IRQ_HANDLED; + + dev_err_ratelimited(mmu->dev, + "Unhandled fault: status 0x%08x iova 0x%08x\n", + status, iova); + + return IRQ_HANDLED; +} + +static irqreturn_t ipmmu_irq(int irq, void *dev) +{ + struct ipmmu_vmsa_device *mmu = dev; + struct iommu_domain *io_domain; + struct ipmmu_vmsa_domain *domain; + + if (!mmu->mapping) + return IRQ_NONE; + + io_domain = mmu->mapping->domain; + domain = io_domain->priv; + + return ipmmu_domain_irq(domain); +} + +/* ----------------------------------------------------------------------------- + * Page Table Management + */ + +static void ipmmu_free_ptes(pmd_t *pmd) +{ + pgtable_t table = pmd_pgtable(*pmd); + __free_page(table); +} + +static void ipmmu_free_pmds(pud_t *pud) +{ + pmd_t *pmd, *pmd_base = pmd_offset(pud, 0); + unsigned int i; + + pmd = pmd_base; + for (i = 0; i < PTRS_PER_PMD; ++i) { + if (pmd_none(*pmd)) + continue; + + ipmmu_free_ptes(pmd); + pmd++; + } + + pmd_free(NULL, pmd_base); +} + +static void ipmmu_free_puds(pgd_t *pgd) +{ + pud_t *pud, *pud_base = pud_offset(pgd, 0); + unsigned int i; + + pud = pud_base; + for (i = 0; i < PTRS_PER_PUD; ++i) { + if (pud_none(*pud)) + continue; + + ipmmu_free_pmds(pud); + pud++; + } + + pud_free(NULL, pud_base); +} + +static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) +{ + pgd_t *pgd, *pgd_base = domain->pgd; + unsigned int i; + + /* + * Recursively free the page tables for this domain. We don't care about + * speculative TLB filling, because the TLB will be nuked next time this + * context bank is re-allocated and no devices currently map to these + * tables. + */ + pgd = pgd_base; + for (i = 0; i < PTRS_PER_PGD; ++i) { + if (pgd_none(*pgd)) + continue; + ipmmu_free_puds(pgd); + pgd++; + } + + kfree(pgd_base); +} + +/* + * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte) + * functions as they would flush the CPU TLB. + */ + +static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, + unsigned long addr, unsigned long end, + phys_addr_t phys, int prot) +{ + unsigned long pfn = __phys_to_pfn(phys); + pteval_t pteval = ARM_VMSA_PTE_PAGE | ARM_VMSA_PTE_NS | ARM_VMSA_PTE_AF + | ARM_VMSA_PTE_XN; + pte_t *pte, *start; + + if (pmd_none(*pmd)) { + /* Allocate a new set of tables */ + pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); + if (!pte) + return -ENOMEM; + + ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE); + *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE); + ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); + + pte += pte_index(addr); + } else + pte = pte_offset_kernel(pmd, addr); + + pteval |= ARM_VMSA_PTE_AP_UNPRIV | ARM_VMSA_PTE_nG; + if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) + pteval |= ARM_VMSA_PTE_AP_RDONLY; + + if (prot & IOMMU_CACHE) + pteval |= (IMMAIR_ATTR_IDX_WBRWA << + ARM_VMSA_PTE_ATTRINDX_SHIFT); + + /* If no access, create a faulting entry to avoid TLB fills */ + if (prot & IOMMU_EXEC) + pteval &= ~ARM_VMSA_PTE_XN; + else if (!(prot & (IOMMU_READ | IOMMU_WRITE))) + pteval &= ~ARM_VMSA_PTE_PAGE; + + pteval |= ARM_VMSA_PTE_SH_IS; + start = pte; + + /* Install the page table entries. */ + do { + *pte++ = pfn_pte(pfn++, __pgprot(pteval)); + addr += PAGE_SIZE; + } while (addr != end); + + ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * (pte - start)); + return 0; +} + +static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, + unsigned long addr, unsigned long end, + phys_addr_t phys, int prot) +{ + unsigned long next; + pmd_t *pmd; + int ret; + +#ifndef __PAGETABLE_PMD_FOLDED + if (pud_none(*pud)) { + pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); + if (!pmd) + return -ENOMEM; + + ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE); + *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE); + ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); + + pmd += pmd_index(addr); + } else +#endif + pmd = pmd_offset(pud, addr); + + do { + next = pmd_addr_end(addr, end); + ret = ipmmu_alloc_init_pte(mmu, pmd, addr, end, phys, prot); + phys += next - addr; + } while (pmd++, addr = next, addr < end); + + return ret; +} + +static int ipmmu_alloc_init_pud(struct ipmmu_vmsa_device *mmu, pgd_t *pgd, + unsigned long addr, unsigned long end, + phys_addr_t phys, int prot) +{ + unsigned long next; + pud_t *pud; + int ret; + +#ifndef __PAGETABLE_PUD_FOLDED + if (pgd_none(*pgd)) { + pud = (pud_t *)get_zeroed_page(GFP_ATOMIC); + if (!pud) + return -ENOMEM; + + ipmmu_flush_pgtable(mmu, pud, PAGE_SIZE); + *pgd = __pgd(__pa(pud) | PMD_NSTABLE | PMD_TYPE_TABLE); + ipmmu_flush_pgtable(mmu, pgd, sizeof(*pgd)); + + pud += pud_index(addr); + } else +#endif + pud = pud_offset(pgd, addr); + + do { + next = pud_addr_end(addr, end); + ret = ipmmu_alloc_init_pmd(mmu, pud, addr, next, phys, prot); + phys += next - addr; + } while (pud++, addr = next, addr < end); + + return ret; +} + +static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, + unsigned long iova, phys_addr_t paddr, + size_t size, int prot) +{ + struct ipmmu_vmsa_device *mmu = domain->mmu; + pgd_t *pgd = domain->pgd; + unsigned long flags; + unsigned long end; + int ret; + + if (!pgd) + return -EINVAL; + + if (size & ~PAGE_MASK) + return -EINVAL; + + if (paddr & ~((1ULL << 40) - 1)) + return -ERANGE; + + spin_lock_irqsave(&domain->lock, flags); + + pgd += pgd_index(iova); + end = iova + size; + + do { + unsigned long next = pgd_addr_end(iova, end); + + ret = ipmmu_alloc_init_pud(mmu, pgd, iova, next, paddr, prot); + if (ret) + break; + + paddr += next - iova; + iova = next; + } while (pgd++, iova != end); + + spin_unlock_irqrestore(&domain->lock, flags); + + ipmmu_tlb_invalidate(domain); + + return ret; +} + +/* ----------------------------------------------------------------------------- + * IOMMU Operations + */ + +static const struct ipmmu_vmsa_master * +ipmmu_find_master(struct ipmmu_vmsa_device *ipmmu, struct device *dev) +{ + const struct ipmmu_vmsa_master *master = ipmmu->pdata->masters; + const char *devname = dev_name(dev); + unsigned int i; + + for (i = 0; i < ipmmu->pdata->num_masters; ++i, ++master) { + if (strcmp(master->name, devname) == 0) + return master; + } + + return NULL; +} + +static int ipmmu_domain_init(struct iommu_domain *io_domain) +{ + struct ipmmu_vmsa_domain *domain; + + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) + return -ENOMEM; + + spin_lock_init(&domain->lock); + + domain->pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL); + if (!domain->pgd) { + kfree(domain); + return -ENOMEM; + } + + io_domain->priv = domain; + domain->io_domain = io_domain; + + return 0; +} + +static void ipmmu_domain_destroy(struct iommu_domain *io_domain) +{ + struct ipmmu_vmsa_domain *domain = io_domain->priv; + + /* + * Free the domain resources. We assume that all devices have already + * been detached. + */ + ipmmu_domain_destroy_context(domain); + ipmmu_free_pgtables(domain); + kfree(domain); +} + +static int ipmmu_attach_device(struct iommu_domain *io_domain, + struct device *dev) +{ + struct ipmmu_vmsa_device *mmu = dev->archdata.iommu; + struct ipmmu_vmsa_domain *domain = io_domain->priv; + const struct ipmmu_vmsa_master *master; + unsigned long flags; + int ret = 0; + + if (!mmu) { + dev_err(dev, "Cannot attach to IPMMU\n"); + return -ENXIO; + } + + spin_lock_irqsave(&domain->lock, flags); + + if (!domain->mmu) { + /* The domain hasn't been used yet, initialize it. */ + domain->mmu = mmu; + ret = ipmmu_domain_init_context(domain); + } else if (domain->mmu != mmu) { + /* + * Something is wrong, we can't attach two devices using + * different IOMMUs to the same domain. + */ + dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", + dev_name(mmu->dev), dev_name(domain->mmu->dev)); + ret = -EINVAL; + } + + spin_unlock_irqrestore(&domain->lock, flags); + + if (ret < 0) + return ret; + + master = ipmmu_find_master(mmu, dev); + if (!master) + return -EINVAL; + + ipmmu_utlb_enable(domain, master); + + return 0; +} + +static void ipmmu_detach_device(struct iommu_domain *io_domain, + struct device *dev) +{ + struct ipmmu_vmsa_domain *domain = io_domain->priv; + const struct ipmmu_vmsa_master *master; + + master = ipmmu_find_master(domain->mmu, dev); + if (!master) + return; + + ipmmu_utlb_disable(domain, master); + + /* + * TODO: Optimize by disabling the context when no device is attached. + */ +} + +static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct ipmmu_vmsa_domain *domain = io_domain->priv; + + if (!domain) + return -ENODEV; + + return ipmmu_handle_mapping(domain, iova, paddr, size, prot); +} + +static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, + size_t size) +{ + struct ipmmu_vmsa_domain *domain = io_domain->priv; + int ret; + + ret = ipmmu_handle_mapping(domain, iova, 0, size, 0); + return ret ? 0 : size; +} + +static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, + dma_addr_t iova) +{ + struct ipmmu_vmsa_domain *domain = io_domain->priv; + pgd_t pgd; + pud_t pud; + pmd_t pmd; + pte_t pte; + + /* TODO: Is locking needed ? */ + + if (!domain->pgd) + return 0; + + pgd = *(domain->pgd + pgd_index(iova)); + if (pgd_none(pgd)) + return 0; + + pud = *pud_offset(&pgd, iova); + if (pud_none(pud)) + return 0; + + pmd = *pmd_offset(&pud, iova); + if (pmd_none(pmd)) + return 0; + + pte = *(pmd_page_vaddr(pmd) + pte_index(iova)); + if (pte_none(pte)) + return 0; + + return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); +} + +static int ipmmu_add_device(struct device *dev) +{ + const struct ipmmu_vmsa_master *master = NULL; + struct ipmmu_vmsa_device *mmu; + struct iommu_group *group; + int ret; + + if (dev->archdata.iommu) { + dev_warn(dev, "IOMMU driver already assigned to device %s\n", + dev_name(dev)); + return -EINVAL; + } + + /* Find the master corresponding to the device. */ + spin_lock(&ipmmu_devices_lock); + + list_for_each_entry(mmu, &ipmmu_devices, list) { + master = ipmmu_find_master(mmu, dev); + if (master) { + /* + * TODO Take a reference to the master to protect + * against device removal. + */ + break; + } + } + + spin_unlock(&ipmmu_devices_lock); + + if (!master) + return -ENODEV; + + if (!master->utlb >= mmu->num_utlbs) + return -EINVAL; + + /* Create a device group and add the device to it. */ + group = iommu_group_alloc(); + if (IS_ERR(group)) { + dev_err(dev, "Failed to allocate IOMMU group\n"); + return PTR_ERR(group); + } + + ret = iommu_group_add_device(group, dev); + iommu_group_put(group); + + if (ret < 0) { + dev_err(dev, "Failed to add device to IPMMU group\n"); + return ret; + } + + dev->archdata.iommu = mmu; + + /* + * Create the ARM mapping, used by the ARM DMA mapping core to allocate + * VAs. This will allocate a corresponding IOMMU domain. + * + * TODO: + * - Create one mapping per context (TLB). + * - Make the mapping size configurable ? We currently use a 2GB mapping + * at a 1GB offset to ensure that NULL VAs will fault. + */ + if (!mmu->mapping) { + struct dma_iommu_mapping *mapping; + + mapping = arm_iommu_create_mapping(&platform_bus_type, + SZ_1G, SZ_2G, 0); + if (IS_ERR(mapping)) { + dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); + return PTR_ERR(mapping); + } + + mmu->mapping = mapping; + } + + /* Attach the ARM VA mapping to the device. */ + ret = arm_iommu_attach_device(dev, mmu->mapping); + if (ret < 0) { + dev_err(dev, "Failed to attach device to VA mapping\n"); + goto error; + } + + return 0; + +error: + dev->archdata.iommu = NULL; + iommu_group_remove_device(dev); + return ret; +} + +static void ipmmu_remove_device(struct device *dev) +{ + arm_iommu_detach_device(dev); + iommu_group_remove_device(dev); + dev->archdata.iommu = NULL; +} + +static struct iommu_ops ipmmu_ops = { + .domain_init = ipmmu_domain_init, + .domain_destroy = ipmmu_domain_destroy, + .attach_dev = ipmmu_attach_device, + .detach_dev = ipmmu_detach_device, + .map = ipmmu_map, + .unmap = ipmmu_unmap, + .iova_to_phys = ipmmu_iova_to_phys, + .add_device = ipmmu_add_device, + .remove_device = ipmmu_remove_device, + .pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K, +}; + +/* ----------------------------------------------------------------------------- + * Probe/remove and init + */ + +static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu) +{ + unsigned int i; + + /* Disable all contexts. */ + for (i = 0; i < 4; ++i) + ipmmu_write(mmu, i * IM_CTX_SIZE + IMCTR, 0); +} + +static int ipmmu_probe(struct platform_device *pdev) +{ + struct ipmmu_vmsa_device *mmu; + struct resource *res; + int irq; + int ret; + + if (!pdev->dev.platform_data) { + dev_err(&pdev->dev, "missing platform data\n"); + return -EINVAL; + } + + mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); + if (!mmu) { + dev_err(&pdev->dev, "cannot allocate device data\n"); + return -ENOMEM; + } + + mmu->dev = &pdev->dev; + mmu->pdata = pdev->dev.platform_data; + mmu->num_utlbs = 32; + + /* Map I/O memory and request IRQ. */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + mmu->base = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(mmu->base)) + return PTR_ERR(mmu->base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) { + dev_err(&pdev->dev, "no IRQ found\n"); + return irq; + } + + ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, + dev_name(&pdev->dev), mmu); + if (ret < 0) { + dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); + return irq; + } + + ipmmu_device_reset(mmu); + + /* + * We can't create the ARM mapping here as it requires the bus to have + * an IOMMU, which only happens when bus_set_iommu() is called in + * ipmmu_init() after the probe function returns. + */ + + spin_lock(&ipmmu_devices_lock); + list_add(&mmu->list, &ipmmu_devices); + spin_unlock(&ipmmu_devices_lock); + + platform_set_drvdata(pdev, mmu); + + return 0; +} + +static int ipmmu_remove(struct platform_device *pdev) +{ + struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); + + spin_lock(&ipmmu_devices_lock); + list_del(&mmu->list); + spin_unlock(&ipmmu_devices_lock); + + arm_iommu_release_mapping(mmu->mapping); + + ipmmu_device_reset(mmu); + + return 0; +} + +static struct platform_driver ipmmu_driver = { + .driver = { + .owner = THIS_MODULE, + .name = "ipmmu-vmsa", + }, + .probe = ipmmu_probe, + .remove = ipmmu_remove, +}; + +static int __init ipmmu_init(void) +{ + int ret; + + ret = platform_driver_register(&ipmmu_driver); + if (ret < 0) + return ret; + + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &ipmmu_ops); + + return 0; +} + +static void __exit ipmmu_exit(void) +{ + return platform_driver_unregister(&ipmmu_driver); +} + +subsys_initcall(ipmmu_init); +module_exit(ipmmu_exit); + +MODULE_DESCRIPTION("IOMMU API for Renesas VMSA-compatible IPMMU"); +MODULE_AUTHOR("Laurent Pinchart "); +MODULE_LICENSE("GPL v2"); diff --git a/include/linux/platform_data/ipmmu-vmsa.h b/include/linux/platform_data/ipmmu-vmsa.h new file mode 100644 index 0000000..5275b3a --- /dev/null +++ b/include/linux/platform_data/ipmmu-vmsa.h @@ -0,0 +1,24 @@ +/* + * IPMMU VMSA Platform Data + * + * Copyright (C) 2014 Renesas Electronics Corporation + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#ifndef __IPMMU_VMSA_H__ +#define __IPMMU_VMSA_H__ + +struct ipmmu_vmsa_master { + const char *name; + unsigned int utlb; +}; + +struct ipmmu_vmsa_platform_data { + const struct ipmmu_vmsa_master *masters; + unsigned int num_masters; +}; + +#endif /* __IPMMU_VMSA_H__ */ -- cgit v0.10.2 From 192d2045707b25b984436eabfbfd3c8f1ada5a56 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:42 +0200 Subject: iommu/ipmmu-vmsa: Refactor micro-TLB lookup Cache the micro-TLB number in archdata allocated in the .add_device handler instead of looking it up when the deviced is attached and detached. This simplifies the .attach_dev and .detach_dev operations and prepares for DT support. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index b084530..1949f3c 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -44,6 +44,11 @@ struct ipmmu_vmsa_domain { pgd_t *pgd; }; +struct ipmmu_vmsa_archdata { + struct ipmmu_vmsa_device *mmu; + unsigned int utlb; +}; + static DEFINE_SPINLOCK(ipmmu_devices_lock); static LIST_HEAD(ipmmu_devices); @@ -265,14 +270,19 @@ static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain) * Enable MMU translation for the microTLB. */ static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, - const struct ipmmu_vmsa_master *master) + unsigned int utlb) { struct ipmmu_vmsa_device *mmu = domain->mmu; + /* + * TODO: Reference-count the microTLB as several bus masters can be + * connected to the same microTLB. + */ + /* TODO: What should we set the ASID to ? */ - ipmmu_write(mmu, IMUASID(master->utlb), 0); + ipmmu_write(mmu, IMUASID(utlb), 0); /* TODO: Do we need to flush the microTLB ? */ - ipmmu_write(mmu, IMUCTR(master->utlb), + ipmmu_write(mmu, IMUCTR(utlb), IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | IMUCTR_MMUEN); } @@ -281,11 +291,11 @@ static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain, * Disable MMU translation for the microTLB. */ static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, - const struct ipmmu_vmsa_master *master) + unsigned int utlb) { struct ipmmu_vmsa_device *mmu = domain->mmu; - ipmmu_write(mmu, IMUCTR(master->utlb), 0); + ipmmu_write(mmu, IMUCTR(utlb), 0); } static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr, @@ -674,21 +684,6 @@ static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, * IOMMU Operations */ -static const struct ipmmu_vmsa_master * -ipmmu_find_master(struct ipmmu_vmsa_device *ipmmu, struct device *dev) -{ - const struct ipmmu_vmsa_master *master = ipmmu->pdata->masters; - const char *devname = dev_name(dev); - unsigned int i; - - for (i = 0; i < ipmmu->pdata->num_masters; ++i, ++master) { - if (strcmp(master->name, devname) == 0) - return master; - } - - return NULL; -} - static int ipmmu_domain_init(struct iommu_domain *io_domain) { struct ipmmu_vmsa_domain *domain; @@ -727,9 +722,9 @@ static void ipmmu_domain_destroy(struct iommu_domain *io_domain) static int ipmmu_attach_device(struct iommu_domain *io_domain, struct device *dev) { - struct ipmmu_vmsa_device *mmu = dev->archdata.iommu; + struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; + struct ipmmu_vmsa_device *mmu = archdata->mmu; struct ipmmu_vmsa_domain *domain = io_domain->priv; - const struct ipmmu_vmsa_master *master; unsigned long flags; int ret = 0; @@ -759,11 +754,7 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, if (ret < 0) return ret; - master = ipmmu_find_master(mmu, dev); - if (!master) - return -EINVAL; - - ipmmu_utlb_enable(domain, master); + ipmmu_utlb_enable(domain, archdata->utlb); return 0; } @@ -771,14 +762,10 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, static void ipmmu_detach_device(struct iommu_domain *io_domain, struct device *dev) { + struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; struct ipmmu_vmsa_domain *domain = io_domain->priv; - const struct ipmmu_vmsa_master *master; - master = ipmmu_find_master(domain->mmu, dev); - if (!master) - return; - - ipmmu_utlb_disable(domain, master); + ipmmu_utlb_disable(domain, archdata->utlb); /* * TODO: Optimize by disabling the context when no device is attached. @@ -839,11 +826,26 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); } +static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev) +{ + const struct ipmmu_vmsa_master *master = mmu->pdata->masters; + const char *devname = dev_name(dev); + unsigned int i; + + for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) { + if (strcmp(master->name, devname) == 0) + return master->utlb; + } + + return -1; +} + static int ipmmu_add_device(struct device *dev) { - const struct ipmmu_vmsa_master *master = NULL; + struct ipmmu_vmsa_archdata *archdata; struct ipmmu_vmsa_device *mmu; struct iommu_group *group; + int utlb = -1; int ret; if (dev->archdata.iommu) { @@ -856,10 +858,10 @@ static int ipmmu_add_device(struct device *dev) spin_lock(&ipmmu_devices_lock); list_for_each_entry(mmu, &ipmmu_devices, list) { - master = ipmmu_find_master(mmu, dev); - if (master) { + utlb = ipmmu_find_utlb(mmu, dev); + if (utlb >= 0) { /* - * TODO Take a reference to the master to protect + * TODO Take a reference to the MMU to protect * against device removal. */ break; @@ -868,10 +870,10 @@ static int ipmmu_add_device(struct device *dev) spin_unlock(&ipmmu_devices_lock); - if (!master) + if (utlb < 0) return -ENODEV; - if (!master->utlb >= mmu->num_utlbs) + if (utlb >= mmu->num_utlbs) return -EINVAL; /* Create a device group and add the device to it. */ @@ -889,7 +891,15 @@ static int ipmmu_add_device(struct device *dev) return ret; } - dev->archdata.iommu = mmu; + archdata = kzalloc(sizeof(*archdata), GFP_KERNEL); + if (!archdata) { + ret = -ENOMEM; + goto error; + } + + archdata->mmu = mmu; + archdata->utlb = utlb; + dev->archdata.iommu = archdata; /* * Create the ARM mapping, used by the ARM DMA mapping core to allocate @@ -923,6 +933,7 @@ static int ipmmu_add_device(struct device *dev) return 0; error: + kfree(dev->archdata.iommu); dev->archdata.iommu = NULL; iommu_group_remove_device(dev); return ret; @@ -932,6 +943,7 @@ static void ipmmu_remove_device(struct device *dev) { arm_iommu_detach_device(dev); iommu_group_remove_device(dev); + kfree(dev->archdata.iommu); dev->archdata.iommu = NULL; } -- cgit v0.10.2 From 251dac410d29c8ab432034e67472a53b6c3e497e Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:44 +0200 Subject: iommu/ipmmu-vmsa: Fix the supported page sizes The hardware supports 2MB page sizes, not 1MB. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 1949f3c..0fd322d 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -957,7 +957,7 @@ static struct iommu_ops ipmmu_ops = { .iova_to_phys = ipmmu_iova_to_phys, .add_device = ipmmu_add_device, .remove_device = ipmmu_remove_device, - .pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K, + .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K, }; /* ----------------------------------------------------------------------------- -- cgit v0.10.2 From bc28191b165bb07f2817219da1b8c20246d022f6 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:45 +0200 Subject: iommu/ipmmu-vmsa: Define driver-specific page directory sizes The PTRS_PER_(PUD|PGD|PMD|PTE) macros evaluate to different values depending on whether LPAE is enabled. The IPMMU driver uses a long descriptor format regardless of LPAE, making those macros mismatch the IPMMU configuration on non-LPAE systems. Replace the macros by driver-specific versions that always evaluate to the right value. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 0fd322d..f8f5b19 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -209,6 +209,11 @@ static LIST_HEAD(ipmmu_devices); #define ARM_VMSA_PTE_MEMATTR_NC (((pteval_t)0x5) << 2) #define ARM_VMSA_PTE_MEMATTR_DEV (((pteval_t)0x1) << 2) +#define IPMMU_PTRS_PER_PTE 512 +#define IPMMU_PTRS_PER_PMD 512 +#define IPMMU_PTRS_PER_PGD 4 +#define IPMMU_PTRS_PER_PUD 1 + /* ----------------------------------------------------------------------------- * Read/Write Access */ @@ -327,7 +332,7 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) /* TTBR0 */ ipmmu_flush_pgtable(domain->mmu, domain->pgd, - PTRS_PER_PGD * sizeof(*domain->pgd)); + IPMMU_PTRS_PER_PGD * sizeof(*domain->pgd)); ttbr = __pa(domain->pgd); ipmmu_ctx_write(domain, IMTTLBR0, ttbr); ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32); @@ -469,7 +474,7 @@ static void ipmmu_free_pmds(pud_t *pud) unsigned int i; pmd = pmd_base; - for (i = 0; i < PTRS_PER_PMD; ++i) { + for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { if (pmd_none(*pmd)) continue; @@ -486,7 +491,7 @@ static void ipmmu_free_puds(pgd_t *pgd) unsigned int i; pud = pud_base; - for (i = 0; i < PTRS_PER_PUD; ++i) { + for (i = 0; i < IPMMU_PTRS_PER_PUD; ++i) { if (pud_none(*pud)) continue; @@ -509,7 +514,7 @@ static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) * tables. */ pgd = pgd_base; - for (i = 0; i < PTRS_PER_PGD; ++i) { + for (i = 0; i < IPMMU_PTRS_PER_PGD; ++i) { if (pgd_none(*pgd)) continue; ipmmu_free_puds(pgd); @@ -694,7 +699,7 @@ static int ipmmu_domain_init(struct iommu_domain *io_domain) spin_lock_init(&domain->lock); - domain->pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL); + domain->pgd = kzalloc(IPMMU_PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL); if (!domain->pgd) { kfree(domain); return -ENOMEM; -- cgit v0.10.2 From 4ee3cc9c4a62659bc5f5ed59ea49a0b98b5ac670 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:46 +0200 Subject: iommu/ipmmu-vmsa: Set the PTE contiguous hint bit when possible The contiguous hint bit signals to the IOMMU that a range of 16 PTEs refer to physically contiguous memory. It improves performances by dividing the number of TLB lookups by 16, effectively implementing 64kB page sizes. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index f8f5b19..e64c616 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -209,6 +209,9 @@ static LIST_HEAD(ipmmu_devices); #define ARM_VMSA_PTE_MEMATTR_NC (((pteval_t)0x5) << 2) #define ARM_VMSA_PTE_MEMATTR_DEV (((pteval_t)0x1) << 2) +#define ARM_VMSA_PTE_CONT_ENTRIES 16 +#define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES) + #define IPMMU_PTRS_PER_PTE 512 #define IPMMU_PTRS_PER_PMD 512 #define IPMMU_PTRS_PER_PGD 4 @@ -569,10 +572,44 @@ static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, pteval |= ARM_VMSA_PTE_SH_IS; start = pte; - /* Install the page table entries. */ + /* + * Install the page table entries. + * + * Set the contiguous hint in the PTEs where possible. The hint + * indicates a series of ARM_VMSA_PTE_CONT_ENTRIES PTEs mapping a + * physically contiguous region with the following constraints: + * + * - The region start is aligned to ARM_VMSA_PTE_CONT_SIZE + * - Each PTE in the region has the contiguous hint bit set + * + * We don't support partial unmapping so there's no need to care about + * clearing the contiguous hint from neighbour PTEs. + */ do { - *pte++ = pfn_pte(pfn++, __pgprot(pteval)); - addr += PAGE_SIZE; + unsigned long chunk_end; + + /* + * If the address is aligned to a contiguous region size and the + * mapping size is large enough, process the largest possible + * number of PTEs multiple of ARM_VMSA_PTE_CONT_ENTRIES. + * Otherwise process the smallest number of PTEs to align the + * address to a contiguous region size or to complete the + * mapping. + */ + if (IS_ALIGNED(addr, ARM_VMSA_PTE_CONT_SIZE) && + end - addr >= ARM_VMSA_PTE_CONT_SIZE) { + chunk_end = round_down(end, ARM_VMSA_PTE_CONT_SIZE); + pteval |= ARM_VMSA_PTE_CONT; + } else { + chunk_end = min(ALIGN(addr, ARM_VMSA_PTE_CONT_SIZE), + end); + pteval &= ~ARM_VMSA_PTE_CONT; + } + + do { + *pte++ = pfn_pte(pfn++, __pgprot(pteval)); + addr += PAGE_SIZE; + } while (addr != chunk_end); } while (addr != end); ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * (pte - start)); -- cgit v0.10.2 From 14e5123ee9bb0d91ceb451d0231b52f8c04af99d Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:47 +0200 Subject: iommu/ipmmu-vmsa: PMD is never folded, PUD always is The driver only supports the 3-level long descriptor format that has no PUD and always has a PMD. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index e64c616..1201afa 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -215,7 +215,6 @@ static LIST_HEAD(ipmmu_devices); #define IPMMU_PTRS_PER_PTE 512 #define IPMMU_PTRS_PER_PMD 512 #define IPMMU_PTRS_PER_PGD 4 -#define IPMMU_PTRS_PER_PUD 1 /* ----------------------------------------------------------------------------- * Read/Write Access @@ -465,6 +464,8 @@ static irqreturn_t ipmmu_irq(int irq, void *dev) * Page Table Management */ +#define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK)) + static void ipmmu_free_ptes(pmd_t *pmd) { pgtable_t table = pmd_pgtable(*pmd); @@ -473,10 +474,10 @@ static void ipmmu_free_ptes(pmd_t *pmd) static void ipmmu_free_pmds(pud_t *pud) { - pmd_t *pmd, *pmd_base = pmd_offset(pud, 0); + pmd_t *pmd = pmd_offset(pud, 0); + pgtable_t table; unsigned int i; - pmd = pmd_base; for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { if (pmd_none(*pmd)) continue; @@ -485,24 +486,8 @@ static void ipmmu_free_pmds(pud_t *pud) pmd++; } - pmd_free(NULL, pmd_base); -} - -static void ipmmu_free_puds(pgd_t *pgd) -{ - pud_t *pud, *pud_base = pud_offset(pgd, 0); - unsigned int i; - - pud = pud_base; - for (i = 0; i < IPMMU_PTRS_PER_PUD; ++i) { - if (pud_none(*pud)) - continue; - - ipmmu_free_pmds(pud); - pud++; - } - - pud_free(NULL, pud_base); + table = pud_pgtable(*pud); + __free_page(table); } static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) @@ -520,7 +505,7 @@ static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) for (i = 0; i < IPMMU_PTRS_PER_PGD; ++i) { if (pgd_none(*pgd)) continue; - ipmmu_free_puds(pgd); + ipmmu_free_pmds((pud_t *)pgd); pgd++; } @@ -624,7 +609,6 @@ static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, pmd_t *pmd; int ret; -#ifndef __PAGETABLE_PMD_FOLDED if (pud_none(*pud)) { pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); if (!pmd) @@ -636,7 +620,6 @@ static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, pmd += pmd_index(addr); } else -#endif pmd = pmd_offset(pud, addr); do { @@ -648,38 +631,6 @@ static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, return ret; } -static int ipmmu_alloc_init_pud(struct ipmmu_vmsa_device *mmu, pgd_t *pgd, - unsigned long addr, unsigned long end, - phys_addr_t phys, int prot) -{ - unsigned long next; - pud_t *pud; - int ret; - -#ifndef __PAGETABLE_PUD_FOLDED - if (pgd_none(*pgd)) { - pud = (pud_t *)get_zeroed_page(GFP_ATOMIC); - if (!pud) - return -ENOMEM; - - ipmmu_flush_pgtable(mmu, pud, PAGE_SIZE); - *pgd = __pgd(__pa(pud) | PMD_NSTABLE | PMD_TYPE_TABLE); - ipmmu_flush_pgtable(mmu, pgd, sizeof(*pgd)); - - pud += pud_index(addr); - } else -#endif - pud = pud_offset(pgd, addr); - - do { - next = pud_addr_end(addr, end); - ret = ipmmu_alloc_init_pmd(mmu, pud, addr, next, phys, prot); - phys += next - addr; - } while (pud++, addr = next, addr < end); - - return ret; -} - static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) @@ -707,7 +658,8 @@ static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, do { unsigned long next = pgd_addr_end(iova, end); - ret = ipmmu_alloc_init_pud(mmu, pgd, iova, next, paddr, prot); + ret = ipmmu_alloc_init_pmd(mmu, (pud_t *)pgd, iova, next, paddr, + prot); if (ret) break; -- cgit v0.10.2 From 9009f256596da78567d63c434691f7e409a99400 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:48 +0200 Subject: iommu/ipmmu-vmsa: Rewrite page table management The IOMMU core will only call us with page sizes advertized as supported by the driver. We can thus simplify the code by removing loops over PGD and PMD entries. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 1201afa..87703c3 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -517,118 +517,97 @@ static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) * functions as they would flush the CPU TLB. */ -static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, - unsigned long addr, unsigned long end, - phys_addr_t phys, int prot) +static pte_t *ipmmu_alloc_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, + unsigned long iova) { - unsigned long pfn = __phys_to_pfn(phys); - pteval_t pteval = ARM_VMSA_PTE_PAGE | ARM_VMSA_PTE_NS | ARM_VMSA_PTE_AF - | ARM_VMSA_PTE_XN; - pte_t *pte, *start; + pte_t *pte; - if (pmd_none(*pmd)) { - /* Allocate a new set of tables */ - pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); - if (!pte) - return -ENOMEM; + if (!pmd_none(*pmd)) + return pte_offset_kernel(pmd, iova); - ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE); - *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE); - ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); + pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); + if (!pte) + return NULL; - pte += pte_index(addr); - } else - pte = pte_offset_kernel(pmd, addr); + ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE); + *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE); + ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); - pteval |= ARM_VMSA_PTE_AP_UNPRIV | ARM_VMSA_PTE_nG; - if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) - pteval |= ARM_VMSA_PTE_AP_RDONLY; + return pte + pte_index(iova); +} - if (prot & IOMMU_CACHE) - pteval |= (IMMAIR_ATTR_IDX_WBRWA << - ARM_VMSA_PTE_ATTRINDX_SHIFT); +static pmd_t *ipmmu_alloc_pmd(struct ipmmu_vmsa_device *mmu, pgd_t *pgd, + unsigned long iova) +{ + pud_t *pud = (pud_t *)pgd; + pmd_t *pmd; - /* If no access, create a faulting entry to avoid TLB fills */ - if (prot & IOMMU_EXEC) - pteval &= ~ARM_VMSA_PTE_XN; - else if (!(prot & (IOMMU_READ | IOMMU_WRITE))) - pteval &= ~ARM_VMSA_PTE_PAGE; + if (!pud_none(*pud)) + return pmd_offset(pud, iova); - pteval |= ARM_VMSA_PTE_SH_IS; - start = pte; + pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); + if (!pmd) + return NULL; - /* - * Install the page table entries. - * - * Set the contiguous hint in the PTEs where possible. The hint - * indicates a series of ARM_VMSA_PTE_CONT_ENTRIES PTEs mapping a - * physically contiguous region with the following constraints: - * - * - The region start is aligned to ARM_VMSA_PTE_CONT_SIZE - * - Each PTE in the region has the contiguous hint bit set - * - * We don't support partial unmapping so there's no need to care about - * clearing the contiguous hint from neighbour PTEs. - */ - do { - unsigned long chunk_end; + ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE); + *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE); + ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); - /* - * If the address is aligned to a contiguous region size and the - * mapping size is large enough, process the largest possible - * number of PTEs multiple of ARM_VMSA_PTE_CONT_ENTRIES. - * Otherwise process the smallest number of PTEs to align the - * address to a contiguous region size or to complete the - * mapping. - */ - if (IS_ALIGNED(addr, ARM_VMSA_PTE_CONT_SIZE) && - end - addr >= ARM_VMSA_PTE_CONT_SIZE) { - chunk_end = round_down(end, ARM_VMSA_PTE_CONT_SIZE); - pteval |= ARM_VMSA_PTE_CONT; - } else { - chunk_end = min(ALIGN(addr, ARM_VMSA_PTE_CONT_SIZE), - end); - pteval &= ~ARM_VMSA_PTE_CONT; - } + return pmd + pmd_index(iova); +} - do { - *pte++ = pfn_pte(pfn++, __pgprot(pteval)); - addr += PAGE_SIZE; - } while (addr != chunk_end); - } while (addr != end); +static u64 ipmmu_page_prot(unsigned int prot, u64 type) +{ + u64 pgprot = ARM_VMSA_PTE_XN | ARM_VMSA_PTE_nG | ARM_VMSA_PTE_AF + | ARM_VMSA_PTE_SH_IS | ARM_VMSA_PTE_AP_UNPRIV + | ARM_VMSA_PTE_NS | type; - ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * (pte - start)); - return 0; + if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) + pgprot |= ARM_VMSA_PTE_AP_RDONLY; + + if (prot & IOMMU_CACHE) + pgprot |= IMMAIR_ATTR_IDX_WBRWA << ARM_VMSA_PTE_ATTRINDX_SHIFT; + + if (prot & IOMMU_EXEC) + pgprot &= ~ARM_VMSA_PTE_XN; + else if (!(prot & (IOMMU_READ | IOMMU_WRITE))) + /* If no access create a faulting entry to avoid TLB fills. */ + pgprot &= ~ARM_VMSA_PTE_PAGE; + + return pgprot; } -static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, - unsigned long addr, unsigned long end, - phys_addr_t phys, int prot) +static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, + unsigned long iova, unsigned long pfn, + size_t size, int prot) { - unsigned long next; - pmd_t *pmd; - int ret; + pteval_t pteval = ipmmu_page_prot(prot, ARM_VMSA_PTE_PAGE); + unsigned int num_ptes = 1; + pte_t *pte, *start; + unsigned int i; - if (pud_none(*pud)) { - pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); - if (!pmd) - return -ENOMEM; + pte = ipmmu_alloc_pte(mmu, pmd, iova); + if (!pte) + return -ENOMEM; + + start = pte; - ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE); - *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE); - ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); + /* + * Install the page table entries. We can be called both for a single + * page or for a block of 16 physically contiguous pages. In the latter + * case set the PTE contiguous hint. + */ + if (size == SZ_64K) { + pteval |= ARM_VMSA_PTE_CONT; + num_ptes = ARM_VMSA_PTE_CONT_ENTRIES; + } - pmd += pmd_index(addr); - } else - pmd = pmd_offset(pud, addr); + for (i = num_ptes; i; --i) + *pte++ = pfn_pte(pfn++, __pgprot(pteval)); - do { - next = pmd_addr_end(addr, end); - ret = ipmmu_alloc_init_pte(mmu, pmd, addr, end, phys, prot); - phys += next - addr; - } while (pmd++, addr = next, addr < end); + ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * num_ptes); - return ret; + return 0; } static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, @@ -638,7 +617,8 @@ static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, struct ipmmu_vmsa_device *mmu = domain->mmu; pgd_t *pgd = domain->pgd; unsigned long flags; - unsigned long end; + unsigned long pfn; + pmd_t *pmd; int ret; if (!pgd) @@ -650,26 +630,25 @@ static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, if (paddr & ~((1ULL << 40) - 1)) return -ERANGE; - spin_lock_irqsave(&domain->lock, flags); - + pfn = __phys_to_pfn(paddr); pgd += pgd_index(iova); - end = iova + size; - do { - unsigned long next = pgd_addr_end(iova, end); + /* Update the page tables. */ + spin_lock_irqsave(&domain->lock, flags); - ret = ipmmu_alloc_init_pmd(mmu, (pud_t *)pgd, iova, next, paddr, - prot); - if (ret) - break; + pmd = ipmmu_alloc_pmd(mmu, pgd, iova); + if (!pmd) { + ret = -ENOMEM; + goto done; + } - paddr += next - iova; - iova = next; - } while (pgd++, iova != end); + ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot); +done: spin_unlock_irqrestore(&domain->lock, flags); - ipmmu_tlb_invalidate(domain); + if (!ret) + ipmmu_tlb_invalidate(domain); return ret; } @@ -951,7 +930,7 @@ static struct iommu_ops ipmmu_ops = { .iova_to_phys = ipmmu_iova_to_phys, .add_device = ipmmu_add_device, .remove_device = ipmmu_remove_device, - .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K, + .pgsize_bitmap = SZ_64K | SZ_4K, }; /* ----------------------------------------------------------------------------- -- cgit v0.10.2 From dda7c2e4d3f160aecf21ca56d73ceb0ff6ede587 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:49 +0200 Subject: iommu/ipmmu-vmsa: Support 2MB mappings Add support for 2MB block mappings at the PMD level. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 87703c3..7f05220 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -479,7 +479,7 @@ static void ipmmu_free_pmds(pud_t *pud) unsigned int i; for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { - if (pmd_none(*pmd)) + if (!pmd_table(*pmd)) continue; ipmmu_free_ptes(pmd); @@ -610,6 +610,18 @@ static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, return 0; } +static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, + unsigned long iova, unsigned long pfn, + int prot) +{ + pmdval_t pmdval = ipmmu_page_prot(prot, PMD_TYPE_SECT); + + *pmd = pfn_pmd(pfn, __pgprot(pmdval)); + ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); + + return 0; +} + static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) @@ -642,7 +654,18 @@ static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, goto done; } - ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot); + switch (size) { + case SZ_2M: + ret = ipmmu_alloc_init_pmd(mmu, pmd, iova, pfn, prot); + break; + case SZ_64K: + case SZ_4K: + ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot); + break; + default: + ret = -EINVAL; + break; + } done: spin_unlock_irqrestore(&domain->lock, flags); @@ -792,6 +815,9 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, if (pmd_none(pmd)) return 0; + if (pmd_sect(pmd)) + return __pfn_to_phys(pmd_pfn(pmd)) | (iova & ~PMD_MASK); + pte = *(pmd_page_vaddr(pmd) + pte_index(iova)); if (pte_none(pte)) return 0; @@ -930,7 +956,7 @@ static struct iommu_ops ipmmu_ops = { .iova_to_phys = ipmmu_iova_to_phys, .add_device = ipmmu_add_device, .remove_device = ipmmu_remove_device, - .pgsize_bitmap = SZ_64K | SZ_4K, + .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K, }; /* ----------------------------------------------------------------------------- -- cgit v0.10.2 From bec0ca0333d7030ef5b9afa7e4fd95b25d3ec338 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:50 +0200 Subject: iommu/ipmmu-vmsa: Remove stage 2 PTE bits definitions We don't support stage 2 translation yet. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 7f05220..76f2550 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -201,14 +201,6 @@ static LIST_HEAD(ipmmu_devices); #define ARM_VMSA_PTE_ATTRINDX_SHIFT 2 #define ARM_VMSA_PTE_nG (((pteval_t)1) << 11) -/* Stage-2 PTE */ -#define ARM_VMSA_PTE_HAP_FAULT (((pteval_t)0) << 6) -#define ARM_VMSA_PTE_HAP_READ (((pteval_t)1) << 6) -#define ARM_VMSA_PTE_HAP_WRITE (((pteval_t)2) << 6) -#define ARM_VMSA_PTE_MEMATTR_OIWB (((pteval_t)0xf) << 2) -#define ARM_VMSA_PTE_MEMATTR_NC (((pteval_t)0x5) << 2) -#define ARM_VMSA_PTE_MEMATTR_DEV (((pteval_t)0x1) << 2) - #define ARM_VMSA_PTE_CONT_ENTRIES 16 #define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES) -- cgit v0.10.2 From 004c5b32fa5e1d0b31d9b4b77160504e416a0ceb Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 15 May 2014 12:40:51 +0200 Subject: iommu/ipmmu-vmsa: Support clearing mappings Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 76f2550..95b819a 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -192,14 +192,22 @@ static LIST_HEAD(ipmmu_devices); #define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8) #define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8) #define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8) +#define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8) #define ARM_VMSA_PTE_NS (((pteval_t)1) << 5) #define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0) /* Stage-1 PTE */ +#define ARM_VMSA_PTE_nG (((pteval_t)1) << 11) #define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6) #define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6) +#define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6) +#define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2) #define ARM_VMSA_PTE_ATTRINDX_SHIFT 2 -#define ARM_VMSA_PTE_nG (((pteval_t)1) << 11) + +#define ARM_VMSA_PTE_ATTRS_MASK \ + (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \ + ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \ + ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK) #define ARM_VMSA_PTE_CONT_ENTRIES 16 #define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES) @@ -614,7 +622,7 @@ static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, return 0; } -static int ipmmu_handle_mapping(struct ipmmu_vmsa_domain *domain, +static int ipmmu_create_mapping(struct ipmmu_vmsa_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -668,6 +676,180 @@ done: return ret; } +static void ipmmu_clear_pud(struct ipmmu_vmsa_device *mmu, pud_t *pud) +{ + /* Free the page table. */ + pgtable_t table = pud_pgtable(*pud); + __free_page(table); + + /* Clear the PUD. */ + *pud = __pud(0); + ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); +} + +static void ipmmu_clear_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, + pmd_t *pmd) +{ + unsigned int i; + + /* Free the page table. */ + if (pmd_table(*pmd)) { + pgtable_t table = pmd_pgtable(*pmd); + __free_page(table); + } + + /* Clear the PMD. */ + *pmd = __pmd(0); + ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); + + /* Check whether the PUD is still needed. */ + pmd = pmd_offset(pud, 0); + for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { + if (!pmd_none(pmd[i])) + return; + } + + /* Clear the parent PUD. */ + ipmmu_clear_pud(mmu, pud); +} + +static void ipmmu_clear_pte(struct ipmmu_vmsa_device *mmu, pud_t *pud, + pmd_t *pmd, pte_t *pte, unsigned int num_ptes) +{ + unsigned int i; + + /* Clear the PTE. */ + for (i = num_ptes; i; --i) + pte[i-1] = __pte(0); + + ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * num_ptes); + + /* Check whether the PMD is still needed. */ + pte = pte_offset_kernel(pmd, 0); + for (i = 0; i < IPMMU_PTRS_PER_PTE; ++i) { + if (!pte_none(pte[i])) + return; + } + + /* Clear the parent PMD. */ + ipmmu_clear_pmd(mmu, pud, pmd); +} + +static int ipmmu_split_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd) +{ + pte_t *pte, *start; + pteval_t pteval; + unsigned long pfn; + unsigned int i; + + pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); + if (!pte) + return -ENOMEM; + + /* Copy the PMD attributes. */ + pteval = (pmd_val(*pmd) & ARM_VMSA_PTE_ATTRS_MASK) + | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_PAGE; + + pfn = pmd_pfn(*pmd); + start = pte; + + for (i = IPMMU_PTRS_PER_PTE; i; --i) + *pte++ = pfn_pte(pfn++, __pgprot(pteval)); + + ipmmu_flush_pgtable(mmu, start, PAGE_SIZE); + *pmd = __pmd(__pa(start) | PMD_NSTABLE | PMD_TYPE_TABLE); + ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); + + return 0; +} + +static void ipmmu_split_pte(struct ipmmu_vmsa_device *mmu, pte_t *pte) +{ + unsigned int i; + + for (i = ARM_VMSA_PTE_CONT_ENTRIES; i; --i) + pte[i-1] = __pte(pte_val(*pte) & ~ARM_VMSA_PTE_CONT); + + ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * ARM_VMSA_PTE_CONT_ENTRIES); +} + +static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain, + unsigned long iova, size_t size) +{ + struct ipmmu_vmsa_device *mmu = domain->mmu; + unsigned long flags; + pgd_t *pgd = domain->pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + int ret = 0; + + if (!pgd) + return -EINVAL; + + if (size & ~PAGE_MASK) + return -EINVAL; + + pgd += pgd_index(iova); + pud = (pud_t *)pgd; + + spin_lock_irqsave(&domain->lock, flags); + + /* If there's no PUD or PMD we're done. */ + if (pud_none(*pud)) + goto done; + + pmd = pmd_offset(pud, iova); + if (pmd_none(*pmd)) + goto done; + + /* + * When freeing a 2MB block just clear the PMD. In the unlikely case the + * block is mapped as individual pages this will free the corresponding + * PTE page table. + */ + if (size == SZ_2M) { + ipmmu_clear_pmd(mmu, pud, pmd); + goto done; + } + + /* + * If the PMD has been mapped as a section remap it as pages to allow + * freeing individual pages. + */ + if (pmd_sect(*pmd)) + ipmmu_split_pmd(mmu, pmd); + + pte = pte_offset_kernel(pmd, iova); + + /* + * When freeing a 64kB block just clear the PTE entries. We don't have + * to care about the contiguous hint of the surrounding entries. + */ + if (size == SZ_64K) { + ipmmu_clear_pte(mmu, pud, pmd, pte, ARM_VMSA_PTE_CONT_ENTRIES); + goto done; + } + + /* + * If the PTE has been mapped with the contiguous hint set remap it and + * its surrounding PTEs to allow unmapping a single page. + */ + if (pte_val(*pte) & ARM_VMSA_PTE_CONT) + ipmmu_split_pte(mmu, pte); + + /* Clear the PTE. */ + ipmmu_clear_pte(mmu, pud, pmd, pte, 1); + +done: + spin_unlock_irqrestore(&domain->lock, flags); + + if (ret) + ipmmu_tlb_invalidate(domain); + + return 0; +} + /* ----------------------------------------------------------------------------- * IOMMU Operations */ @@ -768,7 +950,7 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, if (!domain) return -ENODEV; - return ipmmu_handle_mapping(domain, iova, paddr, size, prot); + return ipmmu_create_mapping(domain, iova, paddr, size, prot); } static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, @@ -777,7 +959,7 @@ static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, struct ipmmu_vmsa_domain *domain = io_domain->priv; int ret; - ret = ipmmu_handle_mapping(domain, iova, 0, size, 0); + ret = ipmmu_clear_mapping(domain, iova, size); return ret ? 0 : size; } -- cgit v0.10.2 From b87d2d7c0b3507b6b17590d741b5be01e8a21b84 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 20 May 2014 23:18:22 +0200 Subject: iommu/amd: Don't access IOMMUv2 state_table directly This is a preparation for converting the state_table into a state_list. Signed-off-by: Joerg Roedel Tested-by: Jay Cornwall diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 203b2e6..5db3675 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -111,13 +111,18 @@ static u16 device_id(struct pci_dev *pdev) return devid; } +static struct device_state *__get_device_state(u16 devid) +{ + return state_table[devid]; +} + static struct device_state *get_device_state(u16 devid) { struct device_state *dev_state; unsigned long flags; spin_lock_irqsave(&state_lock, flags); - dev_state = state_table[devid]; + dev_state = __get_device_state(devid); if (dev_state != NULL) atomic_inc(&dev_state->count); spin_unlock_irqrestore(&state_lock, flags); @@ -841,7 +846,7 @@ void amd_iommu_free_device(struct pci_dev *pdev) spin_lock_irqsave(&state_lock, flags); - dev_state = state_table[devid]; + dev_state = __get_device_state(devid); if (dev_state == NULL) { spin_unlock_irqrestore(&state_lock, flags); return; @@ -874,7 +879,7 @@ int amd_iommu_set_invalid_ppr_cb(struct pci_dev *pdev, spin_lock_irqsave(&state_lock, flags); ret = -EINVAL; - dev_state = state_table[devid]; + dev_state = __get_device_state(devid); if (dev_state == NULL) goto out_unlock; @@ -905,7 +910,7 @@ int amd_iommu_set_invalidate_ctx_cb(struct pci_dev *pdev, spin_lock_irqsave(&state_lock, flags); ret = -EINVAL; - dev_state = state_table[devid]; + dev_state = __get_device_state(devid); if (dev_state == NULL) goto out_unlock; -- cgit v0.10.2 From 741669c765c45cace9bd630f7c8c8047c8b33567 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 20 May 2014 23:18:23 +0200 Subject: iommu/amd: Convert IOMMUv2 state_table into state_list The state_table consumes 512kb of memory and is only sparsly populated. Convert it into a list to save memory. There should be no measurable performance impact. Signed-off-by: Joerg Roedel Tested-by: Jay Cornwall diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 5db3675..569b6a0 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -56,6 +56,8 @@ struct pasid_state { }; struct device_state { + struct list_head list; + u16 devid; atomic_t count; struct pci_dev *pdev; struct pasid_state **states; @@ -81,7 +83,7 @@ struct fault { u16 flags; }; -static struct device_state **state_table; +static LIST_HEAD(state_list); static spinlock_t state_lock; /* List and lock for all pasid_states */ @@ -113,7 +115,14 @@ static u16 device_id(struct pci_dev *pdev) static struct device_state *__get_device_state(u16 devid) { - return state_table[devid]; + struct device_state *dev_state; + + list_for_each_entry(dev_state, &state_list, list) { + if (dev_state->devid == devid) + return dev_state; + } + + return NULL; } static struct device_state *get_device_state(u16 devid) @@ -778,7 +787,8 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) spin_lock_init(&dev_state->lock); init_waitqueue_head(&dev_state->wq); - dev_state->pdev = pdev; + dev_state->pdev = pdev; + dev_state->devid = devid; tmp = pasids; for (dev_state->pasid_levels = 0; (tmp - 1) & ~0x1ff; tmp >>= 9) @@ -808,13 +818,13 @@ int amd_iommu_init_device(struct pci_dev *pdev, int pasids) spin_lock_irqsave(&state_lock, flags); - if (state_table[devid] != NULL) { + if (__get_device_state(devid) != NULL) { spin_unlock_irqrestore(&state_lock, flags); ret = -EBUSY; goto out_free_domain; } - state_table[devid] = dev_state; + list_add_tail(&dev_state->list, &state_list); spin_unlock_irqrestore(&state_lock, flags); @@ -852,7 +862,7 @@ void amd_iommu_free_device(struct pci_dev *pdev) return; } - state_table[devid] = NULL; + list_del(&dev_state->list); spin_unlock_irqrestore(&state_lock, flags); @@ -927,7 +937,6 @@ EXPORT_SYMBOL(amd_iommu_set_invalidate_ctx_cb); static int __init amd_iommu_v2_init(void) { - size_t state_table_size; int ret; pr_info("AMD IOMMUv2 driver by Joerg Roedel \n"); @@ -943,16 +952,10 @@ static int __init amd_iommu_v2_init(void) spin_lock_init(&state_lock); - state_table_size = MAX_DEVICES * sizeof(struct device_state *); - state_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(state_table_size)); - if (state_table == NULL) - return -ENOMEM; - ret = -ENOMEM; iommu_wq = create_workqueue("amd_iommu_v2"); if (iommu_wq == NULL) - goto out_free; + goto out; ret = -ENOMEM; empty_page_table = (u64 *)get_zeroed_page(GFP_KERNEL); @@ -967,16 +970,13 @@ static int __init amd_iommu_v2_init(void) out_destroy_wq: destroy_workqueue(iommu_wq); -out_free: - free_pages((unsigned long)state_table, get_order(state_table_size)); - +out: return ret; } static void __exit amd_iommu_v2_exit(void) { struct device_state *dev_state; - size_t state_table_size; int i; if (!amd_iommu_v2_supported()) @@ -1005,9 +1005,6 @@ static void __exit amd_iommu_v2_exit(void) destroy_workqueue(iommu_wq); - state_table_size = MAX_DEVICES * sizeof(struct device_state *); - free_pages((unsigned long)state_table, get_order(state_table_size)); - free_page((unsigned long)empty_page_table); } -- cgit v0.10.2 From a40d4c67d7dec4e9b04e2fe1105072a843fda5a9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 20 May 2014 23:18:24 +0200 Subject: iommu/amd: Implement mmu_notifier_release call-back Since mmu_notifier call-backs can sleep (because they use SRCU now) we can use them to tear down PASID mappings. This allows us to finally remove the hack to use the task_exit notifier from oprofile to get notified when a process dies. Signed-off-by: Joerg Roedel Tested-by: Jay Cornwall diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 569b6a0..09d342b 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -101,7 +101,6 @@ static u64 *empty_page_table; static void free_pasid_states(struct device_state *dev_state); static void unbind_pasid(struct device_state *dev_state, int pasid); -static int task_exit(struct notifier_block *nb, unsigned long e, void *data); static u16 device_id(struct pci_dev *pdev) { @@ -172,10 +171,6 @@ static void put_device_state_wait(struct device_state *dev_state) free_device_state(dev_state); } -static struct notifier_block profile_nb = { - .notifier_call = task_exit, -}; - static void link_pasid_state(struct pasid_state *pasid_state) { spin_lock(&ps_lock); @@ -393,7 +388,12 @@ static void free_pasid_states(struct device_state *dev_state) continue; put_pasid_state(pasid_state); - unbind_pasid(dev_state, i); + + /* + * This will call the mn_release function and + * unbind the PASID + */ + mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm); } if (dev_state->pasid_levels == 2) @@ -475,7 +475,24 @@ static void mn_invalidate_range_end(struct mmu_notifier *mn, __pa(pasid_state->mm->pgd)); } +static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) +{ + struct pasid_state *pasid_state; + struct device_state *dev_state; + + might_sleep(); + + pasid_state = mn_to_state(mn); + dev_state = pasid_state->device_state; + + if (pasid_state->device_state->inv_ctx_cb) + dev_state->inv_ctx_cb(dev_state->pdev, pasid_state->pasid); + + unbind_pasid(dev_state, pasid_state->pasid); +} + static struct mmu_notifier_ops iommu_mn = { + .release = mn_release, .clear_flush_young = mn_clear_flush_young, .change_pte = mn_change_pte, .invalidate_page = mn_invalidate_page, @@ -620,53 +637,6 @@ static struct notifier_block ppr_nb = { .notifier_call = ppr_notifier, }; -static int task_exit(struct notifier_block *nb, unsigned long e, void *data) -{ - struct pasid_state *pasid_state; - struct task_struct *task; - - task = data; - - /* - * Using this notifier is a hack - but there is no other choice - * at the moment. What I really want is a sleeping notifier that - * is called when an MM goes down. But such a notifier doesn't - * exist yet. The notifier needs to sleep because it has to make - * sure that the device does not use the PASID and the address - * space anymore before it is destroyed. This includes waiting - * for pending PRI requests to pass the workqueue. The - * MMU-Notifiers would be a good fit, but they use RCU and so - * they are not allowed to sleep. Lets see how we can solve this - * in a more intelligent way in the future. - */ -again: - spin_lock(&ps_lock); - list_for_each_entry(pasid_state, &pasid_state_list, list) { - struct device_state *dev_state; - int pasid; - - if (pasid_state->task != task) - continue; - - /* Drop Lock and unbind */ - spin_unlock(&ps_lock); - - dev_state = pasid_state->device_state; - pasid = pasid_state->pasid; - - if (pasid_state->device_state->inv_ctx_cb) - dev_state->inv_ctx_cb(dev_state->pdev, pasid); - - unbind_pasid(dev_state, pasid); - - /* Task may be in the list multiple times */ - goto again; - } - spin_unlock(&ps_lock); - - return NOTIFY_OK; -} - int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid, struct task_struct *task) { @@ -741,6 +711,7 @@ EXPORT_SYMBOL(amd_iommu_bind_pasid); void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid) { + struct pasid_state *pasid_state; struct device_state *dev_state; u16 devid; @@ -757,7 +728,17 @@ void amd_iommu_unbind_pasid(struct pci_dev *pdev, int pasid) if (pasid < 0 || pasid >= dev_state->max_pasids) goto out; - unbind_pasid(dev_state, pasid); + pasid_state = get_pasid_state(dev_state, pasid); + if (pasid_state == NULL) + goto out; + /* + * Drop reference taken here. We are safe because we still hold + * the reference taken in the amd_iommu_bind_pasid function. + */ + put_pasid_state(pasid_state); + + /* This will call the mn_release function and unbind the PASID */ + mmu_notifier_unregister(&pasid_state->mn, pasid_state->mm); out: put_device_state(dev_state); @@ -963,7 +944,6 @@ static int __init amd_iommu_v2_init(void) goto out_destroy_wq; amd_iommu_register_ppr_notifier(&ppr_nb); - profile_event_register(PROFILE_TASK_EXIT, &profile_nb); return 0; @@ -982,7 +962,6 @@ static void __exit amd_iommu_v2_exit(void) if (!amd_iommu_v2_supported()) return; - profile_event_unregister(PROFILE_TASK_EXIT, &profile_nb); amd_iommu_unregister_ppr_notifier(&ppr_nb); flush_workqueue(iommu_wq); -- cgit v0.10.2 From 9163b9013542a688fe4152985118a9c46e2d255d Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 20 May 2014 23:18:25 +0200 Subject: iommu/amd: Remove IOMMUv2 pasid_state_list This list was only used for the task_exit notifier function. Now that it is gone we can remove it. Signed-off-by: Joerg Roedel Tested-by: Jay Cornwall diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 09d342b..c65f3ec 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -86,10 +86,6 @@ struct fault { static LIST_HEAD(state_list); static spinlock_t state_lock; -/* List and lock for all pasid_states */ -static LIST_HEAD(pasid_state_list); -static DEFINE_SPINLOCK(ps_lock); - static struct workqueue_struct *iommu_wq; /* @@ -171,25 +167,6 @@ static void put_device_state_wait(struct device_state *dev_state) free_device_state(dev_state); } -static void link_pasid_state(struct pasid_state *pasid_state) -{ - spin_lock(&ps_lock); - list_add_tail(&pasid_state->list, &pasid_state_list); - spin_unlock(&ps_lock); -} - -static void __unlink_pasid_state(struct pasid_state *pasid_state) -{ - list_del(&pasid_state->list); -} - -static void unlink_pasid_state(struct pasid_state *pasid_state) -{ - spin_lock(&ps_lock); - __unlink_pasid_state(pasid_state); - spin_unlock(&ps_lock); -} - /* Must be called under dev_state->lock */ static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state, int pasid, bool alloc) @@ -346,7 +323,6 @@ static void unbind_pasid(struct device_state *dev_state, int pasid) if (pasid_state == NULL) return; - unlink_pasid_state(pasid_state); __unbind_pasid(pasid_state); put_pasid_state_wait(pasid_state); /* Reference taken in this function */ } @@ -689,8 +665,6 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid, if (ret) goto out_clear_state; - link_pasid_state(pasid_state); - return 0; out_clear_state: -- cgit v0.10.2 From e79df31c60ea79954e854616da233e1b8c6475ab Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 20 May 2014 23:18:26 +0200 Subject: iommu/amd: Handle parallel invalidate_range_start/end calls correctly Add a counter to the pasid_state so that we do not restore the original page-table before all invalidate_range_start to invalidate_range_end sections have finished. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index c65f3ec..d4daa05 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -45,6 +45,8 @@ struct pri_queue { struct pasid_state { struct list_head list; /* For global state-list */ atomic_t count; /* Reference count */ + atomic_t mmu_notifier_count; /* Counting nested mmu_notifier + calls */ struct task_struct *task; /* Task bound to this PASID */ struct mm_struct *mm; /* mm_struct for the faults */ struct mmu_notifier mn; /* mmu_otifier handle */ @@ -433,8 +435,11 @@ static void mn_invalidate_range_start(struct mmu_notifier *mn, pasid_state = mn_to_state(mn); dev_state = pasid_state->device_state; - amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid, - __pa(empty_page_table)); + if (atomic_add_return(1, &pasid_state->mmu_notifier_count) == 1) { + amd_iommu_domain_set_gcr3(dev_state->domain, + pasid_state->pasid, + __pa(empty_page_table)); + } } static void mn_invalidate_range_end(struct mmu_notifier *mn, @@ -447,8 +452,11 @@ static void mn_invalidate_range_end(struct mmu_notifier *mn, pasid_state = mn_to_state(mn); dev_state = pasid_state->device_state; - amd_iommu_domain_set_gcr3(dev_state->domain, pasid_state->pasid, - __pa(pasid_state->mm->pgd)); + if (atomic_dec_and_test(&pasid_state->mmu_notifier_count)) { + amd_iommu_domain_set_gcr3(dev_state->domain, + pasid_state->pasid, + __pa(pasid_state->mm->pgd)); + } } static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) @@ -642,6 +650,7 @@ int amd_iommu_bind_pasid(struct pci_dev *pdev, int pasid, goto out; atomic_set(&pasid_state->count, 1); + atomic_set(&pasid_state->mmu_notifier_count, 0); init_waitqueue_head(&pasid_state->wq); spin_lock_init(&pasid_state->lock); -- cgit v0.10.2 From ecef115d4540e3ef73551a21fb0622270a825110 Mon Sep 17 00:00:00 2001 From: Vaughan Cao Date: Fri, 16 May 2014 15:39:40 +0800 Subject: iommu/amd: Remove duplicate checking code amd_iommu_rlookup_table[devid] != NULL is already guaranteed by check_device called before, it's fine to attach device at this point. Signed-off-by: Vaughan Cao Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 57068e8..bd8003d9 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3514,12 +3514,6 @@ int __init amd_iommu_init_passthrough(void) dev_data = get_dev_data(&dev->dev); dev_data->passthrough = true; - devid = get_device_id(&dev->dev); - - iommu = amd_iommu_rlookup_table[devid]; - if (!iommu) - continue; - attach_device(&dev->dev, pt_domain); } -- cgit v0.10.2 From d6a71bf79da03b835f8df5430eaee6fc46df1038 Mon Sep 17 00:00:00 2001 From: Rickard Strandqvist Date: Sat, 17 May 2014 19:16:44 +0200 Subject: iommu: fsl_pamu.c: Fix for possible null pointer dereference There is otherwise a risk of a possible null pointer dereference. Was largely found by using a static code analysis program called cppcheck. Signed-off-by: Rickard Strandqvist Reviewed-by: Bharat Bhushan Acked-by: Varun Sethi Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index cba0498..b99dd88 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -592,8 +592,7 @@ found_cpu_node: /* advance to next node in cache hierarchy */ node = of_find_node_by_phandle(*prop); if (!node) { - pr_debug("Invalid node for cache hierarchy %s\n", - node->full_name); + pr_debug("Invalid node for cache hierarchy\n"); return ~(u32)0; } } -- cgit v0.10.2 From b28165dae0157869d4a701a6d513d768252938f2 Mon Sep 17 00:00:00 2001 From: Sachin Kamat Date: Thu, 22 May 2014 09:50:54 +0530 Subject: iommu/exynos: Remove invalid symbol dependency EXYNOS_DEV_SYSMMU symbol is not defined anywhere and prevents building the Exynos driver. Remove it. Signed-off-by: Sachin Kamat Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index df56e4c..b067a62 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -178,7 +178,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && EXYNOS_DEV_SYSMMU + depends on ARCH_EXYNOS select IOMMU_API help Support for the IOMMU(System MMU) of Samsung Exynos application -- cgit v0.10.2 From 5455d700e35c914c47cc56e77ef0fb74946aa4b5 Mon Sep 17 00:00:00 2001 From: Sachin Kamat Date: Thu, 22 May 2014 09:50:55 +0530 Subject: iommu/exynos: Fix trivial typo Fix typo and add missing punctuation. Signed-off-by: Sachin Kamat Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index b067a62..422807f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -181,10 +181,10 @@ config EXYNOS_IOMMU depends on ARCH_EXYNOS select IOMMU_API help - Support for the IOMMU(System MMU) of Samsung Exynos application - processor family. This enables H/W multimedia accellerators to see - non-linear physical memory chunks as a linear memory in their - address spaces + Support for the IOMMU (System MMU) of Samsung Exynos application + processor family. This enables H/W multimedia accelerators to see + non-linear physical memory chunks as linear memory in their + address space. If unsure, say N here. @@ -193,9 +193,9 @@ config EXYNOS_IOMMU_DEBUG depends on EXYNOS_IOMMU help Select this to see the detailed log message that shows what - happens in the IOMMU driver + happens in the IOMMU driver. - Say N unless you need kernel log message for IOMMU debugging + Say N unless you need kernel log message for IOMMU debugging. config SHMOBILE_IPMMU bool -- cgit v0.10.2 From 365409db5d8811315297b8d03097d363683bfb5b Mon Sep 17 00:00:00 2001 From: Sachin Kamat Date: Thu, 22 May 2014 09:50:56 +0530 Subject: iommu/exynos: Fix checkpatch warning Silences the following type of warnings: WARNING: Missing a blank line after declarations Signed-off-by: Sachin Kamat Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 09f69b1..99054d2 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -268,6 +268,7 @@ static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, sysmmu_iova_t iova, unsigned int num_inv) { unsigned int i; + for (i = 0; i < num_inv; i++) { __raw_writel((iova & SPAGE_MASK) | 1, sfrbase + REG_MMU_FLUSH_ENTRY); @@ -878,6 +879,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, */ if (need_flush_flpd_cache) { struct exynos_iommu_owner *owner; + spin_lock(&priv->lock); list_for_each_entry(owner, &priv->clients, client) sysmmu_tlb_invalidate_flpdcache( @@ -941,6 +943,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, *pgcnt -= 1; } else { /* size == LPAGE_SIZE */ int i; + for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { if (WARN_ON(!lv2ent_fault(pent))) { if (i > 0) -- cgit v0.10.2 From 720b0cef715ab97b21b33e7f3c328e2863411cab Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Mon, 26 May 2014 13:07:01 +0200 Subject: arm/ipmmu-vmsa: Fix compile error The function arm_iommu_create_mapping lost the order parameter. Remove it from this IOMMU driver too to make it build. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 95b819a..53cde08 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -1087,7 +1087,7 @@ static int ipmmu_add_device(struct device *dev) struct dma_iommu_mapping *mapping; mapping = arm_iommu_create_mapping(&platform_bus_type, - SZ_1G, SZ_2G, 0); + SZ_1G, SZ_2G); if (IS_ERR(mapping)) { dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); return PTR_ERR(mapping); -- cgit v0.10.2 From 3dbc26085383570935e475aa08b67d5cc05be390 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 30 May 2014 20:11:20 +0200 Subject: iommu/amd: Fix recently introduced compile warnings Fix two compile warnings about unused variables introduced by commit ecef115. Reported-by: Paul Bolle Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index bd8003d9..4aec6a2 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3499,8 +3499,6 @@ int __init amd_iommu_init_passthrough(void) { struct iommu_dev_data *dev_data; struct pci_dev *dev = NULL; - struct amd_iommu *iommu; - u16 devid; int ret; ret = alloc_passthrough_domain(); -- cgit v0.10.2 From afba256b14644e4ca5f9e84672e61b14cb71c40e Mon Sep 17 00:00:00 2001 From: Kefeng Wang Date: Tue, 27 May 2014 18:18:42 +0800 Subject: iommu/msm: Use devm_ioremap_resource to simplify code Use devm_ioremap_resource() to make the code simpler, drop unused variable, redundant return value check, and error-handing code. Signed-off-by: Kefeng Wang Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c index 08ba497..61def7cb 100644 --- a/drivers/iommu/msm_iommu_dev.c +++ b/drivers/iommu/msm_iommu_dev.c @@ -127,13 +127,12 @@ static void msm_iommu_reset(void __iomem *base, int ncb) static int msm_iommu_probe(struct platform_device *pdev) { - struct resource *r, *r2; + struct resource *r; struct clk *iommu_clk; struct clk *iommu_pclk; struct msm_iommu_drvdata *drvdata; struct msm_iommu_dev *iommu_dev = pdev->dev.platform_data; void __iomem *regs_base; - resource_size_t len; int ret, irq, par; if (pdev->id == -1) { @@ -178,35 +177,16 @@ static int msm_iommu_probe(struct platform_device *pdev) iommu_clk = NULL; r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase"); - - if (!r) { - ret = -ENODEV; - goto fail_clk; - } - - len = resource_size(r); - - r2 = request_mem_region(r->start, len, r->name); - if (!r2) { - pr_err("Could not request memory region: start=%p, len=%d\n", - (void *) r->start, len); - ret = -EBUSY; + regs_base = devm_ioremap_resource(&pdev->dev, r); + if (IS_ERR(regs_base)) { + ret = PTR_ERR(regs_base); goto fail_clk; } - regs_base = ioremap(r2->start, len); - - if (!regs_base) { - pr_err("Could not ioremap: start=%p, len=%d\n", - (void *) r2->start, len); - ret = -EBUSY; - goto fail_mem; - } - irq = platform_get_irq_byname(pdev, "secure_irq"); if (irq < 0) { ret = -ENODEV; - goto fail_io; + goto fail_clk; } msm_iommu_reset(regs_base, iommu_dev->ncb); @@ -222,14 +202,14 @@ static int msm_iommu_probe(struct platform_device *pdev) if (!par) { pr_err("%s: Invalid PAR value detected\n", iommu_dev->name); ret = -ENODEV; - goto fail_io; + goto fail_clk; } ret = request_irq(irq, msm_iommu_fault_handler, 0, "msm_iommu_secure_irpt_handler", drvdata); if (ret) { pr_err("Request IRQ %d failed with ret=%d\n", irq, ret); - goto fail_io; + goto fail_clk; } @@ -250,10 +230,6 @@ static int msm_iommu_probe(struct platform_device *pdev) clk_disable(iommu_pclk); return 0; -fail_io: - iounmap(regs_base); -fail_mem: - release_mem_region(r->start, len); fail_clk: if (iommu_clk) { clk_disable(iommu_clk); -- cgit v0.10.2