From b8f80bffd51f4ef029051d6898d9c2e3e5637dc3 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Fri, 14 Mar 2014 14:00:56 +0100 Subject: iommu/ipmmu-vmsa: Cleanup failures of ARM mapping creation or attachment The ARM IOMMU mapping needs to be released when attaching the device fails. Add arm_iommu_release_mapping() to the error code path. This is safe to call with a NULL mapping, so no specific check is needed. Cleanup is also missing when failing to create a mapping. Jump to the error code path in that case instead of returning immediately. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 7486931..034293c 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -1090,7 +1090,8 @@ static int ipmmu_add_device(struct device *dev) SZ_1G, SZ_2G); if (IS_ERR(mapping)) { dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); - return PTR_ERR(mapping); + ret = PTR_ERR(mapping); + goto error; } mmu->mapping = mapping; @@ -1106,6 +1107,7 @@ static int ipmmu_add_device(struct device *dev) return 0; error: + arm_iommu_release_mapping(mmu->mapping); kfree(dev->archdata.iommu); dev->archdata.iommu = NULL; iommu_group_remove_device(dev); -- cgit v0.10.2 From 22463cab3f96baf6b568200a35c7648438eea7ff Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 24 Jul 2014 15:34:54 +0200 Subject: iommu/ipmmu-vmsa: Flush P[UM]D entry before freeing the child page table When clearing PUD or PMD entries the child page table (if any) is freed and the PUD or PMD entry is then cleared. This result in a small race condition window during which a free page table could be accessed by the IPMMU. Fix it by clearing and flushing the PUD or PMD entry before freeing the child page table. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 034293c..a32d237 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -678,30 +678,33 @@ done: static void ipmmu_clear_pud(struct ipmmu_vmsa_device *mmu, pud_t *pud) { - /* Free the page table. */ pgtable_t table = pud_pgtable(*pud); - __free_page(table); /* Clear the PUD. */ *pud = __pud(0); ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); + + /* Free the page table. */ + __free_page(table); } static void ipmmu_clear_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, pmd_t *pmd) { + pmd_t pmdval = *pmd; unsigned int i; - /* Free the page table. */ - if (pmd_table(*pmd)) { - pgtable_t table = pmd_pgtable(*pmd); - __free_page(table); - } - /* Clear the PMD. */ *pmd = __pmd(0); ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); + /* Free the page table. */ + if (pmd_table(pmdval)) { + pgtable_t table = pmd_pgtable(pmdval); + + __free_page(table); + } + /* Check whether the PUD is still needed. */ pmd = pmd_offset(pud, 0); for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { -- cgit v0.10.2 From 9eca0a5875403027e10d68dd162df9790d42839e Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 24 Jul 2014 15:34:54 +0200 Subject: iommu/ipmmu-vmsa: Invalidate TLB after unmapping The TLB must be invalidated after unmapping memory to remove stale TLB entries. this was supposed to be performed already, but a bug in the driver prevented the TLB invalidate function from being called. Fix it. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index a32d237..3a61103 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -785,7 +785,6 @@ static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain, pud_t *pud; pmd_t *pmd; pte_t *pte; - int ret = 0; if (!pgd) return -EINVAL; @@ -847,8 +846,7 @@ static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain, done: spin_unlock_irqrestore(&domain->lock, flags); - if (ret) - ipmmu_tlb_invalidate(domain); + ipmmu_tlb_invalidate(domain); return 0; } -- cgit v0.10.2 From 4a93f21d87e769477c0cb2b98d7e7911b8d37c03 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 17 Mar 2014 01:02:46 +0100 Subject: iommu/ipmmu-vmsa: Add device tree bindings documentation Signed-off-by: Laurent Pinchart Acked-by: Geert Uytterhoeven diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt new file mode 100644 index 0000000..cd29083 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt @@ -0,0 +1,41 @@ +* Renesas VMSA-Compatible IOMMU + +The IPMMU is an IOMMU implementation compatible with the ARM VMSA page tables. +It provides address translation for bus masters outside of the CPU, each +connected to the IPMMU through a port called micro-TLB. + + +Required Properties: + + - compatible: Must contain "renesas,ipmmu-vmsa". + - reg: Base address and size of the IPMMU registers. + - interrupts: Specifiers for the MMU fault interrupts. For instances that + support secure mode two interrupts must be specified, for non-secure and + secure mode, in that order. For instances that don't support secure mode a + single interrupt must be specified. + + - #iommu-cells: Must be 1. + +Each bus master connected to an IPMMU must reference the IPMMU in its device +node with the following property: + + - iommus: A reference to the IPMMU in two cells. The first cell is a phandle + to the IPMMU and the second cell the number of the micro-TLB that the + device is connected to. + + +Example: R8A7791 IPMMU-MX and VSP1-D0 bus master + + ipmmu_mx: mmu@fe951000 { + compatible = "renasas,ipmmu-vmsa"; + reg = <0 0xfe951000 0 0x1000>; + interrupts = <0 222 IRQ_TYPE_LEVEL_HIGH>, + <0 221 IRQ_TYPE_LEVEL_HIGH>; + #iommu-cells = <1>; + }; + + vsp1@fe928000 { + ... + iommus = <&ipmmu_mx 13>; + ... + }; -- cgit v0.10.2 From 275f5053c7786285d2f20d2dd12908f834c47ad8 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 17 Mar 2014 01:02:46 +0100 Subject: iommu/ipmmu-vmsa: Add device tree support Make platform data optional when the device is instantiated from DT and look up the micro-TLB number in the bus master DT node. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 3a61103..368c852 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +59,8 @@ static LIST_HEAD(ipmmu_devices); * Registers Definition */ +#define IM_NS_ALIAS_OFFSET 0x800 + #define IM_CTX_SIZE 0x40 #define IMCTR 0x0000 @@ -1002,16 +1005,33 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev) { - const struct ipmmu_vmsa_master *master = mmu->pdata->masters; - const char *devname = dev_name(dev); - unsigned int i; + struct of_phandle_args args; + int ret; + + if (mmu->pdata) { + const struct ipmmu_vmsa_master *master = mmu->pdata->masters; + const char *devname = dev_name(dev); + unsigned int i; - for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) { - if (strcmp(master->name, devname) == 0) - return master->utlb; + for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) { + if (strcmp(master->name, devname) == 0) + return master->utlb; + } + + return -1; } - return -1; + ret = of_parse_phandle_with_args(dev->of_node, "iommus", + "#iommu-cells", 0, &args); + if (ret < 0) + return -1; + + of_node_put(args.np); + + if (args.np != mmu->dev->of_node || args.args_count != 1) + return -1; + + return args.args[0]; } static int ipmmu_add_device(struct device *dev) @@ -1157,7 +1177,7 @@ static int ipmmu_probe(struct platform_device *pdev) int irq; int ret; - if (!pdev->dev.platform_data) { + if (!IS_ENABLED(CONFIG_OF) && !pdev->dev.platform_data) { dev_err(&pdev->dev, "missing platform data\n"); return -EINVAL; } @@ -1178,6 +1198,20 @@ static int ipmmu_probe(struct platform_device *pdev) if (IS_ERR(mmu->base)) return PTR_ERR(mmu->base); + /* + * The IPMMU has two register banks, for secure and non-secure modes. + * The bank mapped at the beginning of the IPMMU address space + * corresponds to the running mode of the CPU. When running in secure + * mode the non-secure register bank is also available at an offset. + * + * Secure mode operation isn't clearly documented and is thus currently + * not implemented in the driver. Furthermore, preliminary tests of + * non-secure operation with the main register bank were not successful. + * Offset the registers base unconditionally to point to the non-secure + * alias space for now. + */ + mmu->base += IM_NS_ALIAS_OFFSET; + irq = platform_get_irq(pdev, 0); if (irq < 0) { dev_err(&pdev->dev, "no IRQ found\n"); @@ -1223,9 +1257,14 @@ static int ipmmu_remove(struct platform_device *pdev) return 0; } +static const struct of_device_id ipmmu_of_ids[] = { + { .compatible = "renesas,ipmmu-vmsa", }, +}; + static struct platform_driver ipmmu_driver = { .driver = { .name = "ipmmu-vmsa", + .of_match_table = of_match_ptr(ipmmu_of_ids), }, .probe = ipmmu_probe, .remove = ipmmu_remove, -- cgit v0.10.2 From a166d31ee56e8fc56ce2497d8de9da5359f4ee41 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 24 Jul 2014 01:36:43 +0200 Subject: iommu/ipmmu-vmsa: Support multiple micro TLBs per device Devices such as the system DMA controller are connected to multiple micro TLBs of the same IOMMU. Support this. Selective enabling of micro TLBs based on runtime device usage isn't possible at the moment due to lack of support in the IOMMU and DMA mapping APIs. Support for devices connected to different IOMMUs is also unsupported for the same reason. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 368c852..5d080cf 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -47,7 +47,8 @@ struct ipmmu_vmsa_domain { struct ipmmu_vmsa_archdata { struct ipmmu_vmsa_device *mmu; - unsigned int utlb; + unsigned int *utlbs; + unsigned int num_utlbs; }; static DEFINE_SPINLOCK(ipmmu_devices_lock); @@ -900,6 +901,7 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, struct ipmmu_vmsa_device *mmu = archdata->mmu; struct ipmmu_vmsa_domain *domain = io_domain->priv; unsigned long flags; + unsigned int i; int ret = 0; if (!mmu) { @@ -928,7 +930,8 @@ static int ipmmu_attach_device(struct iommu_domain *io_domain, if (ret < 0) return ret; - ipmmu_utlb_enable(domain, archdata->utlb); + for (i = 0; i < archdata->num_utlbs; ++i) + ipmmu_utlb_enable(domain, archdata->utlbs[i]); return 0; } @@ -938,8 +941,10 @@ static void ipmmu_detach_device(struct iommu_domain *io_domain, { struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; struct ipmmu_vmsa_domain *domain = io_domain->priv; + unsigned int i; - ipmmu_utlb_disable(domain, archdata->utlb); + for (i = 0; i < archdata->num_utlbs; ++i) + ipmmu_utlb_disable(domain, archdata->utlbs[i]); /* * TODO: Optimize by disabling the context when no device is attached. @@ -1003,10 +1008,12 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); } -static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev) +static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev, + unsigned int **_utlbs) { - struct of_phandle_args args; - int ret; + unsigned int *utlbs; + unsigned int i; + int count; if (mmu->pdata) { const struct ipmmu_vmsa_master *master = mmu->pdata->masters; @@ -1014,32 +1021,64 @@ static int ipmmu_find_utlb(struct ipmmu_vmsa_device *mmu, struct device *dev) unsigned int i; for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) { - if (strcmp(master->name, devname) == 0) - return master->utlb; + if (strcmp(master->name, devname) == 0) { + utlbs = kmalloc(sizeof(*utlbs), GFP_KERNEL); + if (!utlbs) + return -ENOMEM; + + utlbs[0] = master->utlb; + + *_utlbs = utlbs; + return 1; + } } - return -1; + return -EINVAL; } - ret = of_parse_phandle_with_args(dev->of_node, "iommus", - "#iommu-cells", 0, &args); - if (ret < 0) - return -1; + count = of_count_phandle_with_args(dev->of_node, "iommus", + "#iommu-cells"); + if (count < 0) + return -EINVAL; + + utlbs = kcalloc(count, sizeof(*utlbs), GFP_KERNEL); + if (!utlbs) + return -ENOMEM; + + for (i = 0; i < count; ++i) { + struct of_phandle_args args; + int ret; + + ret = of_parse_phandle_with_args(dev->of_node, "iommus", + "#iommu-cells", i, &args); + if (ret < 0) + goto error; + + of_node_put(args.np); + + if (args.np != mmu->dev->of_node || args.args_count != 1) + goto error; + + utlbs[i] = args.args[0]; + } - of_node_put(args.np); + *_utlbs = utlbs; - if (args.np != mmu->dev->of_node || args.args_count != 1) - return -1; + return count; - return args.args[0]; +error: + kfree(utlbs); + return -EINVAL; } static int ipmmu_add_device(struct device *dev) { struct ipmmu_vmsa_archdata *archdata; struct ipmmu_vmsa_device *mmu; - struct iommu_group *group; - int utlb = -1; + struct iommu_group *group = NULL; + unsigned int *utlbs = NULL; + unsigned int i; + int num_utlbs = 0; int ret; if (dev->archdata.iommu) { @@ -1052,8 +1091,8 @@ static int ipmmu_add_device(struct device *dev) spin_lock(&ipmmu_devices_lock); list_for_each_entry(mmu, &ipmmu_devices, list) { - utlb = ipmmu_find_utlb(mmu, dev); - if (utlb >= 0) { + num_utlbs = ipmmu_find_utlbs(mmu, dev, &utlbs); + if (num_utlbs) { /* * TODO Take a reference to the MMU to protect * against device removal. @@ -1064,17 +1103,22 @@ static int ipmmu_add_device(struct device *dev) spin_unlock(&ipmmu_devices_lock); - if (utlb < 0) + if (num_utlbs <= 0) return -ENODEV; - if (utlb >= mmu->num_utlbs) - return -EINVAL; + for (i = 0; i < num_utlbs; ++i) { + if (utlbs[i] >= mmu->num_utlbs) { + ret = -EINVAL; + goto error; + } + } /* Create a device group and add the device to it. */ group = iommu_group_alloc(); if (IS_ERR(group)) { dev_err(dev, "Failed to allocate IOMMU group\n"); - return PTR_ERR(group); + ret = PTR_ERR(group); + goto error; } ret = iommu_group_add_device(group, dev); @@ -1082,7 +1126,8 @@ static int ipmmu_add_device(struct device *dev) if (ret < 0) { dev_err(dev, "Failed to add device to IPMMU group\n"); - return ret; + group = NULL; + goto error; } archdata = kzalloc(sizeof(*archdata), GFP_KERNEL); @@ -1092,7 +1137,8 @@ static int ipmmu_add_device(struct device *dev) } archdata->mmu = mmu; - archdata->utlb = utlb; + archdata->utlbs = utlbs; + archdata->num_utlbs = num_utlbs; dev->archdata.iommu = archdata; /* @@ -1129,17 +1175,28 @@ static int ipmmu_add_device(struct device *dev) error: arm_iommu_release_mapping(mmu->mapping); + kfree(dev->archdata.iommu); + kfree(utlbs); + dev->archdata.iommu = NULL; - iommu_group_remove_device(dev); + + if (!IS_ERR_OR_NULL(group)) + iommu_group_remove_device(dev); + return ret; } static void ipmmu_remove_device(struct device *dev) { + struct ipmmu_vmsa_archdata *archdata = dev->archdata.iommu; + arm_iommu_detach_device(dev); iommu_group_remove_device(dev); - kfree(dev->archdata.iommu); + + kfree(archdata->utlbs); + kfree(archdata); + dev->archdata.iommu = NULL; } -- cgit v0.10.2 From 78e1f974dd351ac82d978e3aac2d27422013f914 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sun, 14 Dec 2014 02:37:09 +0200 Subject: iommu/ipmmu-vmsa: Remove platform data support No board file instantiates the IPMMU using platform data. Now that we have DT support, get rid of platform data. Signed-off-by: Laurent Pinchart diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 5d080cf..791c3da 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -30,7 +29,6 @@ struct ipmmu_vmsa_device { void __iomem *base; struct list_head list; - const struct ipmmu_vmsa_platform_data *pdata; unsigned int num_utlbs; struct dma_iommu_mapping *mapping; @@ -1015,27 +1013,6 @@ static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev, unsigned int i; int count; - if (mmu->pdata) { - const struct ipmmu_vmsa_master *master = mmu->pdata->masters; - const char *devname = dev_name(dev); - unsigned int i; - - for (i = 0; i < mmu->pdata->num_masters; ++i, ++master) { - if (strcmp(master->name, devname) == 0) { - utlbs = kmalloc(sizeof(*utlbs), GFP_KERNEL); - if (!utlbs) - return -ENOMEM; - - utlbs[0] = master->utlb; - - *_utlbs = utlbs; - return 1; - } - } - - return -EINVAL; - } - count = of_count_phandle_with_args(dev->of_node, "iommus", "#iommu-cells"); if (count < 0) @@ -1246,7 +1223,6 @@ static int ipmmu_probe(struct platform_device *pdev) } mmu->dev = &pdev->dev; - mmu->pdata = pdev->dev.platform_data; mmu->num_utlbs = 32; /* Map I/O memory and request IRQ. */ diff --git a/include/linux/platform_data/ipmmu-vmsa.h b/include/linux/platform_data/ipmmu-vmsa.h deleted file mode 100644 index 5275b3a..0000000 --- a/include/linux/platform_data/ipmmu-vmsa.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * IPMMU VMSA Platform Data - * - * Copyright (C) 2014 Renesas Electronics Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - */ - -#ifndef __IPMMU_VMSA_H__ -#define __IPMMU_VMSA_H__ - -struct ipmmu_vmsa_master { - const char *name; - unsigned int utlb; -}; - -struct ipmmu_vmsa_platform_data { - const struct ipmmu_vmsa_master *masters; - unsigned int num_masters; -}; - -#endif /* __IPMMU_VMSA_H__ */ -- cgit v0.10.2 From 114150d8f4fc9e7a003be2bdb0efd31796d241ff Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jan 2015 17:51:13 +0000 Subject: iommu: Allow building iova.c independently In preparation for sharing the IOVA allocator, split it out under its own Kconfig symbol. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 325188e..a839ca9 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -13,6 +13,9 @@ menuconfig IOMMU_SUPPORT if IOMMU_SUPPORT +config IOMMU_IOVA + bool + config OF_IOMMU def_bool y depends on OF && IOMMU_API @@ -91,6 +94,7 @@ config INTEL_IOMMU bool "Support for Intel IOMMU using DMA Remapping Devices" depends on PCI_MSI && ACPI && (X86 || IA64_GENERIC) select IOMMU_API + select IOMMU_IOVA select DMAR_TABLE help DMA remapping (DMAR) devices support enables independent address diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 7b976f2..0b1b94e 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,13 +1,14 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o +obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm-smmu.o obj-$(CONFIG_DMAR_TABLE) += dmar.o -obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o +obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o -- cgit v0.10.2 From 85b4545629663486b7f71047ce3b54fa0ad3eb28 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jan 2015 17:51:14 +0000 Subject: iommu: Consolidate IOVA allocator code In order to share the IOVA allocator with other architectures, break the unnecssary dependency on the Intel IOMMU driver and move the remaining IOVA internals to iova.c Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 40dfbc0..e758d8e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -485,7 +485,6 @@ __setup("intel_iommu=", intel_iommu_setup); static struct kmem_cache *iommu_domain_cache; static struct kmem_cache *iommu_devinfo_cache; -static struct kmem_cache *iommu_iova_cache; static inline void *alloc_pgtable_page(int node) { @@ -523,16 +522,6 @@ static inline void free_devinfo_mem(void *vaddr) kmem_cache_free(iommu_devinfo_cache, vaddr); } -struct iova *alloc_iova_mem(void) -{ - return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC); -} - -void free_iova_mem(struct iova *iova) -{ - kmem_cache_free(iommu_iova_cache, iova); -} - static inline int domain_type_is_vm(struct dmar_domain *domain) { return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE; @@ -3427,23 +3416,6 @@ static inline int iommu_devinfo_cache_init(void) return ret; } -static inline int iommu_iova_cache_init(void) -{ - int ret = 0; - - iommu_iova_cache = kmem_cache_create("iommu_iova", - sizeof(struct iova), - 0, - SLAB_HWCACHE_ALIGN, - NULL); - if (!iommu_iova_cache) { - printk(KERN_ERR "Couldn't create iova cache\n"); - ret = -ENOMEM; - } - - return ret; -} - static int __init iommu_init_mempool(void) { int ret; @@ -3461,7 +3433,7 @@ static int __init iommu_init_mempool(void) kmem_cache_destroy(iommu_domain_cache); domain_error: - kmem_cache_destroy(iommu_iova_cache); + iommu_iova_cache_destroy(); return -ENOMEM; } @@ -3470,8 +3442,7 @@ static void __init iommu_exit_mempool(void) { kmem_cache_destroy(iommu_devinfo_cache); kmem_cache_destroy(iommu_domain_cache); - kmem_cache_destroy(iommu_iova_cache); - + iommu_iova_cache_destroy(); } static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev) diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index f6b17e6..520b8c8 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -18,6 +18,41 @@ */ #include +#include + +static struct kmem_cache *iommu_iova_cache; + +int iommu_iova_cache_init(void) +{ + int ret = 0; + + iommu_iova_cache = kmem_cache_create("iommu_iova", + sizeof(struct iova), + 0, + SLAB_HWCACHE_ALIGN, + NULL); + if (!iommu_iova_cache) { + pr_err("Couldn't create iova cache\n"); + ret = -ENOMEM; + } + + return ret; +} + +void iommu_iova_cache_destroy(void) +{ + kmem_cache_destroy(iommu_iova_cache); +} + +struct iova *alloc_iova_mem(void) +{ + return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC); +} + +void free_iova_mem(struct iova *iova) +{ + kmem_cache_free(iommu_iova_cache, iova); +} void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit) diff --git a/include/linux/iova.h b/include/linux/iova.h index 19e81d5..ad0507c 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -39,6 +39,9 @@ static inline unsigned long iova_size(struct iova *iova) return iova->pfn_hi - iova->pfn_lo + 1; } +int iommu_iova_cache_init(void); +void iommu_iova_cache_destroy(void); + struct iova *alloc_iova_mem(void); void free_iova_mem(struct iova *iova); void free_iova(struct iova_domain *iovad, unsigned long pfn); -- cgit v0.10.2 From 1b72250076dde4276acecf3a7da722b185703e78 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jan 2015 17:51:15 +0000 Subject: iommu: Make IOVA domain low limit flexible To share the IOVA allocator with other architectures, it needs to accommodate more general aperture restrictions; move the lower limit from a compile-time constant to a runtime domain property to allow IOVA domains with different requirements to co-exist. Also reword the slightly unclear description of alloc_iova since we're touching it anyway. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index e758d8e..86f9e82 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -71,6 +71,9 @@ __DOMAIN_MAX_PFN(gaw), (unsigned long)-1)) #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT) +/* IO virtual address start page frame number */ +#define IOVA_START_PFN (1) + #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT) #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32)) #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64)) @@ -1632,7 +1635,7 @@ static int dmar_init_reserved_ranges(void) struct iova *iova; int i; - init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN); + init_iova_domain(&reserved_iova_list, IOVA_START_PFN, DMA_32BIT_PFN); lockdep_set_class(&reserved_iova_list.iova_rbtree_lock, &reserved_rbtree_key); @@ -1690,7 +1693,7 @@ static int domain_init(struct dmar_domain *domain, int guest_width) int adjust_width, agaw; unsigned long sagaw; - init_iova_domain(&domain->iovad, DMA_32BIT_PFN); + init_iova_domain(&domain->iovad, IOVA_START_PFN, DMA_32BIT_PFN); domain_reserve_special_ranges(domain); /* calculate AGAW */ @@ -4313,7 +4316,7 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width) { int adjust_width; - init_iova_domain(&domain->iovad, DMA_32BIT_PFN); + init_iova_domain(&domain->iovad, IOVA_START_PFN, DMA_32BIT_PFN); domain_reserve_special_ranges(domain); /* calculate AGAW */ diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 520b8c8..a3dbba8 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -55,11 +55,13 @@ void free_iova_mem(struct iova *iova) } void -init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit) +init_iova_domain(struct iova_domain *iovad, unsigned long start_pfn, + unsigned long pfn_32bit) { spin_lock_init(&iovad->iova_rbtree_lock); iovad->rbroot = RB_ROOT; iovad->cached32_node = NULL; + iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = pfn_32bit; } @@ -162,7 +164,7 @@ move_left: if (!curr) { if (size_aligned) pad_size = iova_get_pad_size(size, limit_pfn); - if ((IOVA_START_PFN + size + pad_size) > limit_pfn) { + if ((iovad->start_pfn + size + pad_size) > limit_pfn) { spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); return -ENOMEM; } @@ -237,8 +239,8 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova) * @size: - size of page frames to allocate * @limit_pfn: - max limit address * @size_aligned: - set if size_aligned address range is required - * This function allocates an iova in the range limit_pfn to IOVA_START_PFN - * looking from limit_pfn instead from IOVA_START_PFN. If the size_aligned + * This function allocates an iova in the range iovad->start_pfn to limit_pfn, + * searching top-down from limit_pfn to iovad->start_pfn. If the size_aligned * flag is set then the allocated address iova->pfn_lo will be naturally * aligned on roundup_power_of_two(size). */ diff --git a/include/linux/iova.h b/include/linux/iova.h index ad0507c..591b196 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -16,9 +16,6 @@ #include #include -/* IO virtual address start page frame number */ -#define IOVA_START_PFN (1) - /* iova structure */ struct iova { struct rb_node node; @@ -31,6 +28,7 @@ struct iova_domain { spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */ struct rb_root rbroot; /* iova domain rbtree root */ struct rb_node *cached32_node; /* Save last alloced node */ + unsigned long start_pfn; /* Lower limit for this domain */ unsigned long dma_32bit_pfn; }; @@ -52,7 +50,8 @@ struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to); -void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit); +void init_iova_domain(struct iova_domain *iovad, unsigned long start_pfn, + unsigned long pfn_32bit); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); struct iova *split_and_remove_iova(struct iova_domain *iovad, -- cgit v0.10.2 From 0fb5fe874c42942e16c450ae05da453e13a1c09e Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 12 Jan 2015 17:51:16 +0000 Subject: iommu: Make IOVA domain page size explicit Systems may contain heterogeneous IOMMUs supporting differing minimum page sizes, which may also not be common with the CPU page size. Thus it is practical to have an explicit notion of IOVA granularity to simplify handling of mapping and allocation constraints. As an initial step, move the IOVA page granularity from an implicit compile-time constant to a per-domain property so we can make use of it in IOVA domain context at runtime. To keep the abstraction tidy, extend the little API of inline iova_* helpers to parallel some of the equivalent PAGE_* macros. Signed-off-by: Robin Murphy Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 86f9e82..ae4c1a8 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1635,7 +1635,8 @@ static int dmar_init_reserved_ranges(void) struct iova *iova; int i; - init_iova_domain(&reserved_iova_list, IOVA_START_PFN, DMA_32BIT_PFN); + init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN, + DMA_32BIT_PFN); lockdep_set_class(&reserved_iova_list.iova_rbtree_lock, &reserved_rbtree_key); @@ -1693,7 +1694,8 @@ static int domain_init(struct dmar_domain *domain, int guest_width) int adjust_width, agaw; unsigned long sagaw; - init_iova_domain(&domain->iovad, IOVA_START_PFN, DMA_32BIT_PFN); + init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN, + DMA_32BIT_PFN); domain_reserve_special_ranges(domain); /* calculate AGAW */ @@ -4316,7 +4318,8 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width) { int adjust_width; - init_iova_domain(&domain->iovad, IOVA_START_PFN, DMA_32BIT_PFN); + init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN, + DMA_32BIT_PFN); domain_reserve_special_ranges(domain); /* calculate AGAW */ diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index a3dbba8..9dd8208 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -55,12 +55,20 @@ void free_iova_mem(struct iova *iova) } void -init_iova_domain(struct iova_domain *iovad, unsigned long start_pfn, - unsigned long pfn_32bit) +init_iova_domain(struct iova_domain *iovad, unsigned long granule, + unsigned long start_pfn, unsigned long pfn_32bit) { + /* + * IOVA granularity will normally be equal to the smallest + * supported IOMMU page size; both *must* be capable of + * representing individual CPU pages exactly. + */ + BUG_ON((granule > PAGE_SIZE) || !is_power_of_2(granule)); + spin_lock_init(&iovad->iova_rbtree_lock); iovad->rbroot = RB_ROOT; iovad->cached32_node = NULL; + iovad->granule = granule; iovad->start_pfn = start_pfn; iovad->dma_32bit_pfn = pfn_32bit; } diff --git a/include/linux/iova.h b/include/linux/iova.h index 591b196..3920a19 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -28,6 +28,7 @@ struct iova_domain { spinlock_t iova_rbtree_lock; /* Lock to protect update of rbtree */ struct rb_root rbroot; /* iova domain rbtree root */ struct rb_node *cached32_node; /* Save last alloced node */ + unsigned long granule; /* pfn granularity for this domain */ unsigned long start_pfn; /* Lower limit for this domain */ unsigned long dma_32bit_pfn; }; @@ -37,6 +38,36 @@ static inline unsigned long iova_size(struct iova *iova) return iova->pfn_hi - iova->pfn_lo + 1; } +static inline unsigned long iova_shift(struct iova_domain *iovad) +{ + return __ffs(iovad->granule); +} + +static inline unsigned long iova_mask(struct iova_domain *iovad) +{ + return iovad->granule - 1; +} + +static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova) +{ + return iova & iova_mask(iovad); +} + +static inline size_t iova_align(struct iova_domain *iovad, size_t size) +{ + return ALIGN(size, iovad->granule); +} + +static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova) +{ + return (dma_addr_t)iova->pfn_lo << iova_shift(iovad); +} + +static inline unsigned long iova_pfn(struct iova_domain *iovad, dma_addr_t iova) +{ + return iova >> iova_shift(iovad); +} + int iommu_iova_cache_init(void); void iommu_iova_cache_destroy(void); @@ -50,8 +81,8 @@ struct iova *alloc_iova(struct iova_domain *iovad, unsigned long size, struct iova *reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi); void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to); -void init_iova_domain(struct iova_domain *iovad, unsigned long start_pfn, - unsigned long pfn_32bit); +void init_iova_domain(struct iova_domain *iovad, unsigned long granule, + unsigned long start_pfn, unsigned long pfn_32bit); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); struct iova *split_and_remove_iova(struct iova_domain *iovad, -- cgit v0.10.2 From 6fd492fd746d9858a41dc85eef44bd627b809109 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Fri, 16 Jan 2015 16:47:19 -0700 Subject: iommu: Fix trace_unmap() to report original iova iommu_unmap() calls trace_unmap() with changed iova and original size. trace_unmap() should report original iova instead. Change iommu_unmap() to call trace_unmap() with original iova. Signed-off-by: Shuah Khan Reported-by: Alex Williamson Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f7718d7..d4c3db5 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1094,6 +1094,7 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { size_t unmapped_page, unmapped = 0; unsigned int min_pagesz; + unsigned long orig_iova = iova; if (unlikely(domain->ops->unmap == NULL || domain->ops->pgsize_bitmap == 0UL)) @@ -1133,7 +1134,7 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) unmapped += unmapped_page; } - trace_unmap(iova, 0, size); + trace_unmap(orig_iova, 0, size); return unmapped; } EXPORT_SYMBOL_GPL(iommu_unmap); -- cgit v0.10.2 From db8614d35bb8fc6d032792c801bd5b38ce860f19 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Fri, 16 Jan 2015 20:53:17 -0700 Subject: iommu: Change trace unmap api to report unmapped size Currently map and unmap are implemented as events under a common trace class declaration. The common class forces trace_unmap() to require a bogus physical address argument that it doesn't use. Changing unmap to report unmapped size will provide useful information for debugging. Remove common map_unmap trace class and change map and unmap into separate events as opposed to events under the same class to allow for differences in the reporting information. In addition, map and unmap are changed to handle size value as size_t instead of int to match the passed size value and avoid overflow. Signed-off-by: Shuah Khan Suggested-by: Alex Williamson Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d4c3db5..3a4fb62 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1134,7 +1134,7 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) unmapped += unmapped_page; } - trace_unmap(orig_iova, 0, size); + trace_unmap(orig_iova, size, unmapped); return unmapped; } EXPORT_SYMBOL_GPL(iommu_unmap); diff --git a/include/trace/events/iommu.h b/include/trace/events/iommu.h index a8f5c32..2c7befb 100644 --- a/include/trace/events/iommu.h +++ b/include/trace/events/iommu.h @@ -83,7 +83,7 @@ DEFINE_EVENT(iommu_device_event, detach_device_from_domain, TP_ARGS(dev) ); -DECLARE_EVENT_CLASS(iommu_map_unmap, +TRACE_EVENT(map, TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size), @@ -92,7 +92,7 @@ DECLARE_EVENT_CLASS(iommu_map_unmap, TP_STRUCT__entry( __field(u64, iova) __field(u64, paddr) - __field(int, size) + __field(size_t, size) ), TP_fast_assign( @@ -101,26 +101,31 @@ DECLARE_EVENT_CLASS(iommu_map_unmap, __entry->size = size; ), - TP_printk("IOMMU: iova=0x%016llx paddr=0x%016llx size=0x%x", + TP_printk("IOMMU: iova=0x%016llx paddr=0x%016llx size=%zu", __entry->iova, __entry->paddr, __entry->size ) ); -DEFINE_EVENT(iommu_map_unmap, map, +TRACE_EVENT(unmap, - TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size), - - TP_ARGS(iova, paddr, size) -); + TP_PROTO(unsigned long iova, size_t size, size_t unmapped_size), -DEFINE_EVENT_PRINT(iommu_map_unmap, unmap, + TP_ARGS(iova, size, unmapped_size), - TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size), + TP_STRUCT__entry( + __field(u64, iova) + __field(size_t, size) + __field(size_t, unmapped_size) + ), - TP_ARGS(iova, paddr, size), + TP_fast_assign( + __entry->iova = iova; + __entry->size = size; + __entry->unmapped_size = unmapped_size; + ), - TP_printk("IOMMU: iova=0x%016llx size=0x%x", - __entry->iova, __entry->size + TP_printk("IOMMU: iova=0x%016llx size=%zu unmapped_size=%zu", + __entry->iova, __entry->size, __entry->unmapped_size ) ); -- cgit v0.10.2 From fdb1d7be7c4d452e9735aeb2b60ae8a2fcf0a514 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 14 Nov 2014 17:16:49 +0000 Subject: iommu: introduce generic page table allocation framework This patch introduces a generic framework for allocating page tables for an IOMMU. There are a number of reasons we want to do this: - It avoids duplication of complex table management code in IOMMU drivers that use the same page table format - It removes any coupling with the CPU table format (and even the architecture!) - It defines an API for IOMMU TLB maintenance Tested-by: Laurent Pinchart Signed-off-by: Will Deacon diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 325188e..3faaa41 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -13,6 +13,14 @@ menuconfig IOMMU_SUPPORT if IOMMU_SUPPORT +menu "Generic IOMMU Pagetable Support" + +# Selected by the actual pagetable implementations +config IOMMU_IO_PGTABLE + bool + +endmenu + config OF_IOMMU def_bool y depends on OF && IOMMU_API diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 7b976f2..701c951 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -1,6 +1,7 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o +obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c new file mode 100644 index 0000000..f664a1c --- /dev/null +++ b/drivers/iommu/io-pgtable.c @@ -0,0 +1,71 @@ +/* + * Generic page table allocator for IOMMUs. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon + */ + +#include +#include +#include + +#include "io-pgtable.h" + +static const struct io_pgtable_init_fns * +io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = +{ +}; + +struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct io_pgtable_cfg *cfg, + void *cookie) +{ + struct io_pgtable *iop; + const struct io_pgtable_init_fns *fns; + + if (fmt >= IO_PGTABLE_NUM_FMTS) + return NULL; + + fns = io_pgtable_init_table[fmt]; + if (!fns) + return NULL; + + iop = fns->alloc(cfg, cookie); + if (!iop) + return NULL; + + iop->fmt = fmt; + iop->cookie = cookie; + iop->cfg = *cfg; + + return &iop->ops; +} + +/* + * It is the IOMMU driver's responsibility to ensure that the page table + * is no longer accessible to the walker by this point. + */ +void free_io_pgtable_ops(struct io_pgtable_ops *ops) +{ + struct io_pgtable *iop; + + if (!ops) + return; + + iop = container_of(ops, struct io_pgtable, ops); + iop->cfg.tlb->tlb_flush_all(iop->cookie); + io_pgtable_init_table[iop->fmt]->free(iop); +} diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h new file mode 100644 index 0000000..fdd792c --- /dev/null +++ b/drivers/iommu/io-pgtable.h @@ -0,0 +1,128 @@ +#ifndef __IO_PGTABLE_H +#define __IO_PGTABLE_H + +/* + * Public API for use by IOMMU drivers + */ +enum io_pgtable_fmt { + IO_PGTABLE_NUM_FMTS, +}; + +/** + * struct iommu_gather_ops - IOMMU callbacks for TLB and page table management. + * + * @tlb_flush_all: Synchronously invalidate the entire TLB context. + * @tlb_add_flush: Queue up a TLB invalidation for a virtual address range. + * @tlb_sync: Ensure any queue TLB invalidation has taken effect. + * @flush_pgtable: Ensure page table updates are visible to the IOMMU. + * + * Note that these can all be called in atomic context and must therefore + * not block. + */ +struct iommu_gather_ops { + void (*tlb_flush_all)(void *cookie); + void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf, + void *cookie); + void (*tlb_sync)(void *cookie); + void (*flush_pgtable)(void *ptr, size_t size, void *cookie); +}; + +/** + * struct io_pgtable_cfg - Configuration data for a set of page tables. + * + * @quirks: A bitmap of hardware quirks that require some special + * action by the low-level page table allocator. + * @pgsize_bitmap: A bitmap of page sizes supported by this set of page + * tables. + * @ias: Input address (iova) size, in bits. + * @oas: Output address (paddr) size, in bits. + * @tlb: TLB management callbacks for this set of tables. + */ +struct io_pgtable_cfg { + int quirks; /* IO_PGTABLE_QUIRK_* */ + unsigned long pgsize_bitmap; + unsigned int ias; + unsigned int oas; + const struct iommu_gather_ops *tlb; + + /* Low-level data specific to the table format */ + union { + }; +}; + +/** + * struct io_pgtable_ops - Page table manipulation API for IOMMU drivers. + * + * @map: Map a physically contiguous memory region. + * @unmap: Unmap a physically contiguous memory region. + * @iova_to_phys: Translate iova to physical address. + * + * These functions map directly onto the iommu_ops member functions with + * the same names. + */ +struct io_pgtable_ops { + int (*map)(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot); + int (*unmap)(struct io_pgtable_ops *ops, unsigned long iova, + size_t size); + phys_addr_t (*iova_to_phys)(struct io_pgtable_ops *ops, + unsigned long iova); +}; + +/** + * alloc_io_pgtable_ops() - Allocate a page table allocator for use by an IOMMU. + * + * @fmt: The page table format. + * @cfg: The page table configuration. This will be modified to represent + * the configuration actually provided by the allocator (e.g. the + * pgsize_bitmap may be restricted). + * @cookie: An opaque token provided by the IOMMU driver and passed back to + * the callback routines in cfg->tlb. + */ +struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct io_pgtable_cfg *cfg, + void *cookie); + +/** + * free_io_pgtable_ops() - Free an io_pgtable_ops structure. The caller + * *must* ensure that the page table is no longer + * live, but the TLB can be dirty. + * + * @ops: The ops returned from alloc_io_pgtable_ops. + */ +void free_io_pgtable_ops(struct io_pgtable_ops *ops); + + +/* + * Internal structures for page table allocator implementations. + */ + +/** + * struct io_pgtable - Internal structure describing a set of page tables. + * + * @fmt: The page table format. + * @cookie: An opaque token provided by the IOMMU driver and passed back to + * any callback routines. + * @cfg: A copy of the page table configuration. + * @ops: The page table operations in use for this set of page tables. + */ +struct io_pgtable { + enum io_pgtable_fmt fmt; + void *cookie; + struct io_pgtable_cfg cfg; + struct io_pgtable_ops ops; +}; + +/** + * struct io_pgtable_init_fns - Alloc/free a set of page tables for a + * particular format. + * + * @alloc: Allocate a set of page tables described by cfg. + * @free: Free the page tables associated with iop. + */ +struct io_pgtable_init_fns { + struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); + void (*free)(struct io_pgtable *iop); +}; + +#endif /* __IO_PGTABLE_H */ -- cgit v0.10.2 From e1d3c0fd701df831169b116cd5c5d6203ac07f70 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 14 Nov 2014 17:18:23 +0000 Subject: iommu: add ARM LPAE page table allocator A number of IOMMUs found in ARM SoCs can walk architecture-compatible page tables. This patch adds a generic allocator for Stage-1 and Stage-2 v7/v8 long-descriptor page tables. 4k, 16k and 64k pages are supported, with up to 4-levels of walk to cover a 48-bit address space. Tested-by: Laurent Pinchart Signed-off-by: Will Deacon diff --git a/MAINTAINERS b/MAINTAINERS index 3589d67..00b2786 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1589,6 +1589,7 @@ M: Will Deacon L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained F: drivers/iommu/arm-smmu.c +F: drivers/iommu/io-pgtable-arm.c ARM64 PORT (AARCH64 ARCHITECTURE) M: Catalin Marinas diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 3faaa41..306454f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -19,6 +19,15 @@ menu "Generic IOMMU Pagetable Support" config IOMMU_IO_PGTABLE bool +config IOMMU_IO_PGTABLE_LPAE + bool "ARMv7/v8 Long Descriptor Format" + select IOMMU_IO_PGTABLE + help + Enable support for the ARM long descriptor pagetable format. + This allocator supports 4K/2M/1G, 16K/32M and 64K/512M page + sizes at both stage-1 and stage-2, as well as address spaces + up to 48-bits in size. + endmenu config OF_IOMMU diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 701c951..d6889b4 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_IOMMU_API) += iommu.o obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o +obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c new file mode 100644 index 0000000..dbe6178 --- /dev/null +++ b/drivers/iommu/io-pgtable-arm.c @@ -0,0 +1,781 @@ +/* + * CPU-agnostic ARM page table allocator. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon + */ + +#define pr_fmt(fmt) "arm-lpae io-pgtable: " fmt + +#include +#include +#include +#include +#include + +#include "io-pgtable.h" + +#define ARM_LPAE_MAX_ADDR_BITS 48 +#define ARM_LPAE_S2_MAX_CONCAT_PAGES 16 +#define ARM_LPAE_MAX_LEVELS 4 + +/* Struct accessors */ +#define io_pgtable_to_data(x) \ + container_of((x), struct arm_lpae_io_pgtable, iop) + +#define io_pgtable_ops_to_pgtable(x) \ + container_of((x), struct io_pgtable, ops) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +/* + * For consistency with the architecture, we always consider + * ARM_LPAE_MAX_LEVELS levels, with the walk starting at level n >=0 + */ +#define ARM_LPAE_START_LVL(d) (ARM_LPAE_MAX_LEVELS - (d)->levels) + +/* + * Calculate the right shift amount to get to the portion describing level l + * in a virtual address mapped by the pagetable in d. + */ +#define ARM_LPAE_LVL_SHIFT(l,d) \ + ((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \ + * (d)->bits_per_level) + (d)->pg_shift) + +#define ARM_LPAE_PAGES_PER_PGD(d) ((d)->pgd_size >> (d)->pg_shift) + +/* + * Calculate the index at level l used to map virtual address a using the + * pagetable in d. + */ +#define ARM_LPAE_PGD_IDX(l,d) \ + ((l) == ARM_LPAE_START_LVL(d) ? ilog2(ARM_LPAE_PAGES_PER_PGD(d)) : 0) + +#define ARM_LPAE_LVL_IDX(a,l,d) \ + (((a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \ + ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1)) + +/* Calculate the block/page mapping size at level l for pagetable in d. */ +#define ARM_LPAE_BLOCK_SIZE(l,d) \ + (1 << (ilog2(sizeof(arm_lpae_iopte)) + \ + ((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level))) + +/* Page table bits */ +#define ARM_LPAE_PTE_TYPE_SHIFT 0 +#define ARM_LPAE_PTE_TYPE_MASK 0x3 + +#define ARM_LPAE_PTE_TYPE_BLOCK 1 +#define ARM_LPAE_PTE_TYPE_TABLE 3 +#define ARM_LPAE_PTE_TYPE_PAGE 3 + +#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) +#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) +#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) +#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) +#define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) +#define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) + +#define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) +/* Ignore the contiguous bit for block splitting */ +#define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52) +#define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK | \ + ARM_LPAE_PTE_ATTR_HI_MASK) + +/* Stage-1 PTE */ +#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6) +#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2 +#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11) + +/* Stage-2 PTE */ +#define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6) +#define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6) +#define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6) +#define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2) +#define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2) +#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2) + +/* Register bits */ +#define ARM_32_LPAE_TCR_EAE (1 << 31) +#define ARM_64_LPAE_S2_TCR_RES1 (1 << 31) + +#define ARM_LPAE_TCR_TG0_4K (0 << 14) +#define ARM_LPAE_TCR_TG0_64K (1 << 14) +#define ARM_LPAE_TCR_TG0_16K (2 << 14) + +#define ARM_LPAE_TCR_SH0_SHIFT 12 +#define ARM_LPAE_TCR_SH0_MASK 0x3 +#define ARM_LPAE_TCR_SH_NS 0 +#define ARM_LPAE_TCR_SH_OS 2 +#define ARM_LPAE_TCR_SH_IS 3 + +#define ARM_LPAE_TCR_ORGN0_SHIFT 10 +#define ARM_LPAE_TCR_IRGN0_SHIFT 8 +#define ARM_LPAE_TCR_RGN_MASK 0x3 +#define ARM_LPAE_TCR_RGN_NC 0 +#define ARM_LPAE_TCR_RGN_WBWA 1 +#define ARM_LPAE_TCR_RGN_WT 2 +#define ARM_LPAE_TCR_RGN_WB 3 + +#define ARM_LPAE_TCR_SL0_SHIFT 6 +#define ARM_LPAE_TCR_SL0_MASK 0x3 + +#define ARM_LPAE_TCR_T0SZ_SHIFT 0 +#define ARM_LPAE_TCR_SZ_MASK 0xf + +#define ARM_LPAE_TCR_PS_SHIFT 16 +#define ARM_LPAE_TCR_PS_MASK 0x7 + +#define ARM_LPAE_TCR_IPS_SHIFT 32 +#define ARM_LPAE_TCR_IPS_MASK 0x7 + +#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL +#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL +#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL +#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL +#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL +#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL + +#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define ARM_LPAE_MAIR_ATTR_MASK 0xff +#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04 +#define ARM_LPAE_MAIR_ATTR_NC 0x44 +#define ARM_LPAE_MAIR_ATTR_WBRWA 0xff +#define ARM_LPAE_MAIR_ATTR_IDX_NC 0 +#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1 +#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2 + +/* IOPTE accessors */ +#define iopte_deref(pte,d) \ + (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ + & ~((1ULL << (d)->pg_shift) - 1))) + +#define iopte_type(pte,l) \ + (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) + +#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK) + +#define iopte_leaf(pte,l) \ + (l == (ARM_LPAE_MAX_LEVELS - 1) ? \ + (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_PAGE) : \ + (iopte_type(pte,l) == ARM_LPAE_PTE_TYPE_BLOCK)) + +#define iopte_to_pfn(pte,d) \ + (((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) >> (d)->pg_shift) + +#define pfn_to_iopte(pfn,d) \ + (((pfn) << (d)->pg_shift) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1)) + +struct arm_lpae_io_pgtable { + struct io_pgtable iop; + + int levels; + size_t pgd_size; + unsigned long pg_shift; + unsigned long bits_per_level; + + void *pgd; +}; + +typedef u64 arm_lpae_iopte; + +static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, + arm_lpae_iopte prot, int lvl, + arm_lpae_iopte *ptep) +{ + arm_lpae_iopte pte = prot; + + /* We require an unmap first */ + if (WARN_ON(iopte_leaf(*ptep, lvl))) + return -EEXIST; + + if (lvl == ARM_LPAE_MAX_LEVELS - 1) + pte |= ARM_LPAE_PTE_TYPE_PAGE; + else + pte |= ARM_LPAE_PTE_TYPE_BLOCK; + + pte |= ARM_LPAE_PTE_AF | ARM_LPAE_PTE_SH_IS; + pte |= pfn_to_iopte(paddr >> data->pg_shift, data); + + *ptep = pte; + data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), data->iop.cookie); + return 0; +} + +static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, arm_lpae_iopte prot, + int lvl, arm_lpae_iopte *ptep) +{ + arm_lpae_iopte *cptep, pte; + void *cookie = data->iop.cookie; + size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + + /* Find our entry at the current level */ + ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); + + /* If we can install a leaf entry at this level, then do so */ + if (size == block_size && (size & data->iop.cfg.pgsize_bitmap)) + return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep); + + /* We can't allocate tables at the final level */ + if (WARN_ON(lvl >= ARM_LPAE_MAX_LEVELS - 1)) + return -EINVAL; + + /* Grab a pointer to the next level */ + pte = *ptep; + if (!pte) { + cptep = alloc_pages_exact(1UL << data->pg_shift, + GFP_ATOMIC | __GFP_ZERO); + if (!cptep) + return -ENOMEM; + + data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift, + cookie); + pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; + *ptep = pte; + data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + } else { + cptep = iopte_deref(pte, data); + } + + /* Rinse, repeat */ + return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep); +} + +static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, + int prot) +{ + arm_lpae_iopte pte; + + if (data->iop.fmt == ARM_64_LPAE_S1 || + data->iop.fmt == ARM_32_LPAE_S1) { + pte = ARM_LPAE_PTE_AP_UNPRIV | ARM_LPAE_PTE_nG; + + if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) + pte |= ARM_LPAE_PTE_AP_RDONLY; + + if (prot & IOMMU_CACHE) + pte |= (ARM_LPAE_MAIR_ATTR_IDX_CACHE + << ARM_LPAE_PTE_ATTRINDX_SHIFT); + } else { + pte = ARM_LPAE_PTE_HAP_FAULT; + if (prot & IOMMU_READ) + pte |= ARM_LPAE_PTE_HAP_READ; + if (prot & IOMMU_WRITE) + pte |= ARM_LPAE_PTE_HAP_WRITE; + if (prot & IOMMU_CACHE) + pte |= ARM_LPAE_PTE_MEMATTR_OIWB; + else + pte |= ARM_LPAE_PTE_MEMATTR_NC; + } + + if (prot & IOMMU_NOEXEC) + pte |= ARM_LPAE_PTE_XN; + + return pte; +} + +static int arm_lpae_map(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int iommu_prot) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte *ptep = data->pgd; + int lvl = ARM_LPAE_START_LVL(data); + arm_lpae_iopte prot; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + prot = arm_lpae_prot_to_pte(data, iommu_prot); + return __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep); +} + +static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, + arm_lpae_iopte *ptep) +{ + arm_lpae_iopte *start, *end; + unsigned long table_size; + + /* Only leaf entries at the last level */ + if (lvl == ARM_LPAE_MAX_LEVELS - 1) + return; + + if (lvl == ARM_LPAE_START_LVL(data)) + table_size = data->pgd_size; + else + table_size = 1UL << data->pg_shift; + + start = ptep; + end = (void *)ptep + table_size; + + while (ptep != end) { + arm_lpae_iopte pte = *ptep++; + + if (!pte || iopte_leaf(pte, lvl)) + continue; + + __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); + } + + free_pages_exact(start, table_size); +} + +static void arm_lpae_free_pgtable(struct io_pgtable *iop) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop); + + __arm_lpae_free_pgtable(data, ARM_LPAE_START_LVL(data), data->pgd); + kfree(data); +} + +static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, + unsigned long iova, size_t size, + arm_lpae_iopte prot, int lvl, + arm_lpae_iopte *ptep, size_t blk_size) +{ + unsigned long blk_start, blk_end; + phys_addr_t blk_paddr; + arm_lpae_iopte table = 0; + void *cookie = data->iop.cookie; + const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; + + blk_start = iova & ~(blk_size - 1); + blk_end = blk_start + blk_size; + blk_paddr = iopte_to_pfn(*ptep, data) << data->pg_shift; + + for (; blk_start < blk_end; blk_start += size, blk_paddr += size) { + arm_lpae_iopte *tablep; + + /* Unmap! */ + if (blk_start == iova) + continue; + + /* __arm_lpae_map expects a pointer to the start of the table */ + tablep = &table - ARM_LPAE_LVL_IDX(blk_start, lvl, data); + if (__arm_lpae_map(data, blk_start, blk_paddr, size, prot, lvl, + tablep) < 0) { + if (table) { + /* Free the table we allocated */ + tablep = iopte_deref(table, data); + __arm_lpae_free_pgtable(data, lvl + 1, tablep); + } + return 0; /* Bytes unmapped */ + } + } + + *ptep = table; + tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + iova &= ~(blk_size - 1); + tlb->tlb_add_flush(iova, blk_size, true, cookie); + return size; +} + +static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_lpae_iopte *ptep) +{ + arm_lpae_iopte pte; + const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; + void *cookie = data->iop.cookie; + size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + + ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); + pte = *ptep; + + /* Something went horribly wrong and we ran out of page table */ + if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS))) + return 0; + + /* If the size matches this level, we're in the right place */ + if (size == blk_size) { + *ptep = 0; + tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); + + if (!iopte_leaf(pte, lvl)) { + /* Also flush any partial walks */ + tlb->tlb_add_flush(iova, size, false, cookie); + tlb->tlb_sync(data->iop.cookie); + ptep = iopte_deref(pte, data); + __arm_lpae_free_pgtable(data, lvl + 1, ptep); + } else { + tlb->tlb_add_flush(iova, size, true, cookie); + } + + return size; + } else if (iopte_leaf(pte, lvl)) { + /* + * Insert a table at the next level to map the old region, + * minus the part we want to unmap + */ + return arm_lpae_split_blk_unmap(data, iova, size, + iopte_prot(pte), lvl, ptep, + blk_size); + } + + /* Keep on walkin' */ + ptep = iopte_deref(pte, data); + return __arm_lpae_unmap(data, iova, size, lvl + 1, ptep); +} + +static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) +{ + size_t unmapped; + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable *iop = &data->iop; + arm_lpae_iopte *ptep = data->pgd; + int lvl = ARM_LPAE_START_LVL(data); + + unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); + if (unmapped) + iop->cfg.tlb->tlb_sync(iop->cookie); + + return unmapped; +} + +static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_lpae_iopte pte, *ptep = data->pgd; + int lvl = ARM_LPAE_START_LVL(data); + + do { + /* Valid IOPTE pointer? */ + if (!ptep) + return 0; + + /* Grab the IOPTE we're interested in */ + pte = *(ptep + ARM_LPAE_LVL_IDX(iova, lvl, data)); + + /* Valid entry? */ + if (!pte) + return 0; + + /* Leaf entry? */ + if (iopte_leaf(pte,lvl)) + goto found_translation; + + /* Take it to the next level */ + ptep = iopte_deref(pte, data); + } while (++lvl < ARM_LPAE_MAX_LEVELS); + + /* Ran out of page tables to walk */ + return 0; + +found_translation: + iova &= ((1 << data->pg_shift) - 1); + return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; +} + +static void arm_lpae_restrict_pgsizes(struct io_pgtable_cfg *cfg) +{ + unsigned long granule; + + /* + * We need to restrict the supported page sizes to match the + * translation regime for a particular granule. Aim to match + * the CPU page size if possible, otherwise prefer smaller sizes. + * While we're at it, restrict the block sizes to match the + * chosen granule. + */ + if (cfg->pgsize_bitmap & PAGE_SIZE) + granule = PAGE_SIZE; + else if (cfg->pgsize_bitmap & ~PAGE_MASK) + granule = 1UL << __fls(cfg->pgsize_bitmap & ~PAGE_MASK); + else if (cfg->pgsize_bitmap & PAGE_MASK) + granule = 1UL << __ffs(cfg->pgsize_bitmap & PAGE_MASK); + else + granule = 0; + + switch (granule) { + case SZ_4K: + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + break; + case SZ_16K: + cfg->pgsize_bitmap &= (SZ_16K | SZ_32M); + break; + case SZ_64K: + cfg->pgsize_bitmap &= (SZ_64K | SZ_512M); + break; + default: + cfg->pgsize_bitmap = 0; + } +} + +static struct arm_lpae_io_pgtable * +arm_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg) +{ + unsigned long va_bits, pgd_bits; + struct arm_lpae_io_pgtable *data; + + arm_lpae_restrict_pgsizes(cfg); + + if (!(cfg->pgsize_bitmap & (SZ_4K | SZ_16K | SZ_64K))) + return NULL; + + if (cfg->ias > ARM_LPAE_MAX_ADDR_BITS) + return NULL; + + if (cfg->oas > ARM_LPAE_MAX_ADDR_BITS) + return NULL; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + data->pg_shift = __ffs(cfg->pgsize_bitmap); + data->bits_per_level = data->pg_shift - ilog2(sizeof(arm_lpae_iopte)); + + va_bits = cfg->ias - data->pg_shift; + data->levels = DIV_ROUND_UP(va_bits, data->bits_per_level); + + /* Calculate the actual size of our pgd (without concatenation) */ + pgd_bits = va_bits - (data->bits_per_level * (data->levels - 1)); + data->pgd_size = 1UL << (pgd_bits + ilog2(sizeof(arm_lpae_iopte))); + + data->iop.ops = (struct io_pgtable_ops) { + .map = arm_lpae_map, + .unmap = arm_lpae_unmap, + .iova_to_phys = arm_lpae_iova_to_phys, + }; + + return data; +} + +static struct io_pgtable * +arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) +{ + u64 reg; + struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); + + if (!data) + return NULL; + + /* TCR */ + reg = (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + + switch (1 << data->pg_shift) { + case SZ_4K: + reg |= ARM_LPAE_TCR_TG0_4K; + break; + case SZ_16K: + reg |= ARM_LPAE_TCR_TG0_16K; + break; + case SZ_64K: + reg |= ARM_LPAE_TCR_TG0_64K; + break; + } + + switch (cfg->oas) { + case 32: + reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; + case 36: + reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; + case 40: + reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; + case 42: + reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; + case 44: + reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; + case 48: + reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_IPS_SHIFT); + break; + default: + goto out_free_data; + } + + reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; + cfg->arm_lpae_s1_cfg.tcr = reg; + + /* MAIRs */ + reg = (ARM_LPAE_MAIR_ATTR_NC + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_NC)) | + (ARM_LPAE_MAIR_ATTR_WBRWA + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_CACHE)) | + (ARM_LPAE_MAIR_ATTR_DEVICE + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); + + cfg->arm_lpae_s1_cfg.mair[0] = reg; + cfg->arm_lpae_s1_cfg.mair[1] = 0; + + /* Looking good; allocate a pgd */ + data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); + if (!data->pgd) + goto out_free_data; + + cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + + /* TTBRs */ + cfg->arm_lpae_s1_cfg.ttbr[0] = virt_to_phys(data->pgd); + cfg->arm_lpae_s1_cfg.ttbr[1] = 0; + return &data->iop; + +out_free_data: + kfree(data); + return NULL; +} + +static struct io_pgtable * +arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) +{ + u64 reg, sl; + struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); + + if (!data) + return NULL; + + /* + * Concatenate PGDs at level 1 if possible in order to reduce + * the depth of the stage-2 walk. + */ + if (data->levels == ARM_LPAE_MAX_LEVELS) { + unsigned long pgd_pages; + + pgd_pages = data->pgd_size >> ilog2(sizeof(arm_lpae_iopte)); + if (pgd_pages <= ARM_LPAE_S2_MAX_CONCAT_PAGES) { + data->pgd_size = pgd_pages << data->pg_shift; + data->levels--; + } + } + + /* VTCR */ + reg = ARM_64_LPAE_S2_TCR_RES1 | + (ARM_LPAE_TCR_SH_IS << ARM_LPAE_TCR_SH0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | + (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); + + sl = ARM_LPAE_START_LVL(data); + + switch (1 << data->pg_shift) { + case SZ_4K: + reg |= ARM_LPAE_TCR_TG0_4K; + sl++; /* SL0 format is different for 4K granule size */ + break; + case SZ_16K: + reg |= ARM_LPAE_TCR_TG0_16K; + break; + case SZ_64K: + reg |= ARM_LPAE_TCR_TG0_64K; + break; + } + + switch (cfg->oas) { + case 32: + reg |= (ARM_LPAE_TCR_PS_32_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; + case 36: + reg |= (ARM_LPAE_TCR_PS_36_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; + case 40: + reg |= (ARM_LPAE_TCR_PS_40_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; + case 42: + reg |= (ARM_LPAE_TCR_PS_42_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; + case 44: + reg |= (ARM_LPAE_TCR_PS_44_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; + case 48: + reg |= (ARM_LPAE_TCR_PS_48_BIT << ARM_LPAE_TCR_PS_SHIFT); + break; + default: + goto out_free_data; + } + + reg |= (64ULL - cfg->ias) << ARM_LPAE_TCR_T0SZ_SHIFT; + reg |= (~sl & ARM_LPAE_TCR_SL0_MASK) << ARM_LPAE_TCR_SL0_SHIFT; + cfg->arm_lpae_s2_cfg.vtcr = reg; + + /* Allocate pgd pages */ + data->pgd = alloc_pages_exact(data->pgd_size, GFP_KERNEL | __GFP_ZERO); + if (!data->pgd) + goto out_free_data; + + cfg->tlb->flush_pgtable(data->pgd, data->pgd_size, cookie); + + /* VTTBR */ + cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); + return &data->iop; + +out_free_data: + kfree(data); + return NULL; +} + +static struct io_pgtable * +arm_32_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct io_pgtable *iop; + + if (cfg->ias > 32 || cfg->oas > 40) + return NULL; + + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + iop = arm_64_lpae_alloc_pgtable_s1(cfg, cookie); + if (iop) { + cfg->arm_lpae_s1_cfg.tcr |= ARM_32_LPAE_TCR_EAE; + cfg->arm_lpae_s1_cfg.tcr &= 0xffffffff; + } + + return iop; +} + +static struct io_pgtable * +arm_32_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) +{ + struct io_pgtable *iop; + + if (cfg->ias > 40 || cfg->oas > 40) + return NULL; + + cfg->pgsize_bitmap &= (SZ_4K | SZ_2M | SZ_1G); + iop = arm_64_lpae_alloc_pgtable_s2(cfg, cookie); + if (iop) + cfg->arm_lpae_s2_cfg.vtcr &= 0xffffffff; + + return iop; +} + +struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { + .alloc = arm_64_lpae_alloc_pgtable_s1, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { + .alloc = arm_64_lpae_alloc_pgtable_s2, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { + .alloc = arm_32_lpae_alloc_pgtable_s1, + .free = arm_lpae_free_pgtable, +}; + +struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { + .alloc = arm_32_lpae_alloc_pgtable_s2, + .free = arm_lpae_free_pgtable, +}; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index f664a1c..6436fe2 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -24,9 +24,20 @@ #include "io-pgtable.h" +extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; + static const struct io_pgtable_init_fns * io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE + [ARM_32_LPAE_S1] = &io_pgtable_arm_32_lpae_s1_init_fns, + [ARM_32_LPAE_S2] = &io_pgtable_arm_32_lpae_s2_init_fns, + [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, + [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, +#endif }; struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index fdd792c..05c4e59 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -5,6 +5,10 @@ * Public API for use by IOMMU drivers */ enum io_pgtable_fmt { + ARM_32_LPAE_S1, + ARM_32_LPAE_S2, + ARM_64_LPAE_S1, + ARM_64_LPAE_S2, IO_PGTABLE_NUM_FMTS, }; @@ -47,6 +51,16 @@ struct io_pgtable_cfg { /* Low-level data specific to the table format */ union { + struct { + u64 ttbr[2]; + u64 tcr; + u64 mair[2]; + } arm_lpae_s1_cfg; + + struct { + u64 vttbr; + u64 vtcr; + } arm_lpae_s2_cfg; }; }; -- cgit v0.10.2 From fe4b991dcd84e0104cf2e29223a819335ed048a7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 17 Nov 2014 23:31:12 +0000 Subject: iommu: add self-consistency tests to ARM LPAE IO page table allocator This patch adds a series of basic self-consistency tests to the ARM LPAE IO page table allocator that exercise corner cases in map/unmap, as well as testing all valid configurations of pagesize, ias and stage. Signed-off-by: Will Deacon diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 306454f..9fd9909 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -28,6 +28,15 @@ config IOMMU_IO_PGTABLE_LPAE sizes at both stage-1 and stage-2, as well as address spaces up to 48-bits in size. +config IOMMU_IO_PGTABLE_LPAE_SELFTEST + bool "LPAE selftests" + depends on IOMMU_IO_PGTABLE_LPAE + help + Enable self-tests for LPAE page table allocator. This performs + a series of page-table consistency checks during boot. + + If unsure, say N here. + endmenu config OF_IOMMU diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index dbe6178..52fb214 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -193,6 +193,8 @@ struct arm_lpae_io_pgtable { typedef u64 arm_lpae_iopte; +static bool selftest_running = false; + static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, unsigned long iova, phys_addr_t paddr, arm_lpae_iopte prot, int lvl, @@ -201,8 +203,10 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, arm_lpae_iopte pte = prot; /* We require an unmap first */ - if (WARN_ON(iopte_leaf(*ptep, lvl))) + if (iopte_leaf(*ptep, lvl)) { + WARN_ON(!selftest_running); return -EEXIST; + } if (lvl == ARM_LPAE_MAX_LEVELS - 1) pte |= ARM_LPAE_PTE_TYPE_PAGE; @@ -779,3 +783,197 @@ struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { .alloc = arm_32_lpae_alloc_pgtable_s2, .free = arm_lpae_free_pgtable, }; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_LPAE_SELFTEST + +static struct io_pgtable_cfg *cfg_cookie; + +static void dummy_tlb_flush_all(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf, + void *cookie) +{ + WARN_ON(cookie != cfg_cookie); + WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); +} + +static void dummy_tlb_sync(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_flush_pgtable(void *ptr, size_t size, void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static struct iommu_gather_ops dummy_tlb_ops __initdata = { + .tlb_flush_all = dummy_tlb_flush_all, + .tlb_add_flush = dummy_tlb_add_flush, + .tlb_sync = dummy_tlb_sync, + .flush_pgtable = dummy_flush_pgtable, +}; + +static void __init arm_lpae_dump_ops(struct io_pgtable_ops *ops) +{ + struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + + pr_err("cfg: pgsize_bitmap 0x%lx, ias %u-bit\n", + cfg->pgsize_bitmap, cfg->ias); + pr_err("data: %d levels, 0x%zx pgd_size, %lu pg_shift, %lu bits_per_level, pgd @ %p\n", + data->levels, data->pgd_size, data->pg_shift, + data->bits_per_level, data->pgd); +} + +#define __FAIL(ops, i) ({ \ + WARN(1, "selftest: test failed for fmt idx %d\n", (i)); \ + arm_lpae_dump_ops(ops); \ + selftest_running = false; \ + -EFAULT; \ +}) + +static int __init arm_lpae_run_tests(struct io_pgtable_cfg *cfg) +{ + static const enum io_pgtable_fmt fmts[] = { + ARM_64_LPAE_S1, + ARM_64_LPAE_S2, + }; + + int i, j; + unsigned long iova; + size_t size; + struct io_pgtable_ops *ops; + + selftest_running = true; + + for (i = 0; i < ARRAY_SIZE(fmts); ++i) { + cfg_cookie = cfg; + ops = alloc_io_pgtable_ops(fmts[i], cfg, cfg); + if (!ops) { + pr_err("selftest: failed to allocate io pgtable ops\n"); + return -ENOMEM; + } + + /* + * Initial sanity checks. + * Empty page tables shouldn't provide any translations. + */ + if (ops->iova_to_phys(ops, 42)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, SZ_1G + 42)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, SZ_2G + 42)) + return __FAIL(ops, i); + + /* + * Distinct mappings of different granule sizes. + */ + iova = 0; + j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG); + while (j != BITS_PER_LONG) { + size = 1UL << j; + + if (ops->map(ops, iova, iova, size, IOMMU_READ | + IOMMU_WRITE | + IOMMU_NOEXEC | + IOMMU_CACHE)) + return __FAIL(ops, i); + + /* Overlapping mappings */ + if (!ops->map(ops, iova, iova + size, size, + IOMMU_READ | IOMMU_NOEXEC)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops, i); + + iova += SZ_1G; + j++; + j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j); + } + + /* Partial unmap */ + size = 1UL << __ffs(cfg->pgsize_bitmap); + if (ops->unmap(ops, SZ_1G + size, size) != size) + return __FAIL(ops, i); + + /* Remap of partial unmap */ + if (ops->map(ops, SZ_1G + size, size, size, IOMMU_READ)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, SZ_1G + size + 42) != (size + 42)) + return __FAIL(ops, i); + + /* Full unmap */ + iova = 0; + j = find_first_bit(&cfg->pgsize_bitmap, BITS_PER_LONG); + while (j != BITS_PER_LONG) { + size = 1UL << j; + + if (ops->unmap(ops, iova, size) != size) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42)) + return __FAIL(ops, i); + + /* Remap full block */ + if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) + return __FAIL(ops, i); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops, i); + + iova += SZ_1G; + j++; + j = find_next_bit(&cfg->pgsize_bitmap, BITS_PER_LONG, j); + } + + free_io_pgtable_ops(ops); + } + + selftest_running = false; + return 0; +} + +static int __init arm_lpae_do_selftests(void) +{ + static const unsigned long pgsize[] = { + SZ_4K | SZ_2M | SZ_1G, + SZ_16K | SZ_32M, + SZ_64K | SZ_512M, + }; + + static const unsigned int ias[] = { + 32, 36, 40, 42, 44, 48, + }; + + int i, j, pass = 0, fail = 0; + struct io_pgtable_cfg cfg = { + .tlb = &dummy_tlb_ops, + .oas = 48, + }; + + for (i = 0; i < ARRAY_SIZE(pgsize); ++i) { + for (j = 0; j < ARRAY_SIZE(ias); ++j) { + cfg.pgsize_bitmap = pgsize[i]; + cfg.ias = ias[j]; + pr_info("selftest: pgsize_bitmap 0x%08lx, IAS %u\n", + pgsize[i], ias[j]); + if (arm_lpae_run_tests(&cfg)) + fail++; + else + pass++; + } + } + + pr_info("selftest: completed with %d PASS %d FAIL\n", pass, fail); + return fail ? -EFAULT : 0; +} +subsys_initcall(arm_lpae_do_selftests); +#endif -- cgit v0.10.2 From c896c132b01895fd1445d178e36155b671c6f9ee Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sun, 14 Dec 2014 23:34:50 +0200 Subject: iommu: io-pgtable-arm: add non-secure quirk The quirk causes the Non-Secure bit to be set in all page table entries. Signed-off-by: Laurent Pinchart Signed-off-by: Will Deacon diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 52fb214..5a500ed 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -82,11 +82,13 @@ #define ARM_LPAE_PTE_TYPE_TABLE 3 #define ARM_LPAE_PTE_TYPE_PAGE 3 +#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63) #define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53) #define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10) #define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8) #define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8) #define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8) +#define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5) #define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0) #define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2) @@ -208,6 +210,9 @@ static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data, return -EEXIST; } + if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + pte |= ARM_LPAE_PTE_NS; + if (lvl == ARM_LPAE_MAX_LEVELS - 1) pte |= ARM_LPAE_PTE_TYPE_PAGE; else @@ -251,6 +256,8 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, data->iop.cfg.tlb->flush_pgtable(cptep, 1UL << data->pg_shift, cookie); pte = __pa(cptep) | ARM_LPAE_PTE_TYPE_TABLE; + if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS) + pte |= ARM_LPAE_PTE_NSTABLE; *ptep = pte; data->iop.cfg.tlb->flush_pgtable(ptep, sizeof(*ptep), cookie); } else { diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 05c4e59..10e32f6 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -43,7 +43,8 @@ struct iommu_gather_ops { * @tlb: TLB management callbacks for this set of tables. */ struct io_pgtable_cfg { - int quirks; /* IO_PGTABLE_QUIRK_* */ + #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */ + int quirks; unsigned long pgsize_bitmap; unsigned int ias; unsigned int oas; -- cgit v0.10.2 From 518f7136244c167538f732691be589959310b295 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Fri, 14 Nov 2014 17:17:54 +0000 Subject: iommu/arm-smmu: make use of generic LPAE allocator The ARM SMMU can walk LPAE page tables, so make use of the generic allocator. Signed-off-by: Will Deacon diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index b1f9a20..528c3fd 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -349,7 +349,6 @@ config ARM64_VA_BITS_42 config ARM64_VA_BITS_48 bool "48-bit" - depends on !ARM_SMMU endchoice diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 9fd9909..87060ad 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -330,13 +330,13 @@ config SPAPR_TCE_IOMMU config ARM_SMMU bool "ARM Ltd. System MMU (SMMU) Support" - depends on ARM64 || (ARM_LPAE && OF) + depends on ARM64 || ARM select IOMMU_API + select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU if ARM help Support for implementations of the ARM System MMU architecture - versions 1 and 2. The driver supports both v7l and v8l table - formats with 4k and 64k page sizes. + versions 1 and 2. Say Y here if your SoC includes an IOMMU device implementing the ARM SMMU architecture. diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 6cd47b7..919ba43 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -23,8 +23,6 @@ * - Stream-matching and stream-indexing * - v7/v8 long-descriptor format * - Non-secure access to the SMMU - * - 4k and 64k pages, with contiguous pte hints. - * - Up to 48-bit addressing (dependent on VA_BITS) * - Context fault reporting */ @@ -36,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -46,7 +43,7 @@ #include -#include +#include "io-pgtable.h" /* Maximum number of stream IDs assigned to a single device */ #define MAX_MASTER_STREAMIDS MAX_PHANDLE_ARGS @@ -71,40 +68,6 @@ ((smmu->options & ARM_SMMU_OPT_SECURE_CFG_ACCESS) \ ? 0x400 : 0)) -/* Page table bits */ -#define ARM_SMMU_PTE_XN (((pteval_t)3) << 53) -#define ARM_SMMU_PTE_CONT (((pteval_t)1) << 52) -#define ARM_SMMU_PTE_AF (((pteval_t)1) << 10) -#define ARM_SMMU_PTE_SH_NS (((pteval_t)0) << 8) -#define ARM_SMMU_PTE_SH_OS (((pteval_t)2) << 8) -#define ARM_SMMU_PTE_SH_IS (((pteval_t)3) << 8) -#define ARM_SMMU_PTE_PAGE (((pteval_t)3) << 0) - -#if PAGE_SIZE == SZ_4K -#define ARM_SMMU_PTE_CONT_ENTRIES 16 -#elif PAGE_SIZE == SZ_64K -#define ARM_SMMU_PTE_CONT_ENTRIES 32 -#else -#define ARM_SMMU_PTE_CONT_ENTRIES 1 -#endif - -#define ARM_SMMU_PTE_CONT_SIZE (PAGE_SIZE * ARM_SMMU_PTE_CONT_ENTRIES) -#define ARM_SMMU_PTE_CONT_MASK (~(ARM_SMMU_PTE_CONT_SIZE - 1)) - -/* Stage-1 PTE */ -#define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6) -#define ARM_SMMU_PTE_AP_RDONLY (((pteval_t)2) << 6) -#define ARM_SMMU_PTE_ATTRINDX_SHIFT 2 -#define ARM_SMMU_PTE_nG (((pteval_t)1) << 11) - -/* Stage-2 PTE */ -#define ARM_SMMU_PTE_HAP_FAULT (((pteval_t)0) << 6) -#define ARM_SMMU_PTE_HAP_READ (((pteval_t)1) << 6) -#define ARM_SMMU_PTE_HAP_WRITE (((pteval_t)2) << 6) -#define ARM_SMMU_PTE_MEMATTR_OIWB (((pteval_t)0xf) << 2) -#define ARM_SMMU_PTE_MEMATTR_NC (((pteval_t)0x5) << 2) -#define ARM_SMMU_PTE_MEMATTR_DEV (((pteval_t)0x1) << 2) - /* Configuration registers */ #define ARM_SMMU_GR0_sCR0 0x0 #define sCR0_CLIENTPD (1 << 0) @@ -132,17 +95,11 @@ #define ARM_SMMU_GR0_sGFSYNR0 0x50 #define ARM_SMMU_GR0_sGFSYNR1 0x54 #define ARM_SMMU_GR0_sGFSYNR2 0x58 -#define ARM_SMMU_GR0_PIDR0 0xfe0 -#define ARM_SMMU_GR0_PIDR1 0xfe4 -#define ARM_SMMU_GR0_PIDR2 0xfe8 #define ID0_S1TS (1 << 30) #define ID0_S2TS (1 << 29) #define ID0_NTS (1 << 28) #define ID0_SMS (1 << 27) -#define ID0_PTFS_SHIFT 24 -#define ID0_PTFS_MASK 0x2 -#define ID0_PTFS_V8_ONLY 0x2 #define ID0_CTTW (1 << 14) #define ID0_NUMIRPT_SHIFT 16 #define ID0_NUMIRPT_MASK 0xff @@ -169,9 +126,6 @@ #define ID2_PTFS_16K (1 << 13) #define ID2_PTFS_64K (1 << 14) -#define PIDR2_ARCH_SHIFT 4 -#define PIDR2_ARCH_MASK 0xf - /* Global TLB invalidation */ #define ARM_SMMU_GR0_STLBIALL 0x60 #define ARM_SMMU_GR0_TLBIVMID 0x64 @@ -231,13 +185,20 @@ #define ARM_SMMU_CB_TTBCR2 0x10 #define ARM_SMMU_CB_TTBR0_LO 0x20 #define ARM_SMMU_CB_TTBR0_HI 0x24 +#define ARM_SMMU_CB_TTBR1_LO 0x28 +#define ARM_SMMU_CB_TTBR1_HI 0x2c #define ARM_SMMU_CB_TTBCR 0x30 #define ARM_SMMU_CB_S1_MAIR0 0x38 +#define ARM_SMMU_CB_S1_MAIR1 0x3c #define ARM_SMMU_CB_FSR 0x58 #define ARM_SMMU_CB_FAR_LO 0x60 #define ARM_SMMU_CB_FAR_HI 0x64 #define ARM_SMMU_CB_FSYNR0 0x68 +#define ARM_SMMU_CB_S1_TLBIVA 0x600 #define ARM_SMMU_CB_S1_TLBIASID 0x610 +#define ARM_SMMU_CB_S1_TLBIVAL 0x620 +#define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 +#define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 #define SCTLR_S1_ASIDPNE (1 << 12) #define SCTLR_CFCFG (1 << 7) @@ -252,44 +213,9 @@ #define RESUME_RETRY (0 << 0) #define RESUME_TERMINATE (1 << 0) -#define TTBCR_EAE (1 << 31) - -#define TTBCR_PASIZE_SHIFT 16 -#define TTBCR_PASIZE_MASK 0x7 - -#define TTBCR_TG0_4K (0 << 14) -#define TTBCR_TG0_64K (1 << 14) - -#define TTBCR_SH0_SHIFT 12 -#define TTBCR_SH0_MASK 0x3 -#define TTBCR_SH_NS 0 -#define TTBCR_SH_OS 2 -#define TTBCR_SH_IS 3 - -#define TTBCR_ORGN0_SHIFT 10 -#define TTBCR_IRGN0_SHIFT 8 -#define TTBCR_RGN_MASK 0x3 -#define TTBCR_RGN_NC 0 -#define TTBCR_RGN_WBWA 1 -#define TTBCR_RGN_WT 2 -#define TTBCR_RGN_WB 3 - -#define TTBCR_SL0_SHIFT 6 -#define TTBCR_SL0_MASK 0x3 -#define TTBCR_SL0_LVL_2 0 -#define TTBCR_SL0_LVL_1 1 - -#define TTBCR_T1SZ_SHIFT 16 -#define TTBCR_T0SZ_SHIFT 0 -#define TTBCR_SZ_MASK 0xf - #define TTBCR2_SEP_SHIFT 15 #define TTBCR2_SEP_MASK 0x7 -#define TTBCR2_PASIZE_SHIFT 0 -#define TTBCR2_PASIZE_MASK 0x7 - -/* Common definitions for PASize and SEP fields */ #define TTBCR2_ADDR_32 0 #define TTBCR2_ADDR_36 1 #define TTBCR2_ADDR_40 2 @@ -297,16 +223,7 @@ #define TTBCR2_ADDR_44 4 #define TTBCR2_ADDR_48 5 -#define TTBRn_HI_ASID_SHIFT 16 - -#define MAIR_ATTR_SHIFT(n) ((n) << 3) -#define MAIR_ATTR_MASK 0xff -#define MAIR_ATTR_DEVICE 0x04 -#define MAIR_ATTR_NC 0x44 -#define MAIR_ATTR_WBRWA 0xff -#define MAIR_ATTR_IDX_NC 0 -#define MAIR_ATTR_IDX_CACHE 1 -#define MAIR_ATTR_IDX_DEV 2 +#define TTBRn_HI_ASID_SHIFT 16 #define FSR_MULTI (1 << 31) #define FSR_SS (1 << 30) @@ -380,10 +297,9 @@ struct arm_smmu_device { u32 num_mapping_groups; DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS); - unsigned long s1_input_size; - unsigned long s1_output_size; - unsigned long s2_input_size; - unsigned long s2_output_size; + unsigned long va_size; + unsigned long ipa_size; + unsigned long pa_size; u32 num_global_irqs; u32 num_context_irqs; @@ -397,7 +313,6 @@ struct arm_smmu_cfg { u8 cbndx; u8 irptndx; u32 cbar; - pgd_t *pgd; }; #define INVALID_IRPTNDX 0xff @@ -412,11 +327,15 @@ enum arm_smmu_domain_stage { struct arm_smmu_domain { struct arm_smmu_device *smmu; + struct io_pgtable_ops *pgtbl_ops; + spinlock_t pgtbl_lock; struct arm_smmu_cfg cfg; enum arm_smmu_domain_stage stage; - spinlock_t lock; + struct mutex init_mutex; /* Protects smmu pointer */ }; +static struct iommu_ops arm_smmu_ops; + static DEFINE_SPINLOCK(arm_smmu_devices_lock); static LIST_HEAD(arm_smmu_devices); @@ -597,7 +516,7 @@ static void __arm_smmu_free_bitmap(unsigned long *map, int idx) } /* Wait for any pending TLB invalidations to complete */ -static void arm_smmu_tlb_sync(struct arm_smmu_device *smmu) +static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu) { int count = 0; void __iomem *gr0_base = ARM_SMMU_GR0(smmu); @@ -615,12 +534,19 @@ static void arm_smmu_tlb_sync(struct arm_smmu_device *smmu) } } -static void arm_smmu_tlb_inv_context(struct arm_smmu_domain *smmu_domain) +static void arm_smmu_tlb_sync(void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + __arm_smmu_tlb_sync(smmu_domain->smmu); +} + +static void arm_smmu_tlb_inv_context(void *cookie) { + struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; struct arm_smmu_device *smmu = smmu_domain->smmu; - void __iomem *base = ARM_SMMU_GR0(smmu); bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; + void __iomem *base; if (stage1) { base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); @@ -632,9 +558,76 @@ static void arm_smmu_tlb_inv_context(struct arm_smmu_domain *smmu_domain) base + ARM_SMMU_GR0_TLBIVMID); } - arm_smmu_tlb_sync(smmu); + __arm_smmu_tlb_sync(smmu); +} + +static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, + bool leaf, void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct arm_smmu_device *smmu = smmu_domain->smmu; + bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; + void __iomem *reg; + + if (stage1) { + reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); + reg += leaf ? ARM_SMMU_CB_S1_TLBIVAL : ARM_SMMU_CB_S1_TLBIVA; + + if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { + iova &= ~12UL; + iova |= ARM_SMMU_CB_ASID(cfg); + writel_relaxed(iova, reg); +#ifdef CONFIG_64BIT + } else { + iova >>= 12; + iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48; + writeq_relaxed(iova, reg); +#endif + } +#ifdef CONFIG_64BIT + } else if (smmu->version == ARM_SMMU_V2) { + reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); + reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : + ARM_SMMU_CB_S2_TLBIIPAS2; + writeq_relaxed(iova >> 12, reg); +#endif + } else { + reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; + writel_relaxed(ARM_SMMU_CB_VMID(cfg), reg); + } +} + +static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_device *smmu = smmu_domain->smmu; + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + + + /* Ensure new page tables are visible to the hardware walker */ + if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { + dsb(ishst); + } else { + /* + * If the SMMU can't walk tables in the CPU caches, treat them + * like non-coherent DMA since we need to flush the new entries + * all the way out to memory. There's no possibility of + * recursion here as the SMMU table walker will not be wired + * through another SMMU. + */ + dma_map_page(smmu->dev, virt_to_page(addr), offset, size, + DMA_TO_DEVICE); + } } +static struct iommu_gather_ops arm_smmu_gather_ops = { + .tlb_flush_all = arm_smmu_tlb_inv_context, + .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, + .tlb_sync = arm_smmu_tlb_sync, + .flush_pgtable = arm_smmu_flush_pgtable, +}; + static irqreturn_t arm_smmu_context_fault(int irq, void *dev) { int flags, ret; @@ -712,29 +705,8 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev) return IRQ_HANDLED; } -static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr, - size_t size) -{ - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; - - - /* Ensure new page tables are visible to the hardware walker */ - if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) { - dsb(ishst); - } else { - /* - * If the SMMU can't walk tables in the CPU caches, treat them - * like non-coherent DMA since we need to flush the new entries - * all the way out to memory. There's no possibility of - * recursion here as the SMMU table walker will not be wired - * through another SMMU. - */ - dma_map_page(smmu->dev, virt_to_page(addr), offset, size, - DMA_TO_DEVICE); - } -} - -static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) +static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, + struct io_pgtable_cfg *pgtbl_cfg) { u32 reg; bool stage1; @@ -771,124 +743,68 @@ static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) #else reg = CBA2R_RW64_32BIT; #endif - writel_relaxed(reg, - gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); - - /* TTBCR2 */ - switch (smmu->s1_input_size) { - case 32: - reg = (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT); - break; - case 36: - reg = (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT); - break; - case 39: - case 40: - reg = (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT); - break; - case 42: - reg = (TTBCR2_ADDR_42 << TTBCR2_SEP_SHIFT); - break; - case 44: - reg = (TTBCR2_ADDR_44 << TTBCR2_SEP_SHIFT); - break; - case 48: - reg = (TTBCR2_ADDR_48 << TTBCR2_SEP_SHIFT); - break; - } - - switch (smmu->s1_output_size) { - case 32: - reg |= (TTBCR2_ADDR_32 << TTBCR2_PASIZE_SHIFT); - break; - case 36: - reg |= (TTBCR2_ADDR_36 << TTBCR2_PASIZE_SHIFT); - break; - case 39: - case 40: - reg |= (TTBCR2_ADDR_40 << TTBCR2_PASIZE_SHIFT); - break; - case 42: - reg |= (TTBCR2_ADDR_42 << TTBCR2_PASIZE_SHIFT); - break; - case 44: - reg |= (TTBCR2_ADDR_44 << TTBCR2_PASIZE_SHIFT); - break; - case 48: - reg |= (TTBCR2_ADDR_48 << TTBCR2_PASIZE_SHIFT); - break; - } - - if (stage1) - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2); + writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBA2R(cfg->cbndx)); } - /* TTBR0 */ - arm_smmu_flush_pgtable(smmu, cfg->pgd, - PTRS_PER_PGD * sizeof(pgd_t)); - reg = __pa(cfg->pgd); - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); - reg = (phys_addr_t)__pa(cfg->pgd) >> 32; - if (stage1) + /* TTBRs */ + if (stage1) { + reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); + reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0] >> 32; reg |= ARM_SMMU_CB_ASID(cfg) << TTBRn_HI_ASID_SHIFT; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); - - /* - * TTBCR - * We use long descriptor, with inner-shareable WBWA tables in TTBR0. - */ - if (smmu->version > ARM_SMMU_V1) { - if (PAGE_SIZE == SZ_4K) - reg = TTBCR_TG0_4K; - else - reg = TTBCR_TG0_64K; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); - if (!stage1) { - reg |= (64 - smmu->s2_input_size) << TTBCR_T0SZ_SHIFT; + reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1]; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1_LO); + reg = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[1] >> 32; + reg |= ARM_SMMU_CB_ASID(cfg) << TTBRn_HI_ASID_SHIFT; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR1_HI); + } else { + reg = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); + reg = pgtbl_cfg->arm_lpae_s2_cfg.vttbr >> 32; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); + } - switch (smmu->s2_output_size) { + /* TTBCR */ + if (stage1) { + reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); + if (smmu->version > ARM_SMMU_V1) { + reg = pgtbl_cfg->arm_lpae_s1_cfg.tcr >> 32; + switch (smmu->va_size) { case 32: - reg |= (TTBCR2_ADDR_32 << TTBCR_PASIZE_SHIFT); + reg |= (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT); break; case 36: - reg |= (TTBCR2_ADDR_36 << TTBCR_PASIZE_SHIFT); + reg |= (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT); break; case 40: - reg |= (TTBCR2_ADDR_40 << TTBCR_PASIZE_SHIFT); + reg |= (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT); break; case 42: - reg |= (TTBCR2_ADDR_42 << TTBCR_PASIZE_SHIFT); + reg |= (TTBCR2_ADDR_42 << TTBCR2_SEP_SHIFT); break; case 44: - reg |= (TTBCR2_ADDR_44 << TTBCR_PASIZE_SHIFT); + reg |= (TTBCR2_ADDR_44 << TTBCR2_SEP_SHIFT); break; case 48: - reg |= (TTBCR2_ADDR_48 << TTBCR_PASIZE_SHIFT); + reg |= (TTBCR2_ADDR_48 << TTBCR2_SEP_SHIFT); break; } - } else { - reg |= (64 - smmu->s1_input_size) << TTBCR_T0SZ_SHIFT; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2); } } else { - reg = 0; + reg = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); } - reg |= TTBCR_EAE | - (TTBCR_SH_IS << TTBCR_SH0_SHIFT) | - (TTBCR_RGN_WBWA << TTBCR_ORGN0_SHIFT) | - (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT); - - if (!stage1) - reg |= (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT); - - writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); - - /* MAIR0 (stage-1 only) */ + /* MAIRs (stage-1 only) */ if (stage1) { - reg = (MAIR_ATTR_NC << MAIR_ATTR_SHIFT(MAIR_ATTR_IDX_NC)) | - (MAIR_ATTR_WBRWA << MAIR_ATTR_SHIFT(MAIR_ATTR_IDX_CACHE)) | - (MAIR_ATTR_DEVICE << MAIR_ATTR_SHIFT(MAIR_ATTR_IDX_DEV)); + reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0); + reg = pgtbl_cfg->arm_lpae_s1_cfg.mair[1]; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR1); } /* SCTLR */ @@ -905,11 +821,14 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, struct arm_smmu_device *smmu) { int irq, start, ret = 0; - unsigned long flags; + unsigned long ias, oas; + struct io_pgtable_ops *pgtbl_ops; + struct io_pgtable_cfg pgtbl_cfg; + enum io_pgtable_fmt fmt; struct arm_smmu_domain *smmu_domain = domain->priv; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - spin_lock_irqsave(&smmu_domain->lock, flags); + mutex_lock(&smmu_domain->init_mutex); if (smmu_domain->smmu) goto out_unlock; @@ -940,6 +859,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, case ARM_SMMU_DOMAIN_S1: cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; start = smmu->num_s2_context_banks; + ias = smmu->va_size; + oas = smmu->ipa_size; + if (IS_ENABLED(CONFIG_64BIT)) + fmt = ARM_64_LPAE_S1; + else + fmt = ARM_32_LPAE_S1; break; case ARM_SMMU_DOMAIN_NESTED: /* @@ -949,6 +874,12 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, case ARM_SMMU_DOMAIN_S2: cfg->cbar = CBAR_TYPE_S2_TRANS; start = 0; + ias = smmu->ipa_size; + oas = smmu->pa_size; + if (IS_ENABLED(CONFIG_64BIT)) + fmt = ARM_64_LPAE_S2; + else + fmt = ARM_32_LPAE_S2; break; default: ret = -EINVAL; @@ -968,10 +899,30 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, cfg->irptndx = cfg->cbndx; } - ACCESS_ONCE(smmu_domain->smmu) = smmu; - arm_smmu_init_context_bank(smmu_domain); - spin_unlock_irqrestore(&smmu_domain->lock, flags); + pgtbl_cfg = (struct io_pgtable_cfg) { + .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, + .ias = ias, + .oas = oas, + .tlb = &arm_smmu_gather_ops, + }; + + smmu_domain->smmu = smmu; + pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); + if (!pgtbl_ops) { + ret = -ENOMEM; + goto out_clear_smmu; + } + + /* Update our support page sizes to reflect the page table format */ + arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + + /* Initialise the context bank with our page table cfg */ + arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); + /* + * Request context fault interrupt. Do this last to avoid the + * handler seeing a half-initialised domain state. + */ irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, "arm-smmu-context-fault", domain); @@ -981,10 +932,16 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, cfg->irptndx = INVALID_IRPTNDX; } + mutex_unlock(&smmu_domain->init_mutex); + + /* Publish page table ops for map/unmap */ + smmu_domain->pgtbl_ops = pgtbl_ops; return 0; +out_clear_smmu: + smmu_domain->smmu = NULL; out_unlock: - spin_unlock_irqrestore(&smmu_domain->lock, flags); + mutex_unlock(&smmu_domain->init_mutex); return ret; } @@ -999,23 +956,27 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) if (!smmu) return; - /* Disable the context bank and nuke the TLB before freeing it. */ + /* + * Disable the context bank and free the page tables before freeing + * it. + */ cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); writel_relaxed(0, cb_base + ARM_SMMU_CB_SCTLR); - arm_smmu_tlb_inv_context(smmu_domain); if (cfg->irptndx != INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; free_irq(irq, domain); } + if (smmu_domain->pgtbl_ops) + free_io_pgtable_ops(smmu_domain->pgtbl_ops); + __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); } static int arm_smmu_domain_init(struct iommu_domain *domain) { struct arm_smmu_domain *smmu_domain; - pgd_t *pgd; /* * Allocate the domain and initialise some of its data structures. @@ -1026,81 +987,10 @@ static int arm_smmu_domain_init(struct iommu_domain *domain) if (!smmu_domain) return -ENOMEM; - pgd = kcalloc(PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL); - if (!pgd) - goto out_free_domain; - smmu_domain->cfg.pgd = pgd; - - spin_lock_init(&smmu_domain->lock); + mutex_init(&smmu_domain->init_mutex); + spin_lock_init(&smmu_domain->pgtbl_lock); domain->priv = smmu_domain; return 0; - -out_free_domain: - kfree(smmu_domain); - return -ENOMEM; -} - -static void arm_smmu_free_ptes(pmd_t *pmd) -{ - pgtable_t table = pmd_pgtable(*pmd); - - __free_page(table); -} - -static void arm_smmu_free_pmds(pud_t *pud) -{ - int i; - pmd_t *pmd, *pmd_base = pmd_offset(pud, 0); - - pmd = pmd_base; - for (i = 0; i < PTRS_PER_PMD; ++i) { - if (pmd_none(*pmd)) - continue; - - arm_smmu_free_ptes(pmd); - pmd++; - } - - pmd_free(NULL, pmd_base); -} - -static void arm_smmu_free_puds(pgd_t *pgd) -{ - int i; - pud_t *pud, *pud_base = pud_offset(pgd, 0); - - pud = pud_base; - for (i = 0; i < PTRS_PER_PUD; ++i) { - if (pud_none(*pud)) - continue; - - arm_smmu_free_pmds(pud); - pud++; - } - - pud_free(NULL, pud_base); -} - -static void arm_smmu_free_pgtables(struct arm_smmu_domain *smmu_domain) -{ - int i; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - pgd_t *pgd, *pgd_base = cfg->pgd; - - /* - * Recursively free the page tables for this domain. We don't - * care about speculative TLB filling because the tables should - * not be active in any context bank at this point (SCTLR.M is 0). - */ - pgd = pgd_base; - for (i = 0; i < PTRS_PER_PGD; ++i) { - if (pgd_none(*pgd)) - continue; - arm_smmu_free_puds(pgd); - pgd++; - } - - kfree(pgd_base); } static void arm_smmu_domain_destroy(struct iommu_domain *domain) @@ -1112,7 +1002,6 @@ static void arm_smmu_domain_destroy(struct iommu_domain *domain) * already been detached. */ arm_smmu_destroy_domain_context(domain); - arm_smmu_free_pgtables(smmu_domain); kfree(smmu_domain); } @@ -1244,7 +1133,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; struct arm_smmu_domain *smmu_domain = domain->priv; - struct arm_smmu_device *smmu, *dom_smmu; + struct arm_smmu_device *smmu; struct arm_smmu_master_cfg *cfg; smmu = find_smmu_for_device(dev); @@ -1258,21 +1147,16 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -EEXIST; } + /* Ensure that the domain is finalised */ + ret = arm_smmu_init_domain_context(domain, smmu); + if (IS_ERR_VALUE(ret)) + return ret; + /* * Sanity check the domain. We don't support domains across * different SMMUs. */ - dom_smmu = ACCESS_ONCE(smmu_domain->smmu); - if (!dom_smmu) { - /* Now that we have a master, we can finalise the domain */ - ret = arm_smmu_init_domain_context(domain, smmu); - if (IS_ERR_VALUE(ret)) - return ret; - - dom_smmu = smmu_domain->smmu; - } - - if (dom_smmu != smmu) { + if (smmu_domain->smmu != smmu) { dev_err(dev, "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); @@ -1303,293 +1187,55 @@ static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) arm_smmu_domain_remove_master(smmu_domain, cfg); } -static bool arm_smmu_pte_is_contiguous_range(unsigned long addr, - unsigned long end) -{ - return !(addr & ~ARM_SMMU_PTE_CONT_MASK) && - (addr + ARM_SMMU_PTE_CONT_SIZE <= end); -} - -static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd, - unsigned long addr, unsigned long end, - unsigned long pfn, int prot, int stage) -{ - pte_t *pte, *start; - pteval_t pteval = ARM_SMMU_PTE_PAGE | ARM_SMMU_PTE_AF; - - if (pmd_none(*pmd)) { - /* Allocate a new set of tables */ - pgtable_t table = alloc_page(GFP_ATOMIC|__GFP_ZERO); - - if (!table) - return -ENOMEM; - - arm_smmu_flush_pgtable(smmu, page_address(table), PAGE_SIZE); - pmd_populate(NULL, pmd, table); - arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd)); - } - - if (stage == 1) { - pteval |= ARM_SMMU_PTE_AP_UNPRIV | ARM_SMMU_PTE_nG; - if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) - pteval |= ARM_SMMU_PTE_AP_RDONLY; - - if (prot & IOMMU_CACHE) - pteval |= (MAIR_ATTR_IDX_CACHE << - ARM_SMMU_PTE_ATTRINDX_SHIFT); - } else { - pteval |= ARM_SMMU_PTE_HAP_FAULT; - if (prot & IOMMU_READ) - pteval |= ARM_SMMU_PTE_HAP_READ; - if (prot & IOMMU_WRITE) - pteval |= ARM_SMMU_PTE_HAP_WRITE; - if (prot & IOMMU_CACHE) - pteval |= ARM_SMMU_PTE_MEMATTR_OIWB; - else - pteval |= ARM_SMMU_PTE_MEMATTR_NC; - } - - if (prot & IOMMU_NOEXEC) - pteval |= ARM_SMMU_PTE_XN; - - /* If no access, create a faulting entry to avoid TLB fills */ - if (!(prot & (IOMMU_READ | IOMMU_WRITE))) - pteval &= ~ARM_SMMU_PTE_PAGE; - - pteval |= ARM_SMMU_PTE_SH_IS; - start = pmd_page_vaddr(*pmd) + pte_index(addr); - pte = start; - - /* - * Install the page table entries. This is fairly complicated - * since we attempt to make use of the contiguous hint in the - * ptes where possible. The contiguous hint indicates a series - * of ARM_SMMU_PTE_CONT_ENTRIES ptes mapping a physically - * contiguous region with the following constraints: - * - * - The region start is aligned to ARM_SMMU_PTE_CONT_SIZE - * - Each pte in the region has the contiguous hint bit set - * - * This complicates unmapping (also handled by this code, when - * neither IOMMU_READ or IOMMU_WRITE are set) because it is - * possible, yet highly unlikely, that a client may unmap only - * part of a contiguous range. This requires clearing of the - * contiguous hint bits in the range before installing the new - * faulting entries. - * - * Note that re-mapping an address range without first unmapping - * it is not supported, so TLB invalidation is not required here - * and is instead performed at unmap and domain-init time. - */ - do { - int i = 1; - - pteval &= ~ARM_SMMU_PTE_CONT; - - if (arm_smmu_pte_is_contiguous_range(addr, end)) { - i = ARM_SMMU_PTE_CONT_ENTRIES; - pteval |= ARM_SMMU_PTE_CONT; - } else if (pte_val(*pte) & - (ARM_SMMU_PTE_CONT | ARM_SMMU_PTE_PAGE)) { - int j; - pte_t *cont_start; - unsigned long idx = pte_index(addr); - - idx &= ~(ARM_SMMU_PTE_CONT_ENTRIES - 1); - cont_start = pmd_page_vaddr(*pmd) + idx; - for (j = 0; j < ARM_SMMU_PTE_CONT_ENTRIES; ++j) - pte_val(*(cont_start + j)) &= - ~ARM_SMMU_PTE_CONT; - - arm_smmu_flush_pgtable(smmu, cont_start, - sizeof(*pte) * - ARM_SMMU_PTE_CONT_ENTRIES); - } - - do { - *pte = pfn_pte(pfn, __pgprot(pteval)); - } while (pte++, pfn++, addr += PAGE_SIZE, --i); - } while (addr != end); - - arm_smmu_flush_pgtable(smmu, start, sizeof(*pte) * (pte - start)); - return 0; -} - -static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud, - unsigned long addr, unsigned long end, - phys_addr_t phys, int prot, int stage) -{ - int ret; - pmd_t *pmd; - unsigned long next, pfn = __phys_to_pfn(phys); - -#ifndef __PAGETABLE_PMD_FOLDED - if (pud_none(*pud)) { - pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); - if (!pmd) - return -ENOMEM; - - arm_smmu_flush_pgtable(smmu, pmd, PAGE_SIZE); - pud_populate(NULL, pud, pmd); - arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud)); - - pmd += pmd_index(addr); - } else -#endif - pmd = pmd_offset(pud, addr); - - do { - next = pmd_addr_end(addr, end); - ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, next, pfn, - prot, stage); - phys += next - addr; - pfn = __phys_to_pfn(phys); - } while (pmd++, addr = next, addr < end); - - return ret; -} - -static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd, - unsigned long addr, unsigned long end, - phys_addr_t phys, int prot, int stage) -{ - int ret = 0; - pud_t *pud; - unsigned long next; - -#ifndef __PAGETABLE_PUD_FOLDED - if (pgd_none(*pgd)) { - pud = (pud_t *)get_zeroed_page(GFP_ATOMIC); - if (!pud) - return -ENOMEM; - - arm_smmu_flush_pgtable(smmu, pud, PAGE_SIZE); - pgd_populate(NULL, pgd, pud); - arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd)); - - pud += pud_index(addr); - } else -#endif - pud = pud_offset(pgd, addr); - - do { - next = pud_addr_end(addr, end); - ret = arm_smmu_alloc_init_pmd(smmu, pud, addr, next, phys, - prot, stage); - phys += next - addr; - } while (pud++, addr = next, addr < end); - - return ret; -} - -static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, - unsigned long iova, phys_addr_t paddr, - size_t size, int prot) -{ - int ret, stage; - unsigned long end; - phys_addr_t input_mask, output_mask; - struct arm_smmu_device *smmu = smmu_domain->smmu; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; - pgd_t *pgd = cfg->pgd; - unsigned long flags; - - if (cfg->cbar == CBAR_TYPE_S2_TRANS) { - stage = 2; - input_mask = (1ULL << smmu->s2_input_size) - 1; - output_mask = (1ULL << smmu->s2_output_size) - 1; - } else { - stage = 1; - input_mask = (1ULL << smmu->s1_input_size) - 1; - output_mask = (1ULL << smmu->s1_output_size) - 1; - } - - if (!pgd) - return -EINVAL; - - if (size & ~PAGE_MASK) - return -EINVAL; - - if ((phys_addr_t)iova & ~input_mask) - return -ERANGE; - - if (paddr & ~output_mask) - return -ERANGE; - - spin_lock_irqsave(&smmu_domain->lock, flags); - pgd += pgd_index(iova); - end = iova + size; - do { - unsigned long next = pgd_addr_end(iova, end); - - ret = arm_smmu_alloc_init_pud(smmu, pgd, iova, next, paddr, - prot, stage); - if (ret) - goto out_unlock; - - paddr += next - iova; - iova = next; - } while (pgd++, iova != end); - -out_unlock: - spin_unlock_irqrestore(&smmu_domain->lock, flags); - - return ret; -} - static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { + int ret; + unsigned long flags; struct arm_smmu_domain *smmu_domain = domain->priv; + struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; - if (!smmu_domain) + if (!ops) return -ENODEV; - return arm_smmu_handle_mapping(smmu_domain, iova, paddr, size, prot); + spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); + ret = ops->map(ops, iova, paddr, size, prot); + spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + return ret; } static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) { - int ret; + size_t ret; + unsigned long flags; struct arm_smmu_domain *smmu_domain = domain->priv; + struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; - ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0); - arm_smmu_tlb_inv_context(smmu_domain); - return ret ? 0 : size; + if (!ops) + return 0; + + spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); + ret = ops->unmap(ops, iova, size); + spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + return ret; } static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) { - pgd_t *pgdp, pgd; - pud_t pud; - pmd_t pmd; - pte_t pte; + phys_addr_t ret; + unsigned long flags; struct arm_smmu_domain *smmu_domain = domain->priv; - struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; - pgdp = cfg->pgd; - if (!pgdp) + if (!ops) return 0; - pgd = *(pgdp + pgd_index(iova)); - if (pgd_none(pgd)) - return 0; - - pud = *pud_offset(&pgd, iova); - if (pud_none(pud)) - return 0; - - pmd = *pmd_offset(&pud, iova); - if (pmd_none(pmd)) - return 0; - - pte = *(pmd_page_vaddr(pmd) + pte_index(iova)); - if (pte_none(pte)) - return 0; - - return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); + spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); + ret = ops->iova_to_phys(ops, iova); + spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + return ret; } static bool arm_smmu_capable(enum iommu_cap cap) @@ -1698,24 +1344,34 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain, static int arm_smmu_domain_set_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) { + int ret = 0; struct arm_smmu_domain *smmu_domain = domain->priv; + mutex_lock(&smmu_domain->init_mutex); + switch (attr) { case DOMAIN_ATTR_NESTING: - if (smmu_domain->smmu) - return -EPERM; + if (smmu_domain->smmu) { + ret = -EPERM; + goto out_unlock; + } + if (*(int *)data) smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; else smmu_domain->stage = ARM_SMMU_DOMAIN_S1; - return 0; + break; default: - return -ENODEV; + ret = -ENODEV; } + +out_unlock: + mutex_unlock(&smmu_domain->init_mutex); + return ret; } -static const struct iommu_ops arm_smmu_ops = { +static struct iommu_ops arm_smmu_ops = { .capable = arm_smmu_capable, .domain_init = arm_smmu_domain_init, .domain_destroy = arm_smmu_domain_destroy, @@ -1729,9 +1385,7 @@ static const struct iommu_ops arm_smmu_ops = { .remove_device = arm_smmu_remove_device, .domain_get_attr = arm_smmu_domain_get_attr, .domain_set_attr = arm_smmu_domain_set_attr, - .pgsize_bitmap = (SECTION_SIZE | - ARM_SMMU_PTE_CONT_SIZE | - PAGE_SIZE), + .pgsize_bitmap = -1UL, /* Restricted during device attach */ }; static void arm_smmu_device_reset(struct arm_smmu_device *smmu) @@ -1782,7 +1436,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) reg &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); /* Push the button */ - arm_smmu_tlb_sync(smmu); + __arm_smmu_tlb_sync(smmu); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); } @@ -1816,12 +1470,6 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) /* ID0 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); -#ifndef CONFIG_64BIT - if (((id >> ID0_PTFS_SHIFT) & ID0_PTFS_MASK) == ID0_PTFS_V8_ONLY) { - dev_err(smmu->dev, "\tno v7 descriptor support!\n"); - return -ENODEV; - } -#endif /* Restrict available stages based on module parameter */ if (force_stage == 1) @@ -1894,16 +1542,14 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) smmu->pgshift = (id & ID1_PAGESIZE) ? 16 : 12; /* Check for size mismatch of SMMU address space from mapped region */ - size = 1 << - (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); + size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); size *= 2 << smmu->pgshift; if (smmu->size != size) dev_warn(smmu->dev, "SMMU address space size (0x%lx) differs from mapped region size (0x%lx)!\n", size, smmu->size); - smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & - ID1_NUMS2CB_MASK; + smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & ID1_NUMS2CB_MASK; smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; if (smmu->num_s2_context_banks > smmu->num_context_banks) { dev_err(smmu->dev, "impossible number of S2 context banks!\n"); @@ -1915,46 +1561,40 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) /* ID2 */ id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); - smmu->s1_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size); + smmu->ipa_size = size; - /* Stage-2 input size limited due to pgd allocation (PTRS_PER_PGD) */ -#ifdef CONFIG_64BIT - smmu->s2_input_size = min_t(unsigned long, VA_BITS, size); -#else - smmu->s2_input_size = min(32UL, size); -#endif - - /* The stage-2 output mask is also applied for bypass */ + /* The output mask is also applied for bypass */ size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); - smmu->s2_output_size = min_t(unsigned long, PHYS_MASK_SHIFT, size); + smmu->pa_size = size; if (smmu->version == ARM_SMMU_V1) { - smmu->s1_input_size = 32; + smmu->va_size = smmu->ipa_size; + size = SZ_4K | SZ_2M | SZ_1G; } else { -#ifdef CONFIG_64BIT size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; - size = min(VA_BITS, arm_smmu_id_size_to_bits(size)); -#else - size = 32; + smmu->va_size = arm_smmu_id_size_to_bits(size); +#ifndef CONFIG_64BIT + smmu->va_size = min(32UL, smmu->va_size); #endif - smmu->s1_input_size = size; - - if ((PAGE_SIZE == SZ_4K && !(id & ID2_PTFS_4K)) || - (PAGE_SIZE == SZ_64K && !(id & ID2_PTFS_64K)) || - (PAGE_SIZE != SZ_4K && PAGE_SIZE != SZ_64K)) { - dev_err(smmu->dev, "CPU page size 0x%lx unsupported\n", - PAGE_SIZE); - return -ENODEV; - } + size = 0; + if (id & ID2_PTFS_4K) + size |= SZ_4K | SZ_2M | SZ_1G; + if (id & ID2_PTFS_16K) + size |= SZ_16K | SZ_32M; + if (id & ID2_PTFS_64K) + size |= SZ_64K | SZ_512M; } + arm_smmu_ops.pgsize_bitmap &= size; + dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", size); + if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", - smmu->s1_input_size, smmu->s1_output_size); + smmu->va_size, smmu->ipa_size); if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", - smmu->s2_input_size, smmu->s2_output_size); + smmu->ipa_size, smmu->pa_size); return 0; } -- cgit v0.10.2 From 3c8567d1cac0fa4c44ddcf0a956cae0bf2a348f3 Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Tue, 23 Dec 2014 17:39:22 +0000 Subject: iommu/arm-smmu: don't touch the secure STLBIALL register Currently we do a STLBIALL when we initialize the SMMU. However, in some configurations that register is not supposed to be touched and is marked as "Secure only" in the spec. Rip it out. Signed-off-by: Mitchel Humpherys Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 919ba43..006f006 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -127,7 +127,6 @@ #define ID2_PTFS_64K (1 << 14) /* Global TLB invalidation */ -#define ARM_SMMU_GR0_STLBIALL 0x60 #define ARM_SMMU_GR0_TLBIVMID 0x64 #define ARM_SMMU_GR0_TLBIALLNSNH 0x68 #define ARM_SMMU_GR0_TLBIALLH 0x6c @@ -1414,7 +1413,6 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) } /* Invalidate the TLB, just in case */ - writel_relaxed(0, gr0_base + ARM_SMMU_GR0_STLBIALL); writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH); writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH); -- cgit v0.10.2 From 54c523127bcca986c6f9b04c7b56a949ea011899 Mon Sep 17 00:00:00 2001 From: Matt Wagantall Date: Mon, 15 Dec 2014 23:47:23 +0000 Subject: iopoll: Introduce memory-mapped IO polling macros It is sometimes necessary to poll a memory-mapped register until its value satisfies some condition. Introduce a family of convenience macros that do this. Tight-looping, sleeping, and timing out can all be accomplished using these macros. Cc: Thierry Reding Cc: Andrew Morton Cc: Robert Elliott Acked-by: Arnd Bergmann Acked-by: Will Deacon Signed-off-by: Matt Wagantall Signed-off-by: Mitchel Humpherys Signed-off-by: Will Deacon diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h new file mode 100644 index 0000000..1c30014 --- /dev/null +++ b/include/linux/iopoll.h @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2012-2014 The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + */ + +#ifndef _LINUX_IOPOLL_H +#define _LINUX_IOPOLL_H + +#include +#include +#include +#include +#include +#include + +/** + * readx_poll_timeout - Periodically poll an address until a condition is met or a timeout occurs + * @op: accessor function (takes @addr as its only argument) + * @addr: Address to poll + * @val: Variable to read the value into + * @cond: Break condition (usually involving @val) + * @sleep_us: Maximum time to sleep between reads in us (0 + * tight-loops). Should be less than ~20ms since usleep_range + * is used (see Documentation/timers/timers-howto.txt). + * @timeout_us: Timeout in us, 0 means never timeout + * + * Returns 0 on success and -ETIMEDOUT upon a timeout. In either + * case, the last read value at @addr is stored in @val. Must not + * be called from atomic context if sleep_us or timeout_us are used. + * + * When available, you'll probably want to use one of the specialized + * macros defined below rather than this macro directly. + */ +#define readx_poll_timeout(op, addr, val, cond, sleep_us, timeout_us) \ +({ \ + ktime_t timeout = ktime_add_us(ktime_get(), timeout_us); \ + might_sleep_if(sleep_us); \ + for (;;) { \ + (val) = op(addr); \ + if (cond) \ + break; \ + if (timeout_us && ktime_compare(ktime_get(), timeout) > 0) { \ + (val) = op(addr); \ + break; \ + } \ + if (sleep_us) \ + usleep_range((sleep_us >> 2) + 1, sleep_us); \ + } \ + (cond) ? 0 : -ETIMEDOUT; \ +}) + +/** + * readx_poll_timeout_atomic - Periodically poll an address until a condition is met or a timeout occurs + * @op: accessor function (takes @addr as its only argument) + * @addr: Address to poll + * @val: Variable to read the value into + * @cond: Break condition (usually involving @val) + * @delay_us: Time to udelay between reads in us (0 tight-loops). Should + * be less than ~10us since udelay is used (see + * Documentation/timers/timers-howto.txt). + * @timeout_us: Timeout in us, 0 means never timeout + * + * Returns 0 on success and -ETIMEDOUT upon a timeout. In either + * case, the last read value at @addr is stored in @val. + * + * When available, you'll probably want to use one of the specialized + * macros defined below rather than this macro directly. + */ +#define readx_poll_timeout_atomic(op, addr, val, cond, delay_us, timeout_us) \ +({ \ + ktime_t timeout = ktime_add_us(ktime_get(), timeout_us); \ + for (;;) { \ + (val) = op(addr); \ + if (cond) \ + break; \ + if (timeout_us && ktime_compare(ktime_get(), timeout) > 0) { \ + (val) = op(addr); \ + break; \ + } \ + if (delay_us) \ + udelay(delay_us); \ + } \ + (cond) ? 0 : -ETIMEDOUT; \ +}) + + +#define readb_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readb, addr, val, cond, delay_us, timeout_us) + +#define readb_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readb, addr, val, cond, delay_us, timeout_us) + +#define readw_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readw, addr, val, cond, delay_us, timeout_us) + +#define readw_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readw, addr, val, cond, delay_us, timeout_us) + +#define readl_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readl, addr, val, cond, delay_us, timeout_us) + +#define readl_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readl, addr, val, cond, delay_us, timeout_us) + +#define readq_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readq, addr, val, cond, delay_us, timeout_us) + +#define readq_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readq, addr, val, cond, delay_us, timeout_us) + +#define readb_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readb_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readb_relaxed_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readb_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readw_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readw_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readw_relaxed_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readw_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readl_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readl_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readl_relaxed_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readl_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readq_relaxed_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout(readq_relaxed, addr, val, cond, delay_us, timeout_us) + +#define readq_relaxed_poll_timeout_atomic(addr, val, cond, delay_us, timeout_us) \ + readx_poll_timeout_atomic(readq_relaxed, addr, val, cond, delay_us, timeout_us) + +#endif /* _LINUX_IOPOLL_H */ -- cgit v0.10.2 From 859a732e4f713270152c78df6e09accbde006734 Mon Sep 17 00:00:00 2001 From: Mitchel Humpherys Date: Wed, 29 Oct 2014 21:13:40 +0000 Subject: iommu/arm-smmu: add support for iova_to_phys through ATS1PR Currently, we provide the iommu_ops.iova_to_phys service by doing a table walk in software to translate IO virtual addresses to physical addresses. On SMMUs that support it, it can be useful to ask the SMMU itself to do the translation. This can be used to warm the TLBs for an SMMU. It can also be useful for testing and hardware validation. Since the address translation registers are optional on SMMUv2, only enable hardware translations when using SMMUv1 or when SMMU_IDR0.S1TS=1 and SMMU_IDR0.ATOSNS=0, as described in the ARM SMMU v1-v2 spec. Signed-off-by: Mitchel Humpherys [will: reworked on top of generic iopgtbl changes] Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 006f006..1d6d43b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -100,6 +101,7 @@ #define ID0_S2TS (1 << 29) #define ID0_NTS (1 << 28) #define ID0_SMS (1 << 27) +#define ID0_ATOSNS (1 << 26) #define ID0_CTTW (1 << 14) #define ID0_NUMIRPT_SHIFT 16 #define ID0_NUMIRPT_MASK 0xff @@ -189,6 +191,8 @@ #define ARM_SMMU_CB_TTBCR 0x30 #define ARM_SMMU_CB_S1_MAIR0 0x38 #define ARM_SMMU_CB_S1_MAIR1 0x3c +#define ARM_SMMU_CB_PAR_LO 0x50 +#define ARM_SMMU_CB_PAR_HI 0x54 #define ARM_SMMU_CB_FSR 0x58 #define ARM_SMMU_CB_FAR_LO 0x60 #define ARM_SMMU_CB_FAR_HI 0x64 @@ -198,6 +202,9 @@ #define ARM_SMMU_CB_S1_TLBIVAL 0x620 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 +#define ARM_SMMU_CB_ATS1PR_LO 0x800 +#define ARM_SMMU_CB_ATS1PR_HI 0x804 +#define ARM_SMMU_CB_ATSR 0x8f0 #define SCTLR_S1_ASIDPNE (1 << 12) #define SCTLR_CFCFG (1 << 7) @@ -209,6 +216,10 @@ #define SCTLR_M (1 << 0) #define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE) +#define CB_PAR_F (1 << 0) + +#define ATSR_ACTIVE (1 << 0) + #define RESUME_RETRY (0 << 0) #define RESUME_TERMINATE (1 << 0) @@ -282,6 +293,7 @@ struct arm_smmu_device { #define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) #define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) #define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) +#define ARM_SMMU_FEAT_TRANS_OPS (1 << 5) u32 features; #define ARM_SMMU_OPT_SECURE_CFG_ACCESS (1 << 0) @@ -1220,8 +1232,52 @@ static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, return ret; } +static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cfg *cfg = &smmu_domain->cfg; + struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; + struct device *dev = smmu->dev; + void __iomem *cb_base; + u32 tmp; + u64 phys; + + cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); + + if (smmu->version == 1) { + u32 reg = iova & ~0xfff; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); + } else { + u32 reg = iova & ~0xfff; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); + reg = (iova & ~0xfff) >> 32; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_HI); + } + + if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, + !(tmp & ATSR_ACTIVE), 5, 50)) { + dev_err(dev, + "iova to phys timed out on 0x%pad. Falling back to software table walk.\n", + &iova); + return ops->iova_to_phys(ops, iova); + } + + phys = readl_relaxed(cb_base + ARM_SMMU_CB_PAR_LO); + phys |= ((u64)readl_relaxed(cb_base + ARM_SMMU_CB_PAR_HI)) << 32; + + if (phys & CB_PAR_F) { + dev_err(dev, "translation fault!\n"); + dev_err(dev, "PAR = 0x%llx\n", phys); + return 0; + } + + return (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); +} + static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, - dma_addr_t iova) + dma_addr_t iova) { phys_addr_t ret; unsigned long flags; @@ -1232,8 +1288,12 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, return 0; spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); - ret = ops->iova_to_phys(ops, iova); + if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS) + ret = arm_smmu_iova_to_phys_hard(domain, iova); + else + ret = ops->iova_to_phys(ops, iova); spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + return ret; } @@ -1496,6 +1556,11 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) return -ENODEV; } + if (smmu->version == 1 || (!(id & ID0_ATOSNS) && (id & ID0_S1TS))) { + smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; + dev_notice(smmu->dev, "\taddress translation ops\n"); + } + if (id & ID0_CTTW) { smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; dev_notice(smmu->dev, "\tcoherent table walk\n"); -- cgit v0.10.2 From 860cd64d102d9b6c97830e09c447a9a850ea7641 Mon Sep 17 00:00:00 2001 From: Shuah Khan Date: Thu, 15 Jan 2015 19:29:43 -0700 Subject: iommu: Fix trace_map() to report original iova and original size iommu_map() calls trace_map() with iova and size. trace_map() should report original iova and original size as opposed to iova and size after they get changed during mapping. size is always zero at the end of mapping which is useless to report and iova as it gets incremented, it is not as useful as the original iova. Change iommu_map() to call trace_map() to report original iova and original size. Signed-off-by: Shuah Khan Reported-by: Alex Williamson Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3a4fb62..9e0dcdb 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1084,7 +1084,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, if (ret) iommu_unmap(domain, orig_iova, orig_size - size); else - trace_map(iova, paddr, size); + trace_map(orig_iova, paddr, orig_size); return ret; } -- cgit v0.10.2 From f20ed39f53145e45edd27ff9fba5a60429c40bac Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Tue, 20 Jan 2015 18:30:04 +0200 Subject: iommu/ipmmu-vmsa: Use the ARM LPAE page table allocator Replace the custom page table allocation implementation with the standard allocator. The driver loses the ability to map 64kB chunkgs using the PTE contiguous hint, hence the removal of the SZ_64K page size from the IOMMU page sizes bitmap. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 87060ad..b7c656c84f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -313,6 +313,7 @@ config IPMMU_VMSA depends on ARM_LPAE depends on ARCH_SHMOBILE || COMPILE_TEST select IOMMU_API + select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU help Support for the Renesas VMSA-compatible IPMMU Renesas found in the diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 791c3da..3d7e732 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -24,6 +24,8 @@ #include #include +#include "io-pgtable.h" + struct ipmmu_vmsa_device { struct device *dev; void __iomem *base; @@ -38,9 +40,11 @@ struct ipmmu_vmsa_domain { struct ipmmu_vmsa_device *mmu; struct iommu_domain *io_domain; + struct io_pgtable_cfg cfg; + struct io_pgtable_ops *iop; + unsigned int context_id; spinlock_t lock; /* Protects mappings */ - pgd_t *pgd; }; struct ipmmu_vmsa_archdata { @@ -173,52 +177,6 @@ static LIST_HEAD(ipmmu_devices); #define IMUASID_ASID0_SHIFT 0 /* ----------------------------------------------------------------------------- - * Page Table Bits - */ - -/* - * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory access, - * Long-descriptor format" that the NStable bit being set in a table descriptor - * will result in the NStable and NS bits of all child entries being ignored and - * considered as being set. The IPMMU seems not to comply with this, as it - * generates a secure access page fault if any of the NStable and NS bits isn't - * set when running in non-secure mode. - */ -#ifndef PMD_NSTABLE -#define PMD_NSTABLE (_AT(pmdval_t, 1) << 63) -#endif - -#define ARM_VMSA_PTE_XN (((pteval_t)3) << 53) -#define ARM_VMSA_PTE_CONT (((pteval_t)1) << 52) -#define ARM_VMSA_PTE_AF (((pteval_t)1) << 10) -#define ARM_VMSA_PTE_SH_NS (((pteval_t)0) << 8) -#define ARM_VMSA_PTE_SH_OS (((pteval_t)2) << 8) -#define ARM_VMSA_PTE_SH_IS (((pteval_t)3) << 8) -#define ARM_VMSA_PTE_SH_MASK (((pteval_t)3) << 8) -#define ARM_VMSA_PTE_NS (((pteval_t)1) << 5) -#define ARM_VMSA_PTE_PAGE (((pteval_t)3) << 0) - -/* Stage-1 PTE */ -#define ARM_VMSA_PTE_nG (((pteval_t)1) << 11) -#define ARM_VMSA_PTE_AP_UNPRIV (((pteval_t)1) << 6) -#define ARM_VMSA_PTE_AP_RDONLY (((pteval_t)2) << 6) -#define ARM_VMSA_PTE_AP_MASK (((pteval_t)3) << 6) -#define ARM_VMSA_PTE_ATTRINDX_MASK (((pteval_t)3) << 2) -#define ARM_VMSA_PTE_ATTRINDX_SHIFT 2 - -#define ARM_VMSA_PTE_ATTRS_MASK \ - (ARM_VMSA_PTE_XN | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_nG | \ - ARM_VMSA_PTE_AF | ARM_VMSA_PTE_SH_MASK | ARM_VMSA_PTE_AP_MASK | \ - ARM_VMSA_PTE_NS | ARM_VMSA_PTE_ATTRINDX_MASK) - -#define ARM_VMSA_PTE_CONT_ENTRIES 16 -#define ARM_VMSA_PTE_CONT_SIZE (PAGE_SIZE * ARM_VMSA_PTE_CONT_ENTRIES) - -#define IPMMU_PTRS_PER_PTE 512 -#define IPMMU_PTRS_PER_PMD 512 -#define IPMMU_PTRS_PER_PGD 4 - -/* ----------------------------------------------------------------------------- * Read/Write Access */ @@ -307,18 +265,39 @@ static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain, ipmmu_write(mmu, IMUCTR(utlb), 0); } -static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr, - size_t size) +static void ipmmu_tlb_flush_all(void *cookie) { - unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + struct ipmmu_vmsa_domain *domain = cookie; + + ipmmu_tlb_invalidate(domain); +} + +static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, bool leaf, + void *cookie) +{ + /* The hardware doesn't support selective TLB flush. */ +} + +static void ipmmu_flush_pgtable(void *ptr, size_t size, void *cookie) +{ + unsigned long offset = (unsigned long)ptr & ~PAGE_MASK; + struct ipmmu_vmsa_domain *domain = cookie; /* * TODO: Add support for coherent walk through CCI with DVM and remove * cache handling. */ - dma_map_page(mmu->dev, virt_to_page(addr), offset, size, DMA_TO_DEVICE); + dma_map_page(domain->mmu->dev, virt_to_page(ptr), offset, size, + DMA_TO_DEVICE); } +static struct iommu_gather_ops ipmmu_gather_ops = { + .tlb_flush_all = ipmmu_tlb_flush_all, + .tlb_add_flush = ipmmu_tlb_add_flush, + .tlb_sync = ipmmu_tlb_flush_all, + .flush_pgtable = ipmmu_flush_pgtable, +}; + /* ----------------------------------------------------------------------------- * Domain/Context Management */ @@ -326,7 +305,28 @@ static void ipmmu_flush_pgtable(struct ipmmu_vmsa_device *mmu, void *addr, static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) { phys_addr_t ttbr; - u32 reg; + + /* + * Allocate the page table operations. + * + * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory + * access, Long-descriptor format" that the NStable bit being set in a + * table descriptor will result in the NStable and NS bits of all child + * entries being ignored and considered as being set. The IPMMU seems + * not to comply with this, as it generates a secure access page fault + * if any of the NStable and NS bits isn't set when running in + * non-secure mode. + */ + domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; + domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, + domain->cfg.ias = 32; + domain->cfg.oas = 40; + domain->cfg.tlb = &ipmmu_gather_ops; + + domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, + domain); + if (!domain->iop) + return -EINVAL; /* * TODO: When adding support for multiple contexts, find an unused @@ -335,9 +335,7 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) domain->context_id = 0; /* TTBR0 */ - ipmmu_flush_pgtable(domain->mmu, domain->pgd, - IPMMU_PTRS_PER_PGD * sizeof(*domain->pgd)); - ttbr = __pa(domain->pgd); + ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; ipmmu_ctx_write(domain, IMTTLBR0, ttbr); ipmmu_ctx_write(domain, IMTTUBR0, ttbr >> 32); @@ -350,15 +348,8 @@ static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain) IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA | IMTTBCR_IRGN0_WB_WA | IMTTBCR_SL0_LVL_1); - /* - * MAIR0 - * We need three attributes only, non-cacheable, write-back read/write - * allocate and device memory. - */ - reg = (IMMAIR_ATTR_NC << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_NC)) - | (IMMAIR_ATTR_WBRWA << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_WBRWA)) - | (IMMAIR_ATTR_DEVICE << IMMAIR_ATTR_SHIFT(IMMAIR_ATTR_IDX_DEV)); - ipmmu_ctx_write(domain, IMMAIR0, reg); + /* MAIR0 */ + ipmmu_ctx_write(domain, IMMAIR0, domain->cfg.arm_lpae_s1_cfg.mair[0]); /* IMBUSCR */ ipmmu_ctx_write(domain, IMBUSCR, @@ -463,397 +454,6 @@ static irqreturn_t ipmmu_irq(int irq, void *dev) } /* ----------------------------------------------------------------------------- - * Page Table Management - */ - -#define pud_pgtable(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK)) - -static void ipmmu_free_ptes(pmd_t *pmd) -{ - pgtable_t table = pmd_pgtable(*pmd); - __free_page(table); -} - -static void ipmmu_free_pmds(pud_t *pud) -{ - pmd_t *pmd = pmd_offset(pud, 0); - pgtable_t table; - unsigned int i; - - for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { - if (!pmd_table(*pmd)) - continue; - - ipmmu_free_ptes(pmd); - pmd++; - } - - table = pud_pgtable(*pud); - __free_page(table); -} - -static void ipmmu_free_pgtables(struct ipmmu_vmsa_domain *domain) -{ - pgd_t *pgd, *pgd_base = domain->pgd; - unsigned int i; - - /* - * Recursively free the page tables for this domain. We don't care about - * speculative TLB filling, because the TLB will be nuked next time this - * context bank is re-allocated and no devices currently map to these - * tables. - */ - pgd = pgd_base; - for (i = 0; i < IPMMU_PTRS_PER_PGD; ++i) { - if (pgd_none(*pgd)) - continue; - ipmmu_free_pmds((pud_t *)pgd); - pgd++; - } - - kfree(pgd_base); -} - -/* - * We can't use the (pgd|pud|pmd|pte)_populate or the set_(pgd|pud|pmd|pte) - * functions as they would flush the CPU TLB. - */ - -static pte_t *ipmmu_alloc_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, - unsigned long iova) -{ - pte_t *pte; - - if (!pmd_none(*pmd)) - return pte_offset_kernel(pmd, iova); - - pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); - if (!pte) - return NULL; - - ipmmu_flush_pgtable(mmu, pte, PAGE_SIZE); - *pmd = __pmd(__pa(pte) | PMD_NSTABLE | PMD_TYPE_TABLE); - ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); - - return pte + pte_index(iova); -} - -static pmd_t *ipmmu_alloc_pmd(struct ipmmu_vmsa_device *mmu, pgd_t *pgd, - unsigned long iova) -{ - pud_t *pud = (pud_t *)pgd; - pmd_t *pmd; - - if (!pud_none(*pud)) - return pmd_offset(pud, iova); - - pmd = (pmd_t *)get_zeroed_page(GFP_ATOMIC); - if (!pmd) - return NULL; - - ipmmu_flush_pgtable(mmu, pmd, PAGE_SIZE); - *pud = __pud(__pa(pmd) | PMD_NSTABLE | PMD_TYPE_TABLE); - ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); - - return pmd + pmd_index(iova); -} - -static u64 ipmmu_page_prot(unsigned int prot, u64 type) -{ - u64 pgprot = ARM_VMSA_PTE_nG | ARM_VMSA_PTE_AF - | ARM_VMSA_PTE_SH_IS | ARM_VMSA_PTE_AP_UNPRIV - | ARM_VMSA_PTE_NS | type; - - if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) - pgprot |= ARM_VMSA_PTE_AP_RDONLY; - - if (prot & IOMMU_CACHE) - pgprot |= IMMAIR_ATTR_IDX_WBRWA << ARM_VMSA_PTE_ATTRINDX_SHIFT; - - if (prot & IOMMU_NOEXEC) - pgprot |= ARM_VMSA_PTE_XN; - else if (!(prot & (IOMMU_READ | IOMMU_WRITE))) - /* If no access create a faulting entry to avoid TLB fills. */ - pgprot &= ~ARM_VMSA_PTE_PAGE; - - return pgprot; -} - -static int ipmmu_alloc_init_pte(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, - unsigned long iova, unsigned long pfn, - size_t size, int prot) -{ - pteval_t pteval = ipmmu_page_prot(prot, ARM_VMSA_PTE_PAGE); - unsigned int num_ptes = 1; - pte_t *pte, *start; - unsigned int i; - - pte = ipmmu_alloc_pte(mmu, pmd, iova); - if (!pte) - return -ENOMEM; - - start = pte; - - /* - * Install the page table entries. We can be called both for a single - * page or for a block of 16 physically contiguous pages. In the latter - * case set the PTE contiguous hint. - */ - if (size == SZ_64K) { - pteval |= ARM_VMSA_PTE_CONT; - num_ptes = ARM_VMSA_PTE_CONT_ENTRIES; - } - - for (i = num_ptes; i; --i) - *pte++ = pfn_pte(pfn++, __pgprot(pteval)); - - ipmmu_flush_pgtable(mmu, start, sizeof(*pte) * num_ptes); - - return 0; -} - -static int ipmmu_alloc_init_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd, - unsigned long iova, unsigned long pfn, - int prot) -{ - pmdval_t pmdval = ipmmu_page_prot(prot, PMD_TYPE_SECT); - - *pmd = pfn_pmd(pfn, __pgprot(pmdval)); - ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); - - return 0; -} - -static int ipmmu_create_mapping(struct ipmmu_vmsa_domain *domain, - unsigned long iova, phys_addr_t paddr, - size_t size, int prot) -{ - struct ipmmu_vmsa_device *mmu = domain->mmu; - pgd_t *pgd = domain->pgd; - unsigned long flags; - unsigned long pfn; - pmd_t *pmd; - int ret; - - if (!pgd) - return -EINVAL; - - if (size & ~PAGE_MASK) - return -EINVAL; - - if (paddr & ~((1ULL << 40) - 1)) - return -ERANGE; - - pfn = __phys_to_pfn(paddr); - pgd += pgd_index(iova); - - /* Update the page tables. */ - spin_lock_irqsave(&domain->lock, flags); - - pmd = ipmmu_alloc_pmd(mmu, pgd, iova); - if (!pmd) { - ret = -ENOMEM; - goto done; - } - - switch (size) { - case SZ_2M: - ret = ipmmu_alloc_init_pmd(mmu, pmd, iova, pfn, prot); - break; - case SZ_64K: - case SZ_4K: - ret = ipmmu_alloc_init_pte(mmu, pmd, iova, pfn, size, prot); - break; - default: - ret = -EINVAL; - break; - } - -done: - spin_unlock_irqrestore(&domain->lock, flags); - - if (!ret) - ipmmu_tlb_invalidate(domain); - - return ret; -} - -static void ipmmu_clear_pud(struct ipmmu_vmsa_device *mmu, pud_t *pud) -{ - pgtable_t table = pud_pgtable(*pud); - - /* Clear the PUD. */ - *pud = __pud(0); - ipmmu_flush_pgtable(mmu, pud, sizeof(*pud)); - - /* Free the page table. */ - __free_page(table); -} - -static void ipmmu_clear_pmd(struct ipmmu_vmsa_device *mmu, pud_t *pud, - pmd_t *pmd) -{ - pmd_t pmdval = *pmd; - unsigned int i; - - /* Clear the PMD. */ - *pmd = __pmd(0); - ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); - - /* Free the page table. */ - if (pmd_table(pmdval)) { - pgtable_t table = pmd_pgtable(pmdval); - - __free_page(table); - } - - /* Check whether the PUD is still needed. */ - pmd = pmd_offset(pud, 0); - for (i = 0; i < IPMMU_PTRS_PER_PMD; ++i) { - if (!pmd_none(pmd[i])) - return; - } - - /* Clear the parent PUD. */ - ipmmu_clear_pud(mmu, pud); -} - -static void ipmmu_clear_pte(struct ipmmu_vmsa_device *mmu, pud_t *pud, - pmd_t *pmd, pte_t *pte, unsigned int num_ptes) -{ - unsigned int i; - - /* Clear the PTE. */ - for (i = num_ptes; i; --i) - pte[i-1] = __pte(0); - - ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * num_ptes); - - /* Check whether the PMD is still needed. */ - pte = pte_offset_kernel(pmd, 0); - for (i = 0; i < IPMMU_PTRS_PER_PTE; ++i) { - if (!pte_none(pte[i])) - return; - } - - /* Clear the parent PMD. */ - ipmmu_clear_pmd(mmu, pud, pmd); -} - -static int ipmmu_split_pmd(struct ipmmu_vmsa_device *mmu, pmd_t *pmd) -{ - pte_t *pte, *start; - pteval_t pteval; - unsigned long pfn; - unsigned int i; - - pte = (pte_t *)get_zeroed_page(GFP_ATOMIC); - if (!pte) - return -ENOMEM; - - /* Copy the PMD attributes. */ - pteval = (pmd_val(*pmd) & ARM_VMSA_PTE_ATTRS_MASK) - | ARM_VMSA_PTE_CONT | ARM_VMSA_PTE_PAGE; - - pfn = pmd_pfn(*pmd); - start = pte; - - for (i = IPMMU_PTRS_PER_PTE; i; --i) - *pte++ = pfn_pte(pfn++, __pgprot(pteval)); - - ipmmu_flush_pgtable(mmu, start, PAGE_SIZE); - *pmd = __pmd(__pa(start) | PMD_NSTABLE | PMD_TYPE_TABLE); - ipmmu_flush_pgtable(mmu, pmd, sizeof(*pmd)); - - return 0; -} - -static void ipmmu_split_pte(struct ipmmu_vmsa_device *mmu, pte_t *pte) -{ - unsigned int i; - - for (i = ARM_VMSA_PTE_CONT_ENTRIES; i; --i) - pte[i-1] = __pte(pte_val(*pte) & ~ARM_VMSA_PTE_CONT); - - ipmmu_flush_pgtable(mmu, pte, sizeof(*pte) * ARM_VMSA_PTE_CONT_ENTRIES); -} - -static int ipmmu_clear_mapping(struct ipmmu_vmsa_domain *domain, - unsigned long iova, size_t size) -{ - struct ipmmu_vmsa_device *mmu = domain->mmu; - unsigned long flags; - pgd_t *pgd = domain->pgd; - pud_t *pud; - pmd_t *pmd; - pte_t *pte; - - if (!pgd) - return -EINVAL; - - if (size & ~PAGE_MASK) - return -EINVAL; - - pgd += pgd_index(iova); - pud = (pud_t *)pgd; - - spin_lock_irqsave(&domain->lock, flags); - - /* If there's no PUD or PMD we're done. */ - if (pud_none(*pud)) - goto done; - - pmd = pmd_offset(pud, iova); - if (pmd_none(*pmd)) - goto done; - - /* - * When freeing a 2MB block just clear the PMD. In the unlikely case the - * block is mapped as individual pages this will free the corresponding - * PTE page table. - */ - if (size == SZ_2M) { - ipmmu_clear_pmd(mmu, pud, pmd); - goto done; - } - - /* - * If the PMD has been mapped as a section remap it as pages to allow - * freeing individual pages. - */ - if (pmd_sect(*pmd)) - ipmmu_split_pmd(mmu, pmd); - - pte = pte_offset_kernel(pmd, iova); - - /* - * When freeing a 64kB block just clear the PTE entries. We don't have - * to care about the contiguous hint of the surrounding entries. - */ - if (size == SZ_64K) { - ipmmu_clear_pte(mmu, pud, pmd, pte, ARM_VMSA_PTE_CONT_ENTRIES); - goto done; - } - - /* - * If the PTE has been mapped with the contiguous hint set remap it and - * its surrounding PTEs to allow unmapping a single page. - */ - if (pte_val(*pte) & ARM_VMSA_PTE_CONT) - ipmmu_split_pte(mmu, pte); - - /* Clear the PTE. */ - ipmmu_clear_pte(mmu, pud, pmd, pte, 1); - -done: - spin_unlock_irqrestore(&domain->lock, flags); - - ipmmu_tlb_invalidate(domain); - - return 0; -} - -/* ----------------------------------------------------------------------------- * IOMMU Operations */ @@ -867,12 +467,6 @@ static int ipmmu_domain_init(struct iommu_domain *io_domain) spin_lock_init(&domain->lock); - domain->pgd = kzalloc(IPMMU_PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL); - if (!domain->pgd) { - kfree(domain); - return -ENOMEM; - } - io_domain->priv = domain; domain->io_domain = io_domain; @@ -888,7 +482,7 @@ static void ipmmu_domain_destroy(struct iommu_domain *io_domain) * been detached. */ ipmmu_domain_destroy_context(domain); - ipmmu_free_pgtables(domain); + free_io_pgtable_ops(domain->iop); kfree(domain); } @@ -957,53 +551,25 @@ static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova, if (!domain) return -ENODEV; - return ipmmu_create_mapping(domain, iova, paddr, size, prot); + return domain->iop->map(domain->iop, iova, paddr, size, prot); } static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova, size_t size) { struct ipmmu_vmsa_domain *domain = io_domain->priv; - int ret; - ret = ipmmu_clear_mapping(domain, iova, size); - return ret ? 0 : size; + return domain->iop->unmap(domain->iop, iova, size); } static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, dma_addr_t iova) { struct ipmmu_vmsa_domain *domain = io_domain->priv; - pgd_t pgd; - pud_t pud; - pmd_t pmd; - pte_t pte; /* TODO: Is locking needed ? */ - if (!domain->pgd) - return 0; - - pgd = *(domain->pgd + pgd_index(iova)); - if (pgd_none(pgd)) - return 0; - - pud = *pud_offset(&pgd, iova); - if (pud_none(pud)) - return 0; - - pmd = *pmd_offset(&pud, iova); - if (pmd_none(pmd)) - return 0; - - if (pmd_sect(pmd)) - return __pfn_to_phys(pmd_pfn(pmd)) | (iova & ~PMD_MASK); - - pte = *(pmd_page_vaddr(pmd) + pte_index(iova)); - if (pte_none(pte)) - return 0; - - return __pfn_to_phys(pte_pfn(pte)) | (iova & ~PAGE_MASK); + return domain->iop->iova_to_phys(domain->iop, iova); } static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev, @@ -1188,7 +754,7 @@ static const struct iommu_ops ipmmu_ops = { .iova_to_phys = ipmmu_iova_to_phys, .add_device = ipmmu_add_device, .remove_device = ipmmu_remove_device, - .pgsize_bitmap = SZ_2M | SZ_64K | SZ_4K, + .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K, }; /* ----------------------------------------------------------------------------- -- cgit v0.10.2 From 0f1fb99b62ce226f8d818852f812c5d79071ce58 Mon Sep 17 00:00:00 2001 From: Emil Medve Date: Thu, 22 Jan 2015 08:47:20 -0600 Subject: iommu/fsl: Fix section mismatch Section mismatch in reference from the variable fsl_of_pamu_driver to the function .init.text:fsl_pamu_probe() The variable fsl_of_pamu_driver references the function __init fsl_pamu_probe() If the reference is valid then annotate the variable with __init* or __refdata (see linux/init.h) or name the variable: *_template, *_timer, *_sht, *_ops, *_probe, *_probe_one, *_console Signed-off-by: Emil Medve Acked-by: Varun Sethi Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 80ac68d..d478ad8 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -1224,7 +1224,7 @@ static const struct of_device_id fsl_of_pamu_ids[] = { {}, }; -static struct platform_driver fsl_of_pamu_driver = { +static struct platform_driver fsl_of_pamu_driver __initdata = { .driver = { .name = "fsl-of-pamu", }, -- cgit v0.10.2 From ca4f55878732306cb47bd76bfa803c255b4a0e6f Mon Sep 17 00:00:00 2001 From: Emil Medve Date: Thu, 22 Jan 2015 08:47:21 -0600 Subject: iommu/fsl: Remove unused fsl_of_pamu_ids[] Signed-off-by: Emil Medve Acked-by: Varun Sethi Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index d478ad8..113fb7b 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -1214,16 +1214,6 @@ error: return ret; } -static const struct of_device_id fsl_of_pamu_ids[] = { - { - .compatible = "fsl,p4080-pamu", - }, - { - .compatible = "fsl,pamu", - }, - {}, -}; - static struct platform_driver fsl_of_pamu_driver __initdata = { .driver = { .name = "fsl-of-pamu", -- cgit v0.10.2 From e5144c9375851948f50e13db80c3150e3a7521b8 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Wed, 28 Jan 2015 15:45:53 +0100 Subject: iommu: Disable on !MMU builds A lot of the IOMMU support code does not build if the CPU does not have an MMU itself, and it's not clear if there is any use case for it, so let's just disable it and wait for anybody to need it. This avoids randconfig errors like ../arch/arm/mm/dma-mapping.c: In function '__iommu_alloc_remap': ../arch/arm/mm/dma-mapping.c:1278:34: error: 'VM_ARM_DMA_CONSISTENT' undeclared (first use in this function) area = get_vm_area_caller(size, VM_ARM_DMA_CONSISTENT | VM_USERMAP, ^ ../arch/arm/mm/dma-mapping.c:1278:34: note: each undeclared identifier is reported only once for each function it appears in ../arch/arm/mm/dma-mapping.c: In function '__atomic_get_pages': ../arch/arm/mm/dma-mapping.c:1358:27: error: 'atomic_pool' undeclared (first use in this function) struct dma_pool *pool = &atomic_pool; Signed-off-by: Arnd Bergmann Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a839ca9..5189cc1 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -4,6 +4,7 @@ config IOMMU_API menuconfig IOMMU_SUPPORT bool "IOMMU Hardware Support" + depends on MMU default y ---help--- Say Y here if you want to compile device drivers for IO Memory @@ -144,7 +145,7 @@ config IRQ_REMAP # OMAP IOMMU support config OMAP_IOMMU bool "OMAP IOMMU Support" - depends on ARCH_OMAP2PLUS + depends on ARCH_OMAP2PLUS && MMU select IOMMU_API config OMAP_IOMMU_DEBUG @@ -191,7 +192,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && ARM + depends on ARCH_EXYNOS && ARM && MMU select IOMMU_API select ARM_DMA_USE_IOMMU help @@ -220,7 +221,7 @@ config SHMOBILE_IPMMU_TLB config SHMOBILE_IOMMU bool "IOMMU for Renesas IPMMU/IPMMUI" default n - depends on ARM + depends on ARM && MMU depends on ARCH_SHMOBILE || COMPILE_TEST select IOMMU_API select ARM_DMA_USE_IOMMU @@ -309,6 +310,7 @@ config SPAPR_TCE_IOMMU config ARM_SMMU bool "ARM Ltd. System MMU (SMMU) Support" depends on ARM64 || (ARM_LPAE && OF) + depends on MMU select IOMMU_API select ARM_DMA_USE_IOMMU if ARM help -- cgit v0.10.2 From bb590c9011a7c4e195af486438ead666023f96bd Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Sat, 24 Jan 2015 23:13:50 +0200 Subject: iommu/ipmmu-vmsa: Fix IOMMU lookup when multiple IOMMUs are registered When adding a new device the driver loops over all registered IOMMUs and calls the ipmmu_find_utlbs() function to parse the DT iommus attribute. The function returns an error when the IOMMU referenced in DT doesn't match the current IOMMU. The caller incorrectly breaks from the loop immediately when the error is reported, resulting in only the first IOMMU being considered. Fix this, and while at it move code that isn't specific to an IOMMU instance out of the loop. Signed-off-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 3d7e732..10186ca 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -573,45 +573,28 @@ static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain, } static int ipmmu_find_utlbs(struct ipmmu_vmsa_device *mmu, struct device *dev, - unsigned int **_utlbs) + unsigned int *utlbs, unsigned int num_utlbs) { - unsigned int *utlbs; unsigned int i; - int count; - - count = of_count_phandle_with_args(dev->of_node, "iommus", - "#iommu-cells"); - if (count < 0) - return -EINVAL; - - utlbs = kcalloc(count, sizeof(*utlbs), GFP_KERNEL); - if (!utlbs) - return -ENOMEM; - for (i = 0; i < count; ++i) { + for (i = 0; i < num_utlbs; ++i) { struct of_phandle_args args; int ret; ret = of_parse_phandle_with_args(dev->of_node, "iommus", "#iommu-cells", i, &args); if (ret < 0) - goto error; + return ret; of_node_put(args.np); if (args.np != mmu->dev->of_node || args.args_count != 1) - goto error; + return -EINVAL; utlbs[i] = args.args[0]; } - *_utlbs = utlbs; - - return count; - -error: - kfree(utlbs); - return -EINVAL; + return 0; } static int ipmmu_add_device(struct device *dev) @@ -619,10 +602,10 @@ static int ipmmu_add_device(struct device *dev) struct ipmmu_vmsa_archdata *archdata; struct ipmmu_vmsa_device *mmu; struct iommu_group *group = NULL; - unsigned int *utlbs = NULL; + unsigned int *utlbs; unsigned int i; - int num_utlbs = 0; - int ret; + int num_utlbs; + int ret = -ENODEV; if (dev->archdata.iommu) { dev_warn(dev, "IOMMU driver already assigned to device %s\n", @@ -631,11 +614,21 @@ static int ipmmu_add_device(struct device *dev) } /* Find the master corresponding to the device. */ + + num_utlbs = of_count_phandle_with_args(dev->of_node, "iommus", + "#iommu-cells"); + if (num_utlbs < 0) + return -ENODEV; + + utlbs = kcalloc(num_utlbs, sizeof(*utlbs), GFP_KERNEL); + if (!utlbs) + return -ENOMEM; + spin_lock(&ipmmu_devices_lock); list_for_each_entry(mmu, &ipmmu_devices, list) { - num_utlbs = ipmmu_find_utlbs(mmu, dev, &utlbs); - if (num_utlbs) { + ret = ipmmu_find_utlbs(mmu, dev, utlbs, num_utlbs); + if (!ret) { /* * TODO Take a reference to the MMU to protect * against device removal. @@ -646,7 +639,7 @@ static int ipmmu_add_device(struct device *dev) spin_unlock(&ipmmu_devices_lock); - if (num_utlbs <= 0) + if (ret < 0) return -ENODEV; for (i = 0; i < num_utlbs; ++i) { -- cgit v0.10.2 From 477ab7a19cec8409e4e2dd10e7348e4cac3c06e5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 20 Jan 2015 16:13:33 +0100 Subject: iommu: Make more drivers depend on COMPILE_TEST For easier compile testing of these iommu drivers. Acked-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 5189cc1..db98eb7 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -23,7 +23,8 @@ config OF_IOMMU config FSL_PAMU bool "Freescale IOMMU support" - depends on PPC_E500MC + depends on PPC32 + depends on PPC_E500MC || COMPILE_TEST select IOMMU_API select GENERIC_ALLOCATOR help @@ -34,7 +35,8 @@ config FSL_PAMU # MSM IOMMU support config MSM_IOMMU bool "MSM IOMMU Support" - depends on ARCH_MSM8X60 || ARCH_MSM8960 + depends on ARM + depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST select IOMMU_API help Support for the IOMMUs found on certain Qualcomm SOCs. @@ -145,7 +147,8 @@ config IRQ_REMAP # OMAP IOMMU support config OMAP_IOMMU bool "OMAP IOMMU Support" - depends on ARCH_OMAP2PLUS && MMU + depends on ARM && MMU + depends on ARCH_OMAP2PLUS || COMPILE_TEST select IOMMU_API config OMAP_IOMMU_DEBUG -- cgit v0.10.2 From 1d7f449c898b4180b9db79007f23f599b5bb879e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 22 Jan 2015 14:42:06 +0100 Subject: iommu/omap: Print phys_addr_t using %pa Fixes this compile warning: drivers/iommu/omap-iommu.c: In function 'omap_iommu_map': drivers/iommu/omap-iommu.c:1139:2: warning: format '%x' expects argument of type 'unsigned int', but argument 5 has type 'phys_addr_t' [-Wformat=] dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); Acked-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index bbb7dce..f59f857 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1126,7 +1126,7 @@ static int omap_iommu_map(struct iommu_domain *domain, unsigned long da, return -EINVAL; } - dev_dbg(dev, "mapping da 0x%lx to pa 0x%x size 0x%x\n", da, pa, bytes); + dev_dbg(dev, "mapping da 0x%lx to pa %pa size 0x%x\n", da, &pa, bytes); iotlb_init_entry(&e, da, pa, omap_pgsz); -- cgit v0.10.2 From 8cd4f75183446d00432235fedceffae03a6332db Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 22 Jan 2015 14:50:39 +0100 Subject: iommu/fsl: Use %pa to print phys_addr_t Fix compile warnings. Acked-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 113fb7b..6ee947e 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -352,7 +352,7 @@ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, unsigned long fspi; if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { - pr_debug("window size too small or not a power of two %llx\n", win_size); + pr_debug("window size too small or not a power of two %pa\n", &win_size); return -EINVAL; } @@ -1121,8 +1121,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); - dev_dbg(&pdev->dev, "ppaact virt=%p phys=0x%llx\n", ppaact, - (unsigned long long) ppaact_phys); + dev_dbg(&pdev->dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); /* Check to see if we need to implement the work-around on this SOC */ @@ -1138,7 +1137,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) if (csd_port_id) { dev_dbg(&pdev->dev, "creating coherency subdomain at address " - "0x%llx, size %zu, port id 0x%08x", ppaact_phys, + "%pa, size %zu, port id 0x%08x", &ppaact_phys, mem_size, csd_port_id); ret = create_csd(ppaact_phys, mem_size, csd_port_id); -- cgit v0.10.2 From cd70d4659ff3ab7e25b29a6f70be5bcfd078db1f Mon Sep 17 00:00:00 2001 From: Emil Medve Date: Wed, 28 Jan 2015 08:34:33 -0600 Subject: iommu/fsl: Various cleanups Currently a PAMU driver patch is very likely to receive some checkpatch complaints about the code in the context of the patch. This patch is an attempt to fix most of that and make the driver more readable Also fixed a subset of the sparse and coccinelle reported issues. Signed-off-by: Emil Medve Signed-off-by: Joerg Roedel diff --git a/arch/powerpc/include/asm/fsl_pamu_stash.h b/arch/powerpc/include/asm/fsl_pamu_stash.h index caa1b21..38311c9 100644 --- a/arch/powerpc/include/asm/fsl_pamu_stash.h +++ b/arch/powerpc/include/asm/fsl_pamu_stash.h @@ -32,8 +32,8 @@ enum pamu_stash_target { */ struct pamu_stash_attribute { - u32 cpu; /* cpu number */ - u32 cache; /* cache to stash to: L1,L2,L3 */ + u32 cpu; /* cpu number */ + u32 cache; /* cache to stash to: L1,L2,L3 */ }; #endif /* __FSL_PAMU_STASH_H */ diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 6ee947e..abeedc9 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -18,22 +18,13 @@ #define pr_fmt(fmt) "fsl-pamu: %s: " fmt, __func__ -#include -#include -#include -#include -#include -#include +#include "fsl_pamu.h" + #include -#include -#include -#include #include -#include -#include -#include -#include "fsl_pamu.h" +#include +#include /* define indexes for each operation mapping scenario */ #define OMI_QMAN 0x00 @@ -44,13 +35,13 @@ #define make64(high, low) (((u64)(high) << 32) | (low)) struct pamu_isr_data { - void __iomem *pamu_reg_base; /* Base address of PAMU regs*/ + void __iomem *pamu_reg_base; /* Base address of PAMU regs */ unsigned int count; /* The number of PAMUs */ }; static struct paace *ppaact; static struct paace *spaact; -static struct ome *omt; +static struct ome *omt __initdata; /* * Table for matching compatible strings, for device tree @@ -58,14 +49,13 @@ static struct ome *omt; * "fsl,qoriq-device-config-2.0" corresponds to T4 & B4 * SOCs. For the older SOCs "fsl,qoriq-device-config-1.0" * string would be used. -*/ -static const struct of_device_id guts_device_ids[] = { + */ +static const struct of_device_id guts_device_ids[] __initconst = { { .compatible = "fsl,qoriq-device-config-1.0", }, { .compatible = "fsl,qoriq-device-config-2.0", }, {} }; - /* * Table for matching compatible strings, for device tree * L3 cache controller node. @@ -73,7 +63,7 @@ static const struct of_device_id guts_device_ids[] = { * "fsl,b4860-l3-cache-controller" corresponds to B4 & * "fsl,p4080-l3-cache-controller" corresponds to other, * SOCs. -*/ + */ static const struct of_device_id l3_device_ids[] = { { .compatible = "fsl,t4240-l3-cache-controller", }, { .compatible = "fsl,b4860-l3-cache-controller", }, @@ -85,7 +75,7 @@ static const struct of_device_id l3_device_ids[] = { static u32 max_subwindow_count; /* Pool for fspi allocation */ -struct gen_pool *spaace_pool; +static struct gen_pool *spaace_pool; /** * pamu_get_max_subwin_cnt() - Return the maximum supported @@ -170,7 +160,7 @@ int pamu_disable_liodn(int liodn) static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) { /* Bug if not a power of 2 */ - BUG_ON((addrspace_size & (addrspace_size - 1))); + BUG_ON(addrspace_size & (addrspace_size - 1)); /* window size is 2^(WSE+1) bytes */ return fls64(addrspace_size) - 2; @@ -179,8 +169,8 @@ static unsigned int map_addrspace_size_to_wse(phys_addr_t addrspace_size) /* Derive the PAACE window count encoding for the subwindow count */ static unsigned int map_subwindow_cnt_to_wce(u32 subwindow_cnt) { - /* window count is 2^(WCE+1) bytes */ - return __ffs(subwindow_cnt) - 1; + /* window count is 2^(WCE+1) bytes */ + return __ffs(subwindow_cnt) - 1; } /* @@ -241,7 +231,7 @@ static struct paace *pamu_get_spaace(struct paace *paace, u32 wnum) * If no SPAACE entry is available or the allocator can not reserve the required * number of contiguous entries function returns ULONG_MAX indicating a failure. * -*/ + */ static unsigned long pamu_get_fspi_and_allocate(u32 subwin_cnt) { unsigned long spaace_addr; @@ -288,9 +278,8 @@ int pamu_update_paace_stash(int liodn, u32 subwin, u32 value) } if (subwin) { paace = pamu_get_spaace(paace, subwin - 1); - if (!paace) { + if (!paace) return -ENOENT; - } } set_bf(paace->impl_attr, PAACE_IA_CID, value); @@ -311,14 +300,12 @@ int pamu_disable_spaace(int liodn, u32 subwin) } if (subwin) { paace = pamu_get_spaace(paace, subwin - 1); - if (!paace) { + if (!paace) return -ENOENT; - } - set_bf(paace->addr_bitfields, PAACE_AF_V, - PAACE_V_INVALID); + set_bf(paace->addr_bitfields, PAACE_AF_V, PAACE_V_INVALID); } else { set_bf(paace->addr_bitfields, PAACE_AF_AP, - PAACE_AP_PERMS_DENIED); + PAACE_AP_PERMS_DENIED); } mb(); @@ -326,7 +313,6 @@ int pamu_disable_spaace(int liodn, u32 subwin) return 0; } - /** * pamu_config_paace() - Sets up PPAACE entry for specified liodn * @@ -352,7 +338,8 @@ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, unsigned long fspi; if ((win_size & (win_size - 1)) || win_size < PAMU_PAGE_SIZE) { - pr_debug("window size too small or not a power of two %pa\n", &win_size); + pr_debug("window size too small or not a power of two %pa\n", + &win_size); return -EINVAL; } @@ -362,13 +349,12 @@ int pamu_config_ppaace(int liodn, phys_addr_t win_addr, phys_addr_t win_size, } ppaace = pamu_get_ppaace(liodn); - if (!ppaace) { + if (!ppaace) return -ENOENT; - } /* window size is 2^(WSE+1) bytes */ set_bf(ppaace->addr_bitfields, PPAACE_AF_WSE, - map_addrspace_size_to_wse(win_size)); + map_addrspace_size_to_wse(win_size)); pamu_init_ppaace(ppaace); @@ -442,7 +428,6 @@ int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin, { struct paace *paace; - /* setup sub-windows */ if (!subwin_cnt) { pr_debug("Invalid subwindow count\n"); @@ -510,11 +495,11 @@ int pamu_config_spaace(int liodn, u32 subwin_cnt, u32 subwin, } /** -* get_ome_index() - Returns the index in the operation mapping table -* for device. -* @*omi_index: pointer for storing the index value -* -*/ + * get_ome_index() - Returns the index in the operation mapping table + * for device. + * @*omi_index: pointer for storing the index value + * + */ void get_ome_index(u32 *omi_index, struct device *dev) { if (of_device_is_compatible(dev->of_node, "fsl,qman-portal")) @@ -544,9 +529,10 @@ u32 get_stash_id(u32 stash_dest_hint, u32 vcpu) if (stash_dest_hint == PAMU_ATTR_CACHE_L3) { node = of_find_matching_node(NULL, l3_device_ids); if (node) { - prop = of_get_property(node, "cache-stash-id", 0); + prop = of_get_property(node, "cache-stash-id", NULL); if (!prop) { - pr_debug("missing cache-stash-id at %s\n", node->full_name); + pr_debug("missing cache-stash-id at %s\n", + node->full_name); of_node_put(node); return ~(u32)0; } @@ -570,9 +556,10 @@ found_cpu_node: /* find the hwnode that represents the cache */ for (cache_level = PAMU_ATTR_CACHE_L1; (cache_level < PAMU_ATTR_CACHE_L3) && found; cache_level++) { if (stash_dest_hint == cache_level) { - prop = of_get_property(node, "cache-stash-id", 0); + prop = of_get_property(node, "cache-stash-id", NULL); if (!prop) { - pr_debug("missing cache-stash-id at %s\n", node->full_name); + pr_debug("missing cache-stash-id at %s\n", + node->full_name); of_node_put(node); return ~(u32)0; } @@ -580,10 +567,10 @@ found_cpu_node: return be32_to_cpup(prop); } - prop = of_get_property(node, "next-level-cache", 0); + prop = of_get_property(node, "next-level-cache", NULL); if (!prop) { pr_debug("can't find next-level-cache at %s\n", - node->full_name); + node->full_name); of_node_put(node); return ~(u32)0; /* can't traverse any further */ } @@ -598,7 +585,7 @@ found_cpu_node: } pr_debug("stash dest not found for %d on vcpu %d\n", - stash_dest_hint, vcpu); + stash_dest_hint, vcpu); return ~(u32)0; } @@ -612,7 +599,7 @@ found_cpu_node: * Memory accesses to QMAN and BMAN private memory need not be coherent, so * clear the PAACE entry coherency attribute for them. */ -static void setup_qbman_paace(struct paace *ppaace, int paace_type) +static void __init setup_qbman_paace(struct paace *ppaace, int paace_type) { switch (paace_type) { case QMAN_PAACE: @@ -626,7 +613,7 @@ static void setup_qbman_paace(struct paace *ppaace, int paace_type) case QMAN_PORTAL_PAACE: set_bf(ppaace->impl_attr, PAACE_IA_OTM, PAACE_OTM_INDEXED); ppaace->op_encode.index_ot.omi = OMI_QMAN; - /*Set DQRR and Frame stashing for the L3 cache */ + /* Set DQRR and Frame stashing for the L3 cache */ set_bf(ppaace->impl_attr, PAACE_IA_CID, get_stash_id(PAMU_ATTR_CACHE_L3, 0)); break; case BMAN_PAACE: @@ -679,7 +666,7 @@ static void __init setup_omt(struct ome *omt) * Get the maximum number of PAACT table entries * and subwindows supported by PAMU */ -static void get_pamu_cap_values(unsigned long pamu_reg_base) +static void __init get_pamu_cap_values(unsigned long pamu_reg_base) { u32 pc_val; @@ -689,9 +676,9 @@ static void get_pamu_cap_values(unsigned long pamu_reg_base) } /* Setup PAMU registers pointing to PAACT, SPAACT and OMT */ -int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, - phys_addr_t ppaact_phys, phys_addr_t spaact_phys, - phys_addr_t omt_phys) +static int __init setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, + phys_addr_t ppaact_phys, phys_addr_t spaact_phys, + phys_addr_t omt_phys) { u32 *pc; struct pamu_mmap_regs *pamu_regs; @@ -727,7 +714,7 @@ int setup_one_pamu(unsigned long pamu_reg_base, unsigned long pamu_reg_size, */ out_be32((u32 *)(pamu_reg_base + PAMU_PICS), - PAMU_ACCESS_VIOLATION_ENABLE); + PAMU_ACCESS_VIOLATION_ENABLE); out_be32(pc, PAMU_PC_PE | PAMU_PC_OCE | PAMU_PC_SPCC | PAMU_PC_PPCC); return 0; } @@ -757,9 +744,9 @@ static void __init setup_liodns(void) ppaace->wbah = 0; set_bf(ppaace->addr_bitfields, PPAACE_AF_WBAL, 0); set_bf(ppaace->impl_attr, PAACE_IA_ATM, - PAACE_ATM_NO_XLATE); + PAACE_ATM_NO_XLATE); set_bf(ppaace->addr_bitfields, PAACE_AF_AP, - PAACE_AP_PERMS_ALL); + PAACE_AP_PERMS_ALL); if (of_device_is_compatible(node, "fsl,qman-portal")) setup_qbman_paace(ppaace, QMAN_PORTAL_PAACE); if (of_device_is_compatible(node, "fsl,qman")) @@ -772,7 +759,7 @@ static void __init setup_liodns(void) } } -irqreturn_t pamu_av_isr(int irq, void *arg) +static irqreturn_t pamu_av_isr(int irq, void *arg) { struct pamu_isr_data *data = arg; phys_addr_t phys; @@ -792,14 +779,16 @@ irqreturn_t pamu_av_isr(int irq, void *arg) pr_emerg("POES2=%08x\n", in_be32(p + PAMU_POES2)); pr_emerg("AVS1=%08x\n", avs1); pr_emerg("AVS2=%08x\n", in_be32(p + PAMU_AVS2)); - pr_emerg("AVA=%016llx\n", make64(in_be32(p + PAMU_AVAH), - in_be32(p + PAMU_AVAL))); + pr_emerg("AVA=%016llx\n", + make64(in_be32(p + PAMU_AVAH), + in_be32(p + PAMU_AVAL))); pr_emerg("UDAD=%08x\n", in_be32(p + PAMU_UDAD)); - pr_emerg("POEA=%016llx\n", make64(in_be32(p + PAMU_POEAH), - in_be32(p + PAMU_POEAL))); + pr_emerg("POEA=%016llx\n", + make64(in_be32(p + PAMU_POEAH), + in_be32(p + PAMU_POEAL))); phys = make64(in_be32(p + PAMU_POEAH), - in_be32(p + PAMU_POEAL)); + in_be32(p + PAMU_POEAL)); /* Assume that POEA points to a PAACE */ if (phys) { @@ -807,11 +796,12 @@ irqreturn_t pamu_av_isr(int irq, void *arg) /* Only the first four words are relevant */ for (j = 0; j < 4; j++) - pr_emerg("PAACE[%u]=%08x\n", j, in_be32(paace + j)); + pr_emerg("PAACE[%u]=%08x\n", + j, in_be32(paace + j)); } /* clear access violation condition */ - out_be32((p + PAMU_AVS1), avs1 & PAMU_AV_MASK); + out_be32(p + PAMU_AVS1, avs1 & PAMU_AV_MASK); paace = pamu_get_ppaace(avs1 >> PAMU_AVS1_LIODN_SHIFT); BUG_ON(!paace); /* check if we got a violation for a disabled LIODN */ @@ -827,13 +817,13 @@ irqreturn_t pamu_av_isr(int irq, void *arg) /* Disable the LIODN */ ret = pamu_disable_liodn(avs1 >> PAMU_AVS1_LIODN_SHIFT); BUG_ON(ret); - pr_emerg("Disabling liodn %x\n", avs1 >> PAMU_AVS1_LIODN_SHIFT); + pr_emerg("Disabling liodn %x\n", + avs1 >> PAMU_AVS1_LIODN_SHIFT); } out_be32((p + PAMU_PICS), pics); } } - return IRQ_HANDLED; } @@ -952,7 +942,7 @@ static int __init create_csd(phys_addr_t phys, size_t size, u32 csd_port_id) } if (i == 0 || i == num_laws) { - /* This should never happen*/ + /* This should never happen */ ret = -ENOENT; goto error; } @@ -998,26 +988,27 @@ error: static const struct { u32 svr; u32 port_id; -} port_id_map[] = { - {0x82100010, 0xFF000000}, /* P2040 1.0 */ - {0x82100011, 0xFF000000}, /* P2040 1.1 */ - {0x82100110, 0xFF000000}, /* P2041 1.0 */ - {0x82100111, 0xFF000000}, /* P2041 1.1 */ - {0x82110310, 0xFF000000}, /* P3041 1.0 */ - {0x82110311, 0xFF000000}, /* P3041 1.1 */ - {0x82010020, 0xFFF80000}, /* P4040 2.0 */ - {0x82000020, 0xFFF80000}, /* P4080 2.0 */ - {0x82210010, 0xFC000000}, /* P5010 1.0 */ - {0x82210020, 0xFC000000}, /* P5010 2.0 */ - {0x82200010, 0xFC000000}, /* P5020 1.0 */ - {0x82050010, 0xFF800000}, /* P5021 1.0 */ - {0x82040010, 0xFF800000}, /* P5040 1.0 */ +} port_id_map[] __initconst = { + {(SVR_P2040 << 8) | 0x10, 0xFF000000}, /* P2040 1.0 */ + {(SVR_P2040 << 8) | 0x11, 0xFF000000}, /* P2040 1.1 */ + {(SVR_P2041 << 8) | 0x10, 0xFF000000}, /* P2041 1.0 */ + {(SVR_P2041 << 8) | 0x11, 0xFF000000}, /* P2041 1.1 */ + {(SVR_P3041 << 8) | 0x10, 0xFF000000}, /* P3041 1.0 */ + {(SVR_P3041 << 8) | 0x11, 0xFF000000}, /* P3041 1.1 */ + {(SVR_P4040 << 8) | 0x20, 0xFFF80000}, /* P4040 2.0 */ + {(SVR_P4080 << 8) | 0x20, 0xFFF80000}, /* P4080 2.0 */ + {(SVR_P5010 << 8) | 0x10, 0xFC000000}, /* P5010 1.0 */ + {(SVR_P5010 << 8) | 0x20, 0xFC000000}, /* P5010 2.0 */ + {(SVR_P5020 << 8) | 0x10, 0xFC000000}, /* P5020 1.0 */ + {(SVR_P5021 << 8) | 0x10, 0xFF800000}, /* P5021 1.0 */ + {(SVR_P5040 << 8) | 0x10, 0xFF800000}, /* P5040 1.0 */ }; #define SVR_SECURITY 0x80000 /* The Security (E) bit */ static int __init fsl_pamu_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; void __iomem *pamu_regs = NULL; struct ccsr_guts __iomem *guts_regs = NULL; u32 pamubypenr, pamu_counter; @@ -1042,22 +1033,21 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) * NOTE : All PAMUs share the same LIODN tables. */ - pamu_regs = of_iomap(pdev->dev.of_node, 0); + pamu_regs = of_iomap(dev->of_node, 0); if (!pamu_regs) { - dev_err(&pdev->dev, "ioremap of PAMU node failed\n"); + dev_err(dev, "ioremap of PAMU node failed\n"); return -ENOMEM; } - of_get_address(pdev->dev.of_node, 0, &size, NULL); + of_get_address(dev->of_node, 0, &size, NULL); - irq = irq_of_parse_and_map(pdev->dev.of_node, 0); + irq = irq_of_parse_and_map(dev->of_node, 0); if (irq == NO_IRQ) { - dev_warn(&pdev->dev, "no interrupts listed in PAMU node\n"); + dev_warn(dev, "no interrupts listed in PAMU node\n"); goto error; } - data = kzalloc(sizeof(struct pamu_isr_data), GFP_KERNEL); + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) { - dev_err(&pdev->dev, "PAMU isr data memory allocation failed\n"); ret = -ENOMEM; goto error; } @@ -1067,15 +1057,14 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) /* The ISR needs access to the regs, so we won't iounmap them */ ret = request_irq(irq, pamu_av_isr, 0, "pamu", data); if (ret < 0) { - dev_err(&pdev->dev, "error %i installing ISR for irq %i\n", - ret, irq); + dev_err(dev, "error %i installing ISR for irq %i\n", ret, irq); goto error; } guts_node = of_find_matching_node(NULL, guts_device_ids); if (!guts_node) { - dev_err(&pdev->dev, "could not find GUTS node %s\n", - pdev->dev.of_node->full_name); + dev_err(dev, "could not find GUTS node %s\n", + dev->of_node->full_name); ret = -ENODEV; goto error; } @@ -1083,7 +1072,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) guts_regs = of_iomap(guts_node, 0); of_node_put(guts_node); if (!guts_regs) { - dev_err(&pdev->dev, "ioremap of GUTS node failed\n"); + dev_err(dev, "ioremap of GUTS node failed\n"); ret = -ENODEV; goto error; } @@ -1103,7 +1092,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) p = alloc_pages(GFP_KERNEL | __GFP_ZERO, order); if (!p) { - dev_err(&pdev->dev, "unable to allocate PAACT/SPAACT/OMT block\n"); + dev_err(dev, "unable to allocate PAACT/SPAACT/OMT block\n"); ret = -ENOMEM; goto error; } @@ -1113,7 +1102,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) /* Make sure the memory is naturally aligned */ if (ppaact_phys & ((PAGE_SIZE << order) - 1)) { - dev_err(&pdev->dev, "PAACT/OMT block is unaligned\n"); + dev_err(dev, "PAACT/OMT block is unaligned\n"); ret = -ENOMEM; goto error; } @@ -1121,7 +1110,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) spaact = (void *)ppaact + (PAGE_SIZE << get_order(PAACT_SIZE)); omt = (void *)spaact + (PAGE_SIZE << get_order(SPAACT_SIZE)); - dev_dbg(&pdev->dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); + dev_dbg(dev, "ppaact virt=%p phys=%pa\n", ppaact, &ppaact_phys); /* Check to see if we need to implement the work-around on this SOC */ @@ -1129,21 +1118,19 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) for (i = 0; i < ARRAY_SIZE(port_id_map); i++) { if (port_id_map[i].svr == (mfspr(SPRN_SVR) & ~SVR_SECURITY)) { csd_port_id = port_id_map[i].port_id; - dev_dbg(&pdev->dev, "found matching SVR %08x\n", + dev_dbg(dev, "found matching SVR %08x\n", port_id_map[i].svr); break; } } if (csd_port_id) { - dev_dbg(&pdev->dev, "creating coherency subdomain at address " - "%pa, size %zu, port id 0x%08x", &ppaact_phys, - mem_size, csd_port_id); + dev_dbg(dev, "creating coherency subdomain at address %pa, size %zu, port id 0x%08x", + &ppaact_phys, mem_size, csd_port_id); ret = create_csd(ppaact_phys, mem_size, csd_port_id); if (ret) { - dev_err(&pdev->dev, "could not create coherence " - "subdomain\n"); + dev_err(dev, "could not create coherence subdomain\n"); return ret; } } @@ -1154,7 +1141,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) spaace_pool = gen_pool_create(ilog2(sizeof(struct paace)), -1); if (!spaace_pool) { ret = -ENOMEM; - dev_err(&pdev->dev, "PAMU : failed to allocate spaace gen pool\n"); + dev_err(dev, "Failed to allocate spaace gen pool\n"); goto error; } @@ -1167,9 +1154,9 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) for (pamu_reg_off = 0, pamu_counter = 0x80000000; pamu_reg_off < size; pamu_reg_off += PAMU_OFFSET, pamu_counter >>= 1) { - pamu_reg_base = (unsigned long) pamu_regs + pamu_reg_off; + pamu_reg_base = (unsigned long)pamu_regs + pamu_reg_off; setup_one_pamu(pamu_reg_base, pamu_reg_off, ppaact_phys, - spaact_phys, omt_phys); + spaact_phys, omt_phys); /* Disable PAMU bypass for this PAMU */ pamubypenr &= ~pamu_counter; } @@ -1181,7 +1168,7 @@ static int __init fsl_pamu_probe(struct platform_device *pdev) iounmap(guts_regs); - /* Enable DMA for the LIODNs in the device tree*/ + /* Enable DMA for the LIODNs in the device tree */ setup_liodns(); diff --git a/drivers/iommu/fsl_pamu.h b/drivers/iommu/fsl_pamu.h index 8fc1a12..aab723f 100644 --- a/drivers/iommu/fsl_pamu.h +++ b/drivers/iommu/fsl_pamu.h @@ -19,13 +19,15 @@ #ifndef __FSL_PAMU_H #define __FSL_PAMU_H +#include + #include /* Bit Field macros * v = bit field variable; m = mask, m##_SHIFT = shift, x = value to load */ -#define set_bf(v, m, x) (v = ((v) & ~(m)) | (((x) << (m##_SHIFT)) & (m))) -#define get_bf(v, m) (((v) & (m)) >> (m##_SHIFT)) +#define set_bf(v, m, x) (v = ((v) & ~(m)) | (((x) << m##_SHIFT) & (m))) +#define get_bf(v, m) (((v) & (m)) >> m##_SHIFT) /* PAMU CCSR space */ #define PAMU_PGC 0x00000000 /* Allows all peripheral accesses */ @@ -65,7 +67,7 @@ struct pamu_mmap_regs { #define PAMU_AVS1_GCV 0x2000 #define PAMU_AVS1_PDV 0x4000 #define PAMU_AV_MASK (PAMU_AVS1_AV | PAMU_AVS1_OTV | PAMU_AVS1_APV | PAMU_AVS1_WAV \ - | PAMU_AVS1_LAV | PAMU_AVS1_GCV | PAMU_AVS1_PDV) + | PAMU_AVS1_LAV | PAMU_AVS1_GCV | PAMU_AVS1_PDV) #define PAMU_AVS1_LIODN_SHIFT 16 #define PAMU_LAV_LIODN_NOT_IN_PPAACT 0x400 @@ -198,8 +200,7 @@ struct pamu_mmap_regs { #define PAACE_ATM_NO_XLATE 0x00 #define PAACE_ATM_WINDOW_XLATE 0x01 #define PAACE_ATM_PAGE_XLATE 0x02 -#define PAACE_ATM_WIN_PG_XLATE \ - (PAACE_ATM_WINDOW_XLATE | PAACE_ATM_PAGE_XLATE) +#define PAACE_ATM_WIN_PG_XLATE (PAACE_ATM_WINDOW_XLATE | PAACE_ATM_PAGE_XLATE) #define PAACE_OTM_NO_XLATE 0x00 #define PAACE_OTM_IMMEDIATE 0x01 #define PAACE_OTM_INDEXED 0x02 @@ -219,7 +220,7 @@ struct pamu_mmap_regs { #define PAACE_TCEF_FORMAT0_8B 0x00 #define PAACE_TCEF_FORMAT1_RSVD 0x01 /* - * Hard coded value for the PAACT size to accomodate + * Hard coded value for the PAACT size to accommodate * maximum LIODN value generated by u-boot. */ #define PAACE_NUMBER_ENTRIES 0x500 @@ -332,7 +333,7 @@ struct paace { #define NUM_MOE 128 struct ome { u8 moe[NUM_MOE]; -} __attribute__((packed)); +} __packed; #define PAACT_SIZE (sizeof(struct paace) * PAACE_NUMBER_ENTRIES) #define SPAACT_SIZE (sizeof(struct paace) * SPAACE_NUMBER_ENTRIES) diff --git a/drivers/iommu/fsl_pamu_domain.c b/drivers/iommu/fsl_pamu_domain.c index c828f80..ceebd28 100644 --- a/drivers/iommu/fsl_pamu_domain.c +++ b/drivers/iommu/fsl_pamu_domain.c @@ -19,26 +19,10 @@ #define pr_fmt(fmt) "fsl-pamu-domain: %s: " fmt, __func__ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - #include "fsl_pamu_domain.h" +#include + /* * Global spinlock that needs to be held while * configuring PAMU. @@ -51,23 +35,21 @@ static DEFINE_SPINLOCK(device_domain_lock); static int __init iommu_init_mempool(void) { - fsl_pamu_domain_cache = kmem_cache_create("fsl_pamu_domain", - sizeof(struct fsl_dma_domain), - 0, - SLAB_HWCACHE_ALIGN, - - NULL); + sizeof(struct fsl_dma_domain), + 0, + SLAB_HWCACHE_ALIGN, + NULL); if (!fsl_pamu_domain_cache) { pr_debug("Couldn't create fsl iommu_domain cache\n"); return -ENOMEM; } iommu_devinfo_cache = kmem_cache_create("iommu_devinfo", - sizeof(struct device_domain_info), - 0, - SLAB_HWCACHE_ALIGN, - NULL); + sizeof(struct device_domain_info), + 0, + SLAB_HWCACHE_ALIGN, + NULL); if (!iommu_devinfo_cache) { pr_debug("Couldn't create devinfo cache\n"); kmem_cache_destroy(fsl_pamu_domain_cache); @@ -80,8 +62,7 @@ static int __init iommu_init_mempool(void) static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t iova) { u32 win_cnt = dma_domain->win_cnt; - struct dma_window *win_ptr = - &dma_domain->win_arr[0]; + struct dma_window *win_ptr = &dma_domain->win_arr[0]; struct iommu_domain_geometry *geom; geom = &dma_domain->iommu_domain->geometry; @@ -103,22 +84,20 @@ static phys_addr_t get_phys_addr(struct fsl_dma_domain *dma_domain, dma_addr_t i } if (win_ptr->valid) - return (win_ptr->paddr + (iova & (win_ptr->size - 1))); + return win_ptr->paddr + (iova & (win_ptr->size - 1)); return 0; } static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) { - struct dma_window *sub_win_ptr = - &dma_domain->win_arr[0]; + struct dma_window *sub_win_ptr = &dma_domain->win_arr[0]; int i, ret; unsigned long rpn, flags; for (i = 0; i < dma_domain->win_cnt; i++) { if (sub_win_ptr[i].valid) { - rpn = sub_win_ptr[i].paddr >> - PAMU_PAGE_SHIFT; + rpn = sub_win_ptr[i].paddr >> PAMU_PAGE_SHIFT; spin_lock_irqsave(&iommu_lock, flags); ret = pamu_config_spaace(liodn, dma_domain->win_cnt, i, sub_win_ptr[i].size, @@ -130,7 +109,7 @@ static int map_subwins(int liodn, struct fsl_dma_domain *dma_domain) sub_win_ptr[i].prot); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) { - pr_debug("PAMU SPAACE configuration failed for liodn %d\n", + pr_debug("SPAACE configuration failed for liodn %d\n", liodn); return ret; } @@ -156,8 +135,7 @@ static int map_win(int liodn, struct fsl_dma_domain *dma_domain) 0, wnd->prot); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) - pr_debug("PAMU PAACE configuration failed for liodn %d\n", - liodn); + pr_debug("PAACE configuration failed for liodn %d\n", liodn); return ret; } @@ -169,7 +147,6 @@ static int map_liodn(int liodn, struct fsl_dma_domain *dma_domain) return map_subwins(liodn, dma_domain); else return map_win(liodn, dma_domain); - } /* Update window/subwindow mapping for the LIODN */ @@ -190,7 +167,8 @@ static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr (wnd_nr > 0) ? 1 : 0, wnd->prot); if (ret) - pr_debug("Subwindow reconfiguration failed for liodn %d\n", liodn); + pr_debug("Subwindow reconfiguration failed for liodn %d\n", + liodn); } else { phys_addr_t wnd_addr; @@ -200,10 +178,11 @@ static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr wnd->size, ~(u32)0, wnd->paddr >> PAMU_PAGE_SHIFT, - dma_domain->snoop_id, dma_domain->stash_id, - 0, wnd->prot); + dma_domain->snoop_id, dma_domain->stash_id, + 0, wnd->prot); if (ret) - pr_debug("Window reconfiguration failed for liodn %d\n", liodn); + pr_debug("Window reconfiguration failed for liodn %d\n", + liodn); } spin_unlock_irqrestore(&iommu_lock, flags); @@ -212,14 +191,15 @@ static int update_liodn(int liodn, struct fsl_dma_domain *dma_domain, u32 wnd_nr } static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, - u32 val) + u32 val) { int ret = 0, i; unsigned long flags; spin_lock_irqsave(&iommu_lock, flags); if (!dma_domain->win_arr) { - pr_debug("Windows not configured, stash destination update failed for liodn %d\n", liodn); + pr_debug("Windows not configured, stash destination update failed for liodn %d\n", + liodn); spin_unlock_irqrestore(&iommu_lock, flags); return -EINVAL; } @@ -227,7 +207,8 @@ static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, for (i = 0; i < dma_domain->win_cnt; i++) { ret = pamu_update_paace_stash(liodn, i, val); if (ret) { - pr_debug("Failed to update SPAACE %d field for liodn %d\n ", i, liodn); + pr_debug("Failed to update SPAACE %d field for liodn %d\n ", + i, liodn); spin_unlock_irqrestore(&iommu_lock, flags); return ret; } @@ -240,9 +221,9 @@ static int update_liodn_stash(int liodn, struct fsl_dma_domain *dma_domain, /* Set the geometry parameters for a LIODN */ static int pamu_set_liodn(int liodn, struct device *dev, - struct fsl_dma_domain *dma_domain, - struct iommu_domain_geometry *geom_attr, - u32 win_cnt) + struct fsl_dma_domain *dma_domain, + struct iommu_domain_geometry *geom_attr, + u32 win_cnt) { phys_addr_t window_addr, window_size; phys_addr_t subwin_size; @@ -268,7 +249,8 @@ static int pamu_set_liodn(int liodn, struct device *dev, dma_domain->stash_id, win_cnt, 0); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) { - pr_debug("PAMU PAACE configuration failed for liodn %d, win_cnt =%d\n", liodn, win_cnt); + pr_debug("PAACE configuration failed for liodn %d, win_cnt =%d\n", + liodn, win_cnt); return ret; } @@ -285,7 +267,8 @@ static int pamu_set_liodn(int liodn, struct device *dev, 0, 0); spin_unlock_irqrestore(&iommu_lock, flags); if (ret) { - pr_debug("PAMU SPAACE configuration failed for liodn %d\n", liodn); + pr_debug("SPAACE configuration failed for liodn %d\n", + liodn); return ret; } } @@ -301,13 +284,13 @@ static int check_size(u64 size, dma_addr_t iova) * to PAMU page size. */ if ((size & (size - 1)) || size < PAMU_PAGE_SIZE) { - pr_debug("%s: size too small or not a power of two\n", __func__); + pr_debug("Size too small or not a power of two\n"); return -EINVAL; } - /* iova must be page size aligned*/ + /* iova must be page size aligned */ if (iova & (size - 1)) { - pr_debug("%s: address is not aligned with window size\n", __func__); + pr_debug("Address is not aligned with window size\n"); return -EINVAL; } @@ -396,16 +379,15 @@ static void attach_device(struct fsl_dma_domain *dma_domain, int liodn, struct d if (!dev->archdata.iommu_domain) dev->archdata.iommu_domain = info; spin_unlock_irqrestore(&device_domain_lock, flags); - } static phys_addr_t fsl_pamu_iova_to_phys(struct iommu_domain *domain, - dma_addr_t iova) + dma_addr_t iova) { struct fsl_dma_domain *dma_domain = domain->priv; - if ((iova < domain->geometry.aperture_start) || - iova > (domain->geometry.aperture_end)) + if (iova < domain->geometry.aperture_start || + iova > domain->geometry.aperture_end) return 0; return get_phys_addr(dma_domain, iova); @@ -460,7 +442,7 @@ static int pamu_set_domain_geometry(struct fsl_dma_domain *dma_domain, list_for_each_entry(info, &dma_domain->devices, link) { ret = pamu_set_liodn(info->liodn, info->dev, dma_domain, - geom_attr, win_cnt); + geom_attr, win_cnt); if (ret) break; } @@ -543,7 +525,6 @@ static void fsl_pamu_window_disable(struct iommu_domain *domain, u32 wnd_nr) } spin_unlock_irqrestore(&dma_domain->domain_lock, flags); - } static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, @@ -576,7 +557,7 @@ static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, win_size = dma_domain->geom_size >> ilog2(dma_domain->win_cnt); if (size > win_size) { - pr_debug("Invalid window size \n"); + pr_debug("Invalid window size\n"); spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return -EINVAL; } @@ -622,8 +603,8 @@ static int fsl_pamu_window_enable(struct iommu_domain *domain, u32 wnd_nr, * and window mappings. */ static int handle_attach_device(struct fsl_dma_domain *dma_domain, - struct device *dev, const u32 *liodn, - int num) + struct device *dev, const u32 *liodn, + int num) { unsigned long flags; struct iommu_domain *domain = dma_domain->iommu_domain; @@ -632,11 +613,10 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain, spin_lock_irqsave(&dma_domain->domain_lock, flags); for (i = 0; i < num; i++) { - /* Ensure that LIODN value is valid */ if (liodn[i] >= PAACE_NUMBER_ENTRIES) { pr_debug("Invalid liodn %d, attach device failed for %s\n", - liodn[i], dev->of_node->full_name); + liodn[i], dev->of_node->full_name); ret = -EINVAL; break; } @@ -649,9 +629,9 @@ static int handle_attach_device(struct fsl_dma_domain *dma_domain, */ if (dma_domain->win_arr) { u32 win_cnt = dma_domain->win_cnt > 1 ? dma_domain->win_cnt : 0; + ret = pamu_set_liodn(liodn[i], dev, dma_domain, - &domain->geometry, - win_cnt); + &domain->geometry, win_cnt); if (ret) break; if (dma_domain->mapped) { @@ -698,19 +678,18 @@ static int fsl_pamu_attach_device(struct iommu_domain *domain, liodn = of_get_property(dev->of_node, "fsl,liodn", &len); if (liodn) { liodn_cnt = len / sizeof(u32); - ret = handle_attach_device(dma_domain, dev, - liodn, liodn_cnt); + ret = handle_attach_device(dma_domain, dev, liodn, liodn_cnt); } else { pr_debug("missing fsl,liodn property at %s\n", - dev->of_node->full_name); - ret = -EINVAL; + dev->of_node->full_name); + ret = -EINVAL; } return ret; } static void fsl_pamu_detach_device(struct iommu_domain *domain, - struct device *dev) + struct device *dev) { struct fsl_dma_domain *dma_domain = domain->priv; const u32 *prop; @@ -738,7 +717,7 @@ static void fsl_pamu_detach_device(struct iommu_domain *domain, detach_device(dev, dma_domain); else pr_debug("missing fsl,liodn property at %s\n", - dev->of_node->full_name); + dev->of_node->full_name); } static int configure_domain_geometry(struct iommu_domain *domain, void *data) @@ -754,10 +733,10 @@ static int configure_domain_geometry(struct iommu_domain *domain, void *data) * DMA outside of the geometry. */ if (check_size(geom_size, geom_attr->aperture_start) || - !geom_attr->force_aperture) { - pr_debug("Invalid PAMU geometry attributes\n"); - return -EINVAL; - } + !geom_attr->force_aperture) { + pr_debug("Invalid PAMU geometry attributes\n"); + return -EINVAL; + } spin_lock_irqsave(&dma_domain->domain_lock, flags); if (dma_domain->enabled) { @@ -786,7 +765,7 @@ static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) spin_lock_irqsave(&dma_domain->domain_lock, flags); memcpy(&dma_domain->dma_stash, stash_attr, - sizeof(struct pamu_stash_attribute)); + sizeof(struct pamu_stash_attribute)); dma_domain->stash_id = get_stash_id(stash_attr->cache, stash_attr->cpu); @@ -803,7 +782,7 @@ static int configure_domain_stash(struct fsl_dma_domain *dma_domain, void *data) return ret; } -/* Configure domain dma state i.e. enable/disable DMA*/ +/* Configure domain dma state i.e. enable/disable DMA */ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool enable) { struct device_domain_info *info; @@ -819,8 +798,7 @@ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool en } dma_domain->enabled = enable; - list_for_each_entry(info, &dma_domain->devices, - link) { + list_for_each_entry(info, &dma_domain->devices, link) { ret = (enable) ? pamu_enable_liodn(info->liodn) : pamu_disable_liodn(info->liodn); if (ret) @@ -833,12 +811,11 @@ static int configure_domain_dma_state(struct fsl_dma_domain *dma_domain, bool en } static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, - enum iommu_attr attr_type, void *data) + enum iommu_attr attr_type, void *data) { struct fsl_dma_domain *dma_domain = domain->priv; int ret = 0; - switch (attr_type) { case DOMAIN_ATTR_GEOMETRY: ret = configure_domain_geometry(domain, data); @@ -853,22 +830,21 @@ static int fsl_pamu_set_domain_attr(struct iommu_domain *domain, pr_debug("Unsupported attribute type\n"); ret = -EINVAL; break; - }; + } return ret; } static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, - enum iommu_attr attr_type, void *data) + enum iommu_attr attr_type, void *data) { struct fsl_dma_domain *dma_domain = domain->priv; int ret = 0; - switch (attr_type) { case DOMAIN_ATTR_FSL_PAMU_STASH: - memcpy((struct pamu_stash_attribute *) data, &dma_domain->dma_stash, - sizeof(struct pamu_stash_attribute)); + memcpy(data, &dma_domain->dma_stash, + sizeof(struct pamu_stash_attribute)); break; case DOMAIN_ATTR_FSL_PAMU_ENABLE: *(int *)data = dma_domain->enabled; @@ -880,7 +856,7 @@ static int fsl_pamu_get_domain_attr(struct iommu_domain *domain, pr_debug("Unsupported attribute type\n"); ret = -EINVAL; break; - }; + } return ret; } @@ -903,11 +879,8 @@ static bool check_pci_ctl_endpt_part(struct pci_controller *pci_ctl) /* Check the PCI controller version number by readding BRR1 register */ version = in_be32(pci_ctl->cfg_addr + (PCI_FSL_BRR1 >> 2)); version &= PCI_FSL_BRR1_VER; - /* If PCI controller version is >= 0x204 we can partition endpoints*/ - if (version >= 0x204) - return 1; - - return 0; + /* If PCI controller version is >= 0x204 we can partition endpoints */ + return version >= 0x204; } /* Get iommu group information from peer devices or devices on the parent bus */ @@ -968,8 +941,9 @@ static struct iommu_group *get_pci_device_group(struct pci_dev *pdev) if (pci_ctl->parent->iommu_group) { group = get_device_iommu_group(pci_ctl->parent); iommu_group_remove_device(pci_ctl->parent); - } else + } else { group = get_shared_pci_device_group(pdev); + } } if (!group) @@ -1055,11 +1029,12 @@ static int fsl_pamu_set_windows(struct iommu_domain *domain, u32 w_count) } ret = pamu_set_domain_geometry(dma_domain, &domain->geometry, - ((w_count > 1) ? w_count : 0)); + w_count > 1 ? w_count : 0); if (!ret) { kfree(dma_domain->win_arr); - dma_domain->win_arr = kzalloc(sizeof(struct dma_window) * - w_count, GFP_ATOMIC); + dma_domain->win_arr = kcalloc(w_count, + sizeof(*dma_domain->win_arr), + GFP_ATOMIC); if (!dma_domain->win_arr) { spin_unlock_irqrestore(&dma_domain->domain_lock, flags); return -ENOMEM; @@ -1095,7 +1070,7 @@ static const struct iommu_ops fsl_pamu_ops = { .remove_device = fsl_pamu_remove_device, }; -int pamu_domain_init(void) +int __init pamu_domain_init(void) { int ret = 0; -- cgit v0.10.2 From a4188beee59763b3507939968677776561adbba5 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Fri, 30 Jan 2015 22:55:55 +0100 Subject: iommu/arm-smmu: Avoid build warning ARM allmodconfig gained a new warning when dma_addr_t is 32-bit wide: drivers/iommu/arm-smmu.c: In function 'arm_smmu_iova_to_phys_hard': drivers/iommu/arm-smmu.c:1255:3: warning: right shift count >= width of type This changes the calculation so that the effective type is always 64-bit. Signed-off-by: Arnd Bergmann Fixes: 859a732e4f713 ("iommu/arm-smmu: add support for iova_to_phys through ATS1PR") Acked-by: Will Deacon Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 1d6d43b..fc13dd5 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1252,7 +1252,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, } else { u32 reg = iova & ~0xfff; writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); - reg = (iova & ~0xfff) >> 32; + reg = ((u64)iova & ~0xfff) >> 32; writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_HI); } -- cgit v0.10.2 From 91f65facba5add493fffb643acdb258bd9f54eb2 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Tue, 3 Feb 2015 13:25:51 +0100 Subject: iommu/amd: Fix amd_iommu_free_device() put_device_state_wait() doesn't loop on the condition and a spurious wakeup will have it free the device state even though there might still be references out to it. Fix this by using 'normal' wait primitives. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 90f70d0..b6398d7 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -151,18 +151,6 @@ static void put_device_state(struct device_state *dev_state) wake_up(&dev_state->wq); } -static void put_device_state_wait(struct device_state *dev_state) -{ - DEFINE_WAIT(wait); - - prepare_to_wait(&dev_state->wq, &wait, TASK_UNINTERRUPTIBLE); - if (!atomic_dec_and_test(&dev_state->count)) - schedule(); - finish_wait(&dev_state->wq, &wait); - - free_device_state(dev_state); -} - /* Must be called under dev_state->lock */ static struct pasid_state **__get_pasid_state_ptr(struct device_state *dev_state, int pasid, bool alloc) @@ -851,7 +839,13 @@ void amd_iommu_free_device(struct pci_dev *pdev) /* Get rid of any remaining pasid states */ free_pasid_states(dev_state); - put_device_state_wait(dev_state); + put_device_state(dev_state); + /* + * Wait until the last reference is dropped before freeing + * the device state. + */ + wait_event(dev_state->wq, !atomic_read(&dev_state->count)); + free_device_state(dev_state); } EXPORT_SYMBOL(amd_iommu_free_device); -- cgit v0.10.2 From a1bec062c90456983225054d39c8a601db48e638 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 4 Feb 2015 15:50:38 +0100 Subject: iommu/amd: Use wait_event in put_pasid_state_wait Now that I learned about possible spurious wakeups this place needs fixing too. Replace the self-coded sleep variant with the generic wait_event() helper. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index b6398d7..5cc1409 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -266,14 +266,7 @@ static void put_pasid_state(struct pasid_state *pasid_state) static void put_pasid_state_wait(struct pasid_state *pasid_state) { - DEFINE_WAIT(wait); - - prepare_to_wait(&pasid_state->wq, &wait, TASK_UNINTERRUPTIBLE); - - if (!atomic_dec_and_test(&pasid_state->count)) - schedule(); - - finish_wait(&pasid_state->wq, &wait); + wait_event(pasid_state->wq, !atomic_read(&pasid_state->count)); free_pasid_state(pasid_state); } -- cgit v0.10.2 From 63ce3ae889db917cff0cf0c65c837ca7160c8a83 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 4 Feb 2015 16:12:55 +0100 Subject: iommu: Update my email address The AMD address is dead for a long time already, replace it with a working one. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 9802485..6b6dc72 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1,6 +1,6 @@ /* * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * Leo Duran * * This program is free software; you can redistribute it and/or modify it diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index b0522f1..e93eb8c 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1,6 +1,6 @@ /* * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * Leo Duran * * This program is free software; you can redistribute it and/or modify it diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h index 95ed6de..b62ff54 100644 --- a/drivers/iommu/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2009-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index cec51a8..c4fffb7 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * Leo Duran * * This program is free software; you can redistribute it and/or modify it diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 90f70d0..d3a3caf 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -1,6 +1,6 @@ /* * Copyright (C) 2010-2012 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -31,7 +31,7 @@ #include "amd_iommu_proto.h" MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Joerg Roedel "); +MODULE_AUTHOR("Joerg Roedel "); #define MAX_DEVICES 0x10000 #define PRI_QUEUE_SIZE 512 @@ -921,7 +921,7 @@ static int __init amd_iommu_v2_init(void) { int ret; - pr_info("AMD IOMMUv2 driver by Joerg Roedel \n"); + pr_info("AMD IOMMUv2 driver by Joerg Roedel \n"); if (!amd_iommu_v2_supported()) { pr_info("AMD IOMMUv2 functionality not available on this system\n"); diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 9e0dcdb..72e683d 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1,6 +1,6 @@ /* * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h index fde250f..a2b7501 100644 --- a/drivers/iommu/irq_remapping.h +++ b/drivers/iommu/irq_remapping.h @@ -1,6 +1,6 @@ /* * Copyright (C) 2012 Advanced Micro Devices, Inc. - * Author: Joerg Roedel + * Author: Joerg Roedel * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published -- cgit v0.10.2 From ae0cbbb1cde1f1bc2454a138a5cab1887d0e103c Mon Sep 17 00:00:00 2001 From: Quentin Lambert Date: Wed, 4 Feb 2015 11:40:07 +0100 Subject: iommu/amd: Convert non-returned local variable to boolean when relevant This patch was produced using Coccinelle. A simplified version of the semantic patch is: @r exists@ identifier f; local idexpression u8 x; identifier xname; @@ f(...) { ...when any ( x@xname = 1; | x@xname = 0; ) ...when any } @bad exists@ identifier r.f; local idexpression u8 r.x expression e1 != {0, 1}, e2; @@ f(...) { ...when any ( x = e1; | x + e2 ) ...when any } @depends on !bad@ identifier r.f; local idexpression u8 r.x; identifier r.xname; @@ f(...) { ... ++ bool xname; - int xname; <... ( x = - 1 + true | x = - -1 + false ) ...> } Signed-off-by: Quentin Lambert Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 9802485..5ac2d11 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -843,10 +843,10 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, size_t size, u16 domid, int pde) { u64 pages; - int s; + bool s; pages = iommu_num_pages(address, size, PAGE_SIZE); - s = 0; + s = false; if (pages > 1) { /* @@ -854,7 +854,7 @@ static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, * TLB entries for this domain */ address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - s = 1; + s = true; } address &= PAGE_MASK; @@ -874,10 +874,10 @@ static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, u64 address, size_t size) { u64 pages; - int s; + bool s; pages = iommu_num_pages(address, size, PAGE_SIZE); - s = 0; + s = false; if (pages > 1) { /* @@ -885,7 +885,7 @@ static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, * TLB entries for this domain */ address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; - s = 1; + s = true; } address &= PAGE_MASK; -- cgit v0.10.2