diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/Kconfig | 20 | ||||
| -rw-r--r-- | drivers/iommu/Makefile | 3 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_init.c | 16 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu-v3.c | 3 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu.c | 28 | ||||
| -rw-r--r-- | drivers/iommu/dmar.c | 21 | ||||
| -rw-r--r-- | drivers/iommu/exynos-iommu.c | 107 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 29 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 29 | ||||
| -rw-r--r-- | drivers/iommu/iova.c | 8 | ||||
| -rw-r--r-- | drivers/iommu/msm_iommu.c | 870 | ||||
| -rw-r--r-- | drivers/iommu/msm_iommu.h | 73 | ||||
| -rw-r--r-- | drivers/iommu/msm_iommu_dev.c | 381 | ||||
| -rw-r--r-- | drivers/iommu/mtk_iommu.c | 49 | ||||
| -rw-r--r-- | drivers/iommu/mtk_iommu.h | 77 | ||||
| -rw-r--r-- | drivers/iommu/mtk_iommu_v1.c | 727 | ||||
| -rw-r--r-- | drivers/iommu/rockchip-iommu.c | 183 |
18 files changed, 1589 insertions, 1037 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 0b1aabb..d432ca8 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -89,8 +89,8 @@ config MSM_IOMMU bool "MSM IOMMU Support" depends on ARM depends on ARCH_MSM8X60 || ARCH_MSM8960 || COMPILE_TEST - depends on BROKEN select IOMMU_API + select IOMMU_IO_PGTABLE_ARMV7S help Support for the IOMMUs found on certain Qualcomm SOCs. These IOMMUs allow virtualization of the address space used by most @@ -344,4 +344,22 @@ config MTK_IOMMU If unsure, say N here. +config MTK_IOMMU_V1 + bool "MTK IOMMU Version 1 (M4U gen1) Support" + depends on ARM + depends on ARCH_MEDIATEK || COMPILE_TEST + select ARM_DMA_USE_IOMMU + select IOMMU_API + select MEMORY + select MTK_SMI + select COMMON_CLK_MT2701_MMSYS + select COMMON_CLK_MT2701_IMGSYS + select COMMON_CLK_MT2701_VDECSYS + help + Support for the M4U on certain Mediatek SoCs. M4U generation 1 HW is + Multimedia Memory Managememt Unit. This option enables remapping of + DMA memory accesses for the multimedia subsystem. + + if unsure, say N here. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index c6edb31..195f7b9 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -7,7 +7,7 @@ obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o -obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o +obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm-smmu.o @@ -18,6 +18,7 @@ obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o +obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 9e00341..59741ea 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1107,13 +1107,13 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, break; } + devid = e->devid; DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", hid, uid, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); - devid = e->devid; flags = e->flags; ret = add_acpi_hid_device(hid, uid, &devid, false); @@ -1568,13 +1568,23 @@ static int __init amd_iommu_init_pci(void) break; } + /* + * Order is important here to make sure any unity map requirements are + * fulfilled. The unity mappings are created and written to the device + * table during the amd_iommu_init_api() call. + * + * After that we call init_device_table_dma() to make sure any + * uninitialized DTE will block DMA, and in the end we flush the caches + * of all IOMMUs to make sure the changes to the device table are + * active. + */ + ret = amd_iommu_init_api(); + init_device_table_dma(); for_each_iommu(iommu) iommu_flush_all_caches(iommu); - ret = amd_iommu_init_api(); - if (!ret) print_iommu_info(); diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 94b6821..ce80117 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1941,6 +1941,7 @@ static struct iommu_ops arm_smmu_ops = { .attach_dev = arm_smmu_attach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, + .map_sg = default_iommu_map_sg, .iova_to_phys = arm_smmu_iova_to_phys, .add_device = arm_smmu_add_device, .remove_device = arm_smmu_remove_device, @@ -2686,6 +2687,8 @@ static int __init arm_smmu_init(void) if (ret) return ret; + pci_request_acs(); + return bus_set_iommu(&pci_bus_type, &arm_smmu_ops); } diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 9345a3f..4f49fe2 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -987,8 +987,8 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain, * handler seeing a half-initialised domain state. */ irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; - ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, - "arm-smmu-context-fault", domain); + ret = devm_request_irq(smmu->dev, irq, arm_smmu_context_fault, + IRQF_SHARED, "arm-smmu-context-fault", domain); if (ret < 0) { dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", cfg->irptndx, irq); @@ -1028,7 +1028,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) if (cfg->irptndx != INVALID_IRPTNDX) { irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; - free_irq(irq, domain); + devm_free_irq(smmu->dev, irq, domain); } free_io_pgtable_ops(smmu_domain->pgtbl_ops); @@ -1986,15 +1986,15 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) } for (i = 0; i < smmu->num_global_irqs; ++i) { - err = request_irq(smmu->irqs[i], - arm_smmu_global_fault, - IRQF_SHARED, - "arm-smmu global fault", - smmu); + err = devm_request_irq(smmu->dev, smmu->irqs[i], + arm_smmu_global_fault, + IRQF_SHARED, + "arm-smmu global fault", + smmu); if (err) { dev_err(dev, "failed to request global IRQ %d (%u)\n", i, smmu->irqs[i]); - goto out_free_irqs; + goto out_put_masters; } } @@ -2006,10 +2006,6 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) arm_smmu_device_reset(smmu); return 0; -out_free_irqs: - while (i--) - free_irq(smmu->irqs[i], smmu); - out_put_masters: for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { struct arm_smmu_master *master @@ -2050,7 +2046,7 @@ static int arm_smmu_device_remove(struct platform_device *pdev) dev_err(dev, "removing device with active domains!\n"); for (i = 0; i < smmu->num_global_irqs; ++i) - free_irq(smmu->irqs[i], smmu); + devm_free_irq(smmu->dev, smmu->irqs[i], smmu); /* Turn the thing off */ writel(sCR0_CLIENTPD, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0); @@ -2096,8 +2092,10 @@ static int __init arm_smmu_init(void) #endif #ifdef CONFIG_PCI - if (!iommu_present(&pci_bus_type)) + if (!iommu_present(&pci_bus_type)) { + pci_request_acs(); bus_set_iommu(&pci_bus_type, &arm_smmu_ops); + } #endif return 0; diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 6a86b5d..1344e29 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -241,8 +241,20 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, if (!dmar_match_pci_path(info, scope->bus, path, level)) continue; - if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^ - (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) { + /* + * We expect devices with endpoint scope to have normal PCI + * headers, and devices with bridge scope to have bridge PCI + * headers. However PCI NTB devices may be listed in the + * DMAR table with bridge scope, even though they have a + * normal PCI header. NTB devices are identified by class + * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch + * for this special case. + */ + if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && + info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) || + (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE && + (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL && + info->dev->class >> 8 != PCI_CLASS_BRIDGE_OTHER))) { pr_warn("Device scope type does not match for %s\n", pci_name(info->dev)); return -EINVAL; @@ -1155,8 +1167,6 @@ static int qi_check_fault(struct intel_iommu *iommu, int index) (unsigned long long)qi->desc[index].high); memcpy(&qi->desc[index], &qi->desc[wait_index], sizeof(struct qi_desc)); - __iommu_flush_cache(iommu, &qi->desc[index], - sizeof(struct qi_desc)); writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG); return -EINVAL; } @@ -1231,9 +1241,6 @@ restart: hw[wait_index] = wait_desc; - __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc)); - __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc)); - qi->free_head = (qi->free_head + 2) % QI_LENGTH; qi->free_cnt -= 2; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 5ecc86c..33dcc29 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -54,6 +54,10 @@ typedef u32 sysmmu_pte_t; #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) +#ifdef CONFIG_BIG_ENDIAN +#warning "revisit driver if we can enable big-endian ptes" +#endif + /* * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces * v5.0 introduced support for 36bit physical address space by shifting @@ -322,14 +326,27 @@ static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) __sysmmu_tlb_invalidate(data); } +static void __sysmmu_enable_clocks(struct sysmmu_drvdata *data) +{ + BUG_ON(clk_prepare_enable(data->clk_master)); + BUG_ON(clk_prepare_enable(data->clk)); + BUG_ON(clk_prepare_enable(data->pclk)); + BUG_ON(clk_prepare_enable(data->aclk)); +} + +static void __sysmmu_disable_clocks(struct sysmmu_drvdata *data) +{ + clk_disable_unprepare(data->aclk); + clk_disable_unprepare(data->pclk); + clk_disable_unprepare(data->clk); + clk_disable_unprepare(data->clk_master); +} + static void __sysmmu_get_version(struct sysmmu_drvdata *data) { u32 ver; - clk_enable(data->clk_master); - clk_enable(data->clk); - clk_enable(data->pclk); - clk_enable(data->aclk); + __sysmmu_enable_clocks(data); ver = readl(data->sfrbase + REG_MMU_VERSION); @@ -342,10 +359,7 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) dev_dbg(data->sysmmu, "hardware version: %d.%d\n", MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); - clk_disable(data->aclk); - clk_disable(data->pclk); - clk_disable(data->clk); - clk_disable(data->clk_master); + __sysmmu_disable_clocks(data); } static void show_fault_information(struct sysmmu_drvdata *data, @@ -427,10 +441,7 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); writel(0, data->sfrbase + REG_MMU_CFG); - clk_disable(data->aclk); - clk_disable(data->pclk); - clk_disable(data->clk); - clk_disable(data->clk_master); + __sysmmu_disable_clocks(data); } static bool __sysmmu_disable(struct sysmmu_drvdata *data) @@ -475,10 +486,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { - clk_enable(data->clk_master); - clk_enable(data->clk); - clk_enable(data->pclk); - clk_enable(data->aclk); + __sysmmu_enable_clocks(data); writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); @@ -488,6 +496,12 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + /* + * SYSMMU driver keeps master's clock enabled only for the short + * time, while accessing the registers. For performing address + * translation during DMA transaction it relies on the client + * driver to enable it. + */ clk_disable(data->clk_master); } @@ -524,16 +538,15 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, { unsigned long flags; - clk_enable(data->clk_master); spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) { - if (data->version >= MAKE_MMU_VER(3, 3)) - __sysmmu_tlb_invalidate_entry(data, iova, 1); + if (is_sysmmu_active(data) && data->version >= MAKE_MMU_VER(3, 3)) { + clk_enable(data->clk_master); + __sysmmu_tlb_invalidate_entry(data, iova, 1); + clk_disable(data->clk_master); } spin_unlock_irqrestore(&data->lock, flags); - clk_disable(data->clk_master); } static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -572,6 +585,8 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, spin_unlock_irqrestore(&data->lock, flags); } +static struct iommu_ops exynos_iommu_ops; + static int __init exynos_sysmmu_probe(struct platform_device *pdev) { int irq, ret; @@ -602,37 +617,22 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk = devm_clk_get(dev, "sysmmu"); - if (!IS_ERR(data->clk)) { - ret = clk_prepare(data->clk); - if (ret) { - dev_err(dev, "Failed to prepare clk\n"); - return ret; - } - } else { + if (PTR_ERR(data->clk) == -ENOENT) data->clk = NULL; - } + else if (IS_ERR(data->clk)) + return PTR_ERR(data->clk); data->aclk = devm_clk_get(dev, "aclk"); - if (!IS_ERR(data->aclk)) { - ret = clk_prepare(data->aclk); - if (ret) { - dev_err(dev, "Failed to prepare aclk\n"); - return ret; - } - } else { + if (PTR_ERR(data->aclk) == -ENOENT) data->aclk = NULL; - } + else if (IS_ERR(data->aclk)) + return PTR_ERR(data->aclk); data->pclk = devm_clk_get(dev, "pclk"); - if (!IS_ERR(data->pclk)) { - ret = clk_prepare(data->pclk); - if (ret) { - dev_err(dev, "Failed to prepare pclk\n"); - return ret; - } - } else { + if (PTR_ERR(data->pclk) == -ENOENT) data->pclk = NULL; - } + else if (IS_ERR(data->pclk)) + return PTR_ERR(data->pclk); if (!data->clk && (!data->aclk || !data->pclk)) { dev_err(dev, "Failed to get device clock(s)!\n"); @@ -640,15 +640,10 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk_master = devm_clk_get(dev, "master"); - if (!IS_ERR(data->clk_master)) { - ret = clk_prepare(data->clk_master); - if (ret) { - dev_err(dev, "Failed to prepare master's clk\n"); - return ret; - } - } else { + if (PTR_ERR(data->clk_master) == -ENOENT) data->clk_master = NULL; - } + else if (IS_ERR(data->clk_master)) + return PTR_ERR(data->clk_master); data->sysmmu = dev; spin_lock_init(&data->lock); @@ -665,6 +660,8 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) pm_runtime_enable(dev); + of_iommu_set_ops(dev->of_node, &exynos_iommu_ops); + return 0; } @@ -709,6 +706,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { .name = "exynos-sysmmu", .of_match_table = sysmmu_of_match, .pm = &sysmmu_pm_ops, + .suppress_bind_attrs = true, } }; @@ -716,7 +714,7 @@ static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) { dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); - *ent = val; + *ent = cpu_to_le32(val); dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), DMA_TO_DEVICE); } @@ -1357,7 +1355,6 @@ static int __init exynos_iommu_of_setup(struct device_node *np) if (!dma_dev) dma_dev = &pdev->dev; - of_iommu_set_ops(np, &exynos_iommu_ops); return 0; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index a644d0c..9d2938e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1672,7 +1672,7 @@ static int iommu_init_domains(struct intel_iommu *iommu) return -ENOMEM; } - size = ((ndomains >> 8) + 1) * sizeof(struct dmar_domain **); + size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **); iommu->domains = kzalloc(size, GFP_KERNEL); if (iommu->domains) { @@ -1737,7 +1737,7 @@ static void disable_dmar_iommu(struct intel_iommu *iommu) static void free_dmar_iommu(struct intel_iommu *iommu) { if ((iommu->domains) && (iommu->domain_ids)) { - int elems = (cap_ndoms(iommu->cap) >> 8) + 1; + int elems = ALIGN(cap_ndoms(iommu->cap), 256) >> 8; int i; for (i = 0; i < elems; i++) @@ -2076,7 +2076,7 @@ out_unlock: spin_unlock(&iommu->lock); spin_unlock_irqrestore(&device_domain_lock, flags); - return 0; + return ret; } struct domain_context_mapping_data { @@ -3222,11 +3222,6 @@ static int __init init_dmars(void) } } - iommu_flush_write_buffer(iommu); - iommu_set_root_entry(iommu); - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); - if (!ecap_pass_through(iommu->ecap)) hw_pass_through = 0; #ifdef CONFIG_INTEL_IOMMU_SVM @@ -3235,6 +3230,18 @@ static int __init init_dmars(void) #endif } + /* + * Now that qi is enabled on all iommus, set the root entry and flush + * caches. This is required on some Intel X58 chipsets, otherwise the + * flush_context function will loop forever and the boot hangs. + */ + for_each_active_iommu(iommu, drhd) { + iommu_flush_write_buffer(iommu); + iommu_set_root_entry(iommu); + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); + } + if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; @@ -4595,13 +4602,13 @@ static void free_all_cpu_cached_iovas(unsigned int cpu) for (i = 0; i < g_num_of_iommus; i++) { struct intel_iommu *iommu = g_iommus[i]; struct dmar_domain *domain; - u16 did; + int did; if (!iommu) continue; - for (did = 0; did < 0xffff; did++) { - domain = get_iommu_domain(iommu, did); + for (did = 0; did < cap_ndoms(iommu->cap); did++) { + domain = get_iommu_domain(iommu, (u16)did); if (!domain) continue; diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index a1ed1b7..f5c90e1 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -576,7 +576,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, return 0; found_translation: - iova &= (ARM_LPAE_GRANULE(data) - 1); + iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1); return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; } diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index e8d2fb0..b06d935 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -34,8 +34,7 @@ #include <trace/events/iommu.h> static struct kset *iommu_group_kset; -static struct ida iommu_group_ida; -static struct mutex iommu_group_mutex; +static DEFINE_IDA(iommu_group_ida); struct iommu_callback_data { const struct iommu_ops *ops; @@ -144,9 +143,7 @@ static void iommu_group_release(struct kobject *kobj) if (group->iommu_data_release) group->iommu_data_release(group->iommu_data); - mutex_lock(&iommu_group_mutex); - ida_remove(&iommu_group_ida, group->id); - mutex_unlock(&iommu_group_mutex); + ida_simple_remove(&iommu_group_ida, group->id); if (group->default_domain) iommu_domain_free(group->default_domain); @@ -186,26 +183,17 @@ struct iommu_group *iommu_group_alloc(void) INIT_LIST_HEAD(&group->devices); BLOCKING_INIT_NOTIFIER_HEAD(&group->notifier); - mutex_lock(&iommu_group_mutex); - -again: - if (unlikely(0 == ida_pre_get(&iommu_group_ida, GFP_KERNEL))) { + ret = ida_simple_get(&iommu_group_ida, 0, 0, GFP_KERNEL); + if (ret < 0) { kfree(group); - mutex_unlock(&iommu_group_mutex); - return ERR_PTR(-ENOMEM); + return ERR_PTR(ret); } - - if (-EAGAIN == ida_get_new(&iommu_group_ida, &group->id)) - goto again; - - mutex_unlock(&iommu_group_mutex); + group->id = ret; ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype, NULL, "%d", group->id); if (ret) { - mutex_lock(&iommu_group_mutex); - ida_remove(&iommu_group_ida, group->id); - mutex_unlock(&iommu_group_mutex); + ida_simple_remove(&iommu_group_ida, group->id); kfree(group); return ERR_PTR(ret); } @@ -1486,9 +1474,6 @@ static int __init iommu_init(void) { iommu_group_kset = kset_create_and_add("iommu_groups", NULL, kernel_kobj); - ida_init(&iommu_group_ida); - mutex_init(&iommu_group_mutex); - BUG_ON(!iommu_group_kset); return 0; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index ba764a0..e23001b 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -420,8 +420,10 @@ retry: /* Try replenishing IOVAs by flushing rcache. */ flushed_rcache = true; + preempt_disable(); for_each_online_cpu(cpu) free_cpu_cached_iovas(cpu, iovad); + preempt_enable(); goto retry; } @@ -749,7 +751,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, bool can_insert = false; unsigned long flags; - cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches); spin_lock_irqsave(&cpu_rcache->lock, flags); if (!iova_magazine_full(cpu_rcache->loaded)) { @@ -779,6 +781,7 @@ static bool __iova_rcache_insert(struct iova_domain *iovad, iova_magazine_push(cpu_rcache->loaded, iova_pfn); spin_unlock_irqrestore(&cpu_rcache->lock, flags); + put_cpu_ptr(rcache->cpu_rcaches); if (mag_to_free) { iova_magazine_free_pfns(mag_to_free, iovad); @@ -812,7 +815,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, bool has_pfn = false; unsigned long flags; - cpu_rcache = this_cpu_ptr(rcache->cpu_rcaches); + cpu_rcache = get_cpu_ptr(rcache->cpu_rcaches); spin_lock_irqsave(&cpu_rcache->lock, flags); if (!iova_magazine_empty(cpu_rcache->loaded)) { @@ -834,6 +837,7 @@ static unsigned long __iova_rcache_get(struct iova_rcache *rcache, iova_pfn = iova_magazine_pop(cpu_rcache->loaded, limit_pfn); spin_unlock_irqrestore(&cpu_rcache->lock, flags); + put_cpu_ptr(rcache->cpu_rcaches); return iova_pfn; } diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index e321fa5..b09692b 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -27,32 +27,35 @@ #include <linux/slab.h> #include <linux/iommu.h> #include <linux/clk.h> +#include <linux/err.h> +#include <linux/of_iommu.h> #include <asm/cacheflush.h> #include <asm/sizes.h> #include "msm_iommu_hw-8xxx.h" #include "msm_iommu.h" +#include "io-pgtable.h" #define MRC(reg, processor, op1, crn, crm, op2) \ __asm__ __volatile__ ( \ " mrc " #processor "," #op1 ", %0," #crn "," #crm "," #op2 "\n" \ : "=r" (reg)) -#define RCP15_PRRR(reg) MRC(reg, p15, 0, c10, c2, 0) -#define RCP15_NMRR(reg) MRC(reg, p15, 0, c10, c2, 1) - /* bitmap of the page sizes currently supported */ #define MSM_IOMMU_PGSIZES (SZ_4K | SZ_64K | SZ_1M | SZ_16M) -static int msm_iommu_tex_class[4]; - DEFINE_SPINLOCK(msm_iommu_lock); +static LIST_HEAD(qcom_iommu_devices); +static struct iommu_ops msm_iommu_ops; struct msm_priv { - unsigned long *pgtable; struct list_head list_attached; struct iommu_domain domain; + struct io_pgtable_cfg cfg; + struct io_pgtable_ops *iop; + struct device *dev; + spinlock_t pgtlock; /* pagetable lock */ }; static struct msm_priv *to_msm_priv(struct iommu_domain *dom) @@ -60,67 +63,183 @@ static struct msm_priv *to_msm_priv(struct iommu_domain *dom) return container_of(dom, struct msm_priv, domain); } -static int __enable_clocks(struct msm_iommu_drvdata *drvdata) +static int __enable_clocks(struct msm_iommu_dev *iommu) { int ret; - ret = clk_enable(drvdata->pclk); + ret = clk_enable(iommu->pclk); if (ret) goto fail; - if (drvdata->clk) { - ret = clk_enable(drvdata->clk); + if (iommu->clk) { + ret = clk_enable(iommu->clk); if (ret) - clk_disable(drvdata->pclk); + clk_disable(iommu->pclk); } fail: return ret; } -static void __disable_clocks(struct msm_iommu_drvdata *drvdata) +static void __disable_clocks(struct msm_iommu_dev *iommu) { - clk_disable(drvdata->clk); - clk_disable(drvdata->pclk); + if (iommu->clk) + clk_disable(iommu->clk); + clk_disable(iommu->pclk); } -static int __flush_iotlb(struct iommu_domain *domain) +static void msm_iommu_reset(void __iomem *base, int ncb) { - struct msm_priv *priv = to_msm_priv(domain); - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - int ret = 0; -#ifndef CONFIG_IOMMU_PGTABLES_L2 - unsigned long *fl_table = priv->pgtable; - int i; - - if (!list_empty(&priv->list_attached)) { - dmac_flush_range(fl_table, fl_table + SZ_16K); - - for (i = 0; i < NUM_FL_PTE; i++) - if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) { - void *sl_table = __va(fl_table[i] & - FL_BASE_MASK); - dmac_flush_range(sl_table, sl_table + SZ_4K); - } + int ctx; + + SET_RPUE(base, 0); + SET_RPUEIE(base, 0); + SET_ESRRESTORE(base, 0); + SET_TBE(base, 0); + SET_CR(base, 0); + SET_SPDMBE(base, 0); + SET_TESTBUSCR(base, 0); + SET_TLBRSW(base, 0); + SET_GLOBAL_TLBIALL(base, 0); + SET_RPU_ACR(base, 0); + SET_TLBLKCRWE(base, 1); + + for (ctx = 0; ctx < ncb; ctx++) { + SET_BPRCOSH(base, ctx, 0); + SET_BPRCISH(base, ctx, 0); + SET_BPRCNSH(base, ctx, 0); + SET_BPSHCFG(base, ctx, 0); + SET_BPMTCFG(base, ctx, 0); + SET_ACTLR(base, ctx, 0); + SET_SCTLR(base, ctx, 0); + SET_FSRRESTORE(base, ctx, 0); + SET_TTBR0(base, ctx, 0); + SET_TTBR1(base, ctx, 0); + SET_TTBCR(base, ctx, 0); + SET_BFBCR(base, ctx, 0); + SET_PAR(base, ctx, 0); + SET_FAR(base, ctx, 0); + SET_CTX_TLBIALL(base, ctx, 0); + SET_TLBFLPTER(base, ctx, 0); + SET_TLBSLPTER(base, ctx, 0); + SET_TLBLKCR(base, ctx, 0); + SET_CONTEXTIDR(base, ctx, 0); } -#endif +} - list_for_each_entry(ctx_drvdata, &priv->list_attached, attached_elm) { +static void __flush_iotlb(void *cookie) +{ + struct msm_priv *priv = cookie; + struct msm_iommu_dev *iommu = NULL; + struct msm_iommu_ctx_dev *master; + int ret = 0; + + list_for_each_entry(iommu, &priv->list_attached, dom_node) { + ret = __enable_clocks(iommu); + if (ret) + goto fail; - BUG_ON(!ctx_drvdata->pdev || !ctx_drvdata->pdev->dev.parent); + list_for_each_entry(master, &iommu->ctx_list, list) + SET_CTX_TLBIALL(iommu->base, master->num, 0); - iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); - BUG_ON(!iommu_drvdata); + __disable_clocks(iommu); + } +fail: + return; +} - ret = __enable_clocks(iommu_drvdata); +static void __flush_iotlb_range(unsigned long iova, size_t size, + size_t granule, bool leaf, void *cookie) +{ + struct msm_priv *priv = cookie; + struct msm_iommu_dev *iommu = NULL; + struct msm_iommu_ctx_dev *master; + int ret = 0; + int temp_size; + + list_for_each_entry(iommu, &priv->list_attached, dom_node) { + ret = __enable_clocks(iommu); if (ret) goto fail; - SET_CTX_TLBIALL(iommu_drvdata->base, ctx_drvdata->num, 0); - __disable_clocks(iommu_drvdata); + list_for_each_entry(master, &iommu->ctx_list, list) { + temp_size = size; + do { + iova &= TLBIVA_VA; + iova |= GET_CONTEXTIDR_ASID(iommu->base, + master->num); + SET_TLBIVA(iommu->base, master->num, iova); + iova += granule; + } while (temp_size -= granule); + } + + __disable_clocks(iommu); } + fail: - return ret; + return; +} + +static void __flush_iotlb_sync(void *cookie) +{ + /* + * Nothing is needed here, the barrier to guarantee + * completion of the tlb sync operation is implicitly + * taken care when the iommu client does a writel before + * kick starting the other master. + */ +} + +static const struct iommu_gather_ops msm_iommu_gather_ops = { + .tlb_flush_all = __flush_iotlb, + .tlb_add_flush = __flush_iotlb_range, + .tlb_sync = __flush_iotlb_sync, +}; + +static int msm_iommu_alloc_ctx(unsigned long *map, int start, int end) +{ + int idx; + + do { + idx = find_next_zero_bit(map, end, start); + if (idx == end) + return -ENOSPC; + } while (test_and_set_bit(idx, map)); + + return idx; +} + +static void msm_iommu_free_ctx(unsigned long *map, int idx) +{ + clear_bit(idx, map); +} + +static void config_mids(struct msm_iommu_dev *iommu, + struct msm_iommu_ctx_dev *master) +{ + int mid, ctx, i; + + for (i = 0; i < master->num_mids; i++) { + mid = master->mids[i]; + ctx = master->num; + + SET_M2VCBR_N(iommu->base, mid, 0); + SET_CBACR_N(iommu->base, ctx, 0); + + /* Set VMID = 0 */ + SET_VMID(iommu->base, mid, 0); + + /* Set the context number for that MID to this context */ + SET_CBNDX(iommu->base, mid, ctx); + + /* Set MID associated with this context bank to 0*/ + SET_CBVMID(iommu->base, ctx, 0); + + /* Set the ASID for TLB tagging for this context */ + SET_CONTEXTIDR_ASID(iommu->base, ctx, ctx); + + /* Set security bit override to be Non-secure */ + SET_NSCFG(iommu->base, mid, 3); + } } static void __reset_context(void __iomem *base, int ctx) @@ -143,15 +262,17 @@ static void __reset_context(void __iomem *base, int ctx) SET_TLBFLPTER(base, ctx, 0); SET_TLBSLPTER(base, ctx, 0); SET_TLBLKCR(base, ctx, 0); - SET_PRRR(base, ctx, 0); - SET_NMRR(base, ctx, 0); } -static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) +static void __program_context(void __iomem *base, int ctx, + struct msm_priv *priv) { - unsigned int prrr, nmrr; __reset_context(base, ctx); + /* Turn on TEX Remap */ + SET_TRE(base, ctx, 1); + SET_AFE(base, ctx, 1); + /* Set up HTW mode */ /* TLB miss configuration: perform HTW on miss */ SET_TLBMCFG(base, ctx, 0x3); @@ -159,8 +280,13 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) /* V2P configuration: HTW for access */ SET_V2PCFG(base, ctx, 0x3); - SET_TTBCR(base, ctx, 0); - SET_TTBR0_PA(base, ctx, (pgtable >> 14)); + SET_TTBCR(base, ctx, priv->cfg.arm_v7s_cfg.tcr); + SET_TTBR0(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[0]); + SET_TTBR1(base, ctx, priv->cfg.arm_v7s_cfg.ttbr[1]); + + /* Set prrr and nmrr */ + SET_PRRR(base, ctx, priv->cfg.arm_v7s_cfg.prrr); + SET_NMRR(base, ctx, priv->cfg.arm_v7s_cfg.nmrr); /* Invalidate the TLB for this context */ SET_CTX_TLBIALL(base, ctx, 0); @@ -179,38 +305,9 @@ static void __program_context(void __iomem *base, int ctx, phys_addr_t pgtable) SET_RCOSH(base, ctx, 1); SET_RCNSH(base, ctx, 1); - /* Turn on TEX Remap */ - SET_TRE(base, ctx, 1); - - /* Set TEX remap attributes */ - RCP15_PRRR(prrr); - RCP15_NMRR(nmrr); - SET_PRRR(base, ctx, prrr); - SET_NMRR(base, ctx, nmrr); - /* Turn on BFB prefetch */ SET_BFBDFE(base, ctx, 1); -#ifdef CONFIG_IOMMU_PGTABLES_L2 - /* Configure page tables as inner-cacheable and shareable to reduce - * the TLB miss penalty. - */ - SET_TTBR0_SH(base, ctx, 1); - SET_TTBR1_SH(base, ctx, 1); - - SET_TTBR0_NOS(base, ctx, 1); - SET_TTBR1_NOS(base, ctx, 1); - - SET_TTBR0_IRGNH(base, ctx, 0); /* WB, WA */ - SET_TTBR0_IRGNL(base, ctx, 1); - - SET_TTBR1_IRGNH(base, ctx, 0); /* WB, WA */ - SET_TTBR1_IRGNL(base, ctx, 1); - - SET_TTBR0_ORGN(base, ctx, 1); /* WB, WA */ - SET_TTBR1_ORGN(base, ctx, 1); /* WB, WA */ -#endif - /* Enable the MMU */ SET_M(base, ctx, 1); } @@ -227,13 +324,6 @@ static struct iommu_domain *msm_iommu_domain_alloc(unsigned type) goto fail_nomem; INIT_LIST_HEAD(&priv->list_attached); - priv->pgtable = (unsigned long *)__get_free_pages(GFP_KERNEL, - get_order(SZ_16K)); - - if (!priv->pgtable) - goto fail_nomem; - - memset(priv->pgtable, 0, SZ_16K); priv->domain.geometry.aperture_start = 0; priv->domain.geometry.aperture_end = (1ULL << 32) - 1; @@ -250,304 +340,137 @@ static void msm_iommu_domain_free(struct iommu_domain *domain) { struct msm_priv *priv; unsigned long flags; - unsigned long *fl_table; - int i; spin_lock_irqsave(&msm_iommu_lock, flags); priv = to_msm_priv(domain); + kfree(priv); + spin_unlock_irqrestore(&msm_iommu_lock, flags); +} - fl_table = priv->pgtable; - - for (i = 0; i < NUM_FL_PTE; i++) - if ((fl_table[i] & 0x03) == FL_TYPE_TABLE) - free_page((unsigned long) __va(((fl_table[i]) & - FL_BASE_MASK))); +static int msm_iommu_domain_config(struct msm_priv *priv) +{ + spin_lock_init(&priv->pgtlock); + + priv->cfg = (struct io_pgtable_cfg) { + .quirks = IO_PGTABLE_QUIRK_TLBI_ON_MAP, + .pgsize_bitmap = msm_iommu_ops.pgsize_bitmap, + .ias = 32, + .oas = 32, + .tlb = &msm_iommu_gather_ops, + .iommu_dev = priv->dev, + }; + + priv->iop = alloc_io_pgtable_ops(ARM_V7S, &priv->cfg, priv); + if (!priv->iop) { + dev_err(priv->dev, "Failed to allocate pgtable\n"); + return -EINVAL; + } - free_pages((unsigned long)priv->pgtable, get_order(SZ_16K)); - priv->pgtable = NULL; + msm_iommu_ops.pgsize_bitmap = priv->cfg.pgsize_bitmap; - kfree(priv); - spin_unlock_irqrestore(&msm_iommu_lock, flags); + return 0; } static int msm_iommu_attach_dev(struct iommu_domain *domain, struct device *dev) { - struct msm_priv *priv; - struct msm_iommu_ctx_dev *ctx_dev; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - struct msm_iommu_ctx_drvdata *tmp_drvdata; int ret = 0; unsigned long flags; + struct msm_iommu_dev *iommu; + struct msm_priv *priv = to_msm_priv(domain); + struct msm_iommu_ctx_dev *master; - spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = to_msm_priv(domain); - - if (!dev) { - ret = -EINVAL; - goto fail; - } - - iommu_drvdata = dev_get_drvdata(dev->parent); - ctx_drvdata = dev_get_drvdata(dev); - ctx_dev = dev->platform_data; - - if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) { - ret = -EINVAL; - goto fail; - } - - if (!list_empty(&ctx_drvdata->attached_elm)) { - ret = -EBUSY; - goto fail; - } + priv->dev = dev; + msm_iommu_domain_config(priv); - list_for_each_entry(tmp_drvdata, &priv->list_attached, attached_elm) - if (tmp_drvdata == ctx_drvdata) { - ret = -EBUSY; - goto fail; + spin_lock_irqsave(&msm_iommu_lock, flags); + list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) { + master = list_first_entry(&iommu->ctx_list, + struct msm_iommu_ctx_dev, + list); + if (master->of_node == dev->of_node) { + ret = __enable_clocks(iommu); + if (ret) + goto fail; + + list_for_each_entry(master, &iommu->ctx_list, list) { + if (master->num) { + dev_err(dev, "domain already attached"); + ret = -EEXIST; + goto fail; + } + master->num = + msm_iommu_alloc_ctx(iommu->context_map, + 0, iommu->ncb); + if (IS_ERR_VALUE(master->num)) { + ret = -ENODEV; + goto fail; + } + config_mids(iommu, master); + __program_context(iommu->base, master->num, + priv); + } + __disable_clocks(iommu); + list_add(&iommu->dom_node, &priv->list_attached); } - - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; - - __program_context(iommu_drvdata->base, ctx_dev->num, - __pa(priv->pgtable)); - - __disable_clocks(iommu_drvdata); - list_add(&(ctx_drvdata->attached_elm), &priv->list_attached); - ret = __flush_iotlb(domain); + } fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); + return ret; } static void msm_iommu_detach_dev(struct iommu_domain *domain, struct device *dev) { - struct msm_priv *priv; - struct msm_iommu_ctx_dev *ctx_dev; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; + struct msm_priv *priv = to_msm_priv(domain); unsigned long flags; + struct msm_iommu_dev *iommu; + struct msm_iommu_ctx_dev *master; int ret; - spin_lock_irqsave(&msm_iommu_lock, flags); - priv = to_msm_priv(domain); - - if (!dev) - goto fail; - - iommu_drvdata = dev_get_drvdata(dev->parent); - ctx_drvdata = dev_get_drvdata(dev); - ctx_dev = dev->platform_data; - - if (!iommu_drvdata || !ctx_drvdata || !ctx_dev) - goto fail; - - ret = __flush_iotlb(domain); - if (ret) - goto fail; - - ret = __enable_clocks(iommu_drvdata); - if (ret) - goto fail; + free_io_pgtable_ops(priv->iop); - __reset_context(iommu_drvdata->base, ctx_dev->num); - __disable_clocks(iommu_drvdata); - list_del_init(&ctx_drvdata->attached_elm); + spin_lock_irqsave(&msm_iommu_lock, flags); + list_for_each_entry(iommu, &priv->list_attached, dom_node) { + ret = __enable_clocks(iommu); + if (ret) + goto fail; + list_for_each_entry(master, &iommu->ctx_list, list) { + msm_iommu_free_ctx(iommu->context_map, master->num); + __reset_context(iommu->base, master->num); + } + __disable_clocks(iommu); + } fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); } -static int msm_iommu_map(struct iommu_domain *domain, unsigned long va, +static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t pa, size_t len, int prot) { - struct msm_priv *priv; + struct msm_priv *priv = to_msm_priv(domain); unsigned long flags; - unsigned long *fl_table; - unsigned long *fl_pte; - unsigned long fl_offset; - unsigned long *sl_table; - unsigned long *sl_pte; - unsigned long sl_offset; - unsigned int pgprot; - int ret = 0, tex, sh; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - sh = (prot & MSM_IOMMU_ATTR_SH) ? 1 : 0; - tex = msm_iommu_tex_class[prot & MSM_IOMMU_CP_MASK]; - - if (tex < 0 || tex > NUM_TEX_CLASS - 1) { - ret = -EINVAL; - goto fail; - } - - priv = to_msm_priv(domain); - - fl_table = priv->pgtable; - - if (len != SZ_16M && len != SZ_1M && - len != SZ_64K && len != SZ_4K) { - pr_debug("Bad size: %d\n", len); - ret = -EINVAL; - goto fail; - } - - if (!fl_table) { - pr_debug("Null page table\n"); - ret = -EINVAL; - goto fail; - } - - if (len == SZ_16M || len == SZ_1M) { - pgprot = sh ? FL_SHARED : 0; - pgprot |= tex & 0x01 ? FL_BUFFERABLE : 0; - pgprot |= tex & 0x02 ? FL_CACHEABLE : 0; - pgprot |= tex & 0x04 ? FL_TEX0 : 0; - } else { - pgprot = sh ? SL_SHARED : 0; - pgprot |= tex & 0x01 ? SL_BUFFERABLE : 0; - pgprot |= tex & 0x02 ? SL_CACHEABLE : 0; - pgprot |= tex & 0x04 ? SL_TEX0 : 0; - } - - fl_offset = FL_OFFSET(va); /* Upper 12 bits */ - fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ - - if (len == SZ_16M) { - int i = 0; - for (i = 0; i < 16; i++) - *(fl_pte+i) = (pa & 0xFF000000) | FL_SUPERSECTION | - FL_AP_READ | FL_AP_WRITE | FL_TYPE_SECT | - FL_SHARED | FL_NG | pgprot; - } - - if (len == SZ_1M) - *fl_pte = (pa & 0xFFF00000) | FL_AP_READ | FL_AP_WRITE | FL_NG | - FL_TYPE_SECT | FL_SHARED | pgprot; - - /* Need a 2nd level table */ - if ((len == SZ_4K || len == SZ_64K) && (*fl_pte) == 0) { - unsigned long *sl; - sl = (unsigned long *) __get_free_pages(GFP_ATOMIC, - get_order(SZ_4K)); - - if (!sl) { - pr_debug("Could not allocate second level table\n"); - ret = -ENOMEM; - goto fail; - } - - memset(sl, 0, SZ_4K); - *fl_pte = ((((int)__pa(sl)) & FL_BASE_MASK) | FL_TYPE_TABLE); - } - - sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); - sl_offset = SL_OFFSET(va); - sl_pte = sl_table + sl_offset; - - - if (len == SZ_4K) - *sl_pte = (pa & SL_BASE_MASK_SMALL) | SL_AP0 | SL_AP1 | SL_NG | - SL_SHARED | SL_TYPE_SMALL | pgprot; - - if (len == SZ_64K) { - int i; + int ret; - for (i = 0; i < 16; i++) - *(sl_pte+i) = (pa & SL_BASE_MASK_LARGE) | SL_AP0 | - SL_NG | SL_AP1 | SL_SHARED | SL_TYPE_LARGE | pgprot; - } + spin_lock_irqsave(&priv->pgtlock, flags); + ret = priv->iop->map(priv->iop, iova, pa, len, prot); + spin_unlock_irqrestore(&priv->pgtlock, flags); - ret = __flush_iotlb(domain); -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); return ret; } -static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long va, - size_t len) +static size_t msm_iommu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t len) { - struct msm_priv *priv; + struct msm_priv *priv = to_msm_priv(domain); unsigned long flags; - unsigned long *fl_table; - unsigned long *fl_pte; - unsigned long fl_offset; - unsigned long *sl_table; - unsigned long *sl_pte; - unsigned long sl_offset; - int i, ret = 0; - - spin_lock_irqsave(&msm_iommu_lock, flags); - - priv = to_msm_priv(domain); - fl_table = priv->pgtable; + spin_lock_irqsave(&priv->pgtlock, flags); + len = priv->iop->unmap(priv->iop, iova, len); + spin_unlock_irqrestore(&priv->pgtlock, flags); - if (len != SZ_16M && len != SZ_1M && - len != SZ_64K && len != SZ_4K) { - pr_debug("Bad length: %d\n", len); - goto fail; - } - - if (!fl_table) { - pr_debug("Null page table\n"); - goto fail; - } - - fl_offset = FL_OFFSET(va); /* Upper 12 bits */ - fl_pte = fl_table + fl_offset; /* int pointers, 4 bytes */ - - if (*fl_pte == 0) { - pr_debug("First level PTE is 0\n"); - goto fail; - } - - /* Unmap supersection */ - if (len == SZ_16M) - for (i = 0; i < 16; i++) - *(fl_pte+i) = 0; - - if (len == SZ_1M) - *fl_pte = 0; - - sl_table = (unsigned long *) __va(((*fl_pte) & FL_BASE_MASK)); - sl_offset = SL_OFFSET(va); - sl_pte = sl_table + sl_offset; - - if (len == SZ_64K) { - for (i = 0; i < 16; i++) - *(sl_pte+i) = 0; - } - - if (len == SZ_4K) - *sl_pte = 0; - - if (len == SZ_4K || len == SZ_64K) { - int used = 0; - - for (i = 0; i < NUM_SL_PTE; i++) - if (sl_table[i]) - used = 1; - if (!used) { - free_page((unsigned long)sl_table); - *fl_pte = 0; - } - } - - ret = __flush_iotlb(domain); - -fail: - spin_unlock_irqrestore(&msm_iommu_lock, flags); - - /* the IOMMU API requires us to return how many bytes were unmapped */ - len = ret ? 0 : len; return len; } @@ -555,47 +478,46 @@ static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t va) { struct msm_priv *priv; - struct msm_iommu_drvdata *iommu_drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; + struct msm_iommu_dev *iommu; + struct msm_iommu_ctx_dev *master; unsigned int par; unsigned long flags; - void __iomem *base; phys_addr_t ret = 0; - int ctx; spin_lock_irqsave(&msm_iommu_lock, flags); priv = to_msm_priv(domain); - if (list_empty(&priv->list_attached)) - goto fail; + iommu = list_first_entry(&priv->list_attached, + struct msm_iommu_dev, dom_node); - ctx_drvdata = list_entry(priv->list_attached.next, - struct msm_iommu_ctx_drvdata, attached_elm); - iommu_drvdata = dev_get_drvdata(ctx_drvdata->pdev->dev.parent); + if (list_empty(&iommu->ctx_list)) + goto fail; - base = iommu_drvdata->base; - ctx = ctx_drvdata->num; + master = list_first_entry(&iommu->ctx_list, + struct msm_iommu_ctx_dev, list); + if (!master) + goto fail; - ret = __enable_clocks(iommu_drvdata); + ret = __enable_clocks(iommu); if (ret) goto fail; /* Invalidate context TLB */ - SET_CTX_TLBIALL(base, ctx, 0); - SET_V2PPR(base, ctx, va & V2Pxx_VA); + SET_CTX_TLBIALL(iommu->base, master->num, 0); + SET_V2PPR(iommu->base, master->num, va & V2Pxx_VA); - par = GET_PAR(base, ctx); + par = GET_PAR(iommu->base, master->num); /* We are dealing with a supersection */ - if (GET_NOFAULT_SS(base, ctx)) + if (GET_NOFAULT_SS(iommu->base, master->num)) ret = (par & 0xFF000000) | (va & 0x00FFFFFF); else /* Upper 20 bits from PAR, lower 12 from VA */ ret = (par & 0xFFFFF000) | (va & 0x00000FFF); - if (GET_FAULT(base, ctx)) + if (GET_FAULT(iommu->base, master->num)) ret = 0; - __disable_clocks(iommu_drvdata); + __disable_clocks(iommu); fail: spin_unlock_irqrestore(&msm_iommu_lock, flags); return ret; @@ -629,49 +551,92 @@ static void print_ctx_regs(void __iomem *base, int ctx) GET_TTBR0(base, ctx), GET_TTBR1(base, ctx)); pr_err("SCTLR = %08x ACTLR = %08x\n", GET_SCTLR(base, ctx), GET_ACTLR(base, ctx)); - pr_err("PRRR = %08x NMRR = %08x\n", - GET_PRRR(base, ctx), GET_NMRR(base, ctx)); +} + +static void insert_iommu_master(struct device *dev, + struct msm_iommu_dev **iommu, + struct of_phandle_args *spec) +{ + struct msm_iommu_ctx_dev *master = dev->archdata.iommu; + int sid; + + if (list_empty(&(*iommu)->ctx_list)) { + master = kzalloc(sizeof(*master), GFP_ATOMIC); + master->of_node = dev->of_node; + list_add(&master->list, &(*iommu)->ctx_list); + dev->archdata.iommu = master; + } + + for (sid = 0; sid < master->num_mids; sid++) + if (master->mids[sid] == spec->args[0]) { + dev_warn(dev, "Stream ID 0x%hx repeated; ignoring\n", + sid); + return; + } + + master->mids[master->num_mids++] = spec->args[0]; +} + +static int qcom_iommu_of_xlate(struct device *dev, + struct of_phandle_args *spec) +{ + struct msm_iommu_dev *iommu; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&msm_iommu_lock, flags); + list_for_each_entry(iommu, &qcom_iommu_devices, dev_node) + if (iommu->dev->of_node == spec->np) + break; + + if (!iommu || iommu->dev->of_node != spec->np) { + ret = -ENODEV; + goto fail; + } + + insert_iommu_master(dev, &iommu, spec); +fail: + spin_unlock_irqrestore(&msm_iommu_lock, flags); + + return ret; } irqreturn_t msm_iommu_fault_handler(int irq, void *dev_id) { - struct msm_iommu_drvdata *drvdata = dev_id; - void __iomem *base; + struct msm_iommu_dev *iommu = dev_id; unsigned int fsr; int i, ret; spin_lock(&msm_iommu_lock); - if (!drvdata) { + if (!iommu) { pr_err("Invalid device ID in context interrupt handler\n"); goto fail; } - base = drvdata->base; - pr_err("Unexpected IOMMU page fault!\n"); - pr_err("base = %08x\n", (unsigned int) base); + pr_err("base = %08x\n", (unsigned int)iommu->base); - ret = __enable_clocks(drvdata); + ret = __enable_clocks(iommu); if (ret) goto fail; - for (i = 0; i < drvdata->ncb; i++) { - fsr = GET_FSR(base, i); + for (i = 0; i < iommu->ncb; i++) { + fsr = GET_FSR(iommu->base, i); if (fsr) { pr_err("Fault occurred in context %d.\n", i); pr_err("Interesting registers:\n"); - print_ctx_regs(base, i); - SET_FSR(base, i, 0x4000000F); + print_ctx_regs(iommu->base, i); + SET_FSR(iommu->base, i, 0x4000000F); } } - __disable_clocks(drvdata); + __disable_clocks(iommu); fail: spin_unlock(&msm_iommu_lock); return 0; } -static const struct iommu_ops msm_iommu_ops = { +static struct iommu_ops msm_iommu_ops = { .capable = msm_iommu_capable, .domain_alloc = msm_iommu_domain_alloc, .domain_free = msm_iommu_domain_free, @@ -682,54 +647,163 @@ static const struct iommu_ops msm_iommu_ops = { .map_sg = default_iommu_map_sg, .iova_to_phys = msm_iommu_iova_to_phys, .pgsize_bitmap = MSM_IOMMU_PGSIZES, + .of_xlate = qcom_iommu_of_xlate, }; -static int __init get_tex_class(int icp, int ocp, int mt, int nos) +static int msm_iommu_probe(struct platform_device *pdev) { - int i = 0; - unsigned int prrr = 0; - unsigned int nmrr = 0; - int c_icp, c_ocp, c_mt, c_nos; - - RCP15_PRRR(prrr); - RCP15_NMRR(nmrr); - - for (i = 0; i < NUM_TEX_CLASS; i++) { - c_nos = PRRR_NOS(prrr, i); - c_mt = PRRR_MT(prrr, i); - c_icp = NMRR_ICP(nmrr, i); - c_ocp = NMRR_OCP(nmrr, i); - - if (icp == c_icp && ocp == c_ocp && c_mt == mt && c_nos == nos) - return i; + struct resource *r; + struct msm_iommu_dev *iommu; + int ret, par, val; + + iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL); + if (!iommu) + return -ENODEV; + + iommu->dev = &pdev->dev; + INIT_LIST_HEAD(&iommu->ctx_list); + + iommu->pclk = devm_clk_get(iommu->dev, "smmu_pclk"); + if (IS_ERR(iommu->pclk)) { + dev_err(iommu->dev, "could not get smmu_pclk\n"); + return PTR_ERR(iommu->pclk); + } + + ret = clk_prepare(iommu->pclk); + if (ret) { + dev_err(iommu->dev, "could not prepare smmu_pclk\n"); + return ret; + } + + iommu->clk = devm_clk_get(iommu->dev, "iommu_clk"); + if (IS_ERR(iommu->clk)) { + dev_err(iommu->dev, "could not get iommu_clk\n"); + clk_unprepare(iommu->pclk); + return PTR_ERR(iommu->clk); + } + + ret = clk_prepare(iommu->clk); + if (ret) { + dev_err(iommu->dev, "could not prepare iommu_clk\n"); + clk_unprepare(iommu->pclk); + return ret; + } + + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + iommu->base = devm_ioremap_resource(iommu->dev, r); + if (IS_ERR(iommu->base)) { + dev_err(iommu->dev, "could not get iommu base\n"); + ret = PTR_ERR(iommu->base); + goto fail; } - return -ENODEV; + iommu->irq = platform_get_irq(pdev, 0); + if (iommu->irq < 0) { + dev_err(iommu->dev, "could not get iommu irq\n"); + ret = -ENODEV; + goto fail; + } + + ret = of_property_read_u32(iommu->dev->of_node, "qcom,ncb", &val); + if (ret) { + dev_err(iommu->dev, "could not get ncb\n"); + goto fail; + } + iommu->ncb = val; + + msm_iommu_reset(iommu->base, iommu->ncb); + SET_M(iommu->base, 0, 1); + SET_PAR(iommu->base, 0, 0); + SET_V2PCFG(iommu->base, 0, 1); + SET_V2PPR(iommu->base, 0, 0); + par = GET_PAR(iommu->base, 0); + SET_V2PCFG(iommu->base, 0, 0); + SET_M(iommu->base, 0, 0); + + if (!par) { + pr_err("Invalid PAR value detected\n"); + ret = -ENODEV; + goto fail; + } + + ret = devm_request_threaded_irq(iommu->dev, iommu->irq, NULL, + msm_iommu_fault_handler, + IRQF_ONESHOT | IRQF_SHARED, + "msm_iommu_secure_irpt_handler", + iommu); + if (ret) { + pr_err("Request IRQ %d failed with ret=%d\n", iommu->irq, ret); + goto fail; + } + + list_add(&iommu->dev_node, &qcom_iommu_devices); + of_iommu_set_ops(pdev->dev.of_node, &msm_iommu_ops); + + pr_info("device mapped at %p, irq %d with %d ctx banks\n", + iommu->base, iommu->irq, iommu->ncb); + + return ret; +fail: + clk_unprepare(iommu->clk); + clk_unprepare(iommu->pclk); + return ret; +} + +static const struct of_device_id msm_iommu_dt_match[] = { + { .compatible = "qcom,apq8064-iommu" }, + {} +}; + +static int msm_iommu_remove(struct platform_device *pdev) +{ + struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); + + clk_unprepare(iommu->clk); + clk_unprepare(iommu->pclk); + return 0; } -static void __init setup_iommu_tex_classes(void) +static struct platform_driver msm_iommu_driver = { + .driver = { + .name = "msm_iommu", + .of_match_table = msm_iommu_dt_match, + }, + .probe = msm_iommu_probe, + .remove = msm_iommu_remove, +}; + +static int __init msm_iommu_driver_init(void) { - msm_iommu_tex_class[MSM_IOMMU_ATTR_NONCACHED] = - get_tex_class(CP_NONCACHED, CP_NONCACHED, MT_NORMAL, 1); + int ret; - msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_WA] = - get_tex_class(CP_WB_WA, CP_WB_WA, MT_NORMAL, 1); + ret = platform_driver_register(&msm_iommu_driver); + if (ret != 0) + pr_err("Failed to register IOMMU driver\n"); - msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WB_NWA] = - get_tex_class(CP_WB_NWA, CP_WB_NWA, MT_NORMAL, 1); + return ret; +} - msm_iommu_tex_class[MSM_IOMMU_ATTR_CACHED_WT] = - get_tex_class(CP_WT, CP_WT, MT_NORMAL, 1); +static void __exit msm_iommu_driver_exit(void) +{ + platform_driver_unregister(&msm_iommu_driver); } +subsys_initcall(msm_iommu_driver_init); +module_exit(msm_iommu_driver_exit); + static int __init msm_iommu_init(void) { - setup_iommu_tex_classes(); bus_set_iommu(&platform_bus_type, &msm_iommu_ops); return 0; } -subsys_initcall(msm_iommu_init); +static int __init msm_iommu_of_setup(struct device_node *np) +{ + msm_iommu_init(); + return 0; +} + +IOMMU_OF_DECLARE(msm_iommu_of, "qcom,apq8064-iommu", msm_iommu_of_setup); MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); diff --git a/drivers/iommu/msm_iommu.h b/drivers/iommu/msm_iommu.h index 5c7c955..4ca25d5 100644 --- a/drivers/iommu/msm_iommu.h +++ b/drivers/iommu/msm_iommu.h @@ -42,74 +42,53 @@ */ #define MAX_NUM_MIDS 32 +/* Maximum number of context banks that can be present in IOMMU */ +#define IOMMU_MAX_CBS 128 + /** * struct msm_iommu_dev - a single IOMMU hardware instance - * name Human-readable name given to this IOMMU HW instance * ncb Number of context banks present on this IOMMU HW instance + * dev: IOMMU device + * irq: Interrupt number + * clk: The bus clock for this IOMMU hardware instance + * pclk: The clock for the IOMMU bus interconnect + * dev_node: list head in qcom_iommu_device_list + * dom_node: list head for domain + * ctx_list: list of 'struct msm_iommu_ctx_dev' + * context_map: Bitmap to track allocated context banks */ struct msm_iommu_dev { - const char *name; + void __iomem *base; int ncb; + struct device *dev; + int irq; + struct clk *clk; + struct clk *pclk; + struct list_head dev_node; + struct list_head dom_node; + struct list_head ctx_list; + DECLARE_BITMAP(context_map, IOMMU_MAX_CBS); }; /** * struct msm_iommu_ctx_dev - an IOMMU context bank instance - * name Human-readable name given to this context bank + * of_node node ptr of client device * num Index of this context bank within the hardware * mids List of Machine IDs that are to be mapped into this context * bank, terminated by -1. The MID is a set of signals on the * AXI bus that identifies the function associated with a specific * memory request. (See ARM spec). + * num_mids Total number of mids + * node list head in ctx_list */ struct msm_iommu_ctx_dev { - const char *name; + struct device_node *of_node; int num; int mids[MAX_NUM_MIDS]; + int num_mids; + struct list_head list; }; - -/** - * struct msm_iommu_drvdata - A single IOMMU hardware instance - * @base: IOMMU config port base address (VA) - * @ncb The number of contexts on this IOMMU - * @irq: Interrupt number - * @clk: The bus clock for this IOMMU hardware instance - * @pclk: The clock for the IOMMU bus interconnect - * - * A msm_iommu_drvdata holds the global driver data about a single piece - * of an IOMMU hardware instance. - */ -struct msm_iommu_drvdata { - void __iomem *base; - int irq; - int ncb; - struct clk *clk; - struct clk *pclk; -}; - -/** - * struct msm_iommu_ctx_drvdata - an IOMMU context bank instance - * @num: Hardware context number of this context - * @pdev: Platform device associated wit this HW instance - * @attached_elm: List element for domains to track which devices are - * attached to them - * - * A msm_iommu_ctx_drvdata holds the driver data for a single context bank - * within each IOMMU hardware instance - */ -struct msm_iommu_ctx_drvdata { - int num; - struct platform_device *pdev; - struct list_head attached_elm; -}; - -/* - * Look up an IOMMU context device by its context name. NULL if none found. - * Useful for testing and drivers that do not yet fully have IOMMU stuff in - * their platform devices. - */ -struct device *msm_iommu_get_ctx(const char *ctx_name); - /* * Interrupt handler for the IOMMU context fault interrupt. Hooking the * interrupt is not supported in the API yet, but this will print an error diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c deleted file mode 100644 index 4b09e81..0000000 --- a/drivers/iommu/msm_iommu_dev.c +++ /dev/null @@ -1,381 +0,0 @@ -/* Copyright (c) 2010-2011, Code Aurora Forum. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 and - * only version 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/io.h> -#include <linux/clk.h> -#include <linux/iommu.h> -#include <linux/interrupt.h> -#include <linux/err.h> -#include <linux/slab.h> - -#include "msm_iommu_hw-8xxx.h" -#include "msm_iommu.h" - -struct iommu_ctx_iter_data { - /* input */ - const char *name; - - /* output */ - struct device *dev; -}; - -static struct platform_device *msm_iommu_root_dev; - -static int each_iommu_ctx(struct device *dev, void *data) -{ - struct iommu_ctx_iter_data *res = data; - struct msm_iommu_ctx_dev *c = dev->platform_data; - - if (!res || !c || !c->name || !res->name) - return -EINVAL; - - if (!strcmp(res->name, c->name)) { - res->dev = dev; - return 1; - } - return 0; -} - -static int each_iommu(struct device *dev, void *data) -{ - return device_for_each_child(dev, data, each_iommu_ctx); -} - -struct device *msm_iommu_get_ctx(const char *ctx_name) -{ - struct iommu_ctx_iter_data r; - int found; - - if (!msm_iommu_root_dev) { - pr_err("No root IOMMU device.\n"); - goto fail; - } - - r.name = ctx_name; - found = device_for_each_child(&msm_iommu_root_dev->dev, &r, each_iommu); - - if (!found) { - pr_err("Could not find context <%s>\n", ctx_name); - goto fail; - } - - return r.dev; -fail: - return NULL; -} -EXPORT_SYMBOL(msm_iommu_get_ctx); - -static void msm_iommu_reset(void __iomem *base, int ncb) -{ - int ctx; - - SET_RPUE(base, 0); - SET_RPUEIE(base, 0); - SET_ESRRESTORE(base, 0); - SET_TBE(base, 0); - SET_CR(base, 0); - SET_SPDMBE(base, 0); - SET_TESTBUSCR(base, 0); - SET_TLBRSW(base, 0); - SET_GLOBAL_TLBIALL(base, 0); - SET_RPU_ACR(base, 0); - SET_TLBLKCRWE(base, 1); - - for (ctx = 0; ctx < ncb; ctx++) { - SET_BPRCOSH(base, ctx, 0); - SET_BPRCISH(base, ctx, 0); - SET_BPRCNSH(base, ctx, 0); - SET_BPSHCFG(base, ctx, 0); - SET_BPMTCFG(base, ctx, 0); - SET_ACTLR(base, ctx, 0); - SET_SCTLR(base, ctx, 0); - SET_FSRRESTORE(base, ctx, 0); - SET_TTBR0(base, ctx, 0); - SET_TTBR1(base, ctx, 0); - SET_TTBCR(base, ctx, 0); - SET_BFBCR(base, ctx, 0); - SET_PAR(base, ctx, 0); - SET_FAR(base, ctx, 0); - SET_CTX_TLBIALL(base, ctx, 0); - SET_TLBFLPTER(base, ctx, 0); - SET_TLBSLPTER(base, ctx, 0); - SET_TLBLKCR(base, ctx, 0); - SET_PRRR(base, ctx, 0); - SET_NMRR(base, ctx, 0); - SET_CONTEXTIDR(base, ctx, 0); - } -} - -static int msm_iommu_probe(struct platform_device *pdev) -{ - struct resource *r; - struct clk *iommu_clk; - struct clk *iommu_pclk; - struct msm_iommu_drvdata *drvdata; - struct msm_iommu_dev *iommu_dev = dev_get_platdata(&pdev->dev); - void __iomem *regs_base; - int ret, irq, par; - - if (pdev->id == -1) { - msm_iommu_root_dev = pdev; - return 0; - } - - drvdata = kzalloc(sizeof(*drvdata), GFP_KERNEL); - - if (!drvdata) { - ret = -ENOMEM; - goto fail; - } - - if (!iommu_dev) { - ret = -ENODEV; - goto fail; - } - - iommu_pclk = clk_get(NULL, "smmu_pclk"); - if (IS_ERR(iommu_pclk)) { - ret = -ENODEV; - goto fail; - } - - ret = clk_prepare_enable(iommu_pclk); - if (ret) - goto fail_enable; - - iommu_clk = clk_get(&pdev->dev, "iommu_clk"); - - if (!IS_ERR(iommu_clk)) { - if (clk_get_rate(iommu_clk) == 0) - clk_set_rate(iommu_clk, 1); - - ret = clk_prepare_enable(iommu_clk); - if (ret) { - clk_put(iommu_clk); - goto fail_pclk; - } - } else - iommu_clk = NULL; - - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "physbase"); - regs_base = devm_ioremap_resource(&pdev->dev, r); - if (IS_ERR(regs_base)) { - ret = PTR_ERR(regs_base); - goto fail_clk; - } - - irq = platform_get_irq_byname(pdev, "secure_irq"); - if (irq < 0) { - ret = -ENODEV; - goto fail_clk; - } - - msm_iommu_reset(regs_base, iommu_dev->ncb); - - SET_M(regs_base, 0, 1); - SET_PAR(regs_base, 0, 0); - SET_V2PCFG(regs_base, 0, 1); - SET_V2PPR(regs_base, 0, 0); - par = GET_PAR(regs_base, 0); - SET_V2PCFG(regs_base, 0, 0); - SET_M(regs_base, 0, 0); - - if (!par) { - pr_err("%s: Invalid PAR value detected\n", iommu_dev->name); - ret = -ENODEV; - goto fail_clk; - } - - ret = request_irq(irq, msm_iommu_fault_handler, 0, - "msm_iommu_secure_irpt_handler", drvdata); - if (ret) { - pr_err("Request IRQ %d failed with ret=%d\n", irq, ret); - goto fail_clk; - } - - - drvdata->pclk = iommu_pclk; - drvdata->clk = iommu_clk; - drvdata->base = regs_base; - drvdata->irq = irq; - drvdata->ncb = iommu_dev->ncb; - - pr_info("device %s mapped at %p, irq %d with %d ctx banks\n", - iommu_dev->name, regs_base, irq, iommu_dev->ncb); - - platform_set_drvdata(pdev, drvdata); - - clk_disable(iommu_clk); - - clk_disable(iommu_pclk); - - return 0; -fail_clk: - if (iommu_clk) { - clk_disable(iommu_clk); - clk_put(iommu_clk); - } -fail_pclk: - clk_disable_unprepare(iommu_pclk); -fail_enable: - clk_put(iommu_pclk); -fail: - kfree(drvdata); - return ret; -} - -static int msm_iommu_remove(struct platform_device *pdev) -{ - struct msm_iommu_drvdata *drv = NULL; - - drv = platform_get_drvdata(pdev); - if (drv) { - if (drv->clk) { - clk_unprepare(drv->clk); - clk_put(drv->clk); - } - clk_unprepare(drv->pclk); - clk_put(drv->pclk); - memset(drv, 0, sizeof(*drv)); - kfree(drv); - } - return 0; -} - -static int msm_iommu_ctx_probe(struct platform_device *pdev) -{ - struct msm_iommu_ctx_dev *c = dev_get_platdata(&pdev->dev); - struct msm_iommu_drvdata *drvdata; - struct msm_iommu_ctx_drvdata *ctx_drvdata; - int i, ret; - - if (!c || !pdev->dev.parent) - return -EINVAL; - - drvdata = dev_get_drvdata(pdev->dev.parent); - if (!drvdata) - return -ENODEV; - - ctx_drvdata = kzalloc(sizeof(*ctx_drvdata), GFP_KERNEL); - if (!ctx_drvdata) - return -ENOMEM; - - ctx_drvdata->num = c->num; - ctx_drvdata->pdev = pdev; - - INIT_LIST_HEAD(&ctx_drvdata->attached_elm); - platform_set_drvdata(pdev, ctx_drvdata); - - ret = clk_prepare_enable(drvdata->pclk); - if (ret) - goto fail; - - if (drvdata->clk) { - ret = clk_prepare_enable(drvdata->clk); - if (ret) { - clk_disable_unprepare(drvdata->pclk); - goto fail; - } - } - - /* Program the M2V tables for this context */ - for (i = 0; i < MAX_NUM_MIDS; i++) { - int mid = c->mids[i]; - if (mid == -1) - break; - - SET_M2VCBR_N(drvdata->base, mid, 0); - SET_CBACR_N(drvdata->base, c->num, 0); - - /* Set VMID = 0 */ - SET_VMID(drvdata->base, mid, 0); - - /* Set the context number for that MID to this context */ - SET_CBNDX(drvdata->base, mid, c->num); - - /* Set MID associated with this context bank to 0*/ - SET_CBVMID(drvdata->base, c->num, 0); - - /* Set the ASID for TLB tagging for this context */ - SET_CONTEXTIDR_ASID(drvdata->base, c->num, c->num); - - /* Set security bit override to be Non-secure */ - SET_NSCFG(drvdata->base, mid, 3); - } - - clk_disable(drvdata->clk); - clk_disable(drvdata->pclk); - - dev_info(&pdev->dev, "context %s using bank %d\n", c->name, c->num); - return 0; -fail: - kfree(ctx_drvdata); - return ret; -} - -static int msm_iommu_ctx_remove(struct platform_device *pdev) -{ - struct msm_iommu_ctx_drvdata *drv = NULL; - drv = platform_get_drvdata(pdev); - if (drv) { - memset(drv, 0, sizeof(struct msm_iommu_ctx_drvdata)); - kfree(drv); - } - return 0; -} - -static struct platform_driver msm_iommu_driver = { - .driver = { - .name = "msm_iommu", - }, - .probe = msm_iommu_probe, - .remove = msm_iommu_remove, -}; - -static struct platform_driver msm_iommu_ctx_driver = { - .driver = { - .name = "msm_iommu_ctx", - }, - .probe = msm_iommu_ctx_probe, - .remove = msm_iommu_ctx_remove, -}; - -static struct platform_driver * const drivers[] = { - &msm_iommu_driver, - &msm_iommu_ctx_driver, -}; - -static int __init msm_iommu_driver_init(void) -{ - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); -} - -static void __exit msm_iommu_driver_exit(void) -{ - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); -} - -subsys_initcall(msm_iommu_driver_init); -module_exit(msm_iommu_driver_exit); - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Stepan Moskovchenko <stepanm@codeaurora.org>"); diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index c3043d8..b12c12d 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -34,7 +34,7 @@ #include <dt-bindings/memory/mt8173-larb-port.h> #include <soc/mediatek/smi.h> -#include "io-pgtable.h" +#include "mtk_iommu.h" #define REG_MMU_PT_BASE_ADDR 0x000 @@ -93,20 +93,6 @@ #define MTK_PROTECT_PA_ALIGN 128 -struct mtk_iommu_suspend_reg { - u32 standard_axi_mode; - u32 dcm_dis; - u32 ctrl_reg; - u32 int_control0; - u32 int_main_control; -}; - -struct mtk_iommu_client_priv { - struct list_head client; - unsigned int mtk_m4u_id; - struct device *m4udev; -}; - struct mtk_iommu_domain { spinlock_t pgtlock; /* lock for page table */ @@ -116,19 +102,6 @@ struct mtk_iommu_domain { struct iommu_domain domain; }; -struct mtk_iommu_data { - void __iomem *base; - int irq; - struct device *dev; - struct clk *bclk; - phys_addr_t protect_base; /* protect memory base */ - struct mtk_iommu_suspend_reg reg; - struct mtk_iommu_domain *m4u_dom; - struct iommu_group *m4u_group; - struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ - bool enable_4GB; -}; - static struct iommu_ops mtk_iommu_ops; static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) @@ -455,7 +428,6 @@ static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) if (!dev->archdata.iommu) { /* Get the m4u device */ m4updev = of_find_device_by_node(args->np); - of_node_put(args->np); if (WARN_ON(!m4updev)) return -EINVAL; @@ -552,25 +524,6 @@ static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) return 0; } -static int compare_of(struct device *dev, void *data) -{ - return dev->of_node == data; -} - -static int mtk_iommu_bind(struct device *dev) -{ - struct mtk_iommu_data *data = dev_get_drvdata(dev); - - return component_bind_all(dev, &data->smi_imu); -} - -static void mtk_iommu_unbind(struct device *dev) -{ - struct mtk_iommu_data *data = dev_get_drvdata(dev); - - component_unbind_all(dev, &data->smi_imu); -} - static const struct component_master_ops mtk_iommu_com_ops = { .bind = mtk_iommu_bind, .unbind = mtk_iommu_unbind, diff --git a/drivers/iommu/mtk_iommu.h b/drivers/iommu/mtk_iommu.h new file mode 100644 index 0000000..9ed0a84 --- /dev/null +++ b/drivers/iommu/mtk_iommu.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Honghui Zhang <honghui.zhang@mediatek.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _MTK_IOMMU_H_ +#define _MTK_IOMMU_H_ + +#include <linux/clk.h> +#include <linux/component.h> +#include <linux/device.h> +#include <linux/io.h> +#include <linux/iommu.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <soc/mediatek/smi.h> + +#include "io-pgtable.h" + +struct mtk_iommu_suspend_reg { + u32 standard_axi_mode; + u32 dcm_dis; + u32 ctrl_reg; + u32 int_control0; + u32 int_main_control; +}; + +struct mtk_iommu_client_priv { + struct list_head client; + unsigned int mtk_m4u_id; + struct device *m4udev; +}; + +struct mtk_iommu_domain; + +struct mtk_iommu_data { + void __iomem *base; + int irq; + struct device *dev; + struct clk *bclk; + phys_addr_t protect_base; /* protect memory base */ + struct mtk_iommu_suspend_reg reg; + struct mtk_iommu_domain *m4u_dom; + struct iommu_group *m4u_group; + struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ + bool enable_4GB; +}; + +static int compare_of(struct device *dev, void *data) +{ + return dev->of_node == data; +} + +static int mtk_iommu_bind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + return component_bind_all(dev, &data->smi_imu); +} + +static void mtk_iommu_unbind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + component_unbind_all(dev, &data->smi_imu); +} + +#endif diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c new file mode 100644 index 0000000..b8aeb07 --- /dev/null +++ b/drivers/iommu/mtk_iommu_v1.c @@ -0,0 +1,727 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Honghui Zhang <honghui.zhang@mediatek.com> + * + * Based on driver/iommu/mtk_iommu.c + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include <linux/bootmem.h> +#include <linux/bug.h> +#include <linux/clk.h> +#include <linux/component.h> +#include <linux/device.h> +#include <linux/dma-iommu.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iommu.h> +#include <linux/iopoll.h> +#include <linux/kmemleak.h> +#include <linux/list.h> +#include <linux/of_address.h> +#include <linux/of_iommu.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <asm/barrier.h> +#include <asm/dma-iommu.h> +#include <linux/module.h> +#include <dt-bindings/memory/mt2701-larb-port.h> +#include <soc/mediatek/smi.h> +#include "mtk_iommu.h" + +#define REG_MMU_PT_BASE_ADDR 0x000 + +#define F_ALL_INVLD 0x2 +#define F_MMU_INV_RANGE 0x1 +#define F_INVLD_EN0 BIT(0) +#define F_INVLD_EN1 BIT(1) + +#define F_MMU_FAULT_VA_MSK 0xfffff000 +#define MTK_PROTECT_PA_ALIGN 128 + +#define REG_MMU_CTRL_REG 0x210 +#define F_MMU_CTRL_COHERENT_EN BIT(8) +#define REG_MMU_IVRP_PADDR 0x214 +#define REG_MMU_INT_CONTROL 0x220 +#define F_INT_TRANSLATION_FAULT BIT(0) +#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1) +#define F_INT_INVALID_PA_FAULT BIT(2) +#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3) +#define F_INT_TABLE_WALK_FAULT BIT(4) +#define F_INT_TLB_MISS_FAULT BIT(5) +#define F_INT_PFH_DMA_FIFO_OVERFLOW BIT(6) +#define F_INT_MISS_DMA_FIFO_OVERFLOW BIT(7) + +#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) +#define F_INT_CLR_BIT BIT(12) + +#define REG_MMU_FAULT_ST 0x224 +#define REG_MMU_FAULT_VA 0x228 +#define REG_MMU_INVLD_PA 0x22C +#define REG_MMU_INT_ID 0x388 +#define REG_MMU_INVALIDATE 0x5c0 +#define REG_MMU_INVLD_START_A 0x5c4 +#define REG_MMU_INVLD_END_A 0x5c8 + +#define REG_MMU_INV_SEL 0x5d8 +#define REG_MMU_STANDARD_AXI_MODE 0x5e8 + +#define REG_MMU_DCM 0x5f0 +#define F_MMU_DCM_ON BIT(1) +#define REG_MMU_CPE_DONE 0x60c +#define F_DESC_VALID 0x2 +#define F_DESC_NONSEC BIT(3) +#define MT2701_M4U_TF_LARB(TF) (6 - (((TF) >> 13) & 0x7)) +#define MT2701_M4U_TF_PORT(TF) (((TF) >> 8) & 0xF) +/* MTK generation one iommu HW only support 4K size mapping */ +#define MT2701_IOMMU_PAGE_SHIFT 12 +#define MT2701_IOMMU_PAGE_SIZE (1UL << MT2701_IOMMU_PAGE_SHIFT) + +/* + * MTK m4u support 4GB iova address space, and only support 4K page + * mapping. So the pagetable size should be exactly as 4M. + */ +#define M2701_IOMMU_PGT_SIZE SZ_4M + +struct mtk_iommu_domain { + spinlock_t pgtlock; /* lock for page table */ + struct iommu_domain domain; + u32 *pgt_va; + dma_addr_t pgt_pa; + struct mtk_iommu_data *data; +}; + +static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct mtk_iommu_domain, domain); +} + +static const int mt2701_m4u_in_larb[] = { + LARB0_PORT_OFFSET, LARB1_PORT_OFFSET, + LARB2_PORT_OFFSET, LARB3_PORT_OFFSET +}; + +static inline int mt2701_m4u_to_larb(int id) +{ + int i; + + for (i = ARRAY_SIZE(mt2701_m4u_in_larb) - 1; i >= 0; i--) + if ((id) >= mt2701_m4u_in_larb[i]) + return i; + + return 0; +} + +static inline int mt2701_m4u_to_port(int id) +{ + int larb = mt2701_m4u_to_larb(id); + + return id - mt2701_m4u_in_larb[larb]; +} + +static void mtk_iommu_tlb_flush_all(struct mtk_iommu_data *data) +{ + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, + data->base + REG_MMU_INV_SEL); + writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); + wmb(); /* Make sure the tlb flush all done */ +} + +static void mtk_iommu_tlb_flush_range(struct mtk_iommu_data *data, + unsigned long iova, size_t size) +{ + int ret; + u32 tmp; + + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, + data->base + REG_MMU_INV_SEL); + writel_relaxed(iova & F_MMU_FAULT_VA_MSK, + data->base + REG_MMU_INVLD_START_A); + writel_relaxed((iova + size - 1) & F_MMU_FAULT_VA_MSK, + data->base + REG_MMU_INVLD_END_A); + writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); + + ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, + tmp, tmp != 0, 10, 100000); + if (ret) { + dev_warn(data->dev, + "Partial TLB flush timed out, falling back to full flush\n"); + mtk_iommu_tlb_flush_all(data); + } + /* Clear the CPE status */ + writel_relaxed(0, data->base + REG_MMU_CPE_DONE); +} + +static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) +{ + struct mtk_iommu_data *data = dev_id; + struct mtk_iommu_domain *dom = data->m4u_dom; + u32 int_state, regval, fault_iova, fault_pa; + unsigned int fault_larb, fault_port; + + /* Read error information from registers */ + int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST); + fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA); + + fault_iova &= F_MMU_FAULT_VA_MSK; + fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA); + regval = readl_relaxed(data->base + REG_MMU_INT_ID); + fault_larb = MT2701_M4U_TF_LARB(regval); + fault_port = MT2701_M4U_TF_PORT(regval); + + /* + * MTK v1 iommu HW could not determine whether the fault is read or + * write fault, report as read fault. + */ + if (report_iommu_fault(&dom->domain, data->dev, fault_iova, + IOMMU_FAULT_READ)) + dev_err_ratelimited(data->dev, + "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d\n", + int_state, fault_iova, fault_pa, + fault_larb, fault_port); + + /* Interrupt clear */ + regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL); + regval |= F_INT_CLR_BIT; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); + + mtk_iommu_tlb_flush_all(data); + + return IRQ_HANDLED; +} + +static void mtk_iommu_config(struct mtk_iommu_data *data, + struct device *dev, bool enable) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + struct mtk_smi_larb_iommu *larb_mmu; + unsigned int larbid, portid; + + head = dev->archdata.iommu; + list_for_each_entry_safe(cur, next, &head->client, client) { + larbid = mt2701_m4u_to_larb(cur->mtk_m4u_id); + portid = mt2701_m4u_to_port(cur->mtk_m4u_id); + larb_mmu = &data->smi_imu.larb_imu[larbid]; + + dev_dbg(dev, "%s iommu port: %d\n", + enable ? "enable" : "disable", portid); + + if (enable) + larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); + else + larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + } +} + +static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) +{ + struct mtk_iommu_domain *dom = data->m4u_dom; + + spin_lock_init(&dom->pgtlock); + + dom->pgt_va = dma_zalloc_coherent(data->dev, + M2701_IOMMU_PGT_SIZE, + &dom->pgt_pa, GFP_KERNEL); + if (!dom->pgt_va) + return -ENOMEM; + + writel(dom->pgt_pa, data->base + REG_MMU_PT_BASE_ADDR); + + dom->data = data; + + return 0; +} + +static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) +{ + struct mtk_iommu_domain *dom; + + if (type != IOMMU_DOMAIN_UNMANAGED) + return NULL; + + dom = kzalloc(sizeof(*dom), GFP_KERNEL); + if (!dom) + return NULL; + + return &dom->domain; +} + +static void mtk_iommu_domain_free(struct iommu_domain *domain) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_data *data = dom->data; + + dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE, + dom->pgt_va, dom->pgt_pa); + kfree(to_mtk_domain(domain)); +} + +static int mtk_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + int ret; + + if (!priv) + return -ENODEV; + + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_dom) { + data->m4u_dom = dom; + ret = mtk_iommu_domain_finalise(data); + if (ret) { + data->m4u_dom = NULL; + return ret; + } + } + + mtk_iommu_config(data, dev, true); + return 0; +} + +static void mtk_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + + if (!priv) + return; + + data = dev_get_drvdata(priv->m4udev); + mtk_iommu_config(data, dev, false); +} + +static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; + unsigned long flags; + unsigned int i; + u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); + u32 pabase = (u32)paddr; + int map_size = 0; + + spin_lock_irqsave(&dom->pgtlock, flags); + for (i = 0; i < page_num; i++) { + if (pgt_base_iova[i]) { + memset(pgt_base_iova, 0, i * sizeof(u32)); + break; + } + pgt_base_iova[i] = pabase | F_DESC_VALID | F_DESC_NONSEC; + pabase += MT2701_IOMMU_PAGE_SIZE; + map_size += MT2701_IOMMU_PAGE_SIZE; + } + + spin_unlock_irqrestore(&dom->pgtlock, flags); + + mtk_iommu_tlb_flush_range(dom->data, iova, size); + + return map_size == size ? 0 : -EEXIST; +} + +static size_t mtk_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + u32 *pgt_base_iova = dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT); + unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT; + + spin_lock_irqsave(&dom->pgtlock, flags); + memset(pgt_base_iova, 0, page_num * sizeof(u32)); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + mtk_iommu_tlb_flush_range(dom->data, iova, size); + + return size; +} + +static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + phys_addr_t pa; + + spin_lock_irqsave(&dom->pgtlock, flags); + pa = *(dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT)); + pa = pa & (~(MT2701_IOMMU_PAGE_SIZE - 1)); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return pa; +} + +/* + * MTK generation one iommu HW only support one iommu domain, and all the client + * sharing the same iova address space. + */ +static int mtk_iommu_create_mapping(struct device *dev, + struct of_phandle_args *args) +{ + struct mtk_iommu_client_priv *head, *priv, *next; + struct platform_device *m4updev; + struct dma_iommu_mapping *mtk_mapping; + struct device *m4udev; + int ret; + + if (args->args_count != 1) { + dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", + args->args_count); + return -EINVAL; + } + + if (!dev->archdata.iommu) { + /* Get the m4u device */ + m4updev = of_find_device_by_node(args->np); + if (WARN_ON(!m4updev)) + return -EINVAL; + + head = kzalloc(sizeof(*head), GFP_KERNEL); + if (!head) + return -ENOMEM; + + dev->archdata.iommu = head; + INIT_LIST_HEAD(&head->client); + head->m4udev = &m4updev->dev; + } else { + head = dev->archdata.iommu; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + ret = -ENOMEM; + goto err_free_mem; + } + priv->mtk_m4u_id = args->args[0]; + list_add_tail(&priv->client, &head->client); + + m4udev = head->m4udev; + mtk_mapping = m4udev->archdata.iommu; + if (!mtk_mapping) { + /* MTK iommu support 4GB iova address space. */ + mtk_mapping = arm_iommu_create_mapping(&platform_bus_type, + 0, 1ULL << 32); + if (IS_ERR(mtk_mapping)) { + ret = PTR_ERR(mtk_mapping); + goto err_free_mem; + } + m4udev->archdata.iommu = mtk_mapping; + } + + ret = arm_iommu_attach_device(dev, mtk_mapping); + if (ret) + goto err_release_mapping; + + return 0; + +err_release_mapping: + arm_iommu_release_mapping(mtk_mapping); + m4udev->archdata.iommu = NULL; +err_free_mem: + list_for_each_entry_safe(priv, next, &head->client, client) + kfree(priv); + kfree(head); + dev->archdata.iommu = NULL; + return ret; +} + +static int mtk_iommu_add_device(struct device *dev) +{ + struct iommu_group *group; + struct of_phandle_args iommu_spec; + struct of_phandle_iterator it; + int err; + + of_for_each_phandle(&it, err, dev->of_node, "iommus", + "#iommu-cells", 0) { + int count = of_phandle_iterator_args(&it, iommu_spec.args, + MAX_PHANDLE_ARGS); + iommu_spec.np = of_node_get(it.node); + iommu_spec.args_count = count; + + mtk_iommu_create_mapping(dev, &iommu_spec); + of_node_put(iommu_spec.np); + } + + if (!dev->archdata.iommu) /* Not a iommu client device */ + return -ENODEV; + + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + iommu_group_put(group); + return 0; +} + +static void mtk_iommu_remove_device(struct device *dev) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + + head = dev->archdata.iommu; + if (!head) + return; + + list_for_each_entry_safe(cur, next, &head->client, client) { + list_del(&cur->client); + kfree(cur); + } + kfree(head); + dev->archdata.iommu = NULL; + + iommu_group_remove_device(dev); +} + +static struct iommu_group *mtk_iommu_device_group(struct device *dev) +{ + struct mtk_iommu_data *data; + struct mtk_iommu_client_priv *priv; + + priv = dev->archdata.iommu; + if (!priv) + return ERR_PTR(-ENODEV); + + /* All the client devices are in the same m4u iommu-group */ + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_group) { + data->m4u_group = iommu_group_alloc(); + if (IS_ERR(data->m4u_group)) + dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + } + return data->m4u_group; +} + +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) +{ + u32 regval; + int ret; + + ret = clk_prepare_enable(data->bclk); + if (ret) { + dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); + return ret; + } + + regval = F_MMU_CTRL_COHERENT_EN | F_MMU_TF_PROTECT_SEL(2); + writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); + + regval = F_INT_TRANSLATION_FAULT | + F_INT_MAIN_MULTI_HIT_FAULT | + F_INT_INVALID_PA_FAULT | + F_INT_ENTRY_REPLACEMENT_FAULT | + F_INT_TABLE_WALK_FAULT | + F_INT_TLB_MISS_FAULT | + F_INT_PFH_DMA_FIFO_OVERFLOW | + F_INT_MISS_DMA_FIFO_OVERFLOW; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL); + + /* protect memory,hw will write here while translation fault */ + writel_relaxed(data->protect_base, + data->base + REG_MMU_IVRP_PADDR); + + writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM); + + if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, + dev_name(data->dev), (void *)data)) { + writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); + clk_disable_unprepare(data->bclk); + dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); + return -ENODEV; + } + + return 0; +} + +static struct iommu_ops mtk_iommu_ops = { + .domain_alloc = mtk_iommu_domain_alloc, + .domain_free = mtk_iommu_domain_free, + .attach_dev = mtk_iommu_attach_device, + .detach_dev = mtk_iommu_detach_device, + .map = mtk_iommu_map, + .unmap = mtk_iommu_unmap, + .map_sg = default_iommu_map_sg, + .iova_to_phys = mtk_iommu_iova_to_phys, + .add_device = mtk_iommu_add_device, + .remove_device = mtk_iommu_remove_device, + .device_group = mtk_iommu_device_group, + .pgsize_bitmap = ~0UL << MT2701_IOMMU_PAGE_SHIFT, +}; + +static const struct of_device_id mtk_iommu_of_ids[] = { + { .compatible = "mediatek,mt2701-m4u", }, + {} +}; + +static const struct component_master_ops mtk_iommu_com_ops = { + .bind = mtk_iommu_bind, + .unbind = mtk_iommu_unbind, +}; + +static int mtk_iommu_probe(struct platform_device *pdev) +{ + struct mtk_iommu_data *data; + struct device *dev = &pdev->dev; + struct resource *res; + struct component_match *match = NULL; + struct of_phandle_args larb_spec; + struct of_phandle_iterator it; + void *protect; + int larb_nr, ret, err; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->dev = dev; + + /* Protect memory. HW will access here while translation fault.*/ + protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, + GFP_KERNEL | GFP_DMA); + if (!protect) + return -ENOMEM; + data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->base)) + return PTR_ERR(data->base); + + data->irq = platform_get_irq(pdev, 0); + if (data->irq < 0) + return data->irq; + + data->bclk = devm_clk_get(dev, "bclk"); + if (IS_ERR(data->bclk)) + return PTR_ERR(data->bclk); + + larb_nr = 0; + of_for_each_phandle(&it, err, dev->of_node, + "mediatek,larbs", NULL, 0) { + struct platform_device *plarbdev; + int count = of_phandle_iterator_args(&it, larb_spec.args, + MAX_PHANDLE_ARGS); + + if (count) + continue; + + larb_spec.np = of_node_get(it.node); + if (!of_device_is_available(larb_spec.np)) + continue; + + plarbdev = of_find_device_by_node(larb_spec.np); + of_node_put(larb_spec.np); + if (!plarbdev) { + plarbdev = of_platform_device_create( + larb_spec.np, NULL, + platform_bus_type.dev_root); + if (!plarbdev) + return -EPROBE_DEFER; + } + + data->smi_imu.larb_imu[larb_nr].dev = &plarbdev->dev; + component_match_add(dev, &match, compare_of, larb_spec.np); + larb_nr++; + } + + data->smi_imu.larb_nr = larb_nr; + + platform_set_drvdata(pdev, data); + + ret = mtk_iommu_hw_init(data); + if (ret) + return ret; + + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + + return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); +} + +static int mtk_iommu_remove(struct platform_device *pdev) +{ + struct mtk_iommu_data *data = platform_get_drvdata(pdev); + + if (iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, NULL); + + clk_disable_unprepare(data->bclk); + devm_free_irq(&pdev->dev, data->irq, data); + component_master_del(&pdev->dev, &mtk_iommu_com_ops); + return 0; +} + +static int __maybe_unused mtk_iommu_suspend(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + reg->standard_axi_mode = readl_relaxed(base + + REG_MMU_STANDARD_AXI_MODE); + reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM); + reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); + reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL); + return 0; +} + +static int __maybe_unused mtk_iommu_resume(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR); + writel_relaxed(reg->standard_axi_mode, + base + REG_MMU_STANDARD_AXI_MODE); + writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM); + writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); + writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL); + writel_relaxed(data->protect_base, base + REG_MMU_IVRP_PADDR); + return 0; +} + +static const struct dev_pm_ops mtk_iommu_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) +}; + +static struct platform_driver mtk_iommu_driver = { + .probe = mtk_iommu_probe, + .remove = mtk_iommu_remove, + .driver = { + .name = "mtk-iommu", + .of_match_table = mtk_iommu_of_ids, + .pm = &mtk_iommu_pm_ops, + } +}; + +static int __init m4u_init(void) +{ + return platform_driver_register(&mtk_iommu_driver); +} + +static void __exit m4u_exit(void) +{ + return platform_driver_unregister(&mtk_iommu_driver); +} + +subsys_initcall(m4u_init); +module_exit(m4u_exit); + +MODULE_DESCRIPTION("IOMMU API for MTK architected m4u v1 implementations"); +MODULE_AUTHOR("Honghui Zhang <honghui.zhang@mediatek.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index c7d6156..9afcbf7 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -4,11 +4,10 @@ * published by the Free Software Foundation. */ -#include <asm/cacheflush.h> -#include <asm/pgtable.h> #include <linux/compiler.h> #include <linux/delay.h> #include <linux/device.h> +#include <linux/dma-iommu.h> #include <linux/errno.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -77,7 +76,9 @@ struct rk_iommu_domain { struct list_head iommus; + struct platform_device *pdev; u32 *dt; /* page directory table */ + dma_addr_t dt_dma; spinlock_t iommus_lock; /* lock for iommus list */ spinlock_t dt_lock; /* lock for modifying page directory table */ @@ -93,14 +94,12 @@ struct rk_iommu { struct iommu_domain *domain; /* domain to which iommu is attached */ }; -static inline void rk_table_flush(u32 *va, unsigned int count) +static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, + unsigned int count) { - phys_addr_t pa_start = virt_to_phys(va); - phys_addr_t pa_end = virt_to_phys(va + count); - size_t size = pa_end - pa_start; + size_t size = count * sizeof(u32); /* count of u32 entry */ - __cpuc_flush_dcache_area(va, size); - outer_flush_range(pa_start, pa_end); + dma_sync_single_for_device(&dom->pdev->dev, dma, size, DMA_TO_DEVICE); } static struct rk_iommu_domain *to_rk_domain(struct iommu_domain *dom) @@ -183,10 +182,9 @@ static inline bool rk_dte_is_pt_valid(u32 dte) return dte & RK_DTE_PT_VALID; } -static u32 rk_mk_dte(u32 *pt) +static inline u32 rk_mk_dte(dma_addr_t pt_dma) { - phys_addr_t pt_phys = virt_to_phys(pt); - return (pt_phys & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; + return (pt_dma & RK_DTE_PT_ADDRESS_MASK) | RK_DTE_PT_VALID; } /* @@ -603,13 +601,16 @@ static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, dma_addr_t iova) { + struct device *dev = &rk_domain->pdev->dev; u32 *page_table, *dte_addr; - u32 dte; + u32 dte_index, dte; phys_addr_t pt_phys; + dma_addr_t pt_dma; assert_spin_locked(&rk_domain->dt_lock); - dte_addr = &rk_domain->dt[rk_iova_dte_index(iova)]; + dte_index = rk_iova_dte_index(iova); + dte_addr = &rk_domain->dt[dte_index]; dte = *dte_addr; if (rk_dte_is_pt_valid(dte)) goto done; @@ -618,19 +619,27 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, if (!page_table) return ERR_PTR(-ENOMEM); - dte = rk_mk_dte(page_table); - *dte_addr = dte; + pt_dma = dma_map_single(dev, page_table, SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(dev, pt_dma)) { + dev_err(dev, "DMA mapping error while allocating page table\n"); + free_page((unsigned long)page_table); + return ERR_PTR(-ENOMEM); + } - rk_table_flush(page_table, NUM_PT_ENTRIES); - rk_table_flush(dte_addr, 1); + dte = rk_mk_dte(pt_dma); + *dte_addr = dte; + rk_table_flush(rk_domain, pt_dma, NUM_PT_ENTRIES); + rk_table_flush(rk_domain, + rk_domain->dt_dma + dte_index * sizeof(u32), 1); done: pt_phys = rk_dte_pt_address(dte); return (u32 *)phys_to_virt(pt_phys); } static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, - u32 *pte_addr, dma_addr_t iova, size_t size) + u32 *pte_addr, dma_addr_t pte_dma, + size_t size) { unsigned int pte_count; unsigned int pte_total = size / SPAGE_SIZE; @@ -645,14 +654,14 @@ static size_t rk_iommu_unmap_iova(struct rk_iommu_domain *rk_domain, pte_addr[pte_count] = rk_mk_pte_invalid(pte); } - rk_table_flush(pte_addr, pte_count); + rk_table_flush(rk_domain, pte_dma, pte_count); return pte_count * SPAGE_SIZE; } static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, - dma_addr_t iova, phys_addr_t paddr, size_t size, - int prot) + dma_addr_t pte_dma, dma_addr_t iova, + phys_addr_t paddr, size_t size, int prot) { unsigned int pte_count; unsigned int pte_total = size / SPAGE_SIZE; @@ -671,7 +680,7 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, paddr += SPAGE_SIZE; } - rk_table_flush(pte_addr, pte_count); + rk_table_flush(rk_domain, pte_dma, pte_total); /* * Zap the first and last iova to evict from iotlb any previously @@ -684,7 +693,8 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, return 0; unwind: /* Unmap the range of iovas that we just mapped */ - rk_iommu_unmap_iova(rk_domain, pte_addr, iova, pte_count * SPAGE_SIZE); + rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, + pte_count * SPAGE_SIZE); iova += pte_count * SPAGE_SIZE; page_phys = rk_pte_page_address(pte_addr[pte_count]); @@ -699,8 +709,9 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, { struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; - dma_addr_t iova = (dma_addr_t)_iova; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; u32 *page_table, *pte_addr; + u32 dte_index, pte_index; int ret; spin_lock_irqsave(&rk_domain->dt_lock, flags); @@ -718,8 +729,13 @@ static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova, return PTR_ERR(page_table); } - pte_addr = &page_table[rk_iova_pte_index(iova)]; - ret = rk_iommu_map_iova(rk_domain, pte_addr, iova, paddr, size, prot); + dte_index = rk_domain->dt[rk_iova_dte_index(iova)]; + pte_index = rk_iova_pte_index(iova); + pte_addr = &page_table[pte_index]; + pte_dma = rk_dte_pt_address(dte_index) + pte_index * sizeof(u32); + ret = rk_iommu_map_iova(rk_domain, pte_addr, pte_dma, iova, + paddr, size, prot); + spin_unlock_irqrestore(&rk_domain->dt_lock, flags); return ret; @@ -730,7 +746,7 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, { struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; - dma_addr_t iova = (dma_addr_t)_iova; + dma_addr_t pte_dma, iova = (dma_addr_t)_iova; phys_addr_t pt_phys; u32 dte; u32 *pte_addr; @@ -754,7 +770,8 @@ static size_t rk_iommu_unmap(struct iommu_domain *domain, unsigned long _iova, pt_phys = rk_dte_pt_address(dte); pte_addr = (u32 *)phys_to_virt(pt_phys) + rk_iova_pte_index(iova); - unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, iova, size); + pte_dma = pt_phys + rk_iova_pte_index(iova) * sizeof(u32); + unmap_size = rk_iommu_unmap_iova(rk_domain, pte_addr, pte_dma, size); spin_unlock_irqrestore(&rk_domain->dt_lock, flags); @@ -787,7 +804,6 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; int ret, i; - phys_addr_t dte_addr; /* * Allow 'virtual devices' (e.g., drm) to attach to domain. @@ -807,15 +823,15 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, iommu->domain = domain; - ret = devm_request_irq(dev, iommu->irq, rk_iommu_irq, + ret = devm_request_irq(iommu->dev, iommu->irq, rk_iommu_irq, IRQF_SHARED, dev_name(dev), iommu); if (ret) return ret; - dte_addr = virt_to_phys(rk_domain->dt); for (i = 0; i < iommu->num_mmu; i++) { - rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr); - rk_iommu_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); + rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, + rk_domain->dt_dma); + rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); } @@ -860,7 +876,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, } rk_iommu_disable_stall(iommu); - devm_free_irq(dev, iommu->irq, iommu); + devm_free_irq(iommu->dev, iommu->irq, iommu); iommu->domain = NULL; @@ -870,14 +886,30 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) { struct rk_iommu_domain *rk_domain; + struct platform_device *pdev; + struct device *iommu_dev; - if (type != IOMMU_DOMAIN_UNMANAGED) + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; - rk_domain = kzalloc(sizeof(*rk_domain), GFP_KERNEL); - if (!rk_domain) + /* Register a pdev per domain, so DMA API can base on this *dev + * even some virtual master doesn't have an iommu slave + */ + pdev = platform_device_register_simple("rk_iommu_domain", + PLATFORM_DEVID_AUTO, NULL, 0); + if (IS_ERR(pdev)) return NULL; + rk_domain = devm_kzalloc(&pdev->dev, sizeof(*rk_domain), GFP_KERNEL); + if (!rk_domain) + goto err_unreg_pdev; + + rk_domain->pdev = pdev; + + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&rk_domain->domain)) + goto err_unreg_pdev; + /* * rk32xx iommus use a 2 level pagetable. * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries. @@ -885,18 +917,36 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) */ rk_domain->dt = (u32 *)get_zeroed_page(GFP_KERNEL | GFP_DMA32); if (!rk_domain->dt) - goto err_dt; + goto err_put_cookie; + + iommu_dev = &pdev->dev; + rk_domain->dt_dma = dma_map_single(iommu_dev, rk_domain->dt, + SPAGE_SIZE, DMA_TO_DEVICE); + if (dma_mapping_error(iommu_dev, rk_domain->dt_dma)) { + dev_err(iommu_dev, "DMA map error for DT\n"); + goto err_free_dt; + } - rk_table_flush(rk_domain->dt, NUM_DT_ENTRIES); + rk_table_flush(rk_domain, rk_domain->dt_dma, NUM_DT_ENTRIES); spin_lock_init(&rk_domain->iommus_lock); spin_lock_init(&rk_domain->dt_lock); INIT_LIST_HEAD(&rk_domain->iommus); + rk_domain->domain.geometry.aperture_start = 0; + rk_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32); + rk_domain->domain.geometry.force_aperture = true; + return &rk_domain->domain; -err_dt: - kfree(rk_domain); +err_free_dt: + free_page((unsigned long)rk_domain->dt); +err_put_cookie: + if (type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&rk_domain->domain); +err_unreg_pdev: + platform_device_unregister(pdev); + return NULL; } @@ -912,12 +962,20 @@ static void rk_iommu_domain_free(struct iommu_domain *domain) if (rk_dte_is_pt_valid(dte)) { phys_addr_t pt_phys = rk_dte_pt_address(dte); u32 *page_table = phys_to_virt(pt_phys); + dma_unmap_single(&rk_domain->pdev->dev, pt_phys, + SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)page_table); } } + dma_unmap_single(&rk_domain->pdev->dev, rk_domain->dt_dma, + SPAGE_SIZE, DMA_TO_DEVICE); free_page((unsigned long)rk_domain->dt); - kfree(rk_domain); + + if (domain->type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&rk_domain->domain); + + platform_device_unregister(rk_domain->pdev); } static bool rk_iommu_is_dev_iommu_master(struct device *dev) @@ -1022,17 +1080,43 @@ static const struct iommu_ops rk_iommu_ops = { .detach_dev = rk_iommu_detach_device, .map = rk_iommu_map, .unmap = rk_iommu_unmap, + .map_sg = default_iommu_map_sg, .add_device = rk_iommu_add_device, .remove_device = rk_iommu_remove_device, .iova_to_phys = rk_iommu_iova_to_phys, .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, }; +static int rk_iommu_domain_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + + dev->dma_parms = devm_kzalloc(dev, sizeof(*dev->dma_parms), GFP_KERNEL); + if (!dev->dma_parms) + return -ENOMEM; + + /* Set dma_ops for dev, otherwise it would be dummy_dma_ops */ + arch_setup_dma_ops(dev, 0, DMA_BIT_MASK(32), NULL, false); + + dma_set_max_seg_size(dev, DMA_BIT_MASK(32)); + dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(32)); + + return 0; +} + +static struct platform_driver rk_iommu_domain_driver = { + .probe = rk_iommu_domain_probe, + .driver = { + .name = "rk_iommu_domain", + }, +}; + static int rk_iommu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct rk_iommu *iommu; struct resource *res; + int num_res = pdev->num_resources; int i; iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); @@ -1042,12 +1126,13 @@ static int rk_iommu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, iommu); iommu->dev = dev; iommu->num_mmu = 0; - iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * iommu->num_mmu, + + iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * num_res, GFP_KERNEL); if (!iommu->bases) return -ENOMEM; - for (i = 0; i < pdev->num_resources; i++) { + for (i = 0; i < num_res; i++) { res = platform_get_resource(pdev, IORESOURCE_MEM, i); if (!res) continue; @@ -1103,11 +1188,19 @@ static int __init rk_iommu_init(void) if (ret) return ret; - return platform_driver_register(&rk_iommu_driver); + ret = platform_driver_register(&rk_iommu_domain_driver); + if (ret) + return ret; + + ret = platform_driver_register(&rk_iommu_driver); + if (ret) + platform_driver_unregister(&rk_iommu_domain_driver); + return ret; } static void __exit rk_iommu_exit(void) { platform_driver_unregister(&rk_iommu_driver); + platform_driver_unregister(&rk_iommu_domain_driver); } subsys_initcall(rk_iommu_init); |