From cd6438c5f8446691afa4829fe1a9d7b656204f11 Mon Sep 17 00:00:00 2001 From: ZhengShunQian Date: Tue, 19 Jan 2016 15:03:00 +0800 Subject: iommu/rockchip: Reconstruct to support multi slaves There are some IPs, such as video encoder/decoder, contains 2 slave iommus, one for reading and the other for writing. They share the same irq and clock with master. This patch reconstructs to support this case by making them share the same Page Directory, Page Tables and even the register operations. That means every instruction to the reading MMU registers would be duplicated to the writing MMU and vice versa. Signed-off-by: ZhengShunQian Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index ebf0adb..a6f593a 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -86,7 +86,8 @@ struct rk_iommu_domain { struct rk_iommu { struct device *dev; - void __iomem *base; + void __iomem **bases; + int num_mmu; int irq; struct list_head node; /* entry in rk_iommu_domain.iommus */ struct iommu_domain *domain; /* domain to which iommu is attached */ @@ -271,47 +272,70 @@ static u32 rk_iova_page_offset(dma_addr_t iova) return (u32)(iova & RK_IOVA_PAGE_MASK) >> RK_IOVA_PAGE_SHIFT; } -static u32 rk_iommu_read(struct rk_iommu *iommu, u32 offset) +static u32 rk_iommu_read(void __iomem *base, u32 offset) { - return readl(iommu->base + offset); + return readl(base + offset); } -static void rk_iommu_write(struct rk_iommu *iommu, u32 offset, u32 value) +static void rk_iommu_write(void __iomem *base, u32 offset, u32 value) { - writel(value, iommu->base + offset); + writel(value, base + offset); } static void rk_iommu_command(struct rk_iommu *iommu, u32 command) { - writel(command, iommu->base + RK_MMU_COMMAND); + int i; + + for (i = 0; i < iommu->num_mmu; i++) + writel(command, iommu->bases[i] + RK_MMU_COMMAND); } +static void rk_iommu_base_command(void __iomem *base, u32 command) +{ + writel(command, base + RK_MMU_COMMAND); +} static void rk_iommu_zap_lines(struct rk_iommu *iommu, dma_addr_t iova, size_t size) { + int i; + dma_addr_t iova_end = iova + size; /* * TODO(djkurtz): Figure out when it is more efficient to shootdown the * entire iotlb rather than iterate over individual iovas. */ - for (; iova < iova_end; iova += SPAGE_SIZE) - rk_iommu_write(iommu, RK_MMU_ZAP_ONE_LINE, iova); + for (i = 0; i < iommu->num_mmu; i++) + for (; iova < iova_end; iova += SPAGE_SIZE) + rk_iommu_write(iommu->bases[i], RK_MMU_ZAP_ONE_LINE, iova); } static bool rk_iommu_is_stall_active(struct rk_iommu *iommu) { - return rk_iommu_read(iommu, RK_MMU_STATUS) & RK_MMU_STATUS_STALL_ACTIVE; + bool active = true; + int i; + + for (i = 0; i < iommu->num_mmu; i++) + active &= rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) & + RK_MMU_STATUS_STALL_ACTIVE; + + return active; } static bool rk_iommu_is_paging_enabled(struct rk_iommu *iommu) { - return rk_iommu_read(iommu, RK_MMU_STATUS) & - RK_MMU_STATUS_PAGING_ENABLED; + bool enable = true; + int i; + + for (i = 0; i < iommu->num_mmu; i++) + enable &= rk_iommu_read(iommu->bases[i], RK_MMU_STATUS) & + RK_MMU_STATUS_PAGING_ENABLED; + + return enable; } static int rk_iommu_enable_stall(struct rk_iommu *iommu) { - int ret; + int ret, i; if (rk_iommu_is_stall_active(iommu)) return 0; @@ -324,15 +348,16 @@ static int rk_iommu_enable_stall(struct rk_iommu *iommu) ret = rk_wait_for(rk_iommu_is_stall_active(iommu), 1); if (ret) - dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", - rk_iommu_read(iommu, RK_MMU_STATUS)); + for (i = 0; i < iommu->num_mmu; i++) + dev_err(iommu->dev, "Enable stall request timed out, status: %#08x\n", + rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); return ret; } static int rk_iommu_disable_stall(struct rk_iommu *iommu) { - int ret; + int ret, i; if (!rk_iommu_is_stall_active(iommu)) return 0; @@ -341,15 +366,16 @@ static int rk_iommu_disable_stall(struct rk_iommu *iommu) ret = rk_wait_for(!rk_iommu_is_stall_active(iommu), 1); if (ret) - dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", - rk_iommu_read(iommu, RK_MMU_STATUS)); + for (i = 0; i < iommu->num_mmu; i++) + dev_err(iommu->dev, "Disable stall request timed out, status: %#08x\n", + rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); return ret; } static int rk_iommu_enable_paging(struct rk_iommu *iommu) { - int ret; + int ret, i; if (rk_iommu_is_paging_enabled(iommu)) return 0; @@ -358,15 +384,16 @@ static int rk_iommu_enable_paging(struct rk_iommu *iommu) ret = rk_wait_for(rk_iommu_is_paging_enabled(iommu), 1); if (ret) - dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", - rk_iommu_read(iommu, RK_MMU_STATUS)); + for (i = 0; i < iommu->num_mmu; i++) + dev_err(iommu->dev, "Enable paging request timed out, status: %#08x\n", + rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); return ret; } static int rk_iommu_disable_paging(struct rk_iommu *iommu) { - int ret; + int ret, i; if (!rk_iommu_is_paging_enabled(iommu)) return 0; @@ -375,41 +402,49 @@ static int rk_iommu_disable_paging(struct rk_iommu *iommu) ret = rk_wait_for(!rk_iommu_is_paging_enabled(iommu), 1); if (ret) - dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", - rk_iommu_read(iommu, RK_MMU_STATUS)); + for (i = 0; i < iommu->num_mmu; i++) + dev_err(iommu->dev, "Disable paging request timed out, status: %#08x\n", + rk_iommu_read(iommu->bases[i], RK_MMU_STATUS)); return ret; } static int rk_iommu_force_reset(struct rk_iommu *iommu) { - int ret; + int ret, i; u32 dte_addr; /* * Check if register DTE_ADDR is working by writing DTE_ADDR_DUMMY * and verifying that upper 5 nybbles are read back. */ - rk_iommu_write(iommu, RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); + for (i = 0; i < iommu->num_mmu; i++) { + rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, DTE_ADDR_DUMMY); - dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); - if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { - dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); - return -EFAULT; + dte_addr = rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR); + if (dte_addr != (DTE_ADDR_DUMMY & RK_DTE_PT_ADDRESS_MASK)) { + dev_err(iommu->dev, "Error during raw reset. MMU_DTE_ADDR is not functioning\n"); + return -EFAULT; + } } rk_iommu_command(iommu, RK_MMU_CMD_FORCE_RESET); - ret = rk_wait_for(rk_iommu_read(iommu, RK_MMU_DTE_ADDR) == 0x00000000, - FORCE_RESET_TIMEOUT); - if (ret) - dev_err(iommu->dev, "FORCE_RESET command timed out\n"); + for (i = 0; i < iommu->num_mmu; i++) { + ret = rk_wait_for(rk_iommu_read(iommu->bases[i], RK_MMU_DTE_ADDR) == 0x00000000, + FORCE_RESET_TIMEOUT); + if (ret) { + dev_err(iommu->dev, "FORCE_RESET command timed out\n"); + return ret; + } + } - return ret; + return 0; } -static void log_iova(struct rk_iommu *iommu, dma_addr_t iova) +static void log_iova(struct rk_iommu *iommu, int index, dma_addr_t iova) { + void __iomem *base = iommu->bases[index]; u32 dte_index, pte_index, page_offset; u32 mmu_dte_addr; phys_addr_t mmu_dte_addr_phys, dte_addr_phys; @@ -425,7 +460,7 @@ static void log_iova(struct rk_iommu *iommu, dma_addr_t iova) pte_index = rk_iova_pte_index(iova); page_offset = rk_iova_page_offset(iova); - mmu_dte_addr = rk_iommu_read(iommu, RK_MMU_DTE_ADDR); + mmu_dte_addr = rk_iommu_read(base, RK_MMU_DTE_ADDR); mmu_dte_addr_phys = (phys_addr_t)mmu_dte_addr; dte_addr_phys = mmu_dte_addr_phys + (4 * dte_index); @@ -460,51 +495,56 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id) u32 status; u32 int_status; dma_addr_t iova; + irqreturn_t ret = IRQ_NONE; + int i; - int_status = rk_iommu_read(iommu, RK_MMU_INT_STATUS); - if (int_status == 0) - return IRQ_NONE; + for (i = 0; i < iommu->num_mmu; i++) { + int_status = rk_iommu_read(iommu->bases[i], RK_MMU_INT_STATUS); + if (int_status == 0) + continue; - iova = rk_iommu_read(iommu, RK_MMU_PAGE_FAULT_ADDR); + ret = IRQ_HANDLED; + iova = rk_iommu_read(iommu->bases[i], RK_MMU_PAGE_FAULT_ADDR); - if (int_status & RK_MMU_IRQ_PAGE_FAULT) { - int flags; + if (int_status & RK_MMU_IRQ_PAGE_FAULT) { + int flags; - status = rk_iommu_read(iommu, RK_MMU_STATUS); - flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? - IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + status = rk_iommu_read(iommu->bases[i], RK_MMU_STATUS); + flags = (status & RK_MMU_STATUS_PAGE_FAULT_IS_WRITE) ? + IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; - dev_err(iommu->dev, "Page fault at %pad of type %s\n", - &iova, - (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); + dev_err(iommu->dev, "Page fault at %pad of type %s\n", + &iova, + (flags == IOMMU_FAULT_WRITE) ? "write" : "read"); - log_iova(iommu, iova); + log_iova(iommu, i, iova); - /* - * Report page fault to any installed handlers. - * Ignore the return code, though, since we always zap cache - * and clear the page fault anyway. - */ - if (iommu->domain) - report_iommu_fault(iommu->domain, iommu->dev, iova, - flags); - else - dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); + /* + * Report page fault to any installed handlers. + * Ignore the return code, though, since we always zap cache + * and clear the page fault anyway. + */ + if (iommu->domain) + report_iommu_fault(iommu->domain, iommu->dev, iova, + flags); + else + dev_err(iommu->dev, "Page fault while iommu not attached to domain?\n"); - rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); - rk_iommu_command(iommu, RK_MMU_CMD_PAGE_FAULT_DONE); - } + rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); + rk_iommu_base_command(iommu->bases[i], RK_MMU_CMD_PAGE_FAULT_DONE); + } - if (int_status & RK_MMU_IRQ_BUS_ERROR) - dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); + if (int_status & RK_MMU_IRQ_BUS_ERROR) + dev_err(iommu->dev, "BUS_ERROR occurred at %pad\n", &iova); - if (int_status & ~RK_MMU_IRQ_MASK) - dev_err(iommu->dev, "unexpected int_status: %#08x\n", - int_status); + if (int_status & ~RK_MMU_IRQ_MASK) + dev_err(iommu->dev, "unexpected int_status: %#08x\n", + int_status); - rk_iommu_write(iommu, RK_MMU_INT_CLEAR, int_status); + rk_iommu_write(iommu->bases[i], RK_MMU_INT_CLEAR, int_status); + } - return IRQ_HANDLED; + return ret; } static phys_addr_t rk_iommu_iova_to_phys(struct iommu_domain *domain, @@ -746,7 +786,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, struct rk_iommu *iommu; struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; - int ret; + int ret, i; phys_addr_t dte_addr; /* @@ -773,9 +813,11 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, return ret; dte_addr = virt_to_phys(rk_domain->dt); - rk_iommu_write(iommu, RK_MMU_DTE_ADDR, dte_addr); - rk_iommu_command(iommu, RK_MMU_CMD_ZAP_CACHE); - rk_iommu_write(iommu, RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); + for (i = 0; i < iommu->num_mmu; i++) { + rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, dte_addr); + rk_iommu_command(iommu->bases[i], RK_MMU_CMD_ZAP_CACHE); + rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, RK_MMU_IRQ_MASK); + } ret = rk_iommu_enable_paging(iommu); if (ret) @@ -798,6 +840,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, struct rk_iommu *iommu; struct rk_iommu_domain *rk_domain = to_rk_domain(domain); unsigned long flags; + int i; /* Allow 'virtual devices' (eg drm) to detach from domain */ iommu = rk_iommu_from_dev(dev); @@ -811,8 +854,10 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, /* Ignore error while disabling, just keep going */ rk_iommu_enable_stall(iommu); rk_iommu_disable_paging(iommu); - rk_iommu_write(iommu, RK_MMU_INT_MASK, 0); - rk_iommu_write(iommu, RK_MMU_DTE_ADDR, 0); + for (i = 0; i < iommu->num_mmu; i++) { + rk_iommu_write(iommu->bases[i], RK_MMU_INT_MASK, 0); + rk_iommu_write(iommu->bases[i], RK_MMU_DTE_ADDR, 0); + } rk_iommu_disable_stall(iommu); devm_free_irq(dev, iommu->irq, iommu); @@ -988,6 +1033,7 @@ static int rk_iommu_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct rk_iommu *iommu; struct resource *res; + int i; iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL); if (!iommu) @@ -995,11 +1041,21 @@ static int rk_iommu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, iommu); iommu->dev = dev; + iommu->num_mmu = 0; + iommu->bases = devm_kzalloc(dev, sizeof(*iommu->bases) * iommu->num_mmu, + GFP_KERNEL); + if (!iommu->bases) + return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - iommu->base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(iommu->base)) - return PTR_ERR(iommu->base); + for (i = 0; i < pdev->num_resources; i++) { + res = platform_get_resource(pdev, IORESOURCE_MEM, i); + iommu->bases[i] = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(iommu->bases[i])) + continue; + iommu->num_mmu++; + } + if (iommu->num_mmu == 0) + return PTR_ERR(iommu->bases[0]); iommu->irq = platform_get_irq(pdev, 0); if (iommu->irq < 0) { -- cgit v0.10.2 From e5fc9753b1a831466be9b45d0bf926eeaa8b84a0 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Jan 2016 17:13:13 +0000 Subject: iommu/io-pgtable: Add ARMv7 short descriptor support Add a nearly-complete ARMv7 short descriptor implementation, omitting only a few legacy and CPU-centric aspects which shouldn't be necessary for IOMMU API use anyway. Reviewed-by: Yong Wu Tested-by: Yong Wu Signed-off-by: Yong Wu Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a1e75cb..dc1aaa5 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -39,6 +39,25 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST If unsure, say N here. +config IOMMU_IO_PGTABLE_ARMV7S + bool "ARMv7/v8 Short Descriptor Format" + select IOMMU_IO_PGTABLE + depends on HAS_DMA && (ARM || ARM64 || COMPILE_TEST) + help + Enable support for the ARM Short-descriptor pagetable format. + This supports 32-bit virtual and physical addresses mapped using + 2-level tables with 4KB pages/1MB sections, and contiguous entries + for 64KB pages/16MB supersections if indicated by the IOMMU driver. + +config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST + bool "ARMv7s selftests" + depends on IOMMU_IO_PGTABLE_ARMV7S + help + Enable self-tests for ARMv7s page table allocator. This performs + a series of page-table consistency checks during boot. + + If unsure, say N here. + endmenu config IOMMU_IOVA diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 42fc0c2..2f9bfbc 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o +obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c new file mode 100644 index 0000000..efc7d1e --- /dev/null +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -0,0 +1,849 @@ +/* + * CPU-agnostic ARM page table allocator. + * + * ARMv7 Short-descriptor format, supporting + * - Basic memory attributes + * - Simplified access permissions (AP[2:1] model) + * - Backwards-compatible TEX remap + * - Large pages/supersections (if indicated by the caller) + * + * Not supporting: + * - Legacy access permissions (AP[2:0] model) + * + * Almost certainly never supporting: + * - PXN + * - Domains + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Copyright (C) 2014-2015 ARM Limited + * Copyright (c) 2014-2015 MediaTek Inc. + */ + +#define pr_fmt(fmt) "arm-v7s io-pgtable: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "io-pgtable.h" + +/* Struct accessors */ +#define io_pgtable_to_data(x) \ + container_of((x), struct arm_v7s_io_pgtable, iop) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +/* + * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2, + * and 12 bits in a page. With some carefully-chosen coefficients we can + * hide the ugly inconsistencies behind these macros and at least let the + * rest of the code pretend to be somewhat sane. + */ +#define ARM_V7S_ADDR_BITS 32 +#define _ARM_V7S_LVL_BITS(lvl) (16 - (lvl) * 4) +#define ARM_V7S_LVL_SHIFT(lvl) (ARM_V7S_ADDR_BITS - (4 + 8 * (lvl))) +#define ARM_V7S_TABLE_SHIFT 10 + +#define ARM_V7S_PTES_PER_LVL(lvl) (1 << _ARM_V7S_LVL_BITS(lvl)) +#define ARM_V7S_TABLE_SIZE(lvl) \ + (ARM_V7S_PTES_PER_LVL(lvl) * sizeof(arm_v7s_iopte)) + +#define ARM_V7S_BLOCK_SIZE(lvl) (1UL << ARM_V7S_LVL_SHIFT(lvl)) +#define ARM_V7S_LVL_MASK(lvl) ((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl))) +#define ARM_V7S_TABLE_MASK ((u32)(~0U << ARM_V7S_TABLE_SHIFT)) +#define _ARM_V7S_IDX_MASK(lvl) (ARM_V7S_PTES_PER_LVL(lvl) - 1) +#define ARM_V7S_LVL_IDX(addr, lvl) ({ \ + int _l = lvl; \ + ((u32)(addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l); \ +}) + +/* + * Large page/supersection entries are effectively a block of 16 page/section + * entries, along the lines of the LPAE contiguous hint, but all with the + * same output address. For want of a better common name we'll call them + * "contiguous" versions of their respective page/section entries here, but + * noting the distinction (WRT to TLB maintenance) that they represent *one* + * entry repeated 16 times, not 16 separate entries (as in the LPAE case). + */ +#define ARM_V7S_CONT_PAGES 16 + +/* PTE type bits: these are all mixed up with XN/PXN bits in most cases */ +#define ARM_V7S_PTE_TYPE_TABLE 0x1 +#define ARM_V7S_PTE_TYPE_PAGE 0x2 +#define ARM_V7S_PTE_TYPE_CONT_PAGE 0x1 + +#define ARM_V7S_PTE_IS_VALID(pte) (((pte) & 0x3) != 0) +#define ARM_V7S_PTE_IS_TABLE(pte, lvl) (lvl == 1 && ((pte) & ARM_V7S_PTE_TYPE_TABLE)) + +/* Page table bits */ +#define ARM_V7S_ATTR_XN(lvl) BIT(4 * (2 - (lvl))) +#define ARM_V7S_ATTR_B BIT(2) +#define ARM_V7S_ATTR_C BIT(3) +#define ARM_V7S_ATTR_NS_TABLE BIT(3) +#define ARM_V7S_ATTR_NS_SECTION BIT(19) + +#define ARM_V7S_CONT_SECTION BIT(18) +#define ARM_V7S_CONT_PAGE_XN_SHIFT 15 + +/* + * The attribute bits are consistently ordered*, but occupy bits [17:10] of + * a level 1 PTE vs. bits [11:4] at level 2. Thus we define the individual + * fields relative to that 8-bit block, plus a total shift relative to the PTE. + */ +#define ARM_V7S_ATTR_SHIFT(lvl) (16 - (lvl) * 6) + +#define ARM_V7S_ATTR_MASK 0xff +#define ARM_V7S_ATTR_AP0 BIT(0) +#define ARM_V7S_ATTR_AP1 BIT(1) +#define ARM_V7S_ATTR_AP2 BIT(5) +#define ARM_V7S_ATTR_S BIT(6) +#define ARM_V7S_ATTR_NG BIT(7) +#define ARM_V7S_TEX_SHIFT 2 +#define ARM_V7S_TEX_MASK 0x7 +#define ARM_V7S_ATTR_TEX(val) (((val) & ARM_V7S_TEX_MASK) << ARM_V7S_TEX_SHIFT) + +/* *well, except for TEX on level 2 large pages, of course :( */ +#define ARM_V7S_CONT_PAGE_TEX_SHIFT 6 +#define ARM_V7S_CONT_PAGE_TEX_MASK (ARM_V7S_TEX_MASK << ARM_V7S_CONT_PAGE_TEX_SHIFT) + +/* Simplified access permissions */ +#define ARM_V7S_PTE_AF ARM_V7S_ATTR_AP0 +#define ARM_V7S_PTE_AP_UNPRIV ARM_V7S_ATTR_AP1 +#define ARM_V7S_PTE_AP_RDONLY ARM_V7S_ATTR_AP2 + +/* Register bits */ +#define ARM_V7S_RGN_NC 0 +#define ARM_V7S_RGN_WBWA 1 +#define ARM_V7S_RGN_WT 2 +#define ARM_V7S_RGN_WB 3 + +#define ARM_V7S_PRRR_TYPE_DEVICE 1 +#define ARM_V7S_PRRR_TYPE_NORMAL 2 +#define ARM_V7S_PRRR_TR(n, type) (((type) & 0x3) << ((n) * 2)) +#define ARM_V7S_PRRR_DS0 BIT(16) +#define ARM_V7S_PRRR_DS1 BIT(17) +#define ARM_V7S_PRRR_NS0 BIT(18) +#define ARM_V7S_PRRR_NS1 BIT(19) +#define ARM_V7S_PRRR_NOS(n) BIT((n) + 24) + +#define ARM_V7S_NMRR_IR(n, attr) (((attr) & 0x3) << ((n) * 2)) +#define ARM_V7S_NMRR_OR(n, attr) (((attr) & 0x3) << ((n) * 2 + 16)) + +#define ARM_V7S_TTBR_S BIT(1) +#define ARM_V7S_TTBR_NOS BIT(5) +#define ARM_V7S_TTBR_ORGN_ATTR(attr) (((attr) & 0x3) << 3) +#define ARM_V7S_TTBR_IRGN_ATTR(attr) \ + ((((attr) & 0x1) << 6) | (((attr) & 0x2) >> 1)) + +#define ARM_V7S_TCR_PD1 BIT(5) + +typedef u32 arm_v7s_iopte; + +static bool selftest_running; + +struct arm_v7s_io_pgtable { + struct io_pgtable iop; + + arm_v7s_iopte *pgd; + struct kmem_cache *l2_tables; +}; + +static dma_addr_t __arm_v7s_dma_addr(void *pages) +{ + return (dma_addr_t)virt_to_phys(pages); +} + +static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl) +{ + if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) + pte &= ARM_V7S_TABLE_MASK; + else + pte &= ARM_V7S_LVL_MASK(lvl); + return phys_to_virt(pte); +} + +static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, + struct arm_v7s_io_pgtable *data) +{ + struct device *dev = data->iop.cfg.iommu_dev; + dma_addr_t dma; + size_t size = ARM_V7S_TABLE_SIZE(lvl); + void *table = NULL; + + if (lvl == 1) + table = (void *)__get_dma_pages(__GFP_ZERO, get_order(size)); + else if (lvl == 2) + table = kmem_cache_zalloc(data->l2_tables, gfp); + if (table && !selftest_running) { + dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + goto out_free; + /* + * We depend on the IOMMU being able to work with any physical + * address directly, so if the DMA layer suggests otherwise by + * translating or truncating them, that bodes very badly... + */ + if (dma != virt_to_phys(table)) + goto out_unmap; + } + kmemleak_ignore(table); + return table; + +out_unmap: + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); +out_free: + if (lvl == 1) + free_pages((unsigned long)table, get_order(size)); + else + kmem_cache_free(data->l2_tables, table); + return NULL; +} + +static void __arm_v7s_free_table(void *table, int lvl, + struct arm_v7s_io_pgtable *data) +{ + struct device *dev = data->iop.cfg.iommu_dev; + size_t size = ARM_V7S_TABLE_SIZE(lvl); + + if (!selftest_running) + dma_unmap_single(dev, __arm_v7s_dma_addr(table), size, + DMA_TO_DEVICE); + if (lvl == 1) + free_pages((unsigned long)table, get_order(size)); + else + kmem_cache_free(data->l2_tables, table); +} + +static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, + struct io_pgtable_cfg *cfg) +{ + if (selftest_running) + return; + + dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), + num_entries * sizeof(*ptep), DMA_TO_DEVICE); +} +static void __arm_v7s_set_pte(arm_v7s_iopte *ptep, arm_v7s_iopte pte, + int num_entries, struct io_pgtable_cfg *cfg) +{ + int i; + + for (i = 0; i < num_entries; i++) + ptep[i] = pte; + + __arm_v7s_pte_sync(ptep, num_entries, cfg); +} + +static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, + struct io_pgtable_cfg *cfg) +{ + bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS); + arm_v7s_iopte pte = ARM_V7S_ATTR_NG | ARM_V7S_ATTR_S | + ARM_V7S_ATTR_TEX(1); + + if (ap) { + pte |= ARM_V7S_PTE_AF | ARM_V7S_PTE_AP_UNPRIV; + if (!(prot & IOMMU_WRITE)) + pte |= ARM_V7S_PTE_AP_RDONLY; + } + pte <<= ARM_V7S_ATTR_SHIFT(lvl); + + if ((prot & IOMMU_NOEXEC) && ap) + pte |= ARM_V7S_ATTR_XN(lvl); + if (prot & IOMMU_CACHE) + pte |= ARM_V7S_ATTR_B | ARM_V7S_ATTR_C; + + return pte; +} + +static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) +{ + int prot = IOMMU_READ; + + if (pte & (ARM_V7S_PTE_AP_RDONLY << ARM_V7S_ATTR_SHIFT(lvl))) + prot |= IOMMU_WRITE; + if (pte & ARM_V7S_ATTR_C) + prot |= IOMMU_CACHE; + + return prot; +} + +static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl) +{ + if (lvl == 1) { + pte |= ARM_V7S_CONT_SECTION; + } else if (lvl == 2) { + arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl); + arm_v7s_iopte tex = pte & ARM_V7S_CONT_PAGE_TEX_MASK; + + pte ^= xn | tex | ARM_V7S_PTE_TYPE_PAGE; + pte |= (xn << ARM_V7S_CONT_PAGE_XN_SHIFT) | + (tex << ARM_V7S_CONT_PAGE_TEX_SHIFT) | + ARM_V7S_PTE_TYPE_CONT_PAGE; + } + return pte; +} + +static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl) +{ + if (lvl == 1) { + pte &= ~ARM_V7S_CONT_SECTION; + } else if (lvl == 2) { + arm_v7s_iopte xn = pte & BIT(ARM_V7S_CONT_PAGE_XN_SHIFT); + arm_v7s_iopte tex = pte & (ARM_V7S_CONT_PAGE_TEX_MASK << + ARM_V7S_CONT_PAGE_TEX_SHIFT); + + pte ^= xn | tex | ARM_V7S_PTE_TYPE_CONT_PAGE; + pte |= (xn >> ARM_V7S_CONT_PAGE_XN_SHIFT) | + (tex >> ARM_V7S_CONT_PAGE_TEX_SHIFT) | + ARM_V7S_PTE_TYPE_PAGE; + } + return pte; +} + +static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl) +{ + if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl)) + return pte & ARM_V7S_CONT_SECTION; + else if (lvl == 2) + return !(pte & ARM_V7S_PTE_TYPE_PAGE); + return false; +} + +static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *, unsigned long, + size_t, int, arm_v7s_iopte *); + +static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, int prot, + int lvl, int num_entries, arm_v7s_iopte *ptep) +{ + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_v7s_iopte pte = arm_v7s_prot_to_pte(prot, lvl, cfg); + int i; + + for (i = 0; i < num_entries; i++) + if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) { + /* + * We need to unmap and free the old table before + * overwriting it with a block entry. + */ + arm_v7s_iopte *tblp; + size_t sz = ARM_V7S_BLOCK_SIZE(lvl); + + tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl); + if (WARN_ON(__arm_v7s_unmap(data, iova + i * sz, + sz, lvl, tblp) != sz)) + return -EINVAL; + } else if (ptep[i]) { + /* We require an unmap first */ + WARN_ON(!selftest_running); + return -EEXIST; + } + + pte |= ARM_V7S_PTE_TYPE_PAGE; + if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) + pte |= ARM_V7S_ATTR_NS_SECTION; + + if (num_entries > 1) + pte = arm_v7s_pte_to_cont(pte, lvl); + + pte |= paddr & ARM_V7S_LVL_MASK(lvl); + + __arm_v7s_set_pte(ptep, pte, num_entries, cfg); + return 0; +} + +static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, + int lvl, arm_v7s_iopte *ptep) +{ + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_v7s_iopte pte, *cptep; + int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); + + /* Find our entry at the current level */ + ptep += ARM_V7S_LVL_IDX(iova, lvl); + + /* If we can install a leaf entry at this level, then do so */ + if (num_entries) + return arm_v7s_init_pte(data, iova, paddr, prot, + lvl, num_entries, ptep); + + /* We can't allocate tables at the final level */ + if (WARN_ON(lvl == 2)) + return -EINVAL; + + /* Grab a pointer to the next level */ + pte = *ptep; + if (!pte) { + cptep = __arm_v7s_alloc_table(lvl + 1, GFP_ATOMIC, data); + if (!cptep) + return -ENOMEM; + + pte = virt_to_phys(cptep) | ARM_V7S_PTE_TYPE_TABLE; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) + pte |= ARM_V7S_ATTR_NS_TABLE; + + __arm_v7s_set_pte(ptep, pte, 1, cfg); + } else { + cptep = iopte_deref(pte, lvl); + } + + /* Rinse, repeat */ + return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep); +} + +static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + const struct iommu_gather_ops *tlb = cfg->tlb; + void *cookie = data->iop.cookie; + int ret; + + /* If no access, then nothing to do */ + if (!(prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + ret = __arm_v7s_map(data, iova, paddr, size, prot, 1, data->pgd); + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + if (cfg->quirks & IO_PGTABLE_QUIRK_TLBI_ON_MAP) { + tlb->tlb_add_flush(iova, size, ARM_V7S_BLOCK_SIZE(2), false, + cookie); + tlb->tlb_sync(cookie); + } else { + wmb(); + } + + return ret; +} + +static void arm_v7s_free_pgtable(struct io_pgtable *iop) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop); + int i; + + for (i = 0; i < ARM_V7S_PTES_PER_LVL(1); i++) { + arm_v7s_iopte pte = data->pgd[i]; + + if (ARM_V7S_PTE_IS_TABLE(pte, 1)) + __arm_v7s_free_table(iopte_deref(pte, 1), 2, data); + } + __arm_v7s_free_table(data->pgd, 1, data); + kmem_cache_destroy(data->l2_tables); + kfree(data); +} + +static void arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, + unsigned long iova, int idx, int lvl, + arm_v7s_iopte *ptep) +{ + struct io_pgtable_cfg *cfg = &data->iop.cfg; + void *cookie = data->iop.cookie; + arm_v7s_iopte pte; + size_t size = ARM_V7S_BLOCK_SIZE(lvl); + int i; + + ptep -= idx & (ARM_V7S_CONT_PAGES - 1); + pte = arm_v7s_cont_to_pte(*ptep, lvl); + for (i = 0; i < ARM_V7S_CONT_PAGES; i++) { + ptep[i] = pte; + pte += size; + } + + __arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, cfg); + + size *= ARM_V7S_CONT_PAGES; + cfg->tlb->tlb_add_flush(iova, size, size, true, cookie); + cfg->tlb->tlb_sync(cookie); +} + +static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, + unsigned long iova, size_t size, + arm_v7s_iopte *ptep) +{ + unsigned long blk_start, blk_end, blk_size; + phys_addr_t blk_paddr; + arm_v7s_iopte table = 0; + struct io_pgtable_cfg *cfg = &data->iop.cfg; + int prot = arm_v7s_pte_to_prot(*ptep, 1); + + blk_size = ARM_V7S_BLOCK_SIZE(1); + blk_start = iova & ARM_V7S_LVL_MASK(1); + blk_end = blk_start + ARM_V7S_BLOCK_SIZE(1); + blk_paddr = *ptep & ARM_V7S_LVL_MASK(1); + + for (; blk_start < blk_end; blk_start += size, blk_paddr += size) { + arm_v7s_iopte *tablep; + + /* Unmap! */ + if (blk_start == iova) + continue; + + /* __arm_v7s_map expects a pointer to the start of the table */ + tablep = &table - ARM_V7S_LVL_IDX(blk_start, 1); + if (__arm_v7s_map(data, blk_start, blk_paddr, size, prot, 1, + tablep) < 0) { + if (table) { + /* Free the table we allocated */ + tablep = iopte_deref(table, 1); + __arm_v7s_free_table(tablep, 2, data); + } + return 0; /* Bytes unmapped */ + } + } + + __arm_v7s_set_pte(ptep, table, 1, cfg); + iova &= ~(blk_size - 1); + cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie); + return size; +} + +static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_v7s_iopte *ptep) +{ + arm_v7s_iopte pte[ARM_V7S_CONT_PAGES]; + struct io_pgtable_cfg *cfg = &data->iop.cfg; + const struct iommu_gather_ops *tlb = cfg->tlb; + void *cookie = data->iop.cookie; + int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); + + /* Something went horribly wrong and we ran out of page table */ + if (WARN_ON(lvl > 2)) + return 0; + + idx = ARM_V7S_LVL_IDX(iova, lvl); + ptep += idx; + do { + if (WARN_ON(!ARM_V7S_PTE_IS_VALID(ptep[i]))) + return 0; + pte[i] = ptep[i]; + } while (++i < num_entries); + + /* + * If we've hit a contiguous 'large page' entry at this level, it + * needs splitting first, unless we're unmapping the whole lot. + */ + if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) + arm_v7s_split_cont(data, iova, idx, lvl, ptep); + + /* If the size matches this level, we're in the right place */ + if (num_entries) { + size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl); + + __arm_v7s_set_pte(ptep, 0, num_entries, cfg); + + for (i = 0; i < num_entries; i++) { + if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) { + /* Also flush any partial walks */ + tlb->tlb_add_flush(iova, blk_size, + ARM_V7S_BLOCK_SIZE(lvl + 1), + false, cookie); + tlb->tlb_sync(cookie); + ptep = iopte_deref(pte[i], lvl); + __arm_v7s_free_table(ptep, lvl + 1, data); + } else { + tlb->tlb_add_flush(iova, blk_size, blk_size, + true, cookie); + } + iova += blk_size; + } + return size; + } else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) { + /* + * Insert a table at the next level to map the old region, + * minus the part we want to unmap + */ + return arm_v7s_split_blk_unmap(data, iova, size, ptep); + } + + /* Keep on walkin' */ + ptep = iopte_deref(pte[0], lvl); + return __arm_v7s_unmap(data, iova, size, lvl + 1, ptep); +} + +static int arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) +{ + size_t unmapped; + struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable *iop = &data->iop; + + unmapped = __arm_v7s_unmap(data, iova, size, 1, data->pgd); + if (unmapped) + iop->cfg.tlb->tlb_sync(iop->cookie); + + return unmapped; +} + +static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_v7s_iopte *ptep = data->pgd, pte; + int lvl = 0; + u32 mask; + + do { + pte = ptep[ARM_V7S_LVL_IDX(iova, ++lvl)]; + ptep = iopte_deref(pte, lvl); + } while (ARM_V7S_PTE_IS_TABLE(pte, lvl)); + + if (!ARM_V7S_PTE_IS_VALID(pte)) + return 0; + + mask = ARM_V7S_LVL_MASK(lvl); + if (arm_v7s_pte_is_cont(pte, lvl)) + mask *= ARM_V7S_CONT_PAGES; + return (pte & mask) | (iova & ~mask); +} + +static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, + void *cookie) +{ + struct arm_v7s_io_pgtable *data; + + if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS) + return NULL; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2", + ARM_V7S_TABLE_SIZE(2), + ARM_V7S_TABLE_SIZE(2), + SLAB_CACHE_DMA, NULL); + if (!data->l2_tables) + goto out_free_data; + + data->iop.ops = (struct io_pgtable_ops) { + .map = arm_v7s_map, + .unmap = arm_v7s_unmap, + .iova_to_phys = arm_v7s_iova_to_phys, + }; + + /* We have to do this early for __arm_v7s_alloc_table to work... */ + data->iop.cfg = *cfg; + + /* + * Unless the IOMMU driver indicates supersection support by + * having SZ_16M set in the initial bitmap, they won't be used. + */ + cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M; + + /* TCR: T0SZ=0, disable TTBR1 */ + cfg->arm_v7s_cfg.tcr = ARM_V7S_TCR_PD1; + + /* + * TEX remap: the indices used map to the closest equivalent types + * under the non-TEX-remap interpretation of those attribute bits, + * excepting various implementation-defined aspects of shareability. + */ + cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) | + ARM_V7S_PRRR_TR(4, ARM_V7S_PRRR_TYPE_NORMAL) | + ARM_V7S_PRRR_TR(7, ARM_V7S_PRRR_TYPE_NORMAL) | + ARM_V7S_PRRR_DS0 | ARM_V7S_PRRR_DS1 | + ARM_V7S_PRRR_NS1 | ARM_V7S_PRRR_NOS(7); + cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) | + ARM_V7S_NMRR_OR(7, ARM_V7S_RGN_WBWA); + + /* Looking good; allocate a pgd */ + data->pgd = __arm_v7s_alloc_table(1, GFP_KERNEL, data); + if (!data->pgd) + goto out_free_data; + + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); + + /* TTBRs */ + cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) | + ARM_V7S_TTBR_S | ARM_V7S_TTBR_NOS | + ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA); + cfg->arm_v7s_cfg.ttbr[1] = 0; + return &data->iop; + +out_free_data: + kmem_cache_destroy(data->l2_tables); + kfree(data); + return NULL; +} + +struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns = { + .alloc = arm_v7s_alloc_pgtable, + .free = arm_v7s_free_pgtable, +}; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S_SELFTEST + +static struct io_pgtable_cfg *cfg_cookie; + +static void dummy_tlb_flush_all(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_tlb_add_flush(unsigned long iova, size_t size, + size_t granule, bool leaf, void *cookie) +{ + WARN_ON(cookie != cfg_cookie); + WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); +} + +static void dummy_tlb_sync(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static struct iommu_gather_ops dummy_tlb_ops = { + .tlb_flush_all = dummy_tlb_flush_all, + .tlb_add_flush = dummy_tlb_add_flush, + .tlb_sync = dummy_tlb_sync, +}; + +#define __FAIL(ops) ({ \ + WARN(1, "selftest: test failed\n"); \ + selftest_running = false; \ + -EFAULT; \ +}) + +static int __init arm_v7s_do_selftests(void) +{ + struct io_pgtable_ops *ops; + struct io_pgtable_cfg cfg = { + .tlb = &dummy_tlb_ops, + .oas = 32, + .ias = 32, + .quirks = IO_PGTABLE_QUIRK_ARM_NS, + .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, + }; + unsigned int iova, size, iova_start; + unsigned int i, loopnr = 0; + + selftest_running = true; + + cfg_cookie = &cfg; + + ops = alloc_io_pgtable_ops(ARM_V7S, &cfg, &cfg); + if (!ops) { + pr_err("selftest: failed to allocate io pgtable ops\n"); + return -EINVAL; + } + + /* + * Initial sanity checks. + * Empty page tables shouldn't provide any translations. + */ + if (ops->iova_to_phys(ops, 42)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, SZ_1G + 42)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, SZ_2G + 42)) + return __FAIL(ops); + + /* + * Distinct mappings of different granule sizes. + */ + iova = 0; + i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG); + while (i != BITS_PER_LONG) { + size = 1UL << i; + if (ops->map(ops, iova, iova, size, IOMMU_READ | + IOMMU_WRITE | + IOMMU_NOEXEC | + IOMMU_CACHE)) + return __FAIL(ops); + + /* Overlapping mappings */ + if (!ops->map(ops, iova, iova + size, size, + IOMMU_READ | IOMMU_NOEXEC)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops); + + iova += SZ_16M; + i++; + i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i); + loopnr++; + } + + /* Partial unmap */ + i = 1; + size = 1UL << __ffs(cfg.pgsize_bitmap); + while (i < loopnr) { + iova_start = i * SZ_16M; + if (ops->unmap(ops, iova_start + size, size) != size) + return __FAIL(ops); + + /* Remap of partial unmap */ + if (ops->map(ops, iova_start + size, size, size, IOMMU_READ)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova_start + size + 42) + != (size + 42)) + return __FAIL(ops); + i++; + } + + /* Full unmap */ + iova = 0; + i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG); + while (i != BITS_PER_LONG) { + size = 1UL << i; + + if (ops->unmap(ops, iova, size) != size) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova + 42)) + return __FAIL(ops); + + /* Remap full block */ + if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops); + + iova += SZ_16M; + i++; + i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i); + } + + free_io_pgtable_ops(ops); + + selftest_running = false; + + pr_info("self test ok\n"); + return 0; +} +subsys_initcall(arm_v7s_do_selftests); +#endif diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 6f2e319..8c615b7 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -33,6 +33,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, #endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S + [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, +#endif }; struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 36673c8..aa57073 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -1,5 +1,6 @@ #ifndef __IO_PGTABLE_H #define __IO_PGTABLE_H +#include /* * Public API for use by IOMMU drivers @@ -9,6 +10,7 @@ enum io_pgtable_fmt { ARM_32_LPAE_S2, ARM_64_LPAE_S1, ARM_64_LPAE_S2, + ARM_V7S, IO_PGTABLE_NUM_FMTS, }; @@ -45,7 +47,9 @@ struct iommu_gather_ops { * page table walker. */ struct io_pgtable_cfg { - #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */ + #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) /* Set NS bit in PTEs */ + #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) /* No AP/XN bits */ + #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) /* TLB Inv. on map */ int quirks; unsigned long pgsize_bitmap; unsigned int ias; @@ -65,6 +69,13 @@ struct io_pgtable_cfg { u64 vttbr; u64 vtcr; } arm_lpae_s2_cfg; + + struct { + u32 ttbr[2]; + u32 tcr; + u32 nmrr; + u32 prrr; + } arm_v7s_cfg; }; }; @@ -149,5 +160,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; #endif /* __IO_PGTABLE_H */ -- cgit v0.10.2 From 507e4c9d194360d07f9cc9ca689a17eb14e370a2 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Jan 2016 17:13:14 +0000 Subject: iommu/io-pgtable: Add helper functions for TLB ops Add some simple wrappers to avoid having the guts of the TLB operations spilled all over the page table implementations, and to provide a point to implement extra common functionality. Acked-by: Will Deacon Acked-by: Laurent Pinchart Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index efc7d1e..d39a021 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -417,9 +417,7 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); - struct io_pgtable_cfg *cfg = &data->iop.cfg; - const struct iommu_gather_ops *tlb = cfg->tlb; - void *cookie = data->iop.cookie; + struct io_pgtable *iop = &data->iop; int ret; /* If no access, then nothing to do */ @@ -431,10 +429,10 @@ static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, * Synchronise all PTE updates for the new mapping before there's * a chance for anything to kick off a table walk for the new iova. */ - if (cfg->quirks & IO_PGTABLE_QUIRK_TLBI_ON_MAP) { - tlb->tlb_add_flush(iova, size, ARM_V7S_BLOCK_SIZE(2), false, - cookie); - tlb->tlb_sync(cookie); + if (iop->cfg.quirks & IO_PGTABLE_QUIRK_TLBI_ON_MAP) { + io_pgtable_tlb_add_flush(iop, iova, size, + ARM_V7S_BLOCK_SIZE(2), false); + io_pgtable_tlb_sync(iop); } else { wmb(); } @@ -462,8 +460,7 @@ static void arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, unsigned long iova, int idx, int lvl, arm_v7s_iopte *ptep) { - struct io_pgtable_cfg *cfg = &data->iop.cfg; - void *cookie = data->iop.cookie; + struct io_pgtable *iop = &data->iop; arm_v7s_iopte pte; size_t size = ARM_V7S_BLOCK_SIZE(lvl); int i; @@ -475,11 +472,11 @@ static void arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, pte += size; } - __arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, cfg); + __arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg); size *= ARM_V7S_CONT_PAGES; - cfg->tlb->tlb_add_flush(iova, size, size, true, cookie); - cfg->tlb->tlb_sync(cookie); + io_pgtable_tlb_add_flush(iop, iova, size, size, true); + io_pgtable_tlb_sync(iop); } static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, @@ -489,7 +486,6 @@ static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, unsigned long blk_start, blk_end, blk_size; phys_addr_t blk_paddr; arm_v7s_iopte table = 0; - struct io_pgtable_cfg *cfg = &data->iop.cfg; int prot = arm_v7s_pte_to_prot(*ptep, 1); blk_size = ARM_V7S_BLOCK_SIZE(1); @@ -517,9 +513,9 @@ static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, } } - __arm_v7s_set_pte(ptep, table, 1, cfg); + __arm_v7s_set_pte(ptep, table, 1, &data->iop.cfg); iova &= ~(blk_size - 1); - cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie); + io_pgtable_tlb_add_flush(&data->iop, iova, blk_size, blk_size, true); return size; } @@ -528,9 +524,7 @@ static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, arm_v7s_iopte *ptep) { arm_v7s_iopte pte[ARM_V7S_CONT_PAGES]; - struct io_pgtable_cfg *cfg = &data->iop.cfg; - const struct iommu_gather_ops *tlb = cfg->tlb; - void *cookie = data->iop.cookie; + struct io_pgtable *iop = &data->iop; int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); /* Something went horribly wrong and we ran out of page table */ @@ -556,20 +550,19 @@ static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, if (num_entries) { size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl); - __arm_v7s_set_pte(ptep, 0, num_entries, cfg); + __arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg); for (i = 0; i < num_entries; i++) { if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) { /* Also flush any partial walks */ - tlb->tlb_add_flush(iova, blk_size, - ARM_V7S_BLOCK_SIZE(lvl + 1), - false, cookie); - tlb->tlb_sync(cookie); + io_pgtable_tlb_add_flush(iop, iova, blk_size, + ARM_V7S_BLOCK_SIZE(lvl + 1), false); + io_pgtable_tlb_sync(iop); ptep = iopte_deref(pte[i], lvl); __arm_v7s_free_table(ptep, lvl + 1, data); } else { - tlb->tlb_add_flush(iova, blk_size, blk_size, - true, cookie); + io_pgtable_tlb_add_flush(iop, iova, blk_size, + blk_size, true); } iova += blk_size; } @@ -590,13 +583,12 @@ static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, static int arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t size) { - size_t unmapped; struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); - struct io_pgtable *iop = &data->iop; + size_t unmapped; unmapped = __arm_v7s_unmap(data, iova, size, 1, data->pgd); if (unmapped) - iop->cfg.tlb->tlb_sync(iop->cookie); + io_pgtable_tlb_sync(&data->iop); return unmapped; } diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 381ca5a..b2bbc84 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -446,7 +446,6 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, unsigned long blk_start, blk_end; phys_addr_t blk_paddr; arm_lpae_iopte table = 0; - struct io_pgtable_cfg *cfg = &data->iop.cfg; blk_start = iova & ~(blk_size - 1); blk_end = blk_start + blk_size; @@ -472,9 +471,9 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, } } - __arm_lpae_set_pte(ptep, table, cfg); + __arm_lpae_set_pte(ptep, table, &data->iop.cfg); iova &= ~(blk_size - 1); - cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie); + io_pgtable_tlb_add_flush(&data->iop, iova, blk_size, blk_size, true); return size; } @@ -483,8 +482,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, arm_lpae_iopte *ptep) { arm_lpae_iopte pte; - const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; - void *cookie = data->iop.cookie; + struct io_pgtable *iop = &data->iop; size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); /* Something went horribly wrong and we ran out of page table */ @@ -498,17 +496,17 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* If the size matches this level, we're in the right place */ if (size == blk_size) { - __arm_lpae_set_pte(ptep, 0, &data->iop.cfg); + __arm_lpae_set_pte(ptep, 0, &iop->cfg); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ - tlb->tlb_add_flush(iova, size, ARM_LPAE_GRANULE(data), - false, cookie); - tlb->tlb_sync(cookie); + io_pgtable_tlb_add_flush(iop, iova, size, + ARM_LPAE_GRANULE(data), false); + io_pgtable_tlb_sync(iop); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); } else { - tlb->tlb_add_flush(iova, size, size, true, cookie); + io_pgtable_tlb_add_flush(iop, iova, size, size, true); } return size; @@ -532,13 +530,12 @@ static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, { size_t unmapped; struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); - struct io_pgtable *iop = &data->iop; arm_lpae_iopte *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); if (unmapped) - iop->cfg.tlb->tlb_sync(iop->cookie); + io_pgtable_tlb_sync(&data->iop); return unmapped; } diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 8c615b7..876f6a7 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -75,6 +75,6 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) return; iop = container_of(ops, struct io_pgtable, ops); - iop->cfg.tlb->tlb_flush_all(iop->cookie); + io_pgtable_tlb_flush_all(iop); io_pgtable_init_table[iop->fmt]->free(iop); } diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index aa57073..95c5565 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -144,6 +144,22 @@ struct io_pgtable { #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) +static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) +{ + iop->cfg.tlb->tlb_flush_all(iop->cookie); +} + +static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, + unsigned long iova, size_t size, size_t granule, bool leaf) +{ + iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); +} + +static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) +{ + iop->cfg.tlb->tlb_sync(iop->cookie); +} + /** * struct io_pgtable_init_fns - Alloc/free a set of page tables for a * particular format. -- cgit v0.10.2 From 88492a4700360a086e55d8874ad786105a5e8b0f Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Jan 2016 17:13:15 +0000 Subject: iommu/io-pgtable: Avoid redundant TLB syncs In certain unmapping situations it is quite possible to end up issuing back-to-back TLB synchronisations, which at best is a waste of time and effort, and at worst causes some hardware to get rather confused. Whilst the pagetable implementations, or the IOMMU drivers, or both, could keep track of things to avoid this happening, it seems to make the most sense to prevent code duplication and add some simple state tracking in the common interface between the two. Reviewed-by: Laurent Pinchart Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 95c5565..4faee7d 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -132,12 +132,14 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops); * @fmt: The page table format. * @cookie: An opaque token provided by the IOMMU driver and passed back to * any callback routines. + * @tlb_sync_pending: Private flag for optimising out redundant syncs. * @cfg: A copy of the page table configuration. * @ops: The page table operations in use for this set of page tables. */ struct io_pgtable { enum io_pgtable_fmt fmt; void *cookie; + bool tlb_sync_pending; struct io_pgtable_cfg cfg; struct io_pgtable_ops ops; }; @@ -147,17 +149,22 @@ struct io_pgtable { static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) { iop->cfg.tlb->tlb_flush_all(iop->cookie); + iop->tlb_sync_pending = true; } static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, unsigned long iova, size_t size, size_t granule, bool leaf) { iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); + iop->tlb_sync_pending = true; } static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) { - iop->cfg.tlb->tlb_sync(iop->cookie); + if (iop->tlb_sync_pending) { + iop->cfg.tlb->tlb_sync(iop->cookie); + iop->tlb_sync_pending = false; + } } /** -- cgit v0.10.2 From 3850db49da87121b54c3ba43c82cdbe5a7c76be5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Fri, 12 Feb 2016 17:09:46 +0000 Subject: iommu/io-pgtable: Rationalise quirk handling As the number of io-pgtable implementations grows beyond 1, it's time to rationalise the quirks mechanism before things have a chance to start getting really ugly and out-of-hand. To that end: - Indicate exactly which quirks each format can/does support. - Fail creating a table if a caller wants unsupported quirks. - Properly document where each quirk applies and why. Reviewed-by: Laurent Pinchart Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index d39a021..9fcceb1 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -623,6 +623,11 @@ static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS) return NULL; + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | + IO_PGTABLE_QUIRK_NO_PERMS | + IO_PGTABLE_QUIRK_TLBI_ON_MAP)) + return NULL; + data = kmalloc(sizeof(*data), GFP_KERNEL); if (!data) return NULL; diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index b2bbc84..f433b51 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -659,8 +659,12 @@ static struct io_pgtable * arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg; - struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); + struct arm_lpae_io_pgtable *data; + + if (cfg->quirks & ~IO_PGTABLE_QUIRK_ARM_NS) + return NULL; + data = arm_lpae_alloc_pgtable(cfg); if (!data) return NULL; @@ -743,8 +747,13 @@ static struct io_pgtable * arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg, sl; - struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); + struct arm_lpae_io_pgtable *data; + + /* The NS quirk doesn't apply at stage 2 */ + if (cfg->quirks) + return NULL; + data = arm_lpae_alloc_pgtable(cfg); if (!data) return NULL; diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 4faee7d..d4f5027 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -47,10 +47,24 @@ struct iommu_gather_ops { * page table walker. */ struct io_pgtable_cfg { - #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) /* Set NS bit in PTEs */ - #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) /* No AP/XN bits */ - #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) /* TLB Inv. on map */ - int quirks; + /* + * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in + * stage 1 PTEs, for hardware which insists on validating them + * even in non-secure state where they should normally be ignored. + * + * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and + * IOMMU_NOEXEC flags and map everything with full access, for + * hardware which does not implement the permissions of a given + * format, and/or requires some format-specific default value. + * + * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid + * (unmapped) entries but the hardware might do so anyway, perform + * TLB maintenance when mapping as well as when unmapping. + */ + #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) + #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) + #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) + unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; unsigned int oas; -- cgit v0.10.2 From c8bff3a68f33b94e53f65aa9a4c0e9be34b6b514 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 17 Feb 2016 14:18:00 +0000 Subject: MAINTAINERS: update ARM SMMU entry Ensure that the short-descriptor page table code is included under the SMMU entry, and add Robin as a designated reviewer. Acked-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/MAINTAINERS b/MAINTAINERS index cc2f753..a01bae1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1793,11 +1793,13 @@ F: drivers/edac/synopsys_edac.c ARM SMMU DRIVERS M: Will Deacon +R: Robin Murphy L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained F: drivers/iommu/arm-smmu.c F: drivers/iommu/arm-smmu-v3.c F: drivers/iommu/io-pgtable-arm.c +F: drivers/iommu/io-pgtable-arm-v7s.c ARM64 PORT (AARCH64 ARCHITECTURE) M: Catalin Marinas -- cgit v0.10.2 From d346180e70b91b3d5a1ae7e5603e65593d4622bc Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Jan 2016 18:06:34 +0000 Subject: iommu/arm-smmu: Treat all device transactions as unprivileged The IOMMU API has no concept of privilege so assumes all devices and mappings are equal, and indeed most non-CPU master devices on an AMBA interconnect make little use of the attribute bits on the bus thus by default perform unprivileged data accesses. Some devices, however, believe themselves more equal than others, such as programmable DMA controllers whose 'master' thread issues bus transactions marked as privileged instruction fetches, while the data accesses of its channel threads (under the control of Linux, at least) are marked as unprivileged. This poses a problem for implementing the DMA API on an IOMMU conforming to ARM VMSAv8, under which a page that is unprivileged-writeable is also implicitly privileged-execute-never. Given that, there is no one set of attributes with which iommu_map() can implement, say, dma_alloc_coherent() that will allow every possible type of access without something running into unexecepted permission faults. Fortunately the SMMU architecture provides a means to mitigate such issues by overriding the incoming attributes of a transaction; make use of that to strip the privileged/unprivileged status off incoming transactions, leaving just the instruction/data dichotomy which the IOMMU API does at least understand; Four states good, two states better. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 59ee4b8..1f9093d 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -167,6 +167,9 @@ #define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT) #define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT) +#define S2CR_PRIVCFG_SHIFT 24 +#define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT) + /* Context bank attribute registers */ #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) #define CBAR_VMID_SHIFT 0 @@ -1083,7 +1086,7 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, u32 idx, s2cr; idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; - s2cr = S2CR_TYPE_TRANS | + s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV | (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT); writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); } -- cgit v0.10.2 From 9adb95949a343dac53b1cd81dc973b5f815c88d4 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 26 Jan 2016 18:06:36 +0000 Subject: iommu/arm-smmu: Support DMA-API domains With DMA mapping ops provided by the iommu-dma code, only a minimal contribution from the IOMMU driver is needed to create a suitable DMA-API domain for them to use. Implement this for the ARM SMMUs. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 2087534..f003b3a 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -21,6 +21,7 @@ */ #include +#include #include #include #include @@ -1396,7 +1397,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { struct arm_smmu_domain *smmu_domain; - if (type != IOMMU_DOMAIN_UNMANAGED) + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; /* @@ -1408,6 +1409,12 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) if (!smmu_domain) return NULL; + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&smmu_domain->domain)) { + kfree(smmu_domain); + return NULL; + } + mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); return &smmu_domain->domain; @@ -1436,6 +1443,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; + iommu_put_dma_cookie(domain); free_io_pgtable_ops(smmu_domain->pgtbl_ops); /* Free the CD and ASID, if we allocated them */ diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 1f9093d..e8e7bcc 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -29,6 +29,7 @@ #define pr_fmt(fmt) "arm-smmu: " fmt #include +#include #include #include #include @@ -966,7 +967,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { struct arm_smmu_domain *smmu_domain; - if (type != IOMMU_DOMAIN_UNMANAGED) + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; /* * Allocate the domain and initialise some of its data structures. @@ -977,6 +978,12 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) if (!smmu_domain) return NULL; + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&smmu_domain->domain)) { + kfree(smmu_domain); + return NULL; + } + mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); @@ -991,6 +998,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) * Free the domain resources. We assume that all devices have * already been detached. */ + iommu_put_dma_cookie(domain); arm_smmu_destroy_domain_context(domain); kfree(smmu_domain); } -- cgit v0.10.2 From 45bb966d3dc7c6d4f0bd7673b5fe51deebcaf70b Mon Sep 17 00:00:00 2001 From: Anup Patel Date: Wed, 27 Jan 2016 10:51:16 +0530 Subject: of: iommu: Increment DT node refcount in of_iommu_set_ops() We are saving pointer to iommu DT node in of_iommu_set_ops() hence we should increment DT node ref count. Reviewed-by: Ray Jui Reviewed-by: Scott Branden Reviewed-by: Robin Murphy Signed-off-by: Anup Patel Signed-off-by: Will Deacon diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 60ba238..5fea665 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -110,6 +110,7 @@ void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) if (WARN_ON(!iommu)) return; + of_node_get(np); INIT_LIST_HEAD(&iommu->list); iommu->np = np; iommu->ops = ops; -- cgit v0.10.2 From 25a1c96cd22c949b50d6f0269c23e1c5cb55e5a5 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 10 Feb 2016 14:25:33 +0000 Subject: iommu/arm-smmu: Allow disabling unmatched stream bypass Borrow the disable_bypass parameter from the SMMUv3 driver as a handy debugging/security feature so that unmatched stream IDs (i.e. devices not attached to an IOMMU domain) may be configured to fault. Rather than introduce unsightly inconsistency, or repeat the existing unnecessary use of module_param_named(), fix that as well in passing. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index e8e7bcc..7012531 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -261,9 +261,13 @@ #define FSYNR0_WNR (1 << 4) static int force_stage; -module_param_named(force_stage, force_stage, int, S_IRUGO); +module_param(force_stage, int, S_IRUGO); MODULE_PARM_DESC(force_stage, "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); +static bool disable_bypass; +module_param(disable_bypass, bool, S_IRUGO); +MODULE_PARM_DESC(disable_bypass, + "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); enum arm_smmu_arch_version { ARM_SMMU_V1 = 1, @@ -1119,9 +1123,9 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, */ for (i = 0; i < cfg->num_streamids; ++i) { u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; + u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS; - writel_relaxed(S2CR_TYPE_BYPASS, - gr0_base + ARM_SMMU_GR0_S2CR(idx)); + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx)); } arm_smmu_master_free_smrs(smmu, cfg); @@ -1484,11 +1488,11 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); - /* Mark all SMRn as invalid and all S2CRn as bypass */ + /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */ + reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS; for (i = 0; i < smmu->num_mapping_groups; ++i) { writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i)); - writel_relaxed(S2CR_TYPE_BYPASS, - gr0_base + ARM_SMMU_GR0_S2CR(i)); + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i)); } /* Make sure all context banks are disabled and clear CB_FSR */ @@ -1510,8 +1514,12 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Disable TLB broadcasting. */ reg |= (sCR0_VMIDPNE | sCR0_PTM); - /* Enable client access, but bypass when no mapping is found */ - reg &= ~(sCR0_CLIENTPD | sCR0_USFCFG); + /* Enable client access, handling unmatched streams as appropriate */ + reg &= ~sCR0_CLIENTPD; + if (disable_bypass) + reg |= sCR0_USFCFG; + else + reg &= ~sCR0_USFCFG; /* Disable forced broadcasting */ reg &= ~sCR0_FB; -- cgit v0.10.2 From bc7f2ce0a7b54ba7703f81995fe434f0926424d2 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 17 Feb 2016 17:41:57 +0000 Subject: iommu/arm-smmu: Don't fail device attach if already attached to a domain The ARM SMMU attach_dev implementations returns -EEXIST if the device being attached is already attached to a domain. This doesn't play nicely with the default domain, resulting in splats such as: WARNING: at drivers/iommu/iommu.c:1257 Modules linked in: CPU: 3 PID: 1939 Comm: virtio-net-tx Tainted: G S 4.5.0-rc4+ #1 Hardware name: FVP Base (DT) task: ffffffc87a9d0000 ti: ffffffc07a278000 task.ti: ffffffc07a278000 PC is at __iommu_detach_group+0x68/0xe8 LR is at __iommu_detach_group+0x48/0xe8 This patch fixes the problem by forcefully detaching the device from its old domain, if present, when attaching to a new one. The unused ->detach_dev callback is also removed the iommu_ops structures. Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index f003b3a..d96d45d 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1638,6 +1638,17 @@ static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group) return 0; } +static void arm_smmu_detach_dev(struct device *dev) +{ + struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); + + smmu_group->ste.bypass = true; + if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group))) + dev_warn(dev, "failed to install bypass STE\n"); + + smmu_group->domain = NULL; +} + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; @@ -1650,7 +1661,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) /* Already attached to a different domain? */ if (smmu_group->domain && smmu_group->domain != smmu_domain) - return -EEXIST; + arm_smmu_detach_dev(dev); smmu = smmu_group->smmu; mutex_lock(&smmu_domain->init_mutex); @@ -1687,25 +1698,6 @@ out_unlock: return ret; } -static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); - - BUG_ON(!smmu_domain); - BUG_ON(!smmu_group); - - mutex_lock(&smmu_domain->init_mutex); - BUG_ON(smmu_group->domain != smmu_domain); - - smmu_group->ste.bypass = true; - if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group))) - dev_warn(dev, "failed to install bypass STE\n"); - - smmu_group->domain = NULL; - mutex_unlock(&smmu_domain->init_mutex); -} - static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -1943,7 +1935,6 @@ static struct iommu_ops arm_smmu_ops = { .domain_alloc = arm_smmu_domain_alloc, .domain_free = arm_smmu_domain_free, .attach_dev = arm_smmu_attach_dev, - .detach_dev = arm_smmu_detach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, .iova_to_phys = arm_smmu_iova_to_phys, diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 7012531..f21332a 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1131,6 +1131,16 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, arm_smmu_master_free_smrs(smmu, cfg); } +static void arm_smmu_detach_dev(struct device *dev, + struct arm_smmu_master_cfg *cfg) +{ + struct iommu_domain *domain = dev->archdata.iommu; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + dev->archdata.iommu = NULL; + arm_smmu_domain_remove_master(smmu_domain, cfg); +} + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; @@ -1144,11 +1154,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -ENXIO; } - if (dev->archdata.iommu) { - dev_err(dev, "already attached to IOMMU domain\n"); - return -EEXIST; - } - /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (IS_ERR_VALUE(ret)) @@ -1170,25 +1175,16 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (!cfg) return -ENODEV; + /* Detach the dev from its current domain */ + if (dev->archdata.iommu) + arm_smmu_detach_dev(dev, cfg); + ret = arm_smmu_domain_add_master(smmu_domain, cfg); if (!ret) dev->archdata.iommu = domain; return ret; } -static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_master_cfg *cfg; - - cfg = find_smmu_master_cfg(dev); - if (!cfg) - return; - - dev->archdata.iommu = NULL; - arm_smmu_domain_remove_master(smmu_domain, cfg); -} - static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -1464,7 +1460,6 @@ static struct iommu_ops arm_smmu_ops = { .domain_alloc = arm_smmu_domain_alloc, .domain_free = arm_smmu_domain_free, .attach_dev = arm_smmu_attach_dev, - .detach_dev = arm_smmu_detach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, .map_sg = default_iommu_map_sg, -- cgit v0.10.2 From cbf8277ef4562075eb4e3932ccd60cc11ee9454c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 18 Feb 2016 12:05:57 +0000 Subject: iommu/arm-smmu: Treat IOMMU_DOMAIN_DMA as bypass for now Until all upstream devices have their DMA ops swizzled to point at the SMMU, we need to treat the IOMMU_DOMAIN_DMA domain as bypass to avoid putting devices into an empty address space when detaching from VFIO. Signed-off-by: Will Deacon diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index d96d45d..4ff73ff 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -1687,7 +1687,12 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) goto out_unlock; smmu_group->domain = smmu_domain; - smmu_group->ste.bypass = false; + + /* + * FIXME: This should always be "false" once we have IOMMU-backed + * DMA ops for all devices behind the SMMU. + */ + smmu_group->ste.bypass = domain->type == IOMMU_DOMAIN_DMA; ret = arm_smmu_install_ste_for_group(smmu_group); if (IS_ERR_VALUE(ret)) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index f21332a..2409e3b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -1094,6 +1094,13 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, if (ret) return ret == -EEXIST ? 0 : ret; + /* + * FIXME: This won't be needed once we have IOMMU-backed DMA ops + * for all devices behind the SMMU. + */ + if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA) + return 0; + for (i = 0; i < cfg->num_streamids; ++i) { u32 idx, s2cr; -- cgit v0.10.2 From 06bfcaa91f0eff1aeab37c74056afa79fb450aea Mon Sep 17 00:00:00 2001 From: Yoshihiro Shimoda Date: Wed, 10 Feb 2016 10:18:04 +0900 Subject: iommu: Fix second argument of trace_map() to report correct paddr Since iommu_map() code added pgsize value to the paddr, trace_map() used wrong paddr. So, this patch adds "orig_paddr" value in the iommu_map() to use for the trace_map(). Signed-off-by: Yoshihiro Shimoda Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 0e3b009..bfd4f7c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1314,6 +1314,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; + phys_addr_t orig_paddr = paddr; int ret = 0; if (unlikely(domain->ops->map == NULL || @@ -1358,7 +1359,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, if (ret) iommu_unmap(domain, orig_iova, orig_size - size); else - trace_map(orig_iova, paddr, orig_size); + trace_map(orig_iova, orig_paddr, orig_size); return ret; } -- cgit v0.10.2 From 6c2ae7e2916b028bd4c0fff04589bd176487a122 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:48 +0100 Subject: iommu/exynos: Rework iommu group initialization This patch replaces custom code in add_device implementation with iommu_group_get_for_dev() call and provides the needed callback. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 97c41b8..4fc0790 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1114,28 +1114,32 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, return phys; } +static struct iommu_group *get_device_iommu_group(struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_get(dev); + if (!group) + group = iommu_group_alloc(); + + return group; +} + static int exynos_iommu_add_device(struct device *dev) { struct iommu_group *group; - int ret; if (!has_sysmmu(dev)) return -ENODEV; - group = iommu_group_get(dev); + group = iommu_group_get_for_dev(dev); - if (!group) { - group = iommu_group_alloc(); - if (IS_ERR(group)) { - dev_err(dev, "Failed to allocate IOMMU group\n"); - return PTR_ERR(group); - } - } + if (IS_ERR(group)) + return PTR_ERR(group); - ret = iommu_group_add_device(group, dev); iommu_group_put(group); - return ret; + return 0; } static void exynos_iommu_remove_device(struct device *dev) @@ -1182,6 +1186,7 @@ static struct iommu_ops exynos_iommu_ops = { .unmap = exynos_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = exynos_iommu_iova_to_phys, + .device_group = get_device_iommu_group, .add_device = exynos_iommu_add_device, .remove_device = exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, -- cgit v0.10.2 From 58c6f6a3dc97ed1a5a5262279a2e9db41bd6d035 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:49 +0100 Subject: iommu/exynos: Add support for IOMMU_DOMAIN_DMA domain type This patch adds support for DMA domain type. Such domain have DMA cookie prepared and can be used by generic DMA-IOMMU glue layer. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 4fc0790..595e0da 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -25,9 +25,9 @@ #include #include #include +#include #include -#include #include typedef u32 sysmmu_iova_t; @@ -662,16 +662,21 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) struct exynos_iommu_domain *domain; int i; - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) return NULL; + if (type == IOMMU_DOMAIN_DMA) { + if (iommu_get_dma_cookie(&domain->domain) != 0) + goto err_pgtable; + } else if (type != IOMMU_DOMAIN_UNMANAGED) { + goto err_pgtable; + } + domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); if (!domain->pgtable) - goto err_pgtable; + goto err_dma_cookie; domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); if (!domain->lv2entcnt) @@ -703,6 +708,9 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) err_counter: free_pages((unsigned long)domain->pgtable, 2); +err_dma_cookie: + if (type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&domain->domain); err_pgtable: kfree(domain); return NULL; @@ -727,6 +735,9 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) spin_unlock_irqrestore(&domain->lock, flags); + if (iommu_domain->type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(iommu_domain); + for (i = 0; i < NUM_LV1ENTRIES; i++) if (lv1ent_page(domain->pgtable + i)) kmem_cache_free(lv2table_kmem_cache, -- cgit v0.10.2 From 5e3435eb7e1d8c9431254f5e0053ce1ad654a591 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:50 +0100 Subject: iommu/exynos: Remove ARM-specific cache flush interface This patch replaces custom ARM-specific code for performing CPU cache flush operations with generic code based on DMA-mapping. Domain managing code is independent of particular SYSMMU device, so the first registered SYSMMU device is used for DMA-mapping calls. This simplification works fine because all SYSMMU controllers are in the same address space (where DMA address equals physical address) and the DMA-mapping calls are done mainly to flush CPU cache to make changes visible to SYSMMU controllers. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 595e0da..8c8a7f7 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -27,9 +27,6 @@ #include #include -#include -#include - typedef u32 sysmmu_iova_t; typedef u32 sysmmu_pte_t; @@ -83,6 +80,7 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); } +#define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t)) #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) @@ -134,6 +132,7 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define has_sysmmu(dev) (dev->archdata.iommu != NULL) +static struct device *dma_dev; static struct kmem_cache *lv2table_kmem_cache; static sysmmu_pte_t *zero_lv2_table; #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table)) @@ -650,16 +649,19 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { } }; -static inline void pgtable_flush(void *vastart, void *vaend) +static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) { - dmac_flush_range(vastart, vaend); - outer_flush_range(virt_to_phys(vastart), - virt_to_phys(vaend)); + dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), + DMA_TO_DEVICE); + *ent = val; + dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), + DMA_TO_DEVICE); } static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) { struct exynos_iommu_domain *domain; + dma_addr_t handle; int i; @@ -694,7 +696,10 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) domain->pgtable[i + 7] = ZERO_LV2LINK; } - pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES); + handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE, + DMA_TO_DEVICE); + /* For mapping page table entries we rely on dma == phys */ + BUG_ON(handle != virt_to_phys(domain->pgtable)); spin_lock_init(&domain->lock); spin_lock_init(&domain->pgtablelock); @@ -738,10 +743,18 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) if (iommu_domain->type == IOMMU_DOMAIN_DMA) iommu_put_dma_cookie(iommu_domain); + dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE, + DMA_TO_DEVICE); + for (i = 0; i < NUM_LV1ENTRIES; i++) - if (lv1ent_page(domain->pgtable + i)) + if (lv1ent_page(domain->pgtable + i)) { + phys_addr_t base = lv2table_base(domain->pgtable + i); + + dma_unmap_single(dma_dev, base, LV2TABLE_SIZE, + DMA_TO_DEVICE); kmem_cache_free(lv2table_kmem_cache, - phys_to_virt(lv2table_base(domain->pgtable + i))); + phys_to_virt(base)); + } free_pages((unsigned long)domain->pgtable, 2); free_pages((unsigned long)domain->lv2entcnt, 1); @@ -834,11 +847,10 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, if (!pent) return ERR_PTR(-ENOMEM); - *sent = mk_lv1ent_page(virt_to_phys(pent)); + update_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); kmemleak_ignore(pent); *pgcounter = NUM_LV2ENTRIES; - pgtable_flush(pent, pent + NUM_LV2ENTRIES); - pgtable_flush(sent, sent + 1); + dma_map_single(dma_dev, pent, LV2TABLE_SIZE, DMA_TO_DEVICE); /* * If pre-fetched SLPD is a faulty SLPD in zero_l2_table, @@ -891,9 +903,7 @@ static int lv1set_section(struct exynos_iommu_domain *domain, *pgcnt = 0; } - *sent = mk_lv1ent_sect(paddr); - - pgtable_flush(sent, sent + 1); + update_pte(sent, mk_lv1ent_sect(paddr)); spin_lock(&domain->lock); if (lv1ent_page_zero(sent)) { @@ -917,12 +927,15 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, if (WARN_ON(!lv2ent_fault(pent))) return -EADDRINUSE; - *pent = mk_lv2ent_spage(paddr); - pgtable_flush(pent, pent + 1); + update_pte(pent, mk_lv2ent_spage(paddr)); *pgcnt -= 1; } else { /* size == LPAGE_SIZE */ int i; + dma_addr_t pent_base = virt_to_phys(pent); + dma_sync_single_for_cpu(dma_dev, pent_base, + sizeof(*pent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { if (WARN_ON(!lv2ent_fault(pent))) { if (i > 0) @@ -932,7 +945,9 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, *pent = mk_lv2ent_lpage(paddr); } - pgtable_flush(pent - SPAGES_PER_LPAGE, pent); + dma_sync_single_for_device(dma_dev, pent_base, + sizeof(*pent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); *pgcnt -= SPAGES_PER_LPAGE; } @@ -1042,8 +1057,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, } /* workaround for h/w bug in System MMU v3.3 */ - *ent = ZERO_LV2LINK; - pgtable_flush(ent, ent + 1); + update_pte(ent, ZERO_LV2LINK); size = SECT_SIZE; goto done; } @@ -1064,9 +1078,8 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, } if (lv2ent_small(ent)) { - *ent = 0; + update_pte(ent, 0); size = SPAGE_SIZE; - pgtable_flush(ent, ent + 1); domain->lv2entcnt[lv1ent_offset(iova)] += 1; goto done; } @@ -1077,9 +1090,13 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, goto err; } + dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), + sizeof(*ent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); - pgtable_flush(ent, ent + SPAGES_PER_LPAGE); - + dma_sync_single_for_device(dma_dev, virt_to_phys(ent), + sizeof(*ent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); size = LPAGE_SIZE; domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; done: @@ -1261,6 +1278,13 @@ static int __init exynos_iommu_of_setup(struct device_node *np) if (IS_ERR(pdev)) return PTR_ERR(pdev); + /* + * use the first registered sysmmu device for performing + * dma mapping operations on iommu page tables (cpu cache flush) + */ + if (!dma_dev) + dma_dev = &pdev->dev; + of_iommu_set_ops(np, &exynos_iommu_ops); return 0; } -- cgit v0.10.2 From b398af216939971d49b105d6d05295b906014711 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:51 +0100 Subject: iommu/exynos: Simplify master clock operations All clock API function can be called on NULL clock, so simplify code avoid checking of master clock presence. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 8c8a7f7..bf6b826 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -333,8 +333,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) spin_lock(&data->lock); - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); itype = (enum exynos_sysmmu_inttype) __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); @@ -366,8 +365,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) sysmmu_unblock(data->sfrbase); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); spin_unlock(&data->lock); @@ -376,15 +374,13 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) { - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); __raw_writel(0, data->sfrbase + REG_MMU_CFG); clk_disable(data->clk); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } static bool __sysmmu_disable(struct sysmmu_drvdata *data) @@ -437,8 +433,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); clk_enable(data->clk); __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); @@ -449,8 +444,7 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, @@ -493,16 +487,14 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, { unsigned long flags; - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) __sysmmu_tlb_invalidate_flpdcache(data, iova); spin_unlock_irqrestore(&data->lock, flags); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -514,8 +506,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, if (is_sysmmu_active(data)) { unsigned int num_inv = 1; - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); /* * L2TLB invalidation required @@ -535,8 +526,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, data->sfrbase, iova, num_inv); sysmmu_unblock(data->sfrbase); } - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } else { dev_dbg(data->master, "disabled. Skipping TLB invalidation @ %#x\n", iova); @@ -593,6 +583,8 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) dev_err(dev, "Failed to prepare master's clk\n"); return ret; } + } else { + data->clk_master = NULL; } data->sysmmu = dev; -- cgit v0.10.2 From 02cdc365cfd0298fed2e757071b98fd2d180c31d Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:52 +0100 Subject: iommu/exynos: Refactor code (no direct register access) This patch changes some internal functions to have access to the state of sysmmu device instead of having only it's registers. This will make the code ready for future extensions. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index bf6b826..4275222 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -254,50 +254,49 @@ static bool is_sysmmu_active(struct sysmmu_drvdata *data) return data->activations > 0; } -static void sysmmu_unblock(void __iomem *sfrbase) +static void sysmmu_unblock(struct sysmmu_drvdata *data) { - __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); + __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); } -static bool sysmmu_block(void __iomem *sfrbase) +static bool sysmmu_block(struct sysmmu_drvdata *data) { int i = 120; - __raw_writel(CTRL_BLOCK, sfrbase + REG_MMU_CTRL); - while ((i > 0) && !(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) + __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + while ((i > 0) && !(__raw_readl(data->sfrbase + REG_MMU_STATUS) & 1)) --i; - if (!(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) { - sysmmu_unblock(sfrbase); + if (!(__raw_readl(data->sfrbase + REG_MMU_STATUS) & 1)) { + sysmmu_unblock(data); return false; } return true; } -static void __sysmmu_tlb_invalidate(void __iomem *sfrbase) +static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) { - __raw_writel(0x1, sfrbase + REG_MMU_FLUSH); + __raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH); } -static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, +static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, sysmmu_iova_t iova, unsigned int num_inv) { unsigned int i; for (i = 0; i < num_inv; i++) { __raw_writel((iova & SPAGE_MASK) | 1, - sfrbase + REG_MMU_FLUSH_ENTRY); + data->sfrbase + REG_MMU_FLUSH_ENTRY); iova += SPAGE_SIZE; } } -static void __sysmmu_set_ptbase(void __iomem *sfrbase, - phys_addr_t pgd) +static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) { - __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR); + __raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); - __sysmmu_tlb_invalidate(sfrbase); + __sysmmu_tlb_invalidate(data); } static void show_fault_information(const char *name, @@ -363,7 +362,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); - sysmmu_unblock(data->sfrbase); + sysmmu_unblock(data); clk_disable(data->clk_master); @@ -440,7 +439,7 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) __sysmmu_init_config(data); - __sysmmu_set_ptbase(data->sfrbase, data->pgtable); + __sysmmu_set_ptbase(data, data->pgtable); __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); @@ -521,10 +520,9 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, if (MMU_MAJ_VER(data->version) == 2) num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); - if (sysmmu_block(data->sfrbase)) { - __sysmmu_tlb_invalidate_entry( - data->sfrbase, iova, num_inv); - sysmmu_unblock(data->sfrbase); + if (sysmmu_block(data)) { + __sysmmu_tlb_invalidate_entry(data, iova, num_inv); + sysmmu_unblock(data); } clk_disable(data->clk_master); } else { -- cgit v0.10.2 From d093fc7e8390c3e46294290c0d76de93422a601f Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:53 +0100 Subject: iommu/exynos: Refactor fault handling code This patch provides a new implementation for page fault handing code. The new implementation is ready for future extensions. No functional changes have been made. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 4275222..3a577a4 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -148,40 +148,25 @@ static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) lv2table_base(sent)) + lv2ent_offset(iova); } -enum exynos_sysmmu_inttype { - SYSMMU_PAGEFAULT, - SYSMMU_AR_MULTIHIT, - SYSMMU_AW_MULTIHIT, - SYSMMU_BUSERROR, - SYSMMU_AR_SECURITY, - SYSMMU_AR_ACCESS, - SYSMMU_AW_SECURITY, - SYSMMU_AW_PROTECTION, /* 7 */ - SYSMMU_FAULT_UNKNOWN, - SYSMMU_FAULTS_NUM -}; - -static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = { - REG_PAGE_FAULT_ADDR, - REG_AR_FAULT_ADDR, - REG_AW_FAULT_ADDR, - REG_DEFAULT_SLAVE_ADDR, - REG_AR_FAULT_ADDR, - REG_AR_FAULT_ADDR, - REG_AW_FAULT_ADDR, - REG_AW_FAULT_ADDR +/* + * IOMMU fault information register + */ +struct sysmmu_fault_info { + unsigned int bit; /* bit number in STATUS register */ + unsigned short addr_reg; /* register to read VA fault address */ + const char *name; /* human readable fault name */ + unsigned int type; /* fault type for report_iommu_fault */ }; -static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { - "PAGE FAULT", - "AR MULTI-HIT FAULT", - "AW MULTI-HIT FAULT", - "BUS ERROR", - "AR SECURITY PROTECTION FAULT", - "AR ACCESS PROTECTION FAULT", - "AW SECURITY PROTECTION FAULT", - "AW ACCESS PROTECTION FAULT", - "UNKNOWN FAULT" +static const struct sysmmu_fault_info sysmmu_faults[] = { + { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, + { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, + { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, + { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, + { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, + { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, + { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, + { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, }; /* @@ -299,24 +284,19 @@ static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) __sysmmu_tlb_invalidate(data); } -static void show_fault_information(const char *name, - enum exynos_sysmmu_inttype itype, - phys_addr_t pgtable_base, sysmmu_iova_t fault_addr) +static void show_fault_information(struct sysmmu_drvdata *data, + const struct sysmmu_fault_info *finfo, + sysmmu_iova_t fault_addr) { sysmmu_pte_t *ent; - if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) - itype = SYSMMU_FAULT_UNKNOWN; - - pr_err("%s occurred at %#x by %s(Page table base: %pa)\n", - sysmmu_fault_name[itype], fault_addr, name, &pgtable_base); - - ent = section_entry(phys_to_virt(pgtable_base), fault_addr); - pr_err("\tLv1 entry: %#x\n", *ent); - + dev_err(data->sysmmu, "%s FAULT occurred at %#x (page table base: %pa)\n", + finfo->name, fault_addr, &data->pgtable); + ent = section_entry(phys_to_virt(data->pgtable), fault_addr); + dev_err(data->sysmmu, "\tLv1 entry: %#x\n", *ent); if (lv1ent_page(ent)) { ent = page_entry(ent, fault_addr); - pr_err("\t Lv2 entry: %#x\n", *ent); + dev_err(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); } } @@ -324,8 +304,10 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { /* SYSMMU is in blocked state when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; - enum exynos_sysmmu_inttype itype; - sysmmu_iova_t addr = -1; + const struct sysmmu_fault_info *finfo = sysmmu_faults; + int i, n = ARRAY_SIZE(sysmmu_faults); + unsigned int itype; + sysmmu_iova_t fault_addr = -1; int ret = -ENOSYS; WARN_ON(!is_sysmmu_active(data)); @@ -334,29 +316,20 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) clk_enable(data->clk_master); - itype = (enum exynos_sysmmu_inttype) - __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); - if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) - itype = SYSMMU_FAULT_UNKNOWN; - else - addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]); + itype = __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); + for (i = 0; i < n; i++, finfo++) + if (finfo->bit == itype) + break; + /* unknown/unsupported fault */ + BUG_ON(i == n); - if (itype == SYSMMU_FAULT_UNKNOWN) { - pr_err("%s: Fault is not occurred by System MMU '%s'!\n", - __func__, dev_name(data->sysmmu)); - pr_err("%s: Please check if IRQ is correctly configured.\n", - __func__); - BUG(); - } else { - unsigned int base = - __raw_readl(data->sfrbase + REG_PT_BASE_ADDR); - show_fault_information(dev_name(data->sysmmu), - itype, base, addr); - if (data->domain) - ret = report_iommu_fault(&data->domain->domain, - data->master, addr, itype); - } + /* print debug message */ + fault_addr = __raw_readl(data->sfrbase + finfo->addr_reg); + show_fault_information(data, finfo, fault_addr); + if (data->domain) + ret = report_iommu_fault(&data->domain->domain, + data->master, fault_addr, finfo->type); /* fault is not recovered by fault handler */ BUG_ON(ret != 0); -- cgit v0.10.2 From 83addecdb7c9a4a533205cf57cb4d5b5631a704b Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:54 +0100 Subject: iommu/exynos: Refactor init config code This patch rewrites sysmmu_init_config function to make it easier to read and understand. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 3a577a4..15787a1 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -383,24 +383,17 @@ static bool __sysmmu_disable(struct sysmmu_drvdata *data) static void __sysmmu_init_config(struct sysmmu_drvdata *data) { - unsigned int cfg = CFG_LRU | CFG_QOS(15); - unsigned int ver; - - ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); - if (MMU_MAJ_VER(ver) == 3) { - if (MMU_MIN_VER(ver) >= 2) { - cfg |= CFG_FLPDCACHE; - if (MMU_MIN_VER(ver) == 3) { - cfg |= CFG_ACGEN; - cfg &= ~CFG_LRU; - } else { - cfg |= CFG_SYSSEL; - } - } - } + unsigned int cfg; + + data->version = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); + if (data->version <= MAKE_MMU_VER(3, 1)) + cfg = CFG_LRU | CFG_QOS(15); + else if (data->version <= MAKE_MMU_VER(3, 2)) + cfg = CFG_LRU | CFG_QOS(15) | CFG_FLPDCACHE | CFG_SYSSEL; + else + cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); - data->version = ver; } static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) -- cgit v0.10.2 From d631ea9809785da06a048cfb25c2714f18a9cd63 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:55 +0100 Subject: iommu/exynos: Unify code for fldp cache invalidation This patch simplifies the code for handling of flpdcache invalidation. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 15787a1..e42a76c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -440,13 +440,6 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, return ret; } -static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, - sysmmu_iova_t iova) -{ - if (data->version == MAKE_MMU_VER(3, 3)) - __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); -} - static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { @@ -455,8 +448,10 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, clk_enable(data->clk_master); spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) - __sysmmu_tlb_invalidate_flpdcache(data, iova); + if (is_sysmmu_active(data)) { + if (data->version >= MAKE_MMU_VER(3, 3)) + __sysmmu_tlb_invalidate_entry(data, iova, 1); + } spin_unlock_irqrestore(&data->lock, flags); clk_disable(data->clk_master); -- cgit v0.10.2 From 850d313e2ed52d19b781e38536fc44ba0b58c434 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:56 +0100 Subject: iommu/exynos: Add support for SYSMMU controller with bogus version reg SYSMMU on some SoCs reports bogus values in VERSION register. Force hardware version to 1.0 for such controllers. This patch also moves reading version register to driver's probe() function. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index e42a76c..8e289e2 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -284,6 +284,28 @@ static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) __sysmmu_tlb_invalidate(data); } +static void __sysmmu_get_version(struct sysmmu_drvdata *data) +{ + u32 ver; + + clk_enable(data->clk_master); + clk_enable(data->clk); + + ver = __raw_readl(data->sfrbase + REG_MMU_VERSION); + + /* controllers on some SoCs don't report proper version */ + if (ver == 0x80000001u) + data->version = MAKE_MMU_VER(1, 0); + else + data->version = MMU_RAW_VER(ver); + + dev_dbg(data->sysmmu, "hardware version: %d.%d\n", + MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); + + clk_disable(data->clk); + clk_disable(data->clk_master); +} + static void show_fault_information(struct sysmmu_drvdata *data, const struct sysmmu_fault_info *finfo, sysmmu_iova_t fault_addr) @@ -385,7 +407,6 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) { unsigned int cfg; - data->version = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); if (data->version <= MAKE_MMU_VER(3, 1)) cfg = CFG_LRU | CFG_QOS(15); else if (data->version <= MAKE_MMU_VER(3, 2)) @@ -551,6 +572,7 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); + __sysmmu_get_version(data); pm_runtime_enable(dev); return 0; -- cgit v0.10.2 From e6802707fbfacfa0a4668d176dc2f0f9dd6637be Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:57 +0100 Subject: iommu/exynos: Update device tree documentation Exynos SYSMMU bindings documentation was merged before generic IOMMU binding have been introduced. This patch updates documentation to match current state. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt index bc620fe..f61ca25 100644 --- a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt +++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt @@ -23,28 +23,23 @@ MMUs. for window 1, 2 and 3. * M2M Scalers and G2D in Exynos5420 has one System MMU on the read channel and the other System MMU on the write channel. -The drivers must consider how to handle those System MMUs. One of the idea is -to implement child devices or sub-devices which are the client devices of the -System MMU. -Note: -The current DT binding for the Exynos System MMU is incomplete. -The following properties can be removed or changed, if found incompatible with -the "Generic IOMMU Binding" support for attaching devices to the IOMMU. +For information on assigning System MMU controller to its peripheral devices, +see generic IOMMU bindings. Required properties: - compatible: Should be "samsung,exynos-sysmmu" - reg: A tuple of base address and size of System MMU registers. +- #iommu-cells: Should be <0>. - interrupt-parent: The phandle of the interrupt controller of System MMU - interrupts: An interrupt specifier for interrupt signal of System MMU, according to the format defined by a particular interrupt controller. - clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock. Optional "master" if the clock to the System MMU is gated by - another gate clock other than "sysmmu". - Exynos4 SoCs, there needs no "master" clock. - Exynos5 SoCs, some System MMUs must have "master" clocks. -- clocks: Required if the System MMU is needed to gate its clock. + another gate clock other than "sysmmu" (usually main gate clock + of peripheral device this SYSMMU belongs to). +- clocks: Phandles for respective clocks described by clock-names. - power-domains: Required if the System MMU is needed to gate its power. Please refer to the following document: Documentation/devicetree/bindings/power/pd-samsung.txt @@ -57,6 +52,7 @@ Examples: power-domains = <&pd_gsc>; clocks = <&clock CLK_GSCL0>; clock-names = "gscl"; + iommus = <&sysmmu_gsc0>; }; sysmmu_gsc0: sysmmu@13E80000 { @@ -67,4 +63,5 @@ Examples: clock-names = "sysmmu", "master"; clocks = <&clock CLK_SMMU_GSCL0>, <&clock CLK_GSCL0>; power-domains = <&pd_gsc>; + #iommu-cells = <0>; }; -- cgit v0.10.2 From 740a01eee9ada98b6ccdd2bcb1a5b2470f292fc7 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:58 +0100 Subject: iommu/exynos: Add support for v5 SYSMMU This patch adds support for v5 of SYSMMU controller, found in Samsung Exynos 5433 SoCs. The main difference of v5 is support for 36-bit physical address space and some changes in register layout and core clocks hanging. This patch also adds support for ARM64 architecture, which is used by Exynos 5433 SoCs. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt index f61ca25..85f0688 100644 --- a/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt +++ b/Documentation/devicetree/bindings/iommu/samsung,sysmmu.txt @@ -35,9 +35,10 @@ Required properties: - interrupts: An interrupt specifier for interrupt signal of System MMU, according to the format defined by a particular interrupt controller. -- clock-names: Should be "sysmmu" if the System MMU is needed to gate its clock. +- clock-names: Should be "sysmmu" or a pair of "aclk" and "pclk" to gate + SYSMMU core clocks. Optional "master" if the clock to the System MMU is gated by - another gate clock other than "sysmmu" (usually main gate clock + another gate clock other core (usually main gate clock of peripheral device this SYSMMU belongs to). - clocks: Phandles for respective clocks described by clock-names. - power-domains: Required if the System MMU is needed to gate its power. diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a1e75cb..1674de1 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -243,7 +243,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && ARM && MMU + depends on ARCH_EXYNOS && MMU select IOMMU_API select ARM_DMA_USE_IOMMU help diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 8e289e2..9c8ce95 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1,6 +1,5 @@ -/* linux/drivers/iommu/exynos_iommu.c - * - * Copyright (c) 2011 Samsung Electronics Co., Ltd. +/* + * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. * http://www.samsung.com * * This program is free software; you can redistribute it and/or modify @@ -55,17 +54,25 @@ typedef u32 sysmmu_pte_t; #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) -static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size) -{ - return iova & (size - 1); -} - -#define section_phys(sent) (*(sent) & SECT_MASK) -#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE) -#define lpage_phys(pent) (*(pent) & LPAGE_MASK) -#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE) -#define spage_phys(pent) (*(pent) & SPAGE_MASK) -#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE) +/* + * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces + * v5.0 introduced support for 36bit physical address space by shifting + * all page entry values by 4 bits. + * All SYSMMU controllers in the system support the address spaces of the same + * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper + * value (0 or 4). + */ +static short PG_ENT_SHIFT = -1; +#define SYSMMU_PG_ENT_SHIFT 0 +#define SYSMMU_V5_PG_ENT_SHIFT 4 + +#define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) +#define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) +#define section_offs(iova) (iova & (SECT_SIZE - 1)) +#define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK) +#define lpage_offs(iova) (iova & (LPAGE_SIZE - 1)) +#define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK) +#define spage_offs(iova) (iova & (SPAGE_SIZE - 1)) #define NUM_LV1ENTRIES 4096 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) @@ -84,13 +91,12 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) +#define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) -#define lv2table_base(sent) (*(sent) & 0xFFFFFC00) - -#define mk_lv1ent_sect(pa) ((pa) | 2) -#define mk_lv1ent_page(pa) ((pa) | 1) -#define mk_lv2ent_lpage(pa) ((pa) | 1) -#define mk_lv2ent_spage(pa) ((pa) | 2) +#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2) +#define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) +#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1) +#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2) #define CTRL_ENABLE 0x5 #define CTRL_BLOCK 0x7 @@ -98,14 +104,23 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CFG_LRU 0x1 #define CFG_QOS(n) ((n & 0xF) << 7) -#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */ #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ +/* common registers */ #define REG_MMU_CTRL 0x000 #define REG_MMU_CFG 0x004 #define REG_MMU_STATUS 0x008 +#define REG_MMU_VERSION 0x034 + +#define MMU_MAJ_VER(val) ((val) >> 7) +#define MMU_MIN_VER(val) ((val) & 0x7F) +#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ + +#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) + +/* v1.x - v3.x registers */ #define REG_MMU_FLUSH 0x00C #define REG_MMU_FLUSH_ENTRY 0x010 #define REG_PT_BASE_ADDR 0x014 @@ -117,18 +132,14 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define REG_AR_FAULT_ADDR 0x02C #define REG_DEFAULT_SLAVE_ADDR 0x030 -#define REG_MMU_VERSION 0x034 - -#define MMU_MAJ_VER(val) ((val) >> 7) -#define MMU_MIN_VER(val) ((val) & 0x7F) -#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ - -#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) - -#define REG_PB0_SADDR 0x04C -#define REG_PB0_EADDR 0x050 -#define REG_PB1_SADDR 0x054 -#define REG_PB1_EADDR 0x058 +/* v5.x registers */ +#define REG_V5_PT_BASE_PFN 0x00C +#define REG_V5_MMU_FLUSH_ALL 0x010 +#define REG_V5_MMU_FLUSH_ENTRY 0x014 +#define REG_V5_INT_STATUS 0x060 +#define REG_V5_INT_CLEAR 0x064 +#define REG_V5_FAULT_AR_VA 0x070 +#define REG_V5_FAULT_AW_VA 0x080 #define has_sysmmu(dev) (dev->archdata.iommu != NULL) @@ -169,6 +180,19 @@ static const struct sysmmu_fault_info sysmmu_faults[] = { { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, }; +static const struct sysmmu_fault_info sysmmu_v5_faults[] = { + { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, + { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, + { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, + { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, + { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, + { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, + { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, + { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, + { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, + { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, +}; + /* * This structure is attached to dev.archdata.iommu of the master device * on device add, contains a list of SYSMMU controllers defined by device tree, @@ -205,6 +229,8 @@ struct sysmmu_drvdata { struct device *master; /* master device (owner) */ void __iomem *sfrbase; /* our registers */ struct clk *clk; /* SYSMMU's clock */ + struct clk *aclk; /* SYSMMU's aclk clock */ + struct clk *pclk; /* SYSMMU's pclk clock */ struct clk *clk_master; /* master's device clock */ int activations; /* number of calls to sysmmu_enable */ spinlock_t lock; /* lock for modyfying state */ @@ -262,7 +288,10 @@ static bool sysmmu_block(struct sysmmu_drvdata *data) static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) { - __raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH); + if (MMU_MAJ_VER(data->version) < 5) + __raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH); + else + __raw_writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); } static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -271,15 +300,23 @@ static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, unsigned int i; for (i = 0; i < num_inv; i++) { - __raw_writel((iova & SPAGE_MASK) | 1, - data->sfrbase + REG_MMU_FLUSH_ENTRY); + if (MMU_MAJ_VER(data->version) < 5) + __raw_writel((iova & SPAGE_MASK) | 1, + data->sfrbase + REG_MMU_FLUSH_ENTRY); + else + __raw_writel((iova & SPAGE_MASK) | 1, + data->sfrbase + REG_V5_MMU_FLUSH_ENTRY); iova += SPAGE_SIZE; } } static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) { - __raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); + if (MMU_MAJ_VER(data->version) < 5) + __raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); + else + __raw_writel(pgd >> PAGE_SHIFT, + data->sfrbase + REG_V5_PT_BASE_PFN); __sysmmu_tlb_invalidate(data); } @@ -290,6 +327,8 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) clk_enable(data->clk_master); clk_enable(data->clk); + clk_enable(data->pclk); + clk_enable(data->aclk); ver = __raw_readl(data->sfrbase + REG_MMU_VERSION); @@ -302,6 +341,8 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) dev_dbg(data->sysmmu, "hardware version: %d.%d\n", MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); + clk_disable(data->aclk); + clk_disable(data->pclk); clk_disable(data->clk); clk_disable(data->clk_master); } @@ -326,19 +367,31 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { /* SYSMMU is in blocked state when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; - const struct sysmmu_fault_info *finfo = sysmmu_faults; - int i, n = ARRAY_SIZE(sysmmu_faults); - unsigned int itype; + const struct sysmmu_fault_info *finfo; + unsigned int i, n, itype; sysmmu_iova_t fault_addr = -1; + unsigned short reg_status, reg_clear; int ret = -ENOSYS; WARN_ON(!is_sysmmu_active(data)); + if (MMU_MAJ_VER(data->version) < 5) { + reg_status = REG_INT_STATUS; + reg_clear = REG_INT_CLEAR; + finfo = sysmmu_faults; + n = ARRAY_SIZE(sysmmu_faults); + } else { + reg_status = REG_V5_INT_STATUS; + reg_clear = REG_V5_INT_CLEAR; + finfo = sysmmu_v5_faults; + n = ARRAY_SIZE(sysmmu_v5_faults); + } + spin_lock(&data->lock); clk_enable(data->clk_master); - itype = __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); + itype = __ffs(__raw_readl(data->sfrbase + reg_status)); for (i = 0; i < n; i++, finfo++) if (finfo->bit == itype) break; @@ -355,7 +408,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) /* fault is not recovered by fault handler */ BUG_ON(ret != 0); - __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); + __raw_writel(1 << itype, data->sfrbase + reg_clear); sysmmu_unblock(data); @@ -373,6 +426,8 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); __raw_writel(0, data->sfrbase + REG_MMU_CFG); + clk_disable(data->aclk); + clk_disable(data->pclk); clk_disable(data->clk); clk_disable(data->clk_master); } @@ -421,6 +476,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { clk_enable(data->clk_master); clk_enable(data->clk); + clk_enable(data->pclk); + clk_enable(data->aclk); __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); @@ -544,22 +601,47 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk = devm_clk_get(dev, "sysmmu"); - if (IS_ERR(data->clk)) { - dev_err(dev, "Failed to get clock!\n"); - return PTR_ERR(data->clk); - } else { + if (!IS_ERR(data->clk)) { ret = clk_prepare(data->clk); if (ret) { dev_err(dev, "Failed to prepare clk\n"); return ret; } + } else { + data->clk = NULL; + } + + data->aclk = devm_clk_get(dev, "aclk"); + if (!IS_ERR(data->aclk)) { + ret = clk_prepare(data->aclk); + if (ret) { + dev_err(dev, "Failed to prepare aclk\n"); + return ret; + } + } else { + data->aclk = NULL; + } + + data->pclk = devm_clk_get(dev, "pclk"); + if (!IS_ERR(data->pclk)) { + ret = clk_prepare(data->pclk); + if (ret) { + dev_err(dev, "Failed to prepare pclk\n"); + return ret; + } + } else { + data->pclk = NULL; + } + + if (!data->clk && (!data->aclk || !data->pclk)) { + dev_err(dev, "Failed to get device clock(s)!\n"); + return -ENOSYS; } data->clk_master = devm_clk_get(dev, "master"); if (!IS_ERR(data->clk_master)) { ret = clk_prepare(data->clk_master); if (ret) { - clk_unprepare(data->clk); dev_err(dev, "Failed to prepare master's clk\n"); return ret; } @@ -573,6 +655,13 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); __sysmmu_get_version(data); + if (PG_ENT_SHIFT < 0) { + if (MMU_MAJ_VER(data->version) < 5) + PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; + else + PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; + } + pm_runtime_enable(dev); return 0; @@ -637,6 +726,8 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) dma_addr_t handle; int i; + /* Check if correct PTE offsets are initialized */ + BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev); domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) @@ -816,7 +907,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, bool need_flush_flpd_cache = lv1ent_zero(sent); pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); - BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1)); + BUG_ON((phys_addr_t)pent & (LV2TABLE_SIZE - 1)); if (!pent) return ERR_PTR(-ENOMEM); -- cgit v0.10.2 From 9b93a409fd317be25dca7118163efe74154bc9e7 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:12:59 +0100 Subject: iommu/exynos: Add Maintainers entry for Exynos SYSMMU driver Add Marek Szyprowski as maintainer for Exynos IOMMU driver. Signed-off-by: Marek Szyprowski Acked-by: Krzysztof Kozlowski Signed-off-by: Joerg Roedel diff --git a/MAINTAINERS b/MAINTAINERS index 4978dc1..27dac84 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4305,6 +4305,12 @@ L: dri-devel@lists.freedesktop.org S: Maintained F: drivers/gpu/drm/exynos/exynos_dp* +EXYNOS SYSMMU (IOMMU) driver +M: Marek Szyprowski +L: iommu@lists.linux-foundation.org +S: Maintained +F: drivers/iommu/exynos-iommu.c + EXYNOS MIPI DISPLAY DRIVERS M: Inki Dae M: Donghwa Lee -- cgit v0.10.2 From 5fa61cbff16224ef8595102552ca49b2c6775ab4 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Thu, 18 Feb 2016 15:13:00 +0100 Subject: iommu/exynos: Support multiple attach_device calls IOMMU core calls attach_device callback without detaching device from the previous domain. This patch adds support for such unballanced calls. Signed-off-by: Marek Szyprowski Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 9c8ce95..b066504 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -201,6 +201,7 @@ static const struct sysmmu_fault_info sysmmu_v5_faults[] = { */ struct exynos_iommu_owner { struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ + struct iommu_domain *domain; /* domain this device is attached */ }; /* @@ -825,6 +826,41 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) kfree(domain); } +static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, + struct device *dev) +{ + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + phys_addr_t pagetable = virt_to_phys(domain->pgtable); + struct sysmmu_drvdata *data, *next; + unsigned long flags; + bool found = false; + + if (!has_sysmmu(dev) || owner->domain != iommu_domain) + return; + + spin_lock_irqsave(&domain->lock, flags); + list_for_each_entry_safe(data, next, &domain->clients, domain_node) { + if (data->master == dev) { + if (__sysmmu_disable(data)) { + data->master = NULL; + list_del_init(&data->domain_node); + } + pm_runtime_put(data->sysmmu); + found = true; + } + } + spin_unlock_irqrestore(&domain->lock, flags); + + owner->domain = NULL; + + if (found) + dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", + __func__, &pagetable); + else + dev_err(dev, "%s: No IOMMU is attached\n", __func__); +} + static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct device *dev) { @@ -838,6 +874,9 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, if (!has_sysmmu(dev)) return -ENODEV; + if (owner->domain) + exynos_iommu_detach_device(owner->domain, dev); + list_for_each_entry(data, &owner->controllers, owner_node) { pm_runtime_get_sync(data->sysmmu); ret = __sysmmu_enable(data, pagetable, domain); @@ -856,44 +895,13 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, return ret; } + owner->domain = iommu_domain; dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n", __func__, &pagetable, (ret == 0) ? "" : ", again"); return ret; } -static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, - struct device *dev) -{ - struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); - phys_addr_t pagetable = virt_to_phys(domain->pgtable); - struct sysmmu_drvdata *data, *next; - unsigned long flags; - bool found = false; - - if (!has_sysmmu(dev)) - return; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry_safe(data, next, &domain->clients, domain_node) { - if (data->master == dev) { - if (__sysmmu_disable(data)) { - data->master = NULL; - list_del_init(&data->domain_node); - } - pm_runtime_put(data->sysmmu); - found = true; - } - } - spin_unlock_irqrestore(&domain->lock, flags); - - if (found) - dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", - __func__, &pagetable); - else - dev_err(dev, "%s: No IOMMU is attached\n", __func__); -} - static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { -- cgit v0.10.2 From 9015ba456c432023630a1a2ce13049d84fd5b7a3 Mon Sep 17 00:00:00 2001 From: Simon Horman Date: Mon, 22 Feb 2016 10:41:35 +0900 Subject: iommu/ipmmu-vmsa: Use ARCH_RENESAS Make use of ARCH_RENESAS in place of ARCH_SHMOBILE. This is part of an ongoing process to migrate from ARCH_SHMOBILE to ARCH_RENESAS the motivation for which being that RENESAS seems to be a more appropriate name than SHMOBILE for the majority of Renesas ARM based SoCs. Signed-off-by: Simon Horman Acked-by: Geert Uytterhoeven Acked-by: Laurent Pinchart Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a1e75cb..d4b38e4 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -266,7 +266,7 @@ config EXYNOS_IOMMU_DEBUG config IPMMU_VMSA bool "Renesas VMSA-compatible IPMMU" depends on ARM_LPAE - depends on ARCH_SHMOBILE || COMPILE_TEST + depends on ARCH_RENESAS || COMPILE_TEST select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU -- cgit v0.10.2 From 120399ba66c0675c2b1f61a5ecadd76a7f208da6 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 23 Feb 2016 01:20:47 +0800 Subject: dt-bindings: iommu: Add binding for mediatek IOMMU This patch add mediatek iommu dts binding document. Signed-off-by: Yong Wu Acked-by: Rob Herring Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt new file mode 100644 index 0000000..cd1b1cd --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/mediatek,iommu.txt @@ -0,0 +1,68 @@ +* Mediatek IOMMU Architecture Implementation + + Some Mediatek SOCs contain a Multimedia Memory Management Unit (M4U) which +uses the ARM Short-Descriptor translation table format for address translation. + + About the M4U Hardware Block Diagram, please check below: + + EMI (External Memory Interface) + | + m4u (Multimedia Memory Management Unit) + | + SMI Common(Smart Multimedia Interface Common) + | + +----------------+------- + | | + | | + SMI larb0 SMI larb1 ... SoCs have several SMI local arbiter(larb). + (display) (vdec) + | | + | | + +-----+-----+ +----+----+ + | | | | | | + | | |... | | | ... There are different ports in each larb. + | | | | | | +OVL0 RDMA0 WDMA0 MC PP VLD + + As above, The Multimedia HW will go through SMI and M4U while it +access EMI. SMI is a bridge between m4u and the Multimedia HW. It contain +smi local arbiter and smi common. It will control whether the Multimedia +HW should go though the m4u for translation or bypass it and talk +directly with EMI. And also SMI help control the power domain and clocks for +each local arbiter. + Normally we specify a local arbiter(larb) for each multimedia HW +like display, video decode, and camera. And there are different ports +in each larb. Take a example, There are many ports like MC, PP, VLD in the +video decode local arbiter, all these ports are according to the video HW. + +Required properties: +- compatible : must be "mediatek,mt8173-m4u". +- reg : m4u register base and size. +- interrupts : the interrupt of m4u. +- clocks : must contain one entry for each clock-names. +- clock-names : must be "bclk", It is the block clock of m4u. +- mediatek,larbs : List of phandle to the local arbiters in the current Socs. + Refer to bindings/memory-controllers/mediatek,smi-larb.txt. It must sort + according to the local arbiter index, like larb0, larb1, larb2... +- iommu-cells : must be 1. This is the mtk_m4u_id according to the HW. + Specifies the mtk_m4u_id as defined in + dt-binding/memory/mt8173-larb-port.h. + +Example: + iommu: iommu@10205000 { + compatible = "mediatek,mt8173-m4u"; + reg = <0 0x10205000 0 0x1000>; + interrupts = ; + clocks = <&infracfg CLK_INFRA_M4U>; + clock-names = "bclk"; + mediatek,larbs = <&larb0 &larb1 &larb2 &larb3 &larb4 &larb5>; + #iommu-cells = <1>; + }; + +Example for a client device: + display { + compatible = "mediatek,mt8173-disp"; + iommus = <&iommu M4U_PORT_DISP_OVL0>, + <&iommu M4U_PORT_DISP_RDMA0>; + ... + }; -- cgit v0.10.2 From fb6e2ceee3e634e42405d9b47080ed21442964d9 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 23 Feb 2016 01:20:48 +0800 Subject: dt-bindings: mediatek: Add smi dts binding This patch add smi binding document and smi local arbiter header file. Signed-off-by: Yong Wu Acked-by: Rob Herring Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt new file mode 100644 index 0000000..06a83ce --- /dev/null +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-common.txt @@ -0,0 +1,24 @@ +SMI (Smart Multimedia Interface) Common + +The hardware block diagram please check bindings/iommu/mediatek,iommu.txt + +Required properties: +- compatible : must be "mediatek,mt8173-smi-common" +- reg : the register and size of the SMI block. +- power-domains : a phandle to the power domain of this local arbiter. +- clocks : Must contain an entry for each entry in clock-names. +- clock-names : must contain 2 entries, as follows: + - "apb" : Advanced Peripheral Bus clock, It's the clock for setting + the register. + - "smi" : It's the clock for transfer data and command. + They may be the same if both source clocks are the same. + +Example: + smi_common: smi@14022000 { + compatible = "mediatek,mt8173-smi-common"; + reg = <0 0x14022000 0 0x1000>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_MM>; + clocks = <&mmsys CLK_MM_SMI_COMMON>, + <&mmsys CLK_MM_SMI_COMMON>; + clock-names = "apb", "smi"; + }; diff --git a/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt new file mode 100644 index 0000000..55ff3b7 --- /dev/null +++ b/Documentation/devicetree/bindings/memory-controllers/mediatek,smi-larb.txt @@ -0,0 +1,25 @@ +SMI (Smart Multimedia Interface) Local Arbiter + +The hardware block diagram please check bindings/iommu/mediatek,iommu.txt + +Required properties: +- compatible : must be "mediatek,mt8173-smi-larb" +- reg : the register and size of this local arbiter. +- mediatek,smi : a phandle to the smi_common node. +- power-domains : a phandle to the power domain of this local arbiter. +- clocks : Must contain an entry for each entry in clock-names. +- clock-names: must contain 2 entries, as follows: + - "apb" : Advanced Peripheral Bus clock, It's the clock for setting + the register. + - "smi" : It's the clock for transfer data and command. + +Example: + larb1: larb@16010000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x16010000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_VDEC>; + clocks = <&vdecsys CLK_VDEC_CKEN>, + <&vdecsys CLK_VDEC_LARB_CKEN>; + clock-names = "apb", "smi"; + }; diff --git a/include/dt-bindings/memory/mt8173-larb-port.h b/include/dt-bindings/memory/mt8173-larb-port.h new file mode 100644 index 0000000..5fef5d1 --- /dev/null +++ b/include/dt-bindings/memory/mt8173-larb-port.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Yong Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef __DTS_IOMMU_PORT_MT8173_H +#define __DTS_IOMMU_PORT_MT8173_H + +#define MTK_M4U_ID(larb, port) (((larb) << 5) | (port)) +/* Local arbiter ID */ +#define MTK_M4U_TO_LARB(id) (((id) >> 5) & 0x7) +/* PortID within the local arbiter */ +#define MTK_M4U_TO_PORT(id) ((id) & 0x1f) + +#define M4U_LARB0_ID 0 +#define M4U_LARB1_ID 1 +#define M4U_LARB2_ID 2 +#define M4U_LARB3_ID 3 +#define M4U_LARB4_ID 4 +#define M4U_LARB5_ID 5 + +/* larb0 */ +#define M4U_PORT_DISP_OVL0 MTK_M4U_ID(M4U_LARB0_ID, 0) +#define M4U_PORT_DISP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 1) +#define M4U_PORT_DISP_WDMA0 MTK_M4U_ID(M4U_LARB0_ID, 2) +#define M4U_PORT_DISP_OD_R MTK_M4U_ID(M4U_LARB0_ID, 3) +#define M4U_PORT_DISP_OD_W MTK_M4U_ID(M4U_LARB0_ID, 4) +#define M4U_PORT_MDP_RDMA0 MTK_M4U_ID(M4U_LARB0_ID, 5) +#define M4U_PORT_MDP_WDMA MTK_M4U_ID(M4U_LARB0_ID, 6) +#define M4U_PORT_MDP_WROT0 MTK_M4U_ID(M4U_LARB0_ID, 7) + +/* larb1 */ +#define M4U_PORT_HW_VDEC_MC_EXT MTK_M4U_ID(M4U_LARB1_ID, 0) +#define M4U_PORT_HW_VDEC_PP_EXT MTK_M4U_ID(M4U_LARB1_ID, 1) +#define M4U_PORT_HW_VDEC_UFO_EXT MTK_M4U_ID(M4U_LARB1_ID, 2) +#define M4U_PORT_HW_VDEC_VLD_EXT MTK_M4U_ID(M4U_LARB1_ID, 3) +#define M4U_PORT_HW_VDEC_VLD2_EXT MTK_M4U_ID(M4U_LARB1_ID, 4) +#define M4U_PORT_HW_VDEC_AVC_MV_EXT MTK_M4U_ID(M4U_LARB1_ID, 5) +#define M4U_PORT_HW_VDEC_PRED_RD_EXT MTK_M4U_ID(M4U_LARB1_ID, 6) +#define M4U_PORT_HW_VDEC_PRED_WR_EXT MTK_M4U_ID(M4U_LARB1_ID, 7) +#define M4U_PORT_HW_VDEC_PPWRAP_EXT MTK_M4U_ID(M4U_LARB1_ID, 8) +#define M4U_PORT_HW_VDEC_TILE MTK_M4U_ID(M4U_LARB1_ID, 9) + +/* larb2 */ +#define M4U_PORT_IMGO MTK_M4U_ID(M4U_LARB2_ID, 0) +#define M4U_PORT_RRZO MTK_M4U_ID(M4U_LARB2_ID, 1) +#define M4U_PORT_AAO MTK_M4U_ID(M4U_LARB2_ID, 2) +#define M4U_PORT_LCSO MTK_M4U_ID(M4U_LARB2_ID, 3) +#define M4U_PORT_ESFKO MTK_M4U_ID(M4U_LARB2_ID, 4) +#define M4U_PORT_IMGO_D MTK_M4U_ID(M4U_LARB2_ID, 5) +#define M4U_PORT_LSCI MTK_M4U_ID(M4U_LARB2_ID, 6) +#define M4U_PORT_LSCI_D MTK_M4U_ID(M4U_LARB2_ID, 7) +#define M4U_PORT_BPCI MTK_M4U_ID(M4U_LARB2_ID, 8) +#define M4U_PORT_BPCI_D MTK_M4U_ID(M4U_LARB2_ID, 9) +#define M4U_PORT_UFDI MTK_M4U_ID(M4U_LARB2_ID, 10) +#define M4U_PORT_IMGI MTK_M4U_ID(M4U_LARB2_ID, 11) +#define M4U_PORT_IMG2O MTK_M4U_ID(M4U_LARB2_ID, 12) +#define M4U_PORT_IMG3O MTK_M4U_ID(M4U_LARB2_ID, 13) +#define M4U_PORT_VIPI MTK_M4U_ID(M4U_LARB2_ID, 14) +#define M4U_PORT_VIP2I MTK_M4U_ID(M4U_LARB2_ID, 15) +#define M4U_PORT_VIP3I MTK_M4U_ID(M4U_LARB2_ID, 16) +#define M4U_PORT_LCEI MTK_M4U_ID(M4U_LARB2_ID, 17) +#define M4U_PORT_RB MTK_M4U_ID(M4U_LARB2_ID, 18) +#define M4U_PORT_RP MTK_M4U_ID(M4U_LARB2_ID, 19) +#define M4U_PORT_WR MTK_M4U_ID(M4U_LARB2_ID, 20) + +/* larb3 */ +#define M4U_PORT_VENC_RCPU MTK_M4U_ID(M4U_LARB3_ID, 0) +#define M4U_PORT_VENC_REC MTK_M4U_ID(M4U_LARB3_ID, 1) +#define M4U_PORT_VENC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 2) +#define M4U_PORT_VENC_SV_COMV MTK_M4U_ID(M4U_LARB3_ID, 3) +#define M4U_PORT_VENC_RD_COMV MTK_M4U_ID(M4U_LARB3_ID, 4) +#define M4U_PORT_JPGENC_RDMA MTK_M4U_ID(M4U_LARB3_ID, 5) +#define M4U_PORT_JPGENC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 6) +#define M4U_PORT_JPGDEC_WDMA MTK_M4U_ID(M4U_LARB3_ID, 7) +#define M4U_PORT_JPGDEC_BSDMA MTK_M4U_ID(M4U_LARB3_ID, 8) +#define M4U_PORT_VENC_CUR_LUMA MTK_M4U_ID(M4U_LARB3_ID, 9) +#define M4U_PORT_VENC_CUR_CHROMA MTK_M4U_ID(M4U_LARB3_ID, 10) +#define M4U_PORT_VENC_REF_LUMA MTK_M4U_ID(M4U_LARB3_ID, 11) +#define M4U_PORT_VENC_REF_CHROMA MTK_M4U_ID(M4U_LARB3_ID, 12) +#define M4U_PORT_VENC_NBM_RDMA MTK_M4U_ID(M4U_LARB3_ID, 13) +#define M4U_PORT_VENC_NBM_WDMA MTK_M4U_ID(M4U_LARB3_ID, 14) + +/* larb4 */ +#define M4U_PORT_DISP_OVL1 MTK_M4U_ID(M4U_LARB4_ID, 0) +#define M4U_PORT_DISP_RDMA1 MTK_M4U_ID(M4U_LARB4_ID, 1) +#define M4U_PORT_DISP_RDMA2 MTK_M4U_ID(M4U_LARB4_ID, 2) +#define M4U_PORT_DISP_WDMA1 MTK_M4U_ID(M4U_LARB4_ID, 3) +#define M4U_PORT_MDP_RDMA1 MTK_M4U_ID(M4U_LARB4_ID, 4) +#define M4U_PORT_MDP_WROT1 MTK_M4U_ID(M4U_LARB4_ID, 5) + +/* larb5 */ +#define M4U_PORT_VENC_RCPU_SET2 MTK_M4U_ID(M4U_LARB5_ID, 0) +#define M4U_PORT_VENC_REC_FRM_SET2 MTK_M4U_ID(M4U_LARB5_ID, 1) +#define M4U_PORT_VENC_REF_LUMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 2) +#define M4U_PORT_VENC_REC_CHROMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 3) +#define M4U_PORT_VENC_BSDMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 4) +#define M4U_PORT_VENC_CUR_LUMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 5) +#define M4U_PORT_VENC_CUR_CHROMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 6) +#define M4U_PORT_VENC_RD_COMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 7) +#define M4U_PORT_VENC_SV_COMA_SET2 MTK_M4U_ID(M4U_LARB5_ID, 8) + +#endif -- cgit v0.10.2 From cc8bbe1a83128ad06457e4dc69907c4f9a6fc1a7 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 23 Feb 2016 01:20:49 +0800 Subject: memory: mediatek: Add SMI driver This patch add SMI(Smart Multimedia Interface) driver. This driver is responsible to enable/disable iommu and control the power domain and clocks of each local arbiter. Signed-off-by: Yong Wu Tested-by: Philipp Zabel Reviewed-by: Daniel Kurtz Tested-by: Daniel Kurtz Signed-off-by: Matthias Brugger Signed-off-by: Joerg Roedel diff --git a/drivers/memory/Kconfig b/drivers/memory/Kconfig index 6f31546..51d5cd2 100644 --- a/drivers/memory/Kconfig +++ b/drivers/memory/Kconfig @@ -114,6 +114,14 @@ config JZ4780_NEMC the Ingenic JZ4780. This controller is used to handle external memory devices such as NAND and SRAM. +config MTK_SMI + bool + depends on ARCH_MEDIATEK || COMPILE_TEST + help + This driver is for the Memory Controller module in MediaTek SoCs, + mainly help enable/disable iommu and control the power domain and + clocks for each local arbiter. + source "drivers/memory/tegra/Kconfig" endif diff --git a/drivers/memory/Makefile b/drivers/memory/Makefile index 1c46af5..890bdf4 100644 --- a/drivers/memory/Makefile +++ b/drivers/memory/Makefile @@ -15,5 +15,6 @@ obj-$(CONFIG_FSL_IFC) += fsl_ifc.o obj-$(CONFIG_MVEBU_DEVBUS) += mvebu-devbus.o obj-$(CONFIG_TEGRA20_MC) += tegra20-mc.o obj-$(CONFIG_JZ4780_NEMC) += jz4780-nemc.o +obj-$(CONFIG_MTK_SMI) += mtk-smi.o obj-$(CONFIG_TEGRA_MC) += tegra/ diff --git a/drivers/memory/mtk-smi.c b/drivers/memory/mtk-smi.c new file mode 100644 index 0000000..089091f --- /dev/null +++ b/drivers/memory/mtk-smi.c @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Yong Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define SMI_LARB_MMU_EN 0xf00 + +struct mtk_smi { + struct device *dev; + struct clk *clk_apb, *clk_smi; +}; + +struct mtk_smi_larb { /* larb: local arbiter */ + struct mtk_smi smi; + void __iomem *base; + struct device *smi_common_dev; + u32 *mmu; +}; + +static int mtk_smi_enable(const struct mtk_smi *smi) +{ + int ret; + + ret = pm_runtime_get_sync(smi->dev); + if (ret < 0) + return ret; + + ret = clk_prepare_enable(smi->clk_apb); + if (ret) + goto err_put_pm; + + ret = clk_prepare_enable(smi->clk_smi); + if (ret) + goto err_disable_apb; + + return 0; + +err_disable_apb: + clk_disable_unprepare(smi->clk_apb); +err_put_pm: + pm_runtime_put_sync(smi->dev); + return ret; +} + +static void mtk_smi_disable(const struct mtk_smi *smi) +{ + clk_disable_unprepare(smi->clk_smi); + clk_disable_unprepare(smi->clk_apb); + pm_runtime_put_sync(smi->dev); +} + +int mtk_smi_larb_get(struct device *larbdev) +{ + struct mtk_smi_larb *larb = dev_get_drvdata(larbdev); + struct mtk_smi *common = dev_get_drvdata(larb->smi_common_dev); + int ret; + + /* Enable the smi-common's power and clocks */ + ret = mtk_smi_enable(common); + if (ret) + return ret; + + /* Enable the larb's power and clocks */ + ret = mtk_smi_enable(&larb->smi); + if (ret) { + mtk_smi_disable(common); + return ret; + } + + /* Configure the iommu info for this larb */ + writel(*larb->mmu, larb->base + SMI_LARB_MMU_EN); + + return 0; +} + +void mtk_smi_larb_put(struct device *larbdev) +{ + struct mtk_smi_larb *larb = dev_get_drvdata(larbdev); + struct mtk_smi *common = dev_get_drvdata(larb->smi_common_dev); + + /* + * Don't de-configure the iommu info for this larb since there may be + * several modules in this larb. + * The iommu info will be reset after power off. + */ + + mtk_smi_disable(&larb->smi); + mtk_smi_disable(common); +} + +static int +mtk_smi_larb_bind(struct device *dev, struct device *master, void *data) +{ + struct mtk_smi_larb *larb = dev_get_drvdata(dev); + struct mtk_smi_iommu *smi_iommu = data; + unsigned int i; + + for (i = 0; i < smi_iommu->larb_nr; i++) { + if (dev == smi_iommu->larb_imu[i].dev) { + /* The 'mmu' may be updated in iommu-attach/detach. */ + larb->mmu = &smi_iommu->larb_imu[i].mmu; + return 0; + } + } + return -ENODEV; +} + +static void +mtk_smi_larb_unbind(struct device *dev, struct device *master, void *data) +{ + /* Do nothing as the iommu is always enabled. */ +} + +static const struct component_ops mtk_smi_larb_component_ops = { + .bind = mtk_smi_larb_bind, + .unbind = mtk_smi_larb_unbind, +}; + +static int mtk_smi_larb_probe(struct platform_device *pdev) +{ + struct mtk_smi_larb *larb; + struct resource *res; + struct device *dev = &pdev->dev; + struct device_node *smi_node; + struct platform_device *smi_pdev; + + if (!dev->pm_domain) + return -EPROBE_DEFER; + + larb = devm_kzalloc(dev, sizeof(*larb), GFP_KERNEL); + if (!larb) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + larb->base = devm_ioremap_resource(dev, res); + if (IS_ERR(larb->base)) + return PTR_ERR(larb->base); + + larb->smi.clk_apb = devm_clk_get(dev, "apb"); + if (IS_ERR(larb->smi.clk_apb)) + return PTR_ERR(larb->smi.clk_apb); + + larb->smi.clk_smi = devm_clk_get(dev, "smi"); + if (IS_ERR(larb->smi.clk_smi)) + return PTR_ERR(larb->smi.clk_smi); + larb->smi.dev = dev; + + smi_node = of_parse_phandle(dev->of_node, "mediatek,smi", 0); + if (!smi_node) + return -EINVAL; + + smi_pdev = of_find_device_by_node(smi_node); + of_node_put(smi_node); + if (smi_pdev) { + larb->smi_common_dev = &smi_pdev->dev; + } else { + dev_err(dev, "Failed to get the smi_common device\n"); + return -EINVAL; + } + + pm_runtime_enable(dev); + platform_set_drvdata(pdev, larb); + return component_add(dev, &mtk_smi_larb_component_ops); +} + +static int mtk_smi_larb_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + component_del(&pdev->dev, &mtk_smi_larb_component_ops); + return 0; +} + +static const struct of_device_id mtk_smi_larb_of_ids[] = { + { .compatible = "mediatek,mt8173-smi-larb",}, + {} +}; + +static struct platform_driver mtk_smi_larb_driver = { + .probe = mtk_smi_larb_probe, + .remove = mtk_smi_larb_remove, + .driver = { + .name = "mtk-smi-larb", + .of_match_table = mtk_smi_larb_of_ids, + } +}; + +static int mtk_smi_common_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct mtk_smi *common; + + if (!dev->pm_domain) + return -EPROBE_DEFER; + + common = devm_kzalloc(dev, sizeof(*common), GFP_KERNEL); + if (!common) + return -ENOMEM; + common->dev = dev; + + common->clk_apb = devm_clk_get(dev, "apb"); + if (IS_ERR(common->clk_apb)) + return PTR_ERR(common->clk_apb); + + common->clk_smi = devm_clk_get(dev, "smi"); + if (IS_ERR(common->clk_smi)) + return PTR_ERR(common->clk_smi); + + pm_runtime_enable(dev); + platform_set_drvdata(pdev, common); + return 0; +} + +static int mtk_smi_common_remove(struct platform_device *pdev) +{ + pm_runtime_disable(&pdev->dev); + return 0; +} + +static const struct of_device_id mtk_smi_common_of_ids[] = { + { .compatible = "mediatek,mt8173-smi-common", }, + {} +}; + +static struct platform_driver mtk_smi_common_driver = { + .probe = mtk_smi_common_probe, + .remove = mtk_smi_common_remove, + .driver = { + .name = "mtk-smi-common", + .of_match_table = mtk_smi_common_of_ids, + } +}; + +static int __init mtk_smi_init(void) +{ + int ret; + + ret = platform_driver_register(&mtk_smi_common_driver); + if (ret != 0) { + pr_err("Failed to register SMI driver\n"); + return ret; + } + + ret = platform_driver_register(&mtk_smi_larb_driver); + if (ret != 0) { + pr_err("Failed to register SMI-LARB driver\n"); + goto err_unreg_smi; + } + return ret; + +err_unreg_smi: + platform_driver_unregister(&mtk_smi_common_driver); + return ret; +} +subsys_initcall(mtk_smi_init); diff --git a/include/soc/mediatek/smi.h b/include/soc/mediatek/smi.h new file mode 100644 index 0000000..8893c5e --- /dev/null +++ b/include/soc/mediatek/smi.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Yong Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#ifndef MTK_IOMMU_SMI_H +#define MTK_IOMMU_SMI_H + +#include +#include + +#ifdef CONFIG_MTK_SMI + +#define MTK_LARB_NR_MAX 8 + +#define MTK_SMI_MMU_EN(port) BIT(port) + +struct mtk_smi_larb_iommu { + struct device *dev; + unsigned int mmu; +}; + +struct mtk_smi_iommu { + unsigned int larb_nr; + struct mtk_smi_larb_iommu larb_imu[MTK_LARB_NR_MAX]; +}; + +/* + * mtk_smi_larb_get: Enable the power domain and clocks for this local arbiter. + * It also initialize some basic setting(like iommu). + * mtk_smi_larb_put: Disable the power domain and clocks for this local arbiter. + * Both should be called in non-atomic context. + * + * Returns 0 if successful, negative on failure. + */ +int mtk_smi_larb_get(struct device *larbdev); +void mtk_smi_larb_put(struct device *larbdev); + +#else + +static inline int mtk_smi_larb_get(struct device *larbdev) +{ + return 0; +} + +static inline void mtk_smi_larb_put(struct device *larbdev) { } + +#endif + +#endif -- cgit v0.10.2 From 0df4fabe208d9576f2671d31e77cf46d20fdcd01 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 23 Feb 2016 01:20:50 +0800 Subject: iommu/mediatek: Add mt8173 IOMMU driver This patch adds support for mediatek m4u (MultiMedia Memory Management Unit). Signed-off-by: Yong Wu Reviewed-by: Robin Murphy Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index dc1aaa5..6df982f 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -337,4 +337,20 @@ config S390_IOMMU help Support for the IOMMU API for s390 PCI devices. +config MTK_IOMMU + bool "MTK IOMMU Support" + depends on ARM || ARM64 + depends on ARCH_MEDIATEK || COMPILE_TEST + select IOMMU_API + select IOMMU_DMA + select IOMMU_IO_PGTABLE_ARMV7S + select MEMORY + select MTK_SMI + help + Support for the M4U on certain Mediatek SOCs. M4U is MultiMedia + Memory Management Unit. This option enables remapping of DMA memory + accesses for the multimedia subsystem. + + If unsure, say N here. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 2f9bfbc..c6edb31 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o +obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c new file mode 100644 index 0000000..721ffdb --- /dev/null +++ b/drivers/iommu/mtk_iommu.c @@ -0,0 +1,737 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Yong Wu + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io-pgtable.h" + +#define REG_MMU_PT_BASE_ADDR 0x000 + +#define REG_MMU_INVALIDATE 0x020 +#define F_ALL_INVLD 0x2 +#define F_MMU_INV_RANGE 0x1 + +#define REG_MMU_INVLD_START_A 0x024 +#define REG_MMU_INVLD_END_A 0x028 + +#define REG_MMU_INV_SEL 0x038 +#define F_INVLD_EN0 BIT(0) +#define F_INVLD_EN1 BIT(1) + +#define REG_MMU_STANDARD_AXI_MODE 0x048 +#define REG_MMU_DCM_DIS 0x050 + +#define REG_MMU_CTRL_REG 0x110 +#define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) +#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) + +#define REG_MMU_IVRP_PADDR 0x114 +#define F_MMU_IVRP_PA_SET(pa) ((pa) >> 1) + +#define REG_MMU_INT_CONTROL0 0x120 +#define F_L2_MULIT_HIT_EN BIT(0) +#define F_TABLE_WALK_FAULT_INT_EN BIT(1) +#define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) +#define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) +#define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) +#define F_MISS_FIFO_ERR_INT_EN BIT(6) +#define F_INT_CLR_BIT BIT(12) + +#define REG_MMU_INT_MAIN_CONTROL 0x124 +#define F_INT_TRANSLATION_FAULT BIT(0) +#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1) +#define F_INT_INVALID_PA_FAULT BIT(2) +#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3) +#define F_INT_TLB_MISS_FAULT BIT(4) +#define F_INT_MISS_TRANSACTION_FIFO_FAULT BIT(5) +#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT BIT(6) + +#define REG_MMU_CPE_DONE 0x12C + +#define REG_MMU_FAULT_ST1 0x134 + +#define REG_MMU_FAULT_VA 0x13c +#define F_MMU_FAULT_VA_MSK 0xfffff000 +#define F_MMU_FAULT_VA_WRITE_BIT BIT(1) +#define F_MMU_FAULT_VA_LAYER_BIT BIT(0) + +#define REG_MMU_INVLD_PA 0x140 +#define REG_MMU_INT_ID 0x150 +#define F_MMU0_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) +#define F_MMU0_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) + +#define MTK_PROTECT_PA_ALIGN 128 + +struct mtk_iommu_suspend_reg { + u32 standard_axi_mode; + u32 dcm_dis; + u32 ctrl_reg; + u32 int_control0; + u32 int_main_control; +}; + +struct mtk_iommu_client_priv { + struct list_head client; + unsigned int mtk_m4u_id; + struct device *m4udev; +}; + +struct mtk_iommu_domain { + spinlock_t pgtlock; /* lock for page table */ + + struct io_pgtable_cfg cfg; + struct io_pgtable_ops *iop; + + struct iommu_domain domain; +}; + +struct mtk_iommu_data { + void __iomem *base; + int irq; + struct device *dev; + struct clk *bclk; + phys_addr_t protect_base; /* protect memory base */ + struct mtk_iommu_suspend_reg reg; + struct mtk_iommu_domain *m4u_dom; + struct iommu_group *m4u_group; + struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ +}; + +static struct iommu_ops mtk_iommu_ops; + +static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct mtk_iommu_domain, domain); +} + +static void mtk_iommu_tlb_flush_all(void *cookie) +{ + struct mtk_iommu_data *data = cookie; + + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); + writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); + wmb(); /* Make sure the tlb flush all done */ +} + +static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size, + size_t granule, bool leaf, + void *cookie) +{ + struct mtk_iommu_data *data = cookie; + + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); + + writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); + writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A); + writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); +} + +static void mtk_iommu_tlb_sync(void *cookie) +{ + struct mtk_iommu_data *data = cookie; + int ret; + u32 tmp; + + ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp, + tmp != 0, 10, 100000); + if (ret) { + dev_warn(data->dev, + "Partial TLB flush timed out, falling back to full flush\n"); + mtk_iommu_tlb_flush_all(cookie); + } + /* Clear the CPE status */ + writel_relaxed(0, data->base + REG_MMU_CPE_DONE); +} + +static const struct iommu_gather_ops mtk_iommu_gather_ops = { + .tlb_flush_all = mtk_iommu_tlb_flush_all, + .tlb_add_flush = mtk_iommu_tlb_add_flush_nosync, + .tlb_sync = mtk_iommu_tlb_sync, +}; + +static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) +{ + struct mtk_iommu_data *data = dev_id; + struct mtk_iommu_domain *dom = data->m4u_dom; + u32 int_state, regval, fault_iova, fault_pa; + unsigned int fault_larb, fault_port; + bool layer, write; + + /* Read error info from registers */ + int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); + fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA); + layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; + write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; + fault_iova &= F_MMU_FAULT_VA_MSK; + fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA); + regval = readl_relaxed(data->base + REG_MMU_INT_ID); + fault_larb = F_MMU0_INT_ID_LARB_ID(regval); + fault_port = F_MMU0_INT_ID_PORT_ID(regval); + + if (report_iommu_fault(&dom->domain, data->dev, fault_iova, + write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { + dev_err_ratelimited( + data->dev, + "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n", + int_state, fault_iova, fault_pa, fault_larb, fault_port, + layer, write ? "write" : "read"); + } + + /* Interrupt clear */ + regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); + regval |= F_INT_CLR_BIT; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + + mtk_iommu_tlb_flush_all(data); + + return IRQ_HANDLED; +} + +static void mtk_iommu_config(struct mtk_iommu_data *data, + struct device *dev, bool enable) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + struct mtk_smi_larb_iommu *larb_mmu; + unsigned int larbid, portid; + + head = dev->archdata.iommu; + list_for_each_entry_safe(cur, next, &head->client, client) { + larbid = MTK_M4U_TO_LARB(cur->mtk_m4u_id); + portid = MTK_M4U_TO_PORT(cur->mtk_m4u_id); + larb_mmu = &data->smi_imu.larb_imu[larbid]; + + dev_dbg(dev, "%s iommu port: %d\n", + enable ? "enable" : "disable", portid); + + if (enable) + larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); + else + larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + } +} + +static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) +{ + struct mtk_iommu_domain *dom = data->m4u_dom; + + spin_lock_init(&dom->pgtlock); + + dom->cfg = (struct io_pgtable_cfg) { + .quirks = IO_PGTABLE_QUIRK_ARM_NS | + IO_PGTABLE_QUIRK_NO_PERMS | + IO_PGTABLE_QUIRK_TLBI_ON_MAP, + .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, + .ias = 32, + .oas = 32, + .tlb = &mtk_iommu_gather_ops, + .iommu_dev = data->dev, + }; + + dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); + if (!dom->iop) { + dev_err(data->dev, "Failed to alloc io pgtable\n"); + return -EINVAL; + } + + /* Update our support page sizes bitmap */ + mtk_iommu_ops.pgsize_bitmap = dom->cfg.pgsize_bitmap; + + writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], + data->base + REG_MMU_PT_BASE_ADDR); + return 0; +} + +static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) +{ + struct mtk_iommu_domain *dom; + + if (type != IOMMU_DOMAIN_DMA) + return NULL; + + dom = kzalloc(sizeof(*dom), GFP_KERNEL); + if (!dom) + return NULL; + + if (iommu_get_dma_cookie(&dom->domain)) { + kfree(dom); + return NULL; + } + + dom->domain.geometry.aperture_start = 0; + dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); + dom->domain.geometry.force_aperture = true; + + return &dom->domain; +} + +static void mtk_iommu_domain_free(struct iommu_domain *domain) +{ + iommu_put_dma_cookie(domain); + kfree(to_mtk_domain(domain)); +} + +static int mtk_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + int ret; + + if (!priv) + return -ENODEV; + + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_dom) { + data->m4u_dom = dom; + ret = mtk_iommu_domain_finalise(data); + if (ret) { + data->m4u_dom = NULL; + return ret; + } + } else if (data->m4u_dom != dom) { + /* All the client devices should be in the same m4u domain */ + dev_err(dev, "try to attach into the error iommu domain\n"); + return -EPERM; + } + + mtk_iommu_config(data, dev, true); + return 0; +} + +static void mtk_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + + if (!priv) + return; + + data = dev_get_drvdata(priv->m4udev); + mtk_iommu_config(data, dev, false); +} + +static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + int ret; + + spin_lock_irqsave(&dom->pgtlock, flags); + ret = dom->iop->map(dom->iop, iova, paddr, size, prot); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return ret; +} + +static size_t mtk_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + size_t unmapsz; + + spin_lock_irqsave(&dom->pgtlock, flags); + unmapsz = dom->iop->unmap(dom->iop, iova, size); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return unmapsz; +} + +static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + phys_addr_t pa; + + spin_lock_irqsave(&dom->pgtlock, flags); + pa = dom->iop->iova_to_phys(dom->iop, iova); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return pa; +} + +static int mtk_iommu_add_device(struct device *dev) +{ + struct iommu_group *group; + + if (!dev->archdata.iommu) /* Not a iommu client device */ + return -ENODEV; + + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + iommu_group_put(group); + return 0; +} + +static void mtk_iommu_remove_device(struct device *dev) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + + head = dev->archdata.iommu; + if (!head) + return; + + list_for_each_entry_safe(cur, next, &head->client, client) { + list_del(&cur->client); + kfree(cur); + } + kfree(head); + dev->archdata.iommu = NULL; + + iommu_group_remove_device(dev); +} + +static struct iommu_group *mtk_iommu_device_group(struct device *dev) +{ + struct mtk_iommu_data *data; + struct mtk_iommu_client_priv *priv; + + priv = dev->archdata.iommu; + if (!priv) + return ERR_PTR(-ENODEV); + + /* All the client devices are in the same m4u iommu-group */ + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_group) { + data->m4u_group = iommu_group_alloc(); + if (IS_ERR(data->m4u_group)) + dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + } + return data->m4u_group; +} + +static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + struct mtk_iommu_client_priv *head, *priv, *next; + struct platform_device *m4updev; + + if (args->args_count != 1) { + dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", + args->args_count); + return -EINVAL; + } + + if (!dev->archdata.iommu) { + /* Get the m4u device */ + m4updev = of_find_device_by_node(args->np); + of_node_put(args->np); + if (WARN_ON(!m4updev)) + return -EINVAL; + + head = kzalloc(sizeof(*head), GFP_KERNEL); + if (!head) + return -ENOMEM; + + dev->archdata.iommu = head; + INIT_LIST_HEAD(&head->client); + head->m4udev = &m4updev->dev; + } else { + head = dev->archdata.iommu; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + goto err_free_mem; + + priv->mtk_m4u_id = args->args[0]; + list_add_tail(&priv->client, &head->client); + + return 0; + +err_free_mem: + list_for_each_entry_safe(priv, next, &head->client, client) + kfree(priv); + kfree(head); + dev->archdata.iommu = NULL; + return -ENOMEM; +} + +static struct iommu_ops mtk_iommu_ops = { + .domain_alloc = mtk_iommu_domain_alloc, + .domain_free = mtk_iommu_domain_free, + .attach_dev = mtk_iommu_attach_device, + .detach_dev = mtk_iommu_detach_device, + .map = mtk_iommu_map, + .unmap = mtk_iommu_unmap, + .map_sg = default_iommu_map_sg, + .iova_to_phys = mtk_iommu_iova_to_phys, + .add_device = mtk_iommu_add_device, + .remove_device = mtk_iommu_remove_device, + .device_group = mtk_iommu_device_group, + .of_xlate = mtk_iommu_of_xlate, + .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, +}; + +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) +{ + u32 regval; + int ret; + + ret = clk_prepare_enable(data->bclk); + if (ret) { + dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); + return ret; + } + + regval = F_MMU_PREFETCH_RT_REPLACE_MOD | + F_MMU_TF_PROTECT_SEL(2); + writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); + + regval = F_L2_MULIT_HIT_EN | + F_TABLE_WALK_FAULT_INT_EN | + F_PREETCH_FIFO_OVERFLOW_INT_EN | + F_MISS_FIFO_OVERFLOW_INT_EN | + F_PREFETCH_FIFO_ERR_INT_EN | + F_MISS_FIFO_ERR_INT_EN; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + + regval = F_INT_TRANSLATION_FAULT | + F_INT_MAIN_MULTI_HIT_FAULT | + F_INT_INVALID_PA_FAULT | + F_INT_ENTRY_REPLACEMENT_FAULT | + F_INT_TLB_MISS_FAULT | + F_INT_MISS_TRANSACTION_FIFO_FAULT | + F_INT_PRETETCH_TRANSATION_FIFO_FAULT; + writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); + + writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base), + data->base + REG_MMU_IVRP_PADDR); + + writel_relaxed(0, data->base + REG_MMU_DCM_DIS); + writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE); + + if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, + dev_name(data->dev), (void *)data)) { + writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); + clk_disable_unprepare(data->bclk); + dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); + return -ENODEV; + } + + return 0; +} + +static int compare_of(struct device *dev, void *data) +{ + return dev->of_node == data; +} + +static int mtk_iommu_bind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + return component_bind_all(dev, &data->smi_imu); +} + +static void mtk_iommu_unbind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + component_unbind_all(dev, &data->smi_imu); +} + +static const struct component_master_ops mtk_iommu_com_ops = { + .bind = mtk_iommu_bind, + .unbind = mtk_iommu_unbind, +}; + +static int mtk_iommu_probe(struct platform_device *pdev) +{ + struct mtk_iommu_data *data; + struct device *dev = &pdev->dev; + struct resource *res; + struct component_match *match = NULL; + void *protect; + unsigned int i, larb_nr; + int ret; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + data->dev = dev; + + /* Protect memory. HW will access here while translation fault.*/ + protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); + if (!protect) + return -ENOMEM; + data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->base)) + return PTR_ERR(data->base); + + data->irq = platform_get_irq(pdev, 0); + if (data->irq < 0) + return data->irq; + + data->bclk = devm_clk_get(dev, "bclk"); + if (IS_ERR(data->bclk)) + return PTR_ERR(data->bclk); + + larb_nr = of_count_phandle_with_args(dev->of_node, + "mediatek,larbs", NULL); + if (larb_nr < 0) + return larb_nr; + data->smi_imu.larb_nr = larb_nr; + + for (i = 0; i < larb_nr; i++) { + struct device_node *larbnode; + struct platform_device *plarbdev; + + larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); + if (!larbnode) + return -EINVAL; + + if (!of_device_is_available(larbnode)) + continue; + + plarbdev = of_find_device_by_node(larbnode); + of_node_put(larbnode); + if (!plarbdev) { + plarbdev = of_platform_device_create( + larbnode, NULL, + platform_bus_type.dev_root); + if (IS_ERR(plarbdev)) + return -EPROBE_DEFER; + } + data->smi_imu.larb_imu[i].dev = &plarbdev->dev; + + component_match_add(dev, &match, compare_of, larbnode); + } + + platform_set_drvdata(pdev, data); + + ret = mtk_iommu_hw_init(data); + if (ret) + return ret; + + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + + return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); +} + +static int mtk_iommu_remove(struct platform_device *pdev) +{ + struct mtk_iommu_data *data = platform_get_drvdata(pdev); + + if (iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, NULL); + + free_io_pgtable_ops(data->m4u_dom->iop); + clk_disable_unprepare(data->bclk); + devm_free_irq(&pdev->dev, data->irq, data); + component_master_del(&pdev->dev, &mtk_iommu_com_ops); + return 0; +} + +static int mtk_iommu_suspend(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + reg->standard_axi_mode = readl_relaxed(base + + REG_MMU_STANDARD_AXI_MODE); + reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); + reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); + reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); + reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); + return 0; +} + +static int mtk_iommu_resume(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + writel_relaxed(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], + base + REG_MMU_PT_BASE_ADDR); + writel_relaxed(reg->standard_axi_mode, + base + REG_MMU_STANDARD_AXI_MODE); + writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); + writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); + writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); + writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); + writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base), + base + REG_MMU_IVRP_PADDR); + return 0; +} + +const struct dev_pm_ops mtk_iommu_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) +}; + +static const struct of_device_id mtk_iommu_of_ids[] = { + { .compatible = "mediatek,mt8173-m4u", }, + {} +}; + +static struct platform_driver mtk_iommu_driver = { + .probe = mtk_iommu_probe, + .remove = mtk_iommu_remove, + .driver = { + .name = "mtk-iommu", + .of_match_table = mtk_iommu_of_ids, + .pm = &mtk_iommu_pm_ops, + } +}; + +static int mtk_iommu_init_fn(struct device_node *np) +{ + int ret; + struct platform_device *pdev; + + pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + + ret = platform_driver_register(&mtk_iommu_driver); + if (ret) { + pr_err("%s: Failed to register driver\n", __func__); + return ret; + } + + of_iommu_set_ops(np, &mtk_iommu_ops); + return 0; +} + +IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn); -- cgit v0.10.2 From 5ff6b3a6d3915b4238cfc2725362d586c9927e92 Mon Sep 17 00:00:00 2001 From: Yong Wu Date: Tue, 23 Feb 2016 01:20:51 +0800 Subject: dts: mt8173: Add iommu/smi nodes for mt8173 This patch add the iommu/larbs nodes for mt8173 Signed-off-by: Yong Wu Reviewed-by: Daniel Kurtz Signed-off-by: Joerg Roedel diff --git a/arch/arm64/boot/dts/mediatek/mt8173.dtsi b/arch/arm64/boot/dts/mediatek/mt8173.dtsi index ec135ea..8048811 100644 --- a/arch/arm64/boot/dts/mediatek/mt8173.dtsi +++ b/arch/arm64/boot/dts/mediatek/mt8173.dtsi @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -277,6 +278,17 @@ reg = <0 0x10200620 0 0x20>; }; + iommu: iommu@10205000 { + compatible = "mediatek,mt8173-m4u"; + reg = <0 0x10205000 0 0x1000>; + interrupts = ; + clocks = <&infracfg CLK_INFRA_M4U>; + clock-names = "bclk"; + mediatek,larbs = <&larb0 &larb1 &larb2 + &larb3 &larb4 &larb5>; + #iommu-cells = <1>; + }; + apmixedsys: clock-controller@10209000 { compatible = "mediatek,mt8173-apmixedsys"; reg = <0 0x10209000 0 0x1000>; @@ -589,29 +601,98 @@ status = "disabled"; }; + larb0: larb@14021000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x14021000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_MM>; + clocks = <&mmsys CLK_MM_SMI_LARB0>, + <&mmsys CLK_MM_SMI_LARB0>; + clock-names = "apb", "smi"; + }; + + smi_common: smi@14022000 { + compatible = "mediatek,mt8173-smi-common"; + reg = <0 0x14022000 0 0x1000>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_MM>; + clocks = <&mmsys CLK_MM_SMI_COMMON>, + <&mmsys CLK_MM_SMI_COMMON>; + clock-names = "apb", "smi"; + }; + + larb4: larb@14027000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x14027000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_MM>; + clocks = <&mmsys CLK_MM_SMI_LARB4>, + <&mmsys CLK_MM_SMI_LARB4>; + clock-names = "apb", "smi"; + }; + imgsys: clock-controller@15000000 { compatible = "mediatek,mt8173-imgsys", "syscon"; reg = <0 0x15000000 0 0x1000>; #clock-cells = <1>; }; + larb2: larb@15001000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x15001000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_ISP>; + clocks = <&imgsys CLK_IMG_LARB2_SMI>, + <&imgsys CLK_IMG_LARB2_SMI>; + clock-names = "apb", "smi"; + }; + vdecsys: clock-controller@16000000 { compatible = "mediatek,mt8173-vdecsys", "syscon"; reg = <0 0x16000000 0 0x1000>; #clock-cells = <1>; }; + larb1: larb@16010000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x16010000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_VDEC>; + clocks = <&vdecsys CLK_VDEC_CKEN>, + <&vdecsys CLK_VDEC_LARB_CKEN>; + clock-names = "apb", "smi"; + }; + vencsys: clock-controller@18000000 { compatible = "mediatek,mt8173-vencsys", "syscon"; reg = <0 0x18000000 0 0x1000>; #clock-cells = <1>; }; + larb3: larb@18001000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x18001000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_VENC>; + clocks = <&vencsys CLK_VENC_CKE1>, + <&vencsys CLK_VENC_CKE0>; + clock-names = "apb", "smi"; + }; + vencltsys: clock-controller@19000000 { compatible = "mediatek,mt8173-vencltsys", "syscon"; reg = <0 0x19000000 0 0x1000>; #clock-cells = <1>; }; + + larb5: larb@19001000 { + compatible = "mediatek,mt8173-smi-larb"; + reg = <0 0x19001000 0 0x1000>; + mediatek,smi = <&smi_common>; + power-domains = <&scpsys MT8173_POWER_DOMAIN_VENC_LT>; + clocks = <&vencltsys CLK_VENCLT_CKE1>, + <&vencltsys CLK_VENCLT_CKE0>; + clock-names = "apb", "smi"; + }; }; }; -- cgit v0.10.2 From dbf6c6efa7dd9bea4db038ad7351b75615249046 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 29 Feb 2016 09:45:59 +0100 Subject: iommu/exynos: Pointers are nto physical addresses The exynos iommu driver changed an incorrect cast from pointer to 'unsigned int' to an equally incorrect cast to a 'phys_addr_t', which results in an obvious compile-time error when phys_addr_t is wider than pointers are: drivers/iommu/exynos-iommu.c: In function 'alloc_lv2entry': drivers/iommu/exynos-iommu.c:918:32: error: cast from pointer to integer of different size [-Werror=pointer-to-int-cast] The code does not actually want the physical address (which would involve using virt_to_phys()), but just checks the alignment, so we can change it to use a cast to uintptr_t instead. Signed-off-by: Arnd Bergmann Fixes: 740a01eee9ad ("iommu/exynos: Add support for v5 SYSMMU") Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index b066504..484b3b3 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -915,7 +915,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, bool need_flush_flpd_cache = lv1ent_zero(sent); pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); - BUG_ON((phys_addr_t)pent & (LV2TABLE_SIZE - 1)); + BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1)); if (!pent) return ERR_PTR(-ENOMEM); -- cgit v0.10.2 From 84bd042865cabfe5d91f9e2e7683702f1656bb15 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Mon, 29 Feb 2016 13:42:57 +0100 Subject: iommu/exynos: Use proper readl/writel register interface Drivers should use generic readl/writel calls to access HW registers, so replace all __raw_readl/writel with generic version. Suggested-by: Arnd Bergmann Signed-off-by: Marek Szyprowski Acked-by: Arnd Bergmann Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 484b3b3..5ecc86c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -268,18 +268,18 @@ static bool is_sysmmu_active(struct sysmmu_drvdata *data) static void sysmmu_unblock(struct sysmmu_drvdata *data) { - __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); } static bool sysmmu_block(struct sysmmu_drvdata *data) { int i = 120; - __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); - while ((i > 0) && !(__raw_readl(data->sfrbase + REG_MMU_STATUS) & 1)) + writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1)) --i; - if (!(__raw_readl(data->sfrbase + REG_MMU_STATUS) & 1)) { + if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) { sysmmu_unblock(data); return false; } @@ -290,9 +290,9 @@ static bool sysmmu_block(struct sysmmu_drvdata *data) static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) { if (MMU_MAJ_VER(data->version) < 5) - __raw_writel(0x1, data->sfrbase + REG_MMU_FLUSH); + writel(0x1, data->sfrbase + REG_MMU_FLUSH); else - __raw_writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); + writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); } static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -302,10 +302,10 @@ static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, for (i = 0; i < num_inv; i++) { if (MMU_MAJ_VER(data->version) < 5) - __raw_writel((iova & SPAGE_MASK) | 1, + writel((iova & SPAGE_MASK) | 1, data->sfrbase + REG_MMU_FLUSH_ENTRY); else - __raw_writel((iova & SPAGE_MASK) | 1, + writel((iova & SPAGE_MASK) | 1, data->sfrbase + REG_V5_MMU_FLUSH_ENTRY); iova += SPAGE_SIZE; } @@ -314,9 +314,9 @@ static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) { if (MMU_MAJ_VER(data->version) < 5) - __raw_writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); + writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); else - __raw_writel(pgd >> PAGE_SHIFT, + writel(pgd >> PAGE_SHIFT, data->sfrbase + REG_V5_PT_BASE_PFN); __sysmmu_tlb_invalidate(data); @@ -331,7 +331,7 @@ static void __sysmmu_get_version(struct sysmmu_drvdata *data) clk_enable(data->pclk); clk_enable(data->aclk); - ver = __raw_readl(data->sfrbase + REG_MMU_VERSION); + ver = readl(data->sfrbase + REG_MMU_VERSION); /* controllers on some SoCs don't report proper version */ if (ver == 0x80000001u) @@ -392,7 +392,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) clk_enable(data->clk_master); - itype = __ffs(__raw_readl(data->sfrbase + reg_status)); + itype = __ffs(readl(data->sfrbase + reg_status)); for (i = 0; i < n; i++, finfo++) if (finfo->bit == itype) break; @@ -400,7 +400,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) BUG_ON(i == n); /* print debug message */ - fault_addr = __raw_readl(data->sfrbase + finfo->addr_reg); + fault_addr = readl(data->sfrbase + finfo->addr_reg); show_fault_information(data, finfo, fault_addr); if (data->domain) @@ -409,7 +409,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) /* fault is not recovered by fault handler */ BUG_ON(ret != 0); - __raw_writel(1 << itype, data->sfrbase + reg_clear); + writel(1 << itype, data->sfrbase + reg_clear); sysmmu_unblock(data); @@ -424,8 +424,8 @@ static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) { clk_enable(data->clk_master); - __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); - __raw_writel(0, data->sfrbase + REG_MMU_CFG); + writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); + writel(0, data->sfrbase + REG_MMU_CFG); clk_disable(data->aclk); clk_disable(data->pclk); @@ -470,7 +470,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) else cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; - __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); + writel(cfg, data->sfrbase + REG_MMU_CFG); } static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) @@ -480,13 +480,13 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) clk_enable(data->pclk); clk_enable(data->aclk); - __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); __sysmmu_init_config(data); __sysmmu_set_ptbase(data, data->pgtable); - __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); clk_disable(data->clk_master); } -- cgit v0.10.2 From 1928832f0241f0c39fcd2ce73aaa39d37db50a97 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 29 Feb 2016 10:19:06 +0100 Subject: iommu/mediatek: Select ARM_DMA_USE_IOMMU The newly added Mediatek IOMMU driver uses the IOMMU_DMA infrastructure, but unlike other such drivers, it does not select 'ARM_DMA_USE_IOMMU', which is a prerequisite, leading to a link error: warning: (MTK_IOMMU) selects IOMMU_DMA which has unmet direct dependencies (IOMMU_SUPPORT && NEED_SG_DMA_LENGTH) drivers/iommu/built-in.o: In function `iommu_put_dma_cookie': mtk_iommu.c:(.text+0x11fe): undefined reference to `put_iova_domain' drivers/iommu/built-in.o: In function `iommu_dma_init_domain': mtk_iommu.c:(.text+0x1316): undefined reference to `init_iova_domain' drivers/iommu/built-in.o: In function `__iommu_dma_unmap': mtk_iommu.c:(.text+0x1380): undefined reference to `find_iova' This adds the same select that the other drivers have. On a related note, I wonder if we should just always select ARM_DMA_USE_IOMMU whenever any IOMMU driver is enabled. Are there any cases where we would enable an IOMMU but not use it? Signed-off-by: Arnd Bergmann Fixes: 0df4fabe208d ("iommu/mediatek: Add mt8173 IOMMU driver") Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 6df982f..624510a 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -341,6 +341,7 @@ config MTK_IOMMU bool "MTK IOMMU Support" depends on ARM || ARM64 depends on ARCH_MEDIATEK || COMPILE_TEST + select ARM_DMA_USE_IOMMU select IOMMU_API select IOMMU_DMA select IOMMU_IO_PGTABLE_ARMV7S -- cgit v0.10.2 From fd99f796a20b1a805c80ee51d7349d65103e7aaf Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 29 Feb 2016 10:19:07 +0100 Subject: iommu/mediatek: Mark PM functions as __maybe_unused When CONFIG_PM is unset, we get a harmless warning for this driver: drivers/iommu/mtk_iommu.c:665:12: error: 'mtk_iommu_suspend' defined but not used [-Werror=unused-function] drivers/iommu/mtk_iommu.c:680:12: error: 'mtk_iommu_resume' defined but not used [-Werror=unused-function] Marking the functions as __maybe_unused gits rid of the two functions and lets the compiler silently drop the object code, while still doing syntax checking on them for build-time verification. Signed-off-by: Arnd Bergmann Fixes: 0df4fabe208d ("iommu/mediatek: Add mt8173 IOMMU driver") Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 721ffdb..f3c160e 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -662,7 +662,7 @@ static int mtk_iommu_remove(struct platform_device *pdev) return 0; } -static int mtk_iommu_suspend(struct device *dev) +static int __maybe_unused mtk_iommu_suspend(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; @@ -677,7 +677,7 @@ static int mtk_iommu_suspend(struct device *dev) return 0; } -static int mtk_iommu_resume(struct device *dev) +static int __maybe_unused mtk_iommu_resume(struct device *dev) { struct mtk_iommu_data *data = dev_get_drvdata(dev); struct mtk_iommu_suspend_reg *reg = &data->reg; -- cgit v0.10.2 From 59a68eb892497d32493a1fcbc4bd8f1ce13f598f Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Mon, 29 Feb 2016 11:13:39 +0000 Subject: iommu/dma: Fix NEED_SG_DMA_LENGTH dependency IOMMU_DMA does indeed depend on scatterlists having a DMA length, but the NEED_SG_DMA_LENGTH symbol should be selected, not depended upon. Reported-by: Arnd Bergmann Signed-off-by: Robin Murphy Acked-by: Arnd Bergmann Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a1e75cb..66f1184 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -51,9 +51,9 @@ config OF_IOMMU # IOMMU-agnostic DMA-mapping layer config IOMMU_DMA bool - depends on NEED_SG_DMA_LENGTH select IOMMU_API select IOMMU_IOVA + select NEED_SG_DMA_LENGTH config FSL_PAMU bool "Freescale IOMMU support" -- cgit v0.10.2 From 0b6c0ad3cfe6bc914af3ae595c9729e38f11a024 Mon Sep 17 00:00:00 2001 From: Andrzej Hajda Date: Tue, 1 Mar 2016 10:36:23 +0100 Subject: iommu/mediatek: Fix handling of of_count_phandle_with_args result The function can return negative value so it should be assigned to signed variable. The patch changes also type of related i variable to make code more compact and coherent. The problem has been detected using patch scripts/coccinelle/tests/unsigned_lesser_than_zero.cocci. Signed-off-by: Andrzej Hajda Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index f3c160e..4b9ae64 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -578,8 +578,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) struct resource *res; struct component_match *match = NULL; void *protect; - unsigned int i, larb_nr; - int ret; + int i, larb_nr, ret; data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); if (!data) -- cgit v0.10.2 From 048b31ca97eaa33a74237ed7ef542503da9c0505 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Tue, 1 Mar 2016 19:07:03 +0000 Subject: iommu/io-pgtable-armv7s: Fix kmem_cache_alloc() flags Whilst the default SLUB allocator happily just merges the original allocation flags from kmem_cache_create() with those passed through kmem_cache_alloc(), there is a code path in the SLAB allocator which will aggressively BUG_ON() if the cache was created with SLAB_CACHE_DMA but GFP_DMA is not specified for an allocation: kernel BUG at mm/slab.c:2536! Internal error: Oops - BUG: 0 [#1] SMP ARM Modules linked in:[ 1.299311] Modules linked in: CPU: 1 PID: 1 Comm: swapper/0 Not tainted 4.5.0-rc6-koelsch-05892-ge7e45ad53ab6795e #2270 Hardware name: Generic R8A7791 (Flattened Device Tree) task: ef422040 ti: ef442000 task.ti: ef442000 PC is at cache_alloc_refill+0x2a0/0x530 LR is at _raw_spin_unlock+0x8/0xc ... [] (cache_alloc_refill) from [] (kmem_cache_alloc+0x7c/0xd4) [] (kmem_cache_alloc) from [] (__arm_v7s_alloc_table+0x5c/0x278) [] (__arm_v7s_alloc_table) from [] (__arm_v7s_map.constprop.6+0x68/0x25c) [] (__arm_v7s_map.constprop.6) from [] (arm_v7s_map+0x34/0xa4) [] (arm_v7s_map) from [] (arm_v7s_do_selftests+0x140/0x418) [] (arm_v7s_do_selftests) from [] (do_one_initcall+0x100/0x1b4) [] (do_one_initcall) from [] (kernel_init_freeable+0x120/0x1e8) [] (kernel_init_freeable) from [] (kernel_init+0x8/0xec) [] (kernel_init) from [] (ret_from_fork+0x14/0x2c) Code: 1a000003 e7f001f2 e3130001 0a000000 (e7f001f2) ---[ end trace 190f6f6b84352efd ]--- Keep the peace by adding GFP_DMA when allocating a table. Reported-by: Geert Uytterhoeven Acked-by: Will Deacon Signed-off-by: Robin Murphy Tested-by: Geert Uytterhoeven Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c index 9fcceb1..9488e3c 100644 --- a/drivers/iommu/io-pgtable-arm-v7s.c +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -192,7 +192,7 @@ static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, if (lvl == 1) table = (void *)__get_dma_pages(__GFP_ZERO, get_order(size)); else if (lvl == 2) - table = kmem_cache_zalloc(data->l2_tables, gfp); + table = kmem_cache_zalloc(data->l2_tables, gfp | GFP_DMA); if (table && !selftest_running) { dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); if (dma_mapping_error(dev, dma)) -- cgit v0.10.2 From 3189e4905c2d08d6618d79af2c486f530bbfb3b4 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 2 Mar 2016 13:10:27 +0300 Subject: iommu/mediatek: Check for NULL instead of IS_ERR() of_platform_device_create() returns NULL on error, it never returns error pointers. Fixes: 0df4fabe208d ('iommu/mediatek: Add mt8173 IOMMU driver') Signed-off-by: Dan Carpenter Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index 4b9ae64..929a66a 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -627,7 +627,7 @@ static int mtk_iommu_probe(struct platform_device *pdev) plarbdev = of_platform_device_create( larbnode, NULL, platform_bus_type.dev_root); - if (IS_ERR(plarbdev)) + if (!plarbdev) return -EPROBE_DEFER; } data->smi_imu.larb_imu[i].dev = &plarbdev->dev; @@ -720,8 +720,8 @@ static int mtk_iommu_init_fn(struct device_node *np) struct platform_device *pdev; pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); + if (!pdev) + return -ENOMEM; ret = platform_driver_register(&mtk_iommu_driver); if (ret) { -- cgit v0.10.2 From d4e42e72e7d121be904087be9df9842b35757588 Mon Sep 17 00:00:00 2001 From: Magnus Damm Date: Mon, 29 Feb 2016 23:33:09 +0900 Subject: iommu/ipmmu-vmsa: Add r8a7795 DT binding Update the IPMMU DT binding documentation to include the r8a7795 compat string as well as the "renesas,ipmmu-main" property that on r8a7795 will be used to describe the topology and the relationship between the various cache IPMMU instances and the main IPMMU. Signed-off-by: Magnus Damm Reviewed-by: Simon Horman Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt index 48ffb38..3ed027c 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt @@ -7,23 +7,34 @@ connected to the IPMMU through a port called micro-TLB. Required Properties: - - compatible: Must contain SoC-specific and generic entries from below. + - compatible: Must contain SoC-specific and generic entry below in case + the device is compatible with the R-Car Gen2 VMSA-compatible IPMMU. - "renesas,ipmmu-r8a73a4" for the R8A73A4 (R-Mobile APE6) IPMMU. - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU. - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU. - "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU. - "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU. + - "renesas,ipmmu-r8a7795" for the R8A7795 (R-Car H3) IPMMU. - "renesas,ipmmu-vmsa" for generic R-Car Gen2 VMSA-compatible IPMMU. - reg: Base address and size of the IPMMU registers. - interrupts: Specifiers for the MMU fault interrupts. For instances that support secure mode two interrupts must be specified, for non-secure and secure mode, in that order. For instances that don't support secure mode a - single interrupt must be specified. + single interrupt must be specified. Not required for cache IPMMUs. - #iommu-cells: Must be 1. +Optional properties: + + - renesas,ipmmu-main: reference to the main IPMMU instance in two cells. + The first cell is a phandle to the main IPMMU and the second cell is + the interrupt bit number associated with the particular cache IPMMU device. + The interrupt bit number needs to match the main IPMMU IMSSTR register. + Only used by cache IPMMU instances. + + Each bus master connected to an IPMMU must reference the IPMMU in its device node with the following property: -- cgit v0.10.2