From 733cac2ade2fbb73740a1d22c05c7b006a6cf3d2 Mon Sep 17 00:00:00 2001 From: Robert Callicotte Date: Thu, 16 Apr 2015 23:32:47 -0500 Subject: iommu: Fix checkpatch warnings for Missing a blank line after declarations Fixed checkpatch warnings for missing blank line after declaration of struct. Signed-off-by: Robert Callicotte Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 9dd8208..b7c3d92 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -227,6 +227,7 @@ iova_insert_rbtree(struct rb_root *root, struct iova *iova) /* Figure out where to put new node */ while (*new) { struct iova *this = container_of(*new, struct iova, node); + parent = *new; if (iova->pfn_lo < this->pfn_lo) @@ -350,6 +351,7 @@ void free_iova(struct iova_domain *iovad, unsigned long pfn) { struct iova *iova = find_iova(iovad, pfn); + if (iova) __free_iova(iovad, iova); @@ -369,6 +371,7 @@ void put_iova_domain(struct iova_domain *iovad) node = rb_first(&iovad->rbroot); while (node) { struct iova *iova = container_of(node, struct iova, node); + rb_erase(node, &iovad->rbroot); free_iova_mem(iova); node = rb_first(&iovad->rbroot); @@ -482,6 +485,7 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to) for (node = rb_first(&from->rbroot); node; node = rb_next(node)) { struct iova *iova = container_of(node, struct iova, node); struct iova *new_iova; + new_iova = reserve_iova(to, iova->pfn_lo, iova->pfn_hi); if (!new_iova) printk(KERN_ERR "Reserve iova range %lx@%lx failed\n", -- cgit v0.10.2 From d4dd920cc2f6a948f07544caaa22bccb63378a3a Mon Sep 17 00:00:00 2001 From: Tomasz Figa Date: Mon, 20 Apr 2015 20:43:44 +0900 Subject: iommu/rockchip: Make sure that page table state is coherent To flush created mappings, current mapping code relies on the fact that during unmap the driver zaps every IOVA being unmapped and that it is enough to zap a single IOVA of page table to remove the entire page table from IOMMU cache. Based on these assumptions the driver was made to simply zap the first IOVA of the mapping being created. This is enough to invalidate first page table, which could be shared with another mapping (and thus could be already present in IOMMU cache), but unfortunately it does not do anything about the last page table that could be shared with other mappings as well. Moreover, the flushing is performed before page table contents are actually modified, so there is a race between the CPU updating the page tables and hardware that could be possibly running at the same time and triggering IOMMU look-ups, which could bring back the page tables back to the cache. To fix both issues, this patch makes the mapping code zap first and last (if they are different) IOVAs of new mapping after the page table is updated. Signed-off-by: Tomasz Figa Reviewed-by: Daniel Kurtz Tested-by: Heiko Stuebner Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 4015560..31004c0 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -551,6 +551,15 @@ static void rk_iommu_zap_iova(struct rk_iommu_domain *rk_domain, spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); } +static void rk_iommu_zap_iova_first_last(struct rk_iommu_domain *rk_domain, + dma_addr_t iova, size_t size) +{ + rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); + if (size > SPAGE_SIZE) + rk_iommu_zap_iova(rk_domain, iova + size - SPAGE_SIZE, + SPAGE_SIZE); +} + static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, dma_addr_t iova) { @@ -575,12 +584,6 @@ static u32 *rk_dte_get_page_table(struct rk_iommu_domain *rk_domain, rk_table_flush(page_table, NUM_PT_ENTRIES); rk_table_flush(dte_addr, 1); - /* - * Zap the first iova of newly allocated page table so iommu evicts - * old cached value of new dte from the iotlb. - */ - rk_iommu_zap_iova(rk_domain, iova, SPAGE_SIZE); - done: pt_phys = rk_dte_pt_address(dte); return (u32 *)phys_to_virt(pt_phys); @@ -630,6 +633,14 @@ static int rk_iommu_map_iova(struct rk_iommu_domain *rk_domain, u32 *pte_addr, rk_table_flush(pte_addr, pte_count); + /* + * Zap the first and last iova to evict from iotlb any previously + * mapped cachelines holding stale values for its dte and pte. + * We only zap the first and last iova, since only they could have + * dte or pte shared with an existing mapping. + */ + rk_iommu_zap_iova_first_last(rk_domain, iova, size); + return 0; unwind: /* Unmap the range of iovas that we just mapped */ -- cgit v0.10.2 From dc3814f404dfb82fc40f403725ad0548c9541d09 Mon Sep 17 00:00:00 2001 From: Colin Cross Date: Fri, 8 May 2015 17:05:44 -0700 Subject: iommu/exynos: Tell kmemleak to ignore 2nd level page tables The pointers to the 2nd level page tables are converted to 1st level page table entries, which means kmemleak can't find them and assumes they have been leaked. Call kmemleak_ignore on the 2nd level page tables to prevent them from showing up in kmemleak reports. Signed-off-by: Colin Cross Signed-off-by: Dmitry Torokhov Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 3e89850..e54c1be 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -862,6 +862,7 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, return ERR_PTR(-ENOMEM); *sent = mk_lv1ent_page(virt_to_phys(pent)); + kmemleak_ignore(pent); *pgcounter = NUM_LV2ENTRIES; pgtable_flush(pent, pent + NUM_LV2ENTRIES); pgtable_flush(sent, sent + 1); -- cgit v0.10.2 From ec4292defd42fe45dc9807cedd22b00ffa9f09ae Mon Sep 17 00:00:00 2001 From: Heiko Stuebner Date: Thu, 21 May 2015 09:57:29 +0200 Subject: iommu/rockchip: Silence attaching and detaching of devices Currently the driver emits a log line every time a device attaches or detaches - which happens at every unblank/blank of the drm for example. The message itself also has no real value to the average user and is merely useful when debugging a problem, so make it a dev_dbg instead. Signed-off-by: Heiko Stuebner Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index 31004c0..37f6a59 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -785,7 +785,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, list_add_tail(&iommu->node, &rk_domain->iommus); spin_unlock_irqrestore(&rk_domain->iommus_lock, flags); - dev_info(dev, "Attached to iommu domain\n"); + dev_dbg(dev, "Attached to iommu domain\n"); rk_iommu_disable_stall(iommu); @@ -819,7 +819,7 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, iommu->domain = NULL; - dev_info(dev, "Detached from iommu domain\n"); + dev_dbg(dev, "Detached from iommu domain\n"); } static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) -- cgit v0.10.2 From d7ef9995f1d9e394f994b9a1755cccb21ba3e421 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:23 +0200 Subject: iommu: Init iommu-groups support earlier, in core_initcall iommu_group_alloc might be called very early in case of iommu controllers activated from of_iommu, so ensure that this part of subsystem is ready when devices are being populated from device-tree (core_initcall seems to be okay for this case). Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d4f527e..37a6aa8 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1207,7 +1207,7 @@ static int __init iommu_init(void) return 0; } -arch_initcall(iommu_init); +core_initcall(iommu_init); int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr attr, void *data) -- cgit v0.10.2 From 512bd0c6cced8264dae4c7439fef54507548a72d Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:24 +0200 Subject: iommu/exynos: Don't read version register on every tlb operation This patch removes reading of REG_MMU_VERSION register on every tlb operation and caches SYSMMU version in driver's internal data. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index e54c1be..829931c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -213,6 +213,7 @@ struct sysmmu_drvdata { spinlock_t lock; struct iommu_domain *domain; phys_addr_t pgtable; + unsigned int version; }; static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) @@ -244,11 +245,6 @@ static void sysmmu_unblock(void __iomem *sfrbase) __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); } -static unsigned int __raw_sysmmu_version(struct sysmmu_drvdata *data) -{ - return MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); -} - static bool sysmmu_block(void __iomem *sfrbase) { int i = 120; @@ -408,7 +404,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) unsigned int cfg = CFG_LRU | CFG_QOS(15); unsigned int ver; - ver = __raw_sysmmu_version(data); + ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); if (MMU_MAJ_VER(ver) == 3) { if (MMU_MIN_VER(ver) >= 2) { cfg |= CFG_FLPDCACHE; @@ -422,6 +418,7 @@ static void __sysmmu_init_config(struct sysmmu_drvdata *data) } __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); + data->version = ver; } static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) @@ -531,7 +528,7 @@ static bool exynos_sysmmu_disable(struct device *dev) static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { - if (__raw_sysmmu_version(data) == MAKE_MMU_VER(3, 3)) + if (data->version == MAKE_MMU_VER(3, 3)) __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); } @@ -580,7 +577,7 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, * 1MB page can be cached in one of all sets. * 64KB page can be one of 16 consecutive sets. */ - if (MMU_MAJ_VER(__raw_sysmmu_version(data)) == 2) + if (MMU_MAJ_VER(data->version) == 2) num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); if (sysmmu_block(data->sfrbase)) { -- cgit v0.10.2 From 3211cf20421c216426a05fc072f6131122800784 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:25 +0200 Subject: iommu/exynos: Remove unused functions This patch removes two unneeded functions, which are not a part of generic IOMMU API and were never used by any other driver. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 829931c..278eb19 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -496,13 +496,6 @@ static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable, return ret; } -int exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable) -{ - BUG_ON(!memblock_is_memory(pgtable)); - - return __exynos_sysmmu_enable(dev, pgtable, NULL); -} - static bool exynos_sysmmu_disable(struct device *dev) { unsigned long flags; @@ -594,30 +587,6 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, spin_unlock_irqrestore(&data->lock, flags); } -void exynos_sysmmu_tlb_invalidate(struct device *dev) -{ - struct exynos_iommu_owner *owner = dev->archdata.iommu; - unsigned long flags; - struct sysmmu_drvdata *data; - - data = dev_get_drvdata(owner->sysmmu); - - spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) { - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); - if (sysmmu_block(data->sfrbase)) { - __sysmmu_tlb_invalidate(data->sfrbase); - sysmmu_unblock(data->sfrbase); - } - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); - } else { - dev_dbg(dev, "disabled. Skipping TLB invalidation\n"); - } - spin_unlock_irqrestore(&data->lock, flags); -} - static int __init exynos_sysmmu_probe(struct platform_device *pdev) { int irq, ret; -- cgit v0.10.2 From 73db56989664c5daccb5424bc99483d3a099659e Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:26 +0200 Subject: iommu/exynos: Remove useless members from exynos_iommu_owner structure This patch removes useless spinlocks and other unused members from struct exynos_iommu_owner. There is no point is protecting this structure by spinlock because content of this structure doesn't change and other structures have their own spinlocks. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 278eb19..311d4ed 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -189,9 +189,6 @@ struct exynos_iommu_owner { struct list_head client; /* entry of exynos_iommu_domain.clients */ struct device *dev; struct device *sysmmu; - struct iommu_domain *domain; - void *vmm_data; /* IO virtual memory manager's data */ - spinlock_t lock; /* Lock to preserve consistency of System MMU */ }; struct exynos_iommu_domain { @@ -477,44 +474,34 @@ static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable, struct iommu_domain *domain) { int ret = 0; - unsigned long flags; struct exynos_iommu_owner *owner = dev->archdata.iommu; struct sysmmu_drvdata *data; BUG_ON(!has_sysmmu(dev)); - spin_lock_irqsave(&owner->lock, flags); - data = dev_get_drvdata(owner->sysmmu); ret = __sysmmu_enable(data, pgtable, domain); if (ret >= 0) data->master = dev; - spin_unlock_irqrestore(&owner->lock, flags); - return ret; } static bool exynos_sysmmu_disable(struct device *dev) { - unsigned long flags; bool disabled = true; struct exynos_iommu_owner *owner = dev->archdata.iommu; struct sysmmu_drvdata *data; BUG_ON(!has_sysmmu(dev)); - spin_lock_irqsave(&owner->lock, flags); - data = dev_get_drvdata(owner->sysmmu); disabled = __sysmmu_disable(data); if (disabled) data->master = NULL; - spin_unlock_irqrestore(&owner->lock, flags); - return disabled; } @@ -762,10 +749,8 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, spin_lock_irqsave(&priv->lock, flags); ret = __exynos_sysmmu_enable(dev, pagetable, domain); - if (ret == 0) { + if (ret == 0) list_add_tail(&owner->client, &priv->clients); - owner->domain = domain; - } spin_unlock_irqrestore(&priv->lock, flags); @@ -793,10 +778,8 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, list_for_each_entry(owner, &priv->clients, client) { if (owner == dev->archdata.iommu) { - if (exynos_sysmmu_disable(dev)) { + if (exynos_sysmmu_disable(dev)) list_del_init(&owner->client); - owner->domain = NULL; - } break; } } -- cgit v0.10.2 From 469acebe4a68902bf5b43daed61465eae1ce04ec Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:27 +0200 Subject: iommu/exynos: Refactor function parameters to simplify code This patch simplifies the code by: - refactoring function parameters from struct device pointer to direct pointer to struct sysmmu drvdata - moving list_head enteries from struct exynos_iommu_owner directly to struct sysmmu_drvdata After above refactoring some functions were never used, so remove also them completely. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 311d4ed..5d7dbd0 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -186,8 +186,6 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { /* attached to dev.archdata.iommu of the master device */ struct exynos_iommu_owner { - struct list_head client; /* entry of exynos_iommu_domain.clients */ - struct device *dev; struct device *sysmmu; }; @@ -209,6 +207,7 @@ struct sysmmu_drvdata { int activations; spinlock_t lock; struct iommu_domain *domain; + struct list_head domain_node; phys_addr_t pgtable; unsigned int version; }; @@ -464,47 +463,6 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data, return ret; } -/* __exynos_sysmmu_enable: Enables System MMU - * - * returns -error if an error occurred and System MMU is not enabled, - * 0 if the System MMU has been just enabled and 1 if System MMU was already - * enabled before. - */ -static int __exynos_sysmmu_enable(struct device *dev, phys_addr_t pgtable, - struct iommu_domain *domain) -{ - int ret = 0; - struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct sysmmu_drvdata *data; - - BUG_ON(!has_sysmmu(dev)); - - data = dev_get_drvdata(owner->sysmmu); - - ret = __sysmmu_enable(data, pgtable, domain); - if (ret >= 0) - data->master = dev; - - return ret; -} - -static bool exynos_sysmmu_disable(struct device *dev) -{ - bool disabled = true; - struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct sysmmu_drvdata *data; - - BUG_ON(!has_sysmmu(dev)); - - data = dev_get_drvdata(owner->sysmmu); - - disabled = __sysmmu_disable(data); - if (disabled) - data->master = NULL; - - return disabled; -} - static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { @@ -512,12 +470,10 @@ static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); } -static void sysmmu_tlb_invalidate_flpdcache(struct device *dev, +static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { unsigned long flags; - struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct sysmmu_drvdata *data = dev_get_drvdata(owner->sysmmu); if (!IS_ERR(data->clk_master)) clk_enable(data->clk_master); @@ -531,14 +487,10 @@ static void sysmmu_tlb_invalidate_flpdcache(struct device *dev, clk_disable(data->clk_master); } -static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, - size_t size) +static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, + sysmmu_iova_t iova, size_t size) { - struct exynos_iommu_owner *owner = dev->archdata.iommu; unsigned long flags; - struct sysmmu_drvdata *data; - - data = dev_get_drvdata(owner->sysmmu); spin_lock_irqsave(&data->lock, flags); if (is_sysmmu_active(data)) { @@ -568,8 +520,8 @@ static void sysmmu_tlb_invalidate_entry(struct device *dev, sysmmu_iova_t iova, if (!IS_ERR(data->clk_master)) clk_disable(data->clk_master); } else { - dev_dbg(dev, "disabled. Skipping TLB invalidation @ %#x\n", - iova); + dev_dbg(data->master, + "disabled. Skipping TLB invalidation @ %#x\n", iova); } spin_unlock_irqrestore(&data->lock, flags); } @@ -709,7 +661,7 @@ err_pgtable: static void exynos_iommu_domain_free(struct iommu_domain *domain) { struct exynos_iommu_domain *priv = to_exynos_domain(domain); - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data, *next; unsigned long flags; int i; @@ -717,14 +669,12 @@ static void exynos_iommu_domain_free(struct iommu_domain *domain) spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry(owner, &priv->clients, client) { - while (!exynos_sysmmu_disable(owner->dev)) - ; /* until System MMU is actually disabled */ + list_for_each_entry_safe(data, next, &priv->clients, domain_node) { + if (__sysmmu_disable(data)) + data->master = NULL; + list_del_init(&data->domain_node); } - while (!list_empty(&priv->clients)) - list_del_init(priv->clients.next); - spin_unlock_irqrestore(&priv->lock, flags); for (i = 0; i < NUM_LV1ENTRIES; i++) @@ -742,17 +692,25 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, { struct exynos_iommu_owner *owner = dev->archdata.iommu; struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct sysmmu_drvdata *data; phys_addr_t pagetable = virt_to_phys(priv->pgtable); unsigned long flags; - int ret; + int ret = -ENODEV; - spin_lock_irqsave(&priv->lock, flags); + if (!has_sysmmu(dev)) + return -ENODEV; - ret = __exynos_sysmmu_enable(dev, pagetable, domain); - if (ret == 0) - list_add_tail(&owner->client, &priv->clients); - - spin_unlock_irqrestore(&priv->lock, flags); + data = dev_get_drvdata(owner->sysmmu); + if (data) { + ret = __sysmmu_enable(data, pagetable, domain); + if (ret >= 0) { + data->master = dev; + + spin_lock_irqsave(&priv->lock, flags); + list_add_tail(&data->domain_node, &priv->clients); + spin_unlock_irqrestore(&priv->lock, flags); + } + } if (ret < 0) { dev_err(dev, "%s: Failed to attach IOMMU with pgtable %pa\n", @@ -769,24 +727,29 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, static void exynos_iommu_detach_device(struct iommu_domain *domain, struct device *dev) { - struct exynos_iommu_owner *owner; struct exynos_iommu_domain *priv = to_exynos_domain(domain); phys_addr_t pagetable = virt_to_phys(priv->pgtable); + struct sysmmu_drvdata *data; unsigned long flags; + bool found = false; - spin_lock_irqsave(&priv->lock, flags); + if (!has_sysmmu(dev)) + return; - list_for_each_entry(owner, &priv->clients, client) { - if (owner == dev->archdata.iommu) { - if (exynos_sysmmu_disable(dev)) - list_del_init(&owner->client); + spin_lock_irqsave(&priv->lock, flags); + list_for_each_entry(data, &priv->clients, domain_node) { + if (data->master == dev) { + if (__sysmmu_disable(data)) { + data->master = NULL; + list_del_init(&data->domain_node); + } + found = true; break; } } - spin_unlock_irqrestore(&priv->lock, flags); - if (owner == dev->archdata.iommu) + if (found) dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", __func__, &pagetable); else @@ -834,12 +797,11 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, * not currently mapped. */ if (need_flush_flpd_cache) { - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data; spin_lock(&priv->lock); - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_flpdcache( - owner->dev, iova); + list_for_each_entry(data, &priv->clients, domain_node) + sysmmu_tlb_invalidate_flpdcache(data, iova); spin_unlock(&priv->lock); } } @@ -874,13 +836,13 @@ static int lv1set_section(struct exynos_iommu_domain *priv, spin_lock(&priv->lock); if (lv1ent_page_zero(sent)) { - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data; /* * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD * entry by speculative prefetch of SLPD which has no mapping. */ - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_flpdcache(owner->dev, iova); + list_for_each_entry(data, &priv->clients, domain_node) + sysmmu_tlb_invalidate_flpdcache(data, iova); } spin_unlock(&priv->lock); @@ -985,13 +947,13 @@ static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv, sysmmu_iova_t iova, size_t size) { - struct exynos_iommu_owner *owner; + struct sysmmu_drvdata *data; unsigned long flags; spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry(owner, &priv->clients, client) - sysmmu_tlb_invalidate_entry(owner->dev, iova, size); + list_for_each_entry(data, &priv->clients, domain_node) + sysmmu_tlb_invalidate_entry(data, iova, size); spin_unlock_irqrestore(&priv->lock, flags); } -- cgit v0.10.2 From bfa004893c55ead8ac1f2784601dac3609deb406 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:28 +0200 Subject: iommu/exynos: Rename variables to reflect their purpose This patch renames some variables to make the code easier to understand. 'domain' is replaced by 'iommu_domain' (more generic entity) and really meaningless 'priv' by 'domain' to reflect its purpose. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 5d7dbd0..f5475dc 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -435,8 +435,8 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) clk_disable(data->clk_master); } -static int __sysmmu_enable(struct sysmmu_drvdata *data, - phys_addr_t pgtable, struct iommu_domain *domain) +static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, + struct iommu_domain *iommu_domain) { int ret = 0; unsigned long flags; @@ -444,7 +444,7 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data, spin_lock_irqsave(&data->lock, flags); if (set_sysmmu_active(data)) { data->pgtable = pgtable; - data->domain = domain; + data->domain = iommu_domain; __sysmmu_enable_nocount(data); @@ -609,91 +609,91 @@ static inline void pgtable_flush(void *vastart, void *vaend) static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) { - struct exynos_iommu_domain *exynos_domain; + struct exynos_iommu_domain *domain; int i; if (type != IOMMU_DOMAIN_UNMANAGED) return NULL; - exynos_domain = kzalloc(sizeof(*exynos_domain), GFP_KERNEL); - if (!exynos_domain) + domain = kzalloc(sizeof(*domain), GFP_KERNEL); + if (!domain) return NULL; - exynos_domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); - if (!exynos_domain->pgtable) + domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); + if (!domain->pgtable) goto err_pgtable; - exynos_domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); - if (!exynos_domain->lv2entcnt) + domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); + if (!domain->lv2entcnt) goto err_counter; /* Workaround for System MMU v3.3 to prevent caching 1MiB mapping */ for (i = 0; i < NUM_LV1ENTRIES; i += 8) { - exynos_domain->pgtable[i + 0] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 1] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 2] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 3] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 4] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 5] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 6] = ZERO_LV2LINK; - exynos_domain->pgtable[i + 7] = ZERO_LV2LINK; + domain->pgtable[i + 0] = ZERO_LV2LINK; + domain->pgtable[i + 1] = ZERO_LV2LINK; + domain->pgtable[i + 2] = ZERO_LV2LINK; + domain->pgtable[i + 3] = ZERO_LV2LINK; + domain->pgtable[i + 4] = ZERO_LV2LINK; + domain->pgtable[i + 5] = ZERO_LV2LINK; + domain->pgtable[i + 6] = ZERO_LV2LINK; + domain->pgtable[i + 7] = ZERO_LV2LINK; } - pgtable_flush(exynos_domain->pgtable, exynos_domain->pgtable + NUM_LV1ENTRIES); + pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES); - spin_lock_init(&exynos_domain->lock); - spin_lock_init(&exynos_domain->pgtablelock); - INIT_LIST_HEAD(&exynos_domain->clients); + spin_lock_init(&domain->lock); + spin_lock_init(&domain->pgtablelock); + INIT_LIST_HEAD(&domain->clients); - exynos_domain->domain.geometry.aperture_start = 0; - exynos_domain->domain.geometry.aperture_end = ~0UL; - exynos_domain->domain.geometry.force_aperture = true; + domain->domain.geometry.aperture_start = 0; + domain->domain.geometry.aperture_end = ~0UL; + domain->domain.geometry.force_aperture = true; - return &exynos_domain->domain; + return &domain->domain; err_counter: - free_pages((unsigned long)exynos_domain->pgtable, 2); + free_pages((unsigned long)domain->pgtable, 2); err_pgtable: - kfree(exynos_domain); + kfree(domain); return NULL; } -static void exynos_iommu_domain_free(struct iommu_domain *domain) +static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); struct sysmmu_drvdata *data, *next; unsigned long flags; int i; - WARN_ON(!list_empty(&priv->clients)); + WARN_ON(!list_empty(&domain->clients)); - spin_lock_irqsave(&priv->lock, flags); + spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry_safe(data, next, &priv->clients, domain_node) { + list_for_each_entry_safe(data, next, &domain->clients, domain_node) { if (__sysmmu_disable(data)) data->master = NULL; list_del_init(&data->domain_node); } - spin_unlock_irqrestore(&priv->lock, flags); + spin_unlock_irqrestore(&domain->lock, flags); for (i = 0; i < NUM_LV1ENTRIES; i++) - if (lv1ent_page(priv->pgtable + i)) + if (lv1ent_page(domain->pgtable + i)) kmem_cache_free(lv2table_kmem_cache, - phys_to_virt(lv2table_base(priv->pgtable + i))); + phys_to_virt(lv2table_base(domain->pgtable + i))); - free_pages((unsigned long)priv->pgtable, 2); - free_pages((unsigned long)priv->lv2entcnt, 1); - kfree(priv); + free_pages((unsigned long)domain->pgtable, 2); + free_pages((unsigned long)domain->lv2entcnt, 1); + kfree(domain); } -static int exynos_iommu_attach_device(struct iommu_domain *domain, +static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct device *dev) { struct exynos_iommu_owner *owner = dev->archdata.iommu; - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); struct sysmmu_drvdata *data; - phys_addr_t pagetable = virt_to_phys(priv->pgtable); + phys_addr_t pagetable = virt_to_phys(domain->pgtable); unsigned long flags; int ret = -ENODEV; @@ -702,13 +702,13 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, data = dev_get_drvdata(owner->sysmmu); if (data) { - ret = __sysmmu_enable(data, pagetable, domain); + ret = __sysmmu_enable(data, pagetable, iommu_domain); if (ret >= 0) { data->master = dev; - spin_lock_irqsave(&priv->lock, flags); - list_add_tail(&data->domain_node, &priv->clients); - spin_unlock_irqrestore(&priv->lock, flags); + spin_lock_irqsave(&domain->lock, flags); + list_add_tail(&data->domain_node, &domain->clients); + spin_unlock_irqrestore(&domain->lock, flags); } } @@ -724,11 +724,11 @@ static int exynos_iommu_attach_device(struct iommu_domain *domain, return ret; } -static void exynos_iommu_detach_device(struct iommu_domain *domain, +static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, struct device *dev) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); - phys_addr_t pagetable = virt_to_phys(priv->pgtable); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + phys_addr_t pagetable = virt_to_phys(domain->pgtable); struct sysmmu_drvdata *data; unsigned long flags; bool found = false; @@ -736,8 +736,8 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, if (!has_sysmmu(dev)) return; - spin_lock_irqsave(&priv->lock, flags); - list_for_each_entry(data, &priv->clients, domain_node) { + spin_lock_irqsave(&domain->lock, flags); + list_for_each_entry(data, &domain->clients, domain_node) { if (data->master == dev) { if (__sysmmu_disable(data)) { data->master = NULL; @@ -747,7 +747,7 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, break; } } - spin_unlock_irqrestore(&priv->lock, flags); + spin_unlock_irqrestore(&domain->lock, flags); if (found) dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", @@ -756,7 +756,7 @@ static void exynos_iommu_detach_device(struct iommu_domain *domain, dev_err(dev, "%s: No IOMMU is attached\n", __func__); } -static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, +static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { if (lv1ent_section(sent)) { @@ -799,17 +799,17 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *priv, if (need_flush_flpd_cache) { struct sysmmu_drvdata *data; - spin_lock(&priv->lock); - list_for_each_entry(data, &priv->clients, domain_node) + spin_lock(&domain->lock); + list_for_each_entry(data, &domain->clients, domain_node) sysmmu_tlb_invalidate_flpdcache(data, iova); - spin_unlock(&priv->lock); + spin_unlock(&domain->lock); } } return page_entry(sent, iova); } -static int lv1set_section(struct exynos_iommu_domain *priv, +static int lv1set_section(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, phys_addr_t paddr, short *pgcnt) { @@ -834,17 +834,17 @@ static int lv1set_section(struct exynos_iommu_domain *priv, pgtable_flush(sent, sent + 1); - spin_lock(&priv->lock); + spin_lock(&domain->lock); if (lv1ent_page_zero(sent)) { struct sysmmu_drvdata *data; /* * Flushing FLPD cache in System MMU v3.3 that may cache a FLPD * entry by speculative prefetch of SLPD which has no mapping. */ - list_for_each_entry(data, &priv->clients, domain_node) + list_for_each_entry(data, &domain->clients, domain_node) sysmmu_tlb_invalidate_flpdcache(data, iova); } - spin_unlock(&priv->lock); + spin_unlock(&domain->lock); return 0; } @@ -904,74 +904,75 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, * than or equal to 128KiB. * - Start address of an I/O virtual region must be aligned by 128KiB. */ -static int exynos_iommu_map(struct iommu_domain *domain, unsigned long l_iova, - phys_addr_t paddr, size_t size, int prot) +static int exynos_iommu_map(struct iommu_domain *iommu_domain, + unsigned long l_iova, phys_addr_t paddr, size_t size, + int prot) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); sysmmu_pte_t *entry; sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; unsigned long flags; int ret = -ENOMEM; - BUG_ON(priv->pgtable == NULL); + BUG_ON(domain->pgtable == NULL); - spin_lock_irqsave(&priv->pgtablelock, flags); + spin_lock_irqsave(&domain->pgtablelock, flags); - entry = section_entry(priv->pgtable, iova); + entry = section_entry(domain->pgtable, iova); if (size == SECT_SIZE) { - ret = lv1set_section(priv, entry, iova, paddr, - &priv->lv2entcnt[lv1ent_offset(iova)]); + ret = lv1set_section(domain, entry, iova, paddr, + &domain->lv2entcnt[lv1ent_offset(iova)]); } else { sysmmu_pte_t *pent; - pent = alloc_lv2entry(priv, entry, iova, - &priv->lv2entcnt[lv1ent_offset(iova)]); + pent = alloc_lv2entry(domain, entry, iova, + &domain->lv2entcnt[lv1ent_offset(iova)]); if (IS_ERR(pent)) ret = PTR_ERR(pent); else ret = lv2set_page(pent, paddr, size, - &priv->lv2entcnt[lv1ent_offset(iova)]); + &domain->lv2entcnt[lv1ent_offset(iova)]); } if (ret) pr_err("%s: Failed(%d) to map %#zx bytes @ %#x\n", __func__, ret, size, iova); - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); return ret; } -static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *priv, - sysmmu_iova_t iova, size_t size) +static void exynos_iommu_tlb_invalidate_entry(struct exynos_iommu_domain *domain, + sysmmu_iova_t iova, size_t size) { struct sysmmu_drvdata *data; unsigned long flags; - spin_lock_irqsave(&priv->lock, flags); + spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(data, &priv->clients, domain_node) + list_for_each_entry(data, &domain->clients, domain_node) sysmmu_tlb_invalidate_entry(data, iova, size); - spin_unlock_irqrestore(&priv->lock, flags); + spin_unlock_irqrestore(&domain->lock, flags); } -static size_t exynos_iommu_unmap(struct iommu_domain *domain, - unsigned long l_iova, size_t size) +static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, + unsigned long l_iova, size_t size) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); sysmmu_iova_t iova = (sysmmu_iova_t)l_iova; sysmmu_pte_t *ent; size_t err_pgsize; unsigned long flags; - BUG_ON(priv->pgtable == NULL); + BUG_ON(domain->pgtable == NULL); - spin_lock_irqsave(&priv->pgtablelock, flags); + spin_lock_irqsave(&domain->pgtablelock, flags); - ent = section_entry(priv->pgtable, iova); + ent = section_entry(domain->pgtable, iova); if (lv1ent_section(ent)) { if (WARN_ON(size < SECT_SIZE)) { @@ -1005,7 +1006,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, *ent = 0; size = SPAGE_SIZE; pgtable_flush(ent, ent + 1); - priv->lv2entcnt[lv1ent_offset(iova)] += 1; + domain->lv2entcnt[lv1ent_offset(iova)] += 1; goto done; } @@ -1019,15 +1020,15 @@ static size_t exynos_iommu_unmap(struct iommu_domain *domain, pgtable_flush(ent, ent + SPAGES_PER_LPAGE); size = LPAGE_SIZE; - priv->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; + domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; done: - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); - exynos_iommu_tlb_invalidate_entry(priv, iova, size); + exynos_iommu_tlb_invalidate_entry(domain, iova, size); return size; err: - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); pr_err("%s: Failed: size(%#zx) @ %#x is smaller than page size %#zx\n", __func__, size, iova, err_pgsize); @@ -1035,17 +1036,17 @@ err: return 0; } -static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, +static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, dma_addr_t iova) { - struct exynos_iommu_domain *priv = to_exynos_domain(domain); + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); sysmmu_pte_t *entry; unsigned long flags; phys_addr_t phys = 0; - spin_lock_irqsave(&priv->pgtablelock, flags); + spin_lock_irqsave(&domain->pgtablelock, flags); - entry = section_entry(priv->pgtable, iova); + entry = section_entry(domain->pgtable, iova); if (lv1ent_section(entry)) { phys = section_phys(entry) + section_offs(iova); @@ -1058,7 +1059,7 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, phys = spage_phys(entry) + spage_offs(iova); } - spin_unlock_irqrestore(&priv->pgtablelock, flags); + spin_unlock_irqrestore(&domain->pgtablelock, flags); return phys; } -- cgit v0.10.2 From a9133b9936ca43a93c45bfeabf0eef2ce091ee63 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:29 +0200 Subject: iommu/exynos: Use struct exynos_iommu_domain in internal structures Replace all remaining usage of struct iommu_domain with struct exynos_iommu_domain in all internal structures and functions. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index f5475dc..ba1271f 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -206,7 +206,7 @@ struct sysmmu_drvdata { struct clk *clk_master; int activations; spinlock_t lock; - struct iommu_domain *domain; + struct exynos_iommu_domain *domain; struct list_head domain_node; phys_addr_t pgtable; unsigned int version; @@ -337,7 +337,7 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) show_fault_information(dev_name(data->sysmmu), itype, base, addr); if (data->domain) - ret = report_iommu_fault(data->domain, + ret = report_iommu_fault(&data->domain->domain, data->master, addr, itype); } @@ -436,7 +436,7 @@ static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) } static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, - struct iommu_domain *iommu_domain) + struct exynos_iommu_domain *domain) { int ret = 0; unsigned long flags; @@ -444,7 +444,7 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, spin_lock_irqsave(&data->lock, flags); if (set_sysmmu_active(data)) { data->pgtable = pgtable; - data->domain = iommu_domain; + data->domain = domain; __sysmmu_enable_nocount(data); @@ -702,7 +702,7 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, data = dev_get_drvdata(owner->sysmmu); if (data) { - ret = __sysmmu_enable(data, pagetable, iommu_domain); + ret = __sysmmu_enable(data, pagetable, domain); if (ret >= 0) { data->master = dev; -- cgit v0.10.2 From 312900c605a7fc108e9d290745a5da6212062bce Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:30 +0200 Subject: iommu/exynos: Remove excessive includes and sort others alphabetically Removed following unused includes: , , and . Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index ba1271f..9d9fea2 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -12,19 +12,15 @@ #define DEBUG #endif -#include -#include -#include -#include -#include #include #include -#include +#include #include -#include +#include #include -#include -#include +#include +#include +#include #include #include -- cgit v0.10.2 From 2860af3c8f7ac9e2569378653c3948c422bd30c2 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:31 +0200 Subject: iommu/exynos: Document internal structures Add a few words of comment to all internal structures used by the driver. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 9d9fea2..f6ed59c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -180,32 +180,49 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { "UNKNOWN FAULT" }; -/* attached to dev.archdata.iommu of the master device */ +/* + * This structure is attached to dev.archdata.iommu of the master device + * on device add, contains a list of SYSMMU controllers defined by device tree, + * which are bound to given master device. It is usually referenced by 'owner' + * pointer. +*/ struct exynos_iommu_owner { - struct device *sysmmu; + struct device *sysmmu; /* sysmmu controller for given master */ }; +/* + * This structure exynos specific generalization of struct iommu_domain. + * It contains list of SYSMMU controllers from all master devices, which has + * been attached to this domain and page tables of IO address space defined by + * it. It is usually referenced by 'domain' pointer. + */ struct exynos_iommu_domain { - struct list_head clients; /* list of sysmmu_drvdata.node */ - sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ - short *lv2entcnt; /* free lv2 entry counter for each section */ - spinlock_t lock; /* lock for this structure */ - spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ + struct list_head clients; /* list of sysmmu_drvdata.domain_node */ + sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ + short *lv2entcnt; /* free lv2 entry counter for each section */ + spinlock_t lock; /* lock for modyfying list of clients */ + spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */ struct iommu_domain domain; /* generic domain data structure */ }; +/* + * This structure hold all data of a single SYSMMU controller, this includes + * hw resources like registers and clocks, pointers and list nodes to connect + * it to all other structures, internal state and parameters read from device + * tree. It is usually referenced by 'data' pointer. + */ struct sysmmu_drvdata { - struct device *sysmmu; /* System MMU's device descriptor */ - struct device *master; /* Owner of system MMU */ - void __iomem *sfrbase; - struct clk *clk; - struct clk *clk_master; - int activations; - spinlock_t lock; - struct exynos_iommu_domain *domain; - struct list_head domain_node; - phys_addr_t pgtable; - unsigned int version; + struct device *sysmmu; /* SYSMMU controller device */ + struct device *master; /* master device (owner) */ + void __iomem *sfrbase; /* our registers */ + struct clk *clk; /* SYSMMU's clock */ + struct clk *clk_master; /* master's device clock */ + int activations; /* number of calls to sysmmu_enable */ + spinlock_t lock; /* lock for modyfying state */ + struct exynos_iommu_domain *domain; /* domain we belong to */ + struct list_head domain_node; /* node for domain clients list */ + phys_addr_t pgtable; /* assigned page table structure */ + unsigned int version; /* our version */ }; static struct exynos_iommu_domain *to_exynos_domain(struct iommu_domain *dom) -- cgit v0.10.2 From 06801db0d3bba2b4e585130981fadf71bbce4159 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:32 +0200 Subject: iommu/exynos: Add/remove callbacks should fail if no iommu is available Return fail if given master device passed to add_device/remove_device callbacks doesn't has associated any sysmmu controller. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index f6ed59c..99ed39c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1082,6 +1082,9 @@ static int exynos_iommu_add_device(struct device *dev) struct iommu_group *group; int ret; + if (!has_sysmmu(dev)) + return -ENODEV; + group = iommu_group_get(dev); if (!group) { @@ -1100,6 +1103,9 @@ static int exynos_iommu_add_device(struct device *dev) static void exynos_iommu_remove_device(struct device *dev) { + if (!has_sysmmu(dev)) + return; + iommu_group_remove_device(dev); } -- cgit v0.10.2 From 1b09205436847897da1826a88e5cefd9cde5eed7 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:33 +0200 Subject: iommu/exynos: Add support for binding more than one sysmmu to master device This patch adds support for assigning more than one SYSMMU controller to the master device. This has been achieved simply by chaning the struct device pointer in struct exynos_iommu_owner into the list of struct sysmmu_drvdata of all controllers assigned to the given master device. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 99ed39c..b7f679c 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -187,7 +187,7 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { * pointer. */ struct exynos_iommu_owner { - struct device *sysmmu; /* sysmmu controller for given master */ + struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ }; /* @@ -221,6 +221,7 @@ struct sysmmu_drvdata { spinlock_t lock; /* lock for modyfying state */ struct exynos_iommu_domain *domain; /* domain we belong to */ struct list_head domain_node; /* node for domain clients list */ + struct list_head owner_node; /* node for owner controllers list */ phys_addr_t pgtable; /* assigned page table structure */ unsigned int version; /* our version */ }; @@ -713,8 +714,7 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, if (!has_sysmmu(dev)) return -ENODEV; - data = dev_get_drvdata(owner->sysmmu); - if (data) { + list_for_each_entry(data, &owner->controllers, owner_node) { ret = __sysmmu_enable(data, pagetable, domain); if (ret >= 0) { data->master = dev; @@ -742,7 +742,7 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, { struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); phys_addr_t pagetable = virt_to_phys(domain->pgtable); - struct sysmmu_drvdata *data; + struct sysmmu_drvdata *data, *next; unsigned long flags; bool found = false; @@ -750,14 +750,13 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, return; spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry(data, &domain->clients, domain_node) { + list_for_each_entry_safe(data, next, &domain->clients, domain_node) { if (data->master == dev) { if (__sysmmu_disable(data)) { data->master = NULL; list_del_init(&data->domain_node); } found = true; - break; } } spin_unlock_irqrestore(&domain->lock, flags); -- cgit v0.10.2 From ce70ca562b8d17cd995e3a3a44ad23d97dfbb38e Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:34 +0200 Subject: iommu/exynos: Add support for runtime_pm This patch fixes support for runtime power management for SYSMMU controllers, so they are enabled when master device is attached. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index b7f679c..c658268 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -715,6 +715,7 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, return -ENODEV; list_for_each_entry(data, &owner->controllers, owner_node) { + pm_runtime_get_sync(data->sysmmu); ret = __sysmmu_enable(data, pagetable, domain); if (ret >= 0) { data->master = dev; @@ -756,6 +757,7 @@ static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, data->master = NULL; list_del_init(&data->domain_node); } + pm_runtime_put(data->sysmmu); found = true; } } -- cgit v0.10.2 From 622015e407b0824805bb07416b8d9cdfbdc00182 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:35 +0200 Subject: iommu/exynos: Add system suspend/resume support When system goes into suspend state, iommu should save it's state and restore after system resume. This is handled by 'late' pm ops to ensure that sysmmu will be suspended after its master devices and restored before them. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index c658268..5d2033b 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -601,6 +601,36 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) return 0; } +#ifdef CONFIG_PM_SLEEP +static int exynos_sysmmu_suspend(struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev); + + dev_dbg(dev, "suspend\n"); + if (is_sysmmu_active(data)) { + __sysmmu_disable_nocount(data); + pm_runtime_put(dev); + } + return 0; +} + +static int exynos_sysmmu_resume(struct device *dev) +{ + struct sysmmu_drvdata *data = dev_get_drvdata(dev); + + dev_dbg(dev, "resume\n"); + if (is_sysmmu_active(data)) { + pm_runtime_get_sync(dev); + __sysmmu_enable_nocount(data); + } + return 0; +} +#endif + +static const struct dev_pm_ops sysmmu_pm_ops = { + SET_LATE_SYSTEM_SLEEP_PM_OPS(exynos_sysmmu_suspend, exynos_sysmmu_resume) +}; + static const struct of_device_id sysmmu_of_match[] __initconst = { { .compatible = "samsung,exynos-sysmmu", }, { }, @@ -611,6 +641,7 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { .driver = { .name = "exynos-sysmmu", .of_match_table = sysmmu_of_match, + .pm = &sysmmu_pm_ops, } }; -- cgit v0.10.2 From 8ed55c812fa8c8ab790bda52dea4a1a87d94fb2b Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:36 +0200 Subject: iommu/exynos: Init from dt-specific callback instead of initcall This patch introduces IOMMU_OF_DECLARE-based initialization to the driver, which replaces subsys_initcall-based procedure. exynos_iommu_of_setup ensures that each sysmmu controller is probed before its master device. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 5d2033b..0fb40ee 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -13,16 +13,21 @@ #endif #include +#include #include #include #include #include #include +#include +#include +#include #include #include #include #include +#include #include typedef u32 sysmmu_iova_t; @@ -1141,7 +1146,7 @@ static void exynos_iommu_remove_device(struct device *dev) iommu_group_remove_device(dev); } -static const struct iommu_ops exynos_iommu_ops = { +static struct iommu_ops exynos_iommu_ops = { .domain_alloc = exynos_iommu_domain_alloc, .domain_free = exynos_iommu_domain_free, .attach_dev = exynos_iommu_attach_device, @@ -1155,17 +1160,12 @@ static const struct iommu_ops exynos_iommu_ops = { .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, }; +static bool init_done; + static int __init exynos_iommu_init(void) { - struct device_node *np; int ret; - np = of_find_matching_node(NULL, sysmmu_of_match); - if (!np) - return 0; - - of_node_put(np); - lv2table_kmem_cache = kmem_cache_create("exynos-iommu-lv2table", LV2TABLE_SIZE, LV2TABLE_SIZE, 0, NULL); if (!lv2table_kmem_cache) { @@ -1194,6 +1194,8 @@ static int __init exynos_iommu_init(void) goto err_set_iommu; } + init_done = true; + return 0; err_set_iommu: kmem_cache_free(lv2table_kmem_cache, zero_lv2_table); @@ -1203,4 +1205,21 @@ err_reg_driver: kmem_cache_destroy(lv2table_kmem_cache); return ret; } -subsys_initcall(exynos_iommu_init); + +static int __init exynos_iommu_of_setup(struct device_node *np) +{ + struct platform_device *pdev; + + if (!init_done) + exynos_iommu_init(); + + pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + + of_iommu_set_ops(np, &exynos_iommu_ops); + return 0; +} + +IOMMU_OF_DECLARE(exynos_iommu_of, "samsung,exynos-sysmmu", + exynos_iommu_of_setup); -- cgit v0.10.2 From aa759fd376fbbfc34d5bf21b8f280ce68c17d064 Mon Sep 17 00:00:00 2001 From: Marek Szyprowski Date: Tue, 19 May 2015 15:20:37 +0200 Subject: iommu/exynos: Add callback for initializing devices from device tree This patch adds implementation of of_xlate callback, which prepares masters device for attaching to IOMMU. This callback is called during creating devices from device tree. Signed-off-by: Marek Szyprowski Tested-by: Javier Martinez Canillas Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 0fb40ee..97c41b8 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1146,6 +1146,33 @@ static void exynos_iommu_remove_device(struct device *dev) iommu_group_remove_device(dev); } +static int exynos_iommu_of_xlate(struct device *dev, + struct of_phandle_args *spec) +{ + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct platform_device *sysmmu = of_find_device_by_node(spec->np); + struct sysmmu_drvdata *data; + + if (!sysmmu) + return -ENODEV; + + data = platform_get_drvdata(sysmmu); + if (!data) + return -ENODEV; + + if (!owner) { + owner = kzalloc(sizeof(*owner), GFP_KERNEL); + if (!owner) + return -ENOMEM; + + INIT_LIST_HEAD(&owner->controllers); + dev->archdata.iommu = owner; + } + + list_add_tail(&data->owner_node, &owner->controllers); + return 0; +} + static struct iommu_ops exynos_iommu_ops = { .domain_alloc = exynos_iommu_domain_alloc, .domain_free = exynos_iommu_domain_free, @@ -1158,6 +1185,7 @@ static struct iommu_ops exynos_iommu_ops = { .add_device = exynos_iommu_add_device, .remove_device = exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, + .of_xlate = exynos_iommu_of_xlate, }; static bool init_done; -- cgit v0.10.2 From 661d962f19c23df492a03f47b583ef6a540d6031 Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 27 May 2015 17:09:34 +0100 Subject: iommu/arm-smmu: Fix ATS1* register writes The ATS1* address translation registers only support being written atomically - in SMMUv2 where they are 64 bits wide, 32-bit writes to the lower half are automatically zero-extended, whilst 32-bit writes to the upper half are ignored. Thus, the current logic of performing 64-bit writes as two 32-bit accesses is wrong. Since we already limit IOVAs to 32 bits on 32-bit ARM, the lack of a suitable writeq() implementation there is not an issue, and we only need a little preprocessor ugliness to safely hide the 64-bit case. Signed-off-by: Robin Murphy Signed-off-by: Will Deacon Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 66a803b..e2a788e 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -202,8 +202,7 @@ #define ARM_SMMU_CB_S1_TLBIVAL 0x620 #define ARM_SMMU_CB_S2_TLBIIPAS2 0x630 #define ARM_SMMU_CB_S2_TLBIIPAS2L 0x638 -#define ARM_SMMU_CB_ATS1PR_LO 0x800 -#define ARM_SMMU_CB_ATS1PR_HI 0x804 +#define ARM_SMMU_CB_ATS1PR 0x800 #define ARM_SMMU_CB_ATSR 0x8f0 #define SCTLR_S1_ASIDPNE (1 << 12) @@ -1229,18 +1228,18 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, void __iomem *cb_base; u32 tmp; u64 phys; + unsigned long va; cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); - if (smmu->version == 1) { - u32 reg = iova & ~0xfff; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); - } else { - u32 reg = iova & ~0xfff; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_LO); - reg = ((u64)iova & ~0xfff) >> 32; - writel_relaxed(reg, cb_base + ARM_SMMU_CB_ATS1PR_HI); - } + /* ATS1 registers can only be written atomically */ + va = iova & ~0xfffUL; +#ifdef CONFIG_64BIT + if (smmu->version == ARM_SMMU_V2) + writeq_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); + else +#endif + writel_relaxed(va, cb_base + ARM_SMMU_CB_ATS1PR); if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp, !(tmp & ATSR_ACTIVE), 5, 50)) { -- cgit v0.10.2 From e3ce0c940e02e4d9d09c1c1377f80d9e90a893a0 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 May 2015 17:09:35 +0100 Subject: iommu/arm-smmu: Make force_stage module param read-only in sysfs Changing force_stage dynamically isn't supported by the driver and it also doesn't make a whole lot of sense to change it once the SMMU is up and running. This patch makes the sysfs entry for the parameter read-only. Signed-off-by: Will Deacon Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index e2a788e..dce041b 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -246,7 +246,7 @@ #define FSYNR0_WNR (1 << 4) static int force_stage; -module_param_named(force_stage, force_stage, int, S_IRUGO | S_IWUSR); +module_param_named(force_stage, force_stage, int, S_IRUGO); MODULE_PARM_DESC(force_stage, "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); -- cgit v0.10.2 From a73e528ab1499505c29e14091b19cb041e20e038 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 May 2015 17:25:58 +0100 Subject: Documentation: dt-bindings: Add device-tree binding for ARM SMMUv3 IOMMU This patch adds device-tree bindings for ARM SMMUv3 IOMMU devices. Cc: Mark Rutland Signed-off-by: Will Deacon Signed-off-by: Joerg Roedel diff --git a/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt new file mode 100644 index 0000000..c03eec1 --- /dev/null +++ b/Documentation/devicetree/bindings/iommu/arm,smmu-v3.txt @@ -0,0 +1,37 @@ +* ARM SMMUv3 Architecture Implementation + +The SMMUv3 architecture is a significant deparature from previous +revisions, replacing the MMIO register interface with in-memory command +and event queues and adding support for the ATS and PRI components of +the PCIe specification. + +** SMMUv3 required properties: + +- compatible : Should include: + + * "arm,smmu-v3" for any SMMUv3 compliant + implementation. This entry should be last in the + compatible list. + +- reg : Base address and size of the SMMU. + +- interrupts : Non-secure interrupt list describing the wired + interrupt sources corresponding to entries in + interrupt-names. If no wired interrupts are + present then this property may be omitted. + +- interrupt-names : When the interrupts property is present, should + include the following: + * "eventq" - Event Queue not empty + * "priq" - PRI Queue not empty + * "cmdq-sync" - CMD_SYNC complete + * "gerror" - Global Error activated + +** SMMUv3 optional properties: + +- dma-coherent : Present if DMA operations made by the SMMU (page + table walks, stream table accesses etc) are cache + coherent with the CPU. + + NOTE: this only applies to the SMMU itself, not + masters connected upstream of the SMMU. -- cgit v0.10.2 From 48ec83bcbcf5090fcdf74a6168f161d247492979 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 May 2015 17:25:59 +0100 Subject: iommu/arm-smmu: Add initial driver support for ARM SMMUv3 devices Version three of the ARM SMMU architecture introduces significant changes and improvements over previous versions of the specification, necessitating a new driver in the Linux kernel. The main change to the programming interface is that the majority of the configuration data has been moved from MMIO registers to in-memory data structures, with communication between the CPU and the SMMU being mediated via in-memory circular queues. This patch adds an initial driver for SMMUv3 to Linux. We currently support pinned stage-1 (DMA) and stage-2 (KVM VFIO) mappings using the generic IO-pgtable code. Cc: Robin Murphy Signed-off-by: Will Deacon Signed-off-by: Joerg Roedel diff --git a/MAINTAINERS b/MAINTAINERS index 474bcb6..469d03b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1634,11 +1634,12 @@ F: drivers/i2c/busses/i2c-cadence.c F: drivers/mmc/host/sdhci-of-arasan.c F: drivers/edac/synopsys_edac.c -ARM SMMU DRIVER +ARM SMMU DRIVERS M: Will Deacon L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) S: Maintained F: drivers/iommu/arm-smmu.c +F: drivers/iommu/arm-smmu-v3.c F: drivers/iommu/io-pgtable-arm.c ARM64 PORT (AARCH64 ARCHITECTURE) diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 1ae4e54..40f37a2 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -339,6 +339,7 @@ config SPAPR_TCE_IOMMU Enables bits of IOMMU API required by VFIO. The iommu_ops is not implemented as it is not necessary for VFIO. +# ARM IOMMU support config ARM_SMMU bool "ARM Ltd. System MMU (SMMU) Support" depends on (ARM64 || ARM) && MMU @@ -352,4 +353,16 @@ config ARM_SMMU Say Y here if your SoC includes an IOMMU device implementing the ARM SMMU architecture. +config ARM_SMMU_V3 + bool "ARM Ltd. System MMU Version 3 (SMMUv3) Support" + depends on ARM64 && PCI + select IOMMU_API + select IOMMU_IO_PGTABLE_LPAE + help + Support for implementations of the ARM System MMU architecture + version 3 providing translation support to a PCIe root complex. + + Say Y here if your system includes an IOMMU device implementing + the ARM SMMUv3 architecture. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 080ffab..c6dcc51 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -9,6 +9,7 @@ obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm-smmu.o +obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c new file mode 100644 index 0000000..f141301 --- /dev/null +++ b/drivers/iommu/arm-smmu-v3.c @@ -0,0 +1,2670 @@ +/* + * IOMMU API for ARM architected SMMUv3 implementations. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Copyright (C) 2015 ARM Limited + * + * Author: Will Deacon + * + * This driver is powered by bad coffee and bombay mix. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io-pgtable.h" + +/* MMIO registers */ +#define ARM_SMMU_IDR0 0x0 +#define IDR0_ST_LVL_SHIFT 27 +#define IDR0_ST_LVL_MASK 0x3 +#define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT) +#define IDR0_STALL_MODEL (3 << 24) +#define IDR0_TTENDIAN_SHIFT 21 +#define IDR0_TTENDIAN_MASK 0x3 +#define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT) +#define IDR0_TTENDIAN_BE (3 << IDR0_TTENDIAN_SHIFT) +#define IDR0_TTENDIAN_MIXED (0 << IDR0_TTENDIAN_SHIFT) +#define IDR0_CD2L (1 << 19) +#define IDR0_VMID16 (1 << 18) +#define IDR0_PRI (1 << 16) +#define IDR0_SEV (1 << 14) +#define IDR0_MSI (1 << 13) +#define IDR0_ASID16 (1 << 12) +#define IDR0_ATS (1 << 10) +#define IDR0_HYP (1 << 9) +#define IDR0_COHACC (1 << 4) +#define IDR0_TTF_SHIFT 2 +#define IDR0_TTF_MASK 0x3 +#define IDR0_TTF_AARCH64 (2 << IDR0_TTF_SHIFT) +#define IDR0_S1P (1 << 1) +#define IDR0_S2P (1 << 0) + +#define ARM_SMMU_IDR1 0x4 +#define IDR1_TABLES_PRESET (1 << 30) +#define IDR1_QUEUES_PRESET (1 << 29) +#define IDR1_REL (1 << 28) +#define IDR1_CMDQ_SHIFT 21 +#define IDR1_CMDQ_MASK 0x1f +#define IDR1_EVTQ_SHIFT 16 +#define IDR1_EVTQ_MASK 0x1f +#define IDR1_PRIQ_SHIFT 11 +#define IDR1_PRIQ_MASK 0x1f +#define IDR1_SSID_SHIFT 6 +#define IDR1_SSID_MASK 0x1f +#define IDR1_SID_SHIFT 0 +#define IDR1_SID_MASK 0x3f + +#define ARM_SMMU_IDR5 0x14 +#define IDR5_STALL_MAX_SHIFT 16 +#define IDR5_STALL_MAX_MASK 0xffff +#define IDR5_GRAN64K (1 << 6) +#define IDR5_GRAN16K (1 << 5) +#define IDR5_GRAN4K (1 << 4) +#define IDR5_OAS_SHIFT 0 +#define IDR5_OAS_MASK 0x7 +#define IDR5_OAS_32_BIT (0 << IDR5_OAS_SHIFT) +#define IDR5_OAS_36_BIT (1 << IDR5_OAS_SHIFT) +#define IDR5_OAS_40_BIT (2 << IDR5_OAS_SHIFT) +#define IDR5_OAS_42_BIT (3 << IDR5_OAS_SHIFT) +#define IDR5_OAS_44_BIT (4 << IDR5_OAS_SHIFT) +#define IDR5_OAS_48_BIT (5 << IDR5_OAS_SHIFT) + +#define ARM_SMMU_CR0 0x20 +#define CR0_CMDQEN (1 << 3) +#define CR0_EVTQEN (1 << 2) +#define CR0_PRIQEN (1 << 1) +#define CR0_SMMUEN (1 << 0) + +#define ARM_SMMU_CR0ACK 0x24 + +#define ARM_SMMU_CR1 0x28 +#define CR1_SH_NSH 0 +#define CR1_SH_OSH 2 +#define CR1_SH_ISH 3 +#define CR1_CACHE_NC 0 +#define CR1_CACHE_WB 1 +#define CR1_CACHE_WT 2 +#define CR1_TABLE_SH_SHIFT 10 +#define CR1_TABLE_OC_SHIFT 8 +#define CR1_TABLE_IC_SHIFT 6 +#define CR1_QUEUE_SH_SHIFT 4 +#define CR1_QUEUE_OC_SHIFT 2 +#define CR1_QUEUE_IC_SHIFT 0 + +#define ARM_SMMU_CR2 0x2c +#define CR2_PTM (1 << 2) +#define CR2_RECINVSID (1 << 1) +#define CR2_E2H (1 << 0) + +#define ARM_SMMU_IRQ_CTRL 0x50 +#define IRQ_CTRL_EVTQ_IRQEN (1 << 2) +#define IRQ_CTRL_GERROR_IRQEN (1 << 0) + +#define ARM_SMMU_IRQ_CTRLACK 0x54 + +#define ARM_SMMU_GERROR 0x60 +#define GERROR_SFM_ERR (1 << 8) +#define GERROR_MSI_GERROR_ABT_ERR (1 << 7) +#define GERROR_MSI_PRIQ_ABT_ERR (1 << 6) +#define GERROR_MSI_EVTQ_ABT_ERR (1 << 5) +#define GERROR_MSI_CMDQ_ABT_ERR (1 << 4) +#define GERROR_PRIQ_ABT_ERR (1 << 3) +#define GERROR_EVTQ_ABT_ERR (1 << 2) +#define GERROR_CMDQ_ERR (1 << 0) +#define GERROR_ERR_MASK 0xfd + +#define ARM_SMMU_GERRORN 0x64 + +#define ARM_SMMU_GERROR_IRQ_CFG0 0x68 +#define ARM_SMMU_GERROR_IRQ_CFG1 0x70 +#define ARM_SMMU_GERROR_IRQ_CFG2 0x74 + +#define ARM_SMMU_STRTAB_BASE 0x80 +#define STRTAB_BASE_RA (1UL << 62) +#define STRTAB_BASE_ADDR_SHIFT 6 +#define STRTAB_BASE_ADDR_MASK 0x3ffffffffffUL + +#define ARM_SMMU_STRTAB_BASE_CFG 0x88 +#define STRTAB_BASE_CFG_LOG2SIZE_SHIFT 0 +#define STRTAB_BASE_CFG_LOG2SIZE_MASK 0x3f +#define STRTAB_BASE_CFG_SPLIT_SHIFT 6 +#define STRTAB_BASE_CFG_SPLIT_MASK 0x1f +#define STRTAB_BASE_CFG_FMT_SHIFT 16 +#define STRTAB_BASE_CFG_FMT_MASK 0x3 +#define STRTAB_BASE_CFG_FMT_LINEAR (0 << STRTAB_BASE_CFG_FMT_SHIFT) +#define STRTAB_BASE_CFG_FMT_2LVL (1 << STRTAB_BASE_CFG_FMT_SHIFT) + +#define ARM_SMMU_CMDQ_BASE 0x90 +#define ARM_SMMU_CMDQ_PROD 0x98 +#define ARM_SMMU_CMDQ_CONS 0x9c + +#define ARM_SMMU_EVTQ_BASE 0xa0 +#define ARM_SMMU_EVTQ_PROD 0x100a8 +#define ARM_SMMU_EVTQ_CONS 0x100ac +#define ARM_SMMU_EVTQ_IRQ_CFG0 0xb0 +#define ARM_SMMU_EVTQ_IRQ_CFG1 0xb8 +#define ARM_SMMU_EVTQ_IRQ_CFG2 0xbc + +#define ARM_SMMU_PRIQ_BASE 0xc0 +#define ARM_SMMU_PRIQ_PROD 0x100c8 +#define ARM_SMMU_PRIQ_CONS 0x100cc +#define ARM_SMMU_PRIQ_IRQ_CFG0 0xd0 +#define ARM_SMMU_PRIQ_IRQ_CFG1 0xd8 +#define ARM_SMMU_PRIQ_IRQ_CFG2 0xdc + +/* Common MSI config fields */ +#define MSI_CFG0_SH_SHIFT 60 +#define MSI_CFG0_SH_NSH (0UL << MSI_CFG0_SH_SHIFT) +#define MSI_CFG0_SH_OSH (2UL << MSI_CFG0_SH_SHIFT) +#define MSI_CFG0_SH_ISH (3UL << MSI_CFG0_SH_SHIFT) +#define MSI_CFG0_MEMATTR_SHIFT 56 +#define MSI_CFG0_MEMATTR_DEVICE_nGnRE (0x1 << MSI_CFG0_MEMATTR_SHIFT) +#define MSI_CFG0_ADDR_SHIFT 2 +#define MSI_CFG0_ADDR_MASK 0x3fffffffffffUL + +#define Q_IDX(q, p) ((p) & ((1 << (q)->max_n_shift) - 1)) +#define Q_WRP(q, p) ((p) & (1 << (q)->max_n_shift)) +#define Q_OVERFLOW_FLAG (1 << 31) +#define Q_OVF(q, p) ((p) & Q_OVERFLOW_FLAG) +#define Q_ENT(q, p) ((q)->base + \ + Q_IDX(q, p) * (q)->ent_dwords) + +#define Q_BASE_RWA (1UL << 62) +#define Q_BASE_ADDR_SHIFT 5 +#define Q_BASE_ADDR_MASK 0xfffffffffffUL +#define Q_BASE_LOG2SIZE_SHIFT 0 +#define Q_BASE_LOG2SIZE_MASK 0x1fUL + +/* + * Stream table. + * + * Linear: Enough to cover 1 << IDR1.SIDSIZE entries + * 2lvl: 8k L1 entries, 256 lazy entries per table (each table covers a PCI bus) + */ +#define STRTAB_L1_SZ_SHIFT 16 +#define STRTAB_SPLIT 8 + +#define STRTAB_L1_DESC_DWORDS 1 +#define STRTAB_L1_DESC_SPAN_SHIFT 0 +#define STRTAB_L1_DESC_SPAN_MASK 0x1fUL +#define STRTAB_L1_DESC_L2PTR_SHIFT 6 +#define STRTAB_L1_DESC_L2PTR_MASK 0x3ffffffffffUL + +#define STRTAB_STE_DWORDS 8 +#define STRTAB_STE_0_V (1UL << 0) +#define STRTAB_STE_0_CFG_SHIFT 1 +#define STRTAB_STE_0_CFG_MASK 0x7UL +#define STRTAB_STE_0_CFG_ABORT (0UL << STRTAB_STE_0_CFG_SHIFT) +#define STRTAB_STE_0_CFG_BYPASS (4UL << STRTAB_STE_0_CFG_SHIFT) +#define STRTAB_STE_0_CFG_S1_TRANS (5UL << STRTAB_STE_0_CFG_SHIFT) +#define STRTAB_STE_0_CFG_S2_TRANS (6UL << STRTAB_STE_0_CFG_SHIFT) + +#define STRTAB_STE_0_S1FMT_SHIFT 4 +#define STRTAB_STE_0_S1FMT_LINEAR (0UL << STRTAB_STE_0_S1FMT_SHIFT) +#define STRTAB_STE_0_S1CTXPTR_SHIFT 6 +#define STRTAB_STE_0_S1CTXPTR_MASK 0x3ffffffffffUL +#define STRTAB_STE_0_S1CDMAX_SHIFT 59 +#define STRTAB_STE_0_S1CDMAX_MASK 0x1fUL + +#define STRTAB_STE_1_S1C_CACHE_NC 0UL +#define STRTAB_STE_1_S1C_CACHE_WBRA 1UL +#define STRTAB_STE_1_S1C_CACHE_WT 2UL +#define STRTAB_STE_1_S1C_CACHE_WB 3UL +#define STRTAB_STE_1_S1C_SH_NSH 0UL +#define STRTAB_STE_1_S1C_SH_OSH 2UL +#define STRTAB_STE_1_S1C_SH_ISH 3UL +#define STRTAB_STE_1_S1CIR_SHIFT 2 +#define STRTAB_STE_1_S1COR_SHIFT 4 +#define STRTAB_STE_1_S1CSH_SHIFT 6 + +#define STRTAB_STE_1_S1STALLD (1UL << 27) + +#define STRTAB_STE_1_EATS_ABT 0UL +#define STRTAB_STE_1_EATS_TRANS 1UL +#define STRTAB_STE_1_EATS_S1CHK 2UL +#define STRTAB_STE_1_EATS_SHIFT 28 + +#define STRTAB_STE_1_STRW_NSEL1 0UL +#define STRTAB_STE_1_STRW_EL2 2UL +#define STRTAB_STE_1_STRW_SHIFT 30 + +#define STRTAB_STE_2_S2VMID_SHIFT 0 +#define STRTAB_STE_2_S2VMID_MASK 0xffffUL +#define STRTAB_STE_2_VTCR_SHIFT 32 +#define STRTAB_STE_2_VTCR_MASK 0x7ffffUL +#define STRTAB_STE_2_S2AA64 (1UL << 51) +#define STRTAB_STE_2_S2ENDI (1UL << 52) +#define STRTAB_STE_2_S2PTW (1UL << 54) +#define STRTAB_STE_2_S2R (1UL << 58) + +#define STRTAB_STE_3_S2TTB_SHIFT 4 +#define STRTAB_STE_3_S2TTB_MASK 0xfffffffffffUL + +/* Context descriptor (stage-1 only) */ +#define CTXDESC_CD_DWORDS 8 +#define CTXDESC_CD_0_TCR_T0SZ_SHIFT 0 +#define ARM64_TCR_T0SZ_SHIFT 0 +#define ARM64_TCR_T0SZ_MASK 0x1fUL +#define CTXDESC_CD_0_TCR_TG0_SHIFT 6 +#define ARM64_TCR_TG0_SHIFT 14 +#define ARM64_TCR_TG0_MASK 0x3UL +#define CTXDESC_CD_0_TCR_IRGN0_SHIFT 8 +#define ARM64_TCR_IRGN0_SHIFT 24 +#define ARM64_TCR_IRGN0_MASK 0x3UL +#define CTXDESC_CD_0_TCR_ORGN0_SHIFT 10 +#define ARM64_TCR_ORGN0_SHIFT 26 +#define ARM64_TCR_ORGN0_MASK 0x3UL +#define CTXDESC_CD_0_TCR_SH0_SHIFT 12 +#define ARM64_TCR_SH0_SHIFT 12 +#define ARM64_TCR_SH0_MASK 0x3UL +#define CTXDESC_CD_0_TCR_EPD0_SHIFT 14 +#define ARM64_TCR_EPD0_SHIFT 7 +#define ARM64_TCR_EPD0_MASK 0x1UL +#define CTXDESC_CD_0_TCR_EPD1_SHIFT 30 +#define ARM64_TCR_EPD1_SHIFT 23 +#define ARM64_TCR_EPD1_MASK 0x1UL + +#define CTXDESC_CD_0_ENDI (1UL << 15) +#define CTXDESC_CD_0_V (1UL << 31) + +#define CTXDESC_CD_0_TCR_IPS_SHIFT 32 +#define ARM64_TCR_IPS_SHIFT 32 +#define ARM64_TCR_IPS_MASK 0x7UL +#define CTXDESC_CD_0_TCR_TBI0_SHIFT 38 +#define ARM64_TCR_TBI0_SHIFT 37 +#define ARM64_TCR_TBI0_MASK 0x1UL + +#define CTXDESC_CD_0_AA64 (1UL << 41) +#define CTXDESC_CD_0_R (1UL << 45) +#define CTXDESC_CD_0_A (1UL << 46) +#define CTXDESC_CD_0_ASET_SHIFT 47 +#define CTXDESC_CD_0_ASET_SHARED (0UL << CTXDESC_CD_0_ASET_SHIFT) +#define CTXDESC_CD_0_ASET_PRIVATE (1UL << CTXDESC_CD_0_ASET_SHIFT) +#define CTXDESC_CD_0_ASID_SHIFT 48 +#define CTXDESC_CD_0_ASID_MASK 0xffffUL + +#define CTXDESC_CD_1_TTB0_SHIFT 4 +#define CTXDESC_CD_1_TTB0_MASK 0xfffffffffffUL + +#define CTXDESC_CD_3_MAIR_SHIFT 0 + +/* Convert between AArch64 (CPU) TCR format and SMMU CD format */ +#define ARM_SMMU_TCR2CD(tcr, fld) \ + (((tcr) >> ARM64_TCR_##fld##_SHIFT & ARM64_TCR_##fld##_MASK) \ + << CTXDESC_CD_0_TCR_##fld##_SHIFT) + +/* Command queue */ +#define CMDQ_ENT_DWORDS 2 +#define CMDQ_MAX_SZ_SHIFT 8 + +#define CMDQ_ERR_SHIFT 24 +#define CMDQ_ERR_MASK 0x7f +#define CMDQ_ERR_CERROR_NONE_IDX 0 +#define CMDQ_ERR_CERROR_ILL_IDX 1 +#define CMDQ_ERR_CERROR_ABT_IDX 2 + +#define CMDQ_0_OP_SHIFT 0 +#define CMDQ_0_OP_MASK 0xffUL +#define CMDQ_0_SSV (1UL << 11) + +#define CMDQ_PREFETCH_0_SID_SHIFT 32 +#define CMDQ_PREFETCH_1_SIZE_SHIFT 0 +#define CMDQ_PREFETCH_1_ADDR_MASK ~0xfffUL + +#define CMDQ_CFGI_0_SID_SHIFT 32 +#define CMDQ_CFGI_0_SID_MASK 0xffffffffUL +#define CMDQ_CFGI_1_LEAF (1UL << 0) +#define CMDQ_CFGI_1_RANGE_SHIFT 0 +#define CMDQ_CFGI_1_RANGE_MASK 0x1fUL + +#define CMDQ_TLBI_0_VMID_SHIFT 32 +#define CMDQ_TLBI_0_ASID_SHIFT 48 +#define CMDQ_TLBI_1_LEAF (1UL << 0) +#define CMDQ_TLBI_1_ADDR_MASK ~0xfffUL + +#define CMDQ_PRI_0_SSID_SHIFT 12 +#define CMDQ_PRI_0_SSID_MASK 0xfffffUL +#define CMDQ_PRI_0_SID_SHIFT 32 +#define CMDQ_PRI_0_SID_MASK 0xffffffffUL +#define CMDQ_PRI_1_GRPID_SHIFT 0 +#define CMDQ_PRI_1_GRPID_MASK 0x1ffUL +#define CMDQ_PRI_1_RESP_SHIFT 12 +#define CMDQ_PRI_1_RESP_DENY (0UL << CMDQ_PRI_1_RESP_SHIFT) +#define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT) +#define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT) + +#define CMDQ_SYNC_0_CS_SHIFT 12 +#define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT) +#define CMDQ_SYNC_0_CS_SEV (2UL << CMDQ_SYNC_0_CS_SHIFT) + +/* Event queue */ +#define EVTQ_ENT_DWORDS 4 +#define EVTQ_MAX_SZ_SHIFT 7 + +#define EVTQ_0_ID_SHIFT 0 +#define EVTQ_0_ID_MASK 0xffUL + +/* PRI queue */ +#define PRIQ_ENT_DWORDS 2 +#define PRIQ_MAX_SZ_SHIFT 8 + +#define PRIQ_0_SID_SHIFT 0 +#define PRIQ_0_SID_MASK 0xffffffffUL +#define PRIQ_0_SSID_SHIFT 32 +#define PRIQ_0_SSID_MASK 0xfffffUL +#define PRIQ_0_OF (1UL << 57) +#define PRIQ_0_PERM_PRIV (1UL << 58) +#define PRIQ_0_PERM_EXEC (1UL << 59) +#define PRIQ_0_PERM_READ (1UL << 60) +#define PRIQ_0_PERM_WRITE (1UL << 61) +#define PRIQ_0_PRG_LAST (1UL << 62) +#define PRIQ_0_SSID_V (1UL << 63) + +#define PRIQ_1_PRG_IDX_SHIFT 0 +#define PRIQ_1_PRG_IDX_MASK 0x1ffUL +#define PRIQ_1_ADDR_SHIFT 12 +#define PRIQ_1_ADDR_MASK 0xfffffffffffffUL + +/* High-level queue structures */ +#define ARM_SMMU_POLL_TIMEOUT_US 100 + +static bool disable_bypass; +module_param_named(disable_bypass, disable_bypass, bool, S_IRUGO); +MODULE_PARM_DESC(disable_bypass, + "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); + +enum pri_resp { + PRI_RESP_DENY, + PRI_RESP_FAIL, + PRI_RESP_SUCC, +}; + +struct arm_smmu_cmdq_ent { + /* Common fields */ + u8 opcode; + bool substream_valid; + + /* Command-specific fields */ + union { + #define CMDQ_OP_PREFETCH_CFG 0x1 + struct { + u32 sid; + u8 size; + u64 addr; + } prefetch; + + #define CMDQ_OP_CFGI_STE 0x3 + #define CMDQ_OP_CFGI_ALL 0x4 + struct { + u32 sid; + union { + bool leaf; + u8 span; + }; + } cfgi; + + #define CMDQ_OP_TLBI_NH_ASID 0x11 + #define CMDQ_OP_TLBI_NH_VA 0x12 + #define CMDQ_OP_TLBI_EL2_ALL 0x20 + #define CMDQ_OP_TLBI_S12_VMALL 0x28 + #define CMDQ_OP_TLBI_S2_IPA 0x2a + #define CMDQ_OP_TLBI_NSNH_ALL 0x30 + struct { + u16 asid; + u16 vmid; + bool leaf; + u64 addr; + } tlbi; + + #define CMDQ_OP_PRI_RESP 0x41 + struct { + u32 sid; + u32 ssid; + u16 grpid; + enum pri_resp resp; + } pri; + + #define CMDQ_OP_CMD_SYNC 0x46 + }; +}; + +struct arm_smmu_queue { + int irq; /* Wired interrupt */ + + __le64 *base; + dma_addr_t base_dma; + u64 q_base; + + size_t ent_dwords; + u32 max_n_shift; + u32 prod; + u32 cons; + + u32 __iomem *prod_reg; + u32 __iomem *cons_reg; +}; + +struct arm_smmu_cmdq { + struct arm_smmu_queue q; + spinlock_t lock; +}; + +struct arm_smmu_evtq { + struct arm_smmu_queue q; + u32 max_stalls; +}; + +struct arm_smmu_priq { + struct arm_smmu_queue q; +}; + +/* High-level stream table and context descriptor structures */ +struct arm_smmu_strtab_l1_desc { + u8 span; + + __le64 *l2ptr; + dma_addr_t l2ptr_dma; +}; + +struct arm_smmu_s1_cfg { + __le64 *cdptr; + dma_addr_t cdptr_dma; + + struct arm_smmu_ctx_desc { + u16 asid; + u64 ttbr; + u64 tcr; + u64 mair; + } cd; +}; + +struct arm_smmu_s2_cfg { + u16 vmid; + u64 vttbr; + u64 vtcr; +}; + +struct arm_smmu_strtab_ent { + bool valid; + + bool bypass; /* Overrides s1/s2 config */ + struct arm_smmu_s1_cfg *s1_cfg; + struct arm_smmu_s2_cfg *s2_cfg; +}; + +struct arm_smmu_strtab_cfg { + __le64 *strtab; + dma_addr_t strtab_dma; + struct arm_smmu_strtab_l1_desc *l1_desc; + unsigned int num_l1_ents; + + u64 strtab_base; + u32 strtab_base_cfg; +}; + +/* An SMMUv3 instance */ +struct arm_smmu_device { + struct device *dev; + void __iomem *base; + +#define ARM_SMMU_FEAT_2_LVL_STRTAB (1 << 0) +#define ARM_SMMU_FEAT_2_LVL_CDTAB (1 << 1) +#define ARM_SMMU_FEAT_TT_LE (1 << 2) +#define ARM_SMMU_FEAT_TT_BE (1 << 3) +#define ARM_SMMU_FEAT_PRI (1 << 4) +#define ARM_SMMU_FEAT_ATS (1 << 5) +#define ARM_SMMU_FEAT_SEV (1 << 6) +#define ARM_SMMU_FEAT_MSI (1 << 7) +#define ARM_SMMU_FEAT_COHERENCY (1 << 8) +#define ARM_SMMU_FEAT_TRANS_S1 (1 << 9) +#define ARM_SMMU_FEAT_TRANS_S2 (1 << 10) +#define ARM_SMMU_FEAT_STALLS (1 << 11) +#define ARM_SMMU_FEAT_HYP (1 << 12) + u32 features; + + struct arm_smmu_cmdq cmdq; + struct arm_smmu_evtq evtq; + struct arm_smmu_priq priq; + + int gerr_irq; + + unsigned long ias; /* IPA */ + unsigned long oas; /* PA */ + +#define ARM_SMMU_MAX_ASIDS (1 << 16) + unsigned int asid_bits; + DECLARE_BITMAP(asid_map, ARM_SMMU_MAX_ASIDS); + +#define ARM_SMMU_MAX_VMIDS (1 << 16) + unsigned int vmid_bits; + DECLARE_BITMAP(vmid_map, ARM_SMMU_MAX_VMIDS); + + unsigned int ssid_bits; + unsigned int sid_bits; + + struct arm_smmu_strtab_cfg strtab_cfg; + struct list_head list; +}; + +/* SMMU private data for an IOMMU group */ +struct arm_smmu_group { + struct arm_smmu_device *smmu; + struct arm_smmu_domain *domain; + int num_sids; + u32 *sids; + struct arm_smmu_strtab_ent ste; +}; + +/* SMMU private data for an IOMMU domain */ +enum arm_smmu_domain_stage { + ARM_SMMU_DOMAIN_S1 = 0, + ARM_SMMU_DOMAIN_S2, + ARM_SMMU_DOMAIN_NESTED, +}; + +struct arm_smmu_domain { + struct arm_smmu_device *smmu; + struct mutex init_mutex; /* Protects smmu pointer */ + + struct io_pgtable_ops *pgtbl_ops; + spinlock_t pgtbl_lock; + + enum arm_smmu_domain_stage stage; + union { + struct arm_smmu_s1_cfg s1_cfg; + struct arm_smmu_s2_cfg s2_cfg; + }; + + struct iommu_domain domain; +}; + +/* Our list of SMMU instances */ +static DEFINE_SPINLOCK(arm_smmu_devices_lock); +static LIST_HEAD(arm_smmu_devices); + +static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct arm_smmu_domain, domain); +} + +/* Low-level queue manipulation functions */ +static bool queue_full(struct arm_smmu_queue *q) +{ + return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && + Q_WRP(q, q->prod) != Q_WRP(q, q->cons); +} + +static bool queue_empty(struct arm_smmu_queue *q) +{ + return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) && + Q_WRP(q, q->prod) == Q_WRP(q, q->cons); +} + +static void queue_sync_cons(struct arm_smmu_queue *q) +{ + q->cons = readl_relaxed(q->cons_reg); +} + +static void queue_inc_cons(struct arm_smmu_queue *q) +{ + u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1; + + q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); + writel(q->cons, q->cons_reg); +} + +static int queue_sync_prod(struct arm_smmu_queue *q) +{ + int ret = 0; + u32 prod = readl_relaxed(q->prod_reg); + + if (Q_OVF(q, prod) != Q_OVF(q, q->prod)) + ret = -EOVERFLOW; + + q->prod = prod; + return ret; +} + +static void queue_inc_prod(struct arm_smmu_queue *q) +{ + u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1; + + q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod); + writel(q->prod, q->prod_reg); +} + +static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until) +{ + if (Q_WRP(q, q->cons) == Q_WRP(q, until)) + return Q_IDX(q, q->cons) < Q_IDX(q, until); + + return Q_IDX(q, q->cons) >= Q_IDX(q, until); +} + +static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe) +{ + ktime_t timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US); + + while (queue_sync_cons(q), __queue_cons_before(q, until)) { + if (ktime_compare(ktime_get(), timeout) > 0) + return -ETIMEDOUT; + + if (wfe) { + wfe(); + } else { + cpu_relax(); + udelay(1); + } + } + + return 0; +} + +static void queue_write(__le64 *dst, u64 *src, size_t n_dwords) +{ + int i; + + for (i = 0; i < n_dwords; ++i) + *dst++ = cpu_to_le64(*src++); +} + +static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent) +{ + if (queue_full(q)) + return -ENOSPC; + + queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords); + queue_inc_prod(q); + return 0; +} + +static void queue_read(__le64 *dst, u64 *src, size_t n_dwords) +{ + int i; + + for (i = 0; i < n_dwords; ++i) + *dst++ = le64_to_cpu(*src++); +} + +static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent) +{ + if (queue_empty(q)) + return -EAGAIN; + + queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords); + queue_inc_cons(q); + return 0; +} + +/* High-level queue accessors */ +static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent) +{ + memset(cmd, 0, CMDQ_ENT_DWORDS << 3); + cmd[0] |= (ent->opcode & CMDQ_0_OP_MASK) << CMDQ_0_OP_SHIFT; + + switch (ent->opcode) { + case CMDQ_OP_TLBI_EL2_ALL: + case CMDQ_OP_TLBI_NSNH_ALL: + break; + case CMDQ_OP_PREFETCH_CFG: + cmd[0] |= (u64)ent->prefetch.sid << CMDQ_PREFETCH_0_SID_SHIFT; + cmd[1] |= ent->prefetch.size << CMDQ_PREFETCH_1_SIZE_SHIFT; + cmd[1] |= ent->prefetch.addr & CMDQ_PREFETCH_1_ADDR_MASK; + break; + case CMDQ_OP_CFGI_STE: + cmd[0] |= (u64)ent->cfgi.sid << CMDQ_CFGI_0_SID_SHIFT; + cmd[1] |= ent->cfgi.leaf ? CMDQ_CFGI_1_LEAF : 0; + break; + case CMDQ_OP_CFGI_ALL: + /* Cover the entire SID range */ + cmd[1] |= CMDQ_CFGI_1_RANGE_MASK << CMDQ_CFGI_1_RANGE_SHIFT; + break; + case CMDQ_OP_TLBI_NH_VA: + cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT; + /* Fallthrough */ + case CMDQ_OP_TLBI_S2_IPA: + cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT; + cmd[1] |= ent->tlbi.leaf ? CMDQ_TLBI_1_LEAF : 0; + cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_ADDR_MASK; + break; + case CMDQ_OP_TLBI_NH_ASID: + cmd[0] |= (u64)ent->tlbi.asid << CMDQ_TLBI_0_ASID_SHIFT; + /* Fallthrough */ + case CMDQ_OP_TLBI_S12_VMALL: + cmd[0] |= (u64)ent->tlbi.vmid << CMDQ_TLBI_0_VMID_SHIFT; + break; + case CMDQ_OP_PRI_RESP: + cmd[0] |= ent->substream_valid ? CMDQ_0_SSV : 0; + cmd[0] |= ent->pri.ssid << CMDQ_PRI_0_SSID_SHIFT; + cmd[0] |= (u64)ent->pri.sid << CMDQ_PRI_0_SID_SHIFT; + cmd[1] |= ent->pri.grpid << CMDQ_PRI_1_GRPID_SHIFT; + switch (ent->pri.resp) { + case PRI_RESP_DENY: + cmd[1] |= CMDQ_PRI_1_RESP_DENY; + break; + case PRI_RESP_FAIL: + cmd[1] |= CMDQ_PRI_1_RESP_FAIL; + break; + case PRI_RESP_SUCC: + cmd[1] |= CMDQ_PRI_1_RESP_SUCC; + break; + default: + return -EINVAL; + } + break; + case CMDQ_OP_CMD_SYNC: + cmd[0] |= CMDQ_SYNC_0_CS_SEV; + break; + default: + return -ENOENT; + } + + return 0; +} + +static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) +{ + static const char *cerror_str[] = { + [CMDQ_ERR_CERROR_NONE_IDX] = "No error", + [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command", + [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch", + }; + + int i; + u64 cmd[CMDQ_ENT_DWORDS]; + struct arm_smmu_queue *q = &smmu->cmdq.q; + u32 cons = readl_relaxed(q->cons_reg); + u32 idx = cons >> CMDQ_ERR_SHIFT & CMDQ_ERR_MASK; + struct arm_smmu_cmdq_ent cmd_sync = { + .opcode = CMDQ_OP_CMD_SYNC, + }; + + dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, + cerror_str[idx]); + + switch (idx) { + case CMDQ_ERR_CERROR_ILL_IDX: + break; + case CMDQ_ERR_CERROR_ABT_IDX: + dev_err(smmu->dev, "retrying command fetch\n"); + case CMDQ_ERR_CERROR_NONE_IDX: + return; + } + + /* + * We may have concurrent producers, so we need to be careful + * not to touch any of the shadow cmdq state. + */ + queue_read(cmd, Q_ENT(q, idx), q->ent_dwords); + dev_err(smmu->dev, "skipping command in error state:\n"); + for (i = 0; i < ARRAY_SIZE(cmd); ++i) + dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); + + /* Convert the erroneous command into a CMD_SYNC */ + if (arm_smmu_cmdq_build_cmd(cmd, &cmd_sync)) { + dev_err(smmu->dev, "failed to convert to CMD_SYNC\n"); + return; + } + + queue_write(cmd, Q_ENT(q, idx), q->ent_dwords); +} + +static void arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, + struct arm_smmu_cmdq_ent *ent) +{ + u32 until; + u64 cmd[CMDQ_ENT_DWORDS]; + bool wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); + struct arm_smmu_queue *q = &smmu->cmdq.q; + + if (arm_smmu_cmdq_build_cmd(cmd, ent)) { + dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", + ent->opcode); + return; + } + + spin_lock(&smmu->cmdq.lock); + while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) { + /* + * Keep the queue locked, otherwise the producer could wrap + * twice and we could see a future consumer pointer that looks + * like it's behind us. + */ + if (queue_poll_cons(q, until, wfe)) + dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); + } + + if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe)) + dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n"); + spin_unlock(&smmu->cmdq.lock); +} + +/* Context descriptor manipulation functions */ +static u64 arm_smmu_cpu_tcr_to_cd(u64 tcr) +{ + u64 val = 0; + + /* Repack the TCR. Just care about TTBR0 for now */ + val |= ARM_SMMU_TCR2CD(tcr, T0SZ); + val |= ARM_SMMU_TCR2CD(tcr, TG0); + val |= ARM_SMMU_TCR2CD(tcr, IRGN0); + val |= ARM_SMMU_TCR2CD(tcr, ORGN0); + val |= ARM_SMMU_TCR2CD(tcr, SH0); + val |= ARM_SMMU_TCR2CD(tcr, EPD0); + val |= ARM_SMMU_TCR2CD(tcr, EPD1); + val |= ARM_SMMU_TCR2CD(tcr, IPS); + val |= ARM_SMMU_TCR2CD(tcr, TBI0); + + return val; +} + +static void arm_smmu_write_ctx_desc(struct arm_smmu_device *smmu, + struct arm_smmu_s1_cfg *cfg) +{ + u64 val; + + /* + * We don't need to issue any invalidation here, as we'll invalidate + * the STE when installing the new entry anyway. + */ + val = arm_smmu_cpu_tcr_to_cd(cfg->cd.tcr) | +#ifdef __BIG_ENDIAN + CTXDESC_CD_0_ENDI | +#endif + CTXDESC_CD_0_R | CTXDESC_CD_0_A | CTXDESC_CD_0_ASET_PRIVATE | + CTXDESC_CD_0_AA64 | (u64)cfg->cd.asid << CTXDESC_CD_0_ASID_SHIFT | + CTXDESC_CD_0_V; + cfg->cdptr[0] = cpu_to_le64(val); + + val = cfg->cd.ttbr & CTXDESC_CD_1_TTB0_MASK << CTXDESC_CD_1_TTB0_SHIFT; + cfg->cdptr[1] = cpu_to_le64(val); + + cfg->cdptr[3] = cpu_to_le64(cfg->cd.mair << CTXDESC_CD_3_MAIR_SHIFT); +} + +/* Stream table manipulation functions */ +static void +arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc) +{ + u64 val = 0; + + val |= (desc->span & STRTAB_L1_DESC_SPAN_MASK) + << STRTAB_L1_DESC_SPAN_SHIFT; + val |= desc->l2ptr_dma & + STRTAB_L1_DESC_L2PTR_MASK << STRTAB_L1_DESC_L2PTR_SHIFT; + + *dst = cpu_to_le64(val); +} + +static void arm_smmu_sync_ste_for_sid(struct arm_smmu_device *smmu, u32 sid) +{ + struct arm_smmu_cmdq_ent cmd = { + .opcode = CMDQ_OP_CFGI_STE, + .cfgi = { + .sid = sid, + .leaf = true, + }, + }; + + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + cmd.opcode = CMDQ_OP_CMD_SYNC; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); +} + +static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, + __le64 *dst, struct arm_smmu_strtab_ent *ste) +{ + /* + * This is hideously complicated, but we only really care about + * three cases at the moment: + * + * 1. Invalid (all zero) -> bypass (init) + * 2. Bypass -> translation (attach) + * 3. Translation -> bypass (detach) + * + * Given that we can't update the STE atomically and the SMMU + * doesn't read the thing in a defined order, that leaves us + * with the following maintenance requirements: + * + * 1. Update Config, return (init time STEs aren't live) + * 2. Write everything apart from dword 0, sync, write dword 0, sync + * 3. Update Config, sync + */ + u64 val = le64_to_cpu(dst[0]); + bool ste_live = false; + struct arm_smmu_cmdq_ent prefetch_cmd = { + .opcode = CMDQ_OP_PREFETCH_CFG, + .prefetch = { + .sid = sid, + }, + }; + + if (val & STRTAB_STE_0_V) { + u64 cfg; + + cfg = val & STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT; + switch (cfg) { + case STRTAB_STE_0_CFG_BYPASS: + break; + case STRTAB_STE_0_CFG_S1_TRANS: + case STRTAB_STE_0_CFG_S2_TRANS: + ste_live = true; + break; + default: + BUG(); /* STE corruption */ + } + } + + /* Nuke the existing Config, as we're going to rewrite it */ + val &= ~(STRTAB_STE_0_CFG_MASK << STRTAB_STE_0_CFG_SHIFT); + + if (ste->valid) + val |= STRTAB_STE_0_V; + else + val &= ~STRTAB_STE_0_V; + + if (ste->bypass) { + val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT + : STRTAB_STE_0_CFG_BYPASS; + dst[0] = cpu_to_le64(val); + dst[2] = 0; /* Nuke the VMID */ + if (ste_live) + arm_smmu_sync_ste_for_sid(smmu, sid); + return; + } + + if (ste->s1_cfg) { + BUG_ON(ste_live); + dst[1] = cpu_to_le64( + STRTAB_STE_1_S1C_CACHE_WBRA + << STRTAB_STE_1_S1CIR_SHIFT | + STRTAB_STE_1_S1C_CACHE_WBRA + << STRTAB_STE_1_S1COR_SHIFT | + STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT | + STRTAB_STE_1_S1STALLD | +#ifdef CONFIG_PCI_ATS + STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT | +#endif + STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT); + + val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK + << STRTAB_STE_0_S1CTXPTR_SHIFT) | + STRTAB_STE_0_CFG_S1_TRANS; + + } + + if (ste->s2_cfg) { + BUG_ON(ste_live); + dst[2] = cpu_to_le64( + ste->s2_cfg->vmid << STRTAB_STE_2_S2VMID_SHIFT | + (ste->s2_cfg->vtcr & STRTAB_STE_2_VTCR_MASK) + << STRTAB_STE_2_VTCR_SHIFT | +#ifdef __BIG_ENDIAN + STRTAB_STE_2_S2ENDI | +#endif + STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2AA64 | + STRTAB_STE_2_S2R); + + dst[3] = cpu_to_le64(ste->s2_cfg->vttbr & + STRTAB_STE_3_S2TTB_MASK << STRTAB_STE_3_S2TTB_SHIFT); + + val |= STRTAB_STE_0_CFG_S2_TRANS; + } + + arm_smmu_sync_ste_for_sid(smmu, sid); + dst[0] = cpu_to_le64(val); + arm_smmu_sync_ste_for_sid(smmu, sid); + + /* It's likely that we'll want to use the new STE soon */ + arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); +} + +static void arm_smmu_init_bypass_stes(u64 *strtab, unsigned int nent) +{ + unsigned int i; + struct arm_smmu_strtab_ent ste = { + .valid = true, + .bypass = true, + }; + + for (i = 0; i < nent; ++i) { + arm_smmu_write_strtab_ent(NULL, -1, strtab, &ste); + strtab += STRTAB_STE_DWORDS; + } +} + +static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) +{ + size_t size; + void *strtab; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; + + if (desc->l2ptr) + return 0; + + size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); + strtab = &cfg->strtab[sid >> STRTAB_SPLIT << STRTAB_L1_DESC_DWORDS]; + + desc->span = STRTAB_SPLIT + 1; + desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma, + GFP_KERNEL); + if (!desc->l2ptr) { + dev_err(smmu->dev, + "failed to allocate l2 stream table for SID %u\n", + sid); + return -ENOMEM; + } + + arm_smmu_init_bypass_stes(desc->l2ptr, 1 << STRTAB_SPLIT); + arm_smmu_write_strtab_l1_desc(strtab, desc); + return 0; +} + +/* IRQ and event handlers */ +static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) +{ + int i; + struct arm_smmu_device *smmu = dev; + struct arm_smmu_queue *q = &smmu->evtq.q; + u64 evt[EVTQ_ENT_DWORDS]; + + while (!queue_remove_raw(q, evt)) { + u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK; + + dev_info(smmu->dev, "event 0x%02x received:\n", id); + for (i = 0; i < ARRAY_SIZE(evt); ++i) + dev_info(smmu->dev, "\t0x%016llx\n", + (unsigned long long)evt[i]); + } + + /* Sync our overflow flag, as we believe we're up to speed */ + q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons); + return IRQ_HANDLED; +} + +static irqreturn_t arm_smmu_evtq_handler(int irq, void *dev) +{ + irqreturn_t ret = IRQ_WAKE_THREAD; + struct arm_smmu_device *smmu = dev; + struct arm_smmu_queue *q = &smmu->evtq.q; + + /* + * Not much we can do on overflow, so scream and pretend we're + * trying harder. + */ + if (queue_sync_prod(q) == -EOVERFLOW) + dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); + else if (queue_empty(q)) + ret = IRQ_NONE; + + return ret; +} + +static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) +{ + struct arm_smmu_device *smmu = dev; + struct arm_smmu_queue *q = &smmu->priq.q; + u64 evt[PRIQ_ENT_DWORDS]; + + while (!queue_remove_raw(q, evt)) { + u32 sid, ssid; + u16 grpid; + bool ssv, last; + + sid = evt[0] >> PRIQ_0_SID_SHIFT & PRIQ_0_SID_MASK; + ssv = evt[0] & PRIQ_0_SSID_V; + ssid = ssv ? evt[0] >> PRIQ_0_SSID_SHIFT & PRIQ_0_SSID_MASK : 0; + last = evt[0] & PRIQ_0_PRG_LAST; + grpid = evt[1] >> PRIQ_1_PRG_IDX_SHIFT & PRIQ_1_PRG_IDX_MASK; + + dev_info(smmu->dev, "unexpected PRI request received:\n"); + dev_info(smmu->dev, + "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n", + sid, ssid, grpid, last ? "L" : "", + evt[0] & PRIQ_0_PERM_PRIV ? "" : "un", + evt[0] & PRIQ_0_PERM_READ ? "R" : "", + evt[0] & PRIQ_0_PERM_WRITE ? "W" : "", + evt[0] & PRIQ_0_PERM_EXEC ? "X" : "", + evt[1] & PRIQ_1_ADDR_MASK << PRIQ_1_ADDR_SHIFT); + + if (last) { + struct arm_smmu_cmdq_ent cmd = { + .opcode = CMDQ_OP_PRI_RESP, + .substream_valid = ssv, + .pri = { + .sid = sid, + .ssid = ssid, + .grpid = grpid, + .resp = PRI_RESP_DENY, + }, + }; + + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + } + } + + /* Sync our overflow flag, as we believe we're up to speed */ + q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons); + return IRQ_HANDLED; +} + +static irqreturn_t arm_smmu_priq_handler(int irq, void *dev) +{ + irqreturn_t ret = IRQ_WAKE_THREAD; + struct arm_smmu_device *smmu = dev; + struct arm_smmu_queue *q = &smmu->priq.q; + + /* PRIQ overflow indicates a programming error */ + if (queue_sync_prod(q) == -EOVERFLOW) + dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); + else if (queue_empty(q)) + ret = IRQ_NONE; + + return ret; +} + +static irqreturn_t arm_smmu_cmdq_sync_handler(int irq, void *dev) +{ + /* We don't actually use CMD_SYNC interrupts for anything */ + return IRQ_HANDLED; +} + +static int arm_smmu_device_disable(struct arm_smmu_device *smmu); + +static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev) +{ + u32 gerror, gerrorn; + struct arm_smmu_device *smmu = dev; + + gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); + gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); + + gerror ^= gerrorn; + if (!(gerror & GERROR_ERR_MASK)) + return IRQ_NONE; /* No errors pending */ + + dev_warn(smmu->dev, + "unexpected global error reported (0x%08x), this could be serious\n", + gerror); + + if (gerror & GERROR_SFM_ERR) { + dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); + arm_smmu_device_disable(smmu); + } + + if (gerror & GERROR_MSI_GERROR_ABT_ERR) + dev_warn(smmu->dev, "GERROR MSI write aborted\n"); + + if (gerror & GERROR_MSI_PRIQ_ABT_ERR) { + dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); + arm_smmu_priq_handler(irq, smmu->dev); + } + + if (gerror & GERROR_MSI_EVTQ_ABT_ERR) { + dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); + arm_smmu_evtq_handler(irq, smmu->dev); + } + + if (gerror & GERROR_MSI_CMDQ_ABT_ERR) { + dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); + arm_smmu_cmdq_sync_handler(irq, smmu->dev); + } + + if (gerror & GERROR_PRIQ_ABT_ERR) + dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); + + if (gerror & GERROR_EVTQ_ABT_ERR) + dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); + + if (gerror & GERROR_CMDQ_ERR) + arm_smmu_cmdq_skip_err(smmu); + + writel(gerror, smmu->base + ARM_SMMU_GERRORN); + return IRQ_HANDLED; +} + +/* IO_PGTABLE API */ +static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu) +{ + struct arm_smmu_cmdq_ent cmd; + + cmd.opcode = CMDQ_OP_CMD_SYNC; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); +} + +static void arm_smmu_tlb_sync(void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + __arm_smmu_tlb_sync(smmu_domain->smmu); +} + +static void arm_smmu_tlb_inv_context(void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cmdq_ent cmd; + + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + cmd.opcode = CMDQ_OP_TLBI_NH_ASID; + cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; + cmd.tlbi.vmid = 0; + } else { + cmd.opcode = CMDQ_OP_TLBI_S12_VMALL; + cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; + } + + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + __arm_smmu_tlb_sync(smmu); +} + +static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, + bool leaf, void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_cmdq_ent cmd = { + .tlbi = { + .leaf = leaf, + .addr = iova, + }, + }; + + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + cmd.opcode = CMDQ_OP_TLBI_NH_VA; + cmd.tlbi.asid = smmu_domain->s1_cfg.cd.asid; + } else { + cmd.opcode = CMDQ_OP_TLBI_S2_IPA; + cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; + } + + arm_smmu_cmdq_issue_cmd(smmu, &cmd); +} + +static void arm_smmu_flush_pgtable(void *addr, size_t size, void *cookie) +{ + struct arm_smmu_domain *smmu_domain = cookie; + struct arm_smmu_device *smmu = smmu_domain->smmu; + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + + if (smmu->features & ARM_SMMU_FEAT_COHERENCY) { + dsb(ishst); + } else { + dma_addr_t dma_addr; + struct device *dev = smmu->dev; + + dma_addr = dma_map_page(dev, virt_to_page(addr), offset, size, + DMA_TO_DEVICE); + + if (dma_mapping_error(dev, dma_addr)) + dev_err(dev, "failed to flush pgtable at %p\n", addr); + else + dma_unmap_page(dev, dma_addr, size, DMA_TO_DEVICE); + } +} + +static struct iommu_gather_ops arm_smmu_gather_ops = { + .tlb_flush_all = arm_smmu_tlb_inv_context, + .tlb_add_flush = arm_smmu_tlb_inv_range_nosync, + .tlb_sync = arm_smmu_tlb_sync, + .flush_pgtable = arm_smmu_flush_pgtable, +}; + +/* IOMMU API */ +static bool arm_smmu_capable(enum iommu_cap cap) +{ + switch (cap) { + case IOMMU_CAP_CACHE_COHERENCY: + return true; + case IOMMU_CAP_INTR_REMAP: + return true; /* MSIs are just memory writes */ + case IOMMU_CAP_NOEXEC: + return true; + default: + return false; + } +} + +static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) +{ + struct arm_smmu_domain *smmu_domain; + + if (type != IOMMU_DOMAIN_UNMANAGED) + return NULL; + + /* + * Allocate the domain and initialise some of its data structures. + * We can't really do anything meaningful until we've added a + * master. + */ + smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); + if (!smmu_domain) + return NULL; + + mutex_init(&smmu_domain->init_mutex); + spin_lock_init(&smmu_domain->pgtbl_lock); + return &smmu_domain->domain; +} + +static int arm_smmu_bitmap_alloc(unsigned long *map, int span) +{ + int idx, size = 1 << span; + + do { + idx = find_first_zero_bit(map, size); + if (idx == size) + return -ENOSPC; + } while (test_and_set_bit(idx, map)); + + return idx; +} + +static void arm_smmu_bitmap_free(unsigned long *map, int idx) +{ + clear_bit(idx, map); +} + +static void arm_smmu_domain_free(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (smmu_domain->pgtbl_ops) + free_io_pgtable_ops(smmu_domain->pgtbl_ops); + + /* Free the CD and ASID, if we allocated them */ + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; + + if (cfg->cdptr) { + dma_free_coherent(smmu_domain->smmu->dev, + CTXDESC_CD_DWORDS << 3, + cfg->cdptr, + cfg->cdptr_dma); + + arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid); + } + } else { + struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; + if (cfg->vmid) + arm_smmu_bitmap_free(smmu->vmid_map, cfg->vmid); + } + + kfree(smmu_domain); +} + +static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, + struct io_pgtable_cfg *pgtbl_cfg) +{ + int ret; + u16 asid; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; + + asid = arm_smmu_bitmap_alloc(smmu->asid_map, smmu->asid_bits); + if (IS_ERR_VALUE(asid)) + return asid; + + cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3, + &cfg->cdptr_dma, GFP_KERNEL); + if (!cfg->cdptr) { + dev_warn(smmu->dev, "failed to allocate context descriptor\n"); + goto out_free_asid; + } + + cfg->cd.asid = asid; + cfg->cd.ttbr = pgtbl_cfg->arm_lpae_s1_cfg.ttbr[0]; + cfg->cd.tcr = pgtbl_cfg->arm_lpae_s1_cfg.tcr; + cfg->cd.mair = pgtbl_cfg->arm_lpae_s1_cfg.mair[0]; + return 0; + +out_free_asid: + arm_smmu_bitmap_free(smmu->asid_map, asid); + return ret; +} + +static int arm_smmu_domain_finalise_s2(struct arm_smmu_domain *smmu_domain, + struct io_pgtable_cfg *pgtbl_cfg) +{ + u16 vmid; + struct arm_smmu_device *smmu = smmu_domain->smmu; + struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg; + + vmid = arm_smmu_bitmap_alloc(smmu->vmid_map, smmu->vmid_bits); + if (IS_ERR_VALUE(vmid)) + return vmid; + + cfg->vmid = vmid; + cfg->vttbr = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; + cfg->vtcr = pgtbl_cfg->arm_lpae_s2_cfg.vtcr; + return 0; +} + +static struct iommu_ops arm_smmu_ops; + +static int arm_smmu_domain_finalise(struct iommu_domain *domain) +{ + int ret; + unsigned long ias, oas; + enum io_pgtable_fmt fmt; + struct io_pgtable_cfg pgtbl_cfg; + struct io_pgtable_ops *pgtbl_ops; + int (*finalise_stage_fn)(struct arm_smmu_domain *, + struct io_pgtable_cfg *); + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + + /* Restrict the stage to what we can actually support */ + if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) + smmu_domain->stage = ARM_SMMU_DOMAIN_S2; + if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) + smmu_domain->stage = ARM_SMMU_DOMAIN_S1; + + switch (smmu_domain->stage) { + case ARM_SMMU_DOMAIN_S1: + ias = VA_BITS; + oas = smmu->ias; + fmt = ARM_64_LPAE_S1; + finalise_stage_fn = arm_smmu_domain_finalise_s1; + break; + case ARM_SMMU_DOMAIN_NESTED: + case ARM_SMMU_DOMAIN_S2: + ias = smmu->ias; + oas = smmu->oas; + fmt = ARM_64_LPAE_S2; + finalise_stage_fn = arm_smmu_domain_finalise_s2; + break; + default: + return -EINVAL; + } + + pgtbl_cfg = (struct io_pgtable_cfg) { + .pgsize_bitmap = arm_smmu_ops.pgsize_bitmap, + .ias = ias, + .oas = oas, + .tlb = &arm_smmu_gather_ops, + }; + + pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); + if (!pgtbl_ops) + return -ENOMEM; + + arm_smmu_ops.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; + smmu_domain->pgtbl_ops = pgtbl_ops; + + ret = finalise_stage_fn(smmu_domain, &pgtbl_cfg); + if (IS_ERR_VALUE(ret)) + free_io_pgtable_ops(pgtbl_ops); + + return ret; +} + +static struct arm_smmu_group *arm_smmu_group_get(struct device *dev) +{ + struct iommu_group *group; + struct arm_smmu_group *smmu_group; + + group = iommu_group_get(dev); + if (!group) + return NULL; + + smmu_group = iommu_group_get_iommudata(group); + iommu_group_put(group); + return smmu_group; +} + +static __le64 *arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) +{ + __le64 *step; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + struct arm_smmu_strtab_l1_desc *l1_desc; + int idx; + + /* Two-level walk */ + idx = (sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS; + l1_desc = &cfg->l1_desc[idx]; + idx = (sid & ((1 << STRTAB_SPLIT) - 1)) * STRTAB_STE_DWORDS; + step = &l1_desc->l2ptr[idx]; + } else { + /* Simple linear lookup */ + step = &cfg->strtab[sid * STRTAB_STE_DWORDS]; + } + + return step; +} + +static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group) +{ + int i; + struct arm_smmu_domain *smmu_domain = smmu_group->domain; + struct arm_smmu_strtab_ent *ste = &smmu_group->ste; + struct arm_smmu_device *smmu = smmu_group->smmu; + + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { + ste->s1_cfg = &smmu_domain->s1_cfg; + ste->s2_cfg = NULL; + arm_smmu_write_ctx_desc(smmu, ste->s1_cfg); + } else { + ste->s1_cfg = NULL; + ste->s2_cfg = &smmu_domain->s2_cfg; + } + + for (i = 0; i < smmu_group->num_sids; ++i) { + u32 sid = smmu_group->sids[i]; + __le64 *step = arm_smmu_get_step_for_sid(smmu, sid); + + arm_smmu_write_strtab_ent(smmu, sid, step, ste); + } + + return 0; +} + +static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) +{ + int ret = 0; + struct arm_smmu_device *smmu; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); + + if (!smmu_group) + return -ENOENT; + + /* Already attached to a different domain? */ + if (smmu_group->domain && smmu_group->domain != smmu_domain) + return -EEXIST; + + smmu = smmu_group->smmu; + mutex_lock(&smmu_domain->init_mutex); + + if (!smmu_domain->smmu) { + smmu_domain->smmu = smmu; + ret = arm_smmu_domain_finalise(domain); + if (ret) { + smmu_domain->smmu = NULL; + goto out_unlock; + } + } else if (smmu_domain->smmu != smmu) { + dev_err(dev, + "cannot attach to SMMU %s (upstream of %s)\n", + dev_name(smmu_domain->smmu->dev), + dev_name(smmu->dev)); + ret = -ENXIO; + goto out_unlock; + } + + /* Group already attached to this domain? */ + if (smmu_group->domain) + goto out_unlock; + + smmu_group->domain = smmu_domain; + smmu_group->ste.bypass = false; + + ret = arm_smmu_install_ste_for_group(smmu_group); + if (IS_ERR_VALUE(ret)) + smmu_group->domain = NULL; + +out_unlock: + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); + + BUG_ON(!smmu_domain); + BUG_ON(!smmu_group); + + mutex_lock(&smmu_domain->init_mutex); + BUG_ON(smmu_group->domain != smmu_domain); + + smmu_group->ste.bypass = true; + if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group))) + dev_warn(dev, "failed to install bypass STE\n"); + + smmu_group->domain = NULL; + mutex_unlock(&smmu_domain->init_mutex); +} + +static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + int ret; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + + if (!ops) + return -ENODEV; + + spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); + ret = ops->map(ops, iova, paddr, size, prot); + spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + return ret; +} + +static size_t +arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) +{ + size_t ret; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + + if (!ops) + return 0; + + spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); + ret = ops->unmap(ops, iova, size); + spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + return ret; +} + +static phys_addr_t +arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) +{ + phys_addr_t ret; + unsigned long flags; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; + + if (!ops) + return 0; + + spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags); + ret = ops->iova_to_phys(ops, iova); + spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags); + + return ret; +} + +static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *sidp) +{ + *(u32 *)sidp = alias; + return 0; /* Continue walking */ +} + +static void __arm_smmu_release_pci_iommudata(void *data) +{ + kfree(data); +} + +static struct arm_smmu_device *arm_smmu_get_for_pci_dev(struct pci_dev *pdev) +{ + struct device_node *of_node; + struct arm_smmu_device *curr, *smmu = NULL; + struct pci_bus *bus = pdev->bus; + + /* Walk up to the root bus */ + while (!pci_is_root_bus(bus)) + bus = bus->parent; + + /* Follow the "iommus" phandle from the host controller */ + of_node = of_parse_phandle(bus->bridge->parent->of_node, "iommus", 0); + if (!of_node) + return NULL; + + /* See if we can find an SMMU corresponding to the phandle */ + spin_lock(&arm_smmu_devices_lock); + list_for_each_entry(curr, &arm_smmu_devices, list) { + if (curr->dev->of_node == of_node) { + smmu = curr; + break; + } + } + spin_unlock(&arm_smmu_devices_lock); + of_node_put(of_node); + return smmu; +} + +static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) +{ + unsigned long limit = smmu->strtab_cfg.num_l1_ents; + + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) + limit *= 1UL << STRTAB_SPLIT; + + return sid < limit; +} + +static int arm_smmu_add_device(struct device *dev) +{ + int i, ret; + u32 sid, *sids; + struct pci_dev *pdev; + struct iommu_group *group; + struct arm_smmu_group *smmu_group; + struct arm_smmu_device *smmu; + + /* We only support PCI, for now */ + if (!dev_is_pci(dev)) + return -ENODEV; + + pdev = to_pci_dev(dev); + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + smmu_group = iommu_group_get_iommudata(group); + if (!smmu_group) { + smmu = arm_smmu_get_for_pci_dev(pdev); + if (!smmu) { + ret = -ENOENT; + goto out_put_group; + } + + smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL); + if (!smmu_group) { + ret = -ENOMEM; + goto out_put_group; + } + + smmu_group->ste.valid = true; + smmu_group->smmu = smmu; + iommu_group_set_iommudata(group, smmu_group, + __arm_smmu_release_pci_iommudata); + } else { + smmu = smmu_group->smmu; + } + + /* Assume SID == RID until firmware tells us otherwise */ + pci_for_each_dma_alias(pdev, __arm_smmu_get_pci_sid, &sid); + for (i = 0; i < smmu_group->num_sids; ++i) { + /* If we already know about this SID, then we're done */ + if (smmu_group->sids[i] == sid) + return 0; + } + + /* Check the SID is in range of the SMMU and our stream table */ + if (!arm_smmu_sid_in_range(smmu, sid)) { + ret = -ERANGE; + goto out_put_group; + } + + /* Ensure l2 strtab is initialised */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + ret = arm_smmu_init_l2_strtab(smmu, sid); + if (ret) + goto out_put_group; + } + + /* Resize the SID array for the group */ + smmu_group->num_sids++; + sids = krealloc(smmu_group->sids, smmu_group->num_sids * sizeof(*sids), + GFP_KERNEL); + if (!sids) { + smmu_group->num_sids--; + ret = -ENOMEM; + goto out_put_group; + } + + /* Add the new SID */ + sids[smmu_group->num_sids - 1] = sid; + smmu_group->sids = sids; + return 0; + +out_put_group: + iommu_group_put(group); + return ret; +} + +static void arm_smmu_remove_device(struct device *dev) +{ + iommu_group_remove_device(dev); +} + +static int arm_smmu_domain_get_attr(struct iommu_domain *domain, + enum iommu_attr attr, void *data) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + switch (attr) { + case DOMAIN_ATTR_NESTING: + *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); + return 0; + default: + return -ENODEV; + } +} + +static int arm_smmu_domain_set_attr(struct iommu_domain *domain, + enum iommu_attr attr, void *data) +{ + int ret = 0; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + mutex_lock(&smmu_domain->init_mutex); + + switch (attr) { + case DOMAIN_ATTR_NESTING: + if (smmu_domain->smmu) { + ret = -EPERM; + goto out_unlock; + } + + if (*(int *)data) + smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; + else + smmu_domain->stage = ARM_SMMU_DOMAIN_S1; + + break; + default: + ret = -ENODEV; + } + +out_unlock: + mutex_unlock(&smmu_domain->init_mutex); + return ret; +} + +static struct iommu_ops arm_smmu_ops = { + .capable = arm_smmu_capable, + .domain_alloc = arm_smmu_domain_alloc, + .domain_free = arm_smmu_domain_free, + .attach_dev = arm_smmu_attach_dev, + .detach_dev = arm_smmu_detach_dev, + .map = arm_smmu_map, + .unmap = arm_smmu_unmap, + .iova_to_phys = arm_smmu_iova_to_phys, + .add_device = arm_smmu_add_device, + .remove_device = arm_smmu_remove_device, + .domain_get_attr = arm_smmu_domain_get_attr, + .domain_set_attr = arm_smmu_domain_set_attr, + .pgsize_bitmap = -1UL, /* Restricted during device attach */ +}; + +/* Probing and initialisation functions */ +static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, + struct arm_smmu_queue *q, + unsigned long prod_off, + unsigned long cons_off, + size_t dwords) +{ + size_t qsz = ((1 << q->max_n_shift) * dwords) << 3; + + q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL); + if (!q->base) { + dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n", + qsz); + return -ENOMEM; + } + + q->prod_reg = smmu->base + prod_off; + q->cons_reg = smmu->base + cons_off; + q->ent_dwords = dwords; + + q->q_base = Q_BASE_RWA; + q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT; + q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK) + << Q_BASE_LOG2SIZE_SHIFT; + + q->prod = q->cons = 0; + return 0; +} + +static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu, + struct arm_smmu_queue *q) +{ + size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3; + + dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma); +} + +static void arm_smmu_free_queues(struct arm_smmu_device *smmu) +{ + arm_smmu_free_one_queue(smmu, &smmu->cmdq.q); + arm_smmu_free_one_queue(smmu, &smmu->evtq.q); + + if (smmu->features & ARM_SMMU_FEAT_PRI) + arm_smmu_free_one_queue(smmu, &smmu->priq.q); +} + +static int arm_smmu_init_queues(struct arm_smmu_device *smmu) +{ + int ret; + + /* cmdq */ + spin_lock_init(&smmu->cmdq.lock); + ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD, + ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS); + if (ret) + goto out; + + /* evtq */ + ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD, + ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS); + if (ret) + goto out_free_cmdq; + + /* priq */ + if (!(smmu->features & ARM_SMMU_FEAT_PRI)) + return 0; + + ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, + ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS); + if (ret) + goto out_free_evtq; + + return 0; + +out_free_evtq: + arm_smmu_free_one_queue(smmu, &smmu->evtq.q); +out_free_cmdq: + arm_smmu_free_one_queue(smmu, &smmu->cmdq.q); +out: + return ret; +} + +static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu) +{ + int i; + size_t size; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + + size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); + for (i = 0; i < cfg->num_l1_ents; ++i) { + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i]; + + if (!desc->l2ptr) + continue; + + dma_free_coherent(smmu->dev, size, desc->l2ptr, + desc->l2ptr_dma); + } +} + +static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) +{ + unsigned int i; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + size_t size = sizeof(*cfg->l1_desc) * cfg->num_l1_ents; + void *strtab = smmu->strtab_cfg.strtab; + + cfg->l1_desc = devm_kzalloc(smmu->dev, size, GFP_KERNEL); + if (!cfg->l1_desc) { + dev_err(smmu->dev, "failed to allocate l1 stream table desc\n"); + return -ENOMEM; + } + + for (i = 0; i < cfg->num_l1_ents; ++i) { + arm_smmu_write_strtab_l1_desc(strtab, &cfg->l1_desc[i]); + strtab += STRTAB_L1_DESC_DWORDS << 3; + } + + return 0; +} + +static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) +{ + void *strtab; + u64 reg; + u32 size; + int ret; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + + /* Calculate the L1 size, capped to the SIDSIZE */ + size = STRTAB_L1_SZ_SHIFT - (ilog2(STRTAB_L1_DESC_DWORDS) + 3); + size = min(size, smmu->sid_bits - STRTAB_SPLIT); + if (size + STRTAB_SPLIT < smmu->sid_bits) + dev_warn(smmu->dev, + "2-level strtab only covers %u/%u bits of SID\n", + size + STRTAB_SPLIT, smmu->sid_bits); + + cfg->num_l1_ents = 1 << size; + size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); + strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma, + GFP_KERNEL); + if (!strtab) { + dev_err(smmu->dev, + "failed to allocate l1 stream table (%u bytes)\n", + size); + return -ENOMEM; + } + cfg->strtab = strtab; + + /* Configure strtab_base_cfg for 2 levels */ + reg = STRTAB_BASE_CFG_FMT_2LVL; + reg |= (size & STRTAB_BASE_CFG_LOG2SIZE_MASK) + << STRTAB_BASE_CFG_LOG2SIZE_SHIFT; + reg |= (STRTAB_SPLIT & STRTAB_BASE_CFG_SPLIT_MASK) + << STRTAB_BASE_CFG_SPLIT_SHIFT; + cfg->strtab_base_cfg = reg; + + ret = arm_smmu_init_l1_strtab(smmu); + if (ret) + dma_free_coherent(smmu->dev, + cfg->num_l1_ents * + (STRTAB_L1_DESC_DWORDS << 3), + strtab, + cfg->strtab_dma); + return ret; +} + +static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) +{ + void *strtab; + u64 reg; + u32 size; + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + + size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); + strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma, + GFP_KERNEL); + if (!strtab) { + dev_err(smmu->dev, + "failed to allocate linear stream table (%u bytes)\n", + size); + return -ENOMEM; + } + cfg->strtab = strtab; + cfg->num_l1_ents = 1 << smmu->sid_bits; + + /* Configure strtab_base_cfg for a linear table covering all SIDs */ + reg = STRTAB_BASE_CFG_FMT_LINEAR; + reg |= (smmu->sid_bits & STRTAB_BASE_CFG_LOG2SIZE_MASK) + << STRTAB_BASE_CFG_LOG2SIZE_SHIFT; + cfg->strtab_base_cfg = reg; + + arm_smmu_init_bypass_stes(strtab, cfg->num_l1_ents); + return 0; +} + +static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) +{ + u64 reg; + int ret; + + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) + ret = arm_smmu_init_strtab_2lvl(smmu); + else + ret = arm_smmu_init_strtab_linear(smmu); + + if (ret) + return ret; + + /* Set the strtab base address */ + reg = smmu->strtab_cfg.strtab_dma & + STRTAB_BASE_ADDR_MASK << STRTAB_BASE_ADDR_SHIFT; + reg |= STRTAB_BASE_RA; + smmu->strtab_cfg.strtab_base = reg; + + /* Allocate the first VMID for stage-2 bypass STEs */ + set_bit(0, smmu->vmid_map); + return 0; +} + +static void arm_smmu_free_strtab(struct arm_smmu_device *smmu) +{ + struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; + u32 size = cfg->num_l1_ents; + + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + arm_smmu_free_l2_strtab(smmu); + size *= STRTAB_L1_DESC_DWORDS << 3; + } else { + size *= STRTAB_STE_DWORDS * 3; + } + + dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma); +} + +static int arm_smmu_init_structures(struct arm_smmu_device *smmu) +{ + int ret; + + ret = arm_smmu_init_queues(smmu); + if (ret) + return ret; + + ret = arm_smmu_init_strtab(smmu); + if (ret) + goto out_free_queues; + + return 0; + +out_free_queues: + arm_smmu_free_queues(smmu); + return ret; +} + +static void arm_smmu_free_structures(struct arm_smmu_device *smmu) +{ + arm_smmu_free_strtab(smmu); + arm_smmu_free_queues(smmu); +} + +static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, + unsigned int reg_off, unsigned int ack_off) +{ + u32 reg; + + writel_relaxed(val, smmu->base + reg_off); + return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, + 1, ARM_SMMU_POLL_TIMEOUT_US); +} + +static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) +{ + int ret, irq; + + /* Disable IRQs first */ + ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, + ARM_SMMU_IRQ_CTRLACK); + if (ret) { + dev_err(smmu->dev, "failed to disable irqs\n"); + return ret; + } + + /* Clear the MSI address regs */ + writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); + writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0); + + /* Request wired interrupt lines */ + irq = smmu->evtq.q.irq; + if (irq) { + ret = devm_request_threaded_irq(smmu->dev, irq, + arm_smmu_evtq_handler, + arm_smmu_evtq_thread, + 0, "arm-smmu-v3-evtq", smmu); + if (IS_ERR_VALUE(ret)) + dev_warn(smmu->dev, "failed to enable evtq irq\n"); + } + + irq = smmu->cmdq.q.irq; + if (irq) { + ret = devm_request_irq(smmu->dev, irq, + arm_smmu_cmdq_sync_handler, 0, + "arm-smmu-v3-cmdq-sync", smmu); + if (IS_ERR_VALUE(ret)) + dev_warn(smmu->dev, "failed to enable cmdq-sync irq\n"); + } + + irq = smmu->gerr_irq; + if (irq) { + ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, + 0, "arm-smmu-v3-gerror", smmu); + if (IS_ERR_VALUE(ret)) + dev_warn(smmu->dev, "failed to enable gerror irq\n"); + } + + if (smmu->features & ARM_SMMU_FEAT_PRI) { + writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); + + irq = smmu->priq.q.irq; + if (irq) { + ret = devm_request_threaded_irq(smmu->dev, irq, + arm_smmu_priq_handler, + arm_smmu_priq_thread, + 0, "arm-smmu-v3-priq", + smmu); + if (IS_ERR_VALUE(ret)) + dev_warn(smmu->dev, + "failed to enable priq irq\n"); + } + } + + /* Enable interrupt generation on the SMMU */ + ret = arm_smmu_write_reg_sync(smmu, + IRQ_CTRL_EVTQ_IRQEN | + IRQ_CTRL_GERROR_IRQEN, + ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK); + if (ret) + dev_warn(smmu->dev, "failed to enable irqs\n"); + + return 0; +} + +static int arm_smmu_device_disable(struct arm_smmu_device *smmu) +{ + int ret; + + ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); + if (ret) + dev_err(smmu->dev, "failed to clear cr0\n"); + + return ret; +} + +static int arm_smmu_device_reset(struct arm_smmu_device *smmu) +{ + int ret; + u32 reg, enables; + struct arm_smmu_cmdq_ent cmd; + + /* Clear CR0 and sync (disables SMMU and queue processing) */ + reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); + if (reg & CR0_SMMUEN) + dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); + + ret = arm_smmu_device_disable(smmu); + if (ret) + return ret; + + /* CR1 (table and queue memory attributes) */ + reg = (CR1_SH_ISH << CR1_TABLE_SH_SHIFT) | + (CR1_CACHE_WB << CR1_TABLE_OC_SHIFT) | + (CR1_CACHE_WB << CR1_TABLE_IC_SHIFT) | + (CR1_SH_ISH << CR1_QUEUE_SH_SHIFT) | + (CR1_CACHE_WB << CR1_QUEUE_OC_SHIFT) | + (CR1_CACHE_WB << CR1_QUEUE_IC_SHIFT); + writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); + + /* CR2 (random crap) */ + reg = CR2_PTM | CR2_RECINVSID | CR2_E2H; + writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); + + /* Stream table */ + writeq_relaxed(smmu->strtab_cfg.strtab_base, + smmu->base + ARM_SMMU_STRTAB_BASE); + writel_relaxed(smmu->strtab_cfg.strtab_base_cfg, + smmu->base + ARM_SMMU_STRTAB_BASE_CFG); + + /* Command queue */ + writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); + writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD); + writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS); + + enables = CR0_CMDQEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable command queue\n"); + return ret; + } + + /* Invalidate any cached configuration */ + cmd.opcode = CMDQ_OP_CFGI_ALL; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + cmd.opcode = CMDQ_OP_CMD_SYNC; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + + /* Invalidate any stale TLB entries */ + if (smmu->features & ARM_SMMU_FEAT_HYP) { + cmd.opcode = CMDQ_OP_TLBI_EL2_ALL; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + } + + cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + cmd.opcode = CMDQ_OP_CMD_SYNC; + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + + /* Event queue */ + writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); + writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD); + writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS); + + enables |= CR0_EVTQEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable event queue\n"); + return ret; + } + + /* PRI queue */ + if (smmu->features & ARM_SMMU_FEAT_PRI) { + writeq_relaxed(smmu->priq.q.q_base, + smmu->base + ARM_SMMU_PRIQ_BASE); + writel_relaxed(smmu->priq.q.prod, + smmu->base + ARM_SMMU_PRIQ_PROD); + writel_relaxed(smmu->priq.q.cons, + smmu->base + ARM_SMMU_PRIQ_CONS); + + enables |= CR0_PRIQEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable PRI queue\n"); + return ret; + } + } + + ret = arm_smmu_setup_irqs(smmu); + if (ret) { + dev_err(smmu->dev, "failed to setup irqs\n"); + return ret; + } + + /* Enable the SMMU interface */ + enables |= CR0_SMMUEN; + ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, + ARM_SMMU_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable SMMU interface\n"); + return ret; + } + + return 0; +} + +static int arm_smmu_device_probe(struct arm_smmu_device *smmu) +{ + u32 reg; + bool coherent; + unsigned long pgsize_bitmap = 0; + + /* IDR0 */ + reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); + + /* 2-level structures */ + if ((reg & IDR0_ST_LVL_MASK << IDR0_ST_LVL_SHIFT) == IDR0_ST_LVL_2LVL) + smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; + + if (reg & IDR0_CD2L) + smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB; + + /* + * Translation table endianness. + * We currently require the same endianness as the CPU, but this + * could be changed later by adding a new IO_PGTABLE_QUIRK. + */ + switch (reg & IDR0_TTENDIAN_MASK << IDR0_TTENDIAN_SHIFT) { + case IDR0_TTENDIAN_MIXED: + smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; + break; +#ifdef __BIG_ENDIAN + case IDR0_TTENDIAN_BE: + smmu->features |= ARM_SMMU_FEAT_TT_BE; + break; +#else + case IDR0_TTENDIAN_LE: + smmu->features |= ARM_SMMU_FEAT_TT_LE; + break; +#endif + default: + dev_err(smmu->dev, "unknown/unsupported TT endianness!\n"); + return -ENXIO; + } + + /* Boolean feature flags */ + if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI) + smmu->features |= ARM_SMMU_FEAT_PRI; + + if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS) + smmu->features |= ARM_SMMU_FEAT_ATS; + + if (reg & IDR0_SEV) + smmu->features |= ARM_SMMU_FEAT_SEV; + + if (reg & IDR0_MSI) + smmu->features |= ARM_SMMU_FEAT_MSI; + + if (reg & IDR0_HYP) + smmu->features |= ARM_SMMU_FEAT_HYP; + + /* + * The dma-coherent property is used in preference to the ID + * register, but warn on mismatch. + */ + coherent = of_dma_is_coherent(smmu->dev->of_node); + if (coherent) + smmu->features |= ARM_SMMU_FEAT_COHERENCY; + + if (!!(reg & IDR0_COHACC) != coherent) + dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n", + coherent ? "true" : "false"); + + if (reg & IDR0_STALL_MODEL) + smmu->features |= ARM_SMMU_FEAT_STALLS; + + if (reg & IDR0_S1P) + smmu->features |= ARM_SMMU_FEAT_TRANS_S1; + + if (reg & IDR0_S2P) + smmu->features |= ARM_SMMU_FEAT_TRANS_S2; + + if (!(reg & (IDR0_S1P | IDR0_S2P))) { + dev_err(smmu->dev, "no translation support!\n"); + return -ENXIO; + } + + /* We only support the AArch64 table format at present */ + if ((reg & IDR0_TTF_MASK << IDR0_TTF_SHIFT) < IDR0_TTF_AARCH64) { + dev_err(smmu->dev, "AArch64 table format not supported!\n"); + return -ENXIO; + } + + /* ASID/VMID sizes */ + smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8; + smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8; + + /* IDR1 */ + reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1); + if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) { + dev_err(smmu->dev, "embedded implementation not supported\n"); + return -ENXIO; + } + + /* Queue sizes, capped at 4k */ + smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT, + reg >> IDR1_CMDQ_SHIFT & IDR1_CMDQ_MASK); + if (!smmu->cmdq.q.max_n_shift) { + /* Odd alignment restrictions on the base, so ignore for now */ + dev_err(smmu->dev, "unit-length command queue not supported\n"); + return -ENXIO; + } + + smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT, + reg >> IDR1_EVTQ_SHIFT & IDR1_EVTQ_MASK); + smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT, + reg >> IDR1_PRIQ_SHIFT & IDR1_PRIQ_MASK); + + /* SID/SSID sizes */ + smmu->ssid_bits = reg >> IDR1_SSID_SHIFT & IDR1_SSID_MASK; + smmu->sid_bits = reg >> IDR1_SID_SHIFT & IDR1_SID_MASK; + + /* IDR5 */ + reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); + + /* Maximum number of outstanding stalls */ + smmu->evtq.max_stalls = reg >> IDR5_STALL_MAX_SHIFT + & IDR5_STALL_MAX_MASK; + + /* Page sizes */ + if (reg & IDR5_GRAN64K) + pgsize_bitmap |= SZ_64K | SZ_512M; + if (reg & IDR5_GRAN16K) + pgsize_bitmap |= SZ_16K | SZ_32M; + if (reg & IDR5_GRAN4K) + pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; + + arm_smmu_ops.pgsize_bitmap &= pgsize_bitmap; + + /* Output address size */ + switch (reg & IDR5_OAS_MASK << IDR5_OAS_SHIFT) { + case IDR5_OAS_32_BIT: + smmu->oas = 32; + break; + case IDR5_OAS_36_BIT: + smmu->oas = 36; + break; + case IDR5_OAS_40_BIT: + smmu->oas = 40; + break; + case IDR5_OAS_42_BIT: + smmu->oas = 42; + break; + case IDR5_OAS_44_BIT: + smmu->oas = 44; + break; + case IDR5_OAS_48_BIT: + smmu->oas = 48; + break; + default: + dev_err(smmu->dev, "unknown output address size!\n"); + return -ENXIO; + } + + /* Set the DMA mask for our table walker */ + if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) + dev_warn(smmu->dev, + "failed to set DMA mask for table walker\n"); + + if (!smmu->ias) + smmu->ias = smmu->oas; + + dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", + smmu->ias, smmu->oas, smmu->features); + return 0; +} + +static int arm_smmu_device_dt_probe(struct platform_device *pdev) +{ + int irq, ret; + struct resource *res; + struct arm_smmu_device *smmu; + struct device *dev = &pdev->dev; + + smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); + if (!smmu) { + dev_err(dev, "failed to allocate arm_smmu_device\n"); + return -ENOMEM; + } + smmu->dev = dev; + + /* Base address */ + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (resource_size(res) + 1 < SZ_128K) { + dev_err(dev, "MMIO region too small (%pr)\n", res); + return -EINVAL; + } + + smmu->base = devm_ioremap_resource(dev, res); + if (IS_ERR(smmu->base)) + return PTR_ERR(smmu->base); + + /* Interrupt lines */ + irq = platform_get_irq_byname(pdev, "eventq"); + if (irq > 0) + smmu->evtq.q.irq = irq; + + irq = platform_get_irq_byname(pdev, "priq"); + if (irq > 0) + smmu->priq.q.irq = irq; + + irq = platform_get_irq_byname(pdev, "cmdq-sync"); + if (irq > 0) + smmu->cmdq.q.irq = irq; + + irq = platform_get_irq_byname(pdev, "gerror"); + if (irq > 0) + smmu->gerr_irq = irq; + + /* Probe the h/w */ + ret = arm_smmu_device_probe(smmu); + if (ret) + return ret; + + /* Initialise in-memory data structures */ + ret = arm_smmu_init_structures(smmu); + if (ret) + return ret; + + /* Reset the device */ + ret = arm_smmu_device_reset(smmu); + if (ret) + goto out_free_structures; + + /* Record our private device structure */ + INIT_LIST_HEAD(&smmu->list); + spin_lock(&arm_smmu_devices_lock); + list_add(&smmu->list, &arm_smmu_devices); + spin_unlock(&arm_smmu_devices_lock); + return 0; + +out_free_structures: + arm_smmu_free_structures(smmu); + return ret; +} + +static int arm_smmu_device_remove(struct platform_device *pdev) +{ + struct arm_smmu_device *curr, *smmu = NULL; + struct device *dev = &pdev->dev; + + spin_lock(&arm_smmu_devices_lock); + list_for_each_entry(curr, &arm_smmu_devices, list) { + if (curr->dev == dev) { + smmu = curr; + list_del(&smmu->list); + break; + } + } + spin_unlock(&arm_smmu_devices_lock); + + if (!smmu) + return -ENODEV; + + arm_smmu_device_disable(smmu); + arm_smmu_free_structures(smmu); + return 0; +} + +static struct of_device_id arm_smmu_of_match[] = { + { .compatible = "arm,smmu-v3", }, + { }, +}; +MODULE_DEVICE_TABLE(of, arm_smmu_of_match); + +static struct platform_driver arm_smmu_driver = { + .driver = { + .name = "arm-smmu-v3", + .of_match_table = of_match_ptr(arm_smmu_of_match), + }, + .probe = arm_smmu_device_dt_probe, + .remove = arm_smmu_device_remove, +}; + +static int __init arm_smmu_init(void) +{ + struct device_node *np; + int ret; + + np = of_find_matching_node(NULL, arm_smmu_of_match); + if (!np) + return 0; + + of_node_put(np); + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) + return ret; + + return bus_set_iommu(&pci_bus_type, &arm_smmu_ops); +} + +static void __exit arm_smmu_exit(void) +{ + return platform_driver_unregister(&arm_smmu_driver); +} + +subsys_initcall(arm_smmu_init); +module_exit(arm_smmu_exit); + +MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations"); +MODULE_AUTHOR("Will Deacon "); +MODULE_LICENSE("GPL v2"); -- cgit v0.10.2 From 8a0a01bff855a6763d937acc70a876cb0bed063d Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 27 May 2015 17:26:00 +0100 Subject: drivers/vfio: Allow type-1 IOMMU instantiation on top of an ARM SMMUv3 The ARM SMMUv3 driver is compatible with the notion of a type-1 IOMMU in VFIO. This patch allows VFIO_IOMMU_TYPE1 to be selected if ARM_SMMU_V3=y. Signed-off-by: Will Deacon Acked-by: Alex Williamson Signed-off-by: Joerg Roedel diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index 7d092dd..4540179 100644 --- a/drivers/vfio/Kconfig +++ b/drivers/vfio/Kconfig @@ -21,7 +21,7 @@ config VFIO_VIRQFD menuconfig VFIO tristate "VFIO Non-Privileged userspace driver framework" depends on IOMMU_API - select VFIO_IOMMU_TYPE1 if (X86 || S390 || ARM_SMMU) + select VFIO_IOMMU_TYPE1 if (X86 || S390 || ARM_SMMU || ARM_SMMU_V3) select VFIO_IOMMU_SPAPR_TCE if (PPC_POWERNV || PPC_PSERIES) select VFIO_SPAPR_EEH if (PPC_POWERNV || PPC_PSERIES) select ANON_INODES -- cgit v0.10.2 From e6aabee05f41c9d18e0b92194819edd84f352ac9 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Wed, 27 May 2015 09:26:09 +0200 Subject: iommu/amd: Handle integer overflow in dma_ops_area_alloc Handle this case to make sure boundary_size does not become 0 and trigger a BUG_ON later. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index aa710b0..4dfadcf 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1699,14 +1699,16 @@ static unsigned long dma_ops_area_alloc(struct device *dev, unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; int i = start >> APERTURE_RANGE_SHIFT; - unsigned long boundary_size; + unsigned long boundary_size, mask; unsigned long address = -1; unsigned long limit; next_bit >>= PAGE_SHIFT; - boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - PAGE_SIZE) >> PAGE_SHIFT; + mask = dma_get_seg_boundary(dev); + + boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT : + 1UL << (BITS_PER_LONG - PAGE_SHIFT); for (;i < max_index; ++i) { unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; -- cgit v0.10.2 From 92e7066fde31d5ac48a9bccc12d3063d251dd079 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:24 +0200 Subject: iommu: Remove function name from pr_fmt() Including the function name is only useful for debugging messages. They don't belong into other messages from the iommu core. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d4f527e..c31bfd0 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -16,7 +16,7 @@ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -#define pr_fmt(fmt) "%s: " fmt, __func__ +#define pr_fmt(fmt) "iommu: " fmt #include #include -- cgit v0.10.2 From 269aa808a990b3fdd0e7ec9e04322284c40748c4 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:25 +0200 Subject: iommu: Add a few printk messages to group handling code Write a message to the kernel log when a device is added or removed from a group and add debug messages to group allocation and release routines. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index c31bfd0..755e488 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -128,6 +128,8 @@ static void iommu_group_release(struct kobject *kobj) { struct iommu_group *group = to_iommu_group(kobj); + pr_debug("Releasing group %d\n", group->id); + if (group->iommu_data_release) group->iommu_data_release(group->iommu_data); @@ -207,6 +209,8 @@ again: */ kobject_put(&group->kobj); + pr_debug("Allocated group %d\n", group->id); + return group; } EXPORT_SYMBOL_GPL(iommu_group_alloc); @@ -372,6 +376,9 @@ rename: IOMMU_GROUP_NOTIFY_ADD_DEVICE, dev); trace_add_device_to_group(group->id, dev); + + pr_info("Adding device %s to group %d\n", dev_name(dev), group->id); + return 0; } EXPORT_SYMBOL_GPL(iommu_group_add_device); @@ -388,6 +395,8 @@ void iommu_group_remove_device(struct device *dev) struct iommu_group *group = dev->iommu_group; struct iommu_device *tmp_device, *device = NULL; + pr_info("Removing device %s from group %d\n", dev_name(dev), group->id); + /* Pre-notify listeners that a device is being removed. */ blocking_notifier_call_chain(&group->notifier, IOMMU_GROUP_NOTIFY_DEL_DEVICE, dev); -- cgit v0.10.2 From 19762d7095e6392b6ec56c363a6f29b2119488c2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:26 +0200 Subject: iommu: Propagate error in add_iommu_group Make sure any errors reported from the IOMMU drivers get progapated back to the IOMMU core. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 755e488..9c9336a 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -750,9 +750,7 @@ static int add_iommu_group(struct device *dev, void *data) WARN_ON(dev->iommu_group); - ops->add_device(dev); - - return 0; + return ops->add_device(dev); } static int iommu_bus_notifier(struct notifier_block *nb, -- cgit v0.10.2 From 8da30142a21e2d7595510892a4c99cf294f7e6f1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:27 +0200 Subject: iommu: Clean up after a failed bus initialization Make sure we call the ->remove_device call-back on all devices already initialized with ->add_device when the bus initialization fails. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 9c9336a..f0e0a23 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -753,6 +753,17 @@ static int add_iommu_group(struct device *dev, void *data) return ops->add_device(dev); } +static int remove_iommu_group(struct device *dev, void *data) +{ + struct iommu_callback_data *cb = data; + const struct iommu_ops *ops = cb->ops; + + if (ops->remove_device && dev->iommu_group) + ops->remove_device(dev); + + return 0; +} + static int iommu_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -821,19 +832,25 @@ static int iommu_bus_init(struct bus_type *bus, const struct iommu_ops *ops) nb->notifier_call = iommu_bus_notifier; err = bus_register_notifier(bus, nb); - if (err) { - kfree(nb); - return err; - } + if (err) + goto out_free; err = bus_for_each_dev(bus, NULL, &cb, add_iommu_group); - if (err) { - bus_unregister_notifier(bus, nb); - kfree(nb); - return err; - } + if (err) + goto out_err; + return 0; + +out_err: + /* Clean up */ + bus_for_each_dev(bus, NULL, &cb, remove_iommu_group); + bus_unregister_notifier(bus, nb); + +out_free: + kfree(nb); + + return err; } /** -- cgit v0.10.2 From 843cb6dc7749a25849797cc9aeeb86f87a8acb84 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:28 +0200 Subject: iommu: Call remove_device call-back after driver release Do not remove the device from the IOMMU while the driver is still attached. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f0e0a23..d69e0ca 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -779,7 +779,7 @@ static int iommu_bus_notifier(struct notifier_block *nb, if (action == BUS_NOTIFY_ADD_DEVICE) { if (ops->add_device) return ops->add_device(dev); - } else if (action == BUS_NOTIFY_DEL_DEVICE) { + } else if (action == BUS_NOTIFY_REMOVED_DEVICE) { if (ops->remove_device && dev->iommu_group) { ops->remove_device(dev); return 0; -- cgit v0.10.2 From 53723dc59ff3ab504c739000b287ded49aeb2019 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:29 +0200 Subject: iommu: Allocate a default domain for iommu groups The default domain will be used (if supported by the iommu driver) when the devices in the iommu group are not attached to any other domain. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d69e0ca..49eb9bf 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -51,6 +51,7 @@ struct iommu_group { void (*iommu_data_release)(void *iommu_data); char *name; int id; + struct iommu_domain *default_domain; }; struct iommu_device { @@ -75,6 +76,9 @@ struct iommu_group_attribute iommu_group_attr_##_name = \ #define to_iommu_group(_kobj) \ container_of(_kobj, struct iommu_group, kobj) +static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, + unsigned type); + static ssize_t iommu_group_attr_show(struct kobject *kobj, struct attribute *__attr, char *buf) { @@ -137,6 +141,9 @@ static void iommu_group_release(struct kobject *kobj) ida_remove(&iommu_group_ida, group->id); mutex_unlock(&iommu_group_mutex); + if (group->default_domain) + iommu_domain_free(group->default_domain); + kfree(group->name); kfree(group); } @@ -701,7 +708,17 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev) return group; /* No shared group found, allocate new */ - return iommu_group_alloc(); + group = iommu_group_alloc(); + if (group) { + /* + * Try to allocate a default domain - needs support from the + * IOMMU driver. + */ + group->default_domain = __iommu_domain_alloc(pdev->dev.bus, + IOMMU_DOMAIN_DMA); + } + + return group; } /** @@ -922,22 +939,28 @@ void iommu_set_fault_handler(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_set_fault_handler); -struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, + unsigned type) { struct iommu_domain *domain; if (bus == NULL || bus->iommu_ops == NULL) return NULL; - domain = bus->iommu_ops->domain_alloc(IOMMU_DOMAIN_UNMANAGED); + domain = bus->iommu_ops->domain_alloc(type); if (!domain) return NULL; domain->ops = bus->iommu_ops; - domain->type = IOMMU_DOMAIN_UNMANAGED; + domain->type = type; return domain; } + +struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) +{ + return __iommu_domain_alloc(bus, IOMMU_DOMAIN_UNMANAGED); +} EXPORT_SYMBOL_GPL(iommu_domain_alloc); void iommu_domain_free(struct iommu_domain *domain) -- cgit v0.10.2 From 426a273834eae65abcfc7132a21a85b3151e0bce Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:30 +0200 Subject: iommu: Limit iommu_attach/detach_device to devices with their own group This patch changes the behavior of the iommu_attach_device and iommu_detach_device functions. With this change these functions only work on devices that have their own group. For all other devices the iommu_group_attach/detach functions must be used. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 49eb9bf..ef73923 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -433,6 +433,17 @@ void iommu_group_remove_device(struct device *dev) } EXPORT_SYMBOL_GPL(iommu_group_remove_device); +static int iommu_group_device_count(struct iommu_group *group) +{ + struct iommu_device *entry; + int ret = 0; + + list_for_each_entry(entry, &group->devices, list) + ret++; + + return ret; +} + /** * iommu_group_for_each_dev - iterate over each device in the group * @group: the group @@ -969,7 +980,8 @@ void iommu_domain_free(struct iommu_domain *domain) } EXPORT_SYMBOL_GPL(iommu_domain_free); -int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +static int __iommu_attach_device(struct iommu_domain *domain, + struct device *dev) { int ret; if (unlikely(domain->ops->attach_dev == NULL)) @@ -980,9 +992,38 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev) trace_attach_device_to_domain(dev); return ret; } + +int iommu_attach_device(struct iommu_domain *domain, struct device *dev) +{ + struct iommu_group *group; + int ret; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return __iommu_attach_device(domain, dev); + + /* + * We have a group - lock it to make sure the device-count doesn't + * change while we are attaching + */ + mutex_lock(&group->mutex); + ret = -EINVAL; + if (iommu_group_device_count(group) != 1) + goto out_unlock; + + ret = __iommu_attach_device(domain, dev); + +out_unlock: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; +} EXPORT_SYMBOL_GPL(iommu_attach_device); -void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +static void __iommu_detach_device(struct iommu_domain *domain, + struct device *dev) { if (unlikely(domain->ops->detach_dev == NULL)) return; @@ -990,6 +1031,28 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev) domain->ops->detach_dev(domain, dev); trace_detach_device_from_domain(dev); } + +void iommu_detach_device(struct iommu_domain *domain, struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return __iommu_detach_device(domain, dev); + + mutex_lock(&group->mutex); + if (iommu_group_device_count(group) != 1) { + WARN_ON(1); + goto out_unlock; + } + + __iommu_detach_device(domain, dev); + +out_unlock: + mutex_unlock(&group->mutex); + iommu_group_put(group); +} EXPORT_SYMBOL_GPL(iommu_detach_device); /* @@ -1006,7 +1069,7 @@ static int iommu_group_do_attach_device(struct device *dev, void *data) { struct iommu_domain *domain = data; - return iommu_attach_device(domain, dev); + return __iommu_attach_device(domain, dev); } int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) @@ -1020,7 +1083,7 @@ static int iommu_group_do_detach_device(struct device *dev, void *data) { struct iommu_domain *domain = data; - iommu_detach_device(domain, dev); + __iommu_detach_device(domain, dev); return 0; } -- cgit v0.10.2 From e39cb8a3aa988a74433a3f26443b454cca033651 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:31 +0200 Subject: iommu: Make sure a device is always attached to a domain Make use of the default domain and re-attach a device to it when it is detached from another domain. Also enforce that a device has to be in the default domain before it can be attached to a different domain. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index ef73923..7bce522 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -52,6 +52,7 @@ struct iommu_group { char *name; int id; struct iommu_domain *default_domain; + struct iommu_domain *domain; }; struct iommu_device { @@ -78,6 +79,12 @@ struct iommu_group_attribute iommu_group_attr_##_name = \ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, unsigned type); +static int __iommu_attach_device(struct iommu_domain *domain, + struct device *dev); +static int __iommu_attach_group(struct iommu_domain *domain, + struct iommu_group *group); +static void __iommu_detach_group(struct iommu_domain *domain, + struct iommu_group *group); static ssize_t iommu_group_attr_show(struct kobject *kobj, struct attribute *__attr, char *buf) @@ -376,6 +383,8 @@ rename: mutex_lock(&group->mutex); list_add_tail(&device->list, &group->devices); + if (group->domain) + __iommu_attach_device(group->domain, dev); mutex_unlock(&group->mutex); /* Notify any listeners about change to group. */ @@ -455,19 +464,30 @@ static int iommu_group_device_count(struct iommu_group *group) * The group->mutex is held across callbacks, which will block calls to * iommu_group_add/remove_device. */ -int iommu_group_for_each_dev(struct iommu_group *group, void *data, - int (*fn)(struct device *, void *)) +static int __iommu_group_for_each_dev(struct iommu_group *group, void *data, + int (*fn)(struct device *, void *)) { struct iommu_device *device; int ret = 0; - mutex_lock(&group->mutex); list_for_each_entry(device, &group->devices, list) { ret = fn(device->dev, data); if (ret) break; } + return ret; +} + + +int iommu_group_for_each_dev(struct iommu_group *group, void *data, + int (*fn)(struct device *, void *)) +{ + int ret; + + mutex_lock(&group->mutex); + ret = __iommu_group_for_each_dev(group, data, fn); mutex_unlock(&group->mutex); + return ret; } EXPORT_SYMBOL_GPL(iommu_group_for_each_dev); @@ -727,6 +747,7 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev) */ group->default_domain = __iommu_domain_alloc(pdev->dev.bus, IOMMU_DOMAIN_DMA); + group->domain = group->default_domain; } return group; @@ -1012,7 +1033,7 @@ int iommu_attach_device(struct iommu_domain *domain, struct device *dev) if (iommu_group_device_count(group) != 1) goto out_unlock; - ret = __iommu_attach_device(domain, dev); + ret = __iommu_attach_group(domain, group); out_unlock: mutex_unlock(&group->mutex); @@ -1047,7 +1068,7 @@ void iommu_detach_device(struct iommu_domain *domain, struct device *dev) goto out_unlock; } - __iommu_detach_device(domain, dev); + __iommu_detach_group(domain, group); out_unlock: mutex_unlock(&group->mutex); @@ -1072,10 +1093,31 @@ static int iommu_group_do_attach_device(struct device *dev, void *data) return __iommu_attach_device(domain, dev); } +static int __iommu_attach_group(struct iommu_domain *domain, + struct iommu_group *group) +{ + int ret; + + if (group->default_domain && group->domain != group->default_domain) + return -EBUSY; + + ret = __iommu_group_for_each_dev(group, domain, + iommu_group_do_attach_device); + if (ret == 0) + group->domain = domain; + + return ret; +} + int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) { - return iommu_group_for_each_dev(group, domain, - iommu_group_do_attach_device); + int ret; + + mutex_lock(&group->mutex); + ret = __iommu_attach_group(domain, group); + mutex_unlock(&group->mutex); + + return ret; } EXPORT_SYMBOL_GPL(iommu_attach_group); @@ -1088,9 +1130,35 @@ static int iommu_group_do_detach_device(struct device *dev, void *data) return 0; } +static void __iommu_detach_group(struct iommu_domain *domain, + struct iommu_group *group) +{ + int ret; + + if (!group->default_domain) { + __iommu_group_for_each_dev(group, domain, + iommu_group_do_detach_device); + group->domain = NULL; + return; + } + + if (group->domain == group->default_domain) + return; + + /* Detach by re-attaching to the default domain */ + ret = __iommu_group_for_each_dev(group, group->default_domain, + iommu_group_do_attach_device); + if (ret != 0) + WARN_ON(1); + else + group->domain = group->default_domain; +} + void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group) { - iommu_group_for_each_dev(group, domain, iommu_group_do_detach_device); + mutex_lock(&group->mutex); + __iommu_detach_group(domain, group); + mutex_unlock(&group->mutex); } EXPORT_SYMBOL_GPL(iommu_detach_group); -- cgit v0.10.2 From 2c1296d92ac0367364bcb73a43c12a0bdfbfee75 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:32 +0200 Subject: iommu: Add iommu_get_domain_for_dev function This function can be used to request the current domain a device is attached to. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 7bce522..a0a38bd 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1076,6 +1076,24 @@ out_unlock: } EXPORT_SYMBOL_GPL(iommu_detach_device); +struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) +{ + struct iommu_domain *domain; + struct iommu_group *group; + + group = iommu_group_get(dev); + /* FIXME: Remove this when groups a mandatory for iommu drivers */ + if (group == NULL) + return NULL; + + domain = group->domain; + + iommu_group_put(group); + + return domain; +} +EXPORT_SYMBOL_GPL(iommu_get_domain_for_dev); + /* * IOMMU groups are really the natrual working unit of the IOMMU, but * the IOMMU API works on domains and devices. Bridge that gap by diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0546b87..683a1c4 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -193,6 +193,7 @@ extern int iommu_attach_device(struct iommu_domain *domain, struct device *dev); extern void iommu_detach_device(struct iommu_domain *domain, struct device *dev); +extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); extern int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot); extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, @@ -332,6 +333,11 @@ static inline void iommu_detach_device(struct iommu_domain *domain, { } +static inline struct iommu_domain *iommu_get_domain_for_dev(struct device *dev) +{ + return NULL; +} + static inline int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, int gfp_order, int prot) { -- cgit v0.10.2 From a1015c2b99b94cf521603b41debf167114031456 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:33 +0200 Subject: iommu: Introduce direct mapped region handling Add two new functions to the IOMMU-API to allow the IOMMU drivers to export the requirements for direct mapped regions per device. This is useful for exporting the information in Intel VT-d's RMRR entries or AMD-Vi's unity mappings. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index a0a38bd..6b8d6e7 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1469,3 +1469,19 @@ int iommu_domain_set_attr(struct iommu_domain *domain, return ret; } EXPORT_SYMBOL_GPL(iommu_domain_set_attr); + +void iommu_get_dm_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->get_dm_regions) + ops->get_dm_regions(dev, list); +} + +void iommu_put_dm_regions(struct device *dev, struct list_head *list) +{ + const struct iommu_ops *ops = dev->bus->iommu_ops; + + if (ops && ops->put_dm_regions) + ops->put_dm_regions(dev, list); +} diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 683a1c4..6894999 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -114,6 +114,20 @@ enum iommu_attr { DOMAIN_ATTR_MAX, }; +/** + * struct iommu_dm_region - descriptor for a direct mapped memory region + * @list: Linked list pointers + * @start: System physical start address of the region + * @length: Length of the region in bytes + * @prot: IOMMU Protection flags (READ/WRITE/...) + */ +struct iommu_dm_region { + struct list_head list; + phys_addr_t start; + size_t length; + int prot; +}; + #ifdef CONFIG_IOMMU_API /** @@ -159,6 +173,10 @@ struct iommu_ops { int (*domain_set_attr)(struct iommu_domain *domain, enum iommu_attr attr, void *data); + /* Request/Free a list of direct mapping requirements for a device */ + void (*get_dm_regions)(struct device *dev, struct list_head *list); + void (*put_dm_regions)(struct device *dev, struct list_head *list); + /* Window handling functions */ int (*domain_window_enable)(struct iommu_domain *domain, u32 wnd_nr, phys_addr_t paddr, u64 size, int prot); @@ -205,6 +223,9 @@ extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t io extern void iommu_set_fault_handler(struct iommu_domain *domain, iommu_fault_handler_t handler, void *token); +extern void iommu_get_dm_regions(struct device *dev, struct list_head *list); +extern void iommu_put_dm_regions(struct device *dev, struct list_head *list); + extern int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group); extern void iommu_detach_group(struct iommu_domain *domain, @@ -379,6 +400,16 @@ static inline void iommu_set_fault_handler(struct iommu_domain *domain, { } +static inline void iommu_get_dm_regions(struct device *dev, + struct list_head *list) +{ +} + +static inline void iommu_put_dm_regions(struct device *dev, + struct list_head *list) +{ +} + static inline int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) { -- cgit v0.10.2 From beed2821b4f42c268222c4c1f1795e53340acf64 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:34 +0200 Subject: iommu: Create direct mappings in default domains Use the information exported by the IOMMU drivers to create direct mapped regions in the default domains. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 6b8d6e7..ffad1ea 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -325,6 +325,52 @@ int iommu_group_set_name(struct iommu_group *group, const char *name) } EXPORT_SYMBOL_GPL(iommu_group_set_name); +static int iommu_group_create_direct_mappings(struct iommu_group *group, + struct device *dev) +{ + struct iommu_domain *domain = group->default_domain; + struct iommu_dm_region *entry; + struct list_head mappings; + unsigned long pg_size; + int ret = 0; + + if (!domain || domain->type != IOMMU_DOMAIN_DMA) + return 0; + + BUG_ON(!domain->ops->pgsize_bitmap); + + pg_size = 1UL << __ffs(domain->ops->pgsize_bitmap); + INIT_LIST_HEAD(&mappings); + + iommu_get_dm_regions(dev, &mappings); + + /* We need to consider overlapping regions for different devices */ + list_for_each_entry(entry, &mappings, list) { + dma_addr_t start, end, addr; + + start = ALIGN(entry->start, pg_size); + end = ALIGN(entry->start + entry->length, pg_size); + + for (addr = start; addr < end; addr += pg_size) { + phys_addr_t phys_addr; + + phys_addr = iommu_iova_to_phys(domain, addr); + if (phys_addr) + continue; + + ret = iommu_map(domain, addr, addr, pg_size, entry->prot); + if (ret) + goto out; + } + + } + +out: + iommu_put_dm_regions(dev, &mappings); + + return ret; +} + /** * iommu_group_add_device - add a device to an iommu group * @group: the group into which to add the device (reference should be held) @@ -381,6 +427,8 @@ rename: dev->iommu_group = group; + iommu_group_create_direct_mappings(group, dev); + mutex_lock(&group->mutex); list_add_tail(&device->list, &group->devices); if (group->domain) -- cgit v0.10.2 From 6827ca83695d5e41ad31b0719788ee65f00ca4b3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:35 +0200 Subject: iommu: Add function to query the default domain of a group This will be used to handle unity mappings in the iommu drivers. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index ffad1ea..224c6dd 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -837,6 +837,11 @@ struct iommu_group *iommu_group_get_for_dev(struct device *dev) return group; } +struct iommu_domain *iommu_group_default_domain(struct iommu_group *group) +{ + return group->default_domain; +} + static int add_iommu_group(struct device *dev, void *data) { struct iommu_callback_data *cb = data; diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 6894999..b944b2b 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -249,6 +249,7 @@ extern int iommu_group_unregister_notifier(struct iommu_group *group, struct notifier_block *nb); extern int iommu_group_id(struct iommu_group *group); extern struct iommu_group *iommu_group_get_for_dev(struct device *dev); +extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); extern int iommu_domain_get_attr(struct iommu_domain *domain, enum iommu_attr, void *data); -- cgit v0.10.2 From d290f1e70d85a9a4d124594c6a3d769329960bdc Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:36 +0200 Subject: iommu: Introduce iommu_request_dm_for_dev() This function can be called by an IOMMU driver to request that a device's default domain is direct mapped. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 224c6dd..3b1a255 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1538,3 +1538,56 @@ void iommu_put_dm_regions(struct device *dev, struct list_head *list) if (ops && ops->put_dm_regions) ops->put_dm_regions(dev, list); } + +/* Request that a device is direct mapped by the IOMMU */ +int iommu_request_dm_for_dev(struct device *dev) +{ + struct iommu_domain *dm_domain; + struct iommu_group *group; + int ret; + + /* Device must already be in a group before calling this function */ + group = iommu_group_get_for_dev(dev); + if (!group) + return -EINVAL; + + mutex_lock(&group->mutex); + + /* Check if the default domain is already direct mapped */ + ret = 0; + if (group->default_domain && + group->default_domain->type == IOMMU_DOMAIN_IDENTITY) + goto out; + + /* Don't change mappings of existing devices */ + ret = -EBUSY; + if (iommu_group_device_count(group) != 1) + goto out; + + /* Allocate a direct mapped domain */ + ret = -ENOMEM; + dm_domain = __iommu_domain_alloc(dev->bus, IOMMU_DOMAIN_IDENTITY); + if (!dm_domain) + goto out; + + /* Attach the device to the domain */ + ret = __iommu_attach_group(dm_domain, group); + if (ret) { + iommu_domain_free(dm_domain); + goto out; + } + + /* Make the direct mapped domain the default for this group */ + if (group->default_domain) + iommu_domain_free(group->default_domain); + group->default_domain = dm_domain; + + pr_info("Using direct mapping for device %s\n", dev_name(dev)); + + ret = 0; +out: + mutex_unlock(&group->mutex); + iommu_group_put(group); + + return ret; +} diff --git a/include/linux/iommu.h b/include/linux/iommu.h index b944b2b..dc767f7 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -225,6 +225,7 @@ extern void iommu_set_fault_handler(struct iommu_domain *domain, extern void iommu_get_dm_regions(struct device *dev, struct list_head *list); extern void iommu_put_dm_regions(struct device *dev, struct list_head *list); +extern int iommu_request_dm_for_dev(struct device *dev); extern int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group); @@ -411,6 +412,11 @@ static inline void iommu_put_dm_regions(struct device *dev, { } +static inline int iommu_request_dm_for_dev(struct device *dev) +{ + return -ENODEV; +} + static inline int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group) { -- cgit v0.10.2 From 35cf248f8860b8d2956516d6cdf7e57ff4ed4cbb Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:37 +0200 Subject: iommu/amd: Implement dm_region call-backs Add the get_dm_regions and put_dm_regions callbacks to the iommu_ops of the AMD IOMMU driver. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index e43d489..22d38e3 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -3412,6 +3412,47 @@ static bool amd_iommu_capable(enum iommu_cap cap) return false; } +static void amd_iommu_get_dm_regions(struct device *dev, + struct list_head *head) +{ + struct unity_map_entry *entry; + u16 devid; + + devid = get_device_id(dev); + + list_for_each_entry(entry, &amd_iommu_unity_map, list) { + struct iommu_dm_region *region; + + if (devid < entry->devid_start || devid > entry->devid_end) + continue; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) { + pr_err("Out of memory allocating dm-regions for %s\n", + dev_name(dev)); + return; + } + + region->start = entry->address_start; + region->length = entry->address_end - entry->address_start; + if (entry->prot & IOMMU_PROT_IR) + region->prot |= IOMMU_READ; + if (entry->prot & IOMMU_PROT_IW) + region->prot |= IOMMU_WRITE; + + list_add_tail(®ion->list, head); + } +} + +static void amd_iommu_put_dm_regions(struct device *dev, + struct list_head *head) +{ + struct iommu_dm_region *entry, *next; + + list_for_each_entry_safe(entry, next, head, list) + kfree(entry); +} + static const struct iommu_ops amd_iommu_ops = { .capable = amd_iommu_capable, .domain_alloc = amd_iommu_domain_alloc, @@ -3422,6 +3463,8 @@ static const struct iommu_ops amd_iommu_ops = { .unmap = amd_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = amd_iommu_iova_to_phys, + .get_dm_regions = amd_iommu_get_dm_regions, + .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, }; -- cgit v0.10.2 From 063071dff53858027e95d3cfcedb1780952302ad Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:38 +0200 Subject: iommu/amd: Use default domain if available for DMA-API Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 22d38e3..ae7d636 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2495,12 +2495,19 @@ void amd_iommu_init_notifier(void) static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; + struct iommu_domain *io_domain; struct dma_ops_domain *dma_dom; u16 devid = get_device_id(dev); if (!check_device(dev)) return ERR_PTR(-EINVAL); + io_domain = iommu_get_domain_for_dev(dev); + if (io_domain) { + domain = to_pdomain(io_domain); + return domain; + } + domain = domain_for_device(dev); if (domain != NULL && !dma_ops_domain(domain)) return ERR_PTR(-EBUSY); -- cgit v0.10.2 From aafd8ba0ca74894b9397e412bbd7f8ea2662ead8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:39 +0200 Subject: iommu/amd: Implement add_device and remove_device Implement these two iommu-ops call-backs to make use of the initialization and notifier features of the iommu core. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index ae7d636..0f87769 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -119,7 +119,7 @@ struct iommu_cmd { struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); -static int __init alloc_passthrough_domain(void); +static int alloc_passthrough_domain(void); /**************************************************************************** * @@ -434,64 +434,15 @@ static void iommu_uninit_device(struct device *dev) /* Unlink from alias, it may change if another device is re-plugged */ dev_data->alias_data = NULL; + /* Remove dma-ops */ + dev->archdata.dma_ops = NULL; + /* * We keep dev_data around for unplugged devices and reuse it when the * device is re-plugged - not doing so would introduce a ton of races. */ } -void __init amd_iommu_uninit_devices(void) -{ - struct iommu_dev_data *dev_data, *n; - struct pci_dev *pdev = NULL; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - iommu_uninit_device(&pdev->dev); - } - - /* Free all of our dev_data structures */ - list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list) - free_dev_data(dev_data); -} - -int __init amd_iommu_init_devices(void) -{ - struct pci_dev *pdev = NULL; - int ret = 0; - - for_each_pci_dev(pdev) { - - if (!check_device(&pdev->dev)) - continue; - - ret = iommu_init_device(&pdev->dev); - if (ret == -ENOTSUPP) - iommu_ignore_device(&pdev->dev); - else if (ret) - goto out_free; - } - - /* - * Initialize IOMMU groups only after iommu_init_device() has - * had a chance to populate any IVRS defined aliases. - */ - for_each_pci_dev(pdev) { - if (check_device(&pdev->dev)) - init_iommu_group(&pdev->dev); - } - - return 0; - -out_free: - - amd_iommu_uninit_devices(); - - return ret; -} #ifdef CONFIG_AMD_IOMMU_STATS /* @@ -2402,81 +2353,79 @@ static struct protection_domain *domain_for_device(struct device *dev) return dom; } -static int device_change_notifier(struct notifier_block *nb, - unsigned long action, void *data) +static int amd_iommu_add_device(struct device *dev) { struct dma_ops_domain *dma_domain; struct protection_domain *domain; struct iommu_dev_data *dev_data; - struct device *dev = data; struct amd_iommu *iommu; unsigned long flags; u16 devid; + int ret; - if (!check_device(dev)) + if (!check_device(dev) || get_dev_data(dev)) return 0; - devid = get_device_id(dev); - iommu = amd_iommu_rlookup_table[devid]; - dev_data = get_dev_data(dev); - - switch (action) { - case BUS_NOTIFY_ADD_DEVICE: - - iommu_init_device(dev); - init_iommu_group(dev); + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; - /* - * dev_data is still NULL and - * got initialized in iommu_init_device - */ - dev_data = get_dev_data(dev); + ret = iommu_init_device(dev); + if (ret == -ENOTSUPP) { + iommu_ignore_device(dev); + goto out; + } + init_iommu_group(dev); - if (iommu_pass_through || dev_data->iommu_v2) { - dev_data->passthrough = true; - attach_device(dev, pt_domain); - break; - } + dev_data = get_dev_data(dev); - domain = domain_for_device(dev); + if (iommu_pass_through || dev_data->iommu_v2) { + /* Make sure passthrough domain is allocated */ + alloc_passthrough_domain(); + dev_data->passthrough = true; + attach_device(dev, pt_domain); + goto out; + } - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; + domain = domain_for_device(dev); - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } + /* allocate a protection domain if a device is added */ + dma_domain = find_protection_domain(devid); + if (!dma_domain) { + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) + goto out; + dma_domain->target_dev = devid; - dev->archdata.dma_ops = &amd_iommu_dma_ops; + init_unity_mappings_for_device(dma_domain, devid); - break; - case BUS_NOTIFY_REMOVED_DEVICE: + spin_lock_irqsave(&iommu_pd_list_lock, flags); + list_add_tail(&dma_domain->list, &iommu_pd_list); + spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + } - iommu_uninit_device(dev); + attach_device(dev, &dma_domain->domain); - default: - goto out; - } + dev->archdata.dma_ops = &amd_iommu_dma_ops; +out: iommu_completion_wait(iommu); -out: return 0; } -static struct notifier_block device_nb = { - .notifier_call = device_change_notifier, -}; - -void amd_iommu_init_notifier(void) +static void amd_iommu_remove_device(struct device *dev) { - bus_register_notifier(&pci_bus_type, &device_nb); + struct amd_iommu *iommu; + u16 devid; + + if (!check_device(dev)) + return; + + devid = get_device_id(dev); + iommu = amd_iommu_rlookup_table[devid]; + + iommu_uninit_device(dev); + iommu_completion_wait(iommu); } /***************************************************************************** @@ -3018,54 +2967,6 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask) return check_device(dev); } -/* - * The function for pre-allocating protection domains. - * - * If the driver core informs the DMA layer if a driver grabs a device - * we don't need to preallocate the protection domains anymore. - * For now we have to. - */ -static void __init prealloc_protection_domains(void) -{ - struct iommu_dev_data *dev_data; - struct dma_ops_domain *dma_dom; - struct pci_dev *dev = NULL; - u16 devid; - - for_each_pci_dev(dev) { - - /* Do we handle this device? */ - if (!check_device(&dev->dev)) - continue; - - dev_data = get_dev_data(&dev->dev); - if (!amd_iommu_force_isolation && dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(&dev->dev, pt_domain); - pr_info("AMD-Vi: Using passthrough domain for device %s\n", - dev_name(&dev->dev)); - } - - /* Is there already any domain for it? */ - if (domain_for_device(&dev->dev)) - continue; - - devid = get_device_id(&dev->dev); - - dma_dom = dma_ops_domain_alloc(); - if (!dma_dom) - continue; - init_unity_mappings_for_device(dma_dom, devid); - dma_dom->target_dev = devid; - - attach_device(&dev->dev, &dma_dom->domain); - - list_add_tail(&dma_dom->list, &iommu_pd_list); - } -} - static struct dma_map_ops amd_iommu_dma_ops = { .alloc = alloc_coherent, .free = free_coherent, @@ -3131,11 +3032,6 @@ int __init amd_iommu_init_dma_ops(void) goto free_domains; } - /* - * Pre-allocate the protection domains for each device. - */ - prealloc_protection_domains(); - iommu_detected = 1; swiotlb = 0; @@ -3228,7 +3124,7 @@ out_err: return NULL; } -static int __init alloc_passthrough_domain(void) +static int alloc_passthrough_domain(void) { if (pt_domain != NULL) return 0; @@ -3470,6 +3366,8 @@ static const struct iommu_ops amd_iommu_ops = { .unmap = amd_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = amd_iommu_iova_to_phys, + .add_device = amd_iommu_add_device, + .remove_device = amd_iommu_remove_device, .get_dm_regions = amd_iommu_get_dm_regions, .put_dm_regions = amd_iommu_put_dm_regions, .pgsize_bitmap = AMD_IOMMU_PGSIZES, diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 450ef50..e4a6e40 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -226,6 +226,7 @@ static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); +static void init_device_table_dma(void); static inline void update_last_devid(u16 devid) { @@ -1385,7 +1386,12 @@ static int __init amd_iommu_init_pci(void) break; } - ret = amd_iommu_init_devices(); + init_device_table_dma(); + + for_each_iommu(iommu) + iommu_flush_all_caches(iommu); + + amd_iommu_init_api(); print_iommu_info(); @@ -1825,8 +1831,6 @@ static bool __init check_ioapic_information(void) static void __init free_dma_resources(void) { - amd_iommu_uninit_devices(); - free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, get_order(MAX_DOMAIN_ID/8)); @@ -2019,27 +2023,10 @@ static bool detect_ivrs(void) static int amd_iommu_init_dma(void) { - struct amd_iommu *iommu; - int ret; - if (iommu_pass_through) - ret = amd_iommu_init_passthrough(); + return amd_iommu_init_passthrough(); else - ret = amd_iommu_init_dma_ops(); - - if (ret) - return ret; - - init_device_table_dma(); - - for_each_iommu(iommu) - iommu_flush_all_caches(iommu); - - amd_iommu_init_api(); - - amd_iommu_init_notifier(); - - return 0; + return amd_iommu_init_dma_ops(); } /**************************************************************************** -- cgit v0.10.2 From 0bb6e243d7fbb39fced5bd4a4c83eb49c6e820ce Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:40 +0200 Subject: iommu/amd: Support IOMMU_DOMAIN_DMA type allocation This enables allocation of DMA-API default domains from the IOMMU core and switches allocation of domain dma-api domain to the IOMMU core too. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 0f87769..27300ae 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -64,10 +64,6 @@ static DEFINE_RWLOCK(amd_iommu_devtable_lock); -/* A list of preallocated protection domains */ -static LIST_HEAD(iommu_pd_list); -static DEFINE_SPINLOCK(iommu_pd_list_lock); - /* List of all available dev_data structures */ static LIST_HEAD(dev_data_list); static DEFINE_SPINLOCK(dev_data_list_lock); @@ -234,31 +230,38 @@ static bool pdev_pri_erratum(struct pci_dev *pdev, u32 erratum) } /* - * In this function the list of preallocated protection domains is traversed to - * find the domain for a specific device + * This function actually applies the mapping to the page table of the + * dma_ops domain. */ -static struct dma_ops_domain *find_protection_domain(u16 devid) +static void alloc_unity_mapping(struct dma_ops_domain *dma_dom, + struct unity_map_entry *e) { - struct dma_ops_domain *entry, *ret = NULL; - unsigned long flags; - u16 alias = amd_iommu_alias_table[devid]; - - if (list_empty(&iommu_pd_list)) - return NULL; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); + u64 addr; - list_for_each_entry(entry, &iommu_pd_list, list) { - if (entry->target_dev == devid || - entry->target_dev == alias) { - ret = entry; - break; - } + for (addr = e->address_start; addr < e->address_end; + addr += PAGE_SIZE) { + if (addr < dma_dom->aperture_size) + __set_bit(addr >> PAGE_SHIFT, + dma_dom->aperture[0]->bitmap); } +} + +/* + * Inits the unity mappings required for a specific device + */ +static void init_unity_mappings_for_device(struct device *dev, + struct dma_ops_domain *dma_dom) +{ + struct unity_map_entry *e; + u16 devid; - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + devid = get_device_id(dev); - return ret; + list_for_each_entry(e, &amd_iommu_unity_map, list) { + if (!(devid >= e->devid_start && devid <= e->devid_end)) + continue; + alloc_unity_mapping(dma_dom, e); + } } /* @@ -290,11 +293,23 @@ static bool check_device(struct device *dev) static void init_iommu_group(struct device *dev) { + struct dma_ops_domain *dma_domain; + struct iommu_domain *domain; struct iommu_group *group; group = iommu_group_get_for_dev(dev); - if (!IS_ERR(group)) - iommu_group_put(group); + if (IS_ERR(group)) + return; + + domain = iommu_group_default_domain(group); + if (!domain) + goto out; + + dma_domain = to_pdomain(domain)->priv; + + init_unity_mappings_for_device(dev, dma_domain); +out: + iommu_group_put(group); } static int __last_alias(struct pci_dev *pdev, u16 alias, void *data) @@ -1414,94 +1429,6 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, return unmapped; } -/* - * This function checks if a specific unity mapping entry is needed for - * this specific IOMMU. - */ -static int iommu_for_unity_map(struct amd_iommu *iommu, - struct unity_map_entry *entry) -{ - u16 bdf, i; - - for (i = entry->devid_start; i <= entry->devid_end; ++i) { - bdf = amd_iommu_alias_table[i]; - if (amd_iommu_rlookup_table[bdf] == iommu) - return 1; - } - - return 0; -} - -/* - * This function actually applies the mapping to the page table of the - * dma_ops domain. - */ -static int dma_ops_unity_map(struct dma_ops_domain *dma_dom, - struct unity_map_entry *e) -{ - u64 addr; - int ret; - - for (addr = e->address_start; addr < e->address_end; - addr += PAGE_SIZE) { - ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot, - PAGE_SIZE); - if (ret) - return ret; - /* - * if unity mapping is in aperture range mark the page - * as allocated in the aperture - */ - if (addr < dma_dom->aperture_size) - __set_bit(addr >> PAGE_SHIFT, - dma_dom->aperture[0]->bitmap); - } - - return 0; -} - -/* - * Init the unity mappings for a specific IOMMU in the system - * - * Basically iterates over all unity mapping entries and applies them to - * the default domain DMA of that IOMMU if necessary. - */ -static int iommu_init_unity_mappings(struct amd_iommu *iommu) -{ - struct unity_map_entry *entry; - int ret; - - list_for_each_entry(entry, &amd_iommu_unity_map, list) { - if (!iommu_for_unity_map(iommu, entry)) - continue; - ret = dma_ops_unity_map(iommu->default_dom, entry); - if (ret) - return ret; - } - - return 0; -} - -/* - * Inits the unity mappings required for a specific device - */ -static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom, - u16 devid) -{ - struct unity_map_entry *e; - int ret; - - list_for_each_entry(e, &amd_iommu_unity_map, list) { - if (!(devid >= e->devid_start && devid <= e->devid_end)) - continue; - ret = dma_ops_unity_map(dma_dom, e); - if (ret) - return ret; - } - - return 0; -} - /**************************************************************************** * * The next functions belong to the address allocator for the dma_ops @@ -2324,42 +2251,9 @@ static void detach_device(struct device *dev) dev_data->ats.enabled = false; } -/* - * Find out the protection domain structure for a given PCI device. This - * will give us the pointer to the page table root for example. - */ -static struct protection_domain *domain_for_device(struct device *dev) -{ - struct iommu_dev_data *dev_data; - struct protection_domain *dom = NULL; - unsigned long flags; - - dev_data = get_dev_data(dev); - - if (dev_data->domain) - return dev_data->domain; - - if (dev_data->alias_data != NULL) { - struct iommu_dev_data *alias_data = dev_data->alias_data; - - read_lock_irqsave(&amd_iommu_devtable_lock, flags); - if (alias_data->domain != NULL) { - __attach_device(dev_data, alias_data->domain); - dom = alias_data->domain; - } - read_unlock_irqrestore(&amd_iommu_devtable_lock, flags); - } - - return dom; -} - static int amd_iommu_add_device(struct device *dev) { - struct dma_ops_domain *dma_domain; - struct protection_domain *domain; - struct iommu_dev_data *dev_data; struct amd_iommu *iommu; - unsigned long flags; u16 devid; int ret; @@ -2376,35 +2270,6 @@ static int amd_iommu_add_device(struct device *dev) } init_iommu_group(dev); - dev_data = get_dev_data(dev); - - if (iommu_pass_through || dev_data->iommu_v2) { - /* Make sure passthrough domain is allocated */ - alloc_passthrough_domain(); - dev_data->passthrough = true; - attach_device(dev, pt_domain); - goto out; - } - - domain = domain_for_device(dev); - - /* allocate a protection domain if a device is added */ - dma_domain = find_protection_domain(devid); - if (!dma_domain) { - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - init_unity_mappings_for_device(dma_domain, devid); - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - } - - attach_device(dev, &dma_domain->domain); - dev->archdata.dma_ops = &amd_iommu_dma_ops; out: @@ -2445,34 +2310,19 @@ static struct protection_domain *get_domain(struct device *dev) { struct protection_domain *domain; struct iommu_domain *io_domain; - struct dma_ops_domain *dma_dom; - u16 devid = get_device_id(dev); if (!check_device(dev)) return ERR_PTR(-EINVAL); io_domain = iommu_get_domain_for_dev(dev); - if (io_domain) { - domain = to_pdomain(io_domain); - return domain; - } + if (!io_domain) + return NULL; - domain = domain_for_device(dev); - if (domain != NULL && !dma_ops_domain(domain)) + domain = to_pdomain(io_domain); + if (!dma_ops_domain(domain)) return ERR_PTR(-EBUSY); - if (domain != NULL) - return domain; - - /* Device not bound yet - bind it */ - dma_dom = find_protection_domain(devid); - if (!dma_dom) - dma_dom = amd_iommu_rlookup_table[devid]->default_dom; - attach_device(dev, &dma_dom->domain); - DUMP_printk("Using protection domain %d for device %s\n", - dma_dom->domain.id, dev_name(dev)); - - return &dma_dom->domain; + return domain; } static void update_device_table(struct protection_domain *domain) @@ -3014,23 +2864,7 @@ void __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { - struct amd_iommu *iommu; - int ret, unhandled; - - /* - * first allocate a default protection domain for every IOMMU we - * found in the system. Devices not assigned to any other - * protection domain will be assigned to the default one. - */ - for_each_iommu(iommu) { - iommu->default_dom = dma_ops_domain_alloc(); - if (iommu->default_dom == NULL) - return -ENOMEM; - iommu->default_dom->domain.flags |= PD_DEFAULT_MASK; - ret = iommu_init_unity_mappings(iommu); - if (ret) - goto free_domains; - } + int unhandled; iommu_detected = 1; swiotlb = 0; @@ -3050,14 +2884,6 @@ int __init amd_iommu_init_dma_ops(void) pr_info("AMD-Vi: Lazy IO/TLB flushing enabled\n"); return 0; - -free_domains: - - for_each_iommu(iommu) { - dma_ops_domain_free(iommu->default_dom); - } - - return ret; } /***************************************************************************** @@ -3142,30 +2968,39 @@ static int alloc_passthrough_domain(void) static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) { struct protection_domain *pdomain; + struct dma_ops_domain *dma_domain; - /* We only support unmanaged domains for now */ - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; + switch (type) { + case IOMMU_DOMAIN_UNMANAGED: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; - pdomain = protection_domain_alloc(); - if (!pdomain) - goto out_free; + pdomain->mode = PAGE_MODE_3_LEVEL; + pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); + if (!pdomain->pt_root) { + protection_domain_free(pdomain); + return NULL; + } - pdomain->mode = PAGE_MODE_3_LEVEL; - pdomain->pt_root = (void *)get_zeroed_page(GFP_KERNEL); - if (!pdomain->pt_root) - goto out_free; + pdomain->domain.geometry.aperture_start = 0; + pdomain->domain.geometry.aperture_end = ~0ULL; + pdomain->domain.geometry.force_aperture = true; - pdomain->domain.geometry.aperture_start = 0; - pdomain->domain.geometry.aperture_end = ~0ULL; - pdomain->domain.geometry.force_aperture = true; + break; + case IOMMU_DOMAIN_DMA: + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) { + pr_err("AMD-Vi: Failed to allocate\n"); + return NULL; + } + pdomain = &dma_domain->domain; + break; + default: + return NULL; + } return &pdomain->domain; - -out_free: - protection_domain_free(pdomain); - - return NULL; } static void amd_iommu_domain_free(struct iommu_domain *dom) diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index 05030e5..fe796cf 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -552,9 +552,6 @@ struct amd_iommu { /* if one, we need to send a completion wait command */ bool need_sync; - /* default dma_ops domain for that IOMMU */ - struct dma_ops_domain *default_dom; - /* IOMMU sysfs device */ struct device *iommu_dev; -- cgit v0.10.2 From 07f643a35d6b50f0f091444f07db1353188e787e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:41 +0200 Subject: iommu/amd: Support IOMMU_DOMAIN_IDENTITY type allocation Add support to allocate direct mapped domains through the IOMMU-API. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 27300ae..188b81d 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2996,6 +2996,13 @@ static struct iommu_domain *amd_iommu_domain_alloc(unsigned type) } pdomain = &dma_domain->domain; break; + case IOMMU_DOMAIN_IDENTITY: + pdomain = protection_domain_alloc(); + if (!pdomain) + return NULL; + + pdomain->mode = PAGE_MODE_NONE; + break; default: return NULL; } -- cgit v0.10.2 From 07ee86948c9111d49583be462500042fedfecb4a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:42 +0200 Subject: iommu/amd: Put IOMMUv2 devices in a direct mapped domain A device that might be used for HSA needs to be in a direct mapped domain so that all DMA-API mappings stay alive when the IOMMUv2 stack is used. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 188b81d..623be17 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2253,6 +2253,8 @@ static void detach_device(struct device *dev) static int amd_iommu_add_device(struct device *dev) { + struct iommu_dev_data *dev_data; + struct iommu_domain *domain; struct amd_iommu *iommu; u16 devid; int ret; @@ -2270,7 +2272,18 @@ static int amd_iommu_add_device(struct device *dev) } init_iommu_group(dev); - dev->archdata.dma_ops = &amd_iommu_dma_ops; + dev_data = get_dev_data(dev); + if (dev_data && dev_data->iommu_v2) + iommu_request_dm_for_dev(dev); + + /* Domains are initialized for this device - have a look what we ended up with */ + domain = iommu_get_domain_for_dev(dev); + if (domain->type == IOMMU_DOMAIN_IDENTITY) { + dev_data->passthrough = true; + dev->archdata.dma_ops = &nommu_dma_ops; + } else { + dev->archdata.dma_ops = &amd_iommu_dma_ops; + } out: iommu_completion_wait(iommu); -- cgit v0.10.2 From 343e9cac9c4a2715e8fa6b89b04e31e9fcbae5e3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:43 +0200 Subject: iommu/amd: Get rid of device_dma_ops_init() With device intialization done in the add_device call-back now there is no reason for this function anymore. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 623be17..fadda2c 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2268,6 +2268,7 @@ static int amd_iommu_add_device(struct device *dev) ret = iommu_init_device(dev); if (ret == -ENOTSUPP) { iommu_ignore_device(dev); + dev->archdata.dma_ops = &nommu_dma_ops; goto out; } init_iommu_group(dev); @@ -2840,36 +2841,6 @@ static struct dma_map_ops amd_iommu_dma_ops = { .dma_supported = amd_iommu_dma_supported, }; -static unsigned device_dma_ops_init(void) -{ - struct iommu_dev_data *dev_data; - struct pci_dev *pdev = NULL; - unsigned unhandled = 0; - - for_each_pci_dev(pdev) { - if (!check_device(&pdev->dev)) { - - iommu_ignore_device(&pdev->dev); - - unhandled += 1; - continue; - } - - dev_data = get_dev_data(&pdev->dev); - - if (!dev_data->passthrough) - pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops; - else - pdev->dev.archdata.dma_ops = &nommu_dma_ops; - } - - return unhandled; -} - -/* - * The function which clues the AMD IOMMU driver into dma_ops. - */ - void __init amd_iommu_init_api(void) { bus_set_iommu(&pci_bus_type, &amd_iommu_ops); @@ -2877,18 +2848,9 @@ void __init amd_iommu_init_api(void) int __init amd_iommu_init_dma_ops(void) { - int unhandled; - iommu_detected = 1; swiotlb = 0; - /* Make the driver finally visible to the drivers */ - unhandled = device_dma_ops_init(); - if (unhandled && max_pfn > MAX_DMA32_PFN) { - /* There are unhandled devices - initialize swiotlb for them */ - swiotlb = 1; - } - amd_iommu_stats_init(); if (amd_iommu_unmap_flush) -- cgit v0.10.2 From 2870b0a4911038fd6aed9093cda2dbe80fd0ee2e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:44 +0200 Subject: iommu/amd: Remove unused fields from struct dma_ops_domain The list_head and target_dev members are not used anymore. Remove them. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index fadda2c..96390b9 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1886,7 +1886,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) goto free_dma_dom; dma_dom->need_flush = false; - dma_dom->target_dev = 0xffff; add_domain_to_list(&dma_dom->domain); diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index fe796cf..bb56560 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -446,8 +446,6 @@ struct aperture_range { * Data container for a dma_ops specific protection domain */ struct dma_ops_domain { - struct list_head list; - /* generic protection domain information */ struct protection_domain domain; @@ -462,12 +460,6 @@ struct dma_ops_domain { /* This will be set to true when TLB needs to be flushed */ bool need_flush; - - /* - * if this is a preallocated domain, keep the device for which it was - * preallocated in this variable - */ - u16 target_dev; }; /* -- cgit v0.10.2 From 3a18404cd952ae529651f72a13e5d6ffee824c2e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 28 May 2015 18:41:45 +0200 Subject: iommu/amd: Propagate errors from amd_iommu_init_api This function can fail. Propagate any errors back to the initialization state machine. Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 96390b9..8bc6f40 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2840,9 +2840,9 @@ static struct dma_map_ops amd_iommu_dma_ops = { .dma_supported = amd_iommu_dma_supported, }; -void __init amd_iommu_init_api(void) +int __init amd_iommu_init_api(void) { - bus_set_iommu(&pci_bus_type, &amd_iommu_ops); + return bus_set_iommu(&pci_bus_type, &amd_iommu_ops); } int __init amd_iommu_init_dma_ops(void) diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index e4a6e40..dbac49c 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -1391,9 +1391,10 @@ static int __init amd_iommu_init_pci(void) for_each_iommu(iommu) iommu_flush_all_caches(iommu); - amd_iommu_init_api(); + ret = amd_iommu_init_api(); - print_iommu_info(); + if (!ret) + print_iommu_info(); return ret; } diff --git a/drivers/iommu/amd_iommu_proto.h b/drivers/iommu/amd_iommu_proto.h index 72b0fd4..9ed1c43 100644 --- a/drivers/iommu/amd_iommu_proto.h +++ b/drivers/iommu/amd_iommu_proto.h @@ -30,7 +30,7 @@ extern void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu); extern int amd_iommu_init_devices(void); extern void amd_iommu_uninit_devices(void); extern void amd_iommu_init_notifier(void); -extern void amd_iommu_init_api(void); +extern int amd_iommu_init_api(void); /* Needed for interrupt remapping */ extern int amd_iommu_prepare(void); -- cgit v0.10.2 From 409e553deeeb08d644ed1110e0f1c97b71cb6409 Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Wed, 10 Jun 2015 13:59:27 +0300 Subject: iommu: Checking for NULL instead of IS_ERR The iommu_group_alloc() and iommu_group_get_for_dev() functions return error pointers, they never return NULL. Signed-off-by: Dan Carpenter Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 3b1a255..89dc50b 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -788,15 +788,16 @@ static struct iommu_group *iommu_group_get_for_pci_dev(struct pci_dev *pdev) /* No shared group found, allocate new */ group = iommu_group_alloc(); - if (group) { - /* - * Try to allocate a default domain - needs support from the - * IOMMU driver. - */ - group->default_domain = __iommu_domain_alloc(pdev->dev.bus, - IOMMU_DOMAIN_DMA); - group->domain = group->default_domain; - } + if (IS_ERR(group)) + return NULL; + + /* + * Try to allocate a default domain - needs support from the + * IOMMU driver. + */ + group->default_domain = __iommu_domain_alloc(pdev->dev.bus, + IOMMU_DOMAIN_DMA); + group->domain = group->default_domain; return group; } @@ -1548,8 +1549,8 @@ int iommu_request_dm_for_dev(struct device *dev) /* Device must already be in a group before calling this function */ group = iommu_group_get_for_dev(dev); - if (!group) - return -EINVAL; + if (IS_ERR(group)) + return PTR_ERR(group); mutex_lock(&group->mutex); -- cgit v0.10.2 From 4d58b8a6de6b8c3611f10124f83e90e5a2406437 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 11 Jun 2015 09:21:39 +0200 Subject: iommu/amd: Handle errors returned from iommu_init_device Without this patch only -ENOTSUPP is handled, but there are other possible errors. Handle them too. Reported-by: Dan Carpenter Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 8bc6f40..e5b77d3 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -2265,7 +2265,11 @@ static int amd_iommu_add_device(struct device *dev) iommu = amd_iommu_rlookup_table[devid]; ret = iommu_init_device(dev); - if (ret == -ENOTSUPP) { + if (ret) { + if (ret != -ENOTSUPP) + pr_err("Failed to initialize device %s - trying to proceed anyway\n", + dev_name(dev)); + iommu_ignore_device(dev); dev->archdata.dma_ops = &nommu_dma_ops; goto out; @@ -2273,7 +2277,10 @@ static int amd_iommu_add_device(struct device *dev) init_iommu_group(dev); dev_data = get_dev_data(dev); - if (dev_data && dev_data->iommu_v2) + + BUG_ON(!dev_data); + + if (dev_data->iommu_v2) iommu_request_dm_for_dev(dev); /* Domains are initialized for this device - have a look what we ended up with */ -- cgit v0.10.2 From 9f10e5bf62f7b9937afeee8ff0a491f87438e2a2 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 09:57:06 +0200 Subject: iommu/vt-d: Cleanup log messages Give them a common prefix that can be grepped for and improve the wording here and there. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 9847613..c588658 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -26,7 +26,7 @@ * These routines are used by both DMA-remapping and Interrupt-remapping */ -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */ +#define pr_fmt(fmt) "DMAR: " fmt #include #include @@ -555,7 +555,7 @@ static int dmar_walk_remapping_entries(struct acpi_dmar_header *start, break; } else if (next > end) { /* Avoid passing table end */ - pr_warn(FW_BUG "record passes table end\n"); + pr_warn(FW_BUG "Record passes table end\n"); ret = -EINVAL; break; } @@ -802,7 +802,7 @@ int __init dmar_table_init(void) ret = parse_dmar_table(); if (ret < 0) { if (ret != -ENODEV) - pr_info("parse DMAR table failure.\n"); + pr_info("Parse DMAR table failure.\n"); } else if (list_empty(&dmar_drhd_units)) { pr_info("No DMAR devices found\n"); ret = -ENODEV; @@ -847,7 +847,7 @@ dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg) else addr = early_ioremap(drhd->address, VTD_PAGE_SIZE); if (!addr) { - pr_warn("IOMMU: can't validate: %llx\n", drhd->address); + pr_warn("Can't validate DRHD address: %llx\n", drhd->address); return -EINVAL; } @@ -921,14 +921,14 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr) iommu->reg_size = VTD_PAGE_SIZE; if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) { - pr_err("IOMMU: can't reserve memory\n"); + pr_err("Can't reserve memory\n"); err = -EBUSY; goto out; } iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size); if (!iommu->reg) { - pr_err("IOMMU: can't map the region\n"); + pr_err("Can't map the region\n"); err = -ENOMEM; goto release; } @@ -952,13 +952,13 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr) iommu->reg_size = map_size; if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) { - pr_err("IOMMU: can't reserve memory\n"); + pr_err("Can't reserve memory\n"); err = -EBUSY; goto out; } iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size); if (!iommu->reg) { - pr_err("IOMMU: can't map the region\n"); + pr_err("Can't map the region\n"); err = -ENOMEM; goto release; } @@ -1014,14 +1014,14 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) return -ENOMEM; if (dmar_alloc_seq_id(iommu) < 0) { - pr_err("IOMMU: failed to allocate seq_id\n"); + pr_err("Failed to allocate seq_id\n"); err = -ENOSPC; goto error; } err = map_iommu(iommu, drhd->reg_base_addr); if (err) { - pr_err("IOMMU: failed to map %s\n", iommu->name); + pr_err("Failed to map %s\n", iommu->name); goto error_free_seq_id; } @@ -1045,8 +1045,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) iommu->node = -1; ver = readl(iommu->reg + DMAR_VER_REG); - pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", - iommu->seq_id, + pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n", + iommu->name, (unsigned long long)drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver), (unsigned long long)iommu->cap, @@ -1644,7 +1644,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu) irq = dmar_alloc_hwirq(); if (irq <= 0) { - pr_err("IOMMU: no free vectors\n"); + pr_err("No free IRQ vectors\n"); return -EINVAL; } @@ -1661,7 +1661,7 @@ int dmar_set_interrupt(struct intel_iommu *iommu) ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu); if (ret) - pr_err("IOMMU: can't request irq\n"); + pr_err("Can't request irq\n"); return ret; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5ecfaf2..4faec33 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -15,8 +15,11 @@ * Shaohua Li , * Anil S Keshavamurthy , * Fenghua Yu + * Joerg Roedel */ +#define pr_fmt(fmt) "DMAR: " fmt + #include #include #include @@ -453,25 +456,21 @@ static int __init intel_iommu_setup(char *str) while (*str) { if (!strncmp(str, "on", 2)) { dmar_disabled = 0; - printk(KERN_INFO "Intel-IOMMU: enabled\n"); + pr_info("IOMMU enabled\n"); } else if (!strncmp(str, "off", 3)) { dmar_disabled = 1; - printk(KERN_INFO "Intel-IOMMU: disabled\n"); + pr_info("IOMMU disabled\n"); } else if (!strncmp(str, "igfx_off", 8)) { dmar_map_gfx = 0; - printk(KERN_INFO - "Intel-IOMMU: disable GFX device mapping\n"); + pr_info("Disable GFX device mapping\n"); } else if (!strncmp(str, "forcedac", 8)) { - printk(KERN_INFO - "Intel-IOMMU: Forcing DAC for PCI devices\n"); + pr_info("Forcing DAC for PCI devices\n"); dmar_forcedac = 1; } else if (!strncmp(str, "strict", 6)) { - printk(KERN_INFO - "Intel-IOMMU: disable batched IOTLB flush\n"); + pr_info("Disable batched IOTLB flush\n"); intel_iommu_strict = 1; } else if (!strncmp(str, "sp_off", 6)) { - printk(KERN_INFO - "Intel-IOMMU: disable supported super page\n"); + pr_info("Disable supported super page\n"); intel_iommu_superpage = 0; } else if (!strncmp(str, "ecs_off", 7)) { printk(KERN_INFO @@ -1132,7 +1131,7 @@ static int iommu_alloc_root_entry(struct intel_iommu *iommu) root = (struct root_entry *)alloc_pgtable_page(iommu->node); if (!root) { - pr_err("IOMMU: allocating root entry for %s failed\n", + pr_err("Allocating root entry for %s failed\n", iommu->name); return -ENOMEM; } @@ -1270,9 +1269,9 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, /* check IOTLB invalidation granularity */ if (DMA_TLB_IAIG(val) == 0) - printk(KERN_ERR"IOMMU: flush IOTLB failed\n"); + pr_err("Flush IOTLB failed\n"); if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type)) - pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n", + pr_debug("TLB flush request %Lx, actual %Lx\n", (unsigned long long)DMA_TLB_IIRG(type), (unsigned long long)DMA_TLB_IAIG(val)); } @@ -1443,8 +1442,8 @@ static int iommu_init_domains(struct intel_iommu *iommu) unsigned long nlongs; ndomains = cap_ndoms(iommu->cap); - pr_debug("IOMMU%d: Number of Domains supported <%ld>\n", - iommu->seq_id, ndomains); + pr_debug("%s: Number of Domains supported <%ld>\n", + iommu->name, ndomains); nlongs = BITS_TO_LONGS(ndomains); spin_lock_init(&iommu->lock); @@ -1454,15 +1453,15 @@ static int iommu_init_domains(struct intel_iommu *iommu) */ iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL); if (!iommu->domain_ids) { - pr_err("IOMMU%d: allocating domain id array failed\n", - iommu->seq_id); + pr_err("%s: Allocating domain id array failed\n", + iommu->name); return -ENOMEM; } iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *), GFP_KERNEL); if (!iommu->domains) { - pr_err("IOMMU%d: allocating domain array failed\n", - iommu->seq_id); + pr_err("%s: Allocating domain array failed\n", + iommu->name); kfree(iommu->domain_ids); iommu->domain_ids = NULL; return -ENOMEM; @@ -1567,7 +1566,7 @@ static int iommu_attach_domain(struct dmar_domain *domain, num = __iommu_attach_domain(domain, iommu); spin_unlock_irqrestore(&iommu->lock, flags); if (num < 0) - pr_err("IOMMU: no free domain ids\n"); + pr_err("%s: No free domain ids\n", iommu->name); return num; } @@ -1659,7 +1658,7 @@ static int dmar_init_reserved_ranges(void) iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START), IOVA_PFN(IOAPIC_RANGE_END)); if (!iova) { - printk(KERN_ERR "Reserve IOAPIC range failed\n"); + pr_err("Reserve IOAPIC range failed\n"); return -ENODEV; } @@ -1675,7 +1674,7 @@ static int dmar_init_reserved_ranges(void) IOVA_PFN(r->start), IOVA_PFN(r->end)); if (!iova) { - printk(KERN_ERR "Reserve iova failed\n"); + pr_err("Reserve iova failed\n"); return -ENODEV; } } @@ -1722,7 +1721,7 @@ static int domain_init(struct dmar_domain *domain, int guest_width) sagaw = cap_sagaw(iommu->cap); if (!test_bit(agaw, &sagaw)) { /* hardware doesn't support it, choose a bigger one */ - pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw); + pr_debug("Hardware doesn't support agaw %d\n", agaw); agaw = find_next_bit(&sagaw, 5, agaw); if (agaw >= 5) return -ENODEV; @@ -1823,7 +1822,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain, id = iommu_attach_vm_domain(domain, iommu); if (id < 0) { spin_unlock_irqrestore(&iommu->lock, flags); - pr_err("IOMMU: no free domain ids\n"); + pr_err("%s: No free domain ids\n", iommu->name); return -EFAULT; } } @@ -2050,8 +2049,8 @@ static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, tmp = cmpxchg64_local(&pte->val, 0ULL, pteval); if (tmp) { static int dumps = 5; - printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", - iov_pfn, tmp, (unsigned long long)pteval); + pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n", + iov_pfn, tmp, (unsigned long long)pteval); if (dumps) { dumps--; debug_dma_dump_mappings(NULL); @@ -2323,7 +2322,7 @@ static int iommu_domain_identity_map(struct dmar_domain *domain, if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn), dma_to_mm_pfn(last_vpfn))) { - printk(KERN_ERR "IOMMU: reserve iova failed\n"); + pr_err("Reserving iova failed\n"); return -ENOMEM; } @@ -2356,15 +2355,14 @@ static int iommu_prepare_identity_map(struct device *dev, range which is reserved in E820, so which didn't get set up to start with in si_domain */ if (domain == si_domain && hw_pass_through) { - printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n", - dev_name(dev), start, end); + pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n", + dev_name(dev), start, end); return 0; } - printk(KERN_INFO - "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n", - dev_name(dev), start, end); - + pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n", + dev_name(dev), start, end); + if (end < start) { WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n" "BIOS vendor: %s; Ver: %s; Product Version: %s\n", @@ -2421,12 +2419,11 @@ static inline void iommu_prepare_isa(void) if (!pdev) return; - printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n"); + pr_info("Prepare 0-16MiB unity mapping for LPC\n"); ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1); if (ret) - printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; " - "floppy might not work\n"); + pr_err("Failed to create 0-16MiB identity map - floppy might not work\n"); pci_dev_put(pdev); } @@ -2470,7 +2467,7 @@ static int __init si_domain_init(int hw) return -EFAULT; } - pr_debug("IOMMU: identity mapping domain is domain %d\n", + pr_debug("Identity mapping domain is domain %d\n", si_domain->id); if (hw) @@ -2670,8 +2667,8 @@ static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw hw ? CONTEXT_TT_PASS_THROUGH : CONTEXT_TT_MULTI_LEVEL); if (!ret) - pr_info("IOMMU: %s identity mapping for device %s\n", - hw ? "hardware" : "software", dev_name(dev)); + pr_info("%s identity mapping for device %s\n", + hw ? "Hardware" : "Software", dev_name(dev)); else if (ret == -ENODEV) /* device not associated with an iommu */ ret = 0; @@ -2748,12 +2745,12 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu) */ iommu->flush.flush_context = __iommu_flush_context; iommu->flush.flush_iotlb = __iommu_flush_iotlb; - pr_info("IOMMU: %s using Register based invalidation\n", + pr_info("%s: Using Register based invalidation\n", iommu->name); } else { iommu->flush.flush_context = qi_flush_context; iommu->flush.flush_iotlb = qi_flush_iotlb; - pr_info("IOMMU: %s using Queued invalidation\n", iommu->name); + pr_info("%s: Using Queued invalidation\n", iommu->name); } } @@ -2781,8 +2778,7 @@ static int __init init_dmars(void) g_num_of_iommus++; continue; } - printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n", - DMAR_UNITS_SUPPORTED); + pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED); } /* Preallocate enough resources for IOMMU hot-addition */ @@ -2792,7 +2788,7 @@ static int __init init_dmars(void) g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *), GFP_KERNEL); if (!g_iommus) { - printk(KERN_ERR "Allocating global iommu array failed\n"); + pr_err("Allocating global iommu array failed\n"); ret = -ENOMEM; goto error; } @@ -2843,7 +2839,7 @@ static int __init init_dmars(void) if (iommu_identity_mapping) { ret = iommu_prepare_static_identity_mapping(hw_pass_through); if (ret) { - printk(KERN_CRIT "Failed to setup IOMMU pass-through\n"); + pr_crit("Failed to setup IOMMU pass-through\n"); goto free_iommu; } } @@ -2861,15 +2857,14 @@ static int __init init_dmars(void) * endfor * endfor */ - printk(KERN_INFO "IOMMU: Setting RMRR:\n"); + pr_info("Setting RMRR:\n"); for_each_rmrr_units(rmrr) { /* some BIOS lists non-exist devices in DMAR table. */ for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, dev) { ret = iommu_prepare_rmrr_dev(rmrr, dev); if (ret) - printk(KERN_ERR - "IOMMU: mapping reserved region failed\n"); + pr_err("Mapping reserved region failed\n"); } } @@ -2944,7 +2939,7 @@ static struct iova *intel_alloc_iova(struct device *dev, } iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1); if (unlikely(!iova)) { - printk(KERN_ERR "Allocating %ld-page iova for %s failed", + pr_err("Allocating %ld-page iova for %s failed", nrpages, dev_name(dev)); return NULL; } @@ -2959,7 +2954,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH); if (!domain) { - printk(KERN_ERR "Allocating domain for %s failed", + pr_err("Allocating domain for %s failed\n", dev_name(dev)); return NULL; } @@ -2968,7 +2963,7 @@ static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev) if (unlikely(!domain_context_mapped(dev))) { ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL); if (ret) { - printk(KERN_ERR "Domain context map for %s failed", + pr_err("Domain context map for %s failed\n", dev_name(dev)); return NULL; } @@ -3010,8 +3005,8 @@ static int iommu_no_mapping(struct device *dev) * to non-identity mapping. */ domain_remove_one_dev_info(si_domain, dev); - printk(KERN_INFO "32bit %s uses non-identity mapping\n", - dev_name(dev)); + pr_info("32bit %s uses non-identity mapping\n", + dev_name(dev)); return 0; } } else { @@ -3026,8 +3021,8 @@ static int iommu_no_mapping(struct device *dev) CONTEXT_TT_PASS_THROUGH : CONTEXT_TT_MULTI_LEVEL); if (!ret) { - printk(KERN_INFO "64bit %s uses identity mapping\n", - dev_name(dev)); + pr_info("64bit %s uses identity mapping\n", + dev_name(dev)); return 1; } } @@ -3096,7 +3091,7 @@ static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr, error: if (iova) __free_iova(&domain->iovad, iova); - printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n", + pr_err("Device %s request: %zx@%llx dir %d --- failed\n", dev_name(dev), size, (unsigned long long)paddr, dir); return 0; } @@ -3411,7 +3406,7 @@ static inline int iommu_domain_cache_init(void) NULL); if (!iommu_domain_cache) { - printk(KERN_ERR "Couldn't create iommu_domain cache\n"); + pr_err("Couldn't create iommu_domain cache\n"); ret = -ENOMEM; } @@ -3428,7 +3423,7 @@ static inline int iommu_devinfo_cache_init(void) SLAB_HWCACHE_ALIGN, NULL); if (!iommu_devinfo_cache) { - printk(KERN_ERR "Couldn't create devinfo cache\n"); + pr_err("Couldn't create devinfo cache\n"); ret = -ENOMEM; } @@ -3805,19 +3800,19 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru) return 0; if (hw_pass_through && !ecap_pass_through(iommu->ecap)) { - pr_warn("IOMMU: %s doesn't support hardware pass through.\n", + pr_warn("%s: Doesn't support hardware pass through.\n", iommu->name); return -ENXIO; } if (!ecap_sc_support(iommu->ecap) && domain_update_iommu_snooping(iommu)) { - pr_warn("IOMMU: %s doesn't support snooping.\n", + pr_warn("%s: Doesn't support snooping.\n", iommu->name); return -ENXIO; } sp = domain_update_iommu_superpage(iommu) - 1; if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) { - pr_warn("IOMMU: %s doesn't support large page.\n", + pr_warn("%s: Doesn't support large page.\n", iommu->name); return -ENXIO; } @@ -4048,7 +4043,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, start = mhp->start_pfn << PAGE_SHIFT; end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1; if (iommu_domain_identity_map(si_domain, start, end)) { - pr_warn("dmar: failed to build identity map for [%llx-%llx]\n", + pr_warn("Failed to build identity map for [%llx-%llx]\n", start, end); return NOTIFY_BAD; } @@ -4066,7 +4061,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, iova = find_iova(&si_domain->iovad, start_vpfn); if (iova == NULL) { - pr_debug("dmar: failed get IOVA for PFN %lx\n", + pr_debug("Failed get IOVA for PFN %lx\n", start_vpfn); break; } @@ -4074,7 +4069,7 @@ static int intel_iommu_memory_notifier(struct notifier_block *nb, iova = split_and_remove_iova(&si_domain->iovad, iova, start_vpfn, last_vpfn); if (iova == NULL) { - pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n", + pr_warn("Failed to split IOVA PFN [%lx-%lx]\n", start_vpfn, last_vpfn); return NOTIFY_BAD; } @@ -4200,10 +4195,10 @@ int __init intel_iommu_init(void) goto out_free_dmar; if (list_empty(&dmar_rmrr_units)) - printk(KERN_INFO "DMAR: No RMRR found\n"); + pr_info("No RMRR found\n"); if (list_empty(&dmar_atsr_units)) - printk(KERN_INFO "DMAR: No ATSR found\n"); + pr_info("No ATSR found\n"); if (dmar_init_reserved_ranges()) { if (force_on) @@ -4217,12 +4212,11 @@ int __init intel_iommu_init(void) if (ret) { if (force_on) panic("tboot: Failed to initialize DMARs\n"); - printk(KERN_ERR "IOMMU: dmar init failed\n"); + pr_err("Initialization failed\n"); goto out_free_reserved_range; } up_write(&dmar_global_lock); - printk(KERN_INFO - "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n"); + pr_info("Intel(R) Virtualization Technology for Directed I/O\n"); init_timer(&unmap_timer); #ifdef CONFIG_SWIOTLB @@ -4364,13 +4358,11 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE); if (!dmar_domain) { - printk(KERN_ERR - "intel_iommu_domain_init: dmar_domain == NULL\n"); + pr_err("Can't allocate dmar_domain\n"); return NULL; } if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) { - printk(KERN_ERR - "intel_iommu_domain_init() failed\n"); + pr_err("Domain initialization failed\n"); domain_exit(dmar_domain); return NULL; } @@ -4429,7 +4421,7 @@ static int intel_iommu_attach_device(struct iommu_domain *domain, addr_width = cap_mgaw(iommu->cap); if (dmar_domain->max_addr > (1LL << addr_width)) { - printk(KERN_ERR "%s: iommu width (%d) is not " + pr_err("%s: iommu width (%d) is not " "sufficient for the mapped address (%llx)\n", __func__, addr_width, dmar_domain->max_addr); return -EFAULT; @@ -4483,7 +4475,7 @@ static int intel_iommu_map(struct iommu_domain *domain, /* check if minimum agaw is sufficient for mapped address */ end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1; if (end < max_addr) { - printk(KERN_ERR "%s: iommu width (%d) is not " + pr_err("%s: iommu width (%d) is not " "sufficient for the mapped address (%llx)\n", __func__, dmar_domain->gaw, max_addr); return -EFAULT; @@ -4624,7 +4616,7 @@ static const struct iommu_ops intel_iommu_ops = { static void quirk_iommu_g4x_gfx(struct pci_dev *dev) { /* G4x/GM45 integrated gfx dmar support is totally busted. */ - printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n"); + pr_info("Disabling IOMMU for graphics on this chipset\n"); dmar_map_gfx = 0; } @@ -4642,7 +4634,7 @@ static void quirk_iommu_rwbf(struct pci_dev *dev) * Mobile 4 Series Chipset neglects to set RWBF capability, * but needs it. Same seems to hold for the desktop versions. */ - printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n"); + pr_info("Forcing write-buffer flush capability\n"); rwbf_quirk = 1; } @@ -4672,11 +4664,11 @@ static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev) return; if (!(ggc & GGC_MEMORY_VT_ENABLED)) { - printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); + pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n"); dmar_map_gfx = 0; } else if (dmar_map_gfx) { /* we have to ensure the gfx device is idle before we flush */ - printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n"); + pr_info("Disabling batched IOTLB flush on Ironlake\n"); intel_iommu_strict = 1; } } @@ -4738,7 +4730,7 @@ static void __init check_tylersburg_isoch(void) iommu_identity_mapping |= IDENTMAP_AZALIA; return; } - - printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", + + pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n", vtisochctrl); } diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 5709ae9..3fe3fc7 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -1,3 +1,6 @@ + +#define pr_fmt(fmt) "DMAR-IR: " fmt + #include #include #include @@ -100,8 +103,7 @@ static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) } if (mask > ecap_max_handle_mask(iommu->ecap)) { - printk(KERN_ERR - "Requested mask %x exceeds the max invalidation handle" + pr_err("Requested mask %x exceeds the max invalidation handle" " mask value %Lx\n", mask, ecap_max_handle_mask(iommu->ecap)); return -1; @@ -333,7 +335,7 @@ static int set_ioapic_sid(struct irte *irte, int apic) up_read(&dmar_global_lock); if (sid == 0) { - pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); + pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); return -1; } @@ -360,7 +362,7 @@ static int set_hpet_sid(struct irte *irte, u8 id) up_read(&dmar_global_lock); if (sid == 0) { - pr_warning("Failed to set source-id of HPET block (%d)\n", id); + pr_warn("Failed to set source-id of HPET block (%d)\n", id); return -1; } @@ -580,7 +582,7 @@ static void __init intel_cleanup_irq_remapping(void) } if (x2apic_supported()) - pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); + pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); } static int __init intel_prepare_irq_remapping(void) @@ -589,8 +591,7 @@ static int __init intel_prepare_irq_remapping(void) struct intel_iommu *iommu; if (irq_remap_broken) { - printk(KERN_WARNING - "This system BIOS has enabled interrupt remapping\n" + pr_warn("This system BIOS has enabled interrupt remapping\n" "on a chipset that contains an erratum making that\n" "feature unstable. To maintain system stability\n" "interrupt remapping is being disabled. Please\n" @@ -606,7 +607,7 @@ static int __init intel_prepare_irq_remapping(void) return -ENODEV; if (parse_ioapics_under_ir() != 1) { - printk(KERN_INFO "Not enabling interrupt remapping\n"); + pr_info("Not enabling interrupt remapping\n"); goto error; } @@ -667,8 +668,8 @@ static int __init intel_enable_irq_remapping(void) */ for_each_iommu(iommu, drhd) if (eim && !ecap_eim_support(iommu->ecap)) { - printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " - " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); + pr_info("DRHD %Lx: EIM not supported by DRHD, " + " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); eim = 0; } eim_mode = eim; @@ -682,7 +683,7 @@ static int __init intel_enable_irq_remapping(void) int ret = dmar_enable_qi(iommu); if (ret) { - printk(KERN_ERR "DRHD %Lx: failed to enable queued, " + pr_err("DRHD %Lx: failed to enable queued, " " invalidation, ecap %Lx, ret %d\n", drhd->reg_base_addr, iommu->ecap, ret); goto error; @@ -1145,14 +1146,12 @@ static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) down_read(&dmar_global_lock); iommu = map_dev_to_ir(dev); if (!iommu) { - printk(KERN_ERR - "Unable to map PCI %s to iommu\n", pci_name(dev)); + pr_err("Unable to map PCI %s to iommu\n", pci_name(dev)); index = -ENOENT; } else { index = alloc_irte(iommu, irq, nvec); if (index < 0) { - printk(KERN_ERR - "Unable to allocate %d IRTE for PCI %s\n", + pr_err("Unable to allocate %d IRTE for PCI %s\n", nvec, pci_name(dev)); index = -ENOSPC; } -- cgit v0.10.2 From b63d80d1e01e949dbe469e1d72fc0b7e173dbdd8 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 09:14:34 +0200 Subject: iommu/vt-d: Init QI before root entry is allocated QI needs to be available when we write the root entry into hardware because flushes might be necessary after this. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 4faec33..989761c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2803,6 +2803,8 @@ static int __init init_dmars(void) for_each_active_iommu(iommu, drhd) { g_iommus[iommu->seq_id] = iommu; + intel_iommu_init_qi(iommu); + ret = iommu_init_domains(iommu); if (ret) goto free_iommu; @@ -2819,9 +2821,6 @@ static int __init init_dmars(void) hw_pass_through = 0; } - for_each_active_iommu(iommu, drhd) - intel_iommu_init_qi(iommu); - if (iommu_pass_through) iommu_identity_mapping |= IDENTMAP_ALL; -- cgit v0.10.2 From 5f0a7f7614a9d99325ac8d618f1cdf7a3014287c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 09:18:53 +0200 Subject: iommu/vt-d: Make root entry visible for hardware right after allocation In case there was an old root entry, make our new one visible immediately after it was allocated. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 989761c..bf3e450 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2817,6 +2817,12 @@ static int __init init_dmars(void) ret = iommu_alloc_root_entry(iommu); if (ret) goto free_iommu; + + iommu_flush_write_buffer(iommu); + iommu_set_root_entry(iommu); + iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); + iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); + if (!ecap_pass_through(iommu->ecap)) hw_pass_through = 0; } @@ -2893,10 +2899,6 @@ static int __init init_dmars(void) if (ret) goto free_iommu; - iommu_set_root_entry(iommu); - - iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); - iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH); iommu_enable_translation(iommu); iommu_disable_protect_mem_regions(iommu); } -- cgit v0.10.2 From 4158c2eca3c77ed3cccdcaeab153aad4e433369c Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 10:14:02 +0200 Subject: iommu/vt-d: Detect pre enabled translation Add code to detect whether translation is already enabled in the IOMMU. Save this state in a flags field added to struct intel_iommu. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index bf3e450..39b9062 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -443,6 +443,20 @@ static LIST_HEAD(device_domain_list); static const struct iommu_ops intel_iommu_ops; +static bool translation_pre_enabled(struct intel_iommu *iommu) +{ + return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED); +} + +static void init_translation_status(struct intel_iommu *iommu) +{ + u32 gsts; + + gsts = readl(iommu->reg + DMAR_GSTS_REG); + if (gsts & DMA_GSTS_TES) + iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED; +} + /* Convert generic 'struct iommu_domain to private struct dmar_domain */ static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom) { @@ -2809,6 +2823,11 @@ static int __init init_dmars(void) if (ret) goto free_iommu; + init_translation_status(iommu); + + if (translation_pre_enabled(iommu)) + pr_info("Translation already enabled - trying to copy translation structures\n"); + /* * TBD: * we could share the same root & context tables diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index a240e61..b85b81a 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -320,6 +320,9 @@ enum { MAX_SR_DMAR_REGS }; +#define VTD_FLAG_TRANS_PRE_ENABLED (1 << 0) +#define VTD_FLAG_IRQ_REMAP_PRE_ENABLED (1 << 1) + struct intel_iommu { void __iomem *reg; /* Pointer to hardware regs, virtual addr */ u64 reg_phys; /* physical address of hw register set */ @@ -351,6 +354,7 @@ struct intel_iommu { #endif struct device *iommu_dev; /* IOMMU-sysfs device */ int node; + u32 flags; /* Software defined flags */ }; static inline void __iommu_flush_cache( -- cgit v0.10.2 From 091d42e43d21b6ca7ec39bf5f9e17bc0bd8d4312 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 11:56:10 +0200 Subject: iommu/vt-d: Copy translation tables from old kernel If we are in a kdump kernel and find translation enabled in the iommu, try to copy the translation tables from the old kernel to preserve the mappings until the device driver takes over. This supports old and the extended root-entry and context-table formats. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 39b9062..1eb743c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -193,7 +194,29 @@ struct root_entry { }; #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) +/* + * Take a root_entry and return the Lower Context Table Pointer (LCTP) + * if marked present. + */ +static phys_addr_t root_entry_lctp(struct root_entry *re) +{ + if (!(re->lo & 1)) + return 0; + + return re->lo & VTD_PAGE_MASK; +} + +/* + * Take a root_entry and return the Upper Context Table Pointer (UCTP) + * if marked present. + */ +static phys_addr_t root_entry_uctp(struct root_entry *re) +{ + if (!(re->hi & 1)) + return 0; + return re->hi & VTD_PAGE_MASK; +} /* * low 64 bits: * 0: present @@ -448,6 +471,11 @@ static bool translation_pre_enabled(struct intel_iommu *iommu) return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED); } +static void clear_translation_pre_enabled(struct intel_iommu *iommu) +{ + iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED; +} + static void init_translation_status(struct intel_iommu *iommu) { u32 gsts; @@ -2768,6 +2796,153 @@ static void intel_iommu_init_qi(struct intel_iommu *iommu) } } +static int copy_context_table(struct intel_iommu *iommu, + struct root_entry *old_re, + struct context_entry **tbl, + int bus, bool ext) +{ + struct context_entry *old_ce = NULL, *new_ce = NULL, ce; + int tbl_idx, pos = 0, idx, devfn, ret = 0; + phys_addr_t old_ce_phys; + + tbl_idx = ext ? bus * 2 : bus; + + for (devfn = 0; devfn < 256; devfn++) { + /* First calculate the correct index */ + idx = (ext ? devfn * 2 : devfn) % 256; + + if (idx == 0) { + /* First save what we may have and clean up */ + if (new_ce) { + tbl[tbl_idx] = new_ce; + __iommu_flush_cache(iommu, new_ce, + VTD_PAGE_SIZE); + pos = 1; + } + + if (old_ce) + iounmap(old_ce); + + ret = 0; + if (devfn < 0x80) + old_ce_phys = root_entry_lctp(old_re); + else + old_ce_phys = root_entry_uctp(old_re); + + if (!old_ce_phys) { + if (ext && devfn == 0) { + /* No LCTP, try UCTP */ + devfn = 0x7f; + continue; + } else { + goto out; + } + } + + ret = -ENOMEM; + old_ce = ioremap_cache(old_ce_phys, PAGE_SIZE); + if (!old_ce) + goto out; + + new_ce = alloc_pgtable_page(iommu->node); + if (!new_ce) + goto out_unmap; + + ret = 0; + } + + /* Now copy the context entry */ + ce = old_ce[idx]; + + if (!context_present(&ce)) + continue; + + new_ce[idx] = ce; + } + + tbl[tbl_idx + pos] = new_ce; + + __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE); + +out_unmap: + iounmap(old_ce); + +out: + return ret; +} + +static int copy_translation_tables(struct intel_iommu *iommu) +{ + struct context_entry **ctxt_tbls; + struct root_entry *old_rt; + phys_addr_t old_rt_phys; + int ctxt_table_entries; + unsigned long flags; + u64 rtaddr_reg; + int bus, ret; + bool ext; + + rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG); + ext = !!(rtaddr_reg & DMA_RTADDR_RTT); + + old_rt_phys = rtaddr_reg & VTD_PAGE_MASK; + if (!old_rt_phys) + return -EINVAL; + + old_rt = ioremap_cache(old_rt_phys, PAGE_SIZE); + if (!old_rt) + return -ENOMEM; + + /* This is too big for the stack - allocate it from slab */ + ctxt_table_entries = ext ? 512 : 256; + ret = -ENOMEM; + ctxt_tbls = kzalloc(ctxt_table_entries * sizeof(void *), GFP_KERNEL); + if (!ctxt_tbls) + goto out_unmap; + + for (bus = 0; bus < 256; bus++) { + ret = copy_context_table(iommu, &old_rt[bus], + ctxt_tbls, bus, ext); + if (ret) { + pr_err("%s: Failed to copy context table for bus %d\n", + iommu->name, bus); + continue; + } + } + + spin_lock_irqsave(&iommu->lock, flags); + + /* Context tables are copied, now write them to the root_entry table */ + for (bus = 0; bus < 256; bus++) { + int idx = ext ? bus * 2 : bus; + u64 val; + + if (ctxt_tbls[idx]) { + val = virt_to_phys(ctxt_tbls[idx]) | 1; + iommu->root_entry[bus].lo = val; + } + + if (!ext || !ctxt_tbls[idx + 1]) + continue; + + val = virt_to_phys(ctxt_tbls[idx + 1]) | 1; + iommu->root_entry[bus].hi = val; + } + + spin_unlock_irqrestore(&iommu->lock, flags); + + kfree(ctxt_tbls); + + __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE); + + ret = 0; + +out_unmap: + iounmap(old_rt); + + return ret; +} + static int __init init_dmars(void) { struct dmar_drhd_unit *drhd; @@ -2825,8 +3000,12 @@ static int __init init_dmars(void) init_translation_status(iommu); - if (translation_pre_enabled(iommu)) - pr_info("Translation already enabled - trying to copy translation structures\n"); + if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { + iommu_disable_translation(iommu); + clear_translation_pre_enabled(iommu); + pr_warn("Translation was enabled for %s but we are not in kdump mode\n", + iommu->name); + } /* * TBD: @@ -2837,6 +3016,30 @@ static int __init init_dmars(void) if (ret) goto free_iommu; + if (translation_pre_enabled(iommu)) { + pr_info("Translation already enabled - trying to copy translation structures\n"); + + ret = copy_translation_tables(iommu); + if (ret) { + /* + * We found the IOMMU with translation + * enabled - but failed to copy over the + * old root-entry table. Try to proceed + * by disabling translation now and + * allocating a clean root-entry table. + * This might cause DMAR faults, but + * probably the dump will still succeed. + */ + pr_err("Failed to copy translation tables from previous kernel for %s\n", + iommu->name); + iommu_disable_translation(iommu); + clear_translation_pre_enabled(iommu); + } else { + pr_info("Copied translation tables from previous kernel for %s\n", + iommu->name); + } + } + iommu_flush_write_buffer(iommu); iommu_set_root_entry(iommu); iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL); -- cgit v0.10.2 From dbcd861f252d727ac856a3da2cba8a480a36ab2e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 12:02:09 +0200 Subject: iommu/vt-d: Do not re-use domain-ids from the old kernel Mark all domain-ids we find as reserved, so that there could be no collision between domains from the previous kernel and our domains in the IOMMU TLB. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1eb743c..4318533 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -273,6 +273,11 @@ static inline void context_set_domain_id(struct context_entry *context, context->hi |= (value & ((1 << 16) - 1)) << 8; } +static inline int context_domain_id(struct context_entry *c) +{ + return((c->hi >> 8) & 0xffff); +} + static inline void context_clear_entry(struct context_entry *context) { context->lo = 0; @@ -2802,7 +2807,7 @@ static int copy_context_table(struct intel_iommu *iommu, int bus, bool ext) { struct context_entry *old_ce = NULL, *new_ce = NULL, ce; - int tbl_idx, pos = 0, idx, devfn, ret = 0; + int tbl_idx, pos = 0, idx, devfn, ret = 0, did; phys_addr_t old_ce_phys; tbl_idx = ext ? bus * 2 : bus; @@ -2857,6 +2862,10 @@ static int copy_context_table(struct intel_iommu *iommu, if (!context_present(&ce)) continue; + did = context_domain_id(&ce); + if (did >= 0 && did < cap_ndoms(iommu->cap)) + set_bit(did, iommu->domain_ids); + new_ce[idx] = ce; } -- cgit v0.10.2 From cf484d0e6939ce287e65880ffe48860165fe5cb5 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 12:21:46 +0200 Subject: iommu/vt-d: Mark copied context entries Mark the context entries we copied over from the old kernel, so that we don't detect them as present in other code paths. This makes sure we safely overwrite old context entries when a new domain is assigned. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 4318533..e5a03f8 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -233,10 +233,38 @@ struct context_entry { u64 hi; }; -static inline bool context_present(struct context_entry *context) +static inline void context_clear_pasid_enable(struct context_entry *context) +{ + context->lo &= ~(1ULL << 11); +} + +static inline bool context_pasid_enabled(struct context_entry *context) +{ + return !!(context->lo & (1ULL << 11)); +} + +static inline void context_set_copied(struct context_entry *context) +{ + context->hi |= (1ull << 3); +} + +static inline bool context_copied(struct context_entry *context) +{ + return !!(context->hi & (1ULL << 3)); +} + +static inline bool __context_present(struct context_entry *context) { return (context->lo & 1); } + +static inline bool context_present(struct context_entry *context) +{ + return context_pasid_enabled(context) ? + __context_present(context) : + __context_present(context) && !context_copied(context); +} + static inline void context_set_present(struct context_entry *context) { context->lo |= 1; @@ -1861,6 +1889,8 @@ static int domain_context_mapping_one(struct dmar_domain *domain, return 0; } + context_clear_entry(context); + id = domain->id; pgd = domain->pgd; @@ -2859,13 +2889,32 @@ static int copy_context_table(struct intel_iommu *iommu, /* Now copy the context entry */ ce = old_ce[idx]; - if (!context_present(&ce)) + if (!__context_present(&ce)) continue; did = context_domain_id(&ce); if (did >= 0 && did < cap_ndoms(iommu->cap)) set_bit(did, iommu->domain_ids); + /* + * We need a marker for copied context entries. This + * marker needs to work for the old format as well as + * for extended context entries. + * + * Bit 67 of the context entry is used. In the old + * format this bit is available to software, in the + * extended format it is the PGE bit, but PGE is ignored + * by HW if PASIDs are disabled (and thus still + * available). + * + * So disable PASIDs first and then mark the entry + * copied. This means that we don't copy PASID + * translations from the old kernel, but this is fine as + * faults there are not fatal. + */ + context_clear_pasid_enable(&ce); + context_set_copied(&ce); + new_ce[idx] = ce; } -- cgit v0.10.2 From 86080ccc223aabf8d0b85a504f4f06aa88e82fb3 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 12:27:16 +0200 Subject: iommu/vt-d: Allocate si_domain in init_dmars() This seperates the allocation of the si_domain from its assignment to devices. It makes sure that the iommu=pt case still works in the kdump kernel, when we have to defer the assignment of devices to domains to device driver initialization time. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index e5a03f8..b43f049 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2763,10 +2763,6 @@ static int __init iommu_prepare_static_identity_mapping(int hw) int i; int ret = 0; - ret = si_domain_init(hw); - if (ret) - return -EFAULT; - for_each_pci_dev(pdev) { ret = dev_prepare_static_identity_mapping(&pdev->dev, hw); if (ret) @@ -2780,7 +2776,7 @@ static int __init iommu_prepare_static_identity_mapping(int hw) if (dev->bus != &acpi_bus_type) continue; - + adev= to_acpi_device(dev); mutex_lock(&adev->physical_node_lock); list_for_each_entry(pn, &adev->physical_node_list, node) { @@ -3114,6 +3110,12 @@ static int __init init_dmars(void) iommu_identity_mapping |= IDENTMAP_GFX; #endif + if (iommu_identity_mapping) { + ret = si_domain_init(hw_pass_through); + if (ret) + goto free_iommu; + } + check_tylersburg_isoch(); /* -- cgit v0.10.2 From a87f491890e994dca4bee64690d7e5183a19264e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 12:32:54 +0200 Subject: iommu/vt-d: Don't do early domain assignment if kdump kernel When we copied over context tables from an old kernel, we need to defer assignment of devices to domains until the device driver takes over. So skip this part of initialization when we copied over translation tables from the old kernel. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index b43f049..3b32aa5 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3001,6 +3001,7 @@ static int __init init_dmars(void) { struct dmar_drhd_unit *drhd; struct dmar_rmrr_unit *rmrr; + bool copied_tables = false; struct device *dev; struct intel_iommu *iommu; int i, ret; @@ -3091,6 +3092,7 @@ static int __init init_dmars(void) } else { pr_info("Copied translation tables from previous kernel for %s\n", iommu->name); + copied_tables = true; } } @@ -3119,6 +3121,15 @@ static int __init init_dmars(void) check_tylersburg_isoch(); /* + * If we copied translations from a previous kernel in the kdump + * case, we can not assign the devices to domains now, as that + * would eliminate the old mappings. So skip this part and defer + * the assignment to device driver initialization time. + */ + if (copied_tables) + goto domains_done; + + /* * If pass through is not set or not enabled, setup context entries for * identity mappings for rmrr, gfx, and isa and may fall back to static * identity mapping if iommu_identity_mapping is set. @@ -3157,6 +3168,8 @@ static int __init init_dmars(void) iommu_prepare_isa(); +domains_done: + /* * for each drhd * enable fault log -- cgit v0.10.2 From c3361f2f6e1d64bc7e7b8148bbd1c66b8007a898 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 12:39:25 +0200 Subject: iommu/vt-d: Don't copy translation tables if RTT bit needs to be changed We can't change the RTT bit when translation is enabled, so don't copy translation tables when we would change the bit with our new root entry. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 3b32aa5..ca7d37c 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -2934,10 +2934,20 @@ static int copy_translation_tables(struct intel_iommu *iommu) unsigned long flags; u64 rtaddr_reg; int bus, ret; - bool ext; + bool new_ext, ext; rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG); ext = !!(rtaddr_reg & DMA_RTADDR_RTT); + new_ext = !!ecap_ecs(iommu->ecap); + + /* + * The RTT bit can only be changed when translation is disabled, + * but disabling translation means to open a window for data + * corruption. So bail out and don't copy anything if we would + * have to change the bit. + */ + if (new_ext != ext) + return -EINVAL; old_rt_phys = rtaddr_reg & VTD_PAGE_MASK; if (!old_rt_phys) -- cgit v0.10.2 From 60b523ecfede05796aa77024b3960fecabab4de7 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 12:44:33 +0200 Subject: iommu/vt-d: Don't disable translation prior to OS handover For all the copy-translation code to run, we have to keep translation enabled in intel_iommu_init(). So remove the code disabling it. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ca7d37c..e40c858 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4484,13 +4484,6 @@ int __init intel_iommu_init(void) goto out_free_dmar; } - /* - * Disable translation if already enabled prior to OS handover. - */ - for_each_active_iommu(iommu, drhd) - if (iommu->gcmd & DMA_GCMD_TE) - iommu_disable_translation(iommu); - if (dmar_dev_scope_init() < 0) { if (force_on) panic("tboot: Failed to initialize DMAR device scope\n"); -- cgit v0.10.2 From 8939ddf6d65264cf9f014ffd7c9bff02ad9626e6 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 14:40:01 +0200 Subject: iommu/vt-d: Enable Translation only if it was previously disabled Do not touch the TE bit unless we know translation is disabled. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index e40c858..a98a7b2 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3204,7 +3204,9 @@ domains_done: if (ret) goto free_iommu; - iommu_enable_translation(iommu); + if (!translation_pre_enabled(iommu)) + iommu_enable_translation(iommu); + iommu_disable_protect_mem_regions(iommu); } -- cgit v0.10.2 From 23256d0b350014a05c1edf0f355546aa1ff2eb55 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 14:15:49 +0200 Subject: iommu/vt-d: Move EIM detection to intel_prepare_irq_remapping We need this to be detected already when we program the irq remapping table pointer to hardware. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 3fe3fc7..12250f7 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -589,6 +589,7 @@ static int __init intel_prepare_irq_remapping(void) { struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; + int eim = 0; if (irq_remap_broken) { pr_warn("This system BIOS has enabled interrupt remapping\n" @@ -616,6 +617,26 @@ static int __init intel_prepare_irq_remapping(void) if (!ecap_ir_support(iommu->ecap)) goto error; + /* Detect remapping mode: lapic or x2apic */ + if (x2apic_supported()) { + eim = !dmar_x2apic_optout(); + if (!eim) { + pr_info("x2apic is disabled because BIOS sets x2apic opt out bit."); + pr_info("Use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); + } + } + + for_each_iommu(iommu, drhd) { + if (eim && !ecap_eim_support(iommu->ecap)) { + pr_info("%s does not support EIM\n", iommu->name); + eim = 0; + } + } + + eim_mode = eim; + if (eim) + pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); + /* Do the allocations early */ for_each_iommu(iommu, drhd) if (intel_setup_irq_remapping(iommu)) @@ -633,13 +654,6 @@ static int __init intel_enable_irq_remapping(void) struct dmar_drhd_unit *drhd; struct intel_iommu *iommu; bool setup = false; - int eim = 0; - - if (x2apic_supported()) { - eim = !dmar_x2apic_optout(); - if (!eim) - pr_info("x2apic is disabled because BIOS sets x2apic opt out bit. You can use 'intremap=no_x2apic_optout' to override the BIOS setting.\n"); - } for_each_iommu(iommu, drhd) { /* @@ -664,19 +678,6 @@ static int __init intel_enable_irq_remapping(void) } /* - * check for the Interrupt-remapping support - */ - for_each_iommu(iommu, drhd) - if (eim && !ecap_eim_support(iommu->ecap)) { - pr_info("DRHD %Lx: EIM not supported by DRHD, " - " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); - eim = 0; - } - eim_mode = eim; - if (eim) - pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); - - /* * Enable queued invalidation for all the DRHD's. */ for_each_iommu(iommu, drhd) { @@ -694,7 +695,7 @@ static int __init intel_enable_irq_remapping(void) * Setup Interrupt-remapping for all the DRHD's now. */ for_each_iommu(iommu, drhd) { - iommu_set_irq_remapping(iommu, eim); + iommu_set_irq_remapping(iommu, eim_mode); setup = true; } @@ -710,9 +711,9 @@ static int __init intel_enable_irq_remapping(void) */ x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries; - pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); + pr_info("Enabled IRQ remapping in %s mode\n", eim_mode ? "x2apic" : "xapic"); - return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; + return eim_mode ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; error: intel_cleanup_irq_remapping(); -- cgit v0.10.2 From 9e4e49dfde3bfac48d431893863ac3b9d3f3774a Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 14:23:56 +0200 Subject: iommu/vt-d: Move QI initializationt to intel_setup_irq_remapping QI needs to be enabled when we program the irq remapping table to hardware in the prepare phase later. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 12250f7..46d17e1 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -507,12 +507,35 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) ir_table->base = page_address(pages); ir_table->bitmap = bitmap; iommu->ir_table = ir_table; + + /* + * If the queued invalidation is already initialized, + * shouldn't disable it. + */ + if (!iommu->qi) { + /* + * Clear previous faults. + */ + dmar_fault(-1, iommu); + dmar_disable_qi(iommu); + + if (dmar_enable_qi(iommu)) { + pr_err("Failed to enable queued invalidation\n"); + goto out_free_bitmap; + } + } + return 0; +out_free_bitmap: + kfree(bitmap); out_free_pages: __free_pages(pages, INTR_REMAP_PAGE_ORDER); out_free_table: kfree(ir_table); + + iommu->ir_table = NULL; + return -ENOMEM; } @@ -637,10 +660,14 @@ static int __init intel_prepare_irq_remapping(void) if (eim) pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n"); - /* Do the allocations early */ - for_each_iommu(iommu, drhd) - if (intel_setup_irq_remapping(iommu)) + /* Do the initializations early */ + for_each_iommu(iommu, drhd) { + if (intel_setup_irq_remapping(iommu)) { + pr_err("Failed to setup irq remapping for %s\n", + iommu->name); goto error; + } + } return 0; @@ -655,42 +682,9 @@ static int __init intel_enable_irq_remapping(void) struct intel_iommu *iommu; bool setup = false; - for_each_iommu(iommu, drhd) { - /* - * If the queued invalidation is already initialized, - * shouldn't disable it. - */ - if (iommu->qi) - continue; - - /* - * Clear previous faults. - */ - dmar_fault(-1, iommu); - - /* - * Disable intr remapping and queued invalidation, if already - * enabled prior to OS handover. - */ + for_each_iommu(iommu, drhd) iommu_disable_irq_remapping(iommu); - dmar_disable_qi(iommu); - } - - /* - * Enable queued invalidation for all the DRHD's. - */ - for_each_iommu(iommu, drhd) { - int ret = dmar_enable_qi(iommu); - - if (ret) { - pr_err("DRHD %Lx: failed to enable queued, " - " invalidation, ecap %Lx, ret %d\n", - drhd->reg_base_addr, iommu->ecap, ret); - goto error; - } - } - /* * Setup Interrupt-remapping for all the DRHD's now. */ @@ -1242,28 +1236,12 @@ static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) /* Setup Interrupt-remapping now. */ ret = intel_setup_irq_remapping(iommu); if (ret) { - pr_err("DRHD %Lx: failed to allocate resource\n", - iommu->reg_phys); - ir_remove_ioapic_hpet_scope(iommu); - return ret; - } - - if (!iommu->qi) { - /* Clear previous faults. */ - dmar_fault(-1, iommu); - iommu_disable_irq_remapping(iommu); - dmar_disable_qi(iommu); - } - - /* Enable queued invalidation */ - ret = dmar_enable_qi(iommu); - if (!ret) { - iommu_set_irq_remapping(iommu, eim); - } else { - pr_err("DRHD %Lx: failed to enable queued invalidation, ecap %Lx, ret %d\n", - iommu->reg_phys, iommu->ecap, ret); + pr_err("Failed to setup irq remapping for %s\n", + iommu->name); intel_teardown_irq_remapping(iommu); ir_remove_ioapic_hpet_scope(iommu); + } else { + iommu_set_irq_remapping(iommu, eim); } return ret; -- cgit v0.10.2 From c676f5876bc088ace35ece98042a3be6d8329530 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 14:25:53 +0200 Subject: iommu/vt-d: Disable IRQ remapping in intel_prepare_irq_remapping Move it to this function for now, so that the copy routines for irq remapping take no effect yet. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 46d17e1..f1711836 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -654,6 +654,9 @@ static int __init intel_prepare_irq_remapping(void) pr_info("%s does not support EIM\n", iommu->name); eim = 0; } + + /* Disable IRQ remapping if it is already enabled */ + iommu_disable_irq_remapping(iommu); } eim_mode = eim; @@ -682,9 +685,6 @@ static int __init intel_enable_irq_remapping(void) struct intel_iommu *iommu; bool setup = false; - for_each_iommu(iommu, drhd) - iommu_disable_irq_remapping(iommu); - /* * Setup Interrupt-remapping for all the DRHD's now. */ -- cgit v0.10.2 From d4d1c0f3d6a9558fe3857853afdf2f93e1679c03 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 14:35:54 +0200 Subject: iommu/vt-d: Set IRTA in intel_setup_irq_remapping This way we can give the hardware the new IR table right after it has been allocated and initialized. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index f1711836..8497028 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -428,9 +428,9 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev) static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) { + unsigned long flags; u64 addr; u32 sts; - unsigned long flags; addr = virt_to_phys((void *)iommu->ir_table->base); @@ -447,10 +447,16 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) raw_spin_unlock_irqrestore(&iommu->register_lock, flags); /* - * global invalidation of interrupt entry cache before enabling - * interrupt-remapping. + * Global invalidation of interrupt entry cache to make sure the + * hardware uses the new irq remapping table. */ qi_global_iec(iommu); +} + +static void iommu_enable_irq_remapping(struct intel_iommu *iommu) +{ + unsigned long flags; + u32 sts; raw_spin_lock_irqsave(&iommu->register_lock, flags); @@ -525,6 +531,8 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) } } + iommu_set_irq_remapping(iommu, eim_mode); + return 0; out_free_bitmap: @@ -689,7 +697,7 @@ static int __init intel_enable_irq_remapping(void) * Setup Interrupt-remapping for all the DRHD's now. */ for_each_iommu(iommu, drhd) { - iommu_set_irq_remapping(iommu, eim_mode); + iommu_enable_irq_remapping(iommu); setup = true; } @@ -926,6 +934,7 @@ static int reenable_irq_remapping(int eim) /* Set up interrupt remapping for iommu.*/ iommu_set_irq_remapping(iommu, eim); + iommu_enable_irq_remapping(iommu); setup = true; } @@ -1241,7 +1250,7 @@ static int dmar_ir_add(struct dmar_drhd_unit *dmaru, struct intel_iommu *iommu) intel_teardown_irq_remapping(iommu); ir_remove_ioapic_hpet_scope(iommu); } else { - iommu_set_irq_remapping(iommu, eim); + iommu_enable_irq_remapping(iommu); } return ret; -- cgit v0.10.2 From af3b358e48115588d905cc07a47b3f356e0d01d1 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 15:00:21 +0200 Subject: iommu/vt-d: Copy IR table from old kernel when in kdump mode When we are booting into a kdump kernel and find IR enabled, copy over the contents of the previous IR table so that spurious interrupts will not be target aborted. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 8497028..2a90121 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -54,8 +55,28 @@ static struct hpet_scope ir_hpet[MAX_HPET_TBS]; */ static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); +static void iommu_disable_irq_remapping(struct intel_iommu *iommu); static int __init parse_ioapics_under_ir(void); +static bool ir_pre_enabled(struct intel_iommu *iommu) +{ + return (iommu->flags & VTD_FLAG_IRQ_REMAP_PRE_ENABLED); +} + +static void clear_ir_pre_enabled(struct intel_iommu *iommu) +{ + iommu->flags &= ~VTD_FLAG_IRQ_REMAP_PRE_ENABLED; +} + +static void init_ir_status(struct intel_iommu *iommu) +{ + u32 gsts; + + gsts = readl(iommu->reg + DMAR_GSTS_REG); + if (gsts & DMA_GSTS_IRES) + iommu->flags |= VTD_FLAG_IRQ_REMAP_PRE_ENABLED; +} + static struct irq_2_iommu *irq_2_iommu(unsigned int irq) { struct irq_cfg *cfg = irq_cfg(irq); @@ -426,6 +447,44 @@ static int set_msi_sid(struct irte *irte, struct pci_dev *dev) return 0; } +static int iommu_load_old_irte(struct intel_iommu *iommu) +{ + struct irte *old_ir_table; + phys_addr_t irt_phys; + size_t size; + u64 irta; + + if (!is_kdump_kernel()) { + pr_warn("IRQ remapping was enabled on %s but we are not in kdump mode\n", + iommu->name); + clear_ir_pre_enabled(iommu); + iommu_disable_irq_remapping(iommu); + return -EINVAL; + } + + /* Check whether the old ir-table has the same size as ours */ + irta = dmar_readq(iommu->reg + DMAR_IRTA_REG); + if ((irta & INTR_REMAP_TABLE_REG_SIZE_MASK) + != INTR_REMAP_TABLE_REG_SIZE) + return -EINVAL; + + irt_phys = irta & VTD_PAGE_MASK; + size = INTR_REMAP_TABLE_ENTRIES*sizeof(struct irte); + + /* Map the old IR table */ + old_ir_table = ioremap_cache(irt_phys, size); + if (!old_ir_table) + return -ENOMEM; + + /* Copy data over */ + memcpy(iommu->ir_table->base, old_ir_table, size); + + __iommu_flush_cache(iommu, iommu->ir_table->base, size); + + return 0; +} + + static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) { unsigned long flags; @@ -531,6 +590,17 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) } } + init_ir_status(iommu); + + if (ir_pre_enabled(iommu)) { + if (iommu_load_old_irte(iommu)) + pr_err("Failed to copy IR table for %s from previous kernel\n", + iommu->name); + else + pr_info("Copied IR table for %s from previous kernel\n", + iommu->name); + } + iommu_set_irq_remapping(iommu, eim_mode); return 0; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index b85b81a..9e14edc 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -296,6 +296,7 @@ struct q_inval { /* 1MB - maximum possible interrupt remapping table size */ #define INTR_REMAP_PAGE_ORDER 8 #define INTR_REMAP_TABLE_REG_SIZE 0xf +#define INTR_REMAP_TABLE_REG_SIZE_MASK 0xf #define INTR_REMAP_TABLE_ENTRIES 65536 -- cgit v0.10.2 From 7c3c9876d98a76b97d16c0f46cb108e95542b212 Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 15:06:26 +0200 Subject: iommu/vt-d: Make sure copied over IR entries are not reused Walk over the copied entries and mark the present ones as allocated. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 2a90121..14e10de 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -451,6 +451,7 @@ static int iommu_load_old_irte(struct intel_iommu *iommu) { struct irte *old_ir_table; phys_addr_t irt_phys; + unsigned int i; size_t size; u64 irta; @@ -481,6 +482,15 @@ static int iommu_load_old_irte(struct intel_iommu *iommu) __iommu_flush_cache(iommu, iommu->ir_table->base, size); + /* + * Now check the table for used entries and mark those as + * allocated in the bitmap + */ + for (i = 0; i < INTR_REMAP_TABLE_ENTRIES; i++) { + if (iommu->ir_table->base[i].present) + bitmap_set(iommu->ir_table->bitmap, i, 1); + } + return 0; } -- cgit v0.10.2 From 571dbbd4d044e11c78bc077acb3ccef4c77b096e Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Fri, 12 Jun 2015 15:15:34 +0200 Subject: iommu/vt-d: Don't disable IR when it was previously enabled Keep it enabled in kdump kernel to guarantee interrupt delivery. Tested-by: ZhenHua Li Tested-by: Baoquan He Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 14e10de..47fcebf 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -742,9 +742,6 @@ static int __init intel_prepare_irq_remapping(void) pr_info("%s does not support EIM\n", iommu->name); eim = 0; } - - /* Disable IRQ remapping if it is already enabled */ - iommu_disable_irq_remapping(iommu); } eim_mode = eim; @@ -777,7 +774,8 @@ static int __init intel_enable_irq_remapping(void) * Setup Interrupt-remapping for all the DRHD's now. */ for_each_iommu(iommu, drhd) { - iommu_enable_irq_remapping(iommu); + if (!ir_pre_enabled(iommu)) + iommu_enable_irq_remapping(iommu); setup = true; } -- cgit v0.10.2 From 0b3fff54bc01e8e6064d222a33e6fa7adabd94cd Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Thu, 18 Jun 2015 10:48:34 +0200 Subject: iommu/amd: Handle large pages correctly in free_pagetable Make sure that we are skipping over large PTEs while walking the page-table tree. Cc: stable@kernel.org Fixes: 5c34c403b723 ("iommu/amd: Fix memory leak in free_pagetable") Signed-off-by: Joerg Roedel diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 4dfadcf..31e90c4 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -1866,9 +1866,15 @@ static void free_pt_##LVL (unsigned long __pt) \ pt = (u64 *)__pt; \ \ for (i = 0; i < 512; ++i) { \ + /* PTE present? */ \ if (!IOMMU_PTE_PRESENT(pt[i])) \ continue; \ \ + /* Large PTE? */ \ + if (PM_PTE_LEVEL(pt[i]) == 0 || \ + PM_PTE_LEVEL(pt[i]) == 7) \ + continue; \ + \ p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ FN(p); \ } \ -- cgit v0.10.2