diff options
Diffstat (limited to 'drivers/iommu')
-rw-r--r-- | drivers/iommu/Kconfig | 86 | ||||
-rw-r--r-- | drivers/iommu/Makefile | 2 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu.c | 192 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_init.c | 206 | ||||
-rw-r--r-- | drivers/iommu/amd_iommu_types.h | 13 | ||||
-rw-r--r-- | drivers/iommu/dmar.c | 31 | ||||
-rw-r--r-- | drivers/iommu/exynos-iommu.c | 4 | ||||
-rw-r--r-- | drivers/iommu/intel-iommu.c | 34 | ||||
-rw-r--r-- | drivers/iommu/intel_irq_remapping.c | 58 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 33 | ||||
-rw-r--r-- | drivers/iommu/irq_remapping.c | 238 | ||||
-rw-r--r-- | drivers/iommu/irq_remapping.h | 3 | ||||
-rw-r--r-- | drivers/iommu/msm_iommu.c | 2 | ||||
-rw-r--r-- | drivers/iommu/omap-iommu.c | 6 | ||||
-rw-r--r-- | drivers/iommu/pci.h | 29 | ||||
-rw-r--r-- | drivers/iommu/shmobile-iommu.c | 395 | ||||
-rw-r--r-- | drivers/iommu/shmobile-ipmmu.c | 136 | ||||
-rw-r--r-- | drivers/iommu/shmobile-ipmmu.h | 34 | ||||
-rw-r--r-- | drivers/iommu/tegra-gart.c | 9 | ||||
-rw-r--r-- | drivers/iommu/tegra-smmu.c | 103 |
20 files changed, 1353 insertions, 261 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 73c0716..951f426 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -85,8 +85,8 @@ config AMD_IOMMU_STATS If unsure, say N. config AMD_IOMMU_V2 - tristate "AMD IOMMU Version 2 driver (EXPERIMENTAL)" - depends on AMD_IOMMU && PROFILING && EXPERIMENTAL + tristate "AMD IOMMU Version 2 driver" + depends on AMD_IOMMU && PROFILING select MMU_NOTIFIER ---help--- This option enables support for the AMD IOMMUv2 features of the IOMMU @@ -139,8 +139,8 @@ config INTEL_IOMMU_FLOPPY_WA 16MiB to make floppy (an ISA device) work. config IRQ_REMAP - bool "Support for Interrupt Remapping (EXPERIMENTAL)" - depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL + bool "Support for Interrupt Remapping" + depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI select DMAR_TABLE ---help--- Supports Interrupt remapping for IO-APIC and MSI devices. @@ -150,7 +150,7 @@ config IRQ_REMAP # OMAP IOMMU support config OMAP_IOMMU bool "OMAP IOMMU Support" - depends on ARCH_OMAP + depends on ARCH_OMAP2PLUS select IOMMU_API config OMAP_IOVMM @@ -178,7 +178,7 @@ config TEGRA_IOMMU_GART config TEGRA_IOMMU_SMMU bool "Tegra SMMU IOMMU Support" - depends on ARCH_TEGRA_3x_SOC && TEGRA_AHB + depends on ARCH_TEGRA && TEGRA_AHB select IOMMU_API help Enables support for remapping discontiguous physical memory @@ -207,4 +207,78 @@ config EXYNOS_IOMMU_DEBUG Say N unless you need kernel log message for IOMMU debugging +config SHMOBILE_IPMMU + bool + +config SHMOBILE_IPMMU_TLB + bool + +config SHMOBILE_IOMMU + bool "IOMMU for Renesas IPMMU/IPMMUI" + default n + depends on (ARM && ARCH_SHMOBILE) + select IOMMU_API + select ARM_DMA_USE_IOMMU + select SHMOBILE_IPMMU + select SHMOBILE_IPMMU_TLB + help + Support for Renesas IPMMU/IPMMUI. This option enables + remapping of DMA memory accesses from all of the IP blocks + on the ICB. + + Warning: Drivers (including userspace drivers of UIO + devices) of the IP blocks on the ICB *must* use addresses + allocated from the IPMMU (iova) for DMA with this option + enabled. + + If unsure, say N. + +choice + prompt "IPMMU/IPMMUI address space size" + default SHMOBILE_IOMMU_ADDRSIZE_2048MB + depends on SHMOBILE_IOMMU + help + This option sets IPMMU/IPMMUI address space size by + adjusting the 1st level page table size. The page table size + is calculated as follows: + + page table size = number of page table entries * 4 bytes + number of page table entries = address space size / 1 MiB + + For example, when the address space size is 2048 MiB, the + 1st level page table size is 8192 bytes. + + config SHMOBILE_IOMMU_ADDRSIZE_2048MB + bool "2 GiB" + + config SHMOBILE_IOMMU_ADDRSIZE_1024MB + bool "1 GiB" + + config SHMOBILE_IOMMU_ADDRSIZE_512MB + bool "512 MiB" + + config SHMOBILE_IOMMU_ADDRSIZE_256MB + bool "256 MiB" + + config SHMOBILE_IOMMU_ADDRSIZE_128MB + bool "128 MiB" + + config SHMOBILE_IOMMU_ADDRSIZE_64MB + bool "64 MiB" + + config SHMOBILE_IOMMU_ADDRSIZE_32MB + bool "32 MiB" + +endchoice + +config SHMOBILE_IOMMU_L1SIZE + int + default 8192 if SHMOBILE_IOMMU_ADDRSIZE_2048MB + default 4096 if SHMOBILE_IOMMU_ADDRSIZE_1024MB + default 2048 if SHMOBILE_IOMMU_ADDRSIZE_512MB + default 1024 if SHMOBILE_IOMMU_ADDRSIZE_256MB + default 512 if SHMOBILE_IOMMU_ADDRSIZE_128MB + default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB + default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 4bc664e..027d1af 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -13,4 +13,6 @@ obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o +obj-$(CONFIG_SHMOBILE_IOMMU) += shmobile-iommu.o +obj-$(CONFIG_SHMOBILE_IPMMU) += shmobile-ipmmu.o obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 44683dd..21d02b0 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -46,6 +46,7 @@ #include "amd_iommu_proto.h" #include "amd_iommu_types.h" #include "irq_remapping.h" +#include "pci.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) @@ -173,7 +174,7 @@ static inline u16 get_device_id(struct device *dev) { struct pci_dev *pdev = to_pci_dev(dev); - return calc_devid(pdev->bus->number, pdev->devfn); + return PCI_DEVID(pdev->bus->number, pdev->devfn); } static struct iommu_dev_data *get_dev_data(struct device *dev) @@ -263,12 +264,6 @@ static bool check_device(struct device *dev) return true; } -static void swap_pci_ref(struct pci_dev **from, struct pci_dev *to) -{ - pci_dev_put(*from); - *from = to; -} - static struct pci_bus *find_hosted_bus(struct pci_bus *bus) { while (!bus->self) { @@ -649,26 +644,26 @@ retry: case EVENT_TYPE_ILL_DEV: printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x " "address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address, flags); dump_dte_entry(devid); break; case EVENT_TYPE_IO_FAULT: printk("IO_PAGE_FAULT device=%02x:%02x.%x " "domain=0x%04x address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), domid, address, flags); break; case EVENT_TYPE_DEV_TAB_ERR: printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x " "address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address, flags); break; case EVENT_TYPE_PAGE_TAB_ERR: printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x " "domain=0x%04x address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), domid, address, flags); break; case EVENT_TYPE_ILL_CMD: @@ -682,13 +677,13 @@ retry: case EVENT_TYPE_IOTLB_INV_TO: printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x " "address=0x%016llx]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address); break; case EVENT_TYPE_INV_DEV_REQ: printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x " "address=0x%016llx flags=0x%04x]\n", - PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), address, flags); break; default: @@ -701,9 +696,6 @@ retry: static void iommu_poll_events(struct amd_iommu *iommu) { u32 head, tail; - unsigned long flags; - - spin_lock_irqsave(&iommu->lock, flags); head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); @@ -714,8 +706,6 @@ static void iommu_poll_events(struct amd_iommu *iommu) } writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); - - spin_unlock_irqrestore(&iommu->lock, flags); } static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw) @@ -740,17 +730,11 @@ static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u64 *raw) static void iommu_poll_ppr_log(struct amd_iommu *iommu) { - unsigned long flags; u32 head, tail; if (iommu->ppr_log == NULL) return; - /* enable ppr interrupts again */ - writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET); - - spin_lock_irqsave(&iommu->lock, flags); - head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); @@ -786,34 +770,50 @@ static void iommu_poll_ppr_log(struct amd_iommu *iommu) head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE; writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); - /* - * Release iommu->lock because ppr-handling might need to - * re-acquire it - */ - spin_unlock_irqrestore(&iommu->lock, flags); - /* Handle PPR entry */ iommu_handle_ppr_entry(iommu, entry); - spin_lock_irqsave(&iommu->lock, flags); - /* Refresh ring-buffer information */ head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); } - - spin_unlock_irqrestore(&iommu->lock, flags); } irqreturn_t amd_iommu_int_thread(int irq, void *data) { - struct amd_iommu *iommu; + struct amd_iommu *iommu = (struct amd_iommu *) data; + u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); - for_each_iommu(iommu) { - iommu_poll_events(iommu); - iommu_poll_ppr_log(iommu); - } + while (status & (MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK)) { + /* Enable EVT and PPR interrupts again */ + writel((MMIO_STATUS_EVT_INT_MASK | MMIO_STATUS_PPR_INT_MASK), + iommu->mmio_base + MMIO_STATUS_OFFSET); + + if (status & MMIO_STATUS_EVT_INT_MASK) { + pr_devel("AMD-Vi: Processing IOMMU Event Log\n"); + iommu_poll_events(iommu); + } + + if (status & MMIO_STATUS_PPR_INT_MASK) { + pr_devel("AMD-Vi: Processing IOMMU PPR Log\n"); + iommu_poll_ppr_log(iommu); + } + /* + * Hardware bug: ERBT1312 + * When re-enabling interrupt (by writing 1 + * to clear the bit), the hardware might also try to set + * the interrupt bit in the event status register. + * In this scenario, the bit will be set, and disable + * subsequent interrupts. + * + * Workaround: The IOMMU driver should read back the + * status register and check if the interrupt bits are cleared. + * If not, driver will need to go through the interrupt handler + * again and re-clear the bits + */ + status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); + } return IRQ_HANDLED; } @@ -2466,18 +2466,16 @@ static int device_change_notifier(struct notifier_block *nb, /* allocate a protection domain if a device is added */ dma_domain = find_protection_domain(devid); - if (dma_domain) - goto out; - dma_domain = dma_ops_domain_alloc(); - if (!dma_domain) - goto out; - dma_domain->target_dev = devid; - - spin_lock_irqsave(&iommu_pd_list_lock, flags); - list_add_tail(&dma_domain->list, &iommu_pd_list); - spin_unlock_irqrestore(&iommu_pd_list_lock, flags); - - dev_data = get_dev_data(dev); + if (!dma_domain) { + dma_domain = dma_ops_domain_alloc(); + if (!dma_domain) + goto out; + dma_domain->target_dev = devid; + + spin_lock_irqsave(&iommu_pd_list_lock, flags); + list_add_tail(&dma_domain->list, &iommu_pd_list); + spin_unlock_irqrestore(&iommu_pd_list_lock, flags); + } dev->archdata.dma_ops = &amd_iommu_dma_ops; @@ -2841,24 +2839,6 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, } /* - * This is a special map_sg function which is used if we should map a - * device which is not handled by an AMD IOMMU in the system. - */ -static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist, - int nelems, int dir) -{ - struct scatterlist *s; - int i; - - for_each_sg(sglist, s, nelems, i) { - s->dma_address = (dma_addr_t)sg_phys(s); - s->dma_length = s->length; - } - - return nelems; -} - -/* * The exported map_sg function for dma_ops (handles scatter-gather * lists). */ @@ -2877,9 +2857,7 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, INC_STATS_COUNTER(cnt_map_sg); domain = get_domain(dev); - if (PTR_ERR(domain) == -EINVAL) - return map_sg_no_iommu(dev, sglist, nelems, dir); - else if (IS_ERR(domain)) + if (IS_ERR(domain)) return 0; dma_mask = *dev->dma_mask; @@ -3187,8 +3165,7 @@ int __init amd_iommu_init_dma_ops(void) free_domains: for_each_iommu(iommu) { - if (iommu->default_dom) - dma_ops_domain_free(iommu->default_dom); + dma_ops_domain_free(iommu->default_dom); } return ret; @@ -3413,7 +3390,7 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, } static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom, - u64 iova) + dma_addr_t iova) { struct protection_domain *domain = dom->priv; unsigned long offset_mask; @@ -3950,6 +3927,9 @@ static struct irq_remap_table *get_irq_table(u16 devid, bool ioapic) if (!table) goto out; + /* Initialize table spin-lock */ + spin_lock_init(&table->lock); + if (ioapic) /* Keep the first 32 indexes free for IOAPIC interrupts */ table->min_index = 32; @@ -4010,17 +3990,17 @@ static int alloc_irq_index(struct irq_cfg *cfg, u16 devid, int count) c = 0; if (c == count) { - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; for (; c != 0; --c) table->table[index - c + 1] = IRTE_ALLOCATED; index -= count - 1; - irte_info = &cfg->irq_2_iommu; - irte_info->sub_handle = devid; - irte_info->irte_index = index; - irte_info->iommu = (void *)cfg; + cfg->remapped = 1; + irte_info = &cfg->irq_2_irte; + irte_info->devid = devid; + irte_info->index = index; goto out; } @@ -4101,7 +4081,7 @@ static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry, struct io_apic_irq_attr *attr) { struct irq_remap_table *table; - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; struct irq_cfg *cfg; union irte irte; int ioapic_id; @@ -4113,7 +4093,7 @@ static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry, if (!cfg) return -EINVAL; - irte_info = &cfg->irq_2_iommu; + irte_info = &cfg->irq_2_irte; ioapic_id = mpc_ioapic_id(attr->ioapic); devid = get_ioapic_devid(ioapic_id); @@ -4127,9 +4107,9 @@ static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry, index = attr->ioapic_pin; /* Setup IRQ remapping info */ - irte_info->sub_handle = devid; - irte_info->irte_index = index; - irte_info->iommu = (void *)cfg; + cfg->remapped = 1; + irte_info->devid = devid; + irte_info->index = index; /* Setup IRTE for IOMMU */ irte.val = 0; @@ -4163,7 +4143,7 @@ static int setup_ioapic_entry(int irq, struct IO_APIC_route_entry *entry, static int set_affinity(struct irq_data *data, const struct cpumask *mask, bool force) { - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; unsigned int dest, irq; struct irq_cfg *cfg; union irte irte; @@ -4174,12 +4154,12 @@ static int set_affinity(struct irq_data *data, const struct cpumask *mask, cfg = data->chip_data; irq = data->irq; - irte_info = &cfg->irq_2_iommu; + irte_info = &cfg->irq_2_irte; if (!cpumask_intersects(mask, cpu_online_mask)) return -EINVAL; - if (get_irte(irte_info->sub_handle, irte_info->irte_index, &irte)) + if (get_irte(irte_info->devid, irte_info->index, &irte)) return -EBUSY; if (assign_irq_vector(irq, cfg, mask)) @@ -4195,7 +4175,7 @@ static int set_affinity(struct irq_data *data, const struct cpumask *mask, irte.fields.vector = cfg->vector; irte.fields.destination = dest; - modify_irte(irte_info->sub_handle, irte_info->irte_index, irte); + modify_irte(irte_info->devid, irte_info->index, irte); if (cfg->move_in_progress) send_cleanup_vector(cfg); @@ -4207,16 +4187,16 @@ static int set_affinity(struct irq_data *data, const struct cpumask *mask, static int free_irq(int irq) { - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; struct irq_cfg *cfg; cfg = irq_get_chip_data(irq); if (!cfg) return -EINVAL; - irte_info = &cfg->irq_2_iommu; + irte_info = &cfg->irq_2_irte; - free_irte(irte_info->sub_handle, irte_info->irte_index); + free_irte(irte_info->devid, irte_info->index); return 0; } @@ -4225,7 +4205,7 @@ static void compose_msi_msg(struct pci_dev *pdev, unsigned int irq, unsigned int dest, struct msi_msg *msg, u8 hpet_id) { - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; struct irq_cfg *cfg; union irte irte; @@ -4233,7 +4213,7 @@ static void compose_msi_msg(struct pci_dev *pdev, if (!cfg) return; - irte_info = &cfg->irq_2_iommu; + irte_info = &cfg->irq_2_irte; irte.val = 0; irte.fields.vector = cfg->vector; @@ -4242,11 +4222,11 @@ static void compose_msi_msg(struct pci_dev *pdev, irte.fields.dm = apic->irq_dest_mode; irte.fields.valid = 1; - modify_irte(irte_info->sub_handle, irte_info->irte_index, irte); + modify_irte(irte_info->devid, irte_info->index, irte); msg->address_hi = MSI_ADDR_BASE_HI; msg->address_lo = MSI_ADDR_BASE_LO; - msg->data = irte_info->irte_index; + msg->data = irte_info->index; } static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec) @@ -4271,7 +4251,7 @@ static int msi_alloc_irq(struct pci_dev *pdev, int irq, int nvec) static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq, int index, int offset) { - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; struct irq_cfg *cfg; u16 devid; @@ -4286,18 +4266,18 @@ static int msi_setup_irq(struct pci_dev *pdev, unsigned int irq, return 0; devid = get_device_id(&pdev->dev); - irte_info = &cfg->irq_2_iommu; + irte_info = &cfg->irq_2_irte; - irte_info->sub_handle = devid; - irte_info->irte_index = index + offset; - irte_info->iommu = (void *)cfg; + cfg->remapped = 1; + irte_info->devid = devid; + irte_info->index = index + offset; return 0; } static int setup_hpet_msi(unsigned int irq, unsigned int id) { - struct irq_2_iommu *irte_info; + struct irq_2_irte *irte_info; struct irq_cfg *cfg; int index, devid; @@ -4305,7 +4285,7 @@ static int setup_hpet_msi(unsigned int irq, unsigned int id) if (!cfg) return -EINVAL; - irte_info = &cfg->irq_2_iommu; + irte_info = &cfg->irq_2_irte; devid = get_hpet_devid(id); if (devid < 0) return devid; @@ -4314,9 +4294,9 @@ static int setup_hpet_msi(unsigned int irq, unsigned int id) if (index < 0) return index; - irte_info->sub_handle = devid; - irte_info->irte_index = index; - irte_info->iommu = (void *)cfg; + cfg->remapped = 1; + irte_info->devid = devid; + irte_info->index = index; return 0; } diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index faf10ba..bf51abb 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -213,6 +213,14 @@ enum iommu_init_state { IOMMU_INIT_ERROR, }; +/* Early ioapic and hpet maps from kernel command line */ +#define EARLY_MAP_SIZE 4 +static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; +static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; +static int __initdata early_ioapic_map_size; +static int __initdata early_hpet_map_size; +static bool __initdata cmdline_maps; + static enum iommu_init_state init_state = IOMMU_START_STATE; static int amd_iommu_enable_interrupts(void); @@ -406,7 +414,7 @@ static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr) u32 cap; cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET); - update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap))); + update_last_devid(PCI_DEVID(MMIO_GET_BUS(cap), MMIO_GET_LD(cap))); return 0; } @@ -423,7 +431,7 @@ static int __init find_last_devid_from_ivhd(struct ivhd_header *h) p += sizeof(*h); end += h->length; - find_last_devid_on_pci(PCI_BUS(h->devid), + find_last_devid_on_pci(PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid), PCI_FUNC(h->devid), h->cap_ptr); @@ -703,31 +711,66 @@ static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, set_iommu_for_device(iommu, devid); } -static int add_special_device(u8 type, u8 id, u16 devid) +static int __init add_special_device(u8 type, u8 id, u16 devid, bool cmd_line) { struct devid_map *entry; struct list_head *list; - if (type != IVHD_SPECIAL_IOAPIC && type != IVHD_SPECIAL_HPET) + if (type == IVHD_SPECIAL_IOAPIC) + list = &ioapic_map; + else if (type == IVHD_SPECIAL_HPET) + list = &hpet_map; + else return -EINVAL; + list_for_each_entry(entry, list, list) { + if (!(entry->id == id && entry->cmd_line)) + continue; + + pr_info("AMD-Vi: Command-line override present for %s id %d - ignoring\n", + type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); + + return 0; + } + entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return -ENOMEM; - entry->id = id; - entry->devid = devid; - - if (type == IVHD_SPECIAL_IOAPIC) - list = &ioapic_map; - else - list = &hpet_map; + entry->id = id; + entry->devid = devid; + entry->cmd_line = cmd_line; list_add_tail(&entry->list, list); return 0; } +static int __init add_early_maps(void) +{ + int i, ret; + + for (i = 0; i < early_ioapic_map_size; ++i) { + ret = add_special_device(IVHD_SPECIAL_IOAPIC, + early_ioapic_map[i].id, + early_ioapic_map[i].devid, + early_ioapic_map[i].cmd_line); + if (ret) + return ret; + } + + for (i = 0; i < early_hpet_map_size; ++i) { + ret = add_special_device(IVHD_SPECIAL_HPET, + early_hpet_map[i].id, + early_hpet_map[i].devid, + early_hpet_map[i].cmd_line); + if (ret) + return ret; + } + + return 0; +} + /* * Reads the device exclusion range from ACPI and initializes the IOMMU with * it @@ -764,6 +807,12 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, u32 dev_i, ext_flags = 0; bool alias = false; struct ivhd_entry *e; + int ret; + + + ret = add_early_maps(); + if (ret) + return ret; /* * First save the recommended feature enable bits from ACPI @@ -784,10 +833,10 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x" " last device %02x:%02x.%x flags: %02x\n", - PCI_BUS(iommu->first_device), + PCI_BUS_NUM(iommu->first_device), PCI_SLOT(iommu->first_device), PCI_FUNC(iommu->first_device), - PCI_BUS(iommu->last_device), + PCI_BUS_NUM(iommu->last_device), PCI_SLOT(iommu->last_device), PCI_FUNC(iommu->last_device), e->flags); @@ -801,7 +850,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x " "flags: %02x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags); @@ -813,7 +862,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_SELECT_RANGE_START\t " "devid: %02x:%02x.%x flags: %02x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags); @@ -827,11 +876,11 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x " "flags: %02x devid_to: %02x:%02x.%x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, - PCI_BUS(e->ext >> 8), + PCI_BUS_NUM(e->ext >> 8), PCI_SLOT(e->ext >> 8), PCI_FUNC(e->ext >> 8)); @@ -846,11 +895,11 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_ALIAS_RANGE\t\t " "devid: %02x:%02x.%x flags: %02x " "devid_to: %02x:%02x.%x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, - PCI_BUS(e->ext >> 8), + PCI_BUS_NUM(e->ext >> 8), PCI_SLOT(e->ext >> 8), PCI_FUNC(e->ext >> 8)); @@ -864,7 +913,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x " "flags: %02x ext: %08x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, e->ext); @@ -877,7 +926,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: " "%02x:%02x.%x flags: %02x ext: %08x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid), e->flags, e->ext); @@ -890,7 +939,7 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, case IVHD_DEV_RANGE_END: DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", - PCI_BUS(e->devid), + PCI_BUS_NUM(e->devid), PCI_SLOT(e->devid), PCI_FUNC(e->devid)); @@ -924,12 +973,12 @@ static int __init init_iommu_from_acpi(struct amd_iommu *iommu, DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n", var, (int)handle, - PCI_BUS(devid), + PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid)); set_dev_entry_from_acpi(iommu, devid, e->flags, 0); - ret = add_special_device(type, handle, devid); + ret = add_special_device(type, handle, devid, false); if (ret) return ret; break; @@ -980,7 +1029,7 @@ static void __init free_iommu_all(void) * BIOS should disable L2B micellaneous clock gating by setting * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b */ -static void __init amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) +static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) { u32 value; @@ -1086,7 +1135,7 @@ static int __init init_iommu_all(struct acpi_table_header *table) DUMP_printk("device: %02x:%02x.%01x cap: %04x " "seg: %d flags: %01x info %04x\n", - PCI_BUS(h->devid), PCI_SLOT(h->devid), + PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid), PCI_FUNC(h->devid), h->cap_ptr, h->pci_seg, h->flags, h->info); DUMP_printk(" mmio-addr: %016llx\n", @@ -1116,7 +1165,7 @@ static int iommu_init_pci(struct amd_iommu *iommu) int cap_ptr = iommu->cap_ptr; u32 range, misc, low, high; - iommu->dev = pci_get_bus_and_slot(PCI_BUS(iommu->devid), + iommu->dev = pci_get_bus_and_slot(PCI_BUS_NUM(iommu->devid), iommu->devid & 0xff); if (!iommu->dev) return -ENODEV; @@ -1128,9 +1177,9 @@ static int iommu_init_pci(struct amd_iommu *iommu) pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET, &misc); - iommu->first_device = calc_devid(MMIO_GET_BUS(range), + iommu->first_device = PCI_DEVID(MMIO_GET_BUS(range), MMIO_GET_FD(range)); - iommu->last_device = calc_devid(MMIO_GET_BUS(range), + iommu->last_device = PCI_DEVID(MMIO_GET_BUS(range), MMIO_GET_LD(range)); if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) @@ -1275,7 +1324,7 @@ static int iommu_setup_msi(struct amd_iommu *iommu) amd_iommu_int_handler, amd_iommu_int_thread, 0, "AMD-Vi", - iommu->dev); + iommu); if (r) { pci_disable_msi(iommu->dev); @@ -1388,8 +1437,8 @@ static int __init init_unity_map_range(struct ivmd_header *m) DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x" " range_start: %016llx range_end: %016llx flags: %x\n", s, - PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start), - PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end), + PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start), + PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end), PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), e->address_start, e->address_end, m->flags); @@ -1638,18 +1687,28 @@ static void __init free_on_init_error(void) static bool __init check_ioapic_information(void) { + const char *fw_bug = FW_BUG; bool ret, has_sb_ioapic; int idx; has_sb_ioapic = false; ret = false; + /* + * If we have map overrides on the kernel command line the + * messages in this function might not describe firmware bugs + * anymore - so be careful + */ + if (cmdline_maps) + fw_bug = ""; + for (idx = 0; idx < nr_ioapics; idx++) { int devid, id = mpc_ioapic_id(idx); devid = get_ioapic_devid(id); if (devid < 0) { - pr_err(FW_BUG "AMD-Vi: IOAPIC[%d] not in IVRS table\n", id); + pr_err("%sAMD-Vi: IOAPIC[%d] not in IVRS table\n", + fw_bug, id); ret = false; } else if (devid == IOAPIC_SB_DEVID) { has_sb_ioapic = true; @@ -1666,11 +1725,11 @@ static bool __init check_ioapic_information(void) * when the BIOS is buggy and provides us the wrong * device id for the IOAPIC in the system. */ - pr_err(FW_BUG "AMD-Vi: No southbridge IOAPIC found in IVRS table\n"); + pr_err("%sAMD-Vi: No southbridge IOAPIC found\n", fw_bug); } if (!ret) - pr_err("AMD-Vi: Disabling interrupt remapping due to BIOS Bug(s)\n"); + pr_err("AMD-Vi: Disabling interrupt remapping\n"); return ret; } @@ -1801,6 +1860,7 @@ static int __init early_amd_iommu_init(void) * Interrupt remapping enabled, create kmem_cache for the * remapping tables. */ + ret = -ENOMEM; amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache", MAX_IRQS_PER_TABLE * sizeof(u32), IRQ_TABLE_ALIGNMENT, @@ -1876,11 +1936,6 @@ static int amd_iommu_init_dma(void) struct amd_iommu *iommu; int ret; - init_device_table_dma(); - - for_each_iommu(iommu) - iommu_flush_all_caches(iommu); - if (iommu_pass_through) ret = amd_iommu_init_passthrough(); else @@ -1889,6 +1944,11 @@ static int amd_iommu_init_dma(void) if (ret) return ret; + init_device_table_dma(); + + for_each_iommu(iommu) + iommu_flush_all_caches(iommu); + amd_iommu_init_api(); amd_iommu_init_notifier(); @@ -2097,8 +2157,70 @@ static int __init parse_amd_iommu_options(char *str) return 1; } -__setup("amd_iommu_dump", parse_amd_iommu_dump); -__setup("amd_iommu=", parse_amd_iommu_options); +static int __init parse_ivrs_ioapic(char *str) +{ + unsigned int bus, dev, fn; + int ret, id, i; + u16 devid; + + ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); + + if (ret != 4) { + pr_err("AMD-Vi: Invalid command line: ivrs_ioapic%s\n", str); + return 1; + } + + if (early_ioapic_map_size == EARLY_MAP_SIZE) { + pr_err("AMD-Vi: Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", + str); + return 1; + } + + devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); + + cmdline_maps = true; + i = early_ioapic_map_size++; + early_ioapic_map[i].id = id; + early_ioapic_map[i].devid = devid; + early_ioapic_map[i].cmd_line = true; + + return 1; +} + +static int __init parse_ivrs_hpet(char *str) +{ + unsigned int bus, dev, fn; + int ret, id, i; + u16 devid; + + ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); + + if (ret != 4) { + pr_err("AMD-Vi: Invalid command line: ivrs_hpet%s\n", str); + return 1; + } + + if (early_hpet_map_size == EARLY_MAP_SIZE) { + pr_err("AMD-Vi: Early HPET map overflow - ignoring ivrs_hpet%s\n", + str); + return 1; + } + + devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); + + cmdline_maps = true; + i = early_hpet_map_size++; + early_hpet_map[i].id = id; + early_hpet_map[i].devid = devid; + early_hpet_map[i].cmd_line = true; + + return 1; +} + +__setup("amd_iommu_dump", parse_amd_iommu_dump); +__setup("amd_iommu=", parse_amd_iommu_options); +__setup("ivrs_ioapic", parse_ivrs_ioapic); +__setup("ivrs_hpet", parse_ivrs_hpet); IOMMU_INIT_FINISH(amd_iommu_detect, gart_iommu_hole_init, diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index e38ab43..0285a21 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -24,6 +24,7 @@ #include <linux/mutex.h> #include <linux/list.h> #include <linux/spinlock.h> +#include <linux/pci.h> /* * Maximum number of IOMMUs supported @@ -99,6 +100,7 @@ #define PASID_MASK 0x000fffff /* MMIO status bits */ +#define MMIO_STATUS_EVT_INT_MASK (1 << 1) #define MMIO_STATUS_COM_WAIT_INT_MASK (1 << 2) #define MMIO_STATUS_PPR_INT_MASK (1 << 6) @@ -315,9 +317,6 @@ #define MAX_DOMAIN_ID 65536 -/* FIXME: move this macro to <linux/pci.h> */ -#define PCI_BUS(x) (((x) >> 8) & 0xff) - /* Protection domain flags */ #define PD_DMA_OPS_MASK (1UL << 0) /* domain used for dma_ops */ #define PD_DEFAULT_MASK (1UL << 1) /* domain is a default dma_ops @@ -591,6 +590,7 @@ struct devid_map { struct list_head list; u8 id; u16 devid; + bool cmd_line; }; /* Map HPET and IOAPIC ids to the devid used by the IOMMU */ @@ -703,13 +703,6 @@ extern int amd_iommu_max_glx_val; */ extern void iommu_flush_all_caches(struct amd_iommu *iommu); -/* takes bus and device/function and returns the device id - * FIXME: should that be in generic PCI code? */ -static inline u16 calc_devid(u8 bus, u8 devfn) -{ - return (((u16)bus) << 8) | devfn; -} - static inline int get_ioapic_devid(int id) { struct devid_map *entry; diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 86e2f4a..a7967ce 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -41,6 +41,8 @@ #include <asm/irq_remapping.h> #include <asm/iommu_table.h> +#include "irq_remapping.h" + /* No locks are needed as DMA remapping hardware unit * list is constructed at boot time and hotplug of * these units are not supported by the architecture. @@ -127,7 +129,8 @@ int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT || scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) (*cnt)++; - else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) { + else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC && + scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) { pr_warn("Unsupported device scope\n"); } start += scope->length; @@ -643,7 +646,7 @@ out: int alloc_iommu(struct dmar_drhd_unit *drhd) { struct intel_iommu *iommu; - u32 ver; + u32 ver, sts; static int iommu_allocated = 0; int agaw = 0; int msagaw = 0; @@ -693,6 +696,15 @@ int alloc_iommu(struct dmar_drhd_unit *drhd) (unsigned long long)iommu->cap, (unsigned long long)iommu->ecap); + /* Reflect status in gcmd */ + sts = readl(iommu->reg + DMAR_GSTS_REG); + if (sts & DMA_GSTS_IRES) + iommu->gcmd |= DMA_GCMD_IRE; + if (sts & DMA_GSTS_TES) + iommu->gcmd |= DMA_GCMD_TE; + if (sts & DMA_GSTS_QIES) + iommu->gcmd |= DMA_GCMD_QIE; + raw_spin_lock_init(&iommu->register_lock); drhd->iommu = iommu; @@ -1040,7 +1052,7 @@ int dmar_enable_qi(struct intel_iommu *iommu) qi->desc = page_address(desc_page); - qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC); + qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC); if (!qi->desc_status) { free_page((unsigned long) qi->desc); kfree(qi); @@ -1081,6 +1093,7 @@ static const char *dma_remap_fault_reasons[] = "non-zero reserved fields in RTP", "non-zero reserved fields in CTP", "non-zero reserved fields in PTE", + "PCE for translation request specifies blocking", }; static const char *irq_remap_fault_reasons[] = @@ -1201,7 +1214,7 @@ irqreturn_t dmar_fault(int irq, void *dev_id) /* TBD: ignore advanced fault log currently */ if (!(fault_status & DMA_FSTS_PPF)) - goto clear_rest; + goto unlock_exit; fault_index = dma_fsts_fault_record_index(fault_status); reg = cap_fault_reg_offset(iommu->cap); @@ -1242,11 +1255,10 @@ irqreturn_t dmar_fault(int irq, void *dev_id) fault_index = 0; raw_spin_lock_irqsave(&iommu->register_lock, flag); } -clear_rest: - /* clear all the other faults */ - fault_status = readl(iommu->reg + DMAR_FSTS_REG); - writel(fault_status, iommu->reg + DMAR_FSTS_REG); + writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG); + +unlock_exit: raw_spin_unlock_irqrestore(&iommu->register_lock, flag); return IRQ_HANDLED; } @@ -1294,6 +1306,7 @@ int __init enable_drhd_fault_handling(void) for_each_drhd_unit(drhd) { int ret; struct intel_iommu *iommu = drhd->iommu; + u32 fault_status; ret = dmar_set_interrupt(iommu); if (ret) { @@ -1306,6 +1319,8 @@ int __init enable_drhd_fault_handling(void) * Clear any previous faults. */ dmar_fault(iommu->irq, iommu); + fault_status = readl(iommu->reg + DMAR_FSTS_REG); + writel(fault_status, iommu->reg + DMAR_FSTS_REG); } return 0; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index a8c2948..3f32d64 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -511,7 +511,7 @@ int exynos_sysmmu_enable(struct device *dev, unsigned long pgtable) return ret; } -bool exynos_sysmmu_disable(struct device *dev) +static bool exynos_sysmmu_disable(struct device *dev) { struct sysmmu_drvdata *data = dev_get_drvdata(dev->archdata.iommu); bool disabled; @@ -1027,7 +1027,7 @@ done: } static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *domain, - u64 iova) + dma_addr_t iova) { struct exynos_iommu_domain *priv = domain->priv; unsigned long *entry; diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 0da9693..b4f0e28 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -46,6 +46,9 @@ #include <asm/cacheflush.h> #include <asm/iommu.h> +#include "irq_remapping.h" +#include "pci.h" + #define ROOT_SIZE VTD_PAGE_SIZE #define CONTEXT_SIZE VTD_PAGE_SIZE @@ -3663,6 +3666,7 @@ static struct notifier_block device_nb = { int __init intel_iommu_init(void) { int ret = 0; + struct dmar_drhd_unit *drhd; /* VT-d is required for a TXT/tboot launch, so enforce that */ force_on = tboot_force_iommu(); @@ -3673,6 +3677,20 @@ int __init intel_iommu_init(void) return -ENODEV; } + /* + * Disable translation if already enabled prior to OS handover. + */ + for_each_drhd_unit(drhd) { + struct intel_iommu *iommu; + + if (drhd->ignored) + continue; + + iommu = drhd->iommu; + if (iommu->gcmd & DMA_GCMD_TE) + iommu_disable_translation(iommu); + } + if (dmar_dev_scope_init() < 0) { if (force_on) panic("tboot: Failed to initialize DMAR device scope\n"); @@ -4109,7 +4127,7 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, } static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain, - u64 iova) + dma_addr_t iova) { struct dmar_domain *dmar_domain = domain->priv; struct dma_pte *pte; @@ -4135,12 +4153,6 @@ static int intel_iommu_domain_has_cap(struct iommu_domain *domain, return 0; } -static void swap_pci_ref(struct pci_dev **from, struct pci_dev *to) -{ - pci_dev_put(*from); - *from = to; -} - #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF) static int intel_iommu_add_device(struct device *dev) @@ -4253,13 +4265,19 @@ static void quirk_iommu_rwbf(struct pci_dev *dev) { /* * Mobile 4 Series Chipset neglects to set RWBF capability, - * but needs it: + * but needs it. Same seems to hold for the desktop versions. */ printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n"); rwbf_quirk = 1; } DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf); +DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf); #define GGC 0x52 #define GGC_MEMORY_SIZE_MASK (0xf << 8) diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index af8904d..5b19b2d 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -68,6 +68,7 @@ static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) { struct ir_table *table = iommu->ir_table; struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + struct irq_cfg *cfg = irq_get_chip_data(irq); u16 index, start_index; unsigned int mask = 0; unsigned long flags; @@ -115,6 +116,7 @@ static int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) for (i = index; i < index + count; i++) table->base[i].present = 1; + cfg->remapped = 1; irq_iommu->iommu = iommu; irq_iommu->irte_index = index; irq_iommu->sub_handle = 0; @@ -155,6 +157,7 @@ static int map_irq_to_irte_handle(int irq, u16 *sub_handle) static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) { struct irq_2_iommu *irq_iommu = irq_2_iommu(irq); + struct irq_cfg *cfg = irq_get_chip_data(irq); unsigned long flags; if (!irq_iommu) @@ -162,6 +165,7 @@ static int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subha raw_spin_lock_irqsave(&irq_2_ir_lock, flags); + cfg->remapped = 1; irq_iommu->iommu = iommu; irq_iommu->irte_index = index; irq_iommu->sub_handle = subhandle; @@ -425,11 +429,22 @@ static void iommu_set_irq_remapping(struct intel_iommu *iommu, int mode) /* Enable interrupt-remapping */ iommu->gcmd |= DMA_GCMD_IRE; + iommu->gcmd &= ~DMA_GCMD_CFI; /* Block compatibility-format MSIs */ writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG); IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_IRES), sts); + /* + * With CFI clear in the Global Command register, we should be + * protected from dangerous (i.e. compatibility) interrupts + * regardless of x2apic status. Check just to be sure. + */ + if (sts & DMA_GSTS_CFIS) + WARN(1, KERN_WARNING + "Compatibility-format IRQs enabled despite intr remapping;\n" + "you are vulnerable to IRQ injection.\n"); + raw_spin_unlock_irqrestore(&iommu->register_lock, flags); } @@ -509,6 +524,16 @@ static int __init intel_irq_remapping_supported(void) if (disable_irq_remap) return 0; + if (irq_remap_broken) { + WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND, + "This system BIOS has enabled interrupt remapping\n" + "on a chipset that contains an erratum making that\n" + "feature unstable. To maintain system stability\n" + "interrupt remapping is being disabled. Please\n" + "contact your BIOS vendor for an update\n"); + disable_irq_remap = 1; + return 0; + } if (!dmar_ir_support()) return 0; @@ -526,20 +551,24 @@ static int __init intel_irq_remapping_supported(void) static int __init intel_enable_irq_remapping(void) { struct dmar_drhd_unit *drhd; + bool x2apic_present; int setup = 0; int eim = 0; + x2apic_present = x2apic_supported(); + if (parse_ioapics_under_ir() != 1) { printk(KERN_INFO "Not enable interrupt remapping\n"); - return -1; + goto error; } - if (x2apic_supported()) { + if (x2apic_present) { eim = !dmar_x2apic_optout(); - WARN(!eim, KERN_WARNING - "Your BIOS is broken and requested that x2apic be disabled\n" - "This will leave your machine vulnerable to irq-injection attacks\n" - "Use 'intremap=no_x2apic_optout' to override BIOS request\n"); + if (!eim) + printk(KERN_WARNING + "Your BIOS is broken and requested that x2apic be disabled.\n" + "This will slightly decrease performance.\n" + "Use 'intremap=no_x2apic_optout' to override BIOS request.\n"); } for_each_drhd_unit(drhd) { @@ -578,7 +607,7 @@ static int __init intel_enable_irq_remapping(void) if (eim && !ecap_eim_support(iommu->ecap)) { printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, " " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap); - return -1; + goto error; } } @@ -594,7 +623,7 @@ static int __init intel_enable_irq_remapping(void) printk(KERN_ERR "DRHD %Lx: failed to enable queued, " " invalidation, ecap %Lx, ret %d\n", drhd->reg_base_addr, iommu->ecap, ret); - return -1; + goto error; } } @@ -617,6 +646,14 @@ static int __init intel_enable_irq_remapping(void) goto error; irq_remapping_enabled = 1; + + /* + * VT-d has a different layout for IO-APIC entries when + * interrupt remapping is enabled. So it needs a special routine + * to print IO-APIC entries for debugging purposes too. + */ + x86_io_apic_ops.print_entries = intel_ir_io_apic_print_entries; + pr_info("Enabled IRQ remapping in %s mode\n", eim ? "x2apic" : "xapic"); return eim ? IRQ_REMAP_X2APIC_MODE : IRQ_REMAP_XAPIC_MODE; @@ -625,6 +662,11 @@ error: /* * handle error condition gracefully here! */ + + if (x2apic_present) + WARN(1, KERN_WARNING + "Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); + return -1; } diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d1317b0..1717402 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -204,6 +204,35 @@ again: } EXPORT_SYMBOL_GPL(iommu_group_alloc); +struct iommu_group *iommu_group_get_by_id(int id) +{ + struct kobject *group_kobj; + struct iommu_group *group; + const char *name; + + if (!iommu_group_kset) + return NULL; + + name = kasprintf(GFP_KERNEL, "%d", id); + if (!name) + return NULL; + + group_kobj = kset_find_obj(iommu_group_kset, name); + kfree(name); + + if (!group_kobj) + return NULL; + + group = container_of(group_kobj, struct iommu_group, kobj); + BUG_ON(group->id != id); + + kobject_get(group->devices_kobj); + kobject_put(&group->kobj); + + return group; +} +EXPORT_SYMBOL_GPL(iommu_group_get_by_id); + /** * iommu_group_get_iommudata - retrieve iommu_data registered for a group * @group: the group @@ -863,13 +892,13 @@ EXPORT_SYMBOL_GPL(iommu_unmap); int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr, - phys_addr_t paddr, u64 size, int iommu_prot) + phys_addr_t paddr, u64 size, int prot) { if (unlikely(domain->ops->domain_window_enable == NULL)) return -ENODEV; return domain->ops->domain_window_enable(domain, wnd_nr, paddr, size, - iommu_prot); + prot); } EXPORT_SYMBOL_GPL(iommu_domain_window_enable); diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index faf85d6..dcfea4e 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -1,22 +1,175 @@ +#include <linux/seq_file.h> +#include <linux/cpumask.h> #include <linux/kernel.h> #include <linux/string.h> -#include <linux/cpumask.h> #include <linux/errno.h> #include <linux/msi.h> +#include <linux/irq.h> +#include <linux/pci.h> #include <asm/hw_irq.h> #include <asm/irq_remapping.h> +#include <asm/processor.h> +#include <asm/x86_init.h> +#include <asm/apic.h> #include "irq_remapping.h" int irq_remapping_enabled; int disable_irq_remap; +int irq_remap_broken; int disable_sourceid_checking; int no_x2apic_optout; static struct irq_remap_ops *remap_ops; +static int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec); +static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle); +static int set_remapped_irq_affinity(struct irq_data *data, + const struct cpumask *mask, + bool force); + +static bool irq_remapped(struct irq_cfg *cfg) +{ + return (cfg->remapped == 1); +} + +static void irq_remapping_disable_io_apic(void) +{ + /* + * With interrupt-remapping, for now we will use virtual wire A + * mode, as virtual wire B is little complex (need to configure + * both IOAPIC RTE as well as interrupt-remapping table entry). + * As this gets called during crash dump, keep this simple for + * now. + */ + if (cpu_has_apic || apic_from_smp_config()) + disconnect_bsp_APIC(0); +} + +static int do_setup_msi_irqs(struct pci_dev *dev, int nvec) +{ + int node, ret, sub_handle, index = 0; + unsigned int irq; + struct msi_desc *msidesc; + + nvec = __roundup_pow_of_two(nvec); + + WARN_ON(!list_is_singular(&dev->msi_list)); + msidesc = list_entry(dev->msi_list.next, struct msi_desc, list); + WARN_ON(msidesc->irq); + WARN_ON(msidesc->msi_attrib.multiple); + + node = dev_to_node(&dev->dev); + irq = __create_irqs(get_nr_irqs_gsi(), nvec, node); + if (irq == 0) + return -ENOSPC; + + msidesc->msi_attrib.multiple = ilog2(nvec); + for (sub_handle = 0; sub_handle < nvec; sub_handle++) { + if (!sub_handle) { + index = msi_alloc_remapped_irq(dev, irq, nvec); + if (index < 0) { + ret = index; + goto error; + } + } else { + ret = msi_setup_remapped_irq(dev, irq + sub_handle, + index, sub_handle); + if (ret < 0) + goto error; + } + ret = setup_msi_irq(dev, msidesc, irq, sub_handle); + if (ret < 0) + goto error; + } + return 0; + +error: + destroy_irqs(irq, nvec); + + /* + * Restore altered MSI descriptor fields and prevent just destroyed + * IRQs from tearing down again in default_teardown_msi_irqs() + */ + msidesc->irq = 0; + msidesc->msi_attrib.multiple = 0; + + return ret; +} + +static int do_setup_msix_irqs(struct pci_dev *dev, int nvec) +{ + int node, ret, sub_handle, index = 0; + struct msi_desc *msidesc; + unsigned int irq; + + node = dev_to_node(&dev->dev); + irq = get_nr_irqs_gsi(); + sub_handle = 0; + + list_for_each_entry(msidesc, &dev->msi_list, list) { + + irq = create_irq_nr(irq, node); + if (irq == 0) + return -1; + + if (sub_handle == 0) + ret = index = msi_alloc_remapped_irq(dev, irq, nvec); + else + ret = msi_setup_remapped_irq(dev, irq, index, sub_handle); + + if (ret < 0) + goto error; + + ret = setup_msi_irq(dev, msidesc, irq, 0); + if (ret < 0) + goto error; + + sub_handle += 1; + irq += 1; + } + + return 0; + +error: + destroy_irq(irq); + return ret; +} + +static int irq_remapping_setup_msi_irqs(struct pci_dev *dev, + int nvec, int type) +{ + if (type == PCI_CAP_ID_MSI) + return do_setup_msi_irqs(dev, nvec); + else + return do_setup_msix_irqs(dev, nvec); +} + +void eoi_ioapic_pin_remapped(int apic, int pin, int vector) +{ + /* + * Intr-remapping uses pin number as the virtual vector + * in the RTE. Actual vector is programmed in + * intr-remapping table entry. Hence for the io-apic + * EOI we use the pin number. + */ + io_apic_eoi(apic, pin); +} + +static void __init irq_remapping_modify_x86_ops(void) +{ + x86_io_apic_ops.disable = irq_remapping_disable_io_apic; + x86_io_apic_ops.set_affinity = set_remapped_irq_affinity; + x86_io_apic_ops.setup_entry = setup_ioapic_remapped_entry; + x86_io_apic_ops.eoi_ioapic_pin = eoi_ioapic_pin_remapped; + x86_msi.setup_msi_irqs = irq_remapping_setup_msi_irqs; + x86_msi.setup_hpet_msi = setup_hpet_msi_remapped; + x86_msi.compose_msi_msg = compose_remapped_msi_msg; +} + static __init int setup_nointremap(char *str) { disable_irq_remap = 1; @@ -58,6 +211,11 @@ void __init setup_irq_remapping_ops(void) #endif } +void set_irq_remapping_broken(void) +{ + irq_remap_broken = 1; +} + int irq_remapping_supported(void) { if (disable_irq_remap) @@ -79,15 +237,24 @@ int __init irq_remapping_prepare(void) int __init irq_remapping_enable(void) { + int ret; + if (!remap_ops || !remap_ops->enable) return -ENODEV; - return remap_ops->enable(); + ret = remap_ops->enable(); + + if (irq_remapping_enabled) + irq_remapping_modify_x86_ops(); + + return ret; } void irq_remapping_disable(void) { - if (!remap_ops || !remap_ops->disable) + if (!irq_remapping_enabled || + !remap_ops || + !remap_ops->disable) return; remap_ops->disable(); @@ -95,7 +262,9 @@ void irq_remapping_disable(void) int irq_remapping_reenable(int mode) { - if (!remap_ops || !remap_ops->reenable) + if (!irq_remapping_enabled || + !remap_ops || + !remap_ops->reenable) return 0; return remap_ops->reenable(mode); @@ -103,6 +272,9 @@ int irq_remapping_reenable(int mode) int __init irq_remap_enable_fault_handling(void) { + if (!irq_remapping_enabled) + return 0; + if (!remap_ops || !remap_ops->enable_faulting) return -ENODEV; @@ -133,23 +305,28 @@ int set_remapped_irq_affinity(struct irq_data *data, const struct cpumask *mask, void free_remapped_irq(int irq) { + struct irq_cfg *cfg = irq_get_chip_data(irq); + if (!remap_ops || !remap_ops->free_irq) return; - remap_ops->free_irq(irq); + if (irq_remapped(cfg)) + remap_ops->free_irq(irq); } void compose_remapped_msi_msg(struct pci_dev *pdev, unsigned int irq, unsigned int dest, struct msi_msg *msg, u8 hpet_id) { - if (!remap_ops || !remap_ops->compose_msi_msg) - return; + struct irq_cfg *cfg = irq_get_chip_data(irq); - remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); + if (!irq_remapped(cfg)) + native_compose_msi_msg(pdev, irq, dest, msg, hpet_id); + else if (remap_ops && remap_ops->compose_msi_msg) + remap_ops->compose_msi_msg(pdev, irq, dest, msg, hpet_id); } -int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) +static int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) { if (!remap_ops || !remap_ops->msi_alloc_irq) return -ENODEV; @@ -157,8 +334,8 @@ int msi_alloc_remapped_irq(struct pci_dev *pdev, int irq, int nvec) return remap_ops->msi_alloc_irq(pdev, irq, nvec); } -int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, - int index, int sub_handle) +static int msi_setup_remapped_irq(struct pci_dev *pdev, unsigned int irq, + int index, int sub_handle) { if (!remap_ops || !remap_ops->msi_setup_irq) return -ENODEV; @@ -173,3 +350,42 @@ int setup_hpet_msi_remapped(unsigned int irq, unsigned int id) return remap_ops->setup_hpet_msi(irq, id); } + +void panic_if_irq_remap(const char *msg) +{ + if (irq_remapping_enabled) + panic(msg); +} + +static void ir_ack_apic_edge(struct irq_data *data) +{ + ack_APIC_irq(); +} + +static void ir_ack_apic_level(struct irq_data *data) +{ + ack_APIC_irq(); + eoi_ioapic_irq(data->irq, data->chip_data); +} + +static void ir_print_prefix(struct irq_data *data, struct seq_file *p) +{ + seq_printf(p, " IR-%s", data->chip->name); +} + +void irq_remap_modify_chip_defaults(struct irq_chip *chip) +{ + chip->irq_print_chip = ir_print_prefix; + chip->irq_ack = ir_ack_apic_edge; + chip->irq_eoi = ir_ack_apic_level; + chip->irq_set_affinity = x86_io_apic_ops.set_affinity; +} + +bool setup_remapped_irq(int irq, struct irq_cfg *cfg, struct irq_chip *chip) +{ + if (!irq_remapped(cfg)) + return false; + irq_set_status_flags(irq, IRQ_MOVE_PCNTXT); + irq_remap_modify_chip_defaults(chip); + return true; +} diff --git a/drivers/iommu/irq_remapping.h b/drivers/iommu/irq_remapping.h index 95363ac..90c4dae 100644 --- a/drivers/iommu/irq_remapping.h +++ b/drivers/iommu/irq_remapping.h @@ -32,8 +32,10 @@ struct pci_dev; struct msi_msg; extern int disable_irq_remap; +extern int irq_remap_broken; extern int disable_sourceid_checking; extern int no_x2apic_optout; +extern int irq_remapping_enabled; struct irq_remap_ops { /* Check whether Interrupt Remapping is supported */ @@ -88,6 +90,7 @@ extern struct irq_remap_ops amd_iommu_irq_ops; #define irq_remapping_enabled 0 #define disable_irq_remap 1 +#define irq_remap_broken 0 #endif /* CONFIG_IRQ_REMAP */ diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index fcd14a3..8ab4f41 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -554,7 +554,7 @@ fail: } static phys_addr_t msm_iommu_iova_to_phys(struct iommu_domain *domain, - u64 va) + dma_addr_t va) { struct msm_priv *priv; struct msm_iommu_drvdata *iommu_drvdata; diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index eb4229d..e02e5d7 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -146,7 +146,7 @@ static int iommu_enable(struct omap_iommu *obj) struct platform_device *pdev = to_platform_device(obj->dev); struct iommu_platform_data *pdata = pdev->dev.platform_data; - if (!obj || !pdata) + if (!pdata) return -EINVAL; if (!arch_iommu) @@ -172,7 +172,7 @@ static void iommu_disable(struct omap_iommu *obj) struct platform_device *pdev = to_platform_device(obj->dev); struct iommu_platform_data *pdata = pdev->dev.platform_data; - if (!obj || !pdata) + if (!pdata) return; arch_iommu->disable(obj); @@ -1219,7 +1219,7 @@ static void omap_iommu_domain_destroy(struct iommu_domain *domain) } static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, - u64 da) + dma_addr_t da) { struct omap_iommu_domain *omap_domain = domain->priv; struct omap_iommu *oiommu = omap_domain->iommu_dev; diff --git a/drivers/iommu/pci.h b/drivers/iommu/pci.h new file mode 100644 index 0000000..352d80a --- /dev/null +++ b/drivers/iommu/pci.h @@ -0,0 +1,29 @@ +/* + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, version 2, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * + * Copyright (C) 2013 Red Hat, Inc. + * Copyright (C) 2013 Freescale Semiconductor, Inc. + * + */ +#ifndef __IOMMU_PCI_H +#define __IOMMU_PCI_H + +/* Helper function for swapping pci device reference */ +static inline void swap_pci_ref(struct pci_dev **from, struct pci_dev *to) +{ + pci_dev_put(*from); + *from = to; +} + +#endif /* __IOMMU_PCI_H */ diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c new file mode 100644 index 0000000..d572863 --- /dev/null +++ b/drivers/iommu/shmobile-iommu.c @@ -0,0 +1,395 @@ +/* + * IOMMU for IPMMU/IPMMUI + * Copyright (C) 2012 Hideki EIRAKU + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include <linux/dma-mapping.h> +#include <linux/io.h> +#include <linux/iommu.h> +#include <linux/platform_device.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <asm/dma-iommu.h> +#include "shmobile-ipmmu.h" + +#define L1_SIZE CONFIG_SHMOBILE_IOMMU_L1SIZE +#define L1_LEN (L1_SIZE / 4) +#define L1_ALIGN L1_SIZE +#define L2_SIZE SZ_1K +#define L2_LEN (L2_SIZE / 4) +#define L2_ALIGN L2_SIZE + +struct shmobile_iommu_domain_pgtable { + uint32_t *pgtable; + dma_addr_t handle; +}; + +struct shmobile_iommu_archdata { + struct list_head attached_list; + struct dma_iommu_mapping *iommu_mapping; + spinlock_t attach_lock; + struct shmobile_iommu_domain *attached; + int num_attached_devices; + struct shmobile_ipmmu *ipmmu; +}; + +struct shmobile_iommu_domain { + struct shmobile_iommu_domain_pgtable l1, l2[L1_LEN]; + spinlock_t map_lock; + spinlock_t attached_list_lock; + struct list_head attached_list; +}; + +static struct shmobile_iommu_archdata *ipmmu_archdata; +static struct kmem_cache *l1cache, *l2cache; + +static int pgtable_alloc(struct shmobile_iommu_domain_pgtable *pgtable, + struct kmem_cache *cache, size_t size) +{ + pgtable->pgtable = kmem_cache_zalloc(cache, GFP_ATOMIC); + if (!pgtable->pgtable) + return -ENOMEM; + pgtable->handle = dma_map_single(NULL, pgtable->pgtable, size, + DMA_TO_DEVICE); + return 0; +} + +static void pgtable_free(struct shmobile_iommu_domain_pgtable *pgtable, + struct kmem_cache *cache, size_t size) +{ + dma_unmap_single(NULL, pgtable->handle, size, DMA_TO_DEVICE); + kmem_cache_free(cache, pgtable->pgtable); +} + +static uint32_t pgtable_read(struct shmobile_iommu_domain_pgtable *pgtable, + unsigned int index) +{ + return pgtable->pgtable[index]; +} + +static void pgtable_write(struct shmobile_iommu_domain_pgtable *pgtable, + unsigned int index, unsigned int count, uint32_t val) +{ + unsigned int i; + + for (i = 0; i < count; i++) + pgtable->pgtable[index + i] = val; + dma_sync_single_for_device(NULL, pgtable->handle + index * sizeof(val), + sizeof(val) * count, DMA_TO_DEVICE); +} + +static int shmobile_iommu_domain_init(struct iommu_domain *domain) +{ + struct shmobile_iommu_domain *sh_domain; + int i, ret; + + sh_domain = kmalloc(sizeof(*sh_domain), GFP_KERNEL); + if (!sh_domain) + return -ENOMEM; + ret = pgtable_alloc(&sh_domain->l1, l1cache, L1_SIZE); + if (ret < 0) { + kfree(sh_domain); + return ret; + } + for (i = 0; i < L1_LEN; i++) + sh_domain->l2[i].pgtable = NULL; + spin_lock_init(&sh_domain->map_lock); + spin_lock_init(&sh_domain->attached_list_lock); + INIT_LIST_HEAD(&sh_domain->attached_list); + domain->priv = sh_domain; + return 0; +} + +static void shmobile_iommu_domain_destroy(struct iommu_domain *domain) +{ + struct shmobile_iommu_domain *sh_domain = domain->priv; + int i; + + for (i = 0; i < L1_LEN; i++) { + if (sh_domain->l2[i].pgtable) + pgtable_free(&sh_domain->l2[i], l2cache, L2_SIZE); + } + pgtable_free(&sh_domain->l1, l1cache, L1_SIZE); + kfree(sh_domain); + domain->priv = NULL; +} + +static int shmobile_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct shmobile_iommu_archdata *archdata = dev->archdata.iommu; + struct shmobile_iommu_domain *sh_domain = domain->priv; + int ret = -EBUSY; + + if (!archdata) + return -ENODEV; + spin_lock(&sh_domain->attached_list_lock); + spin_lock(&archdata->attach_lock); + if (archdata->attached != sh_domain) { + if (archdata->attached) + goto err; + ipmmu_tlb_set(archdata->ipmmu, sh_domain->l1.handle, L1_SIZE, + 0); + ipmmu_tlb_flush(archdata->ipmmu); + archdata->attached = sh_domain; + archdata->num_attached_devices = 0; + list_add(&archdata->attached_list, &sh_domain->attached_list); + } + archdata->num_attached_devices++; + ret = 0; +err: + spin_unlock(&archdata->attach_lock); + spin_unlock(&sh_domain->attached_list_lock); + return ret; +} + +static void shmobile_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct shmobile_iommu_archdata *archdata = dev->archdata.iommu; + struct shmobile_iommu_domain *sh_domain = domain->priv; + + if (!archdata) + return; + spin_lock(&sh_domain->attached_list_lock); + spin_lock(&archdata->attach_lock); + archdata->num_attached_devices--; + if (!archdata->num_attached_devices) { + ipmmu_tlb_set(archdata->ipmmu, 0, 0, 0); + ipmmu_tlb_flush(archdata->ipmmu); + archdata->attached = NULL; + list_del(&archdata->attached_list); + } + spin_unlock(&archdata->attach_lock); + spin_unlock(&sh_domain->attached_list_lock); +} + +static void domain_tlb_flush(struct shmobile_iommu_domain *sh_domain) +{ + struct shmobile_iommu_archdata *archdata; + + spin_lock(&sh_domain->attached_list_lock); + list_for_each_entry(archdata, &sh_domain->attached_list, attached_list) + ipmmu_tlb_flush(archdata->ipmmu); + spin_unlock(&sh_domain->attached_list_lock); +} + +static int l2alloc(struct shmobile_iommu_domain *sh_domain, + unsigned int l1index) +{ + int ret; + + if (!sh_domain->l2[l1index].pgtable) { + ret = pgtable_alloc(&sh_domain->l2[l1index], l2cache, L2_SIZE); + if (ret < 0) + return ret; + } + pgtable_write(&sh_domain->l1, l1index, 1, + sh_domain->l2[l1index].handle | 0x1); + return 0; +} + +static void l2realfree(struct shmobile_iommu_domain_pgtable *l2) +{ + if (l2->pgtable) + pgtable_free(l2, l2cache, L2_SIZE); +} + +static void l2free(struct shmobile_iommu_domain *sh_domain, + unsigned int l1index, + struct shmobile_iommu_domain_pgtable *l2) +{ + pgtable_write(&sh_domain->l1, l1index, 1, 0); + if (sh_domain->l2[l1index].pgtable) { + *l2 = sh_domain->l2[l1index]; + sh_domain->l2[l1index].pgtable = NULL; + } +} + +static int shmobile_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL }; + struct shmobile_iommu_domain *sh_domain = domain->priv; + unsigned int l1index, l2index; + int ret; + + l1index = iova >> 20; + switch (size) { + case SZ_4K: + l2index = (iova >> 12) & 0xff; + spin_lock(&sh_domain->map_lock); + ret = l2alloc(sh_domain, l1index); + if (!ret) + pgtable_write(&sh_domain->l2[l1index], l2index, 1, + paddr | 0xff2); + spin_unlock(&sh_domain->map_lock); + break; + case SZ_64K: + l2index = (iova >> 12) & 0xf0; + spin_lock(&sh_domain->map_lock); + ret = l2alloc(sh_domain, l1index); + if (!ret) + pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, + paddr | 0xff1); + spin_unlock(&sh_domain->map_lock); + break; + case SZ_1M: + spin_lock(&sh_domain->map_lock); + l2free(sh_domain, l1index, &l2); + pgtable_write(&sh_domain->l1, l1index, 1, paddr | 0xc02); + spin_unlock(&sh_domain->map_lock); + ret = 0; + break; + default: + ret = -EINVAL; + } + if (!ret) + domain_tlb_flush(sh_domain); + l2realfree(&l2); + return ret; +} + +static size_t shmobile_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL }; + struct shmobile_iommu_domain *sh_domain = domain->priv; + unsigned int l1index, l2index; + uint32_t l2entry = 0; + size_t ret = 0; + + l1index = iova >> 20; + if (!(iova & 0xfffff) && size >= SZ_1M) { + spin_lock(&sh_domain->map_lock); + l2free(sh_domain, l1index, &l2); + spin_unlock(&sh_domain->map_lock); + ret = SZ_1M; + goto done; + } + l2index = (iova >> 12) & 0xff; + spin_lock(&sh_domain->map_lock); + if (sh_domain->l2[l1index].pgtable) + l2entry = pgtable_read(&sh_domain->l2[l1index], l2index); + switch (l2entry & 3) { + case 1: + if (l2index & 0xf) + break; + pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, 0); + ret = SZ_64K; + break; + case 2: + pgtable_write(&sh_domain->l2[l1index], l2index, 1, 0); + ret = SZ_4K; + break; + } + spin_unlock(&sh_domain->map_lock); +done: + if (ret) + domain_tlb_flush(sh_domain); + l2realfree(&l2); + return ret; +} + +static phys_addr_t shmobile_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct shmobile_iommu_domain *sh_domain = domain->priv; + uint32_t l1entry = 0, l2entry = 0; + unsigned int l1index, l2index; + + l1index = iova >> 20; + l2index = (iova >> 12) & 0xff; + spin_lock(&sh_domain->map_lock); + if (sh_domain->l2[l1index].pgtable) + l2entry = pgtable_read(&sh_domain->l2[l1index], l2index); + else + l1entry = pgtable_read(&sh_domain->l1, l1index); + spin_unlock(&sh_domain->map_lock); + switch (l2entry & 3) { + case 1: + return (l2entry & ~0xffff) | (iova & 0xffff); + case 2: + return (l2entry & ~0xfff) | (iova & 0xfff); + default: + if ((l1entry & 3) == 2) + return (l1entry & ~0xfffff) | (iova & 0xfffff); + return 0; + } +} + +static int find_dev_name(struct shmobile_ipmmu *ipmmu, const char *dev_name) +{ + unsigned int i, n = ipmmu->num_dev_names; + + for (i = 0; i < n; i++) { + if (strcmp(ipmmu->dev_names[i], dev_name) == 0) + return 1; + } + return 0; +} + +static int shmobile_iommu_add_device(struct device *dev) +{ + struct shmobile_iommu_archdata *archdata = ipmmu_archdata; + struct dma_iommu_mapping *mapping; + + if (!find_dev_name(archdata->ipmmu, dev_name(dev))) + return 0; + mapping = archdata->iommu_mapping; + if (!mapping) { + mapping = arm_iommu_create_mapping(&platform_bus_type, 0, + L1_LEN << 20, 0); + if (IS_ERR(mapping)) + return PTR_ERR(mapping); + archdata->iommu_mapping = mapping; + } + dev->archdata.iommu = archdata; + if (arm_iommu_attach_device(dev, mapping)) + pr_err("arm_iommu_attach_device failed\n"); + return 0; +} + +static struct iommu_ops shmobile_iommu_ops = { + .domain_init = shmobile_iommu_domain_init, + .domain_destroy = shmobile_iommu_domain_destroy, + .attach_dev = shmobile_iommu_attach_device, + .detach_dev = shmobile_iommu_detach_device, + .map = shmobile_iommu_map, + .unmap = shmobile_iommu_unmap, + .iova_to_phys = shmobile_iommu_iova_to_phys, + .add_device = shmobile_iommu_add_device, + .pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K, +}; + +int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu) +{ + static struct shmobile_iommu_archdata *archdata; + + l1cache = kmem_cache_create("shmobile-iommu-pgtable1", L1_SIZE, + L1_ALIGN, SLAB_HWCACHE_ALIGN, NULL); + if (!l1cache) + return -ENOMEM; + l2cache = kmem_cache_create("shmobile-iommu-pgtable2", L2_SIZE, + L2_ALIGN, SLAB_HWCACHE_ALIGN, NULL); + if (!l2cache) { + kmem_cache_destroy(l1cache); + return -ENOMEM; + } + archdata = kmalloc(sizeof(*archdata), GFP_KERNEL); + if (!archdata) { + kmem_cache_destroy(l1cache); + kmem_cache_destroy(l2cache); + return -ENOMEM; + } + spin_lock_init(&archdata->attach_lock); + archdata->attached = NULL; + archdata->ipmmu = ipmmu; + ipmmu_archdata = archdata; + bus_set_iommu(&platform_bus_type, &shmobile_iommu_ops); + return 0; +} diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c new file mode 100644 index 0000000..8321f89 --- /dev/null +++ b/drivers/iommu/shmobile-ipmmu.c @@ -0,0 +1,136 @@ +/* + * IPMMU/IPMMUI + * Copyright (C) 2012 Hideki EIRAKU + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#include <linux/err.h> +#include <linux/export.h> +#include <linux/io.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/platform_data/sh_ipmmu.h> +#include "shmobile-ipmmu.h" + +#define IMCTR1 0x000 +#define IMCTR2 0x004 +#define IMASID 0x010 +#define IMTTBR 0x014 +#define IMTTBCR 0x018 + +#define IMCTR1_TLBEN (1 << 0) +#define IMCTR1_FLUSH (1 << 1) + +static void ipmmu_reg_write(struct shmobile_ipmmu *ipmmu, unsigned long reg_off, + unsigned long data) +{ + iowrite32(data, ipmmu->ipmmu_base + reg_off); +} + +void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu) +{ + if (!ipmmu) + return; + + mutex_lock(&ipmmu->flush_lock); + if (ipmmu->tlb_enabled) + ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH | IMCTR1_TLBEN); + else + ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH); + mutex_unlock(&ipmmu->flush_lock); +} + +void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size, + int asid) +{ + if (!ipmmu) + return; + + mutex_lock(&ipmmu->flush_lock); + switch (size) { + default: + ipmmu->tlb_enabled = 0; + break; + case 0x2000: + ipmmu_reg_write(ipmmu, IMTTBCR, 1); + ipmmu->tlb_enabled = 1; + break; + case 0x1000: + ipmmu_reg_write(ipmmu, IMTTBCR, 2); + ipmmu->tlb_enabled = 1; + break; + case 0x800: + ipmmu_reg_write(ipmmu, IMTTBCR, 3); + ipmmu->tlb_enabled = 1; + break; + case 0x400: + ipmmu_reg_write(ipmmu, IMTTBCR, 4); + ipmmu->tlb_enabled = 1; + break; + case 0x200: + ipmmu_reg_write(ipmmu, IMTTBCR, 5); + ipmmu->tlb_enabled = 1; + break; + case 0x100: + ipmmu_reg_write(ipmmu, IMTTBCR, 6); + ipmmu->tlb_enabled = 1; + break; + case 0x80: + ipmmu_reg_write(ipmmu, IMTTBCR, 7); + ipmmu->tlb_enabled = 1; + break; + } + ipmmu_reg_write(ipmmu, IMTTBR, phys); + ipmmu_reg_write(ipmmu, IMASID, asid); + mutex_unlock(&ipmmu->flush_lock); +} + +static int ipmmu_probe(struct platform_device *pdev) +{ + struct shmobile_ipmmu *ipmmu; + struct resource *res; + struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(&pdev->dev, "cannot get platform resources\n"); + return -ENOENT; + } + ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL); + if (!ipmmu) { + dev_err(&pdev->dev, "cannot allocate device data\n"); + return -ENOMEM; + } + mutex_init(&ipmmu->flush_lock); + ipmmu->dev = &pdev->dev; + ipmmu->ipmmu_base = devm_ioremap_nocache(&pdev->dev, res->start, + resource_size(res)); + if (!ipmmu->ipmmu_base) { + dev_err(&pdev->dev, "ioremap_nocache failed\n"); + return -ENOMEM; + } + ipmmu->dev_names = pdata->dev_names; + ipmmu->num_dev_names = pdata->num_dev_names; + platform_set_drvdata(pdev, ipmmu); + ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */ + ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */ + ipmmu_iommu_init(ipmmu); + return 0; +} + +static struct platform_driver ipmmu_driver = { + .probe = ipmmu_probe, + .driver = { + .owner = THIS_MODULE, + .name = "ipmmu", + }, +}; + +static int __init ipmmu_init(void) +{ + return platform_driver_register(&ipmmu_driver); +} +subsys_initcall(ipmmu_init); diff --git a/drivers/iommu/shmobile-ipmmu.h b/drivers/iommu/shmobile-ipmmu.h new file mode 100644 index 0000000..4d53684 --- /dev/null +++ b/drivers/iommu/shmobile-ipmmu.h @@ -0,0 +1,34 @@ +/* shmobile-ipmmu.h + * + * Copyright (C) 2012 Hideki EIRAKU + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + */ + +#ifndef __SHMOBILE_IPMMU_H__ +#define __SHMOBILE_IPMMU_H__ + +struct shmobile_ipmmu { + struct device *dev; + void __iomem *ipmmu_base; + int tlb_enabled; + struct mutex flush_lock; + const char * const *dev_names; + unsigned int num_dev_names; +}; + +#ifdef CONFIG_SHMOBILE_IPMMU_TLB +void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu); +void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size, + int asid); +int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu); +#else +static inline int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu) +{ + return -EINVAL; +} +#endif + +#endif /* __SHMOBILE_IPMMU_H__ */ diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c index ce4d9cd..108c0e9 100644 --- a/drivers/iommu/tegra-gart.c +++ b/drivers/iommu/tegra-gart.c @@ -279,7 +279,7 @@ static size_t gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, } static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, - u64 iova) + dma_addr_t iova) { struct gart_device *gart = domain->priv; unsigned long pte; @@ -295,7 +295,8 @@ static phys_addr_t gart_iommu_iova_to_phys(struct iommu_domain *domain, pa = (pte & GART_PAGE_MASK); if (!pfn_valid(__phys_to_pfn(pa))) { - dev_err(gart->dev, "No entry for %08lx:%08x\n", iova, pa); + dev_err(gart->dev, "No entry for %08llx:%08x\n", + (unsigned long long)iova, pa); gart_dump_table(gart); return -EINVAL; } @@ -430,13 +431,11 @@ const struct dev_pm_ops tegra_gart_pm_ops = { .resume = tegra_gart_resume, }; -#ifdef CONFIG_OF static struct of_device_id tegra_gart_of_match[] = { { .compatible = "nvidia,tegra20-gart", }, { }, }; MODULE_DEVICE_TABLE(of, tegra_gart_of_match); -#endif static struct platform_driver tegra_gart_driver = { .probe = tegra_gart_probe, @@ -445,7 +444,7 @@ static struct platform_driver tegra_gart_driver = { .owner = THIS_MODULE, .name = "tegra-gart", .pm = &tegra_gart_pm_ops, - .of_match_table = of_match_ptr(tegra_gart_of_match), + .of_match_table = tegra_gart_of_match, }, }; diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c index 6ad9639..f6f120e 100644 --- a/drivers/iommu/tegra-smmu.c +++ b/drivers/iommu/tegra-smmu.c @@ -1,7 +1,7 @@ /* * IOMMU API for SMMU in Tegra30 * - * Copyright (c) 2011-2012, NVIDIA CORPORATION. All rights reserved. + * Copyright (c) 2011-2013, NVIDIA CORPORATION. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -19,6 +19,7 @@ #define pr_fmt(fmt) "%s(): " fmt, __func__ +#include <linux/err.h> #include <linux/module.h> #include <linux/platform_device.h> #include <linux/spinlock.h> @@ -293,7 +294,11 @@ struct smmu_debugfs_info { * Per SMMU device - IOMMU device */ struct smmu_device { - void __iomem *regs[NUM_SMMU_REG_BANKS]; + void __iomem *regbase; /* register offset base */ + void __iomem **regs; /* register block start address array */ + void __iomem **rege; /* register block end address array */ + int nregs; /* number of register blocks */ + unsigned long iovmm_base; /* remappable base address */ unsigned long page_count; /* total remappable size */ spinlock_t lock; @@ -323,38 +328,37 @@ static struct smmu_device *smmu_handle; /* unique for a system */ /* * SMMU register accessors */ +static bool inline smmu_valid_reg(struct smmu_device *smmu, + void __iomem *addr) +{ + int i; + + for (i = 0; i < smmu->nregs; i++) { + if (addr < smmu->regs[i]) + break; + if (addr <= smmu->rege[i]) + return true; + } + + return false; +} + static inline u32 smmu_read(struct smmu_device *smmu, size_t offs) { - BUG_ON(offs < 0x10); - if (offs < 0x3c) - return readl(smmu->regs[0] + offs - 0x10); - BUG_ON(offs < 0x1f0); - if (offs < 0x200) - return readl(smmu->regs[1] + offs - 0x1f0); - BUG_ON(offs < 0x228); - if (offs < 0x284) - return readl(smmu->regs[2] + offs - 0x228); - BUG(); + void __iomem *addr = smmu->regbase + offs; + + BUG_ON(!smmu_valid_reg(smmu, addr)); + + return readl(addr); } static inline void smmu_write(struct smmu_device *smmu, u32 val, size_t offs) { - BUG_ON(offs < 0x10); - if (offs < 0x3c) { - writel(val, smmu->regs[0] + offs - 0x10); - return; - } - BUG_ON(offs < 0x1f0); - if (offs < 0x200) { - writel(val, smmu->regs[1] + offs - 0x1f0); - return; - } - BUG_ON(offs < 0x228); - if (offs < 0x284) { - writel(val, smmu->regs[2] + offs - 0x228); - return; - } - BUG(); + void __iomem *addr = smmu->regbase + offs; + + BUG_ON(!smmu_valid_reg(smmu, addr)); + + writel(val, addr); } #define VA_PAGE_TO_PA(va, page) \ @@ -753,7 +757,7 @@ static size_t smmu_iommu_unmap(struct iommu_domain *domain, unsigned long iova, } static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain, - u64 iova) + dma_addr_t iova) { struct smmu_as *as = domain->priv; unsigned long *pte; @@ -768,7 +772,8 @@ static phys_addr_t smmu_iommu_iova_to_phys(struct iommu_domain *domain, pfn = *pte & SMMU_PFN_MASK; WARN_ON(!pfn_valid(pfn)); dev_dbg(as->smmu->dev, - "iova:%08lx pfn:%08lx asid:%d\n", iova, pfn, as->asid); + "iova:%08llx pfn:%08lx asid:%d\n", (unsigned long long)iova, + pfn, as->asid); spin_unlock_irqrestore(&as->lock, flags); return PFN_PHYS(pfn); @@ -964,7 +969,6 @@ static ssize_t smmu_debugfs_stats_write(struct file *file, { struct smmu_debugfs_info *info; struct smmu_device *smmu; - struct dentry *dent; int i; enum { _OFF = 0, @@ -992,8 +996,7 @@ static ssize_t smmu_debugfs_stats_write(struct file *file, if (i == ARRAY_SIZE(command)) return -EINVAL; - dent = file->f_dentry; - info = dent->d_inode->i_private; + info = file_inode(file)->i_private; smmu = info->smmu; offs = SMMU_CACHE_CONFIG(info->cache); @@ -1028,15 +1031,11 @@ static ssize_t smmu_debugfs_stats_write(struct file *file, static int smmu_debugfs_stats_show(struct seq_file *s, void *v) { - struct smmu_debugfs_info *info; - struct smmu_device *smmu; - struct dentry *dent; + struct smmu_debugfs_info *info = s->private; + struct smmu_device *smmu = info->smmu; int i; const char * const stats[] = { "hit", "miss", }; - dent = d_find_alias(s->private); - info = dent->d_inode->i_private; - smmu = info->smmu; for (i = 0; i < ARRAY_SIZE(stats); i++) { u32 val; @@ -1050,14 +1049,12 @@ static int smmu_debugfs_stats_show(struct seq_file *s, void *v) stats[i], val, offs); } seq_printf(s, "\n"); - dput(dent); - return 0; } static int smmu_debugfs_stats_open(struct inode *inode, struct file *file) { - return single_open(file, smmu_debugfs_stats_show, inode); + return single_open(file, smmu_debugfs_stats_show, inode->i_private); } static const struct file_operations smmu_debugfs_stats_fops = { @@ -1170,16 +1167,25 @@ static int tegra_smmu_probe(struct platform_device *pdev) return -ENOMEM; } - for (i = 0; i < ARRAY_SIZE(smmu->regs); i++) { + smmu->nregs = pdev->num_resources; + smmu->regs = devm_kzalloc(dev, 2 * smmu->nregs * sizeof(*smmu->regs), + GFP_KERNEL); + smmu->rege = smmu->regs + smmu->nregs; + if (!smmu->regs) + return -ENOMEM; + for (i = 0; i < smmu->nregs; i++) { struct resource *res; res = platform_get_resource(pdev, IORESOURCE_MEM, i); if (!res) return -ENODEV; - smmu->regs[i] = devm_request_and_ioremap(&pdev->dev, res); - if (!smmu->regs[i]) - return -EBUSY; + smmu->regs[i] = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(smmu->regs[i])) + return PTR_ERR(smmu->regs[i]); + smmu->rege[i] = smmu->regs[i] + resource_size(res) - 1; } + /* Same as "mc" 1st regiter block start address */ + smmu->regbase = (void __iomem *)((u32)smmu->regs[0] & PAGE_MASK); err = of_get_dma_window(dev->of_node, NULL, 0, NULL, &base, &size); if (err) @@ -1216,6 +1222,7 @@ static int tegra_smmu_probe(struct platform_device *pdev) as->pte_attr = _PTE_ATTR; spin_lock_init(&as->lock); + spin_lock_init(&as->client_lock); INIT_LIST_HEAD(&as->client); } spin_lock_init(&smmu->lock); @@ -1254,13 +1261,11 @@ const struct dev_pm_ops tegra_smmu_pm_ops = { .resume = tegra_smmu_resume, }; -#ifdef CONFIG_OF static struct of_device_id tegra_smmu_of_match[] = { { .compatible = "nvidia,tegra30-smmu", }, { }, }; MODULE_DEVICE_TABLE(of, tegra_smmu_of_match); -#endif static struct platform_driver tegra_smmu_driver = { .probe = tegra_smmu_probe, @@ -1269,7 +1274,7 @@ static struct platform_driver tegra_smmu_driver = { .owner = THIS_MODULE, .name = "tegra-smmu", .pm = &tegra_smmu_pm_ops, - .of_match_table = of_match_ptr(tegra_smmu_of_match), + .of_match_table = tegra_smmu_of_match, }, }; |