diff options
Diffstat (limited to 'drivers')
29 files changed, 2915 insertions, 637 deletions
diff --git a/drivers/gpu/drm/drm_edid_load.c b/drivers/gpu/drm/drm_edid_load.c index a4f5ce1..271b42b 100644 --- a/drivers/gpu/drm/drm_edid_load.c +++ b/drivers/gpu/drm/drm_edid_load.c @@ -133,8 +133,8 @@ static u8 generic_edid[GENERIC_EDIDS][128] = { }, }; -static u8 *edid_load(struct drm_connector *connector, char *name, - char *connector_name) +static u8 *edid_load(struct drm_connector *connector, const char *name, + const char *connector_name) { const struct firmware *fw; struct platform_device *pdev; @@ -242,7 +242,7 @@ out: int drm_load_edid_firmware(struct drm_connector *connector) { - char *connector_name = drm_get_connector_name(connector); + const char *connector_name = drm_get_connector_name(connector); char *edidname = edid_firmware, *last, *colon; int ret; struct edid *edid; diff --git a/drivers/ide/delkin_cb.c b/drivers/ide/delkin_cb.c index 7e27d32..300daab 100644 --- a/drivers/ide/delkin_cb.c +++ b/drivers/ide/delkin_cb.c @@ -173,18 +173,7 @@ static struct pci_driver delkin_cb_pci_driver = { .resume = delkin_cb_resume, }; -static int __init delkin_cb_init(void) -{ - return pci_register_driver(&delkin_cb_pci_driver); -} - -static void __exit delkin_cb_exit(void) -{ - pci_unregister_driver(&delkin_cb_pci_driver); -} - -module_init(delkin_cb_init); -module_exit(delkin_cb_exit); +module_pci_driver(delkin_cb_pci_driver); MODULE_AUTHOR("Mark Lord"); MODULE_DESCRIPTION("Basic support for Delkin/ASKA/Workbit Cardbus IDE"); diff --git a/drivers/ide/gayle.c b/drivers/ide/gayle.c index 51beb85..0a8440a 100644 --- a/drivers/ide/gayle.c +++ b/drivers/ide/gayle.c @@ -183,20 +183,7 @@ static struct platform_driver amiga_gayle_ide_driver = { }, }; -static int __init amiga_gayle_ide_init(void) -{ - return platform_driver_probe(&amiga_gayle_ide_driver, - amiga_gayle_ide_probe); -} - -module_init(amiga_gayle_ide_init); - -static void __exit amiga_gayle_ide_exit(void) -{ - platform_driver_unregister(&amiga_gayle_ide_driver); -} - -module_exit(amiga_gayle_ide_exit); +module_platform_driver_probe(amiga_gayle_ide_driver, amiga_gayle_ide_probe); MODULE_LICENSE("GPL"); MODULE_ALIAS("platform:amiga-gayle-ide"); diff --git a/drivers/ide/ide-taskfile.c b/drivers/ide/ide-taskfile.c index 729428e..dabb88b 100644 --- a/drivers/ide/ide-taskfile.c +++ b/drivers/ide/ide-taskfile.c @@ -239,9 +239,6 @@ void ide_pio_bytes(ide_drive_t *drive, struct ide_cmd *cmd, unsigned nr_bytes = min(len, cursg->length - cmd->cursg_ofs); int page_is_high; - if (nr_bytes > PAGE_SIZE) - nr_bytes = PAGE_SIZE; - page = sg_page(cursg); offset = cursg->offset + cmd->cursg_ofs; @@ -249,6 +246,8 @@ void ide_pio_bytes(ide_drive_t *drive, struct ide_cmd *cmd, page = nth_page(page, (offset >> PAGE_SHIFT)); offset %= PAGE_SIZE; + nr_bytes = min_t(unsigned, nr_bytes, (PAGE_SIZE - offset)); + page_is_high = PageHighMem(page); if (page_is_high) local_irq_save(flags); diff --git a/drivers/ide/tx4938ide.c b/drivers/ide/tx4938ide.c index 91d49dd..ede8575 100644 --- a/drivers/ide/tx4938ide.c +++ b/drivers/ide/tx4938ide.c @@ -203,18 +203,7 @@ static struct platform_driver tx4938ide_driver = { .remove = __exit_p(tx4938ide_remove), }; -static int __init tx4938ide_init(void) -{ - return platform_driver_probe(&tx4938ide_driver, tx4938ide_probe); -} - -static void __exit tx4938ide_exit(void) -{ - platform_driver_unregister(&tx4938ide_driver); -} - -module_init(tx4938ide_init); -module_exit(tx4938ide_exit); +module_platform_driver_probe(tx4938ide_driver, tx4938ide_probe); MODULE_DESCRIPTION("TX4938 internal IDE driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/ide/tx4939ide.c b/drivers/ide/tx4939ide.c index c0ab800..4ecdee5 100644 --- a/drivers/ide/tx4939ide.c +++ b/drivers/ide/tx4939ide.c @@ -624,18 +624,7 @@ static struct platform_driver tx4939ide_driver = { .resume = tx4939ide_resume, }; -static int __init tx4939ide_init(void) -{ - return platform_driver_probe(&tx4939ide_driver, tx4939ide_probe); -} - -static void __exit tx4939ide_exit(void) -{ - platform_driver_unregister(&tx4939ide_driver); -} - -module_init(tx4939ide_init); -module_exit(tx4939ide_exit); +module_platform_driver_probe(tx4939ide_driver, tx4939ide_probe); MODULE_DESCRIPTION("TX4939 internal IDE driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 01730b2..820d85c 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -269,4 +269,17 @@ config SPAPR_TCE_IOMMU Enables bits of IOMMU API required by VFIO. The iommu_ops is not implemented as it is not necessary for VFIO. +config ARM_SMMU + bool "ARM Ltd. System MMU (SMMU) Support" + depends on ARM64 || (ARM_LPAE && OF) + select IOMMU_API + select ARM_DMA_USE_IOMMU if ARM + help + Support for implementations of the ARM System MMU architecture + versions 1 and 2. The driver supports both v7l and v8l table + formats with 4k and 64k page sizes. + + Say Y here if your SoC includes an IOMMU device implementing + the ARM SMMU architecture. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index ef0e520..bbe7041 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o msm_iommu_dev.o obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o +obj-$(CONFIG_ARM_SMMU) += arm-smmu.o obj-$(CONFIG_DMAR_TABLE) += dmar.o obj-$(CONFIG_INTEL_IOMMU) += iova.o intel-iommu.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 21d02b0..6dc6594 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -287,14 +287,27 @@ static struct pci_dev *get_isolation_root(struct pci_dev *pdev) /* * If it's a multifunction device that does not support our - * required ACS flags, add to the same group as function 0. + * required ACS flags, add to the same group as lowest numbered + * function that also does not suport the required ACS flags. */ if (dma_pdev->multifunction && - !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) - swap_pci_ref(&dma_pdev, - pci_get_slot(dma_pdev->bus, - PCI_DEVFN(PCI_SLOT(dma_pdev->devfn), - 0))); + !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) { + u8 i, slot = PCI_SLOT(dma_pdev->devfn); + + for (i = 0; i < 8; i++) { + struct pci_dev *tmp; + + tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i)); + if (!tmp) + continue; + + if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) { + swap_pci_ref(&dma_pdev, tmp); + break; + } + pci_dev_put(tmp); + } + } /* * Devices on the root bus go through the iommu. If that's not us, @@ -1484,6 +1497,10 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, /* Large PTE found which maps this address */ unmap_size = PTE_PAGE_SIZE(*pte); + + /* Only unmap from the first pte in the page */ + if ((unmap_size - 1) & bus_addr) + break; count = PAGE_SIZE_PTE_COUNT(unmap_size); for (i = 0; i < count; i++) pte[i] = 0ULL; @@ -1493,7 +1510,7 @@ static unsigned long iommu_unmap_page(struct protection_domain *dom, unmapped += unmap_size; } - BUG_ON(!is_power_of_2(unmapped)); + BUG_ON(unmapped && !is_power_of_2(unmapped)); return unmapped; } @@ -1893,34 +1910,59 @@ static void domain_id_free(int id) write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); } +#define DEFINE_FREE_PT_FN(LVL, FN) \ +static void free_pt_##LVL (unsigned long __pt) \ +{ \ + unsigned long p; \ + u64 *pt; \ + int i; \ + \ + pt = (u64 *)__pt; \ + \ + for (i = 0; i < 512; ++i) { \ + if (!IOMMU_PTE_PRESENT(pt[i])) \ + continue; \ + \ + p = (unsigned long)IOMMU_PTE_PAGE(pt[i]); \ + FN(p); \ + } \ + free_page((unsigned long)pt); \ +} + +DEFINE_FREE_PT_FN(l2, free_page) +DEFINE_FREE_PT_FN(l3, free_pt_l2) +DEFINE_FREE_PT_FN(l4, free_pt_l3) +DEFINE_FREE_PT_FN(l5, free_pt_l4) +DEFINE_FREE_PT_FN(l6, free_pt_l5) + static void free_pagetable(struct protection_domain *domain) { - int i, j; - u64 *p1, *p2, *p3; - - p1 = domain->pt_root; - - if (!p1) - return; - - for (i = 0; i < 512; ++i) { - if (!IOMMU_PTE_PRESENT(p1[i])) - continue; - - p2 = IOMMU_PTE_PAGE(p1[i]); - for (j = 0; j < 512; ++j) { - if (!IOMMU_PTE_PRESENT(p2[j])) - continue; - p3 = IOMMU_PTE_PAGE(p2[j]); - free_page((unsigned long)p3); - } + unsigned long root = (unsigned long)domain->pt_root; - free_page((unsigned long)p2); + switch (domain->mode) { + case PAGE_MODE_NONE: + break; + case PAGE_MODE_1_LEVEL: + free_page(root); + break; + case PAGE_MODE_2_LEVEL: + free_pt_l2(root); + break; + case PAGE_MODE_3_LEVEL: + free_pt_l3(root); + break; + case PAGE_MODE_4_LEVEL: + free_pt_l4(root); + break; + case PAGE_MODE_5_LEVEL: + free_pt_l5(root); + break; + case PAGE_MODE_6_LEVEL: + free_pt_l6(root); + break; + default: + BUG(); } - - free_page((unsigned long)p1); - - domain->pt_root = NULL; } static void free_gcr3_tbl_level1(u64 *tbl) diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c new file mode 100644 index 0000000..ebd0a4c --- /dev/null +++ b/drivers/iommu/arm-smmu.c @@ -0,0 +1,1969 @@ +/* + * IOMMU API for ARM architected SMMU implementations. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Copyright (C) 2013 ARM Limited + * + * Author: Will Deacon <will.deacon@arm.com> + * + * This driver currently supports: + * - SMMUv1 and v2 implementations + * - Stream-matching and stream-indexing + * - v7/v8 long-descriptor format + * - Non-secure access to the SMMU + * - 4k and 64k pages, with contiguous pte hints. + * - Up to 39-bit addressing + * - Context fault reporting + */ + +#define pr_fmt(fmt) "arm-smmu: " fmt + +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iommu.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/spinlock.h> + +#include <linux/amba/bus.h> + +#include <asm/pgalloc.h> + +/* Maximum number of stream IDs assigned to a single device */ +#define MAX_MASTER_STREAMIDS 8 + +/* Maximum number of context banks per SMMU */ +#define ARM_SMMU_MAX_CBS 128 + +/* Maximum number of mapping groups per SMMU */ +#define ARM_SMMU_MAX_SMRS 128 + +/* Number of VMIDs per SMMU */ +#define ARM_SMMU_NUM_VMIDS 256 + +/* SMMU global address space */ +#define ARM_SMMU_GR0(smmu) ((smmu)->base) +#define ARM_SMMU_GR1(smmu) ((smmu)->base + (smmu)->pagesize) + +/* Page table bits */ +#define ARM_SMMU_PTE_PAGE (((pteval_t)3) << 0) +#define ARM_SMMU_PTE_CONT (((pteval_t)1) << 52) +#define ARM_SMMU_PTE_AF (((pteval_t)1) << 10) +#define ARM_SMMU_PTE_SH_NS (((pteval_t)0) << 8) +#define ARM_SMMU_PTE_SH_OS (((pteval_t)2) << 8) +#define ARM_SMMU_PTE_SH_IS (((pteval_t)3) << 8) + +#if PAGE_SIZE == SZ_4K +#define ARM_SMMU_PTE_CONT_ENTRIES 16 +#elif PAGE_SIZE == SZ_64K +#define ARM_SMMU_PTE_CONT_ENTRIES 32 +#else +#define ARM_SMMU_PTE_CONT_ENTRIES 1 +#endif + +#define ARM_SMMU_PTE_CONT_SIZE (PAGE_SIZE * ARM_SMMU_PTE_CONT_ENTRIES) +#define ARM_SMMU_PTE_CONT_MASK (~(ARM_SMMU_PTE_CONT_SIZE - 1)) +#define ARM_SMMU_PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(pte_t)) + +/* Stage-1 PTE */ +#define ARM_SMMU_PTE_AP_UNPRIV (((pteval_t)1) << 6) +#define ARM_SMMU_PTE_AP_RDONLY (((pteval_t)2) << 6) +#define ARM_SMMU_PTE_ATTRINDX_SHIFT 2 + +/* Stage-2 PTE */ +#define ARM_SMMU_PTE_HAP_FAULT (((pteval_t)0) << 6) +#define ARM_SMMU_PTE_HAP_READ (((pteval_t)1) << 6) +#define ARM_SMMU_PTE_HAP_WRITE (((pteval_t)2) << 6) +#define ARM_SMMU_PTE_MEMATTR_OIWB (((pteval_t)0xf) << 2) +#define ARM_SMMU_PTE_MEMATTR_NC (((pteval_t)0x5) << 2) +#define ARM_SMMU_PTE_MEMATTR_DEV (((pteval_t)0x1) << 2) + +/* Configuration registers */ +#define ARM_SMMU_GR0_sCR0 0x0 +#define sCR0_CLIENTPD (1 << 0) +#define sCR0_GFRE (1 << 1) +#define sCR0_GFIE (1 << 2) +#define sCR0_GCFGFRE (1 << 4) +#define sCR0_GCFGFIE (1 << 5) +#define sCR0_USFCFG (1 << 10) +#define sCR0_VMIDPNE (1 << 11) +#define sCR0_PTM (1 << 12) +#define sCR0_FB (1 << 13) +#define sCR0_BSU_SHIFT 14 +#define sCR0_BSU_MASK 0x3 + +/* Identification registers */ +#define ARM_SMMU_GR0_ID0 0x20 +#define ARM_SMMU_GR0_ID1 0x24 +#define ARM_SMMU_GR0_ID2 0x28 +#define ARM_SMMU_GR0_ID3 0x2c +#define ARM_SMMU_GR0_ID4 0x30 +#define ARM_SMMU_GR0_ID5 0x34 +#define ARM_SMMU_GR0_ID6 0x38 +#define ARM_SMMU_GR0_ID7 0x3c +#define ARM_SMMU_GR0_sGFSR 0x48 +#define ARM_SMMU_GR0_sGFSYNR0 0x50 +#define ARM_SMMU_GR0_sGFSYNR1 0x54 +#define ARM_SMMU_GR0_sGFSYNR2 0x58 +#define ARM_SMMU_GR0_PIDR0 0xfe0 +#define ARM_SMMU_GR0_PIDR1 0xfe4 +#define ARM_SMMU_GR0_PIDR2 0xfe8 + +#define ID0_S1TS (1 << 30) +#define ID0_S2TS (1 << 29) +#define ID0_NTS (1 << 28) +#define ID0_SMS (1 << 27) +#define ID0_PTFS_SHIFT 24 +#define ID0_PTFS_MASK 0x2 +#define ID0_PTFS_V8_ONLY 0x2 +#define ID0_CTTW (1 << 14) +#define ID0_NUMIRPT_SHIFT 16 +#define ID0_NUMIRPT_MASK 0xff +#define ID0_NUMSMRG_SHIFT 0 +#define ID0_NUMSMRG_MASK 0xff + +#define ID1_PAGESIZE (1 << 31) +#define ID1_NUMPAGENDXB_SHIFT 28 +#define ID1_NUMPAGENDXB_MASK 7 +#define ID1_NUMS2CB_SHIFT 16 +#define ID1_NUMS2CB_MASK 0xff +#define ID1_NUMCB_SHIFT 0 +#define ID1_NUMCB_MASK 0xff + +#define ID2_OAS_SHIFT 4 +#define ID2_OAS_MASK 0xf +#define ID2_IAS_SHIFT 0 +#define ID2_IAS_MASK 0xf +#define ID2_UBS_SHIFT 8 +#define ID2_UBS_MASK 0xf +#define ID2_PTFS_4K (1 << 12) +#define ID2_PTFS_16K (1 << 13) +#define ID2_PTFS_64K (1 << 14) + +#define PIDR2_ARCH_SHIFT 4 +#define PIDR2_ARCH_MASK 0xf + +/* Global TLB invalidation */ +#define ARM_SMMU_GR0_STLBIALL 0x60 +#define ARM_SMMU_GR0_TLBIVMID 0x64 +#define ARM_SMMU_GR0_TLBIALLNSNH 0x68 +#define ARM_SMMU_GR0_TLBIALLH 0x6c +#define ARM_SMMU_GR0_sTLBGSYNC 0x70 +#define ARM_SMMU_GR0_sTLBGSTATUS 0x74 +#define sTLBGSTATUS_GSACTIVE (1 << 0) +#define TLB_LOOP_TIMEOUT 1000000 /* 1s! */ + +/* Stream mapping registers */ +#define ARM_SMMU_GR0_SMR(n) (0x800 + ((n) << 2)) +#define SMR_VALID (1 << 31) +#define SMR_MASK_SHIFT 16 +#define SMR_MASK_MASK 0x7fff +#define SMR_ID_SHIFT 0 +#define SMR_ID_MASK 0x7fff + +#define ARM_SMMU_GR0_S2CR(n) (0xc00 + ((n) << 2)) +#define S2CR_CBNDX_SHIFT 0 +#define S2CR_CBNDX_MASK 0xff +#define S2CR_TYPE_SHIFT 16 +#define S2CR_TYPE_MASK 0x3 +#define S2CR_TYPE_TRANS (0 << S2CR_TYPE_SHIFT) +#define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT) +#define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT) + +/* Context bank attribute registers */ +#define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) +#define CBAR_VMID_SHIFT 0 +#define CBAR_VMID_MASK 0xff +#define CBAR_S1_MEMATTR_SHIFT 12 +#define CBAR_S1_MEMATTR_MASK 0xf +#define CBAR_S1_MEMATTR_WB 0xf +#define CBAR_TYPE_SHIFT 16 +#define CBAR_TYPE_MASK 0x3 +#define CBAR_TYPE_S2_TRANS (0 << CBAR_TYPE_SHIFT) +#define CBAR_TYPE_S1_TRANS_S2_BYPASS (1 << CBAR_TYPE_SHIFT) +#define CBAR_TYPE_S1_TRANS_S2_FAULT (2 << CBAR_TYPE_SHIFT) +#define CBAR_TYPE_S1_TRANS_S2_TRANS (3 << CBAR_TYPE_SHIFT) +#define CBAR_IRPTNDX_SHIFT 24 +#define CBAR_IRPTNDX_MASK 0xff + +#define ARM_SMMU_GR1_CBA2R(n) (0x800 + ((n) << 2)) +#define CBA2R_RW64_32BIT (0 << 0) +#define CBA2R_RW64_64BIT (1 << 0) + +/* Translation context bank */ +#define ARM_SMMU_CB_BASE(smmu) ((smmu)->base + ((smmu)->size >> 1)) +#define ARM_SMMU_CB(smmu, n) ((n) * (smmu)->pagesize) + +#define ARM_SMMU_CB_SCTLR 0x0 +#define ARM_SMMU_CB_RESUME 0x8 +#define ARM_SMMU_CB_TTBCR2 0x10 +#define ARM_SMMU_CB_TTBR0_LO 0x20 +#define ARM_SMMU_CB_TTBR0_HI 0x24 +#define ARM_SMMU_CB_TTBCR 0x30 +#define ARM_SMMU_CB_S1_MAIR0 0x38 +#define ARM_SMMU_CB_FSR 0x58 +#define ARM_SMMU_CB_FAR_LO 0x60 +#define ARM_SMMU_CB_FAR_HI 0x64 +#define ARM_SMMU_CB_FSYNR0 0x68 + +#define SCTLR_S1_ASIDPNE (1 << 12) +#define SCTLR_CFCFG (1 << 7) +#define SCTLR_CFIE (1 << 6) +#define SCTLR_CFRE (1 << 5) +#define SCTLR_E (1 << 4) +#define SCTLR_AFE (1 << 2) +#define SCTLR_TRE (1 << 1) +#define SCTLR_M (1 << 0) +#define SCTLR_EAE_SBOP (SCTLR_AFE | SCTLR_TRE) + +#define RESUME_RETRY (0 << 0) +#define RESUME_TERMINATE (1 << 0) + +#define TTBCR_EAE (1 << 31) + +#define TTBCR_PASIZE_SHIFT 16 +#define TTBCR_PASIZE_MASK 0x7 + +#define TTBCR_TG0_4K (0 << 14) +#define TTBCR_TG0_64K (1 << 14) + +#define TTBCR_SH0_SHIFT 12 +#define TTBCR_SH0_MASK 0x3 +#define TTBCR_SH_NS 0 +#define TTBCR_SH_OS 2 +#define TTBCR_SH_IS 3 + +#define TTBCR_ORGN0_SHIFT 10 +#define TTBCR_IRGN0_SHIFT 8 +#define TTBCR_RGN_MASK 0x3 +#define TTBCR_RGN_NC 0 +#define TTBCR_RGN_WBWA 1 +#define TTBCR_RGN_WT 2 +#define TTBCR_RGN_WB 3 + +#define TTBCR_SL0_SHIFT 6 +#define TTBCR_SL0_MASK 0x3 +#define TTBCR_SL0_LVL_2 0 +#define TTBCR_SL0_LVL_1 1 + +#define TTBCR_T1SZ_SHIFT 16 +#define TTBCR_T0SZ_SHIFT 0 +#define TTBCR_SZ_MASK 0xf + +#define TTBCR2_SEP_SHIFT 15 +#define TTBCR2_SEP_MASK 0x7 + +#define TTBCR2_PASIZE_SHIFT 0 +#define TTBCR2_PASIZE_MASK 0x7 + +/* Common definitions for PASize and SEP fields */ +#define TTBCR2_ADDR_32 0 +#define TTBCR2_ADDR_36 1 +#define TTBCR2_ADDR_40 2 +#define TTBCR2_ADDR_42 3 +#define TTBCR2_ADDR_44 4 +#define TTBCR2_ADDR_48 5 + +#define MAIR_ATTR_SHIFT(n) ((n) << 3) +#define MAIR_ATTR_MASK 0xff +#define MAIR_ATTR_DEVICE 0x04 +#define MAIR_ATTR_NC 0x44 +#define MAIR_ATTR_WBRWA 0xff +#define MAIR_ATTR_IDX_NC 0 +#define MAIR_ATTR_IDX_CACHE 1 +#define MAIR_ATTR_IDX_DEV 2 + +#define FSR_MULTI (1 << 31) +#define FSR_SS (1 << 30) +#define FSR_UUT (1 << 8) +#define FSR_ASF (1 << 7) +#define FSR_TLBLKF (1 << 6) +#define FSR_TLBMCF (1 << 5) +#define FSR_EF (1 << 4) +#define FSR_PF (1 << 3) +#define FSR_AFF (1 << 2) +#define FSR_TF (1 << 1) + +#define FSR_IGN (FSR_AFF | FSR_ASF | FSR_TLBMCF | \ + FSR_TLBLKF) +#define FSR_FAULT (FSR_MULTI | FSR_SS | FSR_UUT | \ + FSR_EF | FSR_PF | FSR_TF) + +#define FSYNR0_WNR (1 << 4) + +struct arm_smmu_smr { + u8 idx; + u16 mask; + u16 id; +}; + +struct arm_smmu_master { + struct device_node *of_node; + + /* + * The following is specific to the master's position in the + * SMMU chain. + */ + struct rb_node node; + int num_streamids; + u16 streamids[MAX_MASTER_STREAMIDS]; + + /* + * We only need to allocate these on the root SMMU, as we + * configure unmatched streams to bypass translation. + */ + struct arm_smmu_smr *smrs; +}; + +struct arm_smmu_device { + struct device *dev; + struct device_node *parent_of_node; + + void __iomem *base; + unsigned long size; + unsigned long pagesize; + +#define ARM_SMMU_FEAT_COHERENT_WALK (1 << 0) +#define ARM_SMMU_FEAT_STREAM_MATCH (1 << 1) +#define ARM_SMMU_FEAT_TRANS_S1 (1 << 2) +#define ARM_SMMU_FEAT_TRANS_S2 (1 << 3) +#define ARM_SMMU_FEAT_TRANS_NESTED (1 << 4) + u32 features; + int version; + + u32 num_context_banks; + u32 num_s2_context_banks; + DECLARE_BITMAP(context_map, ARM_SMMU_MAX_CBS); + atomic_t irptndx; + + u32 num_mapping_groups; + DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS); + + unsigned long input_size; + unsigned long s1_output_size; + unsigned long s2_output_size; + + u32 num_global_irqs; + u32 num_context_irqs; + unsigned int *irqs; + + DECLARE_BITMAP(vmid_map, ARM_SMMU_NUM_VMIDS); + + struct list_head list; + struct rb_root masters; +}; + +struct arm_smmu_cfg { + struct arm_smmu_device *smmu; + u8 vmid; + u8 cbndx; + u8 irptndx; + u32 cbar; + pgd_t *pgd; +}; + +struct arm_smmu_domain { + /* + * A domain can span across multiple, chained SMMUs and requires + * all devices within the domain to follow the same translation + * path. + */ + struct arm_smmu_device *leaf_smmu; + struct arm_smmu_cfg root_cfg; + phys_addr_t output_mask; + + spinlock_t lock; +}; + +static DEFINE_SPINLOCK(arm_smmu_devices_lock); +static LIST_HEAD(arm_smmu_devices); + +static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu, + struct device_node *dev_node) +{ + struct rb_node *node = smmu->masters.rb_node; + + while (node) { + struct arm_smmu_master *master; + master = container_of(node, struct arm_smmu_master, node); + + if (dev_node < master->of_node) + node = node->rb_left; + else if (dev_node > master->of_node) + node = node->rb_right; + else + return master; + } + + return NULL; +} + +static int insert_smmu_master(struct arm_smmu_device *smmu, + struct arm_smmu_master *master) +{ + struct rb_node **new, *parent; + + new = &smmu->masters.rb_node; + parent = NULL; + while (*new) { + struct arm_smmu_master *this; + this = container_of(*new, struct arm_smmu_master, node); + + parent = *new; + if (master->of_node < this->of_node) + new = &((*new)->rb_left); + else if (master->of_node > this->of_node) + new = &((*new)->rb_right); + else + return -EEXIST; + } + + rb_link_node(&master->node, parent, new); + rb_insert_color(&master->node, &smmu->masters); + return 0; +} + +static int register_smmu_master(struct arm_smmu_device *smmu, + struct device *dev, + struct of_phandle_args *masterspec) +{ + int i; + struct arm_smmu_master *master; + + master = find_smmu_master(smmu, masterspec->np); + if (master) { + dev_err(dev, + "rejecting multiple registrations for master device %s\n", + masterspec->np->name); + return -EBUSY; + } + + if (masterspec->args_count > MAX_MASTER_STREAMIDS) { + dev_err(dev, + "reached maximum number (%d) of stream IDs for master device %s\n", + MAX_MASTER_STREAMIDS, masterspec->np->name); + return -ENOSPC; + } + + master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + master->of_node = masterspec->np; + master->num_streamids = masterspec->args_count; + + for (i = 0; i < master->num_streamids; ++i) + master->streamids[i] = masterspec->args[i]; + + return insert_smmu_master(smmu, master); +} + +static struct arm_smmu_device *find_parent_smmu(struct arm_smmu_device *smmu) +{ + struct arm_smmu_device *parent; + + if (!smmu->parent_of_node) + return NULL; + + spin_lock(&arm_smmu_devices_lock); + list_for_each_entry(parent, &arm_smmu_devices, list) + if (parent->dev->of_node == smmu->parent_of_node) + goto out_unlock; + + parent = NULL; + dev_warn(smmu->dev, + "Failed to find SMMU parent despite parent in DT\n"); +out_unlock: + spin_unlock(&arm_smmu_devices_lock); + return parent; +} + +static int __arm_smmu_alloc_bitmap(unsigned long *map, int start, int end) +{ + int idx; + + do { + idx = find_next_zero_bit(map, end, start); + if (idx == end) + return -ENOSPC; + } while (test_and_set_bit(idx, map)); + + return idx; +} + +static void __arm_smmu_free_bitmap(unsigned long *map, int idx) +{ + clear_bit(idx, map); +} + +/* Wait for any pending TLB invalidations to complete */ +static void arm_smmu_tlb_sync(struct arm_smmu_device *smmu) +{ + int count = 0; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + + writel_relaxed(0, gr0_base + ARM_SMMU_GR0_sTLBGSYNC); + while (readl_relaxed(gr0_base + ARM_SMMU_GR0_sTLBGSTATUS) + & sTLBGSTATUS_GSACTIVE) { + cpu_relax(); + if (++count == TLB_LOOP_TIMEOUT) { + dev_err_ratelimited(smmu->dev, + "TLB sync timed out -- SMMU may be deadlocked\n"); + return; + } + udelay(1); + } +} + +static irqreturn_t arm_smmu_context_fault(int irq, void *dev) +{ + int flags, ret; + u32 fsr, far, fsynr, resume; + unsigned long iova; + struct iommu_domain *domain = dev; + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + struct arm_smmu_device *smmu = root_cfg->smmu; + void __iomem *cb_base; + + cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx); + fsr = readl_relaxed(cb_base + ARM_SMMU_CB_FSR); + + if (!(fsr & FSR_FAULT)) + return IRQ_NONE; + + if (fsr & FSR_IGN) + dev_err_ratelimited(smmu->dev, + "Unexpected context fault (fsr 0x%u)\n", + fsr); + + fsynr = readl_relaxed(cb_base + ARM_SMMU_CB_FSYNR0); + flags = fsynr & FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ; + + far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_LO); + iova = far; +#ifdef CONFIG_64BIT + far = readl_relaxed(cb_base + ARM_SMMU_CB_FAR_HI); + iova |= ((unsigned long)far << 32); +#endif + + if (!report_iommu_fault(domain, smmu->dev, iova, flags)) { + ret = IRQ_HANDLED; + resume = RESUME_RETRY; + } else { + ret = IRQ_NONE; + resume = RESUME_TERMINATE; + } + + /* Clear the faulting FSR */ + writel(fsr, cb_base + ARM_SMMU_CB_FSR); + + /* Retry or terminate any stalled transactions */ + if (fsr & FSR_SS) + writel_relaxed(resume, cb_base + ARM_SMMU_CB_RESUME); + + return ret; +} + +static irqreturn_t arm_smmu_global_fault(int irq, void *dev) +{ + u32 gfsr, gfsynr0, gfsynr1, gfsynr2; + struct arm_smmu_device *smmu = dev; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + + gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR); + gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0); + gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1); + gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2); + + dev_err_ratelimited(smmu->dev, + "Unexpected global fault, this could be serious\n"); + dev_err_ratelimited(smmu->dev, + "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", + gfsr, gfsynr0, gfsynr1, gfsynr2); + + writel(gfsr, gr0_base + ARM_SMMU_GR0_sGFSR); + return IRQ_NONE; +} + +static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain) +{ + u32 reg; + bool stage1; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + struct arm_smmu_device *smmu = root_cfg->smmu; + void __iomem *cb_base, *gr0_base, *gr1_base; + + gr0_base = ARM_SMMU_GR0(smmu); + gr1_base = ARM_SMMU_GR1(smmu); + stage1 = root_cfg->cbar != CBAR_TYPE_S2_TRANS; + cb_base = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, root_cfg->cbndx); + + /* CBAR */ + reg = root_cfg->cbar | + (root_cfg->vmid << CBAR_VMID_SHIFT); + if (smmu->version == 1) + reg |= root_cfg->irptndx << CBAR_IRPTNDX_SHIFT; + + /* Use the weakest memory type, so it is overridden by the pte */ + if (stage1) + reg |= (CBAR_S1_MEMATTR_WB << CBAR_S1_MEMATTR_SHIFT); + writel_relaxed(reg, gr1_base + ARM_SMMU_GR1_CBAR(root_cfg->cbndx)); + + if (smmu->version > 1) { + /* CBA2R */ +#ifdef CONFIG_64BIT + reg = CBA2R_RW64_64BIT; +#else + reg = CBA2R_RW64_32BIT; +#endif + writel_relaxed(reg, + gr1_base + ARM_SMMU_GR1_CBA2R(root_cfg->cbndx)); + + /* TTBCR2 */ + switch (smmu->input_size) { + case 32: + reg = (TTBCR2_ADDR_32 << TTBCR2_SEP_SHIFT); + break; + case 36: + reg = (TTBCR2_ADDR_36 << TTBCR2_SEP_SHIFT); + break; + case 39: + reg = (TTBCR2_ADDR_40 << TTBCR2_SEP_SHIFT); + break; + case 42: + reg = (TTBCR2_ADDR_42 << TTBCR2_SEP_SHIFT); + break; + case 44: + reg = (TTBCR2_ADDR_44 << TTBCR2_SEP_SHIFT); + break; + case 48: + reg = (TTBCR2_ADDR_48 << TTBCR2_SEP_SHIFT); + break; + } + + switch (smmu->s1_output_size) { + case 32: + reg |= (TTBCR2_ADDR_32 << TTBCR2_PASIZE_SHIFT); + break; + case 36: + reg |= (TTBCR2_ADDR_36 << TTBCR2_PASIZE_SHIFT); + break; + case 39: + reg |= (TTBCR2_ADDR_40 << TTBCR2_PASIZE_SHIFT); + break; + case 42: + reg |= (TTBCR2_ADDR_42 << TTBCR2_PASIZE_SHIFT); + break; + case 44: + reg |= (TTBCR2_ADDR_44 << TTBCR2_PASIZE_SHIFT); + break; + case 48: + reg |= (TTBCR2_ADDR_48 << TTBCR2_PASIZE_SHIFT); + break; + } + + if (stage1) + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR2); + } + + /* TTBR0 */ + reg = __pa(root_cfg->pgd); +#ifndef __BIG_ENDIAN + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); + reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); +#else + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_HI); + reg = (phys_addr_t)__pa(root_cfg->pgd) >> 32; + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBR0_LO); +#endif + + /* + * TTBCR + * We use long descriptor, with inner-shareable WBWA tables in TTBR0. + */ + if (smmu->version > 1) { + if (PAGE_SIZE == SZ_4K) + reg = TTBCR_TG0_4K; + else + reg = TTBCR_TG0_64K; + + if (!stage1) { + switch (smmu->s2_output_size) { + case 32: + reg |= (TTBCR2_ADDR_32 << TTBCR_PASIZE_SHIFT); + break; + case 36: + reg |= (TTBCR2_ADDR_36 << TTBCR_PASIZE_SHIFT); + break; + case 40: + reg |= (TTBCR2_ADDR_40 << TTBCR_PASIZE_SHIFT); + break; + case 42: + reg |= (TTBCR2_ADDR_42 << TTBCR_PASIZE_SHIFT); + break; + case 44: + reg |= (TTBCR2_ADDR_44 << TTBCR_PASIZE_SHIFT); + break; + case 48: + reg |= (TTBCR2_ADDR_48 << TTBCR_PASIZE_SHIFT); + break; + } + } else { + reg |= (64 - smmu->s1_output_size) << TTBCR_T0SZ_SHIFT; + } + } else { + reg = 0; + } + + reg |= TTBCR_EAE | + (TTBCR_SH_IS << TTBCR_SH0_SHIFT) | + (TTBCR_RGN_WBWA << TTBCR_ORGN0_SHIFT) | + (TTBCR_RGN_WBWA << TTBCR_IRGN0_SHIFT) | + (TTBCR_SL0_LVL_1 << TTBCR_SL0_SHIFT); + writel_relaxed(reg, cb_base + ARM_SMMU_CB_TTBCR); + + /* MAIR0 (stage-1 only) */ + if (stage1) { + reg = (MAIR_ATTR_NC << MAIR_ATTR_SHIFT(MAIR_ATTR_IDX_NC)) | + (MAIR_ATTR_WBRWA << MAIR_ATTR_SHIFT(MAIR_ATTR_IDX_CACHE)) | + (MAIR_ATTR_DEVICE << MAIR_ATTR_SHIFT(MAIR_ATTR_IDX_DEV)); + writel_relaxed(reg, cb_base + ARM_SMMU_CB_S1_MAIR0); + } + + /* Nuke the TLB */ + writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID); + arm_smmu_tlb_sync(smmu); + + /* SCTLR */ + reg = SCTLR_CFCFG | SCTLR_CFIE | SCTLR_CFRE | SCTLR_M | SCTLR_EAE_SBOP; + if (stage1) + reg |= SCTLR_S1_ASIDPNE; +#ifdef __BIG_ENDIAN + reg |= SCTLR_E; +#endif + writel(reg, cb_base + ARM_SMMU_CB_SCTLR); +} + +static int arm_smmu_init_domain_context(struct iommu_domain *domain, + struct device *dev) +{ + int irq, ret, start; + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + struct arm_smmu_device *smmu, *parent; + + /* + * Walk the SMMU chain to find the root device for this chain. + * We assume that no masters have translations which terminate + * early, and therefore check that the root SMMU does indeed have + * a StreamID for the master in question. + */ + parent = dev->archdata.iommu; + smmu_domain->output_mask = -1; + do { + smmu = parent; + smmu_domain->output_mask &= (1ULL << smmu->s2_output_size) - 1; + } while ((parent = find_parent_smmu(smmu))); + + if (!find_smmu_master(smmu, dev->of_node)) { + dev_err(dev, "unable to find root SMMU for device\n"); + return -ENODEV; + } + + ret = __arm_smmu_alloc_bitmap(smmu->vmid_map, 0, ARM_SMMU_NUM_VMIDS); + if (IS_ERR_VALUE(ret)) + return ret; + + root_cfg->vmid = ret; + if (smmu->features & ARM_SMMU_FEAT_TRANS_NESTED) { + /* + * We will likely want to change this if/when KVM gets + * involved. + */ + root_cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; + start = smmu->num_s2_context_banks; + } else if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) { + root_cfg->cbar = CBAR_TYPE_S2_TRANS; + start = 0; + } else { + root_cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; + start = smmu->num_s2_context_banks; + } + + ret = __arm_smmu_alloc_bitmap(smmu->context_map, start, + smmu->num_context_banks); + if (IS_ERR_VALUE(ret)) + goto out_free_vmid; + + root_cfg->cbndx = ret; + + if (smmu->version == 1) { + root_cfg->irptndx = atomic_inc_return(&smmu->irptndx); + root_cfg->irptndx %= smmu->num_context_irqs; + } else { + root_cfg->irptndx = root_cfg->cbndx; + } + + irq = smmu->irqs[smmu->num_global_irqs + root_cfg->irptndx]; + ret = request_irq(irq, arm_smmu_context_fault, IRQF_SHARED, + "arm-smmu-context-fault", domain); + if (IS_ERR_VALUE(ret)) { + dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", + root_cfg->irptndx, irq); + root_cfg->irptndx = -1; + goto out_free_context; + } + + root_cfg->smmu = smmu; + arm_smmu_init_context_bank(smmu_domain); + return ret; + +out_free_context: + __arm_smmu_free_bitmap(smmu->context_map, root_cfg->cbndx); +out_free_vmid: + __arm_smmu_free_bitmap(smmu->vmid_map, root_cfg->vmid); + return ret; +} + +static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + struct arm_smmu_device *smmu = root_cfg->smmu; + int irq; + + if (!smmu) + return; + + if (root_cfg->irptndx != -1) { + irq = smmu->irqs[smmu->num_global_irqs + root_cfg->irptndx]; + free_irq(irq, domain); + } + + __arm_smmu_free_bitmap(smmu->vmid_map, root_cfg->vmid); + __arm_smmu_free_bitmap(smmu->context_map, root_cfg->cbndx); +} + +static int arm_smmu_domain_init(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain; + pgd_t *pgd; + + /* + * Allocate the domain and initialise some of its data structures. + * We can't really do anything meaningful until we've added a + * master. + */ + smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); + if (!smmu_domain) + return -ENOMEM; + + pgd = kzalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL); + if (!pgd) + goto out_free_domain; + smmu_domain->root_cfg.pgd = pgd; + + spin_lock_init(&smmu_domain->lock); + domain->priv = smmu_domain; + return 0; + +out_free_domain: + kfree(smmu_domain); + return -ENOMEM; +} + +static void arm_smmu_free_ptes(pmd_t *pmd) +{ + pgtable_t table = pmd_pgtable(*pmd); + pgtable_page_dtor(table); + __free_page(table); +} + +static void arm_smmu_free_pmds(pud_t *pud) +{ + int i; + pmd_t *pmd, *pmd_base = pmd_offset(pud, 0); + + pmd = pmd_base; + for (i = 0; i < PTRS_PER_PMD; ++i) { + if (pmd_none(*pmd)) + continue; + + arm_smmu_free_ptes(pmd); + pmd++; + } + + pmd_free(NULL, pmd_base); +} + +static void arm_smmu_free_puds(pgd_t *pgd) +{ + int i; + pud_t *pud, *pud_base = pud_offset(pgd, 0); + + pud = pud_base; + for (i = 0; i < PTRS_PER_PUD; ++i) { + if (pud_none(*pud)) + continue; + + arm_smmu_free_pmds(pud); + pud++; + } + + pud_free(NULL, pud_base); +} + +static void arm_smmu_free_pgtables(struct arm_smmu_domain *smmu_domain) +{ + int i; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + pgd_t *pgd, *pgd_base = root_cfg->pgd; + + /* + * Recursively free the page tables for this domain. We don't + * care about speculative TLB filling, because the TLB will be + * nuked next time this context bank is re-allocated and no devices + * currently map to these tables. + */ + pgd = pgd_base; + for (i = 0; i < PTRS_PER_PGD; ++i) { + if (pgd_none(*pgd)) + continue; + arm_smmu_free_puds(pgd); + pgd++; + } + + kfree(pgd_base); +} + +static void arm_smmu_domain_destroy(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = domain->priv; + arm_smmu_destroy_domain_context(domain); + arm_smmu_free_pgtables(smmu_domain); + kfree(smmu_domain); +} + +static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu, + struct arm_smmu_master *master) +{ + int i; + struct arm_smmu_smr *smrs; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + + if (!(smmu->features & ARM_SMMU_FEAT_STREAM_MATCH)) + return 0; + + if (master->smrs) + return -EEXIST; + + smrs = kmalloc(sizeof(*smrs) * master->num_streamids, GFP_KERNEL); + if (!smrs) { + dev_err(smmu->dev, "failed to allocate %d SMRs for master %s\n", + master->num_streamids, master->of_node->name); + return -ENOMEM; + } + + /* Allocate the SMRs on the root SMMU */ + for (i = 0; i < master->num_streamids; ++i) { + int idx = __arm_smmu_alloc_bitmap(smmu->smr_map, 0, + smmu->num_mapping_groups); + if (IS_ERR_VALUE(idx)) { + dev_err(smmu->dev, "failed to allocate free SMR\n"); + goto err_free_smrs; + } + + smrs[i] = (struct arm_smmu_smr) { + .idx = idx, + .mask = 0, /* We don't currently share SMRs */ + .id = master->streamids[i], + }; + } + + /* It worked! Now, poke the actual hardware */ + for (i = 0; i < master->num_streamids; ++i) { + u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT | + smrs[i].mask << SMR_MASK_SHIFT; + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx)); + } + + master->smrs = smrs; + return 0; + +err_free_smrs: + while (--i >= 0) + __arm_smmu_free_bitmap(smmu->smr_map, smrs[i].idx); + kfree(smrs); + return -ENOSPC; +} + +static void arm_smmu_master_free_smrs(struct arm_smmu_device *smmu, + struct arm_smmu_master *master) +{ + int i; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + struct arm_smmu_smr *smrs = master->smrs; + + /* Invalidate the SMRs before freeing back to the allocator */ + for (i = 0; i < master->num_streamids; ++i) { + u8 idx = smrs[i].idx; + writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(idx)); + __arm_smmu_free_bitmap(smmu->smr_map, idx); + } + + master->smrs = NULL; + kfree(smrs); +} + +static void arm_smmu_bypass_stream_mapping(struct arm_smmu_device *smmu, + struct arm_smmu_master *master) +{ + int i; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + + for (i = 0; i < master->num_streamids; ++i) { + u16 sid = master->streamids[i]; + writel_relaxed(S2CR_TYPE_BYPASS, + gr0_base + ARM_SMMU_GR0_S2CR(sid)); + } +} + +static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master *master) +{ + int i, ret; + struct arm_smmu_device *parent, *smmu = smmu_domain->root_cfg.smmu; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + + ret = arm_smmu_master_configure_smrs(smmu, master); + if (ret) + return ret; + + /* Bypass the leaves */ + smmu = smmu_domain->leaf_smmu; + while ((parent = find_parent_smmu(smmu))) { + /* + * We won't have a StreamID match for anything but the root + * smmu, so we only need to worry about StreamID indexing, + * where we must install bypass entries in the S2CRs. + */ + if (smmu->features & ARM_SMMU_FEAT_STREAM_MATCH) + continue; + + arm_smmu_bypass_stream_mapping(smmu, master); + smmu = parent; + } + + /* Now we're at the root, time to point at our context bank */ + for (i = 0; i < master->num_streamids; ++i) { + u32 idx, s2cr; + idx = master->smrs ? master->smrs[i].idx : master->streamids[i]; + s2cr = (S2CR_TYPE_TRANS << S2CR_TYPE_SHIFT) | + (smmu_domain->root_cfg.cbndx << S2CR_CBNDX_SHIFT); + writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); + } + + return 0; +} + +static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, + struct arm_smmu_master *master) +{ + struct arm_smmu_device *smmu = smmu_domain->root_cfg.smmu; + + /* + * We *must* clear the S2CR first, because freeing the SMR means + * that it can be re-allocated immediately. + */ + arm_smmu_bypass_stream_mapping(smmu, master); + arm_smmu_master_free_smrs(smmu, master); +} + +static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) +{ + int ret = -EINVAL; + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_device *device_smmu = dev->archdata.iommu; + struct arm_smmu_master *master; + + if (!device_smmu) { + dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); + return -ENXIO; + } + + /* + * Sanity check the domain. We don't currently support domains + * that cross between different SMMU chains. + */ + spin_lock(&smmu_domain->lock); + if (!smmu_domain->leaf_smmu) { + /* Now that we have a master, we can finalise the domain */ + ret = arm_smmu_init_domain_context(domain, dev); + if (IS_ERR_VALUE(ret)) + goto err_unlock; + + smmu_domain->leaf_smmu = device_smmu; + } else if (smmu_domain->leaf_smmu != device_smmu) { + dev_err(dev, + "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", + dev_name(smmu_domain->leaf_smmu->dev), + dev_name(device_smmu->dev)); + goto err_unlock; + } + spin_unlock(&smmu_domain->lock); + + /* Looks ok, so add the device to the domain */ + master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node); + if (!master) + return -ENODEV; + + return arm_smmu_domain_add_master(smmu_domain, master); + +err_unlock: + spin_unlock(&smmu_domain->lock); + return ret; +} + +static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) +{ + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_master *master; + + master = find_smmu_master(smmu_domain->leaf_smmu, dev->of_node); + if (master) + arm_smmu_domain_remove_master(smmu_domain, master); +} + +static void arm_smmu_flush_pgtable(struct arm_smmu_device *smmu, void *addr, + size_t size) +{ + unsigned long offset = (unsigned long)addr & ~PAGE_MASK; + + /* + * If the SMMU can't walk tables in the CPU caches, treat them + * like non-coherent DMA since we need to flush the new entries + * all the way out to memory. There's no possibility of recursion + * here as the SMMU table walker will not be wired through another + * SMMU. + */ + if (!(smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)) + dma_map_page(smmu->dev, virt_to_page(addr), offset, size, + DMA_TO_DEVICE); +} + +static bool arm_smmu_pte_is_contiguous_range(unsigned long addr, + unsigned long end) +{ + return !(addr & ~ARM_SMMU_PTE_CONT_MASK) && + (addr + ARM_SMMU_PTE_CONT_SIZE <= end); +} + +static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd, + unsigned long addr, unsigned long end, + unsigned long pfn, int flags, int stage) +{ + pte_t *pte, *start; + pteval_t pteval = ARM_SMMU_PTE_PAGE | ARM_SMMU_PTE_AF; + + if (pmd_none(*pmd)) { + /* Allocate a new set of tables */ + pgtable_t table = alloc_page(PGALLOC_GFP); + if (!table) + return -ENOMEM; + + arm_smmu_flush_pgtable(smmu, page_address(table), + ARM_SMMU_PTE_HWTABLE_SIZE); + pgtable_page_ctor(table); + pmd_populate(NULL, pmd, table); + arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd)); + } + + if (stage == 1) { + pteval |= ARM_SMMU_PTE_AP_UNPRIV; + if (!(flags & IOMMU_WRITE) && (flags & IOMMU_READ)) + pteval |= ARM_SMMU_PTE_AP_RDONLY; + + if (flags & IOMMU_CACHE) + pteval |= (MAIR_ATTR_IDX_CACHE << + ARM_SMMU_PTE_ATTRINDX_SHIFT); + } else { + pteval |= ARM_SMMU_PTE_HAP_FAULT; + if (flags & IOMMU_READ) + pteval |= ARM_SMMU_PTE_HAP_READ; + if (flags & IOMMU_WRITE) + pteval |= ARM_SMMU_PTE_HAP_WRITE; + if (flags & IOMMU_CACHE) + pteval |= ARM_SMMU_PTE_MEMATTR_OIWB; + else + pteval |= ARM_SMMU_PTE_MEMATTR_NC; + } + + /* If no access, create a faulting entry to avoid TLB fills */ + if (!(flags & (IOMMU_READ | IOMMU_WRITE))) + pteval &= ~ARM_SMMU_PTE_PAGE; + + pteval |= ARM_SMMU_PTE_SH_IS; + start = pmd_page_vaddr(*pmd) + pte_index(addr); + pte = start; + + /* + * Install the page table entries. This is fairly complicated + * since we attempt to make use of the contiguous hint in the + * ptes where possible. The contiguous hint indicates a series + * of ARM_SMMU_PTE_CONT_ENTRIES ptes mapping a physically + * contiguous region with the following constraints: + * + * - The region start is aligned to ARM_SMMU_PTE_CONT_SIZE + * - Each pte in the region has the contiguous hint bit set + * + * This complicates unmapping (also handled by this code, when + * neither IOMMU_READ or IOMMU_WRITE are set) because it is + * possible, yet highly unlikely, that a client may unmap only + * part of a contiguous range. This requires clearing of the + * contiguous hint bits in the range before installing the new + * faulting entries. + * + * Note that re-mapping an address range without first unmapping + * it is not supported, so TLB invalidation is not required here + * and is instead performed at unmap and domain-init time. + */ + do { + int i = 1; + pteval &= ~ARM_SMMU_PTE_CONT; + + if (arm_smmu_pte_is_contiguous_range(addr, end)) { + i = ARM_SMMU_PTE_CONT_ENTRIES; + pteval |= ARM_SMMU_PTE_CONT; + } else if (pte_val(*pte) & + (ARM_SMMU_PTE_CONT | ARM_SMMU_PTE_PAGE)) { + int j; + pte_t *cont_start; + unsigned long idx = pte_index(addr); + + idx &= ~(ARM_SMMU_PTE_CONT_ENTRIES - 1); + cont_start = pmd_page_vaddr(*pmd) + idx; + for (j = 0; j < ARM_SMMU_PTE_CONT_ENTRIES; ++j) + pte_val(*(cont_start + j)) &= ~ARM_SMMU_PTE_CONT; + + arm_smmu_flush_pgtable(smmu, cont_start, + sizeof(*pte) * + ARM_SMMU_PTE_CONT_ENTRIES); + } + + do { + *pte = pfn_pte(pfn, __pgprot(pteval)); + } while (pte++, pfn++, addr += PAGE_SIZE, --i); + } while (addr != end); + + arm_smmu_flush_pgtable(smmu, start, sizeof(*pte) * (pte - start)); + return 0; +} + +static int arm_smmu_alloc_init_pmd(struct arm_smmu_device *smmu, pud_t *pud, + unsigned long addr, unsigned long end, + phys_addr_t phys, int flags, int stage) +{ + int ret; + pmd_t *pmd; + unsigned long next, pfn = __phys_to_pfn(phys); + +#ifndef __PAGETABLE_PMD_FOLDED + if (pud_none(*pud)) { + pmd = pmd_alloc_one(NULL, addr); + if (!pmd) + return -ENOMEM; + } else +#endif + pmd = pmd_offset(pud, addr); + + do { + next = pmd_addr_end(addr, end); + ret = arm_smmu_alloc_init_pte(smmu, pmd, addr, end, pfn, + flags, stage); + pud_populate(NULL, pud, pmd); + arm_smmu_flush_pgtable(smmu, pud, sizeof(*pud)); + phys += next - addr; + } while (pmd++, addr = next, addr < end); + + return ret; +} + +static int arm_smmu_alloc_init_pud(struct arm_smmu_device *smmu, pgd_t *pgd, + unsigned long addr, unsigned long end, + phys_addr_t phys, int flags, int stage) +{ + int ret = 0; + pud_t *pud; + unsigned long next; + +#ifndef __PAGETABLE_PUD_FOLDED + if (pgd_none(*pgd)) { + pud = pud_alloc_one(NULL, addr); + if (!pud) + return -ENOMEM; + } else +#endif + pud = pud_offset(pgd, addr); + + do { + next = pud_addr_end(addr, end); + ret = arm_smmu_alloc_init_pmd(smmu, pud, addr, next, phys, + flags, stage); + pgd_populate(NULL, pud, pgd); + arm_smmu_flush_pgtable(smmu, pgd, sizeof(*pgd)); + phys += next - addr; + } while (pud++, addr = next, addr < end); + + return ret; +} + +static int arm_smmu_handle_mapping(struct arm_smmu_domain *smmu_domain, + unsigned long iova, phys_addr_t paddr, + size_t size, int flags) +{ + int ret, stage; + unsigned long end; + phys_addr_t input_mask, output_mask; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + pgd_t *pgd = root_cfg->pgd; + struct arm_smmu_device *smmu = root_cfg->smmu; + + if (root_cfg->cbar == CBAR_TYPE_S2_TRANS) { + stage = 2; + output_mask = (1ULL << smmu->s2_output_size) - 1; + } else { + stage = 1; + output_mask = (1ULL << smmu->s1_output_size) - 1; + } + + if (!pgd) + return -EINVAL; + + if (size & ~PAGE_MASK) + return -EINVAL; + + input_mask = (1ULL << smmu->input_size) - 1; + if ((phys_addr_t)iova & ~input_mask) + return -ERANGE; + + if (paddr & ~output_mask) + return -ERANGE; + + spin_lock(&smmu_domain->lock); + pgd += pgd_index(iova); + end = iova + size; + do { + unsigned long next = pgd_addr_end(iova, end); + + ret = arm_smmu_alloc_init_pud(smmu, pgd, iova, next, paddr, + flags, stage); + if (ret) + goto out_unlock; + + paddr += next - iova; + iova = next; + } while (pgd++, iova != end); + +out_unlock: + spin_unlock(&smmu_domain->lock); + + /* Ensure new page tables are visible to the hardware walker */ + if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) + dsb(); + + return ret; +} + +static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int flags) +{ + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_device *smmu = smmu_domain->leaf_smmu; + + if (!smmu_domain || !smmu) + return -ENODEV; + + /* Check for silent address truncation up the SMMU chain. */ + if ((phys_addr_t)iova & ~smmu_domain->output_mask) + return -ERANGE; + + return arm_smmu_handle_mapping(smmu_domain, iova, paddr, size, flags); +} + +static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, + size_t size) +{ + int ret; + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + struct arm_smmu_device *smmu = root_cfg->smmu; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + + ret = arm_smmu_handle_mapping(smmu_domain, iova, 0, size, 0); + writel_relaxed(root_cfg->vmid, gr0_base + ARM_SMMU_GR0_TLBIVMID); + arm_smmu_tlb_sync(smmu); + return ret ? ret : size; +} + +static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + struct arm_smmu_domain *smmu_domain = domain->priv; + struct arm_smmu_cfg *root_cfg = &smmu_domain->root_cfg; + struct arm_smmu_device *smmu = root_cfg->smmu; + + spin_lock(&smmu_domain->lock); + pgd = root_cfg->pgd; + if (!pgd) + goto err_unlock; + + pgd += pgd_index(iova); + if (pgd_none_or_clear_bad(pgd)) + goto err_unlock; + + pud = pud_offset(pgd, iova); + if (pud_none_or_clear_bad(pud)) + goto err_unlock; + + pmd = pmd_offset(pud, iova); + if (pmd_none_or_clear_bad(pmd)) + goto err_unlock; + + pte = pmd_page_vaddr(*pmd) + pte_index(iova); + if (pte_none(pte)) + goto err_unlock; + + spin_unlock(&smmu_domain->lock); + return __pfn_to_phys(pte_pfn(*pte)) | (iova & ~PAGE_MASK); + +err_unlock: + spin_unlock(&smmu_domain->lock); + dev_warn(smmu->dev, + "invalid (corrupt?) page tables detected for iova 0x%llx\n", + (unsigned long long)iova); + return -EINVAL; +} + +static int arm_smmu_domain_has_cap(struct iommu_domain *domain, + unsigned long cap) +{ + unsigned long caps = 0; + struct arm_smmu_domain *smmu_domain = domain->priv; + + if (smmu_domain->root_cfg.smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) + caps |= IOMMU_CAP_CACHE_COHERENCY; + + return !!(cap & caps); +} + +static int arm_smmu_add_device(struct device *dev) +{ + struct arm_smmu_device *child, *parent, *smmu; + struct arm_smmu_master *master = NULL; + + spin_lock(&arm_smmu_devices_lock); + list_for_each_entry(parent, &arm_smmu_devices, list) { + smmu = parent; + + /* Try to find a child of the current SMMU. */ + list_for_each_entry(child, &arm_smmu_devices, list) { + if (child->parent_of_node == parent->dev->of_node) { + /* Does the child sit above our master? */ + master = find_smmu_master(child, dev->of_node); + if (master) { + smmu = NULL; + break; + } + } + } + + /* We found some children, so keep searching. */ + if (!smmu) { + master = NULL; + continue; + } + + master = find_smmu_master(smmu, dev->of_node); + if (master) + break; + } + spin_unlock(&arm_smmu_devices_lock); + + if (!master) + return -ENODEV; + + dev->archdata.iommu = smmu; + return 0; +} + +static void arm_smmu_remove_device(struct device *dev) +{ + dev->archdata.iommu = NULL; +} + +static struct iommu_ops arm_smmu_ops = { + .domain_init = arm_smmu_domain_init, + .domain_destroy = arm_smmu_domain_destroy, + .attach_dev = arm_smmu_attach_dev, + .detach_dev = arm_smmu_detach_dev, + .map = arm_smmu_map, + .unmap = arm_smmu_unmap, + .iova_to_phys = arm_smmu_iova_to_phys, + .domain_has_cap = arm_smmu_domain_has_cap, + .add_device = arm_smmu_add_device, + .remove_device = arm_smmu_remove_device, + .pgsize_bitmap = (SECTION_SIZE | + ARM_SMMU_PTE_CONT_SIZE | + PAGE_SIZE), +}; + +static void arm_smmu_device_reset(struct arm_smmu_device *smmu) +{ + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + int i = 0; + u32 scr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0); + + /* Mark all SMRn as invalid and all S2CRn as bypass */ + for (i = 0; i < smmu->num_mapping_groups; ++i) { + writel_relaxed(~SMR_VALID, gr0_base + ARM_SMMU_GR0_SMR(i)); + writel_relaxed(S2CR_TYPE_BYPASS, gr0_base + ARM_SMMU_GR0_S2CR(i)); + } + + /* Invalidate the TLB, just in case */ + writel_relaxed(0, gr0_base + ARM_SMMU_GR0_STLBIALL); + writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH); + writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH); + + /* Enable fault reporting */ + scr0 |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE); + + /* Disable TLB broadcasting. */ + scr0 |= (sCR0_VMIDPNE | sCR0_PTM); + + /* Enable client access, but bypass when no mapping is found */ + scr0 &= ~(sCR0_CLIENTPD | sCR0_USFCFG); + + /* Disable forced broadcasting */ + scr0 &= ~sCR0_FB; + + /* Don't upgrade barriers */ + scr0 &= ~(sCR0_BSU_MASK << sCR0_BSU_SHIFT); + + /* Push the button */ + arm_smmu_tlb_sync(smmu); + writel(scr0, gr0_base + ARM_SMMU_GR0_sCR0); +} + +static int arm_smmu_id_size_to_bits(int size) +{ + switch (size) { + case 0: + return 32; + case 1: + return 36; + case 2: + return 40; + case 3: + return 42; + case 4: + return 44; + case 5: + default: + return 48; + } +} + +static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) +{ + unsigned long size; + void __iomem *gr0_base = ARM_SMMU_GR0(smmu); + u32 id; + + dev_notice(smmu->dev, "probing hardware configuration...\n"); + + /* Primecell ID */ + id = readl_relaxed(gr0_base + ARM_SMMU_GR0_PIDR2); + smmu->version = ((id >> PIDR2_ARCH_SHIFT) & PIDR2_ARCH_MASK) + 1; + dev_notice(smmu->dev, "SMMUv%d with:\n", smmu->version); + + /* ID0 */ + id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID0); +#ifndef CONFIG_64BIT + if (((id >> ID0_PTFS_SHIFT) & ID0_PTFS_MASK) == ID0_PTFS_V8_ONLY) { + dev_err(smmu->dev, "\tno v7 descriptor support!\n"); + return -ENODEV; + } +#endif + if (id & ID0_S1TS) { + smmu->features |= ARM_SMMU_FEAT_TRANS_S1; + dev_notice(smmu->dev, "\tstage 1 translation\n"); + } + + if (id & ID0_S2TS) { + smmu->features |= ARM_SMMU_FEAT_TRANS_S2; + dev_notice(smmu->dev, "\tstage 2 translation\n"); + } + + if (id & ID0_NTS) { + smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; + dev_notice(smmu->dev, "\tnested translation\n"); + } + + if (!(smmu->features & + (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2 | + ARM_SMMU_FEAT_TRANS_NESTED))) { + dev_err(smmu->dev, "\tno translation support!\n"); + return -ENODEV; + } + + if (id & ID0_CTTW) { + smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; + dev_notice(smmu->dev, "\tcoherent table walk\n"); + } + + if (id & ID0_SMS) { + u32 smr, sid, mask; + + smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; + smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) & + ID0_NUMSMRG_MASK; + if (smmu->num_mapping_groups == 0) { + dev_err(smmu->dev, + "stream-matching supported, but no SMRs present!\n"); + return -ENODEV; + } + + smr = SMR_MASK_MASK << SMR_MASK_SHIFT; + smr |= (SMR_ID_MASK << SMR_ID_SHIFT); + writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0)); + smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0)); + + mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK; + sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK; + if ((mask & sid) != sid) { + dev_err(smmu->dev, + "SMR mask bits (0x%x) insufficient for ID field (0x%x)\n", + mask, sid); + return -ENODEV; + } + + dev_notice(smmu->dev, + "\tstream matching with %u register groups, mask 0x%x", + smmu->num_mapping_groups, mask); + } + + /* ID1 */ + id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID1); + smmu->pagesize = (id & ID1_PAGESIZE) ? SZ_64K : SZ_4K; + + /* Check that we ioremapped enough */ + size = 1 << (((id >> ID1_NUMPAGENDXB_SHIFT) & ID1_NUMPAGENDXB_MASK) + 1); + size *= (smmu->pagesize << 1); + if (smmu->size < size) + dev_warn(smmu->dev, + "device is 0x%lx bytes but only mapped 0x%lx!\n", + size, smmu->size); + + smmu->num_s2_context_banks = (id >> ID1_NUMS2CB_SHIFT) & + ID1_NUMS2CB_MASK; + smmu->num_context_banks = (id >> ID1_NUMCB_SHIFT) & ID1_NUMCB_MASK; + if (smmu->num_s2_context_banks > smmu->num_context_banks) { + dev_err(smmu->dev, "impossible number of S2 context banks!\n"); + return -ENODEV; + } + dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", + smmu->num_context_banks, smmu->num_s2_context_banks); + + /* ID2 */ + id = readl_relaxed(gr0_base + ARM_SMMU_GR0_ID2); + size = arm_smmu_id_size_to_bits((id >> ID2_IAS_SHIFT) & ID2_IAS_MASK); + + /* + * Stage-1 output limited by stage-2 input size due to pgd + * allocation (PTRS_PER_PGD). + */ +#ifdef CONFIG_64BIT + /* Current maximum output size of 39 bits */ + smmu->s1_output_size = min(39UL, size); +#else + smmu->s1_output_size = min(32UL, size); +#endif + + /* The stage-2 output mask is also applied for bypass */ + size = arm_smmu_id_size_to_bits((id >> ID2_OAS_SHIFT) & ID2_OAS_MASK); + smmu->s2_output_size = min((unsigned long)PHYS_MASK_SHIFT, size); + + if (smmu->version == 1) { + smmu->input_size = 32; + } else { +#ifdef CONFIG_64BIT + size = (id >> ID2_UBS_SHIFT) & ID2_UBS_MASK; + size = min(39, arm_smmu_id_size_to_bits(size)); +#else + size = 32; +#endif + smmu->input_size = size; + + if ((PAGE_SIZE == SZ_4K && !(id & ID2_PTFS_4K)) || + (PAGE_SIZE == SZ_64K && !(id & ID2_PTFS_64K)) || + (PAGE_SIZE != SZ_4K && PAGE_SIZE != SZ_64K)) { + dev_err(smmu->dev, "CPU page size 0x%lx unsupported\n", + PAGE_SIZE); + return -ENODEV; + } + } + + dev_notice(smmu->dev, + "\t%lu-bit VA, %lu-bit IPA, %lu-bit PA\n", + smmu->input_size, smmu->s1_output_size, smmu->s2_output_size); + return 0; +} + +static int arm_smmu_device_dt_probe(struct platform_device *pdev) +{ + struct resource *res; + struct arm_smmu_device *smmu; + struct device_node *dev_node; + struct device *dev = &pdev->dev; + struct rb_node *node; + struct of_phandle_args masterspec; + int num_irqs, i, err; + + smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); + if (!smmu) { + dev_err(dev, "failed to allocate arm_smmu_device\n"); + return -ENOMEM; + } + smmu->dev = dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) { + dev_err(dev, "missing base address/size\n"); + return -ENODEV; + } + + smmu->size = resource_size(res); + smmu->base = devm_request_and_ioremap(dev, res); + if (!smmu->base) + return -EADDRNOTAVAIL; + + if (of_property_read_u32(dev->of_node, "#global-interrupts", + &smmu->num_global_irqs)) { + dev_err(dev, "missing #global-interrupts property\n"); + return -ENODEV; + } + + num_irqs = 0; + while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { + num_irqs++; + if (num_irqs > smmu->num_global_irqs) + smmu->num_context_irqs++; + } + + if (num_irqs < smmu->num_global_irqs) { + dev_warn(dev, "found %d interrupts but expected at least %d\n", + num_irqs, smmu->num_global_irqs); + smmu->num_global_irqs = num_irqs; + } + smmu->num_context_irqs = num_irqs - smmu->num_global_irqs; + + smmu->irqs = devm_kzalloc(dev, sizeof(*smmu->irqs) * num_irqs, + GFP_KERNEL); + if (!smmu->irqs) { + dev_err(dev, "failed to allocate %d irqs\n", num_irqs); + return -ENOMEM; + } + + for (i = 0; i < num_irqs; ++i) { + int irq = platform_get_irq(pdev, i); + if (irq < 0) { + dev_err(dev, "failed to get irq index %d\n", i); + return -ENODEV; + } + smmu->irqs[i] = irq; + } + + i = 0; + smmu->masters = RB_ROOT; + while (!of_parse_phandle_with_args(dev->of_node, "mmu-masters", + "#stream-id-cells", i, + &masterspec)) { + err = register_smmu_master(smmu, dev, &masterspec); + if (err) { + dev_err(dev, "failed to add master %s\n", + masterspec.np->name); + goto out_put_masters; + } + + i++; + } + dev_notice(dev, "registered %d master devices\n", i); + + if ((dev_node = of_parse_phandle(dev->of_node, "smmu-parent", 0))) + smmu->parent_of_node = dev_node; + + err = arm_smmu_device_cfg_probe(smmu); + if (err) + goto out_put_parent; + + if (smmu->version > 1 && + smmu->num_context_banks != smmu->num_context_irqs) { + dev_err(dev, + "found only %d context interrupt(s) but %d required\n", + smmu->num_context_irqs, smmu->num_context_banks); + goto out_put_parent; + } + + arm_smmu_device_reset(smmu); + + for (i = 0; i < smmu->num_global_irqs; ++i) { + err = request_irq(smmu->irqs[i], + arm_smmu_global_fault, + IRQF_SHARED, + "arm-smmu global fault", + smmu); + if (err) { + dev_err(dev, "failed to request global IRQ %d (%u)\n", + i, smmu->irqs[i]); + goto out_free_irqs; + } + } + + INIT_LIST_HEAD(&smmu->list); + spin_lock(&arm_smmu_devices_lock); + list_add(&smmu->list, &arm_smmu_devices); + spin_unlock(&arm_smmu_devices_lock); + return 0; + +out_free_irqs: + while (i--) + free_irq(smmu->irqs[i], smmu); + +out_put_parent: + if (smmu->parent_of_node) + of_node_put(smmu->parent_of_node); + +out_put_masters: + for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { + struct arm_smmu_master *master; + master = container_of(node, struct arm_smmu_master, node); + of_node_put(master->of_node); + } + + return err; +} + +static int arm_smmu_device_remove(struct platform_device *pdev) +{ + int i; + struct device *dev = &pdev->dev; + struct arm_smmu_device *curr, *smmu = NULL; + struct rb_node *node; + + spin_lock(&arm_smmu_devices_lock); + list_for_each_entry(curr, &arm_smmu_devices, list) { + if (curr->dev == dev) { + smmu = curr; + list_del(&smmu->list); + break; + } + } + spin_unlock(&arm_smmu_devices_lock); + + if (!smmu) + return -ENODEV; + + if (smmu->parent_of_node) + of_node_put(smmu->parent_of_node); + + for (node = rb_first(&smmu->masters); node; node = rb_next(node)) { + struct arm_smmu_master *master; + master = container_of(node, struct arm_smmu_master, node); + of_node_put(master->of_node); + } + + if (!bitmap_empty(smmu->vmid_map, ARM_SMMU_NUM_VMIDS)) + dev_err(dev, "removing device with active domains!\n"); + + for (i = 0; i < smmu->num_global_irqs; ++i) + free_irq(smmu->irqs[i], smmu); + + /* Turn the thing off */ + writel(sCR0_CLIENTPD, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_sCR0); + return 0; +} + +#ifdef CONFIG_OF +static struct of_device_id arm_smmu_of_match[] = { + { .compatible = "arm,smmu-v1", }, + { .compatible = "arm,smmu-v2", }, + { .compatible = "arm,mmu-400", }, + { .compatible = "arm,mmu-500", }, + { }, +}; +MODULE_DEVICE_TABLE(of, arm_smmu_of_match); +#endif + +static struct platform_driver arm_smmu_driver = { + .driver = { + .owner = THIS_MODULE, + .name = "arm-smmu", + .of_match_table = of_match_ptr(arm_smmu_of_match), + }, + .probe = arm_smmu_device_dt_probe, + .remove = arm_smmu_device_remove, +}; + +static int __init arm_smmu_init(void) +{ + int ret; + + ret = platform_driver_register(&arm_smmu_driver); + if (ret) + return ret; + + /* Oh, for a proper bus abstraction */ + if (!iommu_present(&platform_bus_type)); + bus_set_iommu(&platform_bus_type, &arm_smmu_ops); + + if (!iommu_present(&amba_bustype)); + bus_set_iommu(&amba_bustype, &arm_smmu_ops); + + return 0; +} + +static void __exit arm_smmu_exit(void) +{ + return platform_driver_unregister(&arm_smmu_driver); +} + +module_init(arm_smmu_init); +module_exit(arm_smmu_exit); + +MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); +MODULE_AUTHOR("Will Deacon <will.deacon@arm.com>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index a7967ce..785675a 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -309,6 +309,7 @@ parse_dmar_table(void) struct acpi_table_dmar *dmar; struct acpi_dmar_header *entry_header; int ret = 0; + int drhd_count = 0; /* * Do it again, earlier dmar_tbl mapping could be mapped with @@ -347,6 +348,7 @@ parse_dmar_table(void) switch (entry_header->type) { case ACPI_DMAR_TYPE_HARDWARE_UNIT: + drhd_count++; ret = dmar_parse_one_drhd(entry_header); break; case ACPI_DMAR_TYPE_RESERVED_MEMORY: @@ -371,6 +373,8 @@ parse_dmar_table(void) entry_header = ((void *)entry_header + entry_header->length); } + if (drhd_count == 0) + pr_warn(FW_BUG "No DRHD structure found in DMAR table\n"); return ret; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index b4f0e28..eec0d3e 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -4182,14 +4182,27 @@ static int intel_iommu_add_device(struct device *dev) /* * If it's a multifunction device that does not support our - * required ACS flags, add to the same group as function 0. + * required ACS flags, add to the same group as lowest numbered + * function that also does not suport the required ACS flags. */ if (dma_pdev->multifunction && - !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) - swap_pci_ref(&dma_pdev, - pci_get_slot(dma_pdev->bus, - PCI_DEVFN(PCI_SLOT(dma_pdev->devfn), - 0))); + !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS)) { + u8 i, slot = PCI_SLOT(dma_pdev->devfn); + + for (i = 0; i < 8; i++) { + struct pci_dev *tmp; + + tmp = pci_get_slot(dma_pdev->bus, PCI_DEVFN(slot, i)); + if (!tmp) + continue; + + if (!pci_acs_enabled(tmp, REQ_ACS_FLAGS)) { + swap_pci_ref(&dma_pdev, tmp); + break; + } + pci_dev_put(tmp); + } + } /* * Devices on the root bus go through the iommu. If that's not us, diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index 5b19b2d..f71673d 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -664,8 +664,7 @@ error: */ if (x2apic_present) - WARN(1, KERN_WARNING - "Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); + pr_warn("Failed to enable irq remapping. You are vulnerable to irq-injection attacks.\n"); return -1; } diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index d8f98b1..fbe9ca7 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -754,6 +754,38 @@ int iommu_domain_has_cap(struct iommu_domain *domain, } EXPORT_SYMBOL_GPL(iommu_domain_has_cap); +static size_t iommu_pgsize(struct iommu_domain *domain, + unsigned long addr_merge, size_t size) +{ + unsigned int pgsize_idx; + size_t pgsize; + + /* Max page size that still fits into 'size' */ + pgsize_idx = __fls(size); + + /* need to consider alignment requirements ? */ + if (likely(addr_merge)) { + /* Max page size allowed by address */ + unsigned int align_pgsize_idx = __ffs(addr_merge); + pgsize_idx = min(pgsize_idx, align_pgsize_idx); + } + + /* build a mask of acceptable page sizes */ + pgsize = (1UL << (pgsize_idx + 1)) - 1; + + /* throw away page sizes not supported by the hardware */ + pgsize &= domain->ops->pgsize_bitmap; + + /* make sure we're still sane */ + BUG_ON(!pgsize); + + /* pick the biggest page */ + pgsize_idx = __fls(pgsize); + pgsize = 1UL << pgsize_idx; + + return pgsize; +} + int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -775,45 +807,18 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, * size of the smallest page supported by the hardware */ if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { - pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz " - "0x%x\n", iova, (unsigned long)paddr, - (unsigned long)size, min_pagesz); + pr_err("unaligned: iova 0x%lx pa 0x%pa size 0x%zx min_pagesz 0x%x\n", + iova, &paddr, size, min_pagesz); return -EINVAL; } - pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova, - (unsigned long)paddr, (unsigned long)size); + pr_debug("map: iova 0x%lx pa 0x%pa size 0x%zx\n", iova, &paddr, size); while (size) { - unsigned long pgsize, addr_merge = iova | paddr; - unsigned int pgsize_idx; - - /* Max page size that still fits into 'size' */ - pgsize_idx = __fls(size); - - /* need to consider alignment requirements ? */ - if (likely(addr_merge)) { - /* Max page size allowed by both iova and paddr */ - unsigned int align_pgsize_idx = __ffs(addr_merge); - - pgsize_idx = min(pgsize_idx, align_pgsize_idx); - } - - /* build a mask of acceptable page sizes */ - pgsize = (1UL << (pgsize_idx + 1)) - 1; - - /* throw away page sizes not supported by the hardware */ - pgsize &= domain->ops->pgsize_bitmap; - - /* make sure we're still sane */ - BUG_ON(!pgsize); - - /* pick the biggest page */ - pgsize_idx = __fls(pgsize); - pgsize = 1UL << pgsize_idx; + size_t pgsize = iommu_pgsize(domain, iova | paddr, size); - pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova, - (unsigned long)paddr, pgsize); + pr_debug("mapping: iova 0x%lx pa 0x%pa pgsize 0x%zx\n", + iova, &paddr, pgsize); ret = domain->ops->map(domain, iova, paddr, pgsize, prot); if (ret) @@ -850,27 +855,26 @@ size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) * by the hardware */ if (!IS_ALIGNED(iova | size, min_pagesz)) { - pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n", - iova, (unsigned long)size, min_pagesz); + pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", + iova, size, min_pagesz); return -EINVAL; } - pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova, - (unsigned long)size); + pr_debug("unmap this: iova 0x%lx size 0x%zx\n", iova, size); /* * Keep iterating until we either unmap 'size' bytes (or more) * or we hit an area that isn't mapped. */ while (unmapped < size) { - size_t left = size - unmapped; + size_t pgsize = iommu_pgsize(domain, iova, size - unmapped); - unmapped_page = domain->ops->unmap(domain, iova, left); + unmapped_page = domain->ops->unmap(domain, iova, pgsize); if (!unmapped_page) break; - pr_debug("unmapped: iova 0x%lx size %lx\n", iova, - (unsigned long)unmapped_page); + pr_debug("unmapped: iova 0x%lx size 0x%zx\n", + iova, unmapped_page); iova += unmapped_page; unmapped += unmapped_page; diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index e02e5d7..0ba3766 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -833,16 +833,15 @@ static irqreturn_t iommu_fault_handler(int irq, void *data) iopgd = iopgd_offset(obj, da); if (!iopgd_is_table(*iopgd)) { - dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p " - "*pgd:px%08x\n", obj->name, errs, da, iopgd, *iopgd); + dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:px%08x\n", + obj->name, errs, da, iopgd, *iopgd); return IRQ_NONE; } iopte = iopte_offset(iopgd, da); - dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x " - "pte:0x%p *pte:0x%08x\n", obj->name, errs, da, iopgd, *iopgd, - iopte, *iopte); + dev_err(obj->dev, "%s: errs:0x%08x da:0x%08x pgd:0x%p *pgd:0x%08x pte:0x%p *pte:0x%08x\n", + obj->name, errs, da, iopgd, *iopgd, iopte, *iopte); return IRQ_NONE; } @@ -1235,14 +1234,16 @@ static phys_addr_t omap_iommu_iova_to_phys(struct iommu_domain *domain, else if (iopte_is_large(*pte)) ret = omap_iommu_translate(*pte, da, IOLARGE_MASK); else - dev_err(dev, "bogus pte 0x%x, da 0x%lx", *pte, da); + dev_err(dev, "bogus pte 0x%x, da 0x%llx", *pte, + (unsigned long long)da); } else { if (iopgd_is_section(*pgd)) ret = omap_iommu_translate(*pgd, da, IOSECTION_MASK); else if (iopgd_is_super(*pgd)) ret = omap_iommu_translate(*pgd, da, IOSUPER_MASK); else - dev_err(dev, "bogus pgd 0x%x, da 0x%lx", *pgd, da); + dev_err(dev, "bogus pgd 0x%x, da 0x%llx", *pgd, + (unsigned long long)da); } return ret; diff --git a/drivers/iommu/omap-iopgtable.h b/drivers/iommu/omap-iopgtable.h index cd4ae9e..f4003d5 100644 --- a/drivers/iommu/omap-iopgtable.h +++ b/drivers/iommu/omap-iopgtable.h @@ -95,4 +95,4 @@ static inline phys_addr_t omap_iommu_translate(u32 d, u32 va, u32 mask) #define iopte_offset(iopgd, da) (iopgd_page_vaddr(iopgd) + iopte_index(da)) #define to_iommu(dev) \ - (struct omap_iommu *)platform_get_drvdata(to_platform_device(dev)) + ((struct omap_iommu *)platform_get_drvdata(to_platform_device(dev))) diff --git a/drivers/iommu/omap-iovmm.c b/drivers/iommu/omap-iovmm.c index 46d87569..d147259 100644 --- a/drivers/iommu/omap-iovmm.c +++ b/drivers/iommu/omap-iovmm.c @@ -102,8 +102,8 @@ static size_t sgtable_len(const struct sg_table *sgt) } if (i && sg->offset) { - pr_err("%s: sg[%d] offset not allowed in internal " - "entries\n", __func__, i); + pr_err("%s: sg[%d] offset not allowed in internal entries\n", + __func__, i); return 0; } diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 42d670a..3d2a90a 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -902,7 +902,6 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) struct scatterlist sg; struct virtio_net_ctrl_mq s; struct net_device *dev = vi->dev; - int i; if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) return 0; @@ -916,10 +915,8 @@ static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) queue_pairs); return -EINVAL; } else { - for (i = vi->curr_queue_pairs; i < queue_pairs; i++) - if (!try_fill_recv(&vi->rq[i], GFP_KERNEL)) - schedule_delayed_work(&vi->refill, 0); vi->curr_queue_pairs = queue_pairs; + schedule_delayed_work(&vi->refill, 0); } return 0; diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c index 259ad28..c488da5 100644 --- a/drivers/vfio/vfio.c +++ b/drivers/vfio/vfio.c @@ -76,6 +76,7 @@ struct vfio_group { struct notifier_block nb; struct list_head vfio_next; struct list_head container_next; + atomic_t opened; }; struct vfio_device { @@ -206,6 +207,7 @@ static struct vfio_group *vfio_create_group(struct iommu_group *iommu_group) INIT_LIST_HEAD(&group->device_list); mutex_init(&group->device_lock); atomic_set(&group->container_users, 0); + atomic_set(&group->opened, 0); group->iommu_group = iommu_group; group->nb.notifier_call = vfio_iommu_group_notifier; @@ -1236,12 +1238,22 @@ static long vfio_group_fops_compat_ioctl(struct file *filep, static int vfio_group_fops_open(struct inode *inode, struct file *filep) { struct vfio_group *group; + int opened; group = vfio_group_get_from_minor(iminor(inode)); if (!group) return -ENODEV; + /* Do we need multiple instances of the group open? Seems not. */ + opened = atomic_cmpxchg(&group->opened, 0, 1); + if (opened) { + vfio_group_put(group); + return -EBUSY; + } + + /* Is something still in use from a previous open? */ if (group->container) { + atomic_dec(&group->opened); vfio_group_put(group); return -EBUSY; } @@ -1259,6 +1271,8 @@ static int vfio_group_fops_release(struct inode *inode, struct file *filep) vfio_group_try_dissolve_container(group); + atomic_dec(&group->opened); + vfio_group_put(group); return 0; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 6f3fbc4..a9807de 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -31,6 +31,7 @@ #include <linux/module.h> #include <linux/mm.h> #include <linux/pci.h> /* pci_bus_type */ +#include <linux/rbtree.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/uaccess.h> @@ -47,19 +48,25 @@ module_param_named(allow_unsafe_interrupts, MODULE_PARM_DESC(allow_unsafe_interrupts, "Enable VFIO IOMMU support for on platforms without interrupt remapping support."); +static bool disable_hugepages; +module_param_named(disable_hugepages, + disable_hugepages, bool, S_IRUGO | S_IWUSR); +MODULE_PARM_DESC(disable_hugepages, + "Disable VFIO IOMMU support for IOMMU hugepages."); + struct vfio_iommu { struct iommu_domain *domain; struct mutex lock; - struct list_head dma_list; + struct rb_root dma_list; struct list_head group_list; bool cache; }; struct vfio_dma { - struct list_head next; + struct rb_node node; dma_addr_t iova; /* Device address */ unsigned long vaddr; /* Process virtual addr */ - long npage; /* Number of pages */ + size_t size; /* Map size (bytes) */ int prot; /* IOMMU_READ/WRITE */ }; @@ -73,7 +80,48 @@ struct vfio_group { * into DMA'ble space using the IOMMU */ -#define NPAGE_TO_SIZE(npage) ((size_t)(npage) << PAGE_SHIFT) +static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, + dma_addr_t start, size_t size) +{ + struct rb_node *node = iommu->dma_list.rb_node; + + while (node) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + + if (start + size <= dma->iova) + node = node->rb_left; + else if (start >= dma->iova + dma->size) + node = node->rb_right; + else + return dma; + } + + return NULL; +} + +static void vfio_insert_dma(struct vfio_iommu *iommu, struct vfio_dma *new) +{ + struct rb_node **link = &iommu->dma_list.rb_node, *parent = NULL; + struct vfio_dma *dma; + + while (*link) { + parent = *link; + dma = rb_entry(parent, struct vfio_dma, node); + + if (new->iova + new->size <= dma->iova) + link = &(*link)->rb_left; + else + link = &(*link)->rb_right; + } + + rb_link_node(&new->node, parent, link); + rb_insert_color(&new->node, &iommu->dma_list); +} + +static void vfio_remove_dma(struct vfio_iommu *iommu, struct vfio_dma *old) +{ + rb_erase(&old->node, &iommu->dma_list); +} struct vwork { struct mm_struct *mm; @@ -100,8 +148,8 @@ static void vfio_lock_acct(long npage) struct vwork *vwork; struct mm_struct *mm; - if (!current->mm) - return; /* process exited */ + if (!current->mm || !npage) + return; /* process exited or nothing to do */ if (down_write_trylock(¤t->mm->mmap_sem)) { current->mm->locked_vm += npage; @@ -173,33 +221,6 @@ static int put_pfn(unsigned long pfn, int prot) return 0; } -/* Unmap DMA region */ -static long __vfio_dma_do_unmap(struct vfio_iommu *iommu, dma_addr_t iova, - long npage, int prot) -{ - long i, unlocked = 0; - - for (i = 0; i < npage; i++, iova += PAGE_SIZE) { - unsigned long pfn; - - pfn = iommu_iova_to_phys(iommu->domain, iova) >> PAGE_SHIFT; - if (pfn) { - iommu_unmap(iommu->domain, iova, PAGE_SIZE); - unlocked += put_pfn(pfn, prot); - } - } - return unlocked; -} - -static void vfio_dma_unmap(struct vfio_iommu *iommu, dma_addr_t iova, - long npage, int prot) -{ - long unlocked; - - unlocked = __vfio_dma_do_unmap(iommu, iova, npage, prot); - vfio_lock_acct(-unlocked); -} - static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn) { struct page *page[1]; @@ -226,198 +247,306 @@ static int vaddr_get_pfn(unsigned long vaddr, int prot, unsigned long *pfn) return ret; } -/* Map DMA region */ -static int __vfio_dma_map(struct vfio_iommu *iommu, dma_addr_t iova, - unsigned long vaddr, long npage, int prot) +/* + * Attempt to pin pages. We really don't want to track all the pfns and + * the iommu can only map chunks of consecutive pfns anyway, so get the + * first page and all consecutive pages with the same locking. + */ +static long vfio_pin_pages(unsigned long vaddr, long npage, + int prot, unsigned long *pfn_base) { - dma_addr_t start = iova; - long i, locked = 0; - int ret; + unsigned long limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + bool lock_cap = capable(CAP_IPC_LOCK); + long ret, i; - /* Verify that pages are not already mapped */ - for (i = 0; i < npage; i++, iova += PAGE_SIZE) - if (iommu_iova_to_phys(iommu->domain, iova)) - return -EBUSY; + if (!current->mm) + return -ENODEV; - iova = start; + ret = vaddr_get_pfn(vaddr, prot, pfn_base); + if (ret) + return ret; - if (iommu->cache) - prot |= IOMMU_CACHE; + if (is_invalid_reserved_pfn(*pfn_base)) + return 1; - /* - * XXX We break mappings into pages and use get_user_pages_fast to - * pin the pages in memory. It's been suggested that mlock might - * provide a more efficient mechanism, but nothing prevents the - * user from munlocking the pages, which could then allow the user - * access to random host memory. We also have no guarantee from the - * IOMMU API that the iommu driver can unmap sub-pages of previous - * mappings. This means we might lose an entire range if a single - * page within it is unmapped. Single page mappings are inefficient, - * but provide the most flexibility for now. - */ - for (i = 0; i < npage; i++, iova += PAGE_SIZE, vaddr += PAGE_SIZE) { + if (!lock_cap && current->mm->locked_vm + 1 > limit) { + put_pfn(*pfn_base, prot); + pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", __func__, + limit << PAGE_SHIFT); + return -ENOMEM; + } + + if (unlikely(disable_hugepages)) { + vfio_lock_acct(1); + return 1; + } + + /* Lock all the consecutive pages from pfn_base */ + for (i = 1, vaddr += PAGE_SIZE; i < npage; i++, vaddr += PAGE_SIZE) { unsigned long pfn = 0; ret = vaddr_get_pfn(vaddr, prot, &pfn); - if (ret) { - __vfio_dma_do_unmap(iommu, start, i, prot); - return ret; - } + if (ret) + break; - /* - * Only add actual locked pages to accounting - * XXX We're effectively marking a page locked for every - * IOVA page even though it's possible the user could be - * backing multiple IOVAs with the same vaddr. This over- - * penalizes the user process, but we currently have no - * easy way to do this properly. - */ - if (!is_invalid_reserved_pfn(pfn)) - locked++; + if (pfn != *pfn_base + i || is_invalid_reserved_pfn(pfn)) { + put_pfn(pfn, prot); + break; + } - ret = iommu_map(iommu->domain, iova, - (phys_addr_t)pfn << PAGE_SHIFT, - PAGE_SIZE, prot); - if (ret) { - /* Back out mappings on error */ + if (!lock_cap && current->mm->locked_vm + i + 1 > limit) { put_pfn(pfn, prot); - __vfio_dma_do_unmap(iommu, start, i, prot); - return ret; + pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", + __func__, limit << PAGE_SHIFT); + break; } } - vfio_lock_acct(locked); - return 0; + + vfio_lock_acct(i); + + return i; } -static inline bool ranges_overlap(dma_addr_t start1, size_t size1, - dma_addr_t start2, size_t size2) +static long vfio_unpin_pages(unsigned long pfn, long npage, + int prot, bool do_accounting) { - if (start1 < start2) - return (start2 - start1 < size1); - else if (start2 < start1) - return (start1 - start2 < size2); - return (size1 > 0 && size2 > 0); + unsigned long unlocked = 0; + long i; + + for (i = 0; i < npage; i++) + unlocked += put_pfn(pfn++, prot); + + if (do_accounting) + vfio_lock_acct(-unlocked); + + return unlocked; } -static struct vfio_dma *vfio_find_dma(struct vfio_iommu *iommu, - dma_addr_t start, size_t size) +static int vfio_unmap_unpin(struct vfio_iommu *iommu, struct vfio_dma *dma, + dma_addr_t iova, size_t *size) { - struct vfio_dma *dma; + dma_addr_t start = iova, end = iova + *size; + long unlocked = 0; - list_for_each_entry(dma, &iommu->dma_list, next) { - if (ranges_overlap(dma->iova, NPAGE_TO_SIZE(dma->npage), - start, size)) - return dma; + while (iova < end) { + size_t unmapped; + phys_addr_t phys; + + /* + * We use the IOMMU to track the physical address. This + * saves us from having a lot more entries in our mapping + * tree. The downside is that we don't track the size + * used to do the mapping. We request unmap of a single + * page, but expect IOMMUs that support large pages to + * unmap a larger chunk. + */ + phys = iommu_iova_to_phys(iommu->domain, iova); + if (WARN_ON(!phys)) { + iova += PAGE_SIZE; + continue; + } + + unmapped = iommu_unmap(iommu->domain, iova, PAGE_SIZE); + if (!unmapped) + break; + + unlocked += vfio_unpin_pages(phys >> PAGE_SHIFT, + unmapped >> PAGE_SHIFT, + dma->prot, false); + iova += unmapped; } - return NULL; + + vfio_lock_acct(-unlocked); + + *size = iova - start; + + return 0; } -static long vfio_remove_dma_overlap(struct vfio_iommu *iommu, dma_addr_t start, - size_t size, struct vfio_dma *dma) +static int vfio_remove_dma_overlap(struct vfio_iommu *iommu, dma_addr_t start, + size_t *size, struct vfio_dma *dma) { + size_t offset, overlap, tmp; struct vfio_dma *split; - long npage_lo, npage_hi; - - /* Existing dma region is completely covered, unmap all */ - if (start <= dma->iova && - start + size >= dma->iova + NPAGE_TO_SIZE(dma->npage)) { - vfio_dma_unmap(iommu, dma->iova, dma->npage, dma->prot); - list_del(&dma->next); - npage_lo = dma->npage; + int ret; + + if (!*size) + return 0; + + /* + * Existing dma region is completely covered, unmap all. This is + * the likely case since userspace tends to map and unmap buffers + * in one shot rather than multiple mappings within a buffer. + */ + if (likely(start <= dma->iova && + start + *size >= dma->iova + dma->size)) { + *size = dma->size; + ret = vfio_unmap_unpin(iommu, dma, dma->iova, size); + if (ret) + return ret; + + /* + * Did we remove more than we have? Should never happen + * since a vfio_dma is contiguous in iova and vaddr. + */ + WARN_ON(*size != dma->size); + + vfio_remove_dma(iommu, dma); kfree(dma); - return npage_lo; + return 0; } /* Overlap low address of existing range */ if (start <= dma->iova) { - size_t overlap; + overlap = start + *size - dma->iova; + ret = vfio_unmap_unpin(iommu, dma, dma->iova, &overlap); + if (ret) + return ret; - overlap = start + size - dma->iova; - npage_lo = overlap >> PAGE_SHIFT; + vfio_remove_dma(iommu, dma); - vfio_dma_unmap(iommu, dma->iova, npage_lo, dma->prot); - dma->iova += overlap; - dma->vaddr += overlap; - dma->npage -= npage_lo; - return npage_lo; + /* + * Check, we may have removed to whole vfio_dma. If not + * fixup and re-insert. + */ + if (overlap < dma->size) { + dma->iova += overlap; + dma->vaddr += overlap; + dma->size -= overlap; + vfio_insert_dma(iommu, dma); + } else + kfree(dma); + + *size = overlap; + return 0; } /* Overlap high address of existing range */ - if (start + size >= dma->iova + NPAGE_TO_SIZE(dma->npage)) { - size_t overlap; + if (start + *size >= dma->iova + dma->size) { + offset = start - dma->iova; + overlap = dma->size - offset; - overlap = dma->iova + NPAGE_TO_SIZE(dma->npage) - start; - npage_hi = overlap >> PAGE_SHIFT; + ret = vfio_unmap_unpin(iommu, dma, start, &overlap); + if (ret) + return ret; - vfio_dma_unmap(iommu, start, npage_hi, dma->prot); - dma->npage -= npage_hi; - return npage_hi; + dma->size -= overlap; + *size = overlap; + return 0; } /* Split existing */ - npage_lo = (start - dma->iova) >> PAGE_SHIFT; - npage_hi = dma->npage - (size >> PAGE_SHIFT) - npage_lo; - split = kzalloc(sizeof *split, GFP_KERNEL); + /* + * Allocate our tracking structure early even though it may not + * be used. An Allocation failure later loses track of pages and + * is more difficult to unwind. + */ + split = kzalloc(sizeof(*split), GFP_KERNEL); if (!split) return -ENOMEM; - vfio_dma_unmap(iommu, start, size >> PAGE_SHIFT, dma->prot); + offset = start - dma->iova; + + ret = vfio_unmap_unpin(iommu, dma, start, size); + if (ret || !*size) { + kfree(split); + return ret; + } + + tmp = dma->size; - dma->npage = npage_lo; + /* Resize the lower vfio_dma in place, before the below insert */ + dma->size = offset; - split->npage = npage_hi; - split->iova = start + size; - split->vaddr = dma->vaddr + NPAGE_TO_SIZE(npage_lo) + size; - split->prot = dma->prot; - list_add(&split->next, &iommu->dma_list); - return size >> PAGE_SHIFT; + /* Insert new for remainder, assuming it didn't all get unmapped */ + if (likely(offset + *size < tmp)) { + split->size = tmp - offset - *size; + split->iova = dma->iova + offset + *size; + split->vaddr = dma->vaddr + offset + *size; + split->prot = dma->prot; + vfio_insert_dma(iommu, split); + } else + kfree(split); + + return 0; } static int vfio_dma_do_unmap(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_unmap *unmap) { - long ret = 0, npage = unmap->size >> PAGE_SHIFT; - struct vfio_dma *dma, *tmp; uint64_t mask; + struct vfio_dma *dma; + size_t unmapped = 0, size; + int ret = 0; mask = ((uint64_t)1 << __ffs(iommu->domain->ops->pgsize_bitmap)) - 1; if (unmap->iova & mask) return -EINVAL; - if (unmap->size & mask) + if (!unmap->size || unmap->size & mask) return -EINVAL; - /* XXX We still break these down into PAGE_SIZE */ WARN_ON(mask & PAGE_MASK); mutex_lock(&iommu->lock); - list_for_each_entry_safe(dma, tmp, &iommu->dma_list, next) { - if (ranges_overlap(dma->iova, NPAGE_TO_SIZE(dma->npage), - unmap->iova, unmap->size)) { - ret = vfio_remove_dma_overlap(iommu, unmap->iova, - unmap->size, dma); - if (ret > 0) - npage -= ret; - if (ret < 0 || npage == 0) - break; - } + while ((dma = vfio_find_dma(iommu, unmap->iova, unmap->size))) { + size = unmap->size; + ret = vfio_remove_dma_overlap(iommu, unmap->iova, &size, dma); + if (ret || !size) + break; + unmapped += size; } + mutex_unlock(&iommu->lock); - return ret > 0 ? 0 : (int)ret; + + /* + * We may unmap more than requested, update the unmap struct so + * userspace can know. + */ + unmap->size = unmapped; + + return ret; +} + +/* + * Turns out AMD IOMMU has a page table bug where it won't map large pages + * to a region that previously mapped smaller pages. This should be fixed + * soon, so this is just a temporary workaround to break mappings down into + * PAGE_SIZE. Better to map smaller pages than nothing. + */ +static int map_try_harder(struct vfio_iommu *iommu, dma_addr_t iova, + unsigned long pfn, long npage, int prot) +{ + long i; + int ret; + + for (i = 0; i < npage; i++, pfn++, iova += PAGE_SIZE) { + ret = iommu_map(iommu->domain, iova, + (phys_addr_t)pfn << PAGE_SHIFT, + PAGE_SIZE, prot); + if (ret) + break; + } + + for (; i < npage && i > 0; i--, iova -= PAGE_SIZE) + iommu_unmap(iommu->domain, iova, PAGE_SIZE); + + return ret; } static int vfio_dma_do_map(struct vfio_iommu *iommu, struct vfio_iommu_type1_dma_map *map) { - struct vfio_dma *dma, *pdma = NULL; - dma_addr_t iova = map->iova; - unsigned long locked, lock_limit, vaddr = map->vaddr; + dma_addr_t end, iova; + unsigned long vaddr = map->vaddr; size_t size = map->size; + long npage; int ret = 0, prot = 0; uint64_t mask; - long npage; + + end = map->iova + map->size; mask = ((uint64_t)1 << __ffs(iommu->domain->ops->pgsize_bitmap)) - 1; @@ -430,104 +559,144 @@ static int vfio_dma_do_map(struct vfio_iommu *iommu, if (!prot) return -EINVAL; /* No READ/WRITE? */ + if (iommu->cache) + prot |= IOMMU_CACHE; + if (vaddr & mask) return -EINVAL; - if (iova & mask) + if (map->iova & mask) return -EINVAL; - if (size & mask) + if (!map->size || map->size & mask) return -EINVAL; - /* XXX We still break these down into PAGE_SIZE */ WARN_ON(mask & PAGE_MASK); /* Don't allow IOVA wrap */ - if (iova + size && iova + size < iova) + if (end && end < map->iova) return -EINVAL; /* Don't allow virtual address wrap */ - if (vaddr + size && vaddr + size < vaddr) - return -EINVAL; - - npage = size >> PAGE_SHIFT; - if (!npage) + if (vaddr + map->size && vaddr + map->size < vaddr) return -EINVAL; mutex_lock(&iommu->lock); - if (vfio_find_dma(iommu, iova, size)) { - ret = -EBUSY; - goto out_lock; + if (vfio_find_dma(iommu, map->iova, map->size)) { + mutex_unlock(&iommu->lock); + return -EEXIST; } - /* account for locked pages */ - locked = current->mm->locked_vm + npage; - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (locked > lock_limit && !capable(CAP_IPC_LOCK)) { - pr_warn("%s: RLIMIT_MEMLOCK (%ld) exceeded\n", - __func__, rlimit(RLIMIT_MEMLOCK)); - ret = -ENOMEM; - goto out_lock; - } + for (iova = map->iova; iova < end; iova += size, vaddr += size) { + struct vfio_dma *dma = NULL; + unsigned long pfn; + long i; + + /* Pin a contiguous chunk of memory */ + npage = vfio_pin_pages(vaddr, (end - iova) >> PAGE_SHIFT, + prot, &pfn); + if (npage <= 0) { + WARN_ON(!npage); + ret = (int)npage; + break; + } - ret = __vfio_dma_map(iommu, iova, vaddr, npage, prot); - if (ret) - goto out_lock; + /* Verify pages are not already mapped */ + for (i = 0; i < npage; i++) { + if (iommu_iova_to_phys(iommu->domain, + iova + (i << PAGE_SHIFT))) { + vfio_unpin_pages(pfn, npage, prot, true); + ret = -EBUSY; + break; + } + } - /* Check if we abut a region below - nothing below 0 */ - if (iova) { - dma = vfio_find_dma(iommu, iova - 1, 1); - if (dma && dma->prot == prot && - dma->vaddr + NPAGE_TO_SIZE(dma->npage) == vaddr) { + ret = iommu_map(iommu->domain, iova, + (phys_addr_t)pfn << PAGE_SHIFT, + npage << PAGE_SHIFT, prot); + if (ret) { + if (ret != -EBUSY || + map_try_harder(iommu, iova, pfn, npage, prot)) { + vfio_unpin_pages(pfn, npage, prot, true); + break; + } + } - dma->npage += npage; - iova = dma->iova; - vaddr = dma->vaddr; - npage = dma->npage; - size = NPAGE_TO_SIZE(npage); + size = npage << PAGE_SHIFT; - pdma = dma; + /* + * Check if we abut a region below - nothing below 0. + * This is the most likely case when mapping chunks of + * physically contiguous regions within a virtual address + * range. Update the abutting entry in place since iova + * doesn't change. + */ + if (likely(iova)) { + struct vfio_dma *tmp; + tmp = vfio_find_dma(iommu, iova - 1, 1); + if (tmp && tmp->prot == prot && + tmp->vaddr + tmp->size == vaddr) { + tmp->size += size; + iova = tmp->iova; + size = tmp->size; + vaddr = tmp->vaddr; + dma = tmp; + } + } + + /* + * Check if we abut a region above - nothing above ~0 + 1. + * If we abut above and below, remove and free. If only + * abut above, remove, modify, reinsert. + */ + if (likely(iova + size)) { + struct vfio_dma *tmp; + tmp = vfio_find_dma(iommu, iova + size, 1); + if (tmp && tmp->prot == prot && + tmp->vaddr == vaddr + size) { + vfio_remove_dma(iommu, tmp); + if (dma) { + dma->size += tmp->size; + kfree(tmp); + } else { + size += tmp->size; + tmp->size = size; + tmp->iova = iova; + tmp->vaddr = vaddr; + vfio_insert_dma(iommu, tmp); + dma = tmp; + } + } } - } - /* Check if we abut a region above - nothing above ~0 + 1 */ - if (iova + size) { - dma = vfio_find_dma(iommu, iova + size, 1); - if (dma && dma->prot == prot && - dma->vaddr == vaddr + size) { + if (!dma) { + dma = kzalloc(sizeof(*dma), GFP_KERNEL); + if (!dma) { + iommu_unmap(iommu->domain, iova, size); + vfio_unpin_pages(pfn, npage, prot, true); + ret = -ENOMEM; + break; + } - dma->npage += npage; + dma->size = size; dma->iova = iova; dma->vaddr = vaddr; - - /* - * If merged above and below, remove previously - * merged entry. New entry covers it. - */ - if (pdma) { - list_del(&pdma->next); - kfree(pdma); - } - pdma = dma; + dma->prot = prot; + vfio_insert_dma(iommu, dma); } } - /* Isolated, new region */ - if (!pdma) { - dma = kzalloc(sizeof *dma, GFP_KERNEL); - if (!dma) { - ret = -ENOMEM; - vfio_dma_unmap(iommu, iova, npage, prot); - goto out_lock; + if (ret) { + struct vfio_dma *tmp; + iova = map->iova; + size = map->size; + while ((tmp = vfio_find_dma(iommu, iova, size))) { + int r = vfio_remove_dma_overlap(iommu, iova, + &size, tmp); + if (WARN_ON(r || !size)) + break; } - - dma->npage = npage; - dma->iova = iova; - dma->vaddr = vaddr; - dma->prot = prot; - list_add(&dma->next, &iommu->dma_list); } -out_lock: mutex_unlock(&iommu->lock); return ret; } @@ -606,7 +775,7 @@ static void *vfio_iommu_type1_open(unsigned long arg) return ERR_PTR(-ENOMEM); INIT_LIST_HEAD(&iommu->group_list); - INIT_LIST_HEAD(&iommu->dma_list); + iommu->dma_list = RB_ROOT; mutex_init(&iommu->lock); /* @@ -640,7 +809,7 @@ static void vfio_iommu_type1_release(void *iommu_data) { struct vfio_iommu *iommu = iommu_data; struct vfio_group *group, *group_tmp; - struct vfio_dma *dma, *dma_tmp; + struct rb_node *node; list_for_each_entry_safe(group, group_tmp, &iommu->group_list, next) { iommu_detach_group(iommu->domain, group->iommu_group); @@ -648,10 +817,12 @@ static void vfio_iommu_type1_release(void *iommu_data) kfree(group); } - list_for_each_entry_safe(dma, dma_tmp, &iommu->dma_list, next) { - vfio_dma_unmap(iommu, dma->iova, dma->npage, dma->prot); - list_del(&dma->next); - kfree(dma); + while ((node = rb_first(&iommu->dma_list))) { + struct vfio_dma *dma = rb_entry(node, struct vfio_dma, node); + size_t size = dma->size; + vfio_remove_dma_overlap(iommu, dma->iova, &size, dma); + if (WARN_ON(!size)) + break; } iommu_domain_free(iommu->domain); @@ -706,6 +877,7 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, } else if (cmd == VFIO_IOMMU_UNMAP_DMA) { struct vfio_iommu_type1_dma_unmap unmap; + long ret; minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size); @@ -715,7 +887,11 @@ static long vfio_iommu_type1_ioctl(void *iommu_data, if (unmap.argsz < minsz || unmap.flags) return -EINVAL; - return vfio_dma_do_unmap(iommu, &unmap); + ret = vfio_dma_do_unmap(iommu, &unmap); + if (ret) + return ret; + + return copy_to_user((void __user *)arg, &unmap, minsz); } return -ENOTTY; diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index 8b9226d..017a1e8 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -1,6 +1,7 @@ config VHOST_NET tristate "Host kernel accelerator for virtio net" depends on NET && EVENTFD && (TUN || !TUN) && (MACVTAP || !MACVTAP) + select VHOST select VHOST_RING ---help--- This kernel module can be loaded in host kernel to accelerate @@ -13,6 +14,7 @@ config VHOST_NET config VHOST_SCSI tristate "VHOST_SCSI TCM fabric driver" depends on TARGET_CORE && EVENTFD && m + select VHOST select VHOST_RING default n ---help--- @@ -24,3 +26,9 @@ config VHOST_RING ---help--- This option is selected by any driver which needs to access the host side of a virtio ring. + +config VHOST + tristate + ---help--- + This option is selected by any driver which needs to access + the core of vhost. diff --git a/drivers/vhost/Makefile b/drivers/vhost/Makefile index 654e9afb..e0441c3 100644 --- a/drivers/vhost/Makefile +++ b/drivers/vhost/Makefile @@ -1,7 +1,8 @@ obj-$(CONFIG_VHOST_NET) += vhost_net.o -vhost_net-y := vhost.o net.o +vhost_net-y := net.o obj-$(CONFIG_VHOST_SCSI) += vhost_scsi.o vhost_scsi-y := scsi.o obj-$(CONFIG_VHOST_RING) += vringh.o +obj-$(CONFIG_VHOST) += vhost.o diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 8ca5ac7..027be91 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -168,7 +168,7 @@ static void vhost_net_clear_ubuf_info(struct vhost_net *n) } } -int vhost_net_set_ubuf_info(struct vhost_net *n) +static int vhost_net_set_ubuf_info(struct vhost_net *n) { bool zcopy; int i; @@ -189,7 +189,7 @@ err: return -ENOMEM; } -void vhost_net_vq_reset(struct vhost_net *n) +static void vhost_net_vq_reset(struct vhost_net *n) { int i; diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 7014202..4264840 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -49,7 +49,6 @@ #include <linux/llist.h> #include <linux/bitmap.h> -#include "vhost.c" #include "vhost.h" #define TCM_VHOST_VERSION "v0.1" @@ -116,7 +115,6 @@ struct tcm_vhost_nacl { struct se_node_acl se_node_acl; }; -struct vhost_scsi; struct tcm_vhost_tpg { /* Vhost port target portal group tag for TCM */ u16 tport_tpgt; @@ -218,7 +216,7 @@ static int iov_num_pages(struct iovec *iov) ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT; } -void tcm_vhost_done_inflight(struct kref *kref) +static void tcm_vhost_done_inflight(struct kref *kref) { struct vhost_scsi_inflight *inflight; @@ -329,11 +327,12 @@ static u32 tcm_vhost_get_default_depth(struct se_portal_group *se_tpg) return 1; } -static u32 tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg, - struct se_node_acl *se_nacl, - struct t10_pr_registration *pr_reg, - int *format_code, - unsigned char *buf) +static u32 +tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg, + struct se_node_acl *se_nacl, + struct t10_pr_registration *pr_reg, + int *format_code, + unsigned char *buf) { struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); @@ -359,10 +358,11 @@ static u32 tcm_vhost_get_pr_transport_id(struct se_portal_group *se_tpg, format_code, buf); } -static u32 tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg, - struct se_node_acl *se_nacl, - struct t10_pr_registration *pr_reg, - int *format_code) +static u32 +tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg, + struct se_node_acl *se_nacl, + struct t10_pr_registration *pr_reg, + int *format_code) { struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); @@ -388,10 +388,11 @@ static u32 tcm_vhost_get_pr_transport_id_len(struct se_portal_group *se_tpg, format_code); } -static char *tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg, - const char *buf, - u32 *out_tid_len, - char **port_nexus_ptr) +static char * +tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg, + const char *buf, + u32 *out_tid_len, + char **port_nexus_ptr) { struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); @@ -417,8 +418,8 @@ static char *tcm_vhost_parse_pr_out_transport_id(struct se_portal_group *se_tpg, port_nexus_ptr); } -static struct se_node_acl *tcm_vhost_alloc_fabric_acl( - struct se_portal_group *se_tpg) +static struct se_node_acl * +tcm_vhost_alloc_fabric_acl(struct se_portal_group *se_tpg) { struct tcm_vhost_nacl *nacl; @@ -431,8 +432,9 @@ static struct se_node_acl *tcm_vhost_alloc_fabric_acl( return &nacl->se_node_acl; } -static void tcm_vhost_release_fabric_acl(struct se_portal_group *se_tpg, - struct se_node_acl *se_nacl) +static void +tcm_vhost_release_fabric_acl(struct se_portal_group *se_tpg, + struct se_node_acl *se_nacl) { struct tcm_vhost_nacl *nacl = container_of(se_nacl, struct tcm_vhost_nacl, se_node_acl); @@ -491,28 +493,28 @@ static int tcm_vhost_get_cmd_state(struct se_cmd *se_cmd) return 0; } -static void vhost_scsi_complete_cmd(struct tcm_vhost_cmd *tv_cmd) +static void vhost_scsi_complete_cmd(struct tcm_vhost_cmd *cmd) { - struct vhost_scsi *vs = tv_cmd->tvc_vhost; + struct vhost_scsi *vs = cmd->tvc_vhost; - llist_add(&tv_cmd->tvc_completion_list, &vs->vs_completion_list); + llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list); vhost_work_queue(&vs->dev, &vs->vs_completion_work); } static int tcm_vhost_queue_data_in(struct se_cmd *se_cmd) { - struct tcm_vhost_cmd *tv_cmd = container_of(se_cmd, + struct tcm_vhost_cmd *cmd = container_of(se_cmd, struct tcm_vhost_cmd, tvc_se_cmd); - vhost_scsi_complete_cmd(tv_cmd); + vhost_scsi_complete_cmd(cmd); return 0; } static int tcm_vhost_queue_status(struct se_cmd *se_cmd) { - struct tcm_vhost_cmd *tv_cmd = container_of(se_cmd, + struct tcm_vhost_cmd *cmd = container_of(se_cmd, struct tcm_vhost_cmd, tvc_se_cmd); - vhost_scsi_complete_cmd(tv_cmd); + vhost_scsi_complete_cmd(cmd); return 0; } @@ -527,8 +529,9 @@ static void tcm_vhost_free_evt(struct vhost_scsi *vs, struct tcm_vhost_evt *evt) kfree(evt); } -static struct tcm_vhost_evt *tcm_vhost_allocate_evt(struct vhost_scsi *vs, - u32 event, u32 reason) +static struct tcm_vhost_evt * +tcm_vhost_allocate_evt(struct vhost_scsi *vs, + u32 event, u32 reason) { struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; struct tcm_vhost_evt *evt; @@ -552,28 +555,28 @@ static struct tcm_vhost_evt *tcm_vhost_allocate_evt(struct vhost_scsi *vs, return evt; } -static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) +static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *cmd) { - struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd; + struct se_cmd *se_cmd = &cmd->tvc_se_cmd; /* TODO locking against target/backend threads? */ transport_generic_free_cmd(se_cmd, 1); - if (tv_cmd->tvc_sgl_count) { + if (cmd->tvc_sgl_count) { u32 i; - for (i = 0; i < tv_cmd->tvc_sgl_count; i++) - put_page(sg_page(&tv_cmd->tvc_sgl[i])); + for (i = 0; i < cmd->tvc_sgl_count; i++) + put_page(sg_page(&cmd->tvc_sgl[i])); - kfree(tv_cmd->tvc_sgl); + kfree(cmd->tvc_sgl); } - tcm_vhost_put_inflight(tv_cmd->inflight); + tcm_vhost_put_inflight(cmd->inflight); - kfree(tv_cmd); + kfree(cmd); } -static void tcm_vhost_do_evt_work(struct vhost_scsi *vs, - struct tcm_vhost_evt *evt) +static void +tcm_vhost_do_evt_work(struct vhost_scsi *vs, struct tcm_vhost_evt *evt) { struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; struct virtio_scsi_event *event = &evt->event; @@ -652,7 +655,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) vs_completion_work); DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ); struct virtio_scsi_cmd_resp v_rsp; - struct tcm_vhost_cmd *tv_cmd; + struct tcm_vhost_cmd *cmd; struct llist_node *llnode; struct se_cmd *se_cmd; int ret, vq; @@ -660,32 +663,32 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) bitmap_zero(signal, VHOST_SCSI_MAX_VQ); llnode = llist_del_all(&vs->vs_completion_list); while (llnode) { - tv_cmd = llist_entry(llnode, struct tcm_vhost_cmd, + cmd = llist_entry(llnode, struct tcm_vhost_cmd, tvc_completion_list); llnode = llist_next(llnode); - se_cmd = &tv_cmd->tvc_se_cmd; + se_cmd = &cmd->tvc_se_cmd; pr_debug("%s tv_cmd %p resid %u status %#02x\n", __func__, - tv_cmd, se_cmd->residual_count, se_cmd->scsi_status); + cmd, se_cmd->residual_count, se_cmd->scsi_status); memset(&v_rsp, 0, sizeof(v_rsp)); v_rsp.resid = se_cmd->residual_count; /* TODO is status_qualifier field needed? */ v_rsp.status = se_cmd->scsi_status; v_rsp.sense_len = se_cmd->scsi_sense_length; - memcpy(v_rsp.sense, tv_cmd->tvc_sense_buf, + memcpy(v_rsp.sense, cmd->tvc_sense_buf, v_rsp.sense_len); - ret = copy_to_user(tv_cmd->tvc_resp, &v_rsp, sizeof(v_rsp)); + ret = copy_to_user(cmd->tvc_resp, &v_rsp, sizeof(v_rsp)); if (likely(ret == 0)) { struct vhost_scsi_virtqueue *q; - vhost_add_used(tv_cmd->tvc_vq, tv_cmd->tvc_vq_desc, 0); - q = container_of(tv_cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); + vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0); + q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); vq = q - vs->vqs; __set_bit(vq, signal); } else pr_err("Faulted on virtio_scsi_cmd_resp\n"); - vhost_scsi_free_cmd(tv_cmd); + vhost_scsi_free_cmd(cmd); } vq = -1; @@ -694,35 +697,35 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) vhost_signal(&vs->dev, &vs->vqs[vq].vq); } -static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd( - struct vhost_virtqueue *vq, - struct tcm_vhost_tpg *tv_tpg, - struct virtio_scsi_cmd_req *v_req, - u32 exp_data_len, - int data_direction) +static struct tcm_vhost_cmd * +vhost_scsi_allocate_cmd(struct vhost_virtqueue *vq, + struct tcm_vhost_tpg *tpg, + struct virtio_scsi_cmd_req *v_req, + u32 exp_data_len, + int data_direction) { - struct tcm_vhost_cmd *tv_cmd; + struct tcm_vhost_cmd *cmd; struct tcm_vhost_nexus *tv_nexus; - tv_nexus = tv_tpg->tpg_nexus; + tv_nexus = tpg->tpg_nexus; if (!tv_nexus) { pr_err("Unable to locate active struct tcm_vhost_nexus\n"); return ERR_PTR(-EIO); } - tv_cmd = kzalloc(sizeof(struct tcm_vhost_cmd), GFP_ATOMIC); - if (!tv_cmd) { + cmd = kzalloc(sizeof(struct tcm_vhost_cmd), GFP_ATOMIC); + if (!cmd) { pr_err("Unable to allocate struct tcm_vhost_cmd\n"); return ERR_PTR(-ENOMEM); } - tv_cmd->tvc_tag = v_req->tag; - tv_cmd->tvc_task_attr = v_req->task_attr; - tv_cmd->tvc_exp_data_len = exp_data_len; - tv_cmd->tvc_data_direction = data_direction; - tv_cmd->tvc_nexus = tv_nexus; - tv_cmd->inflight = tcm_vhost_get_inflight(vq); + cmd->tvc_tag = v_req->tag; + cmd->tvc_task_attr = v_req->task_attr; + cmd->tvc_exp_data_len = exp_data_len; + cmd->tvc_data_direction = data_direction; + cmd->tvc_nexus = tv_nexus; + cmd->inflight = tcm_vhost_get_inflight(vq); - return tv_cmd; + return cmd; } /* @@ -730,8 +733,11 @@ static struct tcm_vhost_cmd *vhost_scsi_allocate_cmd( * * Returns the number of scatterlist entries used or -errno on error. */ -static int vhost_scsi_map_to_sgl(struct scatterlist *sgl, - unsigned int sgl_count, struct iovec *iov, int write) +static int +vhost_scsi_map_to_sgl(struct scatterlist *sgl, + unsigned int sgl_count, + struct iovec *iov, + int write) { unsigned int npages = 0, pages_nr, offset, nbytes; struct scatterlist *sg = sgl; @@ -775,8 +781,11 @@ out: return ret; } -static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *tv_cmd, - struct iovec *iov, unsigned int niov, int write) +static int +vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *cmd, + struct iovec *iov, + unsigned int niov, + int write) { int ret; unsigned int i; @@ -792,25 +801,25 @@ static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *tv_cmd, /* TODO overflow checking */ - sg = kmalloc(sizeof(tv_cmd->tvc_sgl[0]) * sgl_count, GFP_ATOMIC); + sg = kmalloc(sizeof(cmd->tvc_sgl[0]) * sgl_count, GFP_ATOMIC); if (!sg) return -ENOMEM; pr_debug("%s sg %p sgl_count %u is_err %d\n", __func__, sg, sgl_count, !sg); sg_init_table(sg, sgl_count); - tv_cmd->tvc_sgl = sg; - tv_cmd->tvc_sgl_count = sgl_count; + cmd->tvc_sgl = sg; + cmd->tvc_sgl_count = sgl_count; pr_debug("Mapping %u iovecs for %u pages\n", niov, sgl_count); for (i = 0; i < niov; i++) { ret = vhost_scsi_map_to_sgl(sg, sgl_count, &iov[i], write); if (ret < 0) { - for (i = 0; i < tv_cmd->tvc_sgl_count; i++) - put_page(sg_page(&tv_cmd->tvc_sgl[i])); - kfree(tv_cmd->tvc_sgl); - tv_cmd->tvc_sgl = NULL; - tv_cmd->tvc_sgl_count = 0; + for (i = 0; i < cmd->tvc_sgl_count; i++) + put_page(sg_page(&cmd->tvc_sgl[i])); + kfree(cmd->tvc_sgl); + cmd->tvc_sgl = NULL; + cmd->tvc_sgl_count = 0; return ret; } @@ -822,15 +831,15 @@ static int vhost_scsi_map_iov_to_sgl(struct tcm_vhost_cmd *tv_cmd, static void tcm_vhost_submission_work(struct work_struct *work) { - struct tcm_vhost_cmd *tv_cmd = + struct tcm_vhost_cmd *cmd = container_of(work, struct tcm_vhost_cmd, work); struct tcm_vhost_nexus *tv_nexus; - struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd; + struct se_cmd *se_cmd = &cmd->tvc_se_cmd; struct scatterlist *sg_ptr, *sg_bidi_ptr = NULL; int rc, sg_no_bidi = 0; - if (tv_cmd->tvc_sgl_count) { - sg_ptr = tv_cmd->tvc_sgl; + if (cmd->tvc_sgl_count) { + sg_ptr = cmd->tvc_sgl; /* FIXME: Fix BIDI operation in tcm_vhost_submission_work() */ #if 0 if (se_cmd->se_cmd_flags & SCF_BIDI) { @@ -841,13 +850,13 @@ static void tcm_vhost_submission_work(struct work_struct *work) } else { sg_ptr = NULL; } - tv_nexus = tv_cmd->tvc_nexus; + tv_nexus = cmd->tvc_nexus; rc = target_submit_cmd_map_sgls(se_cmd, tv_nexus->tvn_se_sess, - tv_cmd->tvc_cdb, &tv_cmd->tvc_sense_buf[0], - tv_cmd->tvc_lun, tv_cmd->tvc_exp_data_len, - tv_cmd->tvc_task_attr, tv_cmd->tvc_data_direction, - 0, sg_ptr, tv_cmd->tvc_sgl_count, + cmd->tvc_cdb, &cmd->tvc_sense_buf[0], + cmd->tvc_lun, cmd->tvc_exp_data_len, + cmd->tvc_task_attr, cmd->tvc_data_direction, + 0, sg_ptr, cmd->tvc_sgl_count, sg_bidi_ptr, sg_no_bidi); if (rc < 0) { transport_send_check_condition_and_sense(se_cmd, @@ -856,8 +865,10 @@ static void tcm_vhost_submission_work(struct work_struct *work) } } -static void vhost_scsi_send_bad_target(struct vhost_scsi *vs, - struct vhost_virtqueue *vq, int head, unsigned out) +static void +vhost_scsi_send_bad_target(struct vhost_scsi *vs, + struct vhost_virtqueue *vq, + int head, unsigned out) { struct virtio_scsi_cmd_resp __user *resp; struct virtio_scsi_cmd_resp rsp; @@ -873,13 +884,13 @@ static void vhost_scsi_send_bad_target(struct vhost_scsi *vs, pr_err("Faulted on virtio_scsi_cmd_resp\n"); } -static void vhost_scsi_handle_vq(struct vhost_scsi *vs, - struct vhost_virtqueue *vq) +static void +vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq) { struct tcm_vhost_tpg **vs_tpg; struct virtio_scsi_cmd_req v_req; - struct tcm_vhost_tpg *tv_tpg; - struct tcm_vhost_cmd *tv_cmd; + struct tcm_vhost_tpg *tpg; + struct tcm_vhost_cmd *cmd; u32 exp_data_len, data_first, data_num, data_direction; unsigned out, in, i; int head, ret; @@ -964,10 +975,10 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs, /* Extract the tpgt */ target = v_req.lun[1]; - tv_tpg = ACCESS_ONCE(vs_tpg[target]); + tpg = ACCESS_ONCE(vs_tpg[target]); /* Target does not exist, fail the request */ - if (unlikely(!tv_tpg)) { + if (unlikely(!tpg)) { vhost_scsi_send_bad_target(vs, vq, head, out); continue; } @@ -976,46 +987,46 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs, for (i = 0; i < data_num; i++) exp_data_len += vq->iov[data_first + i].iov_len; - tv_cmd = vhost_scsi_allocate_cmd(vq, tv_tpg, &v_req, + cmd = vhost_scsi_allocate_cmd(vq, tpg, &v_req, exp_data_len, data_direction); - if (IS_ERR(tv_cmd)) { + if (IS_ERR(cmd)) { vq_err(vq, "vhost_scsi_allocate_cmd failed %ld\n", - PTR_ERR(tv_cmd)); + PTR_ERR(cmd)); goto err_cmd; } pr_debug("Allocated tv_cmd: %p exp_data_len: %d, data_direction" - ": %d\n", tv_cmd, exp_data_len, data_direction); + ": %d\n", cmd, exp_data_len, data_direction); - tv_cmd->tvc_vhost = vs; - tv_cmd->tvc_vq = vq; - tv_cmd->tvc_resp = vq->iov[out].iov_base; + cmd->tvc_vhost = vs; + cmd->tvc_vq = vq; + cmd->tvc_resp = vq->iov[out].iov_base; /* - * Copy in the recieved CDB descriptor into tv_cmd->tvc_cdb + * Copy in the recieved CDB descriptor into cmd->tvc_cdb * that will be used by tcm_vhost_new_cmd_map() and down into * target_setup_cmd_from_cdb() */ - memcpy(tv_cmd->tvc_cdb, v_req.cdb, TCM_VHOST_MAX_CDB_SIZE); + memcpy(cmd->tvc_cdb, v_req.cdb, TCM_VHOST_MAX_CDB_SIZE); /* * Check that the recieved CDB size does not exceeded our * hardcoded max for tcm_vhost */ /* TODO what if cdb was too small for varlen cdb header? */ - if (unlikely(scsi_command_size(tv_cmd->tvc_cdb) > + if (unlikely(scsi_command_size(cmd->tvc_cdb) > TCM_VHOST_MAX_CDB_SIZE)) { vq_err(vq, "Received SCSI CDB with command_size: %d that" " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", - scsi_command_size(tv_cmd->tvc_cdb), + scsi_command_size(cmd->tvc_cdb), TCM_VHOST_MAX_CDB_SIZE); goto err_free; } - tv_cmd->tvc_lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; + cmd->tvc_lun = ((v_req.lun[2] << 8) | v_req.lun[3]) & 0x3FFF; pr_debug("vhost_scsi got command opcode: %#02x, lun: %d\n", - tv_cmd->tvc_cdb[0], tv_cmd->tvc_lun); + cmd->tvc_cdb[0], cmd->tvc_lun); if (data_direction != DMA_NONE) { - ret = vhost_scsi_map_iov_to_sgl(tv_cmd, + ret = vhost_scsi_map_iov_to_sgl(cmd, &vq->iov[data_first], data_num, data_direction == DMA_TO_DEVICE); if (unlikely(ret)) { @@ -1029,22 +1040,22 @@ static void vhost_scsi_handle_vq(struct vhost_scsi *vs, * complete the virtio-scsi request in TCM callback context via * tcm_vhost_queue_data_in() and tcm_vhost_queue_status() */ - tv_cmd->tvc_vq_desc = head; + cmd->tvc_vq_desc = head; /* * Dispatch tv_cmd descriptor for cmwq execution in process * context provided by tcm_vhost_workqueue. This also ensures * tv_cmd is executed on the same kworker CPU as this vhost * thread to gain positive L2 cache locality effects.. */ - INIT_WORK(&tv_cmd->work, tcm_vhost_submission_work); - queue_work(tcm_vhost_workqueue, &tv_cmd->work); + INIT_WORK(&cmd->work, tcm_vhost_submission_work); + queue_work(tcm_vhost_workqueue, &cmd->work); } mutex_unlock(&vq->mutex); return; err_free: - vhost_scsi_free_cmd(tv_cmd); + vhost_scsi_free_cmd(cmd); err_cmd: vhost_scsi_send_bad_target(vs, vq, head, out); mutex_unlock(&vq->mutex); @@ -1055,8 +1066,12 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) pr_debug("%s: The handling func for control queue.\n", __func__); } -static void tcm_vhost_send_evt(struct vhost_scsi *vs, struct tcm_vhost_tpg *tpg, - struct se_lun *lun, u32 event, u32 reason) +static void +tcm_vhost_send_evt(struct vhost_scsi *vs, + struct tcm_vhost_tpg *tpg, + struct se_lun *lun, + u32 event, + u32 reason) { struct tcm_vhost_evt *evt; @@ -1146,12 +1161,12 @@ static void vhost_scsi_flush(struct vhost_scsi *vs) * The lock nesting rule is: * tcm_vhost_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex */ -static int vhost_scsi_set_endpoint( - struct vhost_scsi *vs, - struct vhost_scsi_target *t) +static int +vhost_scsi_set_endpoint(struct vhost_scsi *vs, + struct vhost_scsi_target *t) { struct tcm_vhost_tport *tv_tport; - struct tcm_vhost_tpg *tv_tpg; + struct tcm_vhost_tpg *tpg; struct tcm_vhost_tpg **vs_tpg; struct vhost_virtqueue *vq; int index, ret, i, len; @@ -1178,32 +1193,32 @@ static int vhost_scsi_set_endpoint( if (vs->vs_tpg) memcpy(vs_tpg, vs->vs_tpg, len); - list_for_each_entry(tv_tpg, &tcm_vhost_list, tv_tpg_list) { - mutex_lock(&tv_tpg->tv_tpg_mutex); - if (!tv_tpg->tpg_nexus) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); + list_for_each_entry(tpg, &tcm_vhost_list, tv_tpg_list) { + mutex_lock(&tpg->tv_tpg_mutex); + if (!tpg->tpg_nexus) { + mutex_unlock(&tpg->tv_tpg_mutex); continue; } - if (tv_tpg->tv_tpg_vhost_count != 0) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); + if (tpg->tv_tpg_vhost_count != 0) { + mutex_unlock(&tpg->tv_tpg_mutex); continue; } - tv_tport = tv_tpg->tport; + tv_tport = tpg->tport; if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) { - if (vs->vs_tpg && vs->vs_tpg[tv_tpg->tport_tpgt]) { + if (vs->vs_tpg && vs->vs_tpg[tpg->tport_tpgt]) { kfree(vs_tpg); - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); ret = -EEXIST; goto out; } - tv_tpg->tv_tpg_vhost_count++; - tv_tpg->vhost_scsi = vs; - vs_tpg[tv_tpg->tport_tpgt] = tv_tpg; + tpg->tv_tpg_vhost_count++; + tpg->vhost_scsi = vs; + vs_tpg[tpg->tport_tpgt] = tpg; smp_mb__after_atomic_inc(); match = true; } - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); } if (match) { @@ -1236,12 +1251,12 @@ out: return ret; } -static int vhost_scsi_clear_endpoint( - struct vhost_scsi *vs, - struct vhost_scsi_target *t) +static int +vhost_scsi_clear_endpoint(struct vhost_scsi *vs, + struct vhost_scsi_target *t) { struct tcm_vhost_tport *tv_tport; - struct tcm_vhost_tpg *tv_tpg; + struct tcm_vhost_tpg *tpg; struct vhost_virtqueue *vq; bool match = false; int index, ret, i; @@ -1264,30 +1279,30 @@ static int vhost_scsi_clear_endpoint( for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) { target = i; - tv_tpg = vs->vs_tpg[target]; - if (!tv_tpg) + tpg = vs->vs_tpg[target]; + if (!tpg) continue; - mutex_lock(&tv_tpg->tv_tpg_mutex); - tv_tport = tv_tpg->tport; + mutex_lock(&tpg->tv_tpg_mutex); + tv_tport = tpg->tport; if (!tv_tport) { ret = -ENODEV; goto err_tpg; } if (strcmp(tv_tport->tport_name, t->vhost_wwpn)) { - pr_warn("tv_tport->tport_name: %s, tv_tpg->tport_tpgt: %hu" + pr_warn("tv_tport->tport_name: %s, tpg->tport_tpgt: %hu" " does not match t->vhost_wwpn: %s, t->vhost_tpgt: %hu\n", - tv_tport->tport_name, tv_tpg->tport_tpgt, + tv_tport->tport_name, tpg->tport_tpgt, t->vhost_wwpn, t->vhost_tpgt); ret = -EINVAL; goto err_tpg; } - tv_tpg->tv_tpg_vhost_count--; - tv_tpg->vhost_scsi = NULL; + tpg->tv_tpg_vhost_count--; + tpg->vhost_scsi = NULL; vs->vs_tpg[target] = NULL; match = true; - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); } if (match) { for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { @@ -1311,7 +1326,7 @@ static int vhost_scsi_clear_endpoint( return 0; err_tpg: - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); err_dev: mutex_unlock(&vs->dev.mutex); mutex_unlock(&tcm_vhost_mutex); @@ -1338,68 +1353,70 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features) static int vhost_scsi_open(struct inode *inode, struct file *f) { - struct vhost_scsi *s; + struct vhost_scsi *vs; struct vhost_virtqueue **vqs; int r, i; - s = kzalloc(sizeof(*s), GFP_KERNEL); - if (!s) + vs = kzalloc(sizeof(*vs), GFP_KERNEL); + if (!vs) return -ENOMEM; vqs = kmalloc(VHOST_SCSI_MAX_VQ * sizeof(*vqs), GFP_KERNEL); if (!vqs) { - kfree(s); + kfree(vs); return -ENOMEM; } - vhost_work_init(&s->vs_completion_work, vhost_scsi_complete_cmd_work); - vhost_work_init(&s->vs_event_work, tcm_vhost_evt_work); + vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work); + vhost_work_init(&vs->vs_event_work, tcm_vhost_evt_work); - s->vs_events_nr = 0; - s->vs_events_missed = false; + vs->vs_events_nr = 0; + vs->vs_events_missed = false; - vqs[VHOST_SCSI_VQ_CTL] = &s->vqs[VHOST_SCSI_VQ_CTL].vq; - vqs[VHOST_SCSI_VQ_EVT] = &s->vqs[VHOST_SCSI_VQ_EVT].vq; - s->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick; - s->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick; + vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq; + vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; + vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick; + vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick; for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) { - vqs[i] = &s->vqs[i].vq; - s->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; + vqs[i] = &vs->vqs[i].vq; + vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; } - r = vhost_dev_init(&s->dev, vqs, VHOST_SCSI_MAX_VQ); + r = vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ); - tcm_vhost_init_inflight(s, NULL); + tcm_vhost_init_inflight(vs, NULL); if (r < 0) { kfree(vqs); - kfree(s); + kfree(vs); return r; } - f->private_data = s; + f->private_data = vs; return 0; } static int vhost_scsi_release(struct inode *inode, struct file *f) { - struct vhost_scsi *s = f->private_data; + struct vhost_scsi *vs = f->private_data; struct vhost_scsi_target t; - mutex_lock(&s->dev.mutex); - memcpy(t.vhost_wwpn, s->vs_vhost_wwpn, sizeof(t.vhost_wwpn)); - mutex_unlock(&s->dev.mutex); - vhost_scsi_clear_endpoint(s, &t); - vhost_dev_stop(&s->dev); - vhost_dev_cleanup(&s->dev, false); + mutex_lock(&vs->dev.mutex); + memcpy(t.vhost_wwpn, vs->vs_vhost_wwpn, sizeof(t.vhost_wwpn)); + mutex_unlock(&vs->dev.mutex); + vhost_scsi_clear_endpoint(vs, &t); + vhost_dev_stop(&vs->dev); + vhost_dev_cleanup(&vs->dev, false); /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */ - vhost_scsi_flush(s); - kfree(s->dev.vqs); - kfree(s); + vhost_scsi_flush(vs); + kfree(vs->dev.vqs); + kfree(vs); return 0; } -static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl, - unsigned long arg) +static long +vhost_scsi_ioctl(struct file *f, + unsigned int ioctl, + unsigned long arg) { struct vhost_scsi *vs = f->private_data; struct vhost_scsi_target backend; @@ -1515,8 +1532,9 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) return "Unknown"; } -static void tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg, - struct se_lun *lun, bool plug) +static void +tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg, + struct se_lun *lun, bool plug) { struct vhost_scsi *vs = tpg->vhost_scsi; @@ -1556,18 +1574,18 @@ static void tcm_vhost_hotunplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun) } static int tcm_vhost_port_link(struct se_portal_group *se_tpg, - struct se_lun *lun) + struct se_lun *lun) { - struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, + struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); mutex_lock(&tcm_vhost_mutex); - mutex_lock(&tv_tpg->tv_tpg_mutex); - tv_tpg->tv_tpg_port_count++; - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_lock(&tpg->tv_tpg_mutex); + tpg->tv_tpg_port_count++; + mutex_unlock(&tpg->tv_tpg_mutex); - tcm_vhost_hotplug(tv_tpg, lun); + tcm_vhost_hotplug(tpg, lun); mutex_unlock(&tcm_vhost_mutex); @@ -1575,26 +1593,26 @@ static int tcm_vhost_port_link(struct se_portal_group *se_tpg, } static void tcm_vhost_port_unlink(struct se_portal_group *se_tpg, - struct se_lun *lun) + struct se_lun *lun) { - struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, + struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); mutex_lock(&tcm_vhost_mutex); - mutex_lock(&tv_tpg->tv_tpg_mutex); - tv_tpg->tv_tpg_port_count--; - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_lock(&tpg->tv_tpg_mutex); + tpg->tv_tpg_port_count--; + mutex_unlock(&tpg->tv_tpg_mutex); - tcm_vhost_hotunplug(tv_tpg, lun); + tcm_vhost_hotunplug(tpg, lun); mutex_unlock(&tcm_vhost_mutex); } -static struct se_node_acl *tcm_vhost_make_nodeacl( - struct se_portal_group *se_tpg, - struct config_group *group, - const char *name) +static struct se_node_acl * +tcm_vhost_make_nodeacl(struct se_portal_group *se_tpg, + struct config_group *group, + const char *name) { struct se_node_acl *se_nacl, *se_nacl_new; struct tcm_vhost_nacl *nacl; @@ -1635,23 +1653,23 @@ static void tcm_vhost_drop_nodeacl(struct se_node_acl *se_acl) kfree(nacl); } -static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg, - const char *name) +static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tpg, + const char *name) { struct se_portal_group *se_tpg; struct tcm_vhost_nexus *tv_nexus; - mutex_lock(&tv_tpg->tv_tpg_mutex); - if (tv_tpg->tpg_nexus) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); - pr_debug("tv_tpg->tpg_nexus already exists\n"); + mutex_lock(&tpg->tv_tpg_mutex); + if (tpg->tpg_nexus) { + mutex_unlock(&tpg->tv_tpg_mutex); + pr_debug("tpg->tpg_nexus already exists\n"); return -EEXIST; } - se_tpg = &tv_tpg->se_tpg; + se_tpg = &tpg->se_tpg; tv_nexus = kzalloc(sizeof(struct tcm_vhost_nexus), GFP_KERNEL); if (!tv_nexus) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); pr_err("Unable to allocate struct tcm_vhost_nexus\n"); return -ENOMEM; } @@ -1660,7 +1678,7 @@ static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg, */ tv_nexus->tvn_se_sess = transport_init_session(); if (IS_ERR(tv_nexus->tvn_se_sess)) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); kfree(tv_nexus); return -ENOMEM; } @@ -1672,7 +1690,7 @@ static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg, tv_nexus->tvn_se_sess->se_node_acl = core_tpg_check_initiator_node_acl( se_tpg, (unsigned char *)name); if (!tv_nexus->tvn_se_sess->se_node_acl) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); pr_debug("core_tpg_check_initiator_node_acl() failed" " for %s\n", name); transport_free_session(tv_nexus->tvn_se_sess); @@ -1685,9 +1703,9 @@ static int tcm_vhost_make_nexus(struct tcm_vhost_tpg *tv_tpg, */ __transport_register_session(se_tpg, tv_nexus->tvn_se_sess->se_node_acl, tv_nexus->tvn_se_sess, tv_nexus); - tv_tpg->tpg_nexus = tv_nexus; + tpg->tpg_nexus = tv_nexus; - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); return 0; } @@ -1740,40 +1758,40 @@ static int tcm_vhost_drop_nexus(struct tcm_vhost_tpg *tpg) } static ssize_t tcm_vhost_tpg_show_nexus(struct se_portal_group *se_tpg, - char *page) + char *page) { - struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, + struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); struct tcm_vhost_nexus *tv_nexus; ssize_t ret; - mutex_lock(&tv_tpg->tv_tpg_mutex); - tv_nexus = tv_tpg->tpg_nexus; + mutex_lock(&tpg->tv_tpg_mutex); + tv_nexus = tpg->tpg_nexus; if (!tv_nexus) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); return -ENODEV; } ret = snprintf(page, PAGE_SIZE, "%s\n", tv_nexus->tvn_se_sess->se_node_acl->initiatorname); - mutex_unlock(&tv_tpg->tv_tpg_mutex); + mutex_unlock(&tpg->tv_tpg_mutex); return ret; } static ssize_t tcm_vhost_tpg_store_nexus(struct se_portal_group *se_tpg, - const char *page, - size_t count) + const char *page, + size_t count) { - struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, + struct tcm_vhost_tpg *tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); - struct tcm_vhost_tport *tport_wwn = tv_tpg->tport; + struct tcm_vhost_tport *tport_wwn = tpg->tport; unsigned char i_port[TCM_VHOST_NAMELEN], *ptr, *port_ptr; int ret; /* * Shutdown the active I_T nexus if 'NULL' is passed.. */ if (!strncmp(page, "NULL", 4)) { - ret = tcm_vhost_drop_nexus(tv_tpg); + ret = tcm_vhost_drop_nexus(tpg); return (!ret) ? count : ret; } /* @@ -1831,7 +1849,7 @@ check_newline: if (i_port[strlen(i_port)-1] == '\n') i_port[strlen(i_port)-1] = '\0'; - ret = tcm_vhost_make_nexus(tv_tpg, port_ptr); + ret = tcm_vhost_make_nexus(tpg, port_ptr); if (ret < 0) return ret; @@ -1845,9 +1863,10 @@ static struct configfs_attribute *tcm_vhost_tpg_attrs[] = { NULL, }; -static struct se_portal_group *tcm_vhost_make_tpg(struct se_wwn *wwn, - struct config_group *group, - const char *name) +static struct se_portal_group * +tcm_vhost_make_tpg(struct se_wwn *wwn, + struct config_group *group, + const char *name) { struct tcm_vhost_tport *tport = container_of(wwn, struct tcm_vhost_tport, tport_wwn); @@ -1903,9 +1922,10 @@ static void tcm_vhost_drop_tpg(struct se_portal_group *se_tpg) kfree(tpg); } -static struct se_wwn *tcm_vhost_make_tport(struct target_fabric_configfs *tf, - struct config_group *group, - const char *name) +static struct se_wwn * +tcm_vhost_make_tport(struct target_fabric_configfs *tf, + struct config_group *group, + const char *name) { struct tcm_vhost_tport *tport; char *ptr; @@ -1975,9 +1995,9 @@ static void tcm_vhost_drop_tport(struct se_wwn *wwn) kfree(tport); } -static ssize_t tcm_vhost_wwn_show_attr_version( - struct target_fabric_configfs *tf, - char *page) +static ssize_t +tcm_vhost_wwn_show_attr_version(struct target_fabric_configfs *tf, + char *page) { return sprintf(page, "TCM_VHOST fabric module %s on %s/%s" "on "UTS_RELEASE"\n", TCM_VHOST_VERSION, utsname()->sysname, diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c index 1ee45bc..a73ea21 100644 --- a/drivers/vhost/test.c +++ b/drivers/vhost/test.c @@ -18,7 +18,7 @@ #include <linux/slab.h> #include "test.h" -#include "vhost.c" +#include "vhost.h" /* Max number of bytes transferred before requeueing the job. * Using this limit prevents one virtqueue from starving others. */ @@ -38,17 +38,19 @@ struct vhost_test { * read-size critical section for our kind of RCU. */ static void handle_vq(struct vhost_test *n) { - struct vhost_virtqueue *vq = &n->dev.vqs[VHOST_TEST_VQ]; + struct vhost_virtqueue *vq = &n->vqs[VHOST_TEST_VQ]; unsigned out, in; int head; size_t len, total_len = 0; void *private; - private = rcu_dereference_check(vq->private_data, 1); - if (!private) + mutex_lock(&vq->mutex); + private = vq->private_data; + if (!private) { + mutex_unlock(&vq->mutex); return; + } - mutex_lock(&vq->mutex); vhost_disable_notify(&n->dev, vq); for (;;) { @@ -102,15 +104,23 @@ static int vhost_test_open(struct inode *inode, struct file *f) { struct vhost_test *n = kmalloc(sizeof *n, GFP_KERNEL); struct vhost_dev *dev; + struct vhost_virtqueue **vqs; int r; if (!n) return -ENOMEM; + vqs = kmalloc(VHOST_TEST_VQ_MAX * sizeof(*vqs), GFP_KERNEL); + if (!vqs) { + kfree(n); + return -ENOMEM; + } dev = &n->dev; + vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ]; n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; - r = vhost_dev_init(dev, n->vqs, VHOST_TEST_VQ_MAX); + r = vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX); if (r < 0) { + kfree(vqs); kfree(n); return r; } @@ -126,9 +136,8 @@ static void *vhost_test_stop_vq(struct vhost_test *n, void *private; mutex_lock(&vq->mutex); - private = rcu_dereference_protected(vq->private_data, - lockdep_is_held(&vq->mutex)); - rcu_assign_pointer(vq->private_data, NULL); + private = vq->private_data; + vq->private_data = NULL; mutex_unlock(&vq->mutex); return private; } @@ -140,7 +149,7 @@ static void vhost_test_stop(struct vhost_test *n, void **privatep) static void vhost_test_flush_vq(struct vhost_test *n, int index) { - vhost_poll_flush(&n->dev.vqs[index].poll); + vhost_poll_flush(&n->vqs[index].poll); } static void vhost_test_flush(struct vhost_test *n) @@ -268,14 +277,14 @@ static long vhost_test_ioctl(struct file *f, unsigned int ioctl, return -EFAULT; return vhost_test_run(n, test); case VHOST_GET_FEATURES: - features = VHOST_NET_FEATURES; + features = VHOST_FEATURES; if (copy_to_user(featurep, &features, sizeof features)) return -EFAULT; return 0; case VHOST_SET_FEATURES: if (copy_from_user(&features, featurep, sizeof features)) return -EFAULT; - if (features & ~VHOST_NET_FEATURES) + if (features & ~VHOST_FEATURES) return -EOPNOTSUPP; return vhost_test_set_features(n, features); case VHOST_RESET_OWNER: diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 60aa5ad..e58cf00 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -25,6 +25,7 @@ #include <linux/slab.h> #include <linux/kthread.h> #include <linux/cgroup.h> +#include <linux/module.h> #include "vhost.h" @@ -66,6 +67,7 @@ void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn) work->flushing = 0; work->queue_seq = work->done_seq = 0; } +EXPORT_SYMBOL_GPL(vhost_work_init); /* Init poll structure */ void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, @@ -79,6 +81,7 @@ void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, vhost_work_init(&poll->work, fn); } +EXPORT_SYMBOL_GPL(vhost_poll_init); /* Start polling a file. We add ourselves to file's wait queue. The caller must * keep a reference to a file until after vhost_poll_stop is called. */ @@ -101,6 +104,7 @@ int vhost_poll_start(struct vhost_poll *poll, struct file *file) return ret; } +EXPORT_SYMBOL_GPL(vhost_poll_start); /* Stop polling a file. After this function returns, it becomes safe to drop the * file reference. You must also flush afterwards. */ @@ -111,6 +115,7 @@ void vhost_poll_stop(struct vhost_poll *poll) poll->wqh = NULL; } } +EXPORT_SYMBOL_GPL(vhost_poll_stop); static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work, unsigned seq) @@ -123,7 +128,7 @@ static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work, return left <= 0; } -static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) +void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) { unsigned seq; int flushing; @@ -138,6 +143,7 @@ static void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) spin_unlock_irq(&dev->work_lock); BUG_ON(flushing < 0); } +EXPORT_SYMBOL_GPL(vhost_work_flush); /* Flush any work that has been scheduled. When calling this, don't hold any * locks that are also used by the callback. */ @@ -145,6 +151,7 @@ void vhost_poll_flush(struct vhost_poll *poll) { vhost_work_flush(poll->dev, &poll->work); } +EXPORT_SYMBOL_GPL(vhost_poll_flush); void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) { @@ -158,11 +165,13 @@ void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) } spin_unlock_irqrestore(&dev->work_lock, flags); } +EXPORT_SYMBOL_GPL(vhost_work_queue); void vhost_poll_queue(struct vhost_poll *poll) { vhost_work_queue(poll->dev, &poll->work); } +EXPORT_SYMBOL_GPL(vhost_poll_queue); static void vhost_vq_reset(struct vhost_dev *dev, struct vhost_virtqueue *vq) @@ -251,17 +260,16 @@ static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) /* Helper to allocate iovec buffers for all vqs. */ static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) { + struct vhost_virtqueue *vq; int i; for (i = 0; i < dev->nvqs; ++i) { - dev->vqs[i]->indirect = kmalloc(sizeof *dev->vqs[i]->indirect * - UIO_MAXIOV, GFP_KERNEL); - dev->vqs[i]->log = kmalloc(sizeof *dev->vqs[i]->log * UIO_MAXIOV, - GFP_KERNEL); - dev->vqs[i]->heads = kmalloc(sizeof *dev->vqs[i]->heads * - UIO_MAXIOV, GFP_KERNEL); - if (!dev->vqs[i]->indirect || !dev->vqs[i]->log || - !dev->vqs[i]->heads) + vq = dev->vqs[i]; + vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV, + GFP_KERNEL); + vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL); + vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL); + if (!vq->indirect || !vq->log || !vq->heads) goto err_nomem; } return 0; @@ -283,6 +291,7 @@ static void vhost_dev_free_iovecs(struct vhost_dev *dev) long vhost_dev_init(struct vhost_dev *dev, struct vhost_virtqueue **vqs, int nvqs) { + struct vhost_virtqueue *vq; int i; dev->vqs = vqs; @@ -297,19 +306,21 @@ long vhost_dev_init(struct vhost_dev *dev, dev->worker = NULL; for (i = 0; i < dev->nvqs; ++i) { - dev->vqs[i]->log = NULL; - dev->vqs[i]->indirect = NULL; - dev->vqs[i]->heads = NULL; - dev->vqs[i]->dev = dev; - mutex_init(&dev->vqs[i]->mutex); - vhost_vq_reset(dev, dev->vqs[i]); - if (dev->vqs[i]->handle_kick) - vhost_poll_init(&dev->vqs[i]->poll, - dev->vqs[i]->handle_kick, POLLIN, dev); + vq = dev->vqs[i]; + vq->log = NULL; + vq->indirect = NULL; + vq->heads = NULL; + vq->dev = dev; + mutex_init(&vq->mutex); + vhost_vq_reset(dev, vq); + if (vq->handle_kick) + vhost_poll_init(&vq->poll, vq->handle_kick, + POLLIN, dev); } return 0; } +EXPORT_SYMBOL_GPL(vhost_dev_init); /* Caller should have device mutex */ long vhost_dev_check_owner(struct vhost_dev *dev) @@ -317,6 +328,7 @@ long vhost_dev_check_owner(struct vhost_dev *dev) /* Are you the owner? If not, I don't think you mean to do that */ return dev->mm == current->mm ? 0 : -EPERM; } +EXPORT_SYMBOL_GPL(vhost_dev_check_owner); struct vhost_attach_cgroups_struct { struct vhost_work work; @@ -348,6 +360,7 @@ bool vhost_dev_has_owner(struct vhost_dev *dev) { return dev->mm; } +EXPORT_SYMBOL_GPL(vhost_dev_has_owner); /* Caller should have device mutex */ long vhost_dev_set_owner(struct vhost_dev *dev) @@ -391,11 +404,13 @@ err_worker: err_mm: return err; } +EXPORT_SYMBOL_GPL(vhost_dev_set_owner); struct vhost_memory *vhost_dev_reset_owner_prepare(void) { return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL); } +EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); /* Caller should have device mutex */ void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory) @@ -406,6 +421,7 @@ void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory) memory->nregions = 0; RCU_INIT_POINTER(dev->memory, memory); } +EXPORT_SYMBOL_GPL(vhost_dev_reset_owner); void vhost_dev_stop(struct vhost_dev *dev) { @@ -418,6 +434,7 @@ void vhost_dev_stop(struct vhost_dev *dev) } } } +EXPORT_SYMBOL_GPL(vhost_dev_stop); /* Caller should have device mutex if and only if locked is set */ void vhost_dev_cleanup(struct vhost_dev *dev, bool locked) @@ -458,6 +475,7 @@ void vhost_dev_cleanup(struct vhost_dev *dev, bool locked) mmput(dev->mm); dev->mm = NULL; } +EXPORT_SYMBOL_GPL(vhost_dev_cleanup); static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz) { @@ -543,6 +561,7 @@ int vhost_log_access_ok(struct vhost_dev *dev) lockdep_is_held(&dev->mutex)); return memory_access_ok(dev, mp, 1); } +EXPORT_SYMBOL_GPL(vhost_log_access_ok); /* Verify access for write logging. */ /* Caller should have vq mutex and device mutex */ @@ -568,6 +587,7 @@ int vhost_vq_access_ok(struct vhost_virtqueue *vq) return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) && vq_log_access_ok(vq->dev, vq, vq->log_base); } +EXPORT_SYMBOL_GPL(vhost_vq_access_ok); static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) { @@ -797,6 +817,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp) vhost_poll_flush(&vq->poll); return r; } +EXPORT_SYMBOL_GPL(vhost_vring_ioctl); /* Caller must have device mutex */ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) @@ -877,6 +898,7 @@ long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) done: return r; } +EXPORT_SYMBOL_GPL(vhost_dev_ioctl); static const struct vhost_memory_region *find_region(struct vhost_memory *mem, __u64 addr, __u32 len) @@ -968,6 +990,7 @@ int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, BUG(); return 0; } +EXPORT_SYMBOL_GPL(vhost_log_write); static int vhost_update_used_flags(struct vhost_virtqueue *vq) { @@ -1019,6 +1042,7 @@ int vhost_init_used(struct vhost_virtqueue *vq) vq->signalled_used_valid = false; return get_user(vq->last_used_idx, &vq->used->idx); } +EXPORT_SYMBOL_GPL(vhost_init_used); static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len, struct iovec iov[], int iov_size) @@ -1295,12 +1319,14 @@ int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq, BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); return head; } +EXPORT_SYMBOL_GPL(vhost_get_vq_desc); /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) { vq->last_avail_idx -= n; } +EXPORT_SYMBOL_GPL(vhost_discard_vq_desc); /* After we've used one of their buffers, we tell them about it. We'll then * want to notify the guest, using eventfd. */ @@ -1349,6 +1375,7 @@ int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) vq->signalled_used_valid = false; return 0; } +EXPORT_SYMBOL_GPL(vhost_add_used); static int __vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, @@ -1418,6 +1445,7 @@ int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, } return r; } +EXPORT_SYMBOL_GPL(vhost_add_used_n); static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) { @@ -1462,6 +1490,7 @@ void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) if (vq->call_ctx && vhost_notify(dev, vq)) eventfd_signal(vq->call_ctx, 1); } +EXPORT_SYMBOL_GPL(vhost_signal); /* And here's the combo meal deal. Supersize me! */ void vhost_add_used_and_signal(struct vhost_dev *dev, @@ -1471,6 +1500,7 @@ void vhost_add_used_and_signal(struct vhost_dev *dev, vhost_add_used(vq, head, len); vhost_signal(dev, vq); } +EXPORT_SYMBOL_GPL(vhost_add_used_and_signal); /* multi-buffer version of vhost_add_used_and_signal */ void vhost_add_used_and_signal_n(struct vhost_dev *dev, @@ -1480,6 +1510,7 @@ void vhost_add_used_and_signal_n(struct vhost_dev *dev, vhost_add_used_n(vq, heads, count); vhost_signal(dev, vq); } +EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); /* OK, now we need to know about added descriptors. */ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) @@ -1517,6 +1548,7 @@ bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) return avail_idx != vq->avail_idx; } +EXPORT_SYMBOL_GPL(vhost_enable_notify); /* We don't need to be notified again. */ void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) @@ -1533,3 +1565,21 @@ void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) &vq->used->flags, r); } } +EXPORT_SYMBOL_GPL(vhost_disable_notify); + +static int __init vhost_init(void) +{ + return 0; +} + +static void __exit vhost_exit(void) +{ +} + +module_init(vhost_init); +module_exit(vhost_exit); + +MODULE_VERSION("0.0.1"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Michael S. Tsirkin"); +MODULE_DESCRIPTION("Host kernel accelerator for virtio"); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 64adcf9..42298cd 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -46,6 +46,8 @@ int vhost_poll_start(struct vhost_poll *poll, struct file *file); void vhost_poll_stop(struct vhost_poll *poll); void vhost_poll_flush(struct vhost_poll *poll); void vhost_poll_queue(struct vhost_poll *poll); +void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work); +long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp); struct vhost_log { u64 addr; diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 0098810..1f572c0 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -192,7 +192,8 @@ static void leak_balloon(struct virtio_balloon *vb, size_t num) * virtio_has_feature(vdev, VIRTIO_BALLOON_F_MUST_TELL_HOST); * is true, we *have* to do it in this order */ - tell_host(vb, vb->deflate_vq); + if (vb->num_pfns != 0) + tell_host(vb, vb->deflate_vq); mutex_unlock(&vb->balloon_lock); release_pages_by_pfn(vb->pfns, vb->num_pfns); } diff --git a/drivers/virtio/virtio_pci.c b/drivers/virtio/virtio_pci.c index a7ce730..1aba255 100644 --- a/drivers/virtio/virtio_pci.c +++ b/drivers/virtio/virtio_pci.c @@ -289,9 +289,9 @@ static void vp_free_vectors(struct virtio_device *vdev) pci_disable_msix(vp_dev->pci_dev); vp_dev->msix_enabled = 0; - vp_dev->msix_vectors = 0; } + vp_dev->msix_vectors = 0; vp_dev->msix_used_vectors = 0; kfree(vp_dev->msix_names); vp_dev->msix_names = NULL; @@ -309,6 +309,8 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, unsigned i, v; int err = -ENOMEM; + vp_dev->msix_vectors = nvectors; + vp_dev->msix_entries = kmalloc(nvectors * sizeof *vp_dev->msix_entries, GFP_KERNEL); if (!vp_dev->msix_entries) @@ -336,7 +338,6 @@ static int vp_request_msix_vectors(struct virtio_device *vdev, int nvectors, err = -ENOSPC; if (err) goto error; - vp_dev->msix_vectors = nvectors; vp_dev->msix_enabled = 1; /* Set the vector used for configuration */ |