diff options
78 files changed, 891 insertions, 1433 deletions
diff --git a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt index cd29083..48ffb38 100644 --- a/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt +++ b/Documentation/devicetree/bindings/iommu/renesas,ipmmu-vmsa.txt @@ -7,7 +7,15 @@ connected to the IPMMU through a port called micro-TLB. Required Properties: - - compatible: Must contain "renesas,ipmmu-vmsa". + - compatible: Must contain SoC-specific and generic entries from below. + + - "renesas,ipmmu-r8a73a4" for the R8A73A4 (R-Mobile APE6) IPMMU. + - "renesas,ipmmu-r8a7790" for the R8A7790 (R-Car H2) IPMMU. + - "renesas,ipmmu-r8a7791" for the R8A7791 (R-Car M2-W) IPMMU. + - "renesas,ipmmu-r8a7793" for the R8A7793 (R-Car M2-N) IPMMU. + - "renesas,ipmmu-r8a7794" for the R8A7794 (R-Car E2) IPMMU. + - "renesas,ipmmu-vmsa" for generic R-Car Gen2 VMSA-compatible IPMMU. + - reg: Base address and size of the IPMMU registers. - interrupts: Specifiers for the MMU fault interrupts. For instances that support secure mode two interrupts must be specified, for non-secure and @@ -27,7 +35,7 @@ node with the following property: Example: R8A7791 IPMMU-MX and VSP1-D0 bus master ipmmu_mx: mmu@fe951000 { - compatible = "renasas,ipmmu-vmsa"; + compatible = "renasas,ipmmu-r8a7791", "renasas,ipmmu-vmsa"; reg = <0 0xfe951000 0 0x1000>; interrupts = <0 222 IRQ_TYPE_LEVEL_HIGH>, <0 221 IRQ_TYPE_LEVEL_HIGH>; diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt index a61be39..904ee42 100644 --- a/Documentation/memory-barriers.txt +++ b/Documentation/memory-barriers.txt @@ -1655,17 +1655,18 @@ macro is a good place to start looking. SMP memory barriers are reduced to compiler barriers on uniprocessor compiled systems because it is assumed that a CPU will appear to be self-consistent, and will order overlapping accesses correctly with respect to itself. +However, see the subsection on "Virtual Machine Guests" below. [!] Note that SMP memory barriers _must_ be used to control the ordering of references to shared memory on SMP systems, though the use of locking instead is sufficient. Mandatory barriers should not be used to control SMP effects, since mandatory -barriers unnecessarily impose overhead on UP systems. They may, however, be -used to control MMIO effects on accesses through relaxed memory I/O windows. -These are required even on non-SMP systems as they affect the order in which -memory operations appear to a device by prohibiting both the compiler and the -CPU from reordering them. +barriers impose unnecessary overhead on both SMP and UP systems. They may, +however, be used to control MMIO effects on accesses through relaxed memory I/O +windows. These barriers are required even on non-SMP systems as they affect +the order in which memory operations appear to a device by prohibiting both the +compiler and the CPU from reordering them. There are some more advanced barrier functions: @@ -2948,6 +2949,23 @@ The Alpha defines the Linux kernel's memory barrier model. See the subsection on "Cache Coherency" above. +VIRTUAL MACHINE GUESTS +------------------- + +Guests running within virtual machines might be affected by SMP effects even if +the guest itself is compiled without SMP support. This is an artifact of +interfacing with an SMP host while running an UP kernel. Using mandatory +barriers for this use-case would be possible but is often suboptimal. + +To handle this case optimally, low-level virt_mb() etc macros are available. +These have the same effect as smp_mb() etc when SMP is enabled, but generate +identical code for SMP and non-SMP systems. For example, virtual machine guests +should use virt_mb() rather than smp_mb() when synchronizing against a +(possibly SMP) host. + +These are equivalent to smp_mb() etc counterparts in all other respects, +in particular, they do not control MMIO effects: to control +MMIO effects, use mandatory barriers. ============ EXAMPLE USES diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index 3ff5642..112cc1a 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h @@ -60,38 +60,11 @@ extern void arm_heavy_mb(void); #define dma_wmb() barrier() #endif -#ifndef CONFIG_SMP -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#else -#define smp_mb() dmb(ish) -#define smp_rmb() smp_mb() -#define smp_wmb() dmb(ishst) -#endif - -#define smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - smp_mb(); \ - WRITE_ONCE(*p, v); \ -} while (0) - -#define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - smp_mb(); \ - ___p1; \ -}) - -#define read_barrier_depends() do { } while(0) -#define smp_read_barrier_depends() do { } while(0) - -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) +#define __smp_mb() dmb(ish) +#define __smp_rmb() __smp_mb() +#define __smp_wmb() dmb(ishst) -#define smp_mb__before_atomic() smp_mb() -#define smp_mb__after_atomic() smp_mb() +#include <asm-generic/barrier.h> #endif /* !__ASSEMBLY__ */ #endif /* __ASM_BARRIER_H */ diff --git a/arch/arm64/include/asm/barrier.h b/arch/arm64/include/asm/barrier.h index 9622eb4..dae5c49 100644 --- a/arch/arm64/include/asm/barrier.h +++ b/arch/arm64/include/asm/barrier.h @@ -35,11 +35,11 @@ #define dma_rmb() dmb(oshld) #define dma_wmb() dmb(oshst) -#define smp_mb() dmb(ish) -#define smp_rmb() dmb(ishld) -#define smp_wmb() dmb(ishst) +#define __smp_mb() dmb(ish) +#define __smp_rmb() dmb(ishld) +#define __smp_wmb() dmb(ishst) -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ switch (sizeof(*p)) { \ @@ -62,7 +62,7 @@ do { \ } \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ union { typeof(*p) __val; char __c[1]; } __u; \ compiletime_assert_atomic_type(*p); \ @@ -91,14 +91,7 @@ do { \ __u.__val; \ }) -#define read_barrier_depends() do { } while(0) -#define smp_read_barrier_depends() do { } while(0) - -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) -#define nop() asm volatile("nop"); - -#define smp_mb__before_atomic() smp_mb() -#define smp_mb__after_atomic() smp_mb() +#include <asm-generic/barrier.h> #endif /* __ASSEMBLY__ */ diff --git a/arch/blackfin/include/asm/barrier.h b/arch/blackfin/include/asm/barrier.h index dfb66fe..7cca51c 100644 --- a/arch/blackfin/include/asm/barrier.h +++ b/arch/blackfin/include/asm/barrier.h @@ -78,8 +78,8 @@ #endif /* !CONFIG_SMP */ -#define smp_mb__before_atomic() barrier() -#define smp_mb__after_atomic() barrier() +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() #include <asm-generic/barrier.h> diff --git a/arch/ia64/include/asm/barrier.h b/arch/ia64/include/asm/barrier.h index 209c4b8..588f161 100644 --- a/arch/ia64/include/asm/barrier.h +++ b/arch/ia64/include/asm/barrier.h @@ -42,34 +42,24 @@ #define dma_rmb() mb() #define dma_wmb() mb() -#ifdef CONFIG_SMP -# define smp_mb() mb() -#else -# define smp_mb() barrier() -#endif +# define __smp_mb() mb() -#define smp_rmb() smp_mb() -#define smp_wmb() smp_mb() - -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) - -#define smp_mb__before_atomic() barrier() -#define smp_mb__after_atomic() barrier() +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() /* * IA64 GCC turns volatile stores into st.rel and volatile loads into ld.acq no * need for asm trickery! */ -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ WRITE_ONCE(*p, v); \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ @@ -77,12 +67,12 @@ do { \ ___p1; \ }) -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) - /* * The group barrier in front of the rsm & ssm are necessary to ensure * that none of the previous instructions in the same group are * affected by the rsm/ssm. */ +#include <asm-generic/barrier.h> + #endif /* _ASM_IA64_BARRIER_H */ diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index d2fae05..90fde5b 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -256,7 +256,7 @@ set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask) } static void -nop (struct irq_data *data) +iosapic_nop (struct irq_data *data) { /* do nothing... */ } @@ -415,7 +415,7 @@ iosapic_unmask_level_irq (struct irq_data *data) #define iosapic_shutdown_level_irq mask_irq #define iosapic_enable_level_irq unmask_irq #define iosapic_disable_level_irq mask_irq -#define iosapic_ack_level_irq nop +#define iosapic_ack_level_irq iosapic_nop static struct irq_chip irq_type_iosapic_level = { .name = "IO-SAPIC-level", @@ -453,7 +453,7 @@ iosapic_ack_edge_irq (struct irq_data *data) } #define iosapic_enable_edge_irq unmask_irq -#define iosapic_disable_edge_irq nop +#define iosapic_disable_edge_irq iosapic_nop static struct irq_chip irq_type_iosapic_edge = { .name = "IO-SAPIC-edge", diff --git a/arch/metag/include/asm/barrier.h b/arch/metag/include/asm/barrier.h index 172b7e5..5418517 100644 --- a/arch/metag/include/asm/barrier.h +++ b/arch/metag/include/asm/barrier.h @@ -44,16 +44,6 @@ static inline void wr_fence(void) #define rmb() barrier() #define wmb() mb() -#define dma_rmb() rmb() -#define dma_wmb() wmb() - -#ifndef CONFIG_SMP -#define fence() do { } while (0) -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#else - #ifdef CONFIG_METAG_SMP_WRITE_REORDERING /* * Write to the atomic memory unlock system event register (command 0). This is @@ -63,45 +53,32 @@ static inline void wr_fence(void) * incoherence). It is therefore ineffective if used after and on the same * thread as a write. */ -static inline void fence(void) +static inline void metag_fence(void) { volatile int *flushptr = (volatile int *) LINSYSEVENT_WR_ATOMIC_UNLOCK; barrier(); *flushptr = 0; barrier(); } -#define smp_mb() fence() -#define smp_rmb() fence() -#define smp_wmb() barrier() +#define __smp_mb() metag_fence() +#define __smp_rmb() metag_fence() +#define __smp_wmb() barrier() #else -#define fence() do { } while (0) -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#endif +#define metag_fence() do { } while (0) +#define __smp_mb() barrier() +#define __smp_rmb() barrier() +#define __smp_wmb() barrier() #endif -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) - -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) - -#define smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - smp_mb(); \ - WRITE_ONCE(*p, v); \ -} while (0) +#ifdef CONFIG_SMP +#define fence() metag_fence() +#else +#define fence() do { } while (0) +#endif -#define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - smp_mb(); \ - ___p1; \ -}) +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() -#define smp_mb__before_atomic() barrier() -#define smp_mb__after_atomic() barrier() +#include <asm-generic/barrier.h> #endif /* _ASM_METAG_BARRIER_H */ diff --git a/arch/mips/include/asm/barrier.h b/arch/mips/include/asm/barrier.h index 752e0b8..d296633 100644 --- a/arch/mips/include/asm/barrier.h +++ b/arch/mips/include/asm/barrier.h @@ -10,9 +10,6 @@ #include <asm/addrspace.h> -#define read_barrier_depends() do { } while(0) -#define smp_read_barrier_depends() do { } while(0) - #ifdef CONFIG_CPU_HAS_SYNC #define __sync() \ __asm__ __volatile__( \ @@ -87,23 +84,21 @@ #define wmb() fast_wmb() #define rmb() fast_rmb() -#define dma_wmb() fast_wmb() -#define dma_rmb() fast_rmb() -#if defined(CONFIG_WEAK_ORDERING) && defined(CONFIG_SMP) +#if defined(CONFIG_WEAK_ORDERING) # ifdef CONFIG_CPU_CAVIUM_OCTEON -# define smp_mb() __sync() -# define smp_rmb() barrier() -# define smp_wmb() __syncw() +# define __smp_mb() __sync() +# define __smp_rmb() barrier() +# define __smp_wmb() __syncw() # else -# define smp_mb() __asm__ __volatile__("sync" : : :"memory") -# define smp_rmb() __asm__ __volatile__("sync" : : :"memory") -# define smp_wmb() __asm__ __volatile__("sync" : : :"memory") +# define __smp_mb() __asm__ __volatile__("sync" : : :"memory") +# define __smp_rmb() __asm__ __volatile__("sync" : : :"memory") +# define __smp_wmb() __asm__ __volatile__("sync" : : :"memory") # endif #else -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() +#define __smp_mb() barrier() +#define __smp_rmb() barrier() +#define __smp_wmb() barrier() #endif #if defined(CONFIG_WEAK_REORDERING_BEYOND_LLSC) && defined(CONFIG_SMP) @@ -112,13 +107,11 @@ #define __WEAK_LLSC_MB " \n" #endif -#define smp_store_mb(var, value) \ - do { WRITE_ONCE(var, value); smp_mb(); } while (0) - #define smp_llsc_mb() __asm__ __volatile__(__WEAK_LLSC_MB : : :"memory") #ifdef CONFIG_CPU_CAVIUM_OCTEON #define smp_mb__before_llsc() smp_wmb() +#define __smp_mb__before_llsc() __smp_wmb() /* Cause previous writes to become visible on all CPUs as soon as possible */ #define nudge_writes() __asm__ __volatile__(".set push\n\t" \ ".set arch=octeon\n\t" \ @@ -126,25 +119,13 @@ ".set pop" : : : "memory") #else #define smp_mb__before_llsc() smp_llsc_mb() +#define __smp_mb__before_llsc() smp_llsc_mb() #define nudge_writes() mb() #endif -#define smp_store_release(p, v) \ -do { \ - compiletime_assert_atomic_type(*p); \ - smp_mb(); \ - WRITE_ONCE(*p, v); \ -} while (0) - -#define smp_load_acquire(p) \ -({ \ - typeof(*p) ___p1 = READ_ONCE(*p); \ - compiletime_assert_atomic_type(*p); \ - smp_mb(); \ - ___p1; \ -}) - -#define smp_mb__before_atomic() smp_mb__before_llsc() -#define smp_mb__after_atomic() smp_llsc_mb() +#define __smp_mb__before_atomic() __smp_mb__before_llsc() +#define __smp_mb__after_atomic() smp_llsc_mb() + +#include <asm-generic/barrier.h> #endif /* __ASM_BARRIER_H */ diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index a7af5fb..c0deafc 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -34,8 +34,6 @@ #define rmb() __asm__ __volatile__ ("sync" : : : "memory") #define wmb() __asm__ __volatile__ ("sync" : : : "memory") -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) - #ifdef __SUBARCH_HAS_LWSYNC # define SMPWMB LWSYNC #else @@ -46,22 +44,11 @@ #define dma_rmb() __lwsync() #define dma_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") -#ifdef CONFIG_SMP -#define smp_lwsync() __lwsync() - -#define smp_mb() mb() -#define smp_rmb() __lwsync() -#define smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") -#else -#define smp_lwsync() barrier() +#define __smp_lwsync() __lwsync() -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#endif /* CONFIG_SMP */ - -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) +#define __smp_mb() mb() +#define __smp_rmb() __lwsync() +#define __smp_wmb() __asm__ __volatile__ (stringify_in_c(SMPWMB) : : :"memory") /* * This is a barrier which prevents following instructions from being @@ -72,23 +59,23 @@ #define data_barrier(x) \ asm volatile("twi 0,%0,0; isync" : : "r" (x) : "memory"); -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ - smp_lwsync(); \ + __smp_lwsync(); \ WRITE_ONCE(*p, v); \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ - smp_lwsync(); \ + __smp_lwsync(); \ ___p1; \ }) -#define smp_mb__before_atomic() smp_mb() -#define smp_mb__after_atomic() smp_mb() #define smp_mb__before_spinlock() smp_mb() +#include <asm-generic/barrier.h> + #endif /* _ASM_POWERPC_BARRIER_H */ diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index 7ffd0b1..5c8db3c 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -26,26 +26,18 @@ #define wmb() barrier() #define dma_rmb() mb() #define dma_wmb() mb() -#define smp_mb() mb() -#define smp_rmb() rmb() -#define smp_wmb() wmb() +#define __smp_mb() mb() +#define __smp_rmb() rmb() +#define __smp_wmb() wmb() -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) - -#define smp_mb__before_atomic() smp_mb() -#define smp_mb__after_atomic() smp_mb() - -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) - -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ WRITE_ONCE(*p, v); \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ @@ -53,4 +45,9 @@ do { \ ___p1; \ }) +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() + +#include <asm-generic/barrier.h> + #endif /* __ASM_BARRIER_H */ diff --git a/arch/sh/include/asm/barrier.h b/arch/sh/include/asm/barrier.h index bf91037..f887c64 100644 --- a/arch/sh/include/asm/barrier.h +++ b/arch/sh/include/asm/barrier.h @@ -32,7 +32,8 @@ #define ctrl_barrier() __asm__ __volatile__ ("nop;nop;nop;nop;nop;nop;nop;nop") #endif -#define smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) +#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) +#define smp_store_mb(var, value) __smp_store_mb(var, value) #include <asm-generic/barrier.h> diff --git a/arch/sh/include/asm/cmpxchg-grb.h b/arch/sh/include/asm/cmpxchg-grb.h index f848dec..2ed557b 100644 --- a/arch/sh/include/asm/cmpxchg-grb.h +++ b/arch/sh/include/asm/cmpxchg-grb.h @@ -23,6 +23,28 @@ static inline unsigned long xchg_u32(volatile u32 *m, unsigned long val) return retval; } +static inline unsigned long xchg_u16(volatile u16 *m, unsigned long val) +{ + unsigned long retval; + + __asm__ __volatile__ ( + " .align 2 \n\t" + " mova 1f, r0 \n\t" /* r0 = end point */ + " mov r15, r1 \n\t" /* r1 = saved sp */ + " mov #-6, r15 \n\t" /* LOGIN */ + " mov.w @%1, %0 \n\t" /* load old value */ + " extu.w %0, %0 \n\t" /* extend as unsigned */ + " mov.w %2, @%1 \n\t" /* store new value */ + "1: mov r1, r15 \n\t" /* LOGOUT */ + : "=&r" (retval), + "+r" (m), + "+r" (val) /* inhibit r15 overloading */ + : + : "memory" , "r0", "r1"); + + return retval; +} + static inline unsigned long xchg_u8(volatile u8 *m, unsigned long val) { unsigned long retval; diff --git a/arch/sh/include/asm/cmpxchg-irq.h b/arch/sh/include/asm/cmpxchg-irq.h index bd11f63..f888772 100644 --- a/arch/sh/include/asm/cmpxchg-irq.h +++ b/arch/sh/include/asm/cmpxchg-irq.h @@ -14,6 +14,17 @@ static inline unsigned long xchg_u32(volatile u32 *m, unsigned long val) return retval; } +static inline unsigned long xchg_u16(volatile u16 *m, unsigned long val) +{ + unsigned long flags, retval; + + local_irq_save(flags); + retval = *m; + *m = val; + local_irq_restore(flags); + return retval; +} + static inline unsigned long xchg_u8(volatile u8 *m, unsigned long val) { unsigned long flags, retval; diff --git a/arch/sh/include/asm/cmpxchg-llsc.h b/arch/sh/include/asm/cmpxchg-llsc.h index 4713666..fcfd322 100644 --- a/arch/sh/include/asm/cmpxchg-llsc.h +++ b/arch/sh/include/asm/cmpxchg-llsc.h @@ -22,29 +22,8 @@ static inline unsigned long xchg_u32(volatile u32 *m, unsigned long val) return retval; } -static inline unsigned long xchg_u8(volatile u8 *m, unsigned long val) -{ - unsigned long retval; - unsigned long tmp; - - __asm__ __volatile__ ( - "1: \n\t" - "movli.l @%2, %0 ! xchg_u8 \n\t" - "mov %0, %1 \n\t" - "mov %3, %0 \n\t" - "movco.l %0, @%2 \n\t" - "bf 1b \n\t" - "synco \n\t" - : "=&z"(tmp), "=&r" (retval) - : "r" (m), "r" (val & 0xff) - : "t", "memory" - ); - - return retval; -} - static inline unsigned long -__cmpxchg_u32(volatile int *m, unsigned long old, unsigned long new) +__cmpxchg_u32(volatile u32 *m, unsigned long old, unsigned long new) { unsigned long retval; unsigned long tmp; @@ -68,4 +47,6 @@ __cmpxchg_u32(volatile int *m, unsigned long old, unsigned long new) return retval; } +#include <asm/cmpxchg-xchg.h> + #endif /* __ASM_SH_CMPXCHG_LLSC_H */ diff --git a/arch/sh/include/asm/cmpxchg-xchg.h b/arch/sh/include/asm/cmpxchg-xchg.h new file mode 100644 index 0000000..7219719 --- /dev/null +++ b/arch/sh/include/asm/cmpxchg-xchg.h @@ -0,0 +1,51 @@ +#ifndef __ASM_SH_CMPXCHG_XCHG_H +#define __ASM_SH_CMPXCHG_XCHG_H + +/* + * Copyright (C) 2016 Red Hat, Inc. + * Author: Michael S. Tsirkin <mst@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2. See the + * file "COPYING" in the main directory of this archive for more details. + */ +#include <linux/bitops.h> +#include <asm/byteorder.h> + +/* + * Portable implementations of 1 and 2 byte xchg using a 4 byte cmpxchg. + * Note: this header isn't self-contained: before including it, __cmpxchg_u32 + * must be defined first. + */ +static inline u32 __xchg_cmpxchg(volatile void *ptr, u32 x, int size) +{ + int off = (unsigned long)ptr % sizeof(u32); + volatile u32 *p = ptr - off; +#ifdef __BIG_ENDIAN + int bitoff = (sizeof(u32) - 1 - off) * BITS_PER_BYTE; +#else + int bitoff = off * BITS_PER_BYTE; +#endif + u32 bitmask = ((0x1 << size * BITS_PER_BYTE) - 1) << bitoff; + u32 oldv, newv; + u32 ret; + + do { + oldv = READ_ONCE(*p); + ret = (oldv & bitmask) >> bitoff; + newv = (oldv & ~bitmask) | (x << bitoff); + } while (__cmpxchg_u32(p, oldv, newv) != oldv); + + return ret; +} + +static inline unsigned long xchg_u16(volatile u16 *m, unsigned long val) +{ + return __xchg_cmpxchg(m, val, sizeof *m); +} + +static inline unsigned long xchg_u8(volatile u8 *m, unsigned long val) +{ + return __xchg_cmpxchg(m, val, sizeof *m); +} + +#endif /* __ASM_SH_CMPXCHG_XCHG_H */ diff --git a/arch/sh/include/asm/cmpxchg.h b/arch/sh/include/asm/cmpxchg.h index 85c97b18..5225916 100644 --- a/arch/sh/include/asm/cmpxchg.h +++ b/arch/sh/include/asm/cmpxchg.h @@ -27,6 +27,9 @@ extern void __xchg_called_with_bad_pointer(void); case 4: \ __xchg__res = xchg_u32(__xchg_ptr, x); \ break; \ + case 2: \ + __xchg__res = xchg_u16(__xchg_ptr, x); \ + break; \ case 1: \ __xchg__res = xchg_u8(__xchg_ptr, x); \ break; \ diff --git a/arch/sparc/include/asm/barrier_32.h b/arch/sparc/include/asm/barrier_32.h index ae69eda..8059130 100644 --- a/arch/sparc/include/asm/barrier_32.h +++ b/arch/sparc/include/asm/barrier_32.h @@ -1,7 +1,6 @@ #ifndef __SPARC_BARRIER_H #define __SPARC_BARRIER_H -#include <asm/processor.h> /* for nop() */ #include <asm-generic/barrier.h> #endif /* !(__SPARC_BARRIER_H) */ diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h index 14a9286..c9f6ee6 100644 --- a/arch/sparc/include/asm/barrier_64.h +++ b/arch/sparc/include/asm/barrier_64.h @@ -37,33 +37,14 @@ do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ #define rmb() __asm__ __volatile__("":::"memory") #define wmb() __asm__ __volatile__("":::"memory") -#define dma_rmb() rmb() -#define dma_wmb() wmb() - -#define smp_store_mb(__var, __value) \ - do { WRITE_ONCE(__var, __value); membar_safe("#StoreLoad"); } while(0) - -#ifdef CONFIG_SMP -#define smp_mb() mb() -#define smp_rmb() rmb() -#define smp_wmb() wmb() -#else -#define smp_mb() __asm__ __volatile__("":::"memory") -#define smp_rmb() __asm__ __volatile__("":::"memory") -#define smp_wmb() __asm__ __volatile__("":::"memory") -#endif - -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) - -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ WRITE_ONCE(*p, v); \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ @@ -71,7 +52,9 @@ do { \ ___p1; \ }) -#define smp_mb__before_atomic() barrier() -#define smp_mb__after_atomic() barrier() +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() + +#include <asm-generic/barrier.h> #endif /* !(__SPARC64_BARRIER_H) */ diff --git a/arch/sparc/include/asm/processor.h b/arch/sparc/include/asm/processor.h index 2fe99e6..9da9646 100644 --- a/arch/sparc/include/asm/processor.h +++ b/arch/sparc/include/asm/processor.h @@ -5,7 +5,4 @@ #else #include <asm/processor_32.h> #endif - -#define nop() __asm__ __volatile__ ("nop") - #endif diff --git a/arch/tile/include/asm/barrier.h b/arch/tile/include/asm/barrier.h index 96a42ae..d552228 100644 --- a/arch/tile/include/asm/barrier.h +++ b/arch/tile/include/asm/barrier.h @@ -79,11 +79,12 @@ mb_incoherent(void) * But after the word is updated, the routine issues an "mf" before returning, * and since it's a function call, we don't even need a compiler barrier. */ -#define smp_mb__before_atomic() smp_mb() -#define smp_mb__after_atomic() do { } while (0) +#define __smp_mb__before_atomic() __smp_mb() +#define __smp_mb__after_atomic() do { } while (0) +#define smp_mb__after_atomic() __smp_mb__after_atomic() #else /* 64 bit */ -#define smp_mb__before_atomic() smp_mb() -#define smp_mb__after_atomic() smp_mb() +#define __smp_mb__before_atomic() __smp_mb() +#define __smp_mb__after_atomic() __smp_mb() #endif #include <asm-generic/barrier.h> diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index 0681d25..a584e1c 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -31,20 +31,10 @@ #endif #define dma_wmb() barrier() -#ifdef CONFIG_SMP -#define smp_mb() mb() -#define smp_rmb() dma_rmb() -#define smp_wmb() barrier() -#define smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) -#else /* !SMP */ -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) -#endif /* SMP */ - -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) +#define __smp_mb() mb() +#define __smp_rmb() dma_rmb() +#define __smp_wmb() barrier() +#define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) #if defined(CONFIG_X86_PPRO_FENCE) @@ -53,31 +43,31 @@ * model and we should fall back to full barriers. */ -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ - smp_mb(); \ + __smp_mb(); \ WRITE_ONCE(*p, v); \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ - smp_mb(); \ + __smp_mb(); \ ___p1; \ }) #else /* regular x86 TSO memory ordering */ -#define smp_store_release(p, v) \ +#define __smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ barrier(); \ WRITE_ONCE(*p, v); \ } while (0) -#define smp_load_acquire(p) \ +#define __smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ @@ -88,7 +78,9 @@ do { \ #endif /* Atomic operations are already serializing on x86 */ -#define smp_mb__before_atomic() barrier() -#define smp_mb__after_atomic() barrier() +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() + +#include <asm-generic/barrier.h> #endif /* _ASM_X86_BARRIER_H */ diff --git a/arch/x86/um/asm/barrier.h b/arch/x86/um/asm/barrier.h index 755481f..174781a 100644 --- a/arch/x86/um/asm/barrier.h +++ b/arch/x86/um/asm/barrier.h @@ -36,13 +36,6 @@ #endif /* CONFIG_X86_PPRO_FENCE */ #define dma_wmb() barrier() -#define smp_mb() barrier() -#define smp_rmb() barrier() -#define smp_wmb() barrier() - -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) - -#define read_barrier_depends() do { } while (0) -#define smp_read_barrier_depends() do { } while (0) +#include <asm-generic/barrier.h> #endif diff --git a/arch/xtensa/include/asm/barrier.h b/arch/xtensa/include/asm/barrier.h index 5b88774..956596e 100644 --- a/arch/xtensa/include/asm/barrier.h +++ b/arch/xtensa/include/asm/barrier.h @@ -13,8 +13,8 @@ #define rmb() barrier() #define wmb() mb() -#define smp_mb__before_atomic() barrier() -#define smp_mb__after_atomic() barrier() +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() #include <asm-generic/barrier.h> diff --git a/block/bio.c b/block/bio.c index 4f184d9..dbabd48 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1125,7 +1125,7 @@ struct bio *bio_copy_user_iov(struct request_queue *q, int i, ret; int nr_pages = 0; unsigned int len = iter->count; - unsigned int offset = map_data ? map_data->offset & ~PAGE_MASK : 0; + unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; for (i = 0; i < iter->nr_segs; i++) { unsigned long uaddr; @@ -1304,7 +1304,7 @@ struct bio *bio_map_user_iov(struct request_queue *q, goto out_unmap; } - offset = uaddr & ~PAGE_MASK; + offset = offset_in_page(uaddr); for (j = cur_page; j < page_limit; j++) { unsigned int bytes = PAGE_SIZE - offset; diff --git a/block/blk-core.c b/block/blk-core.c index 33e2f62..476244d 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -51,7 +51,7 @@ DEFINE_IDA(blk_queue_ida); /* * For the allocated request tables */ -struct kmem_cache *request_cachep = NULL; +struct kmem_cache *request_cachep; /* * For queue allocation @@ -646,7 +646,7 @@ struct request_queue *blk_alloc_queue(gfp_t gfp_mask) } EXPORT_SYMBOL(blk_alloc_queue); -int blk_queue_enter(struct request_queue *q, gfp_t gfp) +int blk_queue_enter(struct request_queue *q, bool nowait) { while (true) { int ret; @@ -654,7 +654,7 @@ int blk_queue_enter(struct request_queue *q, gfp_t gfp) if (percpu_ref_tryget_live(&q->q_usage_counter)) return 0; - if (!gfpflags_allow_blocking(gfp)) + if (nowait) return -EBUSY; ret = wait_event_interruptible(q->mq_freeze_wq, @@ -1292,7 +1292,9 @@ static struct request *blk_old_get_request(struct request_queue *q, int rw, struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask) { if (q->mq_ops) - return blk_mq_alloc_request(q, rw, gfp_mask, false); + return blk_mq_alloc_request(q, rw, + (gfp_mask & __GFP_DIRECT_RECLAIM) ? + 0 : BLK_MQ_REQ_NOWAIT); else return blk_old_get_request(q, rw, gfp_mask); } @@ -2060,8 +2062,7 @@ blk_qc_t generic_make_request(struct bio *bio) do { struct request_queue *q = bdev_get_queue(bio->bi_bdev); - if (likely(blk_queue_enter(q, __GFP_DIRECT_RECLAIM) == 0)) { - + if (likely(blk_queue_enter(q, false) == 0)) { ret = q->make_request_fn(q, bio); blk_queue_exit(q); @@ -3534,7 +3535,7 @@ int __init blk_dev_init(void) request_cachep = kmem_cache_create("blkdev_requests", sizeof(struct request), 0, SLAB_PANIC, NULL); - blk_requestq_cachep = kmem_cache_create("blkdev_queue", + blk_requestq_cachep = kmem_cache_create("request_queue", sizeof(struct request_queue), 0, SLAB_PANIC, NULL); return 0; diff --git a/block/blk-merge.c b/block/blk-merge.c index e01405a..1699df5 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -7,6 +7,8 @@ #include <linux/blkdev.h> #include <linux/scatterlist.h> +#include <trace/events/block.h> + #include "blk.h" static struct bio *blk_bio_discard_split(struct request_queue *q, @@ -81,9 +83,6 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, struct bio *new = NULL; bio_for_each_segment(bv, bio, iter) { - if (sectors + (bv.bv_len >> 9) > queue_max_sectors(q)) - goto split; - /* * If the queue doesn't support SG gaps and adding this * offset would create a gap, disallow it. @@ -91,6 +90,22 @@ static struct bio *blk_bio_segment_split(struct request_queue *q, if (bvprvp && bvec_gap_to_prev(q, bvprvp, bv.bv_offset)) goto split; + if (sectors + (bv.bv_len >> 9) > + blk_max_size_offset(q, bio->bi_iter.bi_sector)) { + /* + * Consider this a new segment if we're splitting in + * the middle of this vector. + */ + if (nsegs < queue_max_segments(q) && + sectors < blk_max_size_offset(q, + bio->bi_iter.bi_sector)) { + nsegs++; + sectors = blk_max_size_offset(q, + bio->bi_iter.bi_sector); + } + goto split; + } + if (bvprvp && blk_queue_cluster(q)) { if (seg_size + bv.bv_len > queue_max_segment_size(q)) goto new_segment; @@ -162,6 +177,7 @@ void blk_queue_split(struct request_queue *q, struct bio **bio, split->bi_rw |= REQ_NOMERGE; bio_chain(split, *bio); + trace_block_split(q, split, (*bio)->bi_iter.bi_sector); generic_make_request(*bio); *bio = split; } diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c index 8764c24..d0634bc 100644 --- a/block/blk-mq-cpumap.c +++ b/block/blk-mq-cpumap.c @@ -113,7 +113,7 @@ int blk_mq_hw_queue_to_node(unsigned int *mq_map, unsigned int index) for_each_possible_cpu(i) { if (index == mq_map[i]) - return cpu_to_node(i); + return local_memory_node(cpu_to_node(i)); } return NUMA_NO_NODE; diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index a07ca34..abdbb47 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -268,7 +268,7 @@ static int bt_get(struct blk_mq_alloc_data *data, if (tag != -1) return tag; - if (!gfpflags_allow_blocking(data->gfp)) + if (data->flags & BLK_MQ_REQ_NOWAIT) return -1; bs = bt_wait_ptr(bt, hctx); @@ -303,7 +303,7 @@ static int bt_get(struct blk_mq_alloc_data *data, data->ctx = blk_mq_get_ctx(data->q); data->hctx = data->q->mq_ops->map_queue(data->q, data->ctx->cpu); - if (data->reserved) { + if (data->flags & BLK_MQ_REQ_RESERVED) { bt = &data->hctx->tags->breserved_tags; } else { last_tag = &data->ctx->last_tag; @@ -349,10 +349,9 @@ static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data) unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) { - if (!data->reserved) - return __blk_mq_get_tag(data); - - return __blk_mq_get_reserved_tag(data); + if (data->flags & BLK_MQ_REQ_RESERVED) + return __blk_mq_get_reserved_tag(data); + return __blk_mq_get_tag(data); } static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt) diff --git a/block/blk-mq.c b/block/blk-mq.c index 6d6f8fe..6889d71 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -229,8 +229,8 @@ __blk_mq_alloc_request(struct blk_mq_alloc_data *data, int rw) return NULL; } -struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, - bool reserved) +struct request *blk_mq_alloc_request(struct request_queue *q, int rw, + unsigned int flags) { struct blk_mq_ctx *ctx; struct blk_mq_hw_ctx *hctx; @@ -238,24 +238,22 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, struct blk_mq_alloc_data alloc_data; int ret; - ret = blk_queue_enter(q, gfp); + ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT); if (ret) return ERR_PTR(ret); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); - blk_mq_set_alloc_data(&alloc_data, q, gfp & ~__GFP_DIRECT_RECLAIM, - reserved, ctx, hctx); + blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); rq = __blk_mq_alloc_request(&alloc_data, rw); - if (!rq && (gfp & __GFP_DIRECT_RECLAIM)) { + if (!rq && !(flags & BLK_MQ_REQ_NOWAIT)) { __blk_mq_run_hw_queue(hctx); blk_mq_put_ctx(ctx); ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); - blk_mq_set_alloc_data(&alloc_data, q, gfp, reserved, ctx, - hctx); + blk_mq_set_alloc_data(&alloc_data, q, flags, ctx, hctx); rq = __blk_mq_alloc_request(&alloc_data, rw); ctx = alloc_data.ctx; } @@ -1175,8 +1173,7 @@ static struct request *blk_mq_map_request(struct request_queue *q, rw |= REQ_SYNC; trace_block_getrq(q, bio, rw); - blk_mq_set_alloc_data(&alloc_data, q, GFP_ATOMIC, false, ctx, - hctx); + blk_mq_set_alloc_data(&alloc_data, q, BLK_MQ_REQ_NOWAIT, ctx, hctx); rq = __blk_mq_alloc_request(&alloc_data, rw); if (unlikely(!rq)) { __blk_mq_run_hw_queue(hctx); @@ -1185,8 +1182,7 @@ static struct request *blk_mq_map_request(struct request_queue *q, ctx = blk_mq_get_ctx(q); hctx = q->mq_ops->map_queue(q, ctx->cpu); - blk_mq_set_alloc_data(&alloc_data, q, - __GFP_RECLAIM|__GFP_HIGH, false, ctx, hctx); + blk_mq_set_alloc_data(&alloc_data, q, 0, ctx, hctx); rq = __blk_mq_alloc_request(&alloc_data, rw); ctx = alloc_data.ctx; hctx = alloc_data.hctx; @@ -1794,7 +1790,7 @@ static void blk_mq_init_cpu_queues(struct request_queue *q, * not, we remain on the home node of the device */ if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE) - hctx->numa_node = cpu_to_node(i); + hctx->numa_node = local_memory_node(cpu_to_node(i)); } } @@ -1854,6 +1850,7 @@ static void blk_mq_map_swqueue(struct request_queue *q, hctx->tags = set->tags[i]; WARN_ON(!hctx->tags); + cpumask_copy(hctx->tags->cpumask, hctx->cpumask); /* * Set the map size to the number of mapped software queues. * This is more accurate and more efficient than looping @@ -1867,14 +1864,6 @@ static void blk_mq_map_swqueue(struct request_queue *q, hctx->next_cpu = cpumask_first(hctx->cpumask); hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH; } - - queue_for_each_ctx(q, ctx, i) { - if (!cpumask_test_cpu(i, online_mask)) - continue; - - hctx = q->mq_ops->map_queue(q, i); - cpumask_set_cpu(i, hctx->tags->cpumask); - } } static void queue_set_hctx_shared(struct request_queue *q, bool shared) diff --git a/block/blk-mq.h b/block/blk-mq.h index 713820b..eaede8e 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -96,8 +96,7 @@ static inline void blk_mq_put_ctx(struct blk_mq_ctx *ctx) struct blk_mq_alloc_data { /* input parameter */ struct request_queue *q; - gfp_t gfp; - bool reserved; + unsigned int flags; /* input & output parameter */ struct blk_mq_ctx *ctx; @@ -105,13 +104,11 @@ struct blk_mq_alloc_data { }; static inline void blk_mq_set_alloc_data(struct blk_mq_alloc_data *data, - struct request_queue *q, gfp_t gfp, bool reserved, - struct blk_mq_ctx *ctx, - struct blk_mq_hw_ctx *hctx) + struct request_queue *q, unsigned int flags, + struct blk_mq_ctx *ctx, struct blk_mq_hw_ctx *hctx) { data->q = q; - data->gfp = gfp; - data->reserved = reserved; + data->flags = flags; data->ctx = ctx; data->hctx = hctx; } diff --git a/block/blk-timeout.c b/block/blk-timeout.c index aa40aa9..3610af5 100644 --- a/block/blk-timeout.c +++ b/block/blk-timeout.c @@ -186,6 +186,7 @@ unsigned long blk_rq_timeout(unsigned long timeout) * Notes: * Each request has its own timer, and as it is added to the queue, we * set up the timer. When the request completes, we cancel the timer. + * Queue lock must be held for the non-mq case, mq case doesn't care. */ void blk_add_timer(struct request *req) { @@ -209,6 +210,11 @@ void blk_add_timer(struct request *req) req->timeout = q->rq_timeout; req->deadline = jiffies + req->timeout; + + /* + * Only the non-mq case needs to add the request to a protected list. + * For the mq case we simply scan the tag map. + */ if (!q->mq_ops) list_add_tail(&req->timeout_list, &req->q->timeout_list); diff --git a/block/genhd.c b/block/genhd.c index 5aaeb2a..9f42526 100644 --- a/block/genhd.c +++ b/block/genhd.c @@ -1449,7 +1449,7 @@ static DEFINE_MUTEX(disk_events_mutex); static LIST_HEAD(disk_events); /* disable in-kernel polling by default */ -static unsigned long disk_events_dfl_poll_msecs = 0; +static unsigned long disk_events_dfl_poll_msecs; static unsigned long disk_events_poll_jiffies(struct gendisk *disk) { diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 34997d8..15bec40 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -173,7 +173,7 @@ static struct mtip_cmd *mtip_get_int_command(struct driver_data *dd) { struct request *rq; - rq = blk_mq_alloc_request(dd->queue, 0, __GFP_RECLAIM, true); + rq = blk_mq_alloc_request(dd->queue, 0, BLK_MQ_REQ_RESERVED); return blk_mq_rq_to_pdu(rq); } diff --git a/drivers/block/null_blk.c b/drivers/block/null_blk.c index 09e3c0d..95dff91 100644 --- a/drivers/block/null_blk.c +++ b/drivers/block/null_blk.c @@ -449,7 +449,7 @@ static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) struct request *rq; struct bio *bio = rqd->bio; - rq = blk_mq_alloc_request(q, bio_rw(bio), GFP_KERNEL, 0); + rq = blk_mq_alloc_request(q, bio_rw(bio), 0); if (IS_ERR(rq)) return -ENOMEM; diff --git a/drivers/char/raw.c b/drivers/char/raw.c index 60316fb..9b9809b 100644 --- a/drivers/char/raw.c +++ b/drivers/char/raw.c @@ -71,7 +71,7 @@ static int raw_open(struct inode *inode, struct file *filp) err = -ENODEV; if (!bdev) goto out; - igrab(bdev->bd_inode); + bdgrab(bdev); err = blkdev_get(bdev, filp->f_mode | FMODE_EXCL, raw_open); if (err) goto out; diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c index 06496a1..4150873 100644 --- a/drivers/gpu/drm/virtio/virtgpu_kms.c +++ b/drivers/gpu/drm/virtio/virtgpu_kms.c @@ -130,7 +130,7 @@ int virtio_gpu_driver_load(struct drm_device *dev, unsigned long flags) static vq_callback_t *callbacks[] = { virtio_gpu_ctrl_ack, virtio_gpu_cursor_ack }; - static const char *names[] = { "control", "cursor" }; + static const char * const names[] = { "control", "cursor" }; struct virtio_gpu_device *vgdev; /* this will expand later */ diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index b9094e9..a1e75cb 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -263,81 +263,6 @@ config EXYNOS_IOMMU_DEBUG Say N unless you need kernel log message for IOMMU debugging. -config SHMOBILE_IPMMU - bool - -config SHMOBILE_IPMMU_TLB - bool - -config SHMOBILE_IOMMU - bool "IOMMU for Renesas IPMMU/IPMMUI" - default n - depends on ARM && MMU - depends on ARCH_SHMOBILE || COMPILE_TEST - select IOMMU_API - select ARM_DMA_USE_IOMMU - select SHMOBILE_IPMMU - select SHMOBILE_IPMMU_TLB - help - Support for Renesas IPMMU/IPMMUI. This option enables - remapping of DMA memory accesses from all of the IP blocks - on the ICB. - - Warning: Drivers (including userspace drivers of UIO - devices) of the IP blocks on the ICB *must* use addresses - allocated from the IPMMU (iova) for DMA with this option - enabled. - - If unsure, say N. - -choice - prompt "IPMMU/IPMMUI address space size" - default SHMOBILE_IOMMU_ADDRSIZE_2048MB - depends on SHMOBILE_IOMMU - help - This option sets IPMMU/IPMMUI address space size by - adjusting the 1st level page table size. The page table size - is calculated as follows: - - page table size = number of page table entries * 4 bytes - number of page table entries = address space size / 1 MiB - - For example, when the address space size is 2048 MiB, the - 1st level page table size is 8192 bytes. - - config SHMOBILE_IOMMU_ADDRSIZE_2048MB - bool "2 GiB" - - config SHMOBILE_IOMMU_ADDRSIZE_1024MB - bool "1 GiB" - - config SHMOBILE_IOMMU_ADDRSIZE_512MB - bool "512 MiB" - - config SHMOBILE_IOMMU_ADDRSIZE_256MB - bool "256 MiB" - - config SHMOBILE_IOMMU_ADDRSIZE_128MB - bool "128 MiB" - - config SHMOBILE_IOMMU_ADDRSIZE_64MB - bool "64 MiB" - - config SHMOBILE_IOMMU_ADDRSIZE_32MB - bool "32 MiB" - -endchoice - -config SHMOBILE_IOMMU_L1SIZE - int - default 8192 if SHMOBILE_IOMMU_ADDRSIZE_2048MB - default 4096 if SHMOBILE_IOMMU_ADDRSIZE_1024MB - default 2048 if SHMOBILE_IOMMU_ADDRSIZE_512MB - default 1024 if SHMOBILE_IOMMU_ADDRSIZE_256MB - default 512 if SHMOBILE_IOMMU_ADDRSIZE_128MB - default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB - default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB - config IPMMU_VMSA bool "Renesas VMSA-compatible IPMMU" depends on ARM_LPAE diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 68faca02..42fc0c2 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -22,7 +22,5 @@ obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o -obj-$(CONFIG_SHMOBILE_IOMMU) += shmobile-iommu.o -obj-$(CONFIG_SHMOBILE_IPMMU) += shmobile-ipmmu.o obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o obj-$(CONFIG_S390_IOMMU) += s390-iommu.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 8b2be1e..539b0de 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -35,6 +35,7 @@ #include <linux/msi.h> #include <linux/dma-contiguous.h> #include <linux/irqdomain.h> +#include <linux/percpu.h> #include <asm/irq_remapping.h> #include <asm/io_apic.h> #include <asm/apic.h> @@ -114,6 +115,45 @@ struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); static int protection_domain_init(struct protection_domain *domain); +/* + * For dynamic growth the aperture size is split into ranges of 128MB of + * DMA address space each. This struct represents one such range. + */ +struct aperture_range { + + spinlock_t bitmap_lock; + + /* address allocation bitmap */ + unsigned long *bitmap; + unsigned long offset; + unsigned long next_bit; + + /* + * Array of PTE pages for the aperture. In this array we save all the + * leaf pages of the domain page table used for the aperture. This way + * we don't need to walk the page table to find a specific PTE. We can + * just calculate its address in constant time. + */ + u64 *pte_pages[64]; +}; + +/* + * Data container for a dma_ops specific protection domain + */ +struct dma_ops_domain { + /* generic protection domain information */ + struct protection_domain domain; + + /* size of the aperture for the mappings */ + unsigned long aperture_size; + + /* aperture index we start searching for free addresses */ + u32 __percpu *next_index; + + /* address space relevant data */ + struct aperture_range *aperture[APERTURE_MAX_RANGES]; +}; + /**************************************************************************** * * Helper functions @@ -1167,11 +1207,21 @@ static u64 *alloc_pte(struct protection_domain *domain, end_lvl = PAGE_SIZE_LEVEL(page_size); while (level > end_lvl) { - if (!IOMMU_PTE_PRESENT(*pte)) { + u64 __pte, __npte; + + __pte = *pte; + + if (!IOMMU_PTE_PRESENT(__pte)) { page = (u64 *)get_zeroed_page(gfp); if (!page) return NULL; - *pte = PM_LEVEL_PDE(level, virt_to_phys(page)); + + __npte = PM_LEVEL_PDE(level, virt_to_phys(page)); + + if (cmpxchg64(pte, __pte, __npte)) { + free_page((unsigned long)page); + continue; + } } /* No level skipping support yet */ @@ -1376,8 +1426,10 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, bool populate, gfp_t gfp) { int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT; - struct amd_iommu *iommu; unsigned long i, old_size, pte_pgsize; + struct aperture_range *range; + struct amd_iommu *iommu; + unsigned long flags; #ifdef CONFIG_IOMMU_STRESS populate = false; @@ -1386,15 +1438,17 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, if (index >= APERTURE_MAX_RANGES) return -ENOMEM; - dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp); - if (!dma_dom->aperture[index]) + range = kzalloc(sizeof(struct aperture_range), gfp); + if (!range) return -ENOMEM; - dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp); - if (!dma_dom->aperture[index]->bitmap) + range->bitmap = (void *)get_zeroed_page(gfp); + if (!range->bitmap) goto out_free; - dma_dom->aperture[index]->offset = dma_dom->aperture_size; + range->offset = dma_dom->aperture_size; + + spin_lock_init(&range->bitmap_lock); if (populate) { unsigned long address = dma_dom->aperture_size; @@ -1407,14 +1461,20 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, if (!pte) goto out_free; - dma_dom->aperture[index]->pte_pages[i] = pte_page; + range->pte_pages[i] = pte_page; address += APERTURE_RANGE_SIZE / 64; } } - old_size = dma_dom->aperture_size; - dma_dom->aperture_size += APERTURE_RANGE_SIZE; + spin_lock_irqsave(&dma_dom->domain.lock, flags); + + /* First take the bitmap_lock and then publish the range */ + spin_lock(&range->bitmap_lock); + + old_size = dma_dom->aperture_size; + dma_dom->aperture[index] = range; + dma_dom->aperture_size += APERTURE_RANGE_SIZE; /* Reserve address range used for MSI messages */ if (old_size < MSI_ADDR_BASE_LO && @@ -1461,62 +1521,123 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom, update_domain(&dma_dom->domain); + spin_unlock(&range->bitmap_lock); + + spin_unlock_irqrestore(&dma_dom->domain.lock, flags); + return 0; out_free: update_domain(&dma_dom->domain); - free_page((unsigned long)dma_dom->aperture[index]->bitmap); + free_page((unsigned long)range->bitmap); - kfree(dma_dom->aperture[index]); - dma_dom->aperture[index] = NULL; + kfree(range); return -ENOMEM; } +static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom, + struct aperture_range *range, + unsigned long pages, + unsigned long dma_mask, + unsigned long boundary_size, + unsigned long align_mask, + bool trylock) +{ + unsigned long offset, limit, flags; + dma_addr_t address; + bool flush = false; + + offset = range->offset >> PAGE_SHIFT; + limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset, + dma_mask >> PAGE_SHIFT); + + if (trylock) { + if (!spin_trylock_irqsave(&range->bitmap_lock, flags)) + return -1; + } else { + spin_lock_irqsave(&range->bitmap_lock, flags); + } + + address = iommu_area_alloc(range->bitmap, limit, range->next_bit, + pages, offset, boundary_size, align_mask); + if (address == -1) { + /* Nothing found, retry one time */ + address = iommu_area_alloc(range->bitmap, limit, + 0, pages, offset, boundary_size, + align_mask); + flush = true; + } + + if (address != -1) + range->next_bit = address + pages; + + spin_unlock_irqrestore(&range->bitmap_lock, flags); + + if (flush) { + domain_flush_tlb(&dom->domain); + domain_flush_complete(&dom->domain); + } + + return address; +} + static unsigned long dma_ops_area_alloc(struct device *dev, struct dma_ops_domain *dom, unsigned int pages, unsigned long align_mask, - u64 dma_mask, - unsigned long start) + u64 dma_mask) { - unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE; - int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT; - int i = start >> APERTURE_RANGE_SHIFT; unsigned long boundary_size, mask; unsigned long address = -1; - unsigned long limit; + bool first = true; + u32 start, i; - next_bit >>= PAGE_SHIFT; + preempt_disable(); mask = dma_get_seg_boundary(dev); +again: + start = this_cpu_read(*dom->next_index); + + /* Sanity check - is it really necessary? */ + if (unlikely(start > APERTURE_MAX_RANGES)) { + start = 0; + this_cpu_write(*dom->next_index, 0); + } + boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT : 1UL << (BITS_PER_LONG - PAGE_SHIFT); - for (;i < max_index; ++i) { - unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT; + for (i = 0; i < APERTURE_MAX_RANGES; ++i) { + struct aperture_range *range; + int index; - if (dom->aperture[i]->offset >= dma_mask) - break; + index = (start + i) % APERTURE_MAX_RANGES; - limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset, - dma_mask >> PAGE_SHIFT); + range = dom->aperture[index]; - address = iommu_area_alloc(dom->aperture[i]->bitmap, - limit, next_bit, pages, 0, - boundary_size, align_mask); + if (!range || range->offset >= dma_mask) + continue; + + address = dma_ops_aperture_alloc(dom, range, pages, + dma_mask, boundary_size, + align_mask, first); if (address != -1) { - address = dom->aperture[i]->offset + - (address << PAGE_SHIFT); - dom->next_address = address + (pages << PAGE_SHIFT); + address = range->offset + (address << PAGE_SHIFT); + this_cpu_write(*dom->next_index, index); break; } + } - next_bit = 0; + if (address == -1 && first) { + first = false; + goto again; } + preempt_enable(); + return address; } @@ -1526,21 +1647,14 @@ static unsigned long dma_ops_alloc_addresses(struct device *dev, unsigned long align_mask, u64 dma_mask) { - unsigned long address; - -#ifdef CONFIG_IOMMU_STRESS - dom->next_address = 0; - dom->need_flush = true; -#endif + unsigned long address = -1; - address = dma_ops_area_alloc(dev, dom, pages, align_mask, - dma_mask, dom->next_address); + while (address == -1) { + address = dma_ops_area_alloc(dev, dom, pages, + align_mask, dma_mask); - if (address == -1) { - dom->next_address = 0; - address = dma_ops_area_alloc(dev, dom, pages, align_mask, - dma_mask, 0); - dom->need_flush = true; + if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC)) + break; } if (unlikely(address == -1)) @@ -1562,6 +1676,7 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom, { unsigned i = address >> APERTURE_RANGE_SHIFT; struct aperture_range *range = dom->aperture[i]; + unsigned long flags; BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL); @@ -1570,12 +1685,18 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom, return; #endif - if (address >= dom->next_address) - dom->need_flush = true; + if (amd_iommu_unmap_flush) { + domain_flush_tlb(&dom->domain); + domain_flush_complete(&dom->domain); + } address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT; + spin_lock_irqsave(&range->bitmap_lock, flags); + if (address + pages > range->next_bit) + range->next_bit = address + pages; bitmap_clear(range->bitmap, address, pages); + spin_unlock_irqrestore(&range->bitmap_lock, flags); } @@ -1755,6 +1876,8 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) if (!dom) return; + free_percpu(dom->next_index); + del_domain_from_list(&dom->domain); free_pagetable(&dom->domain); @@ -1769,6 +1892,23 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) kfree(dom); } +static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom, + int max_apertures) +{ + int ret, i, apertures; + + apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT; + ret = 0; + + for (i = apertures; i < max_apertures; ++i) { + ret = alloc_new_range(dma_dom, false, GFP_KERNEL); + if (ret) + break; + } + + return ret; +} + /* * Allocates a new protection domain usable for the dma_ops functions. * It also initializes the page table and the address allocator data @@ -1777,6 +1917,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom) static struct dma_ops_domain *dma_ops_domain_alloc(void) { struct dma_ops_domain *dma_dom; + int cpu; dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL); if (!dma_dom) @@ -1785,6 +1926,10 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (protection_domain_init(&dma_dom->domain)) goto free_dma_dom; + dma_dom->next_index = alloc_percpu(u32); + if (!dma_dom->next_index) + goto free_dma_dom; + dma_dom->domain.mode = PAGE_MODE_2_LEVEL; dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL); dma_dom->domain.flags = PD_DMA_OPS_MASK; @@ -1792,8 +1937,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) if (!dma_dom->domain.pt_root) goto free_dma_dom; - dma_dom->need_flush = false; - add_domain_to_list(&dma_dom->domain); if (alloc_new_range(dma_dom, true, GFP_KERNEL)) @@ -1804,8 +1947,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void) * a valid dma-address. So we can use 0 as error value */ dma_dom->aperture[0]->bitmap[0] = 1; - dma_dom->next_address = 0; + for_each_possible_cpu(cpu) + *per_cpu_ptr(dma_dom->next_index, cpu) = 0; return dma_dom; @@ -2328,7 +2472,7 @@ static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom, else if (direction == DMA_BIDIRECTIONAL) __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW; - WARN_ON(*pte); + WARN_ON_ONCE(*pte); *pte = __pte; @@ -2357,7 +2501,7 @@ static void dma_ops_domain_unmap(struct dma_ops_domain *dom, pte += PM_LEVEL_INDEX(0, address); - WARN_ON(!*pte); + WARN_ON_ONCE(!*pte); *pte = 0ULL; } @@ -2393,26 +2537,11 @@ static dma_addr_t __map_single(struct device *dev, if (align) align_mask = (1UL << get_order(size)) - 1; -retry: address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask, dma_mask); - if (unlikely(address == DMA_ERROR_CODE)) { - /* - * setting next_address here will let the address - * allocator only scan the new allocated range in the - * first run. This is a small optimization. - */ - dma_dom->next_address = dma_dom->aperture_size; - if (alloc_new_range(dma_dom, false, GFP_ATOMIC)) - goto out; - - /* - * aperture was successfully enlarged by 128 MB, try - * allocation again - */ - goto retry; - } + if (address == DMA_ERROR_CODE) + goto out; start = address; for (i = 0; i < pages; ++i) { @@ -2427,11 +2556,10 @@ retry: ADD_STATS_COUNTER(alloced_io_mem, size); - if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) { - domain_flush_tlb(&dma_dom->domain); - dma_dom->need_flush = false; - } else if (unlikely(amd_iommu_np_cache)) + if (unlikely(amd_iommu_np_cache)) { domain_flush_pages(&dma_dom->domain, address, size); + domain_flush_complete(&dma_dom->domain); + } out: return address; @@ -2478,11 +2606,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, SUB_STATS_COUNTER(alloced_io_mem, size); dma_ops_free_addresses(dma_dom, dma_addr, pages); - - if (amd_iommu_unmap_flush || dma_dom->need_flush) { - domain_flush_pages(&dma_dom->domain, flush_addr, size); - dma_dom->need_flush = false; - } } /* @@ -2493,11 +2616,9 @@ static dma_addr_t map_page(struct device *dev, struct page *page, enum dma_data_direction dir, struct dma_attrs *attrs) { - unsigned long flags; + phys_addr_t paddr = page_to_phys(page) + offset; struct protection_domain *domain; - dma_addr_t addr; u64 dma_mask; - phys_addr_t paddr = page_to_phys(page) + offset; INC_STATS_COUNTER(cnt_map_single); @@ -2509,19 +2630,8 @@ static dma_addr_t map_page(struct device *dev, struct page *page, dma_mask = *dev->dma_mask; - spin_lock_irqsave(&domain->lock, flags); - - addr = __map_single(dev, domain->priv, paddr, size, dir, false, + return __map_single(dev, domain->priv, paddr, size, dir, false, dma_mask); - if (addr == DMA_ERROR_CODE) - goto out; - - domain_flush_complete(domain); - -out: - spin_unlock_irqrestore(&domain->lock, flags); - - return addr; } /* @@ -2530,7 +2640,6 @@ out: static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, struct dma_attrs *attrs) { - unsigned long flags; struct protection_domain *domain; INC_STATS_COUNTER(cnt_unmap_single); @@ -2539,13 +2648,7 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, if (IS_ERR(domain)) return; - spin_lock_irqsave(&domain->lock, flags); - __unmap_single(domain->priv, dma_addr, size, dir); - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); } /* @@ -2556,7 +2659,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) { - unsigned long flags; struct protection_domain *domain; int i; struct scatterlist *s; @@ -2572,8 +2674,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, dma_mask = *dev->dma_mask; - spin_lock_irqsave(&domain->lock, flags); - for_each_sg(sglist, s, nelems, i) { paddr = sg_phys(s); @@ -2588,12 +2688,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, goto unmap; } - domain_flush_complete(domain); - -out: - spin_unlock_irqrestore(&domain->lock, flags); - return mapped_elems; + unmap: for_each_sg(sglist, s, mapped_elems, i) { if (s->dma_address) @@ -2602,9 +2698,7 @@ unmap: s->dma_address = s->dma_length = 0; } - mapped_elems = 0; - - goto out; + return 0; } /* @@ -2615,7 +2709,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction dir, struct dma_attrs *attrs) { - unsigned long flags; struct protection_domain *domain; struct scatterlist *s; int i; @@ -2626,17 +2719,11 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, if (IS_ERR(domain)) return; - spin_lock_irqsave(&domain->lock, flags); - for_each_sg(sglist, s, nelems, i) { __unmap_single(domain->priv, s->dma_address, s->dma_length, dir); s->dma_address = s->dma_length = 0; } - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); } /* @@ -2648,7 +2735,6 @@ static void *alloc_coherent(struct device *dev, size_t size, { u64 dma_mask = dev->coherent_dma_mask; struct protection_domain *domain; - unsigned long flags; struct page *page; INC_STATS_COUNTER(cnt_alloc_coherent); @@ -2680,19 +2766,11 @@ static void *alloc_coherent(struct device *dev, size_t size, if (!dma_mask) dma_mask = *dev->dma_mask; - spin_lock_irqsave(&domain->lock, flags); - *dma_addr = __map_single(dev, domain->priv, page_to_phys(page), size, DMA_BIDIRECTIONAL, true, dma_mask); - if (*dma_addr == DMA_ERROR_CODE) { - spin_unlock_irqrestore(&domain->lock, flags); + if (*dma_addr == DMA_ERROR_CODE) goto out_free; - } - - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); return page_address(page); @@ -2712,7 +2790,6 @@ static void free_coherent(struct device *dev, size_t size, struct dma_attrs *attrs) { struct protection_domain *domain; - unsigned long flags; struct page *page; INC_STATS_COUNTER(cnt_free_coherent); @@ -2724,14 +2801,8 @@ static void free_coherent(struct device *dev, size_t size, if (IS_ERR(domain)) goto free_mem; - spin_lock_irqsave(&domain->lock, flags); - __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); - domain_flush_complete(domain); - - spin_unlock_irqrestore(&domain->lock, flags); - free_mem: if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT)) __free_pages(page, get_order(size)); @@ -2746,14 +2817,43 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask) return check_device(dev); } +static int set_dma_mask(struct device *dev, u64 mask) +{ + struct protection_domain *domain; + int max_apertures = 1; + + domain = get_domain(dev); + if (IS_ERR(domain)) + return PTR_ERR(domain); + + if (mask == DMA_BIT_MASK(64)) + max_apertures = 8; + else if (mask > DMA_BIT_MASK(32)) + max_apertures = 4; + + /* + * To prevent lock contention it doesn't make sense to allocate more + * apertures than online cpus + */ + if (max_apertures > num_online_cpus()) + max_apertures = num_online_cpus(); + + if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures)) + dev_err(dev, "Can't allocate %d iommu apertures\n", + max_apertures); + + return 0; +} + static struct dma_map_ops amd_iommu_dma_ops = { - .alloc = alloc_coherent, - .free = free_coherent, - .map_page = map_page, - .unmap_page = unmap_page, - .map_sg = map_sg, - .unmap_sg = unmap_sg, - .dma_supported = amd_iommu_dma_supported, + .alloc = alloc_coherent, + .free = free_coherent, + .map_page = map_page, + .unmap_page = unmap_page, + .map_sg = map_sg, + .unmap_sg = unmap_sg, + .dma_supported = amd_iommu_dma_supported, + .set_dma_mask = set_dma_mask, }; int __init amd_iommu_init_api(void) @@ -3757,11 +3857,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info) case X86_IRQ_ALLOC_TYPE_MSI: case X86_IRQ_ALLOC_TYPE_MSIX: devid = get_device_id(&info->msi_dev->dev); - if (devid >= 0) { - iommu = amd_iommu_rlookup_table[devid]; - if (iommu) - return iommu->msi_domain; - } + iommu = amd_iommu_rlookup_table[devid]; + if (iommu) + return iommu->msi_domain; break; default: break; diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h index b08cf57..9d32b20 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd_iommu_types.h @@ -425,46 +425,6 @@ struct protection_domain { }; /* - * For dynamic growth the aperture size is split into ranges of 128MB of - * DMA address space each. This struct represents one such range. - */ -struct aperture_range { - - /* address allocation bitmap */ - unsigned long *bitmap; - - /* - * Array of PTE pages for the aperture. In this array we save all the - * leaf pages of the domain page table used for the aperture. This way - * we don't need to walk the page table to find a specific PTE. We can - * just calculate its address in constant time. - */ - u64 *pte_pages[64]; - - unsigned long offset; -}; - -/* - * Data container for a dma_ops specific protection domain - */ -struct dma_ops_domain { - /* generic protection domain information */ - struct protection_domain domain; - - /* size of the aperture for the mappings */ - unsigned long aperture_size; - - /* address we start to search for free addresses */ - unsigned long next_address; - - /* address space relevant data */ - struct aperture_range *aperture[APERTURE_MAX_RANGES]; - - /* This will be set to true when TLB needs to be flushed */ - bool need_flush; -}; - -/* * Structure where we save information about one hardware AMD IOMMU in the * system. */ diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c index 7caf2fa..c865737 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd_iommu_v2.c @@ -432,7 +432,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm) unbind_pasid(pasid_state); } -static struct mmu_notifier_ops iommu_mn = { +static const struct mmu_notifier_ops iommu_mn = { .release = mn_release, .clear_flush_young = mn_clear_flush_young, .invalidate_page = mn_invalidate_page, @@ -513,43 +513,39 @@ static bool access_error(struct vm_area_struct *vma, struct fault *fault) static void do_fault(struct work_struct *work) { struct fault *fault = container_of(work, struct fault, work); - struct mm_struct *mm; struct vm_area_struct *vma; + int ret = VM_FAULT_ERROR; + unsigned int flags = 0; + struct mm_struct *mm; u64 address; - int ret, write; - - write = !!(fault->flags & PPR_FAULT_WRITE); mm = fault->state->mm; address = fault->address; + if (fault->flags & PPR_FAULT_USER) + flags |= FAULT_FLAG_USER; + if (fault->flags & PPR_FAULT_WRITE) + flags |= FAULT_FLAG_WRITE; + down_read(&mm->mmap_sem); vma = find_extend_vma(mm, address); - if (!vma || address < vma->vm_start) { + if (!vma || address < vma->vm_start) /* failed to get a vma in the right range */ - up_read(&mm->mmap_sem); - handle_fault_error(fault); goto out; - } /* Check if we have the right permissions on the vma */ - if (access_error(vma, fault)) { - up_read(&mm->mmap_sem); - handle_fault_error(fault); + if (access_error(vma, fault)) goto out; - } - ret = handle_mm_fault(mm, vma, address, write); - if (ret & VM_FAULT_ERROR) { - /* failed to service fault */ - up_read(&mm->mmap_sem); - handle_fault_error(fault); - goto out; - } + ret = handle_mm_fault(mm, vma, address, flags); +out: up_read(&mm->mmap_sem); -out: + if (ret & VM_FAULT_ERROR) + /* failed to service fault */ + handle_fault_error(fault); + finish_pri_tag(fault->dev_state, fault->state, fault->tag); put_pasid_state(fault->state); diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 4e5118a..2087534 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -40,7 +40,10 @@ #define IDR0_ST_LVL_SHIFT 27 #define IDR0_ST_LVL_MASK 0x3 #define IDR0_ST_LVL_2LVL (1 << IDR0_ST_LVL_SHIFT) -#define IDR0_STALL_MODEL (3 << 24) +#define IDR0_STALL_MODEL_SHIFT 24 +#define IDR0_STALL_MODEL_MASK 0x3 +#define IDR0_STALL_MODEL_STALL (0 << IDR0_STALL_MODEL_SHIFT) +#define IDR0_STALL_MODEL_FORCE (2 << IDR0_STALL_MODEL_SHIFT) #define IDR0_TTENDIAN_SHIFT 21 #define IDR0_TTENDIAN_MASK 0x3 #define IDR0_TTENDIAN_LE (2 << IDR0_TTENDIAN_SHIFT) @@ -253,6 +256,9 @@ #define STRTAB_STE_1_STRW_EL2 2UL #define STRTAB_STE_1_STRW_SHIFT 30 +#define STRTAB_STE_1_SHCFG_INCOMING 1UL +#define STRTAB_STE_1_SHCFG_SHIFT 44 + #define STRTAB_STE_2_S2VMID_SHIFT 0 #define STRTAB_STE_2_S2VMID_MASK 0xffffUL #define STRTAB_STE_2_VTCR_SHIFT 32 @@ -378,7 +384,6 @@ #define PRIQ_0_SID_MASK 0xffffffffUL #define PRIQ_0_SSID_SHIFT 32 #define PRIQ_0_SSID_MASK 0xfffffUL -#define PRIQ_0_OF (1UL << 57) #define PRIQ_0_PERM_PRIV (1UL << 58) #define PRIQ_0_PERM_EXEC (1UL << 59) #define PRIQ_0_PERM_READ (1UL << 60) @@ -855,15 +860,17 @@ static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) }; dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, - cerror_str[idx]); + idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown"); switch (idx) { - case CMDQ_ERR_CERROR_ILL_IDX: - break; case CMDQ_ERR_CERROR_ABT_IDX: dev_err(smmu->dev, "retrying command fetch\n"); case CMDQ_ERR_CERROR_NONE_IDX: return; + case CMDQ_ERR_CERROR_ILL_IDX: + /* Fallthrough */ + default: + break; } /* @@ -1042,6 +1049,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, val |= disable_bypass ? STRTAB_STE_0_CFG_ABORT : STRTAB_STE_0_CFG_BYPASS; dst[0] = cpu_to_le64(val); + dst[1] = cpu_to_le64(STRTAB_STE_1_SHCFG_INCOMING + << STRTAB_STE_1_SHCFG_SHIFT); dst[2] = 0; /* Nuke the VMID */ if (ste_live) arm_smmu_sync_ste_for_sid(smmu, sid); @@ -1056,12 +1065,14 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid, STRTAB_STE_1_S1C_CACHE_WBRA << STRTAB_STE_1_S1COR_SHIFT | STRTAB_STE_1_S1C_SH_ISH << STRTAB_STE_1_S1CSH_SHIFT | - STRTAB_STE_1_S1STALLD | #ifdef CONFIG_PCI_ATS STRTAB_STE_1_EATS_TRANS << STRTAB_STE_1_EATS_SHIFT | #endif STRTAB_STE_1_STRW_NSEL1 << STRTAB_STE_1_STRW_SHIFT); + if (smmu->features & ARM_SMMU_FEAT_STALLS) + dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD); + val |= (ste->s1_cfg->cdptr_dma & STRTAB_STE_0_S1CTXPTR_MASK << STRTAB_STE_0_S1CTXPTR_SHIFT) | STRTAB_STE_0_CFG_S1_TRANS; @@ -1123,8 +1134,8 @@ static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) strtab = &cfg->strtab[(sid >> STRTAB_SPLIT) * STRTAB_L1_DESC_DWORDS]; desc->span = STRTAB_SPLIT + 1; - desc->l2ptr = dma_zalloc_coherent(smmu->dev, size, &desc->l2ptr_dma, - GFP_KERNEL); + desc->l2ptr = dmam_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, + GFP_KERNEL | __GFP_ZERO); if (!desc->l2ptr) { dev_err(smmu->dev, "failed to allocate l2 stream table for SID %u\n", @@ -1250,50 +1261,50 @@ static int arm_smmu_device_disable(struct arm_smmu_device *smmu); static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev) { - u32 gerror, gerrorn; + u32 gerror, gerrorn, active; struct arm_smmu_device *smmu = dev; gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); - gerror ^= gerrorn; - if (!(gerror & GERROR_ERR_MASK)) + active = gerror ^ gerrorn; + if (!(active & GERROR_ERR_MASK)) return IRQ_NONE; /* No errors pending */ dev_warn(smmu->dev, "unexpected global error reported (0x%08x), this could be serious\n", - gerror); + active); - if (gerror & GERROR_SFM_ERR) { + if (active & GERROR_SFM_ERR) { dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); arm_smmu_device_disable(smmu); } - if (gerror & GERROR_MSI_GERROR_ABT_ERR) + if (active & GERROR_MSI_GERROR_ABT_ERR) dev_warn(smmu->dev, "GERROR MSI write aborted\n"); - if (gerror & GERROR_MSI_PRIQ_ABT_ERR) { + if (active & GERROR_MSI_PRIQ_ABT_ERR) { dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); arm_smmu_priq_handler(irq, smmu->dev); } - if (gerror & GERROR_MSI_EVTQ_ABT_ERR) { + if (active & GERROR_MSI_EVTQ_ABT_ERR) { dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); arm_smmu_evtq_handler(irq, smmu->dev); } - if (gerror & GERROR_MSI_CMDQ_ABT_ERR) { + if (active & GERROR_MSI_CMDQ_ABT_ERR) { dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); arm_smmu_cmdq_sync_handler(irq, smmu->dev); } - if (gerror & GERROR_PRIQ_ABT_ERR) + if (active & GERROR_PRIQ_ABT_ERR) dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); - if (gerror & GERROR_EVTQ_ABT_ERR) + if (active & GERROR_EVTQ_ABT_ERR) dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); - if (gerror & GERROR_CMDQ_ERR) + if (active & GERROR_CMDQ_ERR) arm_smmu_cmdq_skip_err(smmu); writel(gerror, smmu->base + ARM_SMMU_GERRORN); @@ -1335,7 +1346,7 @@ static void arm_smmu_tlb_inv_context(void *cookie) } static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, - bool leaf, void *cookie) + size_t granule, bool leaf, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_device *smmu = smmu_domain->smmu; @@ -1354,7 +1365,10 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid; } - arm_smmu_cmdq_issue_cmd(smmu, &cmd); + do { + arm_smmu_cmdq_issue_cmd(smmu, &cmd); + cmd.tlbi.addr += granule; + } while (size -= granule); } static struct iommu_gather_ops arm_smmu_gather_ops = { @@ -1429,10 +1443,10 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) struct arm_smmu_s1_cfg *cfg = &smmu_domain->s1_cfg; if (cfg->cdptr) { - dma_free_coherent(smmu_domain->smmu->dev, - CTXDESC_CD_DWORDS << 3, - cfg->cdptr, - cfg->cdptr_dma); + dmam_free_coherent(smmu_domain->smmu->dev, + CTXDESC_CD_DWORDS << 3, + cfg->cdptr, + cfg->cdptr_dma); arm_smmu_bitmap_free(smmu->asid_map, cfg->cd.asid); } @@ -1457,8 +1471,9 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain, if (IS_ERR_VALUE(asid)) return asid; - cfg->cdptr = dma_zalloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3, - &cfg->cdptr_dma, GFP_KERNEL); + cfg->cdptr = dmam_alloc_coherent(smmu->dev, CTXDESC_CD_DWORDS << 3, + &cfg->cdptr_dma, + GFP_KERNEL | __GFP_ZERO); if (!cfg->cdptr) { dev_warn(smmu->dev, "failed to allocate context descriptor\n"); ret = -ENOMEM; @@ -1804,13 +1819,13 @@ static int arm_smmu_add_device(struct device *dev) smmu = arm_smmu_get_for_pci_dev(pdev); if (!smmu) { ret = -ENOENT; - goto out_put_group; + goto out_remove_dev; } smmu_group = kzalloc(sizeof(*smmu_group), GFP_KERNEL); if (!smmu_group) { ret = -ENOMEM; - goto out_put_group; + goto out_remove_dev; } smmu_group->ste.valid = true; @@ -1826,20 +1841,20 @@ static int arm_smmu_add_device(struct device *dev) for (i = 0; i < smmu_group->num_sids; ++i) { /* If we already know about this SID, then we're done */ if (smmu_group->sids[i] == sid) - return 0; + goto out_put_group; } /* Check the SID is in range of the SMMU and our stream table */ if (!arm_smmu_sid_in_range(smmu, sid)) { ret = -ERANGE; - goto out_put_group; + goto out_remove_dev; } /* Ensure l2 strtab is initialised */ if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { ret = arm_smmu_init_l2_strtab(smmu, sid); if (ret) - goto out_put_group; + goto out_remove_dev; } /* Resize the SID array for the group */ @@ -1849,16 +1864,20 @@ static int arm_smmu_add_device(struct device *dev) if (!sids) { smmu_group->num_sids--; ret = -ENOMEM; - goto out_put_group; + goto out_remove_dev; } /* Add the new SID */ sids[smmu_group->num_sids - 1] = sid; smmu_group->sids = sids; - return 0; out_put_group: iommu_group_put(group); + return 0; + +out_remove_dev: + iommu_group_remove_device(dev); + iommu_group_put(group); return ret; } @@ -1937,7 +1956,7 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, { size_t qsz = ((1 << q->max_n_shift) * dwords) << 3; - q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL); + q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL); if (!q->base) { dev_err(smmu->dev, "failed to allocate queue (0x%zx bytes)\n", qsz); @@ -1957,23 +1976,6 @@ static int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, return 0; } -static void arm_smmu_free_one_queue(struct arm_smmu_device *smmu, - struct arm_smmu_queue *q) -{ - size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3; - - dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma); -} - -static void arm_smmu_free_queues(struct arm_smmu_device *smmu) -{ - arm_smmu_free_one_queue(smmu, &smmu->cmdq.q); - arm_smmu_free_one_queue(smmu, &smmu->evtq.q); - - if (smmu->features & ARM_SMMU_FEAT_PRI) - arm_smmu_free_one_queue(smmu, &smmu->priq.q); -} - static int arm_smmu_init_queues(struct arm_smmu_device *smmu) { int ret; @@ -1983,49 +1985,20 @@ static int arm_smmu_init_queues(struct arm_smmu_device *smmu) ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS, CMDQ_ENT_DWORDS); if (ret) - goto out; + return ret; /* evtq */ ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS, EVTQ_ENT_DWORDS); if (ret) - goto out_free_cmdq; + return ret; /* priq */ if (!(smmu->features & ARM_SMMU_FEAT_PRI)) return 0; - ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, - ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS); - if (ret) - goto out_free_evtq; - - return 0; - -out_free_evtq: - arm_smmu_free_one_queue(smmu, &smmu->evtq.q); -out_free_cmdq: - arm_smmu_free_one_queue(smmu, &smmu->cmdq.q); -out: - return ret; -} - -static void arm_smmu_free_l2_strtab(struct arm_smmu_device *smmu) -{ - int i; - size_t size; - struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - - size = 1 << (STRTAB_SPLIT + ilog2(STRTAB_STE_DWORDS) + 3); - for (i = 0; i < cfg->num_l1_ents; ++i) { - struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[i]; - - if (!desc->l2ptr) - continue; - - dma_free_coherent(smmu->dev, size, desc->l2ptr, - desc->l2ptr_dma); - } + return arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD, + ARM_SMMU_PRIQ_CONS, PRIQ_ENT_DWORDS); } static int arm_smmu_init_l1_strtab(struct arm_smmu_device *smmu) @@ -2054,7 +2027,6 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) void *strtab; u64 reg; u32 size, l1size; - int ret; struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; /* @@ -2077,8 +2049,8 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) size, smmu->sid_bits); l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); - strtab = dma_zalloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, - GFP_KERNEL); + strtab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, + GFP_KERNEL | __GFP_ZERO); if (!strtab) { dev_err(smmu->dev, "failed to allocate l1 stream table (%u bytes)\n", @@ -2095,13 +2067,7 @@ static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) << STRTAB_BASE_CFG_SPLIT_SHIFT; cfg->strtab_base_cfg = reg; - ret = arm_smmu_init_l1_strtab(smmu); - if (ret) - dma_free_coherent(smmu->dev, - l1size, - strtab, - cfg->strtab_dma); - return ret; + return arm_smmu_init_l1_strtab(smmu); } static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) @@ -2112,8 +2078,8 @@ static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; size = (1 << smmu->sid_bits) * (STRTAB_STE_DWORDS << 3); - strtab = dma_zalloc_coherent(smmu->dev, size, &cfg->strtab_dma, - GFP_KERNEL); + strtab = dmam_alloc_coherent(smmu->dev, size, &cfg->strtab_dma, + GFP_KERNEL | __GFP_ZERO); if (!strtab) { dev_err(smmu->dev, "failed to allocate linear stream table (%u bytes)\n", @@ -2157,21 +2123,6 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) return 0; } -static void arm_smmu_free_strtab(struct arm_smmu_device *smmu) -{ - struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; - u32 size = cfg->num_l1_ents; - - if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { - arm_smmu_free_l2_strtab(smmu); - size *= STRTAB_L1_DESC_DWORDS << 3; - } else { - size *= STRTAB_STE_DWORDS * 3; - } - - dma_free_coherent(smmu->dev, size, cfg->strtab, cfg->strtab_dma); -} - static int arm_smmu_init_structures(struct arm_smmu_device *smmu) { int ret; @@ -2180,21 +2131,7 @@ static int arm_smmu_init_structures(struct arm_smmu_device *smmu) if (ret) return ret; - ret = arm_smmu_init_strtab(smmu); - if (ret) - goto out_free_queues; - - return 0; - -out_free_queues: - arm_smmu_free_queues(smmu); - return ret; -} - -static void arm_smmu_free_structures(struct arm_smmu_device *smmu) -{ - arm_smmu_free_strtab(smmu); - arm_smmu_free_queues(smmu); + return arm_smmu_init_strtab(smmu); } static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, @@ -2532,8 +2469,12 @@ static int arm_smmu_device_probe(struct arm_smmu_device *smmu) dev_warn(smmu->dev, "IDR0.COHACC overridden by dma-coherent property (%s)\n", coherent ? "true" : "false"); - if (reg & IDR0_STALL_MODEL) + switch (reg & IDR0_STALL_MODEL_MASK << IDR0_STALL_MODEL_SHIFT) { + case IDR0_STALL_MODEL_STALL: + /* Fallthrough */ + case IDR0_STALL_MODEL_FORCE: smmu->features |= ARM_SMMU_FEAT_STALLS; + } if (reg & IDR0_S1P) smmu->features |= ARM_SMMU_FEAT_TRANS_S1; @@ -2699,15 +2640,7 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev) platform_set_drvdata(pdev, smmu); /* Reset the device */ - ret = arm_smmu_device_reset(smmu); - if (ret) - goto out_free_structures; - - return 0; - -out_free_structures: - arm_smmu_free_structures(smmu); - return ret; + return arm_smmu_device_reset(smmu); } static int arm_smmu_device_remove(struct platform_device *pdev) @@ -2715,7 +2648,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev) struct arm_smmu_device *smmu = platform_get_drvdata(pdev); arm_smmu_device_disable(smmu); - arm_smmu_free_structures(smmu); return 0; } diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 47dc7a7..59ee4b8 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -582,7 +582,7 @@ static void arm_smmu_tlb_inv_context(void *cookie) } static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, - bool leaf, void *cookie) + size_t granule, bool leaf, void *cookie) { struct arm_smmu_domain *smmu_domain = cookie; struct arm_smmu_cfg *cfg = &smmu_domain->cfg; @@ -597,12 +597,18 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, if (!IS_ENABLED(CONFIG_64BIT) || smmu->version == ARM_SMMU_V1) { iova &= ~12UL; iova |= ARM_SMMU_CB_ASID(cfg); - writel_relaxed(iova, reg); + do { + writel_relaxed(iova, reg); + iova += granule; + } while (size -= granule); #ifdef CONFIG_64BIT } else { iova >>= 12; iova |= (u64)ARM_SMMU_CB_ASID(cfg) << 48; - writeq_relaxed(iova, reg); + do { + writeq_relaxed(iova, reg); + iova += granule >> 12; + } while (size -= granule); #endif } #ifdef CONFIG_64BIT @@ -610,7 +616,11 @@ static void arm_smmu_tlb_inv_range_nosync(unsigned long iova, size_t size, reg = ARM_SMMU_CB_BASE(smmu) + ARM_SMMU_CB(smmu, cfg->cbndx); reg += leaf ? ARM_SMMU_CB_S2_TLBIIPAS2L : ARM_SMMU_CB_S2_TLBIIPAS2; - writeq_relaxed(iova >> 12, reg); + iova >>= 12; + do { + writeq_relaxed(iova, reg); + iova += granule >> 12; + } while (size -= granule); #endif } else { reg = ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_TLBIVMID; @@ -945,9 +955,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) free_irq(irq, domain); } - if (smmu_domain->pgtbl_ops) - free_io_pgtable_ops(smmu_domain->pgtbl_ops); - + free_io_pgtable_ops(smmu_domain->pgtbl_ops); __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); } @@ -1357,6 +1365,7 @@ static int arm_smmu_add_device(struct device *dev) if (IS_ERR(group)) return PTR_ERR(group); + iommu_group_put(group); return 0; } diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 80e3c17..62a400c 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -1063,13 +1063,19 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) raw_spin_lock_init(&iommu->register_lock); - drhd->iommu = iommu; - - if (intel_iommu_enabled) + if (intel_iommu_enabled) { iommu->iommu_dev = iommu_device_create(NULL, iommu, intel_iommu_groups, "%s", iommu->name); + if (IS_ERR(iommu->iommu_dev)) { + err = PTR_ERR(iommu->iommu_dev); + goto err_unmap; + } + } + + drhd->iommu = iommu; + return 0; err_unmap: diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 7df9777..8bbcbfe 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -38,9 +38,6 @@ #define io_pgtable_to_data(x) \ container_of((x), struct arm_lpae_io_pgtable, iop) -#define io_pgtable_ops_to_pgtable(x) \ - container_of((x), struct io_pgtable, ops) - #define io_pgtable_ops_to_data(x) \ io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) @@ -58,8 +55,10 @@ ((((d)->levels - ((l) - ARM_LPAE_START_LVL(d) + 1)) \ * (d)->bits_per_level) + (d)->pg_shift) +#define ARM_LPAE_GRANULE(d) (1UL << (d)->pg_shift) + #define ARM_LPAE_PAGES_PER_PGD(d) \ - DIV_ROUND_UP((d)->pgd_size, 1UL << (d)->pg_shift) + DIV_ROUND_UP((d)->pgd_size, ARM_LPAE_GRANULE(d)) /* * Calculate the index at level l used to map virtual address a using the @@ -169,7 +168,7 @@ /* IOPTE accessors */ #define iopte_deref(pte,d) \ (__va((pte) & ((1ULL << ARM_LPAE_MAX_ADDR_BITS) - 1) \ - & ~((1ULL << (d)->pg_shift) - 1))) + & ~(ARM_LPAE_GRANULE(d) - 1ULL))) #define iopte_type(pte,l) \ (((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK) @@ -326,7 +325,7 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, /* Grab a pointer to the next level */ pte = *ptep; if (!pte) { - cptep = __arm_lpae_alloc_pages(1UL << data->pg_shift, + cptep = __arm_lpae_alloc_pages(ARM_LPAE_GRANULE(data), GFP_ATOMIC, cfg); if (!cptep) return -ENOMEM; @@ -405,17 +404,18 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, arm_lpae_iopte *start, *end; unsigned long table_size; - /* Only leaf entries at the last level */ - if (lvl == ARM_LPAE_MAX_LEVELS - 1) - return; - if (lvl == ARM_LPAE_START_LVL(data)) table_size = data->pgd_size; else - table_size = 1UL << data->pg_shift; + table_size = ARM_LPAE_GRANULE(data); start = ptep; - end = (void *)ptep + table_size; + + /* Only leaf entries at the last level */ + if (lvl == ARM_LPAE_MAX_LEVELS - 1) + end = ptep; + else + end = (void *)ptep + table_size; while (ptep != end) { arm_lpae_iopte pte = *ptep++; @@ -473,7 +473,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, __arm_lpae_set_pte(ptep, table, cfg); iova &= ~(blk_size - 1); - cfg->tlb->tlb_add_flush(iova, blk_size, true, data->iop.cookie); + cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie); return size; } @@ -486,11 +486,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, void *cookie = data->iop.cookie; size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); + /* Something went horribly wrong and we ran out of page table */ + if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) + return 0; + ptep += ARM_LPAE_LVL_IDX(iova, lvl, data); pte = *ptep; - - /* Something went horribly wrong and we ran out of page table */ - if (WARN_ON(!pte || (lvl == ARM_LPAE_MAX_LEVELS))) + if (WARN_ON(!pte)) return 0; /* If the size matches this level, we're in the right place */ @@ -499,12 +501,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ - tlb->tlb_add_flush(iova, size, false, cookie); + tlb->tlb_add_flush(iova, size, ARM_LPAE_GRANULE(data), + false, cookie); tlb->tlb_sync(cookie); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); } else { - tlb->tlb_add_flush(iova, size, true, cookie); + tlb->tlb_add_flush(iova, size, size, true, cookie); } return size; @@ -570,7 +573,7 @@ static phys_addr_t arm_lpae_iova_to_phys(struct io_pgtable_ops *ops, return 0; found_translation: - iova &= ((1 << data->pg_shift) - 1); + iova &= (ARM_LPAE_GRANULE(data) - 1); return ((phys_addr_t)iopte_to_pfn(pte,data) << data->pg_shift) | iova; } @@ -668,7 +671,7 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_IRGN0_SHIFT) | (ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT); - switch (1 << data->pg_shift) { + switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: reg |= ARM_LPAE_TCR_TG0_4K; break; @@ -769,7 +772,7 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) sl = ARM_LPAE_START_LVL(data); - switch (1 << data->pg_shift) { + switch (ARM_LPAE_GRANULE(data)) { case SZ_4K: reg |= ARM_LPAE_TCR_TG0_4K; sl++; /* SL0 format is different for 4K granule size */ @@ -889,8 +892,8 @@ static void dummy_tlb_flush_all(void *cookie) WARN_ON(cookie != cfg_cookie); } -static void dummy_tlb_add_flush(unsigned long iova, size_t size, bool leaf, - void *cookie) +static void dummy_tlb_add_flush(unsigned long iova, size_t size, + size_t granule, bool leaf, void *cookie) { WARN_ON(cookie != cfg_cookie); WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index ac9e234..36673c8 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -26,8 +26,8 @@ enum io_pgtable_fmt { */ struct iommu_gather_ops { void (*tlb_flush_all)(void *cookie); - void (*tlb_add_flush)(unsigned long iova, size_t size, bool leaf, - void *cookie); + void (*tlb_add_flush)(unsigned long iova, size_t size, size_t granule, + bool leaf, void *cookie); void (*tlb_sync)(void *cookie); }; @@ -131,6 +131,8 @@ struct io_pgtable { struct io_pgtable_ops ops; }; +#define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) + /** * struct io_pgtable_init_fns - Alloc/free a set of page tables for a * particular format. diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index dfb868e..2fdbac6 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -277,8 +277,8 @@ static void ipmmu_tlb_flush_all(void *cookie) ipmmu_tlb_invalidate(domain); } -static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, bool leaf, - void *cookie) +static void ipmmu_tlb_add_flush(unsigned long iova, size_t size, + size_t granule, bool leaf, void *cookie) { /* The hardware doesn't support selective TLB flush. */ } diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c index b6d01f9..4b09e81 100644 --- a/drivers/iommu/msm_iommu_dev.c +++ b/drivers/iommu/msm_iommu_dev.c @@ -359,30 +359,19 @@ static struct platform_driver msm_iommu_ctx_driver = { .remove = msm_iommu_ctx_remove, }; +static struct platform_driver * const drivers[] = { + &msm_iommu_driver, + &msm_iommu_ctx_driver, +}; + static int __init msm_iommu_driver_init(void) { - int ret; - ret = platform_driver_register(&msm_iommu_driver); - if (ret != 0) { - pr_err("Failed to register IOMMU driver\n"); - goto error; - } - - ret = platform_driver_register(&msm_iommu_ctx_driver); - if (ret != 0) { - platform_driver_unregister(&msm_iommu_driver); - pr_err("Failed to register IOMMU context driver\n"); - goto error; - } - -error: - return ret; + return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); } static void __exit msm_iommu_driver_exit(void) { - platform_driver_unregister(&msm_iommu_ctx_driver); - platform_driver_unregister(&msm_iommu_driver); + platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); } subsys_initcall(msm_iommu_driver_init); diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c index 471ee36..a04d491 100644 --- a/drivers/iommu/s390-iommu.c +++ b/drivers/iommu/s390-iommu.c @@ -49,7 +49,7 @@ static bool s390_iommu_capable(enum iommu_cap cap) } } -struct iommu_domain *s390_domain_alloc(unsigned domain_type) +static struct iommu_domain *s390_domain_alloc(unsigned domain_type) { struct s390_domain *s390_domain; @@ -73,7 +73,7 @@ struct iommu_domain *s390_domain_alloc(unsigned domain_type) return &s390_domain->domain; } -void s390_domain_free(struct iommu_domain *domain) +static void s390_domain_free(struct iommu_domain *domain) { struct s390_domain *s390_domain = to_s390_domain(domain); diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c deleted file mode 100644 index a028751..0000000 --- a/drivers/iommu/shmobile-iommu.c +++ /dev/null @@ -1,402 +0,0 @@ -/* - * IOMMU for IPMMU/IPMMUI - * Copyright (C) 2012 Hideki EIRAKU - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - */ - -#include <linux/dma-mapping.h> -#include <linux/io.h> -#include <linux/iommu.h> -#include <linux/platform_device.h> -#include <linux/sizes.h> -#include <linux/slab.h> -#include <asm/dma-iommu.h> -#include "shmobile-ipmmu.h" - -#define L1_SIZE CONFIG_SHMOBILE_IOMMU_L1SIZE -#define L1_LEN (L1_SIZE / 4) -#define L1_ALIGN L1_SIZE -#define L2_SIZE SZ_1K -#define L2_LEN (L2_SIZE / 4) -#define L2_ALIGN L2_SIZE - -struct shmobile_iommu_domain_pgtable { - uint32_t *pgtable; - dma_addr_t handle; -}; - -struct shmobile_iommu_archdata { - struct list_head attached_list; - struct dma_iommu_mapping *iommu_mapping; - spinlock_t attach_lock; - struct shmobile_iommu_domain *attached; - int num_attached_devices; - struct shmobile_ipmmu *ipmmu; -}; - -struct shmobile_iommu_domain { - struct shmobile_iommu_domain_pgtable l1, l2[L1_LEN]; - spinlock_t map_lock; - spinlock_t attached_list_lock; - struct list_head attached_list; - struct iommu_domain domain; -}; - -static struct shmobile_iommu_archdata *ipmmu_archdata; -static struct kmem_cache *l1cache, *l2cache; - -static struct shmobile_iommu_domain *to_sh_domain(struct iommu_domain *dom) -{ - return container_of(dom, struct shmobile_iommu_domain, domain); -} - -static int pgtable_alloc(struct shmobile_iommu_domain_pgtable *pgtable, - struct kmem_cache *cache, size_t size) -{ - pgtable->pgtable = kmem_cache_zalloc(cache, GFP_ATOMIC); - if (!pgtable->pgtable) - return -ENOMEM; - pgtable->handle = dma_map_single(NULL, pgtable->pgtable, size, - DMA_TO_DEVICE); - return 0; -} - -static void pgtable_free(struct shmobile_iommu_domain_pgtable *pgtable, - struct kmem_cache *cache, size_t size) -{ - dma_unmap_single(NULL, pgtable->handle, size, DMA_TO_DEVICE); - kmem_cache_free(cache, pgtable->pgtable); -} - -static uint32_t pgtable_read(struct shmobile_iommu_domain_pgtable *pgtable, - unsigned int index) -{ - return pgtable->pgtable[index]; -} - -static void pgtable_write(struct shmobile_iommu_domain_pgtable *pgtable, - unsigned int index, unsigned int count, uint32_t val) -{ - unsigned int i; - - for (i = 0; i < count; i++) - pgtable->pgtable[index + i] = val; - dma_sync_single_for_device(NULL, pgtable->handle + index * sizeof(val), - sizeof(val) * count, DMA_TO_DEVICE); -} - -static struct iommu_domain *shmobile_iommu_domain_alloc(unsigned type) -{ - struct shmobile_iommu_domain *sh_domain; - int i, ret; - - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; - - sh_domain = kzalloc(sizeof(*sh_domain), GFP_KERNEL); - if (!sh_domain) - return NULL; - ret = pgtable_alloc(&sh_domain->l1, l1cache, L1_SIZE); - if (ret < 0) { - kfree(sh_domain); - return NULL; - } - for (i = 0; i < L1_LEN; i++) - sh_domain->l2[i].pgtable = NULL; - spin_lock_init(&sh_domain->map_lock); - spin_lock_init(&sh_domain->attached_list_lock); - INIT_LIST_HEAD(&sh_domain->attached_list); - return &sh_domain->domain; -} - -static void shmobile_iommu_domain_free(struct iommu_domain *domain) -{ - struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain); - int i; - - for (i = 0; i < L1_LEN; i++) { - if (sh_domain->l2[i].pgtable) - pgtable_free(&sh_domain->l2[i], l2cache, L2_SIZE); - } - pgtable_free(&sh_domain->l1, l1cache, L1_SIZE); - kfree(sh_domain); -} - -static int shmobile_iommu_attach_device(struct iommu_domain *domain, - struct device *dev) -{ - struct shmobile_iommu_archdata *archdata = dev->archdata.iommu; - struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain); - int ret = -EBUSY; - - if (!archdata) - return -ENODEV; - spin_lock(&sh_domain->attached_list_lock); - spin_lock(&archdata->attach_lock); - if (archdata->attached != sh_domain) { - if (archdata->attached) - goto err; - ipmmu_tlb_set(archdata->ipmmu, sh_domain->l1.handle, L1_SIZE, - 0); - ipmmu_tlb_flush(archdata->ipmmu); - archdata->attached = sh_domain; - archdata->num_attached_devices = 0; - list_add(&archdata->attached_list, &sh_domain->attached_list); - } - archdata->num_attached_devices++; - ret = 0; -err: - spin_unlock(&archdata->attach_lock); - spin_unlock(&sh_domain->attached_list_lock); - return ret; -} - -static void shmobile_iommu_detach_device(struct iommu_domain *domain, - struct device *dev) -{ - struct shmobile_iommu_archdata *archdata = dev->archdata.iommu; - struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain); - - if (!archdata) - return; - spin_lock(&sh_domain->attached_list_lock); - spin_lock(&archdata->attach_lock); - archdata->num_attached_devices--; - if (!archdata->num_attached_devices) { - ipmmu_tlb_set(archdata->ipmmu, 0, 0, 0); - ipmmu_tlb_flush(archdata->ipmmu); - archdata->attached = NULL; - list_del(&archdata->attached_list); - } - spin_unlock(&archdata->attach_lock); - spin_unlock(&sh_domain->attached_list_lock); -} - -static void domain_tlb_flush(struct shmobile_iommu_domain *sh_domain) -{ - struct shmobile_iommu_archdata *archdata; - - spin_lock(&sh_domain->attached_list_lock); - list_for_each_entry(archdata, &sh_domain->attached_list, attached_list) - ipmmu_tlb_flush(archdata->ipmmu); - spin_unlock(&sh_domain->attached_list_lock); -} - -static int l2alloc(struct shmobile_iommu_domain *sh_domain, - unsigned int l1index) -{ - int ret; - - if (!sh_domain->l2[l1index].pgtable) { - ret = pgtable_alloc(&sh_domain->l2[l1index], l2cache, L2_SIZE); - if (ret < 0) - return ret; - } - pgtable_write(&sh_domain->l1, l1index, 1, - sh_domain->l2[l1index].handle | 0x1); - return 0; -} - -static void l2realfree(struct shmobile_iommu_domain_pgtable *l2) -{ - if (l2->pgtable) - pgtable_free(l2, l2cache, L2_SIZE); -} - -static void l2free(struct shmobile_iommu_domain *sh_domain, - unsigned int l1index, - struct shmobile_iommu_domain_pgtable *l2) -{ - pgtable_write(&sh_domain->l1, l1index, 1, 0); - if (sh_domain->l2[l1index].pgtable) { - *l2 = sh_domain->l2[l1index]; - sh_domain->l2[l1index].pgtable = NULL; - } -} - -static int shmobile_iommu_map(struct iommu_domain *domain, unsigned long iova, - phys_addr_t paddr, size_t size, int prot) -{ - struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL }; - struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain); - unsigned int l1index, l2index; - int ret; - - l1index = iova >> 20; - switch (size) { - case SZ_4K: - l2index = (iova >> 12) & 0xff; - spin_lock(&sh_domain->map_lock); - ret = l2alloc(sh_domain, l1index); - if (!ret) - pgtable_write(&sh_domain->l2[l1index], l2index, 1, - paddr | 0xff2); - spin_unlock(&sh_domain->map_lock); - break; - case SZ_64K: - l2index = (iova >> 12) & 0xf0; - spin_lock(&sh_domain->map_lock); - ret = l2alloc(sh_domain, l1index); - if (!ret) - pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, - paddr | 0xff1); - spin_unlock(&sh_domain->map_lock); - break; - case SZ_1M: - spin_lock(&sh_domain->map_lock); - l2free(sh_domain, l1index, &l2); - pgtable_write(&sh_domain->l1, l1index, 1, paddr | 0xc02); - spin_unlock(&sh_domain->map_lock); - ret = 0; - break; - default: - ret = -EINVAL; - } - if (!ret) - domain_tlb_flush(sh_domain); - l2realfree(&l2); - return ret; -} - -static size_t shmobile_iommu_unmap(struct iommu_domain *domain, - unsigned long iova, size_t size) -{ - struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL }; - struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain); - unsigned int l1index, l2index; - uint32_t l2entry = 0; - size_t ret = 0; - - l1index = iova >> 20; - if (!(iova & 0xfffff) && size >= SZ_1M) { - spin_lock(&sh_domain->map_lock); - l2free(sh_domain, l1index, &l2); - spin_unlock(&sh_domain->map_lock); - ret = SZ_1M; - goto done; - } - l2index = (iova >> 12) & 0xff; - spin_lock(&sh_domain->map_lock); - if (sh_domain->l2[l1index].pgtable) - l2entry = pgtable_read(&sh_domain->l2[l1index], l2index); - switch (l2entry & 3) { - case 1: - if (l2index & 0xf) - break; - pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, 0); - ret = SZ_64K; - break; - case 2: - pgtable_write(&sh_domain->l2[l1index], l2index, 1, 0); - ret = SZ_4K; - break; - } - spin_unlock(&sh_domain->map_lock); -done: - if (ret) - domain_tlb_flush(sh_domain); - l2realfree(&l2); - return ret; -} - -static phys_addr_t shmobile_iommu_iova_to_phys(struct iommu_domain *domain, - dma_addr_t iova) -{ - struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain); - uint32_t l1entry = 0, l2entry = 0; - unsigned int l1index, l2index; - - l1index = iova >> 20; - l2index = (iova >> 12) & 0xff; - spin_lock(&sh_domain->map_lock); - if (sh_domain->l2[l1index].pgtable) - l2entry = pgtable_read(&sh_domain->l2[l1index], l2index); - else - l1entry = pgtable_read(&sh_domain->l1, l1index); - spin_unlock(&sh_domain->map_lock); - switch (l2entry & 3) { - case 1: - return (l2entry & ~0xffff) | (iova & 0xffff); - case 2: - return (l2entry & ~0xfff) | (iova & 0xfff); - default: - if ((l1entry & 3) == 2) - return (l1entry & ~0xfffff) | (iova & 0xfffff); - return 0; - } -} - -static int find_dev_name(struct shmobile_ipmmu *ipmmu, const char *dev_name) -{ - unsigned int i, n = ipmmu->num_dev_names; - - for (i = 0; i < n; i++) { - if (strcmp(ipmmu->dev_names[i], dev_name) == 0) - return 1; - } - return 0; -} - -static int shmobile_iommu_add_device(struct device *dev) -{ - struct shmobile_iommu_archdata *archdata = ipmmu_archdata; - struct dma_iommu_mapping *mapping; - - if (!find_dev_name(archdata->ipmmu, dev_name(dev))) - return 0; - mapping = archdata->iommu_mapping; - if (!mapping) { - mapping = arm_iommu_create_mapping(&platform_bus_type, 0, - L1_LEN << 20); - if (IS_ERR(mapping)) - return PTR_ERR(mapping); - archdata->iommu_mapping = mapping; - } - dev->archdata.iommu = archdata; - if (arm_iommu_attach_device(dev, mapping)) - pr_err("arm_iommu_attach_device failed\n"); - return 0; -} - -static const struct iommu_ops shmobile_iommu_ops = { - .domain_alloc = shmobile_iommu_domain_alloc, - .domain_free = shmobile_iommu_domain_free, - .attach_dev = shmobile_iommu_attach_device, - .detach_dev = shmobile_iommu_detach_device, - .map = shmobile_iommu_map, - .unmap = shmobile_iommu_unmap, - .map_sg = default_iommu_map_sg, - .iova_to_phys = shmobile_iommu_iova_to_phys, - .add_device = shmobile_iommu_add_device, - .pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K, -}; - -int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu) -{ - static struct shmobile_iommu_archdata *archdata; - - l1cache = kmem_cache_create("shmobile-iommu-pgtable1", L1_SIZE, - L1_ALIGN, SLAB_HWCACHE_ALIGN, NULL); - if (!l1cache) - return -ENOMEM; - l2cache = kmem_cache_create("shmobile-iommu-pgtable2", L2_SIZE, - L2_ALIGN, SLAB_HWCACHE_ALIGN, NULL); - if (!l2cache) { - kmem_cache_destroy(l1cache); - return -ENOMEM; - } - archdata = kzalloc(sizeof(*archdata), GFP_KERNEL); - if (!archdata) { - kmem_cache_destroy(l1cache); - kmem_cache_destroy(l2cache); - return -ENOMEM; - } - spin_lock_init(&archdata->attach_lock); - archdata->ipmmu = ipmmu; - ipmmu_archdata = archdata; - bus_set_iommu(&platform_bus_type, &shmobile_iommu_ops); - return 0; -} diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c deleted file mode 100644 index 951651a..0000000 --- a/drivers/iommu/shmobile-ipmmu.c +++ /dev/null @@ -1,129 +0,0 @@ -/* - * IPMMU/IPMMUI - * Copyright (C) 2012 Hideki EIRAKU - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - */ - -#include <linux/err.h> -#include <linux/export.h> -#include <linux/io.h> -#include <linux/platform_device.h> -#include <linux/slab.h> -#include <linux/platform_data/sh_ipmmu.h> -#include "shmobile-ipmmu.h" - -#define IMCTR1 0x000 -#define IMCTR2 0x004 -#define IMASID 0x010 -#define IMTTBR 0x014 -#define IMTTBCR 0x018 - -#define IMCTR1_TLBEN (1 << 0) -#define IMCTR1_FLUSH (1 << 1) - -static void ipmmu_reg_write(struct shmobile_ipmmu *ipmmu, unsigned long reg_off, - unsigned long data) -{ - iowrite32(data, ipmmu->ipmmu_base + reg_off); -} - -void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu) -{ - if (!ipmmu) - return; - - spin_lock(&ipmmu->flush_lock); - if (ipmmu->tlb_enabled) - ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH | IMCTR1_TLBEN); - else - ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH); - spin_unlock(&ipmmu->flush_lock); -} - -void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size, - int asid) -{ - if (!ipmmu) - return; - - spin_lock(&ipmmu->flush_lock); - switch (size) { - default: - ipmmu->tlb_enabled = 0; - break; - case 0x2000: - ipmmu_reg_write(ipmmu, IMTTBCR, 1); - ipmmu->tlb_enabled = 1; - break; - case 0x1000: - ipmmu_reg_write(ipmmu, IMTTBCR, 2); - ipmmu->tlb_enabled = 1; - break; - case 0x800: - ipmmu_reg_write(ipmmu, IMTTBCR, 3); - ipmmu->tlb_enabled = 1; - break; - case 0x400: - ipmmu_reg_write(ipmmu, IMTTBCR, 4); - ipmmu->tlb_enabled = 1; - break; - case 0x200: - ipmmu_reg_write(ipmmu, IMTTBCR, 5); - ipmmu->tlb_enabled = 1; - break; - case 0x100: - ipmmu_reg_write(ipmmu, IMTTBCR, 6); - ipmmu->tlb_enabled = 1; - break; - case 0x80: - ipmmu_reg_write(ipmmu, IMTTBCR, 7); - ipmmu->tlb_enabled = 1; - break; - } - ipmmu_reg_write(ipmmu, IMTTBR, phys); - ipmmu_reg_write(ipmmu, IMASID, asid); - spin_unlock(&ipmmu->flush_lock); -} - -static int ipmmu_probe(struct platform_device *pdev) -{ - struct shmobile_ipmmu *ipmmu; - struct resource *res; - struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data; - - ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL); - if (!ipmmu) { - dev_err(&pdev->dev, "cannot allocate device data\n"); - return -ENOMEM; - } - spin_lock_init(&ipmmu->flush_lock); - ipmmu->dev = &pdev->dev; - - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(ipmmu->ipmmu_base)) - return PTR_ERR(ipmmu->ipmmu_base); - - ipmmu->dev_names = pdata->dev_names; - ipmmu->num_dev_names = pdata->num_dev_names; - platform_set_drvdata(pdev, ipmmu); - ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */ - ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */ - return ipmmu_iommu_init(ipmmu); -} - -static struct platform_driver ipmmu_driver = { - .probe = ipmmu_probe, - .driver = { - .name = "ipmmu", - }, -}; - -static int __init ipmmu_init(void) -{ - return platform_driver_register(&ipmmu_driver); -} -subsys_initcall(ipmmu_init); diff --git a/drivers/iommu/shmobile-ipmmu.h b/drivers/iommu/shmobile-ipmmu.h deleted file mode 100644 index 9524743..0000000 --- a/drivers/iommu/shmobile-ipmmu.h +++ /dev/null @@ -1,34 +0,0 @@ -/* shmobile-ipmmu.h - * - * Copyright (C) 2012 Hideki EIRAKU - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - */ - -#ifndef __SHMOBILE_IPMMU_H__ -#define __SHMOBILE_IPMMU_H__ - -struct shmobile_ipmmu { - struct device *dev; - void __iomem *ipmmu_base; - int tlb_enabled; - spinlock_t flush_lock; - const char * const *dev_names; - unsigned int num_dev_names; -}; - -#ifdef CONFIG_SHMOBILE_IPMMU_TLB -void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu); -void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size, - int asid); -int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu); -#else -static inline int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu) -{ - return -EINVAL; -} -#endif - -#endif /* __SHMOBILE_IPMMU_H__ */ diff --git a/drivers/misc/mic/card/mic_virtio.c b/drivers/misc/mic/card/mic_virtio.c index e486a0c..f6ed57d 100644 --- a/drivers/misc/mic/card/mic_virtio.c +++ b/drivers/misc/mic/card/mic_virtio.c @@ -311,7 +311,7 @@ unmap: static int mic_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { struct mic_vdev *mvdev = to_micvdev(vdev); struct mic_device_ctrl __iomem *dc = mvdev->dc; diff --git a/drivers/nvme/host/lightnvm.c b/drivers/nvme/host/lightnvm.c index 15f2acb..1af54ea 100644 --- a/drivers/nvme/host/lightnvm.c +++ b/drivers/nvme/host/lightnvm.c @@ -471,7 +471,7 @@ static int nvme_nvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd) struct bio *bio = rqd->bio; struct nvme_nvm_command *cmd; - rq = blk_mq_alloc_request(q, bio_rw(bio), GFP_KERNEL, 0); + rq = blk_mq_alloc_request(q, bio_rw(bio), 0); if (IS_ERR(rq)) return -ENOMEM; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 0c67b57..f5c0e26 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -1041,7 +1041,7 @@ int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd, struct request *req; int ret; - req = blk_mq_alloc_request(q, write, GFP_KERNEL, false); + req = blk_mq_alloc_request(q, write, 0); if (IS_ERR(req)) return PTR_ERR(req); @@ -1094,7 +1094,8 @@ static int nvme_submit_async_admin_req(struct nvme_dev *dev) struct nvme_cmd_info *cmd_info; struct request *req; - req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, true); + req = blk_mq_alloc_request(dev->admin_q, WRITE, + BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED); if (IS_ERR(req)) return PTR_ERR(req); @@ -1119,7 +1120,7 @@ static int nvme_submit_admin_async_cmd(struct nvme_dev *dev, struct request *req; struct nvme_cmd_info *cmd_rq; - req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false); + req = blk_mq_alloc_request(dev->admin_q, WRITE, 0); if (IS_ERR(req)) return PTR_ERR(req); @@ -1320,8 +1321,8 @@ static void nvme_abort_req(struct request *req) if (!dev->abort_limit) return; - abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, - false); + abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, + BLK_MQ_REQ_NOWAIT); if (IS_ERR(abort_req)) return; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index e1a1023..e44872f 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -147,7 +147,7 @@ static void rproc_virtio_del_vqs(struct virtio_device *vdev) static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { struct rproc *rproc = vdev_to_rproc(vdev); int i, ret; diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index 73354ee..1fcd27c 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -945,7 +945,7 @@ static void rpmsg_ns_cb(struct rpmsg_channel *rpdev, void *data, int len, static int rpmsg_probe(struct virtio_device *vdev) { vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; - const char *names[] = { "input", "output" }; + static const char * const names[] = { "input", "output" }; struct virtqueue *vqs[2]; struct virtproc_info *vrp; void *bufs_va; diff --git a/drivers/s390/virtio/kvm_virtio.c b/drivers/s390/virtio/kvm_virtio.c index 53fb975..1d060fd 100644 --- a/drivers/s390/virtio/kvm_virtio.c +++ b/drivers/s390/virtio/kvm_virtio.c @@ -255,7 +255,7 @@ static void kvm_del_vqs(struct virtio_device *vdev) static int kvm_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { struct kvm_device *kdev = to_kvmdev(vdev); int i; diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 1b83159..bf2d130 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -635,7 +635,7 @@ out: static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { struct virtio_ccw_device *vcdev = to_vc_device(vdev); unsigned long *indicatorp = NULL; diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 7efc329..0c3691f 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -209,8 +209,8 @@ static unsigned leak_balloon(struct virtio_balloon *vb, size_t num) */ if (vb->num_pfns != 0) tell_host(vb, vb->deflate_vq); - mutex_unlock(&vb->balloon_lock); release_pages_balloon(vb); + mutex_unlock(&vb->balloon_lock); return num_freed_pages; } @@ -388,7 +388,7 @@ static int init_vqs(struct virtio_balloon *vb) { struct virtqueue *vqs[3]; vq_callback_t *callbacks[] = { balloon_ack, balloon_ack, stats_request }; - const char *names[] = { "inflate", "deflate", "stats" }; + static const char * const names[] = { "inflate", "deflate", "stats" }; int err, nvqs; /* diff --git a/drivers/virtio/virtio_input.c b/drivers/virtio/virtio_input.c index c96944b..350a2a5 100644 --- a/drivers/virtio/virtio_input.c +++ b/drivers/virtio/virtio_input.c @@ -170,7 +170,7 @@ static int virtinput_init_vqs(struct virtio_input *vi) struct virtqueue *vqs[2]; vq_callback_t *cbs[] = { virtinput_recv_events, virtinput_recv_status }; - static const char *names[] = { "events", "status" }; + static const char * const names[] = { "events", "status" }; int err; err = vi->vdev->config->find_vqs(vi->vdev, 2, vqs, cbs, names); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index f499d9d..745c6ee 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -482,7 +482,7 @@ error_available: static int vm_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); unsigned int irq = platform_get_irq(vm_dev->pdev, 0); diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 78f804a..36205c2 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -296,7 +296,7 @@ void vp_del_vqs(struct virtio_device *vdev) static int vp_try_to_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[], + const char * const names[], bool use_msix, bool per_vq_vectors) { @@ -376,7 +376,7 @@ error_find: int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { int err; diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index b976d96..2cc2522 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -139,7 +139,7 @@ void vp_del_vqs(struct virtio_device *vdev); int vp_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]); + const char * const names[]); const char *vp_bus_name(struct virtio_device *vdev); /* Setup the affinity for a virtqueue: diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 8e5cf19..c0c11fa 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -418,7 +418,7 @@ err_new_queue: static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]) + const char * const names[]) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtqueue *vq; diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index ee663c4..e12e385 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -517,10 +517,10 @@ void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) /* If we expect an interrupt for the next entry, tell host * by writing event index and flush out the write before * the read in the next get_buf call. */ - if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { - vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx); - virtio_mb(vq->weak_barriers); - } + if (!(vq->avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) + virtio_store_mb(vq->weak_barriers, + &vring_used_event(&vq->vring), + cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); #ifdef DEBUG vq->last_add_time_valid = false; @@ -653,8 +653,11 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) } /* TODO: tune this threshold */ bufs = (u16)(vq->avail_idx_shadow - vq->last_used_idx) * 3 / 4; - vring_used_event(&vq->vring) = cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs); - virtio_mb(vq->weak_barriers); + + virtio_store_mb(vq->weak_barriers, + &vring_used_event(&vq->vring), + cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); + if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->vring.used->idx) - vq->last_used_idx) > bufs)) { END_USE(vq); return false; diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c index 96a1b8d..eff2b88 100644 --- a/drivers/xen/events/events_fifo.c +++ b/drivers/xen/events/events_fifo.c @@ -41,6 +41,7 @@ #include <linux/percpu.h> #include <linux/cpu.h> +#include <asm/barrier.h> #include <asm/sync_bitops.h> #include <asm/xen/hypercall.h> #include <asm/xen/hypervisor.h> @@ -296,7 +297,7 @@ static void consume_one_event(unsigned cpu, * control block. */ if (head == 0) { - rmb(); /* Ensure word is up-to-date before reading head. */ + virt_rmb(); /* Ensure word is up-to-date before reading head. */ head = control_block->head[priority]; } diff --git a/drivers/xen/xenbus/xenbus_comms.c b/drivers/xen/xenbus/xenbus_comms.c index fdb0f33..ecdecce 100644 --- a/drivers/xen/xenbus/xenbus_comms.c +++ b/drivers/xen/xenbus/xenbus_comms.c @@ -123,14 +123,14 @@ int xb_write(const void *data, unsigned len) avail = len; /* Must write data /after/ reading the consumer index. */ - mb(); + virt_mb(); memcpy(dst, data, avail); data += avail; len -= avail; /* Other side must not see new producer until data is there. */ - wmb(); + virt_wmb(); intf->req_prod += avail; /* Implies mb(): other side will see the updated producer. */ @@ -180,14 +180,14 @@ int xb_read(void *data, unsigned len) avail = len; /* Must read data /after/ reading the producer index. */ - rmb(); + virt_rmb(); memcpy(data, src, avail); data += avail; len -= avail; /* Other side must not see free space until we've copied out */ - mb(); + virt_mb(); intf->rsp_cons += avail; pr_debug("Finished read of %i bytes (%i to go)\n", avail, len); diff --git a/fs/block_dev.c b/fs/block_dev.c index 530145b..ba762ea 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -400,7 +400,7 @@ int bdev_read_page(struct block_device *bdev, sector_t sector, if (!ops->rw_page || bdev_get_integrity(bdev)) return result; - result = blk_queue_enter(bdev->bd_queue, GFP_KERNEL); + result = blk_queue_enter(bdev->bd_queue, false); if (result) return result; result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, READ); @@ -437,7 +437,7 @@ int bdev_write_page(struct block_device *bdev, sector_t sector, if (!ops->rw_page || bdev_get_integrity(bdev)) return -EOPNOTSUPP; - result = blk_queue_enter(bdev->bd_queue, GFP_NOIO); + result = blk_queue_enter(bdev->bd_queue, false); if (result) return result; @@ -700,7 +700,7 @@ static struct block_device *bd_acquire(struct inode *inode) spin_lock(&bdev_lock); bdev = inode->i_bdev; if (bdev) { - ihold(bdev->bd_inode); + bdgrab(bdev); spin_unlock(&bdev_lock); return bdev; } @@ -716,7 +716,7 @@ static struct block_device *bd_acquire(struct inode *inode) * So, we can access it via ->i_mapping always * without igrab(). */ - ihold(bdev->bd_inode); + bdgrab(bdev); inode->i_bdev = bdev; inode->i_mapping = bdev->bd_inode->i_mapping; list_add(&inode->i_devices, &bdev->bd_inodes); @@ -739,7 +739,7 @@ void bd_forget(struct inode *inode) spin_unlock(&bdev_lock); if (bdev) - iput(bdev->bd_inode); + bdput(bdev); } /** diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h index 0f45f93..1cceca14 100644 --- a/include/asm-generic/barrier.h +++ b/include/asm-generic/barrier.h @@ -54,22 +54,38 @@ #define read_barrier_depends() do { } while (0) #endif +#ifndef __smp_mb +#define __smp_mb() mb() +#endif + +#ifndef __smp_rmb +#define __smp_rmb() rmb() +#endif + +#ifndef __smp_wmb +#define __smp_wmb() wmb() +#endif + +#ifndef __smp_read_barrier_depends +#define __smp_read_barrier_depends() read_barrier_depends() +#endif + #ifdef CONFIG_SMP #ifndef smp_mb -#define smp_mb() mb() +#define smp_mb() __smp_mb() #endif #ifndef smp_rmb -#define smp_rmb() rmb() +#define smp_rmb() __smp_rmb() #endif #ifndef smp_wmb -#define smp_wmb() wmb() +#define smp_wmb() __smp_wmb() #endif #ifndef smp_read_barrier_depends -#define smp_read_barrier_depends() read_barrier_depends() +#define smp_read_barrier_depends() __smp_read_barrier_depends() #endif #else /* !CONFIG_SMP */ @@ -92,32 +108,104 @@ #endif /* CONFIG_SMP */ +#ifndef __smp_store_mb +#define __smp_store_mb(var, value) do { WRITE_ONCE(var, value); __smp_mb(); } while (0) +#endif + +#ifndef __smp_mb__before_atomic +#define __smp_mb__before_atomic() __smp_mb() +#endif + +#ifndef __smp_mb__after_atomic +#define __smp_mb__after_atomic() __smp_mb() +#endif + +#ifndef __smp_store_release +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + __smp_mb(); \ + WRITE_ONCE(*p, v); \ +} while (0) +#endif + +#ifndef __smp_load_acquire +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + __smp_mb(); \ + ___p1; \ +}) +#endif + +#ifdef CONFIG_SMP + +#ifndef smp_store_mb +#define smp_store_mb(var, value) __smp_store_mb(var, value) +#endif + +#ifndef smp_mb__before_atomic +#define smp_mb__before_atomic() __smp_mb__before_atomic() +#endif + +#ifndef smp_mb__after_atomic +#define smp_mb__after_atomic() __smp_mb__after_atomic() +#endif + +#ifndef smp_store_release +#define smp_store_release(p, v) __smp_store_release(p, v) +#endif + +#ifndef smp_load_acquire +#define smp_load_acquire(p) __smp_load_acquire(p) +#endif + +#else /* !CONFIG_SMP */ + #ifndef smp_store_mb -#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); smp_mb(); } while (0) +#define smp_store_mb(var, value) do { WRITE_ONCE(var, value); barrier(); } while (0) #endif #ifndef smp_mb__before_atomic -#define smp_mb__before_atomic() smp_mb() +#define smp_mb__before_atomic() barrier() #endif #ifndef smp_mb__after_atomic -#define smp_mb__after_atomic() smp_mb() +#define smp_mb__after_atomic() barrier() #endif +#ifndef smp_store_release #define smp_store_release(p, v) \ do { \ compiletime_assert_atomic_type(*p); \ - smp_mb(); \ + barrier(); \ WRITE_ONCE(*p, v); \ } while (0) +#endif +#ifndef smp_load_acquire #define smp_load_acquire(p) \ ({ \ typeof(*p) ___p1 = READ_ONCE(*p); \ compiletime_assert_atomic_type(*p); \ - smp_mb(); \ + barrier(); \ ___p1; \ }) +#endif + +#endif + +/* Barriers for virtual machine guests when talking to an SMP host */ +#define virt_mb() __smp_mb() +#define virt_rmb() __smp_rmb() +#define virt_wmb() __smp_wmb() +#define virt_read_barrier_depends() __smp_read_barrier_depends() +#define virt_store_mb(var, value) __smp_store_mb(var, value) +#define virt_mb__before_atomic() __smp_mb__before_atomic() +#define virt_mb__after_atomic() __smp_mb__after_atomic() +#define virt_store_release(p, v) __smp_store_release(p, v) +#define virt_load_acquire(p) __smp_load_acquire(p) #endif /* !__ASSEMBLY__ */ #endif /* __ASM_GENERIC_BARRIER_H */ diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index daf17d7..7fc9296 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -188,8 +188,14 @@ void blk_mq_insert_request(struct request *, bool, bool, bool); void blk_mq_free_request(struct request *rq); void blk_mq_free_hctx_request(struct blk_mq_hw_ctx *, struct request *rq); bool blk_mq_can_queue(struct blk_mq_hw_ctx *); + +enum { + BLK_MQ_REQ_NOWAIT = (1 << 0), /* return when out of requests */ + BLK_MQ_REQ_RESERVED = (1 << 1), /* allocate from reserved pool */ +}; + struct request *blk_mq_alloc_request(struct request_queue *q, int rw, - gfp_t gfp, bool reserved); + unsigned int flags); struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); struct cpumask *blk_mq_tags_cpumask(struct blk_mq_tags *tags); diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index bfb64d6..d372ea8 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -795,7 +795,7 @@ extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t, extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t, struct scsi_ioctl_command __user *); -extern int blk_queue_enter(struct request_queue *q, gfp_t gfp); +extern int blk_queue_enter(struct request_queue *q, bool nowait); extern void blk_queue_exit(struct request_queue *q); extern void blk_start_queue(struct request_queue *q); extern void blk_start_queue_async(struct request_queue *q); diff --git a/include/linux/virtio_config.h b/include/linux/virtio_config.h index e5ce8ab..6e6cb0c 100644 --- a/include/linux/virtio_config.h +++ b/include/linux/virtio_config.h @@ -70,7 +70,7 @@ struct virtio_config_ops { int (*find_vqs)(struct virtio_device *, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char *names[]); + const char * const names[]); void (*del_vqs)(struct virtio_device *); u64 (*get_features)(struct virtio_device *vdev); int (*finalize_features)(struct virtio_device *vdev); diff --git a/include/linux/virtio_ring.h b/include/linux/virtio_ring.h index 8e50888..a156e2b 100644 --- a/include/linux/virtio_ring.h +++ b/include/linux/virtio_ring.h @@ -12,7 +12,7 @@ * anyone care? * * For virtio_pci on SMP, we don't need to order with respect to MMIO - * accesses through relaxed memory I/O windows, so smp_mb() et al are + * accesses through relaxed memory I/O windows, so virt_mb() et al are * sufficient. * * For using virtio to talk to real devices (eg. other heterogeneous @@ -23,18 +23,16 @@ static inline void virtio_mb(bool weak_barriers) { -#ifdef CONFIG_SMP if (weak_barriers) - smp_mb(); + virt_mb(); else -#endif mb(); } static inline void virtio_rmb(bool weak_barriers) { if (weak_barriers) - dma_rmb(); + virt_rmb(); else rmb(); } @@ -42,11 +40,22 @@ static inline void virtio_rmb(bool weak_barriers) static inline void virtio_wmb(bool weak_barriers) { if (weak_barriers) - dma_wmb(); + virt_wmb(); else wmb(); } +static inline void virtio_store_mb(bool weak_barriers, + __virtio16 *p, __virtio16 v) +{ + if (weak_barriers) { + virt_store_mb(*p, v); + } else { + WRITE_ONCE(*p, v); + mb(); + } +} + struct virtio_device; struct virtqueue; diff --git a/include/xen/interface/io/ring.h b/include/xen/interface/io/ring.h index 7dc685b..21f4fbd 100644 --- a/include/xen/interface/io/ring.h +++ b/include/xen/interface/io/ring.h @@ -208,12 +208,12 @@ struct __name##_back_ring { \ #define RING_PUSH_REQUESTS(_r) do { \ - wmb(); /* back sees requests /before/ updated producer index */ \ + virt_wmb(); /* back sees requests /before/ updated producer index */ \ (_r)->sring->req_prod = (_r)->req_prod_pvt; \ } while (0) #define RING_PUSH_RESPONSES(_r) do { \ - wmb(); /* front sees responses /before/ updated producer index */ \ + virt_wmb(); /* front sees responses /before/ updated producer index */ \ (_r)->sring->rsp_prod = (_r)->rsp_prod_pvt; \ } while (0) @@ -250,9 +250,9 @@ struct __name##_back_ring { \ #define RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(_r, _notify) do { \ RING_IDX __old = (_r)->sring->req_prod; \ RING_IDX __new = (_r)->req_prod_pvt; \ - wmb(); /* back sees requests /before/ updated producer index */ \ + virt_wmb(); /* back sees requests /before/ updated producer index */ \ (_r)->sring->req_prod = __new; \ - mb(); /* back sees new requests /before/ we check req_event */ \ + virt_mb(); /* back sees new requests /before/ we check req_event */ \ (_notify) = ((RING_IDX)(__new - (_r)->sring->req_event) < \ (RING_IDX)(__new - __old)); \ } while (0) @@ -260,9 +260,9 @@ struct __name##_back_ring { \ #define RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(_r, _notify) do { \ RING_IDX __old = (_r)->sring->rsp_prod; \ RING_IDX __new = (_r)->rsp_prod_pvt; \ - wmb(); /* front sees responses /before/ updated producer index */ \ + virt_wmb(); /* front sees responses /before/ updated producer index */ \ (_r)->sring->rsp_prod = __new; \ - mb(); /* front sees new responses /before/ we check rsp_event */ \ + virt_mb(); /* front sees new responses /before/ we check rsp_event */ \ (_notify) = ((RING_IDX)(__new - (_r)->sring->rsp_event) < \ (RING_IDX)(__new - __old)); \ } while (0) @@ -271,7 +271,7 @@ struct __name##_back_ring { \ (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \ if (_work_to_do) break; \ (_r)->sring->req_event = (_r)->req_cons + 1; \ - mb(); \ + virt_mb(); \ (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r); \ } while (0) @@ -279,7 +279,7 @@ struct __name##_back_ring { \ (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \ if (_work_to_do) break; \ (_r)->sring->rsp_event = (_r)->rsp_cons + 1; \ - mb(); \ + virt_mb(); \ (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r); \ } while (0) diff --git a/mm/balloon_compaction.c b/mm/balloon_compaction.c index d3116be..300117f 100644 --- a/mm/balloon_compaction.c +++ b/mm/balloon_compaction.c @@ -61,6 +61,7 @@ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) bool dequeued_page; dequeued_page = false; + spin_lock_irqsave(&b_dev_info->pages_lock, flags); list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) { /* * Block others from accessing the 'page' while we get around @@ -75,15 +76,14 @@ struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info) continue; } #endif - spin_lock_irqsave(&b_dev_info->pages_lock, flags); balloon_page_delete(page); __count_vm_event(BALLOON_DEFLATE); - spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); unlock_page(page); dequeued_page = true; break; } } + spin_unlock_irqrestore(&b_dev_info->pages_lock, flags); if (!dequeued_page) { /* diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 2b3c228..c7bf1aa 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -5116,13 +5116,44 @@ sub process { } } # check for memory barriers without a comment. - if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) { + + my $barriers = qr{ + mb| + rmb| + wmb| + read_barrier_depends + }x; + my $barrier_stems = qr{ + mb__before_atomic| + mb__after_atomic| + store_release| + load_acquire| + store_mb| + (?:$barriers) + }x; + my $all_barriers = qr{ + (?:$barriers)| + smp_(?:$barrier_stems)| + virt_(?:$barrier_stems) + }x; + + if ($line =~ /\b(?:$all_barriers)\s*\(/) { if (!ctx_has_comment($first_line, $linenr)) { WARN("MEMORY_BARRIER", "memory barrier without comment\n" . $herecurr); } } + my $underscore_smp_barriers = qr{__smp_(?:$barrier_stems)}x; + + if ($realfile !~ m@^include/asm-generic/@ && + $realfile !~ m@/barrier\.h$@ && + $line =~ m/\b(?:$underscore_smp_barriers)\s*\(/ && + $line !~ m/^.\s*\#\s*define\s+(?:$underscore_smp_barriers)\s*\(/) { + WARN("MEMORY_BARRIER", + "__smp memory barriers shouldn't be used outside barrier.h and asm-generic\n" . $herecurr); + } + # check for waitqueue_active without a comment. if ($line =~ /\bwaitqueue_active\s*\(/) { if (!ctx_has_comment($first_line, $linenr)) { |