From 5ee07612e9e20817bb99256ab6cf1400fd5aa270 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 6 Jan 2015 22:41:46 +0100 Subject: ppc/kvm: Replace ACCESS_ONCE with READ_ONCE ACCESS_ONCE does not work reliably on non-scalar types. For example gcc 4.6 and 4.7 might remove the volatile tag for such accesses during the SRA (scalar replacement of aggregates) step (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58145) Change the ppc/kvm code to replace ACCESS_ONCE with READ_ONCE. Signed-off-by: Christian Borntraeger Acked-by: Alexander Graf diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index 7b066f6..7c22997 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -152,7 +152,7 @@ static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, * in virtual mode. */ do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); /* Down_CPPR */ new_state.cppr = new_cppr; @@ -211,7 +211,7 @@ unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu) * pending priority */ do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); xirr = old_state.xisr | (((u32)old_state.cppr) << 24); if (!old_state.xisr) @@ -277,7 +277,7 @@ int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, * whenever the MFRR is made less favored. */ do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); /* Set_MFRR */ new_state.mfrr = mfrr; @@ -352,7 +352,7 @@ int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) icp_rm_clr_vcpu_irq(icp->vcpu); do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); reject = 0; new_state.cppr = cppr; diff --git a/arch/powerpc/kvm/book3s_xics.c b/arch/powerpc/kvm/book3s_xics.c index 807351f..a4a8d9f 100644 --- a/arch/powerpc/kvm/book3s_xics.c +++ b/arch/powerpc/kvm/book3s_xics.c @@ -327,7 +327,7 @@ static bool icp_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority, icp->server_num); do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); *reject = 0; @@ -512,7 +512,7 @@ static void icp_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, * in virtual mode. */ do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); /* Down_CPPR */ new_state.cppr = new_cppr; @@ -567,7 +567,7 @@ static noinline unsigned long kvmppc_h_xirr(struct kvm_vcpu *vcpu) * pending priority */ do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); xirr = old_state.xisr | (((u32)old_state.cppr) << 24); if (!old_state.xisr) @@ -634,7 +634,7 @@ static noinline int kvmppc_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, * whenever the MFRR is made less favored. */ do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); /* Set_MFRR */ new_state.mfrr = mfrr; @@ -679,7 +679,7 @@ static int kvmppc_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server) if (!icp) return H_PARAMETER; } - state = ACCESS_ONCE(icp->state); + state = READ_ONCE(icp->state); kvmppc_set_gpr(vcpu, 4, ((u32)state.cppr << 24) | state.xisr); kvmppc_set_gpr(vcpu, 5, state.mfrr); return H_SUCCESS; @@ -721,7 +721,7 @@ static noinline void kvmppc_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) BOOK3S_INTERRUPT_EXTERNAL_LEVEL); do { - old_state = new_state = ACCESS_ONCE(icp->state); + old_state = new_state = READ_ONCE(icp->state); reject = 0; new_state.cppr = cppr; @@ -885,7 +885,7 @@ static int xics_debug_show(struct seq_file *m, void *private) if (!icp) continue; - state.raw = ACCESS_ONCE(icp->state.raw); + state.raw = READ_ONCE(icp->state.raw); seq_printf(m, "cpu server %#lx XIRR:%#x PPRI:%#x CPPR:%#x MFRR:%#x OUT:%d NR:%d\n", icp->server_num, state.xisr, state.pending_pri, state.cppr, state.mfrr, @@ -1082,7 +1082,7 @@ int kvmppc_xics_set_icp(struct kvm_vcpu *vcpu, u64 icpval) * the ICS states before the ICP states. */ do { - old_state = ACCESS_ONCE(icp->state); + old_state = READ_ONCE(icp->state); if (new_state.mfrr <= old_state.mfrr) { resend = false; -- cgit v0.10.2 From da1a288d8562739aa8ba0273d4fb6b73b856c0d3 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 6 Jan 2015 22:47:41 +0100 Subject: ppc/hugetlbfs: Replace ACCESS_ONCE with READ_ONCE ACCESS_ONCE does not work reliably on non-scalar types. For example gcc 4.6 and 4.7 might remove the volatile tag for such accesses during the SRA (scalar replacement of aggregates) step (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58145) Change the ppc/hugetlbfs code to replace ACCESS_ONCE with READ_ONCE. Signed-off-by: Christian Borntraeger diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 5ff4e07..620d0ec 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -978,7 +978,7 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift */ pdshift = PUD_SHIFT; pudp = pud_offset(&pgd, ea); - pud = ACCESS_ONCE(*pudp); + pud = READ_ONCE(*pudp); if (pud_none(pud)) return NULL; @@ -990,7 +990,7 @@ pte_t *find_linux_pte_or_hugepte(pgd_t *pgdir, unsigned long ea, unsigned *shift else { pdshift = PMD_SHIFT; pmdp = pmd_offset(&pud, ea); - pmd = ACCESS_ONCE(*pmdp); + pmd = READ_ONCE(*pmdp); /* * A hugepage collapse is captured by pmd_none, because * it mark the pmd none and do a hpte invalidate. -- cgit v0.10.2 From 1760f1eb7ec485197bd3a8a9c13e4160bb740275 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Sun, 7 Dec 2014 22:01:59 +0100 Subject: x86/xen/p2m: Replace ACCESS_ONCE with READ_ONCE ACCESS_ONCE does not work reliably on non-scalar types. For example gcc 4.6 and 4.7 might remove the volatile tag for such accesses during the SRA (scalar replacement of aggregates) step (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58145) Change the p2m code to replace ACCESS_ONCE with READ_ONCE. Signed-off-by: Christian Borntraeger Reviewed-by: Juergen Gross Acked-by: David Vrabel diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index edbc7a6..cb71016 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c @@ -554,7 +554,7 @@ static bool alloc_p2m(unsigned long pfn) mid_mfn = NULL; } - p2m_pfn = pte_pfn(ACCESS_ONCE(*ptep)); + p2m_pfn = pte_pfn(READ_ONCE(*ptep)); if (p2m_pfn == PFN_DOWN(__pa(p2m_identity)) || p2m_pfn == PFN_DOWN(__pa(p2m_missing))) { /* p2m leaf page is missing */ -- cgit v0.10.2 From bccec2a0a25206cb837e939adab94768a990ffa9 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 6 Jan 2015 22:49:54 +0100 Subject: x86/spinlock: Leftover conversion ACCESS_ONCE->READ_ONCE commit 78bff1c8684f ("x86/ticketlock: Fix spin_unlock_wait() livelock") introduced two additional ACCESS_ONCE cases in x86 spinlock.h. Lets change those as well. Signed-off-by: Christian Borntraeger Cc: Oleg Nesterov diff --git a/arch/x86/include/asm/spinlock.h b/arch/x86/include/asm/spinlock.h index 625660f..7050d86 100644 --- a/arch/x86/include/asm/spinlock.h +++ b/arch/x86/include/asm/spinlock.h @@ -183,10 +183,10 @@ static __always_inline void arch_spin_lock_flags(arch_spinlock_t *lock, static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) { - __ticket_t head = ACCESS_ONCE(lock->tickets.head); + __ticket_t head = READ_ONCE(lock->tickets.head); for (;;) { - struct __raw_tickets tmp = ACCESS_ONCE(lock->tickets); + struct __raw_tickets tmp = READ_ONCE(lock->tickets); /* * We need to check "unlocked" in a loop, tmp.head == head * can be false positive because of overflow. -- cgit v0.10.2 From 38c5ce936a0862a6ce2c8d1c72689a3aba301425 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 6 Jan 2015 22:54:46 +0100 Subject: mm/gup: Replace ACCESS_ONCE with READ_ONCE ACCESS_ONCE does not work reliably on non-scalar types. For example gcc 4.6 and 4.7 might remove the volatile tag for such accesses during the SRA (scalar replacement of aggregates) step (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58145) Fixup gup_pmd_range. Signed-off-by: Christian Borntraeger diff --git a/mm/gup.c b/mm/gup.c index a900759..bed30efa 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -926,7 +926,7 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end, pmdp = pmd_offset(&pud, addr); do { - pmd_t pmd = ACCESS_ONCE(*pmdp); + pmd_t pmd = READ_ONCE(*pmdp); next = pmd_addr_end(addr, end); if (pmd_none(pmd) || pmd_trans_splitting(pmd)) -- cgit v0.10.2 From 927609d622a3773995f84bc03b4564f873cf0e22 Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Tue, 25 Nov 2014 10:16:39 +0100 Subject: kernel: tighten rules for ACCESS ONCE Now that all non-scalar users of ACCESS_ONCE have been converted to READ_ONCE or ASSIGN once, lets tighten ACCESS_ONCE to only work on scalar types. This variant was proposed by Alexei Starovoitov. Signed-off-by: Christian Borntraeger Reviewed-by: Paul E. McKenney diff --git a/include/linux/compiler.h b/include/linux/compiler.h index a1c81f8..5e186bf 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -447,12 +447,23 @@ static __always_inline void __assign_once_size(volatile void *p, void *res, int * to make the compiler aware of ordering is to put the two invocations of * ACCESS_ONCE() in different C statements. * - * This macro does absolutely -nothing- to prevent the CPU from reordering, - * merging, or refetching absolutely anything at any time. Its main intended - * use is to mediate communication between process-level code and irq/NMI - * handlers, all running on the same CPU. + * ACCESS_ONCE will only work on scalar types. For union types, ACCESS_ONCE + * on a union member will work as long as the size of the member matches the + * size of the union and the size is smaller than word size. + * + * The major use cases of ACCESS_ONCE used to be (1) Mediating communication + * between process-level code and irq/NMI handlers, all running on the same CPU, + * and (2) Ensuring that the compiler does not fold, spindle, or otherwise + * mutilate accesses that either do not require ordering or that interact + * with an explicit memory barrier or atomic instruction that provides the + * required ordering. + * + * If possible use READ_ONCE/ASSIGN_ONCE instead. */ -#define ACCESS_ONCE(x) (*(volatile typeof(x) *)&(x)) +#define __ACCESS_ONCE(x) ({ \ + __maybe_unused typeof(x) __var = 0; \ + (volatile typeof(x) *)&(x); }) +#define ACCESS_ONCE(x) (*__ACCESS_ONCE(x)) /* Ignore/forbid kprobes attach on very low level functions marked by this attribute: */ #ifdef CONFIG_KPROBES -- cgit v0.10.2 From 378af02b1aecabb3756e19c0cbb8cdd9c3b9637f Mon Sep 17 00:00:00 2001 From: Guenter Roeck Date: Wed, 7 Jan 2015 12:32:28 -0800 Subject: next: sh: Fix compile error Commit 927609d622a3 ("kernel: tighten rules for ACCESS ONCE") results in a compile failure for sh builds with CONFIG_X2TLB enabled. arch/sh/mm/gup.c: In function 'gup_get_pte': arch/sh/mm/gup.c:20:2: error: invalid initializer make[1]: *** [arch/sh/mm/gup.o] Error 1 Replace ACCESS_ONCE with READ_ONCE to fix the problem. Fixes: 927609d622a3 ("kernel: tighten rules for ACCESS ONCE") Cc: Paul E. McKenney Cc: Christian Borntraeger Signed-off-by: Guenter Roeck Reviewed-by: Paul E. McKenney Signed-off-by: Christian Borntraeger diff --git a/arch/sh/mm/gup.c b/arch/sh/mm/gup.c index 37458f3..e113bb4 100644 --- a/arch/sh/mm/gup.c +++ b/arch/sh/mm/gup.c @@ -17,7 +17,7 @@ static inline pte_t gup_get_pte(pte_t *ptep) { #ifndef CONFIG_X2TLB - return ACCESS_ONCE(*ptep); + return READ_ONCE(*ptep); #else /* * With get_user_pages_fast, we walk down the pagetables without -- cgit v0.10.2 From c5b19946eb76c67566aae6a84bf2b10ad59295ea Mon Sep 17 00:00:00 2001 From: Christian Borntraeger Date: Mon, 12 Jan 2015 12:13:39 +0100 Subject: kernel: Fix sparse warning for ACCESS_ONCE Commit 927609d622a3 ("kernel: tighten rules for ACCESS ONCE") results in sparse warnings like "Using plain integer as NULL pointer" - Let's add a type cast to the dummy assignment. To avoid warnings lik "sparse: warning: cast to restricted __hc32" we also use __force on that cast. Fixes: 927609d622a3 ("kernel: tighten rules for ACCESS ONCE") Signed-off-by: Christian Borntraeger diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 5e186bf..7bebf05 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -461,7 +461,7 @@ static __always_inline void __assign_once_size(volatile void *p, void *res, int * If possible use READ_ONCE/ASSIGN_ONCE instead. */ #define __ACCESS_ONCE(x) ({ \ - __maybe_unused typeof(x) __var = 0; \ + __maybe_unused typeof(x) __var = (__force typeof(x)) 0; \ (volatile typeof(x) *)&(x); }) #define ACCESS_ONCE(x) (*__ACCESS_ONCE(x)) -- cgit v0.10.2