From e0acd0bd0594161be44c054bb6b984972f444beb Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:30:36 +0300 Subject: asm-generic: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-1-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index c184aa8..dc1269c 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -163,7 +163,7 @@ static inline __must_check long __copy_to_user(void __user *to, #define put_user(x, ptr) \ ({ \ - might_sleep(); \ + might_fault(); \ access_ok(VERIFY_WRITE, ptr, sizeof(*ptr)) ? \ __put_user(x, ptr) : \ -EFAULT; \ @@ -225,7 +225,7 @@ extern int __put_user_bad(void) __attribute__((noreturn)); #define get_user(x, ptr) \ ({ \ - might_sleep(); \ + might_fault(); \ access_ok(VERIFY_READ, ptr, sizeof(*ptr)) ? \ __get_user(x, ptr) : \ -EFAULT; \ @@ -255,7 +255,7 @@ extern int __get_user_bad(void) __attribute__((noreturn)); static inline long copy_from_user(void *to, const void __user * from, unsigned long n) { - might_sleep(); + might_fault(); if (access_ok(VERIFY_READ, from, n)) return __copy_from_user(to, from, n); else @@ -265,7 +265,7 @@ static inline long copy_from_user(void *to, static inline long copy_to_user(void __user *to, const void *from, unsigned long n) { - might_sleep(); + might_fault(); if (access_ok(VERIFY_WRITE, to, n)) return __copy_to_user(to, from, n); else @@ -336,7 +336,7 @@ __clear_user(void __user *to, unsigned long n) static inline __must_check unsigned long clear_user(void __user *to, unsigned long n) { - might_sleep(); + might_fault(); if (!access_ok(VERIFY_WRITE, to, n)) return n; -- cgit v0.10.2 From 56d2ef789f7c424918abdf6b95d84a64c1473220 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:30:42 +0300 Subject: arm64: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Acked-by: Catalin Marinas Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-2-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 008f848..edb3d5c 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -166,7 +166,7 @@ do { \ #define get_user(x, ptr) \ ({ \ - might_sleep(); \ + might_fault(); \ access_ok(VERIFY_READ, (ptr), sizeof(*(ptr))) ? \ __get_user((x), (ptr)) : \ ((x) = 0, -EFAULT); \ @@ -227,7 +227,7 @@ do { \ #define put_user(x, ptr) \ ({ \ - might_sleep(); \ + might_fault(); \ access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \ __put_user((x), (ptr)) : \ -EFAULT; \ -- cgit v0.10.2 From b607ae78ac8a78f8e5e36817500e7c311519f032 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:30:47 +0300 Subject: frv: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-3-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/frv/include/asm/uaccess.h b/arch/frv/include/asm/uaccess.h index 0b67ec5..3ac9a59 100644 --- a/arch/frv/include/asm/uaccess.h +++ b/arch/frv/include/asm/uaccess.h @@ -280,14 +280,14 @@ extern long __memcpy_user(void *dst, const void *src, unsigned long count); static inline unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n) { - might_sleep(); + might_fault(); return __copy_to_user_inatomic(to, from, n); } static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) { - might_sleep(); + might_fault(); return __copy_from_user_inatomic(to, from, n); } -- cgit v0.10.2 From 01682576d5fd1c92b96d79560b17208a6567c331 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:30:51 +0300 Subject: m32r: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-4-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/m32r/include/asm/uaccess.h b/arch/m32r/include/asm/uaccess.h index 1c7047b..84fe7ba 100644 --- a/arch/m32r/include/asm/uaccess.h +++ b/arch/m32r/include/asm/uaccess.h @@ -216,7 +216,7 @@ extern int fixup_exception(struct pt_regs *regs); ({ \ long __gu_err = 0; \ unsigned long __gu_val; \ - might_sleep(); \ + might_fault(); \ __get_user_size(__gu_val,(ptr),(size),__gu_err); \ (x) = (__typeof__(*(ptr)))__gu_val; \ __gu_err; \ @@ -227,7 +227,7 @@ extern int fixup_exception(struct pt_regs *regs); long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - might_sleep(); \ + might_fault(); \ if (access_ok(VERIFY_READ,__gu_addr,size)) \ __get_user_size(__gu_val,__gu_addr,(size),__gu_err); \ (x) = (__typeof__(*(ptr)))__gu_val; \ @@ -295,7 +295,7 @@ do { \ #define __put_user_nocheck(x,ptr,size) \ ({ \ long __pu_err; \ - might_sleep(); \ + might_fault(); \ __put_user_size((x),(ptr),(size),__pu_err); \ __pu_err; \ }) @@ -305,7 +305,7 @@ do { \ ({ \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - might_sleep(); \ + might_fault(); \ if (access_ok(VERIFY_WRITE,__pu_addr,size)) \ __put_user_size((x),__pu_addr,(size),__pu_err); \ __pu_err; \ @@ -597,7 +597,7 @@ unsigned long __generic_copy_from_user(void *, const void __user *, unsigned lon */ #define copy_to_user(to,from,n) \ ({ \ - might_sleep(); \ + might_fault(); \ __generic_copy_to_user((to),(from),(n)); \ }) @@ -638,7 +638,7 @@ unsigned long __generic_copy_from_user(void *, const void __user *, unsigned lon */ #define copy_from_user(to,from,n) \ ({ \ - might_sleep(); \ + might_fault(); \ __generic_copy_from_user((to),(from),(n)); \ }) -- cgit v0.10.2 From ac093f8d5e76be1f2654acfd7a59d339ba037654 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:30:56 +0300 Subject: microblaze: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-5-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index efe59d8..2fc8bf7 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -145,7 +145,7 @@ static inline unsigned long __must_check __clear_user(void __user *to, static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { - might_sleep(); + might_fault(); if (unlikely(!access_ok(VERIFY_WRITE, to, n))) return n; @@ -371,7 +371,7 @@ extern long __user_bad(void); static inline long copy_from_user(void *to, const void __user *from, unsigned long n) { - might_sleep(); + might_fault(); if (access_ok(VERIFY_READ, from, n)) return __copy_from_user(to, from, n); return n; @@ -385,7 +385,7 @@ static inline long copy_from_user(void *to, static inline long copy_to_user(void __user *to, const void *from, unsigned long n) { - might_sleep(); + might_fault(); if (access_ok(VERIFY_WRITE, to, n)) return __copy_to_user(to, from, n); return n; -- cgit v0.10.2 From 3837a3cfe4a27836e0e9f207eb2d4f00b5a8fcba Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:31:05 +0300 Subject: mn10300: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-6-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/mn10300/include/asm/uaccess.h b/arch/mn10300/include/asm/uaccess.h index 780560b..107508a 100644 --- a/arch/mn10300/include/asm/uaccess.h +++ b/arch/mn10300/include/asm/uaccess.h @@ -471,13 +471,13 @@ extern unsigned long __generic_copy_from_user(void *, const void __user *, #define __copy_to_user(to, from, n) \ ({ \ - might_sleep(); \ + might_fault(); \ __copy_to_user_inatomic((to), (from), (n)); \ }) #define __copy_from_user(to, from, n) \ ({ \ - might_sleep(); \ + might_fault(); \ __copy_from_user_inatomic((to), (from), (n)); \ }) -- cgit v0.10.2 From 1af1717dbf96eba8a74a2d6a99e75a7795075a02 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:31:38 +0300 Subject: powerpc: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Arnd Bergmann suggested that the following code if (!is_kernel_addr((unsigned long)__pu_addr)) might_fault(); can be further simplified by adding a version of might_fault that includes the kernel addr check. Will be considered as a further optimization in future. Signed-off-by: Michael S. Tsirkin Acked-by: Benjamin Herrenschmidt Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-7-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 4db4959..9485b43 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -178,7 +178,7 @@ do { \ long __pu_err; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ if (!is_kernel_addr((unsigned long)__pu_addr)) \ - might_sleep(); \ + might_fault(); \ __chk_user_ptr(ptr); \ __put_user_size((x), __pu_addr, (size), __pu_err); \ __pu_err; \ @@ -188,7 +188,7 @@ do { \ ({ \ long __pu_err = -EFAULT; \ __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - might_sleep(); \ + might_fault(); \ if (access_ok(VERIFY_WRITE, __pu_addr, size)) \ __put_user_size((x), __pu_addr, (size), __pu_err); \ __pu_err; \ @@ -268,7 +268,7 @@ do { \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ __chk_user_ptr(ptr); \ if (!is_kernel_addr((unsigned long)__gu_addr)) \ - might_sleep(); \ + might_fault(); \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__typeof__(*(ptr)))__gu_val; \ __gu_err; \ @@ -282,7 +282,7 @@ do { \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ __chk_user_ptr(ptr); \ if (!is_kernel_addr((unsigned long)__gu_addr)) \ - might_sleep(); \ + might_fault(); \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__typeof__(*(ptr)))__gu_val; \ __gu_err; \ @@ -294,7 +294,7 @@ do { \ long __gu_err = -EFAULT; \ unsigned long __gu_val = 0; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ - might_sleep(); \ + might_fault(); \ if (access_ok(VERIFY_READ, __gu_addr, (size))) \ __get_user_size(__gu_val, __gu_addr, (size), __gu_err); \ (x) = (__typeof__(*(ptr)))__gu_val; \ @@ -419,14 +419,14 @@ static inline unsigned long __copy_to_user_inatomic(void __user *to, static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long size) { - might_sleep(); + might_fault(); return __copy_from_user_inatomic(to, from, size); } static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long size) { - might_sleep(); + might_fault(); return __copy_to_user_inatomic(to, from, size); } @@ -434,7 +434,7 @@ extern unsigned long __clear_user(void __user *addr, unsigned long size); static inline unsigned long clear_user(void __user *addr, unsigned long size) { - might_sleep(); + might_fault(); if (likely(access_ok(VERIFY_WRITE, addr, size))) return __clear_user(addr, size); if ((unsigned long)addr < TASK_SIZE) { -- cgit v0.10.2 From f8abe86cc4fbd4ba083fd151b88e02fb3ce88b9c Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:31:48 +0300 Subject: tile: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Acked-by: Chris Metcalf Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-8-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/tile/include/asm/uaccess.h b/arch/tile/include/asm/uaccess.h index 8a082bc..e4d44bd 100644 --- a/arch/tile/include/asm/uaccess.h +++ b/arch/tile/include/asm/uaccess.h @@ -442,7 +442,7 @@ extern unsigned long __copy_in_user_inatomic( static inline unsigned long __must_check __copy_in_user(void __user *to, const void __user *from, unsigned long n) { - might_sleep(); + might_fault(); return __copy_in_user_inatomic(to, from, n); } -- cgit v0.10.2 From 016be2e55d98aee0b97b94b200d6e0e110c8392a Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:31:55 +0300 Subject: x86: uaccess s/might_sleep/might_fault/ The only reason uaccess routines might sleep is if they fault. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-9-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 142810c..4f7923d 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -235,7 +235,7 @@ extern long __copy_user_nocache(void *dst, const void __user *src, static inline int __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) { - might_sleep(); + might_fault(); return __copy_user_nocache(dst, src, size, 1); } -- cgit v0.10.2 From 114276ac0a3beb9c391a410349bd770653e185ce Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:32:13 +0300 Subject: mm, sched: Drop voluntary schedule from might_fault() might_fault() is called from functions like copy_to_user() which most callers expect to be very fast, like a couple of instructions. So functions like memcpy_toiovec() call them many times in a loop. But might_fault() calls might_sleep() and with CONFIG_PREEMPT_VOLUNTARY this results in a function call. Let's not do this - just call __might_sleep() that produces a diagnostic for sleep within atomic, but drop might_preempt(). Here's a test sending traffic between the VM and the host, host is built with CONFIG_PREEMPT_VOLUNTARY: before: incoming: 7122.77 Mb/s outgoing: 8480.37 Mb/s after: incoming: 8619.24 Mb/s outgoing: 9455.42 Mb/s As a side effect, this fixes an issue pointed out by Ingo: might_fault might schedule differently depending on PROVE_LOCKING. Now there's no preemption point in both cases, so it's consistent. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-10-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e9ef6d6..24719ea 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -198,7 +198,7 @@ void might_fault(void); #else static inline void might_fault(void) { - might_sleep(); + __might_sleep(__FILE__, __LINE__, 0); } #endif diff --git a/mm/memory.c b/mm/memory.c index 6dc1882..c1f190f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4222,7 +4222,8 @@ void might_fault(void) if (segment_eq(get_fs(), KERNEL_DS)) return; - might_sleep(); + __might_sleep(__FILE__, __LINE__, 0); + /* * it would be nicer only to annotate paths which are not under * pagefault_disable, however that requires a larger audit and -- cgit v0.10.2 From 662bbcb2747c2422cf98d3d97619509379eee466 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Sun, 26 May 2013 17:32:23 +0300 Subject: mm, sched: Allow uaccess in atomic with pagefault_disable() This changes might_fault() so that it does not trigger a false positive diagnostic for e.g. the following sequence: spin_lock_irqsave() pagefault_disable() copy_to_user() pagefault_enable() spin_unlock_irqrestore() In particular vhost wants to do this, to call socket ops from under a lock. There are 3 cases to consider: - CONFIG_PROVE_LOCKING - might_fault is non-inline so it's easy to move the in_atomic test to fix up the false positive warning. - CONFIG_DEBUG_ATOMIC_SLEEP - might_fault is currently inline, but we are calling a non-inline __might_sleep anyway, so let's use the non-line version of might_fault that does the right thing. - !CONFIG_DEBUG_ATOMIC_SLEEP && !CONFIG_PROVE_LOCKING __might_sleep is a nop so might_fault is a nop. Make this explicit. Signed-off-by: Michael S. Tsirkin Signed-off-by: Peter Zijlstra Cc: Linus Torvalds Cc: Andrew Morton Cc: Peter Zijlstra Link: http://lkml.kernel.org/r/1369577426-26721-11-git-send-email-mst@redhat.com Signed-off-by: Ingo Molnar diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 24719ea..4c7e2e5 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -193,13 +193,10 @@ extern int _cond_resched(void); (__x < 0) ? -__x : __x; \ }) -#ifdef CONFIG_PROVE_LOCKING +#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) void might_fault(void); #else -static inline void might_fault(void) -{ - __might_sleep(__FILE__, __LINE__, 0); -} +static inline void might_fault(void) { } #endif extern struct atomic_notifier_head panic_notifier_list; diff --git a/mm/memory.c b/mm/memory.c index c1f190f..d7d54a1 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4210,7 +4210,7 @@ void print_vma_addr(char *prefix, unsigned long ip) up_read(&mm->mmap_sem); } -#ifdef CONFIG_PROVE_LOCKING +#if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) void might_fault(void) { /* @@ -4222,14 +4222,17 @@ void might_fault(void) if (segment_eq(get_fs(), KERNEL_DS)) return; - __might_sleep(__FILE__, __LINE__, 0); - /* * it would be nicer only to annotate paths which are not under * pagefault_disable, however that requires a larger audit and * providing helpers like get_user_atomic. */ - if (!in_atomic() && current->mm) + if (in_atomic()) + return; + + __might_sleep(__FILE__, __LINE__, 0); + + if (current->mm) might_lock_read(¤t->mm->mmap_sem); } EXPORT_SYMBOL(might_fault); -- cgit v0.10.2