From 9009906f829477f42291c19b5a0f6f50c2be41a6 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 11 Aug 2011 15:31:31 +0200 Subject: mm: pagefault_disabled() Wrap the test for pagefault_disabled() into a helper, this allows us to remove the need for current->pagefault_disabled on !-rt kernels. Signed-off-by: Peter Zijlstra Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 4d6b120..ee01270 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -107,7 +107,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, /* If we're in an interrupt context, or have no user context, we must not take the fault. */ - if (!mm || in_atomic() || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_ALPHA_LARGE_VMALLOC diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 9e92a6c..b40d4ba 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -277,7 +277,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(regs)) diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c index 541d108..25920d2 100644 --- a/arch/avr32/mm/fault.c +++ b/arch/avr32/mm/fault.c @@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs) * If we're in an interrupt or have no user context, we must * not take the fault... */ - if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) || - current->pagefault_disabled) + if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled()) goto no_context; local_irq_enable(); diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c index 69fe7c7..281e859 100644 --- a/arch/cris/mm/fault.c +++ b/arch/cris/mm/fault.c @@ -113,7 +113,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs, * user context, we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(regs)) diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c index de6a23e..8d9fc16 100644 --- a/arch/frv/mm/fault.c +++ b/arch/frv/mm/fault.c @@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(__frame)) diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index aae5478..164db10 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -96,7 +96,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re /* * If we're in an interrupt or have no user context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_VIRTUAL_MEM_MAP diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c index d351a65..ccb6797 100644 --- a/arch/m32r/mm/fault.c +++ b/arch/m32r/mm/fault.c @@ -114,7 +114,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, * If we're in an interrupt or have no user context or are running in an * atomic region then we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; if (error_code & ACE_USERMODE) diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index 55607e3..76bee37 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -85,7 +85,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(regs)) diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index 21d96de..13d6b07 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -107,7 +107,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { if (kernel_mode(regs)) goto bad_area_nosemaphore; diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 5494999..6a492c0 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -89,7 +89,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; if (user_mode(regs)) diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c index 4886c2d..8bd4425 100644 --- a/arch/mn10300/mm/fault.c +++ b/arch/mn10300/mm/fault.c @@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index 68b56fe..f16f57b 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -177,7 +177,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, int fault; unsigned int flags; - if (in_atomic() || current->pagefault_disabled) + if (pagefault_disabled()) goto no_context; tsk = current; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 41230cd..45aa26e 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -261,7 +261,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, if (!arch_irq_disabled_regs(regs)) local_irq_enable(); - if (in_atomic() || mm == NULL || current->pagefault_disabled) { + if (in_atomic() || mm == NULL || pagefault_disabled()) { if (!user_mode(regs)) { rc = SIGSEGV; goto bail; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index a4742e9..6b0efce 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -296,8 +296,8 @@ static inline int do_exception(struct pt_regs *regs, int access) * user context. */ fault = VM_FAULT_BADCONTEXT; - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - tsk->pagefault_disabled)) + if (unlikely(!user_space_fault(trans_exc_code) || + !mm || pagefault_disabled())) goto out; address = trans_exc_code & __FAIL_ADDR_MASK; @@ -443,8 +443,8 @@ void __kprobes do_asce_exception(struct pt_regs *regs) clear_tsk_thread_flag(current, TIF_PER_TRAP); trans_exc_code = regs->int_parm_long; - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - current->pagefault_disabled())) + if (unlikely(!user_space_fault(trans_exc_code) || !mm || + pagefault_disabled())) goto no_context; down_read(&mm->mmap_sem); diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c index 98384f7..35d339b 100644 --- a/arch/score/mm/fault.c +++ b/arch/score/mm/fault.c @@ -73,7 +73,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; if (user_mode(regs)) diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index a2aa4ff..6589138 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -438,7 +438,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 7d51c05..2eaca28 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -199,7 +199,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index c83c76d..1de7683 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -330,7 +330,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_enabled) + if (!mm || pagefault_disabled()) goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c index d28fe5a..40f30ac 100644 --- a/arch/tile/mm/fault.c +++ b/arch/tile/mm/fault.c @@ -357,7 +357,7 @@ static int handle_page_fault(struct pt_regs *regs, * If we're in an interrupt, have no user context or are running in an * atomic region then we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index 9e9ad49..100a278 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -38,7 +38,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, * If the fault was during atomic operation, don't take the fault, just * fail. */ - if (in_atomic() || current->pagefault_disabled) + if (pagefault_disabled()) goto out_nosemaphore; if (is_user) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index f250a04..b5c8e37 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1098,7 +1098,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code) * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 904c479..ff5c9c7 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs) /* If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { bad_page_fault(regs, address, SIGSEGV); return; } diff --git a/include/linux/sched.h b/include/linux/sched.h index d9b8835..f2532f9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -52,6 +52,7 @@ struct sched_param { #include #include #include +#include #include @@ -1258,7 +1259,9 @@ struct task_struct { /* mutex deadlock detection */ struct mutex_waiter *blocked_on; #endif +#ifdef CONFIG_PREEMPT_RT_FULL int pagefault_disabled; +#endif #ifdef CONFIG_TRACE_IRQFLAGS unsigned int irq_events; unsigned long hardirq_enable_ip; @@ -1444,6 +1447,17 @@ static inline void set_numabalancing_state(bool enabled) } #endif +#ifdef CONFIG_PREEMPT_RT_FULL +static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; } +#else +static inline bool cur_pf_disabled(void) { return false; } +#endif + +static inline bool pagefault_disabled(void) +{ + return in_atomic() || cur_pf_disabled(); +} + static inline struct pid *task_pid(struct task_struct *task) { return task->pids[PIDTYPE_PID].pid; diff --git a/kernel/fork.c b/kernel/fork.c index 4d376e8..599033b 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1307,7 +1307,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->hardirq_context = 0; p->softirq_context = 0; #endif +#ifdef CONFIG_PREEMPT_RT_FULL p->pagefault_disabled = 0; +#endif #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0; -- cgit v0.10.2