From 0eafa46823971b4c368f4cdf19f1d081c4ee52b7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 28 Aug 2014 15:00:30 -0700 Subject: rcu: Remove CONFIG_RCU_CPU_STALL_VERBOSE The CONFIG_RCU_CPU_STALL_VERBOSE Kconfig parameter causes preemptible RCU's CPU stall warnings to dump out any preempted tasks that are blocking the current RCU grace period. This information is useful, and the default has been CONFIG_RCU_CPU_STALL_VERBOSE=y for some years. It is therefore time for this commit to remove this Kconfig parameter, so that future kernel builds will always act as if CONFIG_RCU_CPU_STALL_VERBOSE=y. Signed-off-by: Paul E. McKenney diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt index ef5a2fd..1560efb 100644 --- a/Documentation/RCU/stallwarn.txt +++ b/Documentation/RCU/stallwarn.txt @@ -26,12 +26,6 @@ CONFIG_RCU_CPU_STALL_TIMEOUT Stall-warning messages may be enabled and disabled completely via /sys/module/rcupdate/parameters/rcu_cpu_stall_suppress. -CONFIG_RCU_CPU_STALL_VERBOSE - - This kernel configuration parameter causes the stall warning to - also dump the stacks of any tasks that are blocking the current - RCU-preempt grace period. - CONFIG_RCU_CPU_STALL_INFO This kernel configuration parameter causes the stall warning to diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c1d7f27..d062f4d 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -72,9 +72,6 @@ static void __init rcu_bootup_announce_oddness(void) #ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE pr_info("\tRCU torture testing starts during boot.\n"); #endif -#if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE) - pr_info("\tDump stacks of tasks blocking RCU-preempt GP.\n"); -#endif #if defined(CONFIG_RCU_CPU_STALL_INFO) pr_info("\tAdditional per-CPU info printed with stalls.\n"); #endif @@ -415,8 +412,6 @@ void rcu_read_unlock_special(struct task_struct *t) } } -#ifdef CONFIG_RCU_CPU_STALL_VERBOSE - /* * Dump detailed information for all tasks blocking the current RCU * grace period on the specified rcu_node structure. @@ -451,14 +446,6 @@ static void rcu_print_detail_task_stall(struct rcu_state *rsp) rcu_print_detail_task_stall_rnp(rnp); } -#else /* #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */ - -static void rcu_print_detail_task_stall(struct rcu_state *rsp) -{ -} - -#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */ - #ifdef CONFIG_RCU_CPU_STALL_INFO static void rcu_print_task_stall_begin(struct rcu_node *rnp) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 4e35a5d..04e54cd 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1238,18 +1238,6 @@ config RCU_CPU_STALL_TIMEOUT RCU grace period persists, additional CPU stall warnings are printed at more widely spaced intervals. -config RCU_CPU_STALL_VERBOSE - bool "Print additional per-task information for RCU_CPU_STALL_DETECTOR" - depends on TREE_PREEMPT_RCU - default y - help - This option causes RCU to printk detailed per-task information - for any tasks that are stalling the current RCU grace period. - - Say N if you are unsure. - - Say Y if you want to enable such checks. - config RCU_CPU_STALL_INFO bool "Print additional diagnostics on RCU CPU stall" depends on (TREE_RCU || TREE_PREEMPT_RCU) && DEBUG_KERNEL diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 index 38e3895..a04420e 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 @@ -14,6 +14,5 @@ CONFIG_RCU_NOCB_CPU=y CONFIG_RCU_NOCB_CPU_ZERO=y CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE02 b/tools/testing/selftests/rcutorture/configs/rcu/TREE02 index ea119ba..a4f8bfb 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE02 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE02 @@ -19,6 +19,5 @@ CONFIG_RCU_NOCB_CPU=n CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=y CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T b/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T index 19cf948..fe19aaf 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T @@ -19,6 +19,5 @@ CONFIG_RCU_NOCB_CPU=n CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=y CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE03 b/tools/testing/selftests/rcutorture/configs/rcu/TREE03 index f4567fb..a2a9a9b 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE03 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE03 @@ -15,7 +15,6 @@ CONFIG_RCU_FANOUT_EXACT=n CONFIG_RCU_NOCB_CPU=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_RCU_BOOST=y CONFIG_RCU_BOOST_PRIO=2 CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE04 b/tools/testing/selftests/rcutorture/configs/rcu/TREE04 index 0a262fb..0f84db3 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE04 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE04 @@ -19,5 +19,4 @@ CONFIG_RCU_FANOUT_EXACT=n CONFIG_RCU_NOCB_CPU=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=y -CONFIG_RCU_CPU_STALL_VERBOSE=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE05 b/tools/testing/selftests/rcutorture/configs/rcu/TREE05 index 3a06b97..212e3bf 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE05 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE05 @@ -19,5 +19,4 @@ CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y CONFIG_PROVE_RCU=y CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE06 b/tools/testing/selftests/rcutorture/configs/rcu/TREE06 index 8f084cc..7eee63b 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE06 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE06 @@ -20,5 +20,4 @@ CONFIG_DEBUG_LOCK_ALLOC=y CONFIG_PROVE_LOCKING=y CONFIG_PROVE_RCU=y CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE07 b/tools/testing/selftests/rcutorture/configs/rcu/TREE07 index 8f10176..92a97fa 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE07 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE07 @@ -19,5 +19,4 @@ CONFIG_RCU_FANOUT_EXACT=n CONFIG_RCU_NOCB_CPU=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=y -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE08 b/tools/testing/selftests/rcutorture/configs/rcu/TREE08 index 69a2e25..316aa6c 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE08 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE08 @@ -19,6 +19,5 @@ CONFIG_RCU_NOCB_CPU=y CONFIG_RCU_NOCB_CPU_ALL=y CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T b/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T index a0f32fb..55daf08 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T @@ -19,6 +19,5 @@ CONFIG_RCU_NOCB_CPU=y CONFIG_RCU_NOCB_CPU_ALL=y CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE09 b/tools/testing/selftests/rcutorture/configs/rcu/TREE09 index b7a62a5..13081a8 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE09 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE09 @@ -14,6 +14,5 @@ CONFIG_HIBERNATION=n CONFIG_RCU_NOCB_CPU=n CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=n -CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_RCU_BOOST=n CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt b/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt index 3e588db..0dcde4c 100644 --- a/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt +++ b/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt @@ -16,8 +16,7 @@ CONFIG_PROVE_LOCKING -- Do all but two, covering CONFIG_PROVE_RCU and not. CONFIG_PROVE_RCU -- Do all but one under CONFIG_PROVE_LOCKING. CONFIG_RCU_BOOST -- one of TREE_PREEMPT_RCU. CONFIG_RCU_BOOST_PRIO -- set to 2 for _BOOST testing. -CONFIG_RCU_CPU_STALL_INFO -- do one with and without _VERBOSE. -CONFIG_RCU_CPU_STALL_VERBOSE -- do one with and without _INFO. +CONFIG_RCU_CPU_STALL_INFO -- Do one. CONFIG_RCU_FANOUT -- Cover hierarchy as currently, but overlap with others. CONFIG_RCU_FANOUT_EXACT -- Do one. CONFIG_RCU_FANOUT_LEAF -- Do one non-default. -- cgit v0.10.2 From 4568779f7143f88fc5fbe7a8d619653e28f24ad1 Mon Sep 17 00:00:00 2001 From: Stefan Hengelein Date: Tue, 2 Sep 2014 19:55:11 +0200 Subject: init/Kconfig: move RCU_NOCB_CPU dependencies to choice Every choice item of the "Build-forced no-CBs CPUs" choice had a dependency to RCU_NOCB_CPU. It's more comprehensible if the choice itself has the dependency instead of every choice item. The choice itself doesn't need to be visible if there are no items selectable (i.e. on arch/frv) or RCU_NOCB_CPU is not defined. Signed-off-by: Stefan Hengelein Signed-off-by: Andreas Ruprecht Reviewed-by: Josh Triplett Signed-off-by: Andrew Morton Signed-off-by: Paul E. McKenney diff --git a/init/Kconfig b/init/Kconfig index 3ee28ae..0c9546b 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -739,6 +739,7 @@ config RCU_NOCB_CPU choice prompt "Build-forced no-CBs CPUs" default RCU_NOCB_CPU_NONE + depends on RCU_NOCB_CPU help This option allows no-CBs CPUs (whose RCU callbacks are invoked from kthreads rather than from softirq context) to be specified @@ -747,7 +748,6 @@ choice config RCU_NOCB_CPU_NONE bool "No build_forced no-CBs CPUs" - depends on RCU_NOCB_CPU help This option does not force any of the CPUs to be no-CBs CPUs. Only CPUs designated by the rcu_nocbs= boot parameter will be @@ -761,7 +761,6 @@ config RCU_NOCB_CPU_NONE config RCU_NOCB_CPU_ZERO bool "CPU 0 is a build_forced no-CBs CPU" - depends on RCU_NOCB_CPU help This option forces CPU 0 to be a no-CBs CPU, so that its RCU callbacks are invoked by a per-CPU kthread whose name begins @@ -776,7 +775,6 @@ config RCU_NOCB_CPU_ZERO config RCU_NOCB_CPU_ALL bool "All CPUs are build_forced no-CBs CPUs" - depends on RCU_NOCB_CPU help This option forces all CPUs to be no-CBs CPUs. The rcu_nocbs= boot parameter will be ignored. All CPUs' RCU callbacks will -- cgit v0.10.2 From 61cfd0970ea27764434fba5c41bdaefb26c44183 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 2 Sep 2014 12:30:58 -0700 Subject: rcu: Move RCU_BOOST variable declarations, eliminating #ifdef There are some RCU_BOOST-specific per-CPU variable declarations that are needlessly defined under #ifdef in kernel/rcu/tree.c. This commit therefore moves these declarations into a pre-existing #ifdef in kernel/rcu/tree_plugin.h. Signed-off-by: Paul E. McKenney diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9815447..7f73c5e 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -152,19 +152,6 @@ EXPORT_SYMBOL_GPL(rcu_scheduler_active); */ static int rcu_scheduler_fully_active __read_mostly; -#ifdef CONFIG_RCU_BOOST - -/* - * Control variables for per-CPU and per-rcu_node kthreads. These - * handle all flavors of RCU. - */ -static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); -DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); -DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); -DEFINE_PER_CPU(char, rcu_cpu_has_work); - -#endif /* #ifdef CONFIG_RCU_BOOST */ - static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu); static void invoke_rcu_core(void); static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c1d7f27..3578911 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -33,11 +33,24 @@ #define RCU_KTHREAD_PRIO 1 #ifdef CONFIG_RCU_BOOST + #include "../locking/rtmutex_common.h" #define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO -#else + +/* + * Control variables for per-CPU and per-rcu_node kthreads. These + * handle all flavors of RCU. + */ +static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); +DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); +DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); +DEFINE_PER_CPU(char, rcu_cpu_has_work); + +#else /* #ifdef CONFIG_RCU_BOOST */ + #define RCU_BOOST_PRIO RCU_KTHREAD_PRIO -#endif + +#endif /* #else #ifdef CONFIG_RCU_BOOST */ #ifdef CONFIG_RCU_NOCB_CPU static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ -- cgit v0.10.2 From e0775cefb5ede661dbdc0611d7bf3fcd4640005c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 3 Sep 2014 12:25:56 -0700 Subject: rcu: Avoid IPIing idle CPUs from synchronize_sched_expedited() Currently, synchronize_sched_expedited() sends IPIs to all online CPUs, even those that are idle or executing in nohz_full= userspace. Because idle CPUs and nohz_full= userspace CPUs are in extended quiescent states, there is no need to IPI them in the first place. This commit therefore avoids IPIing CPUs that are already in extended quiescent states. Signed-off-by: Paul E. McKenney diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 7f73c5e..9e3c20f 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2950,6 +2950,9 @@ static int synchronize_sched_expedited_cpu_stop(void *data) */ void synchronize_sched_expedited(void) { + cpumask_var_t cm; + bool cma = false; + int cpu; long firstsnap, s, snap; int trycount = 0; struct rcu_state *rsp = &rcu_sched_state; @@ -2984,11 +2987,26 @@ void synchronize_sched_expedited(void) } WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id())); + /* Offline CPUs, idle CPUs, and any CPU we run on are quiescent. */ + cma = zalloc_cpumask_var(&cm, GFP_KERNEL); + if (cma) { + cpumask_copy(cm, cpu_online_mask); + cpumask_clear_cpu(raw_smp_processor_id(), cm); + for_each_cpu(cpu, cm) { + struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); + + if (!(atomic_add_return(0, &rdtp->dynticks) & 0x1)) + cpumask_clear_cpu(cpu, cm); + } + if (cpumask_weight(cm) == 0) + goto all_cpus_idle; + } + /* * Each pass through the following loop attempts to force a * context switch on each CPU. */ - while (try_stop_cpus(cpu_online_mask, + while (try_stop_cpus(cma ? cm : cpu_online_mask, synchronize_sched_expedited_cpu_stop, NULL) == -EAGAIN) { put_online_cpus(); @@ -3000,6 +3018,7 @@ void synchronize_sched_expedited(void) /* ensure test happens before caller kfree */ smp_mb__before_atomic(); /* ^^^ */ atomic_long_inc(&rsp->expedited_workdone1); + free_cpumask_var(cm); return; } @@ -3009,6 +3028,7 @@ void synchronize_sched_expedited(void) } else { wait_rcu_gp(call_rcu_sched); atomic_long_inc(&rsp->expedited_normal); + free_cpumask_var(cm); return; } @@ -3018,6 +3038,7 @@ void synchronize_sched_expedited(void) /* ensure test happens before caller kfree */ smp_mb__before_atomic(); /* ^^^ */ atomic_long_inc(&rsp->expedited_workdone2); + free_cpumask_var(cm); return; } @@ -3032,6 +3053,7 @@ void synchronize_sched_expedited(void) /* CPU hotplug operation in flight, use normal GP. */ wait_rcu_gp(call_rcu_sched); atomic_long_inc(&rsp->expedited_normal); + free_cpumask_var(cm); return; } snap = atomic_long_read(&rsp->expedited_start); @@ -3039,6 +3061,9 @@ void synchronize_sched_expedited(void) } atomic_long_inc(&rsp->expedited_stoppedcpus); +all_cpus_idle: + free_cpumask_var(cm); + /* * Everyone up to our most recent fetch is covered by our grace * period. Update the counter, but only if our work is still -- cgit v0.10.2 From eca1a08986f622c11b75b3b44d561a1f901c9cec Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 23 Oct 2014 11:41:22 -0700 Subject: signal: Exit RCU read-side critical section on each pass through loop The kill_pid_info() can potentially loop indefinitely if tasks are created and deleted sufficiently quickly, and if this happens, this function will remain in a single RCU read-side critical section indefinitely. This commit therefore exits the RCU read-side critical section on each pass through the loop. Because a race must happen to retry the loop, this should have no performance impact in the common case. Reported-by: Dave Jones Signed-off-by: Paul E. McKenney Acked-by: Oleg Nesterov Reviewed-by: Pranith Kumar diff --git a/kernel/signal.c b/kernel/signal.c index 8f0876f..5482098 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1331,23 +1331,21 @@ int kill_pid_info(int sig, struct siginfo *info, struct pid *pid) int error = -ESRCH; struct task_struct *p; - rcu_read_lock(); -retry: - p = pid_task(pid, PIDTYPE_PID); - if (p) { - error = group_send_sig_info(sig, info, p); - if (unlikely(error == -ESRCH)) - /* - * The task was unhashed in between, try again. - * If it is dead, pid_task() will return NULL, - * if we race with de_thread() it will find the - * new leader. - */ - goto retry; - } - rcu_read_unlock(); + for (;;) { + rcu_read_lock(); + p = pid_task(pid, PIDTYPE_PID); + if (p) + error = group_send_sig_info(sig, info, p); + rcu_read_unlock(); + if (likely(!p || error != -ESRCH)) + return error; - return error; + /* + * The task was unhashed in between, try again. If it + * is dead, pid_task() will return NULL, if we race with + * de_thread() it will find the new leader. + */ + } } int kill_proc_info(int sig, struct siginfo *info, pid_t pid) -- cgit v0.10.2 From 392809b25833548ccfc55e61b76c8451a5073216 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sun, 28 Sep 2014 23:44:18 +0200 Subject: signal: Document the RCU protection of ->sighand __cleanup_sighand() frees sighand without RCU grace period. This is correct but this looks "obviously buggy" and constantly confuses the readers, add the comments to explain how this works. Signed-off-by: Oleg Nesterov Reviewed-by: Steven Rostedt Reviewed-by: Rik van Riel Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/fork.c b/kernel/fork.c index 9b7d746..9ca8418 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1022,11 +1022,14 @@ void __cleanup_sighand(struct sighand_struct *sighand) { if (atomic_dec_and_test(&sighand->count)) { signalfd_cleanup(sighand); + /* + * sighand_cachep is SLAB_DESTROY_BY_RCU so we can free it + * without an RCU grace period, see __lock_task_sighand(). + */ kmem_cache_free(sighand_cachep, sighand); } } - /* * Initialize POSIX timer handling for a thread group. */ diff --git a/kernel/signal.c b/kernel/signal.c index 5482098..19e3513 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1275,7 +1275,17 @@ struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, local_irq_restore(*flags); break; } - + /* + * This sighand can be already freed and even reused, but + * we rely on SLAB_DESTROY_BY_RCU and sighand_ctor() which + * initializes ->siglock: this slab can't go away, it has + * the same object type, ->siglock can't be reinitialized. + * + * We need to ensure that tsk->sighand is still the same + * after we take the lock, we can race with de_thread() or + * __exit_signal(). In the latter case the next iteration + * must see ->sighand == NULL. + */ spin_lock(&sighand->siglock); if (likely(sighand == tsk->sighand)) { rcu_read_unlock(); -- cgit v0.10.2 From 21871d7eff2c96ae67e18e00adf59d56940e2fcc Mon Sep 17 00:00:00 2001 From: Clark Williams Date: Fri, 12 Sep 2014 21:21:09 -0500 Subject: rcu: Unify boost and kthread priorities Rename CONFIG_RCU_BOOST_PRIO to CONFIG_RCU_KTHREAD_PRIO and use this value for both the per-CPU kthreads (rcuc/N) and the rcu boosting threads (rcub/n). Also, create the module_parameter rcutree.kthread_prio to be used on the kernel command line at boot to set a new value (rcutree.kthread_prio=N). Signed-off-by: Clark Williams [ paulmck: Ported to rcu/dev, applied Paul Bolle and Peter Zijlstra feedback. ] Signed-off-by: Paul E. McKenney diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 7dbe5ec..6608520 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -2922,6 +2922,13 @@ bytes respectively. Such letter suffixes can also be entirely omitted. quiescent states. Units are jiffies, minimum value is one, and maximum value is HZ. + rcutree.kthread_prio= [KNL,BOOT] + Set the SCHED_FIFO priority of the RCU + per-CPU kthreads (rcuc/N). This value is also + used for the priority of the RCU boost threads + (rcub/N). Valid values are 1-99 and the default + is 1 (the least-favored priority). + rcutree.rcu_nocb_leader_stride= [KNL] Set the number of NOCB kthread groups, which defaults to the square root of the number of diff --git a/init/Kconfig b/init/Kconfig index 0c9546b..15c299c 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -672,30 +672,31 @@ config RCU_BOOST Say Y here if you are working with real-time apps or heavy loads Say N here if you are unsure. -config RCU_BOOST_PRIO - int "Real-time priority to boost RCU readers to" +config RCU_KTHREAD_PRIO + int "Real-time priority to use for RCU worker threads" range 1 99 depends on RCU_BOOST default 1 help - This option specifies the real-time priority to which long-term - preempted RCU readers are to be boosted. If you are working - with a real-time application that has one or more CPU-bound - threads running at a real-time priority level, you should set - RCU_BOOST_PRIO to a priority higher then the highest-priority - real-time CPU-bound thread. The default RCU_BOOST_PRIO value - of 1 is appropriate in the common case, which is real-time + This option specifies the SCHED_FIFO priority value that will be + assigned to the rcuc/n and rcub/n threads and is also the value + used for RCU_BOOST (if enabled). If you are working with a + real-time application that has one or more CPU-bound threads + running at a real-time priority level, you should set + RCU_KTHREAD_PRIO to a priority higher than the highest-priority + real-time CPU-bound application thread. The default RCU_KTHREAD_PRIO + value of 1 is appropriate in the common case, which is real-time applications that do not have any CPU-bound threads. Some real-time applications might not have a single real-time thread that saturates a given CPU, but instead might have multiple real-time threads that, taken together, fully utilize - that CPU. In this case, you should set RCU_BOOST_PRIO to + that CPU. In this case, you should set RCU_KTHREAD_PRIO to a priority higher than the lowest-priority thread that is conspiring to prevent the CPU from running any non-real-time tasks. For example, if one thread at priority 10 and another thread at priority 5 are between themselves fully consuming - the CPU time on a given CPU, then RCU_BOOST_PRIO should be + the CPU time on a given CPU, then RCU_KTHREAD_PRIO should be set to priority 6 or higher. Specify the real-time priority, or take the default if unsure. diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 3578911..344f0e6 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -30,12 +30,13 @@ #include #include "../time/tick-internal.h" -#define RCU_KTHREAD_PRIO 1 - #ifdef CONFIG_RCU_BOOST #include "../locking/rtmutex_common.h" -#define RCU_BOOST_PRIO CONFIG_RCU_BOOST_PRIO + +/* rcuc/rcub kthread realtime priority */ +static int kthread_prio = CONFIG_RCU_KTHREAD_PRIO; +module_param(kthread_prio, int, 0644); /* * Control variables for per-CPU and per-rcu_node kthreads. These @@ -46,11 +47,7 @@ DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); DEFINE_PER_CPU(char, rcu_cpu_has_work); -#else /* #ifdef CONFIG_RCU_BOOST */ - -#define RCU_BOOST_PRIO RCU_KTHREAD_PRIO - -#endif /* #else #ifdef CONFIG_RCU_BOOST */ +#endif /* #ifdef CONFIG_RCU_BOOST */ #ifdef CONFIG_RCU_NOCB_CPU static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ @@ -98,6 +95,9 @@ static void __init rcu_bootup_announce_oddness(void) pr_info("\tBoot-time adjustment of leaf fanout to %d.\n", rcu_fanout_leaf); if (nr_cpu_ids != NR_CPUS) pr_info("\tRCU restricting CPUs from NR_CPUS=%d to nr_cpu_ids=%d.\n", NR_CPUS, nr_cpu_ids); +#ifdef CONFIG_RCU_BOOST + pr_info("\tRCU kthread priority: %d.\n", kthread_prio); +#endif } #ifdef CONFIG_TREE_PREEMPT_RCU @@ -1339,7 +1339,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, smp_mb__after_unlock_lock(); rnp->boost_kthread_task = t; raw_spin_unlock_irqrestore(&rnp->lock, flags); - sp.sched_priority = RCU_BOOST_PRIO; + sp.sched_priority = kthread_prio; sched_setscheduler_nocheck(t, SCHED_FIFO, &sp); wake_up_process(t); /* get to TASK_INTERRUPTIBLE quickly. */ return 0; @@ -1356,7 +1356,7 @@ static void rcu_cpu_kthread_setup(unsigned int cpu) { struct sched_param sp; - sp.sched_priority = RCU_KTHREAD_PRIO; + sp.sched_priority = kthread_prio; sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); } diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE03 b/tools/testing/selftests/rcutorture/configs/rcu/TREE03 index f4567fb..b95f62e 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE03 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE03 @@ -17,5 +17,5 @@ CONFIG_DEBUG_LOCK_ALLOC=n CONFIG_RCU_CPU_STALL_INFO=n CONFIG_RCU_CPU_STALL_VERBOSE=n CONFIG_RCU_BOOST=y -CONFIG_RCU_BOOST_PRIO=2 +CONFIG_RCU_KTHREAD_PRIO=2 CONFIG_DEBUG_OBJECTS_RCU_HEAD=n diff --git a/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt b/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt index 3e588db..ab6e7b4 100644 --- a/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt +++ b/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt @@ -15,8 +15,8 @@ CONFIG_PREEMPT -- Do half. (First three and #8.) CONFIG_PROVE_LOCKING -- Do all but two, covering CONFIG_PROVE_RCU and not. CONFIG_PROVE_RCU -- Do all but one under CONFIG_PROVE_LOCKING. CONFIG_RCU_BOOST -- one of TREE_PREEMPT_RCU. -CONFIG_RCU_BOOST_PRIO -- set to 2 for _BOOST testing. -CONFIG_RCU_CPU_STALL_INFO -- do one with and without _VERBOSE. +CONFIG_RCU_KTHREAD_PRIO -- set to 2 for _BOOST testing. +CONFIG_RCU_CPU_STALL_INFO -- Do one. CONFIG_RCU_CPU_STALL_VERBOSE -- do one with and without _INFO. CONFIG_RCU_FANOUT -- Cover hierarchy as currently, but overlap with others. CONFIG_RCU_FANOUT_EXACT -- Do one. -- cgit v0.10.2 From 28f6569ab7d036cd4ee94c26bb76dc1b3f3fc056 Mon Sep 17 00:00:00 2001 From: Pranith Kumar Date: Mon, 22 Sep 2014 14:00:48 -0400 Subject: rcu: Remove redundant TREE_PREEMPT_RCU config option PREEMPT_RCU and TREE_PREEMPT_RCU serve the same function after TINY_PREEMPT_RCU has been removed. This patch removes TREE_PREEMPT_RCU and uses PREEMPT_RCU config option in its place. Signed-off-by: Pranith Kumar Signed-off-by: Paul E. McKenney diff --git a/Documentation/RCU/rcu.txt b/Documentation/RCU/rcu.txt index bf77833..745f429 100644 --- a/Documentation/RCU/rcu.txt +++ b/Documentation/RCU/rcu.txt @@ -36,7 +36,7 @@ o How can the updater tell when a grace period has completed executed in user mode, or executed in the idle loop, we can safely free up that item. - Preemptible variants of RCU (CONFIG_TREE_PREEMPT_RCU) get the + Preemptible variants of RCU (CONFIG_PREEMPT_RCU) get the same effect, but require that the readers manipulate CPU-local counters. These counters allow limited types of blocking within RCU read-side critical sections. SRCU also uses CPU-local @@ -81,7 +81,7 @@ o I hear that RCU is patented? What is with that? o I hear that RCU needs work in order to support realtime kernels? This work is largely completed. Realtime-friendly RCU can be - enabled via the CONFIG_TREE_PREEMPT_RCU kernel configuration + enabled via the CONFIG_PREEMPT_RCU kernel configuration parameter. However, work is in progress for enabling priority boosting of preempted RCU read-side critical sections. This is needed if you have CPU-bound realtime threads. diff --git a/Documentation/RCU/stallwarn.txt b/Documentation/RCU/stallwarn.txt index ef5a2fd..783e072 100644 --- a/Documentation/RCU/stallwarn.txt +++ b/Documentation/RCU/stallwarn.txt @@ -77,7 +77,7 @@ This message indicates that CPU 5 detected that it was causing a stall, and that the stall was affecting RCU-sched. This message will normally be followed by a stack dump of the offending CPU. On TREE_RCU kernel builds, RCU and RCU-sched are implemented by the same underlying mechanism, -while on TREE_PREEMPT_RCU kernel builds, RCU is instead implemented +while on PREEMPT_RCU kernel builds, RCU is instead implemented by rcu_preempt_state. On the other hand, if the offending CPU fails to print out a stall-warning @@ -89,7 +89,7 @@ INFO: rcu_bh_state detected stalls on CPUs/tasks: { 3 5 } (detected by 2, 2502 j This message indicates that CPU 2 detected that CPUs 3 and 5 were both causing stalls, and that the stall was affecting RCU-bh. This message will normally be followed by stack dumps for each CPU. Please note that -TREE_PREEMPT_RCU builds can be stalled by tasks as well as by CPUs, +PREEMPT_RCU builds can be stalled by tasks as well as by CPUs, and that the tasks will be indicated by PID, for example, "P3421". It is even possible for a rcu_preempt_state stall to be caused by both CPUs -and- tasks, in which case the offending CPUs and tasks will all @@ -205,10 +205,10 @@ o A CPU-bound real-time task in a CONFIG_PREEMPT kernel, which might o A CPU-bound real-time task in a CONFIG_PREEMPT_RT kernel that is running at a higher priority than the RCU softirq threads. This will prevent RCU callbacks from ever being invoked, - and in a CONFIG_TREE_PREEMPT_RCU kernel will further prevent + and in a CONFIG_PREEMPT_RCU kernel will further prevent RCU grace periods from ever completing. Either way, the system will eventually run out of memory and hang. In the - CONFIG_TREE_PREEMPT_RCU case, you might see stall-warning + CONFIG_PREEMPT_RCU case, you might see stall-warning messages. o A hardware or software issue shuts off the scheduler-clock diff --git a/Documentation/RCU/trace.txt b/Documentation/RCU/trace.txt index 910870b..b63b9bb 100644 --- a/Documentation/RCU/trace.txt +++ b/Documentation/RCU/trace.txt @@ -8,7 +8,7 @@ The following sections describe the debugfs files and formats, first for rcutree and next for rcutiny. -CONFIG_TREE_RCU and CONFIG_TREE_PREEMPT_RCU debugfs Files and Formats +CONFIG_TREE_RCU and CONFIG_PREEMPT_RCU debugfs Files and Formats These implementations of RCU provide several debugfs directories under the top-level directory "rcu": @@ -18,7 +18,7 @@ rcu/rcu_preempt rcu/rcu_sched Each directory contains files for the corresponding flavor of RCU. -Note that rcu/rcu_preempt is only present for CONFIG_TREE_PREEMPT_RCU. +Note that rcu/rcu_preempt is only present for CONFIG_PREEMPT_RCU. For CONFIG_TREE_RCU, the RCU flavor maps onto the RCU-sched flavor, so that activity for both appears in rcu/rcu_sched. diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt index e48c57f..88dfce1 100644 --- a/Documentation/RCU/whatisRCU.txt +++ b/Documentation/RCU/whatisRCU.txt @@ -137,7 +137,7 @@ rcu_read_lock() Used by a reader to inform the reclaimer that the reader is entering an RCU read-side critical section. It is illegal to block while in an RCU read-side critical section, though - kernels built with CONFIG_TREE_PREEMPT_RCU can preempt RCU + kernels built with CONFIG_PREEMPT_RCU can preempt RCU read-side critical sections. Any RCU-protected data structure accessed during an RCU read-side critical section is guaranteed to remain unreclaimed for the full duration of that critical section. diff --git a/include/linux/init_task.h b/include/linux/init_task.h index 77fc43f..d996aef 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -102,7 +102,7 @@ extern struct group_info init_groups; #define INIT_IDS #endif -#ifdef CONFIG_TREE_PREEMPT_RCU +#ifdef CONFIG_PREEMPT_RCU #define INIT_TASK_RCU_TREE_PREEMPT() \ .rcu_blocked_node = NULL, #else diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index a4a819f..295bb45 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -57,7 +57,7 @@ enum rcutorture_type { INVALID_RCU_FLAVOR }; -#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) +#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) void rcutorture_get_gp_data(enum rcutorture_type test_type, int *flags, unsigned long *gpnum, unsigned long *completed); void rcutorture_record_test_transition(void); @@ -365,7 +365,7 @@ typedef void call_rcu_func_t(struct rcu_head *head, void (*func)(struct rcu_head *head)); void wait_rcu_gp(call_rcu_func_t crf); -#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) +#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) #include #elif defined(CONFIG_TINY_RCU) #include @@ -852,7 +852,7 @@ static inline void rcu_preempt_sleep_check(void) * * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU), * it is illegal to block while in an RCU read-side critical section. - * In preemptible RCU implementations (TREE_PREEMPT_RCU) in CONFIG_PREEMPT + * In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPT * kernel builds, RCU read-side critical sections may be preempted, * but explicit blocking is illegal. Finally, in preemptible RCU * implementations in real-time (with -rt patchset) kernel builds, RCU diff --git a/include/linux/sched.h b/include/linux/sched.h index 5e344bb..706a9f7 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1278,9 +1278,9 @@ struct task_struct { union rcu_special rcu_read_unlock_special; struct list_head rcu_node_entry; #endif /* #ifdef CONFIG_PREEMPT_RCU */ -#ifdef CONFIG_TREE_PREEMPT_RCU +#ifdef CONFIG_PREEMPT_RCU struct rcu_node *rcu_blocked_node; -#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ +#endif /* #ifdef CONFIG_PREEMPT_RCU */ #ifdef CONFIG_TASKS_RCU unsigned long rcu_tasks_nvcsw; bool rcu_tasks_holdout; diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index e335e7d..c78e88c 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -36,7 +36,7 @@ TRACE_EVENT(rcu_utilization, #ifdef CONFIG_RCU_TRACE -#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) +#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) /* * Tracepoint for grace-period events. Takes a string identifying the @@ -345,7 +345,7 @@ TRACE_EVENT(rcu_fqs, __entry->cpu, __entry->qsevent) ); -#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) */ +#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) */ /* * Tracepoint for dyntick-idle entry/exit events. These take a string diff --git a/init/Kconfig b/init/Kconfig index 15c299c..5ac596e 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -477,7 +477,7 @@ config TREE_RCU thousands of CPUs. It also scales down nicely to smaller systems. -config TREE_PREEMPT_RCU +config PREEMPT_RCU bool "Preemptible tree-based hierarchical RCU" depends on PREEMPT select IRQ_WORK @@ -501,12 +501,6 @@ config TINY_RCU endchoice -config PREEMPT_RCU - def_bool TREE_PREEMPT_RCU - help - This option enables preemptible-RCU code that is common between - TREE_PREEMPT_RCU and, in the old days, TINY_PREEMPT_RCU. - config TASKS_RCU bool "Task_based RCU implementation using voluntary context switch" default n @@ -518,7 +512,7 @@ config TASKS_RCU If unsure, say N. config RCU_STALL_COMMON - def_bool ( TREE_RCU || TREE_PREEMPT_RCU || RCU_TRACE ) + def_bool ( TREE_RCU || PREEMPT_RCU || RCU_TRACE ) help This option enables RCU CPU stall code that is common between the TINY and TREE variants of RCU. The purpose is to allow @@ -576,7 +570,7 @@ config RCU_FANOUT int "Tree-based hierarchical RCU fanout value" range 2 64 if 64BIT range 2 32 if !64BIT - depends on TREE_RCU || TREE_PREEMPT_RCU + depends on TREE_RCU || PREEMPT_RCU default 64 if 64BIT default 32 if !64BIT help @@ -596,7 +590,7 @@ config RCU_FANOUT_LEAF int "Tree-based hierarchical RCU leaf-level fanout value" range 2 RCU_FANOUT if 64BIT range 2 RCU_FANOUT if !64BIT - depends on TREE_RCU || TREE_PREEMPT_RCU + depends on TREE_RCU || PREEMPT_RCU default 16 help This option controls the leaf-level fanout of hierarchical @@ -621,7 +615,7 @@ config RCU_FANOUT_LEAF config RCU_FANOUT_EXACT bool "Disable tree-based hierarchical RCU auto-balancing" - depends on TREE_RCU || TREE_PREEMPT_RCU + depends on TREE_RCU || PREEMPT_RCU default n help This option forces use of the exact RCU_FANOUT value specified, @@ -652,11 +646,11 @@ config RCU_FAST_NO_HZ Say N if you are unsure. config TREE_RCU_TRACE - def_bool RCU_TRACE && ( TREE_RCU || TREE_PREEMPT_RCU ) + def_bool RCU_TRACE && ( TREE_RCU || PREEMPT_RCU ) select DEBUG_FS help This option provides tracing for the TREE_RCU and - TREE_PREEMPT_RCU implementations, permitting Makefile to + PREEMPT_RCU implementations, permitting Makefile to trivially select kernel/rcutree_trace.c. config RCU_BOOST @@ -716,7 +710,7 @@ config RCU_BOOST_DELAY config RCU_NOCB_CPU bool "Offload RCU callback processing from boot-selected CPUs" - depends on TREE_RCU || TREE_PREEMPT_RCU + depends on TREE_RCU || PREEMPT_RCU default n help Use this option to reduce OS jitter for aggressive HPC or diff --git a/kernel/rcu/Makefile b/kernel/rcu/Makefile index 807ccfb..e6fae50 100644 --- a/kernel/rcu/Makefile +++ b/kernel/rcu/Makefile @@ -1,6 +1,6 @@ obj-y += update.o srcu.o obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o obj-$(CONFIG_TREE_RCU) += tree.o -obj-$(CONFIG_TREE_PREEMPT_RCU) += tree.o +obj-$(CONFIG_PREEMPT_RCU) += tree.o obj-$(CONFIG_TREE_RCU_TRACE) += tree_trace.o obj-$(CONFIG_TINY_RCU) += tiny.o diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index bbdc45d..66cde51 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -139,7 +139,7 @@ struct rcu_node { unsigned long expmask; /* Groups that have ->blkd_tasks */ /* elements that need to drain to allow the */ /* current expedited grace period to */ - /* complete (only for TREE_PREEMPT_RCU). */ + /* complete (only for PREEMPT_RCU). */ unsigned long qsmaskinit; /* Per-GP initial value for qsmask & expmask. */ unsigned long grpmask; /* Mask to apply to parent qsmask. */ @@ -530,10 +530,10 @@ DECLARE_PER_CPU(struct rcu_data, rcu_sched_data); extern struct rcu_state rcu_bh_state; DECLARE_PER_CPU(struct rcu_data, rcu_bh_data); -#ifdef CONFIG_TREE_PREEMPT_RCU +#ifdef CONFIG_PREEMPT_RCU extern struct rcu_state rcu_preempt_state; DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); -#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */ +#endif /* #ifdef CONFIG_PREEMPT_RCU */ #ifdef CONFIG_RCU_BOOST DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); @@ -563,10 +563,10 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, #endif /* #ifdef CONFIG_HOTPLUG_CPU */ static void rcu_preempt_check_callbacks(int cpu); void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); -#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) +#if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PREEMPT_RCU) static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, bool wake); -#endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) */ +#endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_PREEMPT_RCU) */ static void __init __rcu_init_preempt(void); static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 344f0e6..6d07fb4 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -100,7 +100,7 @@ static void __init rcu_bootup_announce_oddness(void) #endif } -#ifdef CONFIG_TREE_PREEMPT_RCU +#ifdef CONFIG_PREEMPT_RCU RCU_STATE_INITIALIZER(rcu_preempt, 'p', call_rcu); static struct rcu_state *rcu_state_p = &rcu_preempt_state; @@ -932,7 +932,7 @@ void exit_rcu(void) __rcu_read_unlock(); } -#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */ +#else /* #ifdef CONFIG_PREEMPT_RCU */ static struct rcu_state *rcu_state_p = &rcu_sched_state; @@ -1083,7 +1083,7 @@ void exit_rcu(void) { } -#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */ +#endif /* #else #ifdef CONFIG_PREEMPT_RCU */ #ifdef CONFIG_RCU_BOOST diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 3ef8ba5..27a5b17 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -306,7 +306,7 @@ struct debug_obj_descr rcuhead_debug_descr = { EXPORT_SYMBOL_GPL(rcuhead_debug_descr); #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */ -#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE) +#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) || defined(CONFIG_RCU_TRACE) void do_trace_rcu_torture_read(const char *rcutorturename, struct rcu_head *rhp, unsigned long secs, unsigned long c_old, unsigned long c) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 4e35a5d..12e7b02 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1252,7 +1252,7 @@ config RCU_CPU_STALL_VERBOSE config RCU_CPU_STALL_INFO bool "Print additional diagnostics on RCU CPU stall" - depends on (TREE_RCU || TREE_PREEMPT_RCU) && DEBUG_KERNEL + depends on (TREE_RCU || PREEMPT_RCU) && DEBUG_KERNEL default n help For each stalled CPU that is aware of the current RCU grace diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 index 38e3895..69f8e58 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE01 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE01 @@ -2,7 +2,7 @@ CONFIG_SMP=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE02 b/tools/testing/selftests/rcutorture/configs/rcu/TREE02 index ea119ba..fc29c62 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE02 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE02 @@ -3,7 +3,7 @@ CONFIG_NR_CPUS=8 CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T b/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T index 19cf948..778ec56 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE02-T @@ -3,7 +3,7 @@ CONFIG_NR_CPUS=8 CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE03 b/tools/testing/selftests/rcutorture/configs/rcu/TREE03 index b95f62e..a022f03 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE03 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE03 @@ -3,7 +3,7 @@ CONFIG_NR_CPUS=8 CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=y CONFIG_NO_HZ_IDLE=n CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE08 b/tools/testing/selftests/rcutorture/configs/rcu/TREE08 index 69a2e25..6f8609d 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE08 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE08 @@ -3,7 +3,7 @@ CONFIG_NR_CPUS=16 CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T b/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T index a0f32fb..4e55bee 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE08-T @@ -3,7 +3,7 @@ CONFIG_NR_CPUS=16 CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/TREE09 b/tools/testing/selftests/rcutorture/configs/rcu/TREE09 index b7a62a5..47e2ecd 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/TREE09 +++ b/tools/testing/selftests/rcutorture/configs/rcu/TREE09 @@ -3,7 +3,7 @@ CONFIG_NR_CPUS=1 CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_HZ_PERIODIC=n CONFIG_NO_HZ_IDLE=y CONFIG_NO_HZ_FULL=n diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P1-S-T-NH-SD-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P1-S-T-NH-SD-SMP-HP index f72402d..dd7bd4d 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P1-S-T-NH-SD-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P1-S-T-NH-SD-SMP-HP @@ -9,7 +9,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P2-2-t-nh-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P2-2-t-nh-sd-SMP-hp index 0f3b667..81d4843 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P2-2-t-nh-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P2-2-t-nh-sd-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P3-3-T-nh-SD-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P3-3-T-nh-SD-SMP-hp index b035e14..16e22b7 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P3-3-T-nh-SD-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P3-3-T-nh-SD-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P4-A-t-NH-sd-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P4-A-t-NH-sd-SMP-HP index 3ccf6a9..ea7ed3b 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P4-A-t-NH-sd-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P4-A-t-NH-sd-SMP-HP @@ -8,7 +8,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_RT_MUTEXES=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P5-U-T-NH-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P5-U-T-NH-sd-SMP-hp index a55c008..24f4d5e 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P5-U-T-NH-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v0.0/P5-U-T-NH-sd-SMP-hp @@ -11,7 +11,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_OBJECTS=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P1-S-T-NH-SD-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P1-S-T-NH-SD-SMP-HP index 9647c44..6880663 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P1-S-T-NH-SD-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P1-S-T-NH-SD-SMP-HP @@ -10,7 +10,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P2-2-t-nh-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P2-2-t-nh-sd-SMP-hp index 0f3b667..81d4843 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P2-2-t-nh-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P2-2-t-nh-sd-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P3-3-T-nh-SD-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P3-3-T-nh-SD-SMP-hp index b035e14..16e22b7 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P3-3-T-nh-SD-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P3-3-T-nh-SD-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P4-A-t-NH-sd-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P4-A-t-NH-sd-SMP-HP index 3ccf6a9..ea7ed3b 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P4-A-t-NH-sd-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P4-A-t-NH-sd-SMP-HP @@ -8,7 +8,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_RT_MUTEXES=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P5-U-T-NH-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P5-U-T-NH-sd-SMP-hp index a55c008..24f4d5e 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P5-U-T-NH-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P5-U-T-NH-sd-SMP-hp @@ -11,7 +11,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_OBJECTS=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P6---t-nh-SD-smp-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P6---t-nh-SD-smp-hp index f4c9175..1be59e4 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P6---t-nh-SD-smp-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P6---t-nh-SD-smp-hp @@ -8,7 +8,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -CONFIG_TREE_PREEMPT_RCU=y +CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP index 77a8c5b..b49ef74 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP @@ -14,7 +14,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_PROVE_LOCKING=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-all b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-all index 0eecebc..a55a6ac 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-all +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-all @@ -14,7 +14,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_PROVE_LOCKING=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-none b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-none index 0eecebc..a55a6ac 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-none +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-HP-none @@ -14,7 +14,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_PROVE_LOCKING=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-hp index 588bc70..3134f46 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.12/P7-4-T-NH-SD-SMP-hp @@ -14,7 +14,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_PROVE_LOCKING=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P1-S-T-NH-SD-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P1-S-T-NH-SD-SMP-HP index 9647c44..6880663 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P1-S-T-NH-SD-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P1-S-T-NH-SD-SMP-HP @@ -10,7 +10,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P2-2-t-nh-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P2-2-t-nh-sd-SMP-hp index 0f3b667..81d4843 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P2-2-t-nh-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P2-2-t-nh-sd-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P3-3-T-nh-SD-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P3-3-T-nh-SD-SMP-hp index b035e14..16e22b7 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P3-3-T-nh-SD-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P3-3-T-nh-SD-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P4-A-t-NH-sd-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P4-A-t-NH-sd-SMP-HP index 3ccf6a9..ea7ed3b 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P4-A-t-NH-sd-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P4-A-t-NH-sd-SMP-HP @@ -8,7 +8,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_RT_MUTEXES=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P5-U-T-NH-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P5-U-T-NH-sd-SMP-hp index a55c008..24f4d5e 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P5-U-T-NH-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.3/P5-U-T-NH-sd-SMP-hp @@ -11,7 +11,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_OBJECTS=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P1-S-T-NH-SD-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P1-S-T-NH-SD-SMP-HP index 9647c44..6880663 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P1-S-T-NH-SD-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P1-S-T-NH-SD-SMP-HP @@ -10,7 +10,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P2-2-t-nh-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P2-2-t-nh-sd-SMP-hp index 0f3b667..81d4843 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P2-2-t-nh-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P2-2-t-nh-sd-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P3-3-T-nh-SD-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P3-3-T-nh-SD-SMP-hp index b035e14..16e22b7 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P3-3-T-nh-SD-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P3-3-T-nh-SD-SMP-hp @@ -10,7 +10,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_SYSFS_DEPRECATED_V2=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P4-A-t-NH-sd-SMP-HP b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P4-A-t-NH-sd-SMP-HP index 3ccf6a9..ea7ed3b 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P4-A-t-NH-sd-SMP-HP +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P4-A-t-NH-sd-SMP-HP @@ -8,7 +8,7 @@ CONFIG_HOTPLUG_CPU=y CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_RCU_TORTURE_TEST=m CONFIG_MODULE_UNLOAD=y CONFIG_RT_MUTEXES=y diff --git a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P5-U-T-NH-sd-SMP-hp b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P5-U-T-NH-sd-SMP-hp index a55c008..24f4d5e 100644 --- a/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P5-U-T-NH-sd-SMP-hp +++ b/tools/testing/selftests/rcutorture/configs/rcu/v3.5/P5-U-T-NH-sd-SMP-hp @@ -11,7 +11,7 @@ CONFIG_HIBERNATION=n CONFIG_PREEMPT_NONE=n CONFIG_PREEMPT_VOLUNTARY=n CONFIG_PREEMPT=y -#CHECK#CONFIG_TREE_PREEMPT_RCU=y +#CHECK#CONFIG_PREEMPT_RCU=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_OBJECTS=y CONFIG_DEBUG_OBJECTS_RCU_HEAD=y diff --git a/tools/testing/selftests/rcutorture/doc/TINY_RCU.txt b/tools/testing/selftests/rcutorture/doc/TINY_RCU.txt index 28db67b..9ef33a7 100644 --- a/tools/testing/selftests/rcutorture/doc/TINY_RCU.txt +++ b/tools/testing/selftests/rcutorture/doc/TINY_RCU.txt @@ -34,7 +34,7 @@ CONFIG_PREEMPT CONFIG_PREEMPT_RCU CONFIG_SMP CONFIG_TINY_RCU -CONFIG_TREE_PREEMPT_RCU +CONFIG_PREEMPT_RCU CONFIG_TREE_RCU All forced by CONFIG_TINY_RCU. diff --git a/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt b/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt index ab6e7b4..f613755 100644 --- a/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt +++ b/tools/testing/selftests/rcutorture/doc/TREE_RCU-kconfig.txt @@ -1,5 +1,5 @@ This document gives a brief rationale for the TREE_RCU-related test -cases, a group that includes TREE_PREEMPT_RCU. +cases, a group that includes PREEMPT_RCU. Kconfig Parameters: @@ -14,7 +14,7 @@ CONFIG_NO_HZ_FULL_SYSIDLE -- Do one. CONFIG_PREEMPT -- Do half. (First three and #8.) CONFIG_PROVE_LOCKING -- Do all but two, covering CONFIG_PROVE_RCU and not. CONFIG_PROVE_RCU -- Do all but one under CONFIG_PROVE_LOCKING. -CONFIG_RCU_BOOST -- one of TREE_PREEMPT_RCU. +CONFIG_RCU_BOOST -- one of PREEMPT_RCU. CONFIG_RCU_KTHREAD_PRIO -- set to 2 for _BOOST testing. CONFIG_RCU_CPU_STALL_INFO -- Do one. CONFIG_RCU_CPU_STALL_VERBOSE -- do one with and without _INFO. @@ -27,7 +27,7 @@ CONFIG_RCU_NOCB_CPU_ALL -- Do one. CONFIG_RCU_NOCB_CPU_NONE -- Do one. CONFIG_RCU_NOCB_CPU_ZERO -- Do one. CONFIG_RCU_TRACE -- Do half. -CONFIG_SMP -- Need one !SMP for TREE_PREEMPT_RCU. +CONFIG_SMP -- Need one !SMP for PREEMPT_RCU. RCU-bh: Do one with PREEMPT and one with !PREEMPT. RCU-sched: Do one with PREEMPT but not BOOST. @@ -77,7 +77,7 @@ CONFIG_RCU_CPU_STALL_TIMEOUT CONFIG_RCU_STALL_COMMON - Implied by TREE_RCU and TREE_PREEMPT_RCU. + Implied by TREE_RCU and PREEMPT_RCU. CONFIG_RCU_TORTURE_TEST CONFIG_RCU_TORTURE_TEST_RUNNABLE @@ -88,7 +88,7 @@ CONFIG_RCU_USER_QS Redundant with CONFIG_NO_HZ_FULL. -CONFIG_TREE_PREEMPT_RCU +CONFIG_PREEMPT_RCU CONFIG_TREE_RCU These are controlled by CONFIG_PREEMPT. -- cgit v0.10.2 From 776d680711f4836964f3005ee53794661ca74629 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 23 Oct 2014 10:50:41 -0700 Subject: rcu: Kick rcuo kthreads after their CPU goes offline If a no-CBs CPU were to post an RCU callback with interrupts disabled after it entered the idle loop for the last time, there might be no deferred wakeup for the corresponding rcuo kthreads. This commit therefore adds a set of calls to do_nocb_deferred_wakeup() after the CPU has gone completely offline. Signed-off-by: Paul E. McKenney diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9e3c20f..07bf4aa 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3497,8 +3497,10 @@ static int rcu_cpu_notify(struct notifier_block *self, case CPU_DEAD_FROZEN: case CPU_UP_CANCELED: case CPU_UP_CANCELED_FROZEN: - for_each_rcu_flavor(rsp) + for_each_rcu_flavor(rsp) { rcu_cleanup_dead_cpu(cpu, rsp); + do_nocb_deferred_wakeup(per_cpu_ptr(rsp->rda, cpu)); + } break; default: break; -- cgit v0.10.2 From bbe5d7a93a39104be146f71b9b99af8fb65a6a2d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Fri, 24 Oct 2014 11:11:08 -0700 Subject: rcu: Fix for rcuo online-time-creation reorganization bug Commit 35ce7f29a44a (rcu: Create rcuo kthreads only for onlined CPUs) contains checks for the case where CPUs are brought online out of order, re-wiring the rcuo leader-follower relationships as needed. Unfortunately, this rewiring was broken. This apparently went undetected due to the tendency of systems to bring CPUs online in order. This commit nevertheless fixes the rewiring. Signed-off-by: Paul E. McKenney diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 6d07fb4..5053dae 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -2586,9 +2586,13 @@ static void rcu_spawn_one_nocb_kthread(struct rcu_state *rsp, int cpu) rdp->nocb_leader = rdp_spawn; if (rdp_last && rdp != rdp_spawn) rdp_last->nocb_next_follower = rdp; - rdp_last = rdp; - rdp = rdp->nocb_next_follower; - rdp_last->nocb_next_follower = NULL; + if (rdp == rdp_spawn) { + rdp = rdp->nocb_next_follower; + } else { + rdp_last = rdp; + rdp = rdp->nocb_next_follower; + rdp_last->nocb_next_follower = NULL; + } } while (rdp); rdp_spawn->nocb_next_follower = rdp_old_leader; } -- cgit v0.10.2 From 28ced795cbb43f01146feb96d03a72facdee9911 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Tue, 2 Sep 2014 14:13:44 -0700 Subject: rcu: Remove rcu_dynticks * parameters when they are always this_cpu_ptr(&rcu_dynticks) For some functions in kernel/rcu/tree* the rdtp parameter is always this_cpu_ptr(rdtp). Remove the parameter if constant and calculate the pointer in function. This will have the advantage that it is obvious that the address are all per cpu offsets and thus it will enable the use of this_cpu_ops in the future. Signed-off-by: Christoph Lameter [ paulmck: Forward-ported to rcu/dev, whitespace adjustment. ] Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 9815447..c0673c5 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -510,11 +510,11 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) * we really have entered idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */ -static void rcu_eqs_enter_common(struct rcu_dynticks *rdtp, long long oldval, - bool user) +static void rcu_eqs_enter_common(long long oldval, bool user) { struct rcu_state *rsp; struct rcu_data *rdp; + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); trace_rcu_dyntick(TPS("Start"), oldval, rdtp->dynticks_nesting); if (!user && !is_idle_task(current)) { @@ -565,7 +565,7 @@ static void rcu_eqs_enter(bool user) WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0); if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE) { rdtp->dynticks_nesting = 0; - rcu_eqs_enter_common(rdtp, oldval, user); + rcu_eqs_enter_common(oldval, user); } else { rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE; } @@ -589,7 +589,7 @@ void rcu_idle_enter(void) local_irq_save(flags); rcu_eqs_enter(false); - rcu_sysidle_enter(this_cpu_ptr(&rcu_dynticks), 0); + rcu_sysidle_enter(0); local_irq_restore(flags); } EXPORT_SYMBOL_GPL(rcu_idle_enter); @@ -639,8 +639,8 @@ void rcu_irq_exit(void) if (rdtp->dynticks_nesting) trace_rcu_dyntick(TPS("--="), oldval, rdtp->dynticks_nesting); else - rcu_eqs_enter_common(rdtp, oldval, true); - rcu_sysidle_enter(rdtp, 1); + rcu_eqs_enter_common(oldval, true); + rcu_sysidle_enter(1); local_irq_restore(flags); } @@ -651,9 +651,10 @@ void rcu_irq_exit(void) * we really have exited idle, and must do the appropriate accounting. * The caller must have disabled interrupts. */ -static void rcu_eqs_exit_common(struct rcu_dynticks *rdtp, long long oldval, - int user) +static void rcu_eqs_exit_common(long long oldval, int user) { + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); + rcu_dynticks_task_exit(); smp_mb__before_atomic(); /* Force ordering w/previous sojourn. */ atomic_inc(&rdtp->dynticks); @@ -691,7 +692,7 @@ static void rcu_eqs_exit(bool user) rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE; } else { rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE; - rcu_eqs_exit_common(rdtp, oldval, user); + rcu_eqs_exit_common(oldval, user); } } @@ -712,7 +713,7 @@ void rcu_idle_exit(void) local_irq_save(flags); rcu_eqs_exit(false); - rcu_sysidle_exit(this_cpu_ptr(&rcu_dynticks), 0); + rcu_sysidle_exit(0); local_irq_restore(flags); } EXPORT_SYMBOL_GPL(rcu_idle_exit); @@ -763,8 +764,8 @@ void rcu_irq_enter(void) if (oldval) trace_rcu_dyntick(TPS("++="), oldval, rdtp->dynticks_nesting); else - rcu_eqs_exit_common(rdtp, oldval, true); - rcu_sysidle_exit(rdtp, 1); + rcu_eqs_exit_common(oldval, true); + rcu_sysidle_exit(1); local_irq_restore(flags); } diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index bbdc45d..8e90562 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -606,8 +606,8 @@ static void __init rcu_organize_nocb_kthreads(struct rcu_state *rsp); #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ static void __maybe_unused rcu_kick_nohz_cpu(int cpu); static bool init_nocb_callback_list(struct rcu_data *rdp); -static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq); -static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq); +static void rcu_sysidle_enter(int irq); +static void rcu_sysidle_exit(int irq); static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle, unsigned long *maxj); static bool is_sysidle_rcu_state(struct rcu_state *rsp); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index c1d7f27..0954714 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -2761,9 +2761,10 @@ static int full_sysidle_state; /* Current system-idle state. */ * to detect full-system idle states, not RCU quiescent states and grace * periods. The caller must have disabled interrupts. */ -static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq) +static void rcu_sysidle_enter(int irq) { unsigned long j; + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); /* If there are no nohz_full= CPUs, no need to track this. */ if (!tick_nohz_full_enabled()) @@ -2832,8 +2833,10 @@ void rcu_sysidle_force_exit(void) * usermode execution does -not- count as idle here! The caller must * have disabled interrupts. */ -static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq) +static void rcu_sysidle_exit(int irq) { + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); + /* If there are no nohz_full= CPUs, no need to track this. */ if (!tick_nohz_full_enabled()) return; @@ -3127,11 +3130,11 @@ static void rcu_sysidle_init_percpu_data(struct rcu_dynticks *rdtp) #else /* #ifdef CONFIG_NO_HZ_FULL_SYSIDLE */ -static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq) +static void rcu_sysidle_enter(int irq) { } -static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq) +static void rcu_sysidle_exit(int irq) { } -- cgit v0.10.2 From 11bbb235c26f93b7c69e441452e44adbf6ed6996 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 4 Sep 2014 12:21:41 -0700 Subject: rcu: Use DEFINE_PER_CPU_SHARED_ALIGNED for rcu_data The rcu_data per-CPU variable has a number of fields that are atomically manipulated, potentially by any CPU. This situation can result in false sharing with per-CPU variables that have the misfortune of being allocated adjacent to rcu_data in memory. This commit therefore changes the DEFINE_PER_CPU() to DEFINE_PER_CPU_SHARED_ALIGNED() in order to avoid this false sharing. Reported-by: Christoph Lameter Signed-off-by: Paul E. McKenney Reviewed-by: Christoph Lameter Reviewed-by: Pranith Kumar diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index c0673c5..ab6fcfb 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -105,7 +105,7 @@ struct rcu_state sname##_state = { \ .name = RCU_STATE_NAME(sname), \ .abbr = sabbr, \ }; \ -DEFINE_PER_CPU(struct rcu_data, sname##_data) +DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data) RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched); RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh); -- cgit v0.10.2 From c3377c2da6e594504c900d5ef72374c109e4ca99 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Oct 2014 07:53:02 -0700 Subject: rcu: Remove "cpu" argument to rcu_check_callbacks() The "cpu" argument was kept around on the off-chance that RCU might offload scheduler-clock interrupts. However, this offload approach has been replaced by NO_HZ_FULL, which offloads -all- RCU processing from qualifying CPUs. It is therefore time to remove the "cpu" argument to rcu_check_callbacks(), which this commit does. Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index a4a819f..b4ad275 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -260,7 +260,7 @@ static inline int rcu_preempt_depth(void) void rcu_init(void); void rcu_sched_qs(void); void rcu_bh_qs(void); -void rcu_check_callbacks(int cpu, int user); +void rcu_check_callbacks(int user); struct notifier_block; void rcu_idle_enter(void); void rcu_idle_exit(void); diff --git a/kernel/rcu/tiny.c b/kernel/rcu/tiny.c index c0623fc..01570c6 100644 --- a/kernel/rcu/tiny.c +++ b/kernel/rcu/tiny.c @@ -247,7 +247,7 @@ void rcu_bh_qs(void) * be called from hardirq context. It is normally called from the * scheduling-clock interrupt. */ -void rcu_check_callbacks(int cpu, int user) +void rcu_check_callbacks(int user) { RCU_TRACE(check_cpu_stalls()); if (user || rcu_is_cpu_rrupt_from_idle()) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index ab6fcfb..3107811 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2388,7 +2388,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) * invoked from the scheduling-clock interrupt. If rcu_pending returns * false, there is no point in invoking rcu_check_callbacks(). */ -void rcu_check_callbacks(int cpu, int user) +void rcu_check_callbacks(int user) { trace_rcu_utilization(TPS("Start scheduler-tick")); increment_cpu_stall_ticks(); @@ -2420,8 +2420,8 @@ void rcu_check_callbacks(int cpu, int user) rcu_bh_qs(); } - rcu_preempt_check_callbacks(cpu); - if (rcu_pending(cpu)) + rcu_preempt_check_callbacks(smp_processor_id()); + if (rcu_pending(smp_processor_id())) invoke_rcu_core(); if (user) rcu_note_voluntary_context_switch(current); diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 3260ffd..2d3f5c5 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1377,12 +1377,11 @@ unsigned long get_next_timer_interrupt(unsigned long now) void update_process_times(int user_tick) { struct task_struct *p = current; - int cpu = smp_processor_id(); /* Note: this timer irq context must be accounted for as well. */ account_process_tick(p, user_tick); run_local_timers(); - rcu_check_callbacks(cpu, user_tick); + rcu_check_callbacks(user_tick); #ifdef CONFIG_IRQ_WORK if (in_irq()) irq_work_tick(); -- cgit v0.10.2 From e3950ecd55383f7bd8a06a9649091ccfbbcf5505 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Oct 2014 08:03:57 -0700 Subject: rcu: Remove "cpu" argument to rcu_pending() Because rcu_pending()'s argument is guaranteed to always be the current CPU, drop the argument and replace per_cpu_ptr() with this_cpu_ptr(). Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 3107811..0670ba3 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -325,7 +325,7 @@ static void force_qs_rnp(struct rcu_state *rsp, unsigned long *maxj), bool *isidle, unsigned long *maxj); static void force_quiescent_state(struct rcu_state *rsp); -static int rcu_pending(int cpu); +static int rcu_pending(void); /* * Return the number of RCU-sched batches processed thus far for debug & stats. @@ -2421,7 +2421,7 @@ void rcu_check_callbacks(int user) rcu_bh_qs(); } rcu_preempt_check_callbacks(smp_processor_id()); - if (rcu_pending(smp_processor_id())) + if (rcu_pending()) invoke_rcu_core(); if (user) rcu_note_voluntary_context_switch(current); @@ -3144,12 +3144,12 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) * by the current CPU, returning 1 if so. This function is part of the * RCU implementation; it is -not- an exported member of the RCU API. */ -static int rcu_pending(int cpu) +static int rcu_pending(void) { struct rcu_state *rsp; for_each_rcu_flavor(rsp) - if (__rcu_pending(rsp, per_cpu_ptr(rsp->rda, cpu))) + if (__rcu_pending(rsp, this_cpu_ptr(rsp->rda))) return 1; return 0; } -- cgit v0.10.2 From 86aea0e6e7d571a9452082a7198d0603cc5dd965 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Oct 2014 08:12:00 -0700 Subject: rcu: Remove "cpu" argument to rcu_preempt_check_callbacks() Because rcu_preempt_check_callbacks()'s argument is guaranteed to always be the current CPU, drop the argument and replace per_cpu() with __this_cpu_read(). Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 0670ba3..1af5e2c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -2420,7 +2420,7 @@ void rcu_check_callbacks(int user) rcu_bh_qs(); } - rcu_preempt_check_callbacks(smp_processor_id()); + rcu_preempt_check_callbacks(); if (rcu_pending()) invoke_rcu_core(); if (user) diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 8e90562..94a26e3 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -561,7 +561,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_data *rdp); #endif /* #ifdef CONFIG_HOTPLUG_CPU */ -static void rcu_preempt_check_callbacks(int cpu); +static void rcu_preempt_check_callbacks(void); void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 0954714..7158814 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -621,7 +621,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, * * Caller must disable hard irqs. */ -static void rcu_preempt_check_callbacks(int cpu) +static void rcu_preempt_check_callbacks(void) { struct task_struct *t = current; @@ -630,8 +630,8 @@ static void rcu_preempt_check_callbacks(int cpu) return; } if (t->rcu_read_lock_nesting > 0 && - per_cpu(rcu_preempt_data, cpu).qs_pending && - !per_cpu(rcu_preempt_data, cpu).passed_quiesce) + __this_cpu_read(rcu_preempt_data.qs_pending) && + !__this_cpu_read(rcu_preempt_data.passed_quiesce)) t->rcu_read_unlock_special.b.need_qs = true; } @@ -1017,7 +1017,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, * Because preemptible RCU does not exist, it never has any callbacks * to check. */ -static void rcu_preempt_check_callbacks(int cpu) +static void rcu_preempt_check_callbacks(void) { } -- cgit v0.10.2 From 38200cf24702e5d79ce6c8f4c62036c41845c62d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Oct 2014 12:50:04 -0700 Subject: rcu: Remove "cpu" argument to rcu_note_context_switch() The "cpu" argument to rcu_note_context_switch() is always the current CPU, so drop it. This in turn allows the "cpu" argument to rcu_preempt_note_context_switch() to be removed, which allows the sole use of "cpu" in both functions to be replaced with a this_cpu_ptr(). Again, the anticipated cross-CPU uses of these functions has been replaced by NO_HZ_FULL. Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index 38cc5b1..0e53662 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -78,7 +78,7 @@ static inline void kfree_call_rcu(struct rcu_head *head, call_rcu(head, func); } -static inline void rcu_note_context_switch(int cpu) +static inline void rcu_note_context_switch(void) { rcu_sched_qs(); } diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 3e2f5d4..7b5484d 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -30,7 +30,7 @@ #ifndef __LINUX_RCUTREE_H #define __LINUX_RCUTREE_H -void rcu_note_context_switch(int cpu); +void rcu_note_context_switch(void); #ifndef CONFIG_RCU_NOCB_CPU_ALL int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies); #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ @@ -43,7 +43,7 @@ void rcu_cpu_stall_reset(void); */ static inline void rcu_virt_note_context_switch(int cpu) { - rcu_note_context_switch(cpu); + rcu_note_context_switch(); } void synchronize_rcu_bh(void); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 1af5e2c..b591f14 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -286,11 +286,11 @@ static void rcu_momentary_dyntick_idle(void) * and requires special handling for preemptible RCU. * The caller must have disabled preemption. */ -void rcu_note_context_switch(int cpu) +void rcu_note_context_switch(void) { trace_rcu_utilization(TPS("Start context switch")); rcu_sched_qs(); - rcu_preempt_note_context_switch(cpu); + rcu_preempt_note_context_switch(); if (unlikely(raw_cpu_read(rcu_sched_qs_mask))) rcu_momentary_dyntick_idle(); trace_rcu_utilization(TPS("End context switch")); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 94a26e3..238ac39 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -547,7 +547,7 @@ DECLARE_PER_CPU(char, rcu_cpu_has_work); /* Forward declarations for rcutree_plugin.h */ static void rcu_bootup_announce(void); long rcu_batches_completed(void); -static void rcu_preempt_note_context_switch(int cpu); +static void rcu_preempt_note_context_switch(void); static int rcu_preempt_blocked_readers_cgp(struct rcu_node *rnp); #ifdef CONFIG_HOTPLUG_CPU static void rcu_report_unblock_qs_rnp(struct rcu_node *rnp, diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 7158814..495d4cc 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -156,7 +156,7 @@ static void rcu_preempt_qs(void) * * Caller must disable preemption. */ -static void rcu_preempt_note_context_switch(int cpu) +static void rcu_preempt_note_context_switch(void) { struct task_struct *t = current; unsigned long flags; @@ -167,7 +167,7 @@ static void rcu_preempt_note_context_switch(int cpu) !t->rcu_read_unlock_special.b.blocked) { /* Possibly blocking in an RCU read-side critical section. */ - rdp = per_cpu_ptr(rcu_preempt_state.rda, cpu); + rdp = this_cpu_ptr(rcu_preempt_state.rda); rnp = rdp->mynode; raw_spin_lock_irqsave(&rnp->lock, flags); smp_mb__after_unlock_lock(); @@ -945,7 +945,7 @@ EXPORT_SYMBOL_GPL(rcu_batches_completed); * Because preemptible RCU does not exist, we never have to check for * CPUs being in quiescent states. */ -static void rcu_preempt_note_context_switch(int cpu) +static void rcu_preempt_note_context_switch(void) { } diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4499950..cc18694 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2802,7 +2802,7 @@ need_resched: preempt_disable(); cpu = smp_processor_id(); rq = cpu_rq(cpu); - rcu_note_context_switch(cpu); + rcu_note_context_switch(); prev = rq->curr; schedule_debug(prev); diff --git a/kernel/softirq.c b/kernel/softirq.c index 0699add..501baa9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -656,7 +656,7 @@ static void run_ksoftirqd(unsigned int cpu) * in the task stack here. */ __do_softirq(); - rcu_note_context_switch(cpu); + rcu_note_context_switch(); local_irq_enable(); cond_resched(); return; -- cgit v0.10.2 From aa6da5140b784ece799f670bf532096f67aa7785 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 21 Oct 2014 13:23:08 -0700 Subject: rcu: Remove "cpu" argument to rcu_needs_cpu() The "cpu" argument to rcu_needs_cpu() is always the current CPU, so drop it. This in turn allows the "cpu" argument to rcu_cpu_has_callbacks() to be removed, which allows the uses of "cpu" in both functions to be replaced with a this_cpu_ptr(). Again, the anticipated cross-CPU uses of these functions has been replaced by NO_HZ_FULL. Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index b4ad275..4eb8108 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1103,7 +1103,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head)) #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) -static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) +static inline int rcu_needs_cpu(unsigned long *delta_jiffies) { *delta_jiffies = ULONG_MAX; return 0; diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 7b5484d..5295379 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -32,7 +32,7 @@ void rcu_note_context_switch(void); #ifndef CONFIG_RCU_NOCB_CPU_ALL -int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies); +int rcu_needs_cpu(unsigned long *delta_jiffies); #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ void rcu_cpu_stall_reset(void); diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index b591f14..d678a98 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -3159,7 +3159,7 @@ static int rcu_pending(void) * non-NULL, store an indication of whether all callbacks are lazy. * (If there are no callbacks, all of them are deemed to be lazy.) */ -static int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy) +static int __maybe_unused rcu_cpu_has_callbacks(bool *all_lazy) { bool al = true; bool hc = false; @@ -3167,7 +3167,7 @@ static int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy) struct rcu_state *rsp; for_each_rcu_flavor(rsp) { - rdp = per_cpu_ptr(rsp->rda, cpu); + rdp = this_cpu_ptr(rsp->rda); if (!rdp->nxtlist) continue; hc = true; diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 495d4cc..1797b76 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1512,10 +1512,10 @@ static void rcu_prepare_kthreads(int cpu) * any flavor of RCU. */ #ifndef CONFIG_RCU_NOCB_CPU_ALL -int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies) +int rcu_needs_cpu(unsigned long *delta_jiffies) { *delta_jiffies = ULONG_MAX; - return rcu_cpu_has_callbacks(cpu, NULL); + return rcu_cpu_has_callbacks(NULL); } #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ @@ -1624,15 +1624,15 @@ static bool __maybe_unused rcu_try_advance_all_cbs(void) * The caller must have disabled interrupts. */ #ifndef CONFIG_RCU_NOCB_CPU_ALL -int rcu_needs_cpu(int cpu, unsigned long *dj) +int rcu_needs_cpu(unsigned long *dj) { - struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); /* Snapshot to detect later posting of non-lazy callback. */ rdtp->nonlazy_posted_snap = rdtp->nonlazy_posted; /* If no callbacks, RCU doesn't need the CPU. */ - if (!rcu_cpu_has_callbacks(cpu, &rdtp->all_lazy)) { + if (!rcu_cpu_has_callbacks(&rdtp->all_lazy)) { *dj = ULONG_MAX; return 0; } @@ -1679,7 +1679,7 @@ static void rcu_prepare_for_idle(int cpu) /* Handle nohz enablement switches conservatively. */ tne = ACCESS_ONCE(tick_nohz_active); if (tne != rdtp->tick_nohz_enabled_snap) { - if (rcu_cpu_has_callbacks(cpu, NULL)) + if (rcu_cpu_has_callbacks(NULL)) invoke_rcu_core(); /* force nohz to see update. */ rdtp->tick_nohz_enabled_snap = tne; return; diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 7b5741f..1f43560 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -585,7 +585,7 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, last_jiffies = jiffies; } while (read_seqretry(&jiffies_lock, seq)); - if (rcu_needs_cpu(cpu, &rcu_delta_jiffies) || + if (rcu_needs_cpu(&rcu_delta_jiffies) || arch_needs_cpu() || irq_work_needs_cpu()) { next_jiffies = last_jiffies + 1; delta_jiffies = 1; -- cgit v0.10.2 From 198bbf8127dd2dec6661622981a4e1f9b8f967b7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Oct 2014 15:03:43 -0700 Subject: rcu: Remove "cpu" argument to rcu_prepare_for_idle() The "cpu" argument to rcu_prepare_for_idle() is always the current CPU, so drop it. This in turn allows two of the uses of "cpu" in this function to be replaced with a this_cpu_ptr() and the third by smp_processor_id(), replacing that of the call to rcu_prepare_for_idle(). Again, the anticipated cross-CPU uses of these functions has been replaced by NO_HZ_FULL. Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index d678a98..8251938 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -531,7 +531,7 @@ static void rcu_eqs_enter_common(long long oldval, bool user) rdp = this_cpu_ptr(rsp->rda); do_nocb_deferred_wakeup(rdp); } - rcu_prepare_for_idle(smp_processor_id()); + rcu_prepare_for_idle(); /* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */ smp_mb__before_atomic(); /* See above. */ atomic_inc(&rdtp->dynticks); diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 238ac39..8bc3cac 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -580,7 +580,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, static void __init rcu_spawn_boost_kthreads(void); static void rcu_prepare_kthreads(int cpu); static void rcu_cleanup_after_idle(int cpu); -static void rcu_prepare_for_idle(int cpu); +static void rcu_prepare_for_idle(void); static void rcu_idle_count_callbacks_posted(void); static void print_cpu_stall_info_begin(void); static void print_cpu_stall_info(struct rcu_state *rsp, int cpu); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 1797b76..5e68ab5 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1531,7 +1531,7 @@ static void rcu_cleanup_after_idle(int cpu) * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n, * is nothing. */ -static void rcu_prepare_for_idle(int cpu) +static void rcu_prepare_for_idle(void) { } @@ -1666,12 +1666,12 @@ int rcu_needs_cpu(unsigned long *dj) * * The caller must have disabled interrupts. */ -static void rcu_prepare_for_idle(int cpu) +static void rcu_prepare_for_idle(void) { #ifndef CONFIG_RCU_NOCB_CPU_ALL bool needwake; struct rcu_data *rdp; - struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu); + struct rcu_dynticks *rdtp = this_cpu_ptr(&rcu_dynticks); struct rcu_node *rnp; struct rcu_state *rsp; int tne; @@ -1688,7 +1688,7 @@ static void rcu_prepare_for_idle(int cpu) return; /* If this is a no-CBs CPU, no callbacks, just return. */ - if (rcu_is_nocb_cpu(cpu)) + if (rcu_is_nocb_cpu(smp_processor_id())) return; /* @@ -1712,7 +1712,7 @@ static void rcu_prepare_for_idle(int cpu) return; rdtp->last_accelerate = jiffies; for_each_rcu_flavor(rsp) { - rdp = per_cpu_ptr(rsp->rda, cpu); + rdp = this_cpu_ptr(rsp->rda); if (!*rdp->nxttail[RCU_DONE_TAIL]) continue; rnp = rdp->mynode; -- cgit v0.10.2 From 8fa7845df539105a8962c3173c866483da74ff6d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Oct 2014 15:07:37 -0700 Subject: rcu: Remove "cpu" argument to rcu_cleanup_after_idle() The "cpu" argument to rcu_cleanup_after_idle() is always the current CPU, so drop it. This moves the smp_processor_id() from the caller to rcu_cleanup_after_idle(), saving argument-passing overhead. Again, the anticipated cross-CPU uses of these functions has been replaced by NO_HZ_FULL. Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index 8251938..fa76fd3c 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -661,7 +661,7 @@ static void rcu_eqs_exit_common(long long oldval, int user) /* CPUs seeing atomic_inc() must see later RCU read-side crit sects */ smp_mb__after_atomic(); /* See above. */ WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1)); - rcu_cleanup_after_idle(smp_processor_id()); + rcu_cleanup_after_idle(); trace_rcu_dyntick(TPS("End"), oldval, rdtp->dynticks_nesting); if (!user && !is_idle_task(current)) { struct task_struct *idle __maybe_unused = diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 8bc3cac..36c3039 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -579,7 +579,7 @@ static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, #endif /* #ifdef CONFIG_RCU_BOOST */ static void __init rcu_spawn_boost_kthreads(void); static void rcu_prepare_kthreads(int cpu); -static void rcu_cleanup_after_idle(int cpu); +static void rcu_cleanup_after_idle(void); static void rcu_prepare_for_idle(void); static void rcu_idle_count_callbacks_posted(void); static void print_cpu_stall_info_begin(void); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 5e68ab5..2443282 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1523,7 +1523,7 @@ int rcu_needs_cpu(unsigned long *delta_jiffies) * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up * after it. */ -static void rcu_cleanup_after_idle(int cpu) +static void rcu_cleanup_after_idle(void) { } @@ -1731,10 +1731,10 @@ static void rcu_prepare_for_idle(void) * any grace periods that elapsed while the CPU was idle, and if any * callbacks are now ready to invoke, initiate invocation. */ -static void rcu_cleanup_after_idle(int cpu) +static void rcu_cleanup_after_idle(void) { #ifndef CONFIG_RCU_NOCB_CPU_ALL - if (rcu_is_nocb_cpu(cpu)) + if (rcu_is_nocb_cpu(smp_processor_id())) return; if (rcu_try_advance_all_cbs()) invoke_rcu_core(); -- cgit v0.10.2 From 62db99f4783ea34531fc344bd8c539d5d186f24d Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 22 Oct 2014 14:51:49 -0700 Subject: cpu: Avoid puts_pending overflow A long string of get_online_cpus() with each followed by a put_online_cpu() that fails to acquire cpu_hotplug.lock can result in overflow of the cpu_hotplug.puts_pending counter. Although this is perhaps improbably, a system with absolutely no CPU-hotplug operations will have an arbitrarily long time in which this overflow could occur. This commit therefore adds overflow checks to get_online_cpus() and try_get_online_cpus(). Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/kernel/cpu.c b/kernel/cpu.c index 90a3d01..5d22023 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -86,6 +86,16 @@ static struct { #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) +static void apply_puts_pending(int max) +{ + int delta; + + if (atomic_read(&cpu_hotplug.puts_pending) >= max) { + delta = atomic_xchg(&cpu_hotplug.puts_pending, 0); + cpu_hotplug.refcount -= delta; + } +} + void get_online_cpus(void) { might_sleep(); @@ -93,6 +103,7 @@ void get_online_cpus(void) return; cpuhp_lock_acquire_read(); mutex_lock(&cpu_hotplug.lock); + apply_puts_pending(65536); cpu_hotplug.refcount++; mutex_unlock(&cpu_hotplug.lock); } @@ -105,6 +116,7 @@ bool try_get_online_cpus(void) if (!mutex_trylock(&cpu_hotplug.lock)) return false; cpuhp_lock_acquire_tryread(); + apply_puts_pending(65536); cpu_hotplug.refcount++; mutex_unlock(&cpu_hotplug.lock); return true; @@ -161,12 +173,7 @@ void cpu_hotplug_begin(void) cpuhp_lock_acquire(); for (;;) { mutex_lock(&cpu_hotplug.lock); - if (atomic_read(&cpu_hotplug.puts_pending)) { - int delta; - - delta = atomic_xchg(&cpu_hotplug.puts_pending, 0); - cpu_hotplug.refcount -= delta; - } + apply_puts_pending(1); if (likely(!cpu_hotplug.refcount)) break; __set_current_state(TASK_UNINTERRUPTIBLE); -- cgit v0.10.2 From 74860feed5ed570659e0f3852dd945be5b046038 Mon Sep 17 00:00:00 2001 From: Pranith Kumar Date: Fri, 19 Sep 2014 11:34:09 -0400 Subject: documentation: Document RCU self test boot params Document the RCU self test boot parameters in kernel-parameters.txt. Signed-off-by: Pranith Kumar Signed-off-by: Paul E. McKenney diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 7dbe5ec..4e2ae0e 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -3071,6 +3071,15 @@ bytes respectively. Such letter suffixes can also be entirely omitted. messages. Disable with a value less than or equal to zero. + rcupdate.rcu_self_test= [KNL] + Run the RCU early boot self tests + + rcupdate.rcu_self_test_bh= [KNL] + Run the RCU bh early boot self tests + + rcupdate.rcu_self_test_sched= [KNL] + Run the RCU sched early boot self tests + rdinit= [KNL] Format: Run specified binary instead of /init from the ramdisk, -- cgit v0.10.2 From 8b19d1dead8413442ba0ff0b4e19b08f69d2f1b7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 12 Oct 2014 07:55:47 -0700 Subject: documentation: Additional restriction for control dependencies Short-circuit booleans are not defences against compilers breaking your intended control dependencies. Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt index 22a969c..1073e01 100644 --- a/Documentation/memory-barriers.txt +++ b/Documentation/memory-barriers.txt @@ -694,6 +694,24 @@ Please note once again that the stores to 'b' differ. If they were identical, as noted earlier, the compiler could pull this store outside of the 'if' statement. +You must also be careful not to rely too much on boolean short-circuit +evaluation. Consider this example: + + q = ACCESS_ONCE(a); + if (a || 1 > 0) + ACCESS_ONCE(b) = 1; + +Because the second condition is always true, the compiler can transform +this example as following, defeating control dependency: + + q = ACCESS_ONCE(a); + ACCESS_ONCE(b) = 1; + +This example underscores the need to ensure that the compiler cannot +out-guess your code. More generally, although ACCESS_ONCE() does force +the compiler to actually emit code for a given load, it does not force +the compiler to use the results. + Finally, control dependencies do -not- provide transitivity. This is demonstrated by two related examples, with the initial values of x and y both being zero: -- cgit v0.10.2 From 1f7870dd8729c64b8472b42440811e7ff94d16a4 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sun, 19 Oct 2014 12:05:22 -0700 Subject: documentation: Add atomic_long_t to atomic_ops.txt Signed-off-by: Paul E. McKenney Reviewed-by: Pranith Kumar diff --git a/Documentation/atomic_ops.txt b/Documentation/atomic_ops.txt index 68542fe..183e41b 100644 --- a/Documentation/atomic_ops.txt +++ b/Documentation/atomic_ops.txt @@ -7,12 +7,13 @@ maintainers on how to implement atomic counter, bitops, and spinlock interfaces properly. - The atomic_t type should be defined as a signed integer. -Also, it should be made opaque such that any kind of cast to a normal -C integer type will fail. Something like the following should -suffice: + The atomic_t type should be defined as a signed integer and +the atomic_long_t type as a signed long integer. Also, they should +be made opaque such that any kind of cast to a normal C integer type +will fail. Something like the following should suffice: typedef struct { int counter; } atomic_t; + typedef struct { long counter; } atomic_long_t; Historically, counter has been declared volatile. This is now discouraged. See Documentation/volatile-considered-harmful.txt for the complete rationale. @@ -37,6 +38,9 @@ initializer is used before runtime. If the initializer is used at runtime, a proper implicit or explicit read memory barrier is needed before reading the value with atomic_read from another thread. +As with all of the atomic_ interfaces, replace the leading "atomic_" +with "atomic_long_" to operate on atomic_long_t. + The second interface can be used at runtime, as in: struct foo { atomic_t counter; }; -- cgit v0.10.2 From 8ab8b3e1837fc580b30263ed3c44dc34798714d9 Mon Sep 17 00:00:00 2001 From: Pranith Kumar Date: Tue, 2 Sep 2014 23:34:29 -0400 Subject: documentation: memory-barriers.txt: Correct example for reorderings Correct the example of memory orderings in memory-barriers.txt Commit 615cc2c9cf95 "Documentation/memory-barriers.txt: fix important typo re memory barriers" changed the assignment to x and y. Change the rest of the example to match this change. Reported-by: Ganesh Rapolu Signed-off-by: Pranith Kumar Signed-off-by: Paul E. McKenney diff --git a/Documentation/memory-barriers.txt b/Documentation/memory-barriers.txt index 1073e01..f7fa635 100644 --- a/Documentation/memory-barriers.txt +++ b/Documentation/memory-barriers.txt @@ -121,22 +121,22 @@ For example, consider the following sequence of events: The set of accesses as seen by the memory system in the middle can be arranged in 24 different combinations: - STORE A=3, STORE B=4, x=LOAD A->3, y=LOAD B->4 - STORE A=3, STORE B=4, y=LOAD B->4, x=LOAD A->3 - STORE A=3, x=LOAD A->3, STORE B=4, y=LOAD B->4 - STORE A=3, x=LOAD A->3, y=LOAD B->2, STORE B=4 - STORE A=3, y=LOAD B->2, STORE B=4, x=LOAD A->3 - STORE A=3, y=LOAD B->2, x=LOAD A->3, STORE B=4 - STORE B=4, STORE A=3, x=LOAD A->3, y=LOAD B->4 + STORE A=3, STORE B=4, y=LOAD A->3, x=LOAD B->4 + STORE A=3, STORE B=4, x=LOAD B->4, y=LOAD A->3 + STORE A=3, y=LOAD A->3, STORE B=4, x=LOAD B->4 + STORE A=3, y=LOAD A->3, x=LOAD B->2, STORE B=4 + STORE A=3, x=LOAD B->2, STORE B=4, y=LOAD A->3 + STORE A=3, x=LOAD B->2, y=LOAD A->3, STORE B=4 + STORE B=4, STORE A=3, y=LOAD A->3, x=LOAD B->4 STORE B=4, ... ... and can thus result in four different combinations of values: - x == 1, y == 2 - x == 1, y == 4 - x == 3, y == 2 - x == 3, y == 4 + x == 2, y == 1 + x == 2, y == 3 + x == 4, y == 1 + x == 4, y == 3 Furthermore, the stores committed by a CPU to the memory system may not be -- cgit v0.10.2 From 1a6c9b2675460718f819def9a272cca35575eeb7 Mon Sep 17 00:00:00 2001 From: Pranith Kumar Date: Thu, 25 Sep 2014 14:03:34 -0400 Subject: rcu: Add sparse check for RCU_INIT_POINTER() Add a sparse check when RCU_INIT_POINTER() is used to assign a non __rcu annotated pointer. Signed-off-by: Pranith Kumar Signed-off-by: Paul E. McKenney diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index a4a819f..a033d8b 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1047,6 +1047,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void) */ #define RCU_INIT_POINTER(p, v) \ do { \ + rcu_dereference_sparse(p, __rcu); \ p = RCU_INITIALIZER(v); \ } while (0) -- cgit v0.10.2 From b6331ae8afe4118884c4b6e14a213758d88422b2 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Sat, 4 Oct 2014 03:43:41 -0700 Subject: rcu: Optimize cond_resched_rcu_qs() The current implementation of cond_resched_rcu_qs() can invoke rcu_note_voluntary_context_switch() twice in the should_resched() case, once via the call to __schedule() and once directly. However, as noted by Joe Lawrence in a patch to the team subsystem, cond_resched() returns an indication as to whether or not the call to __schedule() actually happened. This commit therefore changes cond_resched_rcu_qs() so as to invoke rcu_note_voluntary_context_switch() only when __schedule() was not called. Signed-off-by: Paul E. McKenney diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index a033d8b..36ea3ba 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -348,8 +348,8 @@ extern struct srcu_struct tasks_rcu_exit_srcu; */ #define cond_resched_rcu_qs() \ do { \ - rcu_note_voluntary_context_switch(current); \ - cond_resched(); \ + if (!cond_resched()) \ + rcu_note_voluntary_context_switch(current); \ } while (0) #if defined(CONFIG_DEBUG_LOCK_ALLOC) || defined(CONFIG_RCU_TRACE) || defined(CONFIG_SMP) -- cgit v0.10.2 From ce36f2f3eb6613a73bc6f3a5256bde7dd3f95710 Mon Sep 17 00:00:00 2001 From: Oleg Nesterov Date: Sun, 28 Sep 2014 23:44:21 +0200 Subject: rcu: More info about potential deadlocks with rcu_read_unlock() The comment above rcu_read_unlock() explains the potential deadlock if the caller holds one of the locks taken by rt_mutex_unlock() paths, but it is not clear from this documentation that any lock which can be taken from interrupt can lead to deadlock as well and we need to take rt_mutex_lock() into account too. The problem is that rt_mutex_lock() takes wait_lock without disabling irqs, and thus an interrupt taking some LOCK can obviously race with rcu_read_unlock_special() called with the same LOCK held. Signed-off-by: Oleg Nesterov Signed-off-by: Paul E. McKenney diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 36ea3ba..ae6942a 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -887,7 +887,9 @@ static inline void rcu_read_lock(void) * Unfortunately, this function acquires the scheduler's runqueue and * priority-inheritance spinlocks. This means that deadlock could result * if the caller of rcu_read_unlock() already holds one of these locks or - * any lock that is ever acquired while holding them. + * any lock that is ever acquired while holding them; or any lock which + * can be taken from interrupt context because rcu_boost()->rt_mutex_lock() + * does not disable irqs while taking ->wait_lock. * * That said, RCU readers are never priority boosted unless they were * preempted. Therefore, one way to avoid deadlock is to make sure -- cgit v0.10.2 From 60ced4950c5059eff3f03027926eb5384f9923e1 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 27 Oct 2014 16:04:35 -0700 Subject: rcu: Fix FIXME in rcu_tasks_kthread() This commit affines rcu_tasks_kthread() to the housekeeping CPUs in CONFIG_NO_HZ_FULL builds. This is just a default, so systems administrators are free to put this kthread somewhere else if they wish. Signed-off-by: Paul E. McKenney diff --git a/kernel/rcu/update.c b/kernel/rcu/update.c index 3ef8ba5..8a39e68 100644 --- a/kernel/rcu/update.c +++ b/kernel/rcu/update.c @@ -531,7 +531,8 @@ static int __noreturn rcu_tasks_kthread(void *arg) struct rcu_head *next; LIST_HEAD(rcu_tasks_holdouts); - /* FIXME: Add housekeeping affinity. */ + /* Run on housekeeping CPUs by default. Sysadm can move if desired. */ + housekeeping_affine(current); /* * Each pass through the following loop makes one check for -- cgit v0.10.2