From e075867681ca9b8c0b8823e24d0fb4ce3b4f2655 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 10 Oct 2014 02:44:01 +0200 Subject: jiffies: Remove HZ > USEC_PER_SEC special case HZ never goes much further 1000 and a bit. And if we ever reach one tick per microsecond, we might be having a problem. Lets stop maintaining this special case, just leave a paranoid check. Reviewed-by: Rik van Riel Cc: Christoph Lameter Cc: Ingo Molnar Cc; John Stultz Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index 535fd3b..7c6febe 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -363,18 +363,11 @@ static inline unsigned long msecs_to_jiffies(const unsigned int m) } extern unsigned long __usecs_to_jiffies(const unsigned int u); -#if HZ <= USEC_PER_SEC && !(USEC_PER_SEC % HZ) +#if !(USEC_PER_SEC % HZ) static inline unsigned long _usecs_to_jiffies(const unsigned int u) { return (u + (USEC_PER_SEC / HZ) - 1) / (USEC_PER_SEC / HZ); } -#elif HZ > USEC_PER_SEC && !(HZ % USEC_PER_SEC) -static inline unsigned long _usecs_to_jiffies(const unsigned int u) -{ - return u * (HZ / USEC_PER_SEC); -} -static inline unsigned long _usecs_to_jiffies(const unsigned int u) -{ #else static inline unsigned long _usecs_to_jiffies(const unsigned int u) { diff --git a/kernel/time/time.c b/kernel/time/time.c index 85d5bb1..ad1bf23 100644 --- a/kernel/time/time.c +++ b/kernel/time/time.c @@ -268,10 +268,14 @@ EXPORT_SYMBOL(jiffies_to_msecs); unsigned int jiffies_to_usecs(const unsigned long j) { -#if HZ <= USEC_PER_SEC && !(USEC_PER_SEC % HZ) + /* + * Hz usually doesn't go much further MSEC_PER_SEC. + * jiffies_to_usecs() and usecs_to_jiffies() depend on that. + */ + BUILD_BUG_ON(HZ > USEC_PER_SEC); + +#if !(USEC_PER_SEC % HZ) return (USEC_PER_SEC / HZ) * j; -#elif HZ > USEC_PER_SEC && !(HZ % USEC_PER_SEC) - return (j + (HZ / USEC_PER_SEC) - 1)/(HZ / USEC_PER_SEC); #else # if BITS_PER_LONG == 32 return (HZ_TO_USEC_MUL32 * j) >> HZ_TO_USEC_SHR32; -- cgit v0.10.2 From 031a7f456adecaff692bc040b31a7d2262b4ee56 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 27 Oct 2014 17:16:04 +0100 Subject: apm32: Fix cputime == jiffies assumption That code wrongly assumes that cputime_t wraps jiffies_t. Lets use the correct accessors/mutators. No real harm now as that code can't be used with full dynticks. Reviewed-by: Rik van Riel Cc: Christoph Lameter Cc: Ingo Molnar Cc; John Stultz Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index 927ec92..052c9c3 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c @@ -919,7 +919,7 @@ recalc: } else if (jiffies_since_last_check > idle_period) { unsigned int idle_percentage; - idle_percentage = stime - last_stime; + idle_percentage = cputime_to_jiffies(stime - last_stime); idle_percentage *= 100; idle_percentage /= jiffies_since_last_check; use_apm_idle = (idle_percentage > idle_threshold); -- cgit v0.10.2 From 2019e8a3024bd8fcadd57891a88c838849216a5f Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 27 Oct 2014 17:20:35 +0100 Subject: alpha: Fix jiffies based cputime assumption That code wrongly assumes that cputime_t wraps jiffies_t. Lets use the correct accessors/mutators. In practice there should be no harm yet because alpha currently only support tick based cputime accounting which is always jiffies based. Reviewed-by: Rik van Riel Cc: Richard Henderson Cc: Ivan Kokshaysky Cc: Christoph Lameter Cc: Ingo Molnar Cc; John Stultz Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 36dc91a..6cc0816 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -1138,6 +1138,7 @@ SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru) { struct rusage32 r; cputime_t utime, stime; + unsigned long utime_jiffies, stime_jiffies; if (who != RUSAGE_SELF && who != RUSAGE_CHILDREN) return -EINVAL; @@ -1146,14 +1147,18 @@ SYSCALL_DEFINE2(osf_getrusage, int, who, struct rusage32 __user *, ru) switch (who) { case RUSAGE_SELF: task_cputime(current, &utime, &stime); - jiffies_to_timeval32(utime, &r.ru_utime); - jiffies_to_timeval32(stime, &r.ru_stime); + utime_jiffies = cputime_to_jiffies(utime); + stime_jiffies = cputime_to_jiffies(stime); + jiffies_to_timeval32(utime_jiffies, &r.ru_utime); + jiffies_to_timeval32(stime_jiffies, &r.ru_stime); r.ru_minflt = current->min_flt; r.ru_majflt = current->maj_flt; break; case RUSAGE_CHILDREN: - jiffies_to_timeval32(current->signal->cutime, &r.ru_utime); - jiffies_to_timeval32(current->signal->cstime, &r.ru_stime); + utime_jiffies = cputime_to_jiffies(current->signal->cutime); + stime_jiffies = cputime_to_jiffies(current->signal->cstime); + jiffies_to_timeval32(utime_jiffies, &r.ru_utime); + jiffies_to_timeval32(stime_jiffies, &r.ru_stime); r.ru_minflt = current->signal->cmin_flt; r.ru_majflt = current->signal->cmaj_flt; break; -- cgit v0.10.2 From 03f6199a359e460714b6bd08c10b566760f150a6 Mon Sep 17 00:00:00 2001 From: Chris Metcalf Date: Fri, 10 Jul 2015 15:37:25 -0400 Subject: nohz: Prevent tilegx network driver interrupts Normally the tilegx networking shim sends irqs to all the cores to distribute the load of processing incoming-packet interrupts, so that you can get to multiple Gb's of traffic inbound. However, in nohz_full mode we don't want to interrupt the nohz_full cores by default, so we limit the set of cores we use to only the online housekeeping cores. To make client code easier to read, we introduce a new nohz_full accessor, housekeeping_cpumask(), which returns a pointer to the housekeeping_mask if nohz_full is enabled, and otherwise returns the cpu_possible_mask. Signed-off-by: Chris Metcalf Cc: Christoph Lameter Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/drivers/net/ethernet/tile/tilegx.c b/drivers/net/ethernet/tile/tilegx.c index a3f7610..0a15acc 100644 --- a/drivers/net/ethernet/tile/tilegx.c +++ b/drivers/net/ethernet/tile/tilegx.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -2273,7 +2274,8 @@ static int __init tile_net_init_module(void) tile_net_dev_init(name, mac); if (!network_cpus_init()) - network_cpus_map = *cpu_online_mask; + cpumask_and(&network_cpus_map, housekeeping_cpumask(), + cpu_online_mask); return 0; } diff --git a/include/linux/tick.h b/include/linux/tick.h index edbfc9a..1ca93f2 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -163,6 +163,15 @@ static inline void tick_nohz_full_kick_all(void) { } static inline void __tick_nohz_task_switch(struct task_struct *tsk) { } #endif +static inline const struct cpumask *housekeeping_cpumask(void) +{ +#ifdef CONFIG_NO_HZ_FULL + if (tick_nohz_full_enabled()) + return housekeeping_mask; +#endif + return cpu_possible_mask; +} + static inline bool is_housekeeping_cpu(int cpu) { #ifdef CONFIG_NO_HZ_FULL -- cgit v0.10.2 From 594493594373862ed2a7f91d88a5a2670742faa6 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 27 May 2015 15:42:42 +0200 Subject: nohz: Remove idle task special case On nohz full early days, idle dynticks and full dynticks weren't well integrated and we couldn't risk full dynticks calls on idle without risking messing up tick idle statistics. This is why we prevented such thing to happen. Nowadays full dynticks and idle dynticks are better integrated and interact without known issue. So lets remove that. Reviewed-by: Rik van Riel Cc: Christoph Lameter Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index c792429..d6c8eff 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -208,10 +208,8 @@ void __tick_nohz_full_check(void) struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); if (tick_nohz_full_cpu(smp_processor_id())) { - if (ts->tick_stopped && !is_idle_task(current)) { - if (!can_stop_full_tick()) - tick_nohz_restart_sched_tick(ts, ktime_get()); - } + if (ts->tick_stopped && !can_stop_full_tick()) + tick_nohz_restart_sched_tick(ts, ktime_get()); } } @@ -710,7 +708,7 @@ static void tick_nohz_full_stop_tick(struct tick_sched *ts) #ifdef CONFIG_NO_HZ_FULL int cpu = smp_processor_id(); - if (!tick_nohz_full_cpu(cpu) || is_idle_task(current)) + if (!tick_nohz_full_cpu(cpu)) return; if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) -- cgit v0.10.2 From 73738a95d00467812664b7f86ba3052f5faf96d7 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Wed, 27 May 2015 19:22:08 +0200 Subject: nohz: Restart nohz full tick from irq exit Restart the tick when necessary from the irq exit path. It makes nohz full more flexible, simplify the related IPIs and doesn't bring significant overhead on irq exit. In a longer term view, it will allow us to piggyback the nohz kick on the scheduler IPI in the future instead of sending a dedicated IPI that often doubles the scheduler IPI on task wakeup. This will require more changes though including careful review of resched_curr() callers to include nohz full needs. Reviewed-by: Rik van Riel Cc: Christoph Lameter Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/include/linux/tick.h b/include/linux/tick.h index 1ca93f2..7d35b0f 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -147,7 +147,6 @@ static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) cpumask_or(mask, mask, tick_nohz_full_mask); } -extern void __tick_nohz_full_check(void); extern void tick_nohz_full_kick(void); extern void tick_nohz_full_kick_cpu(int cpu); extern void tick_nohz_full_kick_all(void); @@ -156,7 +155,6 @@ extern void __tick_nohz_task_switch(struct task_struct *tsk); static inline bool tick_nohz_full_enabled(void) { return false; } static inline bool tick_nohz_full_cpu(int cpu) { return false; } static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { } -static inline void __tick_nohz_full_check(void) { } static inline void tick_nohz_full_kick_cpu(int cpu) { } static inline void tick_nohz_full_kick(void) { } static inline void tick_nohz_full_kick_all(void) { } @@ -190,12 +188,6 @@ static inline void housekeeping_affine(struct task_struct *t) #endif } -static inline void tick_nohz_full_check(void) -{ - if (tick_nohz_full_enabled()) - __tick_nohz_full_check(); -} - static inline void tick_nohz_task_switch(struct task_struct *tsk) { if (tick_nohz_full_enabled()) diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index d6c8eff..a06cd4a 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -197,25 +197,9 @@ static bool can_stop_full_tick(void) return true; } -static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now); - -/* - * Re-evaluate the need for the tick on the current CPU - * and restart it if necessary. - */ -void __tick_nohz_full_check(void) -{ - struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - - if (tick_nohz_full_cpu(smp_processor_id())) { - if (ts->tick_stopped && !can_stop_full_tick()) - tick_nohz_restart_sched_tick(ts, ktime_get()); - } -} - static void nohz_full_kick_work_func(struct irq_work *work) { - __tick_nohz_full_check(); + /* Empty, the tick restart happens on tick_nohz_irq_exit() */ } static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = { @@ -250,7 +234,7 @@ void tick_nohz_full_kick_cpu(int cpu) static void nohz_full_kick_ipi(void *info) { - __tick_nohz_full_check(); + /* Empty, the tick restart happens on tick_nohz_irq_exit() */ } /* @@ -703,7 +687,9 @@ out: return tick; } -static void tick_nohz_full_stop_tick(struct tick_sched *ts) +static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now); + +static void tick_nohz_full_update_tick(struct tick_sched *ts) { #ifdef CONFIG_NO_HZ_FULL int cpu = smp_processor_id(); @@ -714,10 +700,10 @@ static void tick_nohz_full_stop_tick(struct tick_sched *ts) if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) return; - if (!can_stop_full_tick()) - return; - - tick_nohz_stop_sched_tick(ts, ktime_get(), cpu); + if (can_stop_full_tick()) + tick_nohz_stop_sched_tick(ts, ktime_get(), cpu); + else if (ts->tick_stopped) + tick_nohz_restart_sched_tick(ts, ktime_get()); #endif } @@ -847,7 +833,7 @@ void tick_nohz_irq_exit(void) if (ts->inidle) __tick_nohz_idle_enter(ts); else - tick_nohz_full_stop_tick(ts); + tick_nohz_full_update_tick(ts); } /** -- cgit v0.10.2 From 59d2c7ca492d7a7093755e4108390e4dac8b6365 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Fri, 29 May 2015 14:42:15 +0200 Subject: nohz: Move tick_nohz_restart_sched_tick() above its users Fix the function declaration/definition dance. Reviewed-by: Rik van Riel Cc: Christoph Lameter Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index a06cd4a..6b0d14d 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -687,7 +687,22 @@ out: return tick; } -static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now); +static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) +{ + /* Update jiffies first */ + tick_do_update_jiffies64(now); + update_cpu_load_nohz(); + + calc_load_exit_idle(); + touch_softlockup_watchdog(); + /* + * Cancel the scheduled timer and restore the tick + */ + ts->tick_stopped = 0; + ts->idle_exittime = now; + + tick_nohz_restart(ts, now); +} static void tick_nohz_full_update_tick(struct tick_sched *ts) { @@ -848,23 +863,6 @@ ktime_t tick_nohz_get_sleep_length(void) return ts->sleep_length; } -static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) -{ - /* Update jiffies first */ - tick_do_update_jiffies64(now); - update_cpu_load_nohz(); - - calc_load_exit_idle(); - touch_softlockup_watchdog(); - /* - * Cancel the scheduled timer and restore the tick - */ - ts->tick_stopped = 0; - ts->idle_exittime = now; - - tick_nohz_restart(ts, now); -} - static void tick_nohz_account_idle_ticks(struct tick_sched *ts) { #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE -- cgit v0.10.2 From de734f89b67c2df30e35a09e7e56a3659e5b6ac6 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Thu, 11 Jun 2015 18:07:12 +0200 Subject: nohz: Remove useless argument on tick_nohz_task_switch() Leftover from early code. Cc: Christoph Lameter Cc: Ingo Molnar Cc: Peter Zijlstra Cc: Preeti U Murthy Cc: Rik van Riel Cc: Thomas Gleixner Cc: Viresh Kumar Signed-off-by: Frederic Weisbecker diff --git a/include/linux/tick.h b/include/linux/tick.h index 7d35b0f..48d901f 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -150,7 +150,7 @@ static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) extern void tick_nohz_full_kick(void); extern void tick_nohz_full_kick_cpu(int cpu); extern void tick_nohz_full_kick_all(void); -extern void __tick_nohz_task_switch(struct task_struct *tsk); +extern void __tick_nohz_task_switch(void); #else static inline bool tick_nohz_full_enabled(void) { return false; } static inline bool tick_nohz_full_cpu(int cpu) { return false; } @@ -158,7 +158,7 @@ static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { } static inline void tick_nohz_full_kick_cpu(int cpu) { } static inline void tick_nohz_full_kick(void) { } static inline void tick_nohz_full_kick_all(void) { } -static inline void __tick_nohz_task_switch(struct task_struct *tsk) { } +static inline void __tick_nohz_task_switch(void) { } #endif static inline const struct cpumask *housekeeping_cpumask(void) @@ -188,10 +188,10 @@ static inline void housekeeping_affine(struct task_struct *t) #endif } -static inline void tick_nohz_task_switch(struct task_struct *tsk) +static inline void tick_nohz_task_switch(void) { if (tick_nohz_full_enabled()) - __tick_nohz_task_switch(tsk); + __tick_nohz_task_switch(); } #endif diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 78b4bad10..4d34035 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2489,7 +2489,7 @@ static struct rq *finish_task_switch(struct task_struct *prev) put_task_struct(prev); } - tick_nohz_task_switch(current); + tick_nohz_task_switch(); return rq; } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 6b0d14d..3319e16 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -258,7 +258,7 @@ void tick_nohz_full_kick_all(void) * It might need the tick due to per task/process properties: * perf events, posix cpu timers, ... */ -void __tick_nohz_task_switch(struct task_struct *tsk) +void __tick_nohz_task_switch(void) { unsigned long flags; -- cgit v0.10.2