summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--drivers/cpufreq/cpufreq.c25
-rw-r--r--drivers/cpufreq/cpufreq_governor.c2
-rw-r--r--drivers/cpufreq/intel_pstate.c4
3 files changed, 20 insertions, 11 deletions
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index dd568aa..6eca12a 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -115,12 +115,15 @@ static DEFINE_PER_CPU(struct update_util_data *, cpufreq_update_util_data);
* to call from cpufreq_update_util(). That function will be called from an RCU
* read-side critical section, so it must not sleep.
*
- * Callers must use RCU callbacks to free any memory that might be accessed
- * via the old update_util_data pointer or invoke synchronize_rcu() right after
- * this function to avoid use-after-free.
+ * Callers must use RCU-sched callbacks to free any memory that might be
+ * accessed via the old update_util_data pointer or invoke synchronize_sched()
+ * right after this function to avoid use-after-free.
*/
void cpufreq_set_update_util_data(int cpu, struct update_util_data *data)
{
+ if (WARN_ON(data && !data->func))
+ return;
+
rcu_assign_pointer(per_cpu(cpufreq_update_util_data, cpu), data);
}
EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data);
@@ -133,18 +136,24 @@ EXPORT_SYMBOL_GPL(cpufreq_set_update_util_data);
*
* This function is called by the scheduler on every invocation of
* update_load_avg() on the CPU whose utilization is being updated.
+ *
+ * It can only be called from RCU-sched read-side critical sections.
*/
void cpufreq_update_util(u64 time, unsigned long util, unsigned long max)
{
struct update_util_data *data;
- rcu_read_lock();
+#ifdef CONFIG_LOCKDEP
+ WARN_ON(debug_locks && !rcu_read_lock_sched_held());
+#endif
- data = rcu_dereference(*this_cpu_ptr(&cpufreq_update_util_data));
- if (data && data->func)
+ data = rcu_dereference_sched(*this_cpu_ptr(&cpufreq_update_util_data));
+ /*
+ * If this isn't inside of an RCU-sched read-side critical section, data
+ * may become NULL after the check below.
+ */
+ if (data)
data->func(data, time, util, max);
-
- rcu_read_unlock();
}
/* Flag to suspend/resume CPUFreq governors */
diff --git a/drivers/cpufreq/cpufreq_governor.c b/drivers/cpufreq/cpufreq_governor.c
index 70079e2..db46190 100644
--- a/drivers/cpufreq/cpufreq_governor.c
+++ b/drivers/cpufreq/cpufreq_governor.c
@@ -280,7 +280,7 @@ static inline void gov_clear_update_util(struct cpufreq_policy *policy)
for_each_cpu(i, policy->cpus)
cpufreq_set_update_util_data(i, NULL);
- synchronize_rcu();
+ synchronize_sched();
}
static void gov_cancel_work(struct cpufreq_policy *policy)
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index f4d85c2..2165d2b 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -1168,7 +1168,7 @@ static void intel_pstate_stop_cpu(struct cpufreq_policy *policy)
pr_debug("intel_pstate: CPU %d exiting\n", cpu_num);
cpufreq_set_update_util_data(cpu_num, NULL);
- synchronize_rcu();
+ synchronize_sched();
if (hwp_active)
return;
@@ -1426,7 +1426,7 @@ out:
for_each_online_cpu(cpu) {
if (all_cpu_data[cpu]) {
cpufreq_set_update_util_data(cpu, NULL);
- synchronize_rcu();
+ synchronize_sched();
kfree(all_cpu_data[cpu]);
}
}