diff options
author | Scott Wood <scottwood@freescale.com> | 2014-04-07 23:49:35 (GMT) |
---|---|---|
committer | Scott Wood <scottwood@freescale.com> | 2014-04-07 23:49:35 (GMT) |
commit | 62b8c978ee6b8d135d9e7953221de58000dba986 (patch) | |
tree | 683b04b2e627f6710c22c151b23c8cc9a165315e /drivers/cpufreq/cpufreq.c | |
parent | 78fd82238d0e5716578c326404184a27ba67fd6e (diff) | |
download | linux-fsl-qoriq-62b8c978ee6b8d135d9e7953221de58000dba986.tar.xz |
Rewind v3.13-rc3+ (78fd82238d0e5716) to v3.12
Diffstat (limited to 'drivers/cpufreq/cpufreq.c')
-rw-r--r-- | drivers/cpufreq/cpufreq.c | 322 |
1 files changed, 147 insertions, 175 deletions
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 02d534d..04548f7 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -47,11 +47,49 @@ static LIST_HEAD(cpufreq_policy_list); static DEFINE_PER_CPU(char[CPUFREQ_NAME_LEN], cpufreq_cpu_governor); #endif -static inline bool has_target(void) -{ - return cpufreq_driver->target_index || cpufreq_driver->target; +/* + * cpu_policy_rwsem is a per CPU reader-writer semaphore designed to cure + * all cpufreq/hotplug/workqueue/etc related lock issues. + * + * The rules for this semaphore: + * - Any routine that wants to read from the policy structure will + * do a down_read on this semaphore. + * - Any routine that will write to the policy structure and/or may take away + * the policy altogether (eg. CPU hotplug), will hold this lock in write + * mode before doing so. + * + * Additional rules: + * - Governor routines that can be called in cpufreq hotplug path should not + * take this sem as top level hotplug notifier handler takes this. + * - Lock should not be held across + * __cpufreq_governor(data, CPUFREQ_GOV_STOP); + */ +static DEFINE_PER_CPU(struct rw_semaphore, cpu_policy_rwsem); + +#define lock_policy_rwsem(mode, cpu) \ +static int lock_policy_rwsem_##mode(int cpu) \ +{ \ + struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); \ + BUG_ON(!policy); \ + down_##mode(&per_cpu(cpu_policy_rwsem, policy->cpu)); \ + \ + return 0; \ +} + +lock_policy_rwsem(read, cpu); +lock_policy_rwsem(write, cpu); + +#define unlock_policy_rwsem(mode, cpu) \ +static void unlock_policy_rwsem_##mode(int cpu) \ +{ \ + struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); \ + BUG_ON(!policy); \ + up_##mode(&per_cpu(cpu_policy_rwsem, policy->cpu)); \ } +unlock_policy_rwsem(read, cpu); +unlock_policy_rwsem(write, cpu); + /* * rwsem to guarantee that cpufreq driver module doesn't unload during critical * sections @@ -97,7 +135,7 @@ static DEFINE_MUTEX(cpufreq_governor_mutex); bool have_governor_per_policy(void) { - return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY); + return cpufreq_driver->have_governor_per_policy; } EXPORT_SYMBOL_GPL(have_governor_per_policy); @@ -145,37 +183,6 @@ u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy) } EXPORT_SYMBOL_GPL(get_cpu_idle_time); -/* - * This is a generic cpufreq init() routine which can be used by cpufreq - * drivers of SMP systems. It will do following: - * - validate & show freq table passed - * - set policies transition latency - * - policy->cpus with all possible CPUs - */ -int cpufreq_generic_init(struct cpufreq_policy *policy, - struct cpufreq_frequency_table *table, - unsigned int transition_latency) -{ - int ret; - - ret = cpufreq_table_validate_and_show(policy, table); - if (ret) { - pr_err("%s: invalid frequency table: %d\n", __func__, ret); - return ret; - } - - policy->cpuinfo.transition_latency = transition_latency; - - /* - * The driver only supports the SMP configuartion where all processors - * share the clock and voltage and clock. - */ - cpumask_setall(policy->cpus); - - return 0; -} -EXPORT_SYMBOL_GPL(cpufreq_generic_init); - struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu) { struct cpufreq_policy *policy = NULL; @@ -356,7 +363,7 @@ static int cpufreq_parse_governor(char *str_governor, unsigned int *policy, *policy = CPUFREQ_POLICY_POWERSAVE; err = 0; } - } else if (has_target()) { + } else if (cpufreq_driver->target) { struct cpufreq_governor *t; mutex_lock(&cpufreq_governor_mutex); @@ -407,7 +414,7 @@ show_one(scaling_min_freq, min); show_one(scaling_max_freq, max); show_one(scaling_cur_freq, cur); -static int cpufreq_set_policy(struct cpufreq_policy *policy, +static int __cpufreq_set_policy(struct cpufreq_policy *policy, struct cpufreq_policy *new_policy); /** @@ -428,7 +435,7 @@ static ssize_t store_##file_name \ if (ret != 1) \ return -EINVAL; \ \ - ret = cpufreq_set_policy(policy, &new_policy); \ + ret = __cpufreq_set_policy(policy, &new_policy); \ policy->user_policy.object = policy->object; \ \ return ret ? ret : count; \ @@ -486,7 +493,11 @@ static ssize_t store_scaling_governor(struct cpufreq_policy *policy, &new_policy.governor)) return -EINVAL; - ret = cpufreq_set_policy(policy, &new_policy); + /* + * Do not use cpufreq_set_policy here or the user_policy.max + * will be wrongly overridden + */ + ret = __cpufreq_set_policy(policy, &new_policy); policy->user_policy.policy = policy->policy; policy->user_policy.governor = policy->governor; @@ -514,7 +525,7 @@ static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy, ssize_t i = 0; struct cpufreq_governor *t; - if (!has_target()) { + if (!cpufreq_driver->target) { i += sprintf(buf, "performance powersave"); goto out; } @@ -642,21 +653,24 @@ static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf) { struct cpufreq_policy *policy = to_policy(kobj); struct freq_attr *fattr = to_attr(attr); - ssize_t ret; + ssize_t ret = -EINVAL; if (!down_read_trylock(&cpufreq_rwsem)) - return -EINVAL; + goto exit; - down_read(&policy->rwsem); + if (lock_policy_rwsem_read(policy->cpu) < 0) + goto up_read; if (fattr->show) ret = fattr->show(policy, buf); else ret = -EIO; - up_read(&policy->rwsem); - up_read(&cpufreq_rwsem); + unlock_policy_rwsem_read(policy->cpu); +up_read: + up_read(&cpufreq_rwsem); +exit: return ret; } @@ -675,15 +689,17 @@ static ssize_t store(struct kobject *kobj, struct attribute *attr, if (!down_read_trylock(&cpufreq_rwsem)) goto unlock; - down_write(&policy->rwsem); + if (lock_policy_rwsem_write(policy->cpu) < 0) + goto up_read; if (fattr->store) ret = fattr->store(policy, buf, count); else ret = -EIO; - up_write(&policy->rwsem); + unlock_policy_rwsem_write(policy->cpu); +up_read: up_read(&cpufreq_rwsem); unlock: put_online_cpus(); @@ -799,7 +815,7 @@ static int cpufreq_add_dev_interface(struct cpufreq_policy *policy, if (ret) goto err_out_kobj_put; } - if (has_target()) { + if (cpufreq_driver->target) { ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr); if (ret) goto err_out_kobj_put; @@ -828,11 +844,11 @@ static void cpufreq_init_policy(struct cpufreq_policy *policy) int ret = 0; memcpy(&new_policy, policy, sizeof(*policy)); - /* assure that the starting sequence is run in cpufreq_set_policy */ + /* assure that the starting sequence is run in __cpufreq_set_policy */ policy->governor = NULL; /* set default policy */ - ret = cpufreq_set_policy(policy, &new_policy); + ret = __cpufreq_set_policy(policy, &new_policy); policy->user_policy.policy = policy->policy; policy->user_policy.governor = policy->governor; @@ -848,10 +864,10 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu, struct device *dev, bool frozen) { - int ret = 0; + int ret = 0, has_target = !!cpufreq_driver->target; unsigned long flags; - if (has_target()) { + if (has_target) { ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); if (ret) { pr_err("%s: Failed to stop governor\n", __func__); @@ -859,7 +875,7 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, } } - down_write(&policy->rwsem); + lock_policy_rwsem_write(policy->cpu); write_lock_irqsave(&cpufreq_driver_lock, flags); @@ -867,9 +883,9 @@ static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, per_cpu(cpufreq_cpu_data, cpu) = policy; write_unlock_irqrestore(&cpufreq_driver_lock, flags); - up_write(&policy->rwsem); + unlock_policy_rwsem_write(policy->cpu); - if (has_target()) { + if (has_target) { if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) || (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) { pr_err("%s: Failed to start governor\n", __func__); @@ -914,8 +930,6 @@ static struct cpufreq_policy *cpufreq_policy_alloc(void) goto err_free_cpumask; INIT_LIST_HEAD(&policy->policy_list); - init_rwsem(&policy->rwsem); - return policy; err_free_cpumask: @@ -935,17 +949,26 @@ static void cpufreq_policy_free(struct cpufreq_policy *policy) static void update_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu) { - if (WARN_ON(cpu == policy->cpu)) + if (cpu == policy->cpu) return; - down_write(&policy->rwsem); + /* + * Take direct locks as lock_policy_rwsem_write wouldn't work here. + * Also lock for last cpu is enough here as contention will happen only + * after policy->cpu is changed and after it is changed, other threads + * will try to acquire lock for new cpu. And policy is already updated + * by then. + */ + down_write(&per_cpu(cpu_policy_rwsem, policy->cpu)); policy->last_cpu = policy->cpu; policy->cpu = cpu; - up_write(&policy->rwsem); + up_write(&per_cpu(cpu_policy_rwsem, policy->last_cpu)); +#ifdef CONFIG_CPU_FREQ_TABLE cpufreq_frequency_table_update_policy_cpu(policy); +#endif blocking_notifier_call_chain(&cpufreq_policy_notifier_list, CPUFREQ_UPDATE_POLICY_CPU, policy); } @@ -1030,14 +1053,6 @@ static int __cpufreq_add_dev(struct device *dev, struct subsys_interface *sif, goto err_set_policy_cpu; } - if (cpufreq_driver->get) { - policy->cur = cpufreq_driver->get(policy->cpu); - if (!policy->cur) { - pr_err("%s: ->get() failed\n", __func__); - goto err_get_freq; - } - } - /* related cpus should atleast have policy->cpus */ cpumask_or(policy->related_cpus, policy->related_cpus, policy->cpus); @@ -1092,9 +1107,6 @@ err_out_unregister: per_cpu(cpufreq_cpu_data, j) = NULL; write_unlock_irqrestore(&cpufreq_driver_lock, flags); -err_get_freq: - if (cpufreq_driver->exit) - cpufreq_driver->exit(policy); err_set_policy_cpu: cpufreq_policy_free(policy); nomem_out: @@ -1135,9 +1147,9 @@ static int cpufreq_nominate_new_policy_cpu(struct cpufreq_policy *policy, if (ret) { pr_err("%s: Failed to move kobj: %d", __func__, ret); - down_write(&policy->rwsem); + WARN_ON(lock_policy_rwsem_write(old_cpu)); cpumask_set_cpu(old_cpu, policy->cpus); - up_write(&policy->rwsem); + unlock_policy_rwsem_write(old_cpu); ret = sysfs_create_link(&cpu_dev->kobj, &policy->kobj, "cpufreq"); @@ -1174,7 +1186,7 @@ static int __cpufreq_remove_dev_prepare(struct device *dev, return -EINVAL; } - if (has_target()) { + if (cpufreq_driver->target) { ret = __cpufreq_governor(policy, CPUFREQ_GOV_STOP); if (ret) { pr_err("%s: Failed to stop governor\n", __func__); @@ -1188,21 +1200,22 @@ static int __cpufreq_remove_dev_prepare(struct device *dev, policy->governor->name, CPUFREQ_NAME_LEN); #endif - down_read(&policy->rwsem); + lock_policy_rwsem_read(cpu); cpus = cpumask_weight(policy->cpus); - up_read(&policy->rwsem); + unlock_policy_rwsem_read(cpu); if (cpu != policy->cpu) { if (!frozen) sysfs_remove_link(&dev->kobj, "cpufreq"); } else if (cpus > 1) { + new_cpu = cpufreq_nominate_new_policy_cpu(policy, cpu, frozen); if (new_cpu >= 0) { update_policy_cpu(policy, new_cpu); if (!frozen) { - pr_debug("%s: policy Kobject moved to cpu: %d from: %d\n", - __func__, new_cpu, cpu); + pr_debug("%s: policy Kobject moved to cpu: %d " + "from: %d\n",__func__, new_cpu, cpu); } } } @@ -1230,16 +1243,16 @@ static int __cpufreq_remove_dev_finish(struct device *dev, return -EINVAL; } - down_write(&policy->rwsem); + WARN_ON(lock_policy_rwsem_write(cpu)); cpus = cpumask_weight(policy->cpus); if (cpus > 1) cpumask_clear_cpu(cpu, policy->cpus); - up_write(&policy->rwsem); + unlock_policy_rwsem_write(cpu); /* If cpu is last user of policy, free policy */ if (cpus == 1) { - if (has_target()) { + if (cpufreq_driver->target) { ret = __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); if (ret) { @@ -1250,10 +1263,10 @@ static int __cpufreq_remove_dev_finish(struct device *dev, } if (!frozen) { - down_read(&policy->rwsem); + lock_policy_rwsem_read(cpu); kobj = &policy->kobj; cmp = &policy->kobj_unregister; - up_read(&policy->rwsem); + unlock_policy_rwsem_read(cpu); kobject_put(kobj); /* @@ -1282,7 +1295,7 @@ static int __cpufreq_remove_dev_finish(struct device *dev, if (!frozen) cpufreq_policy_free(policy); } else { - if (has_target()) { + if (cpufreq_driver->target) { if ((ret = __cpufreq_governor(policy, CPUFREQ_GOV_START)) || (ret = __cpufreq_governor(policy, CPUFREQ_GOV_LIMITS))) { pr_err("%s: Failed to start governor\n", @@ -1297,26 +1310,38 @@ static int __cpufreq_remove_dev_finish(struct device *dev, } /** - * cpufreq_remove_dev - remove a CPU device + * __cpufreq_remove_dev - remove a CPU device * * Removes the cpufreq interface for a CPU device. + * Caller should already have policy_rwsem in write mode for this CPU. + * This routine frees the rwsem before returning. */ -static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) +static inline int __cpufreq_remove_dev(struct device *dev, + struct subsys_interface *sif, + bool frozen) { - unsigned int cpu = dev->id; int ret; - if (cpu_is_offline(cpu)) - return 0; - - ret = __cpufreq_remove_dev_prepare(dev, sif, false); + ret = __cpufreq_remove_dev_prepare(dev, sif, frozen); if (!ret) - ret = __cpufreq_remove_dev_finish(dev, sif, false); + ret = __cpufreq_remove_dev_finish(dev, sif, frozen); return ret; } +static int cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif) +{ + unsigned int cpu = dev->id; + int retval; + + if (cpu_is_offline(cpu)) + return 0; + + retval = __cpufreq_remove_dev(dev, sif, false); + return retval; +} + static void handle_update(struct work_struct *work) { struct cpufreq_policy *policy = @@ -1433,22 +1458,22 @@ static unsigned int __cpufreq_get(unsigned int cpu) */ unsigned int cpufreq_get(unsigned int cpu) { - struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu); unsigned int ret_freq = 0; if (cpufreq_disabled() || !cpufreq_driver) return -ENOENT; - BUG_ON(!policy); - if (!down_read_trylock(&cpufreq_rwsem)) return 0; - down_read(&policy->rwsem); + if (unlikely(lock_policy_rwsem_read(cpu))) + goto out_policy; ret_freq = __cpufreq_get(cpu); - up_read(&policy->rwsem); + unlock_policy_rwsem_read(cpu); + +out_policy: up_read(&cpufreq_rwsem); return ret_freq; @@ -1656,75 +1681,12 @@ int __cpufreq_driver_target(struct cpufreq_policy *policy, pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n", policy->cpu, target_freq, relation, old_target_freq); - /* - * This might look like a redundant call as we are checking it again - * after finding index. But it is left intentionally for cases where - * exactly same freq is called again and so we can save on few function - * calls. - */ if (target_freq == policy->cur) return 0; if (cpufreq_driver->target) retval = cpufreq_driver->target(policy, target_freq, relation); - else if (cpufreq_driver->target_index) { - struct cpufreq_frequency_table *freq_table; - struct cpufreq_freqs freqs; - bool notify; - int index; - - freq_table = cpufreq_frequency_get_table(policy->cpu); - if (unlikely(!freq_table)) { - pr_err("%s: Unable to find freq_table\n", __func__); - goto out; - } - - retval = cpufreq_frequency_table_target(policy, freq_table, - target_freq, relation, &index); - if (unlikely(retval)) { - pr_err("%s: Unable to find matching freq\n", __func__); - goto out; - } - - if (freq_table[index].frequency == policy->cur) { - retval = 0; - goto out; - } - - notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION); - - if (notify) { - freqs.old = policy->cur; - freqs.new = freq_table[index].frequency; - freqs.flags = 0; - - pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n", - __func__, policy->cpu, freqs.old, - freqs.new); - - cpufreq_notify_transition(policy, &freqs, - CPUFREQ_PRECHANGE); - } - - retval = cpufreq_driver->target_index(policy, index); - if (retval) - pr_err("%s: Failed to change cpu frequency: %d\n", - __func__, retval); - - if (notify) { - /* - * Notify with old freq in case we failed to change - * frequency - */ - if (retval) - freqs.new = freqs.old; - - cpufreq_notify_transition(policy, &freqs, - CPUFREQ_POSTCHANGE); - } - } -out: return retval; } EXPORT_SYMBOL_GPL(__cpufreq_driver_target); @@ -1735,12 +1697,14 @@ int cpufreq_driver_target(struct cpufreq_policy *policy, { int ret = -EINVAL; - down_write(&policy->rwsem); + if (unlikely(lock_policy_rwsem_write(policy->cpu))) + goto fail; ret = __cpufreq_driver_target(policy, target_freq, relation); - up_write(&policy->rwsem); + unlock_policy_rwsem_write(policy->cpu); +fail: return ret; } EXPORT_SYMBOL_GPL(cpufreq_driver_target); @@ -1907,10 +1871,10 @@ int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu) EXPORT_SYMBOL(cpufreq_get_policy); /* - * policy : current policy. - * new_policy: policy to be set. + * data : current policy. + * policy : policy to be set. */ -static int cpufreq_set_policy(struct cpufreq_policy *policy, +static int __cpufreq_set_policy(struct cpufreq_policy *policy, struct cpufreq_policy *new_policy) { int ret = 0, failed = 1; @@ -1970,10 +1934,10 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, /* end old governor */ if (policy->governor) { __cpufreq_governor(policy, CPUFREQ_GOV_STOP); - up_write(&policy->rwsem); + unlock_policy_rwsem_write(new_policy->cpu); __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); - down_write(&policy->rwsem); + lock_policy_rwsem_write(new_policy->cpu); } /* start new governor */ @@ -1982,10 +1946,10 @@ static int cpufreq_set_policy(struct cpufreq_policy *policy, if (!__cpufreq_governor(policy, CPUFREQ_GOV_START)) { failed = 0; } else { - up_write(&policy->rwsem); + unlock_policy_rwsem_write(new_policy->cpu); __cpufreq_governor(policy, CPUFREQ_GOV_POLICY_EXIT); - down_write(&policy->rwsem); + lock_policy_rwsem_write(new_policy->cpu); } } @@ -2031,7 +1995,10 @@ int cpufreq_update_policy(unsigned int cpu) goto no_policy; } - down_write(&policy->rwsem); + if (unlikely(lock_policy_rwsem_write(cpu))) { + ret = -EINVAL; + goto fail; + } pr_debug("updating policy for CPU %u\n", cpu); memcpy(&new_policy, policy, sizeof(*policy)); @@ -2050,16 +2017,17 @@ int cpufreq_update_policy(unsigned int cpu) pr_debug("Driver did not initialize current freq"); policy->cur = new_policy.cur; } else { - if (policy->cur != new_policy.cur && has_target()) + if (policy->cur != new_policy.cur && cpufreq_driver->target) cpufreq_out_of_sync(cpu, policy->cur, new_policy.cur); } } - ret = cpufreq_set_policy(policy, &new_policy); + ret = __cpufreq_set_policy(policy, &new_policy); - up_write(&policy->rwsem); + unlock_policy_rwsem_write(cpu); +fail: cpufreq_cpu_put(policy); no_policy: return ret; @@ -2128,8 +2096,7 @@ int cpufreq_register_driver(struct cpufreq_driver *driver_data) return -ENODEV; if (!driver_data || !driver_data->verify || !driver_data->init || - !(driver_data->setpolicy || driver_data->target_index || - driver_data->target)) + ((!driver_data->setpolicy) && (!driver_data->target))) return -EINVAL; pr_debug("trying to register driver %s\n", driver_data->name); @@ -2216,9 +2183,14 @@ EXPORT_SYMBOL_GPL(cpufreq_unregister_driver); static int __init cpufreq_core_init(void) { + int cpu; + if (cpufreq_disabled()) return -ENODEV; + for_each_possible_cpu(cpu) + init_rwsem(&per_cpu(cpu_policy_rwsem, cpu)); + cpufreq_global_kobject = kobject_create(); BUG_ON(!cpufreq_global_kobject); register_syscore_ops(&cpufreq_syscore_ops); |