blob: ee23eaf20f35c67984adc8413cb2837a2ea72507 [file] [log] [blame]
Thomas Gleixnerd2912cb2019-06-04 10:11:33 +02001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/drivers/cpufreq/cpufreq.c
4 *
5 * Copyright (C) 2001 Russell King
6 * (C) 2002 - 2003 Dominik Brodowski <linux@brodo.de>
Viresh Kumarbb176f72013-06-19 14:19:33 +05307 * (C) 2013 Viresh Kumar <viresh.kumar@linaro.org>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008 *
Ashok Rajc32b6b82005-10-30 14:59:54 -08009 * Oct 2005 - Ashok Raj <ashok.raj@intel.com>
Dave Jones32ee8c32006-02-28 00:43:23 -050010 * Added handling for CPU hotplug
Dave Jones8ff69732006-03-05 03:37:23 -050011 * Feb 2006 - Jacob Shin <jacob.shin@amd.com>
12 * Fix handling for CPU hotplug -- affected CPUs
Linus Torvalds1da177e2005-04-16 15:20:36 -070013 */
14
Viresh Kumardb701152012-10-23 01:29:03 +020015#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
16
Viresh Kumar5ff0a262013-08-06 22:53:03 +053017#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070018#include <linux/cpufreq.h>
Amit Kucheria5c238a82019-01-30 10:52:01 +053019#include <linux/cpu_cooling.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/delay.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/device.h>
Viresh Kumar5ff0a262013-08-06 22:53:03 +053022#include <linux/init.h>
23#include <linux/kernel_stat.h>
24#include <linux/module.h>
akpm@osdl.org3fc54d32006-01-13 15:54:22 -080025#include <linux/mutex.h>
Viresh Kumar67d874c2019-07-08 16:27:52 +053026#include <linux/pm_qos.h>
Viresh Kumar5ff0a262013-08-06 22:53:03 +053027#include <linux/slab.h>
Viresh Kumar2f0aea92014-03-04 11:00:26 +080028#include <linux/suspend.h>
Doug Anderson90de2a42014-12-23 22:09:48 -080029#include <linux/syscore_ops.h>
Viresh Kumar5ff0a262013-08-06 22:53:03 +053030#include <linux/tick.h>
Thomas Renninger6f4f2722010-04-20 13:17:36 +020031#include <trace/events/power.h>
32
Viresh Kumarb4f06762015-01-27 14:06:08 +053033static LIST_HEAD(cpufreq_policy_list);
Viresh Kumarf9637352015-05-12 12:20:11 +053034
Viresh Kumarf9637352015-05-12 12:20:11 +053035/* Macros to iterate over CPU policies */
Eric Biggersfd7dc7e2016-02-21 12:53:12 -060036#define for_each_suitable_policy(__policy, __active) \
37 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list) \
38 if ((__active) == !policy_is_inactive(__policy))
Viresh Kumarf9637352015-05-12 12:20:11 +053039
40#define for_each_active_policy(__policy) \
41 for_each_suitable_policy(__policy, true)
42#define for_each_inactive_policy(__policy) \
43 for_each_suitable_policy(__policy, false)
44
45#define for_each_policy(__policy) \
Viresh Kumarb4f06762015-01-27 14:06:08 +053046 list_for_each_entry(__policy, &cpufreq_policy_list, policy_list)
47
Viresh Kumarf7b27062015-01-27 14:06:09 +053048/* Iterate over governors */
49static LIST_HEAD(cpufreq_governor_list);
50#define for_each_governor(__governor) \
51 list_for_each_entry(__governor, &cpufreq_governor_list, governor_list)
52
Linus Torvalds1da177e2005-04-16 15:20:36 -070053/**
Dave Jonescd878472006-08-11 17:59:28 -040054 * The "cpufreq driver" - the arch- or hardware-dependent low
Linus Torvalds1da177e2005-04-16 15:20:36 -070055 * level driver of CPUFreq support, and its spinlock. This lock
56 * also protects the cpufreq_cpu_data array.
57 */
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +020058static struct cpufreq_driver *cpufreq_driver;
Mike Travis7a6aedf2008-03-25 15:06:53 -070059static DEFINE_PER_CPU(struct cpufreq_policy *, cpufreq_cpu_data);
Viresh Kumarbb176f72013-06-19 14:19:33 +053060static DEFINE_RWLOCK(cpufreq_driver_lock);
Viresh Kumarbb176f72013-06-19 14:19:33 +053061
Viresh Kumar2f0aea92014-03-04 11:00:26 +080062/* Flag to suspend/resume CPUFreq governors */
63static bool cpufreq_suspended;
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +053065static inline bool has_target(void)
66{
67 return cpufreq_driver->target_index || cpufreq_driver->target;
68}
69
Linus Torvalds1da177e2005-04-16 15:20:36 -070070/* internal prototypes */
Viresh Kumard92d50a2015-01-02 12:34:29 +053071static unsigned int __cpufreq_get(struct cpufreq_policy *policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +020072static int cpufreq_init_governor(struct cpufreq_policy *policy);
73static void cpufreq_exit_governor(struct cpufreq_policy *policy);
Rafael J. Wysocki0a300762016-03-21 15:45:24 +010074static int cpufreq_start_governor(struct cpufreq_policy *policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +020075static void cpufreq_stop_governor(struct cpufreq_policy *policy);
76static void cpufreq_governor_limits(struct cpufreq_policy *policy);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +020077
Linus Torvalds1da177e2005-04-16 15:20:36 -070078/**
Dave Jones32ee8c32006-02-28 00:43:23 -050079 * Two notifier lists: the "policy" list is involved in the
80 * validation process for a new CPU frequency policy; the
Linus Torvalds1da177e2005-04-16 15:20:36 -070081 * "transition" list for kernel code that needs to handle
82 * changes to devices when the CPU clock speed changes.
83 * The mutex locks both lists.
84 */
Alan Sterne041c682006-03-27 01:16:30 -080085static BLOCKING_NOTIFIER_HEAD(cpufreq_policy_notifier_list);
Sebastian Andrzej Siewiorcc85de32018-05-25 12:19:58 +020086SRCU_NOTIFIER_HEAD_STATIC(cpufreq_transition_notifier_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -070087
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -040088static int off __read_mostly;
Viresh Kumarda584452012-10-26 00:51:32 +020089static int cpufreq_disabled(void)
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -040090{
91 return off;
92}
93void disable_cpufreq(void)
94{
95 off = 1;
96}
Dave Jones29464f22009-01-18 01:37:11 -050097static DEFINE_MUTEX(cpufreq_governor_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -070098
Viresh Kumar4d5dcc42013-03-27 15:58:58 +000099bool have_governor_per_policy(void)
100{
Viresh Kumar0b981e72013-10-02 14:13:18 +0530101 return !!(cpufreq_driver->flags & CPUFREQ_HAVE_GOVERNOR_PER_POLICY);
Viresh Kumar4d5dcc42013-03-27 15:58:58 +0000102}
Viresh Kumar3f869d62013-05-16 05:09:56 +0000103EXPORT_SYMBOL_GPL(have_governor_per_policy);
Viresh Kumar4d5dcc42013-03-27 15:58:58 +0000104
Viresh Kumar944e9a02013-05-16 05:09:57 +0000105struct kobject *get_governor_parent_kobj(struct cpufreq_policy *policy)
106{
107 if (have_governor_per_policy())
108 return &policy->kobj;
109 else
110 return cpufreq_global_kobject;
111}
112EXPORT_SYMBOL_GPL(get_governor_parent_kobj);
113
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000114static inline u64 get_cpu_idle_time_jiffy(unsigned int cpu, u64 *wall)
115{
Frederic Weisbecker57208212019-11-21 03:44:28 +0100116 struct kernel_cpustat kcpustat;
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000117 u64 cur_wall_time;
Frederic Weisbecker57208212019-11-21 03:44:28 +0100118 u64 idle_time;
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000119 u64 busy_time;
120
Frederic Weisbecker7fb13272017-01-31 04:09:19 +0100121 cur_wall_time = jiffies64_to_nsecs(get_jiffies_64());
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000122
Frederic Weisbecker57208212019-11-21 03:44:28 +0100123 kcpustat_cpu_fetch(&kcpustat, cpu);
124
125 busy_time = kcpustat.cpustat[CPUTIME_USER];
126 busy_time += kcpustat.cpustat[CPUTIME_SYSTEM];
127 busy_time += kcpustat.cpustat[CPUTIME_IRQ];
128 busy_time += kcpustat.cpustat[CPUTIME_SOFTIRQ];
129 busy_time += kcpustat.cpustat[CPUTIME_STEAL];
130 busy_time += kcpustat.cpustat[CPUTIME_NICE];
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000131
132 idle_time = cur_wall_time - busy_time;
133 if (wall)
Frederic Weisbecker7fb13272017-01-31 04:09:19 +0100134 *wall = div_u64(cur_wall_time, NSEC_PER_USEC);
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000135
Frederic Weisbecker7fb13272017-01-31 04:09:19 +0100136 return div_u64(idle_time, NSEC_PER_USEC);
Viresh Kumar72a4ce32013-05-17 11:26:32 +0000137}
138
139u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy)
140{
141 u64 idle_time = get_cpu_idle_time_us(cpu, io_busy ? wall : NULL);
142
143 if (idle_time == -1ULL)
144 return get_cpu_idle_time_jiffy(cpu, wall);
145 else if (!io_busy)
146 idle_time += get_cpu_iowait_time_us(cpu, wall);
147
148 return idle_time;
149}
150EXPORT_SYMBOL_GPL(get_cpu_idle_time);
151
Dietmar Eggemanne7d54592017-09-26 17:41:07 +0100152__weak void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
153 unsigned long max_freq)
154{
155}
156EXPORT_SYMBOL_GPL(arch_set_freq_scale);
157
Viresh Kumar70e9e772013-10-03 20:29:07 +0530158/*
159 * This is a generic cpufreq init() routine which can be used by cpufreq
160 * drivers of SMP systems. It will do following:
161 * - validate & show freq table passed
162 * - set policies transition latency
163 * - policy->cpus with all possible CPUs
164 */
Viresh Kumarc4dcc8a2019-07-16 09:36:08 +0530165void cpufreq_generic_init(struct cpufreq_policy *policy,
Viresh Kumar70e9e772013-10-03 20:29:07 +0530166 struct cpufreq_frequency_table *table,
167 unsigned int transition_latency)
168{
Viresh Kumar92c99d12018-02-26 10:38:45 +0530169 policy->freq_table = table;
Viresh Kumar70e9e772013-10-03 20:29:07 +0530170 policy->cpuinfo.transition_latency = transition_latency;
171
172 /*
Shailendra Verma58405af2015-05-22 22:48:22 +0530173 * The driver only supports the SMP configuration where all processors
Viresh Kumar70e9e772013-10-03 20:29:07 +0530174 * share the clock and voltage and clock.
175 */
176 cpumask_setall(policy->cpus);
Viresh Kumar70e9e772013-10-03 20:29:07 +0530177}
178EXPORT_SYMBOL_GPL(cpufreq_generic_init);
179
Rafael J. Wysocki1f0bd442015-09-16 02:17:49 +0200180struct cpufreq_policy *cpufreq_cpu_get_raw(unsigned int cpu)
Viresh Kumar652ed952014-01-09 20:38:43 +0530181{
182 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
183
Viresh Kumar988bed02015-05-08 11:53:45 +0530184 return policy && cpumask_test_cpu(cpu, policy->cpus) ? policy : NULL;
185}
Rafael J. Wysocki1f0bd442015-09-16 02:17:49 +0200186EXPORT_SYMBOL_GPL(cpufreq_cpu_get_raw);
Viresh Kumar988bed02015-05-08 11:53:45 +0530187
188unsigned int cpufreq_generic_get(unsigned int cpu)
189{
190 struct cpufreq_policy *policy = cpufreq_cpu_get_raw(cpu);
191
Viresh Kumar652ed952014-01-09 20:38:43 +0530192 if (!policy || IS_ERR(policy->clk)) {
Joe Perchese837f9b2014-03-11 10:03:00 -0700193 pr_err("%s: No %s associated to cpu: %d\n",
194 __func__, policy ? "clk" : "policy", cpu);
Viresh Kumar652ed952014-01-09 20:38:43 +0530195 return 0;
196 }
197
198 return clk_get_rate(policy->clk) / 1000;
199}
200EXPORT_SYMBOL_GPL(cpufreq_generic_get);
201
Viresh Kumar50e9c852015-02-19 17:02:03 +0530202/**
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100203 * cpufreq_cpu_get - Return policy for a CPU and mark it as busy.
204 * @cpu: CPU to find the policy for.
Viresh Kumar50e9c852015-02-19 17:02:03 +0530205 *
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100206 * Call cpufreq_cpu_get_raw() to obtain a cpufreq policy for @cpu and increment
207 * the kobject reference counter of that policy. Return a valid policy on
208 * success or NULL on failure.
Viresh Kumar50e9c852015-02-19 17:02:03 +0530209 *
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100210 * The policy returned by this function has to be released with the help of
211 * cpufreq_cpu_put() to balance its kobject reference counter properly.
Viresh Kumar50e9c852015-02-19 17:02:03 +0530212 */
Viresh Kumar6eed9402013-08-06 22:53:11 +0530213struct cpufreq_policy *cpufreq_cpu_get(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214{
Viresh Kumar6eed9402013-08-06 22:53:11 +0530215 struct cpufreq_policy *policy = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216 unsigned long flags;
217
Viresh Kumar1b947c902015-02-19 17:02:05 +0530218 if (WARN_ON(cpu >= nr_cpu_ids))
Viresh Kumar6eed9402013-08-06 22:53:11 +0530219 return NULL;
220
Linus Torvalds1da177e2005-04-16 15:20:36 -0700221 /* get the cpufreq driver */
Nathan Zimmer0d1857a2013-02-22 16:24:34 +0000222 read_lock_irqsave(&cpufreq_driver_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223
Viresh Kumar6eed9402013-08-06 22:53:11 +0530224 if (cpufreq_driver) {
225 /* get the CPU */
Viresh Kumar988bed02015-05-08 11:53:45 +0530226 policy = cpufreq_cpu_get_raw(cpu);
Viresh Kumar6eed9402013-08-06 22:53:11 +0530227 if (policy)
228 kobject_get(&policy->kobj);
229 }
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +0200230
Viresh Kumar6eed9402013-08-06 22:53:11 +0530231 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700232
Viresh Kumar3a3e9e02013-08-06 22:53:05 +0530233 return policy;
Stephen Boyda9144432012-07-20 18:14:38 +0000234}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700235EXPORT_SYMBOL_GPL(cpufreq_cpu_get);
236
Viresh Kumar50e9c852015-02-19 17:02:03 +0530237/**
Rafael J. Wysocki5d094fe2019-03-05 11:44:04 +0100238 * cpufreq_cpu_put - Decrement kobject usage counter for cpufreq policy.
239 * @policy: cpufreq policy returned by cpufreq_cpu_get().
Viresh Kumar50e9c852015-02-19 17:02:03 +0530240 */
Viresh Kumar3a3e9e02013-08-06 22:53:05 +0530241void cpufreq_cpu_put(struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700242{
Viresh Kumar6eed9402013-08-06 22:53:11 +0530243 kobject_put(&policy->kobj);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244}
245EXPORT_SYMBOL_GPL(cpufreq_cpu_put);
246
Rafael J. Wysocki540a37582019-03-26 12:16:58 +0100247/**
248 * cpufreq_cpu_release - Unlock a policy and decrement its usage counter.
249 * @policy: cpufreq policy returned by cpufreq_cpu_acquire().
250 */
Rafael J. Wysocki9083e492019-03-26 12:19:52 +0100251void cpufreq_cpu_release(struct cpufreq_policy *policy)
Rafael J. Wysocki540a37582019-03-26 12:16:58 +0100252{
253 if (WARN_ON(!policy))
254 return;
255
256 lockdep_assert_held(&policy->rwsem);
257
258 up_write(&policy->rwsem);
259
260 cpufreq_cpu_put(policy);
261}
262
263/**
264 * cpufreq_cpu_acquire - Find policy for a CPU, mark it as busy and lock it.
265 * @cpu: CPU to find the policy for.
266 *
267 * Call cpufreq_cpu_get() to get a reference on the cpufreq policy for @cpu and
268 * if the policy returned by it is not NULL, acquire its rwsem for writing.
269 * Return the policy if it is active or release it and return NULL otherwise.
270 *
271 * The policy returned by this function has to be released with the help of
272 * cpufreq_cpu_release() in order to release its rwsem and balance its usage
273 * counter properly.
274 */
Rafael J. Wysocki9083e492019-03-26 12:19:52 +0100275struct cpufreq_policy *cpufreq_cpu_acquire(unsigned int cpu)
Rafael J. Wysocki540a37582019-03-26 12:16:58 +0100276{
277 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
278
279 if (!policy)
280 return NULL;
281
282 down_write(&policy->rwsem);
283
284 if (policy_is_inactive(policy)) {
285 cpufreq_cpu_release(policy);
286 return NULL;
287 }
288
289 return policy;
290}
291
Linus Torvalds1da177e2005-04-16 15:20:36 -0700292/*********************************************************************
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 * EXTERNALLY AFFECTING FREQUENCY CHANGES *
294 *********************************************************************/
295
296/**
297 * adjust_jiffies - adjust the system "loops_per_jiffy"
298 *
299 * This function alters the system "loops_per_jiffy" for the clock
300 * speed change. Note that loops_per_jiffy cannot be updated on SMP
Dave Jones32ee8c32006-02-28 00:43:23 -0500301 * systems as each CPU might be scaled differently. So, use the arch
Linus Torvalds1da177e2005-04-16 15:20:36 -0700302 * per-CPU loops_per_jiffy value wherever possible.
303 */
Arjan van de Ven858119e2006-01-14 13:20:43 -0800304static void adjust_jiffies(unsigned long val, struct cpufreq_freqs *ci)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305{
Viresh Kumar39c132e2015-01-02 12:34:34 +0530306#ifndef CONFIG_SMP
307 static unsigned long l_p_j_ref;
308 static unsigned int l_p_j_ref_freq;
309
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310 if (ci->flags & CPUFREQ_CONST_LOOPS)
311 return;
312
313 if (!l_p_j_ref_freq) {
314 l_p_j_ref = loops_per_jiffy;
315 l_p_j_ref_freq = ci->old;
Joe Perchese837f9b2014-03-11 10:03:00 -0700316 pr_debug("saving %lu as reference value for loops_per_jiffy; freq is %u kHz\n",
317 l_p_j_ref, l_p_j_ref_freq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318 }
Viresh Kumar0b443ea2014-03-19 11:24:58 +0530319 if (val == CPUFREQ_POSTCHANGE && ci->old != ci->new) {
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530320 loops_per_jiffy = cpufreq_scale(l_p_j_ref, l_p_j_ref_freq,
321 ci->new);
Joe Perchese837f9b2014-03-11 10:03:00 -0700322 pr_debug("scaling loops_per_jiffy to %lu for frequency %u kHz\n",
323 loops_per_jiffy, ci->new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325#endif
Viresh Kumar39c132e2015-01-02 12:34:34 +0530326}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327
Viresh Kumar20b53242018-05-10 15:00:29 +0530328/**
329 * cpufreq_notify_transition - Notify frequency transition and adjust_jiffies.
330 * @policy: cpufreq policy to enable fast frequency switching for.
331 * @freqs: contain details of the frequency update.
332 * @state: set to CPUFREQ_PRECHANGE or CPUFREQ_POSTCHANGE.
333 *
334 * This function calls the transition notifiers and the "adjust_jiffies"
335 * function. It is called twice on all CPU frequency changes that have
336 * external effects.
337 */
338static void cpufreq_notify_transition(struct cpufreq_policy *policy,
339 struct cpufreq_freqs *freqs,
340 unsigned int state)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700341{
Viresh Kumardf240142019-04-29 15:03:58 +0530342 int cpu;
343
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 BUG_ON(irqs_disabled());
345
Dirk Brandewied5aaffa2013-01-17 16:22:21 +0000346 if (cpufreq_disabled())
347 return;
348
Viresh Kumardf240142019-04-29 15:03:58 +0530349 freqs->policy = policy;
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +0200350 freqs->flags = cpufreq_driver->flags;
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +0200351 pr_debug("notification %u of frequency transition to %u kHz\n",
Joe Perchese837f9b2014-03-11 10:03:00 -0700352 state, freqs->new);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353
Linus Torvalds1da177e2005-04-16 15:20:36 -0700354 switch (state) {
355 case CPUFREQ_PRECHANGE:
Viresh Kumar20b53242018-05-10 15:00:29 +0530356 /*
357 * Detect if the driver reported a value as "old frequency"
Dave Jonese4472cb2006-01-31 15:53:55 -0800358 * which is not equal to what the cpufreq core thinks is
359 * "old frequency".
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360 */
Viresh Kumar98015222019-06-28 10:46:55 +0530361 if (policy->cur && policy->cur != freqs->old) {
362 pr_debug("Warning: CPU frequency is %u, cpufreq assumed %u kHz\n",
363 freqs->old, policy->cur);
364 freqs->old = policy->cur;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365 }
Viresh Kumar20b53242018-05-10 15:00:29 +0530366
Viresh Kumardf240142019-04-29 15:03:58 +0530367 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
368 CPUFREQ_PRECHANGE, freqs);
Viresh Kumar20b53242018-05-10 15:00:29 +0530369
Linus Torvalds1da177e2005-04-16 15:20:36 -0700370 adjust_jiffies(CPUFREQ_PRECHANGE, freqs);
371 break;
Dave Jonese4472cb2006-01-31 15:53:55 -0800372
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373 case CPUFREQ_POSTCHANGE:
374 adjust_jiffies(CPUFREQ_POSTCHANGE, freqs);
Viresh Kumar20b53242018-05-10 15:00:29 +0530375 pr_debug("FREQ: %u - CPUs: %*pbl\n", freqs->new,
376 cpumask_pr_args(policy->cpus));
Viresh Kumarbb176f72013-06-19 14:19:33 +0530377
Viresh Kumardf240142019-04-29 15:03:58 +0530378 for_each_cpu(cpu, policy->cpus)
379 trace_cpu_frequency(freqs->new, cpu);
380
381 srcu_notifier_call_chain(&cpufreq_transition_notifier_list,
382 CPUFREQ_POSTCHANGE, freqs);
Viresh Kumar20b53242018-05-10 15:00:29 +0530383
384 cpufreq_stats_record_transition(policy, freqs->new);
385 policy->cur = freqs->new;
386 }
Viresh Kumarb43a7ff2013-03-24 11:56:43 +0530387}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388
Viresh Kumarf7ba3b42013-12-02 11:04:12 +0530389/* Do post notifications when there are chances that transition has failed */
Viresh Kumar236a9802014-03-24 13:35:46 +0530390static void cpufreq_notify_post_transition(struct cpufreq_policy *policy,
Viresh Kumarf7ba3b42013-12-02 11:04:12 +0530391 struct cpufreq_freqs *freqs, int transition_failed)
392{
393 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
394 if (!transition_failed)
395 return;
396
397 swap(freqs->old, freqs->new);
398 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
399 cpufreq_notify_transition(policy, freqs, CPUFREQ_POSTCHANGE);
400}
Viresh Kumarf7ba3b42013-12-02 11:04:12 +0530401
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530402void cpufreq_freq_transition_begin(struct cpufreq_policy *policy,
403 struct cpufreq_freqs *freqs)
404{
Srivatsa S. Bhatca654dc2014-05-05 12:52:39 +0530405
406 /*
407 * Catch double invocations of _begin() which lead to self-deadlock.
408 * ASYNC_NOTIFICATION drivers are left out because the cpufreq core
409 * doesn't invoke _begin() on their behalf, and hence the chances of
410 * double invocations are very low. Moreover, there are scenarios
411 * where these checks can emit false-positive warnings in these
412 * drivers; so we avoid that by skipping them altogether.
413 */
414 WARN_ON(!(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION)
415 && current == policy->transition_task);
416
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530417wait:
418 wait_event(policy->transition_wait, !policy->transition_ongoing);
419
420 spin_lock(&policy->transition_lock);
421
422 if (unlikely(policy->transition_ongoing)) {
423 spin_unlock(&policy->transition_lock);
424 goto wait;
425 }
426
427 policy->transition_ongoing = true;
Srivatsa S. Bhatca654dc2014-05-05 12:52:39 +0530428 policy->transition_task = current;
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530429
430 spin_unlock(&policy->transition_lock);
431
432 cpufreq_notify_transition(policy, freqs, CPUFREQ_PRECHANGE);
433}
434EXPORT_SYMBOL_GPL(cpufreq_freq_transition_begin);
435
436void cpufreq_freq_transition_end(struct cpufreq_policy *policy,
437 struct cpufreq_freqs *freqs, int transition_failed)
438{
Igor Stoppa0e7ea2f2018-09-07 19:09:55 +0300439 if (WARN_ON(!policy->transition_ongoing))
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530440 return;
441
442 cpufreq_notify_post_transition(policy, freqs, transition_failed);
443
444 policy->transition_ongoing = false;
Srivatsa S. Bhatca654dc2014-05-05 12:52:39 +0530445 policy->transition_task = NULL;
Srivatsa S. Bhat12478cf2014-03-24 13:35:44 +0530446
447 wake_up(&policy->transition_wait);
448}
449EXPORT_SYMBOL_GPL(cpufreq_freq_transition_end);
450
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +0200451/*
452 * Fast frequency switching status count. Positive means "enabled", negative
453 * means "disabled" and 0 means "not decided yet".
454 */
455static int cpufreq_fast_switch_count;
456static DEFINE_MUTEX(cpufreq_fast_switch_lock);
457
458static void cpufreq_list_transition_notifiers(void)
459{
460 struct notifier_block *nb;
461
462 pr_info("Registered transition notifiers:\n");
463
464 mutex_lock(&cpufreq_transition_notifier_list.mutex);
465
466 for (nb = cpufreq_transition_notifier_list.head; nb; nb = nb->next)
Sakari Ailusd75f7732019-03-25 21:32:28 +0200467 pr_info("%pS\n", nb->notifier_call);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +0200468
469 mutex_unlock(&cpufreq_transition_notifier_list.mutex);
470}
471
472/**
473 * cpufreq_enable_fast_switch - Enable fast frequency switching for policy.
474 * @policy: cpufreq policy to enable fast frequency switching for.
475 *
476 * Try to enable fast frequency switching for @policy.
477 *
478 * The attempt will fail if there is at least one transition notifier registered
479 * at this point, as fast frequency switching is quite fundamentally at odds
480 * with transition notifiers. Thus if successful, it will make registration of
481 * transition notifiers fail going forward.
482 */
483void cpufreq_enable_fast_switch(struct cpufreq_policy *policy)
484{
485 lockdep_assert_held(&policy->rwsem);
486
487 if (!policy->fast_switch_possible)
488 return;
489
490 mutex_lock(&cpufreq_fast_switch_lock);
491 if (cpufreq_fast_switch_count >= 0) {
492 cpufreq_fast_switch_count++;
493 policy->fast_switch_enabled = true;
494 } else {
495 pr_warn("CPU%u: Fast frequency switching not enabled\n",
496 policy->cpu);
497 cpufreq_list_transition_notifiers();
498 }
499 mutex_unlock(&cpufreq_fast_switch_lock);
500}
501EXPORT_SYMBOL_GPL(cpufreq_enable_fast_switch);
502
Rafael J. Wysocki6c9d9c82016-04-07 23:38:46 +0200503/**
504 * cpufreq_disable_fast_switch - Disable fast frequency switching for policy.
505 * @policy: cpufreq policy to disable fast frequency switching for.
506 */
507void cpufreq_disable_fast_switch(struct cpufreq_policy *policy)
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +0200508{
509 mutex_lock(&cpufreq_fast_switch_lock);
510 if (policy->fast_switch_enabled) {
511 policy->fast_switch_enabled = false;
512 if (!WARN_ON(cpufreq_fast_switch_count <= 0))
513 cpufreq_fast_switch_count--;
514 }
515 mutex_unlock(&cpufreq_fast_switch_lock);
516}
Rafael J. Wysocki6c9d9c82016-04-07 23:38:46 +0200517EXPORT_SYMBOL_GPL(cpufreq_disable_fast_switch);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700518
Steve Mucklee3c06232016-07-13 13:25:25 -0700519/**
520 * cpufreq_driver_resolve_freq - Map a target frequency to a driver-supported
521 * one.
522 * @target_freq: target frequency to resolve.
523 *
524 * The target to driver frequency mapping is cached in the policy.
525 *
526 * Return: Lowest driver-supported frequency greater than or equal to the
527 * given target_freq, subject to policy (min/max) and driver limitations.
528 */
529unsigned int cpufreq_driver_resolve_freq(struct cpufreq_policy *policy,
530 unsigned int target_freq)
531{
532 target_freq = clamp_val(target_freq, policy->min, policy->max);
533 policy->cached_target_freq = target_freq;
Viresh Kumarabe8bd02016-07-21 14:39:26 -0700534
535 if (cpufreq_driver->target_index) {
536 int idx;
537
538 idx = cpufreq_frequency_table_target(policy, target_freq,
539 CPUFREQ_RELATION_L);
540 policy->cached_resolved_idx = idx;
541 return policy->freq_table[idx].frequency;
542 }
543
Steve Mucklee3c06232016-07-13 13:25:25 -0700544 if (cpufreq_driver->resolve_freq)
545 return cpufreq_driver->resolve_freq(policy, target_freq);
Viresh Kumarabe8bd02016-07-21 14:39:26 -0700546
547 return target_freq;
Steve Mucklee3c06232016-07-13 13:25:25 -0700548}
Steve Muckleae2c1ca2016-07-21 19:24:38 -0700549EXPORT_SYMBOL_GPL(cpufreq_driver_resolve_freq);
Steve Mucklee3c06232016-07-13 13:25:25 -0700550
Viresh Kumaraa7519a2017-07-19 15:42:42 +0530551unsigned int cpufreq_policy_transition_delay_us(struct cpufreq_policy *policy)
552{
553 unsigned int latency;
554
555 if (policy->transition_delay_us)
556 return policy->transition_delay_us;
557
558 latency = policy->cpuinfo.transition_latency / NSEC_PER_USEC;
Viresh Kumare948bc82017-08-17 09:12:27 +0530559 if (latency) {
560 /*
561 * For platforms that can change the frequency very fast (< 10
562 * us), the above formula gives a decent transition delay. But
563 * for platforms where transition_latency is in milliseconds, it
564 * ends up giving unrealistic values.
565 *
566 * Cap the default transition delay to 10 ms, which seems to be
567 * a reasonable amount of time after which we should reevaluate
568 * the frequency.
569 */
570 return min(latency * LATENCY_MULTIPLIER, (unsigned int)10000);
571 }
Viresh Kumaraa7519a2017-07-19 15:42:42 +0530572
573 return LATENCY_MULTIPLIER;
574}
575EXPORT_SYMBOL_GPL(cpufreq_policy_transition_delay_us);
576
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577/*********************************************************************
578 * SYSFS INTERFACE *
579 *********************************************************************/
Rashika Kheria8a5c74a2014-02-26 22:12:42 +0530580static ssize_t show_boost(struct kobject *kobj,
Viresh Kumar625c85a2019-01-25 12:53:07 +0530581 struct kobj_attribute *attr, char *buf)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100582{
583 return sprintf(buf, "%d\n", cpufreq_driver->boost_enabled);
584}
585
Viresh Kumar625c85a2019-01-25 12:53:07 +0530586static ssize_t store_boost(struct kobject *kobj, struct kobj_attribute *attr,
587 const char *buf, size_t count)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100588{
589 int ret, enable;
590
591 ret = sscanf(buf, "%d", &enable);
592 if (ret != 1 || enable < 0 || enable > 1)
593 return -EINVAL;
594
595 if (cpufreq_boost_trigger_state(enable)) {
Joe Perchese837f9b2014-03-11 10:03:00 -0700596 pr_err("%s: Cannot %s BOOST!\n",
597 __func__, enable ? "enable" : "disable");
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100598 return -EINVAL;
599 }
600
Joe Perchese837f9b2014-03-11 10:03:00 -0700601 pr_debug("%s: cpufreq BOOST %s\n",
602 __func__, enable ? "enabled" : "disabled");
Lukasz Majewski6f19efc2013-12-20 15:24:49 +0100603
604 return count;
605}
606define_one_global_rw(boost);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Viresh Kumar42f91fa2015-01-02 12:34:26 +0530608static struct cpufreq_governor *find_governor(const char *str_governor)
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -0700609{
610 struct cpufreq_governor *t;
611
Viresh Kumarf7b27062015-01-27 14:06:09 +0530612 for_each_governor(t)
Rasmus Villemoes7c4f4532014-09-29 15:50:11 +0200613 if (!strncasecmp(str_governor, t->name, CPUFREQ_NAME_LEN))
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -0700614 return t;
615
616 return NULL;
617}
618
Yue Huab05d972019-04-29 15:24:18 +0800619static int cpufreq_parse_policy(char *str_governor,
620 struct cpufreq_policy *policy)
621{
622 if (!strncasecmp(str_governor, "performance", CPUFREQ_NAME_LEN)) {
623 policy->policy = CPUFREQ_POLICY_PERFORMANCE;
624 return 0;
625 }
626 if (!strncasecmp(str_governor, "powersave", CPUFREQ_NAME_LEN)) {
627 policy->policy = CPUFREQ_POLICY_POWERSAVE;
628 return 0;
629 }
630 return -EINVAL;
631}
632
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633/**
Viresh Kumar5ddc6d42019-06-20 08:35:48 +0530634 * cpufreq_parse_governor - parse a governor string only for has_target()
Linus Torvalds1da177e2005-04-16 15:20:36 -0700635 */
Rafael J. Wysockiae0ff892017-11-23 01:24:05 +0100636static int cpufreq_parse_governor(char *str_governor,
637 struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638{
Yue Huab05d972019-04-29 15:24:18 +0800639 struct cpufreq_governor *t;
Rafael J. Wysocki045149e2017-11-23 01:23:16 +0100640
Yue Huab05d972019-04-29 15:24:18 +0800641 mutex_lock(&cpufreq_governor_mutex);
642
643 t = find_governor(str_governor);
644 if (!t) {
645 int ret;
646
647 mutex_unlock(&cpufreq_governor_mutex);
648
649 ret = request_module("cpufreq_%s", str_governor);
650 if (ret)
651 return -EINVAL;
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -0700652
akpm@osdl.org3fc54d32006-01-13 15:54:22 -0800653 mutex_lock(&cpufreq_governor_mutex);
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -0700654
Viresh Kumar42f91fa2015-01-02 12:34:26 +0530655 t = find_governor(str_governor);
Yue Huab05d972019-04-29 15:24:18 +0800656 }
657 if (t && !try_module_get(t->owner))
658 t = NULL;
Jeremy Fitzhardingeea714972006-07-06 12:32:01 -0700659
Yue Huab05d972019-04-29 15:24:18 +0800660 mutex_unlock(&cpufreq_governor_mutex);
Rafael J. Wysocki045149e2017-11-23 01:23:16 +0100661
Yue Huab05d972019-04-29 15:24:18 +0800662 if (t) {
663 policy->governor = t;
664 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700665 }
Rafael J. Wysocki045149e2017-11-23 01:23:16 +0100666
667 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700668}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669
Linus Torvalds1da177e2005-04-16 15:20:36 -0700670/**
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530671 * cpufreq_per_cpu_attr_read() / show_##file_name() -
672 * print out cpufreq information
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 *
674 * Write out information from cpufreq_driver->policy[cpu]; object must be
675 * "unsigned int".
676 */
677
Dave Jones32ee8c32006-02-28 00:43:23 -0500678#define show_one(file_name, object) \
679static ssize_t show_##file_name \
Dave Jones905d77c2008-03-05 14:28:32 -0500680(struct cpufreq_policy *policy, char *buf) \
Dave Jones32ee8c32006-02-28 00:43:23 -0500681{ \
Dave Jones29464f22009-01-18 01:37:11 -0500682 return sprintf(buf, "%u\n", policy->object); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700683}
684
685show_one(cpuinfo_min_freq, cpuinfo.min_freq);
686show_one(cpuinfo_max_freq, cpuinfo.max_freq);
Thomas Renningered129782009-02-04 01:17:41 +0100687show_one(cpuinfo_transition_latency, cpuinfo.transition_latency);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688show_one(scaling_min_freq, min);
689show_one(scaling_max_freq, max);
Dirk Brandewiec034b022014-10-13 08:37:40 -0700690
Len Brownf8475ce2017-06-23 22:11:52 -0700691__weak unsigned int arch_freq_get_on_cpu(int cpu)
692{
693 return 0;
694}
695
Viresh Kumar09347b22015-01-02 12:34:24 +0530696static ssize_t show_scaling_cur_freq(struct cpufreq_policy *policy, char *buf)
Dirk Brandewiec034b022014-10-13 08:37:40 -0700697{
698 ssize_t ret;
Len Brownf8475ce2017-06-23 22:11:52 -0700699 unsigned int freq;
Dirk Brandewiec034b022014-10-13 08:37:40 -0700700
Len Brownf8475ce2017-06-23 22:11:52 -0700701 freq = arch_freq_get_on_cpu(policy->cpu);
702 if (freq)
703 ret = sprintf(buf, "%u\n", freq);
704 else if (cpufreq_driver && cpufreq_driver->setpolicy &&
705 cpufreq_driver->get)
Dirk Brandewiec034b022014-10-13 08:37:40 -0700706 ret = sprintf(buf, "%u\n", cpufreq_driver->get(policy->cpu));
707 else
708 ret = sprintf(buf, "%u\n", policy->cur);
709 return ret;
710}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
712/**
713 * cpufreq_per_cpu_attr_write() / store_##file_name() - sysfs write access
714 */
715#define store_one(file_name, object) \
716static ssize_t store_##file_name \
Dave Jones905d77c2008-03-05 14:28:32 -0500717(struct cpufreq_policy *policy, const char *buf, size_t count) \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700718{ \
Viresh Kumar18c49922019-07-05 16:21:24 +0530719 unsigned long val; \
720 int ret; \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700721 \
Viresh Kumar18c49922019-07-05 16:21:24 +0530722 ret = sscanf(buf, "%lu", &val); \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700723 if (ret != 1) \
724 return -EINVAL; \
725 \
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +0200726 ret = freq_qos_update_request(policy->object##_freq_req, val);\
Viresh Kumar18c49922019-07-05 16:21:24 +0530727 return ret >= 0 ? count : ret; \
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728}
729
Dave Jones29464f22009-01-18 01:37:11 -0500730store_one(scaling_min_freq, min);
731store_one(scaling_max_freq, max);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
733/**
734 * show_cpuinfo_cur_freq - current CPU frequency as detected by hardware
735 */
Dave Jones905d77c2008-03-05 14:28:32 -0500736static ssize_t show_cpuinfo_cur_freq(struct cpufreq_policy *policy,
737 char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738{
Viresh Kumard92d50a2015-01-02 12:34:29 +0530739 unsigned int cur_freq = __cpufreq_get(policy);
Rafael J. Wysocki9b4f6032017-03-15 00:12:16 +0100740
741 if (cur_freq)
742 return sprintf(buf, "%u\n", cur_freq);
743
744 return sprintf(buf, "<unknown>\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745}
746
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747/**
748 * show_scaling_governor - show the current policy for the specified CPU
749 */
Dave Jones905d77c2008-03-05 14:28:32 -0500750static ssize_t show_scaling_governor(struct cpufreq_policy *policy, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751{
Dave Jones29464f22009-01-18 01:37:11 -0500752 if (policy->policy == CPUFREQ_POLICY_POWERSAVE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700753 return sprintf(buf, "powersave\n");
754 else if (policy->policy == CPUFREQ_POLICY_PERFORMANCE)
755 return sprintf(buf, "performance\n");
756 else if (policy->governor)
viresh kumar4b972f02012-10-23 01:23:43 +0200757 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n",
Dave Jones29464f22009-01-18 01:37:11 -0500758 policy->governor->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700759 return -EINVAL;
760}
761
Linus Torvalds1da177e2005-04-16 15:20:36 -0700762/**
763 * store_scaling_governor - store policy for the specified CPU
764 */
Dave Jones905d77c2008-03-05 14:28:32 -0500765static ssize_t store_scaling_governor(struct cpufreq_policy *policy,
766 const char *buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767{
Srivatsa S. Bhat5136fa52013-09-07 01:24:06 +0530768 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700769 char str_governor[16];
770 struct cpufreq_policy new_policy;
771
Viresh Kumar8fa5b632015-08-03 08:36:15 +0530772 memcpy(&new_policy, policy, sizeof(*policy));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
Dave Jones29464f22009-01-18 01:37:11 -0500774 ret = sscanf(buf, "%15s", str_governor);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700775 if (ret != 1)
776 return -EINVAL;
777
Yue Huab05d972019-04-29 15:24:18 +0800778 if (cpufreq_driver->setpolicy) {
779 if (cpufreq_parse_policy(str_governor, &new_policy))
780 return -EINVAL;
781 } else {
782 if (cpufreq_parse_governor(str_governor, &new_policy))
783 return -EINVAL;
784 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700785
Viresh Kumar037ce832013-10-02 14:13:16 +0530786 ret = cpufreq_set_policy(policy, &new_policy);
Rafael J. Wysockia8b149d2017-11-23 14:27:07 +0100787
788 if (new_policy.governor)
789 module_put(new_policy.governor->owner);
790
Viresh Kumar88dc4382015-08-03 08:36:18 +0530791 return ret ? ret : count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700792}
793
794/**
795 * show_scaling_driver - show the cpufreq driver currently loaded
796 */
Dave Jones905d77c2008-03-05 14:28:32 -0500797static ssize_t show_scaling_driver(struct cpufreq_policy *policy, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798{
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +0200799 return scnprintf(buf, CPUFREQ_NAME_PLEN, "%s\n", cpufreq_driver->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800}
801
802/**
803 * show_scaling_available_governors - show the available CPUfreq governors
804 */
Dave Jones905d77c2008-03-05 14:28:32 -0500805static ssize_t show_scaling_available_governors(struct cpufreq_policy *policy,
806 char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700807{
808 ssize_t i = 0;
809 struct cpufreq_governor *t;
810
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +0530811 if (!has_target()) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 i += sprintf(buf, "performance powersave");
813 goto out;
814 }
815
Viresh Kumarf7b27062015-01-27 14:06:09 +0530816 for_each_governor(t) {
Dave Jones29464f22009-01-18 01:37:11 -0500817 if (i >= (ssize_t) ((PAGE_SIZE / sizeof(char))
818 - (CPUFREQ_NAME_LEN + 2)))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819 goto out;
viresh kumar4b972f02012-10-23 01:23:43 +0200820 i += scnprintf(&buf[i], CPUFREQ_NAME_PLEN, "%s ", t->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 }
Dave Jones7d5e3502006-02-02 17:03:42 -0500822out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 i += sprintf(&buf[i], "\n");
824 return i;
825}
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700826
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800827ssize_t cpufreq_show_cpus(const struct cpumask *mask, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700828{
829 ssize_t i = 0;
830 unsigned int cpu;
831
Rusty Russell835481d2009-01-04 05:18:06 -0800832 for_each_cpu(cpu, mask) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833 if (i)
834 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " ");
835 i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu);
836 if (i >= (PAGE_SIZE - 5))
Dave Jones29464f22009-01-18 01:37:11 -0500837 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 }
839 i += sprintf(&buf[i], "\n");
840 return i;
841}
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800842EXPORT_SYMBOL_GPL(cpufreq_show_cpus);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700843
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700844/**
845 * show_related_cpus - show the CPUs affected by each transition even if
846 * hw coordination is in use
847 */
848static ssize_t show_related_cpus(struct cpufreq_policy *policy, char *buf)
849{
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800850 return cpufreq_show_cpus(policy->related_cpus, buf);
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700851}
852
853/**
854 * show_affected_cpus - show the CPUs affected by each transition
855 */
856static ssize_t show_affected_cpus(struct cpufreq_policy *policy, char *buf)
857{
Lan Tianyuf4fd3792013-06-27 15:08:54 +0800858 return cpufreq_show_cpus(policy->cpus, buf);
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700859}
860
Venki Pallipadi9e769882007-10-26 10:18:21 -0700861static ssize_t store_scaling_setspeed(struct cpufreq_policy *policy,
Dave Jones905d77c2008-03-05 14:28:32 -0500862 const char *buf, size_t count)
Venki Pallipadi9e769882007-10-26 10:18:21 -0700863{
864 unsigned int freq = 0;
865 unsigned int ret;
866
CHIKAMA masaki879000f2008-06-05 22:46:33 -0700867 if (!policy->governor || !policy->governor->store_setspeed)
Venki Pallipadi9e769882007-10-26 10:18:21 -0700868 return -EINVAL;
869
870 ret = sscanf(buf, "%u", &freq);
871 if (ret != 1)
872 return -EINVAL;
873
874 policy->governor->store_setspeed(policy, freq);
875
876 return count;
877}
878
879static ssize_t show_scaling_setspeed(struct cpufreq_policy *policy, char *buf)
880{
CHIKAMA masaki879000f2008-06-05 22:46:33 -0700881 if (!policy->governor || !policy->governor->show_setspeed)
Venki Pallipadi9e769882007-10-26 10:18:21 -0700882 return sprintf(buf, "<unsupported>\n");
883
884 return policy->governor->show_setspeed(policy, buf);
885}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886
Thomas Renningere2f74f32009-11-19 12:31:01 +0100887/**
viresh kumar8bf1ac722012-10-23 01:23:33 +0200888 * show_bios_limit - show the current cpufreq HW/BIOS limitation
Thomas Renningere2f74f32009-11-19 12:31:01 +0100889 */
890static ssize_t show_bios_limit(struct cpufreq_policy *policy, char *buf)
891{
892 unsigned int limit;
893 int ret;
Yue Hub23aa312019-04-16 10:40:27 +0800894 ret = cpufreq_driver->bios_limit(policy->cpu, &limit);
895 if (!ret)
896 return sprintf(buf, "%u\n", limit);
Thomas Renningere2f74f32009-11-19 12:31:01 +0100897 return sprintf(buf, "%u\n", policy->cpuinfo.max_freq);
898}
899
Borislav Petkov6dad2a22010-03-31 21:56:46 +0200900cpufreq_freq_attr_ro_perm(cpuinfo_cur_freq, 0400);
901cpufreq_freq_attr_ro(cpuinfo_min_freq);
902cpufreq_freq_attr_ro(cpuinfo_max_freq);
903cpufreq_freq_attr_ro(cpuinfo_transition_latency);
904cpufreq_freq_attr_ro(scaling_available_governors);
905cpufreq_freq_attr_ro(scaling_driver);
906cpufreq_freq_attr_ro(scaling_cur_freq);
907cpufreq_freq_attr_ro(bios_limit);
908cpufreq_freq_attr_ro(related_cpus);
909cpufreq_freq_attr_ro(affected_cpus);
910cpufreq_freq_attr_rw(scaling_min_freq);
911cpufreq_freq_attr_rw(scaling_max_freq);
912cpufreq_freq_attr_rw(scaling_governor);
913cpufreq_freq_attr_rw(scaling_setspeed);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914
Dave Jones905d77c2008-03-05 14:28:32 -0500915static struct attribute *default_attrs[] = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916 &cpuinfo_min_freq.attr,
917 &cpuinfo_max_freq.attr,
Thomas Renningered129782009-02-04 01:17:41 +0100918 &cpuinfo_transition_latency.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700919 &scaling_min_freq.attr,
920 &scaling_max_freq.attr,
921 &affected_cpus.attr,
Darrick J. Wonge8628dd2008-04-18 13:31:12 -0700922 &related_cpus.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923 &scaling_governor.attr,
924 &scaling_driver.attr,
925 &scaling_available_governors.attr,
Venki Pallipadi9e769882007-10-26 10:18:21 -0700926 &scaling_setspeed.attr,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927 NULL
928};
929
Dave Jones29464f22009-01-18 01:37:11 -0500930#define to_policy(k) container_of(k, struct cpufreq_policy, kobj)
931#define to_attr(a) container_of(a, struct freq_attr, attr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932
Dave Jones29464f22009-01-18 01:37:11 -0500933static ssize_t show(struct kobject *kobj, struct attribute *attr, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700934{
Dave Jones905d77c2008-03-05 14:28:32 -0500935 struct cpufreq_policy *policy = to_policy(kobj);
936 struct freq_attr *fattr = to_attr(attr);
Viresh Kumar1b750e32013-10-02 14:13:09 +0530937 ssize_t ret;
Viresh Kumar6eed9402013-08-06 22:53:11 +0530938
viresh kumarad7722d2013-10-18 19:10:15 +0530939 down_read(&policy->rwsem);
Rafael J. Wysocki6541aef2016-02-12 23:56:21 +0100940 ret = fattr->show(policy, buf);
viresh kumarad7722d2013-10-18 19:10:15 +0530941 up_read(&policy->rwsem);
Viresh Kumar1b750e32013-10-02 14:13:09 +0530942
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 return ret;
944}
945
Dave Jones905d77c2008-03-05 14:28:32 -0500946static ssize_t store(struct kobject *kobj, struct attribute *attr,
947 const char *buf, size_t count)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700948{
Dave Jones905d77c2008-03-05 14:28:32 -0500949 struct cpufreq_policy *policy = to_policy(kobj);
950 struct freq_attr *fattr = to_attr(attr);
Dave Jonesa07530b2008-03-05 14:22:25 -0500951 ssize_t ret = -EINVAL;
Viresh Kumar6eed9402013-08-06 22:53:11 +0530952
Waiman Long9b3d9bb2018-07-24 14:26:05 -0400953 /*
954 * cpus_read_trylock() is used here to work around a circular lock
955 * dependency problem with respect to the cpufreq_register_driver().
956 */
957 if (!cpus_read_trylock())
958 return -EBUSY;
Srivatsa S. Bhat4f750c92013-09-07 01:23:43 +0530959
Rafael J. Wysocki6541aef2016-02-12 23:56:21 +0100960 if (cpu_online(policy->cpu)) {
961 down_write(&policy->rwsem);
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530962 ret = fattr->store(policy, buf, count);
Rafael J. Wysocki6541aef2016-02-12 23:56:21 +0100963 up_write(&policy->rwsem);
964 }
Gautham R Shenoye08f5f52006-10-26 16:20:58 +0530965
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +0200966 cpus_read_unlock();
Srivatsa S. Bhat4f750c92013-09-07 01:23:43 +0530967
Linus Torvalds1da177e2005-04-16 15:20:36 -0700968 return ret;
969}
970
Dave Jones905d77c2008-03-05 14:28:32 -0500971static void cpufreq_sysfs_release(struct kobject *kobj)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700972{
Dave Jones905d77c2008-03-05 14:28:32 -0500973 struct cpufreq_policy *policy = to_policy(kobj);
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +0200974 pr_debug("last reference is dropped\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 complete(&policy->kobj_unregister);
976}
977
Emese Revfy52cf25d2010-01-19 02:58:23 +0100978static const struct sysfs_ops sysfs_ops = {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 .show = show,
980 .store = store,
981};
982
983static struct kobj_type ktype_cpufreq = {
984 .sysfs_ops = &sysfs_ops,
985 .default_attrs = default_attrs,
986 .release = cpufreq_sysfs_release,
987};
988
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +0200989static void add_cpu_dev_symlink(struct cpufreq_policy *policy, unsigned int cpu)
Viresh Kumar87549142015-06-10 02:13:21 +0200990{
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +0200991 struct device *dev = get_cpu_device(cpu);
992
Viresh Kumar67d874c2019-07-08 16:27:52 +0530993 if (unlikely(!dev))
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +0200994 return;
995
996 if (cpumask_test_and_set_cpu(cpu, policy->real_cpus))
997 return;
998
Viresh Kumar266198042016-09-12 12:07:05 +0530999 dev_dbg(dev, "%s: Adding symlink\n", __func__);
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001000 if (sysfs_create_link(&dev->kobj, &policy->kobj, "cpufreq"))
1001 dev_err(dev, "cpufreq symlink creation failed\n");
Viresh Kumar87549142015-06-10 02:13:21 +02001002}
1003
Viresh Kumar266198042016-09-12 12:07:05 +05301004static void remove_cpu_dev_symlink(struct cpufreq_policy *policy,
1005 struct device *dev)
Viresh Kumar87549142015-06-10 02:13:21 +02001006{
Viresh Kumar266198042016-09-12 12:07:05 +05301007 dev_dbg(dev, "%s: Removing symlink\n", __func__);
1008 sysfs_remove_link(&dev->kobj, "cpufreq");
Viresh Kumar87549142015-06-10 02:13:21 +02001009}
1010
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001011static int cpufreq_add_dev_interface(struct cpufreq_policy *policy)
Dave Jones909a6942009-07-08 18:05:42 -04001012{
1013 struct freq_attr **drv_attr;
Dave Jones909a6942009-07-08 18:05:42 -04001014 int ret = 0;
Dave Jones909a6942009-07-08 18:05:42 -04001015
Dave Jones909a6942009-07-08 18:05:42 -04001016 /* set up files for this cpu device */
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001017 drv_attr = cpufreq_driver->attr;
Viresh Kumarf13f1182015-01-02 12:34:23 +05301018 while (drv_attr && *drv_attr) {
Dave Jones909a6942009-07-08 18:05:42 -04001019 ret = sysfs_create_file(&policy->kobj, &((*drv_attr)->attr));
1020 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001021 return ret;
Dave Jones909a6942009-07-08 18:05:42 -04001022 drv_attr++;
1023 }
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001024 if (cpufreq_driver->get) {
Dave Jones909a6942009-07-08 18:05:42 -04001025 ret = sysfs_create_file(&policy->kobj, &cpuinfo_cur_freq.attr);
1026 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001027 return ret;
Dave Jones909a6942009-07-08 18:05:42 -04001028 }
Dirk Brandewiec034b022014-10-13 08:37:40 -07001029
1030 ret = sysfs_create_file(&policy->kobj, &scaling_cur_freq.attr);
1031 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001032 return ret;
Dirk Brandewiec034b022014-10-13 08:37:40 -07001033
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001034 if (cpufreq_driver->bios_limit) {
Thomas Renningere2f74f32009-11-19 12:31:01 +01001035 ret = sysfs_create_file(&policy->kobj, &bios_limit.attr);
1036 if (ret)
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001037 return ret;
Thomas Renningere2f74f32009-11-19 12:31:01 +01001038 }
Dave Jones909a6942009-07-08 18:05:42 -04001039
Viresh Kumar266198042016-09-12 12:07:05 +05301040 return 0;
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301041}
1042
Rafael J. Wysockide1df262016-02-05 02:37:42 +01001043__weak struct cpufreq_governor *cpufreq_default_governor(void)
1044{
1045 return NULL;
1046}
1047
Viresh Kumar7f0fa402015-07-08 15:12:16 +05301048static int cpufreq_init_policy(struct cpufreq_policy *policy)
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301049{
Yue Huab05d972019-04-29 15:24:18 +08001050 struct cpufreq_governor *gov = NULL, *def_gov = NULL;
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301051 struct cpufreq_policy new_policy;
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301052
Viresh Kumard5b73cd2013-08-06 22:53:06 +05301053 memcpy(&new_policy, policy, sizeof(*policy));
Jason Barona27a9ab2013-12-19 22:50:50 +00001054
Yue Huab05d972019-04-29 15:24:18 +08001055 def_gov = cpufreq_default_governor();
1056
1057 if (has_target()) {
1058 /*
1059 * Update governor of new_policy to the governor used before
1060 * hotplug
1061 */
1062 gov = find_governor(policy->last_governor);
1063 if (gov) {
1064 pr_debug("Restoring governor %s for cpu %d\n",
viresh kumar6e2c89d2014-03-04 11:43:59 +08001065 policy->governor->name, policy->cpu);
Yue Huab05d972019-04-29 15:24:18 +08001066 } else {
1067 if (!def_gov)
1068 return -ENODATA;
1069 gov = def_gov;
1070 }
1071 new_policy.governor = gov;
Rafael J. Wysockide1df262016-02-05 02:37:42 +01001072 } else {
Yue Huab05d972019-04-29 15:24:18 +08001073 /* Use the default policy if there is no last_policy. */
1074 if (policy->last_policy) {
Srinivas Pandruvada69030dd12015-12-01 16:52:14 -08001075 new_policy.policy = policy->last_policy;
Yue Huab05d972019-04-29 15:24:18 +08001076 } else {
1077 if (!def_gov)
1078 return -ENODATA;
1079 cpufreq_parse_policy(def_gov->name, &new_policy);
1080 }
Srinivas Pandruvada69030dd12015-12-01 16:52:14 -08001081 }
Yue Huab05d972019-04-29 15:24:18 +08001082
Viresh Kumar7f0fa402015-07-08 15:12:16 +05301083 return cpufreq_set_policy(policy, &new_policy);
Dave Jones909a6942009-07-08 18:05:42 -04001084}
1085
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001086static int cpufreq_add_policy_cpu(struct cpufreq_policy *policy, unsigned int cpu)
Viresh Kumarfcf80582013-01-29 14:39:08 +00001087{
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05301088 int ret = 0;
Viresh Kumarfcf80582013-01-29 14:39:08 +00001089
Viresh Kumarbb29ae12015-02-19 17:02:06 +05301090 /* Has this CPU been taken care of already? */
1091 if (cpumask_test_cpu(cpu, policy->cpus))
1092 return 0;
1093
Viresh Kumar49f18562016-02-11 17:31:12 +05301094 down_write(&policy->rwsem);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02001095 if (has_target())
1096 cpufreq_stop_governor(policy);
Viresh Kumarfcf80582013-01-29 14:39:08 +00001097
Viresh Kumarfcf80582013-01-29 14:39:08 +00001098 cpumask_set_cpu(cpu, policy->cpus);
Viresh Kumar2eaa3e22013-02-07 10:55:00 +05301099
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05301100 if (has_target()) {
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01001101 ret = cpufreq_start_governor(policy);
Viresh Kumar49f18562016-02-11 17:31:12 +05301102 if (ret)
Viresh Kumar3de9bde2013-08-06 22:53:13 +05301103 pr_err("%s: Failed to start governor\n", __func__);
Viresh Kumar820c6ca2013-04-22 00:48:03 +02001104 }
Viresh Kumar49f18562016-02-11 17:31:12 +05301105 up_write(&policy->rwsem);
1106 return ret;
Viresh Kumarfcf80582013-01-29 14:39:08 +00001107}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108
Viresh Kumarc57b25b2019-07-04 13:06:22 +05301109void refresh_frequency_limits(struct cpufreq_policy *policy)
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301110{
Viresh Kumar67d874c2019-07-08 16:27:52 +05301111 struct cpufreq_policy new_policy;
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301112
Viresh Kumar67d874c2019-07-08 16:27:52 +05301113 if (!policy_is_inactive(policy)) {
1114 new_policy = *policy;
1115 pr_debug("updating policy for CPU %u\n", policy->cpu);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301116
Viresh Kumar67d874c2019-07-08 16:27:52 +05301117 cpufreq_set_policy(policy, &new_policy);
1118 }
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301119}
Viresh Kumarc57b25b2019-07-04 13:06:22 +05301120EXPORT_SYMBOL(refresh_frequency_limits);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301121
Viresh Kumar11eb69b2016-02-22 16:36:42 +05301122static void handle_update(struct work_struct *work)
1123{
1124 struct cpufreq_policy *policy =
1125 container_of(work, struct cpufreq_policy, update);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301126
1127 pr_debug("handle_update for cpu %u called\n", policy->cpu);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301128 down_write(&policy->rwsem);
Viresh Kumar70a59fd2019-06-20 08:35:50 +05301129 refresh_frequency_limits(policy);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301130 up_write(&policy->rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131}
1132
Viresh Kumar67d874c2019-07-08 16:27:52 +05301133static int cpufreq_notifier_min(struct notifier_block *nb, unsigned long freq,
1134 void *data)
Srivatsa S. Bhat84148092013-07-30 04:25:10 +05301135{
Viresh Kumar67d874c2019-07-08 16:27:52 +05301136 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_min);
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301137
Viresh Kumar67d874c2019-07-08 16:27:52 +05301138 schedule_work(&policy->update);
1139 return 0;
1140}
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301141
Viresh Kumar67d874c2019-07-08 16:27:52 +05301142static int cpufreq_notifier_max(struct notifier_block *nb, unsigned long freq,
1143 void *data)
1144{
1145 struct cpufreq_policy *policy = container_of(nb, struct cpufreq_policy, nb_max);
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301146
Viresh Kumar67d874c2019-07-08 16:27:52 +05301147 schedule_work(&policy->update);
1148 return 0;
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301149}
1150
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301151static void cpufreq_policy_put_kobj(struct cpufreq_policy *policy)
Viresh Kumar42f921a2013-12-20 21:26:02 +05301152{
1153 struct kobject *kobj;
1154 struct completion *cmp;
1155
Viresh Kumar87549142015-06-10 02:13:21 +02001156 down_write(&policy->rwsem);
Rafael J. Wysocki1aefc752016-05-31 22:14:44 +02001157 cpufreq_stats_free_table(policy);
Viresh Kumar42f921a2013-12-20 21:26:02 +05301158 kobj = &policy->kobj;
1159 cmp = &policy->kobj_unregister;
Viresh Kumar87549142015-06-10 02:13:21 +02001160 up_write(&policy->rwsem);
Viresh Kumar42f921a2013-12-20 21:26:02 +05301161 kobject_put(kobj);
1162
1163 /*
1164 * We need to make sure that the underlying kobj is
1165 * actually not referenced anymore by anybody before we
1166 * proceed with unloading.
1167 */
1168 pr_debug("waiting for dropping of refcount\n");
1169 wait_for_completion(cmp);
1170 pr_debug("wait complete\n");
1171}
1172
Viresh Kumar67d874c2019-07-08 16:27:52 +05301173static struct cpufreq_policy *cpufreq_policy_alloc(unsigned int cpu)
1174{
1175 struct cpufreq_policy *policy;
1176 struct device *dev = get_cpu_device(cpu);
1177 int ret;
1178
1179 if (!dev)
1180 return NULL;
1181
1182 policy = kzalloc(sizeof(*policy), GFP_KERNEL);
1183 if (!policy)
1184 return NULL;
1185
1186 if (!alloc_cpumask_var(&policy->cpus, GFP_KERNEL))
1187 goto err_free_policy;
1188
1189 if (!zalloc_cpumask_var(&policy->related_cpus, GFP_KERNEL))
1190 goto err_free_cpumask;
1191
1192 if (!zalloc_cpumask_var(&policy->real_cpus, GFP_KERNEL))
1193 goto err_free_rcpumask;
1194
1195 ret = kobject_init_and_add(&policy->kobj, &ktype_cpufreq,
1196 cpufreq_global_kobject, "policy%u", cpu);
1197 if (ret) {
1198 dev_err(dev, "%s: failed to init policy->kobj: %d\n", __func__, ret);
1199 /*
1200 * The entire policy object will be freed below, but the extra
1201 * memory allocated for the kobject name needs to be freed by
1202 * releasing the kobject.
1203 */
1204 kobject_put(&policy->kobj);
1205 goto err_free_real_cpus;
1206 }
1207
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001208 freq_constraints_init(&policy->constraints);
1209
Viresh Kumar67d874c2019-07-08 16:27:52 +05301210 policy->nb_min.notifier_call = cpufreq_notifier_min;
1211 policy->nb_max.notifier_call = cpufreq_notifier_max;
1212
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001213 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MIN,
1214 &policy->nb_min);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301215 if (ret) {
1216 dev_err(dev, "Failed to register MIN QoS notifier: %d (%*pbl)\n",
1217 ret, cpumask_pr_args(policy->cpus));
1218 goto err_kobj_remove;
1219 }
1220
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001221 ret = freq_qos_add_notifier(&policy->constraints, FREQ_QOS_MAX,
1222 &policy->nb_max);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301223 if (ret) {
1224 dev_err(dev, "Failed to register MAX QoS notifier: %d (%*pbl)\n",
1225 ret, cpumask_pr_args(policy->cpus));
1226 goto err_min_qos_notifier;
1227 }
1228
1229 INIT_LIST_HEAD(&policy->policy_list);
1230 init_rwsem(&policy->rwsem);
1231 spin_lock_init(&policy->transition_lock);
1232 init_waitqueue_head(&policy->transition_wait);
1233 init_completion(&policy->kobj_unregister);
1234 INIT_WORK(&policy->update, handle_update);
1235
1236 policy->cpu = cpu;
1237 return policy;
1238
1239err_min_qos_notifier:
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001240 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1241 &policy->nb_min);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301242err_kobj_remove:
1243 cpufreq_policy_put_kobj(policy);
1244err_free_real_cpus:
1245 free_cpumask_var(policy->real_cpus);
1246err_free_rcpumask:
1247 free_cpumask_var(policy->related_cpus);
1248err_free_cpumask:
1249 free_cpumask_var(policy->cpus);
1250err_free_policy:
1251 kfree(policy);
1252
1253 return NULL;
1254}
1255
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301256static void cpufreq_policy_free(struct cpufreq_policy *policy)
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301257{
Viresh Kumar988bed02015-05-08 11:53:45 +05301258 unsigned long flags;
1259 int cpu;
1260
1261 /* Remove policy from list */
1262 write_lock_irqsave(&cpufreq_driver_lock, flags);
1263 list_del(&policy->policy_list);
1264
1265 for_each_cpu(cpu, policy->related_cpus)
1266 per_cpu(cpufreq_cpu_data, cpu) = NULL;
1267 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1268
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001269 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MAX,
1270 &policy->nb_max);
1271 freq_qos_remove_notifier(&policy->constraints, FREQ_QOS_MIN,
1272 &policy->nb_min);
Viresh Kumar6a149032019-07-23 11:44:01 +05301273
Sudeep Holla69410512019-10-18 11:58:15 +01001274 /* Cancel any pending policy->update work before freeing the policy. */
1275 cancel_work_sync(&policy->update);
Viresh Kumar6a149032019-07-23 11:44:01 +05301276
1277 if (policy->max_freq_req) {
1278 /*
1279 * CPUFREQ_CREATE_POLICY notification is sent only after
1280 * successfully adding max_freq_req request.
1281 */
1282 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1283 CPUFREQ_REMOVE_POLICY, policy);
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001284 freq_qos_remove_request(policy->max_freq_req);
Viresh Kumar6a149032019-07-23 11:44:01 +05301285 }
1286
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001287 freq_qos_remove_request(policy->min_freq_req);
Viresh Kumar18c49922019-07-05 16:21:24 +05301288 kfree(policy->min_freq_req);
Viresh Kumar67d874c2019-07-08 16:27:52 +05301289
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301290 cpufreq_policy_put_kobj(policy);
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001291 free_cpumask_var(policy->real_cpus);
Srivatsa S. Bhate9698cc2013-07-30 04:24:11 +05301292 free_cpumask_var(policy->related_cpus);
1293 free_cpumask_var(policy->cpus);
1294 kfree(policy);
1295}
1296
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001297static int cpufreq_online(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001298{
Viresh Kumar7f0c0202015-01-02 12:34:32 +05301299 struct cpufreq_policy *policy;
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001300 bool new_policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001301 unsigned long flags;
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001302 unsigned int j;
1303 int ret;
Ashok Rajc32b6b82005-10-30 14:59:54 -08001304
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001305 pr_debug("%s: bringing CPU%u online\n", __func__, cpu);
Viresh Kumar6eed9402013-08-06 22:53:11 +05301306
Viresh Kumarbb29ae12015-02-19 17:02:06 +05301307 /* Check if this CPU already has a policy to manage it */
Viresh Kumar9104bb22015-05-12 12:22:12 +05301308 policy = per_cpu(cpufreq_cpu_data, cpu);
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001309 if (policy) {
Viresh Kumar9104bb22015-05-12 12:22:12 +05301310 WARN_ON(!cpumask_test_cpu(cpu, policy->related_cpus));
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001311 if (!policy_is_inactive(policy))
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001312 return cpufreq_add_policy_cpu(policy, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001313
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001314 /* This is the only online CPU for the policy. Start over. */
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001315 new_policy = false;
Rafael J. Wysocki11ce7072015-07-27 23:11:21 +02001316 down_write(&policy->rwsem);
1317 policy->cpu = cpu;
1318 policy->governor = NULL;
1319 up_write(&policy->rwsem);
1320 } else {
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001321 new_policy = true;
Rafael J. Wysockia34e63b2015-07-27 23:11:50 +02001322 policy = cpufreq_policy_alloc(cpu);
Rafael J. Wysocki72368d12013-12-27 01:07:11 +01001323 if (!policy)
Rafael J. Wysockid4d854d2015-07-27 23:11:30 +02001324 return -ENOMEM;
Rafael J. Wysocki72368d12013-12-27 01:07:11 +01001325 }
Srivatsa S. Bhat0d66b912013-09-12 01:42:59 +05301326
Viresh Kumar91a12e92019-02-12 16:36:04 +05301327 if (!new_policy && cpufreq_driver->online) {
1328 ret = cpufreq_driver->online(policy);
1329 if (ret) {
1330 pr_debug("%s: %d: initialization failed\n", __func__,
1331 __LINE__);
1332 goto out_exit_policy;
1333 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001334
Viresh Kumar91a12e92019-02-12 16:36:04 +05301335 /* Recover policy->cpus using related_cpus */
1336 cpumask_copy(policy->cpus, policy->related_cpus);
1337 } else {
1338 cpumask_copy(policy->cpus, cpumask_of(cpu));
Viresh Kumar643ae6e2013-01-12 05:14:38 +00001339
Viresh Kumar91a12e92019-02-12 16:36:04 +05301340 /*
1341 * Call driver. From then on the cpufreq must be able
1342 * to accept all calls to ->verify and ->setpolicy for this CPU.
1343 */
1344 ret = cpufreq_driver->init(policy);
1345 if (ret) {
1346 pr_debug("%s: %d: initialization failed\n", __func__,
1347 __LINE__);
1348 goto out_free_policy;
1349 }
Viresh Kumard417e062018-02-22 11:29:44 +05301350
Viresh Kumar91a12e92019-02-12 16:36:04 +05301351 ret = cpufreq_table_validate_and_sort(policy);
1352 if (ret)
1353 goto out_exit_policy;
Tomeu Vizoso6d4e81e2014-11-24 10:08:03 +01001354
Rafael J. Wysocki4d1f3a52015-07-27 23:11:44 +02001355 /* related_cpus should at least include policy->cpus. */
Viresh Kumar0998a032015-10-15 21:35:21 +05301356 cpumask_copy(policy->related_cpus, policy->cpus);
Rafael J. Wysocki4d1f3a52015-07-27 23:11:44 +02001357 }
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001358
Viresh Kumar91a12e92019-02-12 16:36:04 +05301359 down_write(&policy->rwsem);
Viresh Kumar5a7e56a2014-03-04 11:44:00 +08001360 /*
1361 * affected cpus must always be the one, which are online. We aren't
1362 * managing offline cpus here.
1363 */
1364 cpumask_and(policy->cpus, policy->cpus, cpu_online_mask);
1365
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001366 if (new_policy) {
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001367 for_each_cpu(j, policy->related_cpus) {
Viresh Kumar988bed02015-05-08 11:53:45 +05301368 per_cpu(cpufreq_cpu_data, j) = policy;
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001369 add_cpu_dev_symlink(policy, j);
1370 }
Viresh Kumar18c49922019-07-05 16:21:24 +05301371
1372 policy->min_freq_req = kzalloc(2 * sizeof(*policy->min_freq_req),
1373 GFP_KERNEL);
1374 if (!policy->min_freq_req)
1375 goto out_destroy_policy;
1376
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001377 ret = freq_qos_add_request(&policy->constraints,
1378 policy->min_freq_req, FREQ_QOS_MIN,
1379 policy->min);
Viresh Kumar18c49922019-07-05 16:21:24 +05301380 if (ret < 0) {
1381 /*
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001382 * So we don't call freq_qos_remove_request() for an
Viresh Kumar18c49922019-07-05 16:21:24 +05301383 * uninitialized request.
1384 */
1385 kfree(policy->min_freq_req);
1386 policy->min_freq_req = NULL;
Viresh Kumar18c49922019-07-05 16:21:24 +05301387 goto out_destroy_policy;
1388 }
1389
1390 /*
1391 * This must be initialized right here to avoid calling
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001392 * freq_qos_remove_request() on uninitialized request in case
Viresh Kumar18c49922019-07-05 16:21:24 +05301393 * of errors.
1394 */
1395 policy->max_freq_req = policy->min_freq_req + 1;
1396
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02001397 ret = freq_qos_add_request(&policy->constraints,
1398 policy->max_freq_req, FREQ_QOS_MAX,
1399 policy->max);
Viresh Kumar18c49922019-07-05 16:21:24 +05301400 if (ret < 0) {
1401 policy->max_freq_req = NULL;
Viresh Kumar18c49922019-07-05 16:21:24 +05301402 goto out_destroy_policy;
1403 }
Viresh Kumar6a149032019-07-23 11:44:01 +05301404
1405 blocking_notifier_call_chain(&cpufreq_policy_notifier_list,
1406 CPUFREQ_CREATE_POLICY, policy);
Viresh Kumar988bed02015-05-08 11:53:45 +05301407 }
Viresh Kumar652ed952014-01-09 20:38:43 +05301408
Viresh Kumar5ddc6d42019-06-20 08:35:48 +05301409 if (cpufreq_driver->get && has_target()) {
Viresh Kumarda60ce92013-10-03 20:28:30 +05301410 policy->cur = cpufreq_driver->get(policy->cpu);
1411 if (!policy->cur) {
1412 pr_err("%s: ->get() failed\n", __func__);
Viresh Kumard417e062018-02-22 11:29:44 +05301413 goto out_destroy_policy;
Viresh Kumarda60ce92013-10-03 20:28:30 +05301414 }
1415 }
1416
Viresh Kumard3916692013-12-03 11:20:46 +05301417 /*
1418 * Sometimes boot loaders set CPU frequency to a value outside of
1419 * frequency table present with cpufreq core. In such cases CPU might be
1420 * unstable if it has to run on that frequency for long duration of time
1421 * and so its better to set it to a frequency which is specified in
1422 * freq-table. This also makes cpufreq stats inconsistent as
1423 * cpufreq-stats would fail to register because current frequency of CPU
1424 * isn't found in freq-table.
1425 *
1426 * Because we don't want this change to effect boot process badly, we go
1427 * for the next freq which is >= policy->cur ('cur' must be set by now,
1428 * otherwise we will end up setting freq to lowest of the table as 'cur'
1429 * is initialized to zero).
1430 *
1431 * We are passing target-freq as "policy->cur - 1" otherwise
1432 * __cpufreq_driver_target() would simply fail, as policy->cur will be
1433 * equal to target-freq.
1434 */
1435 if ((cpufreq_driver->flags & CPUFREQ_NEED_INITIAL_FREQ_CHECK)
1436 && has_target()) {
1437 /* Are we running at unknown frequency ? */
1438 ret = cpufreq_frequency_table_get_index(policy, policy->cur);
1439 if (ret == -EINVAL) {
1440 /* Warn user and fix it */
1441 pr_warn("%s: CPU%d: Running at unlisted freq: %u KHz\n",
1442 __func__, policy->cpu, policy->cur);
1443 ret = __cpufreq_driver_target(policy, policy->cur - 1,
1444 CPUFREQ_RELATION_L);
1445
1446 /*
1447 * Reaching here after boot in a few seconds may not
1448 * mean that system will remain stable at "unknown"
1449 * frequency for longer duration. Hence, a BUG_ON().
1450 */
1451 BUG_ON(ret);
1452 pr_warn("%s: CPU%d: Unlisted initial frequency changed to: %u KHz\n",
1453 __func__, policy->cpu, policy->cur);
1454 }
1455 }
1456
Rafael J. Wysocki194d99c2015-07-29 03:08:57 +02001457 if (new_policy) {
Rafael J. Wysockid9612a42015-07-27 23:11:37 +02001458 ret = cpufreq_add_dev_interface(policy);
Srivatsa S. Bhata82fab22013-07-30 04:24:49 +05301459 if (ret)
Viresh Kumard417e062018-02-22 11:29:44 +05301460 goto out_destroy_policy;
Rafael J. Wysocki1aefc752016-05-31 22:14:44 +02001461
1462 cpufreq_stats_create_table(policy);
Dave Jones8ff69732006-03-05 03:37:23 -05001463
Viresh Kumar988bed02015-05-08 11:53:45 +05301464 write_lock_irqsave(&cpufreq_driver_lock, flags);
1465 list_add(&policy->policy_list, &cpufreq_policy_list);
1466 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
1467 }
Viresh Kumar9515f4d2013-08-20 12:08:23 +05301468
Viresh Kumar7f0fa402015-07-08 15:12:16 +05301469 ret = cpufreq_init_policy(policy);
1470 if (ret) {
1471 pr_err("%s: Failed to initialize policy for cpu: %d (%d)\n",
1472 __func__, cpu, ret);
Viresh Kumard417e062018-02-22 11:29:44 +05301473 goto out_destroy_policy;
Viresh Kumar08fd8c1c2013-12-24 07:11:01 +05301474 }
Srivatsa S. Bhate18f1682013-07-30 04:24:23 +05301475
Viresh Kumar4e97b632014-03-04 11:44:01 +08001476 up_write(&policy->rwsem);
Viresh Kumar08fd8c1c2013-12-24 07:11:01 +05301477
Greg Kroah-Hartman038c5b32007-12-17 15:54:39 -04001478 kobject_uevent(&policy->kobj, KOBJ_ADD);
Viresh Kumar7c45cf32014-11-27 06:07:51 +05301479
Viresh Kumar7c45cf32014-11-27 06:07:51 +05301480 /* Callback for handling stuff after policy is ready */
1481 if (cpufreq_driver->ready)
1482 cpufreq_driver->ready(policy);
1483
Daniel Lezcanobcc61562019-06-25 13:32:41 +02001484 if (cpufreq_thermal_control_enabled(cpufreq_driver))
Amit Kucheria5c238a82019-01-30 10:52:01 +05301485 policy->cdev = of_cpufreq_cooling_register(policy);
1486
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02001487 pr_debug("initialization complete\n");
Dave Jones87c32272006-03-29 01:48:37 -05001488
Linus Torvalds1da177e2005-04-16 15:20:36 -07001489 return 0;
1490
Viresh Kumard417e062018-02-22 11:29:44 +05301491out_destroy_policy:
Viresh Kumarb24b6472018-02-22 11:29:43 +05301492 for_each_cpu(j, policy->real_cpus)
1493 remove_cpu_dev_symlink(policy, get_cpu_device(j));
1494
Prarit Bhargava7106e022014-09-10 10:12:08 -04001495 up_write(&policy->rwsem);
1496
Viresh Kumard417e062018-02-22 11:29:44 +05301497out_exit_policy:
Viresh Kumarda60ce92013-10-03 20:28:30 +05301498 if (cpufreq_driver->exit)
1499 cpufreq_driver->exit(policy);
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001500
Viresh Kumar8101f992015-07-08 15:12:15 +05301501out_free_policy:
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301502 cpufreq_policy_free(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001503 return ret;
1504}
1505
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001506/**
1507 * cpufreq_add_dev - the cpufreq interface for a CPU device.
1508 * @dev: CPU device.
1509 * @sif: Subsystem interface structure pointer (not used)
1510 */
1511static int cpufreq_add_dev(struct device *dev, struct subsys_interface *sif)
1512{
Rafael J. Wysockia794d612016-04-07 03:31:57 +02001513 struct cpufreq_policy *policy;
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001514 unsigned cpu = dev->id;
Viresh Kumar266198042016-09-12 12:07:05 +05301515 int ret;
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001516
1517 dev_dbg(dev, "%s: adding CPU%u\n", __func__, cpu);
1518
Viresh Kumar266198042016-09-12 12:07:05 +05301519 if (cpu_online(cpu)) {
1520 ret = cpufreq_online(cpu);
1521 if (ret)
1522 return ret;
1523 }
Rafael J. Wysocki0b275352015-07-29 03:03:44 +02001524
Viresh Kumar266198042016-09-12 12:07:05 +05301525 /* Create sysfs link on CPU registration */
Rafael J. Wysockia794d612016-04-07 03:31:57 +02001526 policy = per_cpu(cpufreq_cpu_data, cpu);
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001527 if (policy)
1528 add_cpu_dev_symlink(policy, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529
Rafael J. Wysocki2f0ba792017-03-27 19:33:09 +02001530 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001531}
1532
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02001533static int cpufreq_offline(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001534{
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05301535 struct cpufreq_policy *policy;
Viresh Kumar69cee712016-02-11 17:31:11 +05301536 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001537
Viresh Kumarb8eed8a2013-01-14 13:23:03 +00001538 pr_debug("%s: unregistering CPU %u\n", __func__, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001539
Viresh Kumar988bed02015-05-08 11:53:45 +05301540 policy = cpufreq_cpu_get_raw(cpu);
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05301541 if (!policy) {
Viresh Kumarb8eed8a2013-01-14 13:23:03 +00001542 pr_debug("%s: No cpu_data found\n", __func__);
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02001543 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001545
Viresh Kumar49f18562016-02-11 17:31:12 +05301546 down_write(&policy->rwsem);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02001547 if (has_target())
1548 cpufreq_stop_governor(policy);
Jacob Shin27ecddc2011-04-27 13:32:11 -05001549
Viresh Kumar9591bec2015-06-10 02:20:23 +02001550 cpumask_clear_cpu(cpu, policy->cpus);
Viresh Kumar45732372015-05-12 12:22:34 +05301551
Viresh Kumar9591bec2015-06-10 02:20:23 +02001552 if (policy_is_inactive(policy)) {
1553 if (has_target())
1554 strncpy(policy->last_governor, policy->governor->name,
1555 CPUFREQ_NAME_LEN);
Srinivas Pandruvada69030dd12015-12-01 16:52:14 -08001556 else
1557 policy->last_policy = policy->policy;
Viresh Kumar9591bec2015-06-10 02:20:23 +02001558 } else if (cpu == policy->cpu) {
1559 /* Nominate new CPU */
1560 policy->cpu = cpumask_any(policy->cpus);
1561 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562
Viresh Kumar9591bec2015-06-10 02:20:23 +02001563 /* Start governor again for active policy */
1564 if (!policy_is_inactive(policy)) {
1565 if (has_target()) {
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01001566 ret = cpufreq_start_governor(policy);
Viresh Kumar9591bec2015-06-10 02:20:23 +02001567 if (ret)
1568 pr_err("%s: Failed to start governor\n", __func__);
1569 }
Viresh Kumar69cee712016-02-11 17:31:11 +05301570
Viresh Kumar49f18562016-02-11 17:31:12 +05301571 goto unlock;
Viresh Kumar69cee712016-02-11 17:31:11 +05301572 }
1573
Daniel Lezcanobcc61562019-06-25 13:32:41 +02001574 if (cpufreq_thermal_control_enabled(cpufreq_driver)) {
Amit Kucheria5c238a82019-01-30 10:52:01 +05301575 cpufreq_cooling_unregister(policy->cdev);
1576 policy->cdev = NULL;
1577 }
1578
Viresh Kumar69cee712016-02-11 17:31:11 +05301579 if (cpufreq_driver->stop_cpu)
Dirk Brandewie367dc4a2014-03-19 08:45:53 -07001580 cpufreq_driver->stop_cpu(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02001582 if (has_target())
1583 cpufreq_exit_governor(policy);
Viresh Kumar87549142015-06-10 02:13:21 +02001584
Viresh Kumar87549142015-06-10 02:13:21 +02001585 /*
Viresh Kumar91a12e92019-02-12 16:36:04 +05301586 * Perform the ->offline() during light-weight tear-down, as
1587 * that allows fast recovery when the CPU comes back.
Viresh Kumar87549142015-06-10 02:13:21 +02001588 */
Viresh Kumar91a12e92019-02-12 16:36:04 +05301589 if (cpufreq_driver->offline) {
1590 cpufreq_driver->offline(policy);
1591 } else if (cpufreq_driver->exit) {
Viresh Kumar87549142015-06-10 02:13:21 +02001592 cpufreq_driver->exit(policy);
Srinivas Pandruvada55582bc2015-10-07 13:50:44 -07001593 policy->freq_table = NULL;
1594 }
Viresh Kumar49f18562016-02-11 17:31:12 +05301595
1596unlock:
1597 up_write(&policy->rwsem);
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02001598 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001599}
1600
Srivatsa S. Bhatcedb70a2013-09-07 01:23:09 +05301601/**
Viresh Kumar27a862e2013-10-02 14:13:14 +05301602 * cpufreq_remove_dev - remove a CPU device
Srivatsa S. Bhatcedb70a2013-09-07 01:23:09 +05301603 *
1604 * Removes the cpufreq interface for a CPU device.
Srivatsa S. Bhatcedb70a2013-09-07 01:23:09 +05301605 */
Viresh Kumar71db87b2015-07-30 15:04:01 +05301606static void cpufreq_remove_dev(struct device *dev, struct subsys_interface *sif)
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001607{
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001608 unsigned int cpu = dev->id;
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001609 struct cpufreq_policy *policy = per_cpu(cpufreq_cpu_data, cpu);
Venki Pallipadiec282972007-03-26 12:03:19 -07001610
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001611 if (!policy)
Linus Torvalds1af115d2015-08-31 08:47:40 -07001612 return;
Viresh Kumar87549142015-06-10 02:13:21 +02001613
Viresh Kumar69cee712016-02-11 17:31:11 +05301614 if (cpu_online(cpu))
1615 cpufreq_offline(cpu);
Viresh Kumar87549142015-06-10 02:13:21 +02001616
Rafael J. Wysocki559ed402015-07-26 02:07:47 +02001617 cpumask_clear_cpu(cpu, policy->real_cpus);
Viresh Kumar266198042016-09-12 12:07:05 +05301618 remove_cpu_dev_symlink(policy, dev);
Viresh Kumarf344dae2015-11-21 09:06:49 +05301619
Viresh Kumar91a12e92019-02-12 16:36:04 +05301620 if (cpumask_empty(policy->real_cpus)) {
1621 /* We did light-weight exit earlier, do full tear down now */
1622 if (cpufreq_driver->offline)
1623 cpufreq_driver->exit(policy);
1624
Viresh Kumarf9f41e3e2017-01-05 10:17:27 +05301625 cpufreq_policy_free(policy);
Viresh Kumar91a12e92019-02-12 16:36:04 +05301626 }
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001627}
1628
Linus Torvalds1da177e2005-04-16 15:20:36 -07001629/**
Viresh Kumarbb176f72013-06-19 14:19:33 +05301630 * cpufreq_out_of_sync - If actual and saved CPU frequency differs, we're
1631 * in deep trouble.
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301632 * @policy: policy managing CPUs
Linus Torvalds1da177e2005-04-16 15:20:36 -07001633 * @new_freq: CPU frequency the CPU actually runs at
1634 *
Dave Jones29464f22009-01-18 01:37:11 -05001635 * We adjust to current frequency first, and need to clean up later.
1636 * So either call to cpufreq_update_policy() or schedule handle_update()).
Linus Torvalds1da177e2005-04-16 15:20:36 -07001637 */
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301638static void cpufreq_out_of_sync(struct cpufreq_policy *policy,
Gautham R Shenoye08f5f52006-10-26 16:20:58 +05301639 unsigned int new_freq)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001640{
1641 struct cpufreq_freqs freqs;
Viresh Kumarb43a7ff2013-03-24 11:56:43 +05301642
Joe Perchese837f9b2014-03-11 10:03:00 -07001643 pr_debug("Warning: CPU frequency out of sync: cpufreq and timing core thinks of %u, is %u kHz\n",
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301644 policy->cur, new_freq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645
Viresh Kumara1e1dc42015-01-02 12:34:28 +05301646 freqs.old = policy->cur;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001647 freqs.new = new_freq;
Viresh Kumarb43a7ff2013-03-24 11:56:43 +05301648
Viresh Kumar8fec0512014-03-24 13:35:45 +05301649 cpufreq_freq_transition_begin(policy, &freqs);
1650 cpufreq_freq_transition_end(policy, &freqs, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001651}
1652
Viresh Kumar59807522019-06-20 08:35:49 +05301653static unsigned int cpufreq_verify_current_freq(struct cpufreq_policy *policy, bool update)
1654{
1655 unsigned int new_freq;
1656
1657 new_freq = cpufreq_driver->get(policy->cpu);
1658 if (!new_freq)
1659 return 0;
1660
1661 /*
1662 * If fast frequency switching is used with the given policy, the check
1663 * against policy->cur is pointless, so skip it in that case.
1664 */
1665 if (policy->fast_switch_enabled || !has_target())
1666 return new_freq;
1667
1668 if (policy->cur != new_freq) {
1669 cpufreq_out_of_sync(policy, new_freq);
1670 if (update)
1671 schedule_work(&policy->update);
1672 }
1673
1674 return new_freq;
1675}
1676
Dave Jones32ee8c32006-02-28 00:43:23 -05001677/**
Dhaval Giani4ab70df2006-12-13 14:49:15 +05301678 * cpufreq_quick_get - get the CPU frequency (in kHz) from policy->cur
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001679 * @cpu: CPU number
1680 *
1681 * This is the last known freq, without actually getting it from the driver.
1682 * Return value will be same as what is shown in scaling_cur_freq in sysfs.
1683 */
1684unsigned int cpufreq_quick_get(unsigned int cpu)
1685{
Dirk Brandewie9e21ba82013-02-06 09:02:08 -08001686 struct cpufreq_policy *policy;
Gautham R Shenoye08f5f52006-10-26 16:20:58 +05301687 unsigned int ret_freq = 0;
Richard Cochranc75361c2016-03-11 09:43:07 +01001688 unsigned long flags;
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001689
Richard Cochranc75361c2016-03-11 09:43:07 +01001690 read_lock_irqsave(&cpufreq_driver_lock, flags);
1691
1692 if (cpufreq_driver && cpufreq_driver->setpolicy && cpufreq_driver->get) {
1693 ret_freq = cpufreq_driver->get(cpu);
1694 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
1695 return ret_freq;
1696 }
1697
1698 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Dirk Brandewie9e21ba82013-02-06 09:02:08 -08001699
1700 policy = cpufreq_cpu_get(cpu);
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001701 if (policy) {
Gautham R Shenoye08f5f52006-10-26 16:20:58 +05301702 ret_freq = policy->cur;
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001703 cpufreq_cpu_put(policy);
1704 }
1705
Dave Jones4d34a672008-02-07 16:33:49 -05001706 return ret_freq;
Venkatesh Pallipadi95235ca2005-12-02 10:43:20 -08001707}
1708EXPORT_SYMBOL(cpufreq_quick_get);
1709
Jesse Barnes3d737102011-06-28 10:59:12 -07001710/**
1711 * cpufreq_quick_get_max - get the max reported CPU frequency for this CPU
1712 * @cpu: CPU number
1713 *
1714 * Just return the max possible frequency for a given CPU.
1715 */
1716unsigned int cpufreq_quick_get_max(unsigned int cpu)
1717{
1718 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
1719 unsigned int ret_freq = 0;
1720
1721 if (policy) {
1722 ret_freq = policy->max;
1723 cpufreq_cpu_put(policy);
1724 }
1725
1726 return ret_freq;
1727}
1728EXPORT_SYMBOL(cpufreq_quick_get_max);
1729
Viresh Kumard92d50a2015-01-02 12:34:29 +05301730static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001731{
Yue Hu4db7c342019-04-19 14:27:58 +08001732 if (unlikely(policy_is_inactive(policy)))
Viresh Kumar59807522019-06-20 08:35:49 +05301733 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001734
Viresh Kumar59807522019-06-20 08:35:49 +05301735 return cpufreq_verify_current_freq(policy, true);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001736}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001737
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001738/**
1739 * cpufreq_get - get the current CPU frequency (in kHz)
1740 * @cpu: CPU number
1741 *
1742 * Get the CPU current (static) CPU frequency
1743 */
1744unsigned int cpufreq_get(unsigned int cpu)
1745{
Aaron Plattner999976e2014-03-04 12:42:15 -08001746 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001747 unsigned int ret_freq = 0;
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08001748
Aaron Plattner999976e2014-03-04 12:42:15 -08001749 if (policy) {
1750 down_read(&policy->rwsem);
Yue Hu4db7c342019-04-19 14:27:58 +08001751 if (cpufreq_driver->get)
1752 ret_freq = __cpufreq_get(policy);
Aaron Plattner999976e2014-03-04 12:42:15 -08001753 up_read(&policy->rwsem);
Viresh Kumar26ca8692013-09-20 22:37:31 +05301754
Aaron Plattner999976e2014-03-04 12:42:15 -08001755 cpufreq_cpu_put(policy);
1756 }
Viresh Kumar6eed9402013-08-06 22:53:11 +05301757
Dave Jones4d34a672008-02-07 16:33:49 -05001758 return ret_freq;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001759}
1760EXPORT_SYMBOL(cpufreq_get);
1761
Kay Sievers8a25a2f2011-12-21 14:29:42 -08001762static struct subsys_interface cpufreq_interface = {
1763 .name = "cpufreq",
1764 .subsys = &cpu_subsys,
1765 .add_dev = cpufreq_add_dev,
1766 .remove_dev = cpufreq_remove_dev,
Rafael J. Wysockie00e56d2011-03-23 22:16:32 +01001767};
1768
Viresh Kumare28867e2014-03-04 11:00:27 +08001769/*
1770 * In case platform wants some specific frequency to be configured
1771 * during suspend..
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001772 */
Viresh Kumare28867e2014-03-04 11:00:27 +08001773int cpufreq_generic_suspend(struct cpufreq_policy *policy)
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001774{
Viresh Kumare28867e2014-03-04 11:00:27 +08001775 int ret;
Dave Jones4bc5d342009-08-04 14:03:25 -04001776
Viresh Kumare28867e2014-03-04 11:00:27 +08001777 if (!policy->suspend_freq) {
Bartlomiej Zolnierkiewicz201f3712015-09-08 18:41:02 +02001778 pr_debug("%s: suspend_freq not defined\n", __func__);
1779 return 0;
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001780 }
1781
Viresh Kumare28867e2014-03-04 11:00:27 +08001782 pr_debug("%s: Setting suspend-freq: %u\n", __func__,
1783 policy->suspend_freq);
1784
1785 ret = __cpufreq_driver_target(policy, policy->suspend_freq,
1786 CPUFREQ_RELATION_H);
1787 if (ret)
1788 pr_err("%s: unable to set suspend-freq: %u. err: %d\n",
1789 __func__, policy->suspend_freq, ret);
1790
Dave Jonesc9060492008-02-07 16:32:18 -05001791 return ret;
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001792}
Viresh Kumare28867e2014-03-04 11:00:27 +08001793EXPORT_SYMBOL(cpufreq_generic_suspend);
Benjamin Herrenschmidt42d4dc32005-04-29 07:40:12 -07001794
1795/**
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001796 * cpufreq_suspend() - Suspend CPUFreq governors
Linus Torvalds1da177e2005-04-16 15:20:36 -07001797 *
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001798 * Called during system wide Suspend/Hibernate cycles for suspending governors
1799 * as some platforms can't change frequency after this point in suspend cycle.
1800 * Because some of the devices (like: i2c, regulators, etc) they use for
1801 * changing frequency are suspended quickly after this point.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802 */
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001803void cpufreq_suspend(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001804{
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05301805 struct cpufreq_policy *policy;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001806
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001807 if (!cpufreq_driver)
Rafael J. Wysockie00e56d2011-03-23 22:16:32 +01001808 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001809
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001810 if (!has_target() && !cpufreq_driver->suspend)
Viresh Kumarb1b12bab2014-09-30 09:33:17 +05301811 goto suspend;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001812
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001813 pr_debug("%s: Suspending Governors\n", __func__);
1814
Viresh Kumarf9637352015-05-12 12:20:11 +05301815 for_each_active_policy(policy) {
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001816 if (has_target()) {
1817 down_write(&policy->rwsem);
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02001818 cpufreq_stop_governor(policy);
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001819 up_write(&policy->rwsem);
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001820 }
1821
1822 if (cpufreq_driver->suspend && cpufreq_driver->suspend(policy))
Florian Fainellie9a7cc12019-08-21 16:16:32 -07001823 pr_err("%s: Failed to suspend driver: %s\n", __func__,
1824 cpufreq_driver->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001825 }
Viresh Kumarb1b12bab2014-09-30 09:33:17 +05301826
1827suspend:
1828 cpufreq_suspended = true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001829}
1830
Linus Torvalds1da177e2005-04-16 15:20:36 -07001831/**
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001832 * cpufreq_resume() - Resume CPUFreq governors
Linus Torvalds1da177e2005-04-16 15:20:36 -07001833 *
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001834 * Called during system wide Suspend/Hibernate cycle for resuming governors that
1835 * are suspended with cpufreq_suspend().
Linus Torvalds1da177e2005-04-16 15:20:36 -07001836 */
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001837void cpufreq_resume(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001838{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001839 struct cpufreq_policy *policy;
Viresh Kumar49f18562016-02-11 17:31:12 +05301840 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001841
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001842 if (!cpufreq_driver)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001843 return;
1844
Bo Yan703cbaa2018-01-23 13:57:55 -08001845 if (unlikely(!cpufreq_suspended))
1846 return;
1847
Lan Tianyu8e304442014-09-18 15:03:07 +08001848 cpufreq_suspended = false;
1849
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001850 if (!has_target() && !cpufreq_driver->resume)
Viresh Kumar2f0aea92014-03-04 11:00:26 +08001851 return;
1852
1853 pr_debug("%s: Resuming Governors\n", __func__);
1854
Viresh Kumarf9637352015-05-12 12:20:11 +05301855 for_each_active_policy(policy) {
Viresh Kumar49f18562016-02-11 17:31:12 +05301856 if (cpufreq_driver->resume && cpufreq_driver->resume(policy)) {
Viresh Kumar0c5aa402014-03-24 12:30:29 +05301857 pr_err("%s: Failed to resume driver: %p\n", __func__,
1858 policy);
Rafael J. Wysockiba41e1b2016-05-02 02:27:19 +02001859 } else if (has_target()) {
Viresh Kumar49f18562016-02-11 17:31:12 +05301860 down_write(&policy->rwsem);
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01001861 ret = cpufreq_start_governor(policy);
Viresh Kumar49f18562016-02-11 17:31:12 +05301862 up_write(&policy->rwsem);
1863
1864 if (ret)
1865 pr_err("%s: Failed to start governor for policy: %p\n",
1866 __func__, policy);
1867 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001868 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870
Borislav Petkov9d950462013-01-20 10:24:28 +00001871/**
1872 * cpufreq_get_current_driver - return current driver's name
1873 *
1874 * Return the name string of the currently loaded cpufreq driver
1875 * or NULL, if none.
1876 */
1877const char *cpufreq_get_current_driver(void)
1878{
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02001879 if (cpufreq_driver)
1880 return cpufreq_driver->name;
1881
1882 return NULL;
Borislav Petkov9d950462013-01-20 10:24:28 +00001883}
1884EXPORT_SYMBOL_GPL(cpufreq_get_current_driver);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001885
Thomas Petazzoni51315cd2014-10-19 11:30:27 +02001886/**
1887 * cpufreq_get_driver_data - return current driver data
1888 *
1889 * Return the private data of the currently loaded cpufreq
1890 * driver, or NULL if no cpufreq driver is loaded.
1891 */
1892void *cpufreq_get_driver_data(void)
1893{
1894 if (cpufreq_driver)
1895 return cpufreq_driver->driver_data;
1896
1897 return NULL;
1898}
1899EXPORT_SYMBOL_GPL(cpufreq_get_driver_data);
1900
Linus Torvalds1da177e2005-04-16 15:20:36 -07001901/*********************************************************************
1902 * NOTIFIER LISTS INTERFACE *
1903 *********************************************************************/
1904
1905/**
1906 * cpufreq_register_notifier - register a driver with cpufreq
1907 * @nb: notifier function to register
1908 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
1909 *
Dave Jones32ee8c32006-02-28 00:43:23 -05001910 * Add a driver to one of two lists: either a list of drivers that
Linus Torvalds1da177e2005-04-16 15:20:36 -07001911 * are notified about clock rate changes (once before and once after
1912 * the transition), or a list of drivers that are notified about
1913 * changes in cpufreq policy.
1914 *
1915 * This function may sleep, and has the same return conditions as
Alan Sterne041c682006-03-27 01:16:30 -08001916 * blocking_notifier_chain_register.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001917 */
1918int cpufreq_register_notifier(struct notifier_block *nb, unsigned int list)
1919{
1920 int ret;
1921
Dirk Brandewied5aaffa2013-01-17 16:22:21 +00001922 if (cpufreq_disabled())
1923 return -EINVAL;
1924
Linus Torvalds1da177e2005-04-16 15:20:36 -07001925 switch (list) {
1926 case CPUFREQ_TRANSITION_NOTIFIER:
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02001927 mutex_lock(&cpufreq_fast_switch_lock);
1928
1929 if (cpufreq_fast_switch_count > 0) {
1930 mutex_unlock(&cpufreq_fast_switch_lock);
1931 return -EBUSY;
1932 }
Alan Sternb4dfdbb2006-10-04 02:17:06 -07001933 ret = srcu_notifier_chain_register(
Alan Sterne041c682006-03-27 01:16:30 -08001934 &cpufreq_transition_notifier_list, nb);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02001935 if (!ret)
1936 cpufreq_fast_switch_count--;
1937
1938 mutex_unlock(&cpufreq_fast_switch_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001939 break;
1940 case CPUFREQ_POLICY_NOTIFIER:
Alan Sterne041c682006-03-27 01:16:30 -08001941 ret = blocking_notifier_chain_register(
1942 &cpufreq_policy_notifier_list, nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001943 break;
1944 default:
1945 ret = -EINVAL;
1946 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001947
1948 return ret;
1949}
1950EXPORT_SYMBOL(cpufreq_register_notifier);
1951
Linus Torvalds1da177e2005-04-16 15:20:36 -07001952/**
1953 * cpufreq_unregister_notifier - unregister a driver with cpufreq
1954 * @nb: notifier block to be unregistered
Viresh Kumarbb176f72013-06-19 14:19:33 +05301955 * @list: CPUFREQ_TRANSITION_NOTIFIER or CPUFREQ_POLICY_NOTIFIER
Linus Torvalds1da177e2005-04-16 15:20:36 -07001956 *
1957 * Remove a driver from the CPU frequency notifier list.
1958 *
1959 * This function may sleep, and has the same return conditions as
Alan Sterne041c682006-03-27 01:16:30 -08001960 * blocking_notifier_chain_unregister.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001961 */
1962int cpufreq_unregister_notifier(struct notifier_block *nb, unsigned int list)
1963{
1964 int ret;
1965
Dirk Brandewied5aaffa2013-01-17 16:22:21 +00001966 if (cpufreq_disabled())
1967 return -EINVAL;
1968
Linus Torvalds1da177e2005-04-16 15:20:36 -07001969 switch (list) {
1970 case CPUFREQ_TRANSITION_NOTIFIER:
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02001971 mutex_lock(&cpufreq_fast_switch_lock);
1972
Alan Sternb4dfdbb2006-10-04 02:17:06 -07001973 ret = srcu_notifier_chain_unregister(
Alan Sterne041c682006-03-27 01:16:30 -08001974 &cpufreq_transition_notifier_list, nb);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02001975 if (!ret && !WARN_ON(cpufreq_fast_switch_count >= 0))
1976 cpufreq_fast_switch_count++;
1977
1978 mutex_unlock(&cpufreq_fast_switch_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001979 break;
1980 case CPUFREQ_POLICY_NOTIFIER:
Alan Sterne041c682006-03-27 01:16:30 -08001981 ret = blocking_notifier_chain_unregister(
1982 &cpufreq_policy_notifier_list, nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001983 break;
1984 default:
1985 ret = -EINVAL;
1986 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001987
1988 return ret;
1989}
1990EXPORT_SYMBOL(cpufreq_unregister_notifier);
1991
1992
1993/*********************************************************************
1994 * GOVERNORS *
1995 *********************************************************************/
1996
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02001997/**
1998 * cpufreq_driver_fast_switch - Carry out a fast CPU frequency switch.
1999 * @policy: cpufreq policy to switch the frequency for.
2000 * @target_freq: New frequency to set (may be approximate).
2001 *
2002 * Carry out a fast frequency switch without sleeping.
2003 *
2004 * The driver's ->fast_switch() callback invoked by this function must be
2005 * suitable for being called from within RCU-sched read-side critical sections
2006 * and it is expected to select the minimum available frequency greater than or
2007 * equal to @target_freq (CPUFREQ_RELATION_L).
2008 *
2009 * This function must not be called if policy->fast_switch_enabled is unset.
2010 *
2011 * Governors calling this function must guarantee that it will never be invoked
2012 * twice in parallel for the same policy and that it will never be called in
2013 * parallel with either ->target() or ->target_index() for the same policy.
2014 *
Viresh Kumar209887e2017-08-09 10:21:46 +05302015 * Returns the actual frequency set for the CPU.
2016 *
2017 * If 0 is returned by the driver's ->fast_switch() callback to indicate an
2018 * error condition, the hardware configuration must be preserved.
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002019 */
2020unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy,
2021 unsigned int target_freq)
2022{
Rafael J. Wysockib9af6942016-06-01 22:36:26 +02002023 target_freq = clamp_val(target_freq, policy->min, policy->max);
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002024
2025 return cpufreq_driver->fast_switch(policy, target_freq);
2026}
2027EXPORT_SYMBOL_GPL(cpufreq_driver_fast_switch);
2028
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302029/* Must set freqs->new to intermediate frequency */
2030static int __target_intermediate(struct cpufreq_policy *policy,
2031 struct cpufreq_freqs *freqs, int index)
2032{
2033 int ret;
2034
2035 freqs->new = cpufreq_driver->get_intermediate(policy, index);
2036
2037 /* We don't need to switch to intermediate freq */
2038 if (!freqs->new)
2039 return 0;
2040
2041 pr_debug("%s: cpu: %d, switching to intermediate freq: oldfreq: %u, intermediate freq: %u\n",
2042 __func__, policy->cpu, freqs->old, freqs->new);
2043
2044 cpufreq_freq_transition_begin(policy, freqs);
2045 ret = cpufreq_driver->target_intermediate(policy, index);
2046 cpufreq_freq_transition_end(policy, freqs, ret);
2047
2048 if (ret)
2049 pr_err("%s: Failed to change to intermediate frequency: %d\n",
2050 __func__, ret);
2051
2052 return ret;
2053}
2054
Viresh Kumar23727842016-06-03 10:58:50 +05302055static int __target_index(struct cpufreq_policy *policy, int index)
Viresh Kumar8d657752014-05-21 14:29:29 +05302056{
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302057 struct cpufreq_freqs freqs = {.old = policy->cur, .flags = 0};
2058 unsigned int intermediate_freq = 0;
Viresh Kumar23727842016-06-03 10:58:50 +05302059 unsigned int newfreq = policy->freq_table[index].frequency;
Viresh Kumar8d657752014-05-21 14:29:29 +05302060 int retval = -EINVAL;
2061 bool notify;
2062
Viresh Kumar23727842016-06-03 10:58:50 +05302063 if (newfreq == policy->cur)
2064 return 0;
2065
Viresh Kumar8d657752014-05-21 14:29:29 +05302066 notify = !(cpufreq_driver->flags & CPUFREQ_ASYNC_NOTIFICATION);
Viresh Kumar8d657752014-05-21 14:29:29 +05302067 if (notify) {
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302068 /* Handle switching to intermediate frequency */
2069 if (cpufreq_driver->get_intermediate) {
2070 retval = __target_intermediate(policy, &freqs, index);
2071 if (retval)
2072 return retval;
Viresh Kumar8d657752014-05-21 14:29:29 +05302073
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302074 intermediate_freq = freqs.new;
2075 /* Set old freq to intermediate */
2076 if (intermediate_freq)
2077 freqs.old = freqs.new;
2078 }
2079
Viresh Kumar23727842016-06-03 10:58:50 +05302080 freqs.new = newfreq;
Viresh Kumar8d657752014-05-21 14:29:29 +05302081 pr_debug("%s: cpu: %d, oldfreq: %u, new freq: %u\n",
2082 __func__, policy->cpu, freqs.old, freqs.new);
2083
2084 cpufreq_freq_transition_begin(policy, &freqs);
2085 }
2086
2087 retval = cpufreq_driver->target_index(policy, index);
2088 if (retval)
2089 pr_err("%s: Failed to change cpu frequency: %d\n", __func__,
2090 retval);
2091
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302092 if (notify) {
Viresh Kumar8d657752014-05-21 14:29:29 +05302093 cpufreq_freq_transition_end(policy, &freqs, retval);
2094
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302095 /*
2096 * Failed after setting to intermediate freq? Driver should have
2097 * reverted back to initial frequency and so should we. Check
2098 * here for intermediate_freq instead of get_intermediate, in
Shailendra Verma58405af2015-05-22 22:48:22 +05302099 * case we haven't switched to intermediate freq at all.
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302100 */
2101 if (unlikely(retval && intermediate_freq)) {
2102 freqs.old = intermediate_freq;
2103 freqs.new = policy->restore_freq;
2104 cpufreq_freq_transition_begin(policy, &freqs);
2105 cpufreq_freq_transition_end(policy, &freqs, 0);
2106 }
2107 }
2108
Viresh Kumar8d657752014-05-21 14:29:29 +05302109 return retval;
2110}
2111
Linus Torvalds1da177e2005-04-16 15:20:36 -07002112int __cpufreq_driver_target(struct cpufreq_policy *policy,
2113 unsigned int target_freq,
2114 unsigned int relation)
2115{
Viresh Kumar72499242012-10-31 01:28:21 +01002116 unsigned int old_target_freq = target_freq;
Viresh Kumard218ed72016-06-03 10:58:51 +05302117 int index;
Ashok Rajc32b6b82005-10-30 14:59:54 -08002118
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002119 if (cpufreq_disabled())
2120 return -ENODEV;
2121
Viresh Kumar72499242012-10-31 01:28:21 +01002122 /* Make sure that target_freq is within supported range */
Viresh Kumar910c6e82016-05-26 11:27:24 +05302123 target_freq = clamp_val(target_freq, policy->min, policy->max);
Viresh Kumar72499242012-10-31 01:28:21 +01002124
2125 pr_debug("target for CPU %u: %u kHz, relation %u, requested %u kHz\n",
Joe Perchese837f9b2014-03-11 10:03:00 -07002126 policy->cpu, target_freq, relation, old_target_freq);
Viresh Kumar5a1c0222012-10-31 01:28:15 +01002127
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302128 /*
2129 * This might look like a redundant call as we are checking it again
2130 * after finding index. But it is left intentionally for cases where
2131 * exactly same freq is called again and so we can save on few function
2132 * calls.
2133 */
Viresh Kumar5a1c0222012-10-31 01:28:15 +01002134 if (target_freq == policy->cur)
2135 return 0;
2136
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302137 /* Save last value to restore later on errors */
2138 policy->restore_freq = policy->cur;
2139
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002140 if (cpufreq_driver->target)
Rafael J. Wysocki6019d232016-02-26 00:03:01 +01002141 return cpufreq_driver->target(policy, target_freq, relation);
Ashok Raj90d45d12005-11-08 21:34:24 -08002142
Rafael J. Wysocki6019d232016-02-26 00:03:01 +01002143 if (!cpufreq_driver->target_index)
2144 return -EINVAL;
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302145
Viresh Kumard218ed72016-06-03 10:58:51 +05302146 index = cpufreq_frequency_table_target(policy, target_freq, relation);
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302147
Viresh Kumar23727842016-06-03 10:58:50 +05302148 return __target_index(policy, index);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002149}
2150EXPORT_SYMBOL_GPL(__cpufreq_driver_target);
2151
Linus Torvalds1da177e2005-04-16 15:20:36 -07002152int cpufreq_driver_target(struct cpufreq_policy *policy,
2153 unsigned int target_freq,
2154 unsigned int relation)
2155{
Colin Ian King62c23a82019-08-13 13:21:21 +01002156 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002157
viresh kumarad7722d2013-10-18 19:10:15 +05302158 down_write(&policy->rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002159
2160 ret = __cpufreq_driver_target(policy, target_freq, relation);
2161
viresh kumarad7722d2013-10-18 19:10:15 +05302162 up_write(&policy->rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002163
Linus Torvalds1da177e2005-04-16 15:20:36 -07002164 return ret;
2165}
2166EXPORT_SYMBOL_GPL(cpufreq_driver_target);
2167
Rafael J. Wysockide1df262016-02-05 02:37:42 +01002168__weak struct cpufreq_governor *cpufreq_fallback_governor(void)
2169{
2170 return NULL;
2171}
2172
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002173static int cpufreq_init_governor(struct cpufreq_policy *policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002174{
Dave Jonescc993ca2005-07-28 09:43:56 -07002175 int ret;
Thomas Renninger6afde102007-10-02 13:28:13 -07002176
Viresh Kumar2f0aea92014-03-04 11:00:26 +08002177 /* Don't start any governor operations if we are entering suspend */
2178 if (cpufreq_suspended)
2179 return 0;
Ethan Zhaocb57720b2014-12-18 15:28:19 +09002180 /*
2181 * Governor might not be initiated here if ACPI _PPC changed
2182 * notification happened, so check it.
2183 */
2184 if (!policy->governor)
2185 return -EINVAL;
Viresh Kumar2f0aea92014-03-04 11:00:26 +08002186
Viresh Kumared4676e2017-07-19 15:42:46 +05302187 /* Platform doesn't want dynamic frequency switching ? */
2188 if (policy->governor->dynamic_switching &&
Viresh Kumarfc4c7092017-07-19 15:42:49 +05302189 cpufreq_driver->flags & CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING) {
Rafael J. Wysockide1df262016-02-05 02:37:42 +01002190 struct cpufreq_governor *gov = cpufreq_fallback_governor();
2191
2192 if (gov) {
Viresh Kumarfe829ed2017-07-19 15:42:48 +05302193 pr_warn("Can't use %s governor as dynamic switching is disallowed. Fallback to %s governor\n",
Joe Perchese837f9b2014-03-11 10:03:00 -07002194 policy->governor->name, gov->name);
Thomas Renninger6afde102007-10-02 13:28:13 -07002195 policy->governor = gov;
Rafael J. Wysockide1df262016-02-05 02:37:42 +01002196 } else {
2197 return -EINVAL;
Thomas Renninger6afde102007-10-02 13:28:13 -07002198 }
Thomas Renninger1c256242007-10-02 13:28:12 -07002199 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002200
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002201 if (!try_module_get(policy->governor->owner))
2202 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002204 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
Xiaoguang Chen95731eb2013-06-19 15:00:07 +08002205
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002206 if (policy->governor->init) {
2207 ret = policy->governor->init(policy);
2208 if (ret) {
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02002209 module_put(policy->governor->owner);
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002210 return ret;
2211 }
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02002212 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002214 return 0;
2215}
2216
2217static void cpufreq_exit_governor(struct cpufreq_policy *policy)
2218{
2219 if (cpufreq_suspended || !policy->governor)
2220 return;
2221
2222 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2223
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002224 if (policy->governor->exit)
2225 policy->governor->exit(policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002226
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002227 module_put(policy->governor->owner);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228}
2229
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002230static int cpufreq_start_governor(struct cpufreq_policy *policy)
2231{
2232 int ret;
2233
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002234 if (cpufreq_suspended)
2235 return 0;
2236
2237 if (!policy->governor)
2238 return -EINVAL;
2239
2240 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2241
Viresh Kumar407d0ff2019-06-20 08:35:46 +05302242 if (cpufreq_driver->get)
Viresh Kumar59807522019-06-20 08:35:49 +05302243 cpufreq_verify_current_freq(policy, false);
Rafael J. Wysocki3bbf8fe2016-03-21 15:47:48 +01002244
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002245 if (policy->governor->start) {
2246 ret = policy->governor->start(policy);
2247 if (ret)
2248 return ret;
2249 }
Rafael J. Wysockid6ff44d2016-05-14 00:59:27 +02002250
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002251 if (policy->governor->limits)
2252 policy->governor->limits(policy);
2253
Rafael J. Wysockid6ff44d2016-05-14 00:59:27 +02002254 return 0;
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002255}
2256
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002257static void cpufreq_stop_governor(struct cpufreq_policy *policy)
2258{
2259 if (cpufreq_suspended || !policy->governor)
2260 return;
2261
2262 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2263
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002264 if (policy->governor->stop)
2265 policy->governor->stop(policy);
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002266}
2267
2268static void cpufreq_governor_limits(struct cpufreq_policy *policy)
2269{
2270 if (cpufreq_suspended || !policy->governor)
2271 return;
2272
2273 pr_debug("%s: for CPU %u\n", __func__, policy->cpu);
2274
Rafael J. Wysockie7888922016-06-02 23:24:15 +02002275 if (policy->governor->limits)
2276 policy->governor->limits(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002277}
2278
Linus Torvalds1da177e2005-04-16 15:20:36 -07002279int cpufreq_register_governor(struct cpufreq_governor *governor)
2280{
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002281 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002282
2283 if (!governor)
2284 return -EINVAL;
2285
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002286 if (cpufreq_disabled())
2287 return -ENODEV;
2288
akpm@osdl.org3fc54d32006-01-13 15:54:22 -08002289 mutex_lock(&cpufreq_governor_mutex);
Dave Jones32ee8c32006-02-28 00:43:23 -05002290
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002291 err = -EBUSY;
Viresh Kumar42f91fa2015-01-02 12:34:26 +05302292 if (!find_governor(governor->name)) {
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002293 err = 0;
2294 list_add(&governor->governor_list, &cpufreq_governor_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002295 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002296
Dave Jones32ee8c32006-02-28 00:43:23 -05002297 mutex_unlock(&cpufreq_governor_mutex);
Jeremy Fitzhardinge3bcb09a2006-07-06 12:30:26 -07002298 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002299}
2300EXPORT_SYMBOL_GPL(cpufreq_register_governor);
2301
Linus Torvalds1da177e2005-04-16 15:20:36 -07002302void cpufreq_unregister_governor(struct cpufreq_governor *governor)
2303{
Viresh Kumar45732372015-05-12 12:22:34 +05302304 struct cpufreq_policy *policy;
2305 unsigned long flags;
Prarit Bhargava90e41ba2009-11-12 09:18:46 -05002306
Linus Torvalds1da177e2005-04-16 15:20:36 -07002307 if (!governor)
2308 return;
2309
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002310 if (cpufreq_disabled())
2311 return;
2312
Viresh Kumar45732372015-05-12 12:22:34 +05302313 /* clear last_governor for all inactive policies */
2314 read_lock_irqsave(&cpufreq_driver_lock, flags);
2315 for_each_inactive_policy(policy) {
Viresh Kumar18bf3a12015-05-12 12:22:51 +05302316 if (!strcmp(policy->last_governor, governor->name)) {
2317 policy->governor = NULL;
Viresh Kumar45732372015-05-12 12:22:34 +05302318 strcpy(policy->last_governor, "\0");
Viresh Kumar18bf3a12015-05-12 12:22:51 +05302319 }
Prarit Bhargava90e41ba2009-11-12 09:18:46 -05002320 }
Viresh Kumar45732372015-05-12 12:22:34 +05302321 read_unlock_irqrestore(&cpufreq_driver_lock, flags);
Prarit Bhargava90e41ba2009-11-12 09:18:46 -05002322
akpm@osdl.org3fc54d32006-01-13 15:54:22 -08002323 mutex_lock(&cpufreq_governor_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324 list_del(&governor->governor_list);
akpm@osdl.org3fc54d32006-01-13 15:54:22 -08002325 mutex_unlock(&cpufreq_governor_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002326}
2327EXPORT_SYMBOL_GPL(cpufreq_unregister_governor);
2328
2329
Linus Torvalds1da177e2005-04-16 15:20:36 -07002330/*********************************************************************
2331 * POLICY INTERFACE *
2332 *********************************************************************/
2333
2334/**
2335 * cpufreq_get_policy - get the current cpufreq_policy
Dave Jones29464f22009-01-18 01:37:11 -05002336 * @policy: struct cpufreq_policy into which the current cpufreq_policy
2337 * is written
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338 *
2339 * Reads the current cpufreq policy.
2340 */
2341int cpufreq_get_policy(struct cpufreq_policy *policy, unsigned int cpu)
2342{
2343 struct cpufreq_policy *cpu_policy;
2344 if (!policy)
2345 return -EINVAL;
2346
2347 cpu_policy = cpufreq_cpu_get(cpu);
2348 if (!cpu_policy)
2349 return -EINVAL;
2350
Viresh Kumard5b73cd2013-08-06 22:53:06 +05302351 memcpy(policy, cpu_policy, sizeof(*policy));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002352
2353 cpufreq_cpu_put(cpu_policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002354 return 0;
2355}
2356EXPORT_SYMBOL(cpufreq_get_policy);
2357
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002358/**
2359 * cpufreq_set_policy - Modify cpufreq policy parameters.
2360 * @policy: Policy object to modify.
2361 * @new_policy: New policy data.
2362 *
Viresh Kumardf0eea42019-07-23 11:44:09 +05302363 * Pass @new_policy to the cpufreq driver's ->verify() callback. Next, copy the
2364 * min and max parameters of @new_policy to @policy and either invoke the
2365 * driver's ->setpolicy() callback (if present) or carry out a governor update
2366 * for @policy. That is, run the current governor's ->limits() callback (if the
2367 * governor field in @new_policy points to the same object as the one in
2368 * @policy) or replace the governor for @policy with the new one stored in
2369 * @new_policy.
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002370 *
2371 * The cpuinfo part of @policy is not updated by this function.
Arjan van de Ven153d7f32006-07-26 15:40:07 +02002372 */
Rafael J. Wysocki9083e492019-03-26 12:19:52 +01002373int cpufreq_set_policy(struct cpufreq_policy *policy,
2374 struct cpufreq_policy *new_policy)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002375{
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002376 struct cpufreq_governor *old_gov;
2377 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002378
Joe Perchese837f9b2014-03-11 10:03:00 -07002379 pr_debug("setting new policy for CPU %u: %u - %u kHz\n",
2380 new_policy->cpu, new_policy->min, new_policy->max);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002381
Viresh Kumard5b73cd2013-08-06 22:53:06 +05302382 memcpy(&new_policy->cpuinfo, &policy->cpuinfo, sizeof(policy->cpuinfo));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002383
Pan Xinhuifba95732015-07-30 18:10:40 +08002384 /*
Viresh Kumar67d874c2019-07-08 16:27:52 +05302385 * PM QoS framework collects all the requests from users and provide us
2386 * the final aggregated value here.
2387 */
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02002388 new_policy->min = freq_qos_read_value(&policy->constraints, FREQ_QOS_MIN);
2389 new_policy->max = freq_qos_read_value(&policy->constraints, FREQ_QOS_MAX);
Mattia Dongili9c9a43e2006-07-05 23:12:20 +02002390
Linus Torvalds1da177e2005-04-16 15:20:36 -07002391 /* verify the cpu speed can be set within this limit */
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05302392 ret = cpufreq_driver->verify(new_policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002393 if (ret)
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002394 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002395
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05302396 policy->min = new_policy->min;
2397 policy->max = new_policy->max;
Ruchi Kandoi601b2182018-07-24 10:35:44 -07002398 trace_cpu_frequency_limits(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399
Steve Mucklee3c06232016-07-13 13:25:25 -07002400 policy->cached_target_freq = UINT_MAX;
2401
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002402 pr_debug("new min and max freqs are %u - %u kHz\n",
Joe Perchese837f9b2014-03-11 10:03:00 -07002403 policy->min, policy->max);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002404
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002405 if (cpufreq_driver->setpolicy) {
Viresh Kumar3a3e9e02013-08-06 22:53:05 +05302406 policy->policy = new_policy->policy;
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002407 pr_debug("setting range\n");
Rafael J. Wysocki167a38d2019-02-20 00:26:30 +01002408 return cpufreq_driver->setpolicy(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002409 }
2410
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002411 if (new_policy->governor == policy->governor) {
Rafael J. Wysocki2bb40592019-02-20 00:25:18 +01002412 pr_debug("governor limits update\n");
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002413 cpufreq_governor_limits(policy);
Rafael J. Wysockid6ff44d2016-05-14 00:59:27 +02002414 return 0;
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002415 }
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002416
2417 pr_debug("governor switch\n");
2418
2419 /* save old, working values */
2420 old_gov = policy->governor;
2421 /* end old governor */
2422 if (old_gov) {
Rafael J. Wysocki45482c72016-05-12 15:14:12 +02002423 cpufreq_stop_governor(policy);
Rafael J. Wysocki36be3412016-05-12 15:13:35 +02002424 cpufreq_exit_governor(policy);
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002425 }
2426
2427 /* start new governor */
2428 policy->governor = new_policy->governor;
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002429 ret = cpufreq_init_governor(policy);
Viresh Kumar4bc384a2015-07-18 11:31:03 +05302430 if (!ret) {
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002431 ret = cpufreq_start_governor(policy);
2432 if (!ret) {
Rafael J. Wysocki2bb40592019-02-20 00:25:18 +01002433 pr_debug("governor change\n");
Quentin Perret531b5c92018-12-03 09:56:21 +00002434 sched_cpufreq_governor_change(policy, old_gov);
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002435 return 0;
2436 }
Rafael J. Wysockib7898fd2016-03-30 03:47:49 +02002437 cpufreq_exit_governor(policy);
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002438 }
2439
2440 /* new governor failed, so re-start old one */
2441 pr_debug("starting governor %s failed\n", policy->governor->name);
2442 if (old_gov) {
2443 policy->governor = old_gov;
Rafael J. Wysockia92604b2016-05-14 01:01:46 +02002444 if (cpufreq_init_governor(policy))
Viresh Kumar4bc384a2015-07-18 11:31:03 +05302445 policy->governor = NULL;
2446 else
Rafael J. Wysocki0a300762016-03-21 15:45:24 +01002447 cpufreq_start_governor(policy);
Rafael J. Wysockid9a789c2014-02-17 22:56:35 +01002448 }
2449
Viresh Kumar4bc384a2015-07-18 11:31:03 +05302450 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002451}
2452
2453/**
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002454 * cpufreq_update_policy - Re-evaluate an existing cpufreq policy.
2455 * @cpu: CPU to re-evaluate the policy for.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002456 *
Rafael J. Wysockia0dbb812019-02-20 00:22:51 +01002457 * Update the current frequency for the cpufreq policy of @cpu and use
Viresh Kumar18c49922019-07-05 16:21:24 +05302458 * cpufreq_set_policy() to re-apply the min and max limits, which triggers the
2459 * evaluation of policy notifiers and the cpufreq driver's ->verify() callback
2460 * for the policy in question, among other things.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002461 */
Rafael J. Wysocki30248fe2016-11-18 13:59:21 +01002462void cpufreq_update_policy(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002463{
Rafael J. Wysocki540a37582019-03-26 12:16:58 +01002464 struct cpufreq_policy *policy = cpufreq_cpu_acquire(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002465
Aaron Plattnerfefa8ff2014-06-18 11:27:32 -07002466 if (!policy)
Rafael J. Wysocki30248fe2016-11-18 13:59:21 +01002467 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002468
Viresh Kumarbb176f72013-06-19 14:19:33 +05302469 /*
2470 * BIOS might change freq behind our back
2471 * -> ask driver for current freq and notify governors about a change
2472 */
Viresh Kumar5ddc6d42019-06-20 08:35:48 +05302473 if (cpufreq_driver->get && has_target() &&
Viresh Kumar59807522019-06-20 08:35:49 +05302474 (cpufreq_suspended || WARN_ON(!cpufreq_verify_current_freq(policy, false))))
Rafael J. Wysocki348a2ec2019-02-20 00:24:25 +01002475 goto unlock;
Rafael J. Wysocki30248fe2016-11-18 13:59:21 +01002476
Viresh Kumar70a59fd2019-06-20 08:35:50 +05302477 refresh_frequency_limits(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002478
Aaron Plattnerfefa8ff2014-06-18 11:27:32 -07002479unlock:
Rafael J. Wysocki540a37582019-03-26 12:16:58 +01002480 cpufreq_cpu_release(policy);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002481}
2482EXPORT_SYMBOL(cpufreq_update_policy);
2483
Rafael J. Wysocki5a25e3f2019-03-26 12:15:13 +01002484/**
2485 * cpufreq_update_limits - Update policy limits for a given CPU.
2486 * @cpu: CPU to update the policy limits for.
2487 *
2488 * Invoke the driver's ->update_limits callback if present or call
2489 * cpufreq_update_policy() for @cpu.
2490 */
2491void cpufreq_update_limits(unsigned int cpu)
2492{
2493 if (cpufreq_driver->update_limits)
2494 cpufreq_driver->update_limits(cpu);
2495 else
2496 cpufreq_update_policy(cpu);
2497}
2498EXPORT_SYMBOL_GPL(cpufreq_update_limits);
2499
Linus Torvalds1da177e2005-04-16 15:20:36 -07002500/*********************************************************************
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002501 * BOOST *
2502 *********************************************************************/
2503static int cpufreq_boost_set_sw(int state)
2504{
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002505 struct cpufreq_policy *policy;
2506 int ret = -EINVAL;
2507
Viresh Kumarf9637352015-05-12 12:20:11 +05302508 for_each_active_policy(policy) {
Viresh Kumarf8bfc112016-06-03 10:58:47 +05302509 if (!policy->freq_table)
2510 continue;
Viresh Kumar49f18562016-02-11 17:31:12 +05302511
Viresh Kumarf8bfc112016-06-03 10:58:47 +05302512 ret = cpufreq_frequency_table_cpuinfo(policy,
2513 policy->freq_table);
2514 if (ret) {
2515 pr_err("%s: Policy frequency update failed\n",
2516 __func__);
2517 break;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002518 }
Viresh Kumarf8bfc112016-06-03 10:58:47 +05302519
Rafael J. Wysocki3000ce32019-10-16 12:47:06 +02002520 ret = freq_qos_update_request(policy->max_freq_req, policy->max);
Viresh Kumare61a4122019-08-09 08:01:17 +05302521 if (ret < 0)
Viresh Kumar18c49922019-07-05 16:21:24 +05302522 break;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002523 }
2524
2525 return ret;
2526}
2527
2528int cpufreq_boost_trigger_state(int state)
2529{
2530 unsigned long flags;
2531 int ret = 0;
2532
2533 if (cpufreq_driver->boost_enabled == state)
2534 return 0;
2535
2536 write_lock_irqsave(&cpufreq_driver_lock, flags);
2537 cpufreq_driver->boost_enabled = state;
2538 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2539
2540 ret = cpufreq_driver->set_boost(state);
2541 if (ret) {
2542 write_lock_irqsave(&cpufreq_driver_lock, flags);
2543 cpufreq_driver->boost_enabled = !state;
2544 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
2545
Joe Perchese837f9b2014-03-11 10:03:00 -07002546 pr_err("%s: Cannot %s BOOST\n",
2547 __func__, state ? "enable" : "disable");
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002548 }
2549
2550 return ret;
2551}
2552
Rafael J. Wysocki41669da2015-12-27 00:23:48 +01002553static bool cpufreq_boost_supported(void)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002554{
Yue Hu89f98d72019-04-09 10:25:36 +08002555 return cpufreq_driver->set_boost;
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002556}
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002557
Viresh Kumar44139ed2015-07-29 16:23:09 +05302558static int create_boost_sysfs_file(void)
2559{
2560 int ret;
2561
Viresh Kumarc82bd442015-10-15 21:35:23 +05302562 ret = sysfs_create_file(cpufreq_global_kobject, &boost.attr);
Viresh Kumar44139ed2015-07-29 16:23:09 +05302563 if (ret)
2564 pr_err("%s: cannot register global BOOST sysfs file\n",
2565 __func__);
2566
2567 return ret;
2568}
2569
2570static void remove_boost_sysfs_file(void)
2571{
2572 if (cpufreq_boost_supported())
Viresh Kumarc82bd442015-10-15 21:35:23 +05302573 sysfs_remove_file(cpufreq_global_kobject, &boost.attr);
Viresh Kumar44139ed2015-07-29 16:23:09 +05302574}
2575
2576int cpufreq_enable_boost_support(void)
2577{
2578 if (!cpufreq_driver)
2579 return -EINVAL;
2580
2581 if (cpufreq_boost_supported())
2582 return 0;
2583
Rafael J. Wysocki7a6c79f2015-12-27 00:27:38 +01002584 cpufreq_driver->set_boost = cpufreq_boost_set_sw;
Viresh Kumar44139ed2015-07-29 16:23:09 +05302585
2586 /* This will get removed on driver unregister */
2587 return create_boost_sysfs_file();
2588}
2589EXPORT_SYMBOL_GPL(cpufreq_enable_boost_support);
2590
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002591int cpufreq_boost_enabled(void)
2592{
2593 return cpufreq_driver->boost_enabled;
2594}
2595EXPORT_SYMBOL_GPL(cpufreq_boost_enabled);
2596
2597/*********************************************************************
Linus Torvalds1da177e2005-04-16 15:20:36 -07002598 * REGISTER / UNREGISTER CPUFREQ DRIVER *
2599 *********************************************************************/
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02002600static enum cpuhp_state hp_online;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002601
Chen Yuc4a3fa22017-04-09 13:45:16 +08002602static int cpuhp_cpufreq_online(unsigned int cpu)
2603{
2604 cpufreq_online(cpu);
2605
2606 return 0;
2607}
2608
2609static int cpuhp_cpufreq_offline(unsigned int cpu)
2610{
2611 cpufreq_offline(cpu);
2612
2613 return 0;
2614}
2615
Linus Torvalds1da177e2005-04-16 15:20:36 -07002616/**
2617 * cpufreq_register_driver - register a CPU Frequency driver
2618 * @driver_data: A struct cpufreq_driver containing the values#
2619 * submitted by the CPU Frequency driver.
2620 *
Viresh Kumarbb176f72013-06-19 14:19:33 +05302621 * Registers a CPU Frequency driver to this core code. This code
Eric Biggers63af4052016-02-20 21:50:01 -06002622 * returns zero on success, -EEXIST when another driver got here first
Dave Jones32ee8c32006-02-28 00:43:23 -05002623 * (and isn't unregistered in the meantime).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002624 *
2625 */
Linus Torvalds221dee22007-02-26 14:55:48 -08002626int cpufreq_register_driver(struct cpufreq_driver *driver_data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002627{
2628 unsigned long flags;
2629 int ret;
2630
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002631 if (cpufreq_disabled())
2632 return -ENODEV;
2633
Linus Torvalds1da177e2005-04-16 15:20:36 -07002634 if (!driver_data || !driver_data->verify || !driver_data->init ||
Viresh Kumar9c0ebcf2013-10-25 19:45:48 +05302635 !(driver_data->setpolicy || driver_data->target_index ||
Rafael J. Wysocki98322352014-03-19 12:48:30 +01002636 driver_data->target) ||
2637 (driver_data->setpolicy && (driver_data->target_index ||
Viresh Kumar1c03a2d2014-06-02 22:49:28 +05302638 driver_data->target)) ||
Viresh Kumara9a22b52019-02-14 16:16:21 +05302639 (!driver_data->get_intermediate != !driver_data->target_intermediate) ||
Viresh Kumar91a12e92019-02-12 16:36:04 +05302640 (!driver_data->online != !driver_data->offline))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002641 return -EINVAL;
2642
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002643 pr_debug("trying to register driver %s\n", driver_data->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002644
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002645 /* Protect against concurrent CPU online/offline. */
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002646 cpus_read_lock();
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002647
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002648 write_lock_irqsave(&cpufreq_driver_lock, flags);
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002649 if (cpufreq_driver) {
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002650 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002651 ret = -EEXIST;
2652 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002653 }
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002654 cpufreq_driver = driver_data;
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002655 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002656
Viresh Kumarbc68b7d2015-01-02 12:34:30 +05302657 if (driver_data->setpolicy)
2658 driver_data->flags |= CPUFREQ_CONST_LOOPS;
2659
Rafael J. Wysocki7a6c79f2015-12-27 00:27:38 +01002660 if (cpufreq_boost_supported()) {
2661 ret = create_boost_sysfs_file();
2662 if (ret)
2663 goto err_null_driver;
2664 }
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002665
Kay Sievers8a25a2f2011-12-21 14:29:42 -08002666 ret = subsys_interface_register(&cpufreq_interface);
Jiri Slaby8f5bc2a2011-03-01 17:41:10 +01002667 if (ret)
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002668 goto err_boost_unreg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002669
Viresh Kumarce1bcfe2015-01-02 12:34:35 +05302670 if (!(cpufreq_driver->flags & CPUFREQ_STICKY) &&
2671 list_empty(&cpufreq_policy_list)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002672 /* if all ->init() calls failed, unregister */
David Arcari6c770032017-05-26 11:37:31 -04002673 ret = -ENODEV;
Viresh Kumarce1bcfe2015-01-02 12:34:35 +05302674 pr_debug("%s: No CPU initialized for driver %s\n", __func__,
2675 driver_data->name);
2676 goto err_if_unreg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002677 }
2678
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002679 ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN,
2680 "cpufreq:online",
2681 cpuhp_cpufreq_online,
2682 cpuhp_cpufreq_offline);
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02002683 if (ret < 0)
2684 goto err_if_unreg;
2685 hp_online = ret;
Sebastian Andrzej Siewior5372e052016-09-20 16:56:28 +02002686 ret = 0;
Sebastian Andrzej Siewior27622b02016-09-06 19:04:48 +02002687
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002688 pr_debug("driver %s up and running\n", driver_data->name);
Pankaj Gupta3834abb2016-05-16 11:07:19 +00002689 goto out;
Rafael J. Wysockifdd320d2015-07-30 01:45:07 +02002690
Kay Sievers8a25a2f2011-12-21 14:29:42 -08002691err_if_unreg:
2692 subsys_interface_unregister(&cpufreq_interface);
Lukasz Majewski6f19efc2013-12-20 15:24:49 +01002693err_boost_unreg:
Viresh Kumar44139ed2015-07-29 16:23:09 +05302694 remove_boost_sysfs_file();
Jiri Slaby8f5bc2a2011-03-01 17:41:10 +01002695err_null_driver:
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002696 write_lock_irqsave(&cpufreq_driver_lock, flags);
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002697 cpufreq_driver = NULL;
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002698 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Pankaj Gupta3834abb2016-05-16 11:07:19 +00002699out:
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002700 cpus_read_unlock();
Pankaj Gupta3834abb2016-05-16 11:07:19 +00002701 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002702}
2703EXPORT_SYMBOL_GPL(cpufreq_register_driver);
2704
Linus Torvalds1da177e2005-04-16 15:20:36 -07002705/**
2706 * cpufreq_unregister_driver - unregister the current CPUFreq driver
2707 *
Viresh Kumarbb176f72013-06-19 14:19:33 +05302708 * Unregister the current CPUFreq driver. Only call this if you have
Linus Torvalds1da177e2005-04-16 15:20:36 -07002709 * the right to do so, i.e. if you have succeeded in initialising before!
2710 * Returns zero if successful, and -EINVAL if the cpufreq_driver is
2711 * currently not initialised.
2712 */
Linus Torvalds221dee22007-02-26 14:55:48 -08002713int cpufreq_unregister_driver(struct cpufreq_driver *driver)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002714{
2715 unsigned long flags;
2716
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002717 if (!cpufreq_driver || (driver != cpufreq_driver))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002718 return -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002719
Dominik Brodowski2d06d8c2011-03-27 15:04:46 +02002720 pr_debug("unregistering driver %s\n", driver->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002721
Sebastian Andrzej Siewior454d3a22015-07-22 17:59:11 +02002722 /* Protect against concurrent cpu hotplug */
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002723 cpus_read_lock();
Kay Sievers8a25a2f2011-12-21 14:29:42 -08002724 subsys_interface_unregister(&cpufreq_interface);
Viresh Kumar44139ed2015-07-29 16:23:09 +05302725 remove_boost_sysfs_file();
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002726 cpuhp_remove_state_nocalls_cpuslocked(hp_online);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002727
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002728 write_lock_irqsave(&cpufreq_driver_lock, flags);
Viresh Kumar6eed9402013-08-06 22:53:11 +05302729
Rafael J. Wysocki1c3d85d2013-04-29 00:08:16 +02002730 cpufreq_driver = NULL;
Viresh Kumar6eed9402013-08-06 22:53:11 +05302731
Nathan Zimmer0d1857a2013-02-22 16:24:34 +00002732 write_unlock_irqrestore(&cpufreq_driver_lock, flags);
Sebastian Andrzej Siewiora92551e2017-05-24 10:15:20 +02002733 cpus_read_unlock();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002734
2735 return 0;
2736}
2737EXPORT_SYMBOL_GPL(cpufreq_unregister_driver);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002738
Viresh Kumarc82bd442015-10-15 21:35:23 +05302739struct kobject *cpufreq_global_kobject;
2740EXPORT_SYMBOL(cpufreq_global_kobject);
2741
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002742static int __init cpufreq_core_init(void)
2743{
Konrad Rzeszutek Wilka7b422c2012-03-13 19:18:39 -04002744 if (cpufreq_disabled())
2745 return -ENODEV;
2746
Viresh Kumar8eec1022015-10-15 21:35:22 +05302747 cpufreq_global_kobject = kobject_create_and_add("cpufreq", &cpu_subsys.dev_root->kobj);
Thomas Renninger8aa84ad2009-07-24 15:25:05 +02002748 BUG_ON(!cpufreq_global_kobject);
2749
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002750 return 0;
2751}
Len Brownd82f2692017-02-28 16:44:16 -05002752module_param(off, int, 0444);
Venkatesh Pallipadi5a01f2e2007-02-05 16:12:44 -08002753core_initcall(cpufreq_core_init);